Merge branch '2.18-maintenance' into ifd-buildStore-2.18

.github/labeler.yml

@@ -20,4 +20,4 @@
   # Unit tests
   - src/*/tests/**/*
   # Functional and integration tests
-  - tests/**/*
+  - tests/functional/**/*

.gitignore

@@ -41,14 +41,14 @@ perl/Makefile.config
 /src/libexpr/parser-tab.hh
 /src/libexpr/parser-tab.output
 /src/libexpr/nix.tbl
-/src/libexpr/tests/libnixexpr-tests
+/tests/unit/libexpr/libnixexpr-tests
 
 # /src/libstore/
 *.gen.*
-/src/libstore/tests/libnixstore-tests
+/tests/unit/libstore/libnixstore-tests
 
 # /src/libutil/
-/src/libutil/tests/libnixutil-tests
+/tests/unit/libutil/libnixutil-tests
 
 /src/nix/nix
 
@@ -79,24 +79,24 @@ perl/Makefile.config
 
 /src/build-remote/build-remote
 
-# /tests/
+# /tests/functional/
-/tests/test-tmp
+/tests/functional/test-tmp
-/tests/common/vars-and-functions.sh
+/tests/functional/common/vars-and-functions.sh
-/tests/result*
+/tests/functional/result*
-/tests/restricted-innocent
+/tests/functional/restricted-innocent
-/tests/shell
+/tests/functional/shell
-/tests/shell.drv
+/tests/functional/shell.drv
-/tests/config.nix
+/tests/functional/config.nix
-/tests/ca/config.nix
+/tests/functional/ca/config.nix
-/tests/dyn-drv/config.nix
+/tests/functional/dyn-drv/config.nix
-/tests/repl-result-out
+/tests/functional/repl-result-out
-/tests/test-libstoreconsumer/test-libstoreconsumer
+/tests/functional/test-libstoreconsumer/test-libstoreconsumer
 
-# /tests/lang/
+# /tests/functional/lang/
-/tests/lang/*.out
+/tests/functional/lang/*.out
-/tests/lang/*.out.xml
+/tests/functional/lang/*.out.xml
-/tests/lang/*.err
+/tests/functional/lang/*.err
-/tests/lang/*.ast
+/tests/functional/lang/*.ast
 
 /perl/lib/Nix/Config.pm
 /perl/lib/Nix/Store.cc

.version

@@ -1 +1 @@
-2.18.0
+2.18.1

CONTRIBUTING.md

@@ -52,7 +52,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy).
 
    - [ ] Fixes an [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) issue
    - [ ] Tests, as appropriate:
-     - Functional tests – [`tests/**.sh`](./tests)
+     - Functional tests – [`tests/functional/**.sh`](./tests/functional)
      - Unit tests – [`src/*/tests`](./src/)
      - Integration tests – [`tests/nixos/*`](./tests/nixos)
    - [ ] User documentation in the [manual](..doc/manual/src)

Makefile

@@ -23,14 +23,17 @@ makefiles = \
 
 ifeq ($(tests), yes)
 makefiles += \
-  src/libutil/tests/local.mk \
+  tests/unit/libutil/local.mk \
-  src/libstore/tests/local.mk \
+  tests/unit/libutil-support/local.mk \
-  src/libexpr/tests/local.mk \
+  tests/unit/libstore/local.mk \
-  tests/local.mk \
+  tests/unit/libstore-support/local.mk \
-  tests/ca/local.mk \
+  tests/unit/libexpr/local.mk \
-  tests/dyn-drv/local.mk \
+  tests/unit/libexpr-support/local.mk \
-  tests/test-libstoreconsumer/local.mk \
+  tests/functional/local.mk \
-  tests/plugins/local.mk
+  tests/functional/ca/local.mk \
+  tests/functional/dyn-drv/local.mk \
+  tests/functional/test-libstoreconsumer/local.mk \
+  tests/functional/plugins/local.mk
 else
 makefiles += \
   mk/disable-tests.mk

doc/internal-api/doxygen.cfg.in

@@ -39,17 +39,21 @@ INPUT                  = \
   src/libcmd \
   src/libexpr \
   src/libexpr/flake \
-  src/libexpr/tests \
+  tests/unit/libexpr \
-  src/libexpr/tests/value \
+  tests/unit/libexpr/value \
+  tests/unit/libexpr/test \
+  tests/unit/libexpr/test/value \
   src/libexpr/value \
   src/libfetchers \
   src/libmain \
   src/libstore \
   src/libstore/build \
   src/libstore/builtins \
-  src/libstore/tests \
+  tests/unit/libstore \
+  tests/unit/libstore/test \
   src/libutil \
-  src/libutil/tests \
+  tests/unit/libutil \
+  tests/unit/libutil/test \
   src/nix \
   src/nix-env \
   src/nix-store

doc/manual/_redirects

@@ -0,0 +1,30 @@
+# redirect rules for paths (server-side) to prevent link rot.
+# see ./redirects.js for redirects based on URL fragments (client-side)
+#
+# concrete user story this supports:
+# - user finds URL to the manual for Nix x.y
+# - Nix x.z (z > y) is the most recent release
+# - updating the version in the URL will show the right thing
+#
+# format documentation:
+# - https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file
+# - https://docs.netlify.com/routing/redirects/redirect-options/
+#
+# conventions:
+# - always force (<CODE>!) since this allows re-using file names
+# - group related paths to ease readability
+# - always append new redirects to the end of the file
+# - redirects that should have been there but are missing can be inserted where they belong
+
+/expressions/expression-language /language/ 301!
+/expressions/language-values /language/values 301!
+/expressions/language-constructs /language/constructs 301!
+/expressions/language-operators /language/operators 301!
+/expressions/* /language/:splat 301!
+
+/package-management/basic-package-mgmt /command-ref/nix-env 301!
+
+/package-management/channels* /command-ref/nix-channel 301!
+
+/package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301!
+

doc/manual/generate-manpage.nix

@@ -5,7 +5,7 @@ let
   inherit (import ./utils.nix) concatStrings optionalString filterAttrs trim squash unique showSettings;
 in
 
-commandDump:
+inlineHTML: commandDump:
 
 let
 
@@ -30,7 +30,7 @@ let
 
         ${maybeSubcommands}
 
-        ${maybeDocumentation}
+        ${maybeStoreDocs}
 
         ${maybeOptions}
       '';
@@ -70,7 +70,7 @@ let
         * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description}
       '';
 
-      maybeDocumentation = optionalString
+      maybeStoreDocs = optionalString
         (details ? doc)
         (replaceStrings ["@stores@"] [storeDocs] details.doc);
 
@@ -91,17 +91,20 @@ let
           listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts));
           showOption = name: option:
             let
+              result = trim ''
+                - ${item}
+                  ${option.description}
+              '';
+              item = if inlineHTML
+                then ''<span id="opt-${name}">[`--${name}`](#opt-${name})</span> ${shortName} ${labels}''
+                else "`--${name}` ${shortName} ${labels}";
               shortName = optionalString
                 (option ? shortName)
                 ("/ `-${option.shortName}`");
               labels = optionalString
                 (option ? labels)
                 (concatStringsSep " " (map (s: "*${s}*") option.labels));
-            in trim ''
+            in result;
-              - <span id="opt-${name}">[`--${name}`](#opt-${name})</span> ${shortName} ${labels}
-
-                ${option.description}
-            '';
           categories = sort lessThan (unique (map (cmd: cmd.category) (attrValues allOptions)));
         in concatStrings (map showCategory categories);
     in squash result;
@@ -162,7 +165,7 @@ let
 
           **Settings**:
 
-          ${showSettings { useAnchors = false; } settings}
+          ${showSettings { inherit inlineHTML; } settings}
         '';
     in concatStrings (attrValues (mapAttrs showStore commandInfo.stores));
 

doc/manual/local.mk

@@ -98,12 +98,12 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/sr
 
 $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix
 	@rm -rf $@ $@.tmp
-	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)'
+	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)'
 	@mv $@.tmp $@
 
 $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
-	$(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
+	$(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { inlineHTML = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
 	@mv $@.tmp $@
 
 $(d)/nix.json: $(bindir)/nix

doc/manual/redirects.js

@@ -1,7 +1,9 @@
-// redirect rules for anchors ensure backwards compatibility of URLs.
+// redirect rules for URL fragments (client-side) to prevent link rot.
-// this must be done on the client side, as web servers do not see the anchor part of the URL.
+// this must be done on the client side, as web servers do not see the fragment part of the URL.
+// it will only work with JavaScript enabled in the browser, but this is the best we can do here.
+// see ./_redirects for path redirects (client-side)
 
-// redirections are declared as follows:
+// redirects are declared as follows:
 // each entry has as its key a path matching the requested URL path, relative to the mdBook document root.
 //
 //     IMPORTANT: it must specify the full path with file name and suffix
@@ -19,6 +21,7 @@ const redirects = {
     "chap-distributed-builds": "advanced-topics/distributed-builds.html",
     "chap-post-build-hook": "advanced-topics/post-build-hook.html",
     "chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats",
+    "chap-writing-nix-expressions": "language/index.html",
     "part-command-ref": "command-ref/command-ref.html",
     "conf-allow-import-from-derivation": "command-ref/conf-file.html#conf-allow-import-from-derivation",
     "conf-allow-new-privileges": "command-ref/conf-file.html#conf-allow-new-privileges",

doc/manual/src/SUMMARY.md.in

@@ -109,7 +109,6 @@
   - [CLI guideline](contributing/cli-guideline.md)
   - [C++ style guide](contributing/cxx.md)
 - [Release Notes](release-notes/release-notes.md)
-  - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
   - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md)
   - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md)
   - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md)

doc/manual/src/contributing/testing.md

@@ -3,16 +3,28 @@
 ## Unit-tests
 
 The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined
-under `src/{library_name}/tests` using the
+under `tests/unit/{library_name}/tests` using the
 [googletest](https://google.github.io/googletest/) and
 [rapidcheck](https://github.com/emil-e/rapidcheck) frameworks.
 
 You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`.
 Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable.
 
+### Unit test support libraries
+
+There are headers and code which are not just used to test the library in question, but also downstream libraries.
+For example, we do [property testing] with the [rapidcheck] library.
+This requires writing `Arbitrary` "instances", which are used to describe how to generate values of a given type for the sake of running property tests.
+Because types contain other types, `Arbitrary` "instances" for some type are not just useful for testing that type, but also any other type that contains it.
+Downstream types frequently contain upstream types, so it is very important that we share arbitrary instances so that downstream libraries' property tests can also use them.
+
+It is important that these testing libraries don't contain any actual tests themselves.
+On some platforms they would be run as part of every test executable that uses them, which is redundant.
+On other platforms they wouldn't be run at all.
+
 ## Functional tests
 
-The functional tests reside under the `tests` directory and are listed in `tests/local.mk`.
+The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`.
 Each test is a bash script.
 
 ### Running the whole test suite
@@ -21,8 +33,8 @@ The whole test suite can be run with:
 
 ```shell-session
 $ make install && make installcheck
-ran test tests/foo.sh... [PASS]
+ran test tests/functional/foo.sh... [PASS]
-ran test tests/bar.sh... [PASS]
+ran test tests/functional/bar.sh... [PASS]
 ...
 ```
 
@@ -30,14 +42,14 @@ ran test tests/bar.sh... [PASS]
 
 Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite.
 Each test group is in a subdirectory of `tests`.
-For example, `tests/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs.
+For example, `tests/functional/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs.
 
 That test group can be run like this:
 
 ```shell-session
 $ make ca.test-group -j50
-ran test tests/ca/nix-run.sh... [PASS]
+ran test tests/functional/ca/nix-run.sh... [PASS]
-ran test tests/ca/import-derivation.sh... [PASS]
+ran test tests/functional/ca/import-derivation.sh... [PASS]
 ...
 ```
 
@@ -56,21 +68,21 @@ install-tests-groups += $(test-group-name)
 Individual tests can be run with `make`:
 
 ```shell-session
-$ make tests/${testName}.sh.test
+$ make tests/functional/${testName}.sh.test
-ran test tests/${testName}.sh... [PASS]
+ran test tests/functional/${testName}.sh... [PASS]
 ```
 
 or without `make`:
 
 ```shell-session
-$ ./mk/run-test.sh tests/${testName}.sh
+$ ./mk/run-test.sh tests/functional/${testName}.sh
-ran test tests/${testName}.sh... [PASS]
+ran test tests/functional/${testName}.sh... [PASS]
 ```
 
 To see the complete output, one can also run:
 
 ```shell-session
-$ ./mk/debug-test.sh tests/${testName}.sh
+$ ./mk/debug-test.sh tests/functional/${testName}.sh
 + foo
 output from foo
 + bar
@@ -105,7 +117,7 @@ edit it like so:
 Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point:
 
 ```shell-session
-$ ./mk/debug-test.sh tests/${testName}.sh
+$ ./mk/debug-test.sh tests/functional/${testName}.sh
 ...
 + gdb blash blub
 GNU gdb (GDB) 12.1
@@ -124,9 +136,11 @@ This technique is to include the exact output/behavior of a former version of Ni
 For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered.
 
 It is frequently useful to regenerate the expected output.
-To do that, rerun the failed test with `_NIX_TEST_ACCEPT=1`.
+To do that, rerun the failed test(s) with `_NIX_TEST_ACCEPT=1`.
-(At least, this is the convention we've used for `tests/lang.sh`.
+For example:
-If we add more characterization testing we should always strive to be consistent.)
+```bash
+_NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test
+```
 
 An interesting situation to document is the case when these tests are "overfitted".
 The language tests are, again, an example of this.

doc/manual/utils.nix

@@ -44,10 +44,10 @@ rec {
 
   optionalString = cond: string: if cond then string else "";
 
-  showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }:
+  showSetting = { inlineHTML }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }:
     let
       result = squash ''
-          - ${if useAnchors
+          - ${if inlineHTML
               then ''<span id="conf-${name}">[`${name}`](#conf-${name})</span>''
               else ''`${name}`''}
 

flake.lock

@@ -34,16 +34,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1695124524,
+        "lastModified": 1700748986,
-        "narHash": "sha256-trXDytVCqf3KryQQQrHOZKUabu1/lB8/ndOAuZKQrOE=",
+        "narHash": "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds=",
-        "owner": "edolstra",
+        "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "a3d30b525535e3158221abc1a957ce798ab159fe",
+        "rev": "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0",
         "type": "github"
       },
       "original": {
-        "owner": "edolstra",
+        "owner": "NixOS",
-        "ref": "fix-aws-sdk-cpp",
+        "ref": "nixos-23.05-small",
         "repo": "nixpkgs",
         "type": "github"
       }

flake.nix

@@ -1,8 +1,7 @@
 {
   description = "The purely functional package manager";
 
-  #inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
-  inputs.nixpkgs.url = "github:edolstra/nixpkgs/fix-aws-sdk-cpp";
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
@@ -12,7 +11,7 @@
     let
       inherit (nixpkgs) lib;
 
-      officialRelease = false;
+      officialRelease = true;
 
       version = lib.fileContents ./.version + versionSuffix;
       versionSuffix =
@@ -59,35 +58,28 @@
 
       nixSrc = fileset.toSource {
         root = ./.;
-        fileset = fileset.intersect baseFiles (
+        fileset = fileset.intersect baseFiles (fileset.unions [
-          fileset.difference
+          ./.version
-            (fileset.unions [
+          ./boehmgc-coroutine-sp-fallback.diff
-              ./.version
+          ./bootstrap.sh
-              ./boehmgc-coroutine-sp-fallback.diff
+          ./configure.ac
-              ./bootstrap.sh
+          ./doc
-              ./configure.ac
+          ./local.mk
-              ./doc
+          ./m4
-              ./local.mk
+          ./Makefile
-              ./m4
+          ./Makefile.config.in
-              ./Makefile
+          ./misc
-              ./Makefile.config.in
+          ./mk
-              ./misc
+          ./precompiled-headers.h
-              ./mk
+          ./src
-              ./precompiled-headers.h
+          ./tests/functional
-              ./src
+          ./tests/unit
-              ./tests
+          ./COPYING
-              ./COPYING
+          ./scripts/local.mk
-              ./scripts/local.mk
+          (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
-              (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
+          # TODO: do we really need README.md? It doesn't seem used in the build.
-              # TODO: do we really need README.md? It doesn't seem used in the build.
+          ./README.md
-              ./README.md
+        ]);
-            ])
-            (fileset.unions [
-              # Removed file sets
-              ./tests/nixos
-              ./tests/installer
-            ])
-        );
       };
 
       # Memoize nixpkgs for different platforms for efficiency.

mk/common-test.sh

@@ -1,11 +1,15 @@
+test_dir=tests/functional
+
+test=$(echo -n "$test" | sed -e "s|^$test_dir/||")
+
 TESTS_ENVIRONMENT=("TEST_NAME=${test%.*}" 'NIX_REMOTE=')
 
 : ${BASH:=/usr/bin/env bash}
 
 init_test () {
-   cd tests && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null
+   cd "$test_dir" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null
 }
 
 run_test_proper () {
-   cd $(dirname $test) && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test)
+   cd "$test_dir/$(dirname $test)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test)
 }

src/libexpr/primops.cc

@@ -1536,7 +1536,9 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args,
     auto path = realisePath(state, pos, arg, { .checkForPureEval = false });
 
     /* SourcePath doesn't know about trailing slash. */
-    auto mustBeDir = arg.type() == nString && arg.str().ends_with("/");
+    auto mustBeDir = arg.type() == nString
+        && (arg.str().ends_with("/")
+            || arg.str().ends_with("/."));
 
     try {
         auto checked = state.checkSourcePath(path);

src/libexpr/primops/fetchTree.cc

@@ -395,7 +395,8 @@ static RegisterPrimOp primop_fetchGit({
 
       - `shallow` (default: `false`)
 
-        A Boolean parameter that specifies whether fetching a shallow clone is allowed.
+        A Boolean parameter that specifies whether fetching from a shallow remote repository is allowed.
+        This still performs a full clone of what is available on the remote.
 
       - `allRefs`
 

src/libexpr/tests/local.mk

@@ -1,19 +0,0 @@
-check: libexpr-tests_RUN
-
-programs += libexpr-tests
-
-libexpr-tests_NAME := libnixexpr-tests
-
-libexpr-tests_DIR := $(d)
-
-libexpr-tests_INSTALL_DIR :=
-
-libexpr-tests_SOURCES := \
-    $(wildcard $(d)/*.cc) \
-    $(wildcard $(d)/value/*.cc)
-
-libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -I src/libfetchers
-
-libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers
-
-libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock

src/libstore/build/create-derivation-and-realise-goal.cc

@@ -1,157 +0,0 @@
-#include "create-derivation-and-realise-goal.hh"
-#include "worker.hh"
-
-namespace nix {
-
-CreateDerivationAndRealiseGoal::CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
-    const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker, DerivedPath::Built { .drvPath = drvReq, .outputs = wantedOutputs })
-    , drvReq(drvReq)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    state = &CreateDerivationAndRealiseGoal::getDerivation;
-    name = fmt(
-        "outer obtaining drv from '%s' and then building outputs %s",
-        drvReq->to_string(worker.store),
-        std::visit(overloaded {
-            [&](const OutputsSpec::All) -> std::string {
-                return "* (all of them)";
-            },
-            [&](const OutputsSpec::Names os) {
-                return concatStringsSep(", ", quoteStrings(os));
-            },
-        }, wantedOutputs.raw));
-    trace("created outer");
-
-    worker.updateProgress();
-}
-
-
-CreateDerivationAndRealiseGoal::~CreateDerivationAndRealiseGoal()
-{
-}
-
-
-static StorePath pathPartOfReq(const SingleDerivedPath & req)
-{
-    return std::visit(overloaded {
-        [&](const SingleDerivedPath::Opaque & bo) {
-            return bo.path;
-        },
-        [&](const SingleDerivedPath::Built & bfd) {
-            return pathPartOfReq(*bfd.drvPath);
-        },
-    }, req.raw());
-}
-
-
-std::string CreateDerivationAndRealiseGoal::key()
-{
-    /* Ensure that derivations get built in order of their name,
-       i.e. a derivation named "aardvark" always comes before "baboon". And
-       substitution goals and inner derivation goals always happen before
-       derivation goals (due to "b$"). */
-    return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store);
-}
-
-
-void CreateDerivationAndRealiseGoal::timedOut(Error && ex)
-{
-}
-
-
-void CreateDerivationAndRealiseGoal::work()
-{
-    (this->*state)();
-}
-
-
-void CreateDerivationAndRealiseGoal::addWantedOutputs(const OutputsSpec & outputs)
-{
-    /* If we already want all outputs, there is nothing to do. */
-    auto newWanted = wantedOutputs.union_(outputs);
-    bool needRestart = !newWanted.isSubsetOf(wantedOutputs);
-    wantedOutputs = newWanted;
-
-    if (!needRestart) return;
-
-    if (!optDrvPath)
-        // haven't started steps where the outputs matter yet
-        return;
-    worker.makeDerivationGoal(*optDrvPath, outputs, buildMode);
-}
-
-
-void CreateDerivationAndRealiseGoal::getDerivation()
-{
-    trace("outer init");
-
-    /* The first thing to do is to make sure that the derivation
-       exists.  If it doesn't, it may be created through a
-       substitute. */
-    if (auto optDrvPath = [this]() -> std::optional<StorePath> {
-        if (buildMode != bmNormal) return std::nullopt;
-
-        auto drvPath = StorePath::dummy;
-        try {
-            drvPath = resolveDerivedPath(worker.store, *drvReq);
-        } catch (MissingRealisation &) {
-            return std::nullopt;
-        }
-        return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath)
-            ? std::optional { drvPath }
-            : std::nullopt;
-    }()) {
-        trace(fmt("already have drv '%s' for '%s', can go straight to building",
-            worker.store.printStorePath(*optDrvPath),
-            drvReq->to_string(worker.store)));
-
-        loadAndBuildDerivation();
-    } else {
-        trace("need to obtain drv we want to build");
-
-        addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq)));
-
-        state = &CreateDerivationAndRealiseGoal::loadAndBuildDerivation;
-        if (waitees.empty()) work();
-    }
-}
-
-
-void CreateDerivationAndRealiseGoal::loadAndBuildDerivation()
-{
-    trace("outer load and build derivation");
-
-    if (nrFailed != 0) {
-        amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
-        return;
-    }
-
-    StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
-    /* Build this step! */
-    concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode);
-    addWaitee(upcast_goal(concreteDrvGoal));
-    state = &CreateDerivationAndRealiseGoal::buildDone;
-    optDrvPath = std::move(drvPath);
-    if (waitees.empty()) work();
-}
-
-
-void CreateDerivationAndRealiseGoal::buildDone()
-{
-    trace("outer build done");
-
-    buildResult = upcast_goal(concreteDrvGoal)->getBuildResult(DerivedPath::Built {
-        .drvPath = drvReq,
-        .outputs = wantedOutputs,
-    });
-
-    if (buildResult.success())
-        amDone(ecSuccess);
-    else
-        amDone(ecFailed, Error("building '%s' failed", drvReq->to_string(worker.store)));
-}
-
-
-}

src/libstore/build/create-derivation-and-realise-goal.hh

@@ -1,96 +0,0 @@
-#pragma once
-
-#include "parsed-derivations.hh"
-#include "lock.hh"
-#include "store-api.hh"
-#include "pathlocks.hh"
-#include "goal.hh"
-
-namespace nix {
-
-struct DerivationGoal;
-
-/**
- * This goal type is essentially the serial composition (like function
- * composition) of a goal for getting a derivation, and then a
- * `DerivationGoal` using the newly-obtained derivation.
- *
- * In the (currently experimental) general inductive case of derivations
- * that are themselves build outputs, that first goal will be *another*
- * `CreateDerivationAndRealiseGoal`. In the (much more common) base-case
- * where the derivation has no provence and is just referred to by
- * (content-addressed) store path, that first goal is a
- * `SubstitutionGoal`.
- *
- * If we already have the derivation (e.g. if the evalutator has created
- * the derivation locally and then instructured the store to build it),
- * we can skip the first goal entirely as a small optimization.
- */
-struct CreateDerivationAndRealiseGoal : public Goal
-{
-    /**
-     * How to obtain a store path of the derivation to build.
-     */
-    ref<SingleDerivedPath> drvReq;
-
-    /**
-     * The path of the derivation, once obtained.
-     **/
-    std::optional<StorePath> optDrvPath;
-
-    /**
-     * The goal for the corresponding concrete derivation.
-     **/
-    std::shared_ptr<DerivationGoal> concreteDrvGoal;
-
-    /**
-     * The specific outputs that we need to build.
-     */
-    OutputsSpec wantedOutputs;
-
-    typedef void (CreateDerivationAndRealiseGoal::*GoalState)();
-    GoalState state;
-
-    /**
-     * The final output paths of the build.
-     *
-     * - For input-addressed derivations, always the precomputed paths
-     *
-     * - For content-addressed derivations, calcuated from whatever the
-     *   hash ends up being. (Note that fixed outputs derivations that
-     *   produce the "wrong" output still install that data under its
-     *   true content-address.)
-     */
-    OutputPathMap finalOutputs;
-
-    BuildMode buildMode;
-
-    CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
-        const OutputsSpec & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    virtual ~CreateDerivationAndRealiseGoal();
-
-    void timedOut(Error && ex) override;
-
-    std::string key() override;
-
-    void work() override;
-
-    /**
-     * Add wanted outputs to an already existing derivation goal.
-     */
-    void addWantedOutputs(const OutputsSpec & outputs);
-
-    /**
-     * The states.
-     */
-    void getDerivation();
-    void loadAndBuildDerivation();
-    void buildDone();
-
-    JobCategory jobCategory() const override {
-        return JobCategory::Administration;
-    };
-};
-
-}

src/libstore/build/derivation-goal.cc

@@ -71,7 +71,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
     , wantedOutputs(wantedOutputs)
     , buildMode(buildMode)
 {
-    state = &DerivationGoal::loadDerivation;
+    state = &DerivationGoal::getDerivation;
     name = fmt(
         "building of '%s' from .drv file",
         DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store));
@@ -164,6 +164,24 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
 }
 
 
+void DerivationGoal::getDerivation()
+{
+    trace("init");
+
+    /* The first thing to do is to make sure that the derivation
+       exists.  If it doesn't, it may be created through a
+       substitute. */
+    if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) {
+        loadDerivation();
+        return;
+    }
+
+    addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
+
+    state = &DerivationGoal::loadDerivation;
+}
+
+
 void DerivationGoal::loadDerivation()
 {
     trace("loading derivation");
@@ -1498,24 +1516,23 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
     if (!useDerivation) return;
     auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
 
-    std::optional info = tryGetConcreteDrvGoal(waitee);
+    auto * dg = dynamic_cast<DerivationGoal *>(&*waitee);
-    if (!info) return;
+    if (!dg) return;
-    const auto & [dg, drvReq] = *info;
 
-    auto * nodeP = fullDrv.inputDrvs.findSlot(drvReq.get());
+    auto * nodeP = fullDrv.inputDrvs.findSlot(DerivedPath::Opaque { .path = dg->drvPath });
     if (!nodeP) return;
     auto & outputs = nodeP->value;
 
     for (auto & outputName : outputs) {
-        auto buildResult = dg.get().getBuildResult(DerivedPath::Built {
+        auto buildResult = dg->getBuildResult(DerivedPath::Built {
-            .drvPath = makeConstantStorePathRef(dg.get().drvPath),
+            .drvPath = makeConstantStorePathRef(dg->drvPath),
             .outputs = OutputsSpec::Names { outputName },
         });
         if (buildResult.success()) {
             auto i = buildResult.builtOutputs.find(outputName);
             if (i != buildResult.builtOutputs.end())
                 inputDrvOutputs.insert_or_assign(
-                    { dg.get().drvPath, outputName },
+                    { dg->drvPath, outputName },
                     i->second.outPath);
         }
     }

src/libstore/build/derivation-goal.hh

@@ -52,10 +52,6 @@ struct InitialOutput {
 
 /**
  * A goal for building some or all of the outputs of a derivation.
- *
- * The derivation must already be present, either in the store in a drv
- * or in memory. If the derivation itself needs to be gotten first, a
- * `CreateDerivationAndRealiseGoal` goal must be used instead.
  */
 struct DerivationGoal : public Goal
 {
@@ -235,6 +231,7 @@ struct DerivationGoal : public Goal
     /**
      * The states.
      */
+    void getDerivation();
     void loadDerivation();
     void haveDerivation();
     void outputsSubstitutionTried();

src/libstore/build/entry-points.cc

@@ -1,6 +1,5 @@
 #include "worker.hh"
 #include "substitution-goal.hh"
-#include "create-derivation-and-realise-goal.hh"
 #include "derivation-goal.hh"
 #include "local-store.hh"
 
@@ -16,7 +15,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
 
     worker.run(goals);
 
-    StringSet failed;
+    StorePathSet failed;
     std::optional<Error> ex;
     for (auto & i : goals) {
         if (i->ex) {
@@ -26,10 +25,10 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
                 ex = std::move(i->ex);
         }
         if (i->exitCode != Goal::ecSuccess) {
-            if (auto i2 = dynamic_cast<CreateDerivationAndRealiseGoal *>(i.get()))
+            if (auto i2 = dynamic_cast<DerivationGoal *>(i.get()))
-                failed.insert(i2->drvReq->to_string(*this));
+                failed.insert(i2->drvPath);
             else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
-                failed.insert(printStorePath(i2->storePath));
+                failed.insert(i2->storePath);
         }
     }
 
@@ -38,7 +37,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
         throw std::move(*ex);
     } else if (!failed.empty()) {
         if (ex) logError(ex->info());
-        throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed)));
+        throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed));
     }
 }
 

src/libstore/build/goal.hh

@@ -49,16 +49,6 @@ enum struct JobCategory {
      * A substitution an arbitrary store object; it will use network resources.
      */
     Substitution,
-    /**
-     * A goal that does no "real" work by itself, and just exists to depend on
-     * other goals which *do* do real work. These goals therefore are not
-     * limited.
-     *
-     * These goals cannot infinitely create themselves, so there is no risk of
-     * a "fork bomb" type situation (which would be a problem even though the
-     * goal do no real work) either.
-     */
-    Administration,
 };
 
 struct Goal : public std::enable_shared_from_this<Goal>

src/libstore/build/worker.cc

@@ -2,7 +2,6 @@
 #include "worker.hh"
 #include "substitution-goal.hh"
 #include "drv-output-substitution-goal.hh"
-#include "create-derivation-and-realise-goal.hh"
 #include "local-derivation-goal.hh"
 #include "hook-instance.hh"
 
@@ -42,24 +41,6 @@ Worker::~Worker()
 }
 
 
-std::shared_ptr<CreateDerivationAndRealiseGoal> Worker::makeCreateDerivationAndRealiseGoal(
-    ref<SingleDerivedPath> drvReq,
-    const OutputsSpec & wantedOutputs,
-    BuildMode buildMode)
-{
-    std::weak_ptr<CreateDerivationAndRealiseGoal> & goal_weak = outerDerivationGoals.ensureSlot(*drvReq).value;
-    std::shared_ptr<CreateDerivationAndRealiseGoal> goal = goal_weak.lock();
-    if (!goal) {
-        goal = std::make_shared<CreateDerivationAndRealiseGoal>(drvReq, wantedOutputs, *this, buildMode);
-        goal_weak = goal;
-        wakeUp(goal);
-    } else {
-        goal->addWantedOutputs(wantedOutputs);
-    }
-    return goal;
-}
-
-
 std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
     const StorePath & drvPath,
     const OutputsSpec & wantedOutputs,
@@ -130,7 +111,10 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
 {
     return std::visit(overloaded {
         [&](const DerivedPath::Built & bfd) -> GoalPtr {
-            return makeCreateDerivationAndRealiseGoal(bfd.drvPath, bfd.outputs, buildMode);
+            if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath))
+                return makeDerivationGoal(bop->path, bfd.outputs, buildMode);
+            else
+                throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented.");
         },
         [&](const DerivedPath::Opaque & bo) -> GoalPtr {
             return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
@@ -139,46 +123,24 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
 }
 
 
-template<typename K, typename V, typename F>
-static void cullMap(std::map<K, V> & goalMap, F f)
-{
-    for (auto i = goalMap.begin(); i != goalMap.end();)
-        if (!f(i->second))
-            i = goalMap.erase(i);
-        else ++i;
-}
-
-
 template<typename K, typename G>
 static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
 {
     /* !!! inefficient */
-    cullMap(goalMap, [&](const std::weak_ptr<G> & gp) -> bool {
+    for (auto i = goalMap.begin();
-        return gp.lock() != goal;
+         i != goalMap.end(); )
-    });
+        if (i->second.lock() == goal) {
-}
+            auto j = i; ++j;
-
+            goalMap.erase(i);
-template<typename K>
+            i = j;
-static void removeGoal(std::shared_ptr<CreateDerivationAndRealiseGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode> & goalMap);
+        }
-
+        else ++i;
-template<typename K>
-static void removeGoal(std::shared_ptr<CreateDerivationAndRealiseGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode> & goalMap)
-{
-    /* !!! inefficient */
-    cullMap(goalMap, [&](DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode & node) -> bool {
-        if (node.value.lock() == goal)
-            node.value.reset();
-        removeGoal(goal, node.childMap);
-        return !node.value.expired() || !node.childMap.empty();
-    });
 }
 
 
 void Worker::removeGoal(GoalPtr goal)
 {
-    if (auto drvGoal = std::dynamic_pointer_cast<CreateDerivationAndRealiseGoal>(goal))
+    if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
-        nix::removeGoal(drvGoal, outerDerivationGoals.map);
-    else if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
         nix::removeGoal(drvGoal, derivationGoals);
     else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
         nix::removeGoal(subGoal, substitutionGoals);
@@ -236,19 +198,8 @@ void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
     child.respectTimeouts = respectTimeouts;
     children.emplace_back(child);
     if (inBuildSlot) {
-        switch (goal->jobCategory()) {
+        if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++;
-        case JobCategory::Substitution:
+        else nrLocalBuilds++;
-            nrSubstitutions++;
-            break;
-        case JobCategory::Build:
-            nrLocalBuilds++;
-            break;
-        case JobCategory::Administration:
-            /* Intentionally not limited, see docs */
-            break;
-        default:
-            abort();
-        }
     }
 }
 
@@ -260,20 +211,12 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
     if (i == children.end()) return;
 
     if (i->inBuildSlot) {
-        switch (goal->jobCategory()) {
+        if (goal->jobCategory() == JobCategory::Substitution) {
-        case JobCategory::Substitution:
             assert(nrSubstitutions > 0);
             nrSubstitutions--;
-            break;
+        } else {
-        case JobCategory::Build:
             assert(nrLocalBuilds > 0);
             nrLocalBuilds--;
-            break;
-        case JobCategory::Administration:
-            /* Intentionally not limited, see docs */
-            break;
-        default:
-            abort();
         }
     }
 
@@ -324,9 +267,9 @@ void Worker::run(const Goals & _topGoals)
 
     for (auto & i : _topGoals) {
         topGoals.insert(i);
-        if (auto goal = dynamic_cast<CreateDerivationAndRealiseGoal *>(i.get())) {
+        if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
             topPaths.push_back(DerivedPath::Built {
-                .drvPath = goal->drvReq,
+                .drvPath = makeConstantStorePathRef(goal->drvPath),
                 .outputs = goal->wantedOutputs,
             });
         } else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
@@ -589,19 +532,4 @@ GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal)
     return subGoal;
 }
 
-GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal)
-{
-    return subGoal;
-}
-
-std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee)
-{
-    auto * odg = dynamic_cast<CreateDerivationAndRealiseGoal *>(&*waitee);
-    if (!odg) return std::nullopt;
-    return {{
-        std::cref(*odg->concreteDrvGoal),
-        std::cref(*odg->drvReq),
-    }};
-}
-
 }

src/libstore/build/worker.hh

@@ -4,7 +4,6 @@
 #include "types.hh"
 #include "lock.hh"
 #include "store-api.hh"
-#include "derived-path-map.hh"
 #include "goal.hh"
 #include "realisation.hh"
 
@@ -14,7 +13,6 @@
 namespace nix {
 
 /* Forward definition. */
-struct CreateDerivationAndRealiseGoal;
 struct DerivationGoal;
 struct PathSubstitutionGoal;
 class DrvOutputSubstitutionGoal;
@@ -33,25 +31,9 @@ class DrvOutputSubstitutionGoal;
  */
 GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
 GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
-GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal);
 
 typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
 
-/**
- * The current implementation of impure derivations has
- * `DerivationGoal`s accumulate realisations from their waitees.
- * Unfortunately, `DerivationGoal`s don't directly depend on other
- * goals, but instead depend on `CreateDerivationAndRealiseGoal`s.
- *
- * We try not to share any of the details of any goal type with any
- * other, for sake of modularity and quicker rebuilds. This means we
- * cannot "just" downcast and fish out the field. So as an escape hatch,
- * we have made the function, written in `worker.cc` where all the goal
- * types are visible, and use it instead.
- */
-
-std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee);
-
 /**
  * A mapping used to remember for each child process to what goal it
  * belongs, and file descriptors for receiving log data and output
@@ -119,9 +101,6 @@ private:
      * Maps used to prevent multiple instantiations of a goal for the
      * same derivation / path.
      */
-
-    DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>> outerDerivationGoals;
-
     std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
     std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
     std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
@@ -209,9 +188,6 @@ public:
      * @ref DerivationGoal "derivation goal"
      */
 private:
-    std::shared_ptr<CreateDerivationAndRealiseGoal> makeCreateDerivationAndRealiseGoal(
-        ref<SingleDerivedPath> drvPath,
-        const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
     std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
         const StorePath & drvPath, const OutputsSpec & wantedOutputs,
         std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);

src/libstore/derived-path-map.cc

@@ -51,11 +51,8 @@ typename DerivedPathMap<V>::ChildNode * DerivedPathMap<V>::findSlot(const Single
 
 // instantiations
 
-#include "create-derivation-and-realise-goal.hh"
 namespace nix {
 
-template struct DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>;
-
 GENERATE_CMP_EXT(
     template<>,
     DerivedPathMap<std::set<std::string>>::ChildNode,

src/libstore/derived-path-map.hh

@@ -20,11 +20,8 @@ namespace nix {
  *
  * @param V A type to instantiate for each output. It should probably
  * should be an "optional" type so not every interior node has to have a
- * value. For example, the scheduler uses
+ * value. `* const Something` or `std::optional<Something>` would be
- * `DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>` to
+ * good choices for "optional" types.
- * remember which goals correspond to which outputs. `* const Something`
- * or `std::optional<Something>` would also be good choices for
- * "optional" types.
  */
 template<typename V>
 struct DerivedPathMap {

src/libstore/gc.cc

@@ -776,7 +776,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         }
     };
 
-    /* Synchronisation point for testing, see tests/gc-concurrent.sh. */
+    /* Synchronisation point for testing, see tests/functional/gc-concurrent.sh. */
     if (auto p = getEnv("_NIX_TEST_GC_SYNC"))
         readFile(*p);
 

src/libstore/globals.cc

@@ -24,6 +24,9 @@
 
 #include "config-impl.hh"
 
+#ifdef __APPLE__
+#include <sys/sysctl.h>
+#endif
 
 namespace nix {
 
@@ -154,6 +157,29 @@ unsigned int Settings::getDefaultCores()
       return concurrency;
 }
 
+#if __APPLE__
+static bool hasVirt() {
+
+    int hasVMM;
+    int hvSupport;
+    size_t size;
+
+    size = sizeof(hasVMM);
+    if (sysctlbyname("kern.hv_vmm_present", &hasVMM, &size, NULL, 0) == 0) {
+        if (hasVMM)
+            return false;
+    }
+
+    // whether the kernel and hardware supports virt
+    size = sizeof(hvSupport);
+    if (sysctlbyname("kern.hv_support", &hvSupport, &size, NULL, 0) == 0) {
+        return hvSupport == 1;
+    } else {
+        return false;
+    }
+}
+#endif
+
 StringSet Settings::getDefaultSystemFeatures()
 {
     /* For backwards compatibility, accept some "features" that are
@@ -170,6 +196,11 @@ StringSet Settings::getDefaultSystemFeatures()
         features.insert("kvm");
     #endif
 
+    #if __APPLE__
+    if (hasVirt())
+        features.insert("apple-virt");
+    #endif
+
     return features;
 }
 

src/libstore/globals.hh

@@ -708,6 +708,7 @@ public:
           `kvm` feature.
 
           This setting by default includes `kvm` if `/dev/kvm` is accessible,
+          `apple-virt` if hardware virtualization is available on macOS,
           and the pseudo-features `nixos-test`, `benchmark` and `big-parallel`
           that are used in Nixpkgs to route builds to specific machines.
         )", {}, false};

src/libstore/path-regex.hh

@@ -3,6 +3,6 @@
 
 namespace nix {
 
-static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-\._\?=]+)";
+static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-_\?=][0-9a-zA-Z\+\-\._\?=]*)";
 
 }

src/libstore/path.cc

@@ -11,6 +11,8 @@ static void checkName(std::string_view path, std::string_view name)
     if (name.size() > StorePath::MaxPathLen)
         throw BadStorePath("store path '%s' has a name longer than %d characters",
             path, StorePath::MaxPathLen);
+    if (name[0] == '.')
+        throw BadStorePath("store path '%s' starts with illegal character '.'", path);
     // See nameRegexStr for the definition
     for (auto c : name)
         if (!((c >= '0' && c <= '9')

src/libstore/tests/local.mk

@@ -1,29 +0,0 @@
-check: libstore-tests-exe_RUN
-
-programs += libstore-tests-exe
-
-libstore-tests-exe_NAME = libnixstore-tests
-
-libstore-tests-exe_DIR := $(d)
-
-libstore-tests-exe_INSTALL_DIR :=
-
-libstore-tests-exe_LIBS = libstore-tests
-
-libstore-tests-exe_LDFLAGS := $(GTEST_LIBS)
-
-libraries += libstore-tests
-
-libstore-tests_NAME = libnixstore-tests
-
-libstore-tests_DIR := $(d)
-
-libstore-tests_INSTALL_DIR :=
-
-libstore-tests_SOURCES := $(wildcard $(d)/*.cc)
-
-libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil
-
-libstore-tests_LIBS = libutil-tests libstore libutil
-
-libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)

src/libutil/tests/local.mk

@@ -1,29 +0,0 @@
-check: libutil-tests_RUN
-
-programs += libutil-tests
-
-libutil-tests-exe_NAME = libnixutil-tests
-
-libutil-tests-exe_DIR := $(d)
-
-libutil-tests-exe_INSTALL_DIR :=
-
-libutil-tests-exe_LIBS = libutil-tests
-
-libutil-tests-exe_LDFLAGS := $(GTEST_LIBS)
-
-libraries += libutil-tests
-
-libutil-tests_NAME = libnixutil-tests
-
-libutil-tests_DIR := $(d)
-
-libutil-tests_INSTALL_DIR :=
-
-libutil-tests_SOURCES := $(wildcard $(d)/*.cc)
-
-libutil-tests_CXXFLAGS += -I src/libutil
-
-libutil-tests_LIBS = libutil
-
-libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)

src/libutil/url-parts.hh

@@ -30,7 +30,7 @@ extern std::regex refRegex;
 
 /// Instead of defining what a good Git Ref is, we define what a bad Git Ref is
 /// This is because of the definition of a ref in refs.c in https://github.com/git/git
-/// See tests/fetchGitRefs.sh for the full definition
+/// See tests/functional/fetchGitRefs.sh for the full definition
 const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$";
 extern std::regex badGitRefRegex;
 

src/nix/main.cc

@@ -218,7 +218,8 @@ static void showHelp(std::vector<std::string> subcommand, NixArgs & toplevel)
     vDump->mkString(toplevel.dumpCli());
 
     auto vRes = state.allocValue();
-    state.callFunction(*vGenerateManpage, *vDump, *vRes, noPos);
+    state.callFunction(*vGenerateManpage, state.getBuiltin("false"), *vRes, noPos);
+    state.callFunction(*vRes, *vDump, *vRes, noPos);
 
     auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md"));
     if (!attr)

tests/dyn-drv/dep-built-drv.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-source common.sh
-
-out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link)
-
-clearStore
-
-out2=$(nix-build ./text-hashed-output.nix -A wrapper --no-out-link)
-
-diff -r $out1 $out2

tests/functional/bash-profile.sh

@@ -1,6 +1,6 @@
 source common.sh
 
-sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh
+sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh
 
 user=$(whoami)
 rm -rf $TEST_HOME $TEST_ROOT/profile-var

tests/functional/build-remote-trustless.sh

@@ -6,7 +6,7 @@ unset NIX_STATE_DIR
 
 remoteDir=$TEST_ROOT/remote
 
-# Note: ssh{-ng}://localhost bypasses ssh. See tests/build-remote.sh for
+# Note: ssh{-ng}://localhost bypasses ssh. See tests/functional/build-remote.sh for
 # more details.
 nix-build $file -o $TEST_ROOT/result --max-jobs 0 \
   --arg busybox $busybox \

tests/functional/ca/local.mk

@@ -25,4 +25,4 @@ clean-files += \
   $(d)/config.nix
 
 test-deps += \
-  tests/ca/config.nix
+  tests/functional/ca/config.nix