Merge remote-tracking branch 'upstream/master' into ca-drv-exotic

.github/PULL_REQUEST_TEMPLATE.md

@@ -23,6 +23,7 @@ Maintainers: tick if completed or explain if not relevant
    - unit tests - `src/*/tests`
    - integration tests - `tests/nixos/*`
  - [ ] documentation in the manual
+ - [ ] documentation in the internal API docs
  - [ ] code and comments are self-explanatory
  - [ ] commit message explains why the change was made
  - [ ] new feature or incompatible change: updated release notes

.github/labeler.yml

@@ -2,5 +2,22 @@
   - doc/manual/*
   - src/nix/**/*.md
 
+"store":
+  - src/libstore/store-api.*
+  - src/libstore/*-store.*
+
+"fetching":
+  - src/libfetchers/**/*
+
+"repl":
+  - src/libcmd/repl.*
+  - src/nix/repl.*
+
+"new-cli":
+  - src/nix/**/*
+
 "tests":
+  # Unit tests
+  - src/*/tests/**/*
+  # Functional and integration tests
   - tests/**/*

.gitignore

@@ -19,9 +19,12 @@ perl/Makefile.config
 /doc/manual/nix.json
 /doc/manual/conf-file.json
 /doc/manual/builtins.json
+/doc/manual/xp-features.json
 /doc/manual/src/SUMMARY.md
 /doc/manual/src/command-ref/new-cli
 /doc/manual/src/command-ref/conf-file.md
+/doc/manual/src/command-ref/experimental-features-shortlist.md
+/doc/manual/src/contributing/experimental-feature-descriptions.md
 /doc/manual/src/language/builtins.md
 
 # /scripts/

.version

@@ -1 +1 @@
-2.15.0
+2.16.0

boehmgc-coroutine-sp-fallback.diff

@@ -1,8 +1,8 @@
 diff --git a/darwin_stop_world.c b/darwin_stop_world.c
-index 3dbaa3fb..36a1d1f7 100644
+index 0468aaec..b348d869 100644
 --- a/darwin_stop_world.c
 +++ b/darwin_stop_world.c
-@@ -352,6 +352,7 @@ GC_INNER void GC_push_all_stacks(void)
+@@ -356,6 +356,7 @@ GC_INNER void GC_push_all_stacks(void)
    int nthreads = 0;
    word total_size = 0;
    mach_msg_type_number_t listcount = (mach_msg_type_number_t)THREAD_TABLE_SZ;
@@ -10,7 +10,7 @@ index 3dbaa3fb..36a1d1f7 100644
    if (!EXPECT(GC_thr_initialized, TRUE))
      GC_thr_init();
  
-@@ -407,6 +408,19 @@ GC_INNER void GC_push_all_stacks(void)
+@@ -411,6 +412,19 @@ GC_INNER void GC_push_all_stacks(void)
              GC_push_all_stack_sections(lo, hi, p->traced_stack_sect);
            }
            if (altstack_lo) {
@@ -30,6 +30,22 @@ index 3dbaa3fb..36a1d1f7 100644
              total_size += altstack_hi - altstack_lo;
              GC_push_all_stack(altstack_lo, altstack_hi);
            }
+diff --git a/include/gc.h b/include/gc.h
+index edab6c22..f2c61282 100644
+--- a/include/gc.h
++++ b/include/gc.h
+@@ -2172,6 +2172,11 @@ GC_API void GC_CALL GC_win32_free_heap(void);
+         (*GC_amiga_allocwrapper_do)(a,GC_malloc_atomic_ignore_off_page)
+ #endif /* _AMIGA && !GC_AMIGA_MAKINGLIB */
+ 
++#if !__APPLE__
++/* Patch doesn't work on apple */
++#define NIX_BOEHM_PATCH_VERSION 1
++#endif
++
+ #ifdef __cplusplus
+   } /* extern "C" */
+ #endif
 diff --git a/pthread_stop_world.c b/pthread_stop_world.c
 index b5d71e62..aed7b0bf 100644
 --- a/pthread_stop_world.c

configure.ac

@@ -184,7 +184,7 @@ fi
 
 # Look for OpenSSL, a required dependency. FIXME: this is only (maybe)
 # used by S3BinaryCacheStore.
-PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
+PKG_CHECK_MODULES([OPENSSL], [libcrypto >= 1.1.1], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
 
 
 # Look for libarchive.
@@ -289,13 +289,24 @@ PKG_CHECK_MODULES([GTEST], [gtest_main])
 
 
 # Look for rapidcheck.
+AC_ARG_VAR([RAPIDCHECK_HEADERS], [include path of gtest headers shipped by RAPIDCHECK])
 # No pkg-config yet, https://github.com/emil-e/rapidcheck/issues/302
 AC_LANG_PUSH(C++)
 AC_SUBST(RAPIDCHECK_HEADERS)
 [CXXFLAGS="-I $RAPIDCHECK_HEADERS $CXXFLAGS"]
+[LIBS="-lrapidcheck -lgtest $LIBS"]
 AC_CHECK_HEADERS([rapidcheck/gtest.h], [], [], [#include <gtest/gtest.h>])
-dnl No good for C++ libs with mangled symbols
+dnl AC_CHECK_LIB doesn't work for C++ libs with mangled symbols
-dnl AC_CHECK_LIB([rapidcheck], [])
+AC_LINK_IFELSE([
+    AC_LANG_PROGRAM([[
+      #include <gtest/gtest.h>
+      #include <rapidcheck/gtest.h>
+    ]], [[
+      return RUN_ALL_TESTS();
+    ]])
+  ],
+  [],
+  [AC_MSG_ERROR([librapidcheck is not found.])])
 AC_LANG_POP(C++)
 
 fi

doc/manual/generate-manpage.nix

@@ -10,7 +10,9 @@ let
 
       result = ''
         > **Warning** \
-        > This program is **experimental** and its interface is subject to change.
+        > This program is
+        > [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command)
+        > and its interface is subject to change.
 
         # Name
 
@@ -29,19 +31,18 @@ let
 
       showSynopsis = command: args:
         let
-          showArgument = arg: "*${arg.label}*" + (if arg ? arity then "" else "...");
+          showArgument = arg: "*${arg.label}*" + optionalString (! arg ? arity) "...";
           arguments = concatStringsSep " " (map showArgument args);
         in ''
          `${command}` [*option*...] ${arguments}
         '';
 
-      maybeSubcommands = if details ? commands && details.commands != {}
+      maybeSubcommands = optionalString (details ? commands && details.commands != {})
-        then ''
+         ''
            where *subcommand* is one of the following:
 
            ${subcommands}
-         ''
+         '';
-        else "";
 
       subcommands = if length categories > 1
         then listCategories
@@ -63,12 +64,11 @@ let
         * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description}
       '';
 
-      maybeDocumentation =
+      maybeDocumentation = optionalString
-        if details ? doc
+        (details ? doc)
-        then replaceStrings ["@stores@"] [storeDocs] details.doc
+        (replaceStrings ["@stores@"] [storeDocs] details.doc);
-        else "";
 
-      maybeOptions = if details.flags == {} then "" else ''
+      maybeOptions = optionalString (details.flags != {}) ''
         # Options
 
         ${showOptions details.flags toplevel.flags}
@@ -78,15 +78,19 @@ let
         let
           allOptions = options // commonOptions;
           showCategory = cat: ''
-            ${if cat != "" then "**${cat}:**" else ""}
+            ${optionalString (cat != "") "**${cat}:**"}
 
             ${listOptions (filterAttrs (n: v: v.category == cat) allOptions)}
             '';
           listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts));
           showOption = name: option:
             let
-              shortName = if option ? shortName then "/ `-${option.shortName}`" else "";
+              shortName = optionalString
-              labels = if option ? labels then (concatStringsSep " " (map (s: "*${s}*") option.labels)) else "";
+                (option ? shortName)
+                ("/ `-${option.shortName}`");
+              labels = optionalString
+                (option ? labels)
+                (concatStringsSep " " (map (s: "*${s}*") option.labels));
             in trim ''
               - `--${name}` ${shortName} ${labels}
 

doc/manual/generate-xp-features-shortlist.nix

@@ -0,0 +1,9 @@
+with builtins;
+with import ./utils.nix;
+
+let
+  showExperimentalFeature = name: doc:
+    ''
+      - [`${name}`](@docroot@/contributing/experimental-features.md#xp-feature-${name})
+    '';
+in xps: indent "  " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps)))

doc/manual/generate-xp-features.nix

@@ -0,0 +1,11 @@
+with builtins;
+with import ./utils.nix;
+
+let
+  showExperimentalFeature = name: doc:
+    squash ''
+      ## [`${name}`]{#xp-feature-${name}}
+
+      ${doc}
+    '';
+in xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps)))

doc/manual/local.mk

@@ -81,19 +81,20 @@ $(d)/%.8: $(d)/src/command-ref/%.md
 $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
 	@printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp
 	@cat $^ >> $^.tmp
+	@$(call process-includes,$^,$^.tmp)
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
+$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
 	@cp $< $@
 	@$(call process-includes,$@,$@)
 
-$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
+$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
+$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
 	@mv $@.tmp $@
@@ -106,6 +107,20 @@ $(d)/conf-file.json: $(bindir)/nix
 	$(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp
 	@mv $@.tmp $@
 
+$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix
+	@rm -rf $@ $@.tmp
+	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))'
+	@mv $@.tmp $@
+
+$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix
+	@rm -rf $@ $@.tmp
+	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))'
+	@mv $@.tmp $@
+
+$(d)/xp-features.json: $(bindir)/nix
+	$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp
+	@mv $@.tmp $@
+
 $(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
 	@cat doc/manual/src/language/builtins-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
@@ -145,7 +160,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 	done
 	@touch $@
 
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \

doc/manual/src/SUMMARY.md.in

@@ -95,9 +95,11 @@
 - [Glossary](glossary.md)
 - [Contributing](contributing/contributing.md)
   - [Hacking](contributing/hacking.md)
+  - [Experimental Features](contributing/experimental-features.md)
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
   - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+  - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md)
   - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md)
   - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md)
   - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md)

doc/manual/src/command-ref/experimental-commands.md

@@ -1,6 +1,6 @@
 # Experimental Commands
 
-This section lists experimental commands.
+This section lists [experimental commands](@docroot@/contributing/experimental-features.md#xp-feature-nix-command).
 
 > **Warning**
 >

doc/manual/src/command-ref/nix-shell.md

@@ -120,7 +120,8 @@ shell in which to build it:
 ```console
 $ nix-shell '<nixpkgs>' -A pan
 [nix-shell]$ eval ${unpackPhase:-unpackPhase}
-[nix-shell]$ cd pan-*
+[nix-shell]$ cd $sourceRoot
+[nix-shell]$ eval ${patchPhase:-patchPhase}
 [nix-shell]$ eval ${configurePhase:-configurePhase}
 [nix-shell]$ eval ${buildPhase:-buildPhase}
 [nix-shell]$ ./pan/gui/pan

doc/manual/src/contributing/experimental-features.md

@@ -0,0 +1,95 @@
+This section describes the notion of *experimental features*, and how it fits into the big picture of the development of Nix.
+
+# What are experimental features?
+
+Experimental features are considered unstable, which means that they can be changed or removed at any time.
+Users must explicitly enable them by toggling the associated [experimental feature flags](@docroot@/command-ref/conf-file.md#conf-experimental-features).
+This allows accessing unstable functionality without unwittingly relying on it.
+
+Experimental feature flags were first introduced in [Nix 2.4](@docroot@/release-notes/rl-2.4.md).
+Before that, Nix did have experimental features, but they were not guarded by flags and were merely documented as unstable.
+This was a source of confusion and controversy.
+
+# When should a new feature be marked experimental?
+
+A change in the Nix codebase should be guarded by an experimental feature flag if it is considered likely to be reverted or adapted in a backwards-incompatible manner after gathering more experience with it in practice.
+
+Examples:
+
+- Changes to the Nix language, such as new built-ins, syntactic or semantic changes, etc.
+- Changes to the command-line interface
+
+# Lifecycle of an experimental feature
+
+Experimental features have to be treated on a case-by-case basis.
+However, the standard workflow for an experimental feature is as follows:
+
+- A new feature is implemented in a *pull request*
+  - It is guarded by an experimental feature flag that is disabled by default
+- The pull request is merged, the *experimental* feature ends up in a release
+    - Using the feature requires explicitly enabling it, signifying awareness of the potential risks
+    - Being experimental, the feature can still be changed arbitrarily
+- The feature can be *removed*
+  - The associated experimental feature flag is also removed
+- The feature can be declared *stable*
+  - The associated experimental feature flag is removed
+  - There should be enough evidence of users having tried the feature, such as feedback, fixed bugs, demonstrations of how it is put to use
+  - Maintainers must feel confident that:
+    - The feature is designed and implemented sensibly, that it is fit for purpose
+    - Potential interactions are well-understood
+    - Stabilising the feature will not incur an outsized maintenance burden in the future
+
+The following diagram illustrates the process:
+
+```
+                  .------.
+                  | idea |
+                  '------'
+                      |
+       discussion, design, implementation
+                      |
+                      |     .-------.
+                      |     |       |
+                      v     v       |
+               .--------------.  review
+               | pull request |     |
+               '--------------'     |
+                   |     ^  |       |
+                   |     |  '-------'
+               .---'     '----.
+               |              |
+             merge       user feedback,
+               |       (breaking) changes
+               |              |
+               '---.     .----'
+                   |     |
+                   v     |
+               +--------------+
+           .---| experimental |----.
+           |   +--------------+    |
+           |                       |
+decision to stabilise      decision against
+           |              keeping the feature
+           |                       |
+           v                       v
+       +--------+             +---------+
+       | stable |             | removed |
+       +--------+             +---------+
+```
+
+# Relation to the RFC process
+
+Experimental features and [RFCs](https://github.com/NixOS/rfcs/) both allow approaching substantial changes while minimizing the risk.
+However they serve different purposes:
+
+- An experimental feature enables developers to iterate on and deliver a new idea without committing to it or requiring a costly long-running fork.
+  It is primarily an issue of *implementation*, targeting Nix developers and early testers.
+- The goal of an RFC is to make explicit all the implications of a change:
+  Explain why it is wanted, which new use-cases it enables, which interface changes it requires, etc.
+  It is primarily an issue of *design* and *communication*, targeting the broader community.
+
+This means that experimental features and RFCs are orthogonal mechanisms, and can be used independently or together as needed.
+
+# Currently available experimental features
+
+{{#include ./experimental-feature-descriptions.md}}

doc/manual/src/glossary.md

@@ -15,7 +15,7 @@
 
     Example: `/nix/store/g946hcz4c8mdvq2g8vxx42z51qb71rvp-git-2.38.1.drv`
 
-    See [`nix show-derivation`](./command-ref/new-cli/nix3-show-derivation.md) (experimental) for displaying the contents of store derivations.
+    See [`nix derivation show`](./command-ref/new-cli/nix3-derivation-show.md) (experimental) for displaying the contents of store derivations.
 
     [store derivation]: #gloss-store-derivation
 
@@ -54,7 +54,7 @@
     invoked, the  Nix store can be  referred to
     as a "_local_" or a "_remote_" one:
 
-    + A *local  store* exists  on the filesystem of
+    + A [local store]{#gloss-local-store} exists on the filesystem of
       the machine where Nix is  invoked. You can use other
       local stores  by passing  the `--store` flag  to the
       `nix` command.  Local stores can be used for building derivations.
@@ -65,17 +65,17 @@
       served by the `nix-serve` Perl script.
 
     [store]: #gloss-store
+    [local store]: #gloss-local-store
 
   - [chroot store]{#gloss-chroot-store}\
-    A local store whose canonical path is anything other than `/nix/store`.
+    A [local store] whose canonical path is anything other than `/nix/store`.
 
   - [binary cache]{#gloss-binary-cache}\
     A *binary cache* is a Nix store which uses a different format: its
     metadata and signatures are kept in `.narinfo` files rather than in a
-    Nix database.  This different format simplifies serving store objects
+    [Nix database]. This different format simplifies serving store objects
     over the network, but cannot host builds. Examples of binary caches
-    include S3 buckets and the [NixOS binary
+    include S3 buckets and the [NixOS binary cache](https://cache.nixos.org).
-    cache](https://cache.nixos.org).
 
   - [store path]{#gloss-store-path}\
     The location of a [store object] in the file system, i.e., an
@@ -108,7 +108,7 @@
     [fixed-output derivations](#gloss-fixed-output-derivation).
 
   - [substitute]{#gloss-substitute}\
-    A substitute is a command invocation stored in the Nix database that
+    A substitute is a command invocation stored in the [Nix database] that
     describes how to build a store object, bypassing the normal build
     mechanism (i.e., derivations). Typically, the substitute builds the
     store object by downloading a pre-built version of the store object
@@ -127,6 +127,14 @@
     builder can rely on external inputs such as the network or the
     system time) but the Nix model assumes it.
 
+  - Nix database{#gloss-nix-database}\
+    An SQlite database to track [reference]s between [store object]s.
+    This is an implementation detail of the [local store].
+
+    Default location: `/nix/var/nix/db`.
+
+    [Nix database]: #gloss-nix-database
+
   - [Nix expression]{#gloss-nix-expression}\
     A high-level description of software packages and compositions
     thereof. Deploying software using Nix entails writing Nix
@@ -175,9 +183,9 @@
   - [validity]{#gloss-validity}\
     A store path is valid if all [store object]s in its [closure] can be read from the [store].
 
-    For a local store, this means:
+    For a [local store], this means:
     - The store path leads to an existing [store object] in that [store].
-    - The store path is listed in the Nix database as being valid.
+    - The store path is listed in the [Nix database] as being valid.
     - All paths in the store path's [closure] are valid.
 
     [validity]: #gloss-validity
@@ -217,3 +225,9 @@
     [string]: ./language/values.md#type-string
     [path]: ./language/values.md#type-path
     [attribute name]: ./language/values.md#attribute-set
+
+  - [experimental feature]{#gloss-experimental-feature}\
+    Not yet stabilized functionality guarded by named experimental feature flags.
+    These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting.
+
+    See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md).

doc/manual/src/language/advanced-attributes.md

@@ -208,12 +208,26 @@ Derivations can declare some infrequently used optional attributes.
     about converting to and from base-32 notation.)
 
   - [`__contentAddressed`]{#adv-attr-__contentAddressed}
-    If this **experimental** attribute is set to true, then the derivation
+    > **Warning**
+    > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md).
+    >
+    > To use this attribute, you must enable the
+    > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature.
+    > For example, in [nix.conf](../command-ref/conf-file.md) you could add:
+    >
+    > ```
+    > extra-experimental-features = ca-derivations
+    > ```
+
+    If this attribute is set to `true`, then the derivation
     outputs will be stored in a content-addressed location rather than the
     traditional input-addressed one.
-    This only has an effect if the `ca-derivations` experimental feature is enabled.
 
-    Setting this attribute also requires setting `outputHashMode` and `outputHashAlgo` like for *fixed-output derivations* (see above).
+    Setting this attribute also requires setting
+    [`outputHashMode`](#adv-attr-outputHashMode)
+    and
+    [`outputHashAlgo`](#adv-attr-outputHashAlgo)
+    like for *fixed-output derivations* (see above).
 
   - [`passAsFile`]{#adv-attr-passAsFile}\
     A list of names of attributes that should be passed via files rather
@@ -307,9 +321,11 @@ Derivations can declare some infrequently used optional attributes.
 
   - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\
     > **Warning**
-    > This is an experimental feature.
+    > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md).
     >
-    > To enable it, add the following to [nix.conf](../command-ref/conf-file.md):
+    > To use this attribute, you must enable the
+    > [`discard-references`](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) experimental feature.
+    > For example, in [nix.conf](../command-ref/conf-file.md) you could add:
     >
     > ```
     > extra-experimental-features = discard-references

doc/manual/src/quick-start.md

@@ -19,7 +19,7 @@ to subsequent chapters.
    channel:
 
    ```console
-   $ nix-env -qaP
+   $ nix-env --query --available --attr-path
    nixpkgs.docbook_xml_dtd_43                    docbook-xml-4.3
    nixpkgs.docbook_xml_dtd_45                    docbook-xml-4.5
    nixpkgs.firefox                               firefox-33.0.2
@@ -31,7 +31,7 @@ to subsequent chapters.
 1. Install some packages from the channel:
 
    ```console
-   $ nix-env -iA nixpkgs.hello
+   $ nix-env --install --attr nixpkgs.hello
    ```
 
    This should download pre-built packages; it should not build them
@@ -49,13 +49,13 @@ to subsequent chapters.
 1. Uninstall a package:
 
    ```console
-   $ nix-env -e hello
+   $ nix-env --uninstall hello
    ```
 
 1. You can also test a package without installing it:
 
    ```console
-   $ nix-shell -p hello
+   $ nix-shell --packages hello
    ```
 
    This builds or downloads GNU Hello and its dependencies, then drops
@@ -76,7 +76,7 @@ to subsequent chapters.
 
    ```console
    $ nix-channel --update nixpkgs
-   $ nix-env -u '*'
+   $ nix-env --upgrade '*'
    ```
 
    The latter command will upgrade each installed package for which
@@ -95,5 +95,5 @@ to subsequent chapters.
    them:
 
    ```console
-   $ nix-collect-garbage -d
+   $ nix-collect-garbage --delete-old
    ```

doc/manual/src/release-notes/rl-2.15.md

@@ -0,0 +1,58 @@
+# Release 2.15 (2023-04-11)
+
+* Commands which take installables on the command line can now read them from the standard input if
+  passed the `--stdin` flag. This is primarily useful when you have a large amount of paths which
+  exceed the OS argument limit.
+
+* The `nix-hash` command now supports Base64 and SRI. Use the flags `--base64`
+  or `--sri` to specify the format of output hash as Base64 or SRI, and `--to-base64`
+  or `--to-sri` to convert a hash to Base64 or SRI format, respectively.
+
+  As the choice of hash formats is no longer binary, the `--base16` flag is also added
+  to explicitly specify the Base16 format, which is still the default.
+
+* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents.
+
+  The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation.
+  Using this is better and more clear than relying on the now-removed `.drv` special handling.
+
+  For example,
+  ```shell-session
+  $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv
+  ```
+
+  now gives info about the derivation itself, while
+
+  ```shell-session
+  $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*
+  ```
+  provides information about each of its outputs.
+
+* The experimental command `nix describe-stores` has been removed.
+
+* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
+
+* Documentation for operations of `nix-store` and `nix-env` are now available on separate pages of the manual.
+  They include all common options that can be specified and common environment variables that affect these commands.
+
+  These pages can be viewed offline with `man` using
+
+  * `man nix-store-<operation>` and `man nix-env-<operation>`
+  * `nix-store --help --<operation>` and `nix-env --help --<operation>`.
+
+* Nix when used as a client now checks whether the store (the server) trusts the client.
+  (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.)
+  This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores.
+
+  `nix store ping` and `nix doctor` now display this information.
+
+* The new command `nix derivation add` allows adding derivations to the store without involving the Nix language.
+  It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store.
+  It uses the same JSON layout as `nix derivation show`, and is its inverse.
+
+* `nix show-derivation` has been renamed to `nix derivation show`.
+  This matches `nix derivation add`, and avoids bloating the top-level namespace.
+  The old name is still kept as an alias for compatibility, however.
+
+* The `nix derivation {add,show}` JSON format now includes the derivation name as a top-level field.
+  This is useful in general, but especially necessary for the `add` direction, as otherwise we would need to pass in the name out of band for certain cases.

doc/manual/src/release-notes/rl-next.md

@@ -1,41 +1,2 @@
 # Release X.Y (202?-??-??)
 
-* Commands which take installables on the command line can now read them from the standard input if
-  passed the `--stdin` flag. This is primarily useful when you have a large amount of paths which
-  exceed the OS arg limit.
-
-* The `nix-hash` command now supports Base64 and SRI. Use the flags `--base64`
-  or `--sri` to specify the format of output hash as Base64 or SRI, and `--to-base64`
-  or `--to-sri` to convert a hash to Base64 or SRI format, respectively.
-
-  As the choice of hash formats is no longer binary, the `--base16` flag is also added
-  to explicitly specify the Base16 format, which is still the default.
-
-* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents.
-
-  The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation.
-  Using this is better and more clear than relying on the now-removed `.drv` special handling.
-
-  For example,
-  ```shell-session
-  $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv
-  ```
-
-  now gives info about the derivation itself, while
-
-  ```shell-session
-  $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*
-  ```
-  provides information about each of its outputs.
-
-* The experimental command `nix describe-stores` has been removed.
-
-* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
-
-* Documentation for operations of `nix-store` and `nix-env` are now available on separate pages of the manual.
-  They include all common options that can be specified and common environment variables that affect these commands.
-
-  These pages can be viewed offline with `man` using
-
-  * `man nix-store-<operation>` and `man nix-env-<operation>`
-  * `nix-store --help --<operation>` and `nix-env --help --<operation>`.

doc/manual/utils.nix

@@ -5,6 +5,9 @@ rec {
 
   concatStrings = concatStringsSep "";
 
+  attrsToList = a:
+    map (name: { inherit name; value = a.${name}; }) (builtins.attrNames a);
+
   replaceStringsRec = from: to: string:
     # recursively replace occurrences of `from` with `to` within `string`
     # example:
@@ -39,7 +42,9 @@ rec {
   filterAttrs = pred: set:
     listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set));
 
-  showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value }:
+  optionalString = cond: string: if cond then string else "";
+
+  showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }:
     let
       result = squash ''
           - ${if useAnchors
@@ -49,10 +54,28 @@ rec {
           ${indent "  " body}
         '';
 
+      experimentalFeatureNote = optionalString (experimentalFeature != null) ''
+          > **Warning**
+          > This setting is part of an
+          > [experimental feature](@docroot@/contributing/experimental-features.md).
+
+          To change this setting, you need to make sure the corresponding experimental feature,
+          [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}),
+          is enabled.
+          For example, include the following in [`nix.conf`](#):
+
+          ```
+          extra-experimental-features = ${experimentalFeature}
+          ${name} = ...
+          ```
+        '';
+
       # separate body to cleanly handle indentation
       body = ''
           ${description}
 
+          ${experimentalFeatureNote}
+
           **Default:** ${showDefault documentDefault defaultValue}
 
           ${showAliases aliases}
@@ -71,13 +94,13 @@ rec {
         else "*machine-specific*";
 
       showAliases = aliases:
-          if aliases == [] then "" else
+          optionalString (aliases != [])
             "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}";
 
+    in result;
+
   indent = prefix: s:
     concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s));
 
-    in result;
-
   showSettings = args: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting args) settingsInfo));
 }

flake.nix

@@ -219,6 +219,7 @@
 
         enableParallelBuilding = true;
 
+        configureFlags = testConfigureFlags; # otherwise configure fails
         dontBuild = true;
         doInstallCheck = true;
 
@@ -472,8 +473,6 @@
       };
 
     in {
-      inherit nixpkgsFor;
-
       # A Nixpkgs overlay that overrides the 'nix' and
       # 'nix.perl-bindings' packages.
       overlays.default = overlayFor (p: p.stdenv);
@@ -583,6 +582,8 @@
 
         tests.nix-copy-closure = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy-closure.nix;
 
+        tests.nix-copy = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy.nix;
+
         tests.nssPreload = runNixOSTestFor "x86_64-linux" ./tests/nixos/nss-preload.nix;
 
         tests.githubFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/github-flakes.nix;

local.mk

@@ -1,6 +1,8 @@
 clean-files += Makefile.config
 
-GLOBAL_CXXFLAGS += -Wno-deprecated-declarations
+GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch
+# Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers.
+ERROR_SWITCH_ENUM = -Werror=switch-enum
 
 $(foreach i, config.h $(wildcard src/lib*/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))

maintainers/README.md

@@ -56,7 +56,7 @@ Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.u
 
 The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19/views/1) for tracking its work.
 
-Issues on the board progress through the following states:
+Items on the board progress through the following states:
 
 - No Status
 
@@ -69,6 +69,7 @@ Issues on the board progress through the following states:
   2. [security](https://github.com/NixOS/nix/labels/security)
   3. [regression](https://github.com/NixOS/nix/labels/regression)
   4. [bug](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Abug+sort%3Areactions-%2B1-desc)
+  5. [tests of existing functionality](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Atests+-label%3Afeature+sort%3Areactions-%2B1-desc)
 
   - [oldest pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Acreated-asc)
   - [most popular pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Areactions-%2B1-desc)
@@ -79,6 +80,9 @@ Issues on the board progress through the following states:
 
   If there is disagreement on the general idea behind an issue or pull request, it is moved to _To discuss_, otherwise to _In review_.
 
+  To ensure process quality and reliability, all non-trivial pull requests must be triaged before merging.
+  What constitutes a trivial pull request is up to maintainers' judgement.
+
 - To discuss
 
   Pull requests and issues that are deemed important and controversial are discussed by the team during discussion meetings.
@@ -91,7 +95,7 @@ Issues on the board progress through the following states:
 
     Contributors who took the time to implement concrete change proposals should not wait indefinitely.
 
-  - Prioritise fixing bugs over documentation, improvements or new features
+  - Prioritise fixing bugs and testing over documentation, improvements or new features
 
     The team values stability and accessibility higher than raw functionality.
 

mk/patterns.mk

@@ -1,10 +1,10 @@
 $(buildprefix)%.o: %.cc
 	@mkdir -p "$(dir $@)"
-	$(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP
 
 $(buildprefix)%.o: %.cpp
 	@mkdir -p "$(dir $@)"
-	$(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP
 
 $(buildprefix)%.o: %.c
 	@mkdir -p "$(dir $@)"

perl/lib/Nix/Store.xs

@@ -27,8 +27,6 @@ static ref<Store> store()
     if (!_store) {
         try {
             initLibStore();
-            loadConfFile();
-            settings.lockCPU = false;
             _store = openStore();
         } catch (Error & e) {
             croak("%s", e.what());

src/libcmd/command-installable-value.hh

@@ -1,10 +1,20 @@
+#pragma once
+///@file
+
 #include "installable-value.hh"
 #include "command.hh"
 
 namespace nix {
 
+/**
+ * An InstallableCommand where the single positional argument must be an
+ * InstallableValue in particular.
+ */
 struct InstallableValueCommand : InstallableCommand
 {
+    /**
+     * Entry point to this command
+     */
     virtual void run(ref<Store> store, ref<InstallableValue> installable) = 0;
 
     void run(ref<Store> store, ref<Installable> installable) override;

src/libcmd/command.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "installable-value.hh"
 #include "args.hh"

src/libcmd/common-eval-args.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "args.hh"
 

src/libcmd/editor-for.hh

@@ -1,11 +1,14 @@
 #pragma once
+///@file
 
 #include "types.hh"
 
 namespace nix {
 
-/* Helper function to generate args that invoke $EDITOR on
+/**
-   filename:lineno. */
+ * Helper function to generate args that invoke $EDITOR on
+ * filename:lineno.
+ */
 Strings editorFor(const Path & file, uint32_t line);
 
 }

src/libcmd/installable-attr-path.hh

@@ -1,3 +1,6 @@
+#pragma once
+///@file
+
 #include "globals.hh"
 #include "installable-value.hh"
 #include "outputs-spec.hh"

src/libcmd/installable-derived-path.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "installables.hh"
 

src/libcmd/installable-flake.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "installable-value.hh"
 

src/libcmd/installable-value.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "installables.hh"
 #include "flake/flake.hh"

src/libcmd/installables.cc

@@ -102,6 +102,28 @@ MixFlakeOptions::MixFlakeOptions()
         }}
     });
 
+    addFlag({
+        .longName = "reference-lock-file",
+        .description = "Read the given lock file instead of `flake.lock` within the top-level flake.",
+        .category = category,
+        .labels = {"flake-lock-path"},
+        .handler = {[&](std::string lockFilePath) {
+            lockFlags.referenceLockFilePath = lockFilePath;
+        }},
+        .completer = completePath
+    });
+
+    addFlag({
+        .longName = "output-lock-file",
+        .description = "Write the given lock file instead of `flake.lock` within the top-level flake.",
+        .category = category,
+        .labels = {"flake-lock-path"},
+        .handler = {[&](std::string lockFilePath) {
+            lockFlags.outputLockFilePath = lockFilePath;
+        }},
+        .completer = completePath
+    });
+
     addFlag({
         .longName = "inputs-from",
         .description = "Use the inputs of the specified flake as registry entries.",

src/libcmd/installables.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "util.hh"
 #include "path.hh"

src/libcmd/legacy.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include <functional>
 #include <map>

src/libcmd/markdown.hh

@@ -1,3 +1,6 @@
+#pragma once
+///@file
+
 #include "types.hh"
 
 namespace nix {

src/libcmd/repl.cc

@@ -40,6 +40,7 @@ extern "C" {
 #include "markdown.hh"
 #include "local-fs-store.hh"
 #include "progress-bar.hh"
+#include "print.hh"
 
 #if HAVE_BOEHMGC
 #define GC_INCLUDE_NEW
@@ -252,7 +253,9 @@ void NixRepl::mainLoop()
     el_hist_size = 1000;
 #endif
     read_history(historyFile.c_str());
+    auto oldRepl = curRepl;
     curRepl = this;
+    Finally restoreRepl([&] { curRepl = oldRepl; });
 #ifndef READLINE
     rl_set_complete_func(completionCallback);
     rl_set_list_possib_func(listPossibleCallback);
@@ -423,6 +426,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
 }
 
 
+// FIXME: DRY and match or use the parser
 static bool isVarName(std::string_view s)
 {
     if (s.size() == 0) return false;
@@ -892,17 +896,6 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
 }
 
 
-std::ostream & printStringValue(std::ostream & str, const char * string) {
-    str << "\"";
-    for (const char * i = string; *i; i++)
-        if (*i == '\"' || *i == '\\') str << "\\" << *i;
-        else if (*i == '\n') str << "\\n";
-        else if (*i == '\r') str << "\\r";
-        else if (*i == '\t') str << "\\t";
-        else str << *i;
-    str << "\"";
-    return str;
-}
 
 
 // FIXME: lot of cut&paste from Nix's eval.cc.
@@ -920,12 +913,14 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         break;
 
     case nBool:
-        str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL;
+        str << ANSI_CYAN;
+        printLiteralBool(str, v.boolean);
+        str << ANSI_NORMAL;
         break;
 
     case nString:
         str << ANSI_WARNING;
-        printStringValue(str, v.string.s);
+        printLiteralString(str, v.string.s);
         str << ANSI_NORMAL;
         break;
 
@@ -962,10 +957,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
                 sorted.emplace(state->symbols[i.name], i.value);
 
             for (auto & i : sorted) {
-                if (isVarName(i.first))
+                printAttributeName(str, i.first);
-                    str << i.first;
-                else
-                    printStringValue(str, i.first.c_str());
                 str << " = ";
                 if (seen.count(i.second))
                     str << "«repeated»";
@@ -1024,6 +1016,8 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         str << v.fpoint;
         break;
 
+    case nThunk:
+    case nExternal:
     default:
         str << ANSI_RED "«unknown»" ANSI_NORMAL;
         break;

src/libcmd/repl.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "eval.hh"
 

src/libexpr/attr-path.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "eval.hh"
 
@@ -16,7 +17,9 @@ std::pair<Value *, PosIdx> findAlongAttrPath(
     Bindings & autoArgs,
     Value & vIn);
 
-/* Heuristic to find the filename and lineno or a nix value. */
+/**
+ * Heuristic to find the filename and lineno or a nix value.
+ */
 std::pair<std::string, uint32_t> findPackageFilename(EvalState & state, Value & v, std::string what);
 
 std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);

src/libexpr/attr-set.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "nixexpr.hh"
 #include "symbol-table.hh"
@@ -12,7 +13,9 @@ namespace nix {
 class EvalState;
 struct Value;
 
-/* Map one attribute name to its value. */
+/**
+ * Map one attribute name to its value.
+ */
 struct Attr
 {
     /* the placement of `name` and `pos` in this struct is important.
@@ -36,10 +39,12 @@ static_assert(sizeof(Attr) == 2 * sizeof(uint32_t) + sizeof(Value *),
     "avoid introducing any padding into Attr if at all possible, and do not "
     "introduce new fields that need not be present for almost every instance.");
 
-/* Bindings contains all the attributes of an attribute set. It is defined
+/**
-   by its size and its capacity, the capacity being the number of Attr
+ * Bindings contains all the attributes of an attribute set. It is defined
-   elements allocated after this structure, while the size corresponds to
+ * by its size and its capacity, the capacity being the number of Attr
-   the number of elements already inserted in this structure. */
+ * elements allocated after this structure, while the size corresponds to
+ * the number of elements already inserted in this structure.
+ */
 class Bindings
 {
 public:
@@ -94,7 +99,9 @@ public:
 
     size_t capacity() { return capacity_; }
 
-    /* Returns the attributes in lexicographically sorted order. */
+    /**
+     * Returns the attributes in lexicographically sorted order.
+     */
     std::vector<const Attr *> lexicographicOrder(const SymbolTable & symbols) const
     {
         std::vector<const Attr *> res;
@@ -111,9 +118,11 @@ public:
     friend class EvalState;
 };
 
-/* A wrapper around Bindings that ensures that its always in sorted
+/**
-   order at the end. The only way to consume a BindingsBuilder is to
+ * A wrapper around Bindings that ensures that its always in sorted
-   call finish(), which sorts the bindings. */
+ * order at the end. The only way to consume a BindingsBuilder is to
+ * call finish(), which sorts the bindings.
+ */
 class BindingsBuilder
 {
     Bindings * bindings;

src/libexpr/eval-cache.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "sync.hh"
 #include "hash.hh"
@@ -109,8 +110,10 @@ public:
 
     ref<AttrCursor> getAttr(std::string_view name);
 
-    /* Get an attribute along a chain of attrsets. Note that this does
+    /**
-       not auto-call functors or functions. */
+     * Get an attribute along a chain of attrsets. Note that this does
+     * not auto-call functors or functions.
+     */
     OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
 
     std::string getString();
@@ -129,7 +132,9 @@ public:
 
     Value & forceValue();
 
-    /* Force creation of the .drv file in the Nix store. */
+    /**
+     * Force creation of the .drv file in the Nix store.
+     */
     StorePath forceDerivation();
 };
 

src/libexpr/eval-inline.hh

@@ -1,10 +1,13 @@
 #pragma once
+///@file
 
 #include "eval.hh"
 
 namespace nix {
 
-/* Note: Various places expect the allocated memory to be zeroed. */
+/**
+ * Note: Various places expect the allocated memory to be zeroed.
+ */
 [[gnu::always_inline]]
 inline void * allocBytes(size_t n)
 {

src/libexpr/eval.cc

@@ -9,6 +9,7 @@
 #include "filetransfer.hh"
 #include "function-trace.hh"
 #include "profiles.hh"
+#include "print.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -104,18 +105,10 @@ void Value::print(const SymbolTable & symbols, std::ostream & str,
         str << integer;
         break;
     case tBool:
-        str << (boolean ? "true" : "false");
+        printLiteralBool(str, boolean);
         break;
     case tString:
-        str << "\"";
+        printLiteralString(str, string.s);
-        for (const char * i = string.s; *i; i++)
-            if (*i == '\"' || *i == '\\') str << "\\" << *i;
-            else if (*i == '\n') str << "\\n";
-            else if (*i == '\r') str << "\\r";
-            else if (*i == '\t') str << "\\t";
-            else if (*i == '$' && *(i+1) == '{') str << "\\" << *i;
-            else str << *i;
-        str << "\"";
         break;
     case tPath:
         str << path; // !!! escaping?
@@ -173,7 +166,17 @@ void Value::print(const SymbolTable & symbols, std::ostream & str,
     case tFloat:
         str << fpoint;
         break;
+    case tBlackhole:
+        // Although we know for sure that it's going to be an infinite recursion
+        // when this value is accessed _in the current context_, it's likely
+        // that the user will misinterpret a simpler «infinite recursion» output
+        // as a definitive statement about the value, while in fact it may be
+        // a valid value after `builtins.trace` and perhaps some other steps
+        // have completed.
+        str << "«potential infinite recursion»";
+        break;
     default:
+        printError("Nix evaluator internal error: Value::print(): invalid value type %1%", internalType);
         abort();
     }
 }
@@ -229,6 +232,9 @@ std::string_view showType(ValueType type)
 
 std::string showType(const Value & v)
 {
+    // Allow selecting a subset of enum values
+    #pragma GCC diagnostic push
+    #pragma GCC diagnostic ignored "-Wswitch-enum"
     switch (v.internalType) {
         case tString: return v.string.context ? "a string with context" : "a string";
         case tPrimOp:
@@ -242,16 +248,21 @@ std::string showType(const Value & v)
     default:
         return std::string(showType(v.type()));
     }
+    #pragma GCC diagnostic pop
 }
 
 PosIdx Value::determinePos(const PosIdx pos) const
 {
+    // Allow selecting a subset of enum values
+    #pragma GCC diagnostic push
+    #pragma GCC diagnostic ignored "-Wswitch-enum"
     switch (internalType) {
         case tAttrs: return attrs->pos;
         case tLambda: return lambda.fun->pos;
         case tApp: return app.left->determinePos(pos);
         default: return pos;
     }
+    #pragma GCC diagnostic pop
 }
 
 bool Value::isTrivial() const
@@ -326,6 +337,22 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env)
     }
 }
 
+#if HAVE_BOEHMGC
+/* Disable GC while this object lives. Used by CoroutineContext.
+ *
+ * Boehm keeps a count of GC_disable() and GC_enable() calls,
+ * and only enables GC when the count matches.
+ */
+class BoehmDisableGC {
+public:
+    BoehmDisableGC() {
+        GC_disable();
+    };
+    ~BoehmDisableGC() {
+        GC_enable();
+    };
+};
+#endif
 
 static bool gcInitialised = false;
 
@@ -350,6 +377,15 @@ void initGC()
 
     StackAllocator::defaultAllocator = &boehmGCStackAllocator;
 
+
+#if NIX_BOEHM_PATCH_VERSION != 1
+    printTalkative("Unpatched BoehmGC, disabling GC inside coroutines");
+    /* Used to disable GC when entering coroutines on macOS */
+    create_coro_gc_hook = []() -> std::shared_ptr<void> {
+        return std::make_shared<BoehmDisableGC>();
+    };
+#endif
+
     /* Set the initial heap size to something fairly big (25% of
        physical RAM, up to a maximum of 384 MiB) so that in most cases
        we don't need to garbage collect at all.  (Collection has a
@@ -2327,6 +2363,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
         case nFloat:
             return v1.fpoint == v2.fpoint;
 
+        case nThunk: // Must not be left by forceValue
         default:
             error("cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).debugThrow<EvalError>();
     }

src/libexpr/eval.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "attr-set.hh"
 #include "types.hh"
@@ -42,7 +43,10 @@ struct PrimOp
 struct Env
 {
     Env * up;
-    unsigned short prevWith:14; // nr of levels up to next `with' environment
+    /**
+     * Number of of levels up to next `with` environment
+     */
+    unsigned short prevWith:14;
     enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
     Value * values[0];
 };
@@ -55,8 +59,10 @@ std::unique_ptr<ValMap> mapStaticEnvBindings(const SymbolTable & st, const Stati
 void copyContext(const Value & v, PathSet & context);
 
 
-/* Cache for calls to addToStore(); maps source paths to the store
+/**
-   paths. */
+ * Cache for calls to addToStore(); maps source paths to the store
+ * paths.
+ */
 typedef std::map<Path, StorePath> SrcToStore;
 
 
@@ -68,7 +74,9 @@ typedef std::pair<std::string, std::string> SearchPathElem;
 typedef std::list<SearchPathElem> SearchPath;
 
 
-/* Initialise the Boehm GC, if applicable. */
+/**
+ * Initialise the Boehm GC, if applicable.
+ */
 void initGC();
 
 
@@ -143,26 +151,36 @@ public:
         sOutputSpecified;
     Symbol sDerivationNix;
 
-    /* If set, force copying files to the Nix store even if they
+    /**
-       already exist there. */
+     * If set, force copying files to the Nix store even if they
+     * already exist there.
+     */
     RepairFlag repair;
 
-    /* The allowed filesystem paths in restricted or pure evaluation
+    /**
-       mode. */
+     * The allowed filesystem paths in restricted or pure evaluation
+     * mode.
+     */
     std::optional<PathSet> allowedPaths;
 
     Bindings emptyBindings;
 
-    /* Store used to materialise .drv files. */
+    /**
+     * Store used to materialise .drv files.
+     */
     const ref<Store> store;
 
-    /* Store used to build stuff. */
+    /**
+     * Store used to build stuff.
+     */
     const ref<Store> buildStore;
 
     RootValue vCallFlake = nullptr;
     RootValue vImportedDrvToDerivation = nullptr;
 
-    /* Debugger */
+    /**
+     * Debugger
+     */
     void (* debugRepl)(ref<EvalState> es, const ValMap & extraEnv);
     bool debugStop;
     bool debugQuit;
@@ -218,7 +236,9 @@ public:
 private:
     SrcToStore srcToStore;
 
-    /* A cache from path names to parse trees. */
+    /**
+     * A cache from path names to parse trees.
+     */
 #if HAVE_BOEHMGC
     typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *>>> FileParseCache;
 #else
@@ -226,7 +246,9 @@ private:
 #endif
     FileParseCache fileParseCache;
 
-    /* A cache from path names to values. */
+    /**
+     * A cache from path names to values.
+     */
 #if HAVE_BOEHMGC
     typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value>>> FileEvalCache;
 #else
@@ -238,17 +260,25 @@ private:
 
     std::map<std::string, std::pair<bool, std::string>> searchPathResolved;
 
-    /* Cache used by checkSourcePath(). */
+    /**
+     * Cache used by checkSourcePath().
+     */
     std::unordered_map<Path, Path> resolvedPaths;
 
-    /* Cache used by prim_match(). */
+    /**
+     * Cache used by prim_match().
+     */
     std::shared_ptr<RegexCache> regexCache;
 
 #if HAVE_BOEHMGC
-    /* Allocation cache for GC'd Value objects. */
+    /**
+     * Allocation cache for GC'd Value objects.
+     */
     std::shared_ptr<void *> valueAllocCache;
 
-    /* Allocation cache for size-1 Env objects. */
+    /**
+     * Allocation cache for size-1 Env objects.
+     */
     std::shared_ptr<void *> env1AllocCache;
 #endif
 
@@ -264,47 +294,65 @@ public:
 
     SearchPath getSearchPath() { return searchPath; }
 
-    /* Allow access to a path. */
+    /**
+     * Allow access to a path.
+     */
     void allowPath(const Path & path);
 
-    /* Allow access to a store path. Note that this gets remapped to
+    /**
-       the real store path if `store` is a chroot store. */
+     * Allow access to a store path. Note that this gets remapped to
+     * the real store path if `store` is a chroot store.
+     */
     void allowPath(const StorePath & storePath);
 
-    /* Allow access to a store path and return it as a string. */
+    /**
+     * Allow access to a store path and return it as a string.
+     */
     void allowAndSetStorePathString(const StorePath & storePath, Value & v);
 
-    /* Check whether access to a path is allowed and throw an error if
+    /**
-       not. Otherwise return the canonicalised path. */
+     * Check whether access to a path is allowed and throw an error if
+     * not. Otherwise return the canonicalised path.
+     */
     Path checkSourcePath(const Path & path);
 
     void checkURI(const std::string & uri);
 
-    /* When using a diverted store and 'path' is in the Nix store, map
+    /**
-       'path' to the diverted location (e.g. /nix/store/foo is mapped
+     * When using a diverted store and 'path' is in the Nix store, map
-       to /home/alice/my-nix/nix/store/foo). However, this is only
+     * 'path' to the diverted location (e.g. /nix/store/foo is mapped
-       done if the context is not empty, since otherwise we're
+     * to /home/alice/my-nix/nix/store/foo). However, this is only
-       probably trying to read from the actual /nix/store. This is
+     * done if the context is not empty, since otherwise we're
-       intended to distinguish between import-from-derivation and
+     * probably trying to read from the actual /nix/store. This is
-       sources stored in the actual /nix/store. */
+     * intended to distinguish between import-from-derivation and
+     * sources stored in the actual /nix/store.
+     */
     Path toRealPath(const Path & path, const PathSet & context);
 
-    /* Parse a Nix expression from the specified file. */
+    /**
+     * Parse a Nix expression from the specified file.
+     */
     Expr * parseExprFromFile(const Path & path);
     Expr * parseExprFromFile(const Path & path, std::shared_ptr<StaticEnv> & staticEnv);
 
-    /* Parse a Nix expression from the specified string. */
+    /**
+     * Parse a Nix expression from the specified string.
+     */
     Expr * parseExprFromString(std::string s, const Path & basePath, std::shared_ptr<StaticEnv> & staticEnv);
     Expr * parseExprFromString(std::string s, const Path & basePath);
 
     Expr * parseStdin();
 
-    /* Evaluate an expression read from the given file to normal
+    /**
-       form. Optionally enforce that the top-level expression is
+     * Evaluate an expression read from the given file to normal
-       trivial (i.e. doesn't require arbitrary computation). */
+     * form. Optionally enforce that the top-level expression is
+     * trivial (i.e. doesn't require arbitrary computation).
+     */
     void evalFile(const Path & path, Value & v, bool mustBeTrivial = false);
 
-    /* Like `evalFile`, but with an already parsed expression. */
+    /**
+     * Like `evalFile`, but with an already parsed expression.
+     */
     void cacheFile(
         const Path & path,
         const Path & resolvedPath,
@@ -314,37 +362,52 @@ public:
 
     void resetFileCache();
 
-    /* Look up a file in the search path. */
+    /**
+     * Look up a file in the search path.
+     */
     Path findFile(const std::string_view path);
     Path findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
 
-    /* If the specified search path element is a URI, download it. */
+    /**
+     * If the specified search path element is a URI, download it.
+     */
     std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem);
 
-    /* Evaluate an expression to normal form, storing the result in
+    /**
-       value `v'. */
+     * Evaluate an expression to normal form
+     *
+     * @param [out] v The resulting is stored here.
+     */
     void eval(Expr * e, Value & v);
 
-    /* Evaluation the expression, then verify that it has the expected
+    /**
-       type. */
+     * Evaluation the expression, then verify that it has the expected
+     * type.
+     */
     inline bool evalBool(Env & env, Expr * e);
     inline bool evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx);
     inline void evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx);
 
-    /* If `v' is a thunk, enter it and overwrite `v' with the result
+    /**
-       of the evaluation of the thunk.  If `v' is a delayed function
+     * If `v` is a thunk, enter it and overwrite `v` with the result
-       application, call the function and overwrite `v' with the
+     * of the evaluation of the thunk.  If `v` is a delayed function
-       result.  Otherwise, this is a no-op. */
+     * application, call the function and overwrite `v` with the
+     * result.  Otherwise, this is a no-op.
+     */
     inline void forceValue(Value & v, const PosIdx pos);
 
     template <typename Callable>
     inline void forceValue(Value & v, Callable getPos);
 
-    /* Force a value, then recursively force list elements and
+    /**
-       attributes. */
+     * Force a value, then recursively force list elements and
+     * attributes.
+     */
     void forceValueDeep(Value & v);
 
-    /* Force `v', and then verify that it has the expected type. */
+    /**
+     * Force `v`, and then verify that it has the expected type.
+     */
     NixInt forceInt(Value & v, const PosIdx pos, std::string_view errorCtx);
     NixFloat forceFloat(Value & v, const PosIdx pos, std::string_view errorCtx);
     bool forceBool(Value & v, const PosIdx pos, std::string_view errorCtx);
@@ -355,7 +418,10 @@ public:
     inline void forceAttrs(Value & v, Callable getPos, std::string_view errorCtx);
 
     inline void forceList(Value & v, const PosIdx pos, std::string_view errorCtx);
-    void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx); // either lambda or primop
+    /**
+     * @param v either lambda or primop
+     */
+    void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx);
     std::string_view forceString(Value & v, const PosIdx pos, std::string_view errorCtx);
     std::string_view forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx);
     std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx);
@@ -366,17 +432,23 @@ public:
     void addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame = false) const;
 
 public:
-    /* Return true iff the value `v' denotes a derivation (i.e. a
+    /**
-       set with attribute `type = "derivation"'). */
+     * @return true iff the value `v` denotes a derivation (i.e. a
+     * set with attribute `type = "derivation"`).
+     */
     bool isDerivation(Value & v);
 
     std::optional<std::string> tryAttrsToString(const PosIdx pos, Value & v,
         PathSet & context, bool coerceMore = false, bool copyToStore = true);
 
-    /* String coercion.  Converts strings, paths and derivations to a
+    /**
-       string.  If `coerceMore' is set, also converts nulls, integers,
+     * String coercion.
-       booleans and lists to a string.  If `copyToStore' is set,
+     *
-       referenced paths are copied to the Nix store as a side effect. */
+     * Converts strings, paths and derivations to a
+     * string.  If `coerceMore` is set, also converts nulls, integers,
+     * booleans and lists to a string.  If `copyToStore` is set,
+     * referenced paths are copied to the Nix store as a side effect.
+     */
     BackedStringView coerceToString(const PosIdx pos, Value & v, PathSet & context,
         std::string_view errorCtx,
         bool coerceMore = false, bool copyToStore = true,
@@ -384,21 +456,31 @@ public:
 
     StorePath copyPathToStore(PathSet & context, const Path & path);
 
-    /* Path coercion.  Converts strings, paths and derivations to a
+    /**
-       path.  The result is guaranteed to be a canonicalised, absolute
+     * Path coercion.
-       path.  Nothing is copied to the store. */
+     *
+     * Converts strings, paths and derivations to a
+     * path.  The result is guaranteed to be a canonicalised, absolute
+     * path.  Nothing is copied to the store.
+     */
     Path coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx);
 
-    /* Like coerceToPath, but the result must be a store path. */
+    /**
+     * Like coerceToPath, but the result must be a store path.
+     */
     StorePath coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx);
 
 public:
 
-    /* The base environment, containing the builtin functions and
+    /**
-       values. */
+     * The base environment, containing the builtin functions and
+     * values.
+     */
     Env & baseEnv;
 
-    /* The same, but used during parsing to resolve variables. */
+    /**
+     * The same, but used during parsing to resolve variables.
+     */
     std::shared_ptr<StaticEnv> staticBaseEnv; // !!! should be private
 
 private:
@@ -448,8 +530,10 @@ private:
 
 public:
 
-    /* Do a deep equality test between two values.  That is, list
+    /**
-       elements and attributes are compared recursively. */
+     * Do a deep equality test between two values.  That is, list
+     * elements and attributes are compared recursively.
+     */
     bool eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx);
 
     bool isFunctor(Value & fun);
@@ -463,11 +547,15 @@ public:
         callFunction(fun, 1, args, vRes, pos);
     }
 
-    /* Automatically call a function for which each argument has a
+    /**
-       default value or has a binding in the `args' map. */
+     * Automatically call a function for which each argument has a
+     * default value or has a binding in the `args` map.
+     */
     void autoCallFunction(Bindings & args, Value & fun, Value & res);
 
-    /* Allocation primitives. */
+    /**
+     * Allocation primitives.
+     */
     inline Value * allocValue();
     inline Env & allocEnv(size_t size);
 
@@ -487,10 +575,13 @@ public:
 
     void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx);
 
-    /* Print statistics. */
+    /**
+     * Print statistics.
+     */
     void printStats();
 
-    /* Realise the given context, and return a mapping from the placeholders
+    /**
+     * Realise the given context, and return a mapping from the placeholders
      * used to construct the associated value to their final store path
      */
     [[nodiscard]] StringMap realiseContext(const PathSet & context);
@@ -550,11 +641,15 @@ struct DebugTraceStacker {
     DebugTrace trace;
 };
 
-/* Return a string representing the type of the value `v'. */
+/**
+ * @return A string representing the type of the value `v`.
+ */
 std::string_view showType(ValueType type);
 std::string showType(const Value & v);
 
-/* If `path' refers to a directory, then append "/default.nix". */
+/**
+ * If `path` refers to a directory, then append "/default.nix".
+ */
 Path resolveExprPath(Path path);
 
 struct InvalidPathError : EvalError

src/libexpr/flake/flake.cc

@@ -125,6 +125,9 @@ static FlakeInput parseFlakeInput(EvalState & state,
                 follows.insert(follows.begin(), lockRootPath.begin(), lockRootPath.end());
                 input.follows = follows;
             } else {
+                // Allow selecting a subset of enum values
+                #pragma GCC diagnostic push
+                #pragma GCC diagnostic ignored "-Wswitch-enum"
                 switch (attr.value->type()) {
                     case nString:
                         attrs.emplace(state.symbols[attr.name], attr.value->string.s);
@@ -139,6 +142,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
                         throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
                             state.symbols[attr.name], showType(*attr.value));
                 }
+                #pragma GCC diagnostic pop
             }
         } catch (Error & e) {
             e.addTrace(
@@ -334,10 +338,14 @@ LockedFlake lockFlake(
     }
 
     try {
+        if (!fetchSettings.allowDirty && lockFlags.referenceLockFilePath) {
+            throw Error("reference lock file was provided, but the `allow-dirty` setting is set to false");
+        }
 
         // FIXME: symlink attack
         auto oldLockFile = LockFile::read(
-            flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock");
+            lockFlags.referenceLockFilePath.value_or(
+                flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"));
 
         debug("old lock file: %s", oldLockFile);
 
@@ -619,13 +627,20 @@ LockedFlake lockFlake(
 
         debug("new lock file: %s", newLockFile);
 
+        auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
+        auto sourcePath = topRef.input.getSourcePath();
+        auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt;
+        if (lockFlags.outputLockFilePath) {
+            outputLockFilePath = lockFlags.outputLockFilePath;
+        }
+
         /* Check whether we need to / can write the new lock file. */
-        if (!(newLockFile == oldLockFile)) {
+        if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) {
 
             auto diff = LockFile::diff(oldLockFile, newLockFile);
 
             if (lockFlags.writeLockFile) {
-                if (auto sourcePath = topRef.input.getSourcePath()) {
+                if (outputLockFilePath) {
                     if (auto unlockedInput = newLockFile.isUnlocked()) {
                         if (fetchSettings.warnDirty)
                             warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput);
@@ -633,25 +648,24 @@ LockedFlake lockFlake(
                         if (!lockFlags.updateLockFile)
                             throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef);
 
-                        auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
+                        bool lockFileExists = pathExists(*outputLockFilePath);
-
-                        auto path = *sourcePath + "/" + relPath;
-
-                        bool lockFileExists = pathExists(path);
 
                         if (lockFileExists) {
                             auto s = chomp(diff);
                             if (s.empty())
-                                warn("updating lock file '%s'", path);
+                                warn("updating lock file '%s'", *outputLockFilePath);
                             else
-                                warn("updating lock file '%s':\n%s", path, s);
+                                warn("updating lock file '%s':\n%s", *outputLockFilePath, s);
                         } else
-                            warn("creating lock file '%s'", path);
+                            warn("creating lock file '%s'", *outputLockFilePath);
 
-                        newLockFile.write(path);
+                        newLockFile.write(*outputLockFilePath);
 
                         std::optional<std::string> commitMessage = std::nullopt;
                         if (lockFlags.commitLockFile) {
+                            if (lockFlags.outputLockFilePath) {
+                                throw Error("--commit-lock-file and --output-lock-file are currently incompatible");
+                            }
                             std::string cm;
 
                             cm = fetchSettings.commitLockFileSummary.get();

src/libexpr/flake/flake.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "types.hh"
 #include "flakeref.hh"
@@ -17,7 +18,8 @@ struct FlakeInput;
 
 typedef std::map<FlakeId, FlakeInput> FlakeInputs;
 
-/* FlakeInput is the 'Flake'-level parsed form of the "input" entries
+/**
+ * FlakeInput is the 'Flake'-level parsed form of the "input" entries
  * in the flake file.
  *
  * A FlakeInput is normally constructed by the 'parseFlakeInput'
@@ -41,7 +43,12 @@ typedef std::map<FlakeId, FlakeInput> FlakeInputs;
 struct FlakeInput
 {
     std::optional<FlakeRef> ref;
-    bool isFlake = true;  // true = process flake to get outputs, false = (fetched) static source path
+    /**
+     * true = process flake to get outputs
+     *
+     * false = (fetched) static source path
+     */
+    bool isFlake = true;
     std::optional<InputPath> follows;
     FlakeInputs overrides;
 };
@@ -55,23 +62,42 @@ struct ConfigFile
     void apply();
 };
 
-/* The contents of a flake.nix file. */
+/**
+ * The contents of a flake.nix file.
+ */
 struct Flake
 {
-    FlakeRef originalRef; // the original flake specification (by the user)
+    /**
-    FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake
+     * The original flake specification (by the user)
-    FlakeRef lockedRef; // the specific local store result of invoking the fetcher
+     */
-    bool forceDirty = false; // pretend that 'lockedRef' is dirty
+    FlakeRef originalRef;
+    /**
+     * registry references and caching resolved to the specific underlying flake
+     */
+    FlakeRef resolvedRef;
+    /**
+     * the specific local store result of invoking the fetcher
+     */
+    FlakeRef lockedRef;
+    /**
+     * pretend that 'lockedRef' is dirty
+     */
+    bool forceDirty = false;
     std::optional<std::string> description;
     std::shared_ptr<const fetchers::Tree> sourceInfo;
     FlakeInputs inputs;
-    ConfigFile config; // 'nixConfig' attribute
+    /**
+     * 'nixConfig' attribute
+     */
+    ConfigFile config;
     ~Flake();
 };
 
 Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup);
 
-/* Fingerprint of a locked flake; used as a cache key. */
+/**
+ * Fingerprint of a locked flake; used as a cache key.
+ */
 typedef Hash Fingerprint;
 
 struct LockedFlake
@@ -84,44 +110,72 @@ struct LockedFlake
 
 struct LockFlags
 {
-    /* Whether to ignore the existing lock file, creating a new one
+    /**
-       from scratch. */
+     * Whether to ignore the existing lock file, creating a new one
+     * from scratch.
+     */
     bool recreateLockFile = false;
 
-    /* Whether to update the lock file at all. If set to false, if any
+    /**
-       change to the lock file is needed (e.g. when an input has been
+     * Whether to update the lock file at all. If set to false, if any
-       added to flake.nix), you get a fatal error. */
+     * change to the lock file is needed (e.g. when an input has been
+     * added to flake.nix), you get a fatal error.
+     */
     bool updateLockFile = true;
 
-    /* Whether to write the lock file to disk. If set to true, if the
+    /**
-       any changes to the lock file are needed and the flake is not
+     * Whether to write the lock file to disk. If set to true, if the
-       writable (i.e. is not a local Git working tree or similar), you
+     * any changes to the lock file are needed and the flake is not
-       get a fatal error. If set to false, Nix will use the modified
+     * writable (i.e. is not a local Git working tree or similar), you
-       lock file in memory only, without writing it to disk. */
+     * get a fatal error. If set to false, Nix will use the modified
+     * lock file in memory only, without writing it to disk.
+     */
     bool writeLockFile = true;
 
-    /* Whether to use the registries to lookup indirect flake
+    /**
-       references like 'nixpkgs'. */
+     * Whether to use the registries to lookup indirect flake
+     * references like 'nixpkgs'.
+     */
     std::optional<bool> useRegistries = std::nullopt;
 
-    /* Whether to apply flake's nixConfig attribute to the configuration */
+    /**
+     * Whether to apply flake's nixConfig attribute to the configuration
+     */
 
     bool applyNixConfig = false;
 
-    /* Whether unlocked flake references (i.e. those without a Git
+    /**
-       revision or similar) without a corresponding lock are
+     * Whether unlocked flake references (i.e. those without a Git
-       allowed. Unlocked flake references with a lock are always
+     * revision or similar) without a corresponding lock are
-       allowed. */
+     * allowed. Unlocked flake references with a lock are always
+     * allowed.
+     */
     bool allowUnlocked = true;
 
-    /* Whether to commit changes to flake.lock. */
+    /**
+     * Whether to commit changes to flake.lock.
+     */
     bool commitLockFile = false;
 
-    /* Flake inputs to be overridden. */
+    /**
+     * The path to a lock file to read instead of the `flake.lock` file in the top-level flake
+     */
+    std::optional<std::string> referenceLockFilePath;
+
+    /**
+     * The path to a lock file to write to instead of the `flake.lock` file in the top-level flake
+     */
+    std::optional<Path> outputLockFilePath;
+
+    /**
+     * Flake inputs to be overridden.
+     */
     std::map<InputPath, FlakeRef> inputOverrides;
 
-    /* Flake inputs to be updated. This means that any existing lock
+    /**
-       for those inputs will be ignored. */
+     * Flake inputs to be updated. This means that any existing lock
+     * for those inputs will be ignored.
+     */
     std::set<InputPath> inputUpdates;
 };
 

src/libexpr/flake/flakeref.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "types.hh"
 #include "hash.hh"
@@ -13,7 +14,8 @@ class Store;
 
 typedef std::string FlakeId;
 
-/* A flake reference specifies how to fetch a flake or raw source
+/**
+ * A flake reference specifies how to fetch a flake or raw source
  * (e.g. from a Git repository).  It is created from a URL-like syntax
  * (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{
  * type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local
@@ -32,14 +34,17 @@ typedef std::string FlakeId;
  * be lazy), but the fetcher can be invoked at any time via the
  * FlakeRef to ensure the store is populated with this input.
  */
-
 struct FlakeRef
 {
-    /* Fetcher-specific representation of the input, sufficient to
+    /**
-       perform the fetch operation. */
+     * Fetcher-specific representation of the input, sufficient to
+     * perform the fetch operation.
+     */
     fetchers::Input input;
 
-    /* sub-path within the fetched input that represents this input */
+    /**
+     * sub-path within the fetched input that represents this input
+     */
     Path subdir;
 
     bool operator==(const FlakeRef & other) const;

src/libexpr/flake/lockfile.cc

@@ -234,6 +234,11 @@ bool LockFile::operator ==(const LockFile & other) const
     return toJSON() == other.toJSON();
 }
 
+bool LockFile::operator !=(const LockFile & other) const
+{
+    return !(*this == other);
+}
+
 InputPath parseInputPath(std::string_view s)
 {
     InputPath path;

src/libexpr/flake/lockfile.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "flakeref.hh"
 
@@ -15,9 +16,11 @@ typedef std::vector<FlakeId> InputPath;
 
 struct LockedNode;
 
-/* A node in the lock file. It has outgoing edges to other nodes (its
+/**
-   inputs). Only the root node has this type; all other nodes have
+ * A node in the lock file. It has outgoing edges to other nodes (its
-   type LockedNode. */
+ * inputs). Only the root node has this type; all other nodes have
+ * type LockedNode.
+ */
 struct Node : std::enable_shared_from_this<Node>
 {
     typedef std::variant<ref<LockedNode>, InputPath> Edge;
@@ -27,7 +30,9 @@ struct Node : std::enable_shared_from_this<Node>
     virtual ~Node() { }
 };
 
-/* A non-root node in the lock file. */
+/**
+ * A non-root node in the lock file.
+ */
 struct LockedNode : Node
 {
     FlakeRef lockedRef, originalRef;
@@ -62,10 +67,15 @@ struct LockFile
 
     void write(const Path & path) const;
 
-    /* Check whether this lock file has any unlocked inputs. */
+    /**
+     * Check whether this lock file has any unlocked inputs.
+     */
     std::optional<FlakeRef> isUnlocked() const;
 
     bool operator ==(const LockFile & other) const;
+    // Needed for old gcc versions that don't synthesize it (like gcc 8.2.2
+    // that is still the default on aarch64-linux)
+    bool operator !=(const LockFile & other) const;
 
     std::shared_ptr<Node> findInput(const InputPath & path);
 
@@ -73,7 +83,9 @@ struct LockFile
 
     static std::string diff(const LockFile & oldLocks, const LockFile & newLocks);
 
-    /* Check that every 'follows' input target exists. */
+    /**
+     * Check that every 'follows' input target exists.
+     */
     void check();
 };
 

src/libexpr/function-trace.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "eval.hh"
 

src/libexpr/get-drvs.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "eval.hh"
 #include "path.hh"
@@ -25,7 +26,10 @@ private:
     mutable std::string outputName;
     Outputs outputs;
 
-    bool failed = false; // set if we get an AssertionError
+    /**
+     * Set if we get an AssertionError
+     */
+    bool failed = false;
 
     Bindings * attrs = nullptr, * meta = nullptr;
 
@@ -34,7 +38,10 @@ private:
     bool checkMeta(Value & v);
 
 public:
-    std::string attrPath; /* path towards the derivation */
+    /**
+     * path towards the derivation
+     */
+    std::string attrPath;
 
     DrvInfo(EvalState & state) : state(&state) { };
     DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs);
@@ -46,8 +53,10 @@ public:
     StorePath requireDrvPath() const;
     StorePath queryOutPath() const;
     std::string queryOutputName() const;
-    /** Return the unordered map of output names to (optional) output paths.
+    /**
-     * The "outputs to install" are determined by `meta.outputsToInstall`. */
+     * Return the unordered map of output names to (optional) output paths.
+     * The "outputs to install" are determined by `meta.outputsToInstall`.
+     */
     Outputs queryOutputs(bool withPaths = true, bool onlyOutputsToInstall = false);
 
     StringSet queryMetaNames();
@@ -79,8 +88,10 @@ typedef std::list<DrvInfo> DrvInfos;
 #endif
 
 
-/* If value `v' denotes a derivation, return a DrvInfo object
+/**
-   describing it. Otherwise return nothing. */
+ * If value `v` denotes a derivation, return a DrvInfo object
+ * describing it. Otherwise return nothing.
+ */
 std::optional<DrvInfo> getDerivation(EvalState & state,
     Value & v, bool ignoreAssertionFailures);
 

src/libexpr/json-to-value.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "eval.hh"
 

src/libexpr/local.mk

@@ -46,3 +46,5 @@ $(foreach i, $(wildcard src/libexpr/flake/*.hh), \
 $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
 
 $(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh
+
+src/libexpr/primops/fromTOML.o:	ERROR_SWITCH_ENUM =

src/libexpr/nixexpr.cc

@@ -3,6 +3,7 @@
 #include "eval.hh"
 #include "symbol-table.hh"
 #include "util.hh"
+#include "print.hh"
 
 #include <cstdlib>
 
@@ -60,45 +61,12 @@ Pos::operator std::shared_ptr<AbstractPos>() const
     return pos;
 }
 
-/* Displaying abstract syntax trees. */
+// FIXME: remove, because *symbols* are abstract and do not have a single
-
+//        textual representation; see printIdentifier()
-static void showString(std::ostream & str, std::string_view s)
-{
-    str << '"';
-    for (auto c : s)
-        if (c == '"' || c == '\\' || c == '$') str << "\\" << c;
-        else if (c == '\n') str << "\\n";
-        else if (c == '\r') str << "\\r";
-        else if (c == '\t') str << "\\t";
-        else str << c;
-    str << '"';
-}
-
 std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol)
 {
     std::string_view s = symbol;
-
+    return printIdentifier(str, s);
-    if (s.empty())
-        str << "\"\"";
-    else if (s == "if") // FIXME: handle other keywords
-        str << '"' << s << '"';
-    else {
-        char c = s[0];
-        if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) {
-            showString(str, s);
-            return str;
-        }
-        for (auto c : s)
-            if (!((c >= 'a' && c <= 'z') ||
-                  (c >= 'A' && c <= 'Z') ||
-                  (c >= '0' && c <= '9') ||
-                  c == '_' || c == '\'' || c == '-')) {
-                showString(str, s);
-                return str;
-            }
-        str << s;
-    }
-    return str;
 }
 
 void Expr::show(const SymbolTable & symbols, std::ostream & str) const
@@ -118,7 +86,7 @@ void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const
 
 void ExprString::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    showString(str, s);
+    printLiteralString(str, s);
 }
 
 void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const

src/libexpr/nixexpr.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include <map>
 #include <vector>
@@ -21,7 +22,9 @@ MakeError(UndefinedVarError, Error);
 MakeError(MissingArgumentError, EvalError);
 MakeError(RestrictedPathError, Error);
 
-/* Position objects. */
+/**
+ * Position objects.
+ */
 struct Pos
 {
     uint32_t line;
@@ -132,7 +135,9 @@ class EvalState;
 struct StaticEnv;
 
 
-/* An attribute path is a sequence of attribute names. */
+/**
+ * An attribute path is a sequence of attribute names.
+ */
 struct AttrName
 {
     Symbol symbol;
@@ -212,11 +217,11 @@ struct ExprVar : Expr
        or function argument) or from a "with". */
     bool fromWith;
 
-    /* In the former case, the value is obtained by going `level'
+    /* In the former case, the value is obtained by going `level`
        levels up from the current environment and getting the
-       `displ'th value in that environment.  In the latter case, the
+       `displ`th value in that environment.  In the latter case, the
-       value is obtained by getting the attribute named `name' from
+       value is obtained by getting the attribute named `name` from
-       the set stored in the environment that is `level' levels up
+       the set stored in the environment that is `level` levels up
        from the current one.*/
     Level level;
     Displacement displ;

src/libexpr/primops.cc

@@ -577,6 +577,9 @@ struct CompareValues
                 return v1->integer < v2->fpoint;
             if (v1->type() != v2->type())
                 state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow<EvalError>();
+            // Allow selecting a subset of enum values
+            #pragma GCC diagnostic push
+            #pragma GCC diagnostic ignored "-Wswitch-enum"
             switch (v1->type()) {
                 case nInt:
                     return v1->integer < v2->integer;
@@ -599,6 +602,7 @@ struct CompareValues
                     }
                 default:
                     state.error("cannot compare %s with %s; values of that type are incomparable", showType(*v1), showType(*v2)).debugThrow<EvalError>();
+            #pragma GCC diagnostic pop
             }
         } catch (Error & e) {
             if (!errorCtx.empty())

src/libexpr/primops.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "eval.hh"
 
@@ -22,9 +23,11 @@ struct RegisterPrimOp
     typedef std::vector<Info> PrimOps;
     static PrimOps * primOps;
 
-    /* You can register a constant by passing an arity of 0. fun
+    /**
-       will get called during EvalState initialization, so there
+     * You can register a constant by passing an arity of 0. fun
-       may be primops not yet added and builtins is not yet sorted. */
+     * will get called during EvalState initialization, so there
+     * may be primops not yet added and builtins is not yet sorted.
+     */
     RegisterPrimOp(
         std::string name,
         size_t arity,
@@ -37,10 +40,14 @@ struct RegisterPrimOp
    may wish to use them in limited contexts without globally enabling
    them. */
 
-/* Load a ValueInitializer from a DSO and return whatever it initializes */
+/**
+ * Load a ValueInitializer from a DSO and return whatever it initializes
+ */
 void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Value & v);
 
-/* Execute a program and parse its output */
+/**
+ * Execute a program and parse its output
+ */
 void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v);
 
 }

src/libexpr/print.cc

@@ -0,0 +1,78 @@
+#include "print.hh"
+
+namespace nix {
+
+std::ostream &
+printLiteralString(std::ostream & str, const std::string_view string)
+{
+    str << "\"";
+    for (auto i = string.begin(); i != string.end(); ++i) {
+        if (*i == '\"' || *i == '\\') str << "\\" << *i;
+        else if (*i == '\n') str << "\\n";
+        else if (*i == '\r') str << "\\r";
+        else if (*i == '\t') str << "\\t";
+        else if (*i == '$' && *(i+1) == '{') str << "\\" << *i;
+        else str << *i;
+    }
+    str << "\"";
+    return str;
+}
+
+std::ostream &
+printLiteralBool(std::ostream & str, bool boolean)
+{
+    str << (boolean ? "true" : "false");
+    return str;
+}
+
+std::ostream &
+printIdentifier(std::ostream & str, std::string_view s) {
+    if (s.empty())
+        str << "\"\"";
+    else if (s == "if") // FIXME: handle other keywords
+        str << '"' << s << '"';
+    else {
+        char c = s[0];
+        if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) {
+            printLiteralString(str, s);
+            return str;
+        }
+        for (auto c : s)
+            if (!((c >= 'a' && c <= 'z') ||
+                  (c >= 'A' && c <= 'Z') ||
+                  (c >= '0' && c <= '9') ||
+                  c == '_' || c == '\'' || c == '-')) {
+                printLiteralString(str, s);
+                return str;
+            }
+        str << s;
+    }
+    return str;
+}
+
+// FIXME: keywords
+static bool isVarName(std::string_view s)
+{
+    if (s.size() == 0) return false;
+    char c = s[0];
+    if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false;
+    for (auto & i : s)
+        if (!((i >= 'a' && i <= 'z') ||
+              (i >= 'A' && i <= 'Z') ||
+              (i >= '0' && i <= '9') ||
+              i == '_' || i == '-' || i == '\''))
+            return false;
+    return true;
+}
+
+std::ostream &
+printAttributeName(std::ostream & str, std::string_view name) {
+    if (isVarName(name))
+        str << name;
+    else
+        printLiteralString(str, name);
+    return str;
+}
+
+
+}

src/libexpr/print.hh

@@ -0,0 +1,48 @@
+#pragma once
+/**
+ * @file
+ * @brief Common printing functions for the Nix language
+ *
+ * While most types come with their own methods for printing, they share some
+ * functions that are placed here.
+ */
+
+#include <iostream>
+
+namespace nix {
+    /**
+     * Print a string as a Nix string literal.
+     *
+     * Quotes and fairly minimal escaping are added.
+     *
+     * @param s The logical string
+     */
+    std::ostream & printLiteralString(std::ostream & o, std::string_view s);
+    inline std::ostream & printLiteralString(std::ostream & o, const char * s) {
+        return printLiteralString(o, std::string_view(s));
+    }
+    inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) {
+        return printLiteralString(o, std::string_view(s));
+    }
+
+    /** Print `true` or `false`. */
+    std::ostream & printLiteralBool(std::ostream & o, bool b);
+
+    /**
+     * Print a string as an attribute name in the Nix expression language syntax.
+     *
+     * Prints a quoted string if necessary.
+     */
+    std::ostream & printAttributeName(std::ostream & o, std::string_view s);
+
+    /**
+     * Print a string as an identifier in the Nix expression language syntax.
+     *
+     * FIXME: "identifier" is ambiguous. Identifiers do not have a single
+     *        textual representation. They can be used in variable references,
+     *        let bindings, left-hand sides or attribute names in a select
+     *        expression, or something else entirely, like JSON. Use one of the
+     *        `print*` functions instead.
+     */
+    std::ostream & printIdentifier(std::ostream & o, std::string_view s);
+}

src/libexpr/symbol-table.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include <list>
 #include <map>
@@ -9,15 +10,11 @@
 
 namespace nix {
 
-/* Symbol table used by the parser and evaluator to represent and look
+/**
-   up identifiers and attributes efficiently.  SymbolTable::create()
+ * This class mainly exists to give us an operator<< for ostreams. We could also
-   converts a string into a symbol.  Symbols have the property that
+ * return plain strings from SymbolTable, but then we'd have to wrap every
-   they can be compared efficiently (using an equality test),
+ * instance of a symbol that is fmt()ed, which is inconvenient and error-prone.
-   because the symbol table stores only one copy of each string. */
+ */
-
-/* This class mainly exists to give us an operator<< for ostreams. We could also
-   return plain strings from SymbolTable, but then we'd have to wrap every
-   instance of a symbol that is fmt()ed, which is inconvenient and error-prone. */
 class SymbolStr
 {
     friend class SymbolTable;
@@ -46,6 +43,11 @@ public:
     friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol);
 };
 
+/**
+ * Symbols have the property that they can be compared efficiently
+ * (using an equality test), because the symbol table stores only one
+ * copy of each string.
+ */
 class Symbol
 {
     friend class SymbolTable;
@@ -65,6 +67,10 @@ public:
     bool operator!=(const Symbol other) const { return id != other.id; }
 };
 
+/**
+ * Symbol table used by the parser and evaluator to represent and look
+ * up identifiers and attributes efficiently.
+ */
 class SymbolTable
 {
 private:
@@ -73,6 +79,9 @@ private:
 
 public:
 
+    /**
+     * converts a string into a symbol.
+     */
     Symbol create(std::string_view s)
     {
         // Most symbols are looked up more than once, so we trade off insertion performance

src/libexpr/tests/libexpr.hh

@@ -1,3 +1,6 @@
+#pragma once
+///@file
+
 #include <gtest/gtest.h>
 #include <gmock/gmock.h>
 

src/libexpr/tests/value/context.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include <rapidcheck/gen/Arbitrary.h>
 

src/libexpr/value-to-json.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "nixexpr.hh"
 #include "eval.hh"

src/libexpr/value-to-xml.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "nixexpr.hh"
 #include "eval.hh"

src/libexpr/value.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include <cassert>
 
@@ -35,9 +36,11 @@ typedef enum {
     tFloat
 } InternalType;
 
-// This type abstracts over all actual value types in the language,
+/**
-// grouping together implementation details like tList*, different function
+ * This type abstracts over all actual value types in the language,
-// types, and types in non-normal form (so thunks and co.)
+ * grouping together implementation details like tList*, different function
+ * types, and types in non-normal form (so thunks and co.)
+ */
 typedef enum {
     nThunk,
     nInt,
@@ -69,38 +72,51 @@ class XMLWriter;
 typedef int64_t NixInt;
 typedef double NixFloat;
 
-/* External values must descend from ExternalValueBase, so that
+/**
+ * External values must descend from ExternalValueBase, so that
  * type-agnostic nix functions (e.g. showType) can be implemented
  */
 class ExternalValueBase
 {
     friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
     protected:
-    /* Print out the value */
+    /**
+     * Print out the value
+     */
     virtual std::ostream & print(std::ostream & str) const = 0;
 
     public:
-    /* Return a simple string describing the type */
+    /**
+     * Return a simple string describing the type
+     */
     virtual std::string showType() const = 0;
 
-    /* Return a string to be used in builtins.typeOf */
+    /**
+     * Return a string to be used in builtins.typeOf
+     */
     virtual std::string typeOf() const = 0;
 
-    /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an
+    /**
+     * Coerce the value to a string. Defaults to uncoercable, i.e. throws an
      * error.
      */
     virtual std::string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const;
 
-    /* Compare to another value of the same type. Defaults to uncomparable,
+    /**
+     * Compare to another value of the same type. Defaults to uncomparable,
      * i.e. always false.
      */
     virtual bool operator ==(const ExternalValueBase & b) const;
 
-    /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */
+    /**
+     * Print the value as JSON. Defaults to unconvertable, i.e. throws an error
+     */
     virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict,
         PathSet & context, bool copyToStore = true) const;
 
-    /* Print the value as XML. Defaults to unevaluated */
+    /**
+     * Print the value as XML. Defaults to unevaluated
+     */
     virtual void printValueAsXML(EvalState & state, bool strict, bool location,
         XMLWriter & doc, PathSet & context, PathSet & drvsSeen,
         const PosIdx pos) const;
@@ -145,26 +161,28 @@ public:
         NixInt integer;
         bool boolean;
 
-        /* Strings in the evaluator carry a so-called `context' which
+        /**
-           is a list of strings representing store paths.  This is to
+         * Strings in the evaluator carry a so-called `context` which
-           allow users to write things like
+         * is a list of strings representing store paths.  This is to
+         * allow users to write things like
 
-             "--with-freetype2-library=" + freetype + "/lib"
+         *   "--with-freetype2-library=" + freetype + "/lib"
 
-           where `freetype' is a derivation (or a source to be copied
+         * where `freetype` is a derivation (or a source to be copied
-           to the store).  If we just concatenated the strings without
+         * to the store).  If we just concatenated the strings without
-           keeping track of the referenced store paths, then if the
+         * keeping track of the referenced store paths, then if the
-           string is used as a derivation attribute, the derivation
+         * string is used as a derivation attribute, the derivation
-           will not have the correct dependencies in its inputDrvs and
+         * will not have the correct dependencies in its inputDrvs and
-           inputSrcs.
+         * inputSrcs.
 
-           The semantics of the context is as follows: when a string
+         * The semantics of the context is as follows: when a string
-           with context C is used as a derivation attribute, then the
+         * with context C is used as a derivation attribute, then the
-           derivations in C will be added to the inputDrvs of the
+         * derivations in C will be added to the inputDrvs of the
-           derivation, and the other store paths in C will be added to
+         * derivation, and the other store paths in C will be added to
-           the inputSrcs of the derivations.
+         * the inputSrcs of the derivations.
 
-           For canonicity, the store paths should be in sorted order. */
+         * For canonicity, the store paths should be in sorted order.
+         */
         struct {
             const char * s;
             const char * * context; // must be in sorted order
@@ -196,8 +214,10 @@ public:
         NixFloat fpoint;
     };
 
-    // Returns the normal type of a Value. This only returns nThunk if the
+    /**
-    // Value hasn't been forceValue'd
+     * Returns the normal type of a Value. This only returns nThunk if
+     * the Value hasn't been forceValue'd
+     */
     inline ValueType type() const
     {
         switch (internalType) {
@@ -216,8 +236,10 @@ public:
         abort();
     }
 
-    /* After overwriting an app node, be sure to clear pointers in the
+    /**
-       Value to ensure that the target isn't kept alive unnecessarily. */
+     * After overwriting an app node, be sure to clear pointers in the
+     * Value to ensure that the target isn't kept alive unnecessarily.
+     */
     inline void clearValue()
     {
         app.left = app.right = 0;
@@ -365,9 +387,11 @@ public:
 
     PosIdx determinePos(const PosIdx pos) const;
 
-    /* Check whether forcing this value requires a trivial amount of
+    /**
-       computation. In particular, function applications are
+     * Check whether forcing this value requires a trivial amount of
-       non-trivial. */
+     * computation. In particular, function applications are
+     * non-trivial.
+     */
     bool isTrivial() const;
 
     NixStringContext getContext(const Store &);
@@ -413,7 +437,9 @@ typedef std::map<Symbol, ValueVector> ValueVectorMap;
 #endif
 
 
-/* A value allocated in traceable memory. */
+/**
+ * A value allocated in traceable memory.
+ */
 typedef std::shared_ptr<Value *> RootValue;
 
 RootValue allocRootValue(Value * v);

src/libexpr/value/context.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "util.hh"
 #include "comparator.hh"
@@ -27,9 +28,10 @@ public:
 
 class Store;
 
-/* Plain opaque path to some store object.
+/**
-
+ * Plain opaque path to some store object.
-   Encoded as just the path: ‘<path>’.
+ *
+ * Encoded as just the path: ‘<path>’.
  */
 struct NixStringContextElem_Opaque {
     StorePath path;
@@ -37,13 +39,14 @@ struct NixStringContextElem_Opaque {
     GENERATE_CMP(NixStringContextElem_Opaque, me->path);
 };
 
-/* Path to a derivation and its entire build closure.
+/**
-
+ * Path to a derivation and its entire build closure.
-   The path doesn't just refer to derivation itself and its closure, but
+ *
-   also all outputs of all derivations in that closure (including the
+ * The path doesn't just refer to derivation itself and its closure, but
-   root derivation).
+ * also all outputs of all derivations in that closure (including the
-
+ * root derivation).
-   Encoded in the form ‘=<drvPath>’.
+ *
+ * Encoded in the form ‘=<drvPath>’.
  */
 struct NixStringContextElem_DrvDeep {
     StorePath drvPath;
@@ -51,9 +54,10 @@ struct NixStringContextElem_DrvDeep {
     GENERATE_CMP(NixStringContextElem_DrvDeep, me->drvPath);
 };
 
-/* Derivation output.
+/**
-
+ * Derivation output.
-   Encoded in the form ‘!<output>!<drvPath>’.
+ *
+ * Encoded in the form ‘!<output>!<drvPath>’.
  */
 struct NixStringContextElem_Built {
     StorePath drvPath;
@@ -83,10 +87,11 @@ struct NixStringContextElem : _NixStringContextElem_Raw {
         return static_cast<Raw &>(*this);
     }
 
-    /* Decode a context string, one of:
+    /**
-       - ‘<path>’
+     * Decode a context string, one of:
-       - ‘=<path>’
+     * - ‘<path>’
-       - ‘!<name>!<path>’
+     * - ‘=<path>’
+     * - ‘!<name>!<path>’
      */
     static NixStringContextElem parse(const Store & store, std::string_view s);
     std::string to_string(const Store & store) const;

src/libfetchers/attrs.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "types.hh"
 

src/libfetchers/cache.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "fetchers.hh"
 

src/libfetchers/fetch-settings.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "types.hh"
 #include "config.hh"

src/libfetchers/fetchers.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "types.hh"
 #include "hash.hh"
@@ -20,14 +21,14 @@ struct Tree
 
 struct InputScheme;
 
-/* The Input object is generated by a specific fetcher, based on the
+/**
+ * The Input object is generated by a specific fetcher, based on the
  * user-supplied input attribute in the flake.nix file, and contains
  * the information that the specific fetcher needs to perform the
  * actual fetch.  The Input object is most commonly created via the
  * "fromURL()" or "fromAttrs()" static functions which are provided
  * the url or attrset specified in the flake file.
  */
-
 struct Input
 {
     friend struct InputScheme;
@@ -37,7 +38,9 @@ struct Input
     bool locked = false;
     bool direct = true;
 
-    /* path of the parent of this input, used for relative path resolution */
+    /**
+     * path of the parent of this input, used for relative path resolution
+     */
     std::optional<Path> parent;
 
 public:
@@ -55,27 +58,35 @@ public:
 
     Attrs toAttrs() const;
 
-    /* Check whether this is a "direct" input, that is, not
+    /**
-       one that goes through a registry. */
+     * Check whether this is a "direct" input, that is, not
+     * one that goes through a registry.
+     */
     bool isDirect() const { return direct; }
 
-    /* Check whether this is a "locked" input, that is,
+    /**
-       one that contains a commit hash or content hash. */
+     * Check whether this is a "locked" input, that is,
+     * one that contains a commit hash or content hash.
+     */
     bool isLocked() const { return locked; }
 
-    /* Check whether the input carries all necessary info required
+    /**
-       for cache insertion and substitution.
+     * Check whether the input carries all necessary info required
-       These fields are used to uniquely identify cached trees
+     * for cache insertion and substitution.
-       within the "tarball TTL" window without necessarily
+     * These fields are used to uniquely identify cached trees
-       indicating that the input's origin is unchanged. */
+     * within the "tarball TTL" window without necessarily
+     * indicating that the input's origin is unchanged.
+     */
     bool hasAllInfo() const;
 
     bool operator ==(const Input & other) const;
 
     bool contains(const Input & other) const;
 
-    /* Fetch the input into the Nix store, returning the location in
+    /**
-       the Nix store and the locked input. */
+     * Fetch the input into the Nix store, returning the location in
+     * the Nix store and the locked input.
+     */
     std::pair<Tree, Input> fetch(ref<Store> store) const;
 
     Input applyOverrides(
@@ -104,7 +115,8 @@ public:
 };
 
 
-/* The InputScheme represents a type of fetcher.  Each fetcher
+/**
+ * The InputScheme represents a type of fetcher.  Each fetcher
  * registers with nix at startup time.  When processing an input for a
  * flake, each scheme is given an opportunity to "recognize" that
  * input from the url or attributes in the flake file's specification

src/libfetchers/registry.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "types.hh"
 #include "fetchers.hh"

src/libmain/common-args.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "args.hh"
 

src/libmain/loggers.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "types.hh"
 

src/libmain/progress-bar.cc

@@ -72,6 +72,7 @@ private:
         uint64_t corruptedPaths = 0, untrustedPaths = 0;
 
         bool active = true;
+        bool paused = false;
         bool haveUpdate = true;
     };
 
@@ -120,6 +121,18 @@ public:
         updateThread.join();
     }
 
+    void pause() override {
+        state_.lock()->paused = true;
+        writeToStderr("\r\e[K");
+    }
+
+    void resume() override {
+        state_.lock()->paused = false;
+        writeToStderr("\r\e[K");
+        state_.lock()->haveUpdate = true;
+        updateCV.notify_one();
+    }
+
     bool isVerbose() override
     {
         return printBuildLogs;
@@ -339,7 +352,7 @@ public:
         auto nextWakeup = std::chrono::milliseconds::max();
 
         state.haveUpdate = false;
-        if (!state.active) return nextWakeup;
+        if (state.paused || !state.active) return nextWakeup;
 
         std::string line;
 

src/libmain/progress-bar.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "logging.hh"
 

src/libmain/shared.cc

@@ -10,7 +10,6 @@
 #include <cctype>
 #include <exception>
 #include <iostream>
-#include <mutex>
 
 #include <cstdlib>
 #include <sys/time.h>
@@ -20,16 +19,9 @@
 #ifdef __linux__
 #include <features.h>
 #endif
-#ifdef __GLIBC__
-#include <gnu/lib-names.h>
-#include <nss.h>
-#include <dlfcn.h>
-#endif
 
 #include <openssl/crypto.h>
 
-#include <sodium.h>
-
 
 namespace nix {
 
@@ -115,57 +107,6 @@ std::string getArg(const std::string & opt,
     return *i;
 }
 
-
-#if OPENSSL_VERSION_NUMBER < 0x10101000L
-/* OpenSSL is not thread-safe by default - it will randomly crash
-   unless the user supplies a mutex locking function. So let's do
-   that. */
-static std::vector<std::mutex> opensslLocks;
-
-static void opensslLockCallback(int mode, int type, const char * file, int line)
-{
-    if (mode & CRYPTO_LOCK)
-        opensslLocks[type].lock();
-    else
-        opensslLocks[type].unlock();
-}
-#endif
-
-static std::once_flag dns_resolve_flag;
-
-static void preloadNSS() {
-    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
-       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
-       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
-       load its lookup libraries in the parent before any child gets a chance to. */
-    std::call_once(dns_resolve_flag, []() {
-#ifdef __GLIBC__
-        /* On linux, glibc will run every lookup through the nss layer.
-         * That means every lookup goes, by default, through nscd, which acts as a local
-         * cache.
-         * Because we run builds in a sandbox, we also remove access to nscd otherwise
-         * lookups would leak into the sandbox.
-         *
-         * But now we have a new problem, we need to make sure the nss_dns backend that
-         * does the dns lookups when nscd is not available is loaded or available.
-         *
-         * We can't make it available without leaking nix's environment, so instead we'll
-         * load the backend, and configure nss so it does not try to run dns lookups
-         * through nscd.
-         *
-         * This is technically only used for builtins:fetch* functions so we only care
-         * about dns.
-         *
-         * All other platforms are unaffected.
-         */
-        if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW))
-            warn("unable to load nss_dns backend");
-        // FIXME: get hosts entry from nsswitch.conf.
-        __nss_configure_lookup("hosts", "files dns");
-#endif
-    });
-}
-
 static void sigHandler(int signo) { }
 
 
@@ -177,16 +118,7 @@ void initNix()
     std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf));
 #endif
 
-#if OPENSSL_VERSION_NUMBER < 0x10101000L
+    initLibStore();
-    /* Initialise OpenSSL locking. */
-    opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks());
-    CRYPTO_set_locking_callback(opensslLockCallback);
-#endif
-
-    if (sodium_init() == -1)
-        throw Error("could not initialise libsodium");
-
-    loadConfFile();
 
     startSignalHandlerThread();
 
@@ -223,7 +155,10 @@ void initNix()
     if (sigaction(SIGTRAP, &act, 0)) throw SysError("handling SIGTRAP");
 #endif
 
-    /* Register a SIGSEGV handler to detect stack overflows. */
+    /* Register a SIGSEGV handler to detect stack overflows.
+       Why not initLibExpr()? initGC() is essentially that, but
+       detectStackOverflow is not an instance of the init function concept, as
+       it may have to be invoked more than once per process. */
     detectStackOverflow();
 
     /* There is no privacy in the Nix system ;-)  At least not for
@@ -236,16 +171,6 @@ void initNix()
     gettimeofday(&tv, 0);
     srandom(tv.tv_usec);
 
-    /* On macOS, don't use the per-session TMPDIR (as set e.g. by
-       sshd). This breaks build users because they don't have access
-       to the TMPDIR, in particular in ‘nix-store --serve’. */
-#if __APPLE__
-    if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
-        unsetenv("TMPDIR");
-#endif
-
-    preloadNSS();
-    initLibStore();
 }
 
 

src/libmain/shared.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "util.hh"
 #include "args.hh"
@@ -24,7 +25,9 @@ public:
 
 int handleExceptions(const std::string & programName, std::function<void()> fun);
 
-/* Don't forget to call initPlugins() after settings are initialized! */
+/**
+ * Don't forget to call initPlugins() after settings are initialized!
+ */
 void initNix();
 
 void parseCmdLine(int argc, char * * argv,
@@ -35,7 +38,9 @@ void parseCmdLine(const std::string & programName, const Strings & args,
 
 void printVersion(const std::string & programName);
 
-/* Ugh.  No better place to put this. */
+/**
+ * Ugh.  No better place to put this.
+ */
 void printGCWarning();
 
 class Store;
@@ -74,11 +79,15 @@ struct LegacyArgs : public MixCommonArgs
 };
 
 
-/* Show the manual page for the specified program. */
+/**
+ * Show the manual page for the specified program.
+ */
 void showManPage(const std::string & name);
 
-/* The constructor of this class starts a pager if stdout is a
+/**
-   terminal and $PAGER is set. Stdout is redirected to the pager. */
+ * The constructor of this class starts a pager if stdout is a
+ * terminal and $PAGER is set. Stdout is redirected to the pager.
+ */
 class RunPager
 {
 public:
@@ -109,28 +118,34 @@ struct PrintFreed
 };
 
 
-/* Install a SIGSEGV handler to detect stack overflows. */
+/**
+ * Install a SIGSEGV handler to detect stack overflows.
+ */
 void detectStackOverflow();
 
-/* Pluggable behavior to run in case of a stack overflow.
+/**
-
+ * Pluggable behavior to run in case of a stack overflow.
-   Default value: defaultStackOverflowHandler.
+ *
-
+ * Default value: defaultStackOverflowHandler.
-   This is called by the handler installed by detectStackOverflow().
+ *
-
+ * This is called by the handler installed by detectStackOverflow().
-   This gives Nix library consumers a limit opportunity to report the error
+ *
-   condition. The handler should exit the process.
+ * This gives Nix library consumers a limit opportunity to report the error
-   See defaultStackOverflowHandler() for a reference implementation.
+ * condition. The handler should exit the process.
-
+ * See defaultStackOverflowHandler() for a reference implementation.
-   NOTE: Use with diligence, because this runs in the signal handler, with very
+ *
-   limited stack space and a potentially a corrupted heap, all while the failed
+ * NOTE: Use with diligence, because this runs in the signal handler, with very
-   thread is blocked indefinitely. All functions called must be reentrant. */
+ * limited stack space and a potentially a corrupted heap, all while the failed
+ * thread is blocked indefinitely. All functions called must be reentrant.
+ */
 extern std::function<void(siginfo_t * info, void * ctx)> stackOverflowHandler;
 
-/* The default, robust implementation of stackOverflowHandler.
+/**
-
+ * The default, robust implementation of stackOverflowHandler.
-   Prints an error message directly to stderr using a syscall instead of the
+ *
-   logger. Exits the process immediately after. */
+ * Prints an error message directly to stderr using a syscall instead of the
+ * logger. Exits the process immediately after.
+ */
 void defaultStackOverflowHandler(siginfo_t * info, void * ctx);
 
 }

src/libstore/binary-cache-store.cc

@@ -315,6 +315,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n
                 },
                 .references = {
                     .others = references,
+                    // caller is not capable of creating a self-reference, because this is content-addressed without modulus
                     .self = false,
                 },
             },
@@ -433,6 +434,7 @@ StorePath BinaryCacheStore::addToStore(
                 },
                 .references = {
                     .others = references,
+                    // caller is not capable of creating a self-reference, because this is content-addressed without modulus
                     .self = false,
                 },
             },

src/libstore/binary-cache-store.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "crypto.hh"
 #include "store-api.hh"
@@ -45,6 +46,11 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
         )"};
 };
 
+
+/**
+ * @note subclasses must implement at least one of the two
+ * virtual getFile() methods.
+ */
 class BinaryCacheStore : public virtual BinaryCacheStoreConfig,
     public virtual Store,
     public virtual LogStore
@@ -74,14 +80,15 @@ public:
         std::string && data,
         const std::string & mimeType);
 
-    /* Note: subclasses must implement at least one of the two
+    /**
-       following getFile() methods. */
+     * Dump the contents of the specified file to a sink.
-
+     */
-    /* Dump the contents of the specified file to a sink. */
     virtual void getFile(const std::string & path, Sink & sink);
 
-    /* Fetch the specified file and call the specified callback with
+    /**
-       the result. A subclass may implement this asynchronously. */
+     * Fetch the specified file and call the specified callback with
+     * the result. A subclass may implement this asynchronously.
+     */
     virtual void getFile(
         const std::string & path,
         Callback<std::optional<std::string>> callback) noexcept;

src/libstore/build-result.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "realisation.hh"
 #include "derived-path.hh"
@@ -11,9 +12,12 @@ namespace nix {
 
 struct BuildResult
 {
-    /* Note: don't remove status codes, and only add new status codes
+    /**
-       at the end of the list, to prevent client/server
+     * @note This is directly used in the nix-store --serve protocol.
-       incompatibilities in the nix-store --serve protocol. */
+     * That means we need to worry about compatability across versions.
+     * Therefore, don't remove status codes, and only add new status
+     * codes at the end of the list.
+     */
     enum Status {
         Built = 0,
         Substituted,
@@ -21,8 +25,10 @@ struct BuildResult
         PermanentFailure,
         InputRejected,
         OutputRejected,
-        TransientFailure, // possibly transient
+        /// possibly transient
-        CachedFailure, // no longer used
+        TransientFailure,
+        /// no longer used
+        CachedFailure,
         TimedOut,
         MiscFailure,
         DependencyFailed,
@@ -32,7 +38,12 @@ struct BuildResult
         NoSubstituters,
     } status = MiscFailure;
 
-    // FIXME: include entire ErrorInfo object.
+    /**
+     * Information about the error if the build failed.
+     *
+     * @todo This should be an entire ErrorInfo object, not just a
+     * string, for richer information.
+     */
     std::string errorMsg;
 
     std::string toString() const {
@@ -52,33 +63,46 @@ struct BuildResult
                 case LogLimitExceeded: return "LogLimitExceeded";
                 case NotDeterministic: return "NotDeterministic";
                 case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid";
+                case NoSubstituters: return "NoSubstituters";
                 default: return "Unknown";
             };
         }();
         return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg);
     }
 
-    /* How many times this build was performed. */
+    /**
+     * How many times this build was performed.
+     */
     unsigned int timesBuilt = 0;
 
-    /* If timesBuilt > 1, whether some builds did not produce the same
+    /**
-       result. (Note that 'isNonDeterministic = false' does not mean
+     * If timesBuilt > 1, whether some builds did not produce the same
-       the build is deterministic, just that we don't have evidence of
+     * result. (Note that 'isNonDeterministic = false' does not mean
-       non-determinism.) */
+     * the build is deterministic, just that we don't have evidence of
+     * non-determinism.)
+     */
     bool isNonDeterministic = false;
 
-    /* The derivation we built or the store path we substituted. */
+    /**
+     * The derivation we built or the store path we substituted.
+     */
     DerivedPath path;
 
-    /* For derivations, a mapping from the names of the wanted outputs
+    /**
-       to actual paths. */
+     * For derivations, a mapping from the names of the wanted outputs
+     * to actual paths.
+     */
     DrvOutputs builtOutputs;
 
-    /* The start/stop times of the build (or one of the rounds, if it
+    /**
-       was repeated). */
+     * The start/stop times of the build (or one of the rounds, if it
+     * was repeated).
+     */
     time_t startTime = 0, stopTime = 0;
 
-    /* User and system CPU time the build took. */
+    /**
+     * User and system CPU time the build took.
+     */
     std::optional<std::chrono::microseconds> cpuUser, cpuSystem;
 
     bool success()

src/libstore/build/derivation-goal.cc

@@ -911,7 +911,11 @@ void DerivationGoal::buildDone()
                     msg += line;
                     msg += "\n";
                 }
-                msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.",
+                auto nixLogCommand = experimentalFeatureSettings.isEnabled(Xp::NixCommand)
+                    ? "nix log"
+                    : "nix-store -l";
+                msg += fmt("For full logs, run '" ANSI_BOLD "%s %s" ANSI_NORMAL "'.",
+                    nixLogCommand,
                     worker.store.printStorePath(drvPath));
             }
 

src/libstore/build/derivation-goal.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "parsed-derivations.hh"
 #include "lock.hh"
@@ -15,8 +16,10 @@ struct HookInstance;
 
 typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
 
-/* Unless we are repairing, we don't both to test validity and just assume it,
+/**
-   so the choices are `Absent` or `Valid`. */
+ * Unless we are repairing, we don't both to test validity and just assume it,
+ * so the choices are `Absent` or `Valid`.
+ */
 enum struct PathStatus {
     Corrupt,
     Absent,
@@ -26,11 +29,15 @@ enum struct PathStatus {
 struct InitialOutputStatus {
     StorePath path;
     PathStatus status;
-    /* Valid in the store, and additionally non-corrupt if we are repairing */
+    /**
+     * Valid in the store, and additionally non-corrupt if we are repairing
+     */
     bool isValid() const {
         return status == PathStatus::Valid;
     }
-    /* Merely present, allowed to be corrupt */
+    /**
+     * Merely present, allowed to be corrupt
+     */
     bool isPresent() const {
         return status == PathStatus::Corrupt
             || status == PathStatus::Valid;
@@ -45,59 +52,87 @@ struct InitialOutput {
 
 struct DerivationGoal : public Goal
 {
-    /* Whether to use an on-disk .drv file. */
+    /**
+     * Whether to use an on-disk .drv file.
+     */
     bool useDerivation;
 
-    /* The path of the derivation. */
+    /** The path of the derivation. */
     StorePath drvPath;
 
-    /* The goal for the corresponding resolved derivation */
+    /**
+     * The goal for the corresponding resolved derivation
+     */
     std::shared_ptr<DerivationGoal> resolvedDrvGoal;
 
-    /* The specific outputs that we need to build.  Empty means all of
+    /**
-       them. */
+     * The specific outputs that we need to build.  Empty means all of
+     * them.
+     */
     OutputsSpec wantedOutputs;
 
-    /* Mapping from input derivations + output names to actual store
+    /**
-       paths. This is filled in by waiteeDone() as each dependency
+     * Mapping from input derivations + output names to actual store
-       finishes, before inputsRealised() is reached, */
+     * paths. This is filled in by waiteeDone() as each dependency
+     * finishes, before inputsRealised() is reached.
+     */
     std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
 
-    /* Whether additional wanted outputs have been added. */
+    /**
+     * Whether additional wanted outputs have been added.
+     */
     bool needRestart = false;
 
-    /* Whether to retry substituting the outputs after building the
+    /**
-       inputs. This is done in case of an incomplete closure. */
+     * Whether to retry substituting the outputs after building the
+     * inputs. This is done in case of an incomplete closure.
+     */
     bool retrySubstitution = false;
 
-    /* Whether we've retried substitution, in which case we won't try
+    /**
-       again. */
+     * Whether we've retried substitution, in which case we won't try
+     * again.
+     */
     bool retriedSubstitution = false;
 
-    /* The derivation stored at drvPath. */
+    /**
+     * The derivation stored at drvPath.
+     */
     std::unique_ptr<Derivation> drv;
 
     std::unique_ptr<ParsedDerivation> parsedDrv;
 
-    /* The remainder is state held during the build. */
+    /**
+     * The remainder is state held during the build.
+     */
 
-    /* Locks on (fixed) output paths. */
+    /**
+     * Locks on (fixed) output paths.
+     */
     PathLocks outputLocks;
 
-    /* All input paths (that is, the union of FS closures of the
+    /**
-       immediate input paths). */
+     * All input paths (that is, the union of FS closures of the
+     * immediate input paths).
+     */
     StorePathSet inputPaths;
 
     std::map<std::string, InitialOutput> initialOutputs;
 
-    /* File descriptor for the log file. */
+    /**
+     * File descriptor for the log file.
+     */
     AutoCloseFD fdLogFile;
     std::shared_ptr<BufferedSink> logFileSink, logSink;
 
-    /* Number of bytes received from the builder's stdout/stderr. */
+    /**
+     * Number of bytes received from the builder's stdout/stderr.
+     */
     unsigned long logSize;
 
-    /* The most recent log lines. */
+    /**
+     * The most recent log lines.
+     */
     std::list<std::string> logTail;
 
     std::string currentLogLine;
@@ -105,10 +140,14 @@ struct DerivationGoal : public Goal
 
     std::string currentHookLine;
 
-    /* The build hook. */
+    /**
+     * The build hook.
+     */
     std::unique_ptr<HookInstance> hook;
 
-    /* The sort of derivation we are building. */
+    /**
+     * The sort of derivation we are building.
+     */
     DerivationType derivationType;
 
     typedef void (DerivationGoal::*GoalState)();
@@ -120,12 +159,16 @@ struct DerivationGoal : public Goal
 
     std::unique_ptr<Activity> act;
 
-    /* Activity that denotes waiting for a lock. */
+    /**
+     * Activity that denotes waiting for a lock.
+     */
     std::unique_ptr<Activity> actLock;
 
     std::map<ActivityId, Activity> builderActivities;
 
-    /* The remote machine on which we're building. */
+    /**
+     * The remote machine on which we're building.
+     */
     std::string machineName;
 
     DerivationGoal(const StorePath & drvPath,
@@ -142,10 +185,14 @@ struct DerivationGoal : public Goal
 
     void work() override;
 
-    /* Add wanted outputs to an already existing derivation goal. */
+    /**
+     * Add wanted outputs to an already existing derivation goal.
+     */
     void addWantedOutputs(const OutputsSpec & outputs);
 
-    /* The states. */
+    /**
+     * The states.
+     */
     void getDerivation();
     void loadDerivation();
     void haveDerivation();
@@ -159,28 +206,42 @@ struct DerivationGoal : public Goal
 
     void resolvedFinished();
 
-    /* Is the build hook willing to perform the build? */
+    /**
+     * Is the build hook willing to perform the build?
+     */
     HookReply tryBuildHook();
 
     virtual int getChildStatus();
 
-    /* Check that the derivation outputs all exist and register them
+    /**
-       as valid. */
+     * Check that the derivation outputs all exist and register them
+     * as valid.
+     */
     virtual DrvOutputs registerOutputs();
 
-    /* Open a log file and a pipe to it. */
+    /**
+     * Open a log file and a pipe to it.
+     */
     Path openLogFile();
 
-    /* Sign the newly built realisation if the store allows it */
+    /**
+     * Sign the newly built realisation if the store allows it
+     */
     virtual void signRealisation(Realisation&) {}
 
-    /* Close the log file. */
+    /**
+     * Close the log file.
+     */
     void closeLogFile();
 
-    /* Close the read side of the logger pipe. */
+    /**
+     * Close the read side of the logger pipe.
+     */
     virtual void closeReadPipes();
 
-    /* Cleanup hooks for buildDone() */
+    /**
+     * Cleanup hooks for buildDone()
+     */
     virtual void cleanupHookFinally();
     virtual void cleanupPreChildKill();
     virtual void cleanupPostChildKill();
@@ -190,30 +251,40 @@ struct DerivationGoal : public Goal
 
     virtual bool isReadDesc(int fd);
 
-    /* Callback used by the worker to write to the log. */
+    /**
+     * Callback used by the worker to write to the log.
+     */
     void handleChildOutput(int fd, std::string_view data) override;
     void handleEOF(int fd) override;
     void flushLine();
 
-    /* Wrappers around the corresponding Store methods that first consult the
+    /**
-       derivation.  This is currently needed because when there is no drv file
+     * Wrappers around the corresponding Store methods that first consult the
-       there also is no DB entry. */
+     * derivation.  This is currently needed because when there is no drv file
+     * there also is no DB entry.
+     */
     std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
     OutputPathMap queryDerivationOutputMap();
 
-    /* Update 'initialOutputs' to determine the current status of the
+    /**
-       outputs of the derivation. Also returns a Boolean denoting
+     * Update 'initialOutputs' to determine the current status of the
-       whether all outputs are valid and non-corrupt, and a
+     * outputs of the derivation. Also returns a Boolean denoting
-       'DrvOutputs' structure containing the valid and wanted
+     * whether all outputs are valid and non-corrupt, and a
-       outputs. */
+     * 'DrvOutputs' structure containing the valid and wanted
+     * outputs.
+     */
     std::pair<bool, DrvOutputs> checkPathValidity();
 
-    /* Aborts if any output is not valid or corrupt, and otherwise
+    /**
-       returns a 'DrvOutputs' structure containing the wanted
+     * Aborts if any output is not valid or corrupt, and otherwise
-       outputs. */
+     * returns a 'DrvOutputs' structure containing the wanted
+     * outputs.
+     */
     DrvOutputs assertPathValidity();
 
-    /* Forcibly kill the child process, if any. */
+    /**
+     * Forcibly kill the child process, if any.
+     */
     virtual void killChild();
 
     void repairClosure();

src/libstore/build/drv-output-substitution-goal.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "store-api.hh"
 #include "goal.hh"
@@ -10,24 +11,34 @@ namespace nix {
 
 class Worker;
 
-// Substitution of a derivation output.
+/**
-// This is done in three steps:
+ * Substitution of a derivation output.
-// 1. Fetch the output info from a substituter
+ * This is done in three steps:
-// 2. Substitute the corresponding output path
+ * 1. Fetch the output info from a substituter
-// 3. Register the output info
+ * 2. Substitute the corresponding output path
+ * 3. Register the output info
+ */
 class DrvOutputSubstitutionGoal : public Goal {
 
-    // The drv output we're trying to substitue
+    /**
+     * The drv output we're trying to substitue
+     */
     DrvOutput id;
 
-    // The realisation corresponding to the given output id.
+    /**
-    // Will be filled once we can get it.
+     * The realisation corresponding to the given output id.
+     * Will be filled once we can get it.
+     */
     std::shared_ptr<const Realisation> outputInfo;
 
-    /* The remaining substituters. */
+    /**
+     * The remaining substituters.
+     */
     std::list<ref<Store>> subs;
 
-    /* The current substituter. */
+    /**
+     * The current substituter.
+     */
     std::shared_ptr<Store> sub;
 
     struct DownloadState
@@ -38,7 +49,9 @@ class DrvOutputSubstitutionGoal : public Goal {
 
     std::shared_ptr<DownloadState> downloadState;
 
-    /* Whether a substituter failed. */
+    /**
+     * Whether a substituter failed.
+     */
     bool substituterFailed = false;
 
 public:

src/libstore/build/goal.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "types.hh"
 #include "store-api.hh"
@@ -6,11 +7,15 @@
 
 namespace nix {
 
-/* Forward definition. */
+/**
+ * Forward definition.
+ */
 struct Goal;
 class Worker;
 
-/* A pointer to a goal. */
+/**
+ * A pointer to a goal.
+ */
 typedef std::shared_ptr<Goal> GoalPtr;
 typedef std::weak_ptr<Goal> WeakGoalPtr;
 
@@ -18,48 +23,72 @@ struct CompareGoalPtrs {
     bool operator() (const GoalPtr & a, const GoalPtr & b) const;
 };
 
-/* Set of goals. */
+/**
+ * Set of goals.
+ */
 typedef std::set<GoalPtr, CompareGoalPtrs> Goals;
 typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
 
-/* A map of paths to goals (and the other way around). */
+/**
+ * A map of paths to goals (and the other way around).
+ */
 typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
 
 struct Goal : public std::enable_shared_from_this<Goal>
 {
     typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
 
-    /* Backlink to the worker. */
+    /**
+     * Backlink to the worker.
+     */
     Worker & worker;
 
-    /* Goals that this goal is waiting for. */
+    /**
+     * Goals that this goal is waiting for.
+     */
     Goals waitees;
 
-    /* Goals waiting for this one to finish.  Must use weak pointers
+    /**
-       here to prevent cycles. */
+     * Goals waiting for this one to finish.  Must use weak pointers
+     * here to prevent cycles.
+     */
     WeakGoals waiters;
 
-    /* Number of goals we are/were waiting for that have failed. */
+    /**
+     * Number of goals we are/were waiting for that have failed.
+     */
     size_t nrFailed = 0;
 
-    /* Number of substitution goals we are/were waiting for that
+    /**
-       failed because there are no substituters. */
+     * Number of substitution goals we are/were waiting for that
+     * failed because there are no substituters.
+     */
     size_t nrNoSubstituters = 0;
 
-    /* Number of substitution goals we are/were waiting for that
+    /**
-       failed because they had unsubstitutable references. */
+     * Number of substitution goals we are/were waiting for that
+     * failed because they had unsubstitutable references.
+     */
     size_t nrIncompleteClosure = 0;
 
-    /* Name of this goal for debugging purposes. */
+    /**
+     * Name of this goal for debugging purposes.
+     */
     std::string name;
 
-    /* Whether the goal is finished. */
+    /**
+     * Whether the goal is finished.
+     */
     ExitCode exitCode = ecBusy;
 
-    /* Build result. */
+    /**
+     * Build result.
+     */
     BuildResult buildResult;
 
-    /* Exception containing an error message, if any. */
+    /**
+     * Exception containing an error message, if any.
+     */
     std::optional<Error> ex;
 
     Goal(Worker & worker, DerivedPath path)
@@ -95,9 +124,11 @@ struct Goal : public std::enable_shared_from_this<Goal>
         return name;
     }
 
-    /* Callback in case of a timeout.  It should wake up its waiters,
+    /**
-       get rid of any running child processes that are being monitored
+     * Callback in case of a timeout.  It should wake up its waiters,
-       by the worker (important!), etc. */
+     * get rid of any running child processes that are being monitored
+     * by the worker (important!), etc.
+     */
     virtual void timedOut(Error && ex) = 0;
 
     virtual std::string key() = 0;

src/libstore/build/hook-instance.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "logging.hh"
 #include "serialise.hh"
@@ -7,16 +8,24 @@ namespace nix {
 
 struct HookInstance
 {
-    /* Pipes for talking to the build hook. */
+    /**
+     * Pipes for talking to the build hook.
+     */
     Pipe toHook;
 
-    /* Pipe for the hook's standard output/error. */
+    /**
+     * Pipe for the hook's standard output/error.
+     */
     Pipe fromHook;
 
-    /* Pipe for the builder's standard output/error. */
+    /**
+     * Pipe for the builder's standard output/error.
+     */
     Pipe builderOut;
 
-    /* The process ID of the hook. */
+    /**
+     * The process ID of the hook.
+     */
     Pid pid;
 
     FdSink sink;

src/libstore/build/local-derivation-goal.cc

@@ -1415,6 +1415,9 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
 
     virtual void addBuildLog(const StorePath & path, std::string_view log) override
     { unsupported("addBuildLog"); }
+
+    std::optional<TrustedFlag> isTrustedClient() override
+    { return NotTrusted; }
 };
 
 
@@ -1467,7 +1470,7 @@ void LocalDerivationGoal::startDaemon()
                 FdSink to(remote.get());
                 try {
                     daemon::processConnection(store, from, to,
-                        daemon::NotTrusted, daemon::Recursive);
+                        NotTrusted, daemon::Recursive);
                     debug("terminated daemon connection");
                 } catch (SysError &) {
                     ignoreException();

src/libstore/build/local-derivation-goal.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "derivation-goal.hh"
 #include "local-store.hh"
@@ -9,49 +10,75 @@ struct LocalDerivationGoal : public DerivationGoal
 {
     LocalStore & getLocalStore();
 
-    /* User selected for running the builder. */
+    /**
+     * User selected for running the builder.
+     */
     std::unique_ptr<UserLock> buildUser;
 
-    /* The process ID of the builder. */
+    /**
+     * The process ID of the builder.
+     */
     Pid pid;
 
-    /* The cgroup of the builder, if any. */
+    /**
+     * The cgroup of the builder, if any.
+     */
     std::optional<Path> cgroup;
 
-    /* The temporary directory. */
+    /**
+     * The temporary directory.
+     */
     Path tmpDir;
 
-    /* The path of the temporary directory in the sandbox. */
+    /**
+     * The path of the temporary directory in the sandbox.
+     */
     Path tmpDirInSandbox;
 
-    /* Master side of the pseudoterminal used for the builder's
+    /**
-       standard output/error. */
+     * Master side of the pseudoterminal used for the builder's
+     * standard output/error.
+     */
     AutoCloseFD builderOut;
 
-    /* Pipe for synchronising updates to the builder namespaces. */
+    /**
+     * Pipe for synchronising updates to the builder namespaces.
+     */
     Pipe userNamespaceSync;
 
-    /* The mount namespace and user namespace of the builder, used to add additional
+    /**
-       paths to the sandbox as a result of recursive Nix calls. */
+     * The mount namespace and user namespace of the builder, used to add additional
+     * paths to the sandbox as a result of recursive Nix calls.
+     */
     AutoCloseFD sandboxMountNamespace;
     AutoCloseFD sandboxUserNamespace;
 
-    /* On Linux, whether we're doing the build in its own user
+    /**
-       namespace. */
+     * On Linux, whether we're doing the build in its own user
+     * namespace.
+     */
     bool usingUserNamespace = true;
 
-    /* Whether we're currently doing a chroot build. */
+    /**
+     * Whether we're currently doing a chroot build.
+     */
     bool useChroot = false;
 
     Path chrootRootDir;
 
-    /* RAII object to delete the chroot directory. */
+    /**
+     * RAII object to delete the chroot directory.
+     */
     std::shared_ptr<AutoDelete> autoDelChroot;
 
-    /* Whether to run the build in a private network namespace. */
+    /**
+     * Whether to run the build in a private network namespace.
+     */
     bool privateNetwork = false;
 
-    /* Stuff we need to pass to initChild(). */
+    /**
+     * Stuff we need to pass to initChild().
+     */
     struct ChrootPath {
         Path source;
         bool optional;
@@ -70,30 +97,35 @@ struct LocalDerivationGoal : public DerivationGoal
     SandboxProfile additionalSandboxProfile;
 #endif
 
-    /* Hash rewriting. */
+    /**
+     * Hash rewriting.
+     */
     StringMap inputRewrites, outputRewrites;
     typedef map<StorePath, StorePath> RedirectedOutputs;
     RedirectedOutputs redirectedOutputs;
 
-    /* The outputs paths used during the build.
+    /**
-
+     * The outputs paths used during the build.
-       - Input-addressed derivations or fixed content-addressed outputs are
+     *
-         sometimes built when some of their outputs already exist, and can not
+     * - Input-addressed derivations or fixed content-addressed outputs are
-         be hidden via sandboxing. We use temporary locations instead and
+     *   sometimes built when some of their outputs already exist, and can not
-         rewrite after the build. Otherwise the regular predetermined paths are
+     *   be hidden via sandboxing. We use temporary locations instead and
-         put here.
+     *   rewrite after the build. Otherwise the regular predetermined paths are
-
+     *   put here.
-       - Floating content-addressed derivations do not know their final build
+     *
-         output paths until the outputs are hashed, so random locations are
+     * - Floating content-addressed derivations do not know their final build
-         used, and then renamed. The randomness helps guard against hidden
+     *   output paths until the outputs are hashed, so random locations are
-         self-references.
+     *   used, and then renamed. The randomness helps guard against hidden
+     *   self-references.
      */
     OutputPathMap scratchOutputs;
 
-    /* Path registration info from the previous round, if we're
+    /**
-       building multiple times. Since this contains the hash, it
+     * Path registration info from the previous round, if we're
-       allows us to compare whether two rounds produced the same
+     * building multiple times. Since this contains the hash, it
-       result. */
+     * allows us to compare whether two rounds produced the same
+     * result.
+     */
     std::map<Path, ValidPathInfo> prevInfos;
 
     uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); }
@@ -101,25 +133,37 @@ struct LocalDerivationGoal : public DerivationGoal
 
     const static Path homeDir;
 
-    /* The recursive Nix daemon socket. */
+    /**
+     * The recursive Nix daemon socket.
+     */
     AutoCloseFD daemonSocket;
 
-    /* The daemon main thread. */
+    /**
+     * The daemon main thread.
+     */
     std::thread daemonThread;
 
-    /* The daemon worker threads. */
+    /**
+     * The daemon worker threads.
+     */
     std::vector<std::thread> daemonWorkerThreads;
 
-    /* Paths that were added via recursive Nix calls. */
+    /**
+     * Paths that were added via recursive Nix calls.
+     */
     StorePathSet addedPaths;
 
-    /* Realisations that were added via recursive Nix calls. */
+    /**
+     * Realisations that were added via recursive Nix calls.
+     */
     std::set<DrvOutput> addedDrvOutputs;
 
-    /* Recursive Nix calls are only allowed to build or realize paths
+    /**
-       in the original input closure or added via a recursive Nix call
+     * Recursive Nix calls are only allowed to build or realize paths
-       (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
+     * in the original input closure or added via a recursive Nix call
-       /nix/store/<bla> is some arbitrary path in a binary cache). */
+     * (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
+     * /nix/store/<bla> is some arbitrary path in a binary cache).
+     */
     bool isAllowed(const StorePath & path)
     {
         return inputPaths.count(path) || addedPaths.count(path);
@@ -137,55 +181,81 @@ struct LocalDerivationGoal : public DerivationGoal
 
     virtual ~LocalDerivationGoal() override;
 
-    /* Whether we need to perform hash rewriting if there are valid output paths. */
+    /**
+     * Whether we need to perform hash rewriting if there are valid output paths.
+     */
     bool needsHashRewrite();
 
-    /* The additional states. */
+    /**
+     * The additional states.
+     */
     void tryLocalBuild() override;
 
-    /* Start building a derivation. */
+    /**
+     * Start building a derivation.
+     */
     void startBuilder();
 
-    /* Fill in the environment for the builder. */
+    /**
+     * Fill in the environment for the builder.
+     */
     void initEnv();
 
-    /* Setup tmp dir location. */
+    /**
+     * Setup tmp dir location.
+     */
     void initTmpDir();
 
-    /* Write a JSON file containing the derivation attributes. */
+    /**
+     * Write a JSON file containing the derivation attributes.
+     */
     void writeStructuredAttrs();
 
     void startDaemon();
 
     void stopDaemon();
 
-    /* Add 'path' to the set of paths that may be referenced by the
+    /**
-       outputs, and make it appear in the sandbox. */
+     * Add 'path' to the set of paths that may be referenced by the
+     * outputs, and make it appear in the sandbox.
+     */
     void addDependency(const StorePath & path);
 
-    /* Make a file owned by the builder. */
+    /**
+     * Make a file owned by the builder.
+     */
     void chownToBuilder(const Path & path);
 
     int getChildStatus() override;
 
-    /* Run the builder's process. */
+    /**
+     * Run the builder's process.
+     */
     void runChild();
 
-    /* Check that the derivation outputs all exist and register them
+    /**
-       as valid. */
+     * Check that the derivation outputs all exist and register them
+     * as valid.
+     */
     DrvOutputs registerOutputs() override;
 
     void signRealisation(Realisation &) override;
 
-    /* Check that an output meets the requirements specified by the
+    /**
-       'outputChecks' attribute (or the legacy
+     * Check that an output meets the requirements specified by the
-       '{allowed,disallowed}{References,Requisites}' attributes). */
+     * 'outputChecks' attribute (or the legacy
+     * '{allowed,disallowed}{References,Requisites}' attributes).
+     */
     void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
 
-    /* Close the read side of the logger pipe. */
+    /**
+     * Close the read side of the logger pipe.
+     */
     void closeReadPipes() override;
 
-    /* Cleanup hooks for buildDone() */
+    /**
+     * Cleanup hooks for buildDone()
+     */
     void cleanupHookFinally() override;
     void cleanupPreChildKill() override;
     void cleanupPostChildKill() override;
@@ -195,24 +265,36 @@ struct LocalDerivationGoal : public DerivationGoal
 
     bool isReadDesc(int fd) override;
 
-    /* Delete the temporary directory, if we have one. */
+    /**
+     * Delete the temporary directory, if we have one.
+     */
     void deleteTmpDir(bool force);
 
-    /* Forcibly kill the child process, if any. */
+    /**
+     * Forcibly kill the child process, if any.
+     */
     void killChild() override;
 
-    /* Kill any processes running under the build user UID or in the
+    /**
-       cgroup of the build. */
+     * Kill any processes running under the build user UID or in the
+     * cgroup of the build.
+     */
     void killSandbox(bool getStats);
 
-    /* Create alternative path calculated from but distinct from the
+    /**
-       input, so we can avoid overwriting outputs (or other store paths)
+     * Create alternative path calculated from but distinct from the
-       that already exist. */
+     * input, so we can avoid overwriting outputs (or other store paths)
+     * that already exist.
+     */
     StorePath makeFallbackPath(const StorePath & path);
-    /* Make a path to another based on the output name along with the
+
-       derivation hash. */
+    /**
-    /* FIXME add option to randomize, so we can audit whether our
+     * Make a path to another based on the output name along with the
-       rewrites caught everything */
+     * derivation hash.
+     *
+     * @todo Add option to randomize, so we can audit whether our
+     * rewrites caught everything
+     */
     StorePath makeFallbackPath(std::string_view outputName);
 };
 

src/libstore/build/personality.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include <string>
 

src/libstore/build/substitution-goal.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "lock.hh"
 #include "store-api.hh"
@@ -10,38 +11,58 @@ class Worker;
 
 struct PathSubstitutionGoal : public Goal
 {
-    /* The store path that should be realised through a substitute. */
+    /**
+     * The store path that should be realised through a substitute.
+     */
     StorePath storePath;
 
-    /* The path the substituter refers to the path as. This will be
+    /**
-       different when the stores have different names. */
+     * The path the substituter refers to the path as. This will be
+     * different when the stores have different names.
+     */
     std::optional<StorePath> subPath;
 
-    /* The remaining substituters. */
+    /**
+     * The remaining substituters.
+     */
     std::list<ref<Store>> subs;
 
-    /* The current substituter. */
+    /**
+     * The current substituter.
+     */
     std::shared_ptr<Store> sub;
 
-    /* Whether a substituter failed. */
+    /**
+     * Whether a substituter failed.
+     */
     bool substituterFailed = false;
 
-    /* Path info returned by the substituter's query info operation. */
+    /**
+     * Path info returned by the substituter's query info operation.
+     */
     std::shared_ptr<const ValidPathInfo> info;
 
-    /* Pipe for the substituter's standard output. */
+    /**
+     * Pipe for the substituter's standard output.
+     */
     Pipe outPipe;
 
-    /* The substituter thread. */
+    /**
+     * The substituter thread.
+     */
     std::thread thr;
 
     std::promise<void> promise;
 
-    /* Whether to try to repair a valid path. */
+    /**
+     * Whether to try to repair a valid path.
+     */
     RepairFlag repair;
 
-    /* Location where we're downloading the substitute.  Differs from
+    /**
-       storePath when doing a repair. */
+     * Location where we're downloading the substitute.  Differs from
+     * storePath when doing a repair.
+     */
     Path destPath;
 
     std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
@@ -50,7 +71,9 @@ struct PathSubstitutionGoal : public Goal
     typedef void (PathSubstitutionGoal::*GoalState)();
     GoalState state;
 
-    /* Content address for recomputing store path */
+    /**
+     * Content address for recomputing store path
+     */
     std::optional<ContentAddress> ca;
 
     void done(
@@ -64,16 +87,20 @@ public:
 
     void timedOut(Error && ex) override { abort(); };
 
+    /**
+     * We prepend "a$" to the key name to ensure substitution goals
+     * happen before derivation goals.
+     */
     std::string key() override
     {
-        /* "a$" ensures substitution goals happen before derivation
-           goals. */
         return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
     }
 
     void work() override;
 
-    /* The states. */
+    /**
+     * The states.
+     */
     void init();
     void tryNext();
     void gotInfo();
@@ -81,7 +108,9 @@ public:
     void tryToRun();
     void finished();
 
-    /* Callback used by the worker to write to the log. */
+    /**
+     * Callback used by the worker to write to the log.
+     */
     void handleChildOutput(int fd, std::string_view data) override;
     void handleEOF(int fd) override;
 

src/libstore/build/worker.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "types.hh"
 #include "lock.hh"
@@ -16,24 +17,29 @@ struct DerivationGoal;
 struct PathSubstitutionGoal;
 class DrvOutputSubstitutionGoal;
 
-/* Workaround for not being able to declare a something like
+/**
-
+ * Workaround for not being able to declare a something like
-     class PathSubstitutionGoal : public Goal;
+ *
-
+ * ```c++
-   even when Goal is a complete type.
+ * class PathSubstitutionGoal : public Goal;
-
+ * ```
-   This is still a static cast. The purpose of exporting it is to define it in
+ * even when Goal is a complete type.
-   a place where `PathSubstitutionGoal` is concrete, and use it in a place where it
+ *
-   is opaque. */
+ * This is still a static cast. The purpose of exporting it is to define it in
+ * a place where `PathSubstitutionGoal` is concrete, and use it in a place where it
+ * is opaque.
+ */
 GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
 GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
 
 typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
 
 
-/* A mapping used to remember for each child process to what goal it
+/**
-   belongs, and file descriptors for receiving log data and output
+ * A mapping used to remember for each child process to what goal it
-   path creation commands. */
+ * belongs, and file descriptors for receiving log data and output
+ * path creation commands.
+ */
 struct Child
 {
     WeakGoalPtr goal;
@@ -41,14 +47,19 @@ struct Child
     std::set<int> fds;
     bool respectTimeouts;
     bool inBuildSlot;
-    steady_time_point lastOutput; /* time we last got output on stdout/stderr */
+    /**
+     * Time we last got output on stdout/stderr
+     */
+    steady_time_point lastOutput;
     steady_time_point timeStarted;
 };
 
 /* Forward definition. */
 struct HookInstance;
 
-/* The worker class. */
+/**
+ * The worker class.
+ */
 class Worker
 {
 private:
@@ -56,38 +67,58 @@ private:
     /* Note: the worker should only have strong pointers to the
        top-level goals. */
 
-    /* The top-level goals of the worker. */
+    /**
+     * The top-level goals of the worker.
+     */
     Goals topGoals;
 
-    /* Goals that are ready to do some work. */
+    /**
+     * Goals that are ready to do some work.
+     */
     WeakGoals awake;
 
-    /* Goals waiting for a build slot. */
+    /**
+     * Goals waiting for a build slot.
+     */
     WeakGoals wantingToBuild;
 
-    /* Child processes currently running. */
+    /**
+     * Child processes currently running.
+     */
     std::list<Child> children;
 
-    /* Number of build slots occupied.  This includes local builds and
+    /**
-       substitutions but not remote builds via the build hook. */
+     * Number of build slots occupied.  This includes local builds and
+     * substitutions but not remote builds via the build hook.
+     */
     unsigned int nrLocalBuilds;
 
-    /* Maps used to prevent multiple instantiations of a goal for the
+    /**
-       same derivation / path. */
+     * Maps used to prevent multiple instantiations of a goal for the
+     * same derivation / path.
+     */
     std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
     std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
     std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
 
-    /* Goals waiting for busy paths to be unlocked. */
+    /**
+     * Goals waiting for busy paths to be unlocked.
+     */
     WeakGoals waitingForAnyGoal;
 
-    /* Goals sleeping for a few seconds (polling a lock). */
+    /**
+     * Goals sleeping for a few seconds (polling a lock).
+     */
     WeakGoals waitingForAWhile;
 
-    /* Last time the goals in `waitingForAWhile' where woken up. */
+    /**
+     * Last time the goals in `waitingForAWhile` where woken up.
+     */
     steady_time_point lastWokenUp;
 
-    /* Cache for pathContentsGood(). */
+    /**
+     * Cache for pathContentsGood().
+     */
     std::map<StorePath, bool> pathContentsGoodCache;
 
 public:
@@ -96,17 +127,25 @@ public:
     const Activity actDerivations;
     const Activity actSubstitutions;
 
-    /* Set if at least one derivation had a BuildError (i.e. permanent
+    /**
-       failure). */
+     * Set if at least one derivation had a BuildError (i.e. permanent
+     * failure).
+     */
     bool permanentFailure;
 
-    /* Set if at least one derivation had a timeout. */
+    /**
+     * Set if at least one derivation had a timeout.
+     */
     bool timedOut;
 
-    /* Set if at least one derivation fails with a hash mismatch. */
+    /**
+     * Set if at least one derivation fails with a hash mismatch.
+     */
     bool hashMismatch;
 
-    /* Set if at least one derivation is not deterministic in check mode. */
+    /**
+     * Set if at least one derivation is not deterministic in check mode.
+     */
     bool checkMismatch;
 
     Store & store;
@@ -128,16 +167,22 @@ public:
     uint64_t expectedNarSize = 0;
     uint64_t doneNarSize = 0;
 
-    /* Whether to ask the build hook if it can build a derivation. If
+    /**
-       it answers with "decline-permanently", we don't try again. */
+     * Whether to ask the build hook if it can build a derivation. If
+     * it answers with "decline-permanently", we don't try again.
+     */
     bool tryBuildHook = true;
 
     Worker(Store & store, Store & evalStore);
     ~Worker();
 
-    /* Make a goal (with caching). */
+    /**
+     * Make a goal (with caching).
+     */
 
-    /* derivation goal */
+    /**
+     * derivation goal
+     */
 private:
     std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
         const StorePath & drvPath, const OutputsSpec & wantedOutputs,
@@ -150,56 +195,80 @@ public:
         const StorePath & drvPath, const BasicDerivation & drv,
         const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
 
-    /* substitution goal */
+    /**
+     * substitution goal
+     */
     std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
     std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
 
-    /* Remove a dead goal. */
+    /**
+     * Remove a dead goal.
+     */
     void removeGoal(GoalPtr goal);
 
-    /* Wake up a goal (i.e., there is something for it to do). */
+    /**
+     * Wake up a goal (i.e., there is something for it to do).
+     */
     void wakeUp(GoalPtr goal);
 
-    /* Return the number of local build and substitution processes
+    /**
-       currently running (but not remote builds via the build
+     * Return the number of local build and substitution processes
-       hook). */
+     * currently running (but not remote builds via the build
+     * hook).
+     */
     unsigned int getNrLocalBuilds();
 
-    /* Registers a running child process.  `inBuildSlot' means that
+    /**
-       the process counts towards the jobs limit. */
+     * Registers a running child process.  `inBuildSlot` means that
+     * the process counts towards the jobs limit.
+     */
     void childStarted(GoalPtr goal, const std::set<int> & fds,
         bool inBuildSlot, bool respectTimeouts);
 
-    /* Unregisters a running child process.  `wakeSleepers' should be
+    /**
-       false if there is no sense in waking up goals that are sleeping
+     * Unregisters a running child process.  `wakeSleepers` should be
-       because they can't run yet (e.g., there is no free build slot,
+     * false if there is no sense in waking up goals that are sleeping
-       or the hook would still say `postpone'). */
+     * because they can't run yet (e.g., there is no free build slot,
+     * or the hook would still say `postpone`).
+     */
     void childTerminated(Goal * goal, bool wakeSleepers = true);
 
-    /* Put `goal' to sleep until a build slot becomes available (which
+    /**
-       might be right away). */
+     * Put `goal` to sleep until a build slot becomes available (which
+     * might be right away).
+     */
     void waitForBuildSlot(GoalPtr goal);
 
-    /* Wait for any goal to finish.  Pretty indiscriminate way to
+    /**
-       wait for some resource that some other goal is holding. */
+     * Wait for any goal to finish.  Pretty indiscriminate way to
+     * wait for some resource that some other goal is holding.
+     */
     void waitForAnyGoal(GoalPtr goal);
 
-    /* Wait for a few seconds and then retry this goal.  Used when
+    /**
-       waiting for a lock held by another process.  This kind of
+     * Wait for a few seconds and then retry this goal.  Used when
-       polling is inefficient, but POSIX doesn't really provide a way
+     * waiting for a lock held by another process.  This kind of
-       to wait for multiple locks in the main select() loop. */
+     * polling is inefficient, but POSIX doesn't really provide a way
+     * to wait for multiple locks in the main select() loop.
+     */
     void waitForAWhile(GoalPtr goal);
 
-    /* Loop until the specified top-level goals have finished. */
+    /**
+     * Loop until the specified top-level goals have finished.
+     */
     void run(const Goals & topGoals);
 
-    /* Wait for input to become available. */
+    /**
+     * Wait for input to become available.
+     */
     void waitForInput();
 
     unsigned int exitStatus();
 
-    /* Check whether the given valid path exists and has the right
+    /**
-       contents. */
+     * Check whether the given valid path exists and has the right
+     * contents.
+     */
     bool pathContentsGood(const StorePath & path);
 
     void markContentsGood(const StorePath & path);

src/libstore/builtins.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "derivations.hh"
 

src/libstore/builtins/buildenv.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "derivations.hh"
 #include "store-api.hh"

src/libstore/content-address.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include <variant>
 #include "hash.hh"

src/libstore/crypto.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "types.hh"
 
@@ -11,8 +12,10 @@ struct Key
     std::string name;
     std::string key;
 
-    /* Construct Key from a string in the format
+    /**
-       ‘<name>:<key-in-base64>’. */
+     * Construct Key from a string in the format
+     * ‘<name>:<key-in-base64>’.
+     */
     Key(std::string_view s);
 
     std::string to_string() const;
@@ -28,7 +31,9 @@ struct SecretKey : Key
 {
     SecretKey(std::string_view s);
 
-    /* Return a detached signature of the given string. */
+    /**
+     * Return a detached signature of the given string.
+     */
     std::string signDetached(std::string_view s) const;
 
     PublicKey toPublicKey() const;
@@ -52,8 +57,10 @@ private:
 
 typedef std::map<std::string, PublicKey> PublicKeys;
 
-/* Return true iff ‘sig’ is a correct signature over ‘data’ using one
+/**
-   of the given public keys. */
+ * @return true iff ‘sig’ is a correct signature over ‘data’ using one
+ * of the given public keys.
+ */
 bool verifyDetached(const std::string & data, const std::string & sig,
     const PublicKeys & publicKeys);
 

src/libstore/daemon.cc

@@ -1036,6 +1036,15 @@ void processConnection(
     if (GET_PROTOCOL_MINOR(clientVersion) >= 33)
         to << nixVersion;
 
+    if (GET_PROTOCOL_MINOR(clientVersion) >= 35) {
+        // We and the underlying store both need to trust the client for
+        // it to be trusted.
+        auto temp = trusted
+            ? store->isTrustedClient()
+            : std::optional { NotTrusted };
+        worker_proto::write(*store, to, temp);
+    }
+
     /* Send startup error messages to the client. */
     tunnelLogger->startWork();
 

src/libstore/daemon.hh

@@ -1,11 +1,11 @@
 #pragma once
+///@file
 
 #include "serialise.hh"
 #include "store-api.hh"
 
 namespace nix::daemon {
 
-enum TrustedFlag : bool { NotTrusted = false, Trusted = true };
 enum RecursiveFlag : bool { NotRecursive = false, Recursive = true };
 
 void processConnection(

src/libstore/derivations.cc

@@ -312,6 +312,15 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi
 }
 
 
+/**
+ * Print a derivation string literal to an `std::string`.
+ *
+ * This syntax does not generalize to the expression language, which needs to
+ * escape `$`.
+ *
+ * @param res Where to print to
+ * @param s Which logical string to print
+ */
 static void printString(std::string & res, std::string_view s)
 {
     boost::container::small_vector<char, 64 * 1024> buffer;
@@ -888,6 +897,67 @@ std::optional<BasicDerivation> Derivation::tryResolve(
     return resolved;
 }
 
+
+void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
+{
+    assert(drvPath.isDerivation());
+    std::string drvName(drvPath.name());
+    drvName = drvName.substr(0, drvName.size() - drvExtension.size());
+
+    if (drvName != name) {
+        throw Error("Derivation '%s' has name '%s' which does not match its path", store.printStorePath(drvPath), name);
+    }
+
+    auto envHasRightPath = [&](const StorePath & actual, const std::string & varName)
+    {
+        auto j = env.find(varName);
+        if (j == env.end() || store.parseStorePath(j->second) != actual)
+            throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'",
+                store.printStorePath(drvPath), varName, store.printStorePath(actual));
+    };
+
+
+    // Don't need the answer, but do this anyways to assert is proper
+    // combination. The code below is more general and naturally allows
+    // combinations that are currently prohibited.
+    type();
+
+    std::optional<DrvHash> hashesModulo;
+    for (auto & i : outputs) {
+        std::visit(overloaded {
+            [&](const DerivationOutput::InputAddressed & doia) {
+                if (!hashesModulo) {
+                    // somewhat expensive so we do lazily
+                    hashesModulo = hashDerivationModulo(store, *this, true);
+                }
+                auto currentOutputHash = get(hashesModulo->hashes, i.first);
+                if (!currentOutputHash)
+                    throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'",
+                        store.printStorePath(drvPath), store.printStorePath(doia.path), i.first);
+                StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName);
+                if (doia.path != recomputed)
+                    throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
+                        store.printStorePath(drvPath), store.printStorePath(doia.path), store.printStorePath(recomputed));
+                envHasRightPath(doia.path, i.first);
+            },
+            [&](const DerivationOutput::CAFixed & dof) {
+                auto path = dof.path(store, drvName, i.first);
+                envHasRightPath(path, i.first);
+            },
+            [&](const DerivationOutput::CAFloating &) {
+                /* Nothing to check */
+            },
+            [&](const DerivationOutput::Deferred &) {
+                /* Nothing to check */
+            },
+            [&](const DerivationOutput::Impure &) {
+                /* Nothing to check */
+            },
+        }, i.second.raw());
+    }
+}
+
+
 const Hash impureOutputHash = hashString(htSHA256, "impure");
 
 nlohmann::json DerivationOutput::toJSON(
@@ -916,10 +986,79 @@ nlohmann::json DerivationOutput::toJSON(
     return res;
 }
 
+
+DerivationOutput DerivationOutput::fromJSON(
+    const Store & store, std::string_view drvName, std::string_view outputName,
+    const nlohmann::json & _json)
+{
+    std::set<std::string_view> keys;
+    auto json = (std::map<std::string, nlohmann::json>) _json;
+
+    for (const auto & [key, _] : json)
+        keys.insert(key);
+
+    auto methodAlgo = [&]() -> std::pair<FileIngestionMethod, HashType> {
+        std::string hashAlgo = json["hashAlgo"];
+        auto method = FileIngestionMethod::Flat;
+        if (hashAlgo.substr(0, 2) == "r:") {
+            method = FileIngestionMethod::Recursive;
+            hashAlgo = hashAlgo.substr(2);
+        }
+        auto hashType = parseHashType(hashAlgo);
+        return { method, hashType };
+    };
+
+    if (keys == (std::set<std::string_view> { "path" })) {
+        return DerivationOutput::InputAddressed {
+            .path = store.parseStorePath((std::string) json["path"]),
+        };
+    }
+
+    else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) {
+        auto [method, hashType] = methodAlgo();
+        auto dof = DerivationOutput::CAFixed {
+            .ca = ContentAddressWithReferences::fromParts(
+                method,
+                Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType),
+                {}),
+        };
+        if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"]))
+            throw Error("Path doesn't match derivation output");
+        return dof;
+    }
+
+    else if (keys == (std::set<std::string_view> { "hashAlgo" })) {
+        auto [method, hashType] = methodAlgo();
+        return DerivationOutput::CAFloating {
+            .method = method,
+            .hashType = hashType,
+        };
+    }
+
+    else if (keys == (std::set<std::string_view> { })) {
+        return DerivationOutput::Deferred {};
+    }
+
+    else if (keys == (std::set<std::string_view> { "hashAlgo", "impure" })) {
+        auto [method, hashType] = methodAlgo();
+        return DerivationOutput::Impure {
+            .method = method,
+            .hashType = hashType,
+        };
+    }
+
+    else {
+        throw Error("invalid JSON for derivation output");
+    }
+}
+
+
 nlohmann::json Derivation::toJSON(const Store & store) const
 {
     nlohmann::json res = nlohmann::json::object();
 
+    res["name"] = name;
+
     {
         nlohmann::json & outputsObj = res["outputs"];
         outputsObj = nlohmann::json::object();
@@ -950,4 +1089,43 @@ nlohmann::json Derivation::toJSON(const Store & store) const
     return res;
 }
 
+
+Derivation Derivation::fromJSON(
+    const Store & store,
+    const nlohmann::json & json)
+{
+    Derivation res;
+
+    res.name = json["name"];
+
+    {
+        auto & outputsObj = json["outputs"];
+        for (auto & [outputName, output] : outputsObj.items()) {
+            res.outputs.insert_or_assign(
+                outputName,
+                DerivationOutput::fromJSON(store, res.name, outputName, output));
+        }
+    }
+
+    {
+        auto & inputsList = json["inputSrcs"];
+        for (auto & input : inputsList)
+            res.inputSrcs.insert(store.parseStorePath(static_cast<const std::string &>(input)));
+    }
+
+    {
+        auto & inputDrvsObj = json["inputDrvs"];
+        for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items())
+            res.inputDrvs[store.parseStorePath(inputDrvPath)] =
+                static_cast<const StringSet &>(inputOutputs);
+    }
+
+    res.platform = json["system"];
+    res.builder = json["builder"];
+    res.args = json["args"];
+    res.env = json["env"];
+
+    return res;
+}
+
 }

src/libstore/derivations.hh

@@ -1,4 +1,5 @@
 #pragma once
+///@file
 
 #include "path.hh"
 #include "types.hh"
@@ -6,6 +7,7 @@
 #include "content-address.hh"
 #include "repair-flag.hh"
 #include "sync.hh"
+#include "comparator.hh"
 
 #include <map>
 #include <variant>
@@ -23,6 +25,8 @@ class Store;
 struct DerivationOutputInputAddressed
 {
     StorePath path;
+
+    GENERATE_CMP(DerivationOutputInputAddressed, me->path);
 };
 
 /**
@@ -43,6 +47,8 @@ struct DerivationOutputCAFixed
      * @param outputName The name of this output.
      */
     StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
+
+    GENERATE_CMP(DerivationOutputCAFixed, me->ca);
 };
 
 /**
@@ -61,13 +67,17 @@ struct DerivationOutputCAFloating
      * How the serialization will be hashed
      */
     HashType hashType;
+
+    GENERATE_CMP(DerivationOutputCAFloating, me->method, me->hashType);
 };
 
 /**
  * Input-addressed output which depends on a (CA) derivation whose hash
  * isn't known yet.
  */
-struct DerivationOutputDeferred {};
+struct DerivationOutputDeferred {
+    GENERATE_CMP(DerivationOutputDeferred);
+};
 
 /**
  * Impure output which is moved to a content-addressed location (like
@@ -84,6 +94,8 @@ struct DerivationOutputImpure
      * How the serialization will be hashed
      */
     HashType hashType;
+
+    GENERATE_CMP(DerivationOutputImpure, me->method, me->hashType);
 };
 
 typedef std::variant<
@@ -124,6 +136,11 @@ struct DerivationOutput : _DerivationOutputRaw
         const Store & store,
         std::string_view drvName,
         std::string_view outputName) const;
+    static DerivationOutput fromJSON(
+        const Store & store,
+        std::string_view drvName,
+        std::string_view outputName,
+        const nlohmann::json & json);
 };
 
 typedef std::map<std::string, DerivationOutput> DerivationOutputs;
@@ -241,8 +258,14 @@ struct DerivationType : _DerivationTypeRaw {
 
 struct BasicDerivation
 {
-    DerivationOutputs outputs; /* keyed on symbolic IDs */
+    /**
-    StorePathSet inputSrcs; /* inputs that are sources */
+     * keyed on symbolic IDs
+     */
+    DerivationOutputs outputs;
+    /**
+     * inputs that are sources
+     */
+    StorePathSet inputSrcs;
     std::string platform;
     Path builder;
     Strings args;
@@ -272,6 +295,15 @@ struct BasicDerivation
     DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const;
 
     static std::string_view nameFromPath(const StorePath & storePath);
+
+    GENERATE_CMP(BasicDerivation,
+        me->outputs,
+        me->inputSrcs,
+        me->platform,
+        me->builder,
+        me->args,
+        me->env,
+        me->name);
 };
 
 struct Derivation : BasicDerivation
@@ -307,11 +339,26 @@ struct Derivation : BasicDerivation
         Store & store,
         const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const;
 
+    /* Check that the derivation is valid and does not present any
+       illegal states.
+
+       This is mainly a matter of checking the outputs, where our C++
+       representation supports all sorts of combinations we do not yet
+       allow. */
+    void checkInvariants(Store & store, const StorePath & drvPath) const;
+
     Derivation() = default;
     Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
     Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
 
     nlohmann::json toJSON(const Store & store) const;
+    static Derivation fromJSON(
+        const Store & store,
+        const nlohmann::json & json);
+
+    GENERATE_CMP(Derivation,
+        static_cast<const BasicDerivation &>(*me),
+        me->inputDrvs);
 };
 
 
@@ -388,12 +435,12 @@ void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
  *
  * A fixed-output derivation is a derivation whose outputs have a
  * specified content hash and hash algorithm. (Currently they must have
- * exactly one output (`out'), which is specified using the `outputHash'
+ * exactly one output (`out`), which is specified using the `outputHash`
- * and `outputHashAlgo' attributes, but the algorithm doesn't assume
+ * and `outputHashAlgo` attributes, but the algorithm doesn't assume
  * this.) We don't want changes to such derivations to propagate upwards
  * through the dependency graph, changing output paths everywhere.
  *
- * For instance, if we change the url in a call to the `fetchurl'
+ * For instance, if we change the url in a call to the `fetchurl`
  * function, we do not want to rebuild everything depending on it---after
  * all, (the hash of) the file being downloaded is unchanged.  So the
  * *output paths* should not change. On the other hand, the *derivation

src/libstore/derived-path.cc

@@ -62,15 +62,31 @@ std::string DerivedPath::Opaque::to_string(const Store & store) const
 std::string DerivedPath::Built::to_string(const Store & store) const
 {
     return store.printStorePath(drvPath)
-        + "!"
+        + '^'
+        + outputs.to_string();
+}
+
+std::string DerivedPath::Built::to_string_legacy(const Store & store) const
+{
+    return store.printStorePath(drvPath)
+        + '!'
         + outputs.to_string();
 }
 
 std::string DerivedPath::to_string(const Store & store) const
 {
-    return std::visit(
+    return std::visit(overloaded {
-        [&](const auto & req) { return req.to_string(store); },
+        [&](const DerivedPath::Built & req) { return req.to_string(store); },
-        this->raw());
+        [&](const DerivedPath::Opaque & req) { return req.to_string(store); },
+    }, this->raw());
+}
+
+std::string DerivedPath::to_string_legacy(const Store & store) const
+{
+    return std::visit(overloaded {
+        [&](const DerivedPath::Built & req) { return req.to_string_legacy(store); },
+        [&](const DerivedPath::Opaque & req) { return req.to_string(store); },
+    }, this->raw());
 }
 
 
@@ -87,14 +103,24 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi
     };
 }
 
-DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
+static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator)
 {
-    size_t n = s.find("!");
+    size_t n = s.find(separator);
     return n == s.npos
         ? (DerivedPath) DerivedPath::Opaque::parse(store, s)
         : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1));
 }
 
+DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
+{
+	return parseWith(store, s, "^");
+}
+
+DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s)
+{
+	return parseWith(store, s, "!");
+}
+
 RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
 {
     RealisedPath::Set res;