Merge "libutil: Add bindPath function from libstore" into main

Change-Id: Ib64d695c50a733e0e739ff193f1ea65ed7cb0a57

.clang-tidy

@@ -16,3 +16,20 @@ Checks:
   - -bugprone-unchecked-optional-access
     # many warnings, seems like a questionable lint
   - -bugprone-branch-clone
+    # all thrown exceptions must derive from std::exception
+  - hicpp-exception-baseclass
+    # capturing async lambdas are dangerous
+  - cppcoreguidelines-avoid-capturing-lambda-coroutines
+    # crimes must be appropriately declared as crimes
+  - cppcoreguidelines-pro-type-cstyle-cast
+  - lix-*
+    # This can not yet be applied to Lix itself since we need to do source
+    # reorganization so that lix/ include paths work.
+  - -lix-fixincludes
+    # This lint is included as an example, but the lib function it replaces is
+    # already gone.
+  - -lix-hasprefixsuffix
+
+
+CheckOptions:
+  bugprone-reserved-identifier.AllowedIdentifiers: '__asan_default_options'

.envrc

@@ -1,9 +1,5 @@
 # shellcheck shell=bash
 source_env_if_exists .envrc.local
-# TODO: `use flake .#native-clangStdenvPackages` on macOS?
-use flake ".#${LIX_SHELL_VARIANT:-default}" "${LIX_SHELL_EXTRA_ARGS[@]}"
-export MAKEFLAGS="$MAKEFLAGS -e"
-if [[ -n "$NIX_BUILD_CORES" ]]; then
-    export MAKEFLAGS="$MAKEFLAGS -j $NIX_BUILD_CORES"
-fi
+# Use native-clangStdenvPackages to get clangd by default.
+use flake ".#${LIX_SHELL_VARIANT:-native-clangStdenvPackages}" "${LIX_SHELL_EXTRA_ARGS[@]}"
 export GTEST_BRIEF=1

.this-is-lix

@@ -0,0 +1 @@
+This is a file used by the dev shell shellHook in package.nix to check that this is actually a Lix repo before installing git hooks. Its contents have no meaning.

README.md

@@ -26,4 +26,4 @@ See our [Hacking guide](https://git.lix.systems/lix-project/lix/src/branch/main/
 
 ## License
 
-Lix is released under the [LGPL v2.1](./COPYING).
+Lix is released under [LGPL-2.1-or-later](./COPYING).

boehmgc-coroutine-sp-fallback.diff

@@ -1,54 +0,0 @@
-diff --git a/pthread_stop_world.c b/pthread_stop_world.c
-index 2b45489..0e6d8ef 100644
---- a/pthread_stop_world.c
-+++ b/pthread_stop_world.c
-@@ -776,6 +776,8 @@ STATIC void GC_restart_handler(int sig)
- /* world is stopped.  Should not fail if it isn't.                      */
- GC_INNER void GC_push_all_stacks(void)
- {
-+    size_t stack_limit;
-+    pthread_attr_t pattr;
-     GC_bool found_me = FALSE;
-     size_t nthreads = 0;
-     int i;
-@@ -868,6 +870,40 @@ GC_INNER void GC_push_all_stacks(void)
-             hi = p->altstack + p->altstack_size;
- #         endif
-           /* FIXME: Need to scan the normal stack too, but how ? */
-+        } else {
-+          #ifdef HAVE_PTHREAD_ATTR_GET_NP
-+          if (pthread_attr_init(&pattr) != 0) {
-+            ABORT("GC_push_all_stacks: pthread_attr_init failed!");
-+          }
-+          if (pthread_attr_get_np(p->id, &pattr) != 0) {
-+            ABORT("GC_push_all_stacks: pthread_attr_get_np failed!");
-+          }
-+          #else
-+          if (pthread_getattr_np(p->id, &pattr)) {
-+            ABORT("GC_push_all_stacks: pthread_getattr_np failed!");
-+          }
-+          #endif
-+          if (pthread_attr_getstacksize(&pattr, &stack_limit)) {
-+            ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!");
-+          }
-+          if (pthread_attr_destroy(&pattr)) {
-+            ABORT("GC_push_all_stacks: pthread_attr_destroy failed!");
-+          }
-+          // When a thread goes into a coroutine, we lose its original sp until
-+          // control flow returns to the thread.
-+          // While in the coroutine, the sp points outside the thread stack,
-+          // so we can detect this and push the entire thread stack instead,
-+          // as an approximation.
-+          // We assume that the coroutine has similarly added its entire stack.
-+          // This could be made accurate by cooperating with the application
-+          // via new functions and/or callbacks.
-+          #ifndef STACK_GROWS_UP
-+            if (lo >= hi || lo < hi - stack_limit) { // sp outside stack
-+              lo = hi - stack_limit;
-+            }
-+          #else
-+          #error "STACK_GROWS_UP not supported in boost_coroutine2 (as of june 2021), so we don't support it in Nix."
-+          #endif
-         }
- #       ifdef STACKPTR_CORRECTOR_AVAILABLE
-           if (GC_sp_corrector != 0)

clang-tidy/meson.build

@@ -1,13 +0,0 @@
-project('lix-clang-tidy', ['cpp', 'c'],
-  version : '0.1',
-  default_options : ['warning_level=3', 'cpp_std=c++20'])
-
-llvm = dependency('Clang', version: '>= 14', modules: ['libclang'])
-sources = files(
-  'HasPrefixSuffix.cc',
-  'LixClangTidyChecks.cc',
-  'FixIncludes.cc',
-)
-
-shared_module('lix-clang-tidy', sources,
-  dependencies: llvm)

doc/internal-api/doxygen.cfg.in

@@ -20,7 +20,7 @@ OUTPUT_DIRECTORY       = @docdir@
 # for a project that appears at the top of each page and should give viewer a
 # quick idea about the purpose of the project. Keep the description short.
 
-PROJECT_BRIEF          = "Nix, the purely functional package manager; unstable internal interfaces"
+PROJECT_BRIEF          = "Lix: A modern, delicious implementation of the Nix package manager; unstable internal interfaces"
 
 # If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
 # The default value is: YES.

doc/manual/change-authors.yml

@@ -3,6 +3,10 @@
 #
 # It's used for crediting people accurately in release notes. The release notes
 # script will link to forgejo, then to GitHub if forgejo is not present.
+#
+# When adding someone from outside the Lix project, you generally want to simply link their GitHub profile without adding a display name unless they are well-known in the community by that display name.
+#
+# See doc/manual/src/contributing/hacking.md for more documentation on this file's format and typical usage.
 9999years:
   display_name: wiggles
   forgejo: rbt
@@ -40,6 +44,11 @@ cole-h:
   display_name: Cole Helbling
   github: cole-h
 
+delan:
+  display_name: delan
+  forgejo: delan
+  github: delan
+
 edolstra:
   display_name: Eelco Dolstra
   github: edolstra
@@ -48,6 +57,11 @@ ericson:
   display_name: John Ericson
   github: ericson2314
 
+goldstein:
+  display_name: goldstein
+  forgejo: goldstein
+  github: GoldsteinE
+
 horrors:
   display_name: eldritch horrors
   forgejo: pennae
@@ -56,10 +70,21 @@ horrors:
 iFreilicht:
   github: iFreilicht
 
+isabelroses:
+  forgejo: isabelroses
+  github: isabelroses
+
 jade:
   forgejo: jade
   github: lf-
 
+kjeremy:
+  github: kjeremy
+
+kloenk:
+  forgejo: kloenk
+  github: kloenk
+
 lovesegfault:
   github: lovesegfault
 
@@ -78,11 +103,20 @@ midnightveil:
 ncfavier:
   github: ncfavier
 
+piegames:
+  display_name: piegames
+  forgejo: piegames
+  github: piegamesde
+
 puck:
   display_name: puck
   forgejo: puck
   github: puckipedia
 
+quantumjump:
+  display_name: Quantum Jump
+  github: QuantumBJump
+
 r-vdp:
   github: r-vdp
 

doc/manual/meson.build

@@ -110,6 +110,7 @@ manual = custom_target(
     builtins_md,
     builtin_constants_md,
     rl_next_generated,
+    nix,
   ],
   output : [
     'manual',
@@ -186,6 +187,7 @@ foreach command : nix_nested_manpages
       ],
       input : [
         manual_md,
+        nix,
       ],
       output : command[0] + '-' + page + '.1',
       install : true,
@@ -298,6 +300,7 @@ foreach page : nix3_manpages
     input : [
       'render-manpage.sh',
       manual_md,
+      nix,
     ],
     output : page + '.1',
     install : true,
@@ -341,6 +344,7 @@ foreach entry : nix_manpages
       'render-manpage.sh',
       manual_md,
       entry.get(3, []),
+      nix,
     ],
     output : '@0@.@1@'.format(entry[0], entry[1]),
     install : true,

doc/manual/process-includes.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 set -euo pipefail
 

doc/manual/redirects.js

@@ -345,7 +345,7 @@ const redirects = {
     "linux": "uninstall.html#linux",
     "macos": "uninstall.html#macos",
     "uninstalling": "uninstall.html",
-  }
+  },
   "contributing/hacking.html": {
     "nix-with-flakes": "#building-nix-with-flakes",
     "classic-nix": "#building-nix",

doc/manual/render-manpage.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 set -euo pipefail
 

doc/manual/src/SUMMARY.md

@@ -197,7 +197,8 @@
 - [Release Notes](release-notes/release-notes.md)
   - [Upcoming release](release-notes/rl-next.md)
     <!-- RELENG-AUTO-INSERTION-MARKER (see releng/release_notes.py) -->
-  - [Lix 2.90 (FIXME date)](release-notes/rl-2.90.md)
+  - [Lix 2.91 (2024-08-12)](release-notes/rl-2.91.md)
+  - [Lix 2.90 (2024-07-10)](release-notes/rl-2.90.md)
   - [Nix 2.18 (2023-09-20)](release-notes/rl-2.18.md)
   - [Nix 2.17 (2023-07-24)](release-notes/rl-2.17.md)
   - [Nix 2.16 (2023-05-31)](release-notes/rl-2.16.md)

doc/manual/src/command-ref/opt-common.md

@@ -78,6 +78,16 @@ Most commands in Lix accept the following command-line options:
 
     Display the raw logs, with the progress bar at the bottom.
 
+  - `multiline`
+
+    Display a progress bar during the builds and in the lines below that one line per activity.
+
+
+  - `multiline-with-logs`
+
+    Displayes the raw logs, with a progress bar and activities each in a new line at the bottom.
+
+
 - <span id="opt-no-build-output">[`--no-build-output`](#opt-no-build-output)</span> / `-Q`
 
   By default, output written by builders to standard output and standard error is echoed to the Lix command's standard error.

doc/manual/src/contributing/hacking.md

@@ -39,17 +39,19 @@ $ nix-shell -A native-clangStdenvPackages
 
 ### Building from the development shell
 
-As always you may run [stdenv's phases by name](https://nixos.org/manual/nixpkgs/unstable/#sec-building-stdenv-package-in-nix-shell), e.g.:
+You can build and test Lix with just:
 
 ```bash
-$ configurePhase
-$ buildPhase
-$ checkPhase
-$ installPhase
-$ installCheckPhase
+$ just setup
+$ just build
+$ just test --suite=check
+$ just install
+$ just test --suite=installcheck
 ```
 
-To build manually, however, use the following:
+(Check and installcheck may both be done after install, allowing you to omit the --suite argument entirely, but this is the order package.nix runs them in.)
+
+You can also build Lix manually:
 
 ```bash
 $ meson setup ./build "--prefix=$out" $mesonFlags
@@ -64,9 +66,7 @@ $ meson install -C build
 $ meson test -C build --suite=installcheck
 ```
 
-(Check and installcheck may both be done after install, allowing you to omit the --suite argument entirely, but this is the order package.nix runs them in.)
-
-This will install Lix to `$PWD/outputs`, the `/bin` of which is prepended to PATH in the development shells.
+In both cases, Lix will be installed to `$PWD/outputs`, the `/bin` of which is prepended to PATH in the development shells.
 
 If the tests fail and Meson helpfully has no output for why, use the `--print-error-logs` option to `meson test`.
 
@@ -168,8 +168,27 @@ or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled
 $ nix build .#packages.aarch64-linux.default
 ```
 
-Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`).
-Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms.
+### Cross compiling using the Lix flake
+
+Lix can also be easily cross compiled to the following arbitrarily-chosen system doubles, which can be useful for bootstrapping Lix on new platforms.
+These are specified in `crossSystems` in `flake.nix`; feel free to submit changes to add new ones if they are useful to you.
+
+- `armv6l-linux`
+- `armv7l-linux`
+- `aarch64-linux`
+- `riscv64-linux`
+
+For example, to cross-compile Lix for `armv6l-linux` from another Linux, use the following:
+
+```console
+$ nix build .#nix-armv6l-linux
+```
+
+It's also possible to cross-compile a tarball of binaries suitable for the Lix installer, for example, for `riscv64-linux`:
+
+```console
+$ nix build .#nix-riscv64-linux.passthru.binaryTarball
+```
 
 ### Building for multiple platforms at once
 
@@ -282,9 +301,8 @@ Regular markdown files used for the manual have a base path of their own and the
 
 ## API documentation
 
-Doxygen API documentation is [available
-online](https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs). You
-can also build and view it yourself:
+Doxygen API documentation will be available online in the future ([tracking issue](https://git.lix.systems/lix-project/lix/issues/422)).
+You can also build and view it yourself:
 
 ```console
 # nix build .#hydraJobs.internal-api-docs
@@ -294,44 +312,50 @@ can also build and view it yourself:
 or inside a `nix develop` shell by running:
 
 ```bash
+$ meson configure build -Dinternal-api-docs=enabled
 $ meson compile -C build internal-api-docs
 $ xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html
 ```
 
 ## Coverage analysis
 
-A coverage analysis report is [available
-online](https://hydra.nixos.org/job/nix/master/coverage/latest/download-by-type/report/coverage). You
-can build it yourself:
+A coverage analysis report will be available online in the future (FIXME(lix-hydra)).
+You can build it yourself:
 
 ```
 # nix build .#hydraJobs.coverage
 # xdg-open ./result/coverage/index.html
 ```
 
-Metrics about the change in line/function coverage over time are also
-[available](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts).
+Metrics about the change in line/function coverage over time will be available in the future (FIXME(lix-hydra)).
 
 ## Add a release note
 
 `doc/manual/rl-next` contains release notes entries for all unreleased changes.
 
 User-visible changes should come with a release note.
+Developer-facing changes should have a release note in the Development category if they are significant and if developers should know about them.
 
 ### Add an entry
 
-Here's what a complete entry looks like. The file name is not incorporated in the document.
+Here's what a complete entry looks like.
+The file name is not incorporated in the final document, and is generally a super brief summary of the change synopsis.
 
-```
+```markdown
 ---
 synopsis: Basically a title
 # 1234 or gh#1234 will refer to CppNix GitHub, fj#1234 will refer to a Lix forgejo issue.
 issues: [1234, fj#1234]
-# Use this *only* if there is a CppNix pull request associated with this change
+# Use this *only* if there is a CppNix pull request associated with this change.
 prs: 1238
-# List of Lix Gerrit changelist numbers; if there is an associated Lix GitHub
-# PR, just put in the Gerrit CL number.
+# List of Lix Gerrit changelist numbers.
+# If there is an associated Lix GitHub PR, just put in the Gerrit CL number.
 cls: [123]
+# Heading that this release note will appear under.
+category: Breaking Changes
+# Add a credit mention in the bottom of the release note.
+# your-name is used as a key into doc/manual/change-authors.yml for metadata
+credits: [your-name]
 ---
 
 Here's one or more paragraphs that describe the change.
@@ -346,6 +370,31 @@ Significant changes should add the following header, which moves them to the top
 significance: significant
 ```
 
+The following categories of release notes are supported (see `maintainers/build-release-notes.py`):
+- Breaking Changes
+- Features
+- Improvements
+- Fixes
+- Packaging
+- Development
+- Miscellany
+
+The `credits` field, if present, gives credit to the author of the patch in the release notes with a message like "Many thanks to (your-name) for this" and linking to GitHub or Forgejo profiles if listed.
+
+If you are forward-porting a change from CppNix, please credit the original author, and optionally credit yourself.
+When adding credits metadata for people external to the project and deciding whether to put in a `display_name`, consider what they are generally known as in the community; even if you know their full name (e.g. from their GitHub profile), we suggest only adding it as a display name if that is what they go by in the community.
+There are multiple reasons we follow this practice, but it boils down to privacy and consent: we would rather not capture full names that are not widely used in the community without the consent of the parties involved, even if they are publicly available.
+As of this writing, the entries with full names as `display_name` are either members of the CppNix team or people who added them themselves.
+
+The names specified in `credits` are used as keys to look up the authorship info in `doc/manual/change-authors.yml`.
+The only mandatory part is that every key appearing in `credits` has an entry present in `change-authors.yml`.
+All of the following properties are optional; you can specify `{}` as the metadata if you want a simple non-hyperlinked mention.
+The following properties are supported:
+
+- `display_name`: display name used in place of the key when showing names, if present.
+- `forgejo`: Forgejo username. The name in the release notes will be a link to this, if present.
+- `github`: GitHub username, used if `forgejo` is not set, again making a link.
+
 ### Build process
 
 Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`.

doc/manual/src/contributing/testing.md

@@ -247,7 +247,6 @@ To ensure that characterization testing doesn't make it harder to intentionally
 
 The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute.
 These tests include everything that needs to interact with external services or run Lix in a non-trivial distributed setup.
-Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on <https://hydra.nixos.org/jobset/nix/master>).
 
 You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`
 
@@ -428,14 +427,12 @@ I grepped `src/` for `get[eE]nv\("` to find the mentions in Lix code.
 - `NIX_SHOW_STATS_PATH` - Writes those statistics into a file at the given path instead of stdout. Undocumented.
 - `NIX_SHOW_SYMBOLS` - Dumps the symbol table into the show-stats json output.
 - `TERM` - If `dumb` or unset, disables ANSI colour output.
+- `FORCE_COLOR`, `CLICOLOR_FORCE` - Enables ANSI colour output if `NO_COLOR`/`NOCOLOR` not set.
 - `NO_COLOR`, `NOCOLOR` - Disables ANSI colour output.
 - `_NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS` - Highlights unknown locations in errors.
 - `NIX_PROFILE` - Selects which profile `nix-env` will operate on. Documented elsewhere.
 - `NIX_SSHOPTS` - Options passed to `ssh(1)` when using a ssh remote store.
   Incorrectly documented on `nix-copy-closure` which is *surely* not the only place they are used??
-- `_NIX_TEST_NO_LSOF` - Used on non-Linux, non-macOS platforms to disable using `lsof` when finding gc roots.
-
-  Since https://git.lix.systems/lix-project/lix/issues/156 was fixed, this should probably just be removed as it was a bad workaround for a macOS issue.
 - `_NIX_TEST_GC_SYNC_1` - Path to a pipe that is used to block the GC briefly to validate invariants from the test suite.
 - `_NIX_TEST_GC_SYNC_2` - Path to a pipe that is used to block the GC briefly to validate invariants from the test suite.
 - `_NIX_TEST_FREE_SPACE_FILE` - Path to a file containing a decimal number with the free space that the GC is to believe it has.

doc/manual/src/language/advanced-attributes.md

@@ -292,6 +292,12 @@ Derivations can declare some infrequently used optional attributes.
     (associative) arrays. For example, the attribute `hardening.format = true`
     ends up as the Bash associative array element `${hardening[format]}`.
 
+    > **Warning**
+    >
+    > If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites),
+    [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), maxSize, and maxClosureSize.
+    will have no effect.
+
   - [`outputChecks`]{#adv-attr-outputChecks}\
     When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks`
     attribute allows defining checks per-output.
@@ -326,7 +332,6 @@ Derivations can declare some infrequently used optional attributes.
     ```
 
   - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\
-
     When using [structured attributes](#adv-attr-structuredAttrs), the
     attribute `unsafeDiscardReferences` is an attribute set with a boolean value for each output name.
     If set to `true`, it disables scanning the output for runtime dependencies.

doc/manual/src/language/operators.md

@@ -26,6 +26,8 @@
 | Logical conjunction (`AND`)            | *bool* `&&` *bool*                         | left          | 12         |
 | Logical disjunction (`OR`)             | *bool* <code>\|\|</code> *bool*            | left          | 13         |
 | [Logical implication]                  | *bool* `->` *bool*                         | none          | 14         |
+| \[Experimental\] [Function piping]     | *expr* |> *func*                           | left          | 15         |
+| \[Experimental\] [Function piping]     | *expr* <| *func*                           | right         | 16         |
 
 [string]: ./values.md#type-string
 [path]: ./values.md#type-path
@@ -59,8 +61,10 @@ The result is a [Boolean] value.
 
 ## Arithmetic
 
-Numbers are type-compatible:
-Pure integer operations will always return integers, whereas any operation involving at least one floating point number return a floating point number.
+Numbers will retain their type unless mixed with other numeric types:
+Pure integer operations will always return integers, whereas any operation involving at least one floating point number returns a floating point number.
+
+Integer overflow (of 64-bit signed integers) and division by zero are defined to throw an error.
 
 See also [Comparison] and [Equality].
 
@@ -143,21 +147,103 @@ All comparison operators are implemented in terms of `<`, and the following equi
 | *a* `>`  *b* |       *b* `<` *a*     |
 | *a* `>=` *b* | `! (` *a* `<` *b* `)` |
 
+Note that the above behaviour violates IEEE 754 for floating point numbers with respect to NaN, for instance.
+This may be fixed in a future major language revision.
+
 [Comparison]: #comparison-operators
 
 ## Equality
 
-- [Attribute sets][attribute set] and [list]s are compared recursively, and therefore are fully evaluated.
-- Comparison of [function]s always returns `false`.
-- Numbers are type-compatible, see [arithmetic] operators.
-- Floating point numbers only differ up to a limited precision.
+The following equality comparison rules are followed in order:
+
+- Comparisons are first, sometimes, performed by identity (pointer value), and whether or not this occurs varies depending on the context in which the comparison is performed; for example, through `builtins.elem`, comparison of lists, or other cases.
+  The exact instances in which this occurs, aside from direct list and attribute set comparisons as discussed below, are too dependent on implementation details to meaningfully document.
+
+  See [note on identity comparison](#identity-comparison) below.
+- Comparisons between a combination of integers and floating point numbers are first converted to floating point then compared as floating point.
+- Comparisons between values of differing types, besides the ones mentioned in the above rule, are unequal.
+- Strings are compared as their string values, disregarding string contexts.
+- Paths are compared as their absolute form (since they are stored as such).
+- [Functions][function] are always considered unequal, including with themselves.
+- The following are compared in the typical manner:
+  - Integers
+  - Floating point numbers have equality comparison per IEEE 754.
+
+    Note that this means that just like in most languages, floating point arithmetic results are not typically equality comparable, and should instead be compared by checking that the absolute difference is less than some error margin.
+  - Booleans
+  - Null
+- [Attribute sets][attribute set] are compared following these rules in order:
+  - If both attribute sets have the same identity (via pointer equality), they are considered equal, regardless of whether the contents have reflexive equality (e.g. even if there are functions contained within).
+
+    See [note on identity comparison](#identity-comparison) below.
+  - If both attribute sets have `type = "derivation"` and have an attribute `outPath` that is equal, they are considered equal.
+
+    This means that two results of `builtins.derivation`, regardless of other things added to their attributes via `//` afterwards (or `passthru` in nixpkgs), will compare equal if they passed the same arguments to `builtins.derivation`.
+  - Otherwise, they are compared element-wise in an unspecified order.
+    Although this order *may* be deterministic in some cases, this is not guaranteed, and correct code must not rely on this ordering behaviour.
+
+    The order determines which elements are evaluated first and thus, if there are throwing values in the attribute set, which of those get evaluated, if any, before the comparison returns an unequal result.
+- Lists are compared following these rules in order:
+  - If both lists have the same identity (via pointer equality), they are considered equal, regardless of whether the contents have reflexive equality (e.g. even if there are functions contained within).
+
+    See [note on identity comparison](#identity-comparison) below.
+  - Otherwise, they are compared element-wise in list order.
 
 [function]: ./constructs.md#functions
 
 [Equality]: #equality
 
+### Identity comparison
+
+In the current revision of the Nix language, values are first compared by identity (pointer equality).
+This means that values that are not reflexively equal (that is, they do not satisfy `a == a`), such as functions, are nonetheless sometimes compared as equal with themselves if they are placed in attribute sets or lists, or are compared through other indirect means.
+
+Whether identity comparison applies to a given usage of the language aside from direct list and attribute set comparison is strongly dependent on implementation details to the point it is not feasible to document the exact instances.
+
+This is rather unfortunate behaviour which is regrettably load-bearing on nixpkgs (such as with the `type` attribute of NixOS options) and cannot be changed for the time being.
+It may be changed in a future major language revision.
+
+Correct code must not rely on this behaviour.
+
+For example:
+
+```
+nix-repl> let f = x: 1; s = { func = f; }; in [ (f == f) (s == s) ]
+[ false true ]
+```
+
 ## Logical implication
 
 Equivalent to `!`*b1* `||` *b2*.
 
 [Logical implication]: #logical-implication
+
+## \[Experimental\] Function piping
+
+*This language feature is still experimental and may change at any time. Enable `--extra-experimental-features pipe-operator` to use it.*
+
+Pipes are a dedicated operator for function application, but with reverse order and a lower binding strength.
+This allows you to chain function calls together in way that is more natural to read and requires less parentheses.
+
+`a |> f b |> g` is equivalent to `g (f b a)`.
+`g <| f b <| a` is equivalent to `g (f b a)`.
+
+Example code snippet:
+
+```nix
+defaultPrefsFile = defaultPrefs
+  |> lib.mapAttrsToList (
+    key: value: ''
+      // ${value.reason}
+      pref("${key}", ${builtins.toJSON value.value});
+    ''
+  )
+  |> lib.concatStringsSep "\n"
+  |> pkgs.writeText "nixos-default-prefs.js";
+```
+
+Note how `mapAttrsToList` is called with two arguments (the lambda and `defaultPrefs`),
+but moving the last argument in front of the rest improves the reading flow.
+This is common for functions with long first argument, including all `map`-like functions.
+
+[Function piping]: #experimental-function-piping

doc/manual/src/language/values.md

@@ -7,13 +7,16 @@
   *Strings* can be written in three ways.
 
   The most common way is to enclose the string between double quotes,
-  e.g., `"foo bar"`. Strings can span multiple lines. The special
-  characters `"` and `\` and the character sequence `${` must be
-  escaped by prefixing them with a backslash (`\`). Newlines, carriage
-  returns and tabs can be written as `\n`, `\r` and `\t`,
-  respectively.
+  e.g., `"foo bar"`. Strings can span multiple lines. The backslash
+  (`\`) can be used to escape characters: newlines, carriage returns
+  and tabs may be written as `\n`, `\r` and `\t` respectively; any
+  other characters can be preceded by a backslash to remove any
+  special meaning they may have, like the special characters `"` and
+  `\` and the character sequence `${`.
 
   You can include the results of other expressions into a string by enclosing them in `${ }`, a feature known as [string interpolation].
+  Due to a parser issue that has since come to be relied upon, the character sequence `$${` is interpreted literally and does not introduce an interpolation.
+  To express a `$` character immediately followed by an interpolation, the former must be escaped.
 
   [string interpolation]: ./string-interpolation.md
 
@@ -43,16 +46,16 @@
   Note that the whitespace and newline following the opening `''` is
   ignored if there is no non-whitespace text on the initial line.
 
-  Indented strings support [string interpolation].
-
   Since `${` and `''` have special meaning in indented strings, you
   need a way to quote them. `$` can be escaped by prefixing it with
   `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped
-  by prefixing it with `'`, i.e., `'''`. `$` removes any special
-  meaning from the following `$`. Linefeed, carriage-return and tab
+  by prefixing it with `'`, i.e., `'''`. Linefeed, carriage-return and tab
   characters can be written as `''\n`, `''\r`, `''\t`, and `''\`
   escapes any other character.
 
+  Indented strings support [string interpolation] using `${ }` the same way regular strings do.
+  `$${` is interpreted literally in indented strings as well, so the `$` character must be escaped if it is to be followed by an interpolation.
+
   Indented strings are primarily useful in that they allow multi-line
   string literals to follow the indentation of the enclosing Nix
   expression, and that less escaping is typically necessary for
@@ -85,6 +88,9 @@
   Numbers, which can be *integers* (like `123`) or *floating point*
   (like `123.43` or `.27e13`).
 
+  Integers in the Nix language are 64-bit signed integers.
+  Integer overflow is defined to throw an error.
+
   See [arithmetic] and [comparison] operators for semantics.
 
   [arithmetic]: ./operators.md#arithmetic

doc/manual/src/release-notes/rl-2.90.md

@@ -1,7 +1,7 @@
-# Lix 2.90 "Vanilla Ice Cream" (FIXME date)
+# Lix 2.90 "Vanilla Ice Cream" (2024-07-10)
 
 
-# Lix 2.90.0 (FIXME date)
+# Lix 2.90.0 (2024-07-10)
 
 ## Breaking Changes
 - Deprecate the online flake registries and vendor the default registry [fj#183](https://git.lix.systems/lix-project/lix/issues/183) [fj#110](https://git.lix.systems/lix-project/lix/issues/110) [fj#116](https://git.lix.systems/lix-project/lix/issues/116) [#8953](https://github.com/NixOS/nix/issues/8953) [#9087](https://github.com/NixOS/nix/issues/9087) [cl/1127](https://gerrit.lix.systems/c/lix/+/1127)

doc/manual/src/release-notes/rl-2.91.md

@@ -0,0 +1,489 @@
+# Lix 2.91 "Dragon's Breath" (2024-08-12)
+
+
+# Lix 2.91.0 (2024-08-12)
+
+## Breaking Changes
+- Block io_uring in the Linux sandbox [cl/1611](https://gerrit.lix.systems/c/lix/+/1611)
+
+  The io\_uring API has the unfortunate property that it is not possible to selectively decide which operations should be allowed.
+  This, together with the fact that new operations are routinely added, makes it a hazard to the proper function of the sandbox.
+
+  Therefore, any access to io\_uring has been made unavailable inside the sandbox.
+  As such, attempts to execute any system calls forming part of this API will fail with the error `ENOSYS`, as if io\_uring support had not been configured into the kernel.
+
+  Many thanks to [alois31](https://git.lix.systems/alois31) for this.
+- The `build-hook` setting is now deprecated
+
+  Build hooks communicate with the daemon using a custom, internal, undocumented protocol that is entirely unversioned and cannot be changed.
+  Since we intend to change it anyway we must unfortunately deprecate the current build hook infrastructure.
+  We do not expect this to impact most users—we have not found any uses of `build-hook` in the wild—but if this does affect you, we'd like to hear from you!
+- Lix no longer speaks the Nix remote-build worker protocol to clients or servers older than CppNix 2.3 [fj#325](https://git.lix.systems/lix-project/lix/issues/325) [cl/1207](https://gerrit.lix.systems/c/lix/+/1207) [cl/1208](https://gerrit.lix.systems/c/lix/+/1208) [cl/1206](https://gerrit.lix.systems/c/lix/+/1206) [cl/1205](https://gerrit.lix.systems/c/lix/+/1205) [cl/1204](https://gerrit.lix.systems/c/lix/+/1204) [cl/1203](https://gerrit.lix.systems/c/lix/+/1203) [cl/1479](https://gerrit.lix.systems/c/lix/+/1479)
+
+  CppNix 2.3 was released in 2019, and is the new oldest supported version. We
+  will increase our support baseline in the future up to a final version of CppNix
+  2.18 (which may happen soon given that it is the only still-packaged and thus
+  still-tested >2.3 version), but this step already removes a significant amount
+  of dead, untested, code paths.
+
+  Lix speaks the same version of the protocol as CppNix 2.18 and that fact will
+  never change in the future; the Lix plans to replace the protocol for evolution
+  will entail a complete incompatible replacement that will be supported in
+  parallel with the old protocol. Lix will thus retain remote build compatibility
+  with CppNix as long as CppNix maintains protocol compatibility with 2.18, and
+  as long as Lix retains legacy protocol support (which will likely be a long
+  time given that we plan to convert it to a frozen-in-time shim).
+
+  Many thanks to [jade](https://git.lix.systems/jade) for this.
+
+## Features
+- Pipe operator `|>` (experimental) [fj#438](https://git.lix.systems/lix-project/lix/issues/438) [cl/1654](https://gerrit.lix.systems/c/lix/+/1654)
+
+  Implementation of the pipe operator (`|>`) in the language as described in [RFC 148](https://github.com/NixOS/rfcs/pull/148).
+  The feature is still marked experimental, enable `--extra-experimental-features pipe-operator` to use it.
+
+  Many thanks to [piegames](https://git.lix.systems/piegames) and [eldritch horrors](https://git.lix.systems/pennae) for this.
+
+## Improvements
+- Trace which part of a `foo.bar.baz` expression errors [cl/1505](https://gerrit.lix.systems/c/lix/+/1505) [cl/1506](https://gerrit.lix.systems/c/lix/+/1506)
+
+  Previously, if an attribute path selection expression like `linux_4_9.meta.description` it wouldn't show you which one of those parts in the attribute path, or even that that line of code is what caused evaluation of the failing expression.
+  The previous error looks like this:
+
+  ```
+  pkgs.linuxKernel.kernels.linux_4_9.meta.description
+
+  error:
+         … while evaluating the attribute 'linuxKernel.kernels.linux_4_9.meta.description'
+           at /nix/store/dk2rpyb6ndvfbf19bkb2plcz5y3k8i5v-source/pkgs/top-level/linux-kernels.nix:278:5:
+            277|   } // lib.optionalAttrs config.allowAliases {
+            278|     linux_4_9 = throw "linux 4.9 was removed because it will reach its end of life within 22.11";
+               |     ^
+            279|     linux_4_14 = throw "linux 4.14 was removed because it will reach its end of life within 23.11";
+
+         … while calling the 'throw' builtin
+           at /nix/store/dk2rpyb6ndvfbf19bkb2plcz5y3k8i5v-source/pkgs/top-level/linux-kernels.nix:278:17:
+            277|   } // lib.optionalAttrs config.allowAliases {
+            278|     linux_4_9 = throw "linux 4.9 was removed because it will reach its end of life within 22.11";
+               |                 ^
+            279|     linux_4_14 = throw "linux 4.14 was removed because it will reach its end of life within 23.11";
+
+         error: linux 4.9 was removed because it will reach its end of life within 22.11
+  ```
+
+  Now, the error will look like this:
+
+  ```
+  pkgs.linuxKernel.kernels.linux_4_9.meta.description
+
+  error:
+         … while evaluating the attribute 'linuxKernel.kernels.linux_4_9.meta.description'
+           at /nix/store/dk2rpyb6ndvfbf19bkb2plcz5y3k8i5v-source/pkgs/top-level/linux-kernels.nix:278:5:
+            277|   } // lib.optionalAttrs config.allowAliases {
+            278|     linux_4_9 = throw "linux 4.9 was removed because it will reach its end of life within 22.11";
+               |     ^
+            279|     linux_4_14 = throw "linux 4.14 was removed because it will reach its end of life within 23.11";
+
+         … while evaluating 'pkgs.linuxKernel.kernels.linux_4_9' to select 'meta' on it
+           at «string»:1:1:
+              1| pkgs.linuxKernel.kernels.linux_4_9.meta.description
+               | ^
+
+         … caused by explicit throw
+           at /nix/store/dk2rpyb6ndvfbf19bkb2plcz5y3k8i5v-source/pkgs/top-level/linux-kernels.nix:278:17:
+            277|   } // lib.optionalAttrs config.allowAliases {
+            278|     linux_4_9 = throw "linux 4.9 was removed because it will reach its end of life within 22.11";
+               |                 ^
+            279|     linux_4_14 = throw "linux 4.14 was removed because it will reach its end of life within 23.11";
+
+         error: linux 4.9 was removed because it will reach its end of life within 22.11
+  ```
+
+  Not only does the line of code that referenced the failing attribute show up in the trace, it also tells you that it was specifically the `linux_4_9` part that failed.
+
+  This includes if the failing part is a top-level binding:
+
+  ```
+  let
+    inherit (pkgs.linuxKernel.kernels) linux_4_9;
+  in linux_4_9.meta.description
+  error:
+         … while evaluating 'linux_4_9' to select 'meta.description' on it
+           at «string»:3:4:
+              2|   inherit (pkgs.linuxKernel.kernels) linux_4_9;
+              3| in linux_4_9.meta.description
+               |    ^
+
+         … while evaluating the attribute 'linux_4_9'
+           at /nix/store/dk2rpyb6ndvfbf19bkb2plcz5y3k8i5v-source/pkgs/top-level/linux-kernels.nix:278:5:
+            277|   } // lib.optionalAttrs config.allowAliases {
+            278|     linux_4_9 = throw "linux 4.9 was removed because it will reach its end of life within 22.11";
+               |     ^
+            279|     linux_4_14 = throw "linux 4.14 was removed because it will reach its end of life within 23.11";
+
+         … caused by explicit throw
+           at /nix/store/dk2rpyb6ndvfbf19bkb2plcz5y3k8i5v-source/pkgs/top-level/linux-kernels.nix:278:17:
+            277|   } // lib.optionalAttrs config.allowAliases {
+            278|     linux_4_9 = throw "linux 4.9 was removed because it will reach its end of life within 22.11";
+               |                 ^
+            279|     linux_4_14 = throw "linux 4.14 was removed because it will reach its end of life within 23.11";
+
+         error: linux 4.9 was removed because it will reach its end of life within 22.11
+  ```
+
+  Many thanks to [Qyriad](https://git.lix.systems/Qyriad) for this.
+- Confusing 'invalid path' errors are now 'path does not exist' [cl/1161](https://gerrit.lix.systems/c/lix/+/1161) [cl/1160](https://gerrit.lix.systems/c/lix/+/1160) [cl/1159](https://gerrit.lix.systems/c/lix/+/1159)
+
+  Previously, if a path did not exist in a Nix store, it was referred to as the internal name "path is invalid".
+  This is, however, very confusing, and there were numerous such errors that were exactly the same, making it hard to debug.
+  These errors are now more specific and refer to the path not existing in the store.
+
+  Many thanks to [julia](https://git.lix.systems/midnightveil) for this.
+- Add a `build-dir` setting to set the backing directory for builds [gh#10303](https://github.com/NixOS/nix/pull/10303) [gh#10312](https://github.com/NixOS/nix/pull/10312) [gh#10883](https://github.com/NixOS/nix/pull/10883) [cl/1514](https://gerrit.lix.systems/c/lix/+/1514)
+
+  `build-dir` can now be set in the Nix configuration to choose the backing directory for the build sandbox.
+  This can be useful on systems with `/tmp` on tmpfs, or simply to relocate large builds to another disk.
+
+  Also, `XDG_RUNTIME_DIR` is no longer considered when selecting the default temporary directory,
+  as it's not intended to be used for large amounts of data.
+
+  Many thanks to [Robert Hensing](https://github.com/roberth) and [Tom Bereknyei](https://github.com/tomberek) for this.
+- Better usage of colour control environment variables [cl/1699](https://gerrit.lix.systems/c/lix/+/1699) [cl/1702](https://gerrit.lix.systems/c/lix/+/1702)
+
+  Lix now heeds `NO_COLOR`/`NOCOLOR` for more output types, such as that used in `nix search`, `nix flake metadata` and similar.
+
+  It also now supports `CLICOLOR_FORCE`/`FORCE_COLOR` to force colours regardless of whether there is a terminal on the other side.
+
+  It now follows rules compatible with those described on <https://bixense.com/clicolors/> with `CLICOLOR` defaulted to enabled.
+
+  That is to say, the following procedure is followed in order:
+  - NO_COLOR or NOCOLOR set
+
+    Always disable colour
+  - CLICOLOR_FORCE or FORCE_COLOR set
+
+    Enable colour
+  - The output is a tty; TERM != "dumb"
+
+    Enable colour
+  - Otherwise
+
+    Disable colour
+
+  Many thanks to [jade](https://git.lix.systems/jade) for this.
+- Distinguish between explicit throws and errors that happened while evaluating a throw [cl/1511](https://gerrit.lix.systems/c/lix/+/1511)
+
+  Previously, errors caused by an expression like `throw "invalid argument"` were treated like an error that happened simply while some builtin function was being called:
+
+  ```
+  let
+    throwMsg = p: throw "${p} isn't the right package";
+  in throwMsg "linuz"
+
+  error:
+         … while calling the 'throw' builtin
+           at «string»:2:17:
+              1| let
+              2|   throwMsg = p: throw "${p} isn't the right package";
+               |                 ^
+              3| in throwMsg "linuz"
+
+         error: linuz isn't the right package
+  ```
+
+  But the error didn't just happen "while" calling the `throw` builtin — it's a throw error!
+  Now it looks like this:
+
+  ```
+  let
+    throwMsg = p: throw "${p} isn't the right package";
+  in throwMsg "linuz"
+
+  error:
+         … caused by explicit throw
+           at «string»:2:17:
+              1| let
+              2|   throwMsg = p: throw "${p} isn't the right package";
+               |                 ^
+              3| in throwMsg "linuz"
+
+         error: linuz isn't the right package
+  ```
+
+  This also means that incorrect usage of `throw` or errors evaluating its arguments are easily distinguishable from explicit throws:
+
+  ```
+  let
+    throwMsg = p: throw "${p} isn't the right package";
+  in throwMsg { attrs = "error when coerced in string interpolation"; }
+
+  error:
+         … while calling the 'throw' builtin
+           at «string»:2:17:
+              1| let
+              2|   throwMsg = p: throw "${p} isn't the right package";
+               |                 ^
+              3| in throwMsg { attrs = "error when coerced in string interpolation"; }
+
+         … while evaluating a path segment
+           at «string»:2:24:
+              1| let
+              2|   throwMsg = p: throw "${p} isn't the right package";
+               |                        ^
+              3| in throwMsg { attrs = "error when coerced in string interpolation"; }
+
+         error: cannot coerce a set to a string: { attrs = "error when coerced in string interpolation"; }
+  ```
+
+  Here, instead of an actual thrown error, a type error happens first (trying to coerce an attribute set to a string), but that type error happened *while* calling `throw`.
+
+  Many thanks to [Qyriad](https://git.lix.systems/Qyriad) for this.
+- `nix flake metadata` prints modified date [cl/1700](https://gerrit.lix.systems/c/lix/+/1700)
+
+  Ever wonder "gee, when *did* I update nixpkgs"?
+  Wonder no more, because `nix flake metadata` now simply tells you the times every locked flake input was updated:
+
+  ```
+  <...>
+  Description:   The purely functional package manager
+  Path:          /nix/store/c91yi8sxakc2ry7y4ac1smzwka4l5p78-source
+  Revision:      c52cff582043838bbe29768e7da232483d52b61d-dirty
+  Last modified: 2024-07-31 22:15:54
+  Inputs:
+  ├───flake-compat: github:edolstra/flake-compat/0f9255e01c2351cc7d116c072cb317785dd33b33
+  │   Last modified: 2023-10-04 06:37:54
+  ├───nix2container: github:nlewo/nix2container/3853e5caf9ad24103b13aa6e0e8bcebb47649fe4
+  │   Last modified: 2024-07-10 13:15:56
+  ├───nixpkgs: github:NixOS/nixpkgs/e21630230c77140bc6478a21cd71e8bb73706fce
+  │   Last modified: 2024-07-25 11:26:27
+  ├───nixpkgs-regression: github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2
+  │   Last modified: 2022-01-24 11:20:45
+  └───pre-commit-hooks: github:cachix/git-hooks.nix/f451c19376071a90d8c58ab1a953c6e9840527fd
+      Last modified: 2024-07-15 04:21:09
+  ```
+
+  Many thanks to [jade](https://git.lix.systems/jade) for this.
+- Hash mismatch diagnostics for fixed-output derivations include the URL [cl/1536](https://gerrit.lix.systems/c/lix/+/1536)
+
+  Now, when building fixed-output derivations, Lix will guess the URL that was used in the derivation using the `url` or `urls` properties in the derivation environment.
+  This is a layering violation but making these diagnostics tractable when there are multiple instances of the `AAAA` hash is too significant of an improvement to pass it up.
+
+  ```
+  error: hash mismatch in fixed-output derivation '/nix/store/sjfw324j4533lwnpmr5z4icpb85r63ai-x1.drv':
+          likely URL: https://meow.puppy.forge/puppy.tar.gz
+           specified: sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+              got:    sha256-a1Qvp3FOOkWpL9kFHgugU1ok5UtRPSu+NwCZKbbaEro=
+  ```
+
+  Many thanks to [jade](https://git.lix.systems/jade) for this.
+- Add log formats `multiline` and `multiline-with-logs` [cl/1369](https://gerrit.lix.systems/c/lix/+/1369)
+
+  Added two new log formats (`multiline` and `multiline-with-logs`) that display
+  current activities below each other for better visibility.
+
+  These formats attempt to use the maximum available lines
+  (defaulting to 25 if unable to determine) and print up to that many lines.
+  The status bar is displayed as the first line, with each subsequent
+  activity on its own line.
+
+  Many thanks to [kloenk](https://git.lix.systems/kloenk) for this.
+- Lix will now show the package descriptions in when running `nix flake show`. [cl/1540](https://gerrit.lix.systems/c/lix/+/1540)
+
+  When running `nix flake show`, Lix will now show the package descriptions, if they exist.
+
+  Before:
+
+  ```shell
+  $ nix flake show
+  path:/home/isabel/dev/lix-show?lastModified=1721736108&narHash=sha256-Zo8HP1ur7Q2b39hKUEG8EAh/opgq8xJ2jvwQ/htwO4Q%3D
+  └───packages
+      └───x86_64-linux
+          ├───aNoDescription: package 'simple'
+          ├───bOneLineDescription: package 'simple'
+          ├───cMultiLineDescription: package 'simple'
+          ├───dLongDescription: package 'simple'
+          └───eEmptyDescription: package 'simple'
+  ```
+
+  After:
+
+  ```shell
+  $ nix flake show
+  path:/home/isabel/dev/lix-show?lastModified=1721736108&narHash=sha256-Zo8HP1ur7Q2b39hKUEG8EAh/opgq8xJ2jvwQ/htwO4Q%3D
+  └───packages
+      └───x86_64-linux
+          ├───aNoDescription: package 'simple'
+          ├───bOneLineDescription: package 'simple' - 'one line'
+          ├───cMultiLineDescription: package 'simple' - 'line one'
+          ├───dLongDescription: package 'simple' - 'abcdefghijklmnopqrstuvwxyz'
+          └───eEmptyDescription: package 'simple'
+  ```
+
+  Many thanks to [kjeremy](https://github.com/kjeremy) and [isabelroses](https://git.lix.systems/isabelroses) for this.
+- Eliminate some pretty-printing surprises [#11100](https://github.com/NixOS/nix/pull/11100) [cl/1616](https://gerrit.lix.systems/c/lix/+/1616) [cl/1617](https://gerrit.lix.systems/c/lix/+/1617) [cl/1618](https://gerrit.lix.systems/c/lix/+/1618)
+
+  Some inconsistent and surprising behaviours have been eliminated from the pretty-printing used by the REPL and `nix eval`:
+  * Lists and attribute sets that contain only a single item without nested structures are no longer sometimes inappropriately indented in the REPL, depending on internal state of the evaluator.
+  * Empty attribute sets and derivations are no longer shown as `«repeated»`, since they are always cheap to print.
+    This matches the existing behaviour of `nix-instantiate` on empty attribute sets.
+    Empty lists were never printed as `«repeated»` already.
+  * The REPL by default does not print nested attribute sets and lists, and indicates elided items with an ellipsis.
+    Previously, the ellipsis was printed even when the structure was empty, so that such items do not in fact exist.
+    Since this behaviour was confusing, it does not happen any more.
+
+  Before:
+  ```
+  nix-repl> :p let x = 1 + 2; in [ [ x ] [ x ] ]
+  [
+    [
+      3
+    ]
+    [ 3 ]
+  ]
+
+  nix-repl> let inherit (import <nixpkgs> { }) hello; in [ hello hello ]
+  [
+    «derivation /nix/store/fqs92lzychkm6p37j7fnj4d65nq9fzla-hello-2.12.1.drv»
+    «repeated»
+  ]
+
+  nix-repl> let x = {}; in [ x ]
+  [
+    { ... }
+  ]
+  ```
+
+  After:
+  ```
+  nix-repl> :p let x = 1 + 2; in [ [ x ] [ x ] ]
+  [
+    [ 3 ]
+    [ 3 ]
+  ]
+
+  nix-repl> let inherit (import <nixpkgs> { }) hello; in [ hello hello ]
+  [
+    «derivation /nix/store/fqs92lzychkm6p37j7fnj4d65nq9fzla-hello-2.12.1.drv»
+    «derivation /nix/store/fqs92lzychkm6p37j7fnj4d65nq9fzla-hello-2.12.1.drv»
+  ]
+
+  nix-repl> let x = {}; in [ x ]
+  [
+    { }
+  ]
+  ```
+
+  Many thanks to [alois31](https://git.lix.systems/alois31) and [Robert Hensing](https://github.com/roberth) for this.
+- `nix registry add` now requires a shorthand flakeref on the 'from' side [cl/1494](https://gerrit.lix.systems/c/lix/+/1494)
+
+  The 'from' argument must now be a shorthand flakeref like `nixpkgs` or `nixpkgs/nixos-20.03`, making it harder to accidentally swap the 'from' and 'to' arguments.
+
+  Registry entries that map from other flake URLs can still be specified in registry.json, the `nix.registry` option in NixOS, or the `--override-flake` option in the CLI, but they are not guaranteed to work correctly.
+
+  Many thanks to [delan](https://git.lix.systems/delan) for this.
+- Allow automatic rejection of configuration options from flakes [cl/1541](https://gerrit.lix.systems/c/lix/+/1541)
+
+  Setting `accept-flake-config` to `false` now respects user choice by automatically rejecting configuration options set by flakes.
+  The old behaviour of asking each time is still available (and default) by setting it to the special value `ask`.
+
+  Many thanks to [alois31](https://git.lix.systems/alois31) for this.
+- `nix repl` now allows tab-completing the special repl :colon commands [cl/1367](https://gerrit.lix.systems/c/lix/+/1367)
+
+  The REPL (`nix repl`) supports pressing `<TAB>` to complete a partial expression, but now also supports completing the special :colon commands as well (`:b`, `:edit`, `:doc`, etc), if the line starts with a colon.
+
+  Many thanks to [Qyriad](https://git.lix.systems/Qyriad) for this.
+- `:edit`ing a file in Nix store no longer reloads the repl [fj#341](https://git.lix.systems/lix-project/lix/issues/341) [cl/1620](https://gerrit.lix.systems/c/lix/+/1620)
+
+  Calling `:edit` from the repl now only reloads if the file being edited was outside of Nix store.
+  That means that all the local variables are now preserved across `:edit`s of store paths.
+  This is always safe because the store is read-only.
+
+  Many thanks to [goldstein](https://git.lix.systems/goldstein) for this.
+- `:log` in repl now works on derivation paths [fj#51](https://git.lix.systems/lix-project/lix/issues/51) [cl/1716](https://gerrit.lix.systems/c/lix/+/1716)
+
+  `:log` can now accept store derivation paths in addition to derivation expressions.
+
+  Many thanks to [goldstein](https://git.lix.systems/goldstein) for this.
+
+## Fixes
+- Define integer overflow in the Nix language as an error [fj#423](https://git.lix.systems/lix-project/lix/issues/423) [cl/1594](https://gerrit.lix.systems/c/lix/+/1594) [cl/1595](https://gerrit.lix.systems/c/lix/+/1595) [cl/1597](https://gerrit.lix.systems/c/lix/+/1597) [cl/1609](https://gerrit.lix.systems/c/lix/+/1609)
+
+  Previously, integer overflow in the Nix language invoked C++ level signed overflow, which was undefined behaviour, but *probably* manifested as wrapping around on overflow.
+
+  Since prior to the public release of Lix, Lix had C++ signed overflow defined to crash the process and nobody noticed this having accidentally removed overflow from the Nix language for three months until it was caught by fiddling around.
+  Given the significant body of actual Nix code that has been evaluated by Lix in that time, it does not appear that nixpkgs or much of importance depends on integer overflow, so it is safe to turn into an error.
+
+  Some other overflows were fixed:
+  - `builtins.fromJSON` of values greater than the maximum representable value in a signed 64-bit integer will generate an error.
+  - `nixConfig` in flakes will no longer accept negative values for configuration options.
+
+  Integer overflow now looks like the following:
+
+  ```
+  » nix eval --expr '9223372036854775807 + 1'
+  error: integer overflow in adding 9223372036854775807 + 1
+  ```
+
+  Many thanks to [jade](https://git.lix.systems/jade) for this.
+- Fix nix-collect-garbage --dry-run [fj#432](https://git.lix.systems/lix-project/lix/issues/432) [cl/1566](https://gerrit.lix.systems/c/lix/+/1566)
+
+  `nix-collect-garbage --dry-run` did not previously give any output - it simply
+  exited without even checking to see what paths would be deleted.
+
+  ```
+  $ nix-collect-garbage --dry-run
+  $
+  ```
+
+  We updated the behaviour of the flag such that instead it prints out how many
+  paths it *would* delete, but doesn't actually delete them.
+
+  ```
+  $ nix-collect-garbage --dry-run
+  finding garbage collector roots...
+  determining live/dead paths...
+  ...
+  <nix store paths>
+  ...
+  2670 store paths deleted, 0.00MiB freed
+  $
+  ```
+
+  Many thanks to [Quantum Jump](https://github.com/QuantumBJump) for this.
+- Fix unexpectedly-successful GC failures on macOS [fj#446](https://git.lix.systems/lix-project/lix/issues/446) [cl/1723](https://gerrit.lix.systems/c/lix/+/1723)
+
+  Has the following happened to you on macOS? This failure has been successfully eliminated, thanks to our successful deployment of advanced successful-failure detection technology (it's just `if (failed && errno == 0)`. Patent pending<sup>not really</sup>):
+
+  ```
+  $ nix-store --gc --print-dead
+  finding garbage collector roots...
+  error: Listing pid 87261 file descriptors: Undefined error: 0
+  ```
+
+  Many thanks to [jade](https://git.lix.systems/jade) for this.
+- `nix copy` is now several times faster at `querying info about /nix/store/...` [fj#366](https://git.lix.systems/lix-project/lix/issues/366) [cl/1462](https://gerrit.lix.systems/c/lix/+/1462)
+
+  We fixed a locking bug that serialized `querying info about /nix/store/...`
+  onto just one thread such that it was eating `O(paths to copy * latency)` time
+  while setting up to copy paths to s3 and other stores. It is now `nproc` times
+  faster.
+
+  Many thanks to [jade](https://git.lix.systems/jade) for this.
+
+## Development
+- clang-tidy support [fj#147](https://git.lix.systems/lix-project/lix/issues/147) [cl/1697](https://gerrit.lix.systems/c/lix/+/1697)
+
+  `clang-tidy` can be used to lint Lix with a limited set of lints using `ninja -C build clang-tidy` and `ninja -C build clang-tidy-fix`.
+  In practice, this fixes the built-in meson rule that was used the same as above being broken ever since precompiled headers were introduced.
+
+  Many thanks to [jade](https://git.lix.systems/jade) for this.
+- Lix now supports building with UndefinedBehaviorSanitizer [cl/1483](https://gerrit.lix.systems/c/lix/+/1483) [cl/1481](https://gerrit.lix.systems/c/lix/+/1481) [cl/1669](https://gerrit.lix.systems/c/lix/+/1669)
+
+  You can now build Lix with the configuration option `-Db_sanitize=undefined,address` and it will both work and pass tests with both AddressSanitizer and UndefinedBehaviorSanitizer enabled.
+  To use ASan specifically, you have to set `-Dgc=disabled`, which an error message will tell you to do if necessary anyhow.
+
+  Furthermore, tests passing with Clang ASan+UBSan is checked on every change in CI.
+
+  For a list of undefined behaviour found by tooling usage, see [the gerrit topic "undefined-behaviour"](https://gerrit.lix.systems/q/topic:%22undefined-behaviour%22).
+
+  Many thanks to [jade](https://git.lix.systems/jade) for this.

flake.lock

@@ -19,11 +19,11 @@
     "nix2container": {
       "flake": false,
       "locked": {
-        "lastModified": 1712990762,
-        "narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=",
+        "lastModified": 1720642556,
+        "narHash": "sha256-qsnqk13UmREKmRT7c8hEnz26X3GFFyIQrqx4EaRc1Is=",
         "owner": "nlewo",
         "repo": "nix2container",
-        "rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e",
+        "rev": "3853e5caf9ad24103b13aa6e0e8bcebb47649fe4",
         "type": "github"
       },
       "original": {
@@ -34,11 +34,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1718111384,
-        "narHash": "sha256-7tSst0S5FOmcgvNtfy6cjZX5w8CabCVAfAeCkhY4OVg=",
+        "lastModified": 1721931987,
+        "narHash": "sha256-1Zg8LY0T5EfXtv0Kf4M6SFnjH7Eto4VV+EKJ/YSnhiI=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "a508a44af0c1b1b57785c34d8b54783536273eeb",
+        "rev": "e21630230c77140bc6478a21cd71e8bb73706fce",
         "type": "github"
       },
       "original": {
@@ -67,11 +67,11 @@
     "pre-commit-hooks": {
       "flake": false,
       "locked": {
-        "lastModified": 1712055707,
-        "narHash": "sha256-4XLvuSIDZJGS17xEwSrNuJLL7UjDYKGJSbK1WWX2AK8=",
+        "lastModified": 1721042469,
+        "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=",
         "owner": "cachix",
         "repo": "git-hooks.nix",
-        "rev": "e35aed5fda3cc79f88ed7f1795021e559582093a",
+        "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd",
         "type": "github"
       },
       "original": {

flake.nix

@@ -1,5 +1,5 @@
 {
-  description = "The purely functional package manager";
+  description = "Lix: A modern, delicious implementation of the Nix package manager";
 
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05-small";
@@ -33,7 +33,7 @@
 
       # This notice gets echoed as a dev shell hook, and can be turned off with
       # `touch .nocontribmsg`
-      sgr = ''['';
+      sgr = builtins.fromJSON ''"\u001b["'';
       freezePage = "https://wiki.lix.systems/books/lix-contributors/page/freezes-and-recommended-contributions";
       codebaseOverview = "https://wiki.lix.systems/books/lix-contributors/page/codebase-overview";
       contribNotice = builtins.toFile "lix-contrib-notice" ''
@@ -59,7 +59,8 @@
         (Run `touch .nocontribmsg` to hide this message.)
       '';
 
-      officialRelease = false;
+      versionJson = builtins.fromJSON (builtins.readFile ./version.json);
+      officialRelease = versionJson.official_release;
 
       # Set to true to build the release notes for the next release.
       buildUnreleasedNotes = true;
@@ -84,10 +85,13 @@
       ];
       systems = linuxSystems ++ darwinSystems;
 
+      # If you add something here, please update the list in doc/manual/src/contributing/hacking.md.
+      # Thanks~
       crossSystems = [
         "armv6l-linux"
         "armv7l-linux"
         "riscv64-linux"
+        "aarch64-linux"
         # FIXME: still broken in 24.05: fails to build rustc(??) due to missing -lstdc++ dep
         # "x86_64-freebsd"
         # FIXME: broken dev shell due to python
@@ -137,10 +141,7 @@
                     system = crossSystem;
                   }
                   // lib.optionalAttrs (crossSystem == "x86_64-freebsd") { useLLVM = true; };
-              overlays = [
-                (overlayFor (p: p.${stdenv}))
-                (final: prev: { nixfmt = final.callPackage ./nix-support/nixfmt.nix { }; })
-              ];
+              overlays = [ (overlayFor (p: p.${stdenv})) ];
             };
           stdenvs = forAllStdenvs (make-pkgs null);
           native = stdenvs.stdenvPackages;
@@ -164,6 +165,7 @@
           nixUnstable = prev.nixUnstable;
 
           check-headers = final.buildPackages.callPackage ./maintainers/check-headers.nix { };
+          check-syscalls = final.buildPackages.callPackage ./maintainers/check-syscalls.nix { };
 
           default-busybox-sandbox-shell = final.busybox.override {
             useMusl = true;
@@ -195,14 +197,19 @@
             busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
           };
 
+          lix-clang-tidy = final.callPackage ./subprojects/lix-clang-tidy { };
+
           # Export the patched version of boehmgc that Lix uses into the overlay
           # for consumers of this flake.
-          boehmgc-nix = final.nix.boehmgc-nix;
+          boehmgc-nix = final.nix.passthru.boehmgc-nix;
           # And same thing for our build-release-notes package.
-          build-release-notes = final.nix.build-release-notes;
+          build-release-notes = final.nix.passthru.build-release-notes;
         };
     in
     {
+      # for repl debugging
+      inherit self;
+
       # A Nixpkgs overlay that overrides the 'nix' and
       # 'nix.perl-bindings' packages.
       overlays.default = overlayFor (p: p.stdenv);
@@ -233,6 +240,11 @@
           }
         );
 
+        # Completion tests for the Nix REPL.
+        repl-completion = forAllSystems (
+          system: nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { }
+        );
+
         # Perl bindings for various platforms.
         perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.passthru.perl-bindings);
 
@@ -266,6 +278,52 @@
 
         # System tests.
         tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // {
+          # This is x86_64-linux only, just because we have significantly
+          # cheaper x86_64-linux compute in CI.
+          # It is clangStdenv because clang's sanitizers are nicer.
+          asanBuild = self.packages.x86_64-linux.nix-clangStdenv.override {
+            # Improve caching of non-code changes by not changing the
+            # derivation name every single time, since this will never be seen
+            # by users anyway.
+            versionSuffix = "";
+            sanitize = [
+              "address"
+              "undefined"
+            ];
+            # it is very hard to make *every* CI build use this option such
+            # that we don't wind up building Lix twice, so we do it here where
+            # we are already doing so.
+            werror = true;
+          };
+
+          # Although this might be nicer to do with pre-commit, that would
+          # require adding 12MB of nodejs to the dev shell, whereas building it
+          # in CI with Nix avoids that at a cost of slower feedback on rarely
+          # touched files.
+          jsSyntaxCheck =
+            let
+              nixpkgs = nixpkgsFor.x86_64-linux.native;
+              inherit (nixpkgs) pkgs;
+              docSources = lib.fileset.toSource {
+                root = ./doc;
+                fileset = lib.fileset.fileFilter (f: f.hasExt "js") ./doc;
+              };
+            in
+            pkgs.runCommand "js-syntax-check" { } ''
+              find ${docSources} -type f -print -exec ${pkgs.nodejs-slim}/bin/node --check '{}' ';'
+              touch $out
+            '';
+
+          # clang-tidy run against the Lix codebase using the Lix clang-tidy plugin
+          clang-tidy =
+            let
+              nixpkgs = nixpkgsFor.x86_64-linux.native;
+              inherit (nixpkgs) pkgs;
+            in
+            pkgs.callPackage ./package.nix {
+              versionSuffix = "";
+              lintInsteadOfBuild = true;
+            };
 
           # Make sure that nix-env still produces the exact same result
           # on a particular version of Nixpkgs.
@@ -316,6 +374,25 @@
         pkgs = nixpkgsFor.x86_64-linux.native;
       };
 
+      releaseTests = lib.foldl lib.recursiveUpdate { } [
+        (lib.genAttrs (linux64BitSystems ++ darwinSystems) (system: {
+          nativeBuild = self.packages.${system}.nix;
+        }))
+        (lib.genAttrs (linux64BitSystems) (system: {
+          staticBuild = self.packages.${system}.nix-static;
+        }))
+        {
+          x86_64-linux = {
+            # TODO add more cross/static release targets?
+            crossBuild.aarch64-linux = self.packages.x86_64-linux.nix-aarch64-linux;
+
+            # TODO wire up a nixos installer test with that lix and
+            # run it, once nixpkgs can actually do that (again). :/
+            # # nix build .#nixosTests.installer.{btrfsSimple,luksroot,lvm,simple,switchToFlake}
+          };
+        }
+      ];
+
       # NOTE *do not* add fresh derivations to checks, always add them to
       # hydraJobs first (so CI will pick them up) and only link them here
       checks = forAvailableSystems (
@@ -330,6 +407,7 @@
           rl-next = self.hydraJobs.rl-next.${system}.user;
           # Will be empty attr set on i686-linux, and filtered out by forAvailableSystems.
           pre-commit = self.hydraJobs.pre-commit.${system};
+          repl-completion = self.hydraJobs.repl-completion.${system};
         }
         // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
           dockerImage = self.hydraJobs.dockerImage.${system};
@@ -341,6 +419,8 @@
         rec {
           inherit (nixpkgsFor.${system}.native) nix;
           default = nix;
+
+          inherit (nixpkgsFor.${system}.native) lix-clang-tidy;
         }
         // (
           lib.optionalAttrs (builtins.elem system linux64BitSystems) {
@@ -377,9 +457,9 @@
             pkgs: stdenv:
             let
               nix = pkgs.callPackage ./package.nix {
-                inherit stdenv officialRelease versionSuffix;
+                inherit stdenv versionSuffix;
                 busybox-sandbox-shell = pkgs.busybox-sandbox-shell or pkgs.default-busybox-sandbox;
-                internalApiDocs = true;
+                internalApiDocs = false;
               };
               pre-commit = self.hydraJobs.pre-commit.${pkgs.system} or { };
             in
@@ -407,7 +487,7 @@
             makeShell pkgs pkgs.stdenv
           ))
           // {
-            default = self.devShells.${system}.native-stdenvPackages;
+            default = self.devShells.${system}.native-clangStdenvPackages;
           }
         );
     };

justfile

@@ -9,8 +9,8 @@ clean:
     rm -rf build
 
 # Prepare meson for building
-setup:
-    meson setup build --prefix="$PWD/outputs/out"
+setup *OPTIONS:
+    meson setup build --prefix="$PWD/outputs/out" $mesonFlags {{ OPTIONS }}
 
 # Build lix
 build *OPTIONS:
@@ -25,3 +25,13 @@ install *OPTIONS: (build OPTIONS)
 # Run tests
 test *OPTIONS:
     meson test -C build --print-errorlogs {{ OPTIONS }}
+
+alias clang-tidy := lint
+
+lint:
+    ninja -C build clang-tidy
+
+alias clang-tidy-fix := lint-fix
+
+lint-fix:
+    ninja -C build clang-tidy-fix

lix-doc/Cargo.lock

@@ -9,13 +9,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
 [[package]]
-name = "cbitset"
-version = "0.2.0"
+name = "countme"
+version = "3.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29b6ad25ae296159fb0da12b970b2fe179b234584d7cd294c891e2bbb284466b"
-dependencies = [
- "num-traits",
-]
+checksum = "7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636"
 
 [[package]]
 name = "dissimilar"
@@ -33,19 +30,26 @@ dependencies = [
  "once_cell",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.14.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
+
 [[package]]
 name = "lix-doc"
 version = "0.0.1"
 dependencies = [
  "expect-test",
  "rnix",
+ "rowan",
 ]
 
 [[package]]
-name = "num-traits"
-version = "0.2.18"
+name = "memoffset"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
 dependencies = [
  "autocfg",
 ]
@@ -56,44 +60,26 @@ version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
 
-[[package]]
-name = "proc-macro2"
-version = "1.0.79"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
-dependencies = [
- "proc-macro2",
-]
-
 [[package]]
 name = "rnix"
-version = "0.8.1"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a9b645f0edba447dbfc6473dd22999f46a1d00ab39e777a2713a1cf34a1597b"
+checksum = "bb35cedbeb70e0ccabef2a31bcff0aebd114f19566086300b8f42c725fc2cb5f"
 dependencies = [
- "cbitset",
  "rowan",
 ]
 
 [[package]]
 name = "rowan"
-version = "0.9.1"
+version = "0.15.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ea7cadf87a9d8432e85cb4eb86bd2e765ace60c24ef86e79084dcae5d1c5a19"
+checksum = "32a58fa8a7ccff2aec4f39cc45bf5f985cec7125ab271cf681c279fd00192b49"
 dependencies = [
+ "countme",
+ "hashbrown",
+ "memoffset",
  "rustc-hash",
- "smol_str",
- "text_unit",
- "thin-dst",
+ "text-size",
 ]
 
 [[package]]
@@ -103,59 +89,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
 
 [[package]]
-name = "serde"
-version = "1.0.197"
+name = "text-size"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2"
-dependencies = [
- "serde_derive",
-]
-
-[[package]]
-name = "serde_derive"
-version = "1.0.197"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "smol_str"
-version = "0.1.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fad6c857cbab2627dcf01ec85a623ca4e7dcb5691cbaa3d7fb7653671f0d09c9"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "syn"
-version = "2.0.53"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7383cd0e49fff4b6b90ca5670bfd3e9d6a733b3f90c686605aa7eec8c4996032"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "text_unit"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20431e104bfecc1a40872578dbc390e10290a0e9c35fffe3ce6f73c15a9dbfc2"
-
-[[package]]
-name = "thin-dst"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db3c46be180f1af9673ebb27bc1235396f61ef6965b3fe0dbb2e624deb604f0e"
-
-[[package]]
-name = "unicode-ident"
-version = "1.0.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+checksum = "f18aa187839b2bdb1ad2fa35ead8c4c2976b64e4363c386d45ac0f7ee85c9233"

lix-doc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 description = "Nix function documentation tool, stripped down into a library"
-edition = "2018"
+edition = "2021"
 name = "lix-doc"
 version = "0.0.1"
 license = "BSD-2-Clause OR MIT"
@@ -12,7 +12,9 @@ repository = "https://github.com/lf-/nix-doc"
 crate_type = ["staticlib"]
 
 [dependencies]
-rnix = "0.8.0"
+rnix = "0.11.0"
+# Necessary because rnix fails to export a critical trait (Rowan's AstNode).
+rowan = "0.15.0"
 
 [dev-dependencies]
 expect-test = "1.1.0"

lix-doc/src/lib.rs

@@ -1,5 +1,5 @@
 // SPDX-FileCopyrightText: 2024 Jade Lovelace
-//
+// SPDX-FileCopyrightText: 2024 Lunaphied
 // SPDX-License-Identifier: BSD-2-Clause OR MIT
 
 //! library components of nix-doc
@@ -7,13 +7,16 @@ pub mod pprint;
 
 use crate::pprint::pprint_args;
 
-use rnix::types::{Lambda, TypedNode};
-use rnix::SyntaxKind::*;
-use rnix::{NodeOrToken, SyntaxNode, TextUnit, WalkEvent};
+use rnix::ast::{self, Lambda};
+use rnix::{NodeOrToken, SyntaxKind};
+use rnix::SyntaxNode;
+
+
+// Needed because rnix fucked up and didn't reexport this, oops.
+use rowan::ast::AstNode;
 
 use std::ffi::{CStr, CString};
 use std::fs;
-use std::iter;
 use std::os::raw::c_char;
 use std::panic;
 
@@ -23,66 +26,104 @@ use std::{fmt::Display, str};
 
 pub type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
 
-const DOC_INDENT: usize = 3;
-
 struct SearchResult {
     /// Name of the function
     identifier: String,
 
-    /// Dedented documentation comments
+    /// Dedented documentation comment
     doc: String,
 
     /// Parameter block for the function
     param_block: String,
 }
 
-fn find_pos(file: &str, line: usize, col: usize) -> usize {
-    let mut lines = 1;
-    let mut line_start = 0;
-    let mut it = file.chars().enumerate().peekable();
-    while let Some((count, ch)) = it.next() {
-        if ch == '\n' || ch == '\r' {
-            lines += 1;
-            let addend = if ch == '\r' && it.peek().map(|x| x.1) == Some('\n') {
-                it.next();
-                1
-            } else {
-                0
-            };
-            line_start = count + addend;
-        }
-
-        let col_diff = ((count as i32) - (line_start as i32)).abs() as usize;
-        if lines == line && col_diff == col {
-            return count;
-        }
-    }
-    unreachable!();
-}
-
 impl SearchResult {
     fn format<P: Display>(&self, filename: P, line: usize) -> String {
         format!(
             "**Synopsis:** `{}` = {}\n\n{}\n\n# {}",
             self.identifier.as_str(),
             self.param_block,
-            indented(&self.doc, DOC_INDENT),
+            self.doc,
             format!("{}:{}", filename, line).as_str(),
         )
     }
 }
 
-/// Emits a string `s` indented by `indent` spaces
-fn indented(s: &str, indent: usize) -> String {
-    let indent_s = iter::repeat(' ').take(indent).collect::<String>();
-    s.split('\n')
-        .map(|line| indent_s.clone() + line)
-        .collect::<Vec<_>>()
-        .join("\n")
+/// Converts Nix compatible line endings (Nix accepts `\r`, `\n`, *and* `\r\n` as endings), to
+/// standard `\n` endings for use within Rust land.
+fn convert_endings(s: &str) -> String {
+    let mut out = String::with_capacity(s.len());
+    let mut it = s.chars().peekable();
+
+    while let Some(ch) = it.next() {
+        if ch == '\n' || ch == '\r' {
+            out.push('\n');
+            if ch == '\r' && it.peek().map(|&c| c == '\n').unwrap_or(false) {
+                // Consume `\n` in `\r\n`.
+                it.next();
+            }
+        } else {
+            out.push(ch);
+        }
+    }
+
+    out
+}
+
+/// Converts the position information from Lix itself into an character index into the file itself.
+/// Expects an input string that's already had it's line endings normalized.
+///
+/// Note that this returns a *byte* offset, not a character offset.
+fn find_pos(s: &str, line: usize, col: usize) -> usize {
+    // Nix line positions are 1-indexed.
+    let mut lines = 1;
+    for (byte_pos, ch) in s.char_indices() {
+        // If we find a newline, increase the line count.
+        if ch == '\n' {
+            lines += 1;
+        }
+
+        // We've arrived at the correct line.
+        if lines == line {
+            // Column position is 1-indexed, and it's a *byte* offset, because Nix doesn't actually
+            // support UTF-8. Rust does though, so we need to convert to a proper byte index to
+            // match rnix. Lix also doesn't consider the line endings part of the column offset so
+            // we implicitly add one to advance to the character *after* that.
+            return byte_pos + col;
+        }
+    }
+
+    // If things never match that should be literally impossible.
+    unreachable!();
+}
+
+/// Represents a forwarded token from rnix's AST over to lix-doc.
+#[derive(Debug, Clone)]
+enum DocToken {
+    Comment(String),
+    Whitespace(String),
+}
+
+/// Determine if a given token string contains more than two newlines, this is used to determine when
+/// we hit blank lines between comments indicating a contextually unrelated comment.
+fn has_empty_line(tok: &DocToken) -> bool {
+    // It's either solely whitespace with two newlines inside somewhere, or it's
+    // contained inside a comment token and we don't want to count that as empty.
+    if let DocToken::Whitespace(s) = tok {
+        s.chars().filter(|&c| c == '\n').take(2).count() == 2
+    } else {
+        false
+    }
 }
 
 /// Cleans up a single line, erasing prefix single line comments but preserving indentation
-fn cleanup_single_line<'a>(s: &'a str) -> &'a str {
+// NOTE: We have a bit of a conflict of interest problem here due to the inconsistent format of
+// doc comments. Some doc comments will use a series of single line comments that may then contain `*`
+// characters to represent a list. Some will be multiline comments that don't prefix individual lines
+// with `*`, only using them for lists directly, and some will prefix lines with `*` as a leading
+// character to mark the block. There's no way to disambiguate all three, but we do our best to
+// make the common case pretty.
+fn cleanup_single_line(s: &str) -> &str {
     let mut cmt_new_start = 0;
     let mut iter = s.char_indices().peekable();
     while let Some((idx, ch)) = iter.next() {
@@ -90,7 +131,9 @@ fn cleanup_single_line<'a>(s: &'a str) -> &'a str {
         let (_, next_ch) = iter.peek().unwrap_or(&(0, '\n'));
 
         // if we find a character, save the byte position after it as our new string start
-        if ch == '#' || (ch == '*' && next_ch.is_whitespace()) {
+        // This has special handling for `>` because some Nixpkgs documentation has `*>` right
+        // after the start of their doc comments, and we want to strip the `*` still.
+        if ch == '#' || (ch == '*' && (*next_ch == '>' || next_ch.is_whitespace())) {
             cmt_new_start = idx + 1;
             break;
         }
@@ -103,15 +146,12 @@ fn cleanup_single_line<'a>(s: &'a str) -> &'a str {
     &s[cmt_new_start..]
 }
 
-/// Erases indents in comments. This is *almost* a normal dedent function, but it starts by looking
-/// at the second line if it can.
+/// Erases indents in comments based on the indentation of the first line.
 fn dedent_comment(s: &str) -> String {
     let mut whitespaces = 0;
-    let mut lines = s.lines();
-    let first = lines.next();
 
     // scan for whitespace
-    for line in lines.chain(first) {
+    for line in s.lines() {
         let line_whitespace = line.chars().take_while(|ch| ch.is_whitespace()).count();
 
         if line_whitespace != line.len() {
@@ -121,16 +161,6 @@ fn dedent_comment(s: &str) -> String {
         }
     }
 
-    // maybe the first considered line we found was indented further, so let's look for more lines
-    // that might have a shorter indent. In the case of one line, do nothing.
-    for line in s.lines().skip(1) {
-        let line_whitespace = line.chars().take_while(|ch| ch.is_whitespace()).count();
-
-        if line_whitespace != line.len() {
-            whitespaces = line_whitespace.min(whitespaces);
-        }
-    }
-
     // delete up to `whitespaces` whitespace characters from each line and reconstitute the string
     let mut out = String::new();
     for line in s.lines() {
@@ -143,69 +173,163 @@ fn dedent_comment(s: &str) -> String {
     out
 }
 
-/// Deletes whitespace and leading comment characters
+/// Takes a series of comment and whitespace strings and output a clean single block of text to use
+/// as the output documentation comment block.
 ///
-/// Oversight we are choosing to ignore: if you put # characters at the beginning of lines in a
-/// multiline comment, they will be deleted.
-fn cleanup_comments<S: AsRef<str>, I: DoubleEndedIterator<Item = S>>(comment: &mut I) -> String {
+/// This function expects to be given the tokens in reverse order (proceeding upwards from the
+/// first comment above the definitions), this allows us to properly enforce the below conditions.
+/// The output from this function will be reordered and ready for display.
+///
+/// The two types of documentation comments we expect are:
+///
+/// - A single multiline comment not whitespace separated from the start.
+/// - A series of back to back single line comments not separated by whitespace.
+///
+/// Any other combination will be filtered out.
+///
+/// Once an empty line is encountered, we know no more valid documentation comments remain and stop.
+fn cleanup_comments<I: Iterator<Item = DocToken>>(tokens: &mut I) -> String {
+    // Keep track of when we've found a single line and multiline comment, we use this to
+    // only process a single multiline or back to back single lines.
+    let mut found_single_line = false;
+
+    // Comments that have survived our filtering phase and should be cleaned up.
+    let mut valid = vec![];
+
+    // Filter out comments that don't meet the characteristics of documentation comments.
+    for tok in tokens {
+        if has_empty_line(&tok) {
+            // Take tokens until we hit whitespace containing an empty line.
+            break;
+        }
+
+        // Only care about comments from this point on.
+        if let DocToken::Comment(comment) = tok {
+            // Now determine if it's a single line comment.
+            let is_single_line = comment.starts_with('#');
+
+            // We've found a single line comment if we've found one before or we just found one.
+            found_single_line |= is_single_line;
+
+            // What we do next is only special when we hit a multiline comment.
+            if !is_single_line {
+                // If we've hit a multiline comment as our first comment, take that one alone.
+                if !found_single_line {
+                    // Otherwise we've hit a multiline comment immediately and this is our
+                    // one and only doc comment to worry about.
+                    valid.push(comment);
+                }
+                // Otherwise we've hit a multiline comment after single line comments, in either
+                // case this means we're done processing comments.
+                break;
+            }
+
+            // Otherwise this is a new single line comment to push to the stack.
+            valid.push(comment);
+        }
+    }
+
+    // Cleanup comments for user consumption.
     dedent_comment(
-        &comment
+        &valid
+            .into_iter()
             .rev()
             .map(|small_comment| {
                 small_comment
-                    .as_ref()
-                    // space before multiline start
-                    .trim_start()
-                    // multiline starts
+                    // Trim off start of multiline comments.
                     .trim_start_matches("/*")
-                    // trailing so we can grab multiline end
-                    .trim_end()
-                    // multiline ends
+                    // Trim off end of multiline comments.
                     .trim_end_matches("*/")
-                    // extra space that was in the multiline
+                    // Trim off any internal whitespace that's trapped inside comments themselves.
                     .trim()
+                    // Split comments by newlines to extract lines of multiline comments.
                     .split('\n')
-                    // erase single line comments and such
+                    // Cleanup single line comments and a few more tweaks for multiline comments.
                     .map(cleanup_single_line)
                     .collect::<Vec<_>>()
+                    // Reconstruct the multiline comment's whitespace.
                     .join("\n")
             })
             .collect::<Vec<_>>()
-            .join("\n"),
+            // We've found that when multiple back to back single line comments are used in Nixpkgs,
+            // they make more sense to represent as if someone inserted line breaks into the Markdown
+            // properly, so we join them with linebreaks that markdown will pass through.
+            .join("\n\n"),
     )
 }
 
-/// Get the docs for a specific function
+/// Get the docs for a specific function.
+// TODO: Improve error reporting?
 pub fn get_function_docs(filename: &str, line: usize, col: usize) -> Option<String> {
     let content = fs::read(filename).ok()?;
-    let decoded = str::from_utf8(&content).ok()?;
+    let decoded = convert_endings(str::from_utf8(&content).ok()?);
     let pos = find_pos(&decoded, line, col);
-    let rowan_pos = TextUnit::from_usize(pos);
-    let tree = rnix::parse(decoded);
+    let rowan_pos = rnix::TextSize::from(pos as u32);
 
+    // The minimum length of a lambda is 4 characters and thus the range we're looking for must be
+    // at least 4 characters long `_: 3` being an example of a minimal length lambda.
+    let rowan_range = rnix::TextRange::at(rowan_pos, 4.into());
+
+    // Parse the file  using rnix.
+    let root = rnix::Root::parse(&decoded).ok().ok()?;
+
+    // Extract the inner expression that represents the Root node and extract the top level expression.
+    let expr = root.expr()?;
+
+    // There are two cases we have to be able to handle
+    // 1. A straightforward definition with an attrset binding to a lambda that's defined inline.
+    // 2. A lambda defined in a standalone file where the attrset binding imports that file directly.
+    // The latter case will not be able to find the binding so we must be able to handle not finding it.
+
+    // Find the deepest node or token that covers the position given by Lix.
+    let covering = expr.syntax().covering_element(rowan_range);
+
+    // Climb up until we find the lambda node that contains that token.
     let mut lambda = None;
-    for node in tree.node().preorder() {
-        match node {
-            WalkEvent::Enter(n) => {
-                if n.text_range().start() >= rowan_pos && n.kind() == NODE_LAMBDA {
-                    lambda = Lambda::cast(n);
+    for ancestor in covering.ancestors() {
+        if ancestor.kind() == SyntaxKind::NODE_LAMBDA {
+            lambda = Some(ancestor);
             break;
         }
     }
-            WalkEvent::Leave(_) => (),
+
+    // There is literally always a lambda or something has gone very very wrong.
+    let lambda =
+        ast::Lambda::cast(
+            lambda.expect("no lambda found; what.")
+        ) .expect("not a rnix::ast::Lambda; what.");
+
+    // Search up, hopefully to find the binding so we can get the identifier name.
+    // TODO: Just provide this directly from the C++ code to make it possible to always have the correct identifier.
+    let mut binding = None;
+    for ancestor in lambda.syntax().ancestors() {
+        if ancestor.kind() == SyntaxKind::NODE_ATTRPATH_VALUE {
+            binding = Some(ancestor);
         }
     }
-    let lambda = lambda?;
-    let res = visit_lambda("func".to_string(), &lambda);
-    Some(res.format(filename, line))
-}
 
-fn visit_lambda(name: String, lambda: &Lambda) -> SearchResult {
+    // Convert the binding to an identifier if it was found, otherwise use a placeholder.
+    let identifier;
+    identifier = match binding.clone() {
+        Some(binding) => ast::AttrpathValue::cast(binding)
+            .expect("not an rnix::ast::AttrpathValue; what")
+            .attrpath()
+            .expect("AttrpathValue has no attrpath; what.")
+            .to_string(),
+        _ => "<unknown binding>".to_string(),
+    };
+
+    // Find all the comments on the binding or the lambda if we have to fall back.
+    let comment_node = binding.as_ref().unwrap_or(lambda.syntax());
+    let comment = find_comment(comment_node).unwrap_or_else(String::new);
+
+    // And display them properly for the markdown function in Lix.
+    Some(visit_lambda(identifier, comment, &lambda).format(filename, line))
+}
+
+fn visit_lambda(name: String, comment: String, lambda: &Lambda) -> SearchResult {
     // grab the arguments
-    let param_block = pprint_args(&lambda);
-
-    // find the doc comment
-    let comment = find_comment(lambda.node().clone()).unwrap_or_else(|| "".to_string());
+    let param_block = pprint_args(lambda);
 
     SearchResult {
         identifier: name,
@@ -214,39 +338,47 @@ fn visit_lambda(name: String, lambda: &Lambda) -> SearchResult {
     }
 }
 
-fn find_comment(node: SyntaxNode) -> Option<String> {
-    let mut node = NodeOrToken::Node(node);
-    let mut comments = Vec::new();
-    loop {
-        loop {
-            if let Some(new) = node.prev_sibling_or_token() {
-                node = new;
-                break;
-            } else {
-                node = NodeOrToken::Node(node.parent()?);
-            }
+fn find_comment(node: &SyntaxNode) -> Option<String> {
+    let mut it = node
+        .siblings_with_tokens(rowan::Direction::Prev)
+        // Skip ourselves as we're always the first token returned.
+        .skip(1)
+        .peekable();
+
+    // Consume up to one whitespace token before the first comment. There might not always be
+    // whitespace such as the (rather unusual) case of `/* meow */x = a: 3`.
+    if matches!(it.peek(), Some(NodeOrToken::Token(token)) if token.kind() == SyntaxKind::TOKEN_WHITESPACE) {
+        it.next();
     }
 
-        match node.kind() {
-            TOKEN_COMMENT => match &node {
-                NodeOrToken::Token(token) => comments.push(token.text().clone()),
-                NodeOrToken::Node(_) => unreachable!(),
-            },
-            // This stuff is found as part of `the-fn = f: ...`
-            // here:                           ^^^^^^^^
-            NODE_KEY | TOKEN_ASSIGN => (),
-            t if t.is_trivia() => (),
-            _ => break,
+    let comments = it.map_while(|element| match element {
+            NodeOrToken::Token(token) => {
+                match token.kind() {
+                    // Map the tokens we're interested in to our internal token type.
+                    SyntaxKind::TOKEN_COMMENT => Some(DocToken::Comment(token.text().to_owned())),
+                    SyntaxKind::TOKEN_WHITESPACE => {
+                        Some(DocToken::Whitespace(token.text().to_owned()))
+                    }
+                    // If we hit a different token type, we know we've gone past relevant comments
+                    // and should stop.
+                    _ => None,
                 }
             }
-    let doc = cleanup_comments(&mut comments.iter().map(|c| c.as_str()));
-    Some(doc).filter(|it| !it.is_empty())
+            // If we hit a node entry we've definitely gone past comments that would be related to
+            // this node and we should retreat.
+            _ => None,
+        });
+
+    // For the curious, `into_iter()` here consumes the binding producing an owned value allowing us to avoid
+    // making the original binding mutable, we don't reuse it later so this is a cute way to handle it, though
+    // there's probably a better way we just can't remember.
+    Some(cleanup_comments(&mut comments.into_iter())).filter(|c| !c.is_empty())
 }
 
 /// Get the docs for a function in the given file path at the given file position and return it as
 /// a C string pointer
 #[no_mangle]
-pub extern "C" fn nd_get_function_docs(
+pub extern "C" fn lixdoc_get_function_docs(
     filename: *const c_char,
     line: usize,
     col: usize,
@@ -269,9 +401,9 @@ pub extern "C" fn nd_get_function_docs(
         .unwrap_or(ptr::null())
 }
 
-/// Call this to free a string from nd_get_function_docs
+/// Call this to free a string from `lixdoc_get_function_docs`.
 #[no_mangle]
-pub extern "C" fn nd_free_string(s: *const c_char) {
+pub extern "C" fn lixdoc_free_string(s: *const c_char) {
     unsafe {
         // cast note: this cast is turning something that was cast to const
         // back to mut
@@ -283,35 +415,57 @@ pub extern "C" fn nd_free_string(s: *const c_char) {
 mod tests {
     use super::*;
 
+    #[test]
+    fn test_line_conversion() {
+        let fakefile = "abc\rdef\r\nghi";
+        assert_eq!(convert_endings(fakefile), "abc\ndef\nghi");
+    }
+
     #[test]
     fn test_bytepos() {
         let fakefile = "abc\ndef\nghi";
         assert_eq!(find_pos(fakefile, 2, 2), 5);
     }
 
+    #[test]
+    fn test_bytepos_unusual() {
+        let fakefile = convert_endings("abc\rdef\r\nghi");
+        assert_eq!(find_pos(&fakefile, 2, 2), 5);
+        assert_eq!(find_pos(&fakefile, 3, 2), 9);
+    }
+
+    /// This test is to check that we correctly resolve byte positions even when inconsistent with
+    /// character positions.
     #[test]
     fn test_bytepos_cursed() {
-        let fakefile = "abc\rdef\r\nghi";
-        assert_eq!(find_pos(fakefile, 2, 2), 5);
-        assert_eq!(find_pos(fakefile, 3, 2), 10);
+        let fakefile = "hello\nwórld";
+        // Try to find the position of the `r` after world, which will be wrong if we don't handle
+        // UTF-8 properly.
+        let pos = find_pos(&fakefile, 2, 4);
+        dbg!(&fakefile[pos..]);
+        assert_eq!(pos, 9)
     }
 
     #[test]
     fn test_comment_stripping() {
-        let ex1 = ["/* blah blah blah\n      foooo baaar\n   blah */"];
+        let ex1 = [DocToken::Comment(
+            "/* blah blah blah\n      foooo baaar\n   blah */".to_string(),
+        )];
         assert_eq!(
-            cleanup_comments(&mut ex1.iter()),
+            cleanup_comments(&mut ex1.into_iter()),
             "blah blah blah\n      foooo baaar\n   blah"
         );
 
-        let ex2 = ["# a1", "#    a2", "# aa"];
-        assert_eq!(cleanup_comments(&mut ex2.iter()), "aa\n   a2\na1");
+        let ex2 = ["# a1", "#    a2", "# aa"]
+            .into_iter()
+            .map(|s| DocToken::Comment(s.to_string()));
+        assert_eq!(cleanup_comments(&mut ex2.into_iter()), "aa\n\n   a2\n\na1");
     }
 
     #[test]
     fn test_dedent() {
         let ex1 = "a\n   b\n   c\n     d";
-        assert_eq!(dedent_comment(ex1), "a\nb\nc\n  d");
+        assert_eq!(dedent_comment(ex1), ex1);
         let ex2 = "a\nb\nc";
         assert_eq!(dedent_comment(ex2), ex2);
         let ex3 = "   a\n   b\n\n     c";
@@ -335,4 +489,31 @@ mod tests {
         let ex1 = "   **Foo**:";
         assert_eq!(cleanup_single_line(ex1), ex1);
     }
+
+    // TODO: Next CL
+    //#[test]
+    //fn comment_test_complex() {
+    //    let testcase = r#"
+    //    rec {
+    //        /*
+    //           Hello
+    //           23
+    //             This is a comment.
+    //             this is another comment.
+    //             and this is a third comment.
+    //                          Way
+    //              go
+    //        */
+    //        meow = { g }: {a, b ? 4, ...}: g: c: 5;
+    //        # And another comment.
+    //        cat = 34;
+    //        # inner layer.
+    //        "inner-layer" = outer: meow;
+    //    }
+    //    "#;
+    //    // Need to find the location of the lambda, we do a quick hack.
+    //    let location = dbg!(testcase.find("{ g }").unwrap() as u32);
+    //
+    //    //get_function_docs(filename, line, col)
+    //}
 }

lix-doc/src/pprint.rs

@@ -1,38 +1,61 @@
 // SPDX-FileCopyrightText: 2024 Jade Lovelace
-//
+// SPDX-FileCopyrightText: 2024 Lunaphied
 // SPDX-License-Identifier: BSD-2-Clause OR MIT
 
-use rnix::types::{Lambda, TypedNode};
-use rnix::SyntaxKind::*;
+use rnix::ast::{Expr, Lambda};
+use rowan::ast::AstNode;
 
 /// Pretty-prints the arguments to a function
 pub fn pprint_args(lambda: &Lambda) -> String {
     // TODO: handle docs directly on NODE_IDENT args (uncommon case)
     let mut lambda = lambda.clone();
+    let mut depth = 0;
     let mut out = String::new();
     loop {
-        let arg = lambda.arg().unwrap();
-        match arg.kind() {
-            NODE_IDENT => {
-                out += &format!("*{}*", &arg.to_string());
+        let arg = lambda.param().unwrap();
+        for child in arg.syntax().children_with_tokens() {
+            //dbg!(child.kind());
+            match child {
+                rowan::NodeOrToken::Node(node) => {
+                    out.push_str(&node.text().to_string());
+                    if node.kind() == rnix::SyntaxKind::NODE_PAT_ENTRY {
+                        out.push_str(&",\n");
+                    }
+                }
+                rowan::NodeOrToken::Token(token) => {
+                    use rnix::SyntaxKind::{
+                        TOKEN_COMMENT, TOKEN_ELLIPSIS, TOKEN_L_BRACE, TOKEN_QUESTION, TOKEN_R_BRACE,
+                    };
+                    match token.kind() {
+                        TOKEN_COMMENT | TOKEN_ELLIPSIS | TOKEN_QUESTION | TOKEN_L_BRACE
+                        | TOKEN_R_BRACE => {
+                            //dbg!(&token);
+                            out.push_str(&token.text().to_string());
+                            if token.kind() == TOKEN_COMMENT {
+                                out.push('\n');
+                            }
+                        }
+                        _ => {}
+                    }
+                    //out.push_str(&token.text().to_string());
+                }
+            }
+        }
         out.push_str(": ");
         let body = lambda.body().unwrap();
-                if body.kind() == NODE_LAMBDA {
-                    lambda = Lambda::cast(body).unwrap();
+        if let Expr::Lambda(inner) = body {
+            lambda = inner;
+            // If we recurse we want the next line of recursion to be indented and on a new line.
+            out.push('\n');
+            for _ in 0..=depth {
+                out.push('\t');
+            }
+            depth += 1;
         } else {
+            // If we don't find an inner lambda we're done with argument handling.
             break;
         }
     }
-            NODE_PATTERN => {
-                out += &format!("*{}*", &arg.to_string());
-                out.push_str(": ");
-                break;
-            }
-            t => {
-                unreachable!("unhandled arg type {:?}", t);
-            }
-        }
-    }
     out.push_str("...");
     out
 

maintainers/build-release-notes.py

@@ -20,6 +20,8 @@ SIGNIFICANCECES = {
 
 # This is just hardcoded for better validation. If you think there should be
 # more of them, feel free to add more.
+#
+# Please update doc/manual/src/contributing/hacking.md if you do. Thanks~
 CATEGORIES = [
     'Breaking Changes',
     'Features',
@@ -104,7 +106,7 @@ def do_category(author_info: AuthorInfoDB, entries: list[Tuple[pathlib.Path, Any
             links = []
             links += [format_issue(str(s)) for s in listify(entry.metadata.get('issues', []))]
             links += [format_pr(str(s)) for s in listify(entry.metadata.get('prs', []))]
-            links += [format_cl(cl) for cl in listify(entry.metadata.get('cls', []))]
+            links += [format_cl(int(cl)) for cl in listify(entry.metadata.get('cls', []))]
             if links != []:
                 header += " " + " ".join(links)
             if header:
@@ -127,7 +129,7 @@ def run_on_dir(author_info: AuthorInfoDB, d):
     entries = defaultdict(list)
     for p in paths:
         try:
-            e = frontmatter.load(p)
+            e = frontmatter.load(p) # type: ignore
             if 'synopsis' not in e.metadata:
                 raise Exception('missing synopsis')
             unknownKeys = set(e.metadata.keys()) - set(KNOWN_KEYS)

maintainers/buildtime_report.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 # Generates a report of build time based on a meson build using -ftime-trace in
 # Clang.

maintainers/check-syscalls.nix

@@ -0,0 +1,16 @@
+{
+  runCommandNoCC,
+  lib,
+  libseccomp,
+  writeShellScriptBin,
+}:
+let
+  syscalls-csv = runCommandNoCC "syscalls.csv" { } ''
+    echo ${lib.escapeShellArg libseccomp.src}
+    tar -xf ${lib.escapeShellArg libseccomp.src} --strip-components=2 ${libseccomp.name}/src/syscalls.csv
+    mv syscalls.csv "$out"
+  '';
+in
+writeShellScriptBin "check-syscalls" ''
+  ${./check-syscalls.sh} ${syscalls-csv}
+''

maintainers/check-syscalls.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -e
+
+diff -u <(awk < src/libstore/platform/linux.cc '/BEGIN extract-syscalls/ { extracting = 1; next }
+match($0, /allowSyscall\(ctx, SCMP_SYS\(([^)]*)\)\);|\/\/ skip ([^ ]*)/, result) { print result[1] result[2] }
+/END extract-syscalls/ { extracting = 0; next }') <(tail -n+2 "$1" | cut -d, -f 1)

meson.build

@@ -142,6 +142,16 @@ else
   cpp_pch = []
 endif
 
+# gcc 12 is known to miscompile some coroutine-based code quite horribly,
+# causing (among other things) copies of move-only objects and the double
+# frees one would expect when the objects are unique_ptrs. these problems
+# often show up as memory corruption when nesting generators (since we do
+# treat generators like owned memory) and will cause inexplicable crashs.
+assert(
+  cxx.get_id() != 'gcc' or cxx.version().version_compare('>=13'),
+  'GCC 12 and earlier are known to miscompile lix coroutines, use GCC 13 or clang.'
+)
+
 
 # Translate some historical and Mesony CPU names to Lixy CPU names.
 # FIXME(Qyriad): the 32-bit x86 code is not tested right now, because cross compilation for Lix
@@ -167,6 +177,7 @@ message('canonical Nix system name:', host_system)
 
 is_linux = host_machine.system() == 'linux'
 is_darwin = host_machine.system() == 'darwin'
+is_freebsd = host_machine.system() == 'freebsd'
 is_x64 = host_machine.cpu_family() == 'x86_64'
 
 # Per-platform arguments that you should probably pass to shared_module() invocations.
@@ -188,23 +199,27 @@ configdata = { }
 # Dependencies
 #
 
-boehm = dependency('bdw-gc', required : get_option('gc'), version : '>=8.2.6')
+gc_opt = get_option('gc').disable_if(
+  'address' in get_option('b_sanitize'),
+  error_message: 'gc does far too many memory crimes for ASan'
+)
+boehm = dependency('bdw-gc', required : gc_opt, version : '>=8.2.6', include_type : 'system')
 configdata += {
   'HAVE_BOEHMGC': boehm.found().to_int(),
 }
 
-boost = dependency('boost', required : true, modules : ['context', 'coroutine', 'container'])
+boost = dependency('boost', required : true, modules : ['container'], include_type : 'system')
 
 # cpuid only makes sense on x86_64
 cpuid_required = is_x64 ? get_option('cpuid') : false
-cpuid = dependency('libcpuid', 'cpuid', required : cpuid_required)
+cpuid = dependency('libcpuid', 'cpuid', required : cpuid_required, include_type : 'system')
 configdata += {
   'HAVE_LIBCPUID': cpuid.found().to_int(),
 }
 
 # seccomp only makes sense on Linux
 seccomp_required = is_linux ? get_option('seccomp-sandboxing') : false
-seccomp = dependency('libseccomp', 'seccomp', required : seccomp_required, version : '>=2.5.5')
+seccomp = dependency('libseccomp', 'seccomp', required : seccomp_required, version : '>=2.5.5', include_type : 'system')
 if is_linux and not seccomp.found()
   warning('Sandbox security is reduced because libseccomp has not been found! Please provide libseccomp if it supports your CPU architecture.')
 endif
@@ -212,18 +227,24 @@ configdata += {
   'HAVE_SECCOMP': seccomp.found().to_int(),
 }
 
-libarchive = dependency('libarchive', required : true)
+libarchive = dependency('libarchive', required : true, include_type : 'system')
 
 brotli = [
-  dependency('libbrotlicommon', required : true),
-  dependency('libbrotlidec', required : true),
-  dependency('libbrotlienc', required : true),
+  dependency('libbrotlicommon', required : true, include_type : 'system'),
+  dependency('libbrotlidec', required : true, include_type : 'system'),
+  dependency('libbrotlienc', required : true, include_type : 'system'),
 ]
 
-openssl = dependency('libcrypto', 'openssl', required : true)
+openssl = dependency('libcrypto', 'openssl', required : true, include_type : 'system')
 
-aws_sdk = dependency('aws-cpp-sdk-core', required : false)
-aws_sdk_transfer = dependency('aws-cpp-sdk-transfer', required : aws_sdk.found(), fallback : ['aws_sdk', 'aws_cpp_sdk_transfer_dep'])
+# FIXME: confirm we actually support such old versions of aws-sdk-cpp
+aws_sdk = dependency('aws-cpp-sdk-core', required : false, version : '>=1.8', include_type : 'system')
+aws_sdk_transfer = dependency(
+  'aws-cpp-sdk-transfer',
+  required : aws_sdk.found(),
+  fallback : ['aws_sdk', 'aws_cpp_sdk_transfer_dep'],
+  include_type : 'system',
+)
 if aws_sdk.found()
   # The AWS pkg-config adds -std=c++11.
   # https://github.com/aws/aws-sdk-cpp/issues/2673
@@ -234,12 +255,6 @@ if aws_sdk.found()
     links : true,
     sources : true,
   )
-  s = aws_sdk.version().split('.')
-  configdata += {
-    'AWS_VERSION_MAJOR': s[0].to_int(),
-    'AWS_VERSION_MINOR': s[1].to_int(),
-    'AWS_VERSION_PATCH': s[2].to_int(),
-  }
   aws_sdk_transfer = aws_sdk_transfer.partial_dependency(
     compile_args : false,
     includes : true,
@@ -249,7 +264,12 @@ if aws_sdk.found()
   )
 endif
 
-aws_s3 = dependency('aws-cpp-sdk-s3', required : aws_sdk.found(), fallback : ['aws_sdk', 'aws_cpp_sdk_s3_dep'])
+aws_s3 = dependency(
+  'aws-cpp-sdk-s3',
+  required : aws_sdk.found(),
+  fallback : ['aws_sdk', 'aws_cpp_sdk_s3_dep'],
+  include_type : 'system',
+)
 if aws_s3.found()
   # The AWS pkg-config adds -std=c++11.
   # https://github.com/aws/aws-sdk-cpp/issues/2673
@@ -266,32 +286,41 @@ configdata += {
   'ENABLE_S3': aws_s3.found().to_int(),
 }
 
-sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19', required : true)
+sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19', required : true, include_type : 'system')
 
-sodium = dependency('libsodium', 'sodium', required : true)
+sodium = dependency('libsodium', 'sodium', required : true, include_type : 'system')
 
-curl = dependency('libcurl', 'curl', required : true)
+curl = dependency('libcurl', 'curl', required : true, include_type : 'system')
 
-editline = dependency('libeditline', 'editline', version : '>=1.14', required : true)
+editline = dependency('libeditline', 'editline', version : '>=1.14', required : true, include_type : 'system')
 
-lowdown = dependency('lowdown', version : '>=0.9.0', required : true)
+lowdown = dependency('lowdown', version : '>=0.9.0', required : true, include_type : 'system')
 
 # HACK(Qyriad): rapidcheck's pkg-config doesn't include the libs lol
 # Note: technically we 'check' for rapidcheck twice, for the internal-api-docs handling above,
 # but Meson will cache the result of the first one, and the required : arguments are different.
-rapidcheck_meson = dependency('rapidcheck', required : enable_tests)
+rapidcheck_meson = dependency('rapidcheck', required : enable_tests, include_type : 'system')
 rapidcheck = declare_dependency(dependencies : rapidcheck_meson, link_args : ['-lrapidcheck'])
 
 gtest = [
-  dependency('gtest', required : enable_tests),
-  dependency('gtest_main', required : enable_tests),
-  dependency('gmock', required : enable_tests),
-  dependency('gmock_main', required : enable_tests),
+  dependency('gtest', required : enable_tests, include_type : 'system'),
+  dependency('gtest_main', required : enable_tests, include_type : 'system'),
+  dependency('gmock', required : enable_tests, include_type : 'system'),
+  dependency('gmock_main', required : enable_tests, include_type : 'system'),
 ]
 
-toml11 = dependency('toml11', version : '>=3.7.0', required : true, method : 'cmake')
+toml11 = dependency('toml11', version : '>=3.7.0', required : true, method : 'cmake', include_type : 'system')
 
-nlohmann_json = dependency('nlohmann_json', required : true)
+pegtl = dependency(
+  'pegtl',
+  version : '>=3.2.7',
+  required : true,
+  method : 'cmake',
+  modules : [ 'taocpp::pegtl' ],
+  include_type : 'system',
+)
+
+nlohmann_json = dependency('nlohmann_json', required : true, include_type : 'system')
 
 # lix-doc is a Rust project provided via buildInputs and unfortunately doesn't have any way to be detected.
 # Just declare it manually to resolve this.
@@ -300,6 +329,10 @@ nlohmann_json = dependency('nlohmann_json', required : true)
 # *absolutely* are not going to make it work)
 lix_doc = declare_dependency(link_args : [ '-llix_doc' ])
 
+if is_freebsd
+  libprocstat = declare_dependency(link_args : [ '-lprocstat' ])
+endif
+
 #
 # Build-time tools
 #
@@ -339,8 +372,6 @@ endif
 # that busybox sh won't run busybox applets as builtins (which would break our sandbox).
 
 lsof = find_program('lsof', native : true)
-bison = find_program('bison', native : true)
-flex = find_program('flex', native : true)
 
 # This is how Nix does generated headers...
 # other instances of header generation use a very similar command.
@@ -386,6 +417,11 @@ check_funcs = [
   'strsignal',
   'sysconf',
 ]
+if target_machine.kernel() in ['linux', 'freebsd']
+  # musl does not have close_range as of 2024-08-10
+  # patch: https://www.openwall.com/lists/musl/2024/08/01/9
+  check_funcs += [ 'close_range' ]
+endif
 foreach funcspec : check_funcs
   define_name = 'HAVE_' + funcspec.underscorify().to_upper()
   define_value = cxx.has_function(funcspec).to_int()
@@ -430,8 +466,10 @@ add_project_arguments(
   '-Wimplicit-fallthrough',
   '-Werror=switch',
   '-Werror=switch-enum',
+  '-Werror=unused-result',
   '-Wdeprecated-copy',
   '-Wignored-qualifiers',
+  '-Werror=suggest-override',
   # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked
   # at ~1% overhead in `nix search`.
   #
@@ -441,7 +479,9 @@ add_project_arguments(
   language : 'cpp',
 )
 
-if cxx.get_id() in ['gcc', 'clang']
+# We turn off the production UBSan if the slower dev UBSan is requested, to
+# give better diagnostics.
+if cxx.get_id() in ['gcc', 'clang'] and 'undefined' not in get_option('b_sanitize')
   # 2024-03-24: jade benchmarked the default sanitize reporting in clang and got
   # a regression of about 10% on hackage-packages.nix with clang. So we are trapping instead.
   #
@@ -456,17 +496,35 @@ if cxx.get_id() in ['gcc', 'clang']
   add_project_arguments(sanitize_args, language: 'cpp')
   add_project_link_arguments(sanitize_args, language: 'cpp')
 endif
+# Clang's default of -no-shared-libsan on Linux causes link errors; on macOS it defaults to shared.
+# GCC defaults to shared libsan so is fine.
+if cxx.get_id() == 'clang' and get_option('b_sanitize') != ''
+  add_project_link_arguments('-shared-libsan', language : 'cpp')
+endif
+
+# Clang gets grumpy about missing libasan symbols if -shared-libasan is not
+# passed when building shared libs, at least on Linux
+if cxx.get_id() == 'clang' and 'address' in get_option('b_sanitize')
+  add_project_link_arguments('-shared-libasan', language : 'cpp')
+endif
 
 add_project_link_arguments('-pthread', language : 'cpp')
+
 if cxx.get_linker_id() in ['ld.bfd', 'ld.gold']
   add_project_link_arguments('-Wl,--no-copy-dt-needed-entries', language : 'cpp')
 endif
 
+if is_freebsd
+  # FreeBSD's `environ` is defined in `crt1.o`, not `libc.so`,
+  # so the linker thinks it's undefined
+  add_project_link_arguments('-Wl,-z,undefs', language: 'cpp')
+endif
+
 # Generate Chromium tracing files for each compiled file, which enables
 # maintainers/buildtime_report.sh BUILD-DIR to simply work in clang builds.
 #
 # They can also be manually viewed at https://ui.perfetto.dev
-if get_option('profile-build').require(meson.get_compiler('cpp').get_id() == 'clang').enabled()
+if get_option('profile-build').require(cxx.get_id() == 'clang').enabled()
   add_project_arguments('-ftime-trace', language: 'cpp')
 endif
 
@@ -506,3 +564,5 @@ if enable_tests
   subdir('tests/unit')
   subdir('tests/functional')
 endif
+
+subdir('meson/clang-tidy')

meson.options

@@ -1,7 +1,7 @@
 # vim: filetype=meson
 
 option('enable-build', type : 'boolean', value : true,
-  description : 'Set to false to not actually build. Only really makes sense with -Dinternal-api-docs=true',
+  description : 'set to false to not actually build. Only really makes sense with -Dinternal-api-docs=true',
 )
 
 option('gc', type : 'feature',
@@ -37,7 +37,7 @@ option('tests-brief', type : 'boolean', value : false,
 )
 
 option('profile-build', type : 'feature', value: 'disabled',
-  description : 'whether to enable -ftime-trace in clang builds, allowing for speeding up the build.'
+  description : 'whether to enable -ftime-trace in clang builds, allowing for diagnosing the cause of build time.'
 )
 
 option('store-dir', type : 'string', value : '/nix/store',
@@ -68,3 +68,7 @@ option('profile-dir', type : 'string', value : 'etc/profile.d',
 option('enable-pch-std', type : 'boolean', value : true,
   description : 'whether to use precompiled headers for C++\'s standard library (breaks clangd if you\'re using GCC)',
 )
+
+option('lix-clang-tidy-checks-path', type : 'string', value : '',
+  description: 'path to lix-clang-tidy-checks library file, if providing it externally. Uses an internal one if this is not set',
+)

meson/clang-tidy/build_required_targets.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+import subprocess
+
+def get_targets_of_rule(build_root: str, rule_name: str) -> list[str]:
+    return subprocess.check_output(['ninja', '-C', build_root, '-t', 'targets', 'rule', rule_name]).decode().strip().splitlines()
+
+def ninja_build(build_root: str, targets: list[str]):
+    subprocess.check_call(['ninja', '-C', build_root, '--', *targets])
+
+def main():
+    import argparse
+    ap = argparse.ArgumentParser(description='Builds required targets for clang-tidy')
+    ap.add_argument('build_root', help='Ninja build root', type=str)
+
+    args = ap.parse_args()
+
+    targets = [t for t in get_targets_of_rule(args.build_root, 'CUSTOM_COMMAND') if t.endswith('gen.hh')]
+    ninja_build(args.build_root, targets)
+
+if __name__ == '__main__':
+    main()

meson/clang-tidy/clean_compdb.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+# Deletes the PCH arguments from a compilation database, to workaround nixpkgs
+# stdenv having a cc-wrapper that is impossible to use for anything except cc
+# itself, for example, clang-tidy.
+
+import json
+import shlex
+
+
+def process_compdb(compdb: list[dict]) -> list[dict]:
+
+    def munch_command(args: list[str]) -> list[str]:
+        out = []
+        eat_next = False
+        for i, arg in enumerate(args):
+            if arg == '-fpch-preprocess':
+                # as used with gcc
+                continue
+            elif arg == '-include-pch' or (arg == '-include' and args[i + 1] == 'precompiled-headers.hh'):
+                # -include-pch some-pch (clang), or -include some-pch (gcc)
+                eat_next = True
+                continue
+            if not eat_next:
+                out.append(arg)
+            eat_next = False
+        return out
+
+    def chomp(item: dict) -> dict:
+        item = item.copy()
+        item['command'] = shlex.join(munch_command(shlex.split(item['command'])))
+        return item
+
+    return [chomp(x) for x in compdb if not x['file'].endswith('precompiled-headers.hh')]
+
+
+def main():
+    import argparse
+    ap = argparse.ArgumentParser(
+        description='Delete pch arguments from compilation database')
+    ap.add_argument('input',
+                    type=argparse.FileType('r'),
+                    help='Input json file')
+    ap.add_argument('output',
+                    type=argparse.FileType('w'),
+                    help='Output json file')
+    args = ap.parse_args()
+
+    input_json = json.load(args.input)
+    json.dump(process_compdb(input_json), args.output, indent=2)
+
+
+if __name__ == '__main__':
+    main()

meson/clang-tidy/meson.build

@@ -0,0 +1,98 @@
+# The clang-tidy target for Lix
+
+run_clang_tidy = find_program('run-clang-tidy', required : false)
+# Although this looks like it wants to be pkg-config, pkg-config does not
+# really work for *plugins*, which are executable-like .so files that also
+# cannot be found via find_program. Fun!
+if get_option('lix-clang-tidy-checks-path') != ''
+  lix_clang_tidy_so = get_option('lix-clang-tidy-checks-path')
+  lix_clang_tidy_so_found = true
+else
+  lix_clang_tidy_subproj = subproject(
+    'lix-clang-tidy',
+    required : false,
+    default_options : {'build-by-default': false}
+  )
+  if lix_clang_tidy_subproj.found()
+    lix_clang_tidy_so = lix_clang_tidy_subproj.get_variable('lix_clang_tidy')
+    lix_clang_tidy_so_found = true
+  else
+    lix_clang_tidy_so_found = false
+  endif
+endif
+
+# Due to numerous problems, such as:
+# - Meson does not expose pch targets, but *fine*, I can just ask Ninja for
+#   them with `ninja -t targets rule cpp_PCH` and build them manually:
+#   https://github.com/mesonbuild/meson/issues/13499
+# - Nixpkgs stdenv buries the cc-wrapper under a giant pile of assumptions
+#   about the cc-wrapper actually being used on the cc of a stdenv, rather than
+#   independently for clang-tidy, and we need to use cc-wrapper to get the
+#   correct hardening flags so that clang-tidy can actually parse the PCH file
+#
+# I give up. I am going to delete the damn PCH args and then it will work.
+meson.add_postconf_script(
+  python,
+  meson.current_source_dir() / 'clean_compdb.py',
+  meson.global_build_root() / 'compile_commands.json',
+  meson.current_build_dir() / 'compile_commands.json',
+)
+
+# Horrible hack to get around not being able to depend on another target's
+# generated headers in any way in the meson DSL
+# https://github.com/mesonbuild/meson/issues/12817 which was incorrectly
+# closed, if you *actually* need to generate the files once.
+# Also related: https://github.com/mesonbuild/meson/issues/3667
+#
+# Or we could ban meson generators because their design is broken.
+build_all_generated_headers = custom_target(
+  command : [
+    python,
+    meson.current_source_dir() / 'build_required_targets.py',
+    meson.global_build_root(),
+  ],
+  output : 'generated_headers.stamp',
+  build_by_default : false,
+  build_always_stale : true,
+)
+
+if lix_clang_tidy_so_found
+  run_clang_tidy_args = [
+    '-load',
+    lix_clang_tidy_so,
+    '-p',
+    # We have to workaround a run-clang-tidy bug too, so we must give the
+    # directory name rather than the actual compdb file.
+    # https://github.com/llvm/llvm-project/issues/101440
+    meson.current_build_dir(),
+    '-quiet',
+  ]
+  run_target(
+    'clang-tidy',
+    command : [
+      # XXX: This explicitly invokes it with python because of a nixpkgs bug
+      # where clang-unwrapped does not patch interpreters in run-clang-tidy.
+      # However, making clang-unwrapped depend on python is also silly, so idk.
+      python,
+      run_clang_tidy,
+      run_clang_tidy_args,
+      '-warnings-as-errors',
+      '*',
+    ],
+    depends : [
+      build_all_generated_headers,
+    ],
+  )
+  run_target(
+    'clang-tidy-fix',
+    command : [
+      python,
+      run_clang_tidy,
+      run_clang_tidy_args,
+      '-fix',
+    ],
+    depends : [
+      build_all_generated_headers,
+    ],
+  )
+endif

meson/cleanup-install.bash

@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-# Meson will call this with an absolute path to Bash.
-# The shebang is just for convenience.
-
-# The parser and lexer tab are generated via custom Meson targets in src/libexpr/meson.build,
-# but Meson doesn't support marking only part of a target for install. The generation creates
-# both headers (parser-tab.hh, lexer-tab.hh) and source files (parser-tab.cc, lexer-tab.cc),
-# and we definitely want the former installed, but not the latter. This script is added to
-# Meson's install steps to correct this, as the logic for it is just complex enough to
-# warrant separate and careful handling, because both Meson's configured include directory
-# may or may not be an absolute path, and DESTDIR may or may not be set at all, but can't be
-# manipulated in Meson logic.
-
-set -euo pipefail
-
-echo "cleanup-install: removing Meson-placed C++ sources from dest includedir"
-
-if [[ "${1/--help/}" != "$1" ]]; then
-	echo "cleanup-install: this script should only be called from the Meson build system"
-	exit 1
-fi
-
-# Ensure the includedir was passed as the first argument
-# (set -u will make this fail otherwise).
-includedir="$1"
-# And then ensure that first argument is a directory that exists.
-if ! [[ -d "$1" ]]; then
-	echo "cleanup-install: this script should only be called from the Meson build system"
-	echo "argv[1] (${1@Q}) is not a directory"
-	exit 2
-fi
-
-# If DESTDIR environment variable is set, prepend it to the include dir.
-# Unfortunately, we cannot do this on the Meson side. We do have an environment variable
-# `MESON_INSTALL_DESTDIR_PREFIX`, but that will not refer to the include directory if
-# includedir has been set separately, which Lix's split-output derivation does.
-# We also cannot simply do an inline bash conditional like "${DESTDIR:=}" or similar,
-# because we need to specifically *join* DESTDIR and includedir with a slash, and *not*
-# have a slash if DESTDIR isn't set at all, since $includedir could be a relative directory.
-# Finally, DESTDIR is only available to us as an environment variable in these install scripts,
-# not in Meson logic.
-# Therefore, our best option is to have Meson pass this script the configured includedir,
-# and perform this dance with it and $DESTDIR.
-if [[ -n "${DESTDIR:-}" ]]; then
-	includedir="$DESTDIR/$includedir"
-fi
-
-# Intentionally not using -f.
-# If these files don't exist then our assumptions have been violated and we should fail.
-rm -v "$includedir/lix/libexpr/parser-tab.cc" "$includedir/lix/libexpr/lexer-tab.cc"

misc/meson.build

@@ -4,3 +4,9 @@ subdir('zsh')
 
 subdir('systemd')
 subdir('flake-registry')
+
+runinpty = configure_file(
+  copy : true,
+  input : meson.current_source_dir() / 'runinpty.py',
+  output : 'runinpty.py',
+)

misc/pre-commit.nix

@@ -106,7 +106,7 @@ pre-commit-run {
     };
     treefmt = {
       enable = true;
-      settings.formatters = [ pkgs.nixfmt ];
+      settings.formatters = [ pkgs.nixfmt-rfc-style ];
     };
   };
 }

misc/runinpty.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation; All Rights Reserved
+# SPDX-FileCopyrightText: 2024 Jade Lovelace
+# SPDX-License-Identifier: LGPL-2.1-or-later
+"""
+This script exists to lose Lix a dependency on expect(1) for the ability to run
+something in a pty.
+
+Yes, it could be replaced by script(1) but macOS and Linux script(1) have
+diverged sufficiently badly that even specifying a subcommand to run is not the
+same.
+"""
+import pty
+import sys
+import os
+from termios import ONLCR, ONLRET, ONOCR, OPOST, TCSAFLUSH, tcgetattr, tcsetattr
+from tty import setraw
+import termios
+
+def setup_terminal():
+    # does not matter which fd we use because we are in a fresh pty
+    modi = tcgetattr(pty.STDOUT_FILENO)
+    [iflag, oflag, cflag, lflag, ispeed, ospeed, cc] = modi
+
+    # Turning \n into \r\n is not cool, Linux!
+    oflag &= ~ONLCR
+    # I don't know what "implementation dependent postprocessing means" but it
+    # sounds bad
+    oflag &= ~OPOST
+    # Assume that NL performs the role of CR; do not insert CRs at column 0
+    oflag |= ONLRET | ONOCR
+
+    modi = [iflag, oflag, cflag, lflag, ispeed, ospeed, cc]
+
+    tcsetattr(pty.STDOUT_FILENO, TCSAFLUSH, modi)
+
+
+def spawn(argv: list[str]):
+    """
+    As opposed to pty.spawn, this one more seriously controls the pty settings.
+    Necessary to turn off such fun functionality as onlcr (LF to CRLF).
+
+    This is essentially copy pasted from pty.spawn, since there is no way to
+    hook the child pre-execve
+    """
+    pid, master_fd = pty.fork()
+    if pid == pty.CHILD:
+        setup_terminal()
+        os.execlp(argv[0], *argv)
+
+    try:
+        mode = tcgetattr(pty.STDIN_FILENO)
+        setraw(pty.STDIN_FILENO)
+        restore = True
+    except termios.error:
+        restore = False
+
+    try:
+        pty._copy(master_fd, pty._read, pty._read) # type: ignore
+    finally:
+        if restore:
+            tcsetattr(pty.STDIN_FILENO, TCSAFLUSH, mode) # type: ignore
+
+    os.close(master_fd)
+    return os.waitpid(pid, 0)[1]
+
+
+def main():
+    if len(sys.argv) == 1:
+        print(f'Usage: {sys.argv[0]} [command args]', file=sys.stderr)
+        sys.exit(1)
+
+    sys.exit(os.waitstatus_to_exitcode(spawn(sys.argv[1:])))
+
+
+if __name__ == '__main__':
+    main()

misc/systemd/nix-daemon.service.in

@@ -1,6 +1,6 @@
 [Unit]
 Description=Nix Daemon
-Documentation=man:nix-daemon https://nixos.org/manual
+Documentation=man:nix-daemon https://docs.lix.systems/manual/lix/stable
 RequiresMountsFor=@storedir@
 RequiresMountsFor=@localstatedir@
 RequiresMountsFor=@localstatedir@/nix/db

nix-support/binary-tarball.nix

@@ -5,16 +5,20 @@
   system,
 }:
 let
-  installerClosureInfo = buildPackages.closureInfo {
   rootPaths = [
     nix
     cacert
   ];
-  };
+  installerClosureInfo = buildPackages.closureInfo { inherit rootPaths; };
 
   meta.description = "Distribution-independent Lix bootstrap binaries for ${system}";
 in
-buildPackages.runCommand "lix-binary-tarball-${nix.version}" { inherit meta; } ''
+buildPackages.runCommand "lix-binary-tarball-${nix.version}"
+  {
+    inherit meta;
+    passthru.rootPaths = rootPaths;
+  }
+  ''
     cp ${installerClosureInfo}/registration $TMPDIR/reginfo
 
     dir=lix-${nix.version}-${system}

nix-support/nixfmt.nix

@@ -1,65 +0,0 @@
-# Copy of `nixfmt-rfc-style` vendored from `nixpkgs` master:
-# https://github.com/NixOS/nixpkgs/blob/ab6071eb54cc9b66dda436111d4f569e4e56cbf4/pkgs/by-name/ni/nixfmt-rfc-style/package.nix
-{
-  haskell,
-  haskellPackages,
-  fetchFromGitHub,
-}:
-let
-  inherit (haskell.lib.compose) justStaticExecutables;
-  raw-pkg = haskellPackages.callPackage (
-    {
-      mkDerivation,
-      base,
-      cmdargs,
-      directory,
-      fetchzip,
-      filepath,
-      lib,
-      megaparsec,
-      mtl,
-      parser-combinators,
-      safe-exceptions,
-      scientific,
-      text,
-      transformers,
-      unix,
-    }:
-    mkDerivation {
-      pname = "nixfmt";
-      version = "0.6.0-unstable-2024-03-14";
-      src = fetchFromGitHub {
-        owner = "serokell";
-        repo = "nixfmt";
-        rev = "8d13b593fa8d8d6e5075f541f3231222a08e84df";
-        hash = "sha256-HtXvzmfN4wk45qiKZ7V+/5WBV7jnTHfd7iBwF4XGl64=";
-      };
-      isLibrary = true;
-      isExecutable = true;
-      libraryHaskellDepends = [
-        base
-        megaparsec
-        mtl
-        parser-combinators
-        scientific
-        text
-        transformers
-      ];
-      executableHaskellDepends = [
-        base
-        cmdargs
-        directory
-        filepath
-        safe-exceptions
-        text
-        unix
-      ];
-      jailbreak = true;
-      homepage = "https://github.com/serokell/nixfmt";
-      description = "An opinionated formatter for Nix";
-      license = lib.licenses.mpl20;
-      mainProgram = "nixfmt";
-    }
-  ) { };
-in
-justStaticExecutables raw-pkg

package.nix

@@ -10,17 +10,16 @@
   boehmgc-nix ? __forDefaults.boehmgc-nix,
   boehmgc,
   nlohmann_json,
-  bison,
   build-release-notes ? __forDefaults.build-release-notes,
   boost,
   brotli,
   bzip2,
+  callPackage,
   cmake,
   curl,
   doxygen,
   editline-lix ? __forDefaults.editline-lix,
   editline,
-  flex,
   git,
   gtest,
   jq,
@@ -28,6 +27,8 @@
   libcpuid,
   libseccomp,
   libsodium,
+  lix-clang-tidy ? null,
+  llvmPackages,
   lsof,
   lowdown,
   mdbook,
@@ -36,6 +37,7 @@
   meson,
   ninja,
   openssl,
+  pegtl,
   pkg-config,
   python3,
   rapidcheck,
@@ -52,32 +54,37 @@
 
   pname ? "lix",
   versionSuffix ? "",
-  officialRelease ? false,
+  officialRelease ? __forDefaults.versionJson.official_release,
   # Set to true to build the release notes for the next release.
   buildUnreleasedNotes ? true,
   internalApiDocs ? false,
 
+  # Support garbage collection in the evaluator.
+  enableGC ? sanitize == null || !builtins.elem "address" sanitize,
+  # List of Meson sanitize options. Accepts values of b_sanitize, e.g.
+  # "address", "undefined", "thread".
+  # Enabling the "address" sanitizer will disable garbage collection in the evaluator.
+  sanitize ? null,
+  # Turn compiler warnings into errors.
+  werror ? false,
+
+  lintInsteadOfBuild ? false,
+
   # Not a real argument, just the only way to approximate let-binding some
   # stuff for argument defaults.
   __forDefaults ? {
     canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
 
-    boehmgc-nix = (boehmgc.override { enableLargeConfig = true; }).overrideAttrs {
-      patches = [
-        # We do *not* include prev.patches (which doesn't exist in normal pkgs.boehmgc anyway)
-        # because if the caller of this package passed a patched boehm as `boehmgc` instead of
-        # `boehmgc-nix` then this will almost certainly have duplicate patches, which means
-        # the patches won't apply and we'll get a build failure.
-        ./boehmgc-coroutine-sp-fallback.diff
-      ];
-    };
+    versionJson = builtins.fromJSON (builtins.readFile ./version.json);
+
+    boehmgc-nix = boehmgc.override { enableLargeConfig = true; };
 
     editline-lix = editline.overrideAttrs (prev: {
       configureFlags = prev.configureFlags or [ ] ++ [ (lib.enableFeature true "sigstop") ];
     });
 
-    lix-doc = pkgs.callPackage ./lix-doc/package.nix { };
-    build-release-notes = pkgs.callPackage ./maintainers/build-release-notes.nix { };
+    lix-doc = callPackage ./lix-doc/package.nix { };
+    build-release-notes = callPackage ./maintainers/build-release-notes.nix { };
   },
 }:
 let
@@ -85,10 +92,13 @@ let
   inherit (lib) fileset;
   inherit (stdenv) hostPlatform buildPlatform;
 
-  versionJson = builtins.fromJSON (builtins.readFile ./version.json);
-  version = versionJson.version + versionSuffix;
+  version = __forDefaults.versionJson.version + versionSuffix;
 
-  aws-sdk-cpp-nix = aws-sdk-cpp.override {
+  aws-sdk-cpp-nix =
+    if aws-sdk-cpp == null then
+      null
+    else
+      aws-sdk-cpp.override {
         apis = [
           "s3"
           "transfer"
@@ -98,19 +108,7 @@ let
 
   # Reimplementation of Nixpkgs' Meson cross file, with some additions to make
   # it actually work.
-  mesonCrossFile =
-    let
-      cpuFamily =
-        platform:
-        with platform;
-        if isAarch32 then
-          "arm"
-        else if isx86_32 then
-          "x86"
-        else
-          platform.uname.processor;
-    in
-    builtins.toFile "lix-cross-file.conf" ''
+  mesonCrossFile = builtins.toFile "lix-cross-file.conf" ''
     [properties]
     # Meson is convinced that if !buildPlatform.canExecute hostPlatform then we cannot
     # build anything at all, which is not at all correct. If we can't execute the host
@@ -126,10 +124,7 @@ let
 
   # The internal API docs need these for the build, but if we're not building
   # Nix itself, then these don't need to be propagated.
-  maybePropagatedInputs = [
-    boehmgc-nix
-    nlohmann_json
-  ];
+  maybePropagatedInputs = lib.optional enableGC boehmgc-nix ++ [ nlohmann_json ];
 
   # .gitignore has already been processed, so any changes in it are irrelevant
   # at this point. It is not represented verbatim for test purposes because
@@ -152,6 +147,7 @@ let
     (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
   ];
 in
+assert (lintInsteadOfBuild -> lix-clang-tidy != null);
 stdenv.mkDerivation (finalAttrs: {
   inherit pname version;
 
@@ -164,13 +160,13 @@ stdenv.mkDerivation (finalAttrs: {
           topLevelBuildFiles
           functionalTestFiles
         ]
-        ++ lib.optionals (!finalAttrs.dontBuild || internalApiDocs) [
-          ./boehmgc-coroutine-sp-fallback.diff
+        ++ lib.optionals (!finalAttrs.dontBuild || internalApiDocs || lintInsteadOfBuild) [
           ./doc
           ./misc
           ./src
           ./COPYING
         ]
+        ++ lib.optionals lintInsteadOfBuild [ ./.clang-tidy ]
       )
     );
   };
@@ -184,9 +180,14 @@ stdenv.mkDerivation (finalAttrs: {
       "doc"
     ];
 
-  dontBuild = false;
+  dontBuild = lintInsteadOfBuild;
 
   mesonFlags =
+    let
+      sanitizeOpts = lib.optional (
+        sanitize != null
+      ) "-Db_sanitize=${builtins.concatStringsSep "," sanitize}";
+    in
     lib.optionals hostPlatform.isLinux [
       # You'd think meson could just find this in PATH, but busybox is in buildInputs,
       # which don't actually get added to PATH. And buildInputs is correct over
@@ -194,24 +195,26 @@ stdenv.mkDerivation (finalAttrs: {
       "-Dsandbox-shell=${lib.getExe' busybox-sandbox-shell "busybox"}"
     ]
     ++ lib.optional hostPlatform.isStatic "-Denable-embedded-sandbox-shell=true"
-    ++ lib.optional (finalAttrs.dontBuild) "-Denable-build=false"
+    ++ lib.optional (finalAttrs.dontBuild && !lintInsteadOfBuild) "-Denable-build=false"
+    ++ lib.optional lintInsteadOfBuild "-Dlix-clang-tidy-checks-path=${lix-clang-tidy}/lib/liblix-clang-tidy.so"
     ++ [
       # mesonConfigurePhase automatically passes -Dauto_features=enabled,
       # so we must explicitly enable or disable features that we are not passing
       # dependencies for.
+      (lib.mesonEnable "gc" enableGC)
       (lib.mesonEnable "internal-api-docs" internalApiDocs)
-      (lib.mesonBool "enable-tests" finalAttrs.finalPackage.doCheck)
+      (lib.mesonBool "enable-tests" (finalAttrs.finalPackage.doCheck || lintInsteadOfBuild))
       (lib.mesonBool "enable-docs" canRunInstalled)
+      (lib.mesonBool "werror" werror)
     ]
-    ++ lib.optional (hostPlatform != buildPlatform) "--cross-file=${mesonCrossFile}";
+    ++ lib.optional (hostPlatform != buildPlatform) "--cross-file=${mesonCrossFile}"
+    ++ sanitizeOpts;
 
   # We only include CMake so that Meson can locate toml11, which only ships CMake dependency metadata.
   dontUseCmakeConfigure = true;
 
   nativeBuildInputs =
     [
-      bison
-      flex
       python3
       meson
       ninja
@@ -233,7 +236,13 @@ stdenv.mkDerivation (finalAttrs: {
     ]
     ++ lib.optional hostPlatform.isLinux util-linuxMinimal
     ++ lib.optional (!officialRelease && buildUnreleasedNotes) build-release-notes
-    ++ lib.optional internalApiDocs doxygen;
+    ++ lib.optional internalApiDocs doxygen
+    ++ lib.optionals lintInsteadOfBuild [
+      # required for a wrapped clang-tidy
+      llvmPackages.clang-tools
+      # required for run-clang-tidy
+      llvmPackages.clang-unwrapped
+    ];
 
   buildInputs =
     [
@@ -250,6 +259,7 @@ stdenv.mkDerivation (finalAttrs: {
       libsodium
       toml11
       lix-doc
+      pegtl
     ]
     ++ lib.optionals hostPlatform.isLinux [
       libseccomp
@@ -259,7 +269,10 @@ stdenv.mkDerivation (finalAttrs: {
     ++ lib.optional hostPlatform.isx86_64 libcpuid
     # There have been issues building these dependencies
     ++ lib.optional (hostPlatform.canExecute buildPlatform) aws-sdk-cpp-nix
-    ++ lib.optionals (finalAttrs.dontBuild) maybePropagatedInputs;
+    ++ lib.optionals (finalAttrs.dontBuild) maybePropagatedInputs
+    # I am so sorry. This is because checkInputs are required to pass
+    # configure, but we don't actually want to *run* the checks here.
+    ++ lib.optionals lintInsteadOfBuild finalAttrs.checkInputs;
 
   checkInputs = [
     gtest
@@ -298,12 +311,8 @@ stdenv.mkDerivation (finalAttrs: {
       install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
     ''
     + ''
-      # Workaround https://github.com/NixOS/nixpkgs/issues/294890.
-      if [[ -n "''${doCheck:-}" ]]; then
-        appendToVar configureFlags "--enable-tests"
-      else
-        appendToVar configureFlags "--disable-tests"
-      fi
+      # Fix up /usr/bin/env shebangs relied on by the build
+      patchShebangs --build tests/ doc/manual/
     '';
 
   mesonBuildType = "debugoptimized";
@@ -312,7 +321,7 @@ stdenv.mkDerivation (finalAttrs: {
 
   enableParallelBuilding = true;
 
-  doCheck = canRunInstalled;
+  doCheck = canRunInstalled && !lintInsteadOfBuild;
 
   mesonCheckFlags = [
     "--suite=check"
@@ -324,8 +333,19 @@ stdenv.mkDerivation (finalAttrs: {
   # Make sure the internal API docs are already built, because mesonInstallPhase
   # won't let us build them there. They would normally be built in buildPhase,
   # but the internal API docs are conventionally built with doBuild = false.
-  preInstall = lib.optional internalApiDocs ''
+  preInstall =
+    (lib.optionalString internalApiDocs ''
       meson ''${mesonBuildFlags:-} compile "$installTargets"
+    '')
+    # evil, but like above, we do not want to run an actual build phase
+    + lib.optionalString lintInsteadOfBuild ''
+      ninja clang-tidy
+    '';
+
+  installPhase = lib.optionalString lintInsteadOfBuild ''
+    runHook preInstall
+    touch $out
+    runHook postInstall
   '';
 
   postInstall =
@@ -379,9 +399,12 @@ stdenv.mkDerivation (finalAttrs: {
   # Export the patched version of boehmgc.
   # flake.nix exports that into its overlay.
   passthru = {
-    inherit (__forDefaults) boehmgc-nix editline-lix build-release-notes;
-
-    inherit officialRelease;
+    inherit (__forDefaults)
+      boehmgc-nix
+      editline-lix
+      build-release-notes
+      pegtl
+      ;
 
     # The collection of dependency logic for this derivation is complicated enough that
     # it's easier to parameterize the devShell off an already called package.nix.
@@ -390,18 +413,18 @@ stdenv.mkDerivation (finalAttrs: {
         mkShell,
 
         bashInteractive,
-        clang-tools,
         clangbuildanalyzer,
+        doxygen,
         glibcLocales,
         just,
-        llvmPackages,
-        nixfmt,
+        nixfmt-rfc-style,
         skopeo,
         xonsh,
 
         # Lix specific packages
         pre-commit-checks,
         contribNotice,
+        check-syscalls,
       }:
       let
         glibcFix = lib.optionalAttrs (buildPlatform.isLinux && glibcLocales != null) {
@@ -415,6 +438,7 @@ stdenv.mkDerivation (finalAttrs: {
             p.python-frontmatter
             p.requests
             p.xdg-base-dirs
+            p.packaging
             (p.toPythonModule xonsh.passthru.unwrapped)
           ]
         );
@@ -446,20 +470,33 @@ stdenv.mkDerivation (finalAttrs: {
             ++ [ (lib.mesonBool "enable-pch-std" stdenv.cc.isClang) ];
 
           packages =
-            lib.optional (stdenv.cc.isClang && hostPlatform == buildPlatform) clang-tools
+            lib.optional (stdenv.cc.isClang && hostPlatform == buildPlatform) llvmPackages.clang-tools
             ++ [
               # Why are we providing a bashInteractive? Well, when you run
               # `bash` from inside `nix develop`, say, because you are using it
               # via direnv, you will by default get bash (unusable edition).
               bashInteractive
+              check-syscalls
               pythonEnv
               # docker image tool
               skopeo
               just
-              nixfmt
+              nixfmt-rfc-style
+              # Included above when internalApiDocs is true, but we set that to
+              # false intentionally to save dev build time.
+              # To build them in a dev shell, you can set -Dinternal-api-docs=enabled when configuring.
+              doxygen
               # Load-bearing order. Must come before clang-unwrapped below, but after clang_tools above.
               stdenv.cc
             ]
+            ++ [
+              pkgs.rust-analyzer
+              pkgs.cargo
+              pkgs.rustc
+              pkgs.rustfmt
+              pkgs.rustPlatform.rustLibSrc
+              pkgs.rustPlatform.rustcSrc
+            ]
             ++ lib.optionals stdenv.cc.isClang [
               # Required for clang-tidy checks.
               llvmPackages.llvm
@@ -476,16 +513,21 @@ stdenv.mkDerivation (finalAttrs: {
               # https://git.lix.systems/lix-project/lix/src/commit/7575db522e9008685c4009423398f6900a16bcce/src/nix/develop.cc#L240-L241
               # this is, of course, absurd.
               if [[ $name != lix-shell-env && $name != lix-shell-env-env ]]; then
-                return;
+                return
               fi
 
-              PATH=$prefix/bin:$PATH
+              PATH=$prefix/bin''${PATH:+:''${PATH}}
               unset PYTHONPATH
-              export MANPATH=$out/share/man:$MANPATH
+              export MANPATH=$out/share/man:''${MANPATH:-}
 
               # Make bash completion work.
               XDG_DATA_DIRS+=:$out/share
 
+              if [[ ! -f ./.this-is-lix ]]; then
+                echo "Dev shell not started from inside a Lix repo, skipping repo setup" >&2
+                return
+              fi
+
               ${lib.optionalString (pre-commit-checks ? shellHook) pre-commit-checks.shellHook}
               # Allow `touch .nocontribmsg` to turn this notice off.
               if ! [[ -f .nocontribmsg ]]; then

perl/configure.ac

@@ -1,84 +0,0 @@
-AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX"]))
-AC_CONFIG_SRCDIR(MANIFEST)
-AC_CONFIG_AUX_DIR(../config)
-
-CFLAGS=
-CXXFLAGS=
-AC_PROG_CC
-AC_PROG_CXX
-
-AC_CANONICAL_HOST
-
-# Use 64-bit file system calls so that we can support files > 2 GiB.
-AC_SYS_LARGEFILE
-
-AC_DEFUN([NEED_PROG],
-[
-AC_PATH_PROG($1, $2)
-if test -z "$$1"; then
-    AC_MSG_ERROR([$2 is required])
-fi
-])
-
-NEED_PROG(perl, perl)
-NEED_PROG(curl, curl)
-NEED_PROG(bzip2, bzip2)
-NEED_PROG(xz, xz)
-
-# Test that Perl has the open/fork feature (Perl 5.8.0 and beyond).
-AC_MSG_CHECKING([whether Perl is recent enough])
-if ! $perl -e 'open(FOO, "-|", "true"); while (<FOO>) { print; }; close FOO or die;'; then
-    AC_MSG_RESULT(no)
-    AC_MSG_ERROR([Your Perl version is too old.  Lix requires Perl 5.8.0 or newer.])
-fi
-AC_MSG_RESULT(yes)
-
-
-# Figure out where to install Perl modules.
-AC_MSG_CHECKING([for the Perl installation prefix])
-perlversion=$($perl -e 'use Config; print $Config{version};')
-perlarchname=$($perl -e 'use Config; print $Config{archname};')
-AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname])
-AC_MSG_RESULT($perllibdir)
-
-# Look for libsodium.
-PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
-
-# Check for the required Perl dependencies (DBI and DBD::SQLite).
-perlFlags="-I$perllibdir"
-
-AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH],
-  [prefix of the Perl DBI library]),
-  perlFlags="$perlFlags -I$withval")
-
-AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH],
-  [prefix of the Perl DBD::SQLite library]),
-  perlFlags="$perlFlags -I$withval")
-
-AC_MSG_CHECKING([whether DBD::SQLite works])
-if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then
-    AC_MSG_RESULT(no)
-    AC_MSG_FAILURE([The Perl modules DBI and/or DBD::SQLite are missing.])
-fi
-AC_MSG_RESULT(yes)
-
-AC_SUBST(perlFlags)
-
-PKG_CHECK_MODULES([NIX], [nix-store])
-
-NEED_PROG([NIX], [nix])
-
-# Expand all variables in config.status.
-test "$prefix" = NONE && prefix=$ac_default_prefix
-test "$exec_prefix" = NONE && exec_prefix='${prefix}'
-for name in $ac_subst_vars; do
-    declare $name="$(eval echo "${!name}")"
-    declare $name="$(eval echo "${!name}")"
-    declare $name="$(eval echo "${!name}")"
-done
-
-rm -f Makefile.config
-ln -sfn ../mk mk
-
-AC_CONFIG_FILES([])
-AC_OUTPUT

perl/lib/Nix/Store.xs

@@ -77,7 +77,7 @@ SV * queryReferences(char * path)
 SV * queryPathHash(char * path)
     PPCODE:
         try {
-            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true);
+            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base::Base32, true);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -103,7 +103,7 @@ SV * queryPathInfo(char * path, int base32)
                 XPUSHs(&PL_sv_undef);
             else
                 XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
-            auto s = info->narHash.to_string(base32 ? Base32 : Base16, true);
+            auto s = info->narHash.to_string(base32 ? Base::Base32 : Base::Base16, true);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
             mXPUSHi(info->registrationTime);
             mXPUSHi(info->narSize);
@@ -205,7 +205,7 @@ SV * hashPath(char * algo, int base32, char * path)
     PPCODE:
         try {
             Hash h = hashPath(parseHashType(algo), path).first;
-            auto s = h.to_string(base32 ? Base32 : Base16, false);
+            auto s = h.to_string(base32 ? Base::Base32 : Base::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -216,7 +216,7 @@ SV * hashFile(char * algo, int base32, char * path)
     PPCODE:
         try {
             Hash h = hashFile(parseHashType(algo), path);
-            auto s = h.to_string(base32 ? Base32 : Base16, false);
+            auto s = h.to_string(base32 ? Base::Base32 : Base::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -227,7 +227,7 @@ SV * hashString(char * algo, int base32, char * s)
     PPCODE:
         try {
             Hash h = hashString(parseHashType(algo), s);
-            auto s = h.to_string(base32 ? Base32 : Base16, false);
+            auto s = h.to_string(base32 ? Base::Base32 : Base::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -238,7 +238,7 @@ SV * convertHash(char * algo, char * s, int toBase32)
     PPCODE:
         try {
             auto h = Hash::parseAny(s, parseHashType(algo));
-            auto s = h.to_string(toBase32 ? Base32 : Base16, false);
+            auto s = h.to_string(toBase32 ? Base::Base32 : Base::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());

perl/meson.build

@@ -65,5 +65,6 @@ if cxx.get_linker_id() in ['ld.bfd', 'ld.gold']
 endif
 
 libstore = dependency('lixstore', 'lix-store', required : true)
+libutil = dependency('lixutil', 'lix-util', required : true)
 
 subdir('lib/Nix')

releng/README.md

@@ -30,7 +30,7 @@ First, we prepare the release. `python -m releng prepare` is used for this.
 Then we tag the release with `python -m releng tag`:
 
 * Git HEAD is detached.
-* `officialRelease = true` is set in `flake.nix`, this is committed, and a
+* `"official_release": true` is set in `version.json`, this is committed, and a
   release is tagged.
 * The tag is merged back into the last branch (either `main` for new releases
   or `release-MAJOR` for maintenance releases) with `git merge -s ours VERSION`
@@ -57,12 +57,10 @@ Next, we do the publication with `python -m releng upload`:
     `nix upgrade-nix`.
   * s3://releases/lix/lix-VERSION/ gets the following contents
     * Binary tarballs
-    * Docs: `manual/` (FIXME: should we actually do this? what about putting it
-      on docs.lix.systems? I think doing both is correct, since the Web site
-      should not be an archive of random old manuals)
+    * Docs: `manual/`, primarily as an archive of old manuals
     * Docs as tarball in addition to web.
     * Source tarball
-    * Docker image (FIXME: upload to forgejo registry and github registry [in the future][upload-docker])
+    * Docker image
   * s3://docs/manual/lix/MAJOR
   * s3://docs/manual/lix/stable
 
@@ -80,6 +78,7 @@ Next, we do the publication with `python -m releng upload`:
   FIXME: automate branch-off to `release-*` branch.
 * **Manually** (FIXME?) switch back to the release branch, which now has the
   correct revision.
+* Deal with the external systems (see sections below).
 * Post!!
   * Merge release blog post to [lix-website].
   * Toot about it! https://chaos.social/@lix_project
@@ -87,22 +86,33 @@ Next, we do the publication with `python -m releng upload`:
 
 [lix-website]: https://git.lix.systems/lix-project/lix-website
 
-[upload-docker]: https://git.lix.systems/lix-project/lix/issues/252
-
 ### Installer
 
-The installer is cross-built to several systems from a Mac using
-`build-all.xsh` and `upload-to-lix.xsh` in the installer repo (FIXME: currently
-at least; maybe this should be moved here?) .
+The installer is cross-built to several systems from a Mac using `build-all.xsh` and `upload-to-lix.xsh` in the installer repo (FIXME: currently at least; maybe this should be moved here?).
 
-It installs a binary tarball (FIXME: [it should be taught to substitute from
-cache instead][installer-substitute])
-from some URL; this is the `hydraJobs.binaryTarball`. The default URLs differ
-by architecture and are [configured here][tarball-urls].
+It installs a binary tarball (FIXME: [it should be taught to substitute from cache instead][installer-substitute]) from some URL; this is the `hydraJobs.binaryTarball`.
+The default URLs differ by architecture and are [configured here][tarball-urls].
+
+To automatically do the file changes for a new version, run `python3 set_version.py NEW_VERSION`, and submit the result for review.
 
 [installer-substitute]: https://git.lix.systems/lix-project/lix-installer/issues/13
 [tarball-urls]: https://git.lix.systems/lix-project/lix-installer/src/commit/693592ed10d421a885bec0a9dd45e87ab87eb90a/src/settings.rs#L14-L28
 
+### Web site
+
+The website has various release-version dependent pieces.
+You can update them with `python3 update_version.py NEW_VERSION`, which will regenerate the affected page sources.
+
+These need the release to have been done first as they need hashes for tarballs and such.
+
+### NixOS module
+
+The NixOS module has underdeveloped releng in it.
+Currently you have to do the whole branch-off dance manually to a `release-VERSION` branch and update the tarball URLs to point to the release versions manually.
+
+FIXME: this should be unified with the `set_version.py` work in `lix-installer` and probably all the releng kept in here, or kept elsewhere.
+Related: https://git.lix.systems/lix-project/lix/issues/439
+
 ## Infrastructure summary
 
 * releases.lix.systems (`s3://releases`):

releng/cli.py

@@ -1,10 +1,13 @@
+import logging
+import argparse
+import sys
+
 from . import create_release
 from . import docker
 from .environment import RelengEnvironment
 from . import environment
-import argparse
-import sys
 
+log = logging.getLogger(__name__)
 
 def do_build(args):
     if args.target == 'all':
@@ -21,6 +24,9 @@ def do_tag(args):
     create_release.do_tag_merge(force_tag=args.force_tag,
                                 no_check_git=args.no_check_git)
 
+    log.info('Merged the release commit into your last branch, and switched to a detached HEAD of the artifact to be released.')
+    log.info('After you are done with releasing, switch to your previous branch and push that branch for review.')
+
 
 def do_upload(env: RelengEnvironment, args):
     create_release.setup_creds(env)

releng/create_release.xsh

@@ -2,6 +2,7 @@ import json
 import subprocess
 import itertools
 import textwrap
+import logging
 from pathlib import Path
 import tempfile
 import hashlib
@@ -11,10 +12,12 @@ from . import environment
 from .environment import RelengEnvironment
 from . import keys
 from . import docker
-from .version import VERSION, RELEASE_NAME, MAJOR
+from .version import VERSION, RELEASE_NAME, MAJOR, OFFICIAL_RELEASE
 from .gitutils import verify_are_on_tag, git_preconditions
 from . import release_notes
 
+log = logging.getLogger(__name__)
+
 $RAISE_SUBPROC_ERROR = True
 $XONSH_SHOW_TRACEBACK = True
 
@@ -39,16 +42,25 @@ def setup_creds(env: RelengEnvironment):
 
 
 def official_release_commit_tag(force_tag=False):
-    print('[+] Setting officialRelease in flake.nix and tagging')
+    print('[+] Setting officialRelease in version.json and tagging')
     prev_branch = $(git symbolic-ref --short HEAD).strip()
 
     git switch --detach
-    sed -i 's/officialRelease = false/officialRelease = true/' flake.nix
-    git add flake.nix
+
+    # Must be done in two parts due to buffering (opening the file immediately
+    # would truncate it).
+    new_version_json = $(jq --indent 4 '.official_release = true' version.json)
+    with open('version.json', 'w') as fh:
+        fh.write(new_version_json)
+    git add version.json
+
     message = f'release: {VERSION} "{RELEASE_NAME}"\n\nRelease produced with releng/create_release.xsh'
     git commit -m @(message)
     git tag @(['-f'] if force_tag else []) -a -m @(message) @(VERSION)
 
+    with open('releng/prev-git-branch.txt', 'w') as fh:
+        fh.write(prev_branch)
+
     return prev_branch
 
 
@@ -106,8 +118,8 @@ def upload_drv_paths_and_outputs(env: RelengEnvironment, paths: list[str]):
         raise subprocess.CalledProcessError(rv, proc.args)
 
 
-def make_manifest(eval_result):
-    manifest = {vs['system']: vs['outputs']['out'] for vs in eval_result}
+def make_manifest(builds_by_system):
+
     def manifest_line(system, out):
         return f'  {system} = "{out}";'
 
@@ -116,7 +128,7 @@ def make_manifest(eval_result):
         {{
         {lines}
         }}
-    """).format(lines='\n'.join(manifest_line(s, p) for (s, p) in manifest.items()))
+    """).format(lines='\n'.join(manifest_line(s, p) for (s, p) in builds_by_system.items()))
 
     return manifest_text
 
@@ -142,6 +154,18 @@ def sha256_file(f: Path):
     return hasher.hexdigest()
 
 
+def extract_builds_by_system(eval_result):
+    # This could be a dictionary comprehension, but we want to be absolutely
+    # sure we don't have duplicates.
+    ret = {}
+    for attr in eval_result:
+        if attr['attrPath'][0] != 'build':
+            continue
+        assert attr['system'] not in ret
+        ret[attr['system']] = attr['outputs']['out']
+    return ret
+
+
 def make_artifacts_dir(eval_result, d: Path):
     d.mkdir(exist_ok=True, parents=True)
     version_dir = d / 'lix' / f'lix-{VERSION}'
@@ -150,12 +174,14 @@ def make_artifacts_dir(eval_result, d: Path):
     tarballs_drv = next(p for p in eval_result if p['attr'] == 'tarballs')
     cp --no-preserve=mode -r @(tarballs_drv['outputs']['out'])/* @(version_dir)
 
+    builds_by_system = extract_builds_by_system(eval_result)
+
     # FIXME: upgrade-nix searches for manifest.nix at root, which is rather annoying
     with open(d / 'manifest.nix', 'w') as h:
-        h.write(make_manifest(eval_result))
+        h.write(make_manifest(builds_by_system))
 
     with open(version_dir / 'manifest.nix', 'w') as h:
-        h.write(make_manifest(eval_result))
+        h.write(make_manifest(builds_by_system))
 
     print('[+] Make sources tarball')
 
@@ -215,6 +241,22 @@ def upload_artifacts(env: RelengEnvironment, noconfirm=False, no_check_git=False
     print('[+] Upload manual')
     upload_manual(env)
 
+    prev_branch = None
+    try:
+        with open('releng/prev-git-branch.txt', 'r') as fh:
+            prev_branch = fh.read().strip()
+    except FileNotFoundError:
+        log.warn('Cannot find previous git branch file, skipping pushing git objects')
+
+    if prev_branch:
+        print('[+] git push to the repo')
+        # We have to push the ref to gerrit for review at least such that the
+        # commit is known, before we can push it as a tag.
+        if env.git_repo_is_gerrit:
+            git push @(env.git_repo) f'{prev_branch}:refs/for/{prev_branch}'
+        else:
+            git push @(env.git_repo) f'{prev_branch}:{prev_branch}'
+
         print('[+] git push tag')
         git push @(['-f'] if force_push_tag else []) @(env.git_repo) f'{VERSION}:refs/tags/{VERSION}'
 
@@ -236,15 +278,14 @@ def build_manual(eval_result):
 
 
 def upload_manual(env: RelengEnvironment):
-    stable = json.loads($(nix eval --json '.#nix.officialRelease'))
-    if stable:
+    if OFFICIAL_RELEASE:
         version = MAJOR
     else:
         version = 'nightly'
 
     print('[+] aws s3 sync manual')
     aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/@(version)/
-    if stable:
+    if OFFICIAL_RELEASE:
         aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/stable/
 
 

releng/docker.xsh

@@ -44,23 +44,8 @@ def upload_docker_images(target: DockerTarget, paths: list[Path]):
 
         for path in paths:
             digest_file = tmp / (path.name + '.digest')
-            tmp_image = tmp / 'tmp-image.tar.gz'
 
-            # insecure-policy: we don't have any signature policy, we are just uploading an image
-            #
-            # Absurd: we copy it into an OCI image first so we can get the hash
-            # we need to upload it untagged, because skopeo has no "don't tag
-            # this" option.
-            # The reason for this is that forgejo's container registry throws
-            # away old versions of tags immediately, so we cannot use a temp
-            # tag, and it *does* reduce confusion to not upload tags that
-            # should not be used.
-            #
-            # Workaround for: https://github.com/containers/skopeo/issues/2354
-            log.info('skopeo copy to temp oci-archive %s', tmp_image)
-            skopeo --insecure-policy copy --format oci --all --digestfile @(digest_file) docker-archive:@(path) oci-archive:@(tmp_image)
-
-            inspection = json.loads($(skopeo inspect oci-archive:@(tmp_image)))
+            inspection = json.loads($(skopeo inspect docker-archive:@(path)))
 
             docker_arch = inspection['Architecture']
             docker_os = inspection['Os']
@@ -68,8 +53,9 @@ def upload_docker_images(target: DockerTarget, paths: list[Path]):
 
             log.info('Pushing image %s for %s to %s', path, docker_arch, target.registry_path)
 
+            # insecure-policy: we don't have any signature policy, we are just uploading an image
+            skopeo --insecure-policy copy --digestfile @(digest_file) --all docker-archive:@(path) f'docker://{target.registry_path}@@unknown-digest@@'
             digest = digest_file.read_text().strip()
-            skopeo --insecure-policy copy --preserve-digests --all oci-archive:@(tmp_image) f'docker://{target.registry_path}@{digest}'
 
             # skopeo doesn't give us the manifest size directly, so we just ask the registry
             metadata = reg.image_info(target.registry_path, digest)

releng/environment.py

@@ -52,6 +52,7 @@ class RelengEnvironment:
     releases_bucket: str
     docs_bucket: str
     git_repo: str
+    git_repo_is_gerrit: bool
 
     docker_targets: list[DockerTarget]
 
@@ -79,6 +80,7 @@ STAGING = RelengEnvironment(
     cache_store_overlay={'secret-key': 'staging.key'},
     releases_bucket='s3://staging-releases',
     git_repo='ssh://git@git.lix.systems/lix-project/lix-releng-staging',
+    git_repo_is_gerrit=False,
     docker_targets=[
         # latest will be auto tagged if appropriate
         DockerTarget('git.lix.systems/lix-project/lix-releng-staging',
@@ -113,6 +115,7 @@ PROD = RelengEnvironment(
     cache_store_overlay={'secret-key': 'prod.key'},
     releases_bucket='s3://releases',
     git_repo=guess_gerrit_remote(),
+    git_repo_is_gerrit=True,
     docker_targets=[
         # latest will be auto tagged if appropriate
         DockerTarget('git.lix.systems/lix-project/lix',

releng/gitutils.xsh

@@ -1,11 +1,24 @@
 import subprocess
-import json
+from packaging.version import Version
 
 from .version import VERSION
 
 
+def remote_is_plausible(url: str) -> bool:
+    return ('git.lix.systems' in url and 'lix-project/lix' in url) or ('gerrit.lix.systems' in url and url.endswith('lix'))
+
+
 def version_compare(v1: str, v2: str):
-    return json.loads($(nix-instantiate --eval --json --argstr v1 @(v1) --argstr v2 @(v2) --expr '{v1, v2}: builtins.compareVersions v1 v2'))
+    v1 = Version(v1)
+    v2 = Version(v2)
+    if v1 < v2:
+        return -1
+    elif v1 > v2:
+        return 1
+    elif v1 == v2:
+        return 0
+    else:
+        raise ValueError('these versions are beyond each others celestial plane')
 
 
 def latest_tag_on_branch(branch: str) -> str:
@@ -13,16 +26,18 @@ def latest_tag_on_branch(branch: str) -> str:
 
 
 def is_maintenance_branch(branch: str) -> bool:
-    try:
-        main_tag = latest_tag_on_branch('main')
+    """
+    Returns whether the given branch is probably a maintenance branch.
+
+    This uses a heuristic: `main` should have a newer tag than a given
+    maintenance branch if there has been a major release since that maintenance
+    branch.
+    """
+    assert remote_is_plausible($(git remote get-url origin).strip())
+    main_tag = latest_tag_on_branch('origin/main')
     current_tag = latest_tag_on_branch(branch)
 
     return version_compare(current_tag, main_tag) < 0
-    except subprocess.CalledProcessError:
-        # This is the case before Lix releases 2.90, since main *has* no
-        # release tag on it.
-        # FIXME: delete this case after 2.91
-        return False
 
 
 def verify_are_on_tag():

releng/release-tests.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+shopt -s inherit_errexit failglob
+
+nixpkgss=(
+    "$(nix eval --impure --raw --expr '(import ./flake.nix).inputs.nixpkgs.url')"
+    "github:NixOS/nixpkgs/nixos-unstable-small"
+)
+jobs=(
+    $(nix eval \
+        --json --apply '
+          let f = n: t:
+            if builtins.isAttrs t
+            then (if t.type or "" == "derivation"
+                  then [ n ]
+                  else builtins.concatMap (m: f "${n}.${m}" t.${m}) (builtins.attrNames t))
+            else [];
+          in f ".#.releaseTests"
+        ' \
+        '.#.releaseTests' \
+        | jq -r '.[]'
+    )
+)
+
+for override in "${nixpkgss}"
+do
+    (
+        set -x
+        nix build \
+            --log-format multiline \
+            --no-link \
+            --override-input nixpkgs "$override" \
+            "${jobs[@]}"
+    )
+done

releng/version.py

@@ -4,3 +4,4 @@ version_json = json.load(open('version.json'))
 VERSION = version_json['version']
 MAJOR = '.'.join(VERSION.split('.')[:2])
 RELEASE_NAME = version_json['release_name']
+OFFICIAL_RELEASE = version_json['official_release']

src/asan-options/asan-options.cc

@@ -0,0 +1,17 @@
+/// @file This is very bothersome code that has to be included in every
+/// executable to get the correct default ASan options. I am so sorry.
+
+extern "C" [[gnu::retain]] const char *__asan_default_options()
+{
+    // We leak a bunch of memory knowingly on purpose. It's not worthwhile to
+    // diagnose that memory being leaked for now.
+    //
+    // Instruction bytes are useful for finding the actual code that
+    // corresponds to an ASan report.
+    //
+    // TODO: setting log_path=asan.log or not: neither works, since you can't
+    // write to the fs in certain places in the testsuite, but you also cannot
+    // write arbitrarily to stderr in other places so the reports get eaten.
+    // pain 🥖
+    return "halt_on_error=1:abort_on_error=1:detect_leaks=0:print_summary=1:dump_instruction_bytes=1";
+}

src/build-remote/build-remote.cc

@@ -42,7 +42,7 @@ static std::string makeLockFilename(const std::string & storeUri) {
     // This avoids issues with the escaped URI being very long and causing
     // path too long errors, while also avoiding any possibility of collision
     // caused by simple truncation.
-    auto hash = hashString(HashType::htSHA256, storeUri).to_string(Base::Base32, false);
+    auto hash = hashString(HashType::SHA256, storeUri).to_string(Base::Base32, false);
     return escapeUri(storeUri).substr(0, 48) + "-" + hash.substr(0, 16);
 }
 
@@ -236,9 +236,9 @@ static int main_build_remote(int argc, char * * argv)
                 }
 
 #if __APPLE__
-                futimes(bestSlotLock.get(), NULL);
+                futimes(bestSlotLock.get(), nullptr);
 #else
-                futimens(bestSlotLock.get(), NULL);
+                futimens(bestSlotLock.get(), nullptr);
 #endif
 
                 lock.reset();

src/libcmd/cmd-profiles.cc

@@ -244,9 +244,9 @@ StorePath ProfileManifest::build(ref<Store> store)
 
     /* Add the symlink tree to the store. */
     StringSink sink;
-    dumpPath(tempDir, sink);
+    sink << dumpPath(tempDir);
 
-    auto narHash = hashString(htSHA256, sink.s);
+    auto narHash = hashString(HashType::SHA256, sink.s);
 
     ValidPathInfo info{
         *store,

src/libcmd/cmd-profiles.hh

@@ -37,7 +37,7 @@ struct ProfileElement
     StorePathSet storePaths;
     std::optional<ProfileElementSource> source;
     bool active = true;
-    int priority = DEFAULT_PRIORITY;
+    NixInt::Inner priority = DEFAULT_PRIORITY;
 
     std::string identifier() const;
 

src/libcmd/command-installable-value.cc

@@ -1,11 +0,0 @@
-#include "command-installable-value.hh"
-
-namespace nix {
-
-void InstallableValueCommand::run(ref<Store> store, ref<Installable> installable)
-{
-    auto installableValue = InstallableValue::require(installable);
-    run(store, installableValue);
-}
-
-}

src/libcmd/command-installable-value.hh

@@ -1,23 +0,0 @@
-#pragma once
-///@file
-
-#include "installable-value.hh"
-#include "command.hh"
-
-namespace nix {
-
-/**
- * An InstallableCommand where the single positional argument must be an
- * InstallableValue in particular.
- */
-struct InstallableValueCommand : InstallableCommand
-{
-    /**
-     * Entry point to this command
-     */
-    virtual void run(ref<Store> store, ref<InstallableValue> installable) = 0;
-
-    void run(ref<Store> store, ref<Installable> installable) override;
-};
-
-}

src/libcmd/installable-attr-path.cc

@@ -1,23 +1,11 @@
-#include "globals.hh"
 #include "installable-attr-path.hh"
 #include "outputs-spec.hh"
 #include "command.hh"
 #include "attr-path.hh"
 #include "common-eval-args.hh"
-#include "derivations.hh"
-#include "eval-inline.hh"
 #include "eval.hh"
 #include "get-drvs.hh"
-#include "store-api.hh"
-#include "shared.hh"
 #include "flake/flake.hh"
-#include "eval-cache.hh"
-#include "url.hh"
-#include "registry.hh"
-#include "build-result.hh"
-
-#include <regex>
-#include <queue>
 
 #include <nlohmann/json.hpp>
 

src/libcmd/installable-attr-path.hh

@@ -1,25 +1,11 @@
 #pragma once
 ///@file
 
-#include "globals.hh"
 #include "installable-value.hh"
 #include "outputs-spec.hh"
 #include "command.hh"
-#include "attr-path.hh"
 #include "common-eval-args.hh"
-#include "derivations.hh"
-#include "eval-inline.hh"
 #include "eval.hh"
-#include "get-drvs.hh"
-#include "store-api.hh"
-#include "shared.hh"
-#include "eval-cache.hh"
-#include "url.hh"
-#include "registry.hh"
-#include "build-result.hh"
-
-#include <regex>
-#include <queue>
 
 #include <nlohmann/json.hpp>
 

src/libcmd/installable-flake.cc

@@ -116,12 +116,12 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
 
     auto drvPath = attr->forceDerivation();
 
-    std::optional<NixInt> priority;
+    std::optional<NixInt::Inner> priority;
 
     if (attr->maybeGetAttr(state->sOutputSpecified)) {
     } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) {
         if (auto aPriority = aMeta->maybeGetAttr("priority"))
-            priority = aPriority->getInt();
+            priority = aPriority->getInt().value;
     }
 
     return {{

src/libcmd/installable-value.hh

@@ -40,7 +40,7 @@ struct ExtraPathInfoValue : ExtraPathInfo
         /**
          * An optional priority for use with "build envs". See Package
          */
-        std::optional<NixInt> priority;
+        std::optional<NixInt::Inner> priority;
 
         /**
          * The attribute path associated with this value. The idea is

src/libcmd/installables.cc

@@ -214,7 +214,7 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s
 
             evalSettings.pureEval = false;
             auto state = getEvalState();
-            Expr *e = state->parseExprFromFile(
+            Expr & e = state->parseExprFromFile(
                 resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file)))
                 );
 
@@ -393,13 +393,10 @@ ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
     std::shared_ptr<flake::LockedFlake> lockedFlake)
 {
-    auto fingerprint = lockedFlake->getFingerprint();
-    return make_ref<nix::eval_cache::EvalCache>(
-        evalSettings.useEvalCache && evalSettings.pureEval
-            ? std::optional { std::cref(fingerprint) }
-            : std::nullopt,
-        state,
-        [&state, lockedFlake]()
+    auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval
+        ? std::make_optional(lockedFlake->getFingerprint())
+        : std::nullopt;
+    auto rootLoader = [&state, lockedFlake]()
         {
             /* For testing whether the evaluation cache is
                complete. */
@@ -415,7 +412,17 @@ ref<eval_cache::EvalCache> openEvalCache(
             assert(aOutputs);
 
             return aOutputs->value;
-        });
+        };
+
+    if (fingerprint) {
+        auto search = state.evalCaches.find(fingerprint.value());
+        if (search == state.evalCaches.end()) {
+            search = state.evalCaches.emplace(fingerprint.value(), make_ref<nix::eval_cache::EvalCache>(fingerprint, state, rootLoader)).first;
+        }
+        return search->second;
+    } else {
+        return make_ref<nix::eval_cache::EvalCache>(std::nullopt, state, rootLoader);
+    }
 }
 
 Installables SourceExprCommand::parseInstallables(
@@ -434,13 +441,13 @@ Installables SourceExprCommand::parseInstallables(
         auto vFile = state->allocValue();
 
         if (file == "-") {
-            auto e = state->parseStdin();
+            auto & e = state->parseStdin();
             state->eval(e, *vFile);
         }
         else if (file)
             state->evalFile(lookupFileArg(*state, *file), *vFile);
         else {
-            auto e = state->parseExprFromString(*expr, state->rootPath(CanonPath::fromCwd()));
+            auto & e = state->parseExprFromString(*expr, state->rootPath(CanonPath::fromCwd()));
             state->eval(e, *vFile);
         }
 

src/libcmd/meson.build

@@ -1,6 +1,5 @@
 libcmd_sources = files(
   'built-path.cc',
-  'command-installable-value.cc',
   'cmd-profiles.cc',
   'command.cc',
   'common-eval-args.cc',
@@ -18,7 +17,6 @@ libcmd_sources = files(
 
 libcmd_headers = files(
   'built-path.hh',
-  'command-installable-value.hh',
   'cmd-profiles.hh',
   'command.hh',
   'common-eval-args.hh',

src/libcmd/repl-interacter.cc

@@ -96,7 +96,7 @@ static int listPossibleCallback(char * s, char *** avp)
         return p;
     };
 
-    vp = check((char **) malloc(possible.size() * sizeof(char *)));
+    vp = check(static_cast<char **>(malloc(possible.size() * sizeof(char *))));
 
     for (auto & p : possible)
         vp[ac++] = check(strdup(p.c_str()));

src/libcmd/repl.cc

@@ -1,7 +1,10 @@
+#include <cstdio>
+#include <editline.h>
 #include <iostream>
 #include <cstdlib>
 #include <cstring>
 #include <climits>
+#include <string_view>
 
 #include "box_ptr.hh"
 #include "repl-interacter.hh"
@@ -29,7 +32,6 @@
 #include "local-fs-store.hh"
 #include "signals.hh"
 #include "print.hh"
-#include "progress-bar.hh"
 #include "gc-small-vector.hh"
 #include "users.hh"
 
@@ -38,24 +40,24 @@
 #include <gc/gc_cpp.h>
 #endif
 
-// XXX: These are for nix-doc features and will be removed in a future rewrite where this functionality is integrated more natively.
+// XXX: These are for lix-doc features and will be removed in a future rewrite where this functionality is integrated more natively.
 extern "C" {
-    char const *nd_get_function_docs(char const *filename, size_t line, size_t col);
-    void nd_free_string(char const *str);
+    char const *lixdoc_get_function_docs(char const *filename, size_t line, size_t col);
+    void lixdoc_free_string(char const *str);
 }
 
 namespace nix {
 
 
 /** Wrapper around std::unique_ptr with a custom deleter for strings from nix-doc **/
-using NdString = std::unique_ptr<const char, decltype(&nd_free_string)>;
+using NdString = std::unique_ptr<const char, decltype(&lixdoc_free_string)>;
 
 /**
  * Fetch a string representing the doc comment using nix-doc and wrap it in an RAII wrapper.
  */
 NdString lambdaDocsForPos(SourcePath const path, nix::Pos const &pos) {
   std::string const file = path.to_string();
-  return NdString{nd_get_function_docs(file.c_str(), pos.line, pos.column), &nd_free_string};
+  return NdString{lixdoc_get_function_docs(file.c_str(), pos.line, pos.column), &lixdoc_free_string};
 }
 
 /**
@@ -79,6 +81,8 @@ enum class ProcessLineResult {
     PromptAgain,
 };
 
+using namespace std::literals::string_view_literals;
+
 struct NixRepl
     : AbstractNixRepl
     , detail::ReplCompleterMixin
@@ -86,6 +90,35 @@ struct NixRepl
     , gc
     #endif
 {
+    /* clang-format: off */
+    static constexpr std::array COMMANDS = {
+        "add"sv, "a"sv,
+        "load"sv, "l"sv,
+        "load-flake"sv, "lf"sv,
+        "reload"sv, "r"sv,
+        "edit"sv, "e"sv,
+        "t"sv,
+        "u"sv,
+        "b"sv,
+        "bl"sv,
+        "i"sv,
+        "sh"sv,
+        "log"sv,
+        "print"sv, "p"sv,
+        "quit"sv, "q"sv,
+        "doc"sv,
+        "te"sv,
+    };
+
+    static constexpr std::array DEBUG_COMMANDS = {
+        "env"sv,
+        "bt"sv, "backtrace"sv,
+        "st"sv,
+        "c"sv, "continue"sv,
+        "s"sv, "step"sv,
+    };
+    /* clang-format: on */
+
     size_t debugTraceIndex;
 
     Strings loadedFiles;
@@ -122,7 +155,7 @@ struct NixRepl
     void reloadFiles();
     void addAttrsToScope(Value & attrs);
     void addVarToScope(const Symbol name, Value & v);
-    Expr * parseString(std::string s);
+    Expr & parseString(std::string s);
     void evalString(std::string s, Value & v);
     void loadDebugTraceEnv(DebugTrace & dt);
 
@@ -209,8 +242,7 @@ NixRepl::NixRepl(const SearchPath & searchPath, nix::ref<Store> store, ref<EvalS
 {
 }
 
-void runNix(Path program, const Strings & args,
-    const std::optional<std::string> & input = {})
+void runNix(Path program, const Strings & args)
 {
     auto subprocessEnv = getEnv();
     subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue();
@@ -219,8 +251,7 @@ void runNix(Path program, const Strings & args,
         .program = settings.nixBinDir+ "/" + program,
         .args = args,
         .environment = subprocessEnv,
-        .input = input,
-    });
+    }).wait();
 
     return;
 }
@@ -268,7 +299,7 @@ ReplExitStatus NixRepl::mainLoop()
 
     /* Stop the progress bar because it interferes with the display of
        the repl. */
-    stopProgressBar();
+    logger->pause();
 
     std::string input;
 
@@ -323,6 +354,36 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
 {
     StringSet completions;
 
+    // We should only complete colon commands if there's a colon at the beginning,
+    // but editline (for... whatever reason) doesn't *give* us the colon in the
+    // completion callback. If the user types :rel<TAB>, `prefix` will only be `rel`.
+    // Luckily, editline provides a global variable for its current buffer, so we can
+    // check for the presence of a colon there.
+    if (rl_line_buffer != nullptr && rl_line_buffer[0] == ':') {
+        for (auto const & colonCmd : this->COMMANDS) {
+            if (colonCmd.starts_with(prefix)) {
+                completions.insert(std::string(colonCmd));
+            }
+        }
+
+        if (state->debugRepl) {
+            for (auto const & colonCmd : this->DEBUG_COMMANDS) {
+                if (colonCmd.starts_with(prefix)) {
+                    completions.insert(std::string(colonCmd));
+                }
+            }
+        }
+
+        // If there were : command completions, then we should only return those,
+        // because otherwise this is not valid Nix syntax.
+        // However if we didn't get any completions, then this could be something
+        // like `:b pkgs.hel<TAB>`, in which case we should do expression completion
+        // as normal.
+        if (!completions.empty()) {
+            return completions;
+        }
+    }
+
     size_t start = prefix.find_last_of(" \n\r\t(){}[]");
     std::string prev, cur;
     if (start == std::string::npos) {
@@ -365,9 +426,9 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
             auto expr = cur.substr(0, dot);
             auto cur2 = cur.substr(dot + 1);
 
-            Expr * e = parseString(expr);
+            Expr & e = parseString(expr);
             Value v;
-            e->eval(*state, *env, v);
+            e.eval(*state, *env, v);
             state->forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)");
 
             for (auto & i : *v.attrs) {
@@ -475,7 +536,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
              << "  :t <expr>                    Describe result of evaluation\n"
              << "  :u <expr>                    Build derivation, then start nix-shell\n"
              << "  :doc <expr>                  Show documentation for the provided function (experimental lambda support)\n"
-             << "  :log <expr>                  Show logs for a derivation\n"
+             << "  :log <expr | .drv path>      Show logs for a derivation\n"
              << "  :te, :trace-enable [bool]    Enable, disable or toggle showing traces for\n"
              << "                               errors\n"
              << "  :?, :help                    Brings up this help menu\n"
@@ -589,12 +650,15 @@ ProcessLineResult NixRepl::processLine(std::string line)
 
         // runProgram redirects stdout to a StringSink,
         // using runProgram2 to allow editors to display their UI
-        runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args });
+        runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }).wait();
 
-        // Reload right after exiting the editor
+        // Reload right after exiting the editor if path is not in store
+        // Store is immutable, so there could be no changes, so there's no need to reload
+        if (!state->store->isInStore(path.resolveSymlinks().path.abs())) {
             state->resetFileCache();
             reloadFiles();
         }
+    }
 
     else if (command == ":t") {
         Value v;
@@ -612,42 +676,19 @@ ProcessLineResult NixRepl::processLine(std::string line)
         runNix("nix-shell", {state->store->printStorePath(drvPath)});
     }
 
-    else if (command == ":b" || command == ":bl" || command == ":i" || command == ":sh" || command == ":log") {
+    else if (command == ":log") {
+        StorePath drvPath = ([&] {
+            auto maybeDrvPath = state->store->maybeParseStorePath(arg);
+            if (maybeDrvPath && maybeDrvPath->isDerivation()) {
+                return std::move(*maybeDrvPath);
+            } else {
                 Value v;
                 evalString(arg, v);
-        StorePath drvPath = getDerivationPath(v);
+                return getDerivationPath(v);
+            }
+        })();
         Path drvPathRaw = state->store->printStorePath(drvPath);
 
-        if (command == ":b" || command == ":bl") {
-            // TODO: this only shows a progress bar for explicitly initiated builds,
-            // not eval-time fetching or builds performed for IFD.
-            // But we can't just show it everywhere, since that would erase partial output from evaluation.
-            startProgressBar();
-            Finally stopLogger([&]() {
-                stopProgressBar();
-            });
-
-            state->store->buildPaths({
-                DerivedPath::Built {
-                    .drvPath = makeConstantStorePathRef(drvPath),
-                    .outputs = OutputsSpec::All { },
-                },
-            });
-            auto drv = state->store->readDerivation(drvPath);
-            logger->cout("\nThis derivation produced the following outputs:");
-            for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath)) {
-                auto localStore = state->store.dynamic_pointer_cast<LocalFSStore>();
-                if (localStore && command == ":bl") {
-                    std::string symlink = "repl-result-" + outputName;
-                    localStore->addPermRoot(outputPath, absPath(symlink));
-                    logger->cout("  ./%s -> %s", symlink, state->store->printStorePath(outputPath));
-                } else {
-                    logger->cout("  %s -> %s", outputName, state->store->printStorePath(outputPath));
-                }
-            }
-        } else if (command == ":i") {
-            runNix("nix-env", {"-i", drvPathRaw});
-        } else if (command == ":log") {
         settings.readOnlyMode = true;
         Finally roModeReset([&]() {
             settings.readOnlyMode = false;
@@ -675,6 +716,44 @@ ProcessLineResult NixRepl::processLine(std::string line)
             }
         }
         if (!foundLog) throw Error("build log of '%s' is not available", drvPathRaw);
+    }
+
+    else if (command == ":b" || command == ":bl" || command == ":i" || command == ":sh") {
+        Value v;
+        evalString(arg, v);
+        StorePath drvPath = getDerivationPath(v);
+        Path drvPathRaw = state->store->printStorePath(drvPath);
+
+        if (command == ":b" || command == ":bl") {
+            // TODO: this only shows a progress bar for explicitly initiated builds,
+            // not eval-time fetching or builds performed for IFD.
+            // But we can't just show it everywhere, since that would erase partial output from evaluation.
+            logger->resetProgress();
+            logger->resume();
+            Finally stopLogger([&]() {
+                logger->pause();
+            });
+
+            state->store->buildPaths({
+                DerivedPath::Built {
+                    .drvPath = makeConstantStorePathRef(drvPath),
+                    .outputs = OutputsSpec::All { },
+                },
+            });
+            auto drv = state->store->readDerivation(drvPath);
+            logger->cout("\nThis derivation produced the following outputs:");
+            for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath)) {
+                auto localStore = state->store.dynamic_pointer_cast<LocalFSStore>();
+                if (localStore && command == ":bl") {
+                    std::string symlink = "repl-result-" + outputName;
+                    localStore->addPermRoot(outputPath, absPath(symlink));
+                    logger->cout("  ./%s -> %s", symlink, state->store->printStorePath(outputPath));
+                } else {
+                    logger->cout("  %s -> %s", outputName, state->store->printStorePath(outputPath));
+                }
+            }
+        } else if (command == ":i") {
+            runNix("nix-env", {"-i", drvPathRaw});
         } else {
             runNix("nix-shell", {drvPathRaw});
         }
@@ -758,7 +837,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
             line[p + 1] != '=' &&
             isVarName(name = removeWhitespace(line.substr(0, p))))
         {
-            Expr * e = parseString(line.substr(p + 1));
+            Expr & e = parseString(line.substr(p + 1));
             Value & v(*state->allocValue());
             v.mkThunk(env, e);
             addVarToScope(state->symbols.create(name), v);
@@ -883,7 +962,7 @@ Value * NixRepl::getReplOverlaysEvalFunction()
     auto code =
         #include "repl-overlays.nix.gen.hh"
         ;
-    auto expr = state->parseExprFromString(
+    auto & expr = state->parseExprFromString(
         code,
         SourcePath(evalReplInitFilesPath),
         state->staticBaseEnv
@@ -991,7 +1070,7 @@ Value * NixRepl::bindingsToAttrs()
 }
 
 
-Expr * NixRepl::parseString(std::string s)
+Expr & NixRepl::parseString(std::string s)
 {
     return state->parseExprFromString(std::move(s), state->rootPath(CanonPath::fromCwd()), staticEnv);
 }
@@ -999,16 +1078,16 @@ Expr * NixRepl::parseString(std::string s)
 
 void NixRepl::evalString(std::string s, Value & v)
 {
-    Expr * e = parseString(s);
-    e->eval(*state, *env, v);
+    Expr & e = parseString(s);
+    e.eval(*state, *env, v);
     state->forceValue(v, v.determinePos(noPos));
 }
 
 Value * NixRepl::evalFile(SourcePath & path)
 {
-    auto expr = state->parseExprFromFile(path, staticEnv);
+    auto & expr = state->parseExprFromFile(path, staticEnv);
     Value * result(state->allocValue());
-    expr->eval(*state, *env, *result);
+    expr.eval(*state, *env, *result);
     state->forceValue(*result, result->determinePos(noPos));
     return result;
 }

src/libexpr/attr-set.cc

@@ -1,5 +1,6 @@
 #include "attr-set.hh"
-#include "eval-inline.hh"
+#include "eval.hh"
+#include "gc-alloc.hh"
 
 #include <algorithm>
 
@@ -19,7 +20,7 @@ Bindings * EvalState::allocBindings(size_t capacity)
         throw Error("attribute set of size %d is too big", capacity);
     nrAttrsets++;
     nrAttrsInAttrsets += capacity;
-    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
+    return new (gcAllocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
 }
 
 

src/libexpr/eval-cache.cc

@@ -50,7 +50,7 @@ struct AttrDb
         Path cacheDir = getCacheDir() + "/nix/eval-cache-v5";
         createDirs(cacheDir);
 
-        Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
+        Path dbPath = cacheDir + "/" + fingerprint.to_string(Base::Base16, false) + ".sqlite";
 
         state->db = SQLite(dbPath);
         state->db.isCache();
@@ -306,7 +306,7 @@ struct AttrDb
             case AttrType::Bool:
                 return {{rowId, queryAttribute.getInt(2) != 0}};
             case AttrType::Int:
-                return {{rowId, int_t{queryAttribute.getInt(2)}}};
+                return {{rowId, int_t{NixInt{queryAttribute.getInt(2)}}}};
             case AttrType::ListOfStrings:
                 return {{rowId, tokenizeString<std::vector<std::string>>(queryAttribute.getStr(2), "\t")}};
             case AttrType::Missing:
@@ -449,7 +449,7 @@ Value & AttrCursor::forceValue()
         else if (v.type() == nBool)
             cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
         else if (v.type() == nInt)
-            cachedValue = {root->db->setInt(getKey(), v.integer), int_t{v.integer}};
+            cachedValue = {root->db->setInt(getKey(), v.integer.value), int_t{v.integer}};
         else if (v.type() == nAttrs)
             ; // FIXME: do something?
         else

src/libexpr/eval-inline.hh

@@ -4,26 +4,10 @@
 #include "print.hh"
 #include "eval.hh"
 #include "eval-error.hh"
+#include "gc-alloc.hh"
 
 namespace nix {
 
-/**
- * Note: Various places expect the allocated memory to be zeroed.
- */
-[[gnu::always_inline]]
-inline void * allocBytes(size_t n)
-{
-    void * p;
-#if HAVE_BOEHMGC
-    p = GC_MALLOC(n);
-#else
-    p = calloc(n, 1);
-#endif
-    if (!p) throw std::bad_alloc();
-    return p;
-}
-
-
 [[gnu::always_inline]]
 Value * EvalState::allocValue()
 {
@@ -43,7 +27,7 @@ Value * EvalState::allocValue()
     *valueAllocCache = GC_NEXT(p);
     GC_NEXT(p) = nullptr;
 #else
-    void * p = allocBytes(sizeof(Value));
+    void * p = gcAllocBytes(sizeof(Value));
 #endif
 
     nrValues++;
@@ -73,7 +57,7 @@ Env & EvalState::allocEnv(size_t size)
         env = (Env *) p;
     } else
 #endif
-        env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
+        env = (Env *) gcAllocBytes(sizeof(Env) + size * sizeof(Value *));
 
     /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
 
@@ -86,11 +70,11 @@ void EvalState::forceValue(Value & v, const PosIdx pos)
 {
     if (v.isThunk()) {
         Env * env = v.thunk.env;
-        Expr * expr = v.thunk.expr;
+        Expr & expr = *v.thunk.expr;
         try {
             v.mkBlackhole();
             //checkInterrupt();
-            expr->eval(*this, *env, v);
+            expr.eval(*this, *env, v);
         } catch (...) {
             v.mkThunk(env, expr);
             tryFixupBlackHolePos(v, pos);

src/libexpr/eval-settings.cc

@@ -63,11 +63,9 @@ Strings EvalSettings::getDefaultNixPath()
         }
     };
 
-    if (!evalSettings.restrictEval && !evalSettings.pureEval) {
     add(getNixDefExpr() + "/channels");
     add(rootChannelsDir() + "/nixpkgs", "nixpkgs");
     add(rootChannelsDir());
-    }
 
     return res;
 }

src/libexpr/eval-settings.hh

@@ -75,8 +75,17 @@ struct EvalSettings : Config
         R"(
           Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state:
 
-          - Restrict file system and network access to files specified by cryptographic hash
-          - Disable [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime)
+          - File system and network access is restricted to accesses to immutable data only:
+            - Path literals relative to the home directory like `~/lix` are rejected at parse time.
+            - Access to absolute paths that did not result from Nix language evaluation is rejected when such paths are given as parameters to builtins like, for example, [`builtins.readFile`](@docroot@/language/builtins.md#builtins-readFile).
+
+              Access is nonetheless allowed to (absolute) paths in the Nix store that are returned by builtins like [`builtins.filterSource`](@docroot@/language/builtins.md#builtins-filterSource), [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball) and similar.
+            - Impure fetches such as not specifying a commit ID for `builtins.fetchGit` or not specifying a hash for `builtins.fetchTarball` are rejected.
+            - In flakes, access to relative paths outside of the root of the flake's source tree (often, a git repository) is rejected.
+          - The evaluator ignores `NIX_PATH`, `-I` and the `nix-path` setting. Thus, [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath) is an empty list.
+          - The builtins [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) are absent from `builtins`.
+          - [`builtins.getEnv`](@docroot@/language/builtin-constants.md#builtins-currentSystem) always returns empty string for any variable.
+          - [`builtins.storePath`](@docroot@/language/builtins.md#builtins-storePath) throws an error (Lix may change this, tracking issue: <https://git.lix.systems/lix-project/lix/issues/402>)
         )"
         };
 
@@ -98,6 +107,7 @@ struct EvalSettings : Config
           allowed to access `https://github.com/NixOS/patchelf.git`.
         )"};
 
+
     Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
         R"(
           If set to `true`, the Nix evaluator will trace every function call.

src/libexpr/eval.cc

@@ -4,10 +4,12 @@
 #include "primops.hh"
 #include "print-options.hh"
 #include "shared.hh"
+#include "suggestions.hh"
 #include "types.hh"
 #include "store-api.hh"
 #include "derivations.hh"
 #include "downstream-placeholder.hh"
+#include "gc-alloc.hh"
 #include "globals.hh"
 #include "eval-inline.hh"
 #include "filetransfer.hh"
@@ -17,10 +19,8 @@
 #include "gc-small-vector.hh"
 #include "fetch-to-store.hh"
 #include "flake/flakeref.hh"
-#include "parser-tab.hh"
 
 #include <algorithm>
-#include <chrono>
 #include <iostream>
 #include <sstream>
 #include <cstring>
@@ -42,58 +42,12 @@
 #include <gc/gc.h>
 #include <gc/gc_cpp.h>
 
-#include <boost/coroutine2/coroutine.hpp>
-#include <boost/coroutine2/protected_fixedsize_stack.hpp>
-#include <boost/context/stack_context.hpp>
-
 #endif
 
 using json = nlohmann::json;
 
 namespace nix {
 
-static char * allocString(size_t size)
-{
-    char * t;
-#if HAVE_BOEHMGC
-    t = (char *) GC_MALLOC_ATOMIC(size);
-#else
-    t = (char *) malloc(size);
-#endif
-    if (!t) throw std::bad_alloc();
-    return t;
-}
-
-
-static char * dupString(const char * s)
-{
-    char * t;
-#if HAVE_BOEHMGC
-    t = GC_STRDUP(s);
-#else
-    t = strdup(s);
-#endif
-    if (!t) throw std::bad_alloc();
-    return t;
-}
-
-
-// When there's no need to write to the string, we can optimize away empty
-// string allocations.
-// This function handles makeImmutableString(std::string_view()) by returning
-// the empty string.
-static const char * makeImmutableString(std::string_view s)
-{
-    const size_t size = s.size();
-    if (size == 0)
-        return "";
-    auto t = allocString(size + 1);
-    memcpy(t, s.data(), size);
-    t[size] = '\0';
-    return t;
-}
-
-
 RootValue allocRootValue(Value * v)
 {
 #if HAVE_BOEHMGC
@@ -116,11 +70,6 @@ std::string printValue(EvalState & state, Value & v)
     return out.str();
 }
 
-void Value::print(EvalState & state, std::ostream & str, PrintOptions options)
-{
-    printValue(state, str, *this, options);
-}
-
 const Value * getPrimOp(const Value &v) {
     const Value * primOp = &v;
     while (primOp->isPrimOpApp()) {
@@ -170,32 +119,6 @@ std::string showType(const Value & v)
     #pragma GCC diagnostic pop
 }
 
-PosIdx Value::determinePos(const PosIdx pos) const
-{
-    // Allow selecting a subset of enum values
-    #pragma GCC diagnostic push
-    #pragma GCC diagnostic ignored "-Wswitch-enum"
-    switch (internalType) {
-        case tAttrs: return attrs->pos;
-        case tLambda: return lambda.fun->pos;
-        case tApp: return app.left->determinePos(pos);
-        default: return pos;
-    }
-    #pragma GCC diagnostic pop
-}
-
-bool Value::isTrivial() const
-{
-    return
-        internalType != tApp
-        && internalType != tPrimOpApp
-        && (internalType != tThunk
-            || (dynamic_cast<ExprAttrs *>(thunk.expr)
-                && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty())
-            || dynamic_cast<ExprLambda *>(thunk.expr)
-            || dynamic_cast<ExprList *>(thunk.expr));
-}
-
 
 #if HAVE_BOEHMGC
 /* Called when the Boehm GC runs out of memory. */
@@ -205,42 +128,6 @@ static void * oomHandler(size_t requested)
     throw std::bad_alloc();
 }
 
-class BoehmGCStackAllocator : public StackAllocator {
-    boost::coroutines2::protected_fixedsize_stack stack {
-        // We allocate 8 MB, the default max stack size on NixOS.
-        // A smaller stack might be quicker to allocate but reduces the stack
-        // depth available for source filter expressions etc.
-        std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024))
-    };
-
-    // This is specific to boost::coroutines2::protected_fixedsize_stack.
-    // The stack protection page is included in sctx.size, so we have to
-    // subtract one page size from the stack size.
-    std::size_t pfss_usable_stack_size(boost::context::stack_context &sctx) {
-        return sctx.size - boost::context::stack_traits::page_size();
-    }
-
-  public:
-    boost::context::stack_context allocate() override {
-        auto sctx = stack.allocate();
-
-        // Stacks generally start at a high address and grow to lower addresses.
-        // Architectures that do the opposite are rare; in fact so rare that
-        // boost_routine does not implement it.
-        // So we subtract the stack size.
-        GC_add_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp);
-        return sctx;
-    }
-
-    void deallocate(boost::context::stack_context sctx) override {
-        GC_remove_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp);
-        stack.deallocate(sctx);
-    }
-
-};
-
-static BoehmGCStackAllocator boehmGCStackAllocator;
-
 #endif
 
 
@@ -256,23 +143,6 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env)
     }
 }
 
-#if HAVE_BOEHMGC
-/* Disable GC while this object lives. Used by CoroutineContext.
- *
- * Boehm keeps a count of GC_disable() and GC_enable() calls,
- * and only enables GC when the count matches.
- */
-class BoehmDisableGC {
-public:
-    BoehmDisableGC() {
-        GC_disable();
-    };
-    ~BoehmDisableGC() {
-        GC_enable();
-    };
-};
-#endif
-
 static bool gcInitialised = false;
 
 void initGC()
@@ -294,17 +164,6 @@ void initGC()
 
     GC_set_oom_fn(oomHandler);
 
-    StackAllocator::defaultAllocator = &boehmGCStackAllocator;
-
-
-#if NIX_BOEHM_PATCH_VERSION != 1
-    printTalkative("Unpatched BoehmGC, disabling GC inside coroutines");
-    /* Used to disable GC when entering coroutines on macOS */
-    create_coro_gc_hook = []() -> std::shared_ptr<void> {
-        return std::make_shared<BoehmDisableGC>();
-    };
-#endif
-
     /* Set the initial heap size to something fairly big (25% of
        physical RAM, up to a maximum of 384 MiB) so that in most cases
        we don't need to garbage collect at all.  (Collection has a
@@ -358,6 +217,12 @@ EvalState::EvalState(
     , sRight(symbols.create("right"))
     , sWrong(symbols.create("wrong"))
     , sStructuredAttrs(symbols.create("__structuredAttrs"))
+    , sAllowedReferences(symbols.create("allowedReferences"))
+    , sAllowedRequisites(symbols.create("allowedRequisites"))
+    , sDisallowedReferences(symbols.create("disallowedReferences"))
+    , sDisallowedRequisites(symbols.create("disallowedRequisites"))
+    , sMaxSize(symbols.create("maxSize"))
+    , sMaxClosureSize(symbols.create("maxClosureSize"))
     , sBuilder(symbols.create("builder"))
     , sArgs(symbols.create("args"))
     , sContentAddressed(symbols.create("__contentAddressed"))
@@ -418,7 +283,7 @@ EvalState::EvalState(
     }
 
     if (evalSettings.restrictEval || evalSettings.pureEval) {
-        allowedPaths = PathSet();
+        allowedPaths = std::optional(PathSet());
 
         for (auto & i : searchPath.elements) {
             auto r = resolveSearchPathPath(i.path);
@@ -602,29 +467,6 @@ std::ostream & operator<<(std::ostream & output, PrimOp & primOp)
     return output;
 }
 
-
-PrimOp * Value::primOpAppPrimOp() const
-{
-    Value * left = primOpApp.left;
-    while (left && !left->isPrimOp()) {
-        left = left->primOpApp.left;
-    }
-
-    if (!left)
-        return nullptr;
-    return left->primOp;
-}
-
-
-void Value::mkPrimOp(PrimOp * p)
-{
-    p->check();
-    clearValue();
-    internalType = tPrimOp;
-    primOp = p;
-}
-
-
 Value * EvalState::addPrimOp(PrimOp && primOp)
 {
     /* Hack to make constants lazy: turn them into a application of
@@ -876,43 +718,6 @@ DebugTraceStacker::DebugTraceStacker(EvalState & evalState, DebugTrace t)
         evalState.runDebugRepl(nullptr, trace.env, trace.expr);
 }
 
-void Value::mkString(std::string_view s)
-{
-    mkString(makeImmutableString(s));
-}
-
-
-static void copyContextToValue(Value & v, const NixStringContext & context)
-{
-    if (!context.empty()) {
-        size_t n = 0;
-        v.string.context = (const char * *)
-            allocBytes((context.size() + 1) * sizeof(char *));
-        for (auto & i : context)
-            v.string.context[n++] = dupString(i.to_string().c_str());
-        v.string.context[n] = 0;
-    }
-}
-
-void Value::mkString(std::string_view s, const NixStringContext & context)
-{
-    mkString(s);
-    copyContextToValue(*this, context);
-}
-
-void Value::mkStringMove(const char * s, const NixStringContext & context)
-{
-    mkString(s);
-    copyContextToValue(*this, context);
-}
-
-
-void Value::mkPath(const SourcePath & path)
-{
-    mkPath(makeImmutableString(path.path.abs()));
-}
-
-
 inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 {
     for (auto l = var.level; l; --l, env = env->up) ;
@@ -943,21 +748,21 @@ void EvalState::mkList(Value & v, size_t size)
 {
     v.mkList(size);
     if (size > 2)
-        v.bigList.elems = (Value * *) allocBytes(size * sizeof(Value *));
+        v.bigList.elems = gcAllocType<Value *>(size);
     nrListElems += size;
 }
 
 
 unsigned long nrThunks = 0;
 
-static inline void mkThunk(Value & v, Env & env, Expr * expr)
+static inline void mkThunk(Value & v, Env & env, Expr & expr)
 {
     v.mkThunk(&env, expr);
     nrThunks++;
 }
 
 
-void EvalState::mkThunk_(Value & v, Expr * expr)
+void EvalState::mkThunk_(Value & v, Expr & expr)
 {
     mkThunk(v, baseEnv, expr);
 }
@@ -1058,7 +863,7 @@ void EvalState::mkSingleDerivedPathString(
 Value * Expr::maybeThunk(EvalState & state, Env & env)
 {
     Value * v = state.allocValue();
-    mkThunk(*v, env, this);
+    mkThunk(*v, env, *this);
     return v;
 }
 
@@ -1122,7 +927,7 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial
         e = j->second;
 
     if (!e)
-        e = parseExprFromFile(checkSourcePath(resolvedPath));
+        e = &parseExprFromFile(checkSourcePath(resolvedPath));
 
     cacheFile(path, resolvedPath, e, v, mustBeTrivial);
 }
@@ -1159,7 +964,7 @@ void EvalState::cacheFile(
         if (mustBeTrivial &&
             !(dynamic_cast<ExprAttrs *>(e)))
             error<EvalError>("file '%s' must be an attribute set", path).debugThrow();
-        eval(e, v);
+        eval(*e, v);
     } catch (Error & e) {
         addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath.to_string());
         throw;
@@ -1170,23 +975,23 @@ void EvalState::cacheFile(
 }
 
 
-void EvalState::eval(Expr * e, Value & v)
+void EvalState::eval(Expr & e, Value & v)
 {
-    e->eval(*this, baseEnv, v);
+    e.eval(*this, baseEnv, v);
 }
 
 
-inline bool EvalState::evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx)
+inline bool EvalState::evalBool(Env & env, Expr & e, const PosIdx pos, std::string_view errorCtx)
 {
     try {
         Value v;
-        e->eval(*this, env, v);
+        e.eval(*this, env, v);
         if (v.type() != nBool)
             error<TypeError>(
                  "expected a Boolean but found %1%: %2%",
                  showType(v),
                  ValuePrinter(*this, v, errorPrintOptions)
-             ).atPos(pos).withFrame(env, *e).debugThrow();
+             ).atPos(pos).withFrame(env, e).debugThrow();
         return v.boolean;
     } catch (Error & e) {
         e.addTrace(positions[pos], errorCtx);
@@ -1195,16 +1000,16 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const PosIdx pos, std::stri
 }
 
 
-inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx)
+inline void EvalState::evalAttrs(Env & env, Expr & e, Value & v, const PosIdx pos, std::string_view errorCtx)
 {
     try {
-        e->eval(*this, env, v);
+        e.eval(*this, env, v);
         if (v.type() != nAttrs)
             error<TypeError>(
                 "expected a set but found %1%: %2%",
                 showType(v),
                 ValuePrinter(*this, v, errorPrintOptions)
-            ).withFrame(env, *e).debugThrow();
+            ).withFrame(env, e).debugThrow();
     } catch (Error & e) {
         e.addTrace(positions[pos], errorCtx);
         throw;
@@ -1247,7 +1052,7 @@ Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up)
     inheritEnv.up = &up;
 
     Displacement displ = 0;
-    for (auto from : *inheritFromExprs)
+    for (auto & from : *inheritFromExprs)
         inheritEnv.values[displ++] = from->maybeThunk(state, up);
 
     return &inheritEnv;
@@ -1277,7 +1082,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
             Value * vAttr;
             if (hasOverrides && i.second.kind != AttrDef::Kind::Inherited) {
                 vAttr = state.allocValue();
-                mkThunk(*vAttr, *i.second.chooseByKind(&env2, &env, inheritEnv), i.second.e);
+                mkThunk(*vAttr, *i.second.chooseByKind(&env2, &env, inheritEnv), *i.second.e);
             } else
                 vAttr = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv));
             env2.values[displ++] = vAttr;
@@ -1426,11 +1231,26 @@ static std::string showAttrPath(EvalState & state, Env & env, const AttrPath & a
 
 void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 {
-    Value vTmp;
-    PosIdx pos2;
-    Value * vAttrs = &vTmp;
+    Value vFirst;
 
-    e->eval(state, env, vTmp);
+    // Pointer to the current attrset Value in this select chain.
+    Value * vCurrent = &vFirst;
+    // Position for the current attrset Value in this select chain.
+    PosIdx posCurrent;
+
+    try {
+        e->eval(state, env, vFirst);
+    } catch (Error & e) {
+        assert(this->e != nullptr);
+        state.addErrorTrace(
+            e,
+            getPos(),
+            "while evaluating '%s' to select '%s' on it",
+            ExprPrinter(state, *this->e),
+            showAttrPath(state.symbols, this->attrPath)
+        );
+        throw;
+    }
 
     try {
         auto dts = state.debugRepl
@@ -1443,48 +1263,112 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
                 showAttrPath(state, env, attrPath))
             : nullptr;
 
-        for (auto & i : attrPath) {
+        for (auto const & [partIdx, currentAttrName] : enumerate(attrPath)) {
             state.nrLookups++;
-            Bindings::iterator j;
-            auto name = getName(i, state, env);
-            if (def) {
-                state.forceValue(*vAttrs, pos);
-                if (vAttrs->type() != nAttrs ||
-                    (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
-                {
-                    def->eval(state, env, v);
+
+            Symbol const name = getName(currentAttrName, state, env);
+
+            // For formatting errors, which should be done only when needed.
+            auto partsSoFar = [&]() -> std::string {
+                std::stringstream ss;
+                // We start with the base thing this ExprSelect is selecting on.
+                assert(this->e != nullptr);
+                this->e->show(state.symbols, ss);
+
+                // Then grab each part of the attr path up to this one.
+                assert(partIdx < attrPath.size());
+                std::span<AttrName> const parts(
+                    attrPath.begin(),
+                    attrPath.begin() + partIdx
+                );
+
+                // And convert them to strings and join them.
+                for (auto const & part : parts) {
+                    auto const partName = getName(part, state, env);
+                    ss << "." << state.symbols[partName];
+                }
+
+                return ss.str();
+            };
+
+            try {
+                state.forceValue(*vCurrent, pos);
+            } catch (Error & e) {
+                state.addErrorTrace(
+                    e,
+                    getPos(),
+                    "while evaluating '%s' to select '%s' on it",
+                    partsSoFar(),
+                    state.symbols[name]
+                );
+                throw;
+            }
+
+            if (vCurrent->type() != nAttrs) {
+
+                // If we have an `or` provided default,
+                // then this is allowed to not be an attrset.
+                if (def != nullptr) {
+                    this->def->eval(state, env, v);
                     return;
                 }
-            } else {
-                state.forceAttrs(*vAttrs, pos, "while selecting an attribute");
-                if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) {
-                    std::set<std::string> allAttrNames;
-                    for (auto & attr : *vAttrs->attrs)
-                        allAttrNames.insert(state.symbols[attr.name]);
-                    auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]);
-                    state.error<EvalError>("attribute '%1%' missing", state.symbols[name])
-                        .atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow();
-                }
-            }
-            vAttrs = j->value;
-            pos2 = j->pos;
-            if (state.countCalls) state.attrSelects[pos2]++;
+
+                // Otherwise, we must type error.
+                state.error<TypeError>(
+                    "expected a set but found %s: %s",
+                    showType(*vCurrent),
+                    ValuePrinter(state, *vCurrent, errorPrintOptions)
+                ).addTrace(
+                    pos,
+                    HintFmt("while selecting '%s' on '%s'", state.symbols[name], partsSoFar())
+                ).debugThrow();
             }
 
-        state.forceValue(*vAttrs, (pos2 ? pos2 : this->pos ) );
+            // Now that we know this is actually an attrset, try to find an attr
+            // with the selected name.
+            Bindings::iterator attrIt = vCurrent->attrs->find(name);
+            if (attrIt == vCurrent->attrs->end()) {
+
+                // If we have an `or` provided default, then we'll use that.
+                if (def != nullptr) {
+                    this->def->eval(state, env, v);
+                    return;
+                }
+
+                // Otherwise, missing attr error.
+                std::set<std::string> allAttrNames;
+                for (auto const & attr : *vCurrent->attrs) {
+                    allAttrNames.insert(state.symbols[attr.name]);
+                }
+                auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]);
+                state.error<EvalError>("attribute '%s' missing", state.symbols[name])
+                    .atPos(pos)
+                    .withSuggestions(suggestions)
+                    .withFrame(env, *this)
+                    .debugThrow();
+            }
+
+            // If we're here, then we successfully found the attribute.
+            // Set our currently operated-on attrset to this one, and keep going.
+            vCurrent = attrIt->value;
+            posCurrent = attrIt->pos;
+            if (state.countCalls) state.attrSelects[posCurrent]++;
+        }
+
+        state.forceValue(*vCurrent, (posCurrent ? posCurrent : this->pos));
 
     } catch (Error & e) {
-        if (pos2) {
-            auto pos2r = state.positions[pos2];
+        if (posCurrent) {
+            auto pos2r = state.positions[posCurrent];
             auto origin = std::get_if<SourcePath>(&pos2r.origin);
             if (!(origin && *origin == state.derivationInternal))
-                state.addErrorTrace(e, pos2, "while evaluating the attribute '%1%'",
+                state.addErrorTrace(e, posCurrent, "while evaluating the attribute '%1%'",
                     showAttrPath(state, env, attrPath));
         }
         throw;
     }
 
-    v = *vAttrs;
+    v = *vCurrent;
 }
 
 
@@ -1533,6 +1417,66 @@ public:
 };
 };
 
+/** Currently these each just take one, but maybe in the future we could have diagnostics
+ * for all unexpected and missing arguments?
+ */
+struct FormalsMatch
+{
+    std::vector<Symbol> missing;
+    std::vector<Symbol> unexpected;
+};
+
+/** Matchup an attribute argument set to a lambda's formal arguments,
+ * or return what arguments were required but not given, or given but not allowed.
+ * (currently returns only one, for each).
+ */
+FormalsMatch matchupFormals(EvalState & state, Env & env, Displacement & displ, ExprLambda const & lambda, Bindings & attrs)
+{
+    size_t attrsUsed = 0;
+
+    for (auto const & formal : lambda.formals->formals) {
+
+        // The attribute whose name matches the name of the formal we're matching up, if it exists.
+        Attr const * matchingArg = attrs.get(formal.name);
+        if (matchingArg) {
+            attrsUsed += 1;
+            env.values[displ] = matchingArg->value;
+            displ += 1;
+
+            // We're done here. Move on to the next formal.
+            continue;
+        }
+
+        // The argument for this formal wasn't given.
+        // If the formal has a default, use it.
+        if (formal.def) {
+            env.values[displ] = formal.def->maybeThunk(state, env);
+            displ += 1;
+        } else {
+            // Otherwise, let our caller know what was missing.
+            return FormalsMatch{
+                .missing = {formal.name},
+            };
+        }
+    }
+
+    // Check for unexpected extra arguments.
+    if (!lambda.formals->ellipsis && attrsUsed != attrs.size()) {
+        // Return the first unexpected argument.
+        for (Attr const & attr : attrs) {
+            if (!lambda.formals->has(attr.name)) {
+                return FormalsMatch{
+                    .unexpected = {attr.name},
+                };
+            }
+        }
+
+        abort(); // unreachable.
+    }
+
+    return FormalsMatch{};
+}
+
 void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos)
 {
     if (callDepth > evalSettings.maxCallDepth)
@@ -1586,53 +1530,42 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 if (lambda.arg)
                     env2.values[displ++] = args[0];
 
-                /* For each formal argument, get the actual argument.  If
-                   there is no matching actual argument but the formal
-                   argument has a default, use the default. */
-                size_t attrsUsed = 0;
-                for (auto & i : lambda.formals->formals) {
-                    auto j = args[0]->attrs->get(i.name);
-                    if (!j) {
-                        if (!i.def) {
-                            error<TypeError>("function '%1%' called without required argument '%2%'",
-                                             (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"),
-                                             symbols[i.name])
+                ///* For each formal argument, get the actual argument.  If
+                //   there is no matching actual argument but the formal
+                //   argument has a default, use the default. */
+                auto const formalsMatch = matchupFormals(
+                    *this,
+                    env2,
+                    displ,
+                    lambda,
+                    *args[0]->attrs
+                );
+                for (auto const & missingArg : formalsMatch.missing) {
+                    auto const missing = symbols[missingArg];
+                    error<TypeError>("function '%s' called without required argument '%s'", lambda.getName(symbols), missing)
                         .atPos(lambda.pos)
                         .withTrace(pos, "from call site")
                         .withFrame(*fun.lambda.env, lambda)
                         .debugThrow();
                 }
-                        env2.values[displ++] = i.def->maybeThunk(*this, env2);
-                    } else {
-                        attrsUsed++;
-                        env2.values[displ++] = j->value;
+                for (auto const & unexpectedArg : formalsMatch.unexpected) {
+                    auto const unex = symbols[unexpectedArg];
+                    std::set<std::string> formalNames;
+                    for (auto const & formal : lambda.formals->formals) {
+                        formalNames.insert(symbols[formal.name]);
                     }
+                    auto sug = Suggestions::bestMatches(formalNames, unex);
+                    error<TypeError>("function '%s' called with unexpected argument '%s'", lambda.getName(symbols), unex)
+                        .atPos(lambda.pos)
+                        .withTrace(pos, "from call site")
+                        .withSuggestions(sug)
+                        .withFrame(*fun.lambda.env, lambda)
+                        .debugThrow();
                 }
 
-                /* Check that each actual argument is listed as a formal
-                   argument (unless the attribute match specifies a `...'). */
-                if (!lambda.formals->ellipsis && attrsUsed != args[0]->attrs->size()) {
-                    /* Nope, so show the first unexpected argument to the
-                       user. */
-                    for (auto & i : *args[0]->attrs)
-                        if (!lambda.formals->has(i.name)) {
-                            std::set<std::string> formalNames;
-                            for (auto & formal : lambda.formals->formals)
-                                formalNames.insert(symbols[formal.name]);
-                            auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]);
-                            error<TypeError>("function '%1%' called with unexpected argument '%2%'",
-                                             (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"),
-                                             symbols[i.name])
-                                .atPos(lambda.pos)
-                                .withTrace(pos, "from call site")
-                                .withSuggestions(suggestions)
-                                .withFrame(*fun.lambda.env, lambda)
-                                .debugThrow();
-                        }
-                    abort(); // can't happen
-                }
             }
 
+
             nrFunctionCalls++;
             if (countCalls) incrFunctionCall(&lambda);
 
@@ -1642,9 +1575,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                     ? makeDebugTraceStacker(
                         *this, *lambda.body, env2, positions[lambda.pos],
                         "while calling %s",
-                        lambda.name
-                        ? concatStrings("'", symbols[lambda.name], "'")
-                        : "anonymous lambda")
+                        lambda.getQuotedName(symbols))
                     : nullptr;
 
                 lambda.body->eval(*this, env2, vCur);
@@ -1654,9 +1585,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                         e,
                         lambda.pos,
                         "while calling %s",
-                        lambda.name
-                        ? concatStrings("'", symbols[lambda.name], "'")
-                        : "anonymous lambda");
+                        lambda.getQuotedName(symbols));
                     if (pos) addErrorTrace(e, pos, "from call site");
                 }
                 throw;
@@ -1683,6 +1612,15 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
 
                 try {
                     fn->fun(*this, vCur.determinePos(noPos), args, vCur);
+                } catch (ThrownError & e) {
+                    // Distinguish between an error that simply happened while "throw"
+                    // was being evaluated and an explicit thrown error.
+                    if (fn->name == "throw") {
+                        addErrorTrace(e, pos, "caused by explicit %s", "throw");
+                    } else {
+                        addErrorTrace(e, pos, "while calling the '%s' builtin", fn->name);
+                    }
+                    throw;
                 } catch (Error & e) {
                     addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
                     throw;
@@ -1849,7 +1787,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
 Nix attempted to evaluate a function as a top level expression; in
 this case it must have its arguments supplied either by default
 values, or passed explicitly with '--arg' or '--argstr'. See
-https://nixos.org/manual/nix/stable/language/constructs.html#functions.)", symbols[i.name])
+https://docs.lix.systems/manual/lix/stable/language/constructs.html#functions)", symbols[i.name])
                     .atPos(i.pos).withFrame(*fun.lambda.env, *fun.lambda.fun).debugThrow();
             }
         }
@@ -1872,13 +1810,13 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
 void ExprIf::eval(EvalState & state, Env & env, Value & v)
 {
     // We cheat in the parser, and pass the position of the condition as the position of the if itself.
-    (state.evalBool(env, cond, pos, "while evaluating a branch condition") ? then : else_)->eval(state, env, v);
+    (state.evalBool(env, *cond, pos, "while evaluating a branch condition") ? *then : *else_).eval(state, env, v);
 }
 
 
 void ExprAssert::eval(EvalState & state, Env & env, Value & v)
 {
-    if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) {
+    if (!state.evalBool(env, *cond, pos, "in the condition of the assert statement")) {
         std::ostringstream out;
         cond->show(state.symbols, out);
         state.error<AssertionError>("assertion '%1%' failed", out.str()).atPos(pos).withFrame(env, *this).debugThrow();
@@ -1889,7 +1827,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
 
 void ExprOpNot::eval(EvalState & state, Env & env, Value & v)
 {
-    v.mkBool(!state.evalBool(env, e, getPos(), "in the argument of the not operator")); // XXX: FIXME: !
+    v.mkBool(!state.evalBool(env, *e, getPos(), "in the argument of the not operator")); // XXX: FIXME: !
 }
 
 
@@ -1911,27 +1849,27 @@ void ExprOpNEq::eval(EvalState & state, Env & env, Value & v)
 
 void ExprOpAnd::eval(EvalState & state, Env & env, Value & v)
 {
-    v.mkBool(state.evalBool(env, e1, pos, "in the left operand of the AND (&&) operator") && state.evalBool(env, e2, pos, "in the right operand of the AND (&&) operator"));
+    v.mkBool(state.evalBool(env, *e1, pos, "in the left operand of the AND (&&) operator") && state.evalBool(env, *e2, pos, "in the right operand of the AND (&&) operator"));
 }
 
 
 void ExprOpOr::eval(EvalState & state, Env & env, Value & v)
 {
-    v.mkBool(state.evalBool(env, e1, pos, "in the left operand of the OR (||) operator") || state.evalBool(env, e2, pos, "in the right operand of the OR (||) operator"));
+    v.mkBool(state.evalBool(env, *e1, pos, "in the left operand of the OR (||) operator") || state.evalBool(env, *e2, pos, "in the right operand of the OR (||) operator"));
 }
 
 
 void ExprOpImpl::eval(EvalState & state, Env & env, Value & v)
 {
-    v.mkBool(!state.evalBool(env, e1, pos, "in the left operand of the IMPL (->) operator") || state.evalBool(env, e2, pos, "in the right operand of the IMPL (->) operator"));
+    v.mkBool(!state.evalBool(env, *e1, pos, "in the left operand of the IMPL (->) operator") || state.evalBool(env, *e2, pos, "in the right operand of the IMPL (->) operator"));
 }
 
 
 void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
 {
     Value v1, v2;
-    state.evalAttrs(env, e1, v1, pos, "in the left operand of the update (//) operator");
-    state.evalAttrs(env, e2, v2, pos, "in the right operand of the update (//) operator");
+    state.evalAttrs(env, *e1, v1, pos, "in the left operand of the update (//) operator");
+    state.evalAttrs(env, *e2, v2, pos, "in the right operand of the update (//) operator");
 
     state.nrOpUpdates++;
 
@@ -2008,7 +1946,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     NixStringContext context;
     std::vector<BackedStringView> s;
     size_t sSize = 0;
-    NixInt n = 0;
+    NixInt n{0};
     NixFloat nf = 0;
 
     bool first = !forceString;
@@ -2024,7 +1962,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
        Value. allocating a GC'd string directly and moving it into a
        Value lets us avoid an allocation and copy. */
     const auto c_str = [&] {
-        char * result = allocString(sSize + 1);
+        char * result = gcAllocString(sSize + 1);
         char * tmp = result;
         for (const auto & part : s) {
             memcpy(tmp, part->data(), part->size());
@@ -2035,10 +1973,10 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     };
 
     // List of returned strings. References to these Values must NOT be persisted.
-    SmallTemporaryValueVector<conservativeStackReservation> values(es->size());
+    SmallTemporaryValueVector<conservativeStackReservation> values(es.size());
     Value * vTmpP = values.data();
 
-    for (auto & [i_pos, i] : *es) {
+    for (auto & [i_pos, i] : es) {
         Value & vTmp = *vTmpP++;
         i->eval(state, env, vTmp);
 
@@ -2052,23 +1990,28 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
 
         if (firstType == nInt) {
             if (vTmp.type() == nInt) {
-                n += vTmp.integer;
+                auto newN = n + vTmp.integer;
+                if (auto checked = newN.valueChecked(); checked.has_value()) {
+                    n = NixInt(*checked);
+                } else {
+                    state.error<EvalError>("integer overflow in adding %1% + %2%", n, vTmp.integer).atPos(i_pos).debugThrow();
+                }
             } else if (vTmp.type() == nFloat) {
                 // Upgrade the type from int to float;
                 firstType = nFloat;
-                nf = n;
+                nf = n.value;
                 nf += vTmp.fpoint;
             } else
                 state.error<EvalError>("cannot add %1% to an integer", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow();
         } else if (firstType == nFloat) {
             if (vTmp.type() == nInt) {
-                nf += vTmp.integer;
+                nf += vTmp.integer.value;
             } else if (vTmp.type() == nFloat) {
                 nf += vTmp.fpoint;
             } else
                 state.error<EvalError>("cannot add %1% to a float", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow();
         } else {
-            if (s.empty()) s.reserve(es->size());
+            if (s.empty()) s.reserve(es.size());
             /* skip canonization of first path, which would only be not
             canonized in the first place if it's coming from a ./${foo} type
             path */
@@ -2187,7 +2130,7 @@ NixFloat EvalState::forceFloat(Value & v, const PosIdx pos, std::string_view err
     try {
         forceValue(v, pos);
         if (v.type() == nInt)
-            return v.integer;
+            return v.integer.value;
         else if (v.type() != nFloat)
             error<TypeError>(
                 "expected a float but found %1%: %2%",
@@ -2374,7 +2317,7 @@ BackedStringView EvalState::coerceToString(
            shell scripting convenience, just like `null'. */
         if (v.type() == nBool && v.boolean) return "1";
         if (v.type() == nBool && !v.boolean) return "";
-        if (v.type() == nInt) return std::to_string(v.integer);
+        if (v.type() == nInt) return std::to_string(v.integer.value);
         if (v.type() == nFloat) return std::to_string(v.fpoint);
         if (v.type() == nNull) return "";
 
@@ -2518,9 +2461,9 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
 
     // Special case type-compatibility between float and int
     if (v1.type() == nInt && v2.type() == nFloat)
-        return v1.integer == v2.fpoint;
+        return v1.integer.value == v2.fpoint;
     if (v1.type() == nFloat && v2.type() == nInt)
-        return v1.fpoint == v2.integer;
+        return v1.fpoint == v2.integer.value;
 
     // All other types are not compatible with each other.
     if (v1.type() != v2.type()) return false;
@@ -2752,39 +2695,39 @@ SourcePath resolveExprPath(SourcePath path)
 }
 
 
-Expr * EvalState::parseExprFromFile(const SourcePath & path)
+Expr & EvalState::parseExprFromFile(const SourcePath & path)
 {
     return parseExprFromFile(path, staticBaseEnv);
 }
 
 
-Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv)
+Expr & EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv)
 {
     auto buffer = path.readFile();
     // readFile hopefully have left some extra space for terminators
     buffer.append("\0\0", 2);
-    return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv);
+    return *parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
+Expr & EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv, const ExperimentalFeatureSettings & xpSettings)
 {
     // NOTE this method (and parseStdin) must take care to *fully copy* their input
     // into their respective Pos::Origin until the parser stops overwriting its input
     // data.
     auto s = make_ref<std::string>(s_);
     s_.append("\0\0", 2);
-    return parse(s_.data(), s_.size(), Pos::String{.source = s}, basePath, staticEnv);
+    return *parse(s_.data(), s_.size(), Pos::String{.source = s}, basePath, staticEnv, xpSettings);
 }
 
 
-Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath)
+Expr & EvalState::parseExprFromString(std::string s, const SourcePath & basePath, const ExperimentalFeatureSettings & xpSettings)
 {
-    return parseExprFromString(std::move(s), basePath, staticBaseEnv);
+    return parseExprFromString(std::move(s), basePath, staticBaseEnv, xpSettings);
 }
 
 
-Expr * EvalState::parseStdin()
+Expr & EvalState::parseStdin()
 {
     // NOTE this method (and parseExprFromString) must take care to *fully copy* their
     // input into their respective Pos::Origin until the parser stops overwriting its
@@ -2794,7 +2737,7 @@ Expr * EvalState::parseStdin()
     // drainFD should have left some extra space for terminators
     auto s = make_ref<std::string>(buffer);
     buffer.append("\0\0", 2);
-    return parse(buffer.data(), buffer.size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv);
+    return *parse(buffer.data(), buffer.size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv);
 }
 
 
@@ -2883,21 +2826,6 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
 }
 
 
-Expr * EvalState::parse(
-    char * text,
-    size_t length,
-    Pos::Origin origin,
-    const SourcePath & basePath,
-    std::shared_ptr<StaticEnv> & staticEnv)
-{
-    auto result = parseExprFromBuf(text, length, origin, basePath, symbols, positions, exprSymbols);
-
-    result->bindVars(*this, staticEnv);
-
-    return result;
-}
-
-
 std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
 {
     state.error<TypeError>(

src/libexpr/eval.hh

@@ -3,20 +3,19 @@
 
 #include "attr-set.hh"
 #include "eval-error.hh"
+#include "gc-alloc.hh"
 #include "types.hh"
 #include "value.hh"
 #include "nixexpr.hh"
 #include "symbol-table.hh"
 #include "config.hh"
 #include "experimental-features.hh"
-#include "input-accessor.hh"
 #include "search-path.hh"
 #include "repl-exit-status.hh"
 
 #include <map>
 #include <optional>
 #include <unordered_map>
-#include <mutex>
 #include <functional>
 
 namespace nix {
@@ -33,12 +32,15 @@ class EvalState;
 class StorePath;
 struct SingleDerivedPath;
 enum RepairFlag : bool;
-
+struct MemoryInputAccessor;
+namespace eval_cache {
+    class EvalCache;
+}
 
 /**
  * Function that implements a primop.
  */
-typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v);
+using PrimOpImpl = void(EvalState & state, PosIdx pos, Value ** args, Value & v);
 
 /**
  * Info about a primitive operation, and its implementation
@@ -72,7 +74,7 @@ struct PrimOp
     /**
      * Implementation of the primop.
      */
-    std::function<std::remove_pointer<PrimOpFun>::type> fun;
+    std::function<PrimOpImpl> fun;
 
     /**
      * Optional experimental for this to be gated on.
@@ -111,11 +113,7 @@ struct Constant
     bool impureOnly = false;
 };
 
-#if HAVE_BOEHMGC
-    typedef std::map<std::string, Value *, std::less<std::string>, traceable_allocator<std::pair<const std::string, Value *> > > ValMap;
-#else
-    typedef std::map<std::string, Value *> ValMap;
-#endif
+using ValMap = GcMap<std::string, Value *>;
 
 struct Env
 {
@@ -163,7 +161,10 @@ public:
     const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue,
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
-        sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
+        sRight, sWrong, sStructuredAttrs,
+        sAllowedReferences, sAllowedRequisites, sDisallowedReferences, sDisallowedRequisites,
+        sMaxSize, sMaxClosureSize,
+        sBuilder, sArgs,
         sContentAddressed, sImpure,
         sOutputHash, sOutputHashAlgo, sOutputHashMode,
         sRecurseForDerivations,
@@ -210,7 +211,7 @@ public:
     /**
      * Debugger
      */
-    ReplExitStatus (* debugRepl)(ref<EvalState> es, const ValMap & extraEnv);
+    std::function<ReplExitStatus(ref<EvalState> es, ValMap const & extraEnv)> debugRepl;
     bool debugStop;
     bool inDebugger = false;
     int trylevel;
@@ -234,6 +235,11 @@ public:
         return *new EvalErrorBuilder<T>(*this, args...);
     }
 
+    /**
+     * A cache for evaluation caches, so as to reuse the same root value if possible
+     */
+    std::map<const Hash, ref<eval_cache::EvalCache>> evalCaches;
+
 private:
 
     /* Cache for calls to addToStore(); maps source paths to the store
@@ -243,21 +249,13 @@ private:
     /**
      * A cache from path names to parse trees.
      */
-#if HAVE_BOEHMGC
-    typedef std::map<SourcePath, Expr *, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Expr *>>> FileParseCache;
-#else
-    typedef std::map<SourcePath, Expr *> FileParseCache;
-#endif
+    using FileParseCache = GcMap<SourcePath, Expr *>;
     FileParseCache fileParseCache;
 
     /**
      * A cache from path names to values.
      */
-#if HAVE_BOEHMGC
-    typedef std::map<SourcePath, Value, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Value>>> FileEvalCache;
-#else
-    typedef std::map<SourcePath, Value> FileEvalCache;
-#endif
+    using FileEvalCache = GcMap<SourcePath, Value>;
     FileEvalCache fileEvalCache;
 
     SearchPath searchPath;
@@ -340,16 +338,16 @@ public:
     /**
      * Parse a Nix expression from the specified file.
      */
-    Expr * parseExprFromFile(const SourcePath & path);
-    Expr * parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv);
+    Expr & parseExprFromFile(const SourcePath & path);
+    Expr & parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv);
 
     /**
      * Parse a Nix expression from the specified string.
      */
-    Expr * parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv);
-    Expr * parseExprFromString(std::string s, const SourcePath & basePath);
+    Expr & parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
+    Expr & parseExprFromString(std::string s, const SourcePath & basePath, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
-    Expr * parseStdin();
+    Expr & parseStdin();
 
     /**
      * Evaluate an expression read from the given file to normal
@@ -390,15 +388,15 @@ public:
      *
      * @param [out] v The resulting is stored here.
      */
-    void eval(Expr * e, Value & v);
+    void eval(Expr & e, Value & v);
 
     /**
      * Evaluation the expression, then verify that it has the expected
      * type.
      */
-    inline bool evalBool(Env & env, Expr * e);
-    inline bool evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx);
-    inline void evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx);
+    inline bool evalBool(Env & env, Expr & e);
+    inline bool evalBool(Env & env, Expr & e, const PosIdx pos, std::string_view errorCtx);
+    inline void evalAttrs(Env & env, Expr & e, Value & v, const PosIdx pos, std::string_view errorCtx);
 
     /**
      * If `v` is a thunk, enter it and overwrite `v` with the result
@@ -570,7 +568,8 @@ private:
         size_t length,
         Pos::Origin origin,
         const SourcePath & basePath,
-        std::shared_ptr<StaticEnv> & staticEnv);
+        std::shared_ptr<StaticEnv> & staticEnv,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
     /**
      * Current Nix call stack depth, used with `max-call-depth` setting to throw stack overflow hopefully before we run out of system stack.
@@ -619,7 +618,7 @@ public:
     }
 
     void mkList(Value & v, size_t length);
-    void mkThunk_(Value & v, Expr * expr);
+    void mkThunk_(Value & v, Expr & expr);
     void mkPos(Value & v, PosIdx pos);
 
     /**
@@ -728,15 +727,15 @@ private:
 
     bool countCalls;
 
-    typedef std::map<std::string, size_t> PrimOpCalls;
+    using PrimOpCalls = std::map<std::string, size_t>;
     PrimOpCalls primOpCalls;
 
-    typedef std::map<ExprLambda *, size_t> FunctionCalls;
+    using FunctionCalls = std::map<ExprLambda *, size_t>;
     FunctionCalls functionCalls;
 
     void incrFunctionCall(ExprLambda * fun);
 
-    typedef std::map<PosIdx, size_t> AttrSelects;
+    using AttrSelects = std::map<PosIdx, size_t>;
     AttrSelects attrSelects;
 
     friend struct ExprOpUpdate;
@@ -778,7 +777,7 @@ std::string showType(const Value & v);
  */
 SourcePath resolveExprPath(SourcePath path);
 
-static const std::string corepkgsPrefix{"/__corepkgs__/"};
+static constexpr std::string_view corepkgsPrefix{"/__corepkgs__/"};
 
 
 }

src/libexpr/flake/config.cc

@@ -51,8 +51,14 @@ void ConfigFile::apply()
         else
             assert(false);
 
-        if (!whitelist.count(baseName) && !nix::fetchSettings.acceptFlakeConfig) {
-            bool trusted = false;
+        bool trusted = whitelist.count(baseName);
+        if (!trusted) {
+            switch (nix::fetchSettings.acceptFlakeConfig.get()) {
+            case AcceptFlakeConfig::True: {
+                trusted = true;
+                break;
+            }
+            case AcceptFlakeConfig::Ask: {
                 auto trustedList = readTrustedList();
                 auto tlname = get(trustedList, name);
                 if (auto saved = tlname ? get(*tlname, valueS) : nullptr) {
@@ -60,21 +66,31 @@ void ConfigFile::apply()
                     printInfo("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name, valueS);
                 } else {
                     // FIXME: filter ANSI escapes, newlines, \r, etc.
-                if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) == 'y') {
+                    if (std::tolower(logger->ask(fmt("Do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)? This may allow the flake to gain root, see the nix.conf manual page.", name, valueS)).value_or('n')) == 'y') {
                         trusted = true;
+                    } else {
+                        warn("you can set '%s' to '%b' to automatically reject configuration options supplied by flakes", "accept-flake-config", false);
                     }
                     if (std::tolower(logger->ask(fmt("do you want to permanently mark this value as %s (y/N)?",  trusted ? "trusted": "untrusted" )).value_or('n')) == 'y') {
                         trustedList[name][valueS] = trusted;
                         writeTrustedList(trustedList);
                     }
                 }
-            if (!trusted) {
-                warn("ignoring untrusted flake configuration setting '%s'.\nPass '%s' to trust it", name, "--accept-flake-config");
-                continue;
+                break;
+            }
+            case nix::AcceptFlakeConfig::False: {
+                trusted = false;
+                break;
+            };
             }
         }
 
+        if (trusted) {
+            debug("accepting trusted flake configuration setting '%s'", name);
             globalConfig.set(name, valueS);
+        } else {
+            warn("ignoring untrusted flake configuration setting '%s', pass '%s' to trust it (may allow the flake to gain root, see the nix.conf manual page)", name, "--accept-flake-config");
+        }
     }
 }
 

src/libexpr/flake/flake.cc

@@ -137,9 +137,16 @@ static FlakeInput parseFlakeInput(EvalState & state,
                     case nBool:
                         attrs.emplace(state.symbols[attr.name], Explicit<bool> { attr.value->boolean });
                         break;
-                    case nInt:
-                        attrs.emplace(state.symbols[attr.name], (long unsigned int)attr.value->integer);
+                    case nInt: {
+                        auto intValue = attr.value->integer.value;
+
+                        if (intValue < 0) {
+                            state.error<EvalError>("negative value given for flake input attribute %1%: %2%", state.symbols[attr.name], intValue).debugThrow();
+                        }
+                        uint64_t asUnsigned = intValue;
+                        attrs.emplace(state.symbols[attr.name], asUnsigned);
                         break;
+                    }
                     default:
                         state.error<TypeError>("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
                             state.symbols[attr.name], showType(*attr.value)).debugThrow();
@@ -284,7 +291,7 @@ static Flake getFlake(
             else if (setting.value->type() == nInt)
                 flake.config.settings.emplace(
                     state.symbols[setting.name],
-                    state.forceInt(*setting.value, setting.pos, ""));
+                    state.forceInt(*setting.value, setting.pos, "").value);
             else if (setting.value->type() == nBool)
                 flake.config.settings.emplace(
                     state.symbols[setting.name],
@@ -873,8 +880,14 @@ static void prim_flakeRefToString(
     for (const auto & attr : *args[0]->attrs) {
         auto t = attr.value->type();
         if (t == nInt) {
-            attrs.emplace(state.symbols[attr.name],
-                          (uint64_t) attr.value->integer);
+            auto intValue = attr.value->integer.value;
+
+            if (intValue < 0) {
+                state.error<EvalError>("negative value given for flake ref attr %1%: %2%", state.symbols[attr.name], intValue).atPos(pos).debugThrow();
+            }
+            uint64_t asUnsigned = intValue;
+
+            attrs.emplace(state.symbols[attr.name], asUnsigned);
         } else if (t == nBool) {
             attrs.emplace(state.symbols[attr.name],
                           Explicit<bool> { attr.value->boolean });
@@ -924,7 +937,7 @@ Fingerprint LockedFlake::getFingerprint() const
     // FIXME: as an optimization, if the flake contains a lock file
     // and we haven't changed it, then it's sufficient to use
     // flake.sourceInfo.storePath for the fingerprint.
-    return hashString(htSHA256,
+    return hashString(HashType::SHA256,
         fmt("%s;%s;%d;%d;%s",
             flake.sourceInfo->storePath.to_string(),
             flake.lockedRef.subdir,

src/libexpr/flake/flakeref.cc

@@ -85,8 +85,8 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
         + "(?:#(" + queryRegex + "))?",
         std::regex::ECMAScript);
 
-    static std::regex flakeRegex(
-        "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)"
+    static std::regex flakeShorthandRegex(
+        flakeShorthandRegexS
         + "(?:#(" + queryRegex + "))?",
         std::regex::ECMAScript);
 
@@ -95,7 +95,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
     /* Check if 'url' is a flake ID. This is an abbreviated syntax for
        'flake:<flake-id>?ref=<ref>&rev=<rev>'. */
 
-    if (std::regex_match(url, match, flakeRegex)) {
+    if (std::regex_match(url, match, flakeShorthandRegex)) {
         auto parsedURL = ParsedURL{
             .url = url,
             .base = "flake:" + match.str(1),
@@ -169,14 +169,13 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
                         if (subdir != "") {
                             if (parsedURL.query.count("dir"))
                                 throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
-                            parsedURL.query.insert_or_assign("dir", subdir);
                         }
 
                         if (pathExists(flakeRoot + "/.git/shallow"))
                             parsedURL.query.insert_or_assign("shallow", "1");
 
                         return std::make_pair(
-                            FlakeRef(Input::fromURL(parsedURL, isFlake), getOr(parsedURL.query, "dir", "")),
+                            FlakeRef(Input::fromURL(parsedURL, isFlake), subdir),
                             fragment);
                     }
 
@@ -204,7 +203,13 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
         std::string fragment;
         std::swap(fragment, parsedURL.fragment);
 
-        auto input = Input::fromURL(parsedURL, isFlake);
+        // This has a special meaning for flakes and must not be passed to libfetchers.
+        // Of course this means that libfetchers cannot have fetchers
+        // expecting an argument `dir` 🫠
+        ParsedURL urlForFetchers(parsedURL);
+        urlForFetchers.query.erase("dir");
+
+        auto input = Input::fromURL(urlForFetchers, isFlake);
         input.parent = baseDir;
 
         return std::make_pair(

src/libexpr/gc-alloc.cc

@@ -0,0 +1,23 @@
+#include "gc-alloc.hh"
+
+#include <cstring>
+#include <string_view>
+
+namespace nix
+{
+
+char const * gcCopyStringIfNeeded(std::string_view toCopyFrom)
+{
+    if (toCopyFrom.empty()) {
+        return "";
+    }
+
+    size_t const size = toCopyFrom.size();
+    char * cstr = gcAllocString(size + 1);
+    memcpy(cstr, toCopyFrom.data(), size);
+    cstr[size] = '\0';
+
+    return cstr;
+}
+
+}

src/libexpr/gc-alloc.hh

@@ -0,0 +1,150 @@
+#pragma once
+/// @file Aliases and wrapper functions that are transparently GC-enabled
+/// if Lix is compiled with BoehmGC enabled.
+
+#include <cstddef>
+#include <cstring>
+#include <list>
+#include <map>
+#include <new>
+#include <string_view>
+#include <vector>
+
+#include "checked-arithmetic.hh"
+
+#if HAVE_BOEHMGC
+#include <functional> // std::less
+#include <utility> // std::pair
+#define GC_INCLUDE_NEW
+#include <gc/gc.h>
+#include <gc/gc_allocator.h>
+#include <gc/gc_cpp.h>
+
+/// calloc, transparently GC-enabled.
+#define LIX_GC_CALLOC(size) GC_MALLOC(size)
+
+/// strdup, transaprently GC-enabled.
+#define LIX_GC_STRDUP(str) GC_STRDUP(str)
+
+/// Atomic GC malloc() with GC enabled, or regular malloc() otherwise.
+#define LIX_GC_MALLOC_ATOMIC(size) GC_MALLOC_ATOMIC(size)
+
+namespace nix
+{
+
+/// Alias for std::map which uses BoehmGC's allocator conditional on this Lix
+/// build having GC enabled.
+template<typename KeyT, typename ValueT>
+using GcMap = std::map<
+    KeyT,
+    ValueT,
+    std::less<KeyT>,
+    traceable_allocator<std::pair<KeyT const, ValueT>>
+>;
+
+/// Alias for std::vector which uses BoehmGC's allocator conditional on this Lix
+/// build having GC enabled.
+template<typename ItemT>
+using GcVector = std::vector<ItemT, traceable_allocator<ItemT>>;
+
+/// Alias for std::list which uses BoehmGC's allocator conditional on this Lix
+/// build having GC enabled.
+template<typename ItemT>
+using GcList = std::list<ItemT, traceable_allocator<ItemT>>;
+
+}
+
+#else
+
+#include <cstdlib>
+
+/// calloc, transparently GC-enabled.
+#define LIX_GC_CALLOC(size) calloc(size, 1)
+
+/// strdup, transparently GC-enabled.
+#define LIX_GC_STRDUP(str) strdup(str)
+
+/// Atomic GC malloc() with GC enabled, or regular malloc() otherwise.
+/// The returned memory must never contain pointers.
+#define LIX_GC_MALLOC_ATOMIC(size) malloc(size)
+
+namespace nix
+{
+
+/// Alias for std::map which uses BoehmGC's allocator conditional on this Lix
+/// build having GC enabled.
+template<typename KeyT, typename ValueT>
+using GcMap = std::map<KeyT, ValueT>;
+
+/// Alias for std::vector which uses BoehmGC's allocator conditional on this Lix
+/// build having GC enabled.
+template<typename ItemT>
+using GcVector = std::vector<ItemT>;
+
+/// Alias for std::list which uses BoehmGC's allocator conditional on this Lix
+/// build having GC enabled.
+template<typename ItemT>
+using GcList = std::list<ItemT>;
+
+}
+
+#endif
+
+namespace nix
+{
+
+[[gnu::always_inline]]
+inline void * gcAllocBytes(size_t n)
+{
+    // Note: various places expect the allocated memory to be zero.
+    // Hence: calloc().
+    void * ptr = LIX_GC_CALLOC(n);
+    if (ptr == nullptr) {
+        throw std::bad_alloc();
+    }
+
+    return ptr;
+}
+
+/// Typed, safe wrapper around calloc() (transparently GC-enabled). Allocates
+/// enough for the requested count of the specified type. Also checks for
+/// nullptr (and throws @ref std::bad_alloc), and casts the void pointer to
+/// a pointer of the specified type, for type-convenient goodness.
+template<typename T>
+[[gnu::always_inline]]
+inline T * gcAllocType(size_t howMany = 1)
+{
+    // NOTE: size_t * size_t, which can definitely overflow.
+    // Unsigned integer overflow is definitely a bug, but isn't undefined
+    // behavior, so we can just check if we overflowed after the fact.
+    // However, people can and do request zero sized allocations, so we need
+    // to check that neither of our multiplicands were zero before complaining
+    // about it.
+    auto checkedSz = checked::Checked<size_t>(howMany) * sizeof(T);
+    size_t sz = checkedSz.valueWrapping();
+    if (checkedSz.overflowed()) {
+        // Congrats, you done did an overflow.
+        throw std::bad_alloc();
+    }
+
+    return static_cast<T *>(gcAllocBytes(sz));
+}
+
+/// GC-transparently allocates a buffer for a C-string of @ref size *bytes*,
+/// meaning you should include the size needed by the NUL terminator in the
+/// passed size. Memory allocated with this function must never contain other
+/// pointers.
+inline char * gcAllocString(size_t size)
+{
+    char * cstr = static_cast<char *>(LIX_GC_MALLOC_ATOMIC(size));
+    if (cstr == nullptr) {
+        throw std::bad_alloc();
+    }
+    return cstr;
+}
+
+/// Returns a C-string copied from @ref toCopyFrom, or a single, static empty
+/// string if @ref toCopyFrom is also empty.
+char const * gcCopyStringIfNeeded(std::string_view toCopyFrom);
+
+}

src/libexpr/get-drvs.cc

@@ -1,6 +1,7 @@
 #include "get-drvs.hh"
 #include "eval-inline.hh"
 #include "derivations.hh"
+#include "eval.hh"
 #include "store-api.hh"
 #include "path-with-outputs.hh"
 
@@ -45,7 +46,7 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
 }
 
 
-std::string DrvInfo::queryName() const
+std::string DrvInfo::queryName()
 {
     if (name == "" && attrs) {
         auto i = attrs->find(state->sName);
@@ -56,7 +57,7 @@ std::string DrvInfo::queryName() const
 }
 
 
-std::string DrvInfo::querySystem() const
+std::string DrvInfo::querySystem()
 {
     if (system == "" && attrs) {
         auto i = attrs->find(state->sSystem);
@@ -66,7 +67,7 @@ std::string DrvInfo::querySystem() const
 }
 
 
-std::optional<StorePath> DrvInfo::queryDrvPath() const
+std::optional<StorePath> DrvInfo::queryDrvPath()
 {
     if (!drvPath && attrs) {
         Bindings::iterator i = attrs->find(state->sDrvPath);
@@ -80,7 +81,7 @@ std::optional<StorePath> DrvInfo::queryDrvPath() const
 }
 
 
-StorePath DrvInfo::requireDrvPath() const
+StorePath DrvInfo::requireDrvPath()
 {
     if (auto drvPath = queryDrvPath())
         return *drvPath;
@@ -88,7 +89,7 @@ StorePath DrvInfo::requireDrvPath() const
 }
 
 
-StorePath DrvInfo::queryOutPath() const
+StorePath DrvInfo::queryOutPath()
 {
     if (!outPath && attrs) {
         Bindings::iterator i = attrs->find(state->sOutPath);
@@ -101,51 +102,120 @@ StorePath DrvInfo::queryOutPath() const
     return *outPath;
 }
 
+void DrvInfo::fillOutputs(bool withPaths)
+{
+    auto fillDefault = [&]() {
+        std::optional<StorePath> outPath = std::nullopt;
+        if (withPaths) {
+            outPath.emplace(this->queryOutPath());
+        }
+        this->outputs.emplace("out", outPath);
+    };
+
+    // lol. lmao even.
+    if (this->attrs == nullptr) {
+        fillDefault();
+        return;
+    }
+
+    Attr * outputs = this->attrs->get(this->state->sOutputs);
+    if (outputs == nullptr) {
+        fillDefault();
+        return;
+    }
+
+    // NOTE(Qyriad): I don't think there is any codepath that can cause this to error.
+    this->state->forceList(
+        *outputs->value,
+        outputs->pos,
+        "while evaluating the 'outputs' attribute of a derivation"
+    );
+
+    for (auto [idx, elem] : enumerate(outputs->value->listItems())) {
+        // NOTE(Qyriad): This error should be *extremely* rare in practice.
+        // It is impossible to construct with `stdenv.mkDerivation`,
+        // `builtins.derivation`, or even `derivationStrict`. As far as we can tell,
+        // it is only possible by overriding a derivation attrset already created by
+        // one of those with `//` to introduce the failing `outputs` entry.
+        auto errMsg = fmt("while evaluating output %d of a derivation", idx);
+        std::string_view outputName = state->forceStringNoCtx(
+            *elem,
+            outputs->pos,
+            errMsg
+        );
+
+        if (withPaths) {
+            // Find the attr with this output's name...
+            Attr * out = this->attrs->get(this->state->symbols.create(outputName));
+            if (out == nullptr) {
+                // FIXME: throw error?
+                continue;
+            }
+
+            // Meanwhile we couldn't figure out any circumstances
+            // that cause this to error.
+            state->forceAttrs(*out->value, outputs->pos, errMsg);
+
+            // ...and evaluate its `outPath` attribute.
+            Attr * outPath = out->value->attrs->get(this->state->sOutPath);
+            if (outPath == nullptr) {
+                continue;
+                // FIXME: throw error?
+            }
+
+            NixStringContext context;
+            // And idk what could possibly cause this one to error
+            // that wouldn't error before here.
+            auto storePath = state->coerceToStorePath(
+                outPath->pos,
+                *outPath->value,
+                context,
+                errMsg
+            );
+            this->outputs.emplace(outputName, storePath);
+        } else {
+            this->outputs.emplace(outputName, std::nullopt);
+        }
+    }
+}
 
 DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall)
 {
+    // If we haven't already cached the outputs set, then do so now.
     if (outputs.empty()) {
-        /* Get the ‘outputs’ list. */
-        Bindings::iterator i;
-        if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) {
-            state->forceList(*i->value, i->pos, "while evaluating the 'outputs' attribute of a derivation");
-
-            /* For each output... */
-            for (auto elem : i->value->listItems()) {
-                std::string output(state->forceStringNoCtx(*elem, i->pos, "while evaluating the name of an output of a derivation"));
-
-                if (withPaths) {
-                    /* Evaluate the corresponding set. */
-                    Bindings::iterator out = attrs->find(state->symbols.create(output));
-                    if (out == attrs->end()) continue; // FIXME: throw error?
-                    state->forceAttrs(*out->value, i->pos, "while evaluating an output of a derivation");
-
-                    /* And evaluate its ‘outPath’ attribute. */
-                    Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
-                    if (outPath == out->value->attrs->end()) continue; // FIXME: throw error?
-                    NixStringContext context;
-                    outputs.emplace(output, state->coerceToStorePath(outPath->pos, *outPath->value, context, "while evaluating an output path of a derivation"));
-                } else
-                    outputs.emplace(output, std::nullopt);
-            }
-        } else
-            outputs.emplace("out", withPaths ? std::optional{queryOutPath()} : std::nullopt);
+        // FIXME: this behavior seems kind of busted, since whether or not this
+        // DrvInfo will have paths is forever determined by the *first* call to
+        // this function??
+        fillOutputs(withPaths);
     }
 
+    // Things that operate on derivations like packages, like `nix-env` and `nix build`,
+    // allow derivations to specify which outputs should be used in those user-facing
+    // cases if the user didn't specify an output explicitly.
+    // If the caller just wanted all the outputs for this derivation, though,
+    // then we're done here.
     if (!onlyOutputsToInstall || !attrs)
         return outputs;
 
-    Bindings::iterator i;
-    if (attrs && (i = attrs->find(state->sOutputSpecified)) != attrs->end() && state->forceBool(*i->value, i->pos, "while evaluating the 'outputSpecified' attribute of a derivation")) {
-        Outputs result;
-        auto out = outputs.find(queryOutputName());
-        if (out == outputs.end())
+    // Regardless of `meta.outputsToInstall`, though, you can select into a derivation
+    // output by its attribute, e.g. `pkgs.lix.dev`, which (lol?) sets the magic
+    // attribute `outputSpecified = true`, and changes the `outputName` attr to the
+    // explicitly selected-into output.
+    if (Attr * outSpecAttr = attrs->get(state->sOutputSpecified)) {
+        bool outputSpecified = this->state->forceBool(
+            *outSpecAttr->value,
+            outSpecAttr->pos,
+            "while evaluating the 'outputSpecified' attribute of a derivation"
+        );
+        if (outputSpecified) {
+            auto maybeOut = outputs.find(queryOutputName());
+            if (maybeOut == outputs.end()) {
                 throw Error("derivation does not have output '%s'", queryOutputName());
-        result.insert(*out);
-        return result;
+            }
+            return Outputs{*maybeOut};
+        }
     }
 
-    else {
     /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */
     const Value * outTI = queryMeta("outputsToInstall");
     if (!outTI) return outputs;
@@ -161,10 +231,9 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall
     }
     return result;
 }
-}
 
 
-std::string DrvInfo::queryOutputName() const
+std::string DrvInfo::queryOutputName()
 {
     if (outputName == "" && attrs) {
         Bindings::iterator i = attrs->find(state->sOutputName);
@@ -241,8 +310,8 @@ NixInt DrvInfo::queryMetaInt(const std::string & name, NixInt def)
     if (v->type() == nString) {
         /* Backwards compatibility with before we had support for
            integer meta fields. */
-        if (auto n = string2Int<NixInt>(v->string.s))
-            return *n;
+        if (auto n = string2Int<NixInt::Inner>(v->string.s))
+            return NixInt{*n};
     }
     return def;
 }
@@ -350,15 +419,47 @@ static void getDerivations(EvalState & state, Value & vIn,
     Value v;
     state.autoCallFunction(autoArgs, vIn, v);
 
-    /* Process the expression. */
-    if (!getDerivation(state, v, pathPrefix, drvs, ignoreAssertionFailures)) ;
+    bool shouldRecurse = getDerivation(state, v, pathPrefix, drvs, ignoreAssertionFailures);
+    if (!shouldRecurse) {
+        // We're done here.
+        return;
+    }
 
-    else if (v.type() == nAttrs) {
+    if (v.type() == nList) {
+        // NOTE we can't really deduplicate here because small lists don't have stable addresses
+        // and can cause spurious duplicate detections due to v being on the stack.
+        for (auto [n, elem] : enumerate(v.listItems())) {
+            std::string joinedAttrPath = addToPath(pathPrefix, fmt("%d", n));
+            bool shouldRecurse = getDerivation(state, *elem, joinedAttrPath, drvs, ignoreAssertionFailures);
+            if (shouldRecurse) {
+                getDerivations(
+                    state,
+                    *elem,
+                    joinedAttrPath,
+                    autoArgs,
+                    drvs,
+                    done,
+                    ignoreAssertionFailures
+                );
+            }
+        }
+
+        return;
+    } else if (v.type() != nAttrs) {
+        state.error<TypeError>(
+            "expression was expected to be a derivation or collection of derivations, but instead was %s",
+            showType(v.type(), true)
+        ).debugThrow();
+    }
 
     /* Dont consider sets we've already seen, e.g. y in
        `rec { x.d = derivation {...}; y = x; }`. */
-        if (!done.insert(v.attrs).second) return;
+    auto const &[_, didInsert] = done.insert(v.attrs);
+    if (!didInsert) {
+        return;
+    }
 
+    // FIXME: what the fuck???
     /* !!! undocumented hackery to support combining channels in
        nix-env.cc. */
     bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end();
@@ -368,38 +469,45 @@ static void getDerivations(EvalState & state, Value & vIn,
        there are names clashes between derivations, the derivation
        bound to the attribute with the "lower" name should take
        precedence). */
-        for (auto & i : v.attrs->lexicographicOrder(state.symbols)) {
-            debug("evaluating attribute '%1%'", state.symbols[i->name]);
-            if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex))
+    for (auto & attr : v.attrs->lexicographicOrder(state.symbols)) {
+        debug("evaluating attribute '%1%'", state.symbols[attr->name]);
+        // FIXME: only consider attrs with identifier-like names?? Why???
+        if (!std::regex_match(std::string(state.symbols[attr->name]), attrRegex)) {
             continue;
-            std::string pathPrefix2 = addToPath(pathPrefix, state.symbols[i->name]);
-            if (combineChannels)
-                getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
-            else if (getDerivation(state, *i->value, pathPrefix2, drvs, ignoreAssertionFailures)) {
+        }
+        std::string joinedAttrPath = addToPath(pathPrefix, state.symbols[attr->name]);
+        if (combineChannels) {
+            getDerivations(state, *attr->value, joinedAttrPath, autoArgs, drvs, done, ignoreAssertionFailures);
+        } else if (getDerivation(state, *attr->value, joinedAttrPath, drvs, ignoreAssertionFailures)) {
             /* If the value of this attribute is itself a set,
                should we recurse into it?  => Only if it has a
                `recurseForDerivations = true' attribute. */
-                if (i->value->type() == nAttrs) {
-                    Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations);
-                    if (j != i->value->attrs->end() && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`"))
-                        getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
-                }
-            }
+            if (attr->value->type() == nAttrs) {
+                Attr * recurseForDrvs = attr->value->attrs->get(state.sRecurseForDerivations);
+                if (recurseForDrvs == nullptr) {
+                    continue;
                 }
+                bool shouldRecurse = state.forceBool(
+                    *recurseForDrvs->value,
+                    attr->pos,
+                    fmt("while evaluating the '%s' attribute", Magenta("recurseForDerivations"))
+                );
+                if (!shouldRecurse) {
+                    continue;
                 }
 
-    else if (v.type() == nList) {
-        // NOTE we can't really deduplicate here because small lists don't have stable addresses
-        // and can cause spurious duplicate detections due to v being on the stack.
-        for (auto [n, elem] : enumerate(v.listItems())) {
-            std::string pathPrefix2 = addToPath(pathPrefix, fmt("%d", n));
-            if (getDerivation(state, *elem, pathPrefix2, drvs, ignoreAssertionFailures))
-                getDerivations(state, *elem, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+                getDerivations(
+                    state,
+                    *attr->value,
+                    joinedAttrPath,
+                    autoArgs,
+                    drvs,
+                    done,
+                    ignoreAssertionFailures
+                );
+            }
         }
     }
-
-    else
-        state.error<TypeError>("expression does not evaluate to a derivation (or a set or list of those)").debugThrow();
 }
 
 

src/libexpr/get-drvs.hh

@@ -19,11 +19,11 @@ public:
 private:
     EvalState * state;
 
-    mutable std::string name;
-    mutable std::string system;
-    mutable std::optional<std::optional<StorePath>> drvPath;
-    mutable std::optional<StorePath> outPath;
-    mutable std::string outputName;
+    std::string name;
+    std::string system;
+    std::optional<std::optional<StorePath>> drvPath;
+    std::optional<StorePath> outPath;
+    std::string outputName;
     Outputs outputs;
 
     /**
@@ -37,22 +37,23 @@ private:
 
     bool checkMeta(Value & v);
 
+    void fillOutputs(bool withPaths = true);
+
 public:
     /**
      * path towards the derivation
      */
     std::string attrPath;
 
-    DrvInfo(EvalState & state) : state(&state) { };
     DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs);
     DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs);
 
-    std::string queryName() const;
-    std::string querySystem() const;
-    std::optional<StorePath> queryDrvPath() const;
-    StorePath requireDrvPath() const;
-    StorePath queryOutPath() const;
-    std::string queryOutputName() const;
+    std::string queryName();
+    std::string querySystem();
+    std::optional<StorePath> queryDrvPath();
+    StorePath requireDrvPath();
+    StorePath queryOutPath();
+    std::string queryOutputName();
     /**
      * Return the unordered map of output names to (optional) output paths.
      * The "outputs to install" are determined by `meta.outputsToInstall`.
@@ -80,13 +81,7 @@ public:
     bool hasFailed() { return failed; };
 };
 
-
-#if HAVE_BOEHMGC
-typedef std::list<DrvInfo, traceable_allocator<DrvInfo>> DrvInfos;
-#else
-typedef std::list<DrvInfo> DrvInfos;
-#endif
-
+using DrvInfos = GcList<DrvInfo>;
 
 /**
  * If value `v` denotes a derivation, return a DrvInfo object

src/libexpr/json-to-value.cc

@@ -2,6 +2,7 @@
 #include "value.hh"
 #include "eval.hh"
 
+#include <limits>
 #include <variant>
 #include <nlohmann/json.hpp>
 
@@ -81,42 +82,46 @@ class JSONSax : nlohmann::json_sax<json> {
 public:
     JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {};
 
-    bool null()
+    bool null() override
     {
         rs->value(state).mkNull();
         rs->add();
         return true;
     }
 
-    bool boolean(bool val)
+    bool boolean(bool val) override
     {
         rs->value(state).mkBool(val);
         rs->add();
         return true;
     }
 
-    bool number_integer(number_integer_t val)
+    bool number_integer(number_integer_t val) override
     {
         rs->value(state).mkInt(val);
         rs->add();
         return true;
     }
 
-    bool number_unsigned(number_unsigned_t val)
+    bool number_unsigned(number_unsigned_t val_) override
     {
+        if (val_ > std::numeric_limits<NixInt::Inner>::max()) {
+            throw Error("unsigned json number %1% outside of Nix integer range", val_);
+        }
+        NixInt::Inner val = val_;
         rs->value(state).mkInt(val);
         rs->add();
         return true;
     }
 
-    bool number_float(number_float_t val, const string_t & s)
+    bool number_float(number_float_t val, const string_t & s) override
     {
         rs->value(state).mkFloat(val);
         rs->add();
         return true;
     }
 
-    bool string(string_t & val)
+    bool string(string_t & val) override
     {
         rs->value(state).mkString(val);
         rs->add();
@@ -124,7 +129,7 @@ public:
     }
 
 #if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8
-    bool binary(binary_t&)
+    bool binary(binary_t&) override
     {
         // This function ought to be unreachable
         assert(false);
@@ -132,35 +137,37 @@ public:
     }
 #endif
 
-    bool start_object(std::size_t len)
+    bool start_object(std::size_t len) override
     {
         rs = std::make_unique<JSONObjectState>(std::move(rs));
         return true;
     }
 
-    bool key(string_t & name)
+    bool key(string_t & name) override
     {
         dynamic_cast<JSONObjectState*>(rs.get())->key(name, state);
         return true;
     }
 
-    bool end_object() {
+    bool end_object() override {
         rs = rs->resolve(state);
         rs->add();
         return true;
     }
 
-    bool end_array() {
+    bool end_array() override {
         return end_object();
     }
 
-    bool start_array(size_t len) {
+    bool start_array(size_t len) override {
         rs = std::make_unique<JSONListState>(std::move(rs),
             len != std::numeric_limits<size_t>::max() ? len : 128);
         return true;
     }
 
-    bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) {
+    bool
+    parse_error(std::size_t, const std::string &, const nlohmann::detail::exception & ex) override
+    {
         throw JSONParseError("%s", ex.what());
     }
 };

src/libexpr/lexer.l

@@ -1,296 +0,0 @@
-%option reentrant bison-bridge bison-locations
-%option align
-%option noyywrap
-%option never-interactive
-%option stack
-%option nodefault
-%option nounput noyy_top_state
-
-
-%s DEFAULT
-%x STRING
-%x IND_STRING
-%x INPATH
-%x INPATH_SLASH
-%x PATH_START
-
-
-%{
-#ifdef __clang__
-#pragma clang diagnostic ignored "-Wunneeded-internal-declaration"
-#endif
-
-// yacc generates code that uses unannotated fallthrough.
-#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
-#ifdef __clang__
-#pragma clang diagnostic ignored "-Wimplicit-fallthrough"
-#endif
-
-#include "nixexpr.hh"
-#include "parser-tab.hh"
-#include "strings.hh"
-
-using namespace nix;
-
-namespace nix {
-
-#define CUR_POS state->at(*yylloc)
-
-static void initLoc(YYLTYPE * loc)
-{
-    loc->first_line = loc->last_line = 0;
-    loc->first_column = loc->last_column = 0;
-}
-
-static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
-{
-    loc->stash();
-
-    loc->first_column = loc->last_column;
-    loc->last_column += len;
-}
-
-
-// we make use of the fact that the parser receives a private copy of the input
-// string and can munge around in it.
-static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
-{
-    char * result = s;
-    char * t = s;
-    char c;
-    // the input string is terminated with *two* NULs, so we can safely take
-    // *one* character after the one being checked against.
-    while ((c = *s++)) {
-        if (c == '\\') {
-            c = *s++;
-            if (c == 'n') *t = '\n';
-            else if (c == 'r') *t = '\r';
-            else if (c == 't') *t = '\t';
-            else *t = c;
-        }
-        else if (c == '\r') {
-            /* Normalise CR and CR/LF into LF. */
-            *t = '\n';
-            if (*s == '\n') s++; /* cr/lf */
-        }
-        else *t = c;
-        t++;
-    }
-    return {result, size_t(t - result)};
-}
-
-
-}
-
-#define YY_USER_INIT initLoc(yylloc)
-#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng);
-
-#define PUSH_STATE(state) yy_push_state(state, yyscanner)
-#define POP_STATE() yy_pop_state(yyscanner)
-
-%}
-
-
-ANY         .|\n
-ID          [a-zA-Z\_][a-zA-Z0-9\_\'\-]*
-INT         [0-9]+
-FLOAT       (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)?
-PATH_CHAR   [a-zA-Z0-9\.\_\-\+]
-PATH        {PATH_CHAR}*(\/{PATH_CHAR}+)+\/?
-PATH_SEG    {PATH_CHAR}*\/
-HPATH       \~(\/{PATH_CHAR}+)+\/?
-HPATH_START \~\/
-SPATH       \<{PATH_CHAR}+(\/{PATH_CHAR}+)*\>
-URI         [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+
-
-
-%%
-
-
-if          { return IF; }
-then        { return THEN; }
-else        { return ELSE; }
-assert      { return ASSERT; }
-with        { return WITH; }
-let         { return LET; }
-in          { return IN; }
-rec         { return REC; }
-inherit     { return INHERIT; }
-or          { return OR_KW; }
-\.\.\.      { return ELLIPSIS; }
-
-\=\=        { return EQ; }
-\!\=        { return NEQ; }
-\<\=        { return LEQ; }
-\>\=        { return GEQ; }
-\&\&        { return AND; }
-\|\|        { return OR; }
-\-\>        { return IMPL; }
-\/\/        { return UPDATE; }
-\+\+        { return CONCAT; }
-
-{ID}        { yylval->id = {yytext, (size_t) yyleng}; return ID; }
-{INT}       { errno = 0;
-              std::optional<int64_t> numMay = string2Int<int64_t>(yytext);
-              if (numMay.has_value()) {
-                  yylval->n = *numMay;
-              } else {
-                  throw ParseError(ErrorInfo{
-                      .msg = HintFmt("invalid integer '%1%'", yytext),
-                      .pos = state->positions[CUR_POS],
-                  });
-              }
-              return INT;
-            }
-{FLOAT}     { errno = 0;
-              yylval->nf = strtod(yytext, 0);
-              if (errno != 0)
-                  throw ParseError(ErrorInfo{
-                      .msg = HintFmt("invalid float '%1%'", yytext),
-                      .pos = state->positions[CUR_POS],
-                  });
-              return FLOAT;
-            }
-
-\$\{        { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
-
-\}          { /* State INITIAL only exists at the bottom of the stack and is
-                 used as a marker. DEFAULT replaces it everywhere else.
-                 Popping when in INITIAL state causes an empty stack exception,
-                 so don't */
-              if (YYSTATE != INITIAL)
-                POP_STATE();
-              return '}';
-            }
-\{          { PUSH_STATE(DEFAULT); return '{'; }
-
-\"          { PUSH_STATE(STRING); return '"'; }
-<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" |
-<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ {
-                /* It is impossible to match strings ending with '$' with one
-                   regex because trailing contexts are only valid at the end
-                   of a rule. (A sane but undocumented limitation.) */
-                yylval->str = unescapeStr(state->symbols, yytext, yyleng);
-                return STR;
-              }
-<STRING>\$\{  { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
-<STRING>\"    { POP_STATE(); return '"'; }
-<STRING>\$|\\|\$\\ {
-                /* This can only occur when we reach EOF, otherwise the above
-                   (...|\$[^\{\"\\]|\\.|\$\\.)+ would have triggered.
-                   This is technically invalid, but we leave the problem to the
-                   parser who fails with exact location. */
-                return EOF;
-              }
-
-\'\'(\ *\n)?     { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
-<IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ {
-                   yylval->str = {yytext, (size_t) yyleng, true};
-                   return IND_STR;
-                 }
-<IND_STRING>\'\'\$ |
-<IND_STRING>\$   {
-                   yylval->str = {"$", 1};
-                   return IND_STR;
-                 }
-<IND_STRING>\'\'\' {
-                   yylval->str = {"''", 2};
-                   return IND_STR;
-                 }
-<IND_STRING>\'\'\\{ANY} {
-                   yylval->str = unescapeStr(state->symbols, yytext + 2, yyleng - 2);
-                   return IND_STR;
-                 }
-<IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
-<IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; }
-<IND_STRING>\'   {
-                   yylval->str = {"'", 1};
-                   return IND_STR;
-                 }
-
-{PATH_SEG}\$\{ |
-{HPATH_START}\$\{ {
-  PUSH_STATE(PATH_START);
-  yyless(0);
-  yylloc->unstash();
-}
-
-<PATH_START>{PATH_SEG} {
-  POP_STATE();
-  PUSH_STATE(INPATH_SLASH);
-  yylval->path = {yytext, (size_t) yyleng};
-  return PATH;
-}
-
-<PATH_START>{HPATH_START} {
-  POP_STATE();
-  PUSH_STATE(INPATH_SLASH);
-  yylval->path = {yytext, (size_t) yyleng};
-  return HPATH;
-}
-
-{PATH} {
-  if (yytext[yyleng-1] == '/')
-    PUSH_STATE(INPATH_SLASH);
-  else
-    PUSH_STATE(INPATH);
-  yylval->path = {yytext, (size_t) yyleng};
-  return PATH;
-}
-{HPATH} {
-  if (yytext[yyleng-1] == '/')
-    PUSH_STATE(INPATH_SLASH);
-  else
-    PUSH_STATE(INPATH);
-  yylval->path = {yytext, (size_t) yyleng};
-  return HPATH;
-}
-
-<INPATH,INPATH_SLASH>\$\{ {
-  POP_STATE();
-  PUSH_STATE(INPATH);
-  PUSH_STATE(DEFAULT);
-  return DOLLAR_CURLY;
-}
-<INPATH,INPATH_SLASH>{PATH}|{PATH_SEG}|{PATH_CHAR}+ {
-  POP_STATE();
-  if (yytext[yyleng-1] == '/')
-      PUSH_STATE(INPATH_SLASH);
-  else
-      PUSH_STATE(INPATH);
-  yylval->str = {yytext, (size_t) yyleng};
-  return STR;
-}
-<INPATH>{ANY} |
-<INPATH><<EOF>> {
-  /* if we encounter a non-path character we inform the parser that the path has
-     ended with a PATH_END token and re-parse this character in the default
-     context (it may be ')', ';', or something of that sort) */
-  POP_STATE();
-  yyless(0);
-  yylloc->unstash();
-  return PATH_END;
-}
-
-<INPATH_SLASH>{ANY} |
-<INPATH_SLASH><<EOF>> {
-  throw ParseError(ErrorInfo{
-      .msg = HintFmt("path has a trailing slash"),
-      .pos = state->positions[CUR_POS],
-  });
-}
-
-{SPATH}     { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }
-{URI}       { yylval->uri = {yytext, (size_t) yyleng}; return URI; }
-
-[ \t\r\n]+    /* eat up whitespace */
-\#[^\r\n]*    /* single-line comments */
-\/\*([^*]|\*+[^*/])*\*+\/  /* long comments */
-
-{ANY}       {
-              /* Don't return a negative number, as this will cause
-                 Bison to stop parsing without an error. */
-              return (unsigned char) yytext[0];
-            }
-
-%%

src/libexpr/meson.build

@@ -1,54 +1,3 @@
-parser_tab = custom_target(
-  input : 'parser.y',
-  output : [
-    'parser-tab.cc',
-    'parser-tab.hh',
-  ],
-  command : [
-    'bison',
-    '-v',
-    '-o',
-    '@OUTPUT0@',
-    '@INPUT@',
-    '-d',
-  ],
-  # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add
-  # an install script below which removes parser-tab.cc.
-  install : true,
-  install_dir : includedir / 'lix/libexpr',
-)
-
-lexer_tab = custom_target(
-  input : [
-    'lexer.l',
-    parser_tab,
-  ],
-  output : [
-    'lexer-tab.cc',
-    'lexer-tab.hh',
-  ],
-  command : [
-    'flex',
-    '--outfile',
-    '@OUTPUT0@',
-    '--header-file=' + '@OUTPUT1@',
-    '@INPUT0@',
-  ],
-  # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add
-  # an install script below which removes lexer-tab.cc.
-  install : true,
-  install_dir : includedir / 'lix/libexpr',
-)
-
-# TODO(Qyriad): When the parser and lexer are rewritten this should be removed.
-# NOTE(Qyriad): We do this this way instead of an inline bash or rm command
-# due to subtleties in Meson. Check the comments in cleanup-install.bash for details.
-meson.add_install_script(
-  bash,
-  meson.project_source_root() / 'meson/cleanup-install.bash',
-  '@0@'.format(includedir),
-)
-
 libexpr_generated_headers = [
   gen_header.process('primops/derivation.nix', preserve_path_from : meson.current_source_dir()),
 ]
@@ -73,13 +22,16 @@ libexpr_sources = files(
   'eval.cc',
   'function-trace.cc',
   'get-drvs.cc',
+  'gc-alloc.cc',
   'json-to-value.cc',
   'nixexpr.cc',
+  'parser/parser.cc',
   'paths.cc',
   'primops.cc',
   'print-ambiguous.cc',
   'print.cc',
   'search-path.cc',
+  'value.cc',
   'value-to-json.cc',
   'value-to-xml.cc',
   'flake/config.cc',
@@ -108,9 +60,12 @@ libexpr_headers = files(
   'function-trace.hh',
   'gc-small-vector.hh',
   'get-drvs.hh',
+  'gc-alloc.hh',
   'json-to-value.hh',
   'nixexpr.hh',
-  'parser-state.hh',
+  'parser/change_head.hh',
+  'parser/grammar.hh',
+  'parser/state.hh',
   'pos-idx.hh',
   'pos-table.hh',
   'primops.hh',
@@ -129,8 +84,6 @@ libexpr_headers = files(
 libexpr = library(
   'lixexpr',
   libexpr_sources,
-  parser_tab,
-  lexer_tab,
   libexpr_generated_headers,
   dependencies : [
     liblixutil,

src/libexpr/nixexpr.cc

@@ -50,6 +50,13 @@ void ExprVar::show(const SymbolTable & symbols, std::ostream & str) const
     str << symbols[name];
 }
 
+void ExprInheritFrom::show(SymbolTable const & symbols, std::ostream & str) const
+{
+    str << "(/* expanded inherit (expr) */ ";
+    fromExpr->show(symbols, str);
+    str << ")";
+}
+
 void ExprSelect::show(const SymbolTable & symbols, std::ostream & str) const
 {
     str << "(";
@@ -79,7 +86,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
         return sa < sb;
     });
     std::vector<Symbol> inherits;
-    std::map<ExprInheritFrom *, std::vector<Symbol>> inheritsFrom;
+    std::map<Displacement, std::vector<Symbol>> inheritsFrom;
     for (auto & i : sorted) {
         switch (i->second.kind) {
         case AttrDef::Kind::Plain:
@@ -90,7 +97,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
         case AttrDef::Kind::InheritedFrom: {
             auto & select = dynamic_cast<ExprSelect &>(*i->second.e);
             auto & from = dynamic_cast<ExprInheritFrom &>(*select.e);
-            inheritsFrom[&from].push_back(i->first);
+            inheritsFrom[from.displ].push_back(i->first);
             break;
         }
         }
@@ -102,7 +109,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
     }
     for (const auto & [from, syms] : inheritsFrom) {
         str << "inherit (";
-        (*inheritFromExprs)[from->displ]->show(symbols, str);
+        (*inheritFromExprs)[from]->show(symbols, str);
         str << ")";
         for (auto sym : syms) str << " " << symbols[sym];
         str << "; ";
@@ -151,7 +158,7 @@ void ExprLambda::show(const SymbolTable & symbols, std::ostream & str) const
         // the natural Symbol ordering is by creation time, which can lead to the
         // same expression being printed in two different ways depending on its
         // context. always use lexicographic ordering to avoid this.
-        for (auto & i : formals->lexicographicOrder(symbols)) {
+        for (const Formal & i : formals->lexicographicOrder(symbols)) {
             if (first) first = false; else str << ", ";
             str << symbols[i.name];
             if (i.def) {
@@ -176,7 +183,7 @@ void ExprCall::show(const SymbolTable & symbols, std::ostream & str) const
 {
     str << '(';
     fun->show(symbols, str);
-    for (auto e : args) {
+    for (auto & e : args) {
         str <<  ' ';
         e->show(symbols, str);
     }
@@ -231,7 +238,7 @@ void ExprConcatStrings::show(const SymbolTable & symbols, std::ostream & str) co
 {
     bool first = true;
     str << "(";
-    for (auto & i : *es) {
+    for (auto & i : es) {
         if (first) first = false; else str << " + ";
         i.second->show(symbols, str);
     }
@@ -375,7 +382,7 @@ std::shared_ptr<const StaticEnv> ExprAttrs::bindInheritSources(
     // not even *have* an expr that grabs anything from this env since it's fully
     // invisible, but the evaluator does not allow for this yet.
     auto inner = std::make_shared<StaticEnv>(nullptr, env.get(), 0);
-    for (auto from : *inheritFromExprs)
+    for (auto & from : *inheritFromExprs)
         from->bindVars(es, env);
 
     return inner;
@@ -462,7 +469,7 @@ void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
         es.exprEnvs.insert(std::make_pair(this, env));
 
     fun->bindVars(es, env);
-    for (auto e : args)
+    for (auto & e : args)
         e->bindVars(es, env);
 }
 
@@ -547,7 +554,7 @@ void ExprConcatStrings::bindVars(EvalState & es, const std::shared_ptr<const Sta
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
-    for (auto & i : *this->es)
+    for (auto & i : this->es)
         i.second->bindVars(es, env);
 }
 

src/libexpr/nixexpr.hh

@@ -28,9 +28,9 @@ struct StaticEnv;
 struct AttrName
 {
     Symbol symbol;
-    Expr * expr;
+    std::unique_ptr<Expr> expr;
     AttrName(Symbol s) : symbol(s) {};
-    AttrName(Expr * e) : expr(e) {};
+    AttrName(std::unique_ptr<Expr> e) : expr(std::move(e)) {};
 };
 
 typedef std::vector<AttrName> AttrPath;
@@ -42,12 +42,21 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath)
 
 struct Expr
 {
+protected:
+    Expr(Expr &&) = default;
+    Expr & operator=(Expr &&) = default;
+
+public:
     struct AstSymbols {
         Symbol sub, lessThan, mul, div, or_, findFile, nixPath, body;
     };
 
 
+    Expr() = default;
+    Expr(const Expr &) = delete;
+    Expr & operator=(const Expr &) = delete;
     virtual ~Expr() { };
+
     virtual void show(const SymbolTable & symbols, std::ostream & str) const;
     virtual void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env);
     virtual void eval(EvalState & state, Env & env, Value & v);
@@ -66,6 +75,7 @@ struct ExprInt : Expr
     NixInt n;
     Value v;
     ExprInt(NixInt n) : n(n) { v.mkInt(n); };
+    ExprInt(NixInt::Inner n) : n(n) { v.mkInt(n); };
     Value * maybeThunk(EvalState & state, Env & env) override;
     COMMON_METHODS
 };
@@ -135,32 +145,46 @@ struct ExprVar : Expr
  */
 struct ExprInheritFrom : ExprVar
 {
-    ExprInheritFrom(PosIdx pos, Displacement displ): ExprVar(pos, {})
+    ref<Expr> fromExpr;
+
+    ExprInheritFrom(PosIdx pos, Displacement displ, ref<Expr> fromExpr)
+          : ExprVar(pos, {}), fromExpr(fromExpr)
     {
         this->level = 0;
         this->displ = displ;
         this->fromWith = nullptr;
     }
 
-    void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env);
+    void show(SymbolTable const & symbols, std::ostream & str) const override;
+    void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) override;
 };
 
 struct ExprSelect : Expr
 {
     PosIdx pos;
-    Expr * e, * def;
+
+    /** The expression attributes are being selected on. e.g. `foo` in `foo.bar.baz`. */
+    std::unique_ptr<Expr> e;
+
+    /** A default value specified with `or`, if the selected attr doesn't exist.
+     * e.g. `bix` in `foo.bar.baz or bix`
+     */
+    std::unique_ptr<Expr> def;
+
+    /** The path of attributes being selected. e.g. `bar.baz` in `foo.bar.baz.` */
     AttrPath attrPath;
-    ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { };
-    ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
+
+    ExprSelect(const PosIdx & pos, std::unique_ptr<Expr> e, AttrPath attrPath, std::unique_ptr<Expr> def) : pos(pos), e(std::move(e)), def(std::move(def)), attrPath(std::move(attrPath)) { };
+    ExprSelect(const PosIdx & pos, std::unique_ptr<Expr> e, Symbol name) : pos(pos), e(std::move(e)) { attrPath.push_back(AttrName(name)); };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
 };
 
 struct ExprOpHasAttr : Expr
 {
-    Expr * e;
+    std::unique_ptr<Expr> e;
     AttrPath attrPath;
-    ExprOpHasAttr(Expr * e, AttrPath attrPath) : e(e), attrPath(std::move(attrPath)) { };
+    ExprOpHasAttr(std::unique_ptr<Expr> e, AttrPath attrPath) : e(std::move(e)), attrPath(std::move(attrPath)) { };
     PosIdx getPos() const override { return e->getPos(); }
     COMMON_METHODS
 };
@@ -180,11 +204,11 @@ struct ExprAttrs : Expr
         };
 
         Kind kind;
-        Expr * e;
+        std::unique_ptr<Expr> e;
         PosIdx pos;
         Displacement displ; // displacement
-        AttrDef(Expr * e, const PosIdx & pos, Kind kind = Kind::Plain)
-            : kind(kind), e(e), pos(pos) { };
+        AttrDef(std::unique_ptr<Expr> e, const PosIdx & pos, Kind kind = Kind::Plain)
+            : kind(kind), e(std::move(e)), pos(pos) { };
         AttrDef() { };
 
         template<typename T>
@@ -203,12 +227,12 @@ struct ExprAttrs : Expr
     };
     typedef std::map<Symbol, AttrDef> AttrDefs;
     AttrDefs attrs;
-    std::unique_ptr<std::vector<Expr *>> inheritFromExprs;
+    std::unique_ptr<std::vector<ref<Expr>>> inheritFromExprs;
     struct DynamicAttrDef {
-        Expr * nameExpr, * valueExpr;
+        std::unique_ptr<Expr> nameExpr, valueExpr;
         PosIdx pos;
-        DynamicAttrDef(Expr * nameExpr, Expr * valueExpr, const PosIdx & pos)
-            : nameExpr(nameExpr), valueExpr(valueExpr), pos(pos) { };
+        DynamicAttrDef(std::unique_ptr<Expr> nameExpr, std::unique_ptr<Expr> valueExpr, const PosIdx & pos)
+            : nameExpr(std::move(nameExpr)), valueExpr(std::move(valueExpr)), pos(pos) { };
     };
     typedef std::vector<DynamicAttrDef> DynamicAttrDefs;
     DynamicAttrDefs dynamicAttrs;
@@ -225,7 +249,7 @@ struct ExprAttrs : Expr
 
 struct ExprList : Expr
 {
-    std::vector<Expr *> elems;
+    std::vector<std::unique_ptr<Expr>> elems;
     ExprList() { };
     COMMON_METHODS
     Value * maybeThunk(EvalState & state, Env & env) override;
@@ -240,7 +264,7 @@ struct Formal
 {
     PosIdx pos;
     Symbol name;
-    Expr * def;
+    std::unique_ptr<Expr> def;
 };
 
 /** Attribute set destructuring in arguments of a lambda, if present */
@@ -257,9 +281,9 @@ struct Formals
         return it != formals.end() && it->name == arg;
     }
 
-    std::vector<Formal> lexicographicOrder(const SymbolTable & symbols) const
+    std::vector<std::reference_wrapper<const Formal>> lexicographicOrder(const SymbolTable & symbols) const
     {
-        std::vector<Formal> result(formals.begin(), formals.end());
+        std::vector<std::reference_wrapper<const Formal>> result(formals.begin(), formals.end());
         std::sort(result.begin(), result.end(),
             [&] (const Formal & a, const Formal & b) {
                 std::string_view sa = symbols[a.name], sb = symbols[b.name];
@@ -283,30 +307,55 @@ struct ExprLambda : Expr
     Symbol arg;
     /** Formals are present when the lambda destructures an attr set as
      * argument, with or without ellipsis */
-    Formals * formals;
-    Expr * body;
-    ExprLambda(PosIdx pos, Symbol arg, Formals * formals, Expr * body)
-        : pos(pos), arg(arg), formals(formals), body(body)
+    std::unique_ptr<Formals> formals;
+    std::unique_ptr<Expr> body;
+    ExprLambda(PosIdx pos, Symbol arg, std::unique_ptr<Formals> formals, std::unique_ptr<Expr> body)
+        : pos(pos), arg(arg), formals(std::move(formals)), body(std::move(body))
     {
     };
-    ExprLambda(PosIdx pos, Formals * formals, Expr * body)
-        : pos(pos), formals(formals), body(body)
+    ExprLambda(PosIdx pos, std::unique_ptr<Formals> formals, std::unique_ptr<Expr> body)
+        : pos(pos), formals(std::move(formals)), body(std::move(body))
     {
     }
     void setName(Symbol name) override;
     std::string showNamePos(const EvalState & state) const;
     inline bool hasFormals() const { return formals != nullptr; }
     PosIdx getPos() const override { return pos; }
+
+    /** Returns the name of the lambda,
+     * or "anonymous lambda" if it doesn't have one.
+     */
+    inline std::string getName(SymbolTable const & symbols) const
+    {
+        if (this->name) {
+            return symbols[this->name];
+        }
+
+        return "anonymous lambda";
+    }
+
+    /** Returns the name of the lambda in single quotes,
+     * or "anonymous lambda" if it doesn't have one.
+     */
+    inline std::string getQuotedName(SymbolTable const & symbols) const
+    {
+        if (this->name) {
+            return concatStrings("'", symbols[this->name], "'");
+        }
+
+        return "anonymous lambda";
+    }
+
     COMMON_METHODS
 };
 
 struct ExprCall : Expr
 {
-    Expr * fun;
-    std::vector<Expr *> args;
+    std::unique_ptr<Expr> fun;
+    std::vector<std::unique_ptr<Expr>> args;
     PosIdx pos;
-    ExprCall(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args)
-        : fun(fun), args(args), pos(pos)
+    ExprCall(const PosIdx & pos, std::unique_ptr<Expr> fun, std::vector<std::unique_ptr<Expr>> && args)
+        : fun(std::move(fun)), args(std::move(args)), pos(pos)
     { }
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
@@ -314,19 +363,19 @@ struct ExprCall : Expr
 
 struct ExprLet : Expr
 {
-    ExprAttrs * attrs;
-    Expr * body;
-    ExprLet(ExprAttrs * attrs, Expr * body) : attrs(attrs), body(body) { };
+    std::unique_ptr<ExprAttrs> attrs;
+    std::unique_ptr<Expr> body;
+    ExprLet(std::unique_ptr<ExprAttrs> attrs, std::unique_ptr<Expr> body) : attrs(std::move(attrs)), body(std::move(body)) { };
     COMMON_METHODS
 };
 
 struct ExprWith : Expr
 {
     PosIdx pos;
-    Expr * attrs, * body;
+    std::unique_ptr<Expr> attrs, body;
     size_t prevWith;
     ExprWith * parentWith;
-    ExprWith(const PosIdx & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { };
+    ExprWith(const PosIdx & pos, std::unique_ptr<Expr> attrs, std::unique_ptr<Expr> body) : pos(pos), attrs(std::move(attrs)), body(std::move(body)) { };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
 };
@@ -334,8 +383,8 @@ struct ExprWith : Expr
 struct ExprIf : Expr
 {
     PosIdx pos;
-    Expr * cond, * then, * else_;
-    ExprIf(const PosIdx & pos, Expr * cond, Expr * then, Expr * else_) : pos(pos), cond(cond), then(then), else_(else_) { };
+    std::unique_ptr<Expr> cond, then, else_;
+    ExprIf(const PosIdx & pos, std::unique_ptr<Expr> cond, std::unique_ptr<Expr> then, std::unique_ptr<Expr> else_) : pos(pos), cond(std::move(cond)), then(std::move(then)), else_(std::move(else_)) { };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
 };
@@ -343,16 +392,16 @@ struct ExprIf : Expr
 struct ExprAssert : Expr
 {
     PosIdx pos;
-    Expr * cond, * body;
-    ExprAssert(const PosIdx & pos, Expr * cond, Expr * body) : pos(pos), cond(cond), body(body) { };
+    std::unique_ptr<Expr> cond, body;
+    ExprAssert(const PosIdx & pos, std::unique_ptr<Expr> cond, std::unique_ptr<Expr> body) : pos(pos), cond(std::move(cond)), body(std::move(body)) { };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
 };
 
 struct ExprOpNot : Expr
 {
-    Expr * e;
-    ExprOpNot(Expr * e) : e(e) { };
+    std::unique_ptr<Expr> e;
+    ExprOpNot(std::unique_ptr<Expr> e) : e(std::move(e)) { };
     PosIdx getPos() const override { return e->getPos(); }
     COMMON_METHODS
 };
@@ -361,9 +410,9 @@ struct ExprOpNot : Expr
     struct name : Expr \
     { \
         PosIdx pos; \
-        Expr * e1, * e2; \
-        name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \
-        name(const PosIdx & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \
+        std::unique_ptr<Expr> e1, e2; \
+        name(std::unique_ptr<Expr> e1, std::unique_ptr<Expr> e2) : e1(std::move(e1)), e2(std::move(e2)) { }; \
+        name(const PosIdx & pos, std::unique_ptr<Expr> e1, std::unique_ptr<Expr> e2) : pos(pos), e1(std::move(e1)), e2(std::move(e2)) { }; \
         void show(const SymbolTable & symbols, std::ostream & str) const override \
         { \
             str << "("; e1->show(symbols, str); str << " " s " "; e2->show(symbols, str); str << ")"; \
@@ -388,9 +437,9 @@ struct ExprConcatStrings : Expr
 {
     PosIdx pos;
     bool forceString;
-    std::vector<std::pair<PosIdx, Expr *>> * es;
-    ExprConcatStrings(const PosIdx & pos, bool forceString, std::vector<std::pair<PosIdx, Expr *>> * es)
-        : pos(pos), forceString(forceString), es(es) { };
+    std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> es;
+    ExprConcatStrings(const PosIdx & pos, bool forceString, std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> es)
+        : pos(pos), forceString(forceString), es(std::move(es)) { };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
 };

src/libexpr/parser.y

@@ -1,452 +0,0 @@
-%glr-parser
-%define api.pure
-%locations
-%define parse.error verbose
-%defines
-/* %no-lines */
-%parse-param { void * scanner }
-%parse-param { nix::ParserState * state }
-%lex-param { void * scanner }
-%lex-param { nix::ParserState * state }
-%expect 1
-%expect-rr 1
-
-%code requires {
-
-#ifndef BISON_HEADER
-#define BISON_HEADER
-
-#include <variant>
-
-#include "finally.hh"
-#include "users.hh"
-
-#include "nixexpr.hh"
-#include "eval.hh"
-#include "eval-settings.hh"
-#include "globals.hh"
-#include "parser-state.hh"
-
-#define YYLTYPE ::nix::ParserLocation
-#define YY_DECL int yylex \
-    (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state)
-
-namespace nix {
-
-Expr * parseExprFromBuf(
-    char * text,
-    size_t length,
-    Pos::Origin origin,
-    const SourcePath & basePath,
-    SymbolTable & symbols,
-    PosTable & positions,
-    const Expr::AstSymbols & astSymbols);
-
-}
-
-#endif
-
-}
-
-%{
-
-#include "parser-tab.hh"
-#include "lexer-tab.hh"
-
-YY_DECL;
-
-using namespace nix;
-
-#define CUR_POS state->at(*yylocp)
-
-
-void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error)
-{
-    if (std::string_view(error).starts_with("syntax error, unexpected end of file")) {
-        loc->first_column = loc->last_column;
-        loc->first_line = loc->last_line;
-    }
-    throw ParseError({
-        .msg = HintFmt(error),
-        .pos = state->positions[state->at(*loc)]
-    });
-}
-
-
-%}
-
-%union {
-  // !!! We're probably leaking stuff here.
-  nix::Expr * e;
-  nix::ExprList * list;
-  nix::ExprAttrs * attrs;
-  nix::Formals * formals;
-  nix::Formal * formal;
-  nix::NixInt n;
-  nix::NixFloat nf;
-  nix::StringToken id; // !!! -> Symbol
-  nix::StringToken path;
-  nix::StringToken uri;
-  nix::StringToken str;
-  std::vector<nix::AttrName> * attrNames;
-  std::vector<std::pair<nix::AttrName, nix::PosIdx>> * inheritAttrs;
-  std::vector<std::pair<nix::PosIdx, nix::Expr *>> * string_parts;
-  std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>> * ind_string_parts;
-}
-
-%type <e> start expr expr_function expr_if expr_op
-%type <e> expr_select expr_simple expr_app
-%type <list> expr_list
-%type <attrs> binds
-%type <formals> formals
-%type <formal> formal
-%type <attrNames> attrpath
-%type <inheritAttrs> attrs
-%type <string_parts> string_parts_interpolated
-%type <ind_string_parts> ind_string_parts
-%type <e> path_start string_parts string_attr
-%type <id> attr
-%token <id> ID
-%token <str> STR IND_STR
-%token <n> INT
-%token <nf> FLOAT
-%token <path> PATH HPATH SPATH PATH_END
-%token <uri> URI
-%token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW
-%token DOLLAR_CURLY /* == ${ */
-%token IND_STRING_OPEN IND_STRING_CLOSE
-%token ELLIPSIS
-
-%right IMPL
-%left OR
-%left AND
-%nonassoc EQ NEQ
-%nonassoc '<' '>' LEQ GEQ
-%right UPDATE
-%left NOT
-%left '+' '-'
-%left '*' '/'
-%right CONCAT
-%nonassoc '?'
-%nonassoc NEGATE
-
-%%
-
-start: expr { state->result = $1; };
-
-expr: expr_function;
-
-expr_function
-  : ID ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); }
-  | '{' formals '}' ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, state->validateFormals($2), $5); }
-  | '{' formals '}' '@' ID ':' expr_function
-    {
-      auto arg = state->symbols.create($5);
-      $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7);
-    }
-  | ID '@' '{' formals '}' ':' expr_function
-    {
-      auto arg = state->symbols.create($1);
-      $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7);
-    }
-  | ASSERT expr ';' expr_function
-    { $$ = new ExprAssert(CUR_POS, $2, $4); }
-  | WITH expr ';' expr_function
-    { $$ = new ExprWith(CUR_POS, $2, $4); }
-  | LET binds IN expr_function
-    { if (!$2->dynamicAttrs.empty())
-        throw ParseError({
-            .msg = HintFmt("dynamic attributes not allowed in let"),
-            .pos = state->positions[CUR_POS]
-        });
-      $$ = new ExprLet($2, $4);
-    }
-  | expr_if
-  ;
-
-expr_if
-  : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); }
-  | expr_op
-  ;
-
-expr_op
-  : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); }
-  | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->s.sub), {new ExprInt(0), $2}); }
-  | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); }
-  | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); }
-  | expr_op '<' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$1, $3}); }
-  | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$3, $1})); }
-  | expr_op '>' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$3, $1}); }
-  | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$1, $3})); }
-  | expr_op AND expr_op { $$ = new ExprOpAnd(state->at(@2), $1, $3); }
-  | expr_op OR expr_op { $$ = new ExprOpOr(state->at(@2), $1, $3); }
-  | expr_op IMPL expr_op { $$ = new ExprOpImpl(state->at(@2), $1, $3); }
-  | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(state->at(@2), $1, $3); }
-  | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, std::move(*$3)); delete $3; }
-  | expr_op '+' expr_op
-    { $$ = new ExprConcatStrings(state->at(@2), false, new std::vector<std::pair<PosIdx, Expr *> >({{state->at(@1), $1}, {state->at(@3), $3}})); }
-  | expr_op '-' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.sub), {$1, $3}); }
-  | expr_op '*' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.mul), {$1, $3}); }
-  | expr_op '/' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.div), {$1, $3}); }
-  | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(state->at(@2), $1, $3); }
-  | expr_app
-  ;
-
-expr_app
-  : expr_app expr_select {
-      if (auto e2 = dynamic_cast<ExprCall *>($1)) {
-          e2->args.push_back($2);
-          $$ = $1;
-      } else
-          $$ = new ExprCall(CUR_POS, $1, {$2});
-  }
-  | expr_select
-  ;
-
-expr_select
-  : expr_simple '.' attrpath
-    { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), nullptr); delete $3; }
-  | expr_simple '.' attrpath OR_KW expr_select
-    { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), $5); delete $3; }
-  | /* Backwards compatibility: because Nixpkgs has a rarely used
-       function named ‘or’, allow stuff like ‘map or [...]’. */
-    expr_simple OR_KW
-    { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, state->s.or_)}); }
-  | expr_simple
-  ;
-
-expr_simple
-  : ID {
-      std::string_view s = "__curPos";
-      if ($1.l == s.size() && strncmp($1.p, s.data(), s.size()) == 0)
-          $$ = new ExprPos(CUR_POS);
-      else
-          $$ = new ExprVar(CUR_POS, state->symbols.create($1));
-  }
-  | INT { $$ = new ExprInt($1); }
-  | FLOAT { $$ = new ExprFloat($1); }
-  | '"' string_parts '"' { $$ = $2; }
-  | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
-      $$ = state->stripIndentation(CUR_POS, std::move(*$2));
-      delete $2;
-  }
-  | path_start PATH_END
-  | path_start string_parts_interpolated PATH_END {
-      $2->insert($2->begin(), {state->at(@1), $1});
-      $$ = new ExprConcatStrings(CUR_POS, false, $2);
-  }
-  | SPATH {
-      std::string path($1.p + 1, $1.l - 2);
-      $$ = new ExprCall(CUR_POS,
-          new ExprVar(state->s.findFile),
-          {new ExprVar(state->s.nixPath),
-           new ExprString(std::move(path))});
-  }
-  | URI {
-      static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals);
-      if (noURLLiterals)
-          throw ParseError({
-              .msg = HintFmt("URL literals are disabled"),
-              .pos = state->positions[CUR_POS]
-          });
-      $$ = new ExprString(std::string($1));
-  }
-  | '(' expr ')' { $$ = $2; }
-  /* Let expressions `let {..., body = ...}' are just desugared
-     into `(rec {..., body = ...}).body'. */
-  | LET '{' binds '}'
-    { $3->recursive = true; $$ = new ExprSelect(noPos, $3, state->s.body); }
-  | REC '{' binds '}'
-    { $3->recursive = true; $$ = $3; }
-  | '{' binds '}'
-    { $$ = $2; }
-  | '[' expr_list ']' { $$ = $2; }
-  ;
-
-string_parts
-  : STR { $$ = new ExprString(std::string($1)); }
-  | string_parts_interpolated { $$ = new ExprConcatStrings(CUR_POS, true, $1); }
-  | { $$ = new ExprString(""); }
-  ;
-
-string_parts_interpolated
-  : string_parts_interpolated STR
-  { $$ = $1; $1->emplace_back(state->at(@2), new ExprString(std::string($2))); }
-  | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); }
-  | DOLLAR_CURLY expr '}' { $$ = new std::vector<std::pair<PosIdx, Expr *>>; $$->emplace_back(state->at(@1), $2); }
-  | STR DOLLAR_CURLY expr '}' {
-      $$ = new std::vector<std::pair<PosIdx, Expr *>>;
-      $$->emplace_back(state->at(@1), new ExprString(std::string($1)));
-      $$->emplace_back(state->at(@2), $3);
-    }
-  ;
-
-path_start
-  : PATH {
-    Path path(absPath({$1.p, $1.l}, state->basePath.path.abs()));
-    /* add back in the trailing '/' to the first segment */
-    if ($1.p[$1.l-1] == '/' && $1.l > 1)
-      path += "/";
-    $$ = new ExprPath(path);
-  }
-  | HPATH {
-    if (evalSettings.pureEval) {
-        throw Error(
-            "the path '%s' can not be resolved in pure mode",
-            std::string_view($1.p, $1.l)
-        );
-    }
-    Path path(getHome() + std::string($1.p + 1, $1.l - 1));
-    $$ = new ExprPath(path);
-  }
-  ;
-
-ind_string_parts
-  : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(state->at(@2), $2); }
-  | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); }
-  | { $$ = new std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>>; }
-  ;
-
-binds
-  : binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->at(@2)); delete $2; }
-  | binds INHERIT attrs ';'
-    { $$ = $1;
-      for (auto & [i, iPos] : *$3) {
-          if ($$->attrs.find(i.symbol) != $$->attrs.end())
-              state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos);
-          $$->attrs.emplace(
-              i.symbol,
-              ExprAttrs::AttrDef(new ExprVar(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited));
-      }
-      delete $3;
-    }
-  | binds INHERIT '(' expr ')' attrs ';'
-    { $$ = $1;
-      if (!$$->inheritFromExprs)
-          $$->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
-      $$->inheritFromExprs->push_back($4);
-      auto from = new nix::ExprInheritFrom(state->at(@4), $$->inheritFromExprs->size() - 1);
-      for (auto & [i, iPos] : *$6) {
-          if ($$->attrs.find(i.symbol) != $$->attrs.end())
-              state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos);
-          $$->attrs.emplace(
-              i.symbol,
-              ExprAttrs::AttrDef(
-                  new ExprSelect(iPos, from, i.symbol),
-                  iPos,
-                  ExprAttrs::AttrDef::Kind::InheritedFrom));
-      }
-      delete $6;
-    }
-  | { $$ = new ExprAttrs(state->at(@0)); }
-  ;
-
-attrs
-  : attrs attr { $$ = $1; $1->emplace_back(AttrName(state->symbols.create($2)), state->at(@2)); }
-  | attrs string_attr
-    { $$ = $1;
-      ExprString * str = dynamic_cast<ExprString *>($2);
-      if (str) {
-          $$->emplace_back(AttrName(state->symbols.create(str->s)), state->at(@2));
-          delete str;
-      } else
-          throw ParseError({
-              .msg = HintFmt("dynamic attributes not allowed in inherit"),
-              .pos = state->positions[state->at(@2)]
-          });
-    }
-  | { $$ = new std::vector<std::pair<AttrName, PosIdx>>; }
-  ;
-
-attrpath
-  : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($3))); }
-  | attrpath '.' string_attr
-    { $$ = $1;
-      ExprString * str = dynamic_cast<ExprString *>($3);
-      if (str) {
-          $$->push_back(AttrName(state->symbols.create(str->s)));
-          delete str;
-      } else
-          $$->push_back(AttrName($3));
-    }
-  | attr { $$ = new std::vector<AttrName>; $$->push_back(AttrName(state->symbols.create($1))); }
-  | string_attr
-    { $$ = new std::vector<AttrName>;
-      ExprString *str = dynamic_cast<ExprString *>($1);
-      if (str) {
-          $$->push_back(AttrName(state->symbols.create(str->s)));
-          delete str;
-      } else
-          $$->push_back(AttrName($1));
-    }
-  ;
-
-attr
-  : ID
-  | OR_KW { $$ = {"or", 2}; }
-  ;
-
-string_attr
-  : '"' string_parts '"' { $$ = $2; }
-  | DOLLAR_CURLY expr '}' { $$ = $2; }
-  ;
-
-expr_list
-  : expr_list expr_select { $$ = $1; $1->elems.push_back($2); /* !!! dangerous */ }
-  | { $$ = new ExprList; }
-  ;
-
-formals
-  : formal ',' formals
-    { $$ = $3; $$->formals.emplace_back(*$1); delete $1; }
-  | formal
-    { $$ = new Formals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; }
-  |
-    { $$ = new Formals; $$->ellipsis = false; }
-  | ELLIPSIS
-    { $$ = new Formals; $$->ellipsis = true; }
-  ;
-
-formal
-  : ID { $$ = new Formal{CUR_POS, state->symbols.create($1), 0}; }
-  | ID '?' expr { $$ = new Formal{CUR_POS, state->symbols.create($1), $3}; }
-  ;
-
-%%
-
-#include "eval.hh"
-
-
-namespace nix {
-
-Expr * parseExprFromBuf(
-    char * text,
-    size_t length,
-    Pos::Origin origin,
-    const SourcePath & basePath,
-    SymbolTable & symbols,
-    PosTable & positions,
-    const Expr::AstSymbols & astSymbols)
-{
-    yyscan_t scanner;
-    ParserState state {
-        .symbols = symbols,
-        .positions = positions,
-        .basePath = basePath,
-        .origin = positions.addOrigin(origin, length),
-        .s = astSymbols,
-    };
-
-    yylex_init(&scanner);
-    Finally _destroy([&] { yylex_destroy(scanner); });
-
-    yy_scan_buffer(text, length, scanner);
-    yyparse(scanner, &state);
-
-    return state.result;
-}
-
-
-}

src/libexpr/parser/change_head.hh

@@ -0,0 +1,66 @@
+#pragma once
+///@file
+
+#include <tao/pegtl.hpp>
+
+namespace nix::parser {
+
+// modified copy of change_state, as the manual suggest for more involved
+// state manipulation. we want to change only the first state parameter,
+// and we care about the *initial* position of a rule application (not the
+// past-the-end position as pegtl change_state provides)
+template<typename NewState>
+struct change_head : tao::pegtl::maybe_nothing
+{
+    template<
+        typename Rule,
+        tao::pegtl::apply_mode A,
+        tao::pegtl::rewind_mode M,
+        template<typename...> class Action,
+        template<typename...> class Control,
+        typename ParseInput,
+        typename State,
+        typename... States
+    >
+    [[nodiscard]] static bool match(ParseInput & in, State && st, States &&... sts)
+    {
+        const auto begin = in.iterator();
+
+        if constexpr (std::is_constructible_v<NewState, State, States...>) {
+            NewState s(st, sts...);
+            if (tao::pegtl::match<Rule, A, M, Action, Control>(in, s, sts...)) {
+                if constexpr (A == tao::pegtl::apply_mode::action) {
+                    _success<Action<Rule>>(0, begin, in, s, st, sts...);
+                }
+                return true;
+            }
+            return false;
+        } else if constexpr (std::is_default_constructible_v<NewState>) {
+            NewState s;
+            if (tao::pegtl::match<Rule, A, M, Action, Control>(in, s, sts...)) {
+                if constexpr (A == tao::pegtl::apply_mode::action) {
+                    _success<Action<Rule>>(0, begin, in, s, st, sts...);
+                }
+                return true;
+            }
+            return false;
+        } else {
+            static_assert(decltype(sizeof(NewState))(), "unable to instantiate new state");
+        }
+    }
+
+    template<typename Target, typename ParseInput, typename... S>
+    static void _success(void *, auto & begin, ParseInput & in, S & ... sts)
+    {
+        const typename ParseInput::action_t at(begin, in);
+        Target::success(at, sts...);
+    }
+
+    template<typename Target, typename... S>
+    static void _success(decltype(Target::success0(std::declval<S &>()...), 0), auto &, auto &, S & ... sts)
+    {
+        Target::success0(sts...);
+    }
+};
+
+}

src/libexpr/parser/grammar.hh

@@ -0,0 +1,714 @@
+#pragma once
+///@file
+
+#include "tao/pegtl.hpp"
+#include <type_traits>
+#include <variant>
+
+#include <boost/container/small_vector.hpp>
+
+// NOTE
+// nix line endings are \n, \r\n, \r. the grammar does not use eol or
+// eolf rules in favor of reproducing the old flex lexer as faithfully as
+// possible, and deferring calculation of positions to downstream users.
+
+namespace nix::parser::grammar {
+
+using namespace tao::pegtl;
+namespace p = tao::pegtl;
+
+// character classes
+namespace c {
+
+struct path : sor<
+    ranges<'a', 'z', 'A', 'Z', '0', '9'>,
+    one<'.', '_', '-', '+'>
+> {};
+struct path_sep : one<'/'> {};
+
+struct id_first : ranges<'a', 'z', 'A', 'Z', '_'> {};
+struct id_rest : sor<
+    ranges<'a', 'z', 'A', 'Z', '0', '9'>,
+    one<'_', '\'', '-'>
+> {};
+
+struct uri_scheme_first : ranges<'a', 'z', 'A', 'Z'> {};
+struct uri_scheme_rest : sor<
+    ranges<'a', 'z', 'A', 'Z', '0', '9'>,
+    one<'+', '-', '.'>
+> {};
+struct uri_sep : one<':'> {};
+struct uri_rest : sor<
+    ranges<'a', 'z', 'A', 'Z', '0', '9'>,
+    one<'%', '/', '?', ':', '@', '&', '=', '+', '$', ',', '-', '_', '.', '!', '~', '*', '\''>
+> {};
+
+}
+
+// "tokens". PEGs don't really care about tokens, we merely use them as a convenient
+// way of writing down keywords and a couple complicated syntax rules.
+namespace t {
+
+struct _extend_as_path : seq<
+    star<c::path>,
+    not_at<TAO_PEGTL_STRING("/*")>,
+    not_at<TAO_PEGTL_STRING("//")>,
+    c::path_sep,
+    sor<c::path, TAO_PEGTL_STRING("${")>
+> {};
+struct _extend_as_uri : seq<
+    star<c::uri_scheme_rest>,
+    c::uri_sep,
+    c::uri_rest
+> {};
+
+// keywords might be extended to identifiers, paths, or uris.
+// NOTE this assumes that keywords are a-zA-Z only, otherwise uri schemes would never
+// match correctly.
+// NOTE not a simple seq<...> because this would report incorrect positions for
+// keywords used inside must<> if a prefix of the keyword matches.
+template<typename S>
+struct _keyword : sor<
+    seq<
+        S,
+        not_at<c::id_rest>,
+        not_at<_extend_as_path>,
+        not_at<_extend_as_uri>
+    >,
+    failure
+> {};
+
+struct kw_if      : _keyword<TAO_PEGTL_STRING("if")> {};
+struct kw_then    : _keyword<TAO_PEGTL_STRING("then")> {};
+struct kw_else    : _keyword<TAO_PEGTL_STRING("else")> {};
+struct kw_assert  : _keyword<TAO_PEGTL_STRING("assert")> {};
+struct kw_with    : _keyword<TAO_PEGTL_STRING("with")> {};
+struct kw_let     : _keyword<TAO_PEGTL_STRING("let")> {};
+struct kw_in      : _keyword<TAO_PEGTL_STRING("in")> {};
+struct kw_rec     : _keyword<TAO_PEGTL_STRING("rec")> {};
+struct kw_inherit : _keyword<TAO_PEGTL_STRING("inherit")> {};
+struct kw_or      : _keyword<TAO_PEGTL_STRING("or")> {};
+
+// `-` can be a unary prefix op, a binary infix op, or the first character
+// of a path or -> (ex 1->1--1)
+// `/` can be a path leader or an operator (ex a?a /a)
+struct op_minus : seq<one<'-'>, not_at<one<'>'>>, not_at<_extend_as_path>> {};
+struct op_div   : seq<one<'/'>, not_at<c::path>> {};
+
+// match a rule, making sure we are not matching it where a keyword would match.
+// using minus like this is a lot faster than flipping the order and using seq.
+template<typename... Rules>
+struct _not_at_any_keyword : minus<
+    seq<Rules...>,
+    sor<
+        TAO_PEGTL_STRING("inherit"),
+        TAO_PEGTL_STRING("assert"),
+        TAO_PEGTL_STRING("else"),
+        TAO_PEGTL_STRING("then"),
+        TAO_PEGTL_STRING("with"),
+        TAO_PEGTL_STRING("let"),
+        TAO_PEGTL_STRING("rec"),
+        TAO_PEGTL_STRING("if"),
+        TAO_PEGTL_STRING("in"),
+        TAO_PEGTL_STRING("or")
+    >
+> {};
+
+// identifiers are kind of horrid:
+//
+//   - uri_scheme_first ⊂ id_first
+//   - uri_scheme_first ⊂ uri_scheme_rest ⊂ path
+//   - id_first ⊂ id_rest ∖ { ' } ⊂ path
+//   - id_first ∩ (path ∖ uri_scheme_first) = { _ }
+//   - uri_sep ∉ ⋃ { id_first, id_rest, uri_scheme_first, uri_scheme_rest, path }
+//   - path_sep ∉ ⋃ { id_first, id_rest, uri_scheme_first, uri_scheme_rest }
+//
+// and we want, without reading the input more than once, a string that
+// matches (id_first id_rest*) and is not followed by any number of
+// characters such that the extended string matches path or uri rules.
+//
+// since the first character must be either _ or a uri scheme character
+// we can ignore path-like bits at the beginning. uri_sep cannot appear anywhere
+// in an identifier, so it's only needed in lookahead checks at the uri-like
+// prefix. likewise path_sep cannot appear anywhere in the idenfier, so it's
+// only needed in lookahead checks in the path-like prefix.
+//
+// in total that gives us a decomposition of
+//
+//     (uri-scheme-like? (?! continues-as-uri) | _)
+//     (path-segment-like? (?! continues-as-path))
+//     id_rest*
+struct identifier : _not_at_any_keyword<
+    // we don't use (at<id_rest>, ...) matches here because identifiers are
+    // a really hot path and rewinding as needed by at<> isn't entirely free.
+    sor<
+        seq<
+            c::uri_scheme_first,
+            star<ranges<'a', 'z', 'A', 'Z', '0', '9', '-'>>,
+            not_at<_extend_as_uri>
+        >,
+        one<'_'>
+    >,
+    star<sor<ranges<'a', 'z', 'A', 'Z', '0', '9'>, one<'_', '-'>>>,
+    not_at<_extend_as_path>,
+    star<c::id_rest>
+> {};
+
+// floats may extend ints, thus these rules are very similar.
+struct integer : seq<
+    sor<
+        seq<range<'1', '9'>, star<digit>, not_at<one<'.'>>>,
+        seq<one<'0'>, not_at<one<'.'>, digit>, star<digit>>
+    >,
+    not_at<_extend_as_path>
+> {};
+
+struct floating : seq<
+    sor<
+        seq<range<'1', '9'>, star<digit>, one<'.'>, star<digit>>,
+        seq<opt<one<'0'>>, one<'.'>, plus<digit>>
+    >,
+    opt<one<'E', 'e'>, opt<one<'+', '-'>>, plus<digit>>,
+    not_at<_extend_as_path>
+> {};
+
+struct uri : seq<
+    c::uri_scheme_first,
+    star<c::uri_scheme_rest>,
+    c::uri_sep,
+    plus<c::uri_rest>
+> {};
+
+struct sep : sor<
+    plus<one<' ', '\t', '\r', '\n'>>,
+    seq<one<'#'>, star<not_one<'\r', '\n'>>>,
+    seq<string<'/', '*'>, until<string<'*', '/'>>>
+> {};
+
+}
+
+
+
+using seps = star<t::sep>;
+
+
+// marker for semantic rules. not handling one of these in an action that cares about
+// semantics is probably an error.
+struct semantic {};
+
+
+struct expr;
+
+struct _string {
+    template<typename... Inner>
+    struct literal : semantic, seq<Inner...> {};
+    struct cr_lf : semantic, seq<one<'\r'>, opt<one<'\n'>>> {};
+    struct interpolation : semantic, seq<
+        p::string<'$', '{'>, seps,
+        must<expr>, seps,
+        must<one<'}'>>
+    > {};
+    struct escape : semantic, must<any> {};
+};
+struct string : _string, seq<
+    one<'"'>,
+    star<
+        sor<
+            _string::literal<plus<not_one<'$', '"', '\\', '\r'>>>,
+            _string::cr_lf,
+            _string::interpolation,
+            _string::literal<one<'$'>, opt<one<'$'>>>,
+            seq<one<'\\'>, _string::escape>
+        >
+    >,
+    must<one<'"'>>
+> {};
+
+struct _ind_string {
+    template<bool Indented, typename... Inner>
+    struct literal : semantic, seq<Inner...> {};
+    struct interpolation : semantic, seq<
+        p::string<'$', '{'>, seps,
+        must<expr>, seps,
+        must<one<'}'>>
+    > {};
+    struct escape : semantic, must<any> {};
+};
+struct ind_string : _ind_string, seq<
+    TAO_PEGTL_STRING("''"),
+    opt<star<one<' '>>, one<'\n'>>,
+    star<
+        sor<
+            _ind_string::literal<
+                true,
+                plus<
+                    sor<
+                        not_one<'$', '\''>,
+                        seq<one<'$'>, not_one<'{', '\''>>,
+                        seq<one<'\''>, not_one<'\'', '$'>>
+                    >
+                >
+            >,
+            _ind_string::interpolation,
+            _ind_string::literal<false, one<'$'>>,
+            _ind_string::literal<false, one<'\''>, not_at<one<'\''>>>,
+            seq<one<'\''>, _ind_string::literal<false, p::string<'\'', '\''>>>,
+            seq<
+                p::string<'\'', '\''>,
+                sor<
+                    _ind_string::literal<false, one<'$'>>,
+                    seq<one<'\\'>, _ind_string::escape>
+                >
+            >
+        >
+    >,
+    must<TAO_PEGTL_STRING("''")>
+> {};
+
+struct _path {
+    // legacy lexer rules. extra l_ to avoid reserved c++ identifiers.
+    struct _l_PATH : seq<star<c::path>, plus<c::path_sep, plus<c::path>>, opt<c::path_sep>> {};
+    struct _l_PATH_SEG : seq<star<c::path>, c::path_sep> {};
+    struct _l_HPATH : seq<one<'~'>, plus<c::path_sep, plus<c::path>>, opt<c::path_sep>> {};
+    struct _l_HPATH_START : TAO_PEGTL_STRING("~/") {};
+    struct _path_str : sor<_l_PATH, _l_PATH_SEG, plus<c::path>> {};
+    // modern rules
+    template<typename... Inner>
+    struct literal : semantic, seq<Inner...> {};
+    struct interpolation : semantic, seq<
+        p::string<'$', '{'>, seps,
+        must<expr>, seps,
+        must<one<'}'>>
+    > {};
+    struct anchor : semantic, sor<
+        _l_PATH,
+        seq<_l_PATH_SEG, at<TAO_PEGTL_STRING("${")>>
+    > {};
+    struct home_anchor : semantic, sor<
+        _l_HPATH,
+        seq<_l_HPATH_START, at<TAO_PEGTL_STRING("${")>>
+    > {};
+    struct searched_path : semantic, list<plus<c::path>, c::path_sep> {};
+    struct forbid_prefix_triple_slash : sor<not_at<c::path_sep>, failure> {};
+    struct forbid_prefix_double_slash_no_interp : sor<
+        not_at<c::path_sep, star<c::path>, not_at<TAO_PEGTL_STRING("${")>>,
+        failure
+    > {};
+    // legacy parser rules
+    struct _str_rest : seq<
+        must<forbid_prefix_double_slash_no_interp>,
+        opt<literal<_path_str>>,
+        must<forbid_prefix_triple_slash>,
+        star<
+            sor<
+                literal<_path_str>,
+                interpolation
+            >
+        >
+    > {};
+};
+struct path : _path, sor<
+    seq<
+        sor<_path::anchor, _path::home_anchor>,
+        _path::_str_rest
+    >,
+    seq<one<'<'>, _path::searched_path, one<'>'>>
+> {};
+
+struct _formal {
+    struct name : semantic, t::identifier {};
+    struct default_value : semantic, must<expr> {};
+};
+struct formal : semantic, _formal, seq<
+    _formal::name,
+    opt<seps, one<'?'>, seps, _formal::default_value>
+> {};
+
+struct _formals {
+    struct ellipsis : semantic, p::ellipsis {};
+};
+struct formals : semantic, _formals, seq<
+    one<'{'>, seps,
+    // formals and attrsets share a two-token head sequence ('{' <id>).
+    // this rule unrolls the formals list a bit to provide better error messages than
+    // "expected '='" at the first ',' if formals are incorrect.
+    sor<
+        one<'}'>,
+        seq<_formals::ellipsis, seps, must<one<'}'>>>,
+        seq<
+            formal, seps,
+            if_then_else<
+                at<one<','>>,
+                seq<
+                    star<one<','>, seps, formal, seps>,
+                    opt<one<','>, seps, opt<_formals::ellipsis, seps>>,
+                    must<one<'}'>>
+                >,
+                one<'}'>
+            >
+        >
+    >
+> {};
+
+struct _attr {
+    struct simple : semantic, sor<t::identifier, t::kw_or> {};
+    struct string : semantic, seq<grammar::string> {};
+    struct expr : semantic, seq<
+        TAO_PEGTL_STRING("${"), seps,
+        must<grammar::expr>, seps,
+        must<one<'}'>>
+    > {};
+};
+struct attr : _attr, sor<
+    _attr::simple,
+    _attr::string,
+    _attr::expr
+> {};
+
+struct attrpath : list<attr, one<'.'>, t::sep> {};
+
+struct _inherit {
+    struct from : semantic, must<expr> {};
+    struct attrs : list<attr, seps> {};
+};
+struct inherit : _inherit, seq<
+    t::kw_inherit, seps,
+    opt<one<'('>, seps, _inherit::from, seps, must<one<')'>>, seps>,
+    opt<_inherit::attrs, seps>,
+    must<one<';'>>
+> {};
+
+struct _binding {
+    struct path : semantic, attrpath {};
+    struct equal : one<'='> {};
+    struct value : semantic, must<expr> {};
+};
+struct binding : _binding, seq<
+    _binding::path, seps,
+    must<_binding::equal>, seps,
+    _binding::value, seps,
+    must<one<';'>>
+> {};
+
+struct bindings : opt<list<sor<inherit, binding>, seps>> {};
+
+struct op {
+    enum class kind {
+        // NOTE non-associativity is *NOT* handled in the grammar structure.
+        // handling it in the grammar itself instead of in semantic actions
+        // slows down the parser significantly and makes the rules *much*
+        // harder to read. maybe this will be different at some point when
+        // ! does not sit between two binary precedence levels.
+        nonAssoc,
+        leftAssoc,
+        rightAssoc,
+        unary,
+    };
+    template<typename Rule, unsigned Precedence, kind Kind = kind::leftAssoc>
+    struct _op : Rule {
+        static constexpr unsigned precedence = Precedence;
+        static constexpr op::kind kind = Kind;
+    };
+
+    struct unary_minus : _op<t::op_minus,           3, kind::unary> {};
+
+    // treating this like a unary postfix operator is sketchy, but that's
+    // the most reasonable way to implement the operator precedence set forth
+    // by the language way back. it'd be much better if `.` and `?` had the same
+    // precedence, but alas.
+    struct has_attr   : _op<seq<one<'?'>, seps, must<attrpath>>, 4> {};
+
+    struct concat     : _op<TAO_PEGTL_STRING("++"),  5, kind::rightAssoc> {};
+    struct mul        : _op<one<'*'>,                6> {};
+    struct div        : _op<t::op_div,               6> {};
+    struct plus       : _op<one<'+'>,                7> {};
+    struct minus      : _op<t::op_minus,             7> {};
+    struct not_       : _op<one<'!'>,                8, kind::unary> {};
+    struct update     : _op<TAO_PEGTL_STRING("//"),  9, kind::rightAssoc> {};
+    struct less_eq    : _op<TAO_PEGTL_STRING("<="), 10, kind::nonAssoc> {};
+    struct greater_eq : _op<TAO_PEGTL_STRING(">="), 10, kind::nonAssoc> {};
+    struct less       : _op<one<'<'>,               10, kind::nonAssoc> {};
+    struct greater    : _op<one<'>'>,               10, kind::nonAssoc> {};
+    struct equals     : _op<TAO_PEGTL_STRING("=="), 11, kind::nonAssoc> {};
+    struct not_equals : _op<TAO_PEGTL_STRING("!="), 11, kind::nonAssoc> {};
+    struct and_       : _op<TAO_PEGTL_STRING("&&"), 12> {};
+    struct or_        : _op<TAO_PEGTL_STRING("||"), 13> {};
+    struct implies    : _op<TAO_PEGTL_STRING("->"), 14, kind::rightAssoc> {};
+    struct pipe_right : _op<TAO_PEGTL_STRING("|>"), 15> {};
+    struct pipe_left  : _op<TAO_PEGTL_STRING("<|"), 16, kind::rightAssoc> {};
+};
+
+struct _expr {
+    template<template<typename...> class OpenMod = seq, typename... Init>
+    struct _attrset : seq<
+        Init...,
+        OpenMod<one<'{'>>, seps,
+        bindings, seps,
+        must<one<'}'>>
+    > {};
+
+    struct select;
+
+    struct id : semantic, t::identifier {};
+    struct int_ : semantic, t::integer {};
+    struct float_ : semantic, t::floating {};
+    struct string : semantic, seq<grammar::string> {};
+    struct ind_string : semantic, seq<grammar::ind_string> {};
+    struct path : semantic, seq<grammar::path> {};
+    struct uri : semantic, t::uri {};
+    struct ancient_let : semantic, _attrset<must, t::kw_let, seps> {};
+    struct rec_set : semantic, _attrset<must, t::kw_rec, seps> {};
+    struct set : semantic, _attrset<> {};
+
+    struct _list {
+        struct entry : semantic, seq<select> {};
+    };
+    struct list : semantic, _list, seq<
+        one<'['>, seps,
+        opt<p::list<_list::entry, seps>, seps>,
+        must<one<']'>>
+    > {};
+
+    struct _simple : sor<
+        id,
+        int_,
+        float_,
+        string,
+        ind_string,
+        path,
+        uri,
+        seq<one<'('>, seps, must<expr>, seps, must<one<')'>>>,
+        ancient_let,
+        rec_set,
+        set,
+        list
+    > {};
+
+    struct _select {
+        struct head : _simple {};
+        struct attr : semantic, seq<attrpath> {};
+        struct attr_or : semantic, must<select> {};
+        struct as_app_or : semantic, t::kw_or {};
+    };
+    struct _app {
+        struct first_arg : semantic, seq<select> {};
+        struct another_arg : semantic, seq<select> {};
+        // can be used to stash a position of the application head node
+        struct select_or_fn : seq<select> {};
+    };
+
+    struct select : _select, seq<
+        _select::head, seps,
+        opt<
+            sor<
+                seq<
+                    one<'.'>, seps, _select::attr,
+                    opt<seps, t::kw_or, seps, _select::attr_or>
+                >,
+                _select::as_app_or
+            >
+        >
+    > {};
+
+    struct app : _app, seq<
+        _app::select_or_fn,
+        opt<seps, _app::first_arg, star<seps, _app::another_arg>>
+    > {};
+
+    template<typename Op>
+    struct operator_ : semantic, Op {};
+
+    struct unary : seq<
+        star<sor<operator_<op::not_>, operator_<op::unary_minus>>, seps>,
+        app
+    > {};
+
+    /* Order matters here. The order is the parsing order, not the precedence order: '<=' must be parsed before '<'. */
+    struct _binary_operator : sor<
+        operator_<op::implies>,
+        operator_<op::update>,
+        operator_<op::concat>,
+        operator_<op::plus>,
+        operator_<op::minus>,
+        operator_<op::mul>,
+        operator_<op::div>,
+        operator_<op::pipe_right>,
+        operator_<op::pipe_left>,
+        operator_<op::less_eq>,
+        operator_<op::greater_eq>,
+        operator_<op::less>,
+        operator_<op::greater>,
+        operator_<op::equals>,
+        operator_<op::not_equals>,
+        operator_<op::or_>,
+        operator_<op::and_>
+    > {};
+
+    struct _binop : seq<
+        unary,
+        star<
+            seps,
+            sor<
+                seq<_binary_operator, seps, must<unary>>,
+                operator_<op::has_attr>
+            >
+        >
+    > {};
+
+    struct _lambda {
+        struct arg : semantic, t::identifier {};
+    };
+    struct lambda : semantic, _lambda, sor<
+        seq<
+            _lambda::arg, seps,
+            sor<
+                seq<one<':'>, seps, must<expr>>,
+                seq<one<'@'>, seps, must<formals, seps, one<':'>, seps, expr>>
+            >
+        >,
+        seq<
+            formals, seps,
+            sor<
+                seq<one<':'>, seps, must<expr>>,
+                seq<one<'@'>, seps, must<_lambda::arg, seps, one<':'>, seps, expr>>
+            >
+        >
+    > {};
+
+    struct assert_ : semantic, seq<
+        t::kw_assert, seps,
+        must<expr>, seps,
+        must<one<';'>>, seps,
+        must<expr>
+    > {};
+    struct with : semantic, seq<
+        t::kw_with, seps,
+        must<expr>, seps,
+        must<one<';'>>, seps,
+        must<expr>
+    > {};
+    struct let : seq<
+        t::kw_let, seps,
+        not_at<one<'{'>>, // exclude ancient_let so we can must<kw_in>
+        bindings, seps,
+        must<t::kw_in>, seps,
+        must<expr>
+    > {};
+    struct if_ : semantic, seq<
+        t::kw_if, seps,
+        must<expr>, seps,
+        must<t::kw_then>, seps,
+        must<expr>, seps,
+        must<t::kw_else>, seps,
+        must<expr>
+    > {};
+};
+struct expr : semantic, _expr, sor<
+    _expr::lambda,
+    _expr::assert_,
+    _expr::with,
+    _expr::let,
+    _expr::if_,
+    _expr::_binop
+> {};
+
+// legacy support: \0 terminates input if passed from flex to bison as a token
+struct eof : sor<p::eof, one<0>> {};
+
+struct root : must<seps, expr, seps, eof> {};
+
+
+
+template<typename Rule>
+struct nothing : p::nothing<Rule> {
+    static_assert(!std::is_base_of_v<semantic, Rule>);
+};
+
+
+
+template<typename Self, typename OpCtx, typename AttrPathT, typename ExprT>
+struct operator_semantics {
+    struct has_attr : grammar::op::has_attr {
+        AttrPathT path;
+    };
+
+    struct OpEntry {
+        OpCtx ctx;
+        uint8_t prec;
+        grammar::op::kind assoc;
+        std::variant<
+            grammar::op::not_,
+            grammar::op::unary_minus,
+            grammar::op::implies,
+            grammar::op::or_,
+            grammar::op::and_,
+            grammar::op::equals,
+            grammar::op::not_equals,
+            grammar::op::less_eq,
+            grammar::op::greater_eq,
+            grammar::op::update,
+            grammar::op::concat,
+            grammar::op::less,
+            grammar::op::greater,
+            grammar::op::plus,
+            grammar::op::minus,
+            grammar::op::mul,
+            grammar::op::div,
+            grammar::op::pipe_right,
+            grammar::op::pipe_left,
+            has_attr
+        > op;
+    };
+
+    // statistics here are taken from nixpkgs commit de502c4d0ba96261e5de803e4d1d1925afd3e22f.
+    // over 99.9% of contexts in nixpkgs need at most 4 slots, ~85% need only 1
+    boost::container::small_vector<ExprT, 4> exprs;
+    // over 99.9% of contexts in nixpkgs need at most 2 slots, ~85% need only 1
+    boost::container::small_vector<OpEntry, 2> ops;
+
+    // derived class is expected to define members:
+    //
+    // ExprT applyOp(OpCtx & pos, auto & op, auto &... args);
+    // [[noreturn]] static void badOperator(OpCtx & pos, auto &... args);
+
+    void reduce(uint8_t toPrecedence, auto &... args) {
+        while (!ops.empty()) {
+            auto & [ctx, precedence, kind, op] = ops.back();
+            // NOTE this relies on associativity not being mixed within a precedence level.
+            if ((precedence > toPrecedence)
+                || (kind != grammar::op::kind::leftAssoc && precedence == toPrecedence))
+                break;
+            std::visit([&, ctx=std::move(ctx)] (auto & op) {
+                exprs.push_back(static_cast<Self &>(*this).applyOp(ctx, op, args...));
+            }, op);
+            ops.pop_back();
+        }
+    }
+
+    ExprT popExpr()
+    {
+        auto r = std::move(exprs.back());
+        exprs.pop_back();
+        return r;
+    }
+
+    void pushOp(OpCtx ctx, auto o, auto &... args)
+    {
+        if (o.kind != grammar::op::kind::unary)
+            reduce(o.precedence, args...);
+        if (!ops.empty() && o.kind == grammar::op::kind::nonAssoc) {
+            auto & [_pos, _prec, _kind, _o] = ops.back();
+            if (_kind == o.kind && _prec == o.precedence)
+                Self::badOperator(ctx, args...);
+        }
+        ops.emplace_back(ctx, o.precedence, o.kind, std::move(o));
+    }
+
+    ExprT finish(auto &... args)
+    {
+        reduce(255, args...);
+        return popExpr();
+    }
+};
+
+}

src/libexpr/parser/parser.cc

@@ -0,0 +1,894 @@
+#include "attr-set.hh"
+#include "error.hh"
+#include "eval-settings.hh"
+#include "eval.hh"
+#include "finally.hh"
+#include "nixexpr.hh"
+#include "symbol-table.hh"
+#include "users.hh"
+
+#include "change_head.hh"
+#include "grammar.hh"
+#include "state.hh"
+
+#include <charconv>
+#include <memory>
+
+// flip this define when doing parser development to enable some g checks.
+#if 0
+#include <tao/pegtl/contrib/analyze.hpp>
+#define ANALYZE_GRAMMAR \
+    ([] { \
+        const std::size_t issues = tao::pegtl::analyze<grammar::root>(); \
+        assert(issues == 0); \
+    })()
+#else
+#define ANALYZE_GRAMMAR ((void) 0)
+#endif
+
+namespace p = tao::pegtl;
+
+namespace nix::parser {
+namespace {
+
+template<typename>
+inline constexpr const char * error_message = nullptr;
+
+#define error_message_for(...) \
+    template<> inline constexpr auto error_message<__VA_ARGS__>
+
+error_message_for(p::one<'{'>) = "expecting '{'";
+error_message_for(p::one<'}'>) = "expecting '}'";
+error_message_for(p::one<'"'>) = "expecting '\"'";
+error_message_for(p::one<';'>) = "expecting ';'";
+error_message_for(p::one<')'>) = "expecting ')'";
+error_message_for(p::one<']'>) = "expecting ']'";
+error_message_for(p::one<':'>) = "expecting ':'";
+error_message_for(p::string<'\'', '\''>) = "expecting \"''\"";
+error_message_for(p::any) = "expecting any character";
+error_message_for(grammar::eof) = "expecting end of file";
+error_message_for(grammar::seps) = "expecting separators";
+error_message_for(grammar::path::forbid_prefix_triple_slash) = "too many slashes in path";
+error_message_for(grammar::path::forbid_prefix_double_slash_no_interp) = "path has a trailing slash";
+error_message_for(grammar::expr) = "expecting expression";
+error_message_for(grammar::expr::unary) = "expecting expression";
+error_message_for(grammar::binding::equal) = "expecting '='";
+error_message_for(grammar::expr::lambda::arg) = "expecting identifier";
+error_message_for(grammar::formals) = "expecting formals";
+error_message_for(grammar::attrpath) = "expecting attribute path";
+error_message_for(grammar::expr::select) = "expecting selection expression";
+error_message_for(grammar::t::kw_then) = "expecting 'then'";
+error_message_for(grammar::t::kw_else) = "expecting 'else'";
+error_message_for(grammar::t::kw_in) = "expecting 'in'";
+
+struct SyntaxErrors
+{
+    template<typename Rule>
+    static constexpr auto message = error_message<Rule>;
+
+    template<typename Rule>
+    static constexpr bool raise_on_failure = false;
+};
+
+template<typename Rule>
+struct Control : p::must_if<SyntaxErrors>::control<Rule>
+{
+    template<typename ParseInput, typename... States>
+    [[noreturn]] static void raise(const ParseInput & in, States &&... st)
+    {
+        if (in.empty()) {
+            std::string expected;
+            if constexpr (constexpr auto msg = error_message<Rule>)
+                expected = fmt(", %s", msg);
+            throw p::parse_error("unexpected end of file" + expected, in);
+        }
+        p::must_if<SyntaxErrors>::control<Rule>::raise(in, st...);
+    }
+};
+
+struct ExprState
+    : grammar::
+          operator_semantics<ExprState, PosIdx, AttrPath, std::pair<PosIdx, std::unique_ptr<Expr>>>
+{
+    std::unique_ptr<Expr> popExprOnly() {
+        return std::move(popExpr().second);
+    }
+
+    template<typename Op, typename... Args>
+    std::unique_ptr<Expr> applyUnary(Args &&... args) {
+        return std::make_unique<Op>(popExprOnly(), std::forward<Args>(args)...);
+    }
+
+    template<typename Op>
+    std::unique_ptr<Expr> applyBinary(PosIdx pos) {
+        auto right = popExprOnly(), left = popExprOnly();
+        return std::make_unique<Op>(pos, std::move(left), std::move(right));
+    }
+
+    std::unique_ptr<Expr> call(PosIdx pos, Symbol fn, bool flip = false)
+    {
+        std::vector<std::unique_ptr<Expr>> args(2);
+        args[flip ? 0 : 1] = popExprOnly();
+        args[flip ? 1 : 0] = popExprOnly();
+        return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(fn), std::move(args));
+    }
+
+    std::unique_ptr<Expr> pipe(PosIdx pos, State & state, bool flip = false)
+    {
+        if (!state.xpSettings.isEnabled(Xp::PipeOperator))
+            throw ParseError({
+                .msg = HintFmt("Pipe operator is disabled"),
+                .pos = state.positions[pos]
+            });
+
+        // Reverse the order compared to normal function application: arg |> fn
+        std::unique_ptr<Expr> fn, arg;
+        if (flip) {
+            fn = popExprOnly();
+            arg = popExprOnly();
+        } else {
+            arg = popExprOnly();
+            fn = popExprOnly();
+        }
+        std::vector<std::unique_ptr<Expr>> args{1};
+        args[0] = std::move(arg);
+
+        return std::make_unique<ExprCall>(pos, std::move(fn), std::move(args));
+    }
+
+    std::unique_ptr<Expr> order(PosIdx pos, bool less, State & state)
+    {
+        return call(pos, state.s.lessThan, !less);
+    }
+
+    std::unique_ptr<Expr> concatStrings(PosIdx pos)
+    {
+        std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> args(2);
+        args[1] = popExpr();
+        args[0] = popExpr();
+        return std::make_unique<ExprConcatStrings>(pos, false, std::move(args));
+    }
+
+    std::unique_ptr<Expr> negate(PosIdx pos, State & state)
+    {
+        std::vector<std::unique_ptr<Expr>> args(2);
+        args[0] = std::make_unique<ExprInt>(0);
+        args[1] = popExprOnly();
+        return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(state.s.sub), std::move(args));
+    }
+
+    std::pair<PosIdx, std::unique_ptr<Expr>> applyOp(PosIdx pos, auto & op, State & state) {
+        using Op = grammar::op;
+
+        auto not_ = [] (auto e) {
+            return std::make_unique<ExprOpNot>(std::move(e));
+        };
+
+        return {
+            pos,
+            (overloaded {
+                [&] (Op::implies)     { return applyBinary<ExprOpImpl>(pos); },
+                [&] (Op::or_)         { return applyBinary<ExprOpOr>(pos); },
+                [&] (Op::and_)        { return applyBinary<ExprOpAnd>(pos); },
+                [&] (Op::equals)      { return applyBinary<ExprOpEq>(pos); },
+                [&] (Op::not_equals)  { return applyBinary<ExprOpNEq>(pos); },
+                [&] (Op::less)        { return order(pos, true, state); },
+                [&] (Op::greater_eq)  { return not_(order(pos, true, state)); },
+                [&] (Op::greater)     { return order(pos, false, state); },
+                [&] (Op::less_eq)     { return not_(order(pos, false, state)); },
+                [&] (Op::update)      { return applyBinary<ExprOpUpdate>(pos); },
+                [&] (Op::not_)        { return applyUnary<ExprOpNot>(); },
+                [&] (Op::plus)        { return concatStrings(pos); },
+                [&] (Op::minus)       { return call(pos, state.s.sub); },
+                [&] (Op::mul)         { return call(pos, state.s.mul); },
+                [&] (Op::div)         { return call(pos, state.s.div); },
+                [&] (Op::concat)      { return applyBinary<ExprOpConcatLists>(pos); },
+                [&] (has_attr & a)    { return applyUnary<ExprOpHasAttr>(std::move(a.path)); },
+                [&] (Op::unary_minus) { return negate(pos, state); },
+                [&] (Op::pipe_right)  { return pipe(pos, state, true); },
+                [&] (Op::pipe_left)   { return pipe(pos, state); },
+            })(op)
+        };
+    }
+
+    // always_inline is needed, otherwise pushOp slows down considerably
+    [[noreturn, gnu::always_inline]]
+    static void badOperator(PosIdx pos, State & state)
+    {
+        throw ParseError({
+            .msg = HintFmt("syntax error, unexpected operator"),
+            .pos = state.positions[pos]
+        });
+    }
+
+    template<typename Expr, typename... Args>
+    Expr & pushExpr(PosIdx pos, Args && ... args)
+    {
+        auto p = std::make_unique<Expr>(std::forward<Args>(args)...);
+        auto & result = *p;
+        exprs.emplace_back(pos, std::move(p));
+        return result;
+    }
+};
+
+struct SubexprState {
+private:
+    ExprState * up;
+
+public:
+    explicit SubexprState(ExprState & up, auto &...) : up(&up) {}
+    operator ExprState &() { return *up; }
+    ExprState * operator->() { return up; }
+};
+
+
+
+template<typename Rule>
+struct BuildAST : grammar::nothing<Rule> {};
+
+struct LambdaState : SubexprState {
+    using SubexprState::SubexprState;
+
+    Symbol arg;
+    std::unique_ptr<Formals> formals;
+};
+
+struct FormalsState : SubexprState {
+    using SubexprState::SubexprState;
+
+    Formals formals{};
+    Formal formal{};
+};
+
+template<> struct BuildAST<grammar::formal::name> {
+    static void apply(const auto & in, FormalsState & s, State & ps) {
+        s.formal = {
+            .pos = ps.at(in),
+            .name = ps.symbols.create(in.string_view()),
+        };
+    }
+};
+
+template<> struct BuildAST<grammar::formal> {
+    static void apply0(FormalsState & s, State &) {
+        s.formals.formals.emplace_back(std::move(s.formal));
+    }
+};
+
+template<> struct BuildAST<grammar::formal::default_value> {
+    static void apply0(FormalsState & s, State & ps) {
+        s.formal.def = s->popExprOnly();
+    }
+};
+
+template<> struct BuildAST<grammar::formals::ellipsis> {
+    static void apply0(FormalsState & s, State &) {
+        s.formals.ellipsis = true;
+    }
+};
+
+template<> struct BuildAST<grammar::formals> : change_head<FormalsState> {
+    static void success0(FormalsState & f, LambdaState & s, State &) {
+        s.formals = std::make_unique<Formals>(std::move(f.formals));
+    }
+};
+
+struct AttrState : SubexprState {
+    using SubexprState::SubexprState;
+
+    std::vector<AttrName> attrs;
+
+    template <typename T>
+    void pushAttr(T && attr, PosIdx) { attrs.emplace_back(std::forward<T>(attr)); }
+};
+
+template<> struct BuildAST<grammar::attr::simple> {
+    static void apply(const auto & in, auto & s, State & ps) {
+        s.pushAttr(ps.symbols.create(in.string_view()), ps.at(in));
+    }
+};
+
+template<> struct BuildAST<grammar::attr::string> {
+    static void apply(const auto & in, auto & s, State & ps) {
+        auto e = s->popExprOnly();
+        if (auto str = dynamic_cast<ExprString *>(e.get()))
+            s.pushAttr(ps.symbols.create(str->s), ps.at(in));
+        else
+            s.pushAttr(std::move(e), ps.at(in));
+    }
+};
+
+template<> struct BuildAST<grammar::attr::expr> : BuildAST<grammar::attr::string> {};
+
+struct BindingsState : SubexprState {
+    using SubexprState::SubexprState;
+
+    ExprAttrs attrs;
+    AttrPath path;
+    std::unique_ptr<Expr> value;
+};
+
+struct InheritState : SubexprState {
+    using SubexprState::SubexprState;
+
+    std::vector<std::pair<AttrName, PosIdx>> attrs;
+    std::unique_ptr<Expr> from;
+    PosIdx fromPos;
+
+    template <typename T>
+    void pushAttr(T && attr, PosIdx pos) { attrs.emplace_back(std::forward<T>(attr), pos); }
+};
+
+template<> struct BuildAST<grammar::inherit::from> {
+    static void apply(const auto & in, InheritState & s, State & ps) {
+        s.from = s->popExprOnly();
+        s.fromPos = ps.at(in);
+    }
+};
+
+template<> struct BuildAST<grammar::inherit> : change_head<InheritState> {
+    static void success0(InheritState & s, BindingsState & b, State & ps) {
+        auto & attrs = b.attrs.attrs;
+        // TODO this should not reuse generic attrpath rules.
+        for (auto & [i, iPos] : s.attrs) {
+            if (i.symbol)
+                continue;
+            if (auto str = dynamic_cast<ExprString *>(i.expr.get()))
+                i = AttrName(ps.symbols.create(str->s));
+            else {
+                throw ParseError({
+                    .msg = HintFmt("dynamic attributes not allowed in inherit"),
+                    .pos = ps.positions[iPos]
+                });
+            }
+        }
+        if (s.from != nullptr) {
+            if (!b.attrs.inheritFromExprs)
+                b.attrs.inheritFromExprs = std::make_unique<std::vector<ref<Expr>>>();
+            auto fromExpr = ref<Expr>(std::move(s.from));
+            b.attrs.inheritFromExprs->push_back(fromExpr);
+            for (auto & [i, iPos] : s.attrs) {
+                if (attrs.find(i.symbol) != attrs.end())
+                    ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos);
+                auto inheritFrom = std::make_unique<ExprInheritFrom>(
+                    s.fromPos,
+                    b.attrs.inheritFromExprs->size() - 1,
+                    fromExpr
+                );
+                attrs.emplace(
+                    i.symbol,
+                    ExprAttrs::AttrDef(
+                        std::make_unique<ExprSelect>(iPos, std::move(inheritFrom), i.symbol),
+                        iPos,
+                        ExprAttrs::AttrDef::Kind::InheritedFrom));
+            }
+        } else {
+            for (auto & [i, iPos] : s.attrs) {
+                if (attrs.find(i.symbol) != attrs.end())
+                    ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos);
+                attrs.emplace(
+                    i.symbol,
+                    ExprAttrs::AttrDef(
+                        std::make_unique<ExprVar>(iPos, i.symbol),
+                        iPos,
+                        ExprAttrs::AttrDef::Kind::Inherited));
+            }
+        }
+    }
+};
+
+template<> struct BuildAST<grammar::binding::path> : change_head<AttrState> {
+    static void success0(AttrState & a, BindingsState & s, State & ps) {
+        s.path = std::move(a.attrs);
+    }
+};
+
+template<> struct BuildAST<grammar::binding::value> {
+    static void apply0(BindingsState & s, State & ps) {
+        s.value = s->popExprOnly();
+    }
+};
+
+template<> struct BuildAST<grammar::binding> {
+    static void apply(const auto & in, BindingsState & s, State & ps) {
+        ps.addAttr(&s.attrs, std::move(s.path), std::move(s.value), ps.at(in));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::id> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        if (in.string_view() == "__curPos")
+            s.pushExpr<ExprPos>(ps.at(in), ps.at(in));
+        else
+            s.pushExpr<ExprVar>(ps.at(in), ps.at(in), ps.symbols.create(in.string_view()));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::int_> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        int64_t v;
+        if (std::from_chars(in.begin(), in.end(), v).ec != std::errc{}) {
+            throw ParseError({
+                .msg = HintFmt("invalid integer '%1%'", in.string_view()),
+                .pos = ps.positions[ps.at(in)],
+            });
+        }
+        s.pushExpr<ExprInt>(noPos, v);
+    }
+};
+
+template<> struct BuildAST<grammar::expr::float_> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        // copy the input into a temporary string so we can call stod.
+        // can't use from_chars because libc++ (thus darwin) does not have it,
+        // and floats are not performance-sensitive anyway. if they were you'd
+        // be in much bigger trouble than this.
+        //
+        // we also get to do a locale-save dance because stod is locale-aware and
+        // something (a plugin?) may have called setlocale or uselocale.
+        static struct locale_hack {
+            locale_t posix;
+            locale_hack(): posix(newlocale(LC_ALL_MASK, "POSIX", 0))
+            {
+                if (posix == 0)
+                    throw SysError("could not get POSIX locale");
+            }
+        } locale;
+
+        auto tmp = in.string();
+        double v = [&] {
+            auto oldLocale = uselocale(locale.posix);
+            Finally resetLocale([=] { uselocale(oldLocale); });
+            try {
+                return std::stod(tmp);
+            } catch (...) {
+                throw ParseError({
+                    .msg = HintFmt("invalid float '%1%'", in.string_view()),
+                    .pos = ps.positions[ps.at(in)],
+                });
+            }
+        }();
+        s.pushExpr<ExprFloat>(noPos, v);
+    }
+};
+
+struct StringState : SubexprState {
+    using SubexprState::SubexprState;
+
+    std::string currentLiteral;
+    PosIdx currentPos;
+    std::vector<std::pair<nix::PosIdx, std::unique_ptr<Expr>>> parts;
+
+    void append(PosIdx pos, std::string_view s)
+    {
+        if (currentLiteral.empty())
+            currentPos = pos;
+        currentLiteral += s;
+    }
+
+    // FIXME this truncates strings on NUL for compat with the old parser. ideally
+    // we should use the decomposition the g gives us instead of iterating over
+    // the entire string again.
+    static void unescapeStr(std::string & str)
+    {
+        char * s = str.data();
+        char * t = s;
+        char c;
+        while ((c = *s++)) {
+            if (c == '\\') {
+                c = *s++;
+                if (c == 'n') *t = '\n';
+                else if (c == 'r') *t = '\r';
+                else if (c == 't') *t = '\t';
+                else *t = c;
+            }
+            else if (c == '\r') {
+                /* Normalise CR and CR/LF into LF. */
+                *t = '\n';
+                if (*s == '\n') s++; /* cr/lf */
+            }
+            else *t = c;
+            t++;
+        }
+        str.resize(t - str.data());
+    }
+
+    void endLiteral()
+    {
+        if (!currentLiteral.empty()) {
+            unescapeStr(currentLiteral);
+            parts.emplace_back(currentPos, std::make_unique<ExprString>(std::move(currentLiteral)));
+        }
+    }
+
+    std::unique_ptr<Expr> finish()
+    {
+        if (parts.empty()) {
+            unescapeStr(currentLiteral);
+            return std::make_unique<ExprString>(std::move(currentLiteral));
+        } else {
+            endLiteral();
+            auto pos = parts[0].first;
+            return std::make_unique<ExprConcatStrings>(pos, true, std::move(parts));
+        }
+    }
+};
+
+template<typename... Content> struct BuildAST<grammar::string::literal<Content...>> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        s.append(ps.at(in), in.string_view());
+    }
+};
+
+template<> struct BuildAST<grammar::string::cr_lf> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        s.append(ps.at(in), in.string_view()); // FIXME compat with old parser
+    }
+};
+
+template<> struct BuildAST<grammar::string::interpolation> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        s.endLiteral();
+        s.parts.emplace_back(ps.at(in), s->popExprOnly());
+    }
+};
+
+template<> struct BuildAST<grammar::string::escape> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        s.append(ps.at(in), "\\"); // FIXME compat with old parser
+        s.append(ps.at(in), in.string_view());
+    }
+};
+
+template<> struct BuildAST<grammar::string> : change_head<StringState> {
+    static void success0(StringState & s, ExprState & e, State &) {
+        e.exprs.emplace_back(noPos, s.finish());
+    }
+};
+
+struct IndStringState : SubexprState {
+    using SubexprState::SubexprState;
+
+    std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>> parts;
+};
+
+template<bool Indented, typename... Content>
+struct BuildAST<grammar::ind_string::literal<Indented, Content...>> {
+    static void apply(const auto & in, IndStringState & s, State & ps) {
+        s.parts.emplace_back(ps.at(in), StringToken{in.string_view(), Indented});
+    }
+};
+
+template<> struct BuildAST<grammar::ind_string::interpolation> {
+    static void apply(const auto & in, IndStringState & s, State & ps) {
+        s.parts.emplace_back(ps.at(in), s->popExprOnly());
+    }
+};
+
+template<> struct BuildAST<grammar::ind_string::escape> {
+    static void apply(const auto & in, IndStringState & s, State & ps) {
+        switch (*in.begin()) {
+        case 'n': s.parts.emplace_back(ps.at(in), StringToken{"\n"}); break;
+        case 'r': s.parts.emplace_back(ps.at(in), StringToken{"\r"}); break;
+        case 't': s.parts.emplace_back(ps.at(in), StringToken{"\t"}); break;
+        default:  s.parts.emplace_back(ps.at(in), StringToken{in.string_view()}); break;
+        }
+    }
+};
+
+template<> struct BuildAST<grammar::ind_string> : change_head<IndStringState> {
+    static void success(const auto & in, IndStringState & s, ExprState & e, State & ps) {
+        e.exprs.emplace_back(noPos, ps.stripIndentation(ps.at(in), std::move(s.parts)));
+    }
+};
+
+template<typename... Content> struct BuildAST<grammar::path::literal<Content...>> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        s.append(ps.at(in), in.string_view());
+        s.endLiteral();
+    }
+};
+
+template<> struct BuildAST<grammar::path::interpolation> : BuildAST<grammar::string::interpolation> {};
+
+template<> struct BuildAST<grammar::path::anchor> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        Path path(absPath(in.string(), ps.basePath.path.abs()));
+        /* add back in the trailing '/' to the first segment */
+        if (in.string_view().ends_with('/') && in.size() > 1)
+            path += "/";
+        s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path)));
+    }
+};
+
+template<> struct BuildAST<grammar::path::home_anchor> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        if (evalSettings.pureEval)
+            throw Error("the path '%s' can not be resolved in pure mode", in.string_view());
+        Path path(getHome() + in.string_view().substr(1));
+        s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path)));
+    }
+};
+
+template<> struct BuildAST<grammar::path::searched_path> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        std::vector<std::unique_ptr<Expr>> args{2};
+        args[0] = std::make_unique<ExprVar>(ps.s.nixPath);
+        args[1] = std::make_unique<ExprString>(in.string());
+        s.parts.emplace_back(
+            ps.at(in),
+            std::make_unique<ExprCall>(
+                ps.at(in),
+                std::make_unique<ExprVar>(ps.s.findFile),
+                std::move(args)));
+    }
+};
+
+template<> struct BuildAST<grammar::path> : change_head<StringState> {
+    template<typename E>
+    static void check_slash(PosIdx end, StringState & s, State & ps) {
+        auto e = dynamic_cast<E *>(s.parts.back().second.get());
+        if (!e || !e->s.ends_with('/'))
+            return;
+        if (s.parts.size() > 1 || e->s != "/")
+            throw ParseError({
+                .msg = HintFmt("path has a trailing slash"),
+                .pos = ps.positions[end],
+            });
+    }
+
+    static void success(const auto & in, StringState & s, ExprState & e, State & ps) {
+        s.endLiteral();
+        check_slash<ExprPath>(ps.atEnd(in), s, ps);
+        check_slash<ExprString>(ps.atEnd(in), s, ps);
+        if (s.parts.size() == 1) {
+            e.exprs.emplace_back(noPos, std::move(s.parts.back().second));
+        } else {
+            e.pushExpr<ExprConcatStrings>(ps.at(in), ps.at(in), false, std::move(s.parts));
+        }
+    }
+};
+
+// strings and paths sare handled fully by the grammar-level rule for now
+template<> struct BuildAST<grammar::expr::string> : p::maybe_nothing {};
+template<> struct BuildAST<grammar::expr::ind_string> : p::maybe_nothing {};
+template<> struct BuildAST<grammar::expr::path> : p::maybe_nothing {};
+
+template<> struct BuildAST<grammar::expr::uri> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+       bool noURLLiterals = ps.xpSettings.isEnabled(Xp::NoUrlLiterals);
+       if (noURLLiterals)
+           throw ParseError({
+               .msg = HintFmt("URL literals are disabled"),
+               .pos = ps.positions[ps.at(in)]
+           });
+       s.pushExpr<ExprString>(ps.at(in), in.string());
+    }
+};
+
+template<> struct BuildAST<grammar::expr::ancient_let> : change_head<BindingsState> {
+    static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) {
+        b.attrs.pos = ps.at(in);
+        b.attrs.recursive = true;
+        s.pushExpr<ExprSelect>(b.attrs.pos, b.attrs.pos, std::make_unique<ExprAttrs>(std::move(b.attrs)), ps.s.body);
+    }
+};
+
+template<> struct BuildAST<grammar::expr::rec_set> : change_head<BindingsState> {
+    static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) {
+        b.attrs.pos = ps.at(in);
+        b.attrs.recursive = true;
+        s.pushExpr<ExprAttrs>(b.attrs.pos, std::move(b.attrs));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::set> : change_head<BindingsState> {
+    static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) {
+        b.attrs.pos = ps.at(in);
+        s.pushExpr<ExprAttrs>(b.attrs.pos, std::move(b.attrs));
+    }
+};
+
+using ListState = std::vector<std::unique_ptr<Expr>>;
+
+template<> struct BuildAST<grammar::expr::list> : change_head<ListState> {
+    static void success(const auto & in, ListState & ls, ExprState & s, State & ps) {
+        auto e = std::make_unique<ExprList>();
+        e->elems = std::move(ls);
+        s.exprs.emplace_back(ps.at(in), std::move(e));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::list::entry> : change_head<ExprState> {
+    static void success0(ExprState & e, ListState & s, State & ps) {
+        s.emplace_back(e.finish(ps).second);
+    }
+};
+
+struct SelectState : SubexprState {
+    using SubexprState::SubexprState;
+
+    PosIdx pos;
+    ExprSelect * e = nullptr;
+};
+
+template<> struct BuildAST<grammar::expr::select::head> {
+    static void apply(const auto & in, SelectState & s, State & ps) {
+        s.pos = ps.at(in);
+    }
+};
+
+template<> struct BuildAST<grammar::expr::select::attr> : change_head<AttrState> {
+    static void success0(AttrState & a, SelectState & s, State &) {
+        s.e = &s->pushExpr<ExprSelect>(s.pos, s.pos, s->popExprOnly(), std::move(a.attrs), nullptr);
+    }
+};
+
+template<> struct BuildAST<grammar::expr::select::attr_or> {
+    static void apply0(SelectState & s, State &) {
+        s.e->def = s->popExprOnly();
+    }
+};
+
+template<> struct BuildAST<grammar::expr::select::as_app_or> {
+    static void apply(const auto & in, SelectState & s, State & ps) {
+        std::vector<std::unique_ptr<Expr>> args(1);
+        args[0] = std::make_unique<ExprVar>(ps.at(in), ps.s.or_);
+        s->pushExpr<ExprCall>(s.pos, s.pos, s->popExprOnly(), std::move(args));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::select> : change_head<SelectState> {
+    static void success0(const auto &...) {}
+};
+
+struct AppState : SubexprState {
+    using SubexprState::SubexprState;
+
+    PosIdx pos;
+    ExprCall * e = nullptr;
+};
+
+template<> struct BuildAST<grammar::expr::app::select_or_fn> {
+    static void apply(const auto & in, AppState & s, State & ps) {
+        s.pos = ps.at(in);
+    }
+};
+
+template<> struct BuildAST<grammar::expr::app::first_arg> {
+    static void apply(auto & in, AppState & s, State & ps) {
+        auto arg = s->popExprOnly(), fn = s->popExprOnly();
+        if ((s.e = dynamic_cast<ExprCall *>(fn.get()))) {
+            // TODO remove.
+            // AST compat with old parser, semantics are the same.
+            // this can happen on occasions such as `<p> <p>` or `a or b or`,
+            // neither of which are super worth optimizing.
+            s.e->args.push_back(std::move(arg));
+            s->exprs.emplace_back(noPos, std::move(fn));
+        } else {
+            std::vector<std::unique_ptr<Expr>> args{1};
+            args[0] = std::move(arg);
+            s.e = &s->pushExpr<ExprCall>(s.pos, s.pos, std::move(fn), std::move(args));
+        }
+    }
+};
+
+template<> struct BuildAST<grammar::expr::app::another_arg> {
+    static void apply0(AppState & s, State & ps) {
+        s.e->args.push_back(s->popExprOnly());
+    }
+};
+
+template<> struct BuildAST<grammar::expr::app> : change_head<AppState> {
+    static void success0(const auto &...) {}
+};
+
+template<typename Op> struct BuildAST<grammar::expr::operator_<Op>> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        s.pushOp(ps.at(in), Op{}, ps);
+    }
+};
+template<> struct BuildAST<grammar::expr::operator_<grammar::op::has_attr>> : change_head<AttrState> {
+    static void success(const auto & in, AttrState & a, ExprState & s, State & ps) {
+        s.pushOp(ps.at(in), ExprState::has_attr{{}, std::move(a.attrs)}, ps);
+    }
+};
+
+template<> struct BuildAST<grammar::expr::lambda::arg> {
+    static void apply(const auto & in, LambdaState & s, State & ps) {
+        s.arg = ps.symbols.create(in.string_view());
+    }
+};
+
+template<> struct BuildAST<grammar::expr::lambda> : change_head<LambdaState> {
+    static void success(const auto & in, LambdaState & l, ExprState & s, State & ps) {
+        if (l.formals)
+            l.formals = ps.validateFormals(std::move(l.formals), ps.at(in), l.arg);
+        s.pushExpr<ExprLambda>(ps.at(in), ps.at(in), l.arg, std::move(l.formals), l->popExprOnly());
+    }
+};
+
+template<> struct BuildAST<grammar::expr::assert_> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        auto body = s.popExprOnly(), cond = s.popExprOnly();
+        s.pushExpr<ExprAssert>(ps.at(in), ps.at(in), std::move(cond), std::move(body));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::with> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        auto body = s.popExprOnly(), scope = s.popExprOnly();
+        s.pushExpr<ExprWith>(ps.at(in), ps.at(in), std::move(scope), std::move(body));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::let> : change_head<BindingsState> {
+    static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) {
+        if (!b.attrs.dynamicAttrs.empty())
+            throw ParseError({
+                .msg = HintFmt("dynamic attributes not allowed in let"),
+                .pos = ps.positions[ps.at(in)]
+            });
+
+        s.pushExpr<ExprLet>(ps.at(in), std::make_unique<ExprAttrs>(std::move(b.attrs)), b->popExprOnly());
+    }
+};
+
+template<> struct BuildAST<grammar::expr::if_> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        auto else_ = s.popExprOnly(), then = s.popExprOnly(), cond = s.popExprOnly();
+        s.pushExpr<ExprIf>(ps.at(in), ps.at(in), std::move(cond), std::move(then), std::move(else_));
+    }
+};
+
+template<> struct BuildAST<grammar::expr> : change_head<ExprState> {
+    static void success0(ExprState & inner, ExprState & outer, State & ps) {
+        outer.exprs.push_back(inner.finish(ps));
+    }
+};
+
+}
+}
+
+namespace nix {
+
+Expr * EvalState::parse(
+    char * text,
+    size_t length,
+    Pos::Origin origin,
+    const SourcePath & basePath,
+    std::shared_ptr<StaticEnv> & staticEnv,
+    const ExperimentalFeatureSettings & xpSettings)
+{
+    parser::State s = {
+        symbols,
+        positions,
+        basePath,
+        positions.addOrigin(origin, length),
+        exprSymbols,
+        xpSettings
+    };
+    parser::ExprState x;
+
+    assert(length >= 2);
+    assert(text[length - 1] == 0);
+    assert(text[length - 2] == 0);
+    length -= 2;
+
+    p::string_input<p::tracking_mode::lazy> inp{std::string_view{text, length}, "input"};
+    try {
+        p::parse<parser::grammar::root, parser::BuildAST, parser::Control>(inp, x, s);
+    } catch (p::parse_error & e) {
+        auto pos = e.positions().back();
+        throw ParseError({
+            .msg = HintFmt("syntax error, %s", e.message()),
+            .pos = positions[s.positions.add(s.origin, pos.byte)]
+        });
+    }
+
+    auto [_pos, result] = x.finish(s);
+    result->bindVars(*this, staticEnv);
+    return result.release();
+}
+
+}

src/libexpr/parser/state.hh

@@ -3,59 +3,45 @@
 
 #include "eval.hh"
 
-namespace nix {
+namespace nix::parser {
 
-/**
- * @note Storing a C-style `char *` and `size_t` allows us to avoid
- * having to define the special members that using string_view here
- * would implicitly delete.
- */
 struct StringToken
 {
-    const char * p;
-    size_t l;
+    std::string_view s;
     bool hasIndentation;
-    operator std::string_view() const { return {p, l}; }
+    operator std::string_view() const { return s; }
 };
 
-struct ParserLocation
-{
-    int first_line, first_column;
-    int last_line, last_column;
-
-    // backup to recover from yyless(0)
-    int stashed_first_column, stashed_last_column;
-
-    void stash() {
-        stashed_first_column = first_column;
-        stashed_last_column = last_column;
-    }
-
-    void unstash() {
-        first_column = stashed_first_column;
-        last_column = stashed_last_column;
-    }
-};
-
-struct ParserState
+struct State
 {
     SymbolTable & symbols;
     PosTable & positions;
-    Expr * result;
     SourcePath basePath;
     PosTable::Origin origin;
     const Expr::AstSymbols & s;
+    const ExperimentalFeatureSettings & xpSettings;
 
     void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos);
     void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos);
-    void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos);
-    Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {});
-    Expr * stripIndentation(const PosIdx pos,
-        std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es);
-    PosIdx at(const ParserLocation & loc);
+    void addAttr(ExprAttrs * attrs, AttrPath && attrPath, std::unique_ptr<Expr> e, const PosIdx pos);
+    std::unique_ptr<Formals> validateFormals(std::unique_ptr<Formals> formals, PosIdx pos = noPos, Symbol arg = {});
+    std::unique_ptr<Expr> stripIndentation(const PosIdx pos,
+        std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>> && es);
+
+    // lazy positioning means we don't get byte offsets directly, in.position() would work
+    // but also requires line and column (which is expensive)
+    PosIdx at(const auto & in)
+    {
+        return positions.add(origin, in.begin() - in.input().begin());
+    }
+
+    PosIdx atEnd(const auto & in)
+    {
+        return positions.add(origin, in.end() - in.input().begin());
+    }
 };
 
-inline void ParserState::dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos)
+inline void State::dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos)
 {
     throw ParseError({
          .msg = HintFmt("attribute '%1%' already defined at %2%",
@@ -64,7 +50,7 @@ inline void ParserState::dupAttr(const AttrPath & attrPath, const PosIdx pos, co
     });
 }
 
-inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos)
+inline void State::dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos)
 {
     throw ParseError({
         .msg = HintFmt("attribute '%1%' already defined at %2%", symbols[attr], positions[prevPos]),
@@ -72,7 +58,7 @@ inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx pre
     });
 }
 
-inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos)
+inline void State::addAttr(ExprAttrs * attrs, AttrPath && attrPath, std::unique_ptr<Expr> e, const PosIdx pos)
 {
     AttrPath::iterator i;
     // All attrpaths have at least one attr
@@ -84,20 +70,25 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr *
             ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
             if (j != attrs->attrs.end()) {
                 if (j->second.kind != ExprAttrs::AttrDef::Kind::Inherited) {
-                    ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e);
-                    if (!attrs2) dupAttr({attrPath.begin(), i + 1}, pos, j->second.pos);
+                    ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e.get());
+                    if (!attrs2) {
+                        attrPath.erase(i + 1, attrPath.end());
+                        dupAttr(attrPath, pos, j->second.pos);
+                    }
                     attrs = attrs2;
-                } else
-                    dupAttr({attrPath.begin(), i + 1}, pos, j->second.pos);
                 } else {
-                ExprAttrs * nested = new ExprAttrs;
-                attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos);
-                attrs = nested;
+                    attrPath.erase(i + 1, attrPath.end());
+                    dupAttr(attrPath, pos, j->second.pos);
                 }
             } else {
-            ExprAttrs *nested = new ExprAttrs;
-            attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos));
-            attrs = nested;
+                auto next = attrs->attrs.emplace(std::piecewise_construct,
+                    std::tuple(i->symbol),
+                    std::tuple(std::make_unique<ExprAttrs>(), pos));
+                attrs = static_cast<ExprAttrs *>(next.first->second.e.get());
+            }
+        } else {
+            auto & next = attrs->dynamicAttrs.emplace_back(std::move(i->expr), std::make_unique<ExprAttrs>(), pos);
+            attrs = static_cast<ExprAttrs *>(next.valueExpr.get());
         }
     }
     // Expr insertion.
@@ -109,41 +100,41 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr *
             // e and the expr pointed by the attr path are two attribute sets,
             // we want to merge them.
             // Otherwise, throw an error.
-            auto ae = dynamic_cast<ExprAttrs *>(e);
-            auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
+            auto * ae = dynamic_cast<ExprAttrs *>(e.get());
+            auto * jAttrs = dynamic_cast<ExprAttrs *>(j->second.e.get());
             if (jAttrs && ae) {
                 if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
-                    jAttrs->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
+                    jAttrs->inheritFromExprs = std::make_unique<std::vector<ref<Expr>>>();
                 for (auto & ad : ae->attrs) {
                     auto j2 = jAttrs->attrs.find(ad.first);
                     if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
-                        dupAttr(ad.first, j2->second.pos, ad.second.pos);
-                    jAttrs->attrs.emplace(ad.first, ad.second);
+                        return dupAttr(ad.first, j2->second.pos, ad.second.pos);
                     if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
                         auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
                         auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
                         from.displ += jAttrs->inheritFromExprs->size();
                     }
+                    jAttrs->attrs.emplace(ad.first, std::move(ad.second));
                 }
-                jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end());
-                if (ae->inheritFromExprs) {
-                    jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(),
-                        ae->inheritFromExprs->begin(), ae->inheritFromExprs->end());
-                }
+                std::ranges::move(ae->dynamicAttrs, std::back_inserter(jAttrs->dynamicAttrs));
+                if (ae->inheritFromExprs)
+                    std::ranges::move(*ae->inheritFromExprs, std::back_inserter(*jAttrs->inheritFromExprs));
             } else {
                 dupAttr(attrPath, pos, j->second.pos);
             }
         } else {
             // This attr path is not defined. Let's create it.
-            attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos));
             e->setName(i->symbol);
+            attrs->attrs.emplace(std::piecewise_construct,
+                std::tuple(i->symbol),
+                std::tuple(std::move(e), pos));
         }
     } else {
-        attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos));
+        attrs->dynamicAttrs.emplace_back(std::move(i->expr), std::move(e), pos);
     }
 }
 
-inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg)
+inline std::unique_ptr<Formals> State::validateFormals(std::unique_ptr<Formals> formals, PosIdx pos, Symbol arg)
 {
     std::sort(formals->formals.begin(), formals->formals.end(),
         [] (const auto & a, const auto & b) {
@@ -172,10 +163,10 @@ inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Sym
     return formals;
 }
 
-inline Expr * ParserState::stripIndentation(const PosIdx pos,
-    std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es)
+inline std::unique_ptr<Expr> State::stripIndentation(const PosIdx pos,
+    std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>> && es)
 {
-    if (es.empty()) return new ExprString("");
+    if (es.empty()) return std::make_unique<ExprString>("");
 
     /* Figure out the minimum indentation.  Note that by design
        whitespace-only final lines are not taken into account.  (So
@@ -193,11 +184,11 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos,
             }
             continue;
         }
-        for (size_t j = 0; j < str->l; ++j) {
+        for (size_t j = 0; j < str->s.size(); ++j) {
             if (atStartOfLine) {
-                if (str->p[j] == ' ')
+                if (str->s[j] == ' ')
                     curIndent++;
-                else if (str->p[j] == '\n') {
+                else if (str->s[j] == '\n') {
                     /* Empty line, doesn't influence minimum
                        indentation. */
                     curIndent = 0;
@@ -205,7 +196,7 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos,
                     atStartOfLine = false;
                     if (curIndent < minIndent) minIndent = curIndent;
                 }
-            } else if (str->p[j] == '\n') {
+            } else if (str->s[j] == '\n') {
                 atStartOfLine = true;
                 curIndent = 0;
             }
@@ -213,35 +204,35 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos,
     }
 
     /* Strip spaces from each line. */
-    auto * es2 = new std::vector<std::pair<PosIdx, Expr *>>;
+    std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> es2;
     atStartOfLine = true;
     size_t curDropped = 0;
     size_t n = es.size();
     auto i = es.begin();
-    const auto trimExpr = [&] (Expr * e) {
+    const auto trimExpr = [&] (std::unique_ptr<Expr> e) {
         atStartOfLine = false;
         curDropped = 0;
-        es2->emplace_back(i->first, e);
+        es2.emplace_back(i->first, std::move(e));
     };
-    const auto trimString = [&] (const StringToken & t) {
+    const auto trimString = [&] (const StringToken t) {
         std::string s2;
-        for (size_t j = 0; j < t.l; ++j) {
+        for (size_t j = 0; j < t.s.size(); ++j) {
             if (atStartOfLine) {
-                if (t.p[j] == ' ') {
+                if (t.s[j] == ' ') {
                     if (curDropped++ >= minIndent)
-                        s2 += t.p[j];
+                        s2 += t.s[j];
                 }
-                else if (t.p[j] == '\n') {
+                else if (t.s[j] == '\n') {
                     curDropped = 0;
-                    s2 += t.p[j];
+                    s2 += t.s[j];
                 } else {
                     atStartOfLine = false;
                     curDropped = 0;
-                    s2 += t.p[j];
+                    s2 += t.s[j];
                 }
             } else {
-                s2 += t.p[j];
-                if (t.p[j] == '\n') atStartOfLine = true;
+                s2 += t.s[j];
+                if (t.s[j] == '\n') atStartOfLine = true;
             }
         }
 
@@ -253,24 +244,17 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos,
                 s2 = std::string(s2, 0, p + 1);
         }
 
-        es2->emplace_back(i->first, new ExprString(std::move(s2)));
+        es2.emplace_back(i->first, std::make_unique<ExprString>(std::move(s2)));
     };
     for (; i != es.end(); ++i, --n) {
-        std::visit(overloaded { trimExpr, trimString }, i->second);
+        std::visit(overloaded { trimExpr, trimString }, std::move(i->second));
     }
 
     /* If this is a single string, then don't do a concatenation. */
-    if (es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second)) {
-        auto *const result = (*es2)[0].second;
-        delete es2;
-        return result;
+    if (es2.size() == 1 && dynamic_cast<ExprString *>(es2[0].second.get())) {
+        return std::move(es2[0].second);
     }
-    return new ExprConcatStrings(pos, true, es2);
-}
-
-inline PosIdx ParserState::at(const ParserLocation & loc)
-{
-    return positions.add(origin, loc.first_column);
+    return std::make_unique<ExprConcatStrings>(pos, true, std::move(es2));
 }
 
 }

src/libexpr/primops.cc

@@ -244,9 +244,9 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
             // args[0]->attrs is already sorted.
 
             debug("evaluating file '%1%'", path);
-            Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv);
+            Expr & e = state.parseExprFromFile(resolveExprPath(path), staticEnv);
 
-            e->eval(state, *env, v);
+            e.eval(state, *env, v);
         }
     }
 }
@@ -346,7 +346,7 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu
         state.error<EvalError>("could not open '%1%': %2%", path, dlerror()).debugThrow();
 
     dlerror();
-    ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str());
+    ValueInitializer func = reinterpret_cast<ValueInitializer>(dlsym(handle, sym.c_str()));
     if(!func) {
         char *message = dlerror();
         if (message)
@@ -390,13 +390,13 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v)
     auto output = runProgram(program, true, commandArgs);
     Expr * parsed;
     try {
-        parsed = state.parseExprFromString(std::move(output), state.rootPath(CanonPath::root));
+        parsed = &state.parseExprFromString(std::move(output), state.rootPath(CanonPath::root));
     } catch (Error & e) {
         e.addTrace(state.positions[pos], "while parsing the output from '%1%'", program);
         throw;
     }
     try {
-        state.eval(parsed, v);
+        state.eval(*parsed, v);
     } catch (Error & e) {
         e.addTrace(state.positions[pos], "while evaluating the output from '%1%'", program);
         throw;
@@ -582,9 +582,9 @@ struct CompareValues
     {
         try {
             if (v1->type() == nFloat && v2->type() == nInt)
-                return v1->fpoint < v2->integer;
+                return v1->fpoint < v2->integer.value;
             if (v1->type() == nInt && v2->type() == nFloat)
-                return v1->integer < v2->fpoint;
+                return v1->integer.value < v2->fpoint;
             if (v1->type() != v2->type())
                 state.error<EvalError>("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow();
             // Allow selecting a subset of enum values
@@ -622,14 +622,13 @@ struct CompareValues
     }
 };
 
-
+/// NOTE: this type must NEVER be outside of GC-scanned memory.
 #if HAVE_BOEHMGC
-typedef std::list<Value *, gc_allocator<Value *>> ValueList;
+using UnsafeValueList = std::list<Value *, gc_allocator<Value *>>;
 #else
-typedef std::list<Value *> ValueList;
+using UnsafeValueList = std::list<Value *>;
 #endif
 
-
 static Bindings::iterator getAttr(
     EvalState & state,
     Symbol attrSym,
@@ -652,7 +651,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * a
 
     state.forceList(*startSet->value, noPos, "while evaluating the 'startSet' attribute passed as argument to builtins.genericClosure");
 
-    ValueList workSet;
+    UnsafeValueList workSet;
     for (auto elem : startSet->value->listItems())
         workSet.push_back(elem);
 
@@ -668,7 +667,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * a
     /* Construct the closure by applying the operator to elements of
        `workSet', adding the result to `workSet', continuing until
        no new elements are found. */
-    ValueList res;
+    UnsafeValueList res;
     // `doneKeys' doesn't need to be a GC root, because its values are
     // reachable from res.
     auto cmp = CompareValues(state, noPos, "while comparing the `key` attributes of two genericClosure elements");
@@ -866,7 +865,7 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va
     /* increment state.trylevel, and decrement it when this function returns. */
     MaintainCount trylevel(state.trylevel);
 
-    ReplExitStatus (* savedDebugRepl)(ref<EvalState> es, const ValMap & extraEnv) = nullptr;
+    std::function<ReplExitStatus(ref<EvalState> es, ValMap const & extraEnv)> savedDebugRepl;
     if (state.debugRepl && evalSettings.ignoreExceptionsDuringTry)
     {
         /* to prevent starting the repl from exceptions withing a tryEval, null it. */
@@ -923,14 +922,15 @@ static RegisterPrimOp primop_getEnv({
     .args = {"s"},
     .doc = R"(
       `getEnv` returns the value of the environment variable *s*, or an
-      empty string if the variable doesn’t exist. This function should be
+      empty string if the variable doesn't exist. This function should be
       used with care, as it can introduce all sorts of nasty environment
       dependencies in your Nix expression.
 
-      `getEnv` is used in Nix Packages to locate the file
-      `~/.nixpkgs/config.nix`, which contains user-local settings for Nix
-      Packages. (That is, it does a `getEnv "HOME"` to locate the user’s
-      home directory.)
+      `getEnv` is used in nixpkgs for evil impurities such as locating the file
+      `~/.config/nixpkgs/config.nix` which contains user-local settings for nixpkgs.
+      (That is, it does a `getEnv "HOME"` to locate the user's home directory.)
+
+      When in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval), this function always returns an empty string.
     )",
     .fun = prim_getEnv,
 });
@@ -1213,6 +1213,20 @@ drvName, Bindings * attrs, Value & v)
                         handleOutputs(ss);
                     }
 
+                    if (i->name == state.sAllowedReferences)
+                        warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks.<output>.allowedReferences' instead", drvName);
+                    if (i->name == state.sAllowedRequisites)
+                        warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks.<output>.allowedRequisites' instead", drvName);
+                    if (i->name == state.sDisallowedReferences)
+                        warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks.<output>.disallowedReferences' instead", drvName);
+                    if (i->name == state.sDisallowedRequisites)
+                        warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks.<output>.disallowedRequisites' instead", drvName);
+                    if (i->name == state.sMaxSize)
+                        warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks.<output>.maxSize' instead", drvName);
+                    if (i->name == state.sMaxClosureSize)
+                        warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks.<output>.maxClosureSize' instead", drvName);
+
+
                 } else {
                     auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned();
                     drv.env.emplace(key, s);
@@ -1322,7 +1336,7 @@ drvName, Bindings * attrs, Value & v)
             state.error<EvalError>("derivation cannot be both content-addressed and impure")
                 .atPos(v).debugThrow();
 
-        auto ht = parseHashTypeOpt(outputHashAlgo).value_or(htSHA256);
+        auto ht = parseHashTypeOpt(outputHashAlgo).value_or(HashType::SHA256);
         auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
 
         for (auto & i : outputs) {
@@ -1506,6 +1520,7 @@ static RegisterPrimOp primop_storePath({
       in a new path (e.g. `/nix/store/ld01dnzc…-source-source`).
 
       Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
+      Lix may change this, tracking issue: <https://git.lix.systems/lix-project/lix/issues/402>
 
       See also [`builtins.fetchClosure`](#builtins-fetchClosure).
     )",
@@ -1749,7 +1764,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, V
 
     auto path = realisePath(state, pos, *args[1]);
 
-    v.mkString(hashString(*ht, path.readFile()).to_string(Base16, false));
+    v.mkString(hashString(*ht, path.readFile()).to_string(Base::Base16, false));
 }
 
 static RegisterPrimOp primop_hashFile({
@@ -2331,7 +2346,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
         else if (n == "recursive")
             method = FileIngestionMethod { state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") };
         else if (n == "sha256")
-            expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), htSHA256);
+            expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashType::SHA256);
         else
             state.error<EvalError>(
                 "unsupported argument '%1%' to 'addPath'",
@@ -2424,6 +2439,8 @@ static void prim_attrValues(EvalState & state, const PosIdx pos, Value * * args,
 
     state.mkList(v, args[0]->attrs->size());
 
+    // FIXME: this is incredibly evil, *why*
+    // NOLINTBEGIN(cppcoreguidelines-pro-type-cstyle-cast)
     unsigned int n = 0;
     for (auto & i : *args[0]->attrs)
         v.listElems()[n++] = (Value *) &i;
@@ -2437,6 +2454,7 @@ static void prim_attrValues(EvalState & state, const PosIdx pos, Value * * args,
 
     for (unsigned int i = 0; i < n; ++i)
         v.listElems()[i] = ((Attr *) v.listElems()[i])->value;
+    // NOLINTEND(cppcoreguidelines-pro-type-cstyle-cast)
 }
 
 static RegisterPrimOp primop_attrValues({
@@ -2512,13 +2530,13 @@ static struct LazyPosAcessors {
     PrimOp primop_lineOfPos{
         .arity = 1,
         .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) {
-            v.mkInt(state.positions[PosIdx(args[0]->integer)].line);
+            v.mkInt(state.positions[PosIdx(args[0]->integer.value)].line);
         }
     };
     PrimOp primop_columnOfPos{
         .arity = 1,
         .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) {
-            v.mkInt(state.positions[PosIdx(args[0]->integer)].column);
+            v.mkInt(state.positions[PosIdx(args[0]->integer.value)].column);
         }
     };
 
@@ -2817,7 +2835,7 @@ static void prim_functionArgs(EvalState & state, const PosIdx pos, Value * * arg
     auto attrs = state.buildBindings(args[0]->lambda.fun->formals->formals.size());
     for (auto & i : args[0]->lambda.fun->formals->formals)
         // !!! should optimise booleans (allocate only once)
-        attrs.alloc(i.name, i.pos).mkBool(i.def);
+        attrs.alloc(i.name, i.pos).mkBool(i.def != nullptr);
     v.mkAttrs(attrs);
 }
 
@@ -2988,7 +3006,8 @@ static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Val
 /* Return the n-1'th element of a list. */
 static void prim_elemAt(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    elemAt(state, pos, *args[0], state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt"), v);
+    NixInt::Inner elem = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt").value;
+    elemAt(state, pos, *args[0], elem, v);
 }
 
 static RegisterPrimOp primop_elemAt({
@@ -3272,17 +3291,19 @@ static RegisterPrimOp primop_all({
 
 static void prim_genList(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    auto len = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList");
+    auto len_ = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList").value;
 
-    if (len < 0)
-        state.error<EvalError>("cannot create list of size %1%", len).atPos(pos).debugThrow();
+    if (len_ < 0)
+        state.error<EvalError>("cannot create list of size %1%", len_).atPos(pos).debugThrow();
+
+    size_t len = len_;
 
     // More strict than striclty (!) necessary, but acceptable
     // as evaluating map without accessing any values makes little sense.
     state.forceFunction(*args[0], noPos, "while evaluating the first argument passed to builtins.genList");
 
     state.mkList(v, len);
-    for (unsigned int n = 0; n < (unsigned int) len; ++n) {
+    for (size_t n = 0; n < len; ++n) {
         auto arg = state.allocValue();
         arg->mkInt(n);
         (v.listElems()[n] = state.allocValue())->mkApp(args[0], arg);
@@ -3532,9 +3553,17 @@ static void prim_add(EvalState & state, const PosIdx pos, Value * * args, Value
     if (args[0]->type() == nFloat || args[1]->type() == nFloat)
         v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the addition")
                 + state.forceFloat(*args[1], pos, "while evaluating the second argument of the addition"));
-    else
-        v.mkInt(  state.forceInt(*args[0], pos, "while evaluating the first argument of the addition")
-                + state.forceInt(*args[1], pos, "while evaluating the second argument of the addition"));
+    else {
+        auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the addition");
+        auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the addition");
+
+        auto result_ = i1 + i2;
+        if (auto result = result_.valueChecked(); result.has_value()) {
+            v.mkInt(*result);
+        } else {
+            state.error<EvalError>("integer overflow in adding %1% + %2%", i1, i2).atPos(pos).debugThrow();
+        }
+    }
 }
 
 static RegisterPrimOp primop_add({
@@ -3553,9 +3582,18 @@ static void prim_sub(EvalState & state, const PosIdx pos, Value * * args, Value
     if (args[0]->type() == nFloat || args[1]->type() == nFloat)
         v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the subtraction")
                 - state.forceFloat(*args[1], pos, "while evaluating the second argument of the subtraction"));
-    else
-        v.mkInt(  state.forceInt(*args[0], pos, "while evaluating the first argument of the subtraction")
-                - state.forceInt(*args[1], pos, "while evaluating the second argument of the subtraction"));
+    else {
+        auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the subtraction");
+        auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the subtraction");
+
+        auto result_ = i1 - i2;
+
+        if (auto result = result_.valueChecked(); result.has_value()) {
+            v.mkInt(*result);
+        } else {
+            state.error<EvalError>("integer overflow in subtracting %1% - %2%", i1, i2).atPos(pos).debugThrow();
+        }
+    }
 }
 
 static RegisterPrimOp primop_sub({
@@ -3574,9 +3612,18 @@ static void prim_mul(EvalState & state, const PosIdx pos, Value * * args, Value
     if (args[0]->type() == nFloat || args[1]->type() == nFloat)
         v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first of the multiplication")
                 * state.forceFloat(*args[1], pos, "while evaluating the second argument of the multiplication"));
-    else
-        v.mkInt(  state.forceInt(*args[0], pos, "while evaluating the first argument of the multiplication")
-                * state.forceInt(*args[1], pos, "while evaluating the second argument of the multiplication"));
+    else {
+        auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the multiplication");
+        auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the multiplication");
+
+        auto result_ = i1 * i2;
+
+        if (auto result = result_.valueChecked(); result.has_value()) {
+            v.mkInt(*result);
+        } else {
+            state.error<EvalError>("integer overflow in multiplying %1% * %2%", i1, i2).atPos(pos).debugThrow();
+        }
+    }
 }
 
 static RegisterPrimOp primop_mul({
@@ -3603,10 +3650,12 @@ static void prim_div(EvalState & state, const PosIdx pos, Value * * args, Value
         NixInt i1 = state.forceInt(*args[0], pos, "while evaluating the first operand of the division");
         NixInt i2 = state.forceInt(*args[1], pos, "while evaluating the second operand of the division");
         /* Avoid division overflow as it might raise SIGFPE. */
-        if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
-            state.error<EvalError>("overflow in integer division").atPos(pos).debugThrow();
-
-        v.mkInt(i1 / i2);
+        auto result_ = i1 / i2;
+        if (auto result = result_.valueChecked(); result.has_value()) {
+            v.mkInt(*result);
+        } else {
+            state.error<EvalError>("integer overflow in dividing %1% / %2%", i1, i2).atPos(pos).debugThrow();
+        }
     }
 }
 
@@ -3621,8 +3670,9 @@ static RegisterPrimOp primop_div({
 
 static void prim_bitAnd(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitAnd")
-            & state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitAnd"));
+    auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitAnd");
+    auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitAnd");
+    v.mkInt(i1.value & i2.value);
 }
 
 static RegisterPrimOp primop_bitAnd({
@@ -3636,8 +3686,10 @@ static RegisterPrimOp primop_bitAnd({
 
 static void prim_bitOr(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitOr")
-            | state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitOr"));
+    auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitOr");
+    auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitOr");
+
+    v.mkInt(i1.value | i2.value);
 }
 
 static RegisterPrimOp primop_bitOr({
@@ -3651,8 +3703,10 @@ static RegisterPrimOp primop_bitOr({
 
 static void prim_bitXor(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitXor")
-            ^ state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitXor"));
+    auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitXor");
+    auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitXor");
+
+    v.mkInt(i1.value ^ i2.value);
 }
 
 static RegisterPrimOp primop_bitXor({
@@ -3732,13 +3786,19 @@ static RegisterPrimOp primop_toString({
    non-negative. */
 static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring");
+    NixInt::Inner start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring").value;
 
     if (start < 0)
         state.error<EvalError>("negative start position in 'substring'").atPos(pos).debugThrow();
 
 
-    int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring");
+    NixInt::Inner len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring").value;
+
+    // Negative length may be idiomatically passed to builtins.substring to get
+    // the tail of the string.
+    if (len < 0) {
+        len = std::numeric_limits<NixInt::Inner>::max();
+    }
 
     // Special-case on empty substring to avoid O(n) strlen
     // This allows for the use of empty substrings to efficently capture string context
@@ -3780,7 +3840,7 @@ static void prim_stringLength(EvalState & state, const PosIdx pos, Value * * arg
 {
     NixStringContext context;
     auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.stringLength");
-    v.mkInt(s->size());
+    v.mkInt(NixInt::Inner(s->size()));
 }
 
 static RegisterPrimOp primop_stringLength({
@@ -3804,7 +3864,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args,
     NixStringContext context; // discarded
     auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString");
 
-    v.mkString(hashString(*ht, s).to_string(Base16, false));
+    v.mkString(hashString(*ht, s).to_string(Base::Base16, false));
 }
 
 static RegisterPrimOp primop_hashString({
@@ -4160,7 +4220,8 @@ static void prim_compareVersions(EvalState & state, const PosIdx pos, Value * *
 {
     auto version1 = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.compareVersions");
     auto version2 = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.compareVersions");
-    v.mkInt(compareVersions(version1, version2));
+    auto result = compareVersions(version1, version2);
+    v.mkInt(result < 0 ? -1 : result > 0 ? 1 : 0);
 }
 
 static RegisterPrimOp primop_compareVersions({
@@ -4494,7 +4555,7 @@ void EvalState::createBaseEnv()
         // the parser needs two NUL bytes as terminators; one of them
         // is implied by being a C string.
         "\0";
-    eval(parse(code, sizeof(code), derivationInternal, {CanonPath::root}, staticBaseEnv), *vDerivation);
+    eval(*parse(code, sizeof(code), derivationInternal, {CanonPath::root}, staticBaseEnv), *vDerivation);
 }
 
 

src/libexpr/primops/fetchMercurial.cc

@@ -31,7 +31,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
                 // be both a revision or a branch/tag name.
                 auto value = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `rev` attribute passed to builtins.fetchMercurial");
                 if (std::regex_match(value.begin(), value.end(), revRegex))
-                    rev = Hash::parseAny(value, htSHA1);
+                    rev = Hash::parseAny(value, HashType::SHA1);
                 else
                     ref = value;
             }
@@ -73,7 +73,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
         attrs2.alloc("branch").mkString(*input2.getRef());
     // Backward compatibility: set 'rev' to
     // 0000000000000000000000000000000000000000 for a dirty tree.
-    auto rev2 = input2.getRev().value_or(Hash(htSHA1));
+    auto rev2 = input2.getRev().value_or(Hash(HashType::SHA1));
     attrs2.alloc("rev").mkString(rev2.gitRev());
     attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12));
     if (auto revCount = input2.getRevCount())

src/libexpr/primops/fetchTree.cc

@@ -32,7 +32,7 @@ void emitTreeAttrs(
 
     auto narHash = input.getNarHash();
     assert(narHash);
-    attrs.alloc("narHash").mkString(narHash->to_string(SRI, true));
+    attrs.alloc("narHash").mkString(narHash->to_string(Base::SRI, true));
 
     if (input.getType() == "git")
         attrs.alloc("submodules").mkBool(
@@ -45,7 +45,7 @@ void emitTreeAttrs(
             attrs.alloc("shortRev").mkString(rev->gitShortRev());
         } else if (emptyRevFallback) {
             // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
-            auto emptyHash = Hash(htSHA1);
+            auto emptyHash = Hash(HashType::SHA1);
             attrs.alloc("rev").mkString(emptyHash.gitRev());
             attrs.alloc("shortRev").mkString(emptyHash.gitShortRev());
         }
@@ -148,9 +148,16 @@ static void fetchTree(
             }
             else if (attr.value->type() == nBool)
                 attrs.emplace(state.symbols[attr.name], Explicit<bool>{attr.value->boolean});
-            else if (attr.value->type() == nInt)
-                attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer));
-            else
+            else if (attr.value->type() == nInt) {
+                auto intValue = attr.value->integer.value;
+
+                if (intValue < 0) {
+                    state.error<EvalError>("negative value given for fetchTree attr %1%: %2%", state.symbols[attr.name], intValue).atPos(pos).debugThrow();
+                }
+                unsigned long asUnsigned = intValue;
+
+                attrs.emplace(state.symbols[attr.name], asUnsigned);
+            } else
                 state.error<TypeError>("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
                     state.symbols[attr.name], showType(*attr.value)).debugThrow();
         }
@@ -219,7 +226,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
             if (n == "url")
                 url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch");
             else if (n == "sha256")
-                expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), htSHA256);
+                expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), HashType::SHA256);
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch");
             else
@@ -245,7 +252,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
         state.error<EvalError>("in pure evaluation mode, '%s' requires a 'sha256' argument", who).atPos(pos).debugThrow();
 
     // early exit if pinned and already in the store
-    if (expectedHash && expectedHash->type == htSHA256) {
+    if (expectedHash && expectedHash->type == HashType::SHA256) {
         auto expectedPath = state.store->makeFixedOutputPath(
             name,
             FixedOutputInfo {
@@ -270,13 +277,13 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
     if (expectedHash) {
         auto hash = unpack
             ? state.store->queryPathInfo(storePath)->narHash
-            : hashFile(htSHA256, state.store->toRealPath(storePath));
+            : hashFile(HashType::SHA256, state.store->toRealPath(storePath));
         if (hash != *expectedHash) {
             state.error<EvalError>(
                 "hash mismatch in file downloaded from '%s':\n  specified: %s\n  got:       %s",
                 *url,
-                expectedHash->to_string(Base32, true),
-                hash.to_string(Base32, true)
+                expectedHash->to_string(Base::Base32, true),
+                hash.to_string(Base::Base32, true)
             ).withExitStatus(102)
             .debugThrow();
         }

src/libexpr/print.cc

@@ -264,22 +264,24 @@ private:
             return true;
         }
 
+        if (options.force) {
+            // The item is going to be forced during printing anyway, but we need its type now.
+            state.forceValue(*item, item->determinePos(noPos));
+        }
+
         // Pretty-print single-item attrsets only if they contain nested
         // structures.
         auto itemType = item->type();
-        return itemType == nList || itemType == nAttrs || itemType == nThunk;
+        return itemType == nList || itemType == nAttrs;
     }
 
     void printAttrs(Value & v, size_t depth)
     {
-        if (seen && !seen->insert(v.attrs).second) {
-            printRepeated();
-            return;
-        }
-
         if (options.force && options.derivationPaths && state.isDerivation(v)) {
             printDerivation(v);
-        } else if (depth < options.maxDepth) {
+        } else if (seen && !v.attrs->empty() && !seen->insert(v.attrs).second) {
+            printRepeated();
+        } else if (depth < options.maxDepth || v.attrs->empty()) {
             increaseIndent();
             output << "{";
 
@@ -335,10 +337,15 @@ private:
             return true;
         }
 
+        if (options.force) {
+            // The item is going to be forced during printing anyway, but we need its type now.
+            state.forceValue(*item, item->determinePos(noPos));
+        }
+
         // Pretty-print single-item lists only if they contain nested
         // structures.
         auto itemType = item->type();
-        return itemType == nList || itemType == nAttrs || itemType == nThunk;
+        return itemType == nList || itemType == nAttrs;
     }
 
     void printList(Value & v, size_t depth)
@@ -348,7 +355,7 @@ private:
             return;
         }
 
-        if (depth < options.maxDepth) {
+        if (depth < options.maxDepth || v.listSize() == 0) {
             increaseIndent();
             output << "[";
             auto listItems = v.listItems();
@@ -574,4 +581,10 @@ fmt_internal::HintFmt & fmt_internal::HintFmt::operator%(const ValuePrinter & va
         return *this;
 }
 
+std::ostream & operator<<(std::ostream & output, ExprPrinter const & printer)
+{
+    printer.expr.show(printer.state.symbols, output);
+    return output;
+}
+
 }

src/libexpr/print.hh

@@ -15,6 +15,8 @@
 
 namespace nix {
 
+struct Expr;
+
 class EvalState;
 struct Value;
 
@@ -50,6 +52,12 @@ void printValue(EvalState & state, std::ostream & str, Value & v, PrintOptions o
 /**
  * A partially-applied form of `printValue` which can be formatted using `<<`
  * without allocating an intermediate string.
+ * This class should not outlive the eval state or it will UAF.
+ * FIXME: This should take `nix::ref`s, to avoid that, but our eval methods all have
+ * EvalState &, not ref<EvalState>, and constructing a new shared_ptr to data that
+ * already has a shared_ptr is a much bigger footgun. In the current architecture of
+ * libexpr, using a ValuePrinter after an EvalState has been destroyed would be
+ * pretty hard.
  */
 class ValuePrinter {
     friend std::ostream & operator << (std::ostream & output, const ValuePrinter & printer);
@@ -73,4 +81,26 @@ std::ostream & operator<<(std::ostream & output, const ValuePrinter & printer);
 template<>
 fmt_internal::HintFmt & fmt_internal::HintFmt::operator%(const ValuePrinter & value);
 
+/**
+ * A partially-applied form of Expr::show(), which can be formatted using `<<`
+ * without allocating an intermediate string.
+ * This class should not outlive the eval state or it will UAF.
+ * FIXME: This should take `nix::ref`s, to avoid that, but our eval methods all have
+ * EvalState &, not ref<EvalState>, and constructing a new shared_ptr to data that
+ * already has a shared_ptr is a much bigger footgun. In the current architecture of
+ * libexpr, using an ExprPrinter after an EvalState has been destroyed would be
+ * pretty hard.
+ */
+class ExprPrinter
+{
+    /** The eval state used to get symbols. */
+    EvalState const & state;
+    /** The expression to print. */
+    Expr const & expr;
+
+public:
+    ExprPrinter(EvalState const & state, Expr const & expr) : state(state), expr(expr) { }
+    friend std::ostream & operator << (std::ostream & output, ExprPrinter const & printer);
+};
+
 }

src/libexpr/value-to-json.cc

@@ -22,7 +22,7 @@ json printValueAsJSON(EvalState & state, bool strict,
     switch (v.type()) {
 
         case nInt:
-            out = v.integer;
+            out = v.integer.value;
             break;
 
         case nBool: