rewrite the parser with pegtl instead of flex/bison

this gives about 20% performance improvements on pure parsing. obviously
it'll be less on full eval, but depending on how much parsing has to be
done (eg whether nixpkgs haskell modules are included or not) it ranges
anywhere from 4% to 10% in our tests.

this has been tested with thousands of core hours of fuzz testing to
ensure that the ASTs produced by the new parser are exactly the same as
the ones produced by the old parser. error messages will
change (sometimes a lot) and are currently not perfect, but we'd rather
leave that open for improvement than having this work rot forever.

Change-Id: Ie66ec2d045dec964632c6541e25f8f0797319ee2

.clang-format

@@ -1,51 +0,0 @@
----
-BasedOnStyle: LLVM
-AccessModifierOffset: -4
-AlignAfterOpenBracket: BlockIndent
-AlignEscapedNewlines: Left
-AlignOperands: DontAlign
-AllowShortBlocksOnASingleLine: Always
-AllowShortFunctionsOnASingleLine: Empty
-AllowShortIfStatementsOnASingleLine: WithoutElse
-AlwaysBreakBeforeMultilineStrings: true
-AlwaysBreakTemplateDeclarations: Yes
-BinPackArguments: false
-BinPackParameters: false
-BitFieldColonSpacing: None
-BraceWrapping:
-  AfterCaseLabel: false
-  AfterClass: true
-  AfterControlStatement: MultiLine
-  AfterEnum: false
-  AfterFunction: true
-  AfterNamespace: false
-  AfterObjCDeclaration: false
-  AfterStruct: true
-  AfterUnion: true
-  AfterExternBlock: false
-  BeforeCatch: false
-  BeforeElse: false
-  BeforeLambdaBody: false
-  BeforeWhile: false
-  IndentBraces: false
-  SplitEmptyFunction: true
-  SplitEmptyRecord: false
-  SplitEmptyNamespace: true
-BreakAfterAttributes: Always
-BreakBeforeBinaryOperators: NonAssignment
-BreakBeforeBraces: Custom
-BreakConstructorInitializers: BeforeComma
-ColumnLimit: 100
-EmptyLineAfterAccessModifier: Leave
-EmptyLineBeforeAccessModifier: Leave
-FixNamespaceComments: false
-IndentWidth: 4
-InsertBraces: true
-InsertTrailingCommas: Wrapped
-LambdaBodyIndentation: Signature
-PackConstructorInitializers: CurrentLine
-PointerAlignment: Middle
-SortIncludes: Never
-SpaceAfterCStyleCast: true
-SpaceAfterTemplateKeyword: false
-

.envrc

@@ -1,9 +0,0 @@
-# shellcheck shell=bash
-source_env_if_exists .envrc.local
-# TODO: `use flake .#native-clangStdenvPackages` on macOS?
-use flake ".#${LIX_SHELL_VARIANT:-default}" "${LIX_SHELL_EXTRA_ARGS[@]}"
-export MAKEFLAGS="$MAKEFLAGS -e"
-if [[ -n "$NIX_BUILD_CORES" ]]; then
-    export MAKEFLAGS="$MAKEFLAGS -j $NIX_BUILD_CORES"
-fi
-export GTEST_BRIEF=1

.github/ISSUE_TEMPLATE/bug_report.md

@@ -25,7 +25,7 @@ you probably want to file an issue at https://github.com/NixOS/nixpkgs/issues.
 
 A clear and concise description of what you expected to happen.
 
-## `nix --version` output
+## `nix-env --version` output
 
 ## Additional context
 

.gitignore

@@ -146,16 +146,9 @@ result
 result-*
 
 .vscode/
-.direnv/
-.envrc.local
 
 # clangd and possibly more
 .cache/
 
 # Mac OS
 .DS_Store
-
-# ClangBuildAnalyzer output, see maintainers/buildtime_report.sh
-buildtime.bin
-
-.envrc.local

Makefile

@@ -20,7 +20,8 @@ makefiles = \
   misc/fish/local.mk \
   misc/zsh/local.mk \
   misc/systemd/local.mk \
-  misc/launchd/local.mk
+  misc/launchd/local.mk \
+  misc/upstart/local.mk
 endif
 
 ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes)
@@ -40,7 +41,6 @@ makefiles += \
   tests/functional/ca/local.mk \
   tests/functional/dyn-drv/local.mk \
   tests/functional/test-libstoreconsumer/local.mk \
-  tests/functional/repl_characterization/local.mk \
   tests/functional/plugins/local.mk
 else
 makefiles += \
@@ -60,7 +60,7 @@ endif
 OPTIMIZE = 1
 
 ifeq ($(OPTIMIZE), 1)
-  GLOBAL_CXXFLAGS += -O2 $(CXXLTO)
+  GLOBAL_CXXFLAGS += -O3 $(CXXLTO)
   GLOBAL_LDFLAGS += $(CXXLTO)
 else
   GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE

clang-tidy/.clang-format

@@ -1 +0,0 @@
-BasedOnStyle: llvm

clang-tidy/HasPrefixSuffix.cc

@@ -1,80 +0,0 @@
-#include "HasPrefixSuffix.hh"
-#include <clang/AST/ASTTypeTraits.h>
-#include <clang/AST/Expr.h>
-#include <clang/AST/PrettyPrinter.h>
-#include <clang/AST/Type.h>
-#include <clang/ASTMatchers/ASTMatchers.h>
-#include <clang/Basic/Diagnostic.h>
-#include <clang/Frontend/FrontendAction.h>
-#include <clang/Frontend/FrontendPluginRegistry.h>
-#include <clang/Tooling/Transformer/SourceCode.h>
-#include <clang/Tooling/Transformer/SourceCodeBuilders.h>
-#include <iostream>
-
-namespace nix::clang_tidy {
-using namespace clang::ast_matchers;
-using namespace clang;
-
-void HasPrefixSuffixCheck::registerMatchers(ast_matchers::MatchFinder *Finder) {
-  Finder->addMatcher(
-      traverse(clang::TK_AsIs,
-               callExpr(callee(functionDecl(anyOf(hasName("hasPrefix"),
-                                                  hasName("hasSuffix")))
-                                   .bind("callee-decl")),
-                        optionally(hasArgument(
-                            0, cxxConstructExpr(
-                                   hasDeclaration(functionDecl(hasParameter(
-                                       0, parmVarDecl(hasType(
-                                              asString("const char *")))))))
-                                   .bind("implicit-cast"))))
-                   .bind("call")),
-      this);
-}
-
-void HasPrefixSuffixCheck::check(
-    const ast_matchers::MatchFinder::MatchResult &Result) {
-
-  const auto *CalleeDecl = Result.Nodes.getNodeAs<FunctionDecl>("callee-decl");
-  auto FuncName = std::string(CalleeDecl->getName());
-  std::string NewName;
-  if (FuncName == "hasPrefix") {
-    NewName = "starts_with";
-  } else if (FuncName == "hasSuffix") {
-    NewName = "ends_with";
-  } else {
-    llvm_unreachable("nix-has-prefix: invalid callee");
-  }
-
-  const auto *MatchedDecl = Result.Nodes.getNodeAs<CallExpr>("call");
-  const auto *ImplicitConvertArg =
-      Result.Nodes.getNodeAs<CXXConstructExpr>("implicit-cast");
-
-  const auto *Lhs = MatchedDecl->getArg(0);
-  const auto *Rhs = MatchedDecl->getArg(1);
-  auto Diag = diag(MatchedDecl->getExprLoc(), FuncName + " is deprecated");
-
-  std::string Text = "";
-
-  // Form possible cast to string_view, or nothing.
-  if (ImplicitConvertArg) {
-    Text = "std::string_view(";
-    Text.append(tooling::getText(*Lhs, *Result.Context));
-    Text.append(").");
-  } else {
-    Text.append(*tooling::buildAccess(*Lhs, *Result.Context));
-  }
-
-  // Call .starts_with.
-  Text.append(NewName);
-  Text.push_back('(');
-  Text.append(tooling::getText(*Rhs, *Result.Context));
-  Text.push_back(')');
-
-  Diag << FixItHint::CreateReplacement(MatchedDecl->getSourceRange(), Text);
-
-  // for (const auto *arg : MatchedDecl->arguments()) {
-  //   arg->dumpColor();
-  //   arg->getType().dump();
-  // }
-}
-}; // namespace nix::clang_tidy

clang-tidy/HasPrefixSuffix.hh

@@ -1,25 +0,0 @@
-#pragma once
-///@file
-/// This is an example of a clang-tidy automated refactoring against the Nix
-/// codebase. The refactoring has been completed in
-/// https://gerrit.lix.systems/c/lix/+/565 so this code is around as
-/// an example.
-
-#include <clang-tidy/ClangTidyCheck.h>
-#include <clang/ASTMatchers/ASTMatchFinder.h>
-#include <llvm/ADT/StringRef.h>
-
-namespace nix::clang_tidy {
-
-using namespace clang;
-using namespace clang::tidy;
-using namespace llvm;
-
-class HasPrefixSuffixCheck : public ClangTidyCheck {
-public:
-  HasPrefixSuffixCheck(StringRef Name, ClangTidyContext *Context)
-      : ClangTidyCheck(Name, Context) {}
-  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
-  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
-};
-}; // namespace nix::clang_tidy

clang-tidy/NixClangTidyChecks.cc

@@ -1,17 +0,0 @@
-#include <clang-tidy/ClangTidyModule.h>
-#include <clang-tidy/ClangTidyModuleRegistry.h>
-#include "HasPrefixSuffix.hh"
-
-namespace nix::clang_tidy {
-using namespace clang;
-using namespace clang::tidy;
-
-class NixClangTidyChecks : public ClangTidyModule {
-    public:
-        void addCheckFactories(ClangTidyCheckFactories &CheckFactories) override {
-            CheckFactories.registerCheck<HasPrefixSuffixCheck>("nix-hasprefixsuffix");
-        }
-};
-
-static ClangTidyModuleRegistry::Add<NixClangTidyChecks> X("nix-module", "Adds nix specific checks");
-};

clang-tidy/README.md

@@ -1,56 +0,0 @@
-# Clang tidy lints for Nix
-
-This is a skeleton of a clang-tidy lints library for Nix.
-
-Currently there is one check (which is already obsolete as it has served its
-goal and is there as an example), `HasPrefixSuffixCheck`.
-
-## Running fixes/checks
-
-One file:
-
-```
-ninja -C build && clang-tidy --checks='-*,nix-*' --load=build/libnix-clang-tidy.so -p ../compile_commands.json --fix ../src/libcmd/installables.cc
-```
-
-Several files, in parallel:
-
-```
-ninja -C build && run-clang-tidy -checks='-*,nix-*' -load=build/libnix-clang-tidy.so -p .. -fix ../src | tee -a clang-tidy-result
-```
-
-## Resources
-
-* https://firefox-source-docs.mozilla.org/code-quality/static-analysis/writing-new/clang-query.html
-* https://clang.llvm.org/docs/LibASTMatchersReference.html
-* https://devblogs.microsoft.com/cppblog/exploring-clang-tooling-part-3-rewriting-code-with-clang-tidy/
-
-## Developing new checks
-
-Put something like so in `myquery.txt`:
-
-```
-set traversal     IgnoreUnlessSpelledInSource
-# ^ Ignore implicit AST nodes. May need to use AsIs depending on how you are
-# working.
-set bind-root     true
-# ^ true unless you use any .bind("foo") commands
-set print-matcher true
-enable output     dump
-match callExpr(callee(functionDecl(hasName("hasPrefix"))), optionally(hasArgument( 0, cxxConstructExpr(hasDeclaration(functionDecl(hasParameter(0, parmVarDecl(hasType(asString("const char *"))).bind("meow2"))))))))
-```
-
-Then run, e.g. `clang-query --preload hasprefix.query -p compile_commands.json src/libcmd/installables.cc`.
-
-With this you can iterate a query before writing it in C++ and suffering from
-C++.
-
-### Tips and tricks for the C++
-
-There is a function `dump()` on many things that will dump to stderr. Also
-`llvm::errs()` lets you print to stderr.
-
-When I wrote `HasPrefixSuffixCheck`, I was not really able to figure out how
-the structured replacement system was supposed to work. In principle you can
-describe the replacement with a nice DSL. Look up the Stencil system in Clang
-for details.

clang-tidy/meson.build

@@ -1,8 +0,0 @@
-project('nix-clang-tidy', ['cpp', 'c'],
-  version : '0.1',
-  default_options : ['warning_level=3', 'cpp_std=c++20'])
-
-llvm = dependency('Clang', version: '>= 14', modules: ['libclang'])
-sources = ['HasPrefixSuffix.cc', 'NixClangTidyChecks.cc']
-shared_module('nix-clang-tidy', sources,
-  dependencies: llvm)

configure.ac

@@ -342,13 +342,16 @@ AC_SUBST(doc_generate)
 # Look for lowdown library.
 PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"])
 
-# Look for toml11, a required dependency.
-AC_ARG_VAR([TOML11_HEADERS], [include path of toml11 headers])
+# Look for pegtl.
+# pegtl has only cmake support, no pkg-config.
+AC_ARG_VAR([PEGTL_HEADERS], [include path of pegtl headers])
 AC_LANG_PUSH(C++)
-[CXXFLAGS="-I $TOML11_HEADERS $CXXFLAGS"]
-AC_CHECK_HEADER([toml.hpp], [], [AC_MSG_ERROR([toml11 is not found.])])
+AC_SUBST(PEGTL_HEADERS)
+[CXXFLAGS="-I $PEGTL_HEADERS $CXXFLAGS"]
+AC_CHECK_HEADER(tao/pegtl.hpp, [], [AC_MSG_ERROR([PEGTL not found.])])
 AC_LANG_POP(C++)
 
+
 # Setuid installations.
 AC_CHECK_FUNCS([setresuid setreuid lchown])
 

doc/manual/local.mk

@@ -151,9 +151,9 @@ $(d)/language.json: $(doc_nix)
 
 # Generate "Upcoming release" notes (or clear it and remove from menu)
 $(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/*
-	@if type -p build-release-notes > /dev/null; then \
+	@if type -p changelog-d > /dev/null; then \
 		echo "  GEN   " $@; \
-		build-release-notes doc/manual/rl-next > $@; \
+		changelog-d doc/manual/rl-next > $@; \
 	else \
 		echo "  NULL  " $@; \
 		true > $@; \

doc/manual/rl-next-dev/build-timing-analysis.md

@@ -1,15 +0,0 @@
----
-synopsis: Clang build timing analysis
-cls: 587
----
-
-We now have Clang build profiling available, which generates Chrome
-tracing files for each compilation unit. To enable it, run `meson configure
-build -Dprofile-build=enabled` then rerun the compilation.
-
-If you want to make the build go faster, do a clang build with meson, then run
-`maintainers/buildtime_report.sh build`, then contemplate how to improve the
-build time.
-
-You can also look at individual object files' traces in
-<https://ui.perfetto.dev>.

doc/manual/rl-next/config

@@ -0,0 +1,2 @@
+organization: NixOS
+repository: nix

doc/manual/rl-next/debugger-locals-for-let-expressions.md

@@ -1,7 +1,7 @@
 ---
 synopsis: "`--debugger` can now access bindings from `let` expressions"
 prs: 9918
-issues: 8827
+issues: 8827.
 ---
 
 Breakpoints and errors in the bindings of a `let` expression can now access

doc/manual/rl-next/drop-vendored-toml11.md

@@ -1,7 +0,0 @@
----
-synopsis: Stop vendoring toml11
-cls: 675
----
-
-We don't apply any patches to it, and vendoring it locks users into
-bugs (it hasn't been updated since its introduction in late 2021).

doc/manual/rl-next/dup-attr-errors.md

@@ -1,22 +0,0 @@
----
-synopsis: Duplicate attribute reports are more accurate
-cls: 557
----
-
-Duplicate attribute errors are now more accurate, showing the path at which an error was detected rather than the full, possibly longer, path that caused the error.
-Error reports are now
-```ShellSession
-$ nix eval --expr '{ a.b = 1; a.b.c.d = 1; }'
-error: attribute 'a.b' already defined at «string»:1:3
-       at «string»:1:12:
-            1| { a.b = 1; a.b.c.d = 1;
-             |            ^
-```
-instead of
-```ShellSession
-$ nix eval --expr '{ a.b = 1; a.b.c.d = 1; }'
-error: attribute 'a.b.c.d' already defined at «string»:1:3
-       at «string»:1:12:
-            1| { a.b = 1; a.b.c.d = 1;
-             |            ^
-```

doc/manual/rl-next/empty-search-regex.md

@@ -1,6 +1,8 @@
----
 synopsis: Disallow empty search regex in `nix search`
-prs: 9481
----
+prs: #9481
+description: {
 
 [`nix search`](@docroot@/command-ref/new-cli/nix3-search.md) now requires a search regex to be passed. To show all packages, use `^`.
+
+}
+

doc/manual/rl-next/nix-flake-update-ux.md

@@ -1,17 +0,0 @@
----
-synopsis: "`Overhaul `nix flake update` and `nix flake lock` UX"
-prs: 8817
----
-
-The interface for creating and updating lock files has been overhauled:
-
-- [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now.
-It will *never* update existing inputs.
-
-- [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) does the same, but *will* update inputs.
-- Passing no arguments will update all inputs of the current flake, just like it already did.
-- Passing input names as arguments will ensure only those are updated. This replaces the functionality of `nix flake lock --update-input`
-- To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`.
-
-- The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables.
-They are superceded by `nix flake update`.

doc/manual/rl-next/nixversion-fake.md

@@ -1,11 +0,0 @@
----
-synopsis: "`builtins.nixVersion` now returns a fixed value \"2.18.3-lix\""
-cls: 558
----
-
-`builtins.nixVersion` now returns a fixed value `"2.18.3-lix"`. This prevents
-feature detection assuming that features that exist in Nix post-Lix-branch-off
-might exist, even though the Lix version is greater than the Nix version.
-
-In the future, check for builtins for feature detection. If a feature cannot be
-detected by *those* means, please file a Lix bug.

doc/manual/rl-next/print-value-in-coercion-error.md

@@ -1,7 +1,7 @@
 ---
 synopsis: Coercion errors include the failing value
-issues: 561
-prs: 9754
+issues: #561
+prs: #9754
 ---
 
 The `error: cannot coerce a <TYPE> to a string` message now includes the value

doc/manual/rl-next/print-value-in-type-error.md

@@ -1,7 +1,7 @@
 ---
 synopsis: Type errors include the failing value
-issues: 561
-prs: 9753
+issues: #561
+prs: #9753
 ---
 
 In errors like `value is an integer while a list was expected`, the message now

doc/manual/rl-next/short-expr-flag.md

@@ -1,14 +0,0 @@
----
-synopsis: reintroduce shortened `-E` form for `--expr` to new CLI
-cls: 605
----
-
-In the past, it was possible to supply a shorter `-E` flag instead of fully
-specifying `--expr` every time you wished to provide an expression that would
-be evaluated to produce the given command's input. This was retained for the
-`--file` flag when the new CLI utilities were written with `-f`, but `-E` was
-dropped.
-
-We now restore the `-E` short form for better UX. This is most useful for
-`nix eval` but most any command that takes an Installable argument should benefit
-from it as well.

doc/manual/rl-next/upstart-removal.md

@@ -1,8 +0,0 @@
----
-synopsis: Upstart scripts removed
-cls: 574
----
-
-Upstart scripts have been removed from Lix, since Upstart is obsolete and has
-not been shipped by any major distributions for many years. If these are
-necessary to your use case, please back port them to your packaging.

doc/manual/src/contributing/hacking.md

@@ -340,6 +340,9 @@ Significant changes should add the following header, which moves them to the top
 significance: significant
 ```
 
+<!-- Keep an eye on https://codeberg.org/fgaz/changelog-d/issues/1 -->
+See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog).
+
 ### Build process
 
 Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`.

flake.nix

@@ -150,9 +150,7 @@
           # Forward from the previous stage as we don’t want it to pick the lowdown override
           nixUnstable = prev.nixUnstable;
 
-          build-release-notes =
-            final.buildPackages.callPackage ./maintainers/build-release-notes.nix { };
-          clangbuildanalyzer = final.buildPackages.callPackage ./misc/clangbuildanalyzer.nix { };
+          changelog-d = final.buildPackages.callPackage ./misc/changelog-d.nix { };
           boehmgc-nix = (final.boehmgc.override {
             enableLargeConfig = true;
           }).overrideAttrs (o: {
@@ -194,6 +192,25 @@
             boehmgc = final.boehmgc-nix;
             busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
           };
+
+          pegtl = final.callPackage (
+            { stdenv, cmake, ninja }:
+
+            stdenv.mkDerivation {
+              pname = "pegtl";
+              version = "3.2.7";
+
+              src = final.fetchFromGitHub {
+                repo = "PEGTL";
+                owner = "taocpp";
+                rev = "refs/tags/3.2.7";
+                hash = "sha256-IV5YNGE4EWVrmg2Sia/rcU8jCuiBynQGJM6n3DCWTQU=";
+              };
+
+              nativeBuildInputs = [ cmake ninja ];
+            }
+          ) {};
+
         };
 
     in {
@@ -206,23 +223,6 @@
         # Binary package for various platforms.
         build = forAllSystems (system: self.packages.${system}.nix);
 
-        # FIXME(Qyriad): remove this when the migration to Meson has been completed.
-        # NOTE: mesonBuildClang depends on mesonBuild depends on build to avoid OOMs
-        # on aarch64 builders caused by too many parallel compiler/linker processes.
-        mesonBuild = forAllSystems (system:
-          (self.packages.${system}.nix.override {
-            buildWithMeson = true;
-          }).overrideAttrs (prev: {
-            buildInputs = prev.buildInputs ++ [ self.packages.${system}.nix ];
-          }));
-        mesonBuildClang = forAllSystems (system:
-          (nixpkgsFor.${system}.stdenvs.clangStdenvPackages.nix.override {
-              buildWithMeson = true;
-          }).overrideAttrs (prev: {
-            buildInputs = prev.buildInputs ++ [ self.hydraJobs.mesonBuild.${system} ];
-          })
-        );
-
         # Perl bindings for various platforms.
         perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings);
 
@@ -240,7 +240,7 @@
 
           nix = pkgs.callPackage ./package.nix {
             inherit versionSuffix fileset officialRelease buildUnreleasedNotes;
-            inherit (pkgs) build-release-notes;
+            inherit (pkgs) changelog-d;
             internalApiDocs = true;
             boehmgc = pkgs.boehmgc-nix;
             busybox-sandbox-shell = pkgs.busybox-sandbox-shell;
@@ -280,21 +280,15 @@
         };
       };
 
-      checks = forAllSystems (system: let
-        rl-next-check = name: dir:
-          let pkgs = nixpkgsFor.${system}.native;
-          in pkgs.buildPackages.runCommand "test-${name}-release-notes" { } ''
-          LANG=C.UTF-8 ${lib.getExe pkgs.build-release-notes} ${dir} >$out
-        '';
-      in {
-        # FIXME(Qyriad): remove this when the migration to Meson has been completed.
-        mesonBuild = self.hydraJobs.mesonBuild.${system};
-        mesonBuildClang = self.hydraJobs.mesonBuildClang.${system};
+      checks = forAllSystems (system: {
         binaryTarball = self.hydraJobs.binaryTarball.${system};
         perlBindings = self.hydraJobs.perlBindings.${system};
         nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system};
-        rl-next = rl-next-check "rl-next" ./doc/manual/rl-next;
-        rl-next-dev = rl-next-check "rl-next-dev" ./doc/manual/rl-next-dev;
+        rl-next =
+          let pkgs = nixpkgsFor.${system}.native;
+          in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } ''
+          LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out
+        '';
       } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
         dockerImage = self.hydraJobs.dockerImage.${system};
       });
@@ -341,35 +335,21 @@
               forDevShell = true;
             };
           in
-            (nix.override {
-              buildUnreleasedNotes = true;
-              officialRelease = false;
-            }).overrideAttrs (prev: {
-              # Required for clang-tidy checks
-              buildInputs = prev.buildInputs ++ lib.optionals (stdenv.cc.isClang) [ pkgs.llvmPackages.llvm pkgs.llvmPackages.clang-unwrapped.dev ];
+            nix.overrideAttrs (prev: {
               nativeBuildInputs = prev.nativeBuildInputs
                 ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear
-                # Required for clang-tidy checks
-                ++ lib.optionals (stdenv.cc.isClang) [ pkgs.buildPackages.cmake pkgs.buildPackages.ninja pkgs.buildPackages.llvmPackages.llvm.dev ]
                 ++ lib.optional
                   (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform)
-                  # for some reason that seems accidental and was changed in
-                  # NixOS 24.05-pre, clang-tools is pinned to LLVM 14 when
-                  # default LLVM is newer.
-                  (pkgs.buildPackages.clang-tools.override { inherit (pkgs.buildPackages) llvmPackages; })
-                ++ [
-                  # FIXME(Qyriad): remove once the migration to Meson is complete.
-                  pkgs.buildPackages.meson
-                  pkgs.buildPackages.ninja
-
-                  pkgs.buildPackages.clangbuildanalyzer
-                ];
+                  pkgs.buildPackages.clang-tools;
 
               src = null;
 
               installFlags = "sysconfdir=$(out)/etc";
               strictDeps = false;
 
+              # Required to make non-NixOS Linux not complain about missing locale files during configure in a dev shell
+              ${if stdenv.isLinux then "LOCALE_ARCHIVE" else null} = "${pkgs.glibcLocales}/lib/locale/locale-archive";
+
               shellHook = ''
                 PATH=$prefix/bin:$PATH
                 unset PYTHONPATH
@@ -378,9 +358,6 @@
                 # Make bash completion work.
                 XDG_DATA_DIRS+=:$out/share
               '';
-            } // lib.optionalAttrs (stdenv.buildPlatform.isLinux && pkgs.glibcLocales != null) {
-              # Required to make non-NixOS Linux not complain about missing locale files during configure in a dev shell
-              LOCALE_ARCHIVE = "${lib.getLib pkgs.glibcLocales}/lib/locale/locale-archive";
             });
         in
         forAllSystems (system:

local.mk

@@ -1,11 +1,4 @@
-# 2024-03-24: jade benchmarked the default sanitize reporting in clang and got
-# a regression of about 10% on hackage-packages.nix with clang. So we are trapping instead.
-#
-# This has an overhead of 0-4% on gcc and unmeasurably little on clang, in
-# Nix evaluation benchmarks.
-DEFAULT_SANITIZE_FLAGS = -fsanitize=signed-integer-overflow -fsanitize-undefined-trap-on-error
-GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch $(DEFAULT_SANITIZE_FLAGS)
-GLOBAL_LDFLAGS += $(DEFAULT_SANITIZE_FLAGS)
+GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch
 # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers.
 ERROR_SWITCH_ENUM = -Werror=switch-enum
 

maintainers/build-release-notes.nix

@@ -1,6 +0,0 @@
-{ lib, python3, writeShellScriptBin }:
-
-writeShellScriptBin "build-release-notes" ''
-  exec ${lib.getExe (python3.withPackages (p: [ p.python-frontmatter ]))} \
-    ${./build-release-notes.py} "$@"
-''

maintainers/build-release-notes.py

@@ -1,66 +0,0 @@
-import frontmatter
-import sys
-import pathlib
-import textwrap
-
-GH_BASE = "https://github.com/NixOS/nix"
-FORGEJO_BASE = "https://git.lix.systems/lix-project/lix"
-GERRIT_BASE = "https://gerrit.lix.systems/c/lix/+"
-
-SIGNIFICANCECES = {
-    None: 0,
-    'significant': 10,
-}
-
-def format_link(ident: str, gh_part: str, fj_part: str) -> str:
-    # FIXME: deprecate github as default
-    if ident.isdigit():
-        num, link, base = int(ident), f"#{ident}", f"{GH_BASE}/{gh_part}"
-    elif ident.startswith("gh#"):
-        num, link, base = int(ident[3:]), ident, f"{GH_BASE}/{gh_part}"
-    elif ident.startswith("fj#"):
-        num, link, base = int(ident[3:]), ident, f"{FORGEJO_BASE}/{fj_part}"
-    else:
-        raise Exception("unrecognized reference format", ident)
-    return f"[{link}]({base}/{num})"
-
-def format_issue(issue: str) -> str:
-    return format_link(issue, "issues", "issues")
-def format_pr(pr: str) -> str:
-    return format_link(pr, "pull", "pulls")
-def format_cl(cl: str) -> str:
-    clid = int(cl)
-    return f"[cl/{clid}]({GERRIT_BASE}/{clid})"
-
-paths = pathlib.Path(sys.argv[1]).glob('*.md')
-entries = []
-for p in paths:
-    try:
-        e = frontmatter.load(p)
-        if 'synopsis' not in e.metadata:
-            raise Exception('missing synposis')
-        unknownKeys = set(e.metadata.keys()) - set(('synopsis', 'cls', 'issues', 'prs', 'significance'))
-        if unknownKeys:
-            raise Exception('unknown keys', unknownKeys)
-        entries.append((p, e))
-    except Exception as e:
-        e.add_note(f"in {p}")
-        raise
-
-for p, entry in sorted(entries, key=lambda e: (-SIGNIFICANCECES[e[1].metadata.get('significance')], e[0])):
-    try:
-        header = entry.metadata['synopsis']
-        links = []
-        links += map(format_issue, str(entry.metadata.get('issues', "")).split())
-        links += map(format_pr, str(entry.metadata.get('prs', "")).split())
-        links += map(format_cl, str(entry.metadata.get('cls', "")).split())
-        if links != []:
-            header += " " + " ".join(links)
-        if header:
-            print(f"- {header}")
-            print()
-        print(textwrap.indent(entry.content, '  '))
-        print()
-    except Exception as e:
-        e.add_note(f"in {p}")
-        raise

maintainers/buildtime_report.sh

@@ -1,20 +0,0 @@
-#!/bin/sh
-
-# Generates a report of build time based on a meson build using -ftime-trace in
-# Clang.
-if [ $# -lt 1 ]; then
-    echo "usage: $0 BUILD-DIR [filename]" >&2
-    exit 1
-fi
-
-scriptdir=$(cd "$(dirname -- "$0")" || exit ; pwd -P)
-filename=${2:-$scriptdir/../buildtime.bin}
-
-if [ "$(meson introspect "$1" --buildoptions | jq -r '.[]  | select(.name == "profile-build") | .value')" != enabled ]; then
-    echo 'This build was not done with profile-build enabled, so cannot generate a report' >&2
-    # shellcheck disable=SC2016
-    echo 'Run `meson configure build -Dprofile-build=enabled` then rebuild, first' >&2
-    exit 1
-fi
-
-ClangBuildAnalyzer --all "$1" "$filename" && ClangBuildAnalyzer --analyze "$filename"

maintainers/issue_import.py

@@ -1,148 +0,0 @@
-import requests
-import textwrap
-import dataclasses
-import logging
-import re
-import os
-
-API_BASE = 'https://git.lix.systems/api/v1'
-API_KEY = os.environ['FORGEJO_API_KEY']
-
-log = logging.getLogger(__name__)
-log.setLevel(logging.INFO)
-
-fmt = logging.Formatter('{asctime} {levelname} {name}: {message}',
-                        datefmt='%b %d %H:%M:%S',
-                        style='{')
-
-if not any(isinstance(h, logging.StreamHandler) for h in log.handlers):
-    hand = logging.StreamHandler()
-    hand.setFormatter(fmt)
-    log.addHandler(hand)
-
-# These are erring in the direction of re-triage, rather than necessarily
-# mapping all metadata of the issue
-LABEL_MAPPING = {
-    'lix-import': 153, # 'imported',
-    'contributor-experience': 148, # 'devx',
-    'bug': 150, # 'bug',
-    'UX': 149, # 'ux',
-    'error-messages': 149, # 'ux',
-    'lix-stability': 146, # 'stability',
-    'performance': 147, # 'performance',
-    'tests': 121, # 'tests',
-}
-
-def api(method, endpoint: str, resp_json=True, **kwargs):
-    log.info('http %s %s', method, endpoint)
-    if not endpoint.startswith('https'):
-        endpoint = API_BASE + endpoint
-    resp = requests.request(method,
-                            endpoint,
-                            headers={'Authorization': f'Bearer {API_KEY}'},
-                            **kwargs)
-    resp.raise_for_status()
-    if resp_json:
-        return resp.json()
-    else:
-        return resp
-
-def paginate(method: str, url: str):
-    while True:
-        resp = api(method, url, resp_json=False)
-        yield from resp.json()
-        next_one = resp.links.get('next')
-        if not next_one:
-            return
-        url = next_one.get('url')
-        if not url:
-            return
-
-class DataClassUnpack:
-    """Taken from: https://stackoverflow.com/a/72164665"""
-    classFieldCache = {}
-
-    @classmethod
-    def instantiate(cls, classToInstantiate, argDict):
-        if classToInstantiate not in cls.classFieldCache:
-            cls.classFieldCache[classToInstantiate] = {
-                f.name
-                for f in getattr(classToInstantiate, dataclasses._FIELDS).values() if f._field_type is not dataclasses._FIELD_CLASSVAR # type: ignore
-            }
-
-        fieldSet = cls.classFieldCache[classToInstantiate]
-        filteredArgDict = {k: v for k, v in argDict.items() if k in fieldSet}
-        return classToInstantiate(**filteredArgDict)
-
-@dataclasses.dataclass
-class Label:
-    name: str
-    description: str
-
-@dataclasses.dataclass
-class Issue:
-    number: int
-    url: str
-    html_url: str
-    title: str
-    body: str
-    labels: dataclasses.InitVar[list[dict]]
-    labels_clean: list[Label] = dataclasses.field(init=False)
-
-    def __post_init__(self, labels):
-        self.labels_clean = [DataClassUnpack.instantiate(Label, l) for l in labels]
-
-def issues_to_import():
-    yield from paginate('GET', '/repos/nixos/nix/issues?state=open&labels=lix-import')
-
-def issues_already_imported():
-    yield from paginate('GET', '/repos/lix-project/lix/issues?state=all&labels=imported')
-
-
-UPSTREAM_ISSUE_RE = re.compile(r'^Upstream-Issue: https://git\.lix\.systems/NixOS/nix/issues/(\d+)$', re.MULTILINE)
-
-def make_already_imported():
-    d = {}
-    for issue in issues_already_imported():
-        iss = DataClassUnpack.instantiate(Issue, issue)
-        print(iss)
-        match = UPSTREAM_ISSUE_RE.search(iss.body)
-        if match:
-            d[int(match.group(1))] = iss
-
-    return d
-
-def new_issue(title, body, labels):
-    api('POST', '/repos/lix-project/lix/issues', resp_json=True, json={
-        'labels': labels,
-        'body': body,
-        'title': title,
-        'dont_notify': True,
-    })
-
-already_imported = make_already_imported()
-
-def import_issue(iss: Issue):
-    if iss.number in already_imported:
-        log.info('Skipping already imported %d', iss.number)
-        return
-    new_body = textwrap.dedent('''
-        Upstream-Issue: {iss}
-
-        {original_body}
-    ''').format(iss=iss.html_url, original_body=iss.body)
-
-    new_labels = [LABEL_MAPPING[l.name] for l in iss.labels_clean if l.name in LABEL_MAPPING]
-
-    new_title = '[Nix#{num}] {title}'.format(num=iss.number, title=iss.title)
-
-    log.info('%s', f'create issue with: {new_labels} {new_title} {new_body}')
-    new_issue(new_title, new_body, new_labels)
-
-def go():
-    log.info('Importing issues!')
-    for issue in issues_to_import():
-        import_issue(DataClassUnpack.instantiate(Issue, issue))
-
-if __name__ == '__main__':
-    go()

maintainers/release-notes

@@ -152,7 +152,7 @@ section_title="Release $version_full ($DATE)"
   # TODO add minor number, and append?
   echo "# $section_title"
   echo
-  build-release-notes doc/manual/rl-next
+  changelog-d doc/manual/rl-next | sed -e 's/ *$//'
 ) | tee -a $file
 
 log "Wrote $file"

meson.build

@@ -1,363 +0,0 @@
-#
-# OUTLINE:
-#
-# The top-level meson.build file (this file) handles general logic for build options,
-# generation of config.h (which is put in the build directory, not the source root
-# like the previous, autoconf-based build system did), the mechanism for header
-# generation, and the few global C++ compiler arguments that are added to all targets in Lix.
-#
-# src/meson.build coordinates each of Lix's subcomponents (the lib dirs in ./src),
-# which each have their own meson.build. Lix's components depend on each other,
-# so each of `src/lib{util,store,fetchers,expr,main,cmd}/meson.build` rely on variables
-# set in earlier `meson.build` files. Each of these also defines the install targets for
-# their headers.
-#
-# src/meson.build also collects the miscellaneous source files that are in further subdirectories
-# that become part of the final Nix command (things like `src/nix-build/*.cc`).
-#
-# Finally, src/nix/meson.build defines the Nix command itself, relying on all prior meson files.
-#
-# Unit tests are setup in tests/unit/meson.build, under the test suite "check".
-#
-# Functional tests are a bit more complicated. Generally they're defined in
-# tests/functional/meson.build, and rely on helper scripts meson/setup-functional-tests.py
-# and meson/run-test.py. Scattered around also are configure_file() invocations, which must
-# be placed in specific directories' meson.build files to create the right directory tree
-# in the build directory.
-
-project('lix', 'cpp',
-  version : run_command('bash', '-c', 'echo -n $(cat ./.version)$VERSION_SUFFIX', check : true).stdout().strip(),
-  default_options : [
-    'cpp_std=c++2a',
-    # TODO(Qyriad): increase the warning level
-    'warning_level=1',
-    'debug=true',
-    'optimization=2',
-    'errorlogs=true', # Please print logs for tests that fail
-  ],
-)
-
-fs = import('fs')
-
-prefix = get_option('prefix')
-# For each of these paths, assume that it is relative to the prefix unless
-# it is already an absolute path (which is the default for store-dir, state-dir, and log-dir).
-path_opts = [
-  # Meson built-ins.
-  'datadir',
-  'sysconfdir',
-  'bindir',
-  'mandir',
-  'libdir',
-  'includedir',
-  # Homecooked Lix directories.
-  'store-dir',
-  'state-dir',
-  'log-dir',
-]
-# For your grepping pleasure, this loop sets the following variables that aren't mentioned
-# literally above:
-# store_dir
-# state_dir
-# log_dir
-foreach optname : path_opts
-  varname = optname.replace('-', '_')
-  path = get_option(optname)
-  if fs.is_absolute(path)
-    set_variable(varname, path)
-  else
-    set_variable(varname, prefix / path)
-  endif
-endforeach
-
-
-enable_tests = get_option('enable-tests')
-
-tests_args = []
-
-if get_option('tests-color')
-  tests_args += '--gtest_color=yes'
-endif
-
-if get_option('tests-brief')
-  tests_args += '--gtest_brief=1'
-endif
-
-
-cxx = meson.get_compiler('cpp')
-
-host_system = host_machine.cpu_family() + '-' + host_machine.system()
-message('canonical Nix system name:', host_system)
-
-is_linux = host_machine.system() == 'linux'
-is_x64 = host_machine.cpu_family() == 'x86_64'
-
-deps = [ ]
-configdata = { }
-
-#
-# Dependencies
-#
-
-boehm = dependency('bdw-gc', required : get_option('gc'))
-if boehm.found()
-  deps += boehm
-endif
-configdata += {
-  'HAVE_BOEHMGC': boehm.found().to_int(),
-}
-
-boost = dependency('boost', required : true, modules : ['context', 'coroutine', 'container'])
-deps += boost
-
-# cpuid only makes sense on x86_64
-cpuid_required = is_x64 ? get_option('cpuid') : false
-cpuid = dependency('libcpuid', 'cpuid', required : cpuid_required)
-configdata += {
-  'HAVE_LIBCPUID': cpuid.found().to_int(),
-}
-deps += cpuid
-
-# seccomp only makes sense on Linux
-seccomp_required = is_linux ? get_option('seccomp-sandboxing') : false
-seccomp = dependency('libseccomp', 'seccomp', required : seccomp_required)
-configdata += {
-  'HAVE_SECCOMP': seccomp.found().to_int(),
-}
-
-libarchive = dependency('libarchive', required : true)
-deps += libarchive
-
-brotli = [
-  dependency('libbrotlicommon', required : true),
-  dependency('libbrotlidec', required : true),
-  dependency('libbrotlienc', required : true),
-]
-deps += brotli
-
-openssl = dependency('libcrypto', 'openssl', required : true)
-deps += openssl
-
-aws_sdk = dependency('aws-cpp-sdk-core', required : false)
-if aws_sdk.found()
-  # The AWS pkg-config adds -std=c++11.
-  # https://github.com/aws/aws-sdk-cpp/issues/2673
-  aws_sdk = aws_sdk.partial_dependency(
-    compile_args : false,
-    includes : true,
-    link_args : true,
-    links : true,
-    sources : true,
-  )
-  deps += aws_sdk
-  s = aws_sdk.version().split('.')
-  configdata += {
-    'AWS_VERSION_MAJOR': s[0].to_int(),
-    'AWS_VERSION_MINOR': s[1].to_int(),
-    'AWS_VERSION_PATCH': s[2].to_int(),
-  }
-  aws_sdk_transfer = dependency('aws-cpp-sdk-transfer', required : true).partial_dependency(
-    compile_args : false,
-    includes : true,
-    link_args : true,
-    links : true,
-    sources : true,
-  )
-endif
-
-aws_s3 = dependency('aws-cpp-sdk-s3', required : false)
-if aws_s3.found()
-  # The AWS pkg-config adds -std=c++11.
-  # https://github.com/aws/aws-sdk-cpp/issues/2673
-  aws_s3 = aws_s3.partial_dependency(
-    compile_args : false,
-    includes : true,
-    link_args : true,
-    links : true,
-    sources : true,
-  )
-  deps += aws_s3
-endif
-
-configdata += {
-  'ENABLE_S3': aws_s3.found().to_int(),
-}
-
-sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19', required : true)
-deps += sqlite
-
-sodium = dependency('libsodium', 'sodium', required : true)
-deps += sodium
-
-curl = dependency('libcurl', 'curl', required : true)
-deps += curl
-
-editline = dependency('libeditline', 'editline', version : '>=1.14', required : true)
-deps += editline
-
-lowdown = dependency('lowdown', version : '>=0.9.0', required : true)
-deps += lowdown
-
-# HACK(Qyriad): rapidcheck's pkg-config doesn't include the libs lol
-rapidcheck_meson = dependency('rapidcheck', required : enable_tests)
-rapidcheck = declare_dependency(dependencies : rapidcheck_meson, link_args : ['-lrapidcheck'])
-deps += rapidcheck
-
-gtest = [
-  dependency('gtest', required : enable_tests),
-  dependency('gtest_main', required : enable_tests),
-  dependency('gmock', required : enable_tests),
-  dependency('gmock_main', required : enable_tests),
-]
-deps += gtest
-
-toml11 = dependency('toml11', version : '>=3.7.0', required : true)
-deps += toml11
-
-nlohmann_json = dependency('nlohmann_json', required : true)
-deps += nlohmann_json
-
-#
-# Build-time tools
-#
-bash = find_program('bash')
-coreutils = find_program('coreutils')
-dot = find_program('dot', required : false)
-pymod = import('python')
-python = pymod.find_installation('python3')
-
-# Used to workaround https://github.com/mesonbuild/meson/issues/2320 in src/nix/meson.build.
-installcmd = find_program('install')
-
-sandbox_shell = get_option('sandbox-shell')
-# Consider it required if we're on Linux and the user explicitly specified a non-default value.
-sandbox_shell_required = sandbox_shell != 'busybox' and host_machine.system() == 'linux'
-# NOTE(Qyriad): package.nix puts busybox in buildInputs for Linux.
-# Most builds should not require setting this.
-busybox = find_program(sandbox_shell, required : sandbox_shell_required, native : false)
-if not busybox.found() and host_machine.system() == 'linux' and sandbox_shell_required
-  warning('busybox not found and other sandbox shell was specified')
-  warning('a sandbox shell is recommended on Linux -- configure with -Dsandbox-shell=/path/to/shell to set')
-endif
-# FIXME(Qyriad): the autoconf system checks that busybox has the "standalone" feature, indicating
-# that busybox sh won't run busybox applets as builtins (which would break our sandbox).
-
-lsof = find_program('lsof')
-bison = find_program('bison')
-flex = find_program('flex')
-
-# This is how Nix does generated headers...
-# other instances of header generation use a very similar command.
-# FIXME(Qyriad): do we really need to use the shell for this?
-gen_header_sh = 'echo \'R"__NIX_STR(\' | cat - @INPUT@ && echo \')__NIX_STR"\''
-gen_header = generator(
-  bash,
-  arguments : [ '-c', gen_header_sh ],
-  capture : true,
-  output : '@PLAINNAME@.gen.hh',
-)
-
-#
-# Configuration
-#
-
-run_command('ln', '-s',
-  meson.project_build_root() / '__nothing_link_target',
-  meson.project_build_root() / '__nothing_symlink',
-  check : true,
-)
-can_link_symlink = run_command('ln',
-  meson.project_build_root() / '__nothing_symlink',
-  meson.project_build_root() / '__nothing_hardlink',
-  check : false,
-).returncode() == 0
-run_command('rm', '-f',
-  meson.project_build_root() / '__nothing_symlink',
-  meson.project_build_root() / '__nothing_hardlink',
-  check : true,
-)
-summary('can hardlink to symlink', can_link_symlink, bool_yn : true)
-configdata += { 'CAN_LINK_SYMLINK': can_link_symlink.to_int() }
-
-
-# Check for each of these functions, and create a define like `#define HAVE_LCHOWN 1`.
-check_funcs = [
-  'lchown',
-  'lutimes',
-  'pipe2',
-  'posix_fallocate',
-  'statvfs',
-  'strsignal',
-  'sysconf',
-]
-foreach funcspec : check_funcs
-  define_name = 'HAVE_' + funcspec.underscorify().to_upper()
-  define_value = cxx.has_function(funcspec).to_int()
-  configdata += {
-    define_name: define_value,
-  }
-endforeach
-
-config_h = configure_file(
-  configuration : {
-    'PACKAGE_NAME': '"' + meson.project_name() + '"',
-    'PACKAGE_VERSION': '"' + meson.project_version() + '"',
-    'PACKAGE_TARNAME': '"' + meson.project_name() + '"',
-    'PACKAGE_STRING': '"' + meson.project_name() + ' ' + meson.project_version() + '"',
-    'HAVE_STRUCT_DIRENT_D_TYPE': 1, # FIXME: actually check this for solaris
-    'SYSTEM': '"' + host_system + '"',
-  } + configdata,
-  output : 'config.h',
-)
-
-install_headers(config_h, subdir : 'nix')
-
-add_project_arguments(
-  # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
-  # It would be nice for our headers to be idempotent instead.
-  '-include', 'config.h',
-  '-Wno-deprecated-declarations',
-  '-Wimplicit-fallthrough',
-  '-Werror=switch',
-  '-Werror=switch-enum',
-  language : 'cpp',
-)
-
-if cxx.get_id() in ['gcc', 'clang']
-  # 2024-03-24: jade benchmarked the default sanitize reporting in clang and got
-  # a regression of about 10% on hackage-packages.nix with clang. So we are trapping instead.
-  #
-  # This has an overhead of 0-4% on gcc and unmeasurably little on clang, in
-  # Nix evaluation benchmarks.
-  #
-  # N.B. Meson generates a completely nonsense warning here:
-  # https://github.com/mesonbuild/meson/issues/9822
-  # Both of these args cannot be written in the default meson configuration.
-  # b_sanitize=signed-integer-overflow is ignored, and
-  # -fsanitize-undefined-trap-on-error is not representable.
-  sanitize_args = ['-fsanitize=signed-integer-overflow', '-fsanitize-undefined-trap-on-error']
-  add_project_arguments(sanitize_args, language: 'cpp')
-  add_project_link_arguments(sanitize_args, language: 'cpp')
-endif
-
-add_project_link_arguments('-pthread', language : 'cpp')
-if cxx.get_linker_id() in ['ld.bfd', 'ld.gold']
-  add_project_link_arguments('-Wl,--no-copy-dt-needed-entries', language : 'cpp')
-endif
-
-# Generate Chromium tracing files for each compiled file, which enables
-# maintainers/buildtime_report.sh BUILD-DIR to simply work in clang builds.
-#
-# They can also be manually viewed at https://ui.perfetto.dev
-if get_option('profile-build').require(meson.get_compiler('cpp').get_id() == 'clang').enabled()
-  add_project_arguments('-ftime-trace', language: 'cpp')
-endif
-
-subdir('src')
-subdir('scripts')
-subdir('misc')
-
-if enable_tests
-  subdir('tests/unit')
-  subdir('tests/functional')
-endif

meson.options

@@ -1,48 +0,0 @@
-# vim: filetype=meson
-option('gc', type : 'feature',
-  description : 'enable garbage collection in the Nix expression evaluator (requires Boehm GC)',
-)
-# TODO(Qyriad): is this feature maintained?
-option('embedded-sandbox-shell', type : 'feature',
-  description : 'include the sandbox shell in the Nix binary',
-)
-
-option('cpuid', type : 'feature',
-  description : 'determine microarchitecture levels with libcpuid (only relevant on x86_64)',
-)
-
-option('seccomp-sandboxing', type : 'feature',
-  description : 'build support for seccomp sandboxing (recommended unless your arch doesn\'t support libseccomp, only relevant on Linux)',
-)
-
-option('sandbox-shell', type : 'string', value : 'busybox',
-  description : 'path to a statically-linked shell to use as /bin/sh in sandboxes (usually busybox)',
-)
-
-option('enable-tests', type : 'boolean', value : true,
-  description : 'whether to enable tests or not (requires rapidcheck and gtest)',
-)
-
-option('tests-color', type : 'boolean', value : true,
-  description : 'set to false to disable color output in gtest',
-)
-
-option('tests-brief', type : 'boolean', value : false,
-  description : 'set to true for shorter tests output',
-)
-
-option('profile-build', type : 'feature', value: 'disabled',
-  description : 'whether to enable -ftime-trace in clang builds, allowing for speeding up the build.'
-)
-
-option('store-dir', type : 'string', value : '/nix/store',
-  description : 'path of the Nix store',
-)
-
-option('state-dir', type : 'string', value : '/nix/var',
-  description : 'path to store state in for Nix',
-)
-
-option('log-dir', type : 'string', value : '/nix/var/log',
-  description : 'path to store logs in for Nix',
-)

meson/cleanup-install.bash

@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-# Meson will call this with an absolute path to Bash.
-# The shebang is just for convenience.
-
-# The parser and lexer tab are generated via custom Meson targets in src/libexpr/meson.build,
-# but Meson doesn't support marking only part of a target for install. The generation creates
-# both headers (parser-tab.hh, lexer-tab.hh) and source files (parser-tab.cc, lexer-tab.cc),
-# and we definitely want the former installed, but not the latter. This script is added to
-# Meson's install steps to correct this, as the logic for it is just complex enough to
-# warrant separate and careful handling, because both Meson's configured include directory
-# may or may not be an absolute path, and DESTDIR may or may not be set at all, but can't be
-# manipulated in Meson logic.
-
-set -euo pipefail
-
-echo "cleanup-install: removing Meson-placed C++ sources from dest includedir"
-
-if [[ "${1/--help/}" != "$1" ]]; then
-	echo "cleanup-install: this script should only be called from the Meson build system"
-	exit 1
-fi
-
-# Ensure the includedir was passed as the first argument
-# (set -u will make this fail otherwise).
-includedir="$1"
-# And then ensure that first argument is a directory that exists.
-if ! [[ -d "$1" ]]; then
-	echo "cleanup-install: this script should only be called from the Meson build system"
-	echo "argv[1] (${1@Q}) is not a directory"
-	exit 2
-fi
-
-# If DESTDIR environment variable is set, prepend it to the include dir.
-# Unfortunately, we cannot do this on the Meson side. We do have an environment variable
-# `MESON_INSTALL_DESTDIR_PREFIX`, but that will not refer to the include directory if
-# includedir has been set separately, which Lix's split-output derivation does.
-# We also cannot simply do an inline bash conditional like "${DESTDIR:=}" or similar,
-# because we need to specifically *join* DESTDIR and includedir with a slash, and *not*
-# have a slash if DESTDIR isn't set at all, since $includedir could be a relative directory.
-# Finally, DESTDIR is only available to us as an environment variable in these install scripts,
-# not in Meson logic.
-# Therefore, our best option is to have Meson pass this script the configured includedir,
-# and perform this dance with it and $DESTDIR.
-if [[ -n "${DESTDIR:-}" ]]; then
-	includedir="$DESTDIR/$includedir"
-fi
-
-# Intentionally not using -f.
-# If these files don't exist then our assumptions have been violated and we should fail.
-rm -v "$includedir/nix/parser-tab.cc" "$includedir/nix/lexer-tab.cc"

meson/run-test.py

@@ -1,87 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-This script is a helper for this project's Meson buildsystem to run Lix's
-functional tests. It is an analogue to mk/run-test.sh in the autoconf+Make
-buildsystem.
-
-These tests are run in the installCheckPhase in Lix's derivation, and as such
-expect to be run after the project has already been "installed" to some extent.
-Look at meson/setup-functional-tests.py for more details.
-"""
-
-import argparse
-from pathlib import Path
-import os
-import shutil
-import subprocess
-import sys
-
-name = 'run-test.py'
-
-if 'MESON_BUILD_ROOT' not in os.environ:
-    raise ValueError(f'{name}: this script must be run from the Meson build system')
-
-def main():
-
-    tests_dir = Path(os.path.join(os.environ['MESON_BUILD_ROOT'], 'tests/functional'))
-
-    parser = argparse.ArgumentParser(name)
-    parser.add_argument('target', help='the script path relative to tests/functional to run')
-    args = parser.parse_args()
-
-    target = Path(args.target)
-    # The test suite considers the test's name to be the path to the test relative to
-    # `tests/functional`, but without the file extension.
-    # e.g. for `tests/functional/flakes/develop.sh`, the test name is `flakes/develop`
-    test_name = target.with_suffix('').as_posix()
-    if not target.is_absolute():
-        target = tests_dir.joinpath(target).resolve()
-
-    assert target.exists(), f'{name}: test {target} does not exist; did you run `meson install`?'
-
-    bash = os.environ.get('BASH', shutil.which('bash'))
-    if bash is None:
-        raise ValueError(f'{name}: bash executable not found and BASH environment variable not set')
-
-    test_environment = os.environ | {
-        'TEST_NAME': test_name,
-        # mk/run-test.sh did this, but I don't know if it has any effect since it seems
-        # like the tests that interact with remote stores set it themselves?
-        'NIX_REMOTE': '',
-    }
-
-    # Initialize testing.
-    init_result = subprocess.run([bash, '-e', 'init.sh'], cwd=tests_dir, env=test_environment)
-    if init_result.returncode != 0:
-        print(f'{name}: internal error initializing {args.target}', file=sys.stderr)
-        print('[ERROR]')
-        # Meson interprets exit code 99 as indicating an *error* in the testing process.
-        return 99
-
-    # Run the test itself.
-    test_result = subprocess.run([bash, '-e', target.name], cwd=target.parent, env=test_environment)
-
-    if test_result.returncode == 0:
-        print('[PASS]')
-    elif test_result.returncode == 99:
-        print('[SKIP]')
-        # Meson interprets exit code 77 as indicating a skipped test.
-        return 77
-    else:
-        print('[FAIL]')
-
-    return test_result.returncode
-
-try:
-    sys.exit(main())
-except AssertionError as e:
-    # This should mean that this test was run not-from-Meson, probably without
-    # having run `meson install` first, which is not an bug in this script.
-    print(e, file=sys.stderr)
-    sys.exit(99)
-except Exception as e:
-    print(f'{name}: INTERNAL ERROR running test ({sys.argv}): {e}', file=sys.stderr)
-    print(f'this is a bug in {name}')
-    sys.exit(99)
-

meson/setup-functional-tests.py

@@ -1,105 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-So like. This script is cursed.
-It's a helper for this project's Meson buildsystem for Lix's functional tests.
-The functional tests are a bunch of bash scripts, that each expect to be run from the
-directory from the directory that that script is in, and also expect modifications to have
-happened to the source tree, and even splork files around. The last is against the spirit
-of Meson (and personally annoying), to have build processes that aren't self-contained in the
-out-of-source build directory, but more problematically we need configured files in the test
-tree.
-So. We copy the tests tree into the build directory.
-Meson doesn't have a good way of doing this natively -- the best you could do is subdir()
-into every directory in the tests tree and configure_file(copy : true) on every file,
-but this won't copy symlinks as symlinks, which we need since the test suite has, well,
-tests on symlinks.
-However, the functional tests are normally run during Lix's derivation's installCheckPhase,
-after Lix has already been "installed" somewhere. So in Meson we setup add this file as an
-install script and copy everything in tests/functional to the build directory, preserving
-things like symlinks, even broken ones (which are intentional).
-
-TODO(Qyriad): when we remove the old build system entirely, we can instead fix the tests.
-"""
-
-from pathlib import Path
-import os, os.path
-import shutil
-import sys
-import traceback
-
-name = 'setup-functional-tests.py'
-
-if 'MESON_SOURCE_ROOT' not in os.environ or 'MESON_BUILD_ROOT' not in os.environ:
-    raise ValueError(f'{name}: this script must be run from the Meson build system')
-
-print(f'{name}: mirroring tests/functional to build directory')
-
-tests_source = Path(os.environ['MESON_SOURCE_ROOT']) / 'tests/functional'
-tests_build = Path(os.environ['MESON_BUILD_ROOT']) / 'tests/functional'
-
-def main():
-
-    os.chdir(tests_build)
-
-    for src_dirpath, src_dirnames, src_filenames in os.walk(tests_source):
-        src_dirpath = Path(src_dirpath)
-        assert src_dirpath.is_absolute(), f'{src_dirpath=} is not absolute'
-
-        # os.walk() gives us the absolute path to the directory we're currently in as src_dirpath.
-        # We want to mirror from the perspective of `tests_source`.
-        rel_subdir = src_dirpath.relative_to(tests_source)
-        assert (not rel_subdir.is_absolute()), f'{rel_subdir=} is not relative'
-
-        # And then join that relative path on `tests_build` to get the absolute
-        # path in the build directory that corresponds to `src_dirpath`.
-        build_dirpath = tests_build / rel_subdir
-        assert build_dirpath.is_absolute(), f'{build_dirpath=} is not absolute'
-
-        # More concretely, for the test file tests/functional/ca/build.sh:
-        # - src_dirpath is `$MESON_SOURCE_ROOT/tests/functional/ca`
-        # - rel_subidr is `ca`
-        # - build_dirpath is `$MESON_BUILD_ROOT/tests/functional/ca`
-
-        # `src_dirname` are directories underneath `src_dirpath`, and will be relative
-        # to `src_dirpath`.
-        for src_dirname in src_dirnames:
-            # Take the name of the directory in the tests source and join it on `src_dirpath`
-            # to get the full path to this specific directory in the tests source.
-            src = src_dirpath / src_dirname
-            # If there isn't *something* here, then our logic is wrong.
-            # Path.exists(follow_symlinks=False) wasn't added until Python 3.12, so we use
-            # os.path.lexists() here.
-            assert os.path.lexists(src), f'{src=} does not exist'
-
-            # Take the name of this directory and join it on `build_dirpath` to get the full
-            # path to the directory in `build/tests/functional` that we need to create.
-            dest = build_dirpath / src_dirname
-            if src.is_symlink():
-                src_target = src.readlink()
-                dest.unlink(missing_ok=True)
-                dest.symlink_to(src_target)
-            else:
-                dest.mkdir(parents=True, exist_ok=True)
-
-        for src_filename in src_filenames:
-            # os.walk() should be giving us relative filenames.
-            # If it isn't, our path joins will be veeeery wrong.
-            assert (not Path(src_filename).is_absolute()), f'{src_filename=} is not relative'
-
-            src = src_dirpath / src_filename
-            dst = build_dirpath / src_filename
-            # Mildly misleading name -- unlink removes ordinary files as well as symlinks.
-            dst.unlink(missing_ok=True)
-            # shutil.copy2() best-effort preserves metadata.
-            shutil.copy2(src, dst, follow_symlinks=False)
-
-
-try:
-    sys.exit(main())
-except Exception as e:
-    # Any error is likely a bug in this script.
-    print(f'{name}: INTERNAL ERROR setting up functional tests: {e}', file=sys.stderr)
-    print(traceback.format_exc())
-    print(f'this is a bug in {name}')
-    sys.exit(1)

misc/bash/meson.build

@@ -1,8 +0,0 @@
-configure_file(
-  input : 'completion.sh',
-  output : 'nix',
-  install : true,
-  install_dir : datadir / 'bash-completion/completions',
-  install_mode : 'rw-r--r--',
-  copy : true,
-)

misc/changelog-d.cabal.nix

@@ -0,0 +1,31 @@
+{ mkDerivation, aeson, base, bytestring, cabal-install-parsers
+, Cabal-syntax, containers, directory, filepath, frontmatter
+, generic-lens-lite, lib, mtl, optparse-applicative, parsec, pretty
+, regex-applicative, text, pkgs
+}:
+let rev = "f30f6969e9cd8b56242309639d58acea21c99d06";
+in
+mkDerivation {
+  pname = "changelog-d";
+  version = "0.1";
+  src = pkgs.fetchurl {
+    name = "changelog-d-${rev}.tar.gz";
+    url = "https://codeberg.org/roberth/changelog-d/archive/${rev}.tar.gz";
+    hash = "sha256-8a2+i5u7YoszAgd5OIEW0eYUcP8yfhtoOIhLJkylYJ4=";
+  } // { inherit rev; };
+  isLibrary = false;
+  isExecutable = true;
+  libraryHaskellDepends = [
+    aeson base bytestring cabal-install-parsers Cabal-syntax containers
+    directory filepath frontmatter generic-lens-lite mtl parsec pretty
+    regex-applicative text
+  ];
+  executableHaskellDepends = [
+    base bytestring Cabal-syntax directory filepath
+    optparse-applicative
+  ];
+  doHaddock = false;
+  description = "Concatenate changelog entries into a single one";
+  license = lib.licenses.gpl3Plus;
+  mainProgram = "changelog-d";
+}

misc/changelog-d.nix

@@ -0,0 +1,31 @@
+# Taken temporarily from <nixpkgs/pkgs/by-name/ch/changelog-d/package.nix>
+{
+  callPackage,
+  lib,
+  haskell,
+  haskellPackages,
+}:
+
+let
+  hsPkg = haskellPackages.callPackage ./changelog-d.cabal.nix { };
+
+  addCompletions = haskellPackages.generateOptparseApplicativeCompletions ["changelog-d"];
+
+  haskellModifications =
+    lib.flip lib.pipe [
+      addCompletions
+      haskell.lib.justStaticExecutables
+    ];
+
+  mkDerivationOverrides = finalAttrs: oldAttrs: {
+
+    version = oldAttrs.version + "-git-${lib.strings.substring 0 7 oldAttrs.src.rev}";
+
+    meta = oldAttrs.meta // {
+      homepage = "https://codeberg.org/roberth/changelog-d";
+      maintainers = [ lib.maintainers.roberth ];
+    };
+
+  };
+in
+  (haskellModifications hsPkg).overrideAttrs mkDerivationOverrides

misc/clangbuildanalyzer.nix

@@ -1,27 +0,0 @@
-# Upstreaming here, can be deleted once it's upstreamed:
-# https://github.com/NixOS/nixpkgs/pull/297102
-{ stdenv, lib, cmake, fetchFromGitHub }:
-stdenv.mkDerivation (finalAttrs: {
-  pname = "clangbuildanalyzer";
-  version = "1.5.0";
-
-  src = fetchFromGitHub {
-    owner = "aras-p";
-    repo = "ClangBuildAnalyzer";
-    rev = "v${finalAttrs.version}";
-    sha256 = "sha256-kmgdk634zM0W0OoRoP/RzepArSipa5bNqdVgdZO9gxo=";
-  };
-
-  nativeBuildInputs = [
-    cmake
-  ];
-
-  meta = {
-    description = "Tool for analyzing Clang's -ftrace-time files";
-    homepage = "https://github.com/aras-p/ClangBuildAnalyzer";
-    maintainers = with lib.maintainers; [ lf- ];
-    license = lib.licenses.unlicense;
-    platforms = lib.platforms.unix;
-    mainProgram = "ClangBuildAnalyzer";
-  };
-})

misc/fish/meson.build

@@ -1,8 +0,0 @@
-configure_file(
-  input : 'completion.fish',
-  output : 'nix.fish',
-  install : true,
-  install_dir : datadir / 'fish/vendor_completions.d',
-  install_mode : 'rw-r--r--',
-  copy : true,
-)

misc/meson.build

@@ -1,5 +0,0 @@
-subdir('bash')
-subdir('fish')
-subdir('zsh')
-
-subdir('systemd')

misc/systemd/meson.build

@@ -1,25 +0,0 @@
-foreach config : [ 'nix-daemon.socket', 'nix-daemon.service' ]
-  configure_file(
-    input : config + '.in',
-    output : config,
-    install : true,
-    install_dir : prefix / 'lib/systemd/system',
-    install_mode : 'rw-r--r--',
-    configuration : {
-      'storedir' : store_dir,
-      'localstatedir' : state_dir,
-      'bindir' : bindir,
-    },
-  )
-endforeach
-
-configure_file(
-  input : 'nix-daemon.conf.in',
-  output : 'nix-daemon.conf',
-  install : true,
-  install_dir : prefix / 'lib/tmpfiles.d',
-  install_mode : 'rw-r--r--',
-  configuration : {
-    'localstatedir' : state_dir,
-  },
-)

misc/upstart/local.mk

@@ -0,0 +1,7 @@
+ifdef HOST_LINUX
+
+  $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(sysconfdir)/init, 0644)))
+
+  clean-files += $(d)/nix-daemon.conf
+
+endif

misc/upstart/nix-daemon.conf.in

@@ -0,0 +1,5 @@
+description "Nix Daemon"
+start on filesystem
+stop on shutdown
+respawn
+exec @bindir@/nix-daemon --daemon

misc/zsh/meson.build

@@ -1,10 +0,0 @@
-foreach script : [ [ 'completion.zsh', '_nix' ], [ 'run-help-nix' ] ]
-  configure_file(
-    input : script[0],
-    output : script.get(1, script[0]),
-    install : true,
-    install_dir : datadir / 'zsh/site-functions',
-    install_mode : 'rw-r--r--',
-    copy : true,
-  )
-endforeach

mk/libraries.mk

@@ -78,7 +78,11 @@ define build-library
         $(1)_LDFLAGS += -undefined suppress -flat_namespace
       endif
     else
-      # -Wl,-z,defs is broken with sanitizers on Linux/clang at least.
+      ifndef HOST_DARWIN
+        ifndef HOST_CYGWIN
+          $(1)_LDFLAGS += -Wl,-z,defs
+        endif
+      endif
     endif
 
     ifndef HOST_DARWIN

package.nix

@@ -7,17 +7,14 @@
   aws-sdk-cpp,
   boehmgc,
   nlohmann_json,
-  bison,
-  build-release-notes,
+  changelog-d,
   boost,
   brotli,
   bzip2,
-  cmake,
   curl,
   doxygen,
   editline,
   fileset,
-  flex,
   git,
   gtest,
   jq,
@@ -25,18 +22,15 @@
   libcpuid,
   libseccomp,
   libsodium,
-  lsof,
   lowdown,
   mdbook,
   mdbook-linkcheck,
   mercurial,
-  meson,
-  ninja,
   openssl,
+  pegtl,
   pkg-config,
   rapidcheck,
   sqlite,
-  toml11,
   util-linuxMinimal ? utillinuxMinimal,
   utillinuxMinimal ? null,
   xz,
@@ -52,10 +46,6 @@
   # Avoid setting things that would interfere with a functioning devShell
   forDevShell ? false,
 
-  # FIXME(Qyriad): build Lix using Meson instead of autoconf and make.
-  # This flag will be removed when the migration to Meson is complete.
-  buildWithMeson ? false,
-
   # Not a real argument, just the only way to approximate let-binding some
   # stuff for argument defaults.
   __forDefaults ? {
@@ -95,17 +85,12 @@
     ./README.md
   ];
 
-  topLevelBuildFiles = fileset.unions ([
+  topLevelBuildFiles = fileset.unions [
     ./local.mk
     ./Makefile
     ./Makefile.config.in
     ./mk
-  ] ++ lib.optionals buildWithMeson [
-    ./meson.build
-    ./meson.options
-    ./meson
-    ./scripts/meson.build
-  ]);
+  ];
 
  functionalTestFiles = fileset.unions [
     ./tests/functional
@@ -140,38 +125,22 @@ in stdenv.mkDerivation (finalAttrs: {
 
   dontBuild = false;
 
-  # FIXME(Qyriad): see if this is still needed once the migration to Meson is completed.
-  mesonFlags = lib.optionals (buildWithMeson && stdenv.hostPlatform.isLinux) [
-    "-Dsandbox-shell=${lib.getBin busybox-sandbox-shell}/bin/busybox"
-  ];
-
-  # We only include CMake so that Meson can locate toml11, which only ships CMake dependency metadata.
-  dontUseCmakeConfigure = true;
-
   nativeBuildInputs = [
-    bison
-    flex
-  ] ++ [
     (lib.getBin lowdown)
     mdbook
     mdbook-linkcheck
     autoconf-archive
-  ] ++ lib.optional (!buildWithMeson) autoreconfHook ++ [
+    autoreconfHook
     pkg-config
 
     # Tests
     git
     mercurial
     jq
-    lsof
   ] ++ lib.optional stdenv.hostPlatform.isLinux util-linuxMinimal
-    ++ lib.optional (!officialRelease && buildUnreleasedNotes) build-release-notes
+    ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d
     ++ lib.optional internalApiDocs doxygen
-    ++ lib.optionals buildWithMeson [
-      meson
-      ninja
-      cmake
-  ];
+  ;
 
   buildInputs = [
     curl
@@ -185,9 +154,8 @@ in stdenv.mkDerivation (finalAttrs: {
     boost
     lowdown
     libsodium
-    toml11
   ]
-    ++ lib.optionals stdenv.hostPlatform.isLinux [ libseccomp busybox-sandbox-shell ]
+    ++ lib.optionals stdenv.isLinux [ libseccomp ]
     ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
     # There have been issues building these dependencies
     ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform) aws-sdk-cpp-nix
@@ -205,13 +173,6 @@ in stdenv.mkDerivation (finalAttrs: {
     boost
   ];
 
-  # Needed for Meson to find Boost.
-  # https://github.com/NixOS/nixpkgs/issues/86131.
-  env = lib.optionalAttrs (buildWithMeson || forDevShell) {
-    BOOST_INCLUDEDIR = "${lib.getDev boost}/include";
-    BOOST_LIBRARYDIR = "${lib.getLib boost}/lib";
-  };
-
   preConfigure = lib.optionalString (!finalAttrs.dontBuild && !stdenv.hostPlatform.isStatic) ''
     # Copy libboost_context so we don't get all of Boost in our closure.
     # https://github.com/NixOS/nixpkgs/issues/45462
@@ -248,11 +209,7 @@ in stdenv.mkDerivation (finalAttrs: {
     ++ lib.optional (!canRunInstalled) "--disable-doc-gen"
     ++ [ (lib.enableFeature internalApiDocs "internal-api-docs") ]
     ++ lib.optional (!forDevShell) "--sysconfdir=/etc"
-    ++ [
-      "TOML11_HEADERS=${lib.getDev toml11}/include"
-    ];
-
-  mesonBuildType = lib.optional (buildWithMeson || forDevShell) "debugoptimized";
+    ++ [ "PEGTL_HEADERS=${lib.getDev pegtl}/include" ];
 
   installTargets = lib.optional internalApiDocs "internal-api-html";
 
@@ -262,10 +219,6 @@ in stdenv.mkDerivation (finalAttrs: {
 
   doCheck = true;
 
-  mesonCheckFlags = lib.optionals (buildWithMeson || forDevShell) [
-    "--suite=check"
-  ];
-
   installFlags = "sysconfdir=$(out)/etc";
 
   postInstall = lib.optionalString (!finalAttrs.dontBuild) ''
@@ -275,12 +228,10 @@ in stdenv.mkDerivation (finalAttrs: {
     mkdir -p $out/nix-support
     echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
   '' + lib.optionalString stdenv.isDarwin ''
-    for lib in libnixutil.dylib libnixexpr.dylib; do
-      install_name_tool \
-        -change "${lib.getLib boost}/lib/libboost_context.dylib" \
-        "$out/lib/libboost_context.dylib" \
-        "$out/lib/$lib"
-    done
+    install_name_tool \
+      -change ${boost}/lib/libboost_context.dylib \
+      $out/lib/libboost_context.dylib \
+      $out/lib/libnixutil.dylib
   '' + lib.optionalString internalApiDocs ''
     mkdir -p $out/nix-support
     echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> "$out/nix-support/hydra-build-products"
@@ -290,28 +241,15 @@ in stdenv.mkDerivation (finalAttrs: {
   installCheckFlags = "sysconfdir=$(out)/etc";
   installCheckTarget = "installcheck"; # work around buggy detection in stdenv
 
-  mesonInstallCheckFlags = [
-    "--suite=installcheck"
-  ];
-
   preInstallCheck = lib.optionalString stdenv.hostPlatform.isDarwin ''
     export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
   '';
 
-  installCheckPhase = lib.optionalString buildWithMeson ''
-    runHook preInstallCheck
-    flagsArray=($mesonInstallCheckFlags "''${mesonInstallCheckFlagsArray[@]}")
-    meson test --no-rebuild "''${flagsArray[@]}"
-    runHook postInstallCheck
-  '';
-
   separateDebugInfo = !stdenv.hostPlatform.isStatic && !finalAttrs.dontBuild;
 
   strictDeps = true;
 
-  # strictoverflow is disabled because we trap on signed overflow instead
-  hardeningDisable = [ "strictoverflow" ]
-    ++ lib.optional stdenv.hostPlatform.isStatic "pie";
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
 
   meta.platforms = lib.platforms.unix;
 

scripts/local.mk

@@ -7,7 +7,5 @@ profiledir = $(sysconfdir)/profile.d
 
 $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644))
 $(eval $(call install-file-as, $(d)/nix-profile.fish, $(profiledir)/nix.fish, 0644))
-$(eval $(call install-file-as, $(d)/nix-profile-daemon.sh, $(profiledir)/nix-daemon.sh, 0644))
-$(eval $(call install-file-as, $(d)/nix-profile-daemon.fish, $(profiledir)/nix-daemon.fish, 0644))
 
 clean-files += $(nix_noinst_scripts)

scripts/meson.build

@@ -1,29 +0,0 @@
-# configures `scripts/nix-profile.sh.in` (and copies the original to the build directory).
-# this is only needed for tests, but running it unconditionally does not hurt enough to care.
-configure_file(
-  input : 'nix-profile.sh.in',
-  output : 'nix-profile.sh',
-  configuration : {
-    'localstatedir': state_dir,
-  }
-)
-
-# https://github.com/mesonbuild/meson/issues/860
-configure_file(
-  input : 'nix-profile.sh.in',
-  output : 'nix-profile.sh.in',
-  copy : true,
-)
-
-foreach rc : [ '.sh', '.fish', '-daemon.sh', '-daemon.fish' ]
-  configure_file(
-    input : 'nix-profile' + rc  + '.in',
-    output : 'nix' + rc,
-    install : true,
-    install_dir : sysconfdir / 'profile.d',
-    install_mode : 'rw-r--r--',
-    configuration : {
-      'localstatedir': state_dir,
-    },
-  )
-endforeach

scripts/nix-profile-daemon.fish.in

@@ -1,57 +0,0 @@
-function add_path --argument-names new_path
-    if type -q fish_add_path
-        # fish 3.2.0 or newer
-	fish_add_path --prepend --global $new_path
-    else
-        # older versions of fish
-        if not contains $new_path $fish_user_paths
-            set --global fish_user_paths $new_path $fish_user_paths
-        end
-    end
-end
-
-# Only execute this file once per shell.
-if test -n "$__ETC_PROFILE_NIX_SOURCED"
-  exit
-end
-
-set __ETC_PROFILE_NIX_SOURCED 1
-
-set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile"
-
-# Populate bash completions, .desktop files, etc
-if test -z "$XDG_DATA_DIRS"
-    # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
-    set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:/nix/var/nix/profiles/default/share"
-else
-    set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:/nix/var/nix/profiles/default/share"
-end
-
-# Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
-if test -n "$NIX_SSH_CERT_FILE"
-  : # Allow users to override the NIX_SSL_CERT_FILE
-else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch
-  set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt
-else if test -e /etc/ssl/ca-bundle.pem # openSUSE Tumbleweed
-  set --export NIX_SSL_CERT_FILE /etc/ssl/ca-bundle.pem
-else if test -e /etc/ssl/certs/ca-bundle.crt # Old NixOS
-  set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-bundle.crt
-else if test -e /etc/pki/tls/certs/ca-bundle.crt # Fedora, CentOS
-  set --export NIX_SSL_CERT_FILE /etc/pki/tls/certs/ca-bundle.crt
-else if test -e "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" # fall back to cacert in Nix profile
-  set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ssl/certs/ca-bundle.crt"
-else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile
-  set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt"
-else
-  # Fall back to what is in the nix profiles, favouring whatever is defined last.
-  for i in $NIX_PROFILES
-    if test -e "$i/etc/ssl/certs/ca-bundle.crt"
-      set --export NIX_SSL_CERT_FILE "$i/etc/ssl/certs/ca-bundle.crt"
-    end
-  end
-end
-
-add_path "@localstatedir@/nix/profiles/default/bin"
-add_path "$HOME/.nix-profile/bin"
-
-functions -e add_path

scripts/nix-profile-daemon.sh.in

@@ -1,72 +0,0 @@
-# Only execute this file once per shell.
-if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi
-__ETC_PROFILE_NIX_SOURCED=1
-
-NIX_LINK=$HOME/.nix-profile
-if [ -n "${XDG_STATE_HOME-}" ]; then
-    NIX_LINK_NEW="$XDG_STATE_HOME/nix/profile"
-else
-    NIX_LINK_NEW=$HOME/.local/state/nix/profile
-fi
-if [ -e "$NIX_LINK_NEW" ]; then
-    NIX_LINK="$NIX_LINK_NEW"
-else
-    if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then
-        warning="\033[1;35mwarning:\033[0m"
-        printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2
-        if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then
-            printf "         Since the profiles match, you can safely delete either of them.\n" 1>&2
-        else
-            # This should be an exceptionally rare occasion: the only way to get it would be to
-            # 1. Update to newer Nix;
-            # 2. Remove .nix-profile;
-            # 3. Set the $NIX_LINK_NEW to something other than the default user profile;
-            # 4. Roll back to older Nix.
-            # If someone did all that, they can probably figure out how to migrate the profile.
-            printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2
-        fi
-    fi
-fi
-
-export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"
-
-# Populate bash completions, .desktop files, etc
-if [ -z "${XDG_DATA_DIRS-}" ]; then
-    # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
-    export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
-else
-    export XDG_DATA_DIRS="$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
-fi
-
-# Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
-if [ -n "${NIX_SSL_CERT_FILE:-}" ]; then
-    : # Allow users to override the NIX_SSL_CERT_FILE
-elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch
-    export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
-elif [ -e /etc/ssl/ca-bundle.pem ]; then # openSUSE Tumbleweed
-    export NIX_SSL_CERT_FILE=/etc/ssl/ca-bundle.pem
-elif [ -e /etc/ssl/certs/ca-bundle.crt ]; then # Old NixOS
-    export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt
-elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS
-    export NIX_SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
-else
-  # Fall back to what is in the nix profiles, favouring whatever is defined last.
-  check_nix_profiles() {
-    if [ -n "$ZSH_VERSION" ]; then
-      # Zsh by default doesn't split words in unquoted parameter expansion.
-      # Set local_options for these options to be reverted at the end of the function
-      # and shwordsplit to force splitting words in $NIX_PROFILES below.
-      setopt local_options shwordsplit
-    fi
-    for i in $NIX_PROFILES; do
-      if [ -e "$i/etc/ssl/certs/ca-bundle.crt" ]; then
-        export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt
-      fi
-    done
-  }
-  check_nix_profiles
-  unset -f check_nix_profiles
-fi
-
-export PATH="$NIX_LINK/bin:@localstatedir@/nix/profiles/default/bin:$PATH"
-unset NIX_LINK

src/build-remote/build-remote.cc

@@ -126,7 +126,7 @@ static int main_build_remote(int argc, char * * argv)
             mkdir(currentLoad.c_str(), 0777);
 
             while (true) {
-                bestSlotLock.reset();
+                bestSlotLock = -1;
                 AutoCloseFD lock = openLockFile(currentLoad + "/main-lock", true);
                 lockFile(lock.get(), ltWrite, true);
 
@@ -185,6 +185,16 @@ static int main_build_remote(int argc, char * * argv)
                         std::cerr << "# postpone\n";
                     else
                     {
+                        // build the hint template.
+                        std::string errorText =
+                            "Failed to find a machine for remote build!\n"
+                            "derivation: %s\nrequired (system, features): (%s, [%s])";
+                        errorText += "\n%s available machines:";
+                        errorText += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)";
+
+                        for (unsigned int i = 0; i < machines.size(); ++i)
+                            errorText += "\n([%s], %s, [%s], [%s])";
+
                         // add the template values.
                         std::string drvstr;
                         if (drvPath.has_value())
@@ -192,30 +202,19 @@ static int main_build_remote(int argc, char * * argv)
                         else
                             drvstr = "<unknown>";
 
-                        std::string machinesFormatted;
+                        auto error = HintFmt(errorText);
+                        error
+                            % drvstr
+                            % neededSystem
+                            % concatStringsSep<StringSet>(", ", requiredFeatures)
+                            % machines.size();
 
-                        for (auto & m : machines) {
-                            machinesFormatted += HintFmt(
-                                "\n([%s], %s, [%s], [%s])",
-                                concatStringsSep<StringSet>(", ", m.systemTypes),
-                                m.maxJobs,
-                                concatStringsSep<StringSet>(", ", m.supportedFeatures),
-                                concatStringsSep<StringSet>(", ", m.mandatoryFeatures)
-                            ).str();
-                        }
-
-                        auto error = HintFmt(
-                            "Failed to find a machine for remote build!\n"
-                            "derivation: %s\n"
-                            "required (system, features): (%s, [%s])\n"
-                            "%s available machines:\n"
-                            "(systems, maxjobs, supportedFeatures, mandatoryFeatures)%s",
-                            drvstr,
-                            neededSystem,
-                            concatStringsSep<StringSet>(", ", requiredFeatures),
-                            machines.size(),
-                            Uncolored(machinesFormatted)
-                       );
+                        for (auto & m : machines)
+                            error
+                                % concatStringsSep<StringSet>(", ", m.systemTypes)
+                                % m.maxJobs
+                                % concatStringsSep<StringSet>(", ", m.supportedFeatures)
+                                % concatStringsSep<StringSet>(", ", m.mandatoryFeatures);
 
                         printMsg(couldBuildLocally ? lvlChatty : lvlWarn, error.str());
 
@@ -230,7 +229,7 @@ static int main_build_remote(int argc, char * * argv)
                 futimens(bestSlotLock.get(), NULL);
 #endif
 
-                lock.reset();
+                lock = -1;
 
                 try {
 
@@ -283,7 +282,7 @@ connected:
             copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute);
         }
 
-        uploadLock.reset();
+        uploadLock = -1;
 
         auto drv = store->readDerivation(*drvPath);
 

src/libcmd/command.hh

@@ -97,6 +97,8 @@ struct MixFlakeOptions : virtual Args, EvalCommand
 {
     flake::LockFlags lockFlags;
 
+    std::optional<std::string> needsFlakeInputCompletion = {};
+
     MixFlakeOptions();
 
     /**
@@ -107,8 +109,12 @@ struct MixFlakeOptions : virtual Args, EvalCommand
      * command is operating with (presumably specified via some other
      * arguments) so that the completions for these flags can use them.
      */
-    virtual std::vector<FlakeRef> getFlakeRefsForCompletion()
+    virtual std::vector<std::string> getFlakesForCompletion()
     { return {}; }
+
+    void completeFlakeInput(std::string_view prefix);
+
+    void completionHook() override;
 };
 
 struct SourceExprCommand : virtual Args, MixFlakeOptions
@@ -131,13 +137,7 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions
     /**
      * Complete an installable from the given prefix.
      */
-    void completeInstallable(AddCompletions & completions, std::string_view prefix);
-
-    /**
-     * Convenience wrapper around the underlying function to make setting the
-     * callback easier.
-     */
-    CompleterClosure getCompleteInstallable();
+    void completeInstallable(std::string_view prefix);
 };
 
 /**
@@ -170,7 +170,7 @@ struct RawInstallablesCommand : virtual Args, SourceExprCommand
 
     bool readFromStdIn = false;
 
-    std::vector<FlakeRef> getFlakeRefsForCompletion() override;
+    std::vector<std::string> getFlakesForCompletion() override;
 
 private:
 
@@ -199,7 +199,10 @@ struct InstallableCommand : virtual Args, SourceExprCommand
 
     void run(ref<Store> store) override;
 
-    std::vector<FlakeRef> getFlakeRefsForCompletion() override;
+    std::vector<std::string> getFlakesForCompletion() override
+    {
+        return {_installable};
+    }
 
 private:
 
@@ -326,16 +329,9 @@ struct MixEnvironment : virtual Args {
     void setEnviron();
 };
 
-void completeFlakeInputPath(
-    AddCompletions & completions,
-    ref<EvalState> evalState,
-    const std::vector<FlakeRef> & flakeRefs,
-    std::string_view prefix);
-
-void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::string_view prefix);
+void completeFlakeRef(ref<Store> store, std::string_view prefix);
 
 void completeFlakeRefWithFragment(
-    AddCompletions & completions,
     ref<EvalState> evalState,
     flake::LockFlags lockFlags,
     Strings attrPathPrefixes,

src/libcmd/common-eval-args.cc

@@ -132,8 +132,8 @@ MixEvalArgs::MixEvalArgs()
             if (to.subdir != "") extraAttrs["dir"] = to.subdir;
             fetchers::overrideRegistry(from.input, to.input, extraAttrs);
         }},
-        .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
-            completeFlakeRef(completions, openStore(), prefix);
+        .completer = {[&](size_t, std::string_view prefix) {
+            completeFlakeRef(openStore(), prefix);
         }}
     });
 
@@ -172,7 +172,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s)
         return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
     }
 
-    else if (s.starts_with("flake:")) {
+    else if (hasPrefix(s, "flake:")) {
         experimentalFeatureSettings.require(Xp::Flakes);
         auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false);
         auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath;

src/libcmd/installable-flake.cc

@@ -28,10 +28,6 @@ namespace nix {
 std::vector<std::string> InstallableFlake::getActualAttrPaths()
 {
     std::vector<std::string> res;
-    if (attrPaths.size() == 1 && attrPaths.front().starts_with(".")){
-        res.push_back(attrPaths.front().substr(1));
-        return res;
-    }
 
     for (auto & prefix : prefixes)
         res.push_back(prefix + *attrPaths.begin());

src/libcmd/installables.cc

@@ -28,24 +28,17 @@
 
 namespace nix {
 
-void completeFlakeInputPath(
-    AddCompletions & completions,
-    ref<EvalState> evalState,
-    const std::vector<FlakeRef> & flakeRefs,
-    std::string_view prefix)
-{
-    for (auto & flakeRef : flakeRefs) {
-        auto flake = flake::getFlake(*evalState, flakeRef, true);
-        for (auto & input : flake.inputs)
-            if (input.first.starts_with(prefix))
-                completions.add(input.first);
-    }
-}
-
 MixFlakeOptions::MixFlakeOptions()
 {
     auto category = "Common flake-related options";
 
+    addFlag({
+        .longName = "recreate-lock-file",
+        .description = "Recreate the flake's lock file from scratch.",
+        .category = category,
+        .handler = {&lockFlags.recreateLockFile, true}
+    });
+
     addFlag({
         .longName = "no-update-lock-file",
         .description = "Do not allow any updates to the flake's lock file.",
@@ -78,6 +71,19 @@ MixFlakeOptions::MixFlakeOptions()
         .handler = {&lockFlags.commitLockFile, true}
     });
 
+    addFlag({
+        .longName = "update-input",
+        .description = "Update a specific flake input (ignoring its previous entry in the lock file).",
+        .category = category,
+        .labels = {"input-path"},
+        .handler = {[&](std::string s) {
+            lockFlags.inputUpdates.insert(flake::parseInputPath(s));
+        }},
+        .completer = {[&](size_t, std::string_view prefix) {
+            needsFlakeInputCompletion = {std::string(prefix)};
+        }}
+    });
+
     addFlag({
         .longName = "override-input",
         .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",
@@ -89,12 +95,11 @@ MixFlakeOptions::MixFlakeOptions()
                 flake::parseInputPath(inputPath),
                 parseFlakeRef(flakeRef, absPath("."), true));
         }},
-        .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) {
-            if (n == 0) {
-                completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
-            } else if (n == 1) {
-                completeFlakeRef(completions, getEvalState()->store, prefix);
-            }
+        .completer = {[&](size_t n, std::string_view prefix) {
+            if (n == 0)
+                needsFlakeInputCompletion = {std::string(prefix)};
+            else if (n == 1)
+                completeFlakeRef(getEvalState()->store, prefix);
         }}
     });
 
@@ -141,12 +146,30 @@ MixFlakeOptions::MixFlakeOptions()
                 }
             }
         }},
-        .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
-            completeFlakeRef(completions, getEvalState()->store, prefix);
+        .completer = {[&](size_t, std::string_view prefix) {
+            completeFlakeRef(getEvalState()->store, prefix);
         }}
     });
 }
 
+void MixFlakeOptions::completeFlakeInput(std::string_view prefix)
+{
+    auto evalState = getEvalState();
+    for (auto & flakeRefS : getFlakesForCompletion()) {
+        auto flakeRef = parseFlakeRefWithFragment(expandTilde(flakeRefS), absPath(".")).first;
+        auto flake = flake::getFlake(*evalState, flakeRef, true);
+        for (auto & input : flake.inputs)
+            if (hasPrefix(input.first, prefix))
+                completions->add(input.first);
+    }
+}
+
+void MixFlakeOptions::completionHook()
+{
+    if (auto & prefix = needsFlakeInputCompletion)
+        completeFlakeInput(*prefix);
+}
+
 SourceExprCommand::SourceExprCommand()
 {
     addFlag({
@@ -164,7 +187,6 @@ SourceExprCommand::SourceExprCommand()
 
     addFlag({
         .longName = "expr",
-        .shortName = 'E',
         .description = "Interpret [*installables*](@docroot@/command-ref/new-cli/nix.md#installables) as attribute paths relative to the Nix expression *expr*.",
         .category = installablesCategory,
         .labels = {"expr"},
@@ -204,22 +226,15 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes()
     };
 }
 
-Args::CompleterClosure SourceExprCommand::getCompleteInstallable()
-{
-    return [this](AddCompletions & completions, size_t, std::string_view prefix) {
-        completeInstallable(completions, prefix);
-    };
-}
-
-void SourceExprCommand::completeInstallable(AddCompletions & completions, std::string_view prefix)
+void SourceExprCommand::completeInstallable(std::string_view prefix)
 {
     try {
         if (file) {
-            completions.setType(AddCompletions::Type::Attrs);
+            completionType = ctAttrs;
 
             evalSettings.pureEval = false;
             auto state = getEvalState();
-            Expr *e = state->parseExprFromFile(
+            Expr & e = state->parseExprFromFile(
                 resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file)))
                 );
 
@@ -250,15 +265,14 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s
                     std::string name = state->symbols[i.name];
                     if (name.find(searchWord) == 0) {
                         if (prefix_ == "")
-                            completions.add(name);
+                            completions->add(name);
                         else
-                            completions.add(prefix_ + "." + name);
+                            completions->add(prefix_ + "." + name);
                     }
                 }
             }
         } else {
             completeFlakeRefWithFragment(
-                completions,
                 getEvalState(),
                 lockFlags,
                 getDefaultFlakeAttrPathPrefixes(),
@@ -271,7 +285,6 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s
 }
 
 void completeFlakeRefWithFragment(
-    AddCompletions & completions,
     ref<EvalState> evalState,
     flake::LockFlags lockFlags,
     Strings attrPathPrefixes,
@@ -283,16 +296,11 @@ void completeFlakeRefWithFragment(
     try {
         auto hash = prefix.find('#');
         if (hash == std::string::npos) {
-            completeFlakeRef(completions, evalState->store, prefix);
+            completeFlakeRef(evalState->store, prefix);
         } else {
-            completions.setType(AddCompletions::Type::Attrs);
+            completionType = ctAttrs;
 
             auto fragment = prefix.substr(hash + 1);
-            std::string prefixRoot = "";
-            if (fragment.starts_with(".")){
-                fragment = fragment.substr(1);
-                prefixRoot = ".";
-            }
             auto flakeRefS = std::string(prefix.substr(0, hash));
             auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath("."));
 
@@ -301,9 +309,6 @@ void completeFlakeRefWithFragment(
 
             auto root = evalCache->getRoot();
 
-            if (prefixRoot == "."){
-                attrPathPrefixes.clear();
-            }
             /* Complete 'fragment' relative to all the
                attrpath prefixes as well as the root of the
                flake. */
@@ -315,7 +320,7 @@ void completeFlakeRefWithFragment(
                 auto attrPath = parseAttrPath(*evalState, attrPathS);
 
                 std::string lastAttr;
-                if (!attrPath.empty() && !attrPathS.ends_with(".")) {
+                if (!attrPath.empty() && !hasSuffix(attrPathS, ".")) {
                     lastAttr = evalState->symbols[attrPath.back()];
                     attrPath.pop_back();
                 }
@@ -324,11 +329,11 @@ void completeFlakeRefWithFragment(
                 if (!attr) continue;
 
                 for (auto & attr2 : (*attr)->getAttrs()) {
-                    if (std::string_view(evalState->symbols[attr2]).starts_with(lastAttr)) {
+                    if (hasPrefix(evalState->symbols[attr2], lastAttr)) {
                         auto attrPath2 = (*attr)->getAttrPath(attr2);
                         /* Strip the attrpath prefix. */
                         attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
-                        completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2)));
+                        completions->add(flakeRefS + "#" + concatStringsSep(".", evalState->symbols.resolve(attrPath2)));
                     }
                 }
             }
@@ -339,7 +344,7 @@ void completeFlakeRefWithFragment(
                 for (auto & attrPath : defaultFlakeAttrPaths) {
                     auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath));
                     if (!attr) continue;
-                    completions.add(flakeRefS + "#" + prefixRoot);
+                    completions->add(flakeRefS + "#");
                 }
             }
         }
@@ -348,27 +353,27 @@ void completeFlakeRefWithFragment(
     }
 }
 
-void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::string_view prefix)
+void completeFlakeRef(ref<Store> store, std::string_view prefix)
 {
     if (!experimentalFeatureSettings.isEnabled(Xp::Flakes))
         return;
 
     if (prefix == "")
-        completions.add(".");
+        completions->add(".");
 
-    Args::completeDir(completions, 0, prefix);
+    completeDir(0, prefix);
 
     /* Look for registry entries that match the prefix. */
     for (auto & registry : fetchers::getRegistries(store)) {
         for (auto & entry : registry->entries) {
             auto from = entry.from.to_string();
-            if (!prefix.starts_with("flake:") && from.starts_with("flake:")) {
+            if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) {
                 std::string from2(from, 6);
-                if (from2.starts_with(prefix))
-                    completions.add(from2);
+                if (hasPrefix(from2, prefix))
+                    completions->add(from2);
             } else {
-                if (from.starts_with(prefix))
-                    completions.add(from);
+                if (hasPrefix(from, prefix))
+                    completions->add(from);
             }
         }
     }
@@ -439,13 +444,13 @@ Installables SourceExprCommand::parseInstallables(
         auto vFile = state->allocValue();
 
         if (file == "-") {
-            auto e = state->parseStdin();
+            auto & e = state->parseStdin();
             state->eval(e, *vFile);
         }
         else if (file)
             state->evalFile(lookupFileArg(*state, *file), *vFile);
         else {
-            auto e = state->parseExprFromString(*expr, state->rootPath(CanonPath::fromCwd()));
+            auto & e = state->parseExprFromString(*expr, state->rootPath(CanonPath::fromCwd()));
             state->eval(e, *vFile);
         }
 
@@ -748,7 +753,9 @@ RawInstallablesCommand::RawInstallablesCommand()
     expectArgs({
         .label = "installables",
         .handler = {&rawInstallables},
-        .completer = getCompleteInstallable(),
+        .completer = {[&](size_t, std::string_view prefix) {
+            completeInstallable(prefix);
+        }}
     });
 }
 
@@ -761,17 +768,6 @@ void RawInstallablesCommand::applyDefaultInstallables(std::vector<std::string> &
     }
 }
 
-std::vector<FlakeRef> RawInstallablesCommand::getFlakeRefsForCompletion()
-{
-    applyDefaultInstallables(rawInstallables);
-    std::vector<FlakeRef> res;
-    for (auto i : rawInstallables)
-        res.push_back(parseFlakeRefWithFragment(
-            expandTilde(i),
-            absPath(".")).first);
-    return res;
-}
-
 void RawInstallablesCommand::run(ref<Store> store)
 {
     if (readFromStdIn && !isatty(STDIN_FILENO)) {
@@ -785,13 +781,10 @@ void RawInstallablesCommand::run(ref<Store> store)
     run(store, std::move(rawInstallables));
 }
 
-std::vector<FlakeRef> InstallableCommand::getFlakeRefsForCompletion()
+std::vector<std::string> RawInstallablesCommand::getFlakesForCompletion()
 {
-    return {
-        parseFlakeRefWithFragment(
-            expandTilde(_installable),
-            absPath(".")).first
-    };
+    applyDefaultInstallables(rawInstallables);
+    return rawInstallables;
 }
 
 void InstallablesCommand::run(ref<Store> store, std::vector<std::string> && rawInstallables)
@@ -807,7 +800,9 @@ InstallableCommand::InstallableCommand()
         .label = "installable",
         .optional = true,
         .handler = {&_installable},
-        .completer = getCompleteInstallable(),
+        .completer = {[&](size_t, std::string_view prefix) {
+            completeInstallable(prefix);
+        }}
     });
 }
 

src/libcmd/meson.build

@@ -1,73 +0,0 @@
-libcmd_sources = files(
-  'built-path.cc',
-  'command-installable-value.cc',
-  'command.cc',
-  'common-eval-args.cc',
-  'editor-for.cc',
-  'installable-attr-path.cc',
-  'installable-derived-path.cc',
-  'installable-flake.cc',
-  'installable-value.cc',
-  'installables.cc',
-  'legacy.cc',
-  'markdown.cc',
-  'repl.cc',
-  'repl-interacter.cc',
-)
-
-libcmd_headers = files(
-  'built-path.hh',
-  'command-installable-value.hh',
-  'command.hh',
-  'common-eval-args.hh',
-  'editor-for.hh',
-  'installable-attr-path.hh',
-  'installable-derived-path.hh',
-  'installable-flake.hh',
-  'installable-value.hh',
-  'installables.hh',
-  'legacy.hh',
-  'markdown.hh',
-  'repl-interacter.hh',
-  'repl.hh',
-)
-
-libcmd = library(
-  'nixcmd',
-  libcmd_sources,
-  dependencies : [
-    liblixutil,
-    liblixstore,
-    liblixexpr,
-    liblixfetchers,
-    liblixmain,
-    boehm,
-    editline,
-    lowdown,
-    nlohmann_json,
-  ],
-  install : true,
-  # FIXME(Qyriad): is this right?
-  install_rpath : libdir,
-)
-
-install_headers(libcmd_headers, subdir : 'nix', preserve_path : true)
-
-liblixcmd = declare_dependency(
-  include_directories : '.',
-  link_with : libcmd,
-)
-
-# FIXME: not using the pkg-config module because it creates way too many deps
-# while meson migration is in progress, and we want to not include boost here
-configure_file(
-  input : 'nix-cmd.pc.in',
-  output : 'nix-cmd.pc',
-  install_dir : libdir / 'pkgconfig',
-  configuration : {
-    'prefix' : prefix,
-    'libdir' : libdir,
-    'includedir' : includedir,
-    'PACKAGE_VERSION' : meson.project_version(),
-  },
-)

src/libcmd/repl.cc

@@ -29,7 +29,6 @@
 #include "local-fs-store.hh"
 #include "signals.hh"
 #include "print.hh"
-#include "progress-bar.hh"
 
 #if HAVE_BOEHMGC
 #define GC_INCLUDE_NEW
@@ -96,7 +95,7 @@ struct NixRepl
     void reloadFiles();
     void addAttrsToScope(Value & attrs);
     void addVarToScope(const Symbol name, Value & v);
-    Expr * parseString(std::string s);
+    Expr & parseString(std::string s);
     void evalString(std::string s, Value & v);
     void loadDebugTraceEnv(DebugTrace & dt);
 
@@ -187,7 +186,7 @@ ReplExitStatus NixRepl::mainLoop()
         if (state->debugRepl) {
             debuggerNotice = " debugger";
         }
-        notice("Lix %1%%2%\nType :? for help.", nixVersion, debuggerNotice);
+        notice("Nix %1%%2%\nType :? for help.", nixVersion, debuggerNotice);
     }
 
     isFirstRepl = false;
@@ -196,13 +195,11 @@ ReplExitStatus NixRepl::mainLoop()
 
     auto _guard = interacter->init(static_cast<detail::ReplCompleterMixin *>(this));
 
-    /* Stop the progress bar because it interferes with the display of
-       the repl. */
-    stopProgressBar();
-
     std::string input;
 
     while (true) {
+        // Hide the progress bar while waiting for user input, so that it won't interfere.
+        logger->pause();
         // When continuing input from previous lines, don't print a prompt, just align to the same
         // number of chars as the prompt.
         if (!interacter->getLine(input, input.empty() ? ReplPromptType::ReplPrompt : ReplPromptType::ContinuationPrompt)) {
@@ -213,6 +210,7 @@ ReplExitStatus NixRepl::mainLoop()
             // the entire program?
             return ReplExitStatus::QuitAll;
         }
+        logger->resume();
         try {
             switch (processLine(input)) {
                 case ProcessLineResult::Quit:
@@ -274,7 +272,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
             auto dir = std::string(cur, 0, slash);
             auto prefix2 = std::string(cur, slash + 1);
             for (auto & entry : readDirectory(dir == "" ? "/" : dir)) {
-                if (entry.name[0] != '.' && entry.name.starts_with(prefix2))
+                if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2))
                     completions.insert(prev + dir + "/" + entry.name);
             }
         } catch (Error &) {
@@ -299,9 +297,9 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
             auto expr = cur.substr(0, dot);
             auto cur2 = cur.substr(dot + 1);
 
-            Expr * e = parseString(expr);
+            Expr & e = parseString(expr);
             Value v;
-            e->eval(*state, *env, v);
+            e.eval(*state, *env, v);
             state->forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)");
 
             for (auto & i : *v.attrs) {
@@ -662,7 +660,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
             line[p + 1] != '=' &&
             isVarName(name = removeWhitespace(line.substr(0, p))))
         {
-            Expr * e = parseString(line.substr(p + 1));
+            Expr & e = parseString(line.substr(p + 1));
             Value & v(*state->allocValue());
             v.mkThunk(env, e);
             addVarToScope(state->symbols.create(name), v);
@@ -778,7 +776,7 @@ void NixRepl::addVarToScope(const Symbol name, Value & v)
 }
 
 
-Expr * NixRepl::parseString(std::string s)
+Expr & NixRepl::parseString(std::string s)
 {
     return state->parseExprFromString(std::move(s), state->rootPath(CanonPath::fromCwd()), staticEnv);
 }
@@ -786,8 +784,8 @@ Expr * NixRepl::parseString(std::string s)
 
 void NixRepl::evalString(std::string s, Value & v)
 {
-    Expr * e = parseString(s);
-    e->eval(*state, *env, v);
+    Expr & e = parseString(s);
+    e.eval(*state, *env, v);
     state->forceValue(v, v.determinePos(noPos));
 }
 

src/libexpr/eval-inline.hh

@@ -86,11 +86,11 @@ void EvalState::forceValue(Value & v, const PosIdx pos)
 {
     if (v.isThunk()) {
         Env * env = v.thunk.env;
-        Expr * expr = v.thunk.expr;
+        Expr & expr = *v.thunk.expr;
         try {
             v.mkBlackhole();
             //checkInterrupt();
-            expr->eval(*this, *env, v);
+            expr.eval(*this, *env, v);
         } catch (...) {
             v.mkThunk(env, expr);
             tryFixupBlackHolePos(v, pos);

src/libexpr/eval-settings.cc

@@ -29,7 +29,7 @@ static Strings parseNixPath(const std::string & s)
 
         if (*p == ':') {
             auto prefix = std::string(start2, s.end());
-            if (EvalSettings::isPseudoUrl(prefix) || prefix.starts_with("flake:")) {
+            if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) {
                 ++p;
                 while (p != s.end() && *p != ':') ++p;
             }
@@ -82,7 +82,7 @@ bool EvalSettings::isPseudoUrl(std::string_view s)
 
 std::string EvalSettings::resolvePseudoUrl(std::string_view url)
 {
-    if (url.starts_with("channel:"))
+    if (hasPrefix(url, "channel:"))
         return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz";
     else
         return std::string(url);

src/libexpr/eval.cc

@@ -18,7 +18,6 @@
 #include "gc-small-vector.hh"
 #include "fetch-to-store.hh"
 #include "flake/flakeref.hh"
-#include "parser-tab.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -484,7 +483,7 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_)
      */
     Path abspath = canonPath(path_.path.abs());
 
-    if (abspath.starts_with(corepkgsPrefix)) return CanonPath(abspath);
+    if (hasPrefix(abspath, corepkgsPrefix)) return CanonPath(abspath);
 
     for (auto & i : *allowedPaths) {
         if (isDirOrInDir(abspath, i)) {
@@ -527,18 +526,18 @@ void EvalState::checkURI(const std::string & uri)
         if (uri == prefix ||
             (uri.size() > prefix.size()
             && prefix.size() > 0
-            && uri.starts_with(prefix)
+            && hasPrefix(uri, prefix)
             && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/')))
             return;
 
     /* If the URI is a path, then check it against allowedPaths as
        well. */
-    if (uri.starts_with("/")) {
+    if (hasPrefix(uri, "/")) {
         checkSourcePath(CanonPath(uri));
         return;
     }
 
-    if (uri.starts_with("file://")) {
+    if (hasPrefix(uri, "file://")) {
         checkSourcePath(CanonPath(std::string(uri, 7)));
         return;
     }
@@ -642,7 +641,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
     }
 
     auto envName = symbols.create(primOp.name);
-    if (primOp.name.starts_with("__"))
+    if (hasPrefix(primOp.name, "__"))
         primOp.name = primOp.name.substr(2);
 
     Value * v = allocValue();
@@ -677,21 +676,12 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
 }
 
 
-static std::set<std::string_view> sortedBindingNames(const SymbolTable & st, const StaticEnv & se)
-{
-    std::set<std::string_view> bindings;
-    for (auto [symbol, displ] : se.vars)
-        bindings.emplace(st[symbol]);
-    return bindings;
-}
-
-
 // just for the current level of StaticEnv, not the whole chain.
 void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se)
 {
     std::cout << ANSI_MAGENTA;
-    for (auto & i : sortedBindingNames(st, se))
-        std::cout << i << " ";
+    for (auto & i : se.vars)
+        std::cout << st[i.first] << " ";
     std::cout << ANSI_NORMAL;
     std::cout << std::endl;
 }
@@ -700,14 +690,13 @@ void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se)
 void printWithBindings(const SymbolTable & st, const Env & env)
 {
     if (!env.values[0]->isThunk()) {
-        std::set<std::string_view> bindings;
-        for (const auto & attr : *env.values[0]->attrs)
-            bindings.emplace(st[attr.name]);
-
         std::cout << "with: ";
         std::cout << ANSI_MAGENTA;
-        for (auto & i : bindings)
-            std::cout << i << " ";
+        Bindings::iterator j = env.values[0]->attrs->begin();
+        while (j != env.values[0]->attrs->end()) {
+            std::cout << st[j->name] << " ";
+            ++j;
+        }
         std::cout << ANSI_NORMAL;
         std::cout << std::endl;
     }
@@ -728,9 +717,9 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env &
         std::cout << ANSI_MAGENTA;
         // for the top level, don't print the double underscore ones;
         // they are in builtins.
-        for (auto & i : sortedBindingNames(st, se))
-            if (!i.starts_with("__"))
-                std::cout << i << " ";
+        for (auto & i : se.vars)
+            if (!hasPrefix(st[i.first], "__"))
+                std::cout << st[i.first] << " ";
         std::cout << ANSI_NORMAL;
         std::cout << std::endl;
         if (se.isWith)
@@ -935,14 +924,14 @@ void EvalState::mkList(Value & v, size_t size)
 
 unsigned long nrThunks = 0;
 
-static inline void mkThunk(Value & v, Env & env, Expr * expr)
+static inline void mkThunk(Value & v, Env & env, Expr & expr)
 {
     v.mkThunk(&env, expr);
     nrThunks++;
 }
 
 
-void EvalState::mkThunk_(Value & v, Expr * expr)
+void EvalState::mkThunk_(Value & v, Expr & expr)
 {
     mkThunk(v, baseEnv, expr);
 }
@@ -1043,7 +1032,7 @@ void EvalState::mkSingleDerivedPathString(
 Value * Expr::maybeThunk(EvalState & state, Env & env)
 {
     Value * v = state.allocValue();
-    mkThunk(*v, env, this);
+    mkThunk(*v, env, *this);
     return v;
 }
 
@@ -1107,7 +1096,7 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial
         e = j->second;
 
     if (!e)
-        e = parseExprFromFile(checkSourcePath(resolvedPath));
+        e = &parseExprFromFile(checkSourcePath(resolvedPath));
 
     cacheFile(path, resolvedPath, e, v, mustBeTrivial);
 }
@@ -1144,7 +1133,7 @@ void EvalState::cacheFile(
         if (mustBeTrivial &&
             !(dynamic_cast<ExprAttrs *>(e)))
             error<EvalError>("file '%s' must be an attribute set", path).debugThrow();
-        eval(e, v);
+        eval(*e, v);
     } catch (Error & e) {
         addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath.to_string());
         throw;
@@ -1155,23 +1144,23 @@ void EvalState::cacheFile(
 }
 
 
-void EvalState::eval(Expr * e, Value & v)
+void EvalState::eval(Expr & e, Value & v)
 {
-    e->eval(*this, baseEnv, v);
+    e.eval(*this, baseEnv, v);
 }
 
 
-inline bool EvalState::evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx)
+inline bool EvalState::evalBool(Env & env, Expr & e, const PosIdx pos, std::string_view errorCtx)
 {
     try {
         Value v;
-        e->eval(*this, env, v);
+        e.eval(*this, env, v);
         if (v.type() != nBool)
             error<TypeError>(
                  "expected a Boolean but found %1%: %2%",
                  showType(v),
                  ValuePrinter(*this, v, errorPrintOptions)
-             ).atPos(pos).withFrame(env, *e).debugThrow();
+             ).atPos(pos).withFrame(env, e).debugThrow();
         return v.boolean;
     } catch (Error & e) {
         e.addTrace(positions[pos], errorCtx);
@@ -1180,16 +1169,16 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const PosIdx pos, std::stri
 }
 
 
-inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx)
+inline void EvalState::evalAttrs(Env & env, Expr & e, Value & v, const PosIdx pos, std::string_view errorCtx)
 {
     try {
-        e->eval(*this, env, v);
+        e.eval(*this, env, v);
         if (v.type() != nAttrs)
             error<TypeError>(
                 "expected a set but found %1%: %2%",
                 showType(v),
                 ValuePrinter(*this, v, errorPrintOptions)
-            ).withFrame(env, *e).debugThrow();
+            ).withFrame(env, e).debugThrow();
     } catch (Error & e) {
         e.addTrace(positions[pos], errorCtx);
         throw;
@@ -1232,7 +1221,7 @@ Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up)
     inheritEnv.up = &up;
 
     Displacement displ = 0;
-    for (auto from : *inheritFromExprs)
+    for (auto & from : *inheritFromExprs)
         inheritEnv.values[displ++] = from->maybeThunk(state, up);
 
     return &inheritEnv;
@@ -1262,7 +1251,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
             Value * vAttr;
             if (hasOverrides && i.second.kind != AttrDef::Kind::Inherited) {
                 vAttr = state.allocValue();
-                mkThunk(*vAttr, *i.second.chooseByKind(&env2, &env, inheritEnv), i.second.e);
+                mkThunk(*vAttr, *i.second.chooseByKind(&env2, &env, inheritEnv), *i.second.e);
             } else
                 vAttr = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv));
             env2.values[displ++] = vAttr;
@@ -1857,13 +1846,13 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
 void ExprIf::eval(EvalState & state, Env & env, Value & v)
 {
     // We cheat in the parser, and pass the position of the condition as the position of the if itself.
-    (state.evalBool(env, cond, pos, "while evaluating a branch condition") ? then : else_)->eval(state, env, v);
+    (state.evalBool(env, *cond, pos, "while evaluating a branch condition") ? *then : *else_).eval(state, env, v);
 }
 
 
 void ExprAssert::eval(EvalState & state, Env & env, Value & v)
 {
-    if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) {
+    if (!state.evalBool(env, *cond, pos, "in the condition of the assert statement")) {
         std::ostringstream out;
         cond->show(state.symbols, out);
         state.error<AssertionError>("assertion '%1%' failed", out.str()).atPos(pos).withFrame(env, *this).debugThrow();
@@ -1874,7 +1863,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
 
 void ExprOpNot::eval(EvalState & state, Env & env, Value & v)
 {
-    v.mkBool(!state.evalBool(env, e, getPos(), "in the argument of the not operator")); // XXX: FIXME: !
+    v.mkBool(!state.evalBool(env, *e, getPos(), "in the argument of the not operator")); // XXX: FIXME: !
 }
 
 
@@ -1896,27 +1885,27 @@ void ExprOpNEq::eval(EvalState & state, Env & env, Value & v)
 
 void ExprOpAnd::eval(EvalState & state, Env & env, Value & v)
 {
-    v.mkBool(state.evalBool(env, e1, pos, "in the left operand of the AND (&&) operator") && state.evalBool(env, e2, pos, "in the right operand of the AND (&&) operator"));
+    v.mkBool(state.evalBool(env, *e1, pos, "in the left operand of the AND (&&) operator") && state.evalBool(env, *e2, pos, "in the right operand of the AND (&&) operator"));
 }
 
 
 void ExprOpOr::eval(EvalState & state, Env & env, Value & v)
 {
-    v.mkBool(state.evalBool(env, e1, pos, "in the left operand of the OR (||) operator") || state.evalBool(env, e2, pos, "in the right operand of the OR (||) operator"));
+    v.mkBool(state.evalBool(env, *e1, pos, "in the left operand of the OR (||) operator") || state.evalBool(env, *e2, pos, "in the right operand of the OR (||) operator"));
 }
 
 
 void ExprOpImpl::eval(EvalState & state, Env & env, Value & v)
 {
-    v.mkBool(!state.evalBool(env, e1, pos, "in the left operand of the IMPL (->) operator") || state.evalBool(env, e2, pos, "in the right operand of the IMPL (->) operator"));
+    v.mkBool(!state.evalBool(env, *e1, pos, "in the left operand of the IMPL (->) operator") || state.evalBool(env, *e2, pos, "in the right operand of the IMPL (->) operator"));
 }
 
 
 void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
 {
     Value v1, v2;
-    state.evalAttrs(env, e1, v1, pos, "in the left operand of the update (//) operator");
-    state.evalAttrs(env, e2, v2, pos, "in the right operand of the update (//) operator");
+    state.evalAttrs(env, *e1, v1, pos, "in the left operand of the update (//) operator");
+    state.evalAttrs(env, *e2, v2, pos, "in the right operand of the update (//) operator");
 
     state.nrOpUpdates++;
 
@@ -2020,10 +2009,10 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     };
 
     // List of returned strings. References to these Values must NOT be persisted.
-    SmallTemporaryValueVector<conservativeStackReservation> values(es->size());
+    SmallTemporaryValueVector<conservativeStackReservation> values(es.size());
     Value * vTmpP = values.data();
 
-    for (auto & [i_pos, i] : *es) {
+    for (auto & [i_pos, i] : es) {
         Value & vTmp = *vTmpP++;
         i->eval(state, env, vTmp);
 
@@ -2053,7 +2042,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
             } else
                 state.error<EvalError>("cannot add %1% to a float", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow();
         } else {
-            if (s.empty()) s.reserve(es->size());
+            if (s.empty()) s.reserve(es.size());
             /* skip canonization of first path, which would only be not
             canonized in the first place if it's coming from a ./${foo} type
             path */
@@ -2737,39 +2726,39 @@ SourcePath resolveExprPath(SourcePath path)
 }
 
 
-Expr * EvalState::parseExprFromFile(const SourcePath & path)
+Expr & EvalState::parseExprFromFile(const SourcePath & path)
 {
     return parseExprFromFile(path, staticBaseEnv);
 }
 
 
-Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv)
+Expr & EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv)
 {
     auto buffer = path.readFile();
     // readFile hopefully have left some extra space for terminators
     buffer.append("\0\0", 2);
-    return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv);
+    return *parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
+Expr & EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
 {
     // NOTE this method (and parseStdin) must take care to *fully copy* their input
     // into their respective Pos::Origin until the parser stops overwriting its input
     // data.
     auto s = make_ref<std::string>(s_);
     s_.append("\0\0", 2);
-    return parse(s_.data(), s_.size(), Pos::String{.source = s}, basePath, staticEnv);
+    return *parse(s_.data(), s_.size(), Pos::String{.source = s}, basePath, staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath)
+Expr & EvalState::parseExprFromString(std::string s, const SourcePath & basePath)
 {
     return parseExprFromString(std::move(s), basePath, staticBaseEnv);
 }
 
 
-Expr * EvalState::parseStdin()
+Expr & EvalState::parseStdin()
 {
     // NOTE this method (and parseExprFromString) must take care to *fully copy* their
     // input into their respective Pos::Origin until the parser stops overwriting its
@@ -2779,7 +2768,7 @@ Expr * EvalState::parseStdin()
     // drainFD should have left some extra space for terminators
     auto s = make_ref<std::string>(buffer);
     buffer.append("\0\0", 2);
-    return parse(buffer.data(), buffer.size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv);
+    return *parse(buffer.data(), buffer.size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv);
 }
 
 
@@ -2805,7 +2794,7 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_
         if (pathExists(res)) return CanonPath(canonPath(res));
     }
 
-    if (path.starts_with("nix/"))
+    if (hasPrefix(path, "nix/"))
         return CanonPath(concatStrings(corepkgsPrefix, path.substr(4)));
 
     error<ThrownError>(
@@ -2838,7 +2827,7 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
         }
     }
 
-    else if (value.starts_with("flake:")) {
+    else if (hasPrefix(value, "flake:")) {
         experimentalFeatureSettings.require(Xp::Flakes);
         auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false);
         debug("fetching flake search path element '%s''", value);
@@ -2868,21 +2857,6 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
 }
 
 
-Expr * EvalState::parse(
-    char * text,
-    size_t length,
-    Pos::Origin origin,
-    const SourcePath & basePath,
-    std::shared_ptr<StaticEnv> & staticEnv)
-{
-    auto result = parseExprFromBuf(text, length, origin, basePath, symbols, positions, exprSymbols);
-
-    result->bindVars(*this, staticEnv);
-
-    return result;
-}
-
-
 std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
 {
     state.error<TypeError>(

src/libexpr/eval.hh

@@ -337,16 +337,16 @@ public:
     /**
      * Parse a Nix expression from the specified file.
      */
-    Expr * parseExprFromFile(const SourcePath & path);
-    Expr * parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv);
+    Expr & parseExprFromFile(const SourcePath & path);
+    Expr & parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv);
 
     /**
      * Parse a Nix expression from the specified string.
      */
-    Expr * parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv);
-    Expr * parseExprFromString(std::string s, const SourcePath & basePath);
+    Expr & parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv);
+    Expr & parseExprFromString(std::string s, const SourcePath & basePath);
 
-    Expr * parseStdin();
+    Expr & parseStdin();
 
     /**
      * Evaluate an expression read from the given file to normal
@@ -387,15 +387,15 @@ public:
      *
      * @param [out] v The resulting is stored here.
      */
-    void eval(Expr * e, Value & v);
+    void eval(Expr & e, Value & v);
 
     /**
      * Evaluation the expression, then verify that it has the expected
      * type.
      */
-    inline bool evalBool(Env & env, Expr * e);
-    inline bool evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx);
-    inline void evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx);
+    inline bool evalBool(Env & env, Expr & e);
+    inline bool evalBool(Env & env, Expr & e, const PosIdx pos, std::string_view errorCtx);
+    inline void evalAttrs(Env & env, Expr & e, Value & v, const PosIdx pos, std::string_view errorCtx);
 
     /**
      * If `v` is a thunk, enter it and overwrite `v` with the result
@@ -616,7 +616,7 @@ public:
     }
 
     void mkList(Value & v, size_t length);
-    void mkThunk_(Value & v, Expr * expr);
+    void mkThunk_(Value & v, Expr & expr);
     void mkPos(Value & v, PosIdx pos);
 
     /**

src/libexpr/flake/config.cc

@@ -35,7 +35,7 @@ void ConfigFile::apply()
 
     for (auto & [name, value] : settings) {
 
-        auto baseName = name.starts_with("extra-") ? std::string(name, 6) : name;
+        auto baseName = hasPrefix(name, "extra-") ? std::string(name, 6) : name;
 
         // FIXME: Move into libutil/config.cc.
         std::string valueS;

src/libexpr/flake/flake.cc

@@ -451,8 +451,8 @@ LockedFlake lockFlake(
 
                     assert(input.ref);
 
-                    /* Do we have an entry in the existing lock file?
-                       And the input is not in updateInputs? */
+                    /* Do we have an entry in the existing lock file? And we
+                       don't have a --update-input flag for this input? */
                     std::shared_ptr<LockedNode> oldLock;
 
                     updatesUsed.insert(inputPath);
@@ -476,8 +476,9 @@ LockedFlake lockFlake(
 
                         node->inputs.insert_or_assign(id, childNode);
 
-                        /* If we have this input in updateInputs, then we
-                           must fetch the flake to update it. */
+                        /* If we have an --update-input flag for an input
+                           of this input, then we must fetch the flake to
+                           update it. */
                         auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
 
                         auto mustRefetch =
@@ -619,14 +620,19 @@ LockedFlake lockFlake(
 
         for (auto & i : lockFlags.inputUpdates)
             if (!updatesUsed.count(i))
-                warn("'%s' does not match any input of this flake", printInputPath(i));
+                warn("the flag '--update-input %s' does not match any input", printInputPath(i));
 
         /* Check 'follows' inputs. */
         newLockFile.check();
 
         debug("new lock file: %s", newLockFile);
 
+        auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
         auto sourcePath = topRef.input.getSourcePath();
+        auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt;
+        if (lockFlags.outputLockFilePath) {
+            outputLockFilePath = lockFlags.outputLockFilePath;
+        }
 
         /* Check whether we need to / can write the new lock file. */
         if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) {
@@ -634,7 +640,7 @@ LockedFlake lockFlake(
             auto diff = LockFile::diff(oldLockFile, newLockFile);
 
             if (lockFlags.writeLockFile) {
-                if (sourcePath || lockFlags.outputLockFilePath) {
+                if (outputLockFilePath) {
                     if (auto unlockedInput = newLockFile.isUnlocked()) {
                         if (fetchSettings.warnDirty)
                             warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput);
@@ -642,48 +648,41 @@ LockedFlake lockFlake(
                         if (!lockFlags.updateLockFile)
                             throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef);
 
-                        auto newLockFileS = fmt("%s\n", newLockFile);
-
-                        if (lockFlags.outputLockFilePath) {
-                            if (lockFlags.commitLockFile)
-                                throw Error("'--commit-lock-file' and '--output-lock-file' are incompatible");
-                            writeFile(*lockFlags.outputLockFilePath, newLockFileS);
-                        } else {
-                            auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
-                            auto outputLockFilePath = *sourcePath + "/" + relPath;
-
-                            bool lockFileExists = pathExists(outputLockFilePath);
+                        bool lockFileExists = pathExists(*outputLockFilePath);
 
+                        if (lockFileExists) {
                             auto s = chomp(diff);
-                            if (lockFileExists) {
-                                if (s.empty())
-                                    warn("updating lock file '%s'", outputLockFilePath);
-                                else
-                                    warn("updating lock file '%s':\n%s", outputLockFilePath, s);
-                            } else
-                                warn("creating lock file '%s':\n%s", outputLockFilePath, s);
+                            if (s.empty())
+                                warn("updating lock file '%s'", *outputLockFilePath);
+                            else
+                                warn("updating lock file '%s':\n%s", *outputLockFilePath, s);
+                        } else
+                            warn("creating lock file '%s'", *outputLockFilePath);
 
-                            std::optional<std::string> commitMessage = std::nullopt;
+                        newLockFile.write(*outputLockFilePath);
 
-                            if (lockFlags.commitLockFile) {
-                                std::string cm;
+                        std::optional<std::string> commitMessage = std::nullopt;
+                        if (lockFlags.commitLockFile) {
+                            if (lockFlags.outputLockFilePath) {
+                                throw Error("--commit-lock-file and --output-lock-file are currently incompatible");
+                            }
+                            std::string cm;
 
-                                cm = fetchSettings.commitLockFileSummary.get();
+                            cm = fetchSettings.commitLockFileSummary.get();
 
-                                if (cm == "") {
-                                    cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add");
-                                }
-
-                                cm += "\n\nFlake lock file updates:\n\n";
-                                cm += filterANSIEscapes(diff, true);
-                                commitMessage = cm;
+                            if (cm == "") {
+                                cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add");
                             }
 
-                            topRef.input.putFile(
-                                CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"),
-                                newLockFileS, commitMessage);
+                            cm += "\n\nFlake lock file updates:\n\n";
+                            cm += filterANSIEscapes(diff, true);
+                            commitMessage = cm;
                         }
 
+                        topRef.input.markChangedFile(
+                            (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
+                            commitMessage);
+
                         /* Rewriting the lockfile changed the top-level
                            repo, so we should re-read it. FIXME: we could
                            also just clear the 'rev' field... */

src/libexpr/flake/flakeref.cc

@@ -186,7 +186,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
             }
 
         } else {
-            if (!path.starts_with("/"))
+            if (!hasPrefix(path, "/"))
                 throw BadURL("flake reference '%s' is not an absolute path", url);
             auto query = decodeQuery(match[2]);
             path = canonPath(path + "/" + getOr(query, "dir", ""));

src/libexpr/flake/meson.build

@@ -1,8 +0,0 @@
-libexpr_generated_headers += custom_target(
-  command : [ 'bash', '-c', 'echo \'R"__NIX_STR(\' | cat - @INPUT@ && echo \')__NIX_STR"\'' ],
-  input : 'call-flake.nix',
-  output : '@PLAINNAME@.gen.hh',
-  capture : true,
-  install : true,
-  install_dir : includedir / 'nix/flake',
-)

src/libexpr/gc-small-vector.hh

@@ -4,7 +4,6 @@
 
 #if HAVE_BOEHMGC
 
-#define GC_INCLUDE_NEW
 #include <gc/gc.h>
 #include <gc/gc_cpp.h>
 #include <gc/gc_allocator.h>
@@ -40,4 +39,4 @@ using SmallValueVector = SmallVector<Value *, nItems>;
 template <size_t nItems>
 using SmallTemporaryValueVector = SmallVector<Value, nItems>;
 
-}
+}

src/libexpr/lexer.l

@@ -1,296 +0,0 @@
-%option reentrant bison-bridge bison-locations
-%option align
-%option noyywrap
-%option never-interactive
-%option stack
-%option nodefault
-%option nounput noyy_top_state
-
-
-%s DEFAULT
-%x STRING
-%x IND_STRING
-%x INPATH
-%x INPATH_SLASH
-%x PATH_START
-
-
-%{
-#ifdef __clang__
-#pragma clang diagnostic ignored "-Wunneeded-internal-declaration"
-#endif
-
-// yacc generates code that uses unannotated fallthrough.
-#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
-#ifdef __clang__
-#pragma clang diagnostic ignored "-Wimplicit-fallthrough"
-#endif
-
-#include <boost/lexical_cast.hpp>
-
-#include "nixexpr.hh"
-#include "parser-tab.hh"
-
-using namespace nix;
-
-namespace nix {
-
-#define CUR_POS state->at(*yylloc)
-
-static void initLoc(YYLTYPE * loc)
-{
-    loc->first_line = loc->last_line = 0;
-    loc->first_column = loc->last_column = 0;
-}
-
-static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
-{
-    loc->stash();
-
-    loc->first_column = loc->last_column;
-    loc->last_column += len;
-}
-
-
-// we make use of the fact that the parser receives a private copy of the input
-// string and can munge around in it.
-static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
-{
-    char * result = s;
-    char * t = s;
-    char c;
-    // the input string is terminated with *two* NULs, so we can safely take
-    // *one* character after the one being checked against.
-    while ((c = *s++)) {
-        if (c == '\\') {
-            c = *s++;
-            if (c == 'n') *t = '\n';
-            else if (c == 'r') *t = '\r';
-            else if (c == 't') *t = '\t';
-            else *t = c;
-        }
-        else if (c == '\r') {
-            /* Normalise CR and CR/LF into LF. */
-            *t = '\n';
-            if (*s == '\n') s++; /* cr/lf */
-        }
-        else *t = c;
-        t++;
-    }
-    return {result, size_t(t - result)};
-}
-
-
-}
-
-#define YY_USER_INIT initLoc(yylloc)
-#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng);
-
-#define PUSH_STATE(state) yy_push_state(state, yyscanner)
-#define POP_STATE() yy_pop_state(yyscanner)
-
-%}
-
-
-ANY         .|\n
-ID          [a-zA-Z\_][a-zA-Z0-9\_\'\-]*
-INT         [0-9]+
-FLOAT       (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)?
-PATH_CHAR   [a-zA-Z0-9\.\_\-\+]
-PATH        {PATH_CHAR}*(\/{PATH_CHAR}+)+\/?
-PATH_SEG    {PATH_CHAR}*\/
-HPATH       \~(\/{PATH_CHAR}+)+\/?
-HPATH_START \~\/
-SPATH       \<{PATH_CHAR}+(\/{PATH_CHAR}+)*\>
-URI         [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+
-
-
-%%
-
-
-if          { return IF; }
-then        { return THEN; }
-else        { return ELSE; }
-assert      { return ASSERT; }
-with        { return WITH; }
-let         { return LET; }
-in          { return IN; }
-rec         { return REC; }
-inherit     { return INHERIT; }
-or          { return OR_KW; }
-\.\.\.      { return ELLIPSIS; }
-
-\=\=        { return EQ; }
-\!\=        { return NEQ; }
-\<\=        { return LEQ; }
-\>\=        { return GEQ; }
-\&\&        { return AND; }
-\|\|        { return OR; }
-\-\>        { return IMPL; }
-\/\/        { return UPDATE; }
-\+\+        { return CONCAT; }
-
-{ID}        { yylval->id = {yytext, (size_t) yyleng}; return ID; }
-{INT}       { errno = 0;
-              try {
-                  yylval->n = boost::lexical_cast<int64_t>(yytext);
-              } catch (const boost::bad_lexical_cast &) {
-                  throw ParseError(ErrorInfo{
-                      .msg = HintFmt("invalid integer '%1%'", yytext),
-                      .pos = state->positions[CUR_POS],
-                  });
-              }
-              return INT;
-            }
-{FLOAT}     { errno = 0;
-              yylval->nf = strtod(yytext, 0);
-              if (errno != 0)
-                  throw ParseError(ErrorInfo{
-                      .msg = HintFmt("invalid float '%1%'", yytext),
-                      .pos = state->positions[CUR_POS],
-                  });
-              return FLOAT;
-            }
-
-\$\{        { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
-
-\}          { /* State INITIAL only exists at the bottom of the stack and is
-                 used as a marker. DEFAULT replaces it everywhere else.
-                 Popping when in INITIAL state causes an empty stack exception,
-                 so don't */
-              if (YYSTATE != INITIAL)
-                POP_STATE();
-              return '}';
-            }
-\{          { PUSH_STATE(DEFAULT); return '{'; }
-
-\"          { PUSH_STATE(STRING); return '"'; }
-<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" |
-<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ {
-                /* It is impossible to match strings ending with '$' with one
-                   regex because trailing contexts are only valid at the end
-                   of a rule. (A sane but undocumented limitation.) */
-                yylval->str = unescapeStr(state->symbols, yytext, yyleng);
-                return STR;
-              }
-<STRING>\$\{  { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
-<STRING>\"    { POP_STATE(); return '"'; }
-<STRING>\$|\\|\$\\ {
-                /* This can only occur when we reach EOF, otherwise the above
-                   (...|\$[^\{\"\\]|\\.|\$\\.)+ would have triggered.
-                   This is technically invalid, but we leave the problem to the
-                   parser who fails with exact location. */
-                return EOF;
-              }
-
-\'\'(\ *\n)?     { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
-<IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ {
-                   yylval->str = {yytext, (size_t) yyleng, true};
-                   return IND_STR;
-                 }
-<IND_STRING>\'\'\$ |
-<IND_STRING>\$   {
-                   yylval->str = {"$", 1};
-                   return IND_STR;
-                 }
-<IND_STRING>\'\'\' {
-                   yylval->str = {"''", 2};
-                   return IND_STR;
-                 }
-<IND_STRING>\'\'\\{ANY} {
-                   yylval->str = unescapeStr(state->symbols, yytext + 2, yyleng - 2);
-                   return IND_STR;
-                 }
-<IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
-<IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; }
-<IND_STRING>\'   {
-                   yylval->str = {"'", 1};
-                   return IND_STR;
-                 }
-
-{PATH_SEG}\$\{ |
-{HPATH_START}\$\{ {
-  PUSH_STATE(PATH_START);
-  yyless(0);
-  yylloc->unstash();
-}
-
-<PATH_START>{PATH_SEG} {
-  POP_STATE();
-  PUSH_STATE(INPATH_SLASH);
-  yylval->path = {yytext, (size_t) yyleng};
-  return PATH;
-}
-
-<PATH_START>{HPATH_START} {
-  POP_STATE();
-  PUSH_STATE(INPATH_SLASH);
-  yylval->path = {yytext, (size_t) yyleng};
-  return HPATH;
-}
-
-{PATH} {
-  if (yytext[yyleng-1] == '/')
-    PUSH_STATE(INPATH_SLASH);
-  else
-    PUSH_STATE(INPATH);
-  yylval->path = {yytext, (size_t) yyleng};
-  return PATH;
-}
-{HPATH} {
-  if (yytext[yyleng-1] == '/')
-    PUSH_STATE(INPATH_SLASH);
-  else
-    PUSH_STATE(INPATH);
-  yylval->path = {yytext, (size_t) yyleng};
-  return HPATH;
-}
-
-<INPATH,INPATH_SLASH>\$\{ {
-  POP_STATE();
-  PUSH_STATE(INPATH);
-  PUSH_STATE(DEFAULT);
-  return DOLLAR_CURLY;
-}
-<INPATH,INPATH_SLASH>{PATH}|{PATH_SEG}|{PATH_CHAR}+ {
-  POP_STATE();
-  if (yytext[yyleng-1] == '/')
-      PUSH_STATE(INPATH_SLASH);
-  else
-      PUSH_STATE(INPATH);
-  yylval->str = {yytext, (size_t) yyleng};
-  return STR;
-}
-<INPATH>{ANY} |
-<INPATH><<EOF>> {
-  /* if we encounter a non-path character we inform the parser that the path has
-     ended with a PATH_END token and re-parse this character in the default
-     context (it may be ')', ';', or something of that sort) */
-  POP_STATE();
-  yyless(0);
-  yylloc->unstash();
-  return PATH_END;
-}
-
-<INPATH_SLASH>{ANY} |
-<INPATH_SLASH><<EOF>> {
-  throw ParseError(ErrorInfo{
-      .msg = HintFmt("path has a trailing slash"),
-      .pos = state->positions[CUR_POS],
-  });
-}
-
-{SPATH}     { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }
-{URI}       { yylval->uri = {yytext, (size_t) yyleng}; return URI; }
-
-[ \t\r\n]+    /* eat up whitespace */
-\#[^\r\n]*    /* single-line comments */
-\/\*([^*]|\*+[^*/])*\*+\/  /* long comments */
-
-{ANY}       {
-              /* Don't return a negative number, as this will cause
-                 Bison to stop parsing without an error. */
-              return (unsigned char) yytext[0];
-            }
-
-%%

src/libexpr/local.mk

@@ -7,12 +7,12 @@ libexpr_DIR := $(d)
 libexpr_SOURCES := \
   $(wildcard $(d)/*.cc) \
   $(wildcard $(d)/value/*.cc) \
+  $(wildcard $(d)/parser/*.cc) \
   $(wildcard $(d)/primops/*.cc) \
-  $(wildcard $(d)/flake/*.cc) \
-  $(d)/lexer-tab.cc \
-  $(d)/parser-tab.cc
+  $(wildcard $(d)/flake/*.cc)
 
-libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
+libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr \
+  -I pegtl/include
 
 libexpr_LIBS = libutil libstore libfetchers
 
@@ -26,16 +26,6 @@ endif
 # because inline functions in libexpr's header files call libgc.
 libexpr_LDFLAGS_PROPAGATED = $(BDW_GC_LIBS)
 
-libexpr_ORDER_AFTER := $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
-
-$(d)/parser-tab.cc $(d)/parser-tab.hh: $(d)/parser.y
-	$(trace-gen) bison -v -o $(libexpr_DIR)/parser-tab.cc $< -d
-
-$(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l
-	$(trace-gen) flex --outfile $(libexpr_DIR)/lexer-tab.cc --header-file=$(libexpr_DIR)/lexer-tab.hh $<
-
-clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
-
 $(eval $(call install-file-in, $(buildprefix)$(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
 
 $(foreach i, $(wildcard src/libexpr/value/*.hh), \

src/libexpr/meson.build

@@ -1,176 +0,0 @@
-parser_tab = custom_target(
-  input : 'parser.y',
-  output : [
-    'parser-tab.cc',
-    'parser-tab.hh',
-  ],
-  command : [
-    'bison',
-    '-v',
-    '-o',
-    '@OUTPUT0@',
-    '@INPUT@',
-    '-d',
-  ],
-  # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add
-  # an install script below which removes parser-tab.cc.
-  install : true,
-  install_dir : includedir / 'nix',
-)
-
-lexer_tab = custom_target(
-  input : [
-    'lexer.l',
-    parser_tab,
-  ],
-  output : [
-    'lexer-tab.cc',
-    'lexer-tab.hh',
-  ],
-  command : [
-    'flex',
-    '--outfile',
-    '@OUTPUT0@',
-    '--header-file=' + '@OUTPUT1@',
-    '@INPUT0@',
-  ],
-  # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add
-  # an install script below which removes lexer-tab.cc.
-  install : true,
-  install_dir : includedir / 'nix',
-)
-
-# TODO(Qyriad): When the parser and lexer are rewritten this should be removed.
-# NOTE(Qyriad): We do this this way instead of an inline bash or rm command
-# due to subtleties in Meson. Check the comments in cleanup-install.bash for details.
-meson.add_install_script(
-  bash,
-  meson.project_source_root() / 'meson/cleanup-install.bash',
-  '@0@'.format(includedir),
-)
-
-libexpr_generated_headers = [
-  gen_header.process('primops/derivation.nix', preserve_path_from : meson.current_source_dir()),
-]
-foreach header : [ 'imported-drv-to-derivation.nix', 'fetchurl.nix' ]
-  libexpr_generated_headers += custom_target(
-    command : [ 'bash', '-c', 'echo \'R"__NIX_STR(\' | cat - @INPUT@ && echo \')__NIX_STR"\'' ],
-    input : header,
-    output : '@PLAINNAME@.gen.hh',
-    capture : true,
-    install : true,
-    install_dir : includedir / 'nix',
-  )
-endforeach
-subdir('flake')
-
-libexpr_sources = files(
-  'attr-path.cc',
-  'attr-set.cc',
-  'eval-cache.cc',
-  'eval-error.cc',
-  'eval-settings.cc',
-  'eval.cc',
-  'function-trace.cc',
-  'get-drvs.cc',
-  'json-to-value.cc',
-  'nixexpr.cc',
-  'paths.cc',
-  'primops.cc',
-  'print-ambiguous.cc',
-  'print.cc',
-  'search-path.cc',
-  'value-to-json.cc',
-  'value-to-xml.cc',
-  'flake/config.cc',
-  'flake/flake.cc',
-  'flake/flakeref.cc',
-  'flake/lockfile.cc',
-  'primops/context.cc',
-  'primops/fetchClosure.cc',
-  'primops/fetchMercurial.cc',
-  'primops/fetchTree.cc',
-  'primops/fromTOML.cc',
-  'value/context.cc',
-)
-
-libexpr_headers = files(
-  'attr-path.hh',
-  'attr-set.hh',
-  'eval-cache.hh',
-  'eval-error.hh',
-  'eval-inline.hh',
-  'eval-settings.hh',
-  'eval.hh',
-  'flake/flake.hh',
-  'flake/flakeref.hh',
-  'flake/lockfile.hh',
-  'function-trace.hh',
-  'gc-small-vector.hh',
-  'get-drvs.hh',
-  'json-to-value.hh',
-  'nixexpr.hh',
-  'parser-state.hh',
-  'pos-idx.hh',
-  'pos-table.hh',
-  'primops.hh',
-  'print-ambiguous.hh',
-  'print-options.hh',
-  'print.hh',
-  'repl-exit-status.hh',
-  'search-path.hh',
-  'symbol-table.hh',
-  'value/context.hh',
-  'value-to-json.hh',
-  'value-to-xml.hh',
-  'value.hh',
-)
-
-libexpr = library(
-  'nixexpr',
-  libexpr_sources,
-  parser_tab,
-  lexer_tab,
-  libexpr_generated_headers,
-  dependencies : [
-    liblixutil,
-    liblixstore,
-    liblixfetchers,
-    boehm,
-    boost,
-    toml11,
-    nlohmann_json,
-  ],
-  # for shared.hh
-  include_directories : [
-    '../libmain',
-  ],
-  install : true,
-  # FIXME(Qyriad): is this right?
-  install_rpath : libdir,
-)
-
-install_headers(
-  libexpr_headers,
-  subdir : 'nix',
-  preserve_path : true,
-)
-
-liblixexpr = declare_dependency(
-  include_directories : include_directories('.'),
-  link_with : libexpr,
-)
-
-# FIXME: not using the pkg-config module because it creates way too many deps
-# while meson migration is in progress, and we want to not include boost here
-configure_file(
-  input : 'nix-expr.pc.in',
-  output : 'nix-expr.pc',
-  install_dir : libdir / 'pkgconfig',
-  configuration : {
-    'prefix' : prefix,
-    'libdir' : libdir,
-    'includedir' : includedir,
-    'PACKAGE_VERSION' : meson.project_version(),
-  },
-)

src/libexpr/nixexpr.cc

@@ -78,7 +78,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
         return sa < sb;
     });
     std::vector<Symbol> inherits;
-    std::map<ExprInheritFrom *, std::vector<Symbol>> inheritsFrom;
+    std::map<Displacement, std::vector<Symbol>> inheritsFrom;
     for (auto & i : sorted) {
         switch (i->second.kind) {
         case AttrDef::Kind::Plain:
@@ -89,7 +89,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
         case AttrDef::Kind::InheritedFrom: {
             auto & select = dynamic_cast<ExprSelect &>(*i->second.e);
             auto & from = dynamic_cast<ExprInheritFrom &>(*select.e);
-            inheritsFrom[&from].push_back(i->first);
+            inheritsFrom[from.displ].push_back(i->first);
             break;
         }
         }
@@ -101,7 +101,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
     }
     for (const auto & [from, syms] : inheritsFrom) {
         str << "inherit (";
-        (*inheritFromExprs)[from->displ]->show(symbols, str);
+        (*inheritFromExprs)[from]->show(symbols, str);
         str << ")";
         for (auto sym : syms) str << " " << symbols[sym];
         str << "; ";
@@ -150,7 +150,7 @@ void ExprLambda::show(const SymbolTable & symbols, std::ostream & str) const
         // the natural Symbol ordering is by creation time, which can lead to the
         // same expression being printed in two different ways depending on its
         // context. always use lexicographic ordering to avoid this.
-        for (auto & i : formals->lexicographicOrder(symbols)) {
+        for (const Formal & i : formals->lexicographicOrder(symbols)) {
             if (first) first = false; else str << ", ";
             str << symbols[i.name];
             if (i.def) {
@@ -175,7 +175,7 @@ void ExprCall::show(const SymbolTable & symbols, std::ostream & str) const
 {
     str << '(';
     fun->show(symbols, str);
-    for (auto e : args) {
+    for (auto & e : args) {
         str <<  ' ';
         e->show(symbols, str);
     }
@@ -230,7 +230,7 @@ void ExprConcatStrings::show(const SymbolTable & symbols, std::ostream & str) co
 {
     bool first = true;
     str << "(";
-    for (auto & i : *es) {
+    for (auto & i : es) {
         if (first) first = false; else str << " + ";
         i.second->show(symbols, str);
     }
@@ -374,7 +374,7 @@ std::shared_ptr<const StaticEnv> ExprAttrs::bindInheritSources(
     // not even *have* an expr that grabs anything from this env since it's fully
     // invisible, but the evaluator does not allow for this yet.
     auto inner = std::make_shared<StaticEnv>(nullptr, env.get(), 0);
-    for (auto from : *inheritFromExprs)
+    for (auto & from : *inheritFromExprs)
         from->bindVars(es, env);
 
     return inner;
@@ -461,7 +461,7 @@ void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
         es.exprEnvs.insert(std::make_pair(this, env));
 
     fun->bindVars(es, env);
-    for (auto e : args)
+    for (auto & e : args)
         e->bindVars(es, env);
 }
 
@@ -483,7 +483,7 @@ void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
         i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
 
     if (es.debugRepl)
-        es.exprEnvs.insert(std::make_pair(this, env));
+        es.exprEnvs.insert(std::make_pair(this, newEnv));
 
     body->bindVars(es, newEnv);
 }
@@ -546,7 +546,7 @@ void ExprConcatStrings::bindVars(EvalState & es, const std::shared_ptr<const Sta
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
-    for (auto & i : *this->es)
+    for (auto & i : this->es)
         i.second->bindVars(es, env);
 }
 
@@ -593,7 +593,7 @@ Pos PosTable::operator[](PosIdx p) const
 
     Pos result{0, 0, origin->origin};
     auto lines = this->lines.lock();
-    auto & linesForInput = (*lines)[origin->offset];
+    auto linesForInput = (*lines)[origin->offset];
 
     if (linesForInput.empty()) {
         auto source = result.getSource().value_or("");

src/libexpr/nixexpr.hh

@@ -28,9 +28,9 @@ struct StaticEnv;
 struct AttrName
 {
     Symbol symbol;
-    Expr * expr;
+    std::unique_ptr<Expr> expr;
     AttrName(Symbol s) : symbol(s) {};
-    AttrName(Expr * e) : expr(e) {};
+    AttrName(std::unique_ptr<Expr> e) : expr(std::move(e)) {};
 };
 
 typedef std::vector<AttrName> AttrPath;
@@ -42,12 +42,21 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath)
 
 struct Expr
 {
+protected:
+    Expr(Expr &&) = default;
+    Expr & operator=(Expr &&) = default;
+
+public:
     struct AstSymbols {
         Symbol sub, lessThan, mul, div, or_, findFile, nixPath, body;
     };
 
 
+    Expr() = default;
+    Expr(const Expr &) = delete;
+    Expr & operator=(const Expr &) = delete;
     virtual ~Expr() { };
+
     virtual void show(const SymbolTable & symbols, std::ostream & str) const;
     virtual void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env);
     virtual void eval(EvalState & state, Env & env, Value & v);
@@ -148,19 +157,19 @@ struct ExprInheritFrom : ExprVar
 struct ExprSelect : Expr
 {
     PosIdx pos;
-    Expr * e, * def;
+    std::unique_ptr<Expr> e, def;
     AttrPath attrPath;
-    ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { };
-    ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
+    ExprSelect(const PosIdx & pos, std::unique_ptr<Expr> e, AttrPath attrPath, std::unique_ptr<Expr> def) : pos(pos), e(std::move(e)), def(std::move(def)), attrPath(std::move(attrPath)) { };
+    ExprSelect(const PosIdx & pos, std::unique_ptr<Expr> e, Symbol name) : pos(pos), e(std::move(e)) { attrPath.push_back(AttrName(name)); };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
 };
 
 struct ExprOpHasAttr : Expr
 {
-    Expr * e;
+    std::unique_ptr<Expr> e;
     AttrPath attrPath;
-    ExprOpHasAttr(Expr * e, AttrPath attrPath) : e(e), attrPath(std::move(attrPath)) { };
+    ExprOpHasAttr(std::unique_ptr<Expr> e, AttrPath attrPath) : e(std::move(e)), attrPath(std::move(attrPath)) { };
     PosIdx getPos() const override { return e->getPos(); }
     COMMON_METHODS
 };
@@ -180,11 +189,11 @@ struct ExprAttrs : Expr
         };
 
         Kind kind;
-        Expr * e;
+        std::unique_ptr<Expr> e;
         PosIdx pos;
         Displacement displ; // displacement
-        AttrDef(Expr * e, const PosIdx & pos, Kind kind = Kind::Plain)
-            : kind(kind), e(e), pos(pos) { };
+        AttrDef(std::unique_ptr<Expr> e, const PosIdx & pos, Kind kind = Kind::Plain)
+            : kind(kind), e(std::move(e)), pos(pos) { };
         AttrDef() { };
 
         template<typename T>
@@ -203,12 +212,12 @@ struct ExprAttrs : Expr
     };
     typedef std::map<Symbol, AttrDef> AttrDefs;
     AttrDefs attrs;
-    std::unique_ptr<std::vector<Expr *>> inheritFromExprs;
+    std::unique_ptr<std::vector<std::unique_ptr<Expr>>> inheritFromExprs;
     struct DynamicAttrDef {
-        Expr * nameExpr, * valueExpr;
+        std::unique_ptr<Expr> nameExpr, valueExpr;
         PosIdx pos;
-        DynamicAttrDef(Expr * nameExpr, Expr * valueExpr, const PosIdx & pos)
-            : nameExpr(nameExpr), valueExpr(valueExpr), pos(pos) { };
+        DynamicAttrDef(std::unique_ptr<Expr> nameExpr, std::unique_ptr<Expr> valueExpr, const PosIdx & pos)
+            : nameExpr(std::move(nameExpr)), valueExpr(std::move(valueExpr)), pos(pos) { };
     };
     typedef std::vector<DynamicAttrDef> DynamicAttrDefs;
     DynamicAttrDefs dynamicAttrs;
@@ -225,7 +234,7 @@ struct ExprAttrs : Expr
 
 struct ExprList : Expr
 {
-    std::vector<Expr *> elems;
+    std::vector<std::unique_ptr<Expr>> elems;
     ExprList() { };
     COMMON_METHODS
     Value * maybeThunk(EvalState & state, Env & env) override;
@@ -240,7 +249,7 @@ struct Formal
 {
     PosIdx pos;
     Symbol name;
-    Expr * def;
+    std::unique_ptr<Expr> def;
 };
 
 struct Formals
@@ -256,9 +265,9 @@ struct Formals
         return it != formals.end() && it->name == arg;
     }
 
-    std::vector<Formal> lexicographicOrder(const SymbolTable & symbols) const
+    std::vector<std::reference_wrapper<const Formal>> lexicographicOrder(const SymbolTable & symbols) const
     {
-        std::vector<Formal> result(formals.begin(), formals.end());
+        std::vector<std::reference_wrapper<const Formal>> result(formals.begin(), formals.end());
         std::sort(result.begin(), result.end(),
             [&] (const Formal & a, const Formal & b) {
                 std::string_view sa = symbols[a.name], sb = symbols[b.name];
@@ -273,14 +282,14 @@ struct ExprLambda : Expr
     PosIdx pos;
     Symbol name;
     Symbol arg;
-    Formals * formals;
-    Expr * body;
-    ExprLambda(PosIdx pos, Symbol arg, Formals * formals, Expr * body)
-        : pos(pos), arg(arg), formals(formals), body(body)
+    std::unique_ptr<Formals> formals;
+    std::unique_ptr<Expr> body;
+    ExprLambda(PosIdx pos, Symbol arg, std::unique_ptr<Formals> formals, std::unique_ptr<Expr> body)
+        : pos(pos), arg(arg), formals(std::move(formals)), body(std::move(body))
     {
     };
-    ExprLambda(PosIdx pos, Formals * formals, Expr * body)
-        : pos(pos), formals(formals), body(body)
+    ExprLambda(PosIdx pos, std::unique_ptr<Formals> formals, std::unique_ptr<Expr> body)
+        : pos(pos), formals(std::move(formals)), body(std::move(body))
     {
     }
     void setName(Symbol name) override;
@@ -292,11 +301,11 @@ struct ExprLambda : Expr
 
 struct ExprCall : Expr
 {
-    Expr * fun;
-    std::vector<Expr *> args;
+    std::unique_ptr<Expr> fun;
+    std::vector<std::unique_ptr<Expr>> args;
     PosIdx pos;
-    ExprCall(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args)
-        : fun(fun), args(args), pos(pos)
+    ExprCall(const PosIdx & pos, std::unique_ptr<Expr> fun, std::vector<std::unique_ptr<Expr>> && args)
+        : fun(std::move(fun)), args(std::move(args)), pos(pos)
     { }
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
@@ -304,19 +313,19 @@ struct ExprCall : Expr
 
 struct ExprLet : Expr
 {
-    ExprAttrs * attrs;
-    Expr * body;
-    ExprLet(ExprAttrs * attrs, Expr * body) : attrs(attrs), body(body) { };
+    std::unique_ptr<ExprAttrs> attrs;
+    std::unique_ptr<Expr> body;
+    ExprLet(std::unique_ptr<ExprAttrs> attrs, std::unique_ptr<Expr> body) : attrs(std::move(attrs)), body(std::move(body)) { };
     COMMON_METHODS
 };
 
 struct ExprWith : Expr
 {
     PosIdx pos;
-    Expr * attrs, * body;
+    std::unique_ptr<Expr> attrs, body;
     size_t prevWith;
     ExprWith * parentWith;
-    ExprWith(const PosIdx & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { };
+    ExprWith(const PosIdx & pos, std::unique_ptr<Expr> attrs, std::unique_ptr<Expr> body) : pos(pos), attrs(std::move(attrs)), body(std::move(body)) { };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
 };
@@ -324,8 +333,8 @@ struct ExprWith : Expr
 struct ExprIf : Expr
 {
     PosIdx pos;
-    Expr * cond, * then, * else_;
-    ExprIf(const PosIdx & pos, Expr * cond, Expr * then, Expr * else_) : pos(pos), cond(cond), then(then), else_(else_) { };
+    std::unique_ptr<Expr> cond, then, else_;
+    ExprIf(const PosIdx & pos, std::unique_ptr<Expr> cond, std::unique_ptr<Expr> then, std::unique_ptr<Expr> else_) : pos(pos), cond(std::move(cond)), then(std::move(then)), else_(std::move(else_)) { };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
 };
@@ -333,16 +342,16 @@ struct ExprIf : Expr
 struct ExprAssert : Expr
 {
     PosIdx pos;
-    Expr * cond, * body;
-    ExprAssert(const PosIdx & pos, Expr * cond, Expr * body) : pos(pos), cond(cond), body(body) { };
+    std::unique_ptr<Expr> cond, body;
+    ExprAssert(const PosIdx & pos, std::unique_ptr<Expr> cond, std::unique_ptr<Expr> body) : pos(pos), cond(std::move(cond)), body(std::move(body)) { };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
 };
 
 struct ExprOpNot : Expr
 {
-    Expr * e;
-    ExprOpNot(Expr * e) : e(e) { };
+    std::unique_ptr<Expr> e;
+    ExprOpNot(std::unique_ptr<Expr> e) : e(std::move(e)) { };
     PosIdx getPos() const override { return e->getPos(); }
     COMMON_METHODS
 };
@@ -351,9 +360,9 @@ struct ExprOpNot : Expr
     struct name : Expr \
     { \
         PosIdx pos; \
-        Expr * e1, * e2; \
-        name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \
-        name(const PosIdx & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \
+        std::unique_ptr<Expr> e1, e2; \
+        name(std::unique_ptr<Expr> e1, std::unique_ptr<Expr> e2) : e1(std::move(e1)), e2(std::move(e2)) { }; \
+        name(const PosIdx & pos, std::unique_ptr<Expr> e1, std::unique_ptr<Expr> e2) : pos(pos), e1(std::move(e1)), e2(std::move(e2)) { }; \
         void show(const SymbolTable & symbols, std::ostream & str) const override \
         { \
             str << "("; e1->show(symbols, str); str << " " s " "; e2->show(symbols, str); str << ")"; \
@@ -378,9 +387,9 @@ struct ExprConcatStrings : Expr
 {
     PosIdx pos;
     bool forceString;
-    std::vector<std::pair<PosIdx, Expr *>> * es;
-    ExprConcatStrings(const PosIdx & pos, bool forceString, std::vector<std::pair<PosIdx, Expr *>> * es)
-        : pos(pos), forceString(forceString), es(es) { };
+    std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> es;
+    ExprConcatStrings(const PosIdx & pos, bool forceString, std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> es)
+        : pos(pos), forceString(forceString), es(std::move(es)) { };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
 };

src/libexpr/parser.y

@@ -1,452 +0,0 @@
-%glr-parser
-%define api.pure
-%locations
-%define parse.error verbose
-%defines
-/* %no-lines */
-%parse-param { void * scanner }
-%parse-param { nix::ParserState * state }
-%lex-param { void * scanner }
-%lex-param { nix::ParserState * state }
-%expect 1
-%expect-rr 1
-
-%code requires {
-
-#ifndef BISON_HEADER
-#define BISON_HEADER
-
-#include <variant>
-
-#include "finally.hh"
-#include "util.hh"
-
-#include "nixexpr.hh"
-#include "eval.hh"
-#include "eval-settings.hh"
-#include "globals.hh"
-#include "parser-state.hh"
-
-#define YYLTYPE ::nix::ParserLocation
-#define YY_DECL int yylex \
-    (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state)
-
-namespace nix {
-
-Expr * parseExprFromBuf(
-    char * text,
-    size_t length,
-    Pos::Origin origin,
-    const SourcePath & basePath,
-    SymbolTable & symbols,
-    PosTable & positions,
-    const Expr::AstSymbols & astSymbols);
-
-}
-
-#endif
-
-}
-
-%{
-
-#include "parser-tab.hh"
-#include "lexer-tab.hh"
-
-YY_DECL;
-
-using namespace nix;
-
-#define CUR_POS state->at(*yylocp)
-
-
-void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error)
-{
-    if (std::string_view(error).starts_with("syntax error, unexpected end of file")) {
-        loc->first_column = loc->last_column;
-        loc->first_line = loc->last_line;
-    }
-    throw ParseError({
-        .msg = HintFmt(error),
-        .pos = state->positions[state->at(*loc)]
-    });
-}
-
-
-%}
-
-%union {
-  // !!! We're probably leaking stuff here.
-  nix::Expr * e;
-  nix::ExprList * list;
-  nix::ExprAttrs * attrs;
-  nix::Formals * formals;
-  nix::Formal * formal;
-  nix::NixInt n;
-  nix::NixFloat nf;
-  nix::StringToken id; // !!! -> Symbol
-  nix::StringToken path;
-  nix::StringToken uri;
-  nix::StringToken str;
-  std::vector<nix::AttrName> * attrNames;
-  std::vector<std::pair<nix::AttrName, nix::PosIdx>> * inheritAttrs;
-  std::vector<std::pair<nix::PosIdx, nix::Expr *>> * string_parts;
-  std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>> * ind_string_parts;
-}
-
-%type <e> start expr expr_function expr_if expr_op
-%type <e> expr_select expr_simple expr_app
-%type <list> expr_list
-%type <attrs> binds
-%type <formals> formals
-%type <formal> formal
-%type <attrNames> attrpath
-%type <inheritAttrs> attrs
-%type <string_parts> string_parts_interpolated
-%type <ind_string_parts> ind_string_parts
-%type <e> path_start string_parts string_attr
-%type <id> attr
-%token <id> ID
-%token <str> STR IND_STR
-%token <n> INT
-%token <nf> FLOAT
-%token <path> PATH HPATH SPATH PATH_END
-%token <uri> URI
-%token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW
-%token DOLLAR_CURLY /* == ${ */
-%token IND_STRING_OPEN IND_STRING_CLOSE
-%token ELLIPSIS
-
-%right IMPL
-%left OR
-%left AND
-%nonassoc EQ NEQ
-%nonassoc '<' '>' LEQ GEQ
-%right UPDATE
-%left NOT
-%left '+' '-'
-%left '*' '/'
-%right CONCAT
-%nonassoc '?'
-%nonassoc NEGATE
-
-%%
-
-start: expr { state->result = $1; };
-
-expr: expr_function;
-
-expr_function
-  : ID ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); }
-  | '{' formals '}' ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, state->validateFormals($2), $5); }
-  | '{' formals '}' '@' ID ':' expr_function
-    {
-      auto arg = state->symbols.create($5);
-      $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7);
-    }
-  | ID '@' '{' formals '}' ':' expr_function
-    {
-      auto arg = state->symbols.create($1);
-      $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7);
-    }
-  | ASSERT expr ';' expr_function
-    { $$ = new ExprAssert(CUR_POS, $2, $4); }
-  | WITH expr ';' expr_function
-    { $$ = new ExprWith(CUR_POS, $2, $4); }
-  | LET binds IN expr_function
-    { if (!$2->dynamicAttrs.empty())
-        throw ParseError({
-            .msg = HintFmt("dynamic attributes not allowed in let"),
-            .pos = state->positions[CUR_POS]
-        });
-      $$ = new ExprLet($2, $4);
-    }
-  | expr_if
-  ;
-
-expr_if
-  : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); }
-  | expr_op
-  ;
-
-expr_op
-  : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); }
-  | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->s.sub), {new ExprInt(0), $2}); }
-  | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); }
-  | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); }
-  | expr_op '<' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$1, $3}); }
-  | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$3, $1})); }
-  | expr_op '>' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$3, $1}); }
-  | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$1, $3})); }
-  | expr_op AND expr_op { $$ = new ExprOpAnd(state->at(@2), $1, $3); }
-  | expr_op OR expr_op { $$ = new ExprOpOr(state->at(@2), $1, $3); }
-  | expr_op IMPL expr_op { $$ = new ExprOpImpl(state->at(@2), $1, $3); }
-  | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(state->at(@2), $1, $3); }
-  | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, std::move(*$3)); delete $3; }
-  | expr_op '+' expr_op
-    { $$ = new ExprConcatStrings(state->at(@2), false, new std::vector<std::pair<PosIdx, Expr *> >({{state->at(@1), $1}, {state->at(@3), $3}})); }
-  | expr_op '-' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.sub), {$1, $3}); }
-  | expr_op '*' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.mul), {$1, $3}); }
-  | expr_op '/' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.div), {$1, $3}); }
-  | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(state->at(@2), $1, $3); }
-  | expr_app
-  ;
-
-expr_app
-  : expr_app expr_select {
-      if (auto e2 = dynamic_cast<ExprCall *>($1)) {
-          e2->args.push_back($2);
-          $$ = $1;
-      } else
-          $$ = new ExprCall(CUR_POS, $1, {$2});
-  }
-  | expr_select
-  ;
-
-expr_select
-  : expr_simple '.' attrpath
-    { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), nullptr); delete $3; }
-  | expr_simple '.' attrpath OR_KW expr_select
-    { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), $5); delete $3; }
-  | /* Backwards compatibility: because Nixpkgs has a rarely used
-       function named ‘or’, allow stuff like ‘map or [...]’. */
-    expr_simple OR_KW
-    { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, state->s.or_)}); }
-  | expr_simple
-  ;
-
-expr_simple
-  : ID {
-      std::string_view s = "__curPos";
-      if ($1.l == s.size() && strncmp($1.p, s.data(), s.size()) == 0)
-          $$ = new ExprPos(CUR_POS);
-      else
-          $$ = new ExprVar(CUR_POS, state->symbols.create($1));
-  }
-  | INT { $$ = new ExprInt($1); }
-  | FLOAT { $$ = new ExprFloat($1); }
-  | '"' string_parts '"' { $$ = $2; }
-  | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
-      $$ = state->stripIndentation(CUR_POS, std::move(*$2));
-      delete $2;
-  }
-  | path_start PATH_END
-  | path_start string_parts_interpolated PATH_END {
-      $2->insert($2->begin(), {state->at(@1), $1});
-      $$ = new ExprConcatStrings(CUR_POS, false, $2);
-  }
-  | SPATH {
-      std::string path($1.p + 1, $1.l - 2);
-      $$ = new ExprCall(CUR_POS,
-          new ExprVar(state->s.findFile),
-          {new ExprVar(state->s.nixPath),
-           new ExprString(std::move(path))});
-  }
-  | URI {
-      static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals);
-      if (noURLLiterals)
-          throw ParseError({
-              .msg = HintFmt("URL literals are disabled"),
-              .pos = state->positions[CUR_POS]
-          });
-      $$ = new ExprString(std::string($1));
-  }
-  | '(' expr ')' { $$ = $2; }
-  /* Let expressions `let {..., body = ...}' are just desugared
-     into `(rec {..., body = ...}).body'. */
-  | LET '{' binds '}'
-    { $3->recursive = true; $$ = new ExprSelect(noPos, $3, state->s.body); }
-  | REC '{' binds '}'
-    { $3->recursive = true; $$ = $3; }
-  | '{' binds '}'
-    { $$ = $2; }
-  | '[' expr_list ']' { $$ = $2; }
-  ;
-
-string_parts
-  : STR { $$ = new ExprString(std::string($1)); }
-  | string_parts_interpolated { $$ = new ExprConcatStrings(CUR_POS, true, $1); }
-  | { $$ = new ExprString(""); }
-  ;
-
-string_parts_interpolated
-  : string_parts_interpolated STR
-  { $$ = $1; $1->emplace_back(state->at(@2), new ExprString(std::string($2))); }
-  | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); }
-  | DOLLAR_CURLY expr '}' { $$ = new std::vector<std::pair<PosIdx, Expr *>>; $$->emplace_back(state->at(@1), $2); }
-  | STR DOLLAR_CURLY expr '}' {
-      $$ = new std::vector<std::pair<PosIdx, Expr *>>;
-      $$->emplace_back(state->at(@1), new ExprString(std::string($1)));
-      $$->emplace_back(state->at(@2), $3);
-    }
-  ;
-
-path_start
-  : PATH {
-    Path path(absPath({$1.p, $1.l}, state->basePath.path.abs()));
-    /* add back in the trailing '/' to the first segment */
-    if ($1.p[$1.l-1] == '/' && $1.l > 1)
-      path += "/";
-    $$ = new ExprPath(path);
-  }
-  | HPATH {
-    if (evalSettings.pureEval) {
-        throw Error(
-            "the path '%s' can not be resolved in pure mode",
-            std::string_view($1.p, $1.l)
-        );
-    }
-    Path path(getHome() + std::string($1.p + 1, $1.l - 1));
-    $$ = new ExprPath(path);
-  }
-  ;
-
-ind_string_parts
-  : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(state->at(@2), $2); }
-  | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); }
-  | { $$ = new std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>>; }
-  ;
-
-binds
-  : binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->at(@2)); delete $2; }
-  | binds INHERIT attrs ';'
-    { $$ = $1;
-      for (auto & [i, iPos] : *$3) {
-          if ($$->attrs.find(i.symbol) != $$->attrs.end())
-              state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos);
-          $$->attrs.emplace(
-              i.symbol,
-              ExprAttrs::AttrDef(new ExprVar(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited));
-      }
-      delete $3;
-    }
-  | binds INHERIT '(' expr ')' attrs ';'
-    { $$ = $1;
-      if (!$$->inheritFromExprs)
-          $$->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
-      $$->inheritFromExprs->push_back($4);
-      auto from = new nix::ExprInheritFrom(state->at(@4), $$->inheritFromExprs->size() - 1);
-      for (auto & [i, iPos] : *$6) {
-          if ($$->attrs.find(i.symbol) != $$->attrs.end())
-              state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos);
-          $$->attrs.emplace(
-              i.symbol,
-              ExprAttrs::AttrDef(
-                  new ExprSelect(iPos, from, i.symbol),
-                  iPos,
-                  ExprAttrs::AttrDef::Kind::InheritedFrom));
-      }
-      delete $6;
-    }
-  | { $$ = new ExprAttrs(state->at(@0)); }
-  ;
-
-attrs
-  : attrs attr { $$ = $1; $1->emplace_back(AttrName(state->symbols.create($2)), state->at(@2)); }
-  | attrs string_attr
-    { $$ = $1;
-      ExprString * str = dynamic_cast<ExprString *>($2);
-      if (str) {
-          $$->emplace_back(AttrName(state->symbols.create(str->s)), state->at(@2));
-          delete str;
-      } else
-          throw ParseError({
-              .msg = HintFmt("dynamic attributes not allowed in inherit"),
-              .pos = state->positions[state->at(@2)]
-          });
-    }
-  | { $$ = new std::vector<std::pair<AttrName, PosIdx>>; }
-  ;
-
-attrpath
-  : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($3))); }
-  | attrpath '.' string_attr
-    { $$ = $1;
-      ExprString * str = dynamic_cast<ExprString *>($3);
-      if (str) {
-          $$->push_back(AttrName(state->symbols.create(str->s)));
-          delete str;
-      } else
-          $$->push_back(AttrName($3));
-    }
-  | attr { $$ = new std::vector<AttrName>; $$->push_back(AttrName(state->symbols.create($1))); }
-  | string_attr
-    { $$ = new std::vector<AttrName>;
-      ExprString *str = dynamic_cast<ExprString *>($1);
-      if (str) {
-          $$->push_back(AttrName(state->symbols.create(str->s)));
-          delete str;
-      } else
-          $$->push_back(AttrName($1));
-    }
-  ;
-
-attr
-  : ID
-  | OR_KW { $$ = {"or", 2}; }
-  ;
-
-string_attr
-  : '"' string_parts '"' { $$ = $2; }
-  | DOLLAR_CURLY expr '}' { $$ = $2; }
-  ;
-
-expr_list
-  : expr_list expr_select { $$ = $1; $1->elems.push_back($2); /* !!! dangerous */ }
-  | { $$ = new ExprList; }
-  ;
-
-formals
-  : formal ',' formals
-    { $$ = $3; $$->formals.emplace_back(*$1); delete $1; }
-  | formal
-    { $$ = new Formals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; }
-  |
-    { $$ = new Formals; $$->ellipsis = false; }
-  | ELLIPSIS
-    { $$ = new Formals; $$->ellipsis = true; }
-  ;
-
-formal
-  : ID { $$ = new Formal{CUR_POS, state->symbols.create($1), 0}; }
-  | ID '?' expr { $$ = new Formal{CUR_POS, state->symbols.create($1), $3}; }
-  ;
-
-%%
-
-#include "eval.hh"
-
-
-namespace nix {
-
-Expr * parseExprFromBuf(
-    char * text,
-    size_t length,
-    Pos::Origin origin,
-    const SourcePath & basePath,
-    SymbolTable & symbols,
-    PosTable & positions,
-    const Expr::AstSymbols & astSymbols)
-{
-    yyscan_t scanner;
-    ParserState state {
-        .symbols = symbols,
-        .positions = positions,
-        .basePath = basePath,
-        .origin = positions.addOrigin(origin, length),
-        .s = astSymbols,
-    };
-
-    yylex_init(&scanner);
-    Finally _destroy([&] { yylex_destroy(scanner); });
-
-    yy_scan_buffer(text, length, scanner);
-    yyparse(scanner, &state);
-
-    return state.result;
-}
-
-
-}

src/libexpr/parser/change_head.hh

@@ -0,0 +1,65 @@
+#pragma once
+
+#include <tao/pegtl.hpp>
+
+namespace nix::parser {
+
+// modified copy of change_state, as the manual suggest for more involved
+// state manipulation. we want to change only the first state parameter,
+// and we care about the *initial* position of a rule application (not the
+// past-the-end position as pegtl change_state provides)
+template<typename NewState>
+struct change_head : tao::pegtl::maybe_nothing
+{
+    template<
+        typename Rule,
+        tao::pegtl::apply_mode A,
+        tao::pegtl::rewind_mode M,
+        template<typename...> class Action,
+        template<typename...> class Control,
+        typename ParseInput,
+        typename State,
+        typename... States
+    >
+    [[nodiscard]] static bool match(ParseInput & in, State && st, States &&... sts)
+    {
+        const auto begin = in.iterator();
+
+        if constexpr (std::is_constructible_v<NewState, State, States...>) {
+            NewState s(st, sts...);
+            if (tao::pegtl::match<Rule, A, M, Action, Control>(in, s, sts...)) {
+                if constexpr (A == tao::pegtl::apply_mode::action) {
+                    _success<Action<Rule>>(0, begin, in, s, st, sts...);
+                }
+                return true;
+            }
+            return false;
+        } else if constexpr (std::is_default_constructible_v<NewState>) {
+            NewState s;
+            if (tao::pegtl::match<Rule, A, M, Action, Control>(in, s, sts...)) {
+                if constexpr (A == tao::pegtl::apply_mode::action) {
+                    _success<Action<Rule>>(0, begin, in, s, st, sts...);
+                }
+                return true;
+            }
+            return false;
+        } else {
+            static_assert(decltype(sizeof(NewState))(), "unable to instantiate new state");
+        }
+    }
+
+    template<typename Target, typename ParseInput, typename... S>
+    static void _success(void *, auto & begin, ParseInput & in, S & ... sts)
+    {
+        const typename ParseInput::action_t at(begin, in);
+        Target::success(at, sts...);
+    }
+
+    template<typename Target, typename... S>
+    static void _success(decltype(Target::success0(std::declval<S &>()...), 0), auto &, auto &, S & ... sts)
+    {
+        Target::success0(sts...);
+    }
+};
+
+}

src/libexpr/parser/grammar.hh

@@ -0,0 +1,706 @@
+#pragma once
+
+#include "tao/pegtl.hpp"
+#include <type_traits>
+#include <variant>
+
+#include <boost/container/small_vector.hpp>
+
+// NOTE
+// nix line endings are \n, \r\n, \r. the grammar does not use eol or
+// eolf rules in favor of reproducing the old flex lexer as faithfully as
+// possible, and deferring calculation of positions to downstream users.
+
+namespace nix::parser::grammar {
+
+using namespace tao::pegtl;
+namespace p = tao::pegtl;
+
+// character classes
+namespace c {
+
+struct path : sor<
+    ranges<'a', 'z', 'A', 'Z', '0', '9'>,
+    one<'.', '_', '-', '+'>
+> {};
+struct path_sep : one<'/'> {};
+
+struct id_first : ranges<'a', 'z', 'A', 'Z', '_'> {};
+struct id_rest : sor<
+    ranges<'a', 'z', 'A', 'Z', '0', '9'>,
+    one<'_', '\'', '-'>
+> {};
+
+struct uri_scheme_first : ranges<'a', 'z', 'A', 'Z'> {};
+struct uri_scheme_rest : sor<
+    ranges<'a', 'z', 'A', 'Z', '0', '9'>,
+    one<'+', '-', '.'>
+> {};
+struct uri_sep : one<':'> {};
+struct uri_rest : sor<
+    ranges<'a', 'z', 'A', 'Z', '0', '9'>,
+    one<'%', '/', '?', ':', '@', '&', '=', '+', '$', ',', '-', '_', '.', '!', '~', '*', '\''>
+> {};
+
+}
+
+// "tokens". PEGs don't really care about tokens, we merely use them as a convenient
+// way of writing down keywords and a couple complicated syntax rules.
+namespace t {
+
+struct _extend_as_path : seq<
+    star<c::path>,
+    not_at<TAO_PEGTL_STRING("/*")>,
+    not_at<TAO_PEGTL_STRING("//")>,
+    c::path_sep,
+    sor<c::path, TAO_PEGTL_STRING("${")>
+> {};
+struct _extend_as_uri : seq<
+    star<c::uri_scheme_rest>,
+    c::uri_sep,
+    c::uri_rest
+> {};
+
+// keywords might be extended to identifiers, paths, or uris.
+// NOTE this assumes that keywords are a-zA-Z only, otherwise uri schemes would never
+// match correctly.
+// NOTE not a simple seq<...> because this would report incorrect positions for
+// keywords used inside must<> if a prefix of the keyword matches.
+template<typename S>
+struct _keyword : sor<
+    seq<
+        S,
+        not_at<c::id_rest>,
+        not_at<_extend_as_path>,
+        not_at<_extend_as_uri>
+    >,
+    failure
+> {};
+
+struct kw_if      : _keyword<TAO_PEGTL_STRING("if")> {};
+struct kw_then    : _keyword<TAO_PEGTL_STRING("then")> {};
+struct kw_else    : _keyword<TAO_PEGTL_STRING("else")> {};
+struct kw_assert  : _keyword<TAO_PEGTL_STRING("assert")> {};
+struct kw_with    : _keyword<TAO_PEGTL_STRING("with")> {};
+struct kw_let     : _keyword<TAO_PEGTL_STRING("let")> {};
+struct kw_in      : _keyword<TAO_PEGTL_STRING("in")> {};
+struct kw_rec     : _keyword<TAO_PEGTL_STRING("rec")> {};
+struct kw_inherit : _keyword<TAO_PEGTL_STRING("inherit")> {};
+struct kw_or      : _keyword<TAO_PEGTL_STRING("or")> {};
+
+// `-` can be a unary prefix op, a binary infix op, or the first character
+// of a path or -> (ex 1->1--1)
+// `/` can be a path leader or an operator (ex a?a /a)
+struct op_minus : seq<one<'-'>, not_at<one<'>'>>, not_at<_extend_as_path>> {};
+struct op_div   : seq<one<'/'>, not_at<c::path>> {};
+
+// match a rule, making sure we are not matching it where a keyword would match.
+// using minus like this is a lot faster than flipping the order and using seq.
+template<typename... Rules>
+struct _not_at_any_keyword : minus<
+    seq<Rules...>,
+    sor<
+        TAO_PEGTL_STRING("inherit"),
+        TAO_PEGTL_STRING("assert"),
+        TAO_PEGTL_STRING("else"),
+        TAO_PEGTL_STRING("then"),
+        TAO_PEGTL_STRING("with"),
+        TAO_PEGTL_STRING("let"),
+        TAO_PEGTL_STRING("rec"),
+        TAO_PEGTL_STRING("if"),
+        TAO_PEGTL_STRING("in"),
+        TAO_PEGTL_STRING("or")
+    >
+> {};
+
+// identifiers are kind of horrid:
+//
+//   - uri_scheme_first ⊂ id_first
+//   - uri_scheme_first ⊂ uri_scheme_rest ⊂ path
+//   - id_first ⊂ id_rest ∖ { ' } ⊂ path
+//   - id_first ∩ (path ∖ uri_scheme_first) = { _ }
+//   - uri_sep ∉ ⋃ { id_first, id_rest, uri_scheme_first, uri_scheme_rest, path }
+//   - path_sep ∉ ⋃ { id_first, id_rest, uri_scheme_first, uri_scheme_rest }
+//
+// and we want, without reading the input more than once, a string that
+// matches (id_first id_rest*) and is not followed by any number of
+// characters such that the extended string matches path or uri rules.
+//
+// since the first character must be either _ or a uri scheme character
+// we can ignore path-like bits at the beginning. uri_sep cannot appear anywhere
+// in an identifier, so it's only needed in lookahead checks at the uri-like
+// prefix. likewise path_sep cannot appear anywhere in the idenfier, so it's
+// only needed in lookahead checks in the path-like prefix.
+//
+// in total that gives us a decomposition of
+//
+//     (uri-scheme-like? (?! continues-as-uri) | _)
+//     (path-segment-like? (?! continues-as-path))
+//     id_rest*
+struct identifier : _not_at_any_keyword<
+    // we don't use (at<id_rest>, ...) matches here because identifiers are
+    // a really hot path and rewinding as needed by at<> isn't entirely free.
+    sor<
+        seq<
+            c::uri_scheme_first,
+            star<ranges<'a', 'z', 'A', 'Z', '0', '9', '-'>>,
+            not_at<_extend_as_uri>
+        >,
+        one<'_'>
+    >,
+    star<sor<ranges<'a', 'z', 'A', 'Z', '0', '9'>, one<'_', '-'>>>,
+    not_at<_extend_as_path>,
+    star<c::id_rest>
+> {};
+
+// floats may extend ints, thus these rules are very similar.
+struct integer : seq<
+    sor<
+        seq<range<'1', '9'>, star<digit>, not_at<one<'.'>>>,
+        seq<one<'0'>, not_at<one<'.'>, digit>, star<digit>>
+    >,
+    not_at<_extend_as_path>
+> {};
+
+struct floating : seq<
+    sor<
+        seq<range<'1', '9'>, star<digit>, one<'.'>, star<digit>>,
+        seq<opt<one<'0'>>, one<'.'>, plus<digit>>
+    >,
+    opt<one<'E', 'e'>, opt<one<'+', '-'>>, plus<digit>>,
+    not_at<_extend_as_path>
+> {};
+
+struct uri : seq<
+    c::uri_scheme_first,
+    star<c::uri_scheme_rest>,
+    c::uri_sep,
+    plus<c::uri_rest>
+> {};
+
+struct sep : sor<
+    plus<one<' ', '\t', '\r', '\n'>>,
+    seq<one<'#'>, star<not_one<'\r', '\n'>>>,
+    seq<string<'/', '*'>, until<string<'*', '/'>>>
+> {};
+
+}
+
+
+
+using seps = star<t::sep>;
+
+
+// marker for semantic rules. not handling one of these in an action that cares about
+// semantics is probably an error.
+struct semantic {};
+
+
+struct expr;
+
+struct _string {
+    template<typename... Inner>
+    struct literal : semantic, seq<Inner...> {};
+    struct cr_lf : semantic, seq<one<'\r'>, opt<one<'\n'>>> {};
+    struct interpolation : semantic, seq<
+        p::string<'$', '{'>, seps,
+        must<expr>, seps,
+        must<one<'}'>>
+    > {};
+    struct escape : semantic, must<any> {};
+};
+struct string : _string, seq<
+    one<'"'>,
+    star<
+        sor<
+            _string::literal<plus<not_one<'$', '"', '\\', '\r'>>>,
+            _string::cr_lf,
+            _string::interpolation,
+            _string::literal<one<'$'>, opt<one<'$'>>>,
+            seq<one<'\\'>, _string::escape>
+        >
+    >,
+    must<one<'"'>>
+> {};
+
+struct _ind_string {
+    template<bool Indented, typename... Inner>
+    struct literal : semantic, seq<Inner...> {};
+    struct interpolation : semantic, seq<
+        p::string<'$', '{'>, seps,
+        must<expr>, seps,
+        must<one<'}'>>
+    > {};
+    struct escape : semantic, must<any> {};
+};
+struct ind_string : _ind_string, seq<
+    TAO_PEGTL_STRING("''"),
+    opt<star<one<' '>>, one<'\n'>>,
+    star<
+        sor<
+            _ind_string::literal<
+                true,
+                plus<
+                    sor<
+                        not_one<'$', '\''>,
+                        seq<one<'$'>, not_one<'{', '\''>>,
+                        seq<one<'\''>, not_one<'\'', '$'>>
+                    >
+                >
+            >,
+            _ind_string::interpolation,
+            _ind_string::literal<false, one<'$'>>,
+            _ind_string::literal<false, one<'\''>, not_at<one<'\''>>>,
+            seq<one<'\''>, _ind_string::literal<false, p::string<'\'', '\''>>>,
+            seq<
+                p::string<'\'', '\''>,
+                sor<
+                    _ind_string::literal<false, one<'$'>>,
+                    seq<one<'\\'>, _ind_string::escape>
+                >
+            >
+        >
+    >,
+    must<TAO_PEGTL_STRING("''")>
+> {};
+
+struct _path {
+    // legacy lexer rules. extra l_ to avoid reserved c++ identifiers.
+    struct _l_PATH : seq<star<c::path>, plus<c::path_sep, plus<c::path>>, opt<c::path_sep>> {};
+    struct _l_PATH_SEG : seq<star<c::path>, c::path_sep> {};
+    struct _l_HPATH : seq<one<'~'>, plus<c::path_sep, plus<c::path>>, opt<c::path_sep>> {};
+    struct _l_HPATH_START : TAO_PEGTL_STRING("~/") {};
+    struct _path_str : sor<_l_PATH, _l_PATH_SEG, plus<c::path>> {};
+    // modern rules
+    template<typename... Inner>
+    struct literal : semantic, seq<Inner...> {};
+    struct interpolation : semantic, seq<
+        p::string<'$', '{'>, seps,
+        must<expr>, seps,
+        must<one<'}'>>
+    > {};
+    struct anchor : semantic, sor<
+        _l_PATH,
+        seq<_l_PATH_SEG, at<TAO_PEGTL_STRING("${")>>
+    > {};
+    struct home_anchor : semantic, sor<
+        _l_HPATH,
+        seq<_l_HPATH_START, at<TAO_PEGTL_STRING("${")>>
+    > {};
+    struct searched_path : semantic, list<plus<c::path>, c::path_sep> {};
+    struct forbid_prefix_triple_slash : sor<not_at<c::path_sep>, failure> {};
+    struct forbid_prefix_double_slash_no_interp : sor<
+        not_at<c::path_sep, star<c::path>, not_at<TAO_PEGTL_STRING("${")>>,
+        failure
+    > {};
+    // legacy parser rules
+    struct _str_rest : seq<
+        must<forbid_prefix_double_slash_no_interp>,
+        opt<literal<_path_str>>,
+        must<forbid_prefix_triple_slash>,
+        star<
+            sor<
+                literal<_path_str>,
+                interpolation
+            >
+        >
+    > {};
+};
+struct path : _path, sor<
+    seq<
+        sor<_path::anchor, _path::home_anchor>,
+        _path::_str_rest
+    >,
+    seq<one<'<'>, _path::searched_path, one<'>'>>
+> {};
+
+struct _formal {
+    struct name : semantic, t::identifier {};
+    struct default_value : semantic, must<expr> {};
+};
+struct formal : semantic, _formal, seq<
+    _formal::name,
+    opt<seps, one<'?'>, seps, _formal::default_value>
+> {};
+
+struct _formals {
+    struct ellipsis : semantic, p::ellipsis {};
+};
+struct formals : semantic, _formals, seq<
+    one<'{'>, seps,
+    // formals and attrsets share a two-token head sequence ('{' <id>).
+    // this rule unrolls the formals list a bit to provide better error messages than
+    // "expected '='" at the first ',' if formals are incorrect.
+    sor<
+        one<'}'>,
+        seq<_formals::ellipsis, seps, must<one<'}'>>>,
+        seq<
+            formal, seps,
+            if_then_else<
+                at<one<','>>,
+                seq<
+                    star<one<','>, seps, formal, seps>,
+                    opt<one<','>, seps, opt<_formals::ellipsis, seps>>,
+                    must<one<'}'>>
+                >,
+                one<'}'>
+            >
+        >
+    >
+> {};
+
+struct _attr {
+    struct simple : semantic, sor<t::identifier, t::kw_or> {};
+    struct string : semantic, seq<grammar::string> {};
+    struct expr : semantic, seq<
+        TAO_PEGTL_STRING("${"), seps,
+        must<grammar::expr>, seps,
+        must<one<'}'>>
+    > {};
+};
+struct attr : _attr, sor<
+    _attr::simple,
+    _attr::string,
+    _attr::expr
+> {};
+
+struct attrpath : list<attr, one<'.'>, t::sep> {};
+
+struct _inherit {
+    struct from : semantic, must<expr> {};
+    struct attrs : list<attr, seps> {};
+};
+struct inherit : _inherit, seq<
+    t::kw_inherit, seps,
+    opt<one<'('>, seps, _inherit::from, seps, must<one<')'>>, seps>,
+    opt<_inherit::attrs, seps>,
+    must<one<';'>>
+> {};
+
+struct _binding {
+    struct path : semantic, attrpath {};
+    struct equal : one<'='> {};
+    struct value : semantic, must<expr> {};
+};
+struct binding : _binding, seq<
+    _binding::path, seps,
+    must<_binding::equal>, seps,
+    _binding::value, seps,
+    must<one<';'>>
+> {};
+
+struct bindings : opt<list<sor<inherit, binding>, seps>> {};
+
+struct op {
+    enum class kind {
+        // NOTE non-associativity is *NOT* handled in the grammar structure.
+        // handling it in the grammar itself instead of in semantic actions
+        // slows down the parser significantly and makes the rules *much*
+        // harder to read. maybe this will be different at some point when
+        // ! does not sit between two binary precedence levels.
+        nonAssoc,
+        leftAssoc,
+        rightAssoc,
+        unary,
+    };
+    template<typename Rule, unsigned Precedence, kind Kind = kind::leftAssoc>
+    struct _op : Rule {
+        static constexpr unsigned precedence = Precedence;
+        static constexpr op::kind kind = Kind;
+    };
+
+    struct unary_minus : _op<t::op_minus,           3, kind::unary> {};
+
+    // treating this like a unary postfix operator is sketchy, but that's
+    // the most reasonable way to implement the operator precedence set forth
+    // by the language way back. it'd be much better if `.` and `?` had the same
+    // precedence, but alas.
+    struct has_attr   : _op<seq<one<'?'>, seps, must<attrpath>>, 4> {};
+
+    struct concat     : _op<TAO_PEGTL_STRING("++"),  5, kind::rightAssoc> {};
+    struct mul        : _op<one<'*'>,                6> {};
+    struct div        : _op<t::op_div,               6> {};
+    struct plus       : _op<one<'+'>,                7> {};
+    struct minus      : _op<t::op_minus,             7> {};
+    struct not_       : _op<one<'!'>,                8, kind::unary> {};
+    struct update     : _op<TAO_PEGTL_STRING("//"),  9, kind::rightAssoc> {};
+    struct less_eq    : _op<TAO_PEGTL_STRING("<="), 10, kind::nonAssoc> {};
+    struct greater_eq : _op<TAO_PEGTL_STRING(">="), 10, kind::nonAssoc> {};
+    struct less       : _op<one<'<'>,               10, kind::nonAssoc> {};
+    struct greater    : _op<one<'>'>,               10, kind::nonAssoc> {};
+    struct equals     : _op<TAO_PEGTL_STRING("=="), 11, kind::nonAssoc> {};
+    struct not_equals : _op<TAO_PEGTL_STRING("!="), 11, kind::nonAssoc> {};
+    struct and_       : _op<TAO_PEGTL_STRING("&&"), 12> {};
+    struct or_        : _op<TAO_PEGTL_STRING("||"), 13> {};
+    struct implies    : _op<TAO_PEGTL_STRING("->"), 14, kind::rightAssoc> {};
+};
+
+struct _expr {
+    template<template<typename...> class OpenMod = seq, typename... Init>
+    struct _attrset : seq<
+        Init...,
+        OpenMod<one<'{'>>, seps,
+        bindings, seps,
+        must<one<'}'>>
+    > {};
+
+    struct select;
+
+    struct id : semantic, t::identifier {};
+    struct int_ : semantic, t::integer {};
+    struct float_ : semantic, t::floating {};
+    struct string : semantic, seq<grammar::string> {};
+    struct ind_string : semantic, seq<grammar::ind_string> {};
+    struct path : semantic, seq<grammar::path> {};
+    struct uri : semantic, t::uri {};
+    struct ancient_let : semantic, _attrset<must, t::kw_let, seps> {};
+    struct rec_set : semantic, _attrset<must, t::kw_rec, seps> {};
+    struct set : semantic, _attrset<> {};
+
+    struct _list {
+        struct entry : semantic, seq<select> {};
+    };
+    struct list : semantic, _list, seq<
+        one<'['>, seps,
+        opt<p::list<_list::entry, seps>, seps>,
+        must<one<']'>>
+    > {};
+
+    struct _simple : sor<
+        id,
+        int_,
+        float_,
+        string,
+        ind_string,
+        path,
+        uri,
+        seq<one<'('>, seps, must<expr>, seps, must<one<')'>>>,
+        ancient_let,
+        rec_set,
+        set,
+        list
+    > {};
+
+    struct _select {
+        struct head : _simple {};
+        struct attr : semantic, seq<attrpath> {};
+        struct attr_or : semantic, must<select> {};
+        struct as_app_or : semantic, t::kw_or {};
+    };
+    struct _app {
+        struct first_arg : semantic, seq<select> {};
+        struct another_arg : semantic, seq<select> {};
+        // can be used to stash a position of the application head node
+        struct select_or_fn : seq<select> {};
+    };
+
+    struct select : _select, seq<
+        _select::head, seps,
+        opt<
+            sor<
+                seq<
+                    one<'.'>, seps, _select::attr,
+                    opt<seps, t::kw_or, seps, _select::attr_or>
+                >,
+                _select::as_app_or
+            >
+        >
+    > {};
+
+    struct app : _app, seq<
+        _app::select_or_fn,
+        opt<seps, _app::first_arg, star<seps, _app::another_arg>>
+    > {};
+
+    template<typename Op>
+    struct operator_ : semantic, Op {};
+
+    struct unary : seq<
+        star<sor<operator_<op::not_>, operator_<op::unary_minus>>, seps>,
+        app
+    > {};
+
+    struct _binary_operator : sor<
+        operator_<op::implies>,
+        operator_<op::update>,
+        operator_<op::concat>,
+        operator_<op::plus>,
+        operator_<op::minus>,
+        operator_<op::mul>,
+        operator_<op::div>,
+        operator_<op::less_eq>,
+        operator_<op::greater_eq>,
+        operator_<op::less>,
+        operator_<op::greater>,
+        operator_<op::equals>,
+        operator_<op::not_equals>,
+        operator_<op::or_>,
+        operator_<op::and_>
+    > {};
+
+    struct _binop : seq<
+        unary,
+        star<
+            seps,
+            sor<
+                seq<_binary_operator, seps, must<unary>>,
+                operator_<op::has_attr>
+            >
+        >
+    > {};
+
+    struct _lambda {
+        struct arg : semantic, t::identifier {};
+    };
+    struct lambda : semantic, _lambda, sor<
+        seq<
+            _lambda::arg, seps,
+            sor<
+                seq<one<':'>, seps, must<expr>>,
+                seq<one<'@'>, seps, must<formals, seps, one<':'>, seps, expr>>
+            >
+        >,
+        seq<
+            formals, seps,
+            sor<
+                seq<one<':'>, seps, must<expr>>,
+                seq<one<'@'>, seps, must<_lambda::arg, seps, one<':'>, seps, expr>>
+            >
+        >
+    > {};
+
+    struct assert_ : semantic, seq<
+        t::kw_assert, seps,
+        must<expr>, seps,
+        must<one<';'>>, seps,
+        must<expr>
+    > {};
+    struct with : semantic, seq<
+        t::kw_with, seps,
+        must<expr>, seps,
+        must<one<';'>>, seps,
+        must<expr>
+    > {};
+    struct let : seq<
+        t::kw_let, seps,
+        not_at<one<'{'>>, // exclude ancient_let so we can must<kw_in>
+        bindings, seps,
+        must<t::kw_in>, seps,
+        must<expr>
+    > {};
+    struct if_ : semantic, seq<
+        t::kw_if, seps,
+        must<expr>, seps,
+        must<t::kw_then>, seps,
+        must<expr>, seps,
+        must<t::kw_else>, seps,
+        must<expr>
+    > {};
+};
+struct expr : semantic, _expr, sor<
+    _expr::lambda,
+    _expr::assert_,
+    _expr::with,
+    _expr::let,
+    _expr::if_,
+    _expr::_binop
+> {};
+
+// legacy support: \0 terminates input if passed from flex to bison as a token
+struct eof : sor<p::eof, one<0>> {};
+
+struct root : must<seps, expr, seps, eof> {};
+
+
+
+template<typename Rule>
+struct nothing : p::nothing<Rule> {
+    static_assert(!std::is_base_of_v<semantic, Rule>);
+};
+
+
+
+template<typename Self, typename OpCtx, typename AttrPathT, typename ExprT>
+struct operator_semantics {
+    struct has_attr : grammar::op::has_attr {
+        AttrPathT path;
+    };
+
+    struct OpEntry {
+        OpCtx ctx;
+        uint8_t prec;
+        grammar::op::kind assoc;
+        std::variant<
+            grammar::op::not_,
+            grammar::op::unary_minus,
+            grammar::op::implies,
+            grammar::op::or_,
+            grammar::op::and_,
+            grammar::op::equals,
+            grammar::op::not_equals,
+            grammar::op::less_eq,
+            grammar::op::greater_eq,
+            grammar::op::update,
+            grammar::op::concat,
+            grammar::op::less,
+            grammar::op::greater,
+            grammar::op::plus,
+            grammar::op::minus,
+            grammar::op::mul,
+            grammar::op::div,
+            has_attr
+        > op;
+    };
+
+    // statistics here are taken from nixpkgs commit de502c4d0ba96261e5de803e4d1d1925afd3e22f.
+    // over 99.9% of contexts in nixpkgs need at most 4 slots, ~85% need only 1
+    boost::container::small_vector<ExprT, 4> exprs;
+    // over 99.9% of contexts in nixpkgs need at most 2 slots, ~85% need only 1
+    boost::container::small_vector<OpEntry, 2> ops;
+
+    // derived class is expected to define members:
+    //
+    // ExprT applyOp(OpCtx & pos, auto & op, auto &... args);
+    // [[noreturn]] static void badOperator(OpCtx & pos, auto &... args);
+
+    void reduce(uint8_t toPrecedence, auto &... args) {
+        while (!ops.empty()) {
+            auto & [ctx, precedence, kind, op] = ops.back();
+            // NOTE this relies on associativity not being mixed within a precedence level.
+            if ((precedence > toPrecedence)
+                || (kind != grammar::op::kind::leftAssoc && precedence == toPrecedence))
+                break;
+            std::visit([&, ctx=std::move(ctx)] (auto & op) {
+                exprs.push_back(static_cast<Self &>(*this).applyOp(ctx, op, args...));
+            }, op);
+            ops.pop_back();
+        }
+    }
+
+    ExprT popExpr()
+    {
+        auto r = std::move(exprs.back());
+        exprs.pop_back();
+        return r;
+    }
+
+    void pushOp(OpCtx ctx, auto o, auto &... args)
+    {
+        if (o.kind != grammar::op::kind::unary)
+            reduce(o.precedence, args...);
+        if (!ops.empty() && o.kind == grammar::op::kind::nonAssoc) {
+            auto & [_pos, _prec, _kind, _o] = ops.back();
+            if (_kind == o.kind && _prec == o.precedence)
+                Self::badOperator(ctx, args...);
+        }
+        ops.emplace_back(ctx, o.precedence, o.kind, std::move(o));
+    }
+
+    ExprT finish(auto &... args)
+    {
+        reduce(255, args...);
+        return popExpr();
+    }
+};
+
+}

src/libexpr/parser/parser.cc

@@ -0,0 +1,852 @@
+#include "attr-set.hh"
+#include "error.hh"
+#include "eval.hh"
+#include "eval-settings.hh"
+#include "finally.hh"
+#include "nixexpr.hh"
+#include "symbol-table.hh"
+
+#include "change_head.hh"
+#include "grammar.hh"
+#include "state.hh"
+
+#include <charconv>
+#include <clocale>
+#include <memory>
+
+// flip this define when doing parser development to enable some g checks.
+#if 0
+#include <tao/pegtl/contrib/analyze.hpp>
+#define ANALYZE_GRAMMAR \
+    ([] { \
+        const std::size_t issues = tao::pegtl::analyze<grammar::root>(); \
+        assert(issues == 0); \
+    })()
+#else
+#define ANALYZE_GRAMMAR ((void) 0)
+#endif
+
+namespace p = tao::pegtl;
+
+namespace nix::parser {
+namespace {
+
+template<typename>
+inline constexpr const char * error_message = nullptr;
+
+#define error_message_for(...) \
+    template<> inline constexpr auto error_message<__VA_ARGS__>
+
+error_message_for(p::one<'{'>) = "expecting '{'";
+error_message_for(p::one<'}'>) = "expecting '}'";
+error_message_for(p::one<'"'>) = "expecting '\"'";
+error_message_for(p::one<';'>) = "expecting ';'";
+error_message_for(p::one<')'>) = "expecting ')'";
+error_message_for(p::one<'='>) = "expecting '='";
+error_message_for(p::one<']'>) = "expecting ']'";
+error_message_for(p::one<':'>) = "expecting ':'";
+error_message_for(p::string<'\'', '\''>) = "expecting \"''\"";
+error_message_for(p::any) = "expecting any character";
+error_message_for(grammar::eof) = "expecting end of file";
+error_message_for(grammar::seps) = "expecting separators";
+error_message_for(grammar::path::forbid_prefix_triple_slash) = "too many slashes in path";
+error_message_for(grammar::path::forbid_prefix_double_slash_no_interp) = "path has a trailing slash";
+error_message_for(grammar::expr) = "expecting expression";
+error_message_for(grammar::expr::unary) = "expecting expression";
+error_message_for(grammar::binding::equal) = "expecting '='";
+error_message_for(grammar::expr::lambda::arg) = "expecting identifier";
+error_message_for(grammar::formals) = "expecting formals";
+error_message_for(grammar::attrpath) = "expecting attribute path";
+error_message_for(grammar::expr::select) = "expecting selection expression";
+error_message_for(grammar::t::kw_then) = "expecting 'then'";
+error_message_for(grammar::t::kw_else) = "expecting 'else'";
+error_message_for(grammar::t::kw_in) = "expecting 'in'";
+
+struct SyntaxErrors
+{
+    template<typename Rule>
+    static constexpr auto message = error_message<Rule>;
+
+    template<typename Rule>
+    static constexpr bool raise_on_failure = false;
+};
+
+template<typename Rule>
+struct Control : p::must_if<SyntaxErrors>::control<Rule>
+{
+    template<typename ParseInput, typename... States>
+    [[noreturn]] static void raise(const ParseInput & in, States &&... st)
+    {
+        if (in.empty()) {
+            std::string expected;
+            if constexpr (constexpr auto msg = error_message<Rule>)
+                expected = fmt(", %s", msg);
+            throw p::parse_error("unexpected end of file" + expected, in);
+        }
+        p::must_if<SyntaxErrors>::control<Rule>::raise(in, st...);
+    }
+};
+
+struct ExprState : grammar::operator_semantics<ExprState, PosIdx, AttrPath, std::unique_ptr<Expr>>
+{
+    template<typename Op, typename... Args>
+    std::unique_ptr<Expr> applyUnary(Args &&... args) {
+        return std::make_unique<Op>(popExpr(), std::forward<Args>(args)...);
+    }
+
+    template<typename Op>
+    std::unique_ptr<Expr> applyBinary(PosIdx pos) {
+        auto right = popExpr(), left = popExpr();
+        return std::make_unique<Op>(pos, std::move(left), std::move(right));
+    }
+
+    std::unique_ptr<Expr> call(PosIdx pos, Symbol fn, bool flip = false)
+    {
+        std::vector<std::unique_ptr<Expr>> args(2);
+        args[flip ? 0 : 1] = popExpr();
+        args[flip ? 1 : 0] = popExpr();
+        return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(fn), std::move(args));
+    }
+
+    std::unique_ptr<Expr> order(PosIdx pos, bool less, State & state)
+    {
+        return call(pos, state.s.lessThan, !less);
+    }
+
+    std::unique_ptr<Expr> concatStrings(PosIdx pos)
+    {
+        std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> args(2);
+        args[1].second = popExpr();
+        args[0].second = popExpr();
+        return std::make_unique<ExprConcatStrings>(pos, false, std::move(args));
+    }
+
+    std::unique_ptr<Expr> negate(PosIdx pos, State & state)
+    {
+        std::vector<std::unique_ptr<Expr>> args(2);
+        args[0] = std::make_unique<ExprInt>(0);
+        args[1] = popExpr();
+        return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(state.s.sub), std::move(args));
+    }
+
+    std::unique_ptr<Expr> applyOp(PosIdx pos, auto & op, State & state) {
+        using Op = grammar::op;
+
+        auto not_ = [] (auto e) {
+            return std::make_unique<ExprOpNot>(std::move(e));
+        };
+
+        return (overloaded {
+            [&] (Op::implies)     { return applyBinary<ExprOpImpl>(pos); },
+            [&] (Op::or_)         { return applyBinary<ExprOpOr>(pos); },
+            [&] (Op::and_)        { return applyBinary<ExprOpAnd>(pos); },
+            [&] (Op::equals)      { return applyBinary<ExprOpEq>(pos); },
+            [&] (Op::not_equals)  { return applyBinary<ExprOpNEq>(pos); },
+            [&] (Op::less)        { return order(pos, true, state); },
+            [&] (Op::greater_eq)  { return not_(order(pos, true, state)); },
+            [&] (Op::greater)     { return order(pos, false, state); },
+            [&] (Op::less_eq)     { return not_(order(pos, false, state)); },
+            [&] (Op::update)      { return applyBinary<ExprOpUpdate>(pos); },
+            [&] (Op::not_)        { return applyUnary<ExprOpNot>(); },
+            [&] (Op::plus)        { return concatStrings(pos); },
+            [&] (Op::minus)       { return call(pos, state.s.sub); },
+            [&] (Op::mul)         { return call(pos, state.s.mul); },
+            [&] (Op::div)         { return call(pos, state.s.div); },
+            [&] (Op::concat)      { return applyBinary<ExprOpConcatLists>(pos); },
+            [&] (has_attr & a)    { return applyUnary<ExprOpHasAttr>(std::move(a.path)); },
+            [&] (Op::unary_minus) { return negate(pos, state); },
+        })(op);
+    }
+
+    // always_inline is needed, otherwise pushOp slows down considerably
+    [[noreturn, gnu::always_inline]]
+    static void badOperator(PosIdx pos, State & state)
+    {
+        throw ParseError({
+            .msg = HintFmt("syntax error, unexpected operator"),
+            .pos = state.positions[pos]
+        });
+    }
+
+    template<typename Expr, typename... Args>
+    Expr & pushExpr(Args && ... args)
+    {
+        auto p = std::make_unique<Expr>(std::forward<Args>(args)...);
+        auto & result = *p;
+        exprs.emplace_back(std::move(p));
+        return result;
+    }
+};
+
+struct SubexprState {
+private:
+    ExprState * up;
+
+public:
+    explicit SubexprState(ExprState & up, auto &...) : up(&up) {}
+    operator ExprState &() { return *up; }
+    ExprState * operator->() { return up; }
+};
+
+
+
+template<typename Rule>
+struct BuildAST : grammar::nothing<Rule> {};
+
+struct LambdaState : SubexprState {
+    using SubexprState::SubexprState;
+
+    Symbol arg;
+    std::unique_ptr<Formals> formals;
+};
+
+struct FormalsState : SubexprState {
+    using SubexprState::SubexprState;
+
+    Formals formals{};
+    Formal formal{};
+};
+
+template<> struct BuildAST<grammar::formal::name> {
+    static void apply(const auto & in, FormalsState & s, State & ps) {
+        s.formal = {
+            .pos = ps.at(in),
+            .name = ps.symbols.create(in.string_view()),
+        };
+    }
+};
+
+template<> struct BuildAST<grammar::formal> {
+    static void apply0(FormalsState & s, State &) {
+        s.formals.formals.emplace_back(std::move(s.formal));
+    }
+};
+
+template<> struct BuildAST<grammar::formal::default_value> {
+    static void apply0(FormalsState & s, State & ps) {
+        s.formal.def = s->popExpr();
+    }
+};
+
+template<> struct BuildAST<grammar::formals::ellipsis> {
+    static void apply0(FormalsState & s, State &) {
+        s.formals.ellipsis = true;
+    }
+};
+
+template<> struct BuildAST<grammar::formals> : change_head<FormalsState> {
+    static void success0(FormalsState & f, LambdaState & s, State &) {
+        s.formals = std::make_unique<Formals>(std::move(f.formals));
+    }
+};
+
+struct AttrState : SubexprState {
+    using SubexprState::SubexprState;
+
+    std::vector<AttrName> attrs;
+
+    void pushAttr(auto && attr, PosIdx) { attrs.emplace_back(std::move(attr)); }
+};
+
+template<> struct BuildAST<grammar::attr::simple> {
+    static void apply(const auto & in, auto & s, State & ps) {
+        s.pushAttr(ps.symbols.create(in.string_view()), ps.at(in));
+    }
+};
+
+template<> struct BuildAST<grammar::attr::string> {
+    static void apply(const auto & in, auto & s, State & ps) {
+        auto e = s->popExpr();
+        if (auto str = dynamic_cast<ExprString *>(e.get()))
+            s.pushAttr(ps.symbols.create(str->s), ps.at(in));
+        else
+            s.pushAttr(std::move(e), ps.at(in));
+    }
+};
+
+template<> struct BuildAST<grammar::attr::expr> : BuildAST<grammar::attr::string> {};
+
+struct BindingsState : SubexprState {
+    using SubexprState::SubexprState;
+
+    ExprAttrs attrs;
+    AttrPath path;
+    std::unique_ptr<Expr> value;
+};
+
+struct InheritState : SubexprState {
+    using SubexprState::SubexprState;
+
+    std::vector<std::pair<AttrName, PosIdx>> attrs;
+    std::unique_ptr<Expr> from;
+    PosIdx fromPos;
+
+    void pushAttr(auto && attr, PosIdx pos) { attrs.emplace_back(std::move(attr), pos); }
+};
+
+template<> struct BuildAST<grammar::inherit::from> {
+    static void apply(const auto & in, InheritState & s, State & ps) {
+        s.from = s->popExpr();
+        s.fromPos = ps.at(in);
+    }
+};
+
+template<> struct BuildAST<grammar::inherit> : change_head<InheritState> {
+    static void success0(InheritState & s, BindingsState & b, State & ps) {
+        auto & attrs = b.attrs.attrs;
+        // TODO this should not reuse generic attrpath rules.
+        for (auto & [i, iPos] : s.attrs) {
+            if (i.symbol)
+                continue;
+            if (auto str = dynamic_cast<ExprString *>(i.expr.get()))
+                i = AttrName(ps.symbols.create(str->s));
+            else {
+                throw ParseError({
+                    .msg = HintFmt("dynamic attributes not allowed in inherit"),
+                    .pos = ps.positions[iPos]
+                });
+            }
+        }
+        if (auto fromE = std::move(s.from)) {
+            if (!b.attrs.inheritFromExprs)
+                b.attrs.inheritFromExprs = std::make_unique<std::vector<std::unique_ptr<Expr>>>();
+            b.attrs.inheritFromExprs->push_back(std::move(fromE));
+            for (auto & [i, iPos] : s.attrs) {
+                if (attrs.find(i.symbol) != attrs.end())
+                    ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos);
+                auto from = std::make_unique<ExprInheritFrom>(s.fromPos, b.attrs.inheritFromExprs->size() - 1);
+                attrs.emplace(
+                    i.symbol,
+                    ExprAttrs::AttrDef(
+                        std::make_unique<ExprSelect>(iPos, std::move(from), i.symbol),
+                        iPos,
+                        ExprAttrs::AttrDef::Kind::InheritedFrom));
+            }
+        } else {
+            for (auto & [i, iPos] : s.attrs) {
+                if (attrs.find(i.symbol) != attrs.end())
+                    ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos);
+                attrs.emplace(
+                    i.symbol,
+                    ExprAttrs::AttrDef(
+                        std::make_unique<ExprVar>(iPos, i.symbol),
+                        iPos,
+                        ExprAttrs::AttrDef::Kind::Inherited));
+            }
+        }
+    }
+};
+
+template<> struct BuildAST<grammar::binding::path> : change_head<AttrState> {
+    static void success0(AttrState & a, BindingsState & s, State & ps) {
+        s.path = std::move(a.attrs);
+    }
+};
+
+template<> struct BuildAST<grammar::binding::value> {
+    static void apply0(BindingsState & s, State & ps) {
+        s.value = s->popExpr();
+    }
+};
+
+template<> struct BuildAST<grammar::binding> {
+    static void apply(const auto & in, BindingsState & s, State & ps) {
+        ps.addAttr(&s.attrs, std::move(s.path), std::move(s.value), ps.at(in));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::id> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        if (in.string_view() == "__curPos")
+            s.pushExpr<ExprPos>(ps.at(in));
+        else
+            s.pushExpr<ExprVar>(ps.at(in), ps.symbols.create(in.string_view()));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::int_> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        int64_t v;
+        if (std::from_chars(in.begin(), in.end(), v).ec != std::errc{}) {
+            throw ParseError({
+                .msg = HintFmt("invalid integer '%1%'", in.string_view()),
+                .pos = ps.positions[ps.at(in)],
+            });
+        }
+        s.pushExpr<ExprInt>(v);
+    }
+};
+
+template<> struct BuildAST<grammar::expr::float_> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        // copy the input into a temporary string so we can call stod.
+        // can't use from_chars because libc++ (thus darwin) does not have it,
+        // and floats are not performance-sensitive anyway. if they were you'd
+        // be in much bigger trouble than this.
+        //
+        // we also get to do a locale-save dance because stod is locale-aware and
+        // something (a plugin?) may have called setlocale or uselocale.
+        static struct locale_hack {
+            locale_t posix;
+            locale_hack(): posix(newlocale(LC_ALL_MASK, "POSIX", 0))
+            {
+                if (posix == 0)
+                    throw SysError("could not get POSIX locale");
+            }
+        } locale;
+
+        auto tmp = in.string();
+        double v = [&] {
+            auto oldLocale = uselocale(locale.posix);
+            Finally resetLocale([=] { uselocale(oldLocale); });
+            try {
+                return std::stod(tmp);
+            } catch (...) {
+                throw ParseError({
+                    .msg = HintFmt("invalid float '%1%'", in.string_view()),
+                    .pos = ps.positions[ps.at(in)],
+                });
+            }
+        }();
+        s.pushExpr<ExprFloat>(v);
+    }
+};
+
+struct StringState : SubexprState {
+    using SubexprState::SubexprState;
+
+    std::string currentLiteral;
+    PosIdx currentPos;
+    std::vector<std::pair<nix::PosIdx, std::unique_ptr<Expr>>> parts;
+
+    void append(PosIdx pos, std::string_view s)
+    {
+        if (currentLiteral.empty())
+            currentPos = pos;
+        currentLiteral += s;
+    }
+
+    // FIXME this truncates strings on NUL for compat with the old parser. ideally
+    // we should use the decomposition the g gives us instead of iterating over
+    // the entire string again.
+    static void unescapeStr(std::string & str)
+    {
+        char * s = str.data();
+        char * t = s;
+        char c;
+        while ((c = *s++)) {
+            if (c == '\\') {
+                c = *s++;
+                if (c == 'n') *t = '\n';
+                else if (c == 'r') *t = '\r';
+                else if (c == 't') *t = '\t';
+                else *t = c;
+            }
+            else if (c == '\r') {
+                /* Normalise CR and CR/LF into LF. */
+                *t = '\n';
+                if (*s == '\n') s++; /* cr/lf */
+            }
+            else *t = c;
+            t++;
+        }
+        str.resize(t - str.data());
+    }
+
+    void endLiteral()
+    {
+        if (!currentLiteral.empty()) {
+            unescapeStr(currentLiteral);
+            parts.emplace_back(currentPos, std::make_unique<ExprString>(std::move(currentLiteral)));
+        }
+    }
+
+    std::unique_ptr<Expr> finish()
+    {
+        if (parts.empty()) {
+            unescapeStr(currentLiteral);
+            return std::make_unique<ExprString>(std::move(currentLiteral));
+        } else {
+            endLiteral();
+            auto pos = parts[0].first;
+            return std::make_unique<ExprConcatStrings>(pos, true, std::move(parts));
+        }
+    }
+};
+
+template<typename... Content> struct BuildAST<grammar::string::literal<Content...>> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        s.append(ps.at(in), in.string_view());
+    }
+};
+
+template<> struct BuildAST<grammar::string::cr_lf> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        s.append(ps.at(in), in.string_view()); // FIXME compat with old parser
+    }
+};
+
+template<> struct BuildAST<grammar::string::interpolation> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        s.endLiteral();
+        s.parts.emplace_back(ps.at(in), s->popExpr());
+    }
+};
+
+template<> struct BuildAST<grammar::string::escape> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        s.append(ps.at(in), "\\"); // FIXME compat with old parser
+        s.append(ps.at(in), in.string_view());
+    }
+};
+
+template<> struct BuildAST<grammar::string> : change_head<StringState> {
+    static void success0(StringState & s, ExprState & e, State &) {
+        e.exprs.push_back(s.finish());
+    }
+};
+
+struct IndStringState : SubexprState {
+    using SubexprState::SubexprState;
+
+    std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>> parts;
+};
+
+template<bool Indented, typename... Content>
+struct BuildAST<grammar::ind_string::literal<Indented, Content...>> {
+    static void apply(const auto & in, IndStringState & s, State & ps) {
+        s.parts.emplace_back(ps.at(in), StringToken{in.string_view(), Indented});
+    }
+};
+
+template<> struct BuildAST<grammar::ind_string::interpolation> {
+    static void apply(const auto & in, IndStringState & s, State & ps) {
+        s.parts.emplace_back(ps.at(in), s->popExpr());
+    }
+};
+
+template<> struct BuildAST<grammar::ind_string::escape> {
+    static void apply(const auto & in, IndStringState & s, State & ps) {
+        switch (*in.begin()) {
+        case 'n': s.parts.emplace_back(ps.at(in), StringToken{"\n"}); break;
+        case 'r': s.parts.emplace_back(ps.at(in), StringToken{"\r"}); break;
+        case 't': s.parts.emplace_back(ps.at(in), StringToken{"\t"}); break;
+        default:  s.parts.emplace_back(ps.at(in), StringToken{in.string_view()}); break;
+        }
+    }
+};
+
+template<> struct BuildAST<grammar::ind_string> : change_head<IndStringState> {
+    static void success(const auto & in, IndStringState & s, ExprState & e, State & ps) {
+        e.exprs.emplace_back(ps.stripIndentation(ps.at(in), std::move(s.parts)));
+    }
+};
+
+template<typename... Content> struct BuildAST<grammar::path::literal<Content...>> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        s.append(ps.at(in), in.string_view());
+        s.endLiteral();
+    }
+};
+
+template<> struct BuildAST<grammar::path::interpolation> : BuildAST<grammar::string::interpolation> {};
+
+template<> struct BuildAST<grammar::path::anchor> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        Path path(absPath(in.string(), ps.basePath.path.abs()));
+        /* add back in the trailing '/' to the first segment */
+        if (in.string_view().ends_with('/') && in.size() > 1)
+            path += "/";
+        s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path)));
+    }
+};
+
+template<> struct BuildAST<grammar::path::home_anchor> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        if (evalSettings.pureEval)
+            throw Error("the path '%s' can not be resolved in pure mode", in.string_view());
+        Path path(getHome() + in.string_view().substr(1));
+        s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path)));
+    }
+};
+
+template<> struct BuildAST<grammar::path::searched_path> {
+    static void apply(const auto & in, StringState & s, State & ps) {
+        std::vector<std::unique_ptr<Expr>> args{2};
+        args[0] = std::make_unique<ExprVar>(ps.s.nixPath);
+        args[1] = std::make_unique<ExprString>(in.string());
+        s.parts.emplace_back(
+            ps.at(in),
+            std::make_unique<ExprCall>(
+                ps.at(in),
+                std::make_unique<ExprVar>(ps.s.findFile),
+                std::move(args)));
+    }
+};
+
+template<> struct BuildAST<grammar::path> : change_head<StringState> {
+    template<typename E>
+    static void check_slash(PosIdx end, StringState & s, State & ps) {
+        auto e = dynamic_cast<E *>(s.parts.back().second.get());
+        if (!e || !e->s.ends_with('/'))
+            return;
+        if (s.parts.size() > 1 || e->s != "/")
+            throw ParseError({
+                .msg = HintFmt("path has a trailing slash"),
+                .pos = ps.positions[end],
+            });
+    }
+
+    static void success(const auto & in, StringState & s, ExprState & e, State & ps) {
+        s.endLiteral();
+        check_slash<ExprPath>(ps.atEnd(in), s, ps);
+        check_slash<ExprString>(ps.atEnd(in), s, ps);
+        if (s.parts.size() == 1) {
+            e.exprs.emplace_back(std::move(s.parts.back().second));
+        } else {
+            e.pushExpr<ExprConcatStrings>(ps.at(in), false, std::move(s.parts));
+        }
+    }
+};
+
+// strings and paths sare handled fully by the grammar-level rule for now
+template<> struct BuildAST<grammar::expr::string> : p::maybe_nothing {};
+template<> struct BuildAST<grammar::expr::ind_string> : p::maybe_nothing {};
+template<> struct BuildAST<grammar::expr::path> : p::maybe_nothing {};
+
+template<> struct BuildAST<grammar::expr::uri> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+       static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals);
+       if (noURLLiterals)
+           throw ParseError({
+               .msg = HintFmt("URL literals are disabled"),
+               .pos = ps.positions[ps.at(in)]
+           });
+       s.pushExpr<ExprString>(in.string());
+    }
+};
+
+template<> struct BuildAST<grammar::expr::ancient_let> : change_head<BindingsState> {
+    static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) {
+        b.attrs.pos = ps.at(in);
+        b.attrs.recursive = true;
+        s.pushExpr<ExprSelect>(b.attrs.pos, std::make_unique<ExprAttrs>(std::move(b.attrs)), ps.s.body);
+    }
+};
+
+template<> struct BuildAST<grammar::expr::rec_set> : change_head<BindingsState> {
+    static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) {
+        b.attrs.pos = ps.at(in);
+        b.attrs.recursive = true;
+        s.pushExpr<ExprAttrs>(std::move(b.attrs));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::set> : change_head<BindingsState> {
+    static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) {
+        b.attrs.pos = ps.at(in);
+        s.pushExpr<ExprAttrs>(std::move(b.attrs));
+    }
+};
+
+using ListState = std::vector<std::unique_ptr<Expr>>;
+
+template<> struct BuildAST<grammar::expr::list> : change_head<ListState> {
+    static void success0(ListState & ls, ExprState & s, State &) {
+        auto e = std::make_unique<ExprList>();
+        e->elems = std::move(ls);
+        s.exprs.push_back(std::move(e));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::list::entry> : change_head<ExprState> {
+    static void success0(ExprState & e, ListState & s, State & ps) {
+        s.emplace_back(e.finish(ps));
+    }
+};
+
+struct SelectState : SubexprState {
+    using SubexprState::SubexprState;
+
+    PosIdx pos;
+    ExprSelect * e = nullptr;
+};
+
+template<> struct BuildAST<grammar::expr::select::head> {
+    static void apply(const auto & in, SelectState & s, State & ps) {
+        s.pos = ps.at(in);
+    }
+};
+
+template<> struct BuildAST<grammar::expr::select::attr> : change_head<AttrState> {
+    static void success0(AttrState & a, SelectState & s, State &) {
+        s.e = &s->pushExpr<ExprSelect>(s.pos, s->popExpr(), std::move(a.attrs), nullptr);
+    }
+};
+
+template<> struct BuildAST<grammar::expr::select::attr_or> {
+    static void apply0(SelectState & s, State &) {
+        s.e->def = s->popExpr();
+    }
+};
+
+template<> struct BuildAST<grammar::expr::select::as_app_or> {
+    static void apply(const auto & in, SelectState & s, State & ps) {
+        std::vector<std::unique_ptr<Expr>> args(1);
+        args[0] = std::make_unique<ExprVar>(ps.at(in), ps.s.or_);
+        s->pushExpr<ExprCall>(s.pos, s->popExpr(), std::move(args));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::select> : change_head<SelectState> {
+    static void success0(const auto &...) {}
+};
+
+struct AppState : SubexprState {
+    using SubexprState::SubexprState;
+
+    PosIdx pos;
+    ExprCall * e = nullptr;
+};
+
+template<> struct BuildAST<grammar::expr::app::select_or_fn> {
+    static void apply(const auto & in, AppState & s, State & ps) {
+        s.pos = ps.at(in);
+    }
+};
+
+template<> struct BuildAST<grammar::expr::app::first_arg> {
+    static void apply(auto & in, AppState & s, State & ps) {
+        auto arg = s->popExpr(), fn = s->popExpr();
+        if ((s.e = dynamic_cast<ExprCall *>(fn.get()))) {
+            // TODO remove.
+            // AST compat with old parser, semantics are the same.
+            // this can happen on occasions such as `<p> <p>` or `a or b or`,
+            // neither of which are super worth optimizing.
+            s.e->args.push_back(std::move(arg));
+            s->exprs.emplace_back(std::move(fn));
+        } else {
+            std::vector<std::unique_ptr<Expr>> args{1};
+            args[0] = std::move(arg);
+            s.e = &s->pushExpr<ExprCall>(s.pos, std::move(fn), std::move(args));
+        }
+    }
+};
+
+template<> struct BuildAST<grammar::expr::app::another_arg> {
+    static void apply0(AppState & s, State & ps) {
+        s.e->args.push_back(s->popExpr());
+    }
+};
+
+template<> struct BuildAST<grammar::expr::app> : change_head<AppState> {
+    static void success0(const auto &...) {}
+};
+
+template<typename Op> struct BuildAST<grammar::expr::operator_<Op>> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        s.pushOp(ps.at(in), Op{}, ps);
+    }
+};
+template<> struct BuildAST<grammar::expr::operator_<grammar::op::has_attr>> : change_head<AttrState> {
+    static void success(const auto & in, AttrState & a, ExprState & s, State & ps) {
+        s.pushOp(ps.at(in), ExprState::has_attr{{}, std::move(a.attrs)}, ps);
+    }
+};
+
+template<> struct BuildAST<grammar::expr::lambda::arg> {
+    static void apply(const auto & in, LambdaState & s, State & ps) {
+        s.arg = ps.symbols.create(in.string_view());
+    }
+};
+
+template<> struct BuildAST<grammar::expr::lambda> : change_head<LambdaState> {
+    static void success(const auto & in, LambdaState & l, ExprState & s, State & ps) {
+        if (l.formals)
+            l.formals = ps.validateFormals(std::move(l.formals), ps.at(in), l.arg);
+        s.pushExpr<ExprLambda>(ps.at(in), l.arg, std::move(l.formals), l->popExpr());
+    }
+};
+
+template<> struct BuildAST<grammar::expr::assert_> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        auto body = s.popExpr(), cond = s.popExpr();
+        s.pushExpr<ExprAssert>(ps.at(in), std::move(cond), std::move(body));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::with> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        auto body = s.popExpr(), scope = s.popExpr();
+        s.pushExpr<ExprWith>(ps.at(in), std::move(scope), std::move(body));
+    }
+};
+
+template<> struct BuildAST<grammar::expr::let> : change_head<BindingsState> {
+    static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) {
+        if (!b.attrs.dynamicAttrs.empty())
+            throw ParseError({
+                .msg = HintFmt("dynamic attributes not allowed in let"),
+                .pos = ps.positions[ps.at(in)]
+            });
+
+        s.pushExpr<ExprLet>(std::make_unique<ExprAttrs>(std::move(b.attrs)), b->popExpr());
+    }
+};
+
+template<> struct BuildAST<grammar::expr::if_> {
+    static void apply(const auto & in, ExprState & s, State & ps) {
+        auto else_ = s.popExpr(), then = s.popExpr(), cond = s.popExpr();
+        s.pushExpr<ExprIf>(ps.at(in), std::move(cond), std::move(then), std::move(else_));
+    }
+};
+
+template<> struct BuildAST<grammar::expr> : change_head<ExprState> {
+    static void success0(ExprState & inner, ExprState & outer, State & ps) {
+        outer.exprs.push_back(inner.finish(ps));
+    }
+};
+
+}
+}
+
+namespace nix {
+
+Expr * EvalState::parse(
+    char * text,
+    size_t length,
+    Pos::Origin origin,
+    const SourcePath & basePath,
+    std::shared_ptr<StaticEnv> & staticEnv)
+{
+    parser::State s = {
+        symbols,
+        positions,
+        basePath,
+        positions.addOrigin(origin, length),
+        exprSymbols,
+    };
+    parser::ExprState x;
+
+    assert(length >= 2);
+    assert(text[length - 1] == 0);
+    assert(text[length - 2] == 0);
+    length -= 2;
+
+    p::string_input<p::tracking_mode::lazy> inp{std::string_view{text, length}, "input"};
+    try {
+        p::parse<parser::grammar::root, parser::BuildAST, parser::Control>(inp, x, s);
+    } catch (p::parse_error & e) {
+        auto pos = e.positions().back();
+        throw ParseError({
+            .msg = HintFmt("syntax error, %s", e.message()),
+            .pos = positions[s.positions.add(s.origin, pos.byte)]
+        });
+    }
+
+    auto result = x.finish(s);
+    result->bindVars(*this, staticEnv);
+    return result.release();
+}
+
+}

src/libexpr/parser/state.hh

@@ -3,59 +3,44 @@
 
 #include "eval.hh"
 
-namespace nix {
+namespace nix::parser {
 
-/**
- * @note Storing a C-style `char *` and `size_t` allows us to avoid
- * having to define the special members that using string_view here
- * would implicitly delete.
- */
 struct StringToken
 {
-    const char * p;
-    size_t l;
+    std::string_view s;
     bool hasIndentation;
-    operator std::string_view() const { return {p, l}; }
+    operator std::string_view() const { return s; }
 };
 
-struct ParserLocation
-{
-    int first_line, first_column;
-    int last_line, last_column;
-
-    // backup to recover from yyless(0)
-    int stashed_first_column, stashed_last_column;
-
-    void stash() {
-        stashed_first_column = first_column;
-        stashed_last_column = last_column;
-    }
-
-    void unstash() {
-        first_column = stashed_first_column;
-        last_column = stashed_last_column;
-    }
-};
-
-struct ParserState
+struct State
 {
     SymbolTable & symbols;
     PosTable & positions;
-    Expr * result;
     SourcePath basePath;
     PosTable::Origin origin;
     const Expr::AstSymbols & s;
 
     void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos);
     void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos);
-    void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos);
-    Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {});
-    Expr * stripIndentation(const PosIdx pos,
-        std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es);
-    PosIdx at(const ParserLocation & loc);
+    void addAttr(ExprAttrs * attrs, AttrPath && attrPath, std::unique_ptr<Expr> e, const PosIdx pos);
+    std::unique_ptr<Formals> validateFormals(std::unique_ptr<Formals> formals, PosIdx pos = noPos, Symbol arg = {});
+    std::unique_ptr<Expr> stripIndentation(const PosIdx pos,
+        std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>> && es);
+
+    // lazy positioning means we don't get byte offsets directly, in.position() would work
+    // but also requires line and column (which is expensive)
+    PosIdx at(const auto & in)
+    {
+        return positions.add(origin, in.begin() - in.input().begin());
+    }
+
+    PosIdx atEnd(const auto & in)
+    {
+        return positions.add(origin, in.end() - in.input().begin());
+    }
 };
 
-inline void ParserState::dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos)
+inline void State::dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos)
 {
     throw ParseError({
          .msg = HintFmt("attribute '%1%' already defined at %2%",
@@ -64,7 +49,7 @@ inline void ParserState::dupAttr(const AttrPath & attrPath, const PosIdx pos, co
     });
 }
 
-inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos)
+inline void State::dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos)
 {
     throw ParseError({
         .msg = HintFmt("attribute '%1%' already defined at %2%", symbols[attr], positions[prevPos]),
@@ -72,7 +57,7 @@ inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx pre
     });
 }
 
-inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos)
+inline void State::addAttr(ExprAttrs * attrs, AttrPath && attrPath, std::unique_ptr<Expr> e, const PosIdx pos)
 {
     AttrPath::iterator i;
     // All attrpaths have at least one attr
@@ -84,20 +69,20 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr *
             ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
             if (j != attrs->attrs.end()) {
                 if (j->second.kind != ExprAttrs::AttrDef::Kind::Inherited) {
-                    ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e);
-                    if (!attrs2) dupAttr({attrPath.begin(), i + 1}, pos, j->second.pos);
+                    ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e.get());
+                    if (!attrs2) dupAttr(attrPath, pos, j->second.pos);
                     attrs = attrs2;
                 } else
-                    dupAttr({attrPath.begin(), i + 1}, pos, j->second.pos);
+                    dupAttr(attrPath, pos, j->second.pos);
             } else {
-                ExprAttrs * nested = new ExprAttrs;
-                attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos);
-                attrs = nested;
+                auto next = attrs->attrs.emplace(std::piecewise_construct,
+                    std::tuple(i->symbol),
+                    std::tuple(std::make_unique<ExprAttrs>(), pos));
+                attrs = static_cast<ExprAttrs *>(next.first->second.e.get());
             }
         } else {
-            ExprAttrs *nested = new ExprAttrs;
-            attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos));
-            attrs = nested;
+            auto & next = attrs->dynamicAttrs.emplace_back(std::move(i->expr), std::make_unique<ExprAttrs>(), pos);
+            attrs = static_cast<ExprAttrs *>(next.valueExpr.get());
         }
     }
     // Expr insertion.
@@ -109,41 +94,41 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr *
             // e and the expr pointed by the attr path are two attribute sets,
             // we want to merge them.
             // Otherwise, throw an error.
-            auto ae = dynamic_cast<ExprAttrs *>(e);
-            auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
+            auto ae = dynamic_cast<ExprAttrs *>(e.get());
+            auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e.get());
             if (jAttrs && ae) {
                 if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
-                    jAttrs->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
+                    jAttrs->inheritFromExprs = std::make_unique<std::vector<std::unique_ptr<Expr>>>();
                 for (auto & ad : ae->attrs) {
                     auto j2 = jAttrs->attrs.find(ad.first);
                     if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
-                        dupAttr(ad.first, j2->second.pos, ad.second.pos);
-                    jAttrs->attrs.emplace(ad.first, ad.second);
+                        return dupAttr(ad.first, j2->second.pos, ad.second.pos);
                     if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
                         auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
                         auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
                         from.displ += jAttrs->inheritFromExprs->size();
                     }
+                    jAttrs->attrs.emplace(ad.first, std::move(ad.second));
                 }
-                jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end());
-                if (ae->inheritFromExprs) {
-                    jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(),
-                        ae->inheritFromExprs->begin(), ae->inheritFromExprs->end());
-                }
+                std::ranges::move(ae->dynamicAttrs, std::back_inserter(jAttrs->dynamicAttrs));
+                if (ae->inheritFromExprs)
+                    std::ranges::move(*ae->inheritFromExprs, std::back_inserter(*jAttrs->inheritFromExprs));
             } else {
                 dupAttr(attrPath, pos, j->second.pos);
             }
         } else {
             // This attr path is not defined. Let's create it.
-            attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos));
             e->setName(i->symbol);
+            attrs->attrs.emplace(std::piecewise_construct,
+                std::tuple(i->symbol),
+                std::tuple(std::move(e), pos));
         }
     } else {
-        attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos));
+        attrs->dynamicAttrs.emplace_back(std::move(i->expr), std::move(e), pos);
     }
 }
 
-inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg)
+inline std::unique_ptr<Formals> State::validateFormals(std::unique_ptr<Formals> formals, PosIdx pos, Symbol arg)
 {
     std::sort(formals->formals.begin(), formals->formals.end(),
         [] (const auto & a, const auto & b) {
@@ -172,10 +157,10 @@ inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Sym
     return formals;
 }
 
-inline Expr * ParserState::stripIndentation(const PosIdx pos,
-    std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es)
+inline std::unique_ptr<Expr> State::stripIndentation(const PosIdx pos,
+    std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>> && es)
 {
-    if (es.empty()) return new ExprString("");
+    if (es.empty()) return std::make_unique<ExprString>("");
 
     /* Figure out the minimum indentation.  Note that by design
        whitespace-only final lines are not taken into account.  (So
@@ -193,11 +178,11 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos,
             }
             continue;
         }
-        for (size_t j = 0; j < str->l; ++j) {
+        for (size_t j = 0; j < str->s.size(); ++j) {
             if (atStartOfLine) {
-                if (str->p[j] == ' ')
+                if (str->s[j] == ' ')
                     curIndent++;
-                else if (str->p[j] == '\n') {
+                else if (str->s[j] == '\n') {
                     /* Empty line, doesn't influence minimum
                        indentation. */
                     curIndent = 0;
@@ -205,7 +190,7 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos,
                     atStartOfLine = false;
                     if (curIndent < minIndent) minIndent = curIndent;
                 }
-            } else if (str->p[j] == '\n') {
+            } else if (str->s[j] == '\n') {
                 atStartOfLine = true;
                 curIndent = 0;
             }
@@ -213,35 +198,35 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos,
     }
 
     /* Strip spaces from each line. */
-    auto * es2 = new std::vector<std::pair<PosIdx, Expr *>>;
+    std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> es2;
     atStartOfLine = true;
     size_t curDropped = 0;
     size_t n = es.size();
     auto i = es.begin();
-    const auto trimExpr = [&] (Expr * e) {
+    const auto trimExpr = [&] (std::unique_ptr<Expr> & e) {
         atStartOfLine = false;
         curDropped = 0;
-        es2->emplace_back(i->first, e);
+        es2.emplace_back(i->first, std::move(e));
     };
     const auto trimString = [&] (const StringToken & t) {
         std::string s2;
-        for (size_t j = 0; j < t.l; ++j) {
+        for (size_t j = 0; j < t.s.size(); ++j) {
             if (atStartOfLine) {
-                if (t.p[j] == ' ') {
+                if (t.s[j] == ' ') {
                     if (curDropped++ >= minIndent)
-                        s2 += t.p[j];
+                        s2 += t.s[j];
                 }
-                else if (t.p[j] == '\n') {
+                else if (t.s[j] == '\n') {
                     curDropped = 0;
-                    s2 += t.p[j];
+                    s2 += t.s[j];
                 } else {
                     atStartOfLine = false;
                     curDropped = 0;
-                    s2 += t.p[j];
+                    s2 += t.s[j];
                 }
             } else {
-                s2 += t.p[j];
-                if (t.p[j] == '\n') atStartOfLine = true;
+                s2 += t.s[j];
+                if (t.s[j] == '\n') atStartOfLine = true;
             }
         }
 
@@ -253,24 +238,17 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos,
                 s2 = std::string(s2, 0, p + 1);
         }
 
-        es2->emplace_back(i->first, new ExprString(std::move(s2)));
+        es2.emplace_back(i->first, new ExprString(std::move(s2)));
     };
     for (; i != es.end(); ++i, --n) {
         std::visit(overloaded { trimExpr, trimString }, i->second);
     }
 
     /* If this is a single string, then don't do a concatenation. */
-    if (es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second)) {
-        auto *const result = (*es2)[0].second;
-        delete es2;
-        return result;
+    if (es2.size() == 1 && dynamic_cast<ExprString *>(es2[0].second.get())) {
+        return std::move(es2[0].second);
     }
-    return new ExprConcatStrings(pos, true, es2);
-}
-
-inline PosIdx ParserState::at(const ParserLocation & loc)
-{
-    return positions.add(origin, loc.first_column);
+    return std::make_unique<ExprConcatStrings>(pos, true, std::move(es2));
 }
 
 }

src/libexpr/pos-table.hh

@@ -59,8 +59,9 @@ public:
         uint32_t offset = 0;
         if (auto it = origins.rbegin(); it != origins.rend())
             offset = it->first + it->second.size;
-        // +1 because all PosIdx are offset by 1 to begin with (because noPos == 0), and
-        // another +1 to ensure that all origins can point to EOF, eg on (invalid) empty inputs.
+        // +1 because all PosIdx are offset by 1 to begin with, and
+        // another +1 to ensure that all origins can point to EOF, eg
+        // on (invalid) empty inputs.
         if (2 + offset + size < offset)
             return Origin{origin, offset, 0};
         return origins.emplace(offset, Origin{origin, offset, size}).first->second;

src/libexpr/primops.cc

@@ -243,9 +243,9 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
             // args[0]->attrs is already sorted.
 
             printTalkative("evaluating file '%1%'", path);
-            Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv);
+            Expr & e = state.parseExprFromFile(resolveExprPath(path), staticEnv);
 
-            e->eval(state, *env, v);
+            e.eval(state, *env, v);
         }
     }
 }
@@ -388,13 +388,13 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v)
     auto output = runProgram(program, true, commandArgs);
     Expr * parsed;
     try {
-        parsed = state.parseExprFromString(std::move(output), state.rootPath(CanonPath::root));
+        parsed = &state.parseExprFromString(std::move(output), state.rootPath(CanonPath::root));
     } catch (Error & e) {
         e.addTrace(state.positions[pos], "while parsing the output from '%1%'", program);
         throw;
     }
     try {
-        state.eval(parsed, v);
+        state.eval(*parsed, v);
     } catch (Error & e) {
         e.addTrace(state.positions[pos], "while evaluating the output from '%1%'", program);
         throw;
@@ -2813,7 +2813,7 @@ static void prim_functionArgs(EvalState & state, const PosIdx pos, Value * * arg
     auto attrs = state.buildBindings(args[0]->lambda.fun->formals->formals.size());
     for (auto & i : args[0]->lambda.fun->formals->formals)
         // !!! should optimise booleans (allocate only once)
-        attrs.alloc(i.name, i.pos).mkBool(i.def);
+        attrs.alloc(i.name, i.pos).mkBool(i.def.get());
     v.mkAttrs(attrs);
 }
 
@@ -4347,16 +4347,25 @@ void EvalState::createBaseEnv()
         .impureOnly = true,
     });
 
-    v.mkString("2.18.3-lix");
+    v.mkString(nixVersion);
     addConstant("__nixVersion", v, {
         .type = nString,
         .doc = R"(
-          Legacy version of Nix. Always returns "2.18.3-lix" on Lix.
+          The version of Nix.
 
-          To determine if features exist, Nix scripts should instead use direct
-          means of feature detection, such as checking for existence of
-          builtins they want to use. Doing so allows for much better compatibility
-          across implementations.
+          For example, where the command line returns the current Nix version,
+
+          ```shell-session
+          $ nix --version
+          nix (Nix) 2.16.0
+          ```
+
+          the Nix language evaluator returns the same value:
+
+          ```nix-repl
+          nix-repl> builtins.nixVersion
+          "2.16.0"
+          ```
         )",
     });
 
@@ -4476,7 +4485,7 @@ void EvalState::createBaseEnv()
         // the parser needs two NUL bytes as terminators; one of them
         // is implied by being a C string.
         "\0";
-    eval(parse(code, sizeof(code), derivationInternal, {CanonPath::root}, staticBaseEnv), *vDerivation);
+    eval(*parse(code, sizeof(code), derivationInternal, {CanonPath::root}, staticBaseEnv), *vDerivation);
 }
 
 

src/libexpr/primops/fromTOML.cc

@@ -1,8 +1,9 @@
 #include "primops.hh"
 #include "eval-inline.hh"
 
+#include "../../toml11/toml.hpp"
+
 #include <sstream>
-#include <toml.hpp>
 
 namespace nix {
 

src/libexpr/print-ambiguous.cc

@@ -93,7 +93,7 @@ void printAmbiguous(
         str << v.fpoint;
         break;
     default:
-        printError("Lix evaluator internal error: printAmbiguous: invalid value type");
+        printError("Nix evaluator internal error: printAmbiguous: invalid value type");
         abort();
     }
 }

src/libexpr/print.cc

@@ -1,5 +1,4 @@
 #include <limits>
-#include <span>
 #include <unordered_set>
 
 #include "print.hh"
@@ -608,7 +607,7 @@ std::ostream & operator<<(std::ostream & output, const ValuePrinter & printer)
 }
 
 template<>
-fmt_internal::HintFmt & fmt_internal::HintFmt::operator%(const ValuePrinter & value)
+HintFmt & HintFmt::operator%(const ValuePrinter & value)
 {
         fmt % value;
         return *this;

src/libexpr/print.hh

@@ -86,6 +86,6 @@ std::ostream & operator<<(std::ostream & output, const ValuePrinter & printer);
  * magenta.
  */
 template<>
-fmt_internal::HintFmt & fmt_internal::HintFmt::operator%(const ValuePrinter & value);
+HintFmt & HintFmt::operator%(const ValuePrinter & value);
 
 }

src/libexpr/value-to-xml.cc

@@ -144,7 +144,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
                 if (v.lambda.fun->arg) attrs["name"] = state.symbols[v.lambda.fun->arg];
                 if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1";
                 XMLOpenElement _(doc, "attrspat", attrs);
-                for (auto & i : v.lambda.fun->formals->lexicographicOrder(state.symbols))
+                for (const Formal & i : v.lambda.fun->formals->lexicographicOrder(state.symbols))
                     doc.writeEmptyElement("attr", singletonAttrs("name", state.symbols[i.name]));
             } else
                 doc.writeEmptyElement("varpat", singletonAttrs("name", state.symbols[v.lambda.fun->arg]));

src/libexpr/value.hh

@@ -323,11 +323,11 @@ public:
         }
     }
 
-    inline void mkThunk(Env * e, Expr * ex)
+    inline void mkThunk(Env * e, Expr & ex)
     {
         internalType = tThunk;
         thunk.env = e;
-        thunk.expr = ex;
+        thunk.expr = &ex;
     }
 
     inline void mkApp(Value * l, Value * r)

src/libfetchers/cache.hh

@@ -2,7 +2,6 @@
 ///@file
 
 #include "fetchers.hh"
-#include "path.hh"
 
 namespace nix::fetchers {
 

src/libfetchers/fetchers.cc

@@ -200,13 +200,12 @@ std::optional<Path> Input::getSourcePath() const
     return scheme->getSourcePath(*this);
 }
 
-void Input::putFile(
-    const CanonPath & path,
-    std::string_view contents,
+void Input::markChangedFile(
+    std::string_view file,
     std::optional<std::string> commitMsg) const
 {
     assert(scheme);
-    return scheme->putFile(*this, path, contents, commitMsg);
+    return scheme->markChangedFile(*this, file, commitMsg);
 }
 
 std::string Input::getName() const
@@ -296,18 +295,14 @@ Input InputScheme::applyOverrides(
     return input;
 }
 
-std::optional<Path> InputScheme::getSourcePath(const Input & input) const
+std::optional<Path> InputScheme::getSourcePath(const Input & input)
 {
     return {};
 }
 
-void InputScheme::putFile(
-    const Input & input,
-    const CanonPath & path,
-    std::string_view contents,
-    std::optional<std::string> commitMsg) const
+void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg)
 {
-    throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path);
+    assert(false);
 }
 
 void InputScheme::clone(const Input & input, const Path & destDir) const

src/libfetchers/fetchers.hh

@@ -3,7 +3,6 @@
 
 #include "types.hh"
 #include "hash.hh"
-#include "canon-path.hh"
 #include "path.hh"
 #include "attrs.hh"
 #include "url.hh"
@@ -98,13 +97,8 @@ public:
 
     std::optional<Path> getSourcePath() const;
 
-    /**
-     * Write a file to this input, for input types that support
-     * writing. Optionally commit the change (for e.g. Git inputs).
-     */
-    void putFile(
-        const CanonPath & path,
-        std::string_view contents,
+    void markChangedFile(
+        std::string_view file,
         std::optional<std::string> commitMsg) const;
 
     std::string getName() const;
@@ -150,13 +144,9 @@ struct InputScheme
 
     virtual void clone(const Input & input, const Path & destDir) const;
 
-    virtual std::optional<Path> getSourcePath(const Input & input) const;
+    virtual std::optional<Path> getSourcePath(const Input & input);
 
-    virtual void putFile(
-        const Input & input,
-        const CanonPath & path,
-        std::string_view contents,
-        std::optional<std::string> commitMsg) const;
+    virtual void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg);
 
     virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) = 0;
 };

src/libfetchers/git.cc

@@ -223,7 +223,7 @@ std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, co
     Path actualPath(absPath(workdir));
 
     PathFilter filter = [&](const Path & p) -> bool {
-        assert(p.starts_with(actualPath));
+        assert(hasPrefix(p, actualPath));
         std::string file(p, actualPath.size() + 1);
 
         auto st = lstat(p);
@@ -231,7 +231,7 @@ std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, co
         if (S_ISDIR(st.st_mode)) {
             auto prefix = file + "/";
             auto i = files.lower_bound(prefix);
-            return i != files.end() && (*i).starts_with(prefix);
+            return i != files.end() && hasPrefix(*i, prefix);
         }
 
         return files.count(file);
@@ -267,7 +267,7 @@ struct GitInputScheme : InputScheme
             url.scheme != "git+file") return {};
 
         auto url2(url);
-        if (url2.scheme.starts_with("git+")) url2.scheme = std::string(url2.scheme, 4);
+        if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
         url2.query.clear();
 
         Attrs attrs;
@@ -363,7 +363,7 @@ struct GitInputScheme : InputScheme
         runProgram("git", true, args, {}, true);
     }
 
-    std::optional<Path> getSourcePath(const Input & input) const override
+    std::optional<Path> getSourcePath(const Input & input) override
     {
         auto url = parseURL(getStrAttr(input.attrs, "url"));
         if (url.scheme == "file" && !input.getRef() && !input.getRev())
@@ -371,30 +371,22 @@ struct GitInputScheme : InputScheme
         return {};
     }
 
-    void putFile(
-        const Input & input,
-        const CanonPath & path,
-        std::string_view contents,
-        std::optional<std::string> commitMsg) const override
+    void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
     {
-        auto root = getSourcePath(input);
-        if (!root)
-            throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string());
-
-        writeFile((CanonPath(*root) + path).abs(), contents);
-
+        auto sourcePath = getSourcePath(input);
+        assert(sourcePath);
         auto gitDir = ".git";
 
         auto result = runProgram(RunOptions {
             .program = "git",
-            .args = {"-C", *root, "--git-dir", gitDir, "check-ignore", "--quiet", std::string(path.rel())},
+            .args = {"-C", *sourcePath, "--git-dir", gitDir, "check-ignore", "--quiet", std::string(file)},
         });
         auto exitCode = WEXITSTATUS(result.first);
 
         if (exitCode != 0) {
             // The path is not `.gitignore`d, we can add the file.
             runProgram("git", true,
-                { "-C", *root, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
+                { "-C", *sourcePath, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(file) });
 
 
             if (commitMsg) {
@@ -402,7 +394,7 @@ struct GitInputScheme : InputScheme
                 logger->pause();
                 Finally restoreLogger([]() { logger->resume(); });
                 runProgram("git", true,
-                    { "-C", *root, "--git-dir", gitDir, "commit", std::string(path.rel()), "-m", *commitMsg });
+                    { "-C", *sourcePath, "--git-dir", gitDir, "commit", std::string(file), "-m", *commitMsg });
             }
         }
     }

src/libfetchers/indirect.cc

@@ -1,6 +1,5 @@
 #include "fetchers.hh"
 #include "url-parts.hh"
-#include "path.hh"
 
 namespace nix::fetchers {
 

src/libfetchers/mercurial.cc

@@ -116,7 +116,7 @@ struct MercurialInputScheme : InputScheme
         return res;
     }
 
-    std::optional<Path> getSourcePath(const Input & input) const override
+    std::optional<Path> getSourcePath(const Input & input) override
     {
         auto url = parseURL(getStrAttr(input.attrs, "url"));
         if (url.scheme == "file" && !input.getRef() && !input.getRev())
@@ -124,27 +124,18 @@ struct MercurialInputScheme : InputScheme
         return {};
     }
 
-    void putFile(
-        const Input & input,
-        const CanonPath & path,
-        std::string_view contents,
-        std::optional<std::string> commitMsg) const override
+    void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
     {
-        auto [isLocal, repoPath] = getActualUrl(input);
-        if (!isLocal)
-            throw Error("cannot commit '%s' to Mercurial repository '%s' because it's not a working tree", path, input.to_string());
-
-        auto absPath = CanonPath(repoPath) + path;
-
-        writeFile(absPath.abs(), contents);
+        auto sourcePath = getSourcePath(input);
+        assert(sourcePath);
 
         // FIXME: shut up if file is already tracked.
         runHg(
-            { "add", absPath.abs() });
+            { "add", *sourcePath + "/" + std::string(file) });
 
         if (commitMsg)
             runHg(
-                { "commit", absPath.abs(), "-m", *commitMsg });
+                { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg });
     }
 
     std::pair<bool, std::string> getActualUrl(const Input & input) const
@@ -190,7 +181,7 @@ struct MercurialInputScheme : InputScheme
                 Path actualPath(absPath(actualUrl));
 
                 PathFilter filter = [&](const Path & p) -> bool {
-                    assert(p.starts_with(actualPath));
+                    assert(hasPrefix(p, actualPath));
                     std::string file(p, actualPath.size() + 1);
 
                     auto st = lstat(p);
@@ -198,7 +189,7 @@ struct MercurialInputScheme : InputScheme
                     if (S_ISDIR(st.st_mode)) {
                         auto prefix = file + "/";
                         auto i = files.lower_bound(prefix);
-                        return i != files.end() && (*i).starts_with(prefix);
+                        return i != files.end() && hasPrefix(*i, prefix);
                     }
 
                     return files.count(file);

src/libfetchers/meson.build

@@ -1,43 +0,0 @@
-libfetchers_sources = files(
-  'attrs.cc',
-  'cache.cc',
-  'fetch-settings.cc',
-  'fetch-to-store.cc',
-  'fetchers.cc',
-  'git.cc',
-  'github.cc',
-  'indirect.cc',
-  'mercurial.cc',
-  'path.cc',
-  'registry.cc',
-  'tarball.cc',
-)
-
-libfetchers_headers = files(
-  'attrs.hh',
-  'cache.hh',
-  'fetch-settings.hh',
-  'fetch-to-store.hh',
-  'fetchers.hh',
-  'registry.hh',
-)
-
-libfetchers = library(
-  'nixfetchers',
-  libfetchers_sources,
-  dependencies : [
-    liblixstore,
-    liblixutil,
-    nlohmann_json,
-  ],
-  install : true,
-  # FIXME(Qyriad): is this right?
-  install_rpath : libdir,
-)
-
-install_headers(libfetchers_headers, subdir : 'nix', preserve_path : true)
-
-liblixfetchers = declare_dependency(
-  include_directories : include_directories('.'),
-  link_with : libfetchers,
-)

src/libfetchers/path.cc

@@ -71,28 +71,14 @@ struct PathInputScheme : InputScheme
         return true;
     }
 
-    std::optional<Path> getSourcePath(const Input & input) const override
+    std::optional<Path> getSourcePath(const Input & input) override
     {
         return getStrAttr(input.attrs, "path");
     }
 
-    void putFile(
-        const Input & input,
-        const CanonPath & path,
-        std::string_view contents,
-        std::optional<std::string> commitMsg) const override
+    void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
     {
-        writeFile((CanonPath(getAbsPath(input)) + path).abs(), contents);
-    }
-
-    CanonPath getAbsPath(const Input & input) const
-    {
-        auto path = getStrAttr(input.attrs, "path");
-
-        if (path[0] == '/')
-            return CanonPath(path);
-
-        throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
+        // nothing to do
     }
 
     std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override

src/libfetchers/registry.cc

@@ -157,7 +157,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
             return std::make_shared<Registry>(Registry::Global); // empty registry
         }
 
-        if (!path.starts_with("/")) {
+        if (!hasPrefix(path, "/")) {
             auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath;
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
                 store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json");

src/libfetchers/tarball.cc

@@ -188,10 +188,10 @@ struct CurlInputScheme : InputScheme
 
     const bool hasTarballExtension(std::string_view path) const
     {
-        return path.ends_with(".zip") || path.ends_with(".tar")
-            || path.ends_with(".tgz") || path.ends_with(".tar.gz")
-            || path.ends_with(".tar.xz") || path.ends_with(".tar.bz2")
-            || path.ends_with(".tar.zst");
+        return hasSuffix(path, ".zip") || hasSuffix(path, ".tar")
+            || hasSuffix(path, ".tgz") || hasSuffix(path, ".tar.gz")
+            || hasSuffix(path, ".tar.xz") || hasSuffix(path, ".tar.bz2")
+            || hasSuffix(path, ".tar.zst");
     }
 
     virtual bool isValidURL(const ParsedURL & url, bool requireTree) const = 0;

src/libmain/common-args.cc

@@ -1,5 +1,4 @@
 #include "common-args.hh"
-#include "args/root.hh"
 #include "globals.hh"
 #include "loggers.hh"
 
@@ -35,21 +34,21 @@ MixCommonArgs::MixCommonArgs(const std::string & programName)
         .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).",
         .category = miscCategory,
         .labels = {"name", "value"},
-        .handler = {[this](std::string name, std::string value) {
+        .handler = {[](std::string name, std::string value) {
             try {
                 globalConfig.set(name, value);
             } catch (UsageError & e) {
-                if (!getRoot().completions)
+                if (!completions)
                     warn(e.what());
             }
         }},
-        .completer = [](AddCompletions & completions, size_t index, std::string_view prefix) {
+        .completer = [](size_t index, std::string_view prefix) {
             if (index == 0) {
                 std::map<std::string, Config::SettingInfo> settings;
                 globalConfig.getSettings(settings);
                 for (auto & s : settings)
-                    if (s.first.starts_with(prefix))
-                        completions.add(s.first, fmt("Set the `%s` setting.", s.first));
+                    if (hasPrefix(s.first, prefix))
+                        completions->add(s.first, fmt("Set the `%s` setting.", s.first));
             }
         }
     });

src/libmain/meson.build

@@ -1,47 +0,0 @@
-libmain_sources = files(
-  'common-args.cc',
-  'loggers.cc',
-  'progress-bar.cc',
-  'shared.cc',
-  'stack.cc',
-)
-
-libmain_headers = files(
-  'common-args.hh',
-  'loggers.hh',
-  'progress-bar.hh',
-  'shared.hh',
-)
-
-libmain = library(
-  'nixmain',
-  libmain_sources,
-  dependencies : [
-    liblixutil,
-    liblixstore,
-  ],
-  install : true,
-  # FIXME(Qyriad): is this right?
-  install_rpath : libdir,
-)
-
-install_headers(libmain_headers, subdir : 'nix', preserve_path : true)
-
-liblixmain = declare_dependency(
-  include_directories : include_directories('.'),
-  link_with : libmain,
-)
-
-# FIXME: not using the pkg-config module because it creates way too many deps
-# while meson migration is in progress, and we want to not include boost here
-configure_file(
-  input : 'nix-main.pc.in',
-  output : 'nix-main.pc',
-  install_dir : libdir / 'pkgconfig',
-  configuration : {
-    'prefix' : prefix,
-    'libdir' : libdir,
-    'includedir' : includedir,
-    'PACKAGE_VERSION' : meson.project_version(),
-  },
-)