Add repl-overlays

Adds a `repl-overlays` option, which specifies files that can overlay
and modify the top-level bindings in `nix repl`. For example, with the
following contents in `~/.config/nix/repl.nix`:

    info: final: prev: let
      optionalAttrs = predicate: attrs:
        if predicate
        then attrs
        else {};
    in
      optionalAttrs (prev ? legacyPackages && prev.legacyPackages ? ${info.currentSystem})
      {
        pkgs = prev.legacyPackages.${info.currentSystem};
      }

We can run `nix repl` and use `pkgs` to refer to `legacyPackages.${currentSystem}`:

    $ nix repl --repl-overlays ~/.config/nix/repl.nix nixpkgs
    Nix 2.21.0pre20240309_4111bb6
    Type :? for help.
    Loading installable 'flake:nixpkgs#'...
    Added 5 variables.
    Loading 'repl-overlays'...
    Added 6 variables.
    nix-repl> pkgs.bash
    «derivation /nix/store/g08b5vkwwh0j8ic9rkmd8mpj878rk62z-bash-5.2p26.drv»

Change-Id: Ic12e0f2f210b2f46e920c33088dfe1083f42391a

doc/manual/rl-next/repl-overlays.md

@@ -0,0 +1,35 @@
+---
+synopsis: Add `repl-overlays` option
+prs: 10203
+---
+
+A `repl-overlays` option has been added, which specifies files that can overlay
+and modify the top-level bindings in `nix repl`. For example, with the
+following contents in `~/.config/nix/repl.nix`:
+
+```nix
+info: final: prev: let
+  optionalAttrs = predicate: attrs:
+    if predicate
+    then attrs
+    else {};
+in
+  optionalAttrs (prev ? legacyPackages && prev.legacyPackages ? ${info.currentSystem})
+  {
+    pkgs = prev.legacyPackages.${info.currentSystem};
+  }
+```
+
+We can run `nix repl` and use `pkgs` to refer to `legacyPackages.${currentSystem}`:
+
+```ShellSession
+$ nix repl --repl-overlays ~/.config/nix/repl.nix nixpkgs
+Nix 2.21.0pre20240309_4111bb6
+Type :? for help.
+Loading installable 'flake:nixpkgs#'...
+Added 5 variables.
+Loading 'repl-overlays'...
+Added 6 variables.
+nix-repl> pkgs.bash
+«derivation /nix/store/g08b5vkwwh0j8ic9rkmd8mpj878rk62z-bash-5.2p26.drv»
+```

src/libcmd/local.mk

@@ -13,3 +13,5 @@ libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread
 libcmd_LIBS = libstore libutil libexpr libmain libfetchers
 
 $(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))
+
+$(d)/repl.cc: $(d)/repl-overlays.nix.gen.hh

src/libcmd/repl-overlays.nix

@@ -0,0 +1,8 @@
+info:
+initial:
+functions:
+let final = builtins.foldl'
+              (prev: function: prev // (function info final prev))
+              initial
+              functions;
+in final

src/libcmd/repl.cc

@@ -29,6 +29,7 @@
 #include "local-fs-store.hh"
 #include "signals.hh"
 #include "print.hh"
+#include "gc-small-vector.hh"
 
 #if HAVE_BOEHMGC
 #define GC_INCLUDE_NEW
@@ -99,6 +100,42 @@ struct NixRepl
     void evalString(std::string s, Value & v);
     void loadDebugTraceEnv(DebugTrace & dt);
 
+    /**
+     * Load the `repl-overlays` and add the resulting AttrSet to the top-level
+     * bindings.
+     */
+    void loadReplOverlays();
+
+    /**
+     * Get a list of each of the `repl-overlays` (parsed and evaluated).
+     */
+    Value * replOverlays();
+
+    /**
+     * Get the Nix function that composes the `repl-overlays` together.
+     */
+    Value * getReplOverlaysEvalFunction();
+
+    /**
+     * Cached return value of `getReplOverlaysEvalFunction`.
+     */
+    Value * replOverlaysEvalFunction;
+
+    /**
+     * Get the `info` AttrSet that's passed as the first argument to each
+     * of the `repl-overlays`.
+     */
+    Value * replInitInfo();
+
+    /**
+     * Get the current top-level bindings as an AttrSet.
+     */
+    Value * bindingsToAttrs();
+    /**
+     * Parse a file, evaluate its result, and force the resulting value.
+     */
+    Value * evalFile(SourcePath & path);
+
     void printValue(std::ostream & str,
                               Value & v,
                               unsigned int maxDepth = std::numeric_limits<unsigned int>::max())
@@ -735,14 +772,95 @@ void NixRepl::loadFiles()
     loadedFiles.clear();
 
     for (auto & i : old) {
-        notice("Loading '%1%'...", i);
+        notice("Loading '%1%'...", Magenta(i));
         loadFile(i);
     }
 
     for (auto & [i, what] : getValues()) {
-        notice("Loading installable '%1%'...", what);
+        notice("Loading installable '%1%'...", Magenta(what));
         addAttrsToScope(*i);
     }
+
+    loadReplOverlays();
+}
+
+void NixRepl::loadReplOverlays()
+{
+    if (!evalSettings.replOverlays) {
+        return;
+    }
+
+    notice("Loading '%1%'...", Magenta("repl-overlays"));
+    auto replInitFilesFunction = getReplOverlaysEvalFunction();
+
+    Value &newAttrs(*state->allocValue());
+    SmallValueVector<3> args = {replInitInfo(), bindingsToAttrs(), replOverlays()};
+    state->callFunction(
+        *replInitFilesFunction,
+        args.size(),
+        args.data(),
+        newAttrs,
+        replInitFilesFunction->determinePos(noPos)
+    );
+
+    addAttrsToScope(newAttrs);
+}
+
+Value * NixRepl::getReplOverlaysEvalFunction()
+{
+    if (replOverlaysEvalFunction) {
+        return replOverlaysEvalFunction;
+    }
+
+    auto evalReplInitFilesPath = CanonPath::root + "repl-overlays.nix";
+    replOverlaysEvalFunction = state->allocValue();
+    auto code =
+        #include "repl-overlays.nix.gen.hh"
+        ;
+    auto expr = state->parseExprFromString(
+        code,
+        SourcePath(evalReplInitFilesPath),
+        state->staticBaseEnv
+    );
+
+    state->eval(expr, *replOverlaysEvalFunction);
+
+    return replOverlaysEvalFunction;
+}
+
+Value * NixRepl::replOverlays()
+{
+    Value * replInits(state->allocValue());
+    state->mkList(*replInits, evalSettings.replOverlays.get().size());
+    Value ** replInitElems = replInits->listElems();
+
+    size_t i = 0;
+    for (auto path : evalSettings.replOverlays.get()) {
+        debug("Loading '%1%' path '%2%'...", "repl-overlays", path);
+        SourcePath sourcePath((CanonPath(path)));
+        replInitElems[i] = evalFile(sourcePath);
+        i++;
+    }
+
+
+    return replInits;
+}
+
+Value * NixRepl::replInitInfo()
+{
+    auto builder = state->buildBindings(2);
+
+    Value * currentSystem(state->allocValue());
+    currentSystem->mkString(evalSettings.getCurrentSystem());
+    builder.insert(state->symbols.create("currentSystem"), currentSystem);
+
+    Value * valueNull(state->allocValue());
+    valueNull->mkNull();
+    builder.insert(state->symbols.create("__pleaseUseDotDotDot"), valueNull);
+
+    Value * info(state->allocValue());
+    info->mkAttrs(builder.finish());
+    return info;
 }
 
 
@@ -775,6 +893,18 @@ void NixRepl::addVarToScope(const Symbol name, Value & v)
     varNames.emplace(state->symbols[name]);
 }
 
+Value * NixRepl::bindingsToAttrs()
+{
+    auto builder = state->buildBindings(staticEnv->vars.size());
+    for (auto & [symbol, displacement] : staticEnv->vars) {
+        builder.insert(symbol, env->values[displacement]);
+    }
+
+    Value * attrs(state->allocValue());
+    attrs->mkAttrs(builder.finish());
+    return attrs;
+}
+
 
 Expr * NixRepl::parseString(std::string s)
 {
@@ -789,6 +919,15 @@ void NixRepl::evalString(std::string s, Value & v)
     state->forceValue(v, v.determinePos(noPos));
 }
 
+Value * NixRepl::evalFile(SourcePath & path)
+{
+    auto expr = state->parseExprFromFile(path, staticEnv);
+    Value * result(state->allocValue());
+    expr->eval(*state, *env, *result);
+    state->forceValue(*result, result->determinePos(noPos));
+    return result;
+}
+
 
 std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
    const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,

src/libexpr/eval-settings.hh

@@ -124,6 +124,42 @@ struct EvalSettings : Config
 
           This is useful for debugging warnings in third-party Nix code.
         )"};
+
+    PathsSetting replOverlays{this, Paths(), "repl-overlays",
+        R"(
+          A list of files containing Nix expressions that can be used to add
+          default bindings to [`nix
+          repl`](@docroot@/command-ref/new-cli/nix3-repl.md) sessions.
+
+          Each file is called with three arguments:
+          1. An [attribute set](@docroot@/language/values.html#attribute-set)
+             containing at least a
+             [`currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem)
+             attribute (this is identical to
+             [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem),
+             except that it's available in
+             [`pure-eval`](@docroot@/command-ref/conf-file.html#conf-pure-eval)
+             mode).
+          2. The top-level bindings produced by the previous `repl-overlays`
+             value (or the default top-level bindings).
+          3. The final top-level bindings produced by calling all
+             `repl-overlays`.
+
+          For example, the following file would alias `pkgs` to
+          `legacyPackages.${info.currentSystem}` (if that attribute is defined):
+
+          ```nix
+          info: prev: final:
+          if prev ? legacyPackages
+             && prev.legacyPackages ? ${info.currentSystem}
+          then
+          {
+            pkgs = prev.legacyPackages.${info.currentSystem};
+          }
+          else
+          { }
+          ```
+        )"};
 };
 
 extern EvalSettings evalSettings;

src/libexpr/nixexpr.cc

@@ -4,6 +4,7 @@
 #include "symbol-table.hh"
 #include "util.hh"
 #include "print.hh"
+#include "escape-string.hh"
 
 #include <cstdlib>
 
@@ -36,7 +37,7 @@ void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const
 
 void ExprString::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    printLiteralString(str, s);
+    escapeString(str, s);
 }
 
 void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const

src/libexpr/print-ambiguous.cc

@@ -2,6 +2,7 @@
 #include "print.hh"
 #include "eval.hh"
 #include "signals.hh"
+#include "escape-string.hh"
 
 namespace nix {
 
@@ -27,7 +28,7 @@ void printAmbiguous(
         printLiteralBool(str, v.boolean);
         break;
     case nString:
-        printLiteralString(str, v.string.s);
+        escapeString(str, v.string.s);
         break;
     case nPath:
         str << v.path().to_string(); // !!! escaping?

src/libexpr/print.cc

@@ -1,6 +1,7 @@
 #include <limits>
 #include <unordered_set>
 
+#include "escape-string.hh"
 #include "print.hh"
 #include "ansicolor.hh"
 #include "store-api.hh"
@@ -27,40 +28,6 @@ void printElided(
 }
 
 
-std::ostream &
-printLiteralString(std::ostream & str, const std::string_view string, size_t maxLength, bool ansiColors)
-{
-    size_t charsPrinted = 0;
-    if (ansiColors)
-        str << ANSI_MAGENTA;
-    str << "\"";
-    for (auto i = string.begin(); i != string.end(); ++i) {
-        if (charsPrinted >= maxLength) {
-            str << "\" ";
-            printElided(str, string.length() - charsPrinted, "byte", "bytes", ansiColors);
-            return str;
-        }
-
-        if (*i == '\"' || *i == '\\') str << "\\" << *i;
-        else if (*i == '\n') str << "\\n";
-        else if (*i == '\r') str << "\\r";
-        else if (*i == '\t') str << "\\t";
-        else if (*i == '$' && *(i+1) == '{') str << "\\" << *i;
-        else str << *i;
-        charsPrinted++;
-    }
-    str << "\"";
-    if (ansiColors)
-        str << ANSI_NORMAL;
-    return str;
-}
-
-std::ostream &
-printLiteralString(std::ostream & str, const std::string_view string)
-{
-    return printLiteralString(str, string, std::numeric_limits<size_t>::max(), false);
-}
-
 std::ostream &
 printLiteralBool(std::ostream & str, bool boolean)
 {
@@ -92,7 +59,7 @@ printIdentifier(std::ostream & str, std::string_view s) {
     else {
         char c = s[0];
         if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) {
-            printLiteralString(str, s);
+            escapeString(str, s);
             return str;
         }
         for (auto c : s)
@@ -100,7 +67,7 @@ printIdentifier(std::ostream & str, std::string_view s) {
                   (c >= 'A' && c <= 'Z') ||
                   (c >= '0' && c <= '9') ||
                   c == '_' || c == '\'' || c == '-')) {
-                printLiteralString(str, s);
+                escapeString(str, s);
                 return str;
             }
         str << s;
@@ -128,7 +95,7 @@ printAttributeName(std::ostream & str, std::string_view name) {
     if (isVarName(name))
         str << name;
     else
-        printLiteralString(str, name);
+        escapeString(str, name);
     return str;
 }
 
@@ -247,7 +214,15 @@ private:
 
     void printString(Value & v)
     {
-        printLiteralString(output, v.string.s, options.maxStringLength, options.ansiColors);
+        escapeString(
+            output,
+            v.string.s,
+            {
+                .maxLength = options.maxStringLength,
+                .ansiColors = options.ansiColors,
+                // NB: Non-printing characters won't be escaped.
+            }
+        );
     }
 
     void printPath(Value & v)

src/libexpr/print.hh

@@ -17,22 +17,6 @@ namespace nix {
 class EvalState;
 struct Value;
 
-/**
- * Print a string as a Nix string literal.
- *
- * Quotes and fairly minimal escaping are added.
- *
- * @param o The output stream to print to
- * @param s The logical string
- */
-std::ostream & printLiteralString(std::ostream & o, std::string_view s);
-inline std::ostream & printLiteralString(std::ostream & o, const char * s) {
-    return printLiteralString(o, std::string_view(s));
-}
-inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) {
-    return printLiteralString(o, std::string_view(s));
-}
-
 /** Print `true` or `false`. */
 std::ostream & printLiteralBool(std::ostream & o, bool b);
 

src/libutil/config.cc

@@ -437,6 +437,34 @@ void OptionalPathSetting::operator =(const std::optional<Path> & v)
     this->assign(v);
 }
 
+PathsSetting::PathsSetting(Config * options,
+    const Paths & def,
+    const std::string & name,
+    const std::string & description,
+    const std::set<std::string> & aliases)
+    : BaseSetting<Paths>(def, true, name, description, aliases)
+{
+    options->addSetting(this);
+}
+
+
+Paths PathsSetting::parse(const std::string & str) const
+{
+    auto strings = tokenizeString<Strings>(str);
+    Paths parsed;
+
+    for (auto str : strings) {
+        parsed.push_back(canonPath(str));
+    }
+
+    return parsed;
+}
+
+PathsSetting::operator bool() const noexcept
+{
+    return !get().empty();
+}
+
 bool GlobalConfig::set(const std::string & name, const std::string & value)
 {
     for (auto & config : *configRegistrations)

src/libutil/config.hh

@@ -375,6 +375,26 @@ public:
     void operator =(const std::optional<Path> & v);
 };
 
+/**
+ * Like `OptionalPathSetting`, but allows multiple paths.
+ */
+class PathsSetting : public BaseSetting<Paths>
+{
+public:
+
+    PathsSetting(Config * options,
+        const Paths & def,
+        const std::string & name,
+        const std::string & description,
+        const std::set<std::string> & aliases = {});
+
+    Paths parse(const std::string & str) const override;
+
+    void operator =(const Paths & v);
+
+    operator bool() const noexcept;
+};
+
 struct GlobalConfig : public AbstractConfig
 {
     typedef std::vector<Config*> ConfigRegistrations;

src/libutil/debug-char.cc

@@ -0,0 +1,22 @@
+#include <boost/io/ios_state.hpp>
+#include <iomanip>
+#include <iostream>
+
+#include "debug-char.hh"
+
+namespace nix {
+
+std::ostream & operator<<(std::ostream & s, DebugChar c)
+{
+    boost::io::ios_flags_saver _ifs(s);
+
+    if (isprint(c.c)) {
+        s << static_cast<char>(c.c);
+    } else {
+        s << "\\x" << std::hex << std::setfill('0') << std::setw(2)
+          << (static_cast<unsigned int>(c.c) & 0xff);
+    }
+    return s;
+}
+
+} // namespace nix

src/libutil/debug-char.hh

@@ -0,0 +1,20 @@
+#include <ostream>
+
+namespace nix {
+
+/**
+ * A struct that prints a debug representation of a character, like `\x1f` for
+ * non-printable characters, or the character itself for other characters.
+ *
+ * Note that these are suitable for human readable output, but further care is
+ * necessary to include them in C++ strings to avoid running into adjacent
+ * hex-like characters. (`"puppy\x1bdoggy"` parses as `"puppy" "\x1bd" "oggy"`
+ * and errors because 0x1bd is too big for a `char`.)
+ */
+struct DebugChar {
+  char c;
+};
+
+std::ostream &operator<<(std::ostream &s, DebugChar c);
+
+} // namespace nix

src/libutil/escape-string.cc

@@ -0,0 +1,80 @@
+#include <boost/io/ios_state.hpp>
+#include <iomanip>
+#include <ostream>
+#include <sstream>
+
+#include "ansicolor.hh"
+#include "debug-char.hh"
+#include "english.hh"
+#include "escape-string.hh"
+
+namespace nix {
+
+void printElided(
+    std::ostream & output,
+    unsigned int value,
+    const std::string_view single,
+    const std::string_view plural,
+    bool ansiColors
+)
+{
+    if (ansiColors) {
+        output << ANSI_FAINT;
+    }
+    output << "«";
+    pluralize(output, value, single, plural);
+    output << " elided»";
+    if (ansiColors) {
+        output << ANSI_NORMAL;
+    }
+}
+
+std::ostream &
+escapeString(std::ostream & output, std::string_view string, EscapeStringOptions options)
+{
+    size_t charsPrinted = 0;
+    if (options.ansiColors) {
+        output << ANSI_MAGENTA;
+    }
+    output << "\"";
+    for (auto i = string.begin(); i != string.end(); ++i) {
+        if (charsPrinted >= options.maxLength) {
+            output << "\" ";
+            printElided(
+                output, string.length() - charsPrinted, "byte", "bytes", options.ansiColors
+            );
+            return output;
+        }
+
+        if (*i == '\"' || *i == '\\') {
+            output << "\\" << *i;
+        } else if (*i == '\n') {
+            output << "\\n";
+        } else if (*i == '\r') {
+            output << "\\r";
+        } else if (*i == '\t') {
+            output << "\\t";
+        } else if (*i == '$' && *(i + 1) == '{') {
+            output << "\\" << *i;
+        } else if (options.escapeNonPrinting && !isprint(*i)) {
+            output << DebugChar{*i};
+        } else {
+            output << *i;
+        }
+        charsPrinted++;
+    }
+    output << "\"";
+    if (options.ansiColors) {
+        output << ANSI_NORMAL;
+    }
+    return output;
+}
+
+std::string escapeString(std::string_view s, EscapeStringOptions options)
+{
+    std::ostringstream output;
+    escapeString(output, s, options);
+    return output.str();
+}
+
+}; // namespace nix

src/libutil/escape-string.hh

@@ -0,0 +1,68 @@
+#pragma once
+
+#include <limits>
+#include <ostream>
+
+namespace nix {
+
+/**
+ * Options for escaping strings in `escapeString`.
+ *
+ * With default optional parameters, the output string will round-trip through
+ * the Nix evaluator (i.e. you can copy/paste this function's output into the
+ * REPL and have it evaluate as the string that got passed in).
+ *
+ * With non-default optional parameters, the output string will be
+ * human-readable.
+ */
+struct EscapeStringOptions {
+  /**
+   * If `maxLength` is decreased, some trailing portion of the string may be
+   * omitted with a message like `«123 bytes elided»`.
+   */
+  size_t maxLength = std::numeric_limits<size_t>::max();
+
+  /**
+   * If `ansiColors` is set, the output will contain ANSI terminal escape
+   * sequences.
+   */
+  bool ansiColors = false;
+
+  /**
+   * If `escapeNonPrinting` is set, non-printing ASCII characters (i.e. with
+   * byte values less than 0x20) will be printed in `\xhh` format, like
+   * `\x1d` (other than those that Nix supports, like `\n`, `\r`, `\t`).
+   * Note that this format is not yet supported by the Lix parser/evaluator!
+   *
+   * See: https://git.lix.systems/lix-project/lix/issues/149
+   */
+  bool escapeNonPrinting = false;
+};
+
+/**
+ * Escape a string for output.
+ *
+ * With default optional parameters, the output string will round-trip through
+ * the Nix evaluator (i.e. you can copy/paste this function's output into the
+ * REPL and have it evaluate as the string that got passed in).
+ *
+ * With non-default optional parameters, the output string will be
+ * human-readable.
+ *
+ * See `EscapeStringOptions` for more details on customizing the output.
+ */
+std::ostream &escapeString(std::ostream &output, std::string_view s,
+                           EscapeStringOptions options = {});
+inline std::ostream &escapeString(std::ostream &output, const char *s) {
+  return escapeString(output, std::string_view(s));
+}
+inline std::ostream &escapeString(std::ostream &output, const std::string &s) {
+  return escapeString(output, std::string_view(s));
+}
+
+/**
+ * Escape a string for output, writing the escaped result to a new string.
+ */
+std::string escapeString(std::string_view s, EscapeStringOptions options = {});
+
+} // namespace nix

src/libutil/shlex.cc

@@ -0,0 +1,77 @@
+#include "shlex.hh"
+#include "util.hh"
+
+namespace nix {
+
+std::vector<std::string> shell_split(const std::string & input)
+{
+    std::vector<std::string> result;
+
+    // Hack: `shell_split` is janky and parses ` a` as `{"", "a"}`, so we trim
+    // whitespace before starting.
+    auto inputTrimmed = trim(input);
+
+    if (inputTrimmed.empty()) {
+        return result;
+    }
+
+    std::regex whitespace("^\\s+");
+    auto begin = inputTrimmed.cbegin();
+    std::string currentToken;
+    enum State { sBegin, sSingleQuote, sDoubleQuote };
+    State state = sBegin;
+    auto iterator = begin;
+
+    for (; iterator != inputTrimmed.cend(); ++iterator) {
+        if (state == sBegin) {
+            std::smatch match;
+            if (regex_search(iterator, inputTrimmed.cend(), match, whitespace)) {
+                currentToken.append(begin, iterator);
+                result.push_back(currentToken);
+                iterator = match[0].second;
+                if (iterator == inputTrimmed.cend()) {
+                    return result;
+                }
+                begin = iterator;
+                currentToken.clear();
+            }
+        }
+
+        switch (*iterator) {
+        case '\'':
+            if (state != sDoubleQuote) {
+                currentToken.append(begin, iterator);
+                begin = iterator + 1;
+                state = state == sBegin ? sSingleQuote : sBegin;
+            }
+            break;
+
+        case '"':
+            if (state != sSingleQuote) {
+                currentToken.append(begin, iterator);
+                begin = iterator + 1;
+                state = state == sBegin ? sDoubleQuote : sBegin;
+            }
+            break;
+
+        case '\\':
+            if (state != sSingleQuote) {
+                // perl shellwords mostly just treats the next char as part
+                // of the string with no special processing
+                currentToken.append(begin, iterator);
+                begin = ++iterator;
+            }
+            break;
+        }
+    }
+
+    if (state != sBegin) {
+        throw ShlexError(input);
+    }
+
+    currentToken.append(begin, iterator);
+    result.push_back(currentToken);
+    return result;
+}
+
+}

src/libutil/shlex.hh

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <regex>
+#include <string>
+#include <vector>
+
+#include "error.hh"
+
+namespace nix {
+
+class ShlexError : public Error
+{
+public:
+    const std::string input;
+
+    ShlexError(const std::string input)
+        : Error("Failed to parse shell arguments (unterminated quote?): %1%", input)
+        , input(input)
+    {
+    }
+};
+
+/**
+ * Parse a string into shell arguments.
+ *
+ * Takes care of whitespace, quotes, and backslashes (at least a bit).
+ */
+std::vector<std::string> shell_split(const std::string & input);
+
+} // namespace nix

src/nix-build/nix-build.cc

@@ -23,70 +23,13 @@
 #include "common-eval-args.hh"
 #include "attr-path.hh"
 #include "legacy.hh"
+#include "shlex.hh"
 
 using namespace nix;
 using namespace std::string_literals;
 
 extern char * * environ __attribute__((weak));
 
-/* Recreate the effect of the perl shellwords function, breaking up a
- * string into arguments like a shell word, including escapes
- */
-static std::vector<std::string> shellwords(const std::string & s)
-{
-    std::regex whitespace("^\\s+");
-    auto begin = s.cbegin();
-    std::vector<std::string> res;
-    std::string cur;
-    enum state {
-        sBegin,
-        sSingleQuote,
-        sDoubleQuote
-    };
-    state st = sBegin;
-    auto it = begin;
-    for (; it != s.cend(); ++it) {
-        if (st == sBegin) {
-            std::smatch match;
-            if (regex_search(it, s.cend(), match, whitespace)) {
-                cur.append(begin, it);
-                res.push_back(cur);
-                it = match[0].second;
-                if (it == s.cend()) return res;
-                begin = it;
-                cur.clear();
-            }
-        }
-        switch (*it) {
-            case '\'':
-                if (st != sDoubleQuote) {
-                    cur.append(begin, it);
-                    begin = it + 1;
-                    st = st == sBegin ? sSingleQuote : sBegin;
-                }
-                break;
-            case '"':
-                if (st != sSingleQuote) {
-                    cur.append(begin, it);
-                    begin = it + 1;
-                    st = st == sBegin ? sDoubleQuote : sBegin;
-                }
-                break;
-            case '\\':
-                if (st != sSingleQuote) {
-                    /* perl shellwords mostly just treats the next char as part of the string with no special processing */
-                    cur.append(begin, it);
-                    begin = ++it;
-                }
-                break;
-        }
-    }
-    if (st != sBegin) throw Error("unterminated quote in shebang line");
-    cur.append(begin, it);
-    res.push_back(cur);
-    return res;
-}
-
 static void main_nix_build(int argc, char * * argv)
 {
     auto dryRun = false;
@@ -143,7 +86,7 @@ static void main_nix_build(int argc, char * * argv)
                     line = chomp(line);
                     std::smatch match;
                     if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell\\s+(.*)$")))
-                        for (const auto & word : shellwords(match[1].str()))
+                        for (const auto & word : shell_split(match[1].str()))
                             args.push_back(word);
                 }
             }

tests/functional/repl_characterization/data/basic.ast

@@ -1,17 +1,28 @@
-Commentary "meow meow meow"
-Command "command"
-Output "output output one"
-Output ""
-Output ""
-Output "output output two"
-Commentary "meow meow"
-Command "command two"
-Output "output output output"
-Commentary "commentary"
-Output "output output output"
-Output ""
-Commentary "the blank below should be chomped"
-Command "command three"
-Commentary ""
-Output "meow output"
-Output ""
+Commentary: "meow meow meow"
+Indent: "  "
+Prompt: "nix-repl> "
+Command: "command"
+Indent: "  "
+Output: "output output one"
+Output: ""
+Commentary: ""
+Indent: "  "
+Output: "output output two"
+Commentary: "meow meow"
+Indent: "  "
+Prompt: "nix-repl> "
+Command: "command two"
+Indent: "  "
+Output: "output output output"
+Commentary: "commentary"
+Indent: "  "
+Output: "output output output"
+Output: ""
+Commentary: "the blank below should be chomped"
+Indent: "  "
+Prompt: "nix-repl> "
+Command: "command three"
+Commentary: ""
+Indent: "  "
+Output: "meow output"
+Output: ""

tests/functional/repl_characterization/data/basic_repl.ast

@@ -0,0 +1,60 @@
+Command "1 + 1"
+Output "2"
+Output ""
+Command ":doc builtins.head"
+Output "Synopsis: builtins.head list"
+Output ""
+Output "    Return the first element of a list; abort evaluation if"
+Output "    the argument isn’t a list or is an empty list. You can"
+Output "    test whether a list is empty by comparing it with []."
+Output ""
+Command "f = a: \"\" + a"
+Commentary ""
+Commentary "Expect the trace to not contain any traceback:"
+Commentary ""
+Command "f 2"
+Output "error:"
+Output "       … while evaluating a path segment"
+Output "         at «string»:1:10:"
+Output "            1|  a: \"\" + a"
+Output "             |          ^"
+Output ""
+Output "       error: cannot coerce an integer to a string: 2"
+Output ""
+Command ":te"
+Output "showing error traces"
+Output ""
+Commentary "Expect the trace to have traceback:"
+Commentary ""
+Command "f 2"
+Output "error:"
+Output "       … from call site"
+Output "         at «string»:1:1:"
+Output "            1| f 2"
+Output "             | ^"
+Output ""
+Output "       … while calling anonymous lambda"
+Output "         at «string»:1:2:"
+Output "            1|  a: \"\" + a"
+Output "             |  ^"
+Output ""
+Output "       … while evaluating a path segment"
+Output "         at «string»:1:10:"
+Output "            1|  a: \"\" + a"
+Output "             |          ^"
+Output ""
+Output "       error: cannot coerce an integer to a string: 2"
+Output ""
+Commentary "Turning it off should also work:"
+Commentary ""
+Command ":te"
+Output "not showing error traces"
+Output ""
+Command "f 2"
+Output "error:"
+Output "       … while evaluating a path segment"
+Output "         at «string»:1:10:"
+Output "            1|  a: \"\" + a"
+Output "             |          ^"
+Output ""
+Output "       error: cannot coerce an integer to a string: 2"

tests/functional/repl_characterization/data/basic_tidied.ast

@@ -1,10 +1,9 @@
-Command "command"
-Output "output output one"
-Output ""
-Output ""
-Output "output output two"
-Command "command two"
-Output "output output output"
-Output "output output output"
-Command "command three"
-Output "meow output"
+Command: "command"
+Output: "output output one"
+Output: ""
+Output: "output output two"
+Command: "command two"
+Output: "output output output"
+Output: "output output output"
+Command: "command three"
+Output: "meow output"

tests/functional/repl_characterization/data/basic_tidied.test

@@ -0,0 +1,11 @@
+command
+output output one
+
+output output two
+command two
+output output output
+output output output
+
+command three
+meow output
+

tests/functional/repl_characterization/data/regression_9917.test

@@ -1,6 +1,7 @@
 https://github.com/NixOS/nix/pull/9917 (Enter debugger more reliably in let expressions and function calls)
 
 This test ensures that continues don't skip opportunities to enter the debugger.
+@args --debugger
   trace: before outer break
   info: breakpoint reached
 
@@ -13,7 +14,7 @@ This test ensures that continues don't skip opportunities to enter the debugger.
   0: error: breakpoint reached
   «none»:0
   1: while calling a function
-  TEST_DATA/regression_9917.nix:3:5
+  $TEST_DATA/regression_9917.nix:3:5
 
        2|   a = builtins.trace "before inner break" (
        3|     builtins.break { msg = "hello"; }
@@ -21,7 +22,7 @@ This test ensures that continues don't skip opportunities to enter the debugger.
        4|   );
 
   2: while calling a function
-  TEST_DATA/regression_9917.nix:2:7
+  $TEST_DATA/regression_9917.nix:2:7
 
        1| let
        2|   a = builtins.trace "before inner break" (

tests/functional/repl_characterization/data/regression_9918.test

@@ -1,3 +1,4 @@
+@args --debugger
   error:
          … while evaluating the error message passed to builtin.throw
 
@@ -14,3 +15,18 @@ We expect to be able to see locals like r in the debugger:
 
   Env level 1
   abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true
+
+  nix-repl> :quit
+  error:
+         … while evaluating the file '$TEST_DATA/regression_9918.nix':
+
+         … while calling the 'throw' builtin
+           at $TEST_DATA/regression_9918.nix:3:7:
+              2|   r = [];
+              3|   x = builtins.throw r;
+               |       ^
+              4| in
+
+         … while evaluating the error message passed to builtin.throw
+
+         error: cannot coerce a list to a string: [ ]

tests/functional/repl_characterization/data/repl_overlays.nix

@@ -0,0 +1,3 @@
+{
+  packages.x86_64-linux.default = "my package";
+}

tests/functional/repl_characterization/data/repl_overlays.test

@@ -0,0 +1,5 @@
+Check basic `repl-overlays` functionality.
+@args --repl-overlays
+@args $PWD/../extra_data/repl-overlay-packages-is-pkgs.nix
+  nix-repl> pkgs
+  { default = "my package"; }

tests/functional/repl_characterization/data/repl_overlays_compose.nix

@@ -0,0 +1,3 @@
+{
+  var = "a";
+}

tests/functional/repl_characterization/data/repl_overlays_compose.test

@@ -0,0 +1,7 @@
+Check that multiple `repl-overlays` can compose together
+@args --repl-overlays
+@args "$PWD/../extra_data/repl-overlays-compose-1.nix $PWD/../extra_data/repl-overlays-compose-2.nix"
+  nix-repl> var
+  "abc"
+  nix-repl> varUsingFinal
+  "final value is: puppy"

tests/functional/repl_characterization/data/repl_overlays_destructure_without_dotdotdot_errors.test

@@ -0,0 +1,26 @@
+`repl-overlays` that try to parse out the `info` argument without a `...` error.
+@args --repl-overlays
+@args $PWD/../extra_data/repl-overlay-no-dotdotdot.nix
+@should-start false
+  error:
+         … while calling the 'foldl'' builtin
+           at «string»:5:13:
+              4| functions:
+              5| let final = builtins.foldl'
+               |             ^
+              6|               (prev: function: prev // (function info final prev))
+
+         … in the right operand of the update (//) operator
+           at «string»:6:37:
+              5| let final = builtins.foldl'
+              6|               (prev: function: prev // (function info final prev))
+               |                                     ^
+              7|               initial
+
+         (stack trace truncated; use '--show-trace' to show the full trace)
+
+         error: function 'anonymous lambda' called with unexpected argument '__pleaseUseDotDotDot'
+         at /Users/wiggles/lix/tests/functional/repl_characterization/extra_data/repl-overlay-no-dotdotdot.nix:1:1:
+              1| {currentSystem}: final: prev: {}
+               | ^
+              2|

tests/functional/repl_characterization/data/repl_overlays_error.test

@@ -0,0 +1,22 @@
+`repl-overlays` that fail to evaluate should error.
+@args --repl-overlays
+@args $PWD/../extra_data/repl-overlay-fail.nix
+@should-start false
+  error:
+         … while calling the 'foldl'' builtin
+           at «string»:5:13:
+              4| functions:
+              5| let final = builtins.foldl'
+               |             ^
+              6|               (prev: function: prev // (function info final prev))
+
+         … in the right operand of the update (//) operator
+           at «string»:6:37:
+              5| let final = builtins.foldl'
+              6|               (prev: function: prev // (function info final prev))
+               |                                     ^
+              7|               initial
+
+         (stack trace truncated; use '--show-trace' to show the full trace)
+
+         error: evaluation aborted with the following error message: 'uh oh!'

tests/functional/repl_characterization/data/stack_vars.test

@@ -1,3 +1,4 @@
+@args --debugger
   trace: before outer break
   info: breakpoint reached
 
@@ -24,7 +25,7 @@ If we :st past the frame in the backtrace with the meow in it, the meow should n
   nix-repl> :st 3
 
   3: while calling a function
-  TEST_DATA/stack_vars.nix:5:7
+  $TEST_DATA/stack_vars.nix:5:7
 
        4|   );
        5|   b = builtins.trace "before outer break" (
@@ -58,9 +59,8 @@ If we :st past the frame in the backtrace with the meow in it, the meow should n
   3
 
   nix-repl> :st 3
-
   3: while calling a function
-  TEST_DATA/stack_vars.nix:2:7
+  $TEST_DATA/stack_vars.nix:2:7
 
        1| let
        2|   a = builtins.trace "before inner break" (
@@ -72,3 +72,21 @@ If we :st past the frame in the backtrace with the meow in it, the meow should n
 
   Env level 1
   abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true
+
+  nix-repl> :quit
+  error:
+         … while calling the 'trace' builtin
+           at $TEST_DATA/stack_vars.nix:2:7:
+              1| let
+              2|   a = builtins.trace "before inner break" (
+               |       ^
+              3|     let meow' = 3; in builtins.break { msg = "hello"; }
+
+         … while calling the 'break' builtin
+           at $TEST_DATA/stack_vars.nix:3:23:
+              2|   a = builtins.trace "before inner break" (
+              3|     let meow' = 3; in builtins.break { msg = "hello"; }
+               |                       ^
+              4|   );
+
+         error: breakpoint reached

tests/functional/repl_characterization/extra_data/repl-overlay-fail.nix

@@ -0,0 +1 @@
+info: final: prev: builtins.abort "uh oh!"

tests/functional/repl_characterization/extra_data/repl-overlay-no-dotdotdot.nix

@@ -0,0 +1 @@
+{currentSystem}: final: prev: {}

tests/functional/repl_characterization/extra_data/repl-overlay-packages-is-pkgs.nix

@@ -0,0 +1,4 @@
+info: final: prev:
+{
+  pkgs = final.packages.x86_64-linux;
+}

tests/functional/repl_characterization/extra_data/repl-overlays-compose-1.nix

@@ -0,0 +1,7 @@
+info: final: prev:
+{
+  var = prev.var + "b";
+
+  # We can access the final value of `var` here even though it isn't defined yet:
+  varUsingFinal = "final value is: " + final.newVar;
+}

tests/functional/repl_characterization/extra_data/repl-overlays-compose-2.nix

@@ -0,0 +1,6 @@
+info: final: prev:
+{
+  var = prev.var + "c";
+
+  newVar = "puppy";
+}

tests/functional/repl_characterization/repl_characterization.cc

@@ -1,25 +1,26 @@
 #include <gtest/gtest.h>
 
+#include <boost/algorithm/string/replace.hpp>
+#include <optional>
 #include <string>
 #include <string_view>
-#include <optional>
 #include <unistd.h>
-#include <boost/algorithm/string/replace.hpp>
 
+#include "escape-string.hh"
 #include "test-session.hh"
-#include "util.hh"
 #include "tests/characterization.hh"
 #include "tests/cli-literate-parser.hh"
 #include "tests/terminal-code-eater.hh"
+#include "util.hh"
 
 using namespace std::string_literals;
 
 namespace nix {
 
-static constexpr const char * REPL_PROMPT = "nix-repl> ";
+static constexpr const std::string_view REPL_PROMPT = "nix-repl> ";
 
 // ASCII ENQ character
-static constexpr const char * AUTOMATION_PROMPT = "\x05";
+static constexpr const std::string_view AUTOMATION_PROMPT = "\x05";
 
 static std::string_view trimOutLog(std::string_view outLog)
 {
@@ -40,91 +41,148 @@ public:
         return unitTestData + "/" + testStem;
     }
 
-    void runReplTest(std::string_view const & content, std::vector<std::string> extraArgs = {}) const
+    void runReplTest(const std::string content, std::vector<std::string> extraArgs = {}) const
     {
-        auto syntax = CLILiterateParser::parse(REPL_PROMPT, content);
+        auto parsed = cli_literate_parser::parse(
+            content, cli_literate_parser::Config{.prompt = std::string(REPL_PROMPT), .indent = 2}
+        );
+        parsed.interpolatePwd(unitTestData);
 
         // FIXME: why does this need two --quiets
-        // show-trace is on by default due to test configuration, but is not a standard
-        Strings args{"--quiet", "repl", "--quiet", "--option", "show-trace", "false", "--offline", "--extra-experimental-features", "repl-automation"};
+        // show-trace is on by default due to test configuration, but is not a
+        // standard
+        Strings args{
+            "--quiet",
+            "repl",
+            "--quiet",
+            "--option",
+            "show-trace",
+            "false",
+            "--offline",
+            "--extra-experimental-features",
+            "repl-automation",
+        };
         args.insert(args.end(), extraArgs.begin(), extraArgs.end());
+        args.insert(args.end(), parsed.args.begin(), parsed.args.end());
 
         auto nixBin = canonPath(getEnvNonEmpty("NIX_BIN_DIR").value_or(NIX_BIN_DIR));
 
         auto process = RunningProcess::start(nixBin + "/nix", args);
-        auto session = TestSession{AUTOMATION_PROMPT, std::move(process)};
+        auto session = TestSession(std::string(AUTOMATION_PROMPT), std::move(process));
 
-        for (auto & bit : syntax) {
-            if (bit.kind != CLILiterateParser::NodeKind::COMMAND) {
-                continue;
+        for (auto & event : parsed.syntax) {
+            std::visit(
+                overloaded{
+                    [&](const cli_literate_parser::Command & e) {
+                        ASSERT_TRUE(session.waitForPrompt());
+                        if (e.text == ":quit") {
+                            // If we quit the repl explicitly, we won't have a
+                            // prompt when we're done.
+                            parsed.shouldStart = false;
                         }
-
-            if (!session.waitForPrompt()) {
-                ASSERT_TRUE(false);
+                        session.runCommand(e.text);
+                    },
+                    [&](const cli_literate_parser::Output & e) {},
+                    [&](const auto & e) {},
+                },
+                event
+            );
         }
-            session.runCommand(bit.text);
-        }
-        if (!session.waitForPrompt()) {
-            ASSERT_TRUE(false);
+        if (parsed.shouldStart) {
+            ASSERT_TRUE(session.waitForPrompt());
         }
         session.close();
 
-        auto replacedOutLog = boost::algorithm::replace_all_copy(session.outLog, unitTestData, "TEST_DATA");
+        auto replacedOutLog =
+            boost::algorithm::replace_all_copy(session.outLog, unitTestData, "$TEST_DATA");
         auto cleanedOutLog = trimOutLog(replacedOutLog);
 
-        auto parsedOutLog = CLILiterateParser::parse(AUTOMATION_PROMPT, cleanedOutLog, 0);
+        auto parsedOutLog = cli_literate_parser::parse(
+            std::string(cleanedOutLog),
+            cli_literate_parser::Config{.prompt = std::string(AUTOMATION_PROMPT), .indent = 0}
+        );
 
-        parsedOutLog = CLILiterateParser::tidyOutputForComparison(std::move(parsedOutLog));
-        syntax = CLILiterateParser::tidyOutputForComparison(std::move(syntax));
+        auto expected = parsed.tidyOutputForComparison();
+        auto actual = parsedOutLog.tidyOutputForComparison();
 
-        ASSERT_EQ(parsedOutLog, syntax);
+        ASSERT_EQ(expected, actual);
+    }
+
+    void runReplTestPath(const std::string_view & nameBase, std::vector<std::string> extraArgs)
+    {
+        auto nixPath = goldenMaster(nameBase + ".nix");
+        if (pathExists(nixPath)) {
+            extraArgs.push_back("-f");
+            extraArgs.push_back(nixPath);
+        }
+        readTest(nameBase + ".test", [this, extraArgs](std::string input) {
+            runReplTest(input, extraArgs);
+        });
+    }
+
+    void runReplTestPath(const std::string_view & nameBase)
+    {
+        runReplTestPath(nameBase, {});
+    }
+
+    void runDebuggerTest(const std::string_view & nameBase)
+    {
+        runReplTestPath(nameBase, {"--debugger"});
     }
 };
 
-TEST_F(ReplSessionTest, parses)
+TEST_F(ReplSessionTest, round_trip)
+{
+    writeTest("basic.test", [this]() {
+        const std::string content = readFile(goldenMaster("basic.test"));
+        auto parsed = cli_literate_parser::parse(
+            content, cli_literate_parser::Config{.prompt = std::string(REPL_PROMPT)}
+        );
+
+        std::ostringstream out{};
+        for (auto & node : parsed.syntax) {
+            cli_literate_parser::unparseNode(out, node, true);
+        }
+        return out.str();
+    });
+}
+
+TEST_F(ReplSessionTest, tidy)
 {
     writeTest("basic.ast", [this]() {
         const std::string content = readFile(goldenMaster("basic.test"));
-        auto parser = CLILiterateParser{REPL_PROMPT};
-        parser.feed(content);
-
+        auto parsed = cli_literate_parser::parse(
+            content, cli_literate_parser::Config{.prompt = std::string(REPL_PROMPT)}
+        );
         std::ostringstream out{};
-        for (auto & bit : parser.syntax()) {
-            out << bit.print() << "\n";
+        for (auto & node : parsed.syntax) {
+            out << debugNode(node) << "\n";
         }
         return out.str();
     });
-
     writeTest("basic_tidied.ast", [this]() {
         const std::string content = readFile(goldenMaster("basic.test"));
-        auto syntax = CLILiterateParser::parse(REPL_PROMPT, content);
-
-        syntax = CLILiterateParser::tidyOutputForComparison(std::move(syntax));
-
+        auto parsed = cli_literate_parser::parse(
+            content, cli_literate_parser::Config{.prompt = std::string(REPL_PROMPT)}
+        );
+        auto tidied = parsed.tidyOutputForComparison();
         std::ostringstream out{};
-        for (auto & bit : syntax) {
-            out << bit.print() << "\n";
+        for (auto & node : tidied) {
+            out << debugNode(node) << "\n";
         }
         return out.str();
     });
 }
 
-TEST_F(ReplSessionTest, repl_basic)
-{
-    readTest("basic_repl.test", [this](std::string input) { runReplTest(input); });
-}
-
-#define DEBUGGER_TEST(name) \
+#define REPL_TEST(name)           \
     TEST_F(ReplSessionTest, name) \
     {                             \
-        readTest(#name ".test", [this](std::string input) { \
-            runReplTest(input, {"--debugger", "-f", goldenMaster(#name ".nix")}); \
-        }); \
+        runReplTestPath(#name);   \
     }
 
-DEBUGGER_TEST(regression_9918);
-DEBUGGER_TEST(regression_9917);
-DEBUGGER_TEST(regression_l145);
-DEBUGGER_TEST(stack_vars);
+REPL_TEST(regression_9918);
+REPL_TEST(regression_9917);
+REPL_TEST(stack_vars);
+REPL_TEST(basic_repl);
 
-};
+}; // namespace nix

tests/functional/repl_characterization/test-session.cc

@@ -1,9 +1,10 @@
 #include <iostream>
+#include <span>
 #include <unistd.h>
 
 #include "test-session.hh"
 #include "util.hh"
-#include "tests/debug-char.hh"
+#include "debug-char.hh"
 
 namespace nix {
 
@@ -21,14 +22,17 @@ RunningProcess RunningProcess::start(std::string executable, Strings args)
 
     // This is separate from runProgram2 because we have different IO requirements
     pid_t pid = startProcess([&]() {
-        if (dup2(procStdout.writeSide.get(), STDOUT_FILENO) == -1)
+        if (dup2(procStdout.writeSide.get(), STDOUT_FILENO) == -1) {
             throw SysError("dupping stdout");
-        if (dup2(procStdin.readSide.get(), STDIN_FILENO) == -1)
+        }
+        if (dup2(procStdin.readSide.get(), STDIN_FILENO) == -1) {
             throw SysError("dupping stdin");
+        }
         procStdin.writeSide.close();
         procStdout.readSide.close();
-        if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1)
+        if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) {
             throw SysError("dupping stderr");
+        }
         execv(executable.c_str(), stringsToCharPtrs(args).data());
         throw SysError("exec did not happen");
     });
@@ -44,7 +48,8 @@ RunningProcess RunningProcess::start(std::string executable, Strings args)
 }
 
 [[gnu::unused]]
-std::ostream & operator<<(std::ostream & os, ReplOutputParser::State s)
+std::ostream &
+operator<<(std::ostream & os, ReplOutputParser::State s)
 {
     switch (s) {
     case ReplOutputParser::State::Prompt:
@@ -91,8 +96,7 @@ bool ReplOutputParser::feed(char c)
     return false;
 }
 
-/** Waits for the prompt and then returns if a prompt was found */
-bool TestSession::waitForPrompt()
+bool TestSession::readOutThen(ReadOutThenCallback cb)
 {
     std::vector<char> buf(1024);
 
@@ -106,38 +110,67 @@ bool TestSession::waitForPrompt()
             return false;
         }
 
+        switch (cb(std::span(buf.data(), res))) {
+        case ReadOutThenCallbackResult::Stop:
+            return true;
+        case ReadOutThenCallbackResult::Continue:
+            continue;
+        }
+    }
+}
+
+bool TestSession::waitForPrompt()
+{
+    bool notEof = readOutThen([&](std::span<char> s) -> ReadOutThenCallbackResult {
         bool foundPrompt = false;
-        for (ssize_t i = 0; i < res; ++i) {
+
+        for (auto ch : s) {
             // foundPrompt = foundPrompt || outputParser.feed(buf[i]);
             bool wasEaten = true;
-            eater.feed(buf[i], [&](char c) {
+            eater.feed(ch, [&](char c) {
                 wasEaten = false;
-                foundPrompt = outputParser.feed(buf[i]) || foundPrompt;
+                foundPrompt = outputParser.feed(ch) || foundPrompt;
 
                 outLog.push_back(c);
             });
 
             if constexpr (DEBUG_REPL_PARSER) {
-                std::cerr << "raw " << DebugChar{buf[i]} << (wasEaten ? " [eaten]" : "") << "\n";
+                std::cerr << "raw " << DebugChar{ch} << (wasEaten ? " [eaten]" : "") << "\n";
             }
         }
 
-        if (foundPrompt) {
-            return true;
-        }
+        return foundPrompt ? ReadOutThenCallbackResult::Stop : ReadOutThenCallbackResult::Continue;
+    });
+
+    return notEof;
+}
+
+void TestSession::wait()
+{
+    readOutThen([&](std::span<char> s) {
+        for (auto ch : s) {
+            eater.feed(ch, [&](char c) {
+                outputParser.feed(c);
+                outLog.push_back(c);
+            });
         }
+        // just keep reading till we hit eof
+        return ReadOutThenCallbackResult::Continue;
+    });
 }
 
 void TestSession::close()
 {
     proc.procStdin.close();
+    wait();
     proc.procStdout.close();
 }
 
 void TestSession::runCommand(std::string command)
 {
-    if constexpr (DEBUG_REPL_PARSER)
+    if constexpr (DEBUG_REPL_PARSER) {
         std::cerr << "runCommand " << command << "\n";
+    }
     command += "\n";
     // We have to feed a newline into the output parser, since Nix might not
     // give us a newline before a prompt in all cases (it might clear line

tests/functional/repl_characterization/test-session.hh

@@ -1,7 +1,9 @@
 #pragma once
 ///@file
 
+#include <functional>
 #include <sched.h>
+#include <span>
 #include <string>
 
 #include "util.hh"
@@ -22,8 +24,7 @@ struct RunningProcess
 class ReplOutputParser
 {
 public:
-    ReplOutputParser(std::string prompt)
-        : prompt(prompt)
+    ReplOutputParser(std::string prompt) : prompt(prompt)
     {
         assert(!prompt.empty());
     }
@@ -60,10 +61,27 @@ struct TestSession
     {
     }
 
+    /** Waits for the prompt and then returns if a prompt was found */
     bool waitForPrompt();
 
+    /** Feeds a line of input into the command */
     void runCommand(std::string command);
 
+    /** Closes the session, closing standard input and waiting for standard
+     * output to close, capturing any remaining output. */
     void close();
+
+private:
+    /** Waits until the command closes its output */
+    void wait();
+
+    enum class ReadOutThenCallbackResult { Stop, Continue };
+    using ReadOutThenCallback = std::function<ReadOutThenCallbackResult(std::span<char>)>;
+    /** Reads some chunks of output, calling the callback provided for each
+     * chunk and stopping if it returns Stop.
+     *
+     * @returns false if EOF, true if the callback requested we stop first.
+     * */
+    bool readOutThen(ReadOutThenCallback cb);
 };
 };

tests/unit/libutil-support/local.mk

@@ -8,7 +8,7 @@ libutil-test-support_INSTALL_DIR :=
 
 libutil-test-support_SOURCES := $(wildcard $(d)/tests/*.cc)
 
-libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES)
+libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) -I src/libutil
 
 # libexpr so we can steal their string printer from print.cc
 libutil-test-support_LIBS = libutil libexpr

tests/unit/libutil-support/tests/characterization.hh

@@ -74,20 +74,20 @@ public:
     {
         auto file = goldenMaster(testStem);
 
-        auto got = test();
+        auto actual = test();
 
         if (testAccept())
         {
             createDirs(dirOf(file));
-            writeFile2(file, got);
+            writeFile2(file, actual);
             GTEST_SKIP()
                 << "Updating golden master "
                 << file;
         }
         else
         {
-            decltype(got) expected = readFile2(file);
-            ASSERT_EQ(got, expected);
+            decltype(actual) expected = readFile2(file);
+            ASSERT_EQ(expected, actual);
         }
     }
 

tests/unit/libutil-support/tests/cli-literate-parser.cc

@@ -1,246 +1,447 @@
-#include "cli-literate-parser.hh"
-#include "libexpr/print.hh"
-#include "debug-char.hh"
-#include "types.hh"
-#include "util.hh"
+#include <boost/algorithm/string/replace.hpp>
+#include <boost/algorithm/string/trim.hpp>
 #include <iostream>
 #include <memory>
-#include <boost/algorithm/string/trim.hpp>
+#include <ranges>
+#include <sstream>
+#include <variant>
 
-using namespace std::string_literals;
-
-namespace nix {
+#include "cli-literate-parser.hh"
+#include "escape-string.hh"
+#include "fmt.hh"
+#include "libexpr/print.hh"
+#include "shlex.hh"
+#include "types.hh"
+#include "util.hh"
 
 static constexpr const bool DEBUG_PARSER = false;
 
-constexpr auto CLILiterateParser::stateDebug(State const & s) -> const char *
-{
-    return std::visit(
-        overloaded{// clang-format off
-            [](Indent const&) -> const char * { return "indent"; },
-            [](Commentary const&) -> const char * { return "indent"; },
-            [](Prompt const&) -> const char * { return "prompt"; },
-            [](Command const&) -> const char * { return "command"; },
-            [](OutputLine const&) -> const char * { return "output_line"; }},
-        // clang-format on
-        s);
-}
+using namespace std::string_literals;
+using namespace boost::algorithm;
 
-auto CLILiterateParser::Node::print() const -> std::string
-{
-    std::ostringstream s{};
-    switch (kind) {
-    case NodeKind::COMMENTARY:
-        s << "Commentary ";
-        break;
-    case NodeKind::COMMAND:
-        s << "Command ";
-        break;
-    case NodeKind::OUTPUT:
-        s << "Output ";
-        break;
-    }
-    printLiteralString(s, this->text);
-    return s.str();
-}
+namespace nix {
 
-void PrintTo(std::vector<CLILiterateParser::Node> const & nodes, std::ostream * os)
-{
-    for (auto & node : nodes) {
-        *os << node.print() << "\\n";
-    }
-}
+namespace cli_literate_parser {
 
-auto CLILiterateParser::parse(std::string prompt, std::string_view const & input, size_t indent) -> std::vector<Node>
-{
-    CLILiterateParser p{std::move(prompt), indent};
-    p.feed(input);
-    return std::move(p).intoSyntax();
-}
-
-auto CLILiterateParser::intoSyntax() && -> std::vector<Node>
-{
-    return std::move(this->syntax_);
-}
-
-CLILiterateParser::CLILiterateParser(std::string prompt, size_t indent)
-    : state_(indent == 0 ? State(Prompt{}) : State(Indent{}))
-    , prompt_(prompt)
-    , indent_(indent)
-    , lastWasOutput_(false)
-    , syntax_{}
+struct Parser
 {
+    Parser(const std::string input, Config config)
+        : input(input)
+        , rest(std::string_view(input.begin(), input.end()))
+        , prompt(config.prompt)
+        , indentString(std::string(config.indent, ' '))
+        , lastWasOutput(false)
+        , syntax{}
+    {
         assert(!prompt.empty());
-}
-
-void CLILiterateParser::feed(char c)
-{
-    if constexpr (DEBUG_PARSER) {
-        std::cout << stateDebug(state_) << " " << DebugChar{c} << "\n";
     }
 
-    if (c == '\n') {
-        onNewline();
+    const std::string input;
+    std::string_view rest;
+    const std::string prompt;
+    const std::string indentString;
+
+    /** Last line was output, so we consider a blank to be part of the output */
+    bool lastWasOutput;
+
+    /**
+     * Nodes of syntax being built.
+     */
+    std::vector<Node> syntax;
+
+    auto dbg(std::string_view state) -> void
+    {
+        std::cout << state << ": ";
+        escapeString(
+            std::cout,
+            rest,
+            {
+                .maxLength = 40,
+                .ansiColors = true,
+                .escapeNonPrinting = true,
+            }
+        );
+        std::cout << std::endl;
+    }
+
+    template<typename T>
+    auto pushNode(T node) -> void
+    {
+        if constexpr (DEBUG_PARSER) {
+            std::cout << debugNode(node);
+        }
+        syntax.emplace_back(node);
+    }
+
+    auto parseLiteral(const char c) -> bool
+    {
+        if (!rest.empty() && rest[0] == c) {
+            rest.remove_prefix(1);
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    auto parseLiteral(const std::string_view & literal) -> bool
+    {
+        if (rest.starts_with(literal)) {
+            rest.remove_prefix(literal.length());
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    auto parseBool() -> bool
+    {
+        if (rest.starts_with("true")) {
+            return true;
+        } else if (rest.starts_with("false")) {
+            return false;
+        } else {
+            throw ParseError("true or false", std::string(rest));
+        }
+        auto untilNewline = parseUntilNewline();
+        if (!untilNewline.empty()) {
+            throw ParseError("nothing after true or false", std::string(rest));
+        }
+    }
+
+    auto parseUntilNewline() -> std::string
+    {
+        auto pos = rest.find('\n');
+        if (pos == std::string_view::npos) {
+            throw ParseError("text and then newline", std::string(rest));
+        } else {
+            // `parseOutput()` sets this to true anyways.
+            lastWasOutput = false;
+            auto result = std::string(rest, 0, pos);
+            rest.remove_prefix(pos + 1);
+            return result;
+        }
+    }
+
+    auto parseIndent() -> bool
+    {
+        if constexpr (DEBUG_PARSER) {
+            dbg("indent");
+        }
+        if (indentString.empty()) {
+            return true;
+        }
+
+        if (parseLiteral(indentString)) {
+            pushNode(Indent(indentString));
+            return true;
+        } else {
+            if constexpr (DEBUG_PARSER) {
+                dbg("indent failed");
+            }
+            return false;
+        }
+    }
+
+    auto parseCommand() -> void
+    {
+        if constexpr (DEBUG_PARSER) {
+            dbg("command");
+        }
+        auto untilNewline = parseUntilNewline();
+        pushNode(Command(untilNewline));
+        parseStartOfLine();
+    }
+
+    auto parsePrompt() -> void
+    {
+        if constexpr (DEBUG_PARSER) {
+            dbg("prompt");
+        }
+        if (parseLiteral(prompt)) {
+            pushNode(Prompt(prompt));
+            if (rest.empty()) {
+                return;
+            }
+            parseCommand();
+        } else {
+            parseOutput();
+        }
+    }
+
+    auto parseOutput() -> void
+    {
+        if constexpr (DEBUG_PARSER) {
+            dbg("output");
+        }
+        auto untilNewline = parseUntilNewline();
+        pushNode(Output(untilNewline));
+        lastWasOutput = true;
+        parseStartOfLine();
+    }
+
+    auto parseAtSign() -> void
+    {
+        if constexpr (DEBUG_PARSER) {
+            dbg("@ symbol");
+        }
+        if (!parseLiteral('@')) {
+            parseOutputOrCommentary();
+        }
+
+        if (parseLiteral("args ")) {
+            parseArgs();
+        } else if (parseLiteral("should-start ")) {
+            if constexpr (DEBUG_PARSER) {
+                dbg("@should-start");
+            }
+            auto shouldStart = parseBool();
+            pushNode(ShouldStart{shouldStart});
+            parseStartOfLine();
+        }
+    }
+
+    auto parseArgs() -> void
+    {
+        if constexpr (DEBUG_PARSER) {
+            dbg("@args");
+        }
+        auto untilNewline = parseUntilNewline();
+        pushNode(Args(untilNewline));
+        parseStartOfLine();
+    }
+
+    auto parseOutputOrCommentary() -> void
+    {
+        if constexpr (DEBUG_PARSER) {
+            dbg("output/commentary");
+        }
+        auto oldLastWasOutput = lastWasOutput;
+        auto untilNewline = parseUntilNewline();
+
+        auto trimmed = trim_right_copy(untilNewline);
+
+        if (oldLastWasOutput && trimmed.empty()) {
+            pushNode(Output{trimmed});
+        } else {
+            pushNode(Commentary{untilNewline});
+        }
+        parseStartOfLine();
+    }
+
+    auto parseStartOfLine() -> void
+    {
+        if constexpr (DEBUG_PARSER) {
+            dbg("start of line");
+        }
+        if (rest.empty()) {
             return;
         }
 
+        if (parseIndent()) {
+            parsePrompt();
+        } else {
+            parseAtSign();
+        }
+    }
+
+    auto parse() && -> ParseResult
+    {
+        // Begin the recursive descent parser at the start of a new line.
+        parseStartOfLine();
+        if (!rest.empty()) {
+            throw ParseError("all input to be parsed", std::string(rest));
+        }
+        return std::move(*this).intoParseResult();
+    }
+
+    auto intoParseResult() && -> ParseResult
+    {
+        // Do another pass over the nodes to produce auxiliary results like parsed
+        // command line arguments.
+        std::vector<std::string> args;
+        std::vector<Node> newSyntax;
+        auto shouldStart = true;
+
+        for (auto it = syntax.begin(); it != syntax.end(); ++it) {
+            Node node = std::move(*it);
             std::visit(
                 overloaded{
-            [&](Indent & s) {
-                if (c == ' ') {
-                    if (++s.pos >= indent_) {
-                        transition(Prompt{});
-                    }
-                } else {
-                    transition(Commentary{AccumulatingState{.lineAccumulator = std::string{c}}});
-                }
+                    [&](Args & e) {
+                        auto split = shell_split(std::string(e.text));
+                        args.insert(args.end(), split.begin(), split.end());
                     },
-            [&](Prompt & s) {
-                if (s.pos >= prompt_.length()) {
-                    transition(Command{AccumulatingState{.lineAccumulator = std::string{c}}});
-                    return;
-                } else if (c == prompt_[s.pos]) {
-                    // good prompt character
-                    ++s.pos;
-                } else {
-                    // didn't match the prompt, so it must have actually been output.
-                    s.lineAccumulator.push_back(c);
-                    transition(OutputLine{AccumulatingState{.lineAccumulator = std::move(s.lineAccumulator)}});
-                    return;
-                }
-                s.lineAccumulator.push_back(c);
+                    [&](ShouldStart & e) { shouldStart = e.shouldStart; },
+                    [&](auto & e) {},
                 },
-            [&](AccumulatingState & s) { s.lineAccumulator.push_back(c); }},
-        state_);
-}
+                node
+            );
 
-void CLILiterateParser::onNewline()
+            newSyntax.push_back(node);
+        }
+
+        return ParseResult{
+            .syntax = std::move(newSyntax),
+            .args = std::move(args),
+            .shouldStart = shouldStart,
+        };
+    }
+};
+
+template<typename View>
+auto tidySyntax(View syntax) -> std::vector<Node>
 {
-    State lastState = std::move(state_);
-    bool newLastWasOutput = false;
+    // Note: Setting `lastWasCommand` lets us trim blank lines at the start and
+    // end of the output stream.
+    auto lastWasCommand = true;
+    std::vector<Node> newSyntax;
 
-    syntax_.push_back(std::visit(
+    for (auto it = syntax.begin(); it != syntax.end(); ++it) {
+        Node node = *it;
+        // Only compare `Command` and `Output` nodes.
+        if (std::visit([&](auto && e) { return !e.shouldCompare(); }, node)) {
+            continue;
+        }
+
+        auto shouldKeep = true;
+
+        // Remove blank lines before and after commands. This lets us keep nice
+        // whitespace in the test files.
+        std::visit(
             overloaded{
-            [&](Indent & s) {
-                // XXX: technically this eats trailing spaces
-
-                // a newline following output is considered part of that output
-                if (lastWasOutput_) {
-                    newLastWasOutput = true;
-                    return Node::mkOutput("");
+                [&](Command & e) {
+                    lastWasCommand = true;
+                    auto trimmed = trim_right_copy(e.text);
+                    if (trimmed.empty()) {
+                        shouldKeep = false;
+                    } else {
+                        e.text = trimmed;
                     }
-                return Node::mkCommentary("");
                 },
-            [&](Commentary & s) { return Node::mkCommentary(std::move(s.lineAccumulator)); },
-            [&](Command & s) { return Node::mkCommand(std::move(s.lineAccumulator)); },
-            [&](OutputLine & s) {
-                newLastWasOutput = true;
-                return Node::mkOutput(std::move(s.lineAccumulator));
+                [&](Output & e) {
+                    std::string trimmed = trim_right_copy(e.text);
+                    if (lastWasCommand && trimmed.empty()) {
+                        // NB: Keep `lastWasCommand` true in this branch so we
+                        // can keep pruning empty output lines.
+                        shouldKeep = false;
+                    } else {
+                        e.text = trimmed;
+                        lastWasCommand = false;
+                    }
                 },
-            [&](Prompt & s) {
-                // INDENT followed by newline is also considered a blank output line
-                return Node::mkOutput(std::move(s.lineAccumulator));
-            }},
-        lastState));
+                [&](auto & e) {
+                    lastWasCommand = false;
+                    shouldKeep = false;
+                },
+            },
+            node
+        );
 
-    transition(Indent{});
-    lastWasOutput_ = newLastWasOutput;
-}
-
-void CLILiterateParser::feed(std::string_view s)
-{
-    for (char ch : s) {
-        feed(ch);
-    }
-}
-
-void CLILiterateParser::transition(State new_state)
-{
-    // When we expect INDENT and we are parsing without indents, commentary
-    // cannot exist, so we want to transition directly into PROMPT before
-    // resuming normal processing.
-    if (Indent * i = std::get_if<Indent>(&new_state); i != nullptr && indent_ == 0) {
-        new_state = Prompt{AccumulatingState{}, i->pos};
-    }
-
-    state_ = new_state;
-}
-
-auto CLILiterateParser::syntax() const -> std::vector<Node> const &
-{
-    return syntax_;
-}
-
-auto CLILiterateParser::unparse(const std::string & prompt, const std::vector<Node> & syntax, size_t indent)
-    -> std::string
-{
-    std::string indent_str(indent, ' ');
-    std::ostringstream out{};
-
-    for (auto & node : syntax) {
-        switch (node.kind) {
-        case NodeKind::COMMENTARY:
-            out << node.text << "\n";
-            break;
-        case NodeKind::COMMAND:
-            out << indent_str << prompt << node.text << "\n";
-            break;
-        case NodeKind::OUTPUT:
-            out << indent_str << node.text << "\n";
-            break;
+        if (shouldKeep) {
+            newSyntax.push_back(node);
         }
     }
 
-    return out.str();
-}
-
-auto CLILiterateParser::tidyOutputForComparison(std::vector<Node> && syntax) -> std::vector<Node>
-{
-    std::vector<Node> newSyntax{};
-
-    // Eat trailing newlines, so assume that the very end was actually a command
-    bool lastWasCommand = true;
-    bool newLastWasCommand = true;
-
-    auto v = std::ranges::reverse_view(syntax);
-
-    for (auto it = v.begin(); it != v.end(); ++it) {
-        Node item = std::move(*it);
-
-        lastWasCommand = newLastWasCommand;
-        // chomp commentary
-        if (item.kind == NodeKind::COMMENTARY) {
-            continue;
-        }
-
-        if (item.kind == NodeKind::COMMAND) {
-            newLastWasCommand = true;
-
-            if (item.text == "") {
-                // chomp empty commands
-                continue;
-            }
-        }
-
-        if (item.kind == NodeKind::OUTPUT) {
-            // TODO: horrible
-            bool nextIsCommand = (it + 1 == v.end()) ? false : (it + 1)->kind == NodeKind::COMMAND;
-            std::string trimmedText = boost::algorithm::trim_right_copy(item.text);
-            if ((lastWasCommand || nextIsCommand) && trimmedText == "") {
-                // chomp empty text above or directly below commands
-                continue;
-            }
-
-            // real output, stop chomping
-            newLastWasCommand = false;
-
-            item = Node::mkOutput(std::move(trimmedText));
-        }
-        newSyntax.push_back(std::move(item));
-    }
-
-    std::reverse(newSyntax.begin(), newSyntax.end());
     return newSyntax;
 }
 
-};
+auto ParseResult::tidyOutputForComparison() -> std::vector<Node>
+{
+    auto reversed = tidySyntax(std::ranges::reverse_view(syntax));
+    auto unreversed = tidySyntax(std::ranges::reverse_view(reversed));
+    return unreversed;
+}
+
+void ParseResult::interpolatePwd(std::string_view pwd)
+{
+    std::vector<std::string> newArgs;
+    for (auto & arg : args) {
+        newArgs.push_back(replaceStrings(arg, "$PWD", pwd));
+    }
+    args = std::move(newArgs);
+}
+
+const char * ParseError::what() const noexcept
+{
+    if (what_) {
+        return what_->c_str();
+    } else {
+        std::ostringstream escaped;
+        escapeString(escaped, rest, {.maxLength = 256, .escapeNonPrinting = true});
+        auto hint =
+            new HintFmt("Parse error: Expected %1%, got:\n%2%", expected, Uncolored(escaped.str()));
+        what_ = hint->str();
+        return what_->c_str();
+    }
+}
+
+auto parse(const std::string input, Config config) -> ParseResult
+{
+    return Parser(input, config).parse();
+}
+
+std::ostream & operator<<(std::ostream & output, const Args & node)
+{
+    return output << "@args " << node.text;
+}
+
+std::ostream & operator<<(std::ostream & output, const ShouldStart & node)
+{
+    return output << "@should-start " << (node.shouldStart ? "true" : "false");
+}
+
+std::ostream & operator<<(std::ostream & output, const TextNode & rhs)
+{
+    return output << rhs.text;
+}
+
+void unparseNode(std::ostream & output, const Node & node, bool withNewline)
+{
+    std::visit(
+        [&](const auto & n) { output << n << (withNewline && n.emitNewlineAfter() ? "\n" : ""); },
+        node
+    );
+}
+
+template<typename T>
+std::string gtestFormat(T & value)
+{
+    std::ostringstream formatted;
+    unparseNode(formatted, value, true);
+    auto str = formatted.str();
+    // Needs to be the literal string `\n` and not a newline character to
+    // trigger gtest diff printing. Yes seriously.
+    boost::algorithm::replace_all(str, "\n", "\\n");
+    return str;
+}
+
+void PrintTo(const std::vector<Node> & nodes, std::ostream * output)
+{
+    for (auto & node : nodes) {
+        *output << gtestFormat(node);
+    }
+}
+
+std::string debugNode(const Node & node)
+{
+    std::ostringstream output;
+    output << std::visit([](const auto & n) { return n.kind(); }, node) << ": ";
+    std::ostringstream contents;
+    unparseNode(contents, node, false);
+    escapeString(output, contents.str(), {.escapeNonPrinting = true});
+    return output.str();
+}
+
+auto ParseResult::debugPrint(std::ostream & output) -> void
+{
+    ::nix::cli_literate_parser::debugPrint(output, syntax);
+}
+
+void debugPrint(std::ostream & output, std::vector<Node> & nodes)
+{
+    for (auto & node : nodes) {
+        output << debugNode(node) << std::endl;
+    }
+}
+
+} // namespace cli_literate_parser
+} // namespace nix

tests/unit/libutil-support/tests/cli-literate-parser.hh

@@ -3,132 +3,197 @@
 
 #include <compare>
 #include <memory>
+#include <optional>
 #include <sstream>
+#include <string>
 #include <variant>
 #include <vector>
-#include <string>
 
 namespace nix {
+namespace cli_literate_parser {
+
+// ------------------------- NODES -------------------------
+//
+// To update golden test files while preserving commentary output and other `@`
+// directives, we need to keep commentary output around after parsing.
+
+struct BaseNode {
+  virtual ~BaseNode() = default;
+  BaseNode() {}
+  BaseNode(const BaseNode &) = default;
+
+  virtual auto shouldCompare() const -> bool { return false; }
+
+  virtual auto kind() const -> std::string = 0;
+  virtual auto emitNewlineAfter() const -> bool = 0;
+
+  auto operator<=>(const BaseNode &rhs) const = default;
+};
+
+/**
+ * A node containing text. The text should be identical to how the node was
+ * written in the input file.
+ */
+struct TextNode : BaseNode {
+  std::string text;
+
+  TextNode(std::string text) : BaseNode(), text(text) {}
+};
+
+std::ostream &operator<<(std::ostream &output, const TextNode &node);
+
+#define DECLARE_TEXT_NODE(NAME, NEEDS_NEWLINE, SHOULD_COMPARE)                 \
+  struct NAME : TextNode {                                                     \
+    NAME(std::string text) : TextNode(text) {}                                 \
+    ~NAME() override = default;                                                \
+                                                                               \
+    auto kind() const -> std::string override { return #NAME; }                \
+    auto emitNewlineAfter() const -> bool override { return NEEDS_NEWLINE; }   \
+    auto shouldCompare() const -> bool override { return SHOULD_COMPARE; }     \
+  };
+
+/* name, needsNewline, shouldCompare */
+DECLARE_TEXT_NODE(Prompt, false, false)
+DECLARE_TEXT_NODE(Command, true, true)
+DECLARE_TEXT_NODE(Output, true, true)
+DECLARE_TEXT_NODE(Commentary, true, false)
+DECLARE_TEXT_NODE(Args, true, false)
+DECLARE_TEXT_NODE(Indent, false, false)
+
+#undef DECLARE_TEXT_NODE
+
+struct ShouldStart : BaseNode {
+  bool shouldStart;
+
+  ShouldStart(bool shouldStart) : shouldStart(shouldStart) {}
+  ~ShouldStart() override = default;
+  auto emitNewlineAfter() const -> bool override { return true; }
+  auto kind() const -> std::string override { return "should-start"; }
+
+  auto operator<=>(const ShouldStart &rhs) const = default;
+};
+std::ostream &operator<<(std::ostream &output, const ShouldStart &node);
+
+/**
+ * Any syntax node, including those that are cosmetic.
+ */
+using Node = std::variant<Prompt, Command, Output, Commentary, Args,
+                          ShouldStart, Indent>;
+
+/** Unparses a node into the exact text that would have created it, including a
+ * newline at the end if present, if withNewline is set */
+void unparseNode(std::ostream &output, const Node &node,
+                 bool withNewline = true);
+
+std::string debugNode(const Node &node);
+void debugPrint(std::ostream &output, std::vector<Node> &nodes);
+
+/**
+ * Override gtest printing for lists of nodes.
+ */
+void PrintTo(std::vector<Node> const &nodes, std::ostream *output);
+
+/**
+ * The result of parsing a test file.
+ */
+struct ParseResult {
+  /**
+   * A set of nodes that can be used to reproduce the input file. This is used
+   * to implement updating the test files.
+   */
+  std::vector<Node> syntax;
+
+  /**
+   * Extra CLI arguments.
+   */
+  std::vector<std::string> args;
+
+  /**
+   * Should the program start successfully?
+   */
+  bool shouldStart = false;
+
+  /**
+   * Replace `$PWD` with the given value in `args`.
+   */
+  void interpolatePwd(std::string_view pwd);
+
+  /**
+   * Tidy `syntax` to remove unnecessary nodes.
+   */
+  auto tidyOutputForComparison() -> std::vector<Node>;
+
+  auto debugPrint(std::ostream &output) -> void;
+};
+
+/**
+ * A parse error.
+ */
+struct ParseError : std::exception {
+  std::string expected;
+  std::string rest;
+
+  ParseError(std::string expected, std::string rest)
+      : expected(expected), rest(rest) {}
+
+  const char *what() const noexcept override;
+
+private:
+  /**
+   * Cached formatted contents of `what()`.
+   */
+  mutable std::optional<std::string> what_;
+};
+
+struct Config {
+  /**
+   * The prompt string to look for.
+   */
+  std::string prompt;
+  /**
+   * The number of spaces of indent for commands and output.
+   */
+  size_t indent = 2;
+};
+
 /*
- * A DFA parser for literate test cases for CLIs.
+ * A recursive descent parser for literate test cases for CLIs.
  *
  * FIXME: implement merging of these, so you can auto update cases that have
  * comments.
  *
- * Format:
- * COMMENTARY
- * INDENT PROMPT COMMAND
- * INDENT OUTPUT
+ * Syntax:
+ * ```
+ * ( COMMENTARY
+ * | INDENT PROMPT COMMAND
+ * | INDENT OUTPUT
+ * | @args ARGS
+ * | @should-start ( true | false )) *
+ * ```
  *
  * e.g.
+ * ```
  * commentary commentary commentary
+ * @args --foo
+ * @should-start false
  *   nix-repl> :t 1
  *   an integer
+ * ```
  *
- * Yields:
+ * Yields something like:
+ * ```
  * Commentary "commentary commentary commentary"
+ * Args "--foo"
+ * ShouldStart false
  * Command ":t 1"
  * Output "an integer"
+ * ```
  *
  * Note: one Output line is generated for each line of the sources, because
  * this is effectively necessary to be able to align them in the future to
  * auto-update tests.
  */
-class CLILiterateParser
-{
-public:
+auto parse(std::string input, Config config) -> ParseResult;
 
-    enum class NodeKind {
-        COMMENTARY,
-        COMMAND,
-        OUTPUT,
-    };
-
-    struct Node
-    {
-        NodeKind kind;
-        std::string text;
-        std::strong_ordering operator<=>(Node const &) const = default;
-
-        static Node mkCommentary(std::string text)
-        {
-            return Node{.kind = NodeKind::COMMENTARY, .text = text};
-        }
-
-        static Node mkCommand(std::string text)
-        {
-            return Node{.kind = NodeKind::COMMAND, .text = text};
-        }
-
-        static Node mkOutput(std::string text)
-        {
-            return Node{.kind = NodeKind::OUTPUT, .text = text};
-        }
-
-        auto print() const -> std::string;
-    };
-
-    CLILiterateParser(std::string prompt, size_t indent = 2);
-
-    auto syntax() const -> std::vector<Node> const &;
-
-    /** Feeds a character into the parser */
-    void feed(char c);
-
-    /** Feeds a string into the parser */
-    void feed(std::string_view s);
-
-    /** Parses an input in a non-streaming fashion */
-    static auto parse(std::string prompt, std::string_view const & input, size_t indent = 2) -> std::vector<Node>;
-
-    /** Returns, losslessly, the string that would have generated a syntax tree */
-    static auto unparse(std::string const & prompt, std::vector<Node> const & syntax, size_t indent = 2) -> std::string;
-
-    /** Consumes a CLILiterateParser and gives you the syntax out of it */
-    auto intoSyntax() && -> std::vector<Node>;
-
-    /** Tidies syntax to remove trailing whitespace from outputs and remove any
-     * empty prompts */
-    static auto tidyOutputForComparison(std::vector<Node> && syntax) -> std::vector<Node>;
-
-private:
-
-    struct AccumulatingState
-    {
-        std::string lineAccumulator;
-    };
-    struct Indent
-    {
-        size_t pos = 0;
-    };
-    struct Commentary : public AccumulatingState
-    {};
-    struct Prompt : AccumulatingState
-    {
-        size_t pos = 0;
-    };
-    struct Command : public AccumulatingState
-    {};
-    struct OutputLine : public AccumulatingState
-    {};
-
-    using State = std::variant<Indent, Commentary, Prompt, Command, OutputLine>;
-    State state_;
-
-    constexpr static auto stateDebug(State const&) -> const char *;
-
-    const std::string prompt_;
-    const size_t indent_;
-
-    /** Last line was output, so we consider a blank to be part of the output */
-    bool lastWasOutput_;
-
-    std::vector<Node> syntax_;
-
-    void transition(State newState);
-    void onNewline();
-};
-
-// Override gtest printing for lists of nodes
-void PrintTo(std::vector<CLILiterateParser::Node> const & nodes, std::ostream * os);
-};
+}; // namespace cli_literate_parser
+}; // namespace nix

tests/unit/libutil-support/tests/debug-char.hh

@@ -1,24 +0,0 @@
-///@file
-#include <ostream>
-#include <boost/io/ios_state.hpp>
-
-namespace nix {
-
-struct DebugChar
-{
-    char c;
-};
-
-inline std::ostream & operator<<(std::ostream & s, DebugChar c)
-{
-    boost::io::ios_flags_saver _ifs(s);
-
-    if (isprint(c.c)) {
-        s << static_cast<char>(c.c);
-    } else {
-        s << std::hex << "0x" << (static_cast<unsigned int>(c.c) & 0xff);
-    }
-    return s;
-}
-
-}

tests/unit/libutil/escape-string.cc

@@ -0,0 +1,35 @@
+#include "escape-string.hh"
+#include "ansicolor.hh"
+#include <gtest/gtest.h>
+
+namespace nix {
+
+TEST(EscapeString, simple) {
+  auto escaped = escapeString("puppy");
+  ASSERT_EQ(escaped, "\"puppy\"");
+}
+
+TEST(EscapeString, escaping) {
+  auto escaped = escapeString("\n\r\t \" \\ ${ooga booga}");
+  ASSERT_EQ(escaped, "\"\\n\\r\\t \\\" \\\\ \\${ooga booga}\"");
+}
+
+TEST(EscapeString, maxLength) {
+  auto escaped = escapeString("puppy", {.maxLength = 5});
+  ASSERT_EQ(escaped, "\"puppy\"");
+
+  escaped = escapeString("puppy doggy", {.maxLength = 5});
+  ASSERT_EQ(escaped, "\"puppy\" «6 bytes elided»");
+}
+
+TEST(EscapeString, ansiColors) {
+  auto escaped = escapeString("puppy doggy", {.maxLength = 5, .ansiColors = true});
+  ASSERT_EQ(escaped, ANSI_MAGENTA "\"puppy\" " ANSI_FAINT "«6 bytes elided»" ANSI_NORMAL);
+}
+
+TEST(EscapeString, escapeNonPrinting) {
+  auto escaped = escapeString("puppy\u0005doggy", {.escapeNonPrinting = true});
+  ASSERT_EQ(escaped, "\"puppy\\x05doggy\"");
+}
+
+} // namespace nix

tests/unit/libutil/paths_setting.cc

@@ -0,0 +1,72 @@
+#include "config.hh"
+
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+#include <sstream>
+
+using testing::Eq;
+
+namespace nix {
+
+class PathsSettingTestConfig : public Config
+{
+public:
+    PathsSettingTestConfig()
+        : Config()
+    { }
+
+    PathsSetting paths{this, Paths(), "paths", "documentation"};
+};
+
+struct PathsSettingTest : public ::testing::Test {
+public:
+    PathsSettingTestConfig mkConfig()
+    {
+        return PathsSettingTestConfig();
+    }
+};
+
+TEST_F(PathsSettingTest, parse) {
+    auto config = mkConfig();
+    // Not an absolute path:
+    ASSERT_THROW(config.paths.parse("puppy.nix"), Error);
+
+    ASSERT_THAT(
+        config.paths.parse("/puppy.nix"),
+        Eq<Paths>({"/puppy.nix"})
+    );
+
+    // Splits on whitespace:
+    ASSERT_THAT(
+        config.paths.parse("/puppy.nix /doggy.nix"),
+        Eq<Paths>({"/puppy.nix", "/doggy.nix"})
+    );
+
+    // Splits on _any_ whitespace:
+    ASSERT_THAT(
+        config.paths.parse("/puppy.nix \t  /doggy.nix\n\n\n/borzoi.nix\r/goldie.nix"),
+        Eq<Paths>({"/puppy.nix", "/doggy.nix", "/borzoi.nix", "/goldie.nix"})
+    );
+
+    // Canonicizes paths:
+    ASSERT_THAT(
+        config.paths.parse("/puppy/../doggy.nix"),
+        Eq<Paths>({"/doggy.nix"})
+    );
+}
+
+TEST_F(PathsSettingTest, bool) {
+    auto config = mkConfig();
+    // No paths:
+    ASSERT_FALSE(config.paths);
+    // Set a path:
+    config.set("paths", "/puppy.nix");
+    // Now there are paths:
+    ASSERT_TRUE(config.paths);
+
+    // Multiple paths count too:
+    config.set("paths", "/puppy.nix /doggy.nix");
+    ASSERT_TRUE(config.paths);
+}
+
+} // namespace nix

tests/unit/libutil/shlex.cc

@@ -0,0 +1,57 @@
+#include "shlex.hh"
+
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+#include <sstream>
+
+using testing::Eq;
+
+namespace nix {
+
+TEST(Shlex, shell_split) {
+    ASSERT_THAT(shell_split(""), Eq<std::vector<std::string>>({}));
+    ASSERT_THAT(shell_split("  "), Eq<std::vector<std::string>>({}));
+
+    ASSERT_THAT(
+        shell_split("puppy doggy"),
+        Eq<std::vector<std::string>>({
+            "puppy",
+            "doggy",
+        })
+    );
+
+    ASSERT_THAT(
+        shell_split("goldie \"puppy 'doggy'\" sweety"),
+        Eq<std::vector<std::string>>({
+            "goldie",
+            "puppy 'doggy'",
+            "sweety",
+        })
+    );
+
+    ASSERT_THAT(
+        shell_split("\"pupp\\\"y\""),
+        Eq<std::vector<std::string>>({ "pupp\"y" })
+    );
+
+    ASSERT_THAT(
+        shell_split("goldie 'puppy' doggy"),
+        Eq<std::vector<std::string>>({
+            "goldie",
+            "puppy",
+            "doggy",
+        })
+    );
+
+    ASSERT_THAT(
+        shell_split("'pupp\\\"y'"),
+        Eq<std::vector<std::string>>({
+            "pupp\\\"y",
+        })
+    );
+
+    ASSERT_THROW(shell_split("\"puppy"), ShlexError);
+    ASSERT_THROW(shell_split("'puppy"), ShlexError);
+}
+
+} // namespace nix