Compare commits

..

6 commits

Author SHA1 Message Date
Rebecca Turner efb15e3dd5 Add repl-overlays
Adds a `repl-overlays` option, which specifies files that can overlay
and modify the top-level bindings in `nix repl`. For example, with the
following contents in `~/.config/nix/repl.nix`:

    info: final: prev: let
      optionalAttrs = predicate: attrs:
        if predicate
        then attrs
        else {};
    in
      optionalAttrs (prev ? legacyPackages && prev.legacyPackages ? ${info.currentSystem})
      {
        pkgs = prev.legacyPackages.${info.currentSystem};
      }

We can run `nix repl` and use `pkgs` to refer to `legacyPackages.${currentSystem}`:

    $ nix repl --repl-overlays ~/.config/nix/repl.nix nixpkgs
    Nix 2.21.0pre20240309_4111bb6
    Type :? for help.
    Loading installable 'flake:nixpkgs#'...
    Added 5 variables.
    Loading 'repl-overlays'...
    Added 6 variables.
    nix-repl> pkgs.bash
    «derivation /nix/store/g08b5vkwwh0j8ic9rkmd8mpj878rk62z-bash-5.2p26.drv»

Change-Id: Ic12e0f2f210b2f46e920c33088dfe1083f42391a
2024-03-29 16:26:36 -07:00
Rebecca Turner f3b71e1e62 Add PathsSetting
Change-Id: I1165f6ef033a5f757ca3716d3f8008ba36b01fd0
2024-03-29 16:26:35 -07:00
Rebecca Turner b1bfc20e56 Rewrite REPL test parser
- Use a recursive descent parser so that it's easy to extend.
- Add `@args` to enable customizing command-line arguments
- Add `@should-start` to enable `nix repl` tests that error before
  entering the REPL
- Make sure to read all stdout output before comparing. This catches
  some extra output we were tossing out before!

Change-Id: I5522555df4c313024ab15cd10f9f04e7293bda3a
2024-03-29 16:26:35 -07:00
Rebecca Turner f3f7d81988 Make repl_characterization.cc constants string_views
Change-Id: I96455ee365799ae87d9ab433cf102c691616a45e
2024-03-29 16:26:35 -07:00
Rebecca Turner c41ec4e64c Add EscapeStringOptions and escapeString tests
Change-Id: I86ead2f969c9e03c9edfa51bbc92ee06393fd7d6
2024-03-29 16:26:34 -07:00
Rebecca Turner a5a25894c1 Move escapeString to its own file
Change-Id: Ie5c954ec73c46c9d3c679ef99a83a29cc7a08352
2024-03-29 16:26:29 -07:00
43 changed files with 1516 additions and 496 deletions

View file

@ -0,0 +1,35 @@
---
synopsis: Add `repl-overlays` option
prs: 10203
---
A `repl-overlays` option has been added, which specifies files that can overlay
and modify the top-level bindings in `nix repl`. For example, with the
following contents in `~/.config/nix/repl.nix`:
```nix
info: final: prev: let
optionalAttrs = predicate: attrs:
if predicate
then attrs
else {};
in
optionalAttrs (prev ? legacyPackages && prev.legacyPackages ? ${info.currentSystem})
{
pkgs = prev.legacyPackages.${info.currentSystem};
}
```
We can run `nix repl` and use `pkgs` to refer to `legacyPackages.${currentSystem}`:
```ShellSession
$ nix repl --repl-overlays ~/.config/nix/repl.nix nixpkgs
Nix 2.21.0pre20240309_4111bb6
Type :? for help.
Loading installable 'flake:nixpkgs#'...
Added 5 variables.
Loading 'repl-overlays'...
Added 6 variables.
nix-repl> pkgs.bash
«derivation /nix/store/g08b5vkwwh0j8ic9rkmd8mpj878rk62z-bash-5.2p26.drv»
```

View file

@ -13,3 +13,5 @@ libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread
libcmd_LIBS = libstore libutil libexpr libmain libfetchers
$(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))
$(d)/repl.cc: $(d)/repl-overlays.nix.gen.hh

View file

@ -0,0 +1,8 @@
info:
initial:
functions:
let final = builtins.foldl'
(prev: function: prev // (function info final prev))
initial
functions;
in final

View file

@ -30,6 +30,7 @@
#include "signals.hh"
#include "print.hh"
#include "progress-bar.hh"
#include "gc-small-vector.hh"
#if HAVE_BOEHMGC
#define GC_INCLUDE_NEW
@ -100,6 +101,42 @@ struct NixRepl
void evalString(std::string s, Value & v);
void loadDebugTraceEnv(DebugTrace & dt);
/**
* Load the `repl-overlays` and add the resulting AttrSet to the top-level
* bindings.
*/
void loadReplOverlays();
/**
* Get a list of each of the `repl-overlays` (parsed and evaluated).
*/
Value * replOverlays();
/**
* Get the Nix function that composes the `repl-overlays` together.
*/
Value * getReplOverlaysEvalFunction();
/**
* Cached return value of `getReplOverlaysEvalFunction`.
*/
Value * replOverlaysEvalFunction;
/**
* Get the `info` AttrSet that's passed as the first argument to each
* of the `repl-overlays`.
*/
Value * replInitInfo();
/**
* Get the current top-level bindings as an AttrSet.
*/
Value * bindingsToAttrs();
/**
* Parse a file, evaluate its result, and force the resulting value.
*/
Value * evalFile(SourcePath & path);
void printValue(std::ostream & str,
Value & v,
unsigned int maxDepth = std::numeric_limits<unsigned int>::max())
@ -737,14 +774,95 @@ void NixRepl::loadFiles()
loadedFiles.clear();
for (auto & i : old) {
notice("Loading '%1%'...", i);
notice("Loading '%1%'...", Magenta(i));
loadFile(i);
}
for (auto & [i, what] : getValues()) {
notice("Loading installable '%1%'...", what);
notice("Loading installable '%1%'...", Magenta(what));
addAttrsToScope(*i);
}
loadReplOverlays();
}
void NixRepl::loadReplOverlays()
{
if (!evalSettings.replOverlays) {
return;
}
notice("Loading '%1%'...", Magenta("repl-overlays"));
auto replInitFilesFunction = getReplOverlaysEvalFunction();
Value &newAttrs(*state->allocValue());
SmallValueVector<3> args = {replInitInfo(), bindingsToAttrs(), replOverlays()};
state->callFunction(
*replInitFilesFunction,
args.size(),
args.data(),
newAttrs,
replInitFilesFunction->determinePos(noPos)
);
addAttrsToScope(newAttrs);
}
Value * NixRepl::getReplOverlaysEvalFunction()
{
if (replOverlaysEvalFunction) {
return replOverlaysEvalFunction;
}
auto evalReplInitFilesPath = CanonPath::root + "repl-overlays.nix";
replOverlaysEvalFunction = state->allocValue();
auto code =
#include "repl-overlays.nix.gen.hh"
;
auto expr = state->parseExprFromString(
code,
SourcePath(evalReplInitFilesPath),
state->staticBaseEnv
);
state->eval(expr, *replOverlaysEvalFunction);
return replOverlaysEvalFunction;
}
Value * NixRepl::replOverlays()
{
Value * replInits(state->allocValue());
state->mkList(*replInits, evalSettings.replOverlays.get().size());
Value ** replInitElems = replInits->listElems();
size_t i = 0;
for (auto path : evalSettings.replOverlays.get()) {
debug("Loading '%1%' path '%2%'...", "repl-overlays", path);
SourcePath sourcePath((CanonPath(path)));
replInitElems[i] = evalFile(sourcePath);
i++;
}
return replInits;
}
Value * NixRepl::replInitInfo()
{
auto builder = state->buildBindings(2);
Value * currentSystem(state->allocValue());
currentSystem->mkString(evalSettings.getCurrentSystem());
builder.insert(state->symbols.create("currentSystem"), currentSystem);
Value * valueNull(state->allocValue());
valueNull->mkNull();
builder.insert(state->symbols.create("__pleaseUseDotDotDot"), valueNull);
Value * info(state->allocValue());
info->mkAttrs(builder.finish());
return info;
}
@ -777,6 +895,18 @@ void NixRepl::addVarToScope(const Symbol name, Value & v)
varNames.emplace(state->symbols[name]);
}
Value * NixRepl::bindingsToAttrs()
{
auto builder = state->buildBindings(staticEnv->vars.size());
for (auto & [symbol, displacement] : staticEnv->vars) {
builder.insert(symbol, env->values[displacement]);
}
Value * attrs(state->allocValue());
attrs->mkAttrs(builder.finish());
return attrs;
}
Expr * NixRepl::parseString(std::string s)
{
@ -791,6 +921,15 @@ void NixRepl::evalString(std::string s, Value & v)
state->forceValue(v, v.determinePos(noPos));
}
Value * NixRepl::evalFile(SourcePath & path)
{
auto expr = state->parseExprFromFile(path, staticEnv);
Value * result(state->allocValue());
expr->eval(*state, *env, *result);
state->forceValue(*result, result->determinePos(noPos));
return result;
}
std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,

View file

@ -124,6 +124,42 @@ struct EvalSettings : Config
This is useful for debugging warnings in third-party Nix code.
)"};
PathsSetting replOverlays{this, Paths(), "repl-overlays",
R"(
A list of files containing Nix expressions that can be used to add
default bindings to [`nix
repl`](@docroot@/command-ref/new-cli/nix3-repl.md) sessions.
Each file is called with three arguments:
1. An [attribute set](@docroot@/language/values.html#attribute-set)
containing at least a
[`currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem)
attribute (this is identical to
[`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem),
except that it's available in
[`pure-eval`](@docroot@/command-ref/conf-file.html#conf-pure-eval)
mode).
2. The top-level bindings produced by the previous `repl-overlays`
value (or the default top-level bindings).
3. The final top-level bindings produced by calling all
`repl-overlays`.
For example, the following file would alias `pkgs` to
`legacyPackages.${info.currentSystem}` (if that attribute is defined):
```nix
info: prev: final:
if prev ? legacyPackages
&& prev.legacyPackages ? ${info.currentSystem}
then
{
pkgs = prev.legacyPackages.${info.currentSystem};
}
else
{ }
```
)"};
};
extern EvalSettings evalSettings;

View file

@ -4,6 +4,7 @@
#include "symbol-table.hh"
#include "util.hh"
#include "print.hh"
#include "escape-string.hh"
#include <cstdlib>
@ -36,7 +37,7 @@ void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const
void ExprString::show(const SymbolTable & symbols, std::ostream & str) const
{
printLiteralString(str, s);
escapeString(str, s);
}
void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const

View file

@ -2,6 +2,7 @@
#include "print.hh"
#include "eval.hh"
#include "signals.hh"
#include "escape-string.hh"
namespace nix {
@ -27,7 +28,7 @@ void printAmbiguous(
printLiteralBool(str, v.boolean);
break;
case nString:
printLiteralString(str, v.string.s);
escapeString(str, v.string.s);
break;
case nPath:
str << v.path().to_string(); // !!! escaping?

View file

@ -2,6 +2,7 @@
#include <span>
#include <unordered_set>
#include "escape-string.hh"
#include "print.hh"
#include "ansicolor.hh"
#include "store-api.hh"
@ -11,57 +12,6 @@
namespace nix {
void printElided(
std::ostream & output,
unsigned int value,
const std::string_view single,
const std::string_view plural,
bool ansiColors)
{
if (ansiColors)
output << ANSI_FAINT;
output << "«";
pluralize(output, value, single, plural);
output << " elided»";
if (ansiColors)
output << ANSI_NORMAL;
}
std::ostream &
printLiteralString(std::ostream & str, const std::string_view string, size_t maxLength, bool ansiColors)
{
size_t charsPrinted = 0;
if (ansiColors)
str << ANSI_MAGENTA;
str << "\"";
for (auto i = string.begin(); i != string.end(); ++i) {
if (charsPrinted >= maxLength) {
str << "\" ";
printElided(str, string.length() - charsPrinted, "byte", "bytes", ansiColors);
return str;
}
if (*i == '\"' || *i == '\\') str << "\\" << *i;
else if (*i == '\n') str << "\\n";
else if (*i == '\r') str << "\\r";
else if (*i == '\t') str << "\\t";
else if (*i == '$' && *(i+1) == '{') str << "\\" << *i;
else str << *i;
charsPrinted++;
}
str << "\"";
if (ansiColors)
str << ANSI_NORMAL;
return str;
}
std::ostream &
printLiteralString(std::ostream & str, const std::string_view string)
{
return printLiteralString(str, string, std::numeric_limits<size_t>::max(), false);
}
std::ostream &
printLiteralBool(std::ostream & str, bool boolean)
{
@ -93,7 +43,7 @@ printIdentifier(std::ostream & str, std::string_view s) {
else {
char c = s[0];
if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) {
printLiteralString(str, s);
escapeString(str, s);
return str;
}
for (auto c : s)
@ -101,7 +51,7 @@ printIdentifier(std::ostream & str, std::string_view s) {
(c >= 'A' && c <= 'Z') ||
(c >= '0' && c <= '9') ||
c == '_' || c == '\'' || c == '-')) {
printLiteralString(str, s);
escapeString(str, s);
return str;
}
str << s;
@ -129,7 +79,7 @@ printAttributeName(std::ostream & str, std::string_view name) {
if (isVarName(name))
str << name;
else
printLiteralString(str, name);
escapeString(str, name);
return str;
}
@ -248,7 +198,15 @@ private:
void printString(Value & v)
{
printLiteralString(output, v.string.s, options.maxStringLength, options.ansiColors);
escapeString(
output,
v.string.s,
{
.maxLength = options.maxStringLength,
.ansiColors = options.ansiColors,
// NB: Non-printing characters won't be escaped.
}
);
}
void printPath(Value & v)

View file

@ -11,28 +11,13 @@
#include "fmt.hh"
#include "print-options.hh"
#include "print-elided.hh"
namespace nix {
class EvalState;
struct Value;
/**
* Print a string as a Nix string literal.
*
* Quotes and fairly minimal escaping are added.
*
* @param o The output stream to print to
* @param s The logical string
*/
std::ostream & printLiteralString(std::ostream & o, std::string_view s);
inline std::ostream & printLiteralString(std::ostream & o, const char * s) {
return printLiteralString(o, std::string_view(s));
}
inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) {
return printLiteralString(o, std::string_view(s));
}
/** Print `true` or `false`. */
std::ostream & printLiteralBool(std::ostream & o, bool b);

View file

@ -437,6 +437,34 @@ void OptionalPathSetting::operator =(const std::optional<Path> & v)
this->assign(v);
}
PathsSetting::PathsSetting(Config * options,
const Paths & def,
const std::string & name,
const std::string & description,
const std::set<std::string> & aliases)
: BaseSetting<Paths>(def, true, name, description, aliases)
{
options->addSetting(this);
}
Paths PathsSetting::parse(const std::string & str) const
{
auto strings = tokenizeString<Strings>(str);
Paths parsed;
for (auto str : strings) {
parsed.push_back(canonPath(str));
}
return parsed;
}
PathsSetting::operator bool() const noexcept
{
return !get().empty();
}
bool GlobalConfig::set(const std::string & name, const std::string & value)
{
for (auto & config : *configRegistrations)

View file

@ -375,6 +375,26 @@ public:
void operator =(const std::optional<Path> & v);
};
/**
* Like `OptionalPathSetting`, but allows multiple paths.
*/
class PathsSetting : public BaseSetting<Paths>
{
public:
PathsSetting(Config * options,
const Paths & def,
const std::string & name,
const std::string & description,
const std::set<std::string> & aliases = {});
Paths parse(const std::string & str) const override;
void operator =(const Paths & v);
operator bool() const noexcept;
};
struct GlobalConfig : public AbstractConfig
{
typedef std::vector<Config*> ConfigRegistrations;

View file

@ -0,0 +1,61 @@
#include <iomanip>
#include <ostream>
#include <sstream>
#include "ansicolor.hh"
#include "escape-char.hh"
#include "english.hh"
#include "escape-string.hh"
#include "print-elided.hh"
namespace nix {
std::ostream &
escapeString(std::ostream & output, std::string_view string, EscapeStringOptions options)
{
size_t charsPrinted = 0;
if (options.ansiColors) {
output << ANSI_MAGENTA;
}
output << "\"";
for (auto i = string.begin(); i != string.end(); ++i) {
if (charsPrinted >= options.maxLength) {
output << "\" ";
printElided(
output, string.length() - charsPrinted, "byte", "bytes", options.ansiColors
);
return output;
}
if (*i == '\"' || *i == '\\') {
output << "\\" << *i;
} else if (*i == '\n') {
output << "\\n";
} else if (*i == '\r') {
output << "\\r";
} else if (*i == '\t') {
output << "\\t";
} else if (*i == '$' && *(i + 1) == '{') {
output << "\\" << *i;
} else if (options.escapeNonPrinting && !isprint(*i)) {
output << MaybeHexEscapedChar{*i};
} else {
output << *i;
}
charsPrinted++;
}
output << "\"";
if (options.ansiColors) {
output << ANSI_NORMAL;
}
return output;
}
std::string escapeString(std::string_view s, EscapeStringOptions options)
{
std::ostringstream output;
escapeString(output, s, options);
return output.str();
}
}; // namespace nix

View file

@ -0,0 +1,78 @@
#pragma once
#include <limits>
#include <ostream>
namespace nix {
/**
* Options for escaping strings in `escapeString`.
*
* With default optional parameters, the output string will round-trip through
* the Nix evaluator (i.e. you can copy/paste this function's output into the
* REPL and have it evaluate as the string that got passed in).
*
* With non-default optional parameters, the output string will be
* human-readable.
*/
struct EscapeStringOptions
{
/**
* If `maxLength` is decreased, some trailing portion of the string may be
* omitted with a message like `«123 bytes elided»`.
*/
size_t maxLength = std::numeric_limits<size_t>::max();
/**
* If `ansiColors` is set, the output will contain ANSI terminal escape
* sequences.
*/
bool ansiColors = false;
/**
* If `escapeNonPrinting` is set, non-printing ASCII characters (i.e. with
* byte values less than 0x20) will be printed in `\xhh` format, like
* `\x1d` (other than those that Nix supports, like `\n`, `\r`, `\t`).
* Note that this format is not yet supported by the Lix parser/evaluator!
*
* See: https://git.lix.systems/lix-project/lix/issues/149
*/
bool escapeNonPrinting = false;
};
/**
* Escape a string for output.
*
* With default optional parameters, the output string will round-trip through
* the Nix evaluator (i.e. you can copy/paste this function's output into the
* REPL and have it evaluate as the string that got passed in).
*
* With non-default optional parameters, the output string will be
* human-readable.
*
* See `EscapeStringOptions` for more details on customizing the output.
*/
std::ostream &
escapeString(std::ostream & output, std::string_view s, EscapeStringOptions options = {});
inline std::ostream & escapeString(std::ostream & output, const char * s)
{
return escapeString(output, std::string_view(s));
}
inline std::ostream & escapeString(std::ostream & output, const std::string & s)
{
return escapeString(output, std::string_view(s));
}
/**
* Escape a string for output, writing the escaped result to a new string.
*/
std::string escapeString(std::string_view s, EscapeStringOptions options = {});
inline std::string escapeString(const char * s, EscapeStringOptions options = {})
{
return escapeString(std::string_view(s), options);
}
} // namespace nix

View file

@ -9,6 +9,7 @@ libutil_sources = files(
'english.cc',
'error.cc',
'escape-char.cc',
'escape-string.cc',
'exit.cc',
'experimental-features.cc',
'filesystem.cc',
@ -19,6 +20,7 @@ libutil_sources = files(
'logging.cc',
'namespaces.cc',
'position.cc',
'print-elided.cc',
'references.cc',
'serialise.cc',
'shlex.cc',
@ -52,6 +54,7 @@ libutil_headers = files(
'english.hh',
'error.hh',
'escape-char.hh',
'escape-string.hh',
'exit.hh',
'experimental-features.hh',
'experimental-features-json.hh',
@ -70,6 +73,7 @@ libutil_headers = files(
'namespaces.hh',
'pool.hh',
'position.hh',
'print-elided.hh',
'ref.hh',
'references.hh',
'regex-combinators.hh',

View file

@ -0,0 +1,23 @@
#include "print-elided.hh"
#include "ansicolor.hh"
#include "english.hh"
namespace nix {
void printElided(
std::ostream & output,
unsigned int value,
const std::string_view single,
const std::string_view plural,
bool ansiColors)
{
if (ansiColors)
output << ANSI_FAINT;
output << "«";
pluralize(output, value, single, plural);
output << " elided»";
if (ansiColors)
output << ANSI_NORMAL;
}
}

View file

@ -0,0 +1,23 @@
#pragma once
#include <ostream>
namespace nix {
/**
* Print an `«... elided»` placeholder.
*
* Arguments are forwarded to `pluralize`.
*
* If `ansiColors` is set, the output will be wrapped in `ANSI_FAINT`.
*/
void printElided(
std::ostream & output,
unsigned int value,
const std::string_view single,
const std::string_view plural,
bool ansiColors
);
}

View file

@ -1,17 +1,28 @@
Commentary "meow meow meow"
Command "command"
Output "output output one"
Output ""
Output ""
Output "output output two"
Commentary "meow meow"
Command "command two"
Output "output output output"
Commentary "commentary"
Output "output output output"
Output ""
Commentary "the blank below should be chomped"
Command "command three"
Commentary ""
Output "meow output"
Output ""
Commentary: "meow meow meow"
Indent: " "
Prompt: "nix-repl> "
Command: "command"
Indent: " "
Output: "output output one"
Output: ""
Commentary: ""
Indent: " "
Output: "output output two"
Commentary: "meow meow"
Indent: " "
Prompt: "nix-repl> "
Command: "command two"
Indent: " "
Output: "output output output"
Commentary: "commentary"
Indent: " "
Output: "output output output"
Output: ""
Commentary: "the blank below should be chomped"
Indent: " "
Prompt: "nix-repl> "
Command: "command three"
Commentary: ""
Indent: " "
Output: "meow output"
Output: ""

View file

@ -1,10 +1,9 @@
Command "command"
Output "output output one"
Output ""
Output ""
Output "output output two"
Command "command two"
Output "output output output"
Output "output output output"
Command "command three"
Output "meow output"
Command: "command"
Output: "output output one"
Output: ""
Output: "output output two"
Command: "command two"
Output: "output output output"
Output: "output output output"
Command: "command three"
Output: "meow output"

View file

@ -0,0 +1,11 @@
command
output output one
output output two
command two
output output output
output output output
command three
meow output

View file

@ -1,6 +1,7 @@
https://github.com/NixOS/nix/pull/9917 (Enter debugger more reliably in let expressions and function calls)
This test ensures that continues don't skip opportunities to enter the debugger.
@args --debugger
trace: before outer break
info: breakpoint reached
@ -13,7 +14,7 @@ This test ensures that continues don't skip opportunities to enter the debugger.
0: error: breakpoint reached
«none»:0
1: while calling a function
TEST_DATA/regression_9917.nix:3:5
$TEST_DATA/regression_9917.nix:3:5
2| a = builtins.trace "before inner break" (
3| builtins.break { msg = "hello"; }
@ -21,7 +22,7 @@ This test ensures that continues don't skip opportunities to enter the debugger.
4| );
2: while calling a function
TEST_DATA/regression_9917.nix:2:7
$TEST_DATA/regression_9917.nix:2:7
1| let
2| a = builtins.trace "before inner break" (

View file

@ -1,3 +1,4 @@
@args --debugger
error:
… while evaluating the error message passed to builtin.throw
@ -14,3 +15,18 @@ We expect to be able to see locals like r in the debugger:
Env level 1
abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true
nix-repl> :quit
error:
… while evaluating the file '$TEST_DATA/regression_9918.nix':
… while calling the 'throw' builtin
at $TEST_DATA/regression_9918.nix:3:7:
2| r = [];
3| x = builtins.throw r;
| ^
4| in
… while evaluating the error message passed to builtin.throw
error: cannot coerce a list to a string: [ ]

View file

@ -0,0 +1,3 @@
{
packages.x86_64-linux.default = "my package";
}

View file

@ -0,0 +1,5 @@
Check basic `repl-overlays` functionality.
@args --repl-overlays
@args ${PWD}/../extra_data/repl-overlay-packages-is-pkgs.nix
nix-repl> pkgs
{ default = "my package"; }

View file

@ -0,0 +1,3 @@
{
var = "a";
}

View file

@ -0,0 +1,7 @@
Check that multiple `repl-overlays` can compose together
@args --repl-overlays
@args "${PWD}/../extra_data/repl-overlays-compose-1.nix ${PWD}/../extra_data/repl-overlays-compose-2.nix"
nix-repl> var
"abc"
nix-repl> varUsingFinal
"final value is: puppy"

View file

@ -0,0 +1,26 @@
`repl-overlays` that try to parse out the `info` argument without a `...` error.
@args --repl-overlays
@args ${PWD}/../extra_data/repl-overlay-no-dotdotdot.nix
@should-start false
error:
… while calling the 'foldl'' builtin
at «string»:5:13:
4| functions:
5| let final = builtins.foldl'
| ^
6| (prev: function: prev // (function info final prev))
… in the right operand of the update (//) operator
at «string»:6:37:
5| let final = builtins.foldl'
6| (prev: function: prev // (function info final prev))
| ^
7| initial
(stack trace truncated; use '--show-trace' to show the full trace)
error: function 'anonymous lambda' called with unexpected argument '__pleaseUseDotDotDot'
at /Users/wiggles/lix/tests/functional/repl_characterization/extra_data/repl-overlay-no-dotdotdot.nix:1:1:
1| {currentSystem}: final: prev: {}
| ^
2|

View file

@ -0,0 +1,22 @@
`repl-overlays` that fail to evaluate should error.
@args --repl-overlays
@args ${PWD}/../extra_data/repl-overlay-fail.nix
@should-start false
error:
… while calling the 'foldl'' builtin
at «string»:5:13:
4| functions:
5| let final = builtins.foldl'
| ^
6| (prev: function: prev // (function info final prev))
… in the right operand of the update (//) operator
at «string»:6:37:
5| let final = builtins.foldl'
6| (prev: function: prev // (function info final prev))
| ^
7| initial
(stack trace truncated; use '--show-trace' to show the full trace)
error: evaluation aborted with the following error message: 'uh oh!'

View file

@ -1,3 +1,4 @@
@args --debugger
trace: before outer break
info: breakpoint reached
@ -24,7 +25,7 @@ If we :st past the frame in the backtrace with the meow in it, the meow should n
nix-repl> :st 3
3: while calling a function
TEST_DATA/stack_vars.nix:5:7
$TEST_DATA/stack_vars.nix:5:7
4| );
5| b = builtins.trace "before outer break" (
@ -58,9 +59,8 @@ If we :st past the frame in the backtrace with the meow in it, the meow should n
3
nix-repl> :st 3
3: while calling a function
TEST_DATA/stack_vars.nix:2:7
$TEST_DATA/stack_vars.nix:2:7
1| let
2| a = builtins.trace "before inner break" (
@ -72,3 +72,21 @@ If we :st past the frame in the backtrace with the meow in it, the meow should n
Env level 1
abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true
nix-repl> :quit
error:
… while calling the 'trace' builtin
at $TEST_DATA/stack_vars.nix:2:7:
1| let
2| a = builtins.trace "before inner break" (
| ^
3| let meow' = 3; in builtins.break { msg = "hello"; }
… while calling the 'break' builtin
at $TEST_DATA/stack_vars.nix:3:23:
2| a = builtins.trace "before inner break" (
3| let meow' = 3; in builtins.break { msg = "hello"; }
| ^
4| );
error: breakpoint reached

View file

@ -0,0 +1 @@
info: final: prev: builtins.abort "uh oh!"

View file

@ -0,0 +1 @@
{currentSystem}: final: prev: {}

View file

@ -0,0 +1,4 @@
info: final: prev:
{
pkgs = final.packages.x86_64-linux;
}

View file

@ -0,0 +1,7 @@
info: final: prev:
{
var = prev.var + "b";
# We can access the final value of `var` here even though it isn't defined yet:
varUsingFinal = "final value is: " + final.newVar;
}

View file

@ -0,0 +1,6 @@
info: final: prev:
{
var = prev.var + "c";
newVar = "puppy";
}

View file

@ -1,25 +1,26 @@
#include <gtest/gtest.h>
#include <boost/algorithm/string/replace.hpp>
#include <optional>
#include <string>
#include <string_view>
#include <optional>
#include <unistd.h>
#include <boost/algorithm/string/replace.hpp>
#include "escape-string.hh"
#include "test-session.hh"
#include "util.hh"
#include "tests/characterization.hh"
#include "tests/cli-literate-parser.hh"
#include "tests/terminal-code-eater.hh"
#include "util.hh"
using namespace std::string_literals;
namespace nix {
static constexpr const char * REPL_PROMPT = "nix-repl> ";
static constexpr const std::string_view REPL_PROMPT = "nix-repl> ";
// ASCII ENQ character
static constexpr const char * AUTOMATION_PROMPT = "\x05";
static constexpr const std::string_view AUTOMATION_PROMPT = "\x05";
static std::string_view trimOutLog(std::string_view outLog)
{
@ -40,91 +41,147 @@ public:
return unitTestData + "/" + testStem;
}
void runReplTest(std::string_view const & content, std::vector<std::string> extraArgs = {}) const
void runReplTest(const std::string content, std::vector<std::string> extraArgs = {}) const
{
auto syntax = CLILiterateParser::parse(REPL_PROMPT, content);
auto parsed = cli_literate_parser::parse(
content, cli_literate_parser::Config{.prompt = std::string(REPL_PROMPT), .indent = 2}
);
parsed.interpolatePwd(unitTestData);
// FIXME: why does this need two --quiets
// show-trace is on by default due to test configuration, but is not a standard
Strings args{"--quiet", "repl", "--quiet", "--option", "show-trace", "false", "--offline", "--extra-experimental-features", "repl-automation"};
// show-trace is on by default due to test configuration, but is not a
// standard
Strings args{
"--quiet",
"repl",
"--quiet",
"--option",
"show-trace",
"false",
"--offline",
"--extra-experimental-features",
"repl-automation",
};
args.insert(args.end(), extraArgs.begin(), extraArgs.end());
args.insert(args.end(), parsed.args.begin(), parsed.args.end());
auto nixBin = canonPath(getEnvNonEmpty("NIX_BIN_DIR").value_or(NIX_BIN_DIR));
auto process = RunningProcess::start(nixBin + "/nix", args);
auto session = TestSession{AUTOMATION_PROMPT, std::move(process)};
auto session = TestSession(std::string(AUTOMATION_PROMPT), std::move(process));
for (auto & bit : syntax) {
if (bit.kind != CLILiterateParser::NodeKind::COMMAND) {
continue;
}
if (!session.waitForPrompt()) {
ASSERT_TRUE(false);
}
session.runCommand(bit.text);
for (auto & event : parsed.syntax) {
std::visit(
overloaded{
[&](const cli_literate_parser::Command & e) {
ASSERT_TRUE(session.waitForPrompt());
if (e.text == ":quit") {
// If we quit the repl explicitly, we won't have a
// prompt when we're done.
parsed.shouldStart = false;
}
session.runCommand(e.text);
},
[&](const auto & e) {},
},
event
);
}
if (!session.waitForPrompt()) {
ASSERT_TRUE(false);
if (parsed.shouldStart) {
ASSERT_TRUE(session.waitForPrompt());
}
session.close();
auto replacedOutLog = boost::algorithm::replace_all_copy(session.outLog, unitTestData, "TEST_DATA");
auto replacedOutLog =
boost::algorithm::replace_all_copy(session.outLog, unitTestData, "$TEST_DATA");
auto cleanedOutLog = trimOutLog(replacedOutLog);
auto parsedOutLog = CLILiterateParser::parse(AUTOMATION_PROMPT, cleanedOutLog, 0);
auto parsedOutLog = cli_literate_parser::parse(
std::string(cleanedOutLog),
cli_literate_parser::Config{.prompt = std::string(AUTOMATION_PROMPT), .indent = 0}
);
parsedOutLog = CLILiterateParser::tidyOutputForComparison(std::move(parsedOutLog));
syntax = CLILiterateParser::tidyOutputForComparison(std::move(syntax));
auto expected = parsed.tidyOutputForComparison();
auto actual = parsedOutLog.tidyOutputForComparison();
ASSERT_EQ(parsedOutLog, syntax);
ASSERT_EQ(expected, actual);
}
void runReplTestPath(const std::string_view & nameBase, std::vector<std::string> extraArgs)
{
auto nixPath = goldenMaster(nameBase + ".nix");
if (pathExists(nixPath)) {
extraArgs.push_back("-f");
extraArgs.push_back(nixPath);
}
readTest(nameBase + ".test", [this, extraArgs](std::string input) {
runReplTest(input, extraArgs);
});
}
void runReplTestPath(const std::string_view & nameBase)
{
runReplTestPath(nameBase, {});
}
void runDebuggerTest(const std::string_view & nameBase)
{
runReplTestPath(nameBase, {"--debugger"});
}
};
TEST_F(ReplSessionTest, parses)
TEST_F(ReplSessionTest, round_trip)
{
writeTest("basic.test", [this]() {
const std::string content = readFile(goldenMaster("basic.test"));
auto parsed = cli_literate_parser::parse(
content, cli_literate_parser::Config{.prompt = std::string(REPL_PROMPT)}
);
std::ostringstream out{};
for (auto & node : parsed.syntax) {
cli_literate_parser::unparseNode(out, node, true);
}
return out.str();
});
}
TEST_F(ReplSessionTest, tidy)
{
writeTest("basic.ast", [this]() {
const std::string content = readFile(goldenMaster("basic.test"));
auto parser = CLILiterateParser{REPL_PROMPT};
parser.feed(content);
auto parsed = cli_literate_parser::parse(
content, cli_literate_parser::Config{.prompt = std::string(REPL_PROMPT)}
);
std::ostringstream out{};
for (auto & bit : parser.syntax()) {
out << bit.print() << "\n";
for (auto & node : parsed.syntax) {
out << debugNode(node) << "\n";
}
return out.str();
});
writeTest("basic_tidied.ast", [this]() {
const std::string content = readFile(goldenMaster("basic.test"));
auto syntax = CLILiterateParser::parse(REPL_PROMPT, content);
syntax = CLILiterateParser::tidyOutputForComparison(std::move(syntax));
auto parsed = cli_literate_parser::parse(
content, cli_literate_parser::Config{.prompt = std::string(REPL_PROMPT)}
);
auto tidied = parsed.tidyOutputForComparison();
std::ostringstream out{};
for (auto & bit : syntax) {
out << bit.print() << "\n";
for (auto & node : tidied) {
out << debugNode(node) << "\n";
}
return out.str();
});
}
TEST_F(ReplSessionTest, repl_basic)
{
readTest("basic_repl.test", [this](std::string input) { runReplTest(input); });
}
#define DEBUGGER_TEST(name) \
#define REPL_TEST(name) \
TEST_F(ReplSessionTest, name) \
{ \
readTest(#name ".test", [this](std::string input) { \
runReplTest(input, {"--debugger", "-f", goldenMaster(#name ".nix")}); \
}); \
{ \
runReplTestPath(#name); \
}
DEBUGGER_TEST(regression_9918);
DEBUGGER_TEST(regression_9917);
DEBUGGER_TEST(regression_l145);
DEBUGGER_TEST(stack_vars);
REPL_TEST(regression_9918);
REPL_TEST(regression_9917);
REPL_TEST(stack_vars);
REPL_TEST(basic_repl);
};
}; // namespace nix

View file

@ -1,4 +1,5 @@
#include <iostream>
#include <span>
#include <unistd.h>
#include "test-session.hh"
@ -21,14 +22,17 @@ RunningProcess RunningProcess::start(std::string executable, Strings args)
// This is separate from runProgram2 because we have different IO requirements
pid_t pid = startProcess([&]() {
if (dup2(procStdout.writeSide.get(), STDOUT_FILENO) == -1)
if (dup2(procStdout.writeSide.get(), STDOUT_FILENO) == -1) {
throw SysError("dupping stdout");
if (dup2(procStdin.readSide.get(), STDIN_FILENO) == -1)
}
if (dup2(procStdin.readSide.get(), STDIN_FILENO) == -1) {
throw SysError("dupping stdin");
}
procStdin.writeSide.close();
procStdout.readSide.close();
if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1)
if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) {
throw SysError("dupping stderr");
}
execv(executable.c_str(), stringsToCharPtrs(args).data());
throw SysError("exec did not happen");
});
@ -44,7 +48,8 @@ RunningProcess RunningProcess::start(std::string executable, Strings args)
}
[[gnu::unused]]
std::ostream & operator<<(std::ostream & os, ReplOutputParser::State s)
std::ostream &
operator<<(std::ostream & os, ReplOutputParser::State s)
{
switch (s) {
case ReplOutputParser::State::Prompt:
@ -91,8 +96,7 @@ bool ReplOutputParser::feed(char c)
return false;
}
/** Waits for the prompt and then returns if a prompt was found */
bool TestSession::waitForPrompt()
bool TestSession::readOutThen(ReadOutThenCallback cb)
{
std::vector<char> buf(1024);
@ -106,38 +110,67 @@ bool TestSession::waitForPrompt()
return false;
}
switch (cb(std::span(buf.data(), res))) {
case ReadOutThenCallbackResult::Stop:
return true;
case ReadOutThenCallbackResult::Continue:
continue;
}
}
}
bool TestSession::waitForPrompt()
{
bool notEof = readOutThen([&](std::span<char> s) -> ReadOutThenCallbackResult {
bool foundPrompt = false;
for (ssize_t i = 0; i < res; ++i) {
for (auto ch : s) {
// foundPrompt = foundPrompt || outputParser.feed(buf[i]);
bool wasEaten = true;
eater.feed(buf[i], [&](char c) {
eater.feed(ch, [&](char c) {
wasEaten = false;
foundPrompt = outputParser.feed(buf[i]) || foundPrompt;
foundPrompt = outputParser.feed(ch) || foundPrompt;
outLog.push_back(c);
});
if constexpr (DEBUG_REPL_PARSER) {
std::cerr << "raw " << MaybeHexEscapedChar{buf[i]} << (wasEaten ? " [eaten]" : "") << "\n";
std::cerr << "raw " << MaybeHexEscapedChar{ch} << (wasEaten ? " [eaten]" : "") << "\n";
}
}
if (foundPrompt) {
return true;
return foundPrompt ? ReadOutThenCallbackResult::Stop : ReadOutThenCallbackResult::Continue;
});
return notEof;
}
void TestSession::wait()
{
readOutThen([&](std::span<char> s) {
for (auto ch : s) {
eater.feed(ch, [&](char c) {
outputParser.feed(c);
outLog.push_back(c);
});
}
}
// just keep reading till we hit eof
return ReadOutThenCallbackResult::Continue;
});
}
void TestSession::close()
{
proc.procStdin.close();
wait();
proc.procStdout.close();
}
void TestSession::runCommand(std::string command)
{
if constexpr (DEBUG_REPL_PARSER)
if constexpr (DEBUG_REPL_PARSER) {
std::cerr << "runCommand " << command << "\n";
}
command += "\n";
// We have to feed a newline into the output parser, since Nix might not
// give us a newline before a prompt in all cases (it might clear line

View file

@ -1,7 +1,9 @@
#pragma once
///@file
#include <functional>
#include <sched.h>
#include <span>
#include <string>
#include "util.hh"
@ -22,8 +24,7 @@ struct RunningProcess
class ReplOutputParser
{
public:
ReplOutputParser(std::string prompt)
: prompt(prompt)
ReplOutputParser(std::string prompt) : prompt(prompt)
{
assert(!prompt.empty());
}
@ -60,10 +61,27 @@ struct TestSession
{
}
/** Waits for the prompt and then returns if a prompt was found */
bool waitForPrompt();
/** Feeds a line of input into the command */
void runCommand(std::string command);
/** Closes the session, closing standard input and waiting for standard
* output to close, capturing any remaining output. */
void close();
private:
/** Waits until the command closes its output */
void wait();
enum class ReadOutThenCallbackResult { Stop, Continue };
using ReadOutThenCallback = std::function<ReadOutThenCallbackResult(std::span<char>)>;
/** Reads some chunks of output, calling the callback provided for each
* chunk and stopping if it returns Stop.
*
* @returns false if EOF, true if the callback requested we stop first.
* */
bool readOutThen(ReadOutThenCallback cb);
};
};

View file

@ -8,7 +8,7 @@ libutil-test-support_INSTALL_DIR :=
libutil-test-support_SOURCES := $(wildcard $(d)/tests/*.cc)
libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES)
libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) -I src/libutil
# libexpr so we can steal their string printer from print.cc
libutil-test-support_LIBS = libutil libexpr

View file

@ -74,20 +74,20 @@ public:
{
auto file = goldenMaster(testStem);
auto got = test();
auto actual = test();
if (testAccept())
{
createDirs(dirOf(file));
writeFile2(file, got);
writeFile2(file, actual);
GTEST_SKIP()
<< "Updating golden master "
<< file;
}
else
{
decltype(got) expected = readFile2(file);
ASSERT_EQ(got, expected);
decltype(actual) expected = readFile2(file);
ASSERT_EQ(expected, actual);
}
}

View file

@ -1,247 +1,444 @@
#include "cli-literate-parser.hh"
#include "libexpr/print.hh"
#include "escape-string.hh"
#include "escape-char.hh"
#include "libexpr/print.hh"
#include "types.hh"
#include "util.hh"
#include <ranges>
#include <boost/algorithm/string/replace.hpp>
#include <boost/algorithm/string/trim.hpp>
#include <iostream>
#include <memory>
#include <boost/algorithm/string/trim.hpp>
#include <sstream>
#include <variant>
using namespace std::string_literals;
namespace nix {
#include "cli-literate-parser.hh"
#include "escape-string.hh"
#include "fmt.hh"
#include "libexpr/print.hh"
#include "shlex.hh"
#include "types.hh"
#include "util.hh"
static constexpr const bool DEBUG_PARSER = false;
constexpr auto CLILiterateParser::stateDebug(State const & s) -> const char *
{
return std::visit(
overloaded{// clang-format off
[](Indent const&) -> const char * { return "indent"; },
[](Commentary const&) -> const char * { return "indent"; },
[](Prompt const&) -> const char * { return "prompt"; },
[](Command const&) -> const char * { return "command"; },
[](OutputLine const&) -> const char * { return "output_line"; }},
// clang-format on
s);
}
using namespace std::string_literals;
using namespace boost::algorithm;
auto CLILiterateParser::Node::print() const -> std::string
{
std::ostringstream s{};
switch (kind) {
case NodeKind::COMMENTARY:
s << "Commentary ";
break;
case NodeKind::COMMAND:
s << "Command ";
break;
case NodeKind::OUTPUT:
s << "Output ";
break;
}
printLiteralString(s, this->text);
return s.str();
}
namespace nix {
void PrintTo(std::vector<CLILiterateParser::Node> const & nodes, std::ostream * os)
{
for (auto & node : nodes) {
*os << node.print() << "\\n";
}
}
namespace cli_literate_parser {
auto CLILiterateParser::parse(std::string prompt, std::string_view const & input, size_t indent) -> std::vector<Node>
struct Parser
{
CLILiterateParser p{std::move(prompt), indent};
p.feed(input);
return std::move(p).intoSyntax();
}
auto CLILiterateParser::intoSyntax() && -> std::vector<Node>
{
return std::move(this->syntax_);
}
CLILiterateParser::CLILiterateParser(std::string prompt, size_t indent)
: state_(indent == 0 ? State(Prompt{}) : State(Indent{}))
, prompt_(prompt)
, indent_(indent)
, lastWasOutput_(false)
, syntax_{}
{
assert(!prompt.empty());
}
void CLILiterateParser::feed(char c)
{
if constexpr (DEBUG_PARSER) {
std::cout << stateDebug(state_) << " " << MaybeHexEscapedChar{c} << "\n";
Parser(const std::string input, Config config)
: input(input)
, rest(this->input)
, prompt(config.prompt)
, indentString(std::string(config.indent, ' '))
, lastWasOutput(false)
, syntax{}
{
assert(!prompt.empty());
}
if (c == '\n') {
onNewline();
return;
const std::string input;
std::string_view rest;
const std::string prompt;
const std::string indentString;
/** Last line was output, so we consider a blank to be part of the output */
bool lastWasOutput;
/**
* Nodes of syntax being built.
*/
std::vector<Node> syntax;
auto dbg(std::string_view state) -> void
{
std::cout << state << ": ";
escapeString(
std::cout,
rest,
{
.maxLength = 40,
.ansiColors = true,
.escapeNonPrinting = true,
}
);
std::cout << std::endl;
}
std::visit(
overloaded{
[&](Indent & s) {
if (c == ' ') {
if (++s.pos >= indent_) {
transition(Prompt{});
}
} else {
transition(Commentary{AccumulatingState{.lineAccumulator = std::string{c}}});
}
},
[&](Prompt & s) {
if (s.pos >= prompt_.length()) {
transition(Command{AccumulatingState{.lineAccumulator = std::string{c}}});
return;
} else if (c == prompt_[s.pos]) {
// good prompt character
++s.pos;
} else {
// didn't match the prompt, so it must have actually been output.
s.lineAccumulator.push_back(c);
transition(OutputLine{AccumulatingState{.lineAccumulator = std::move(s.lineAccumulator)}});
return;
}
s.lineAccumulator.push_back(c);
},
[&](AccumulatingState & s) { s.lineAccumulator.push_back(c); }},
state_);
}
void CLILiterateParser::onNewline()
{
State lastState = std::move(state_);
bool newLastWasOutput = false;
syntax_.push_back(std::visit(
overloaded{
[&](Indent & s) {
// XXX: technically this eats trailing spaces
// a newline following output is considered part of that output
if (lastWasOutput_) {
newLastWasOutput = true;
return Node::mkOutput("");
}
return Node::mkCommentary("");
},
[&](Commentary & s) { return Node::mkCommentary(std::move(s.lineAccumulator)); },
[&](Command & s) { return Node::mkCommand(std::move(s.lineAccumulator)); },
[&](OutputLine & s) {
newLastWasOutput = true;
return Node::mkOutput(std::move(s.lineAccumulator));
},
[&](Prompt & s) {
// INDENT followed by newline is also considered a blank output line
return Node::mkOutput(std::move(s.lineAccumulator));
}},
lastState));
transition(Indent{});
lastWasOutput_ = newLastWasOutput;
}
void CLILiterateParser::feed(std::string_view s)
{
for (char ch : s) {
feed(ch);
}
}
void CLILiterateParser::transition(State new_state)
{
// When we expect INDENT and we are parsing without indents, commentary
// cannot exist, so we want to transition directly into PROMPT before
// resuming normal processing.
if (Indent * i = std::get_if<Indent>(&new_state); i != nullptr && indent_ == 0) {
new_state = Prompt{AccumulatingState{}, i->pos};
template<typename T>
auto pushNode(T node) -> void
{
if constexpr (DEBUG_PARSER) {
std::cout << debugNode(node);
}
syntax.emplace_back(node);
}
state_ = new_state;
}
auto CLILiterateParser::syntax() const -> std::vector<Node> const &
{
return syntax_;
}
auto CLILiterateParser::unparse(const std::string & prompt, const std::vector<Node> & syntax, size_t indent)
-> std::string
{
std::string indent_str(indent, ' ');
std::ostringstream out{};
for (auto & node : syntax) {
switch (node.kind) {
case NodeKind::COMMENTARY:
out << node.text << "\n";
break;
case NodeKind::COMMAND:
out << indent_str << prompt << node.text << "\n";
break;
case NodeKind::OUTPUT:
out << indent_str << node.text << "\n";
break;
auto parseLiteral(const char c) -> bool
{
if (rest.starts_with(c)) {
rest.remove_prefix(1);
return true;
} else {
return false;
}
}
return out.str();
}
auto parseLiteral(const std::string_view & literal) -> bool
{
if (rest.starts_with(literal)) {
rest.remove_prefix(literal.length());
return true;
} else {
return false;
}
}
auto CLILiterateParser::tidyOutputForComparison(std::vector<Node> && syntax) -> std::vector<Node>
auto parseBool() -> bool
{
auto result = false;
if (parseLiteral("true")) {
result = true;
} else if (parseLiteral("false")) {
result = false;
} else {
throw ParseError("true or false", std::string(rest));
}
auto untilNewline = parseUntilNewline();
if (!untilNewline.empty()) {
throw ParseError("nothing after true or false", untilNewline);
}
return result;
}
auto parseUntilNewline() -> std::string
{
auto pos = rest.find('\n');
if (pos == std::string_view::npos) {
throw ParseError("text and then newline", std::string(rest));
} else {
// `parseOutput()` sets this to true anyways.
lastWasOutput = false;
auto result = std::string(rest, 0, pos);
rest.remove_prefix(pos + 1);
return result;
}
}
auto parseIndent() -> bool
{
if constexpr (DEBUG_PARSER) {
dbg("indent");
}
if (indentString.empty()) {
return true;
}
if (parseLiteral(indentString)) {
pushNode(Indent(indentString));
return true;
} else {
if constexpr (DEBUG_PARSER) {
dbg("indent failed");
}
return false;
}
}
auto parseCommand() -> void
{
if constexpr (DEBUG_PARSER) {
dbg("command");
}
auto untilNewline = parseUntilNewline();
pushNode(Command(untilNewline));
}
auto parsePrompt() -> void
{
if constexpr (DEBUG_PARSER) {
dbg("prompt");
}
if (parseLiteral(prompt)) {
pushNode(Prompt(prompt));
if (rest.empty()) {
return;
}
parseCommand();
} else {
parseOutput();
}
}
auto parseOutput() -> void
{
if constexpr (DEBUG_PARSER) {
dbg("output");
}
auto untilNewline = parseUntilNewline();
pushNode(Output(untilNewline));
lastWasOutput = true;
}
auto parseAtSign() -> void
{
if constexpr (DEBUG_PARSER) {
dbg("@ symbol");
}
if (!parseLiteral('@')) {
parseOutputOrCommentary();
}
if (parseLiteral("args ")) {
parseArgs();
} else if (parseLiteral("should-start ")) {
if constexpr (DEBUG_PARSER) {
dbg("@should-start");
}
auto shouldStart = parseBool();
pushNode(ShouldStart{shouldStart});
}
}
auto parseArgs() -> void
{
if constexpr (DEBUG_PARSER) {
dbg("@args");
}
auto untilNewline = parseUntilNewline();
pushNode(Args(untilNewline));
}
auto parseOutputOrCommentary() -> void
{
if constexpr (DEBUG_PARSER) {
dbg("output/commentary");
}
auto oldLastWasOutput = lastWasOutput;
auto untilNewline = parseUntilNewline();
auto trimmed = trim_right_copy(untilNewline);
if (oldLastWasOutput && trimmed.empty()) {
pushNode(Output{trimmed});
} else {
pushNode(Commentary{untilNewline});
}
}
auto parseStartOfLine() -> void
{
if constexpr (DEBUG_PARSER) {
dbg("start of line");
}
if (parseIndent()) {
parsePrompt();
} else {
parseAtSign();
}
}
auto parse() && -> ParseResult
{
// Begin the recursive descent parser at the start of a new line.
while (!rest.empty()) {
parseStartOfLine();
}
return std::move(*this).intoParseResult();
}
auto intoParseResult() && -> ParseResult
{
// Do another pass over the nodes to produce auxiliary results like parsed
// command line arguments.
std::vector<std::string> args;
std::vector<Node> newSyntax;
auto shouldStart = true;
for (auto it = syntax.begin(); it != syntax.end(); ++it) {
Node node = std::move(*it);
std::visit(
overloaded{
[&](Args & e) {
auto split = shell_split(std::string(e.text));
args.insert(args.end(), split.begin(), split.end());
},
[&](ShouldStart & e) { shouldStart = e.shouldStart; },
[&](auto & e) {},
},
node
);
newSyntax.push_back(node);
}
return ParseResult{
.syntax = std::move(newSyntax),
.args = std::move(args),
.shouldStart = shouldStart,
};
}
};
template<typename View>
auto tidySyntax(View syntax) -> std::vector<Node>
{
std::vector<Node> newSyntax{};
// Note: Setting `lastWasCommand` lets us trim blank lines at the start and
// end of the output stream.
auto lastWasCommand = true;
std::vector<Node> newSyntax;
// Eat trailing newlines, so assume that the very end was actually a command
bool lastWasCommand = true;
bool newLastWasCommand = true;
auto v = std::ranges::reverse_view(syntax);
for (auto it = v.begin(); it != v.end(); ++it) {
Node item = std::move(*it);
lastWasCommand = newLastWasCommand;
// chomp commentary
if (item.kind == NodeKind::COMMENTARY) {
for (auto it = syntax.begin(); it != syntax.end(); ++it) {
Node node = *it;
// Only compare `Command` and `Output` nodes.
if (std::visit([&](auto && e) { return !e.shouldCompare(); }, node)) {
continue;
}
if (item.kind == NodeKind::COMMAND) {
newLastWasCommand = true;
// Remove blank lines before and after commands. This lets us keep nice
// whitespace in the test files.
auto shouldKeep = std::visit(
overloaded{
[&](Command & e) {
lastWasCommand = true;
auto trimmed = trim_right_copy(e.text);
if (trimmed.empty()) {
return false;
} else {
e.text = trimmed;
return true;
}
},
[&](Output & e) {
std::string trimmed = trim_right_copy(e.text);
if (lastWasCommand && trimmed.empty()) {
// NB: Keep `lastWasCommand` true in this branch so we
// can keep pruning empty output lines.
return false;
} else {
e.text = trimmed;
lastWasCommand = false;
return true;
}
},
[&](auto & e) {
lastWasCommand = false;
return false;
},
},
node
);
if (item.text == "") {
// chomp empty commands
continue;
}
if (shouldKeep) {
newSyntax.push_back(node);
}
if (item.kind == NodeKind::OUTPUT) {
// TODO: horrible
bool nextIsCommand = (it + 1 == v.end()) ? false : (it + 1)->kind == NodeKind::COMMAND;
std::string trimmedText = boost::algorithm::trim_right_copy(item.text);
if ((lastWasCommand || nextIsCommand) && trimmedText == "") {
// chomp empty text above or directly below commands
continue;
}
// real output, stop chomping
newLastWasCommand = false;
item = Node::mkOutput(std::move(trimmedText));
}
newSyntax.push_back(std::move(item));
}
std::reverse(newSyntax.begin(), newSyntax.end());
return newSyntax;
}
};
auto ParseResult::tidyOutputForComparison() -> std::vector<Node>
{
auto reversed = tidySyntax(std::ranges::reverse_view(syntax));
auto unreversed = tidySyntax(std::ranges::reverse_view(reversed));
return unreversed;
}
void ParseResult::interpolatePwd(std::string_view pwd)
{
std::vector<std::string> newArgs;
for (auto & arg : args) {
newArgs.push_back(replaceStrings(arg, "${PWD}", pwd));
}
args = std::move(newArgs);
}
const char * ParseError::what() const noexcept
{
if (what_) {
return what_->c_str();
} else {
auto escaped = escapeString(rest, {.maxLength = 256, .escapeNonPrinting = true});
auto hint =
new HintFmt("Parse error: Expected %1%, got:\n%2%", expected, Uncolored(escaped));
what_ = hint->str();
return what_->c_str();
}
}
auto parse(const std::string input, Config config) -> ParseResult
{
return Parser(input, config).parse();
}
std::ostream & operator<<(std::ostream & output, const Args & node)
{
return output << "@args " << node.text;
}
std::ostream & operator<<(std::ostream & output, const ShouldStart & node)
{
return output << "@should-start " << (node.shouldStart ? "true" : "false");
}
std::ostream & operator<<(std::ostream & output, const TextNode & rhs)
{
return output << rhs.text;
}
void unparseNode(std::ostream & output, const Node & node, bool withNewline)
{
std::visit(
[&](const auto & n) { output << n << (withNewline && n.emitNewlineAfter() ? "\n" : ""); },
node
);
}
template<typename T>
std::string gtestFormat(T & value)
{
std::ostringstream formatted;
unparseNode(formatted, value, true);
auto str = formatted.str();
// Needs to be the literal string `\n` and not a newline character to
// trigger gtest diff printing. Yes seriously.
boost::algorithm::replace_all(str, "\n", "\\n");
return str;
}
void PrintTo(const std::vector<Node> & nodes, std::ostream * output)
{
for (auto & node : nodes) {
*output << gtestFormat(node);
}
}
std::string debugNode(const Node & node)
{
std::ostringstream output;
output << std::visit([](const auto & n) { return n.kind(); }, node) << ": ";
std::ostringstream contents;
unparseNode(contents, node, false);
escapeString(output, contents.str(), {.escapeNonPrinting = true});
return output.str();
}
auto ParseResult::debugPrint(std::ostream & output) -> void
{
::nix::cli_literate_parser::debugPrint(output, syntax);
}
void debugPrint(std::ostream & output, std::vector<Node> & nodes)
{
for (auto & node : nodes) {
output << debugNode(node) << std::endl;
}
}
} // namespace cli_literate_parser
} // namespace nix

View file

@ -3,132 +3,195 @@
#include <compare>
#include <memory>
#include <optional>
#include <sstream>
#include <string>
#include <variant>
#include <vector>
#include <string>
namespace nix {
namespace cli_literate_parser {
// ------------------------- NODES -------------------------
//
// To update golden test files while preserving commentary output and other `@`
// directives, we need to keep commentary output around after parsing.
struct BaseNode {
virtual ~BaseNode() = default;
virtual auto shouldCompare() const -> bool { return false; }
virtual auto kind() const -> std::string = 0;
virtual auto emitNewlineAfter() const -> bool = 0;
auto operator<=>(const BaseNode &rhs) const = default;
};
/**
* A node containing text. The text should be identical to how the node was
* written in the input file.
*/
struct TextNode : BaseNode {
std::string text;
explicit TextNode(std::string text) : text(text) {}
};
std::ostream &operator<<(std::ostream &output, const TextNode &node);
#define DECLARE_TEXT_NODE(NAME, NEEDS_NEWLINE, SHOULD_COMPARE) \
struct NAME : TextNode { \
using TextNode::TextNode; \
~NAME() override = default; \
\
auto kind() const -> std::string override { return #NAME; } \
auto emitNewlineAfter() const -> bool override { return NEEDS_NEWLINE; } \
auto shouldCompare() const -> bool override { return SHOULD_COMPARE; } \
};
/* name, needsNewline, shouldCompare */
DECLARE_TEXT_NODE(Prompt, false, false)
DECLARE_TEXT_NODE(Command, true, true)
DECLARE_TEXT_NODE(Output, true, true)
DECLARE_TEXT_NODE(Commentary, true, false)
DECLARE_TEXT_NODE(Args, true, false)
DECLARE_TEXT_NODE(Indent, false, false)
#undef DECLARE_TEXT_NODE
struct ShouldStart : BaseNode {
bool shouldStart;
ShouldStart(bool shouldStart) : shouldStart(shouldStart) {}
~ShouldStart() override = default;
auto emitNewlineAfter() const -> bool override { return true; }
auto kind() const -> std::string override { return "should-start"; }
auto operator<=>(const ShouldStart &rhs) const = default;
};
std::ostream &operator<<(std::ostream &output, const ShouldStart &node);
/**
* Any syntax node, including those that are cosmetic.
*/
using Node = std::variant<Prompt, Command, Output, Commentary, Args,
ShouldStart, Indent>;
/** Unparses a node into the exact text that would have created it, including a
* newline at the end if present, if withNewline is set */
void unparseNode(std::ostream &output, const Node &node,
bool withNewline = true);
std::string debugNode(const Node &node);
void debugPrint(std::ostream &output, std::vector<Node> &nodes);
/**
* Override gtest printing for lists of nodes.
*/
void PrintTo(std::vector<Node> const &nodes, std::ostream *output);
/**
* The result of parsing a test file.
*/
struct ParseResult {
/**
* A set of nodes that can be used to reproduce the input file. This is used
* to implement updating the test files.
*/
std::vector<Node> syntax;
/**
* Extra CLI arguments.
*/
std::vector<std::string> args;
/**
* Should the program start successfully?
*/
bool shouldStart = false;
/**
* Replace `$PWD` with the given value in `args`.
*/
void interpolatePwd(std::string_view pwd);
/**
* Tidy `syntax` to remove unnecessary nodes.
*/
auto tidyOutputForComparison() -> std::vector<Node>;
auto debugPrint(std::ostream &output) -> void;
};
/**
* A parse error.
*/
struct ParseError : std::exception {
std::string expected;
std::string rest;
ParseError(std::string expected, std::string rest)
: expected(expected), rest(rest) {}
const char *what() const noexcept override;
private:
/**
* Cached formatted contents of `what()`.
*/
mutable std::optional<std::string> what_;
};
struct Config {
/**
* The prompt string to look for.
*/
std::string prompt;
/**
* The number of spaces of indent for commands and output.
*/
size_t indent = 2;
};
/*
* A DFA parser for literate test cases for CLIs.
* A recursive descent parser for literate test cases for CLIs.
*
* FIXME: implement merging of these, so you can auto update cases that have
* comments.
*
* Format:
* COMMENTARY
* INDENT PROMPT COMMAND
* INDENT OUTPUT
* Syntax:
* ```
* ( COMMENTARY
* | INDENT PROMPT COMMAND
* | INDENT OUTPUT
* | @args ARGS
* | @should-start ( true | false )) *
* ```
*
* e.g.
* ```
* commentary commentary commentary
* @args --foo
* @should-start false
* nix-repl> :t 1
* an integer
* ```
*
* Yields:
* Yields something like:
* ```
* Commentary "commentary commentary commentary"
* Args "--foo"
* ShouldStart false
* Command ":t 1"
* Output "an integer"
* ```
*
* Note: one Output line is generated for each line of the sources, because
* this is effectively necessary to be able to align them in the future to
* auto-update tests.
*/
class CLILiterateParser
{
public:
auto parse(std::string input, Config config) -> ParseResult;
enum class NodeKind {
COMMENTARY,
COMMAND,
OUTPUT,
};
struct Node
{
NodeKind kind;
std::string text;
std::strong_ordering operator<=>(Node const &) const = default;
static Node mkCommentary(std::string text)
{
return Node{.kind = NodeKind::COMMENTARY, .text = text};
}
static Node mkCommand(std::string text)
{
return Node{.kind = NodeKind::COMMAND, .text = text};
}
static Node mkOutput(std::string text)
{
return Node{.kind = NodeKind::OUTPUT, .text = text};
}
auto print() const -> std::string;
};
CLILiterateParser(std::string prompt, size_t indent = 2);
auto syntax() const -> std::vector<Node> const &;
/** Feeds a character into the parser */
void feed(char c);
/** Feeds a string into the parser */
void feed(std::string_view s);
/** Parses an input in a non-streaming fashion */
static auto parse(std::string prompt, std::string_view const & input, size_t indent = 2) -> std::vector<Node>;
/** Returns, losslessly, the string that would have generated a syntax tree */
static auto unparse(std::string const & prompt, std::vector<Node> const & syntax, size_t indent = 2) -> std::string;
/** Consumes a CLILiterateParser and gives you the syntax out of it */
auto intoSyntax() && -> std::vector<Node>;
/** Tidies syntax to remove trailing whitespace from outputs and remove any
* empty prompts */
static auto tidyOutputForComparison(std::vector<Node> && syntax) -> std::vector<Node>;
private:
struct AccumulatingState
{
std::string lineAccumulator;
};
struct Indent
{
size_t pos = 0;
};
struct Commentary : public AccumulatingState
{};
struct Prompt : AccumulatingState
{
size_t pos = 0;
};
struct Command : public AccumulatingState
{};
struct OutputLine : public AccumulatingState
{};
using State = std::variant<Indent, Commentary, Prompt, Command, OutputLine>;
State state_;
constexpr static auto stateDebug(State const&) -> const char *;
const std::string prompt_;
const size_t indent_;
/** Last line was output, so we consider a blank to be part of the output */
bool lastWasOutput_;
std::vector<Node> syntax_;
void transition(State newState);
void onNewline();
};
// Override gtest printing for lists of nodes
void PrintTo(std::vector<CLILiterateParser::Node> const & nodes, std::ostream * os);
};
}; // namespace cli_literate_parser
}; // namespace nix

View file

@ -0,0 +1,35 @@
#include "escape-string.hh"
#include "ansicolor.hh"
#include <gtest/gtest.h>
namespace nix {
TEST(EscapeString, simple) {
auto escaped = escapeString("puppy");
ASSERT_EQ(escaped, "\"puppy\"");
}
TEST(EscapeString, escaping) {
auto escaped = escapeString("\n\r\t \" \\ ${ooga booga}");
ASSERT_EQ(escaped, "\"\\n\\r\\t \\\" \\\\ \\${ooga booga}\"");
}
TEST(EscapeString, maxLength) {
auto escaped = escapeString("puppy", {.maxLength = 5});
ASSERT_EQ(escaped, "\"puppy\"");
escaped = escapeString("puppy doggy", {.maxLength = 5});
ASSERT_EQ(escaped, "\"puppy\" «6 bytes elided»");
}
TEST(EscapeString, ansiColors) {
auto escaped = escapeString("puppy doggy", {.maxLength = 5, .ansiColors = true});
ASSERT_EQ(escaped, ANSI_MAGENTA "\"puppy\" " ANSI_FAINT "«6 bytes elided»" ANSI_NORMAL);
}
TEST(EscapeString, escapeNonPrinting) {
auto escaped = escapeString("puppy\u0005doggy", {.escapeNonPrinting = true});
ASSERT_EQ(escaped, "\"puppy\\x05doggy\"");
}
} // namespace nix

View file

@ -0,0 +1,72 @@
#include "config.hh"
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include <sstream>
using testing::Eq;
namespace nix {
class PathsSettingTestConfig : public Config
{
public:
PathsSettingTestConfig()
: Config()
{ }
PathsSetting paths{this, Paths(), "paths", "documentation"};
};
struct PathsSettingTest : public ::testing::Test {
public:
PathsSettingTestConfig mkConfig()
{
return PathsSettingTestConfig();
}
};
TEST_F(PathsSettingTest, parse) {
auto config = mkConfig();
// Not an absolute path:
ASSERT_THROW(config.paths.parse("puppy.nix"), Error);
ASSERT_THAT(
config.paths.parse("/puppy.nix"),
Eq<Paths>({"/puppy.nix"})
);
// Splits on whitespace:
ASSERT_THAT(
config.paths.parse("/puppy.nix /doggy.nix"),
Eq<Paths>({"/puppy.nix", "/doggy.nix"})
);
// Splits on _any_ whitespace:
ASSERT_THAT(
config.paths.parse("/puppy.nix \t /doggy.nix\n\n\n/borzoi.nix\r/goldie.nix"),
Eq<Paths>({"/puppy.nix", "/doggy.nix", "/borzoi.nix", "/goldie.nix"})
);
// Canonicizes paths:
ASSERT_THAT(
config.paths.parse("/puppy/../doggy.nix"),
Eq<Paths>({"/doggy.nix"})
);
}
TEST_F(PathsSettingTest, bool) {
auto config = mkConfig();
// No paths:
ASSERT_FALSE(config.paths);
// Set a path:
config.set("paths", "/puppy.nix");
// Now there are paths:
ASSERT_TRUE(config.paths);
// Multiple paths count too:
config.set("paths", "/puppy.nix /doggy.nix");
ASSERT_TRUE(config.paths);
}
} // namespace nix

View file

@ -39,12 +39,14 @@ libutil_tests_sources = files(
'libutil/closure.cc',
'libutil/compression.cc',
'libutil/config.cc',
'libutil/escape-string.cc',
'libutil/git.cc',
'libutil/hash.cc',
'libutil/hilite.cc',
'libutil/json-utils.cc',
'libutil/logging.cc',
'libutil/lru-cache.cc',
'libutil/paths-setting.cc',
'libutil/pool.cc',
'libutil/references.cc',
'libutil/suggestions.cc',