libexpr: Add experimental pipe operator

The |> operator is a reverse function operator with low binding strength
to replace lib.pipe. Implements RFC 148, see the RFC text for more
details. Closes #438.

Change-Id: I21df66e8014e0d4dd9753dd038560a2b0b7fd805

doc/manual/change-authors.yml

@@ -91,6 +91,11 @@ midnightveil:
 ncfavier:
   github: ncfavier
 
+piegames:
+  display_name: piegames
+  forgejo: piegames
+  github: piegamesde
+
 puck:
   display_name: puck
   forgejo: puck

doc/manual/rl-next/pipe-operator.md

@@ -0,0 +1,10 @@
+---
+synopsis: Pipe operator `|>` (experimental)
+issues: [fj#438]
+cls: [1654]
+category: Features
+credits: [piegames, horrors]
+---
+
+Implementation of the pipe operator (`|>`) in the language as described in [RFC 148](https://github.com/NixOS/rfcs/pull/148).
+The feature is still marked experimental, enable `--extra-experimental-features pipe-operator` to use it.

doc/manual/src/language/operators.md

@@ -26,6 +26,8 @@
 | Logical conjunction (`AND`)            | *bool* `&&` *bool*                         | left          | 12         |
 | Logical disjunction (`OR`)             | *bool* <code>\|\|</code> *bool*            | left          | 13         |
 | [Logical implication]                  | *bool* `->` *bool*                         | none          | 14         |
+| \[Experimental\] [Function piping]     | *expr* |> *func*                           | left          | 15         |
+| \[Experimental\] [Function piping]     | *expr* <| *func*                           | right         | 16         |
 
 [string]: ./values.md#type-string
 [path]: ./values.md#type-path
@@ -215,3 +217,33 @@ nix-repl> let f = x: 1; s = { func = f; }; in [ (f == f) (s == s) ]
 Equivalent to `!`*b1* `||` *b2*.
 
 [Logical implication]: #logical-implication
+
+## \[Experimental\] Function piping
+
+*This language feature is still experimental and may change at any time. Enable `--extra-experimental-features pipe-operator` to use it.*
+
+Pipes are a dedicated operator for function application, but with reverse order and a lower binding strength.
+This allows you to chain function calls together in way that is more natural to read and requires less parentheses.
+
+`a |> f b |> g` is equivalent to `g (f b a)`.
+`g <| f b <| a` is equivalent to `g (f b a)`.
+
+Example code snippet:
+
+```nix
+defaultPrefsFile = defaultPrefs
+  |> lib.mapAttrsToList (
+    key: value: ''
+      // ${value.reason}
+      pref("${key}", ${builtins.toJSON value.value});
+    ''
+  )
+  |> lib.concatStringsSep "\n"
+  |> pkgs.writeText "nixos-default-prefs.js";
+```
+
+Note how `mapAttrsToList` is called with two arguments (the lambda and `defaultPrefs`),
+but moving the last argument in front of the rest improves the reading flow.
+This is common for functions with long first argument, including all `map`-like functions.
+
+[Function piping]: #experimental-function-piping

src/libexpr/eval.cc

@@ -2795,20 +2795,20 @@ Expr & EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<Sta
 }
 
 
-Expr & EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
+Expr & EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv, const ExperimentalFeatureSettings & xpSettings)
 {
     // NOTE this method (and parseStdin) must take care to *fully copy* their input
     // into their respective Pos::Origin until the parser stops overwriting its input
     // data.
     auto s = make_ref<std::string>(s_);
     s_.append("\0\0", 2);
-    return *parse(s_.data(), s_.size(), Pos::String{.source = s}, basePath, staticEnv);
+    return *parse(s_.data(), s_.size(), Pos::String{.source = s}, basePath, staticEnv, xpSettings);
 }
 
 
-Expr & EvalState::parseExprFromString(std::string s, const SourcePath & basePath)
+Expr & EvalState::parseExprFromString(std::string s, const SourcePath & basePath, const ExperimentalFeatureSettings & xpSettings)
 {
-    return parseExprFromString(std::move(s), basePath, staticBaseEnv);
+    return parseExprFromString(std::move(s), basePath, staticBaseEnv, xpSettings);
 }
 
 

src/libexpr/eval.hh

@@ -342,8 +342,8 @@ public:
     /**
      * Parse a Nix expression from the specified string.
      */
-    Expr & parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv);
+    Expr & parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
-    Expr & parseExprFromString(std::string s, const SourcePath & basePath);
+    Expr & parseExprFromString(std::string s, const SourcePath & basePath, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
     Expr & parseStdin();
 
@@ -566,7 +566,8 @@ private:
         size_t length,
         Pos::Origin origin,
         const SourcePath & basePath,
-        std::shared_ptr<StaticEnv> & staticEnv);
+        std::shared_ptr<StaticEnv> & staticEnv,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
     /**
      * Current Nix call stack depth, used with `max-call-depth` setting to throw stack overflow hopefully before we run out of system stack.

src/libexpr/parser/grammar.hh

@@ -434,6 +434,8 @@ struct op {
     struct and_       : _op<TAO_PEGTL_STRING("&&"), 12> {};
     struct or_        : _op<TAO_PEGTL_STRING("||"), 13> {};
     struct implies    : _op<TAO_PEGTL_STRING("->"), 14, kind::rightAssoc> {};
+    struct pipe_right : _op<TAO_PEGTL_STRING("|>"), 15> {};
+    struct pipe_left  : _op<TAO_PEGTL_STRING("<|"), 16, kind::rightAssoc> {};
 };
 
 struct _expr {
@@ -521,6 +523,7 @@ struct _expr {
         app
     > {};
 
+    /* Order matters here. The order is the parsing order, not the precedence order: '<=' must be parsed before '<'. */
     struct _binary_operator : sor<
         operator_<op::implies>,
         operator_<op::update>,
@@ -529,6 +532,8 @@ struct _expr {
         operator_<op::minus>,
         operator_<op::mul>,
         operator_<op::div>,
+        operator_<op::pipe_right>,
+        operator_<op::pipe_left>,
         operator_<op::less_eq>,
         operator_<op::greater_eq>,
         operator_<op::less>,
@@ -649,6 +654,8 @@ struct operator_semantics {
             grammar::op::minus,
             grammar::op::mul,
             grammar::op::div,
+            grammar::op::pipe_right,
+            grammar::op::pipe_left,
             has_attr
         > op;
     };

src/libexpr/parser/parser.cc

@@ -114,6 +114,29 @@ struct ExprState
         return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(fn), std::move(args));
     }
 
+    std::unique_ptr<Expr> pipe(PosIdx pos, State & state, bool flip = false)
+    {
+        if (!state.xpSettings.isEnabled(Xp::PipeOperator))
+            throw ParseError({
+                .msg = HintFmt("Pipe operator is disabled"),
+                .pos = state.positions[pos]
+            });
+
+        // Reverse the order compared to normal function application: arg |> fn
+        std::unique_ptr<Expr> fn, arg;
+        if (flip) {
+            fn = popExprOnly();
+            arg = popExprOnly();
+        } else {
+            arg = popExprOnly();
+            fn = popExprOnly();
+        }
+        std::vector<std::unique_ptr<Expr>> args{1};
+        args[0] = std::move(arg);
+
+        return std::make_unique<ExprCall>(pos, std::move(fn), std::move(args));
+    }
+
     std::unique_ptr<Expr> order(PosIdx pos, bool less, State & state)
     {
         return call(pos, state.s.lessThan, !less);
@@ -163,6 +186,8 @@ struct ExprState
                 [&] (Op::concat)      { return applyBinary<ExprOpConcatLists>(pos); },
                 [&] (has_attr & a)    { return applyUnary<ExprOpHasAttr>(std::move(a.path)); },
                 [&] (Op::unary_minus) { return negate(pos, state); },
+                [&] (Op::pipe_right)  { return pipe(pos, state, true); },
+                [&] (Op::pipe_left)   { return pipe(pos, state); },
             })(op)
         };
     }
@@ -630,7 +655,7 @@ template<> struct BuildAST<grammar::expr::path> : p::maybe_nothing {};
 
 template<> struct BuildAST<grammar::expr::uri> {
     static void apply(const auto & in, ExprState & s, State & ps) {
-       static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals);
+       bool noURLLiterals = ps.xpSettings.isEnabled(Xp::NoUrlLiterals);
        if (noURLLiterals)
            throw ParseError({
                .msg = HintFmt("URL literals are disabled"),
@@ -831,7 +856,8 @@ Expr * EvalState::parse(
     size_t length,
     Pos::Origin origin,
     const SourcePath & basePath,
-    std::shared_ptr<StaticEnv> & staticEnv)
+    std::shared_ptr<StaticEnv> & staticEnv,
+    const ExperimentalFeatureSettings & xpSettings)
 {
     parser::State s = {
         symbols,
@@ -839,6 +865,7 @@ Expr * EvalState::parse(
         basePath,
         positions.addOrigin(origin, length),
         exprSymbols,
+        xpSettings
     };
     parser::ExprState x;
 

src/libexpr/parser/state.hh

@@ -19,6 +19,7 @@ struct State
     SourcePath basePath;
     PosTable::Origin origin;
     const Expr::AstSymbols & s;
+    const ExperimentalFeatureSettings & xpSettings;
 
     void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos);
     void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos);

src/libutil/experimental-features.cc

@@ -166,6 +166,16 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
             may confuse external tooling.
         )",
     },
+    {
+        .tag = Xp::PipeOperator,
+        .name = "pipe-operator",
+        .description = R"(
+            Enable new operators for function application to "pipe" arguments through a chain of functions similar to `lib.pipe`.
+            This implementation is based on Nix [RFC 148](https://github.com/NixOS/rfcs/pull/148).
+
+            Tracking issue: https://git.lix.systems/lix-project/lix/issues/438
+        )",
+    },
     {
         .tag = Xp::FetchClosure,
         .name = "fetch-closure",

src/libutil/experimental-features.hh

@@ -21,6 +21,7 @@ enum struct ExperimentalFeature
     NixCommand,
     RecursiveNix,
     NoUrlLiterals,
+    PipeOperator,
     FetchClosure,
     ReplFlake,
     AutoAllocateUids,

tests/unit/libexpr-support/tests/libexpr.hh

@@ -26,9 +26,9 @@ namespace nix {
                 , state({}, store)
             {
             }
-            Value eval(std::string input, bool forceValue = true) {
+            Value eval(std::string input, bool forceValue = true, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) {
                 Value v;
-                Expr & e = state.parseExprFromString(input, state.rootPath(CanonPath::root));
+                Expr & e = state.parseExprFromString(input, state.rootPath(CanonPath::root), xpSettings);
                 state.eval(e, v);
                 if (forceValue)
                     state.forceValue(v, noPos);

tests/unit/libexpr/trivial.cc

@@ -59,6 +59,11 @@ namespace nix {
         ASSERT_THAT(v, IsFloatEq(1.234));
     }
 
+    TEST_F(TrivialExpressionTest, pointfloat) {
+        auto v = eval(".234");
+        ASSERT_THAT(v, IsFloatEq(0.234));
+    }
+
     TEST_F(TrivialExpressionTest, updateAttrs) {
         auto v = eval("{ a = 1; } // { b = 2; a = 3; }");
         ASSERT_THAT(v, IsAttrsOfSize(2));
@@ -81,6 +86,18 @@ namespace nix {
         ASSERT_THAT(v, IsTrue());
     }
 
+    TEST_F(TrivialExpressionTest, urlLiteral) {
+        auto v = eval("https://nixos.org");
+        ASSERT_THAT(v, IsStringEq("https://nixos.org"));
+    }
+
+    TEST_F(TrivialExpressionTest, noUrlLiteral) {
+        ExperimentalFeatureSettings mockXpSettings;
+        mockXpSettings.set("experimental-features", "no-url-literals");
+
+        ASSERT_THROW(eval("https://nixos.org", true, mockXpSettings), Error);
+    }
+
     TEST_F(TrivialExpressionTest, withFound) {
         auto v = eval("with { a = 23; }; a");
         ASSERT_THAT(v, IsIntEq(23));
@@ -193,4 +210,40 @@ namespace nix {
     TEST_F(TrivialExpressionTest, orCantBeUsed) {
         ASSERT_THROW(eval("let or = 1; in or"), Error);
     }
+
+    // pipes are gated behind an experimental feature flag
+    TEST_F(TrivialExpressionTest, pipeDisabled) {
+        ASSERT_THROW(eval("let add = l: r: l + r; in ''a'' |> add ''b''"), Error);
+        ASSERT_THROW(eval("let add = l: r: l + r; in ''a'' <| add ''b''"), Error);
+    }
+
+    TEST_F(TrivialExpressionTest, pipeRight) {
+        ExperimentalFeatureSettings mockXpSettings;
+        mockXpSettings.set("experimental-features", "pipe-operator");
+
+        auto v = eval("let add = l: r: l + r; in ''a'' |> add ''b''", true, mockXpSettings);
+        ASSERT_THAT(v, IsStringEq("ba"));
+        v = eval("let add = l: r: l + r; in ''a'' |> add ''b'' |> add ''c''", true, mockXpSettings);
+        ASSERT_THAT(v, IsStringEq("cba"));
+    }
+
+    TEST_F(TrivialExpressionTest, pipeLeft) {
+        ExperimentalFeatureSettings mockXpSettings;
+        mockXpSettings.set("experimental-features", "pipe-operator");
+
+        auto v = eval("let add = l: r: l + r; in add ''a'' <| ''b''", true, mockXpSettings);
+        ASSERT_THAT(v, IsStringEq("ab"));
+        v = eval("let add = l: r: l + r; in add ''a'' <| add ''b'' <| ''c''", true, mockXpSettings);
+        ASSERT_THAT(v, IsStringEq("abc"));
+    }
+
+    TEST_F(TrivialExpressionTest, pipeMixed) {
+        ExperimentalFeatureSettings mockXpSettings;
+        mockXpSettings.set("experimental-features", "pipe-operator");
+
+        auto v = eval("let add = l: r: l + r; in add ''a'' <| ''b'' |> add ''c''", true, mockXpSettings);
+        ASSERT_THAT(v, IsStringEq("acb"));
+        v = eval("let add = l: r: l + r; in ''a'' |> add <| ''c''", true, mockXpSettings);
+        ASSERT_THAT(v, IsStringEq("ac"));
+    }
 } /* namespace nix */