From a0e38c16bc7773d0fc62771a9935e079c00899ef Mon Sep 17 00:00:00 2001
From: Tuomas Tynkkynen <tuomas@tuxera.com>
Date: Sun, 25 Feb 2018 23:51:47 +0200
Subject: [PATCH] libexpr: Recognize newline in more places in lexer

Flex's regexes have an annoying feature: the dot matches everything
except a newline. This causes problems for expressions like:

"${0}\
"

where the backslash-newline combination matches this rule instead of the
intended one mentioned in the comment:

    <STRING>\$|\\|\$\\ {
                    /* This can only occur when we reach EOF, otherwise the above
                    (...|\$[^\{\"\\]|\\.|\$\\.)+ would have triggered.
                    This is technically invalid, but we leave the problem to the
                    parser who fails with exact location. */
                    return STR;
                }
However, the parser actually accepts the resulting token sequence
('"' DOLLAR_CURLY 0 '}' STR '"'), which is a problem because the lexer
rule didn't assign anything to yylval. Ultimately this leads to a crash
when dereferencing a NULL pointer in ExprConcatStrings::bindVars().

The fix does change the syntax of the language in some corner cases
but I think it's only turning previously invalid (or crashing) syntax
to valid syntax. E.g.

"a\
b"

and

''a''\
b''

were previously syntax errors but now both result in "a\nb".

Found by afl-fuzz.
---
 src/libexpr/lexer.l                          | 9 +++++----
 tests/lang/eval-okay-backslash-newline-1.exp | 1 +
 tests/lang/eval-okay-backslash-newline-1.nix | 2 ++
 tests/lang/eval-okay-backslash-newline-2.exp | 1 +
 tests/lang/eval-okay-backslash-newline-2.nix | 2 ++
 5 files changed, 11 insertions(+), 4 deletions(-)
 create mode 100644 tests/lang/eval-okay-backslash-newline-1.exp
 create mode 100644 tests/lang/eval-okay-backslash-newline-1.nix
 create mode 100644 tests/lang/eval-okay-backslash-newline-2.exp
 create mode 100644 tests/lang/eval-okay-backslash-newline-2.nix

diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index e5e01fb58..1e9c29afa 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -85,6 +85,7 @@ static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length)
 %}
 
 
+ANY         .|\n
 ID          [a-zA-Z\_][a-zA-Z0-9\_\'\-]*
 INT         [0-9]+
 FLOAT       (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)?
@@ -146,8 +147,8 @@ or          { return OR_KW; }
 <INITIAL,INSIDE_DOLLAR_CURLY>\" {
                 PUSH_STATE(STRING); return '"';
               }
-<STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)*\$/\" |
-<STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)+ {
+<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" |
+<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ {
                 /* It is impossible to match strings ending with '$' with one
                    regex because trailing contexts are only valid at the end
                    of a rule. (A sane but undocumented limitation.) */
@@ -178,7 +179,7 @@ or          { return OR_KW; }
                    yylval->e = new ExprIndStr("''");
                    return IND_STR;
                  }
-<IND_STRING>\'\'\\. {
+<IND_STRING>\'\'\\{ANY} {
                    yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2);
                    return IND_STR;
                  }
@@ -208,7 +209,7 @@ or          { return OR_KW; }
 \#[^\r\n]*    /* single-line comments */
 \/\*([^*]|\*+[^*/])*\*+\/  /* long comments */
 
-.           return yytext[0];
+{ANY}           return yytext[0];
 
 }
 
diff --git a/tests/lang/eval-okay-backslash-newline-1.exp b/tests/lang/eval-okay-backslash-newline-1.exp
new file mode 100644
index 000000000..3e754364c
--- /dev/null
+++ b/tests/lang/eval-okay-backslash-newline-1.exp
@@ -0,0 +1 @@
+"a\nb"
diff --git a/tests/lang/eval-okay-backslash-newline-1.nix b/tests/lang/eval-okay-backslash-newline-1.nix
new file mode 100644
index 000000000..7fef3dddd
--- /dev/null
+++ b/tests/lang/eval-okay-backslash-newline-1.nix
@@ -0,0 +1,2 @@
+"a\
+b"
diff --git a/tests/lang/eval-okay-backslash-newline-2.exp b/tests/lang/eval-okay-backslash-newline-2.exp
new file mode 100644
index 000000000..3e754364c
--- /dev/null
+++ b/tests/lang/eval-okay-backslash-newline-2.exp
@@ -0,0 +1 @@
+"a\nb"
diff --git a/tests/lang/eval-okay-backslash-newline-2.nix b/tests/lang/eval-okay-backslash-newline-2.nix
new file mode 100644
index 000000000..35ddf495c
--- /dev/null
+++ b/tests/lang/eval-okay-backslash-newline-2.nix
@@ -0,0 +1,2 @@
+''a''\
+b''