repl_characterization: eat newlines after commands and source-dir paths

This is because they are unrepresentable in the source files with
commentary but not in the output, so we should just eat them in
normalization. It's ok.

Change-Id: I2cb7e8b3fc7b00874885bb287cbaa200b41cb16b

tests/functional/repl_characterization/data/basic.ast

@@ -9,3 +9,9 @@ Command "command two"
 Output "output output output"
 Commentary "commentary"
 Output "output output output"
+Output ""
+Commentary "the blank below should be chomped"
+Command "command three"
+Commentary ""
+Output "meow output"
+Output ""

tests/functional/repl_characterization/data/basic.test

@@ -9,3 +9,9 @@ meow meow
   output output output
 commentary
   output output output
+
+the blank below should be chomped
+  nix-repl> command three
+
+  meow output
+

tests/functional/repl_characterization/data/basic_tidied.ast

@@ -0,0 +1,10 @@
+Command "command"
+Output "output output one"
+Output ""
+Output ""
+Output "output output two"
+Command "command two"
+Output "output output output"
+Output "output output output"
+Command "command three"
+Output "meow output"

tests/functional/repl_characterization/repl_characterization.cc

@@ -4,6 +4,7 @@
 #include <string_view>
 #include <optional>
 #include <unistd.h>
+#include <boost/algorithm/string/replace.hpp>
 
 #include "test-session.hh"
 #include "util.hh"
@@ -68,7 +69,10 @@ public:
         }
         session.close();
 
-        auto parsedOutLog = CLILiterateParser::parse(AUTOMATION_PROMPT, trimOutLog(session.outLog), 0);
+        auto replacedOutLog = boost::algorithm::replace_all_copy(session.outLog, unitTestData, "TEST_DATA");
+        auto cleanedOutLog = trimOutLog(replacedOutLog);
+
+        auto parsedOutLog = CLILiterateParser::parse(AUTOMATION_PROMPT, cleanedOutLog, 0);
 
         parsedOutLog = CLILiterateParser::tidyOutputForComparison(std::move(parsedOutLog));
         syntax = CLILiterateParser::tidyOutputForComparison(std::move(syntax));
@@ -90,6 +94,19 @@ TEST_F(ReplSessionTest, parses)
         }
         return out.str();
     });
+
+    writeTest("basic_tidied.ast", [this]() {
+        const std::string content = readFile(goldenMaster("basic.test"));
+        auto syntax = CLILiterateParser::parse(REPL_PROMPT, content);
+
+        syntax = CLILiterateParser::tidyOutputForComparison(std::move(syntax));
+
+        std::ostringstream out{};
+        for (auto & bit : syntax) {
+            out << bit.print() << "\n";
+        }
+        return out.str();
+    });
 }
 
 TEST_F(ReplSessionTest, repl_basic)