diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 425ff16c1..0c6952929 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -18,6 +18,7 @@
 
 #include <variant>
 
+#include "finally.hh"
 #include "util.hh"
 
 #include "nixexpr.hh"
@@ -59,7 +60,6 @@ namespace nix {
         Expr * result;
         SourcePath basePath;
         PosTable::Origin origin;
-        std::optional<ErrorInfo> error;
     };
 
 }
@@ -314,10 +314,10 @@ static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data)
 
 void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error)
 {
-    data->error = {
+    throw ParseError({
         .msg = hintfmt(error),
         .errPos = data->state.positions[makeCurPos(*loc, data)]
-    };
+    });
 }
 
 
@@ -685,11 +685,10 @@ Expr * EvalState::parse(
     };
 
     yylex_init(&scanner);
-    yy_scan_buffer(text, length, scanner);
-    int res = yyparse(scanner, &data);
-    yylex_destroy(scanner);
+    Finally _destroy([&] { yylex_destroy(scanner); });
 
-    if (res) throw ParseError(data.error.value());
+    yy_scan_buffer(text, length, scanner);
+    yyparse(scanner, &data);
 
     data.result->bindVars(*this, staticEnv);