From 143b73f52dabb35cd56551c24caef95466bce488 Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <guillaume.maudoux@tweag.io>
Date: Fri, 29 Apr 2022 11:14:08 +0200
Subject: [PATCH 1/3] tests: Distinguish crashes from expected failures

---
 tests/lang.sh | 58 ++++++++++++++++++++++++++++++++-------------------
 1 file changed, 36 insertions(+), 22 deletions(-)

diff --git a/tests/lang.sh b/tests/lang.sh
index 61bb444ba..6cb4b6e2b 100644
--- a/tests/lang.sh
+++ b/tests/lang.sh
@@ -10,32 +10,49 @@ nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw
 set +x
 
 fail=0
+just_failed=0
+
+safe_expect () {
+    expected=$1 name=$2 msg=$3 cmd=$4
+    just_failed=0
+    res=0
+    eval "$cmd" || res=$?
+
+    if [ $res -ge 2 ]; then
+        [ $res -le 128 ] || { echo "FAIL: $name returned unexpected code '$res' (should be 0 or 1)"; just_failed=1; }
+        [ $res -gt 128 ] || { echo "FAIL: $name crashed with $res ($(kill -l $(($res - 128))))"; just_failed=1; }
+    elif [ $expected = "fail" ]; then
+        [ $res -ne 0 ] || { echo "FAIL: $name $msg"; just_failed=1; }
+    elif [ $expected = "pass" ]; then
+        [ $res -eq 0 ] || { echo "FAIL: $name $msg"; just_failed=1; }
+    else
+        echo "usage: safe_expect [pass|fail] name msg command"
+        exit 1
+    fi
+
+    [ $just_failed -eq 0 ] || fail=1
+}
+
 
 for i in lang/parse-fail-*.nix; do
     echo "parsing $i (should fail)";
     i=$(basename $i .nix)
-    if nix-instantiate --parse - < lang/$i.nix; then
-        echo "FAIL: $i shouldn't parse"
-        fail=1
-    fi
+    safe_expect fail "$i" "shouldn't parse" \
+        "nix-instantiate --parse - < lang/$i.nix"
 done
 
 for i in lang/parse-okay-*.nix; do
     echo "parsing $i (should succeed)";
     i=$(basename $i .nix)
-    if ! nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then
-        echo "FAIL: $i should parse"
-        fail=1
-    fi
+    safe_expect pass "$i" "should parse" \
+        "nix-instantiate --parse - < lang/$i.nix > lang/$i.out"
 done
 
 for i in lang/eval-fail-*.nix; do
     echo "evaluating $i (should fail)";
     i=$(basename $i .nix)
-    if nix-instantiate --eval lang/$i.nix; then
-        echo "FAIL: $i shouldn't evaluate"
-        fail=1
-    fi
+    safe_expect fail "$i" "shouldn't evaluate" \
+        "nix-instantiate --eval lang/$i.nix"
 done
 
 for i in lang/eval-okay-*.nix; do
@@ -45,23 +62,20 @@ for i in lang/eval-okay-*.nix; do
     if test -e lang/$i.exp; then
         flags=
         if test -e lang/$i.flags; then
-            flags=$(cat lang/$i.flags)
+            flags='$'"(cat lang/$i.flags)" # because vi does not highlight "\$(...)" properly ;-).
         fi
-        if ! NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then
-            echo "FAIL: $i should evaluate"
-            fail=1
-        elif ! diff lang/$i.out lang/$i.exp; then
+        safe_expect pass "$i" "should evaluate" \
+            "NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out"
+        if [ $just_failed -eq 0 ] && ! diff lang/$i.out lang/$i.exp; then
             echo "FAIL: evaluation result of $i not as expected"
             fail=1
         fi
     fi
 
     if test -e lang/$i.exp.xml; then
-        if ! nix-instantiate --eval --xml --no-location --strict \
-                lang/$i.nix > lang/$i.out.xml; then
-            echo "FAIL: $i should evaluate"
-            fail=1
-        elif ! cmp -s lang/$i.out.xml lang/$i.exp.xml; then
+        safe_expect pass "$i" "should evaluate" \
+            "nix-instantiate --eval --xml --no-location --strict lang/$i.nix > lang/$i.out.xml"
+        if [ $just_failed -eq 0 ] && ! cmp -s lang/$i.out.xml lang/$i.exp.xml; then
             echo "FAIL: XML evaluation result of $i not as expected"
             fail=1
         fi

From 4845886bed18bb12dbfb7bf909b24cc49abd8da6 Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <guillaume.maudoux@tweag.io>
Date: Mon, 2 May 2022 14:22:00 +0200
Subject: [PATCH 2/3] Revert "tests: Distinguish crashes from expected
 failures"

This reverts commit 143b73f52dabb35cd56551c24caef95466bce488.
---
 tests/lang.sh | 58 +++++++++++++++++++--------------------------------
 1 file changed, 22 insertions(+), 36 deletions(-)

diff --git a/tests/lang.sh b/tests/lang.sh
index 6cb4b6e2b..61bb444ba 100644
--- a/tests/lang.sh
+++ b/tests/lang.sh
@@ -10,49 +10,32 @@ nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw
 set +x
 
 fail=0
-just_failed=0
-
-safe_expect () {
-    expected=$1 name=$2 msg=$3 cmd=$4
-    just_failed=0
-    res=0
-    eval "$cmd" || res=$?
-
-    if [ $res -ge 2 ]; then
-        [ $res -le 128 ] || { echo "FAIL: $name returned unexpected code '$res' (should be 0 or 1)"; just_failed=1; }
-        [ $res -gt 128 ] || { echo "FAIL: $name crashed with $res ($(kill -l $(($res - 128))))"; just_failed=1; }
-    elif [ $expected = "fail" ]; then
-        [ $res -ne 0 ] || { echo "FAIL: $name $msg"; just_failed=1; }
-    elif [ $expected = "pass" ]; then
-        [ $res -eq 0 ] || { echo "FAIL: $name $msg"; just_failed=1; }
-    else
-        echo "usage: safe_expect [pass|fail] name msg command"
-        exit 1
-    fi
-
-    [ $just_failed -eq 0 ] || fail=1
-}
-
 
 for i in lang/parse-fail-*.nix; do
     echo "parsing $i (should fail)";
     i=$(basename $i .nix)
-    safe_expect fail "$i" "shouldn't parse" \
-        "nix-instantiate --parse - < lang/$i.nix"
+    if nix-instantiate --parse - < lang/$i.nix; then
+        echo "FAIL: $i shouldn't parse"
+        fail=1
+    fi
 done
 
 for i in lang/parse-okay-*.nix; do
     echo "parsing $i (should succeed)";
     i=$(basename $i .nix)
-    safe_expect pass "$i" "should parse" \
-        "nix-instantiate --parse - < lang/$i.nix > lang/$i.out"
+    if ! nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then
+        echo "FAIL: $i should parse"
+        fail=1
+    fi
 done
 
 for i in lang/eval-fail-*.nix; do
     echo "evaluating $i (should fail)";
     i=$(basename $i .nix)
-    safe_expect fail "$i" "shouldn't evaluate" \
-        "nix-instantiate --eval lang/$i.nix"
+    if nix-instantiate --eval lang/$i.nix; then
+        echo "FAIL: $i shouldn't evaluate"
+        fail=1
+    fi
 done
 
 for i in lang/eval-okay-*.nix; do
@@ -62,20 +45,23 @@ for i in lang/eval-okay-*.nix; do
     if test -e lang/$i.exp; then
         flags=
         if test -e lang/$i.flags; then
-            flags='$'"(cat lang/$i.flags)" # because vi does not highlight "\$(...)" properly ;-).
+            flags=$(cat lang/$i.flags)
         fi
-        safe_expect pass "$i" "should evaluate" \
-            "NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out"
-        if [ $just_failed -eq 0 ] && ! diff lang/$i.out lang/$i.exp; then
+        if ! NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then
+            echo "FAIL: $i should evaluate"
+            fail=1
+        elif ! diff lang/$i.out lang/$i.exp; then
             echo "FAIL: evaluation result of $i not as expected"
             fail=1
         fi
     fi
 
     if test -e lang/$i.exp.xml; then
-        safe_expect pass "$i" "should evaluate" \
-            "nix-instantiate --eval --xml --no-location --strict lang/$i.nix > lang/$i.out.xml"
-        if [ $just_failed -eq 0 ] && ! cmp -s lang/$i.out.xml lang/$i.exp.xml; then
+        if ! nix-instantiate --eval --xml --no-location --strict \
+                lang/$i.nix > lang/$i.out.xml; then
+            echo "FAIL: $i should evaluate"
+            fail=1
+        elif ! cmp -s lang/$i.out.xml lang/$i.exp.xml; then
             echo "FAIL: XML evaluation result of $i not as expected"
             fail=1
         fi

From 275f8561eb80a0f341fa2a03d731b239b1783da0 Mon Sep 17 00:00:00 2001
From: Guillaume Maudoux <guillaume.maudoux@tweag.io>
Date: Mon, 2 May 2022 15:12:50 +0200
Subject: [PATCH 3/3] tests/lang: Distinguish crashes from expected failures

---
 tests/common.sh.in | 11 ++++++-----
 tests/lang.sh      | 11 ++++++-----
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/tests/common.sh.in b/tests/common.sh.in
index 8ce28d318..6cb579e0d 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -157,11 +157,12 @@ expect() {
     local expected res
     expected="$1"
     shift
-    set +e
-    "$@"
-    res="$?"
-    set -e
-    [[ $res -eq $expected ]]
+    "$@" || res="$?"
+    if [[ $res -ne $expected ]]; then
+        echo "Expected '$expected' but got '$res' while running '$*'"
+        return 1
+    fi
+    return 0
 }
 
 needLocalStore() {
diff --git a/tests/lang.sh b/tests/lang.sh
index 61bb444ba..f09eaeb31 100644
--- a/tests/lang.sh
+++ b/tests/lang.sh
@@ -4,6 +4,7 @@ export TEST_VAR=foo # for eval-okay-getenv.nix
 export NIX_REMOTE=dummy://
 
 nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grep -q Hello
+nix-instantiate --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1
 (! nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grep -q Hello)
 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' 2>&1 | grep -q Hello
 
@@ -14,7 +15,7 @@ fail=0
 for i in lang/parse-fail-*.nix; do
     echo "parsing $i (should fail)";
     i=$(basename $i .nix)
-    if nix-instantiate --parse - < lang/$i.nix; then
+    if ! expect 1 nix-instantiate --parse - < lang/$i.nix; then
         echo "FAIL: $i shouldn't parse"
         fail=1
     fi
@@ -23,7 +24,7 @@ done
 for i in lang/parse-okay-*.nix; do
     echo "parsing $i (should succeed)";
     i=$(basename $i .nix)
-    if ! nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then
+    if ! expect 0 nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then
         echo "FAIL: $i should parse"
         fail=1
     fi
@@ -32,7 +33,7 @@ done
 for i in lang/eval-fail-*.nix; do
     echo "evaluating $i (should fail)";
     i=$(basename $i .nix)
-    if nix-instantiate --eval lang/$i.nix; then
+    if ! expect 1 nix-instantiate --eval lang/$i.nix; then
         echo "FAIL: $i shouldn't evaluate"
         fail=1
     fi
@@ -47,7 +48,7 @@ for i in lang/eval-okay-*.nix; do
         if test -e lang/$i.flags; then
             flags=$(cat lang/$i.flags)
         fi
-        if ! NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then
+        if ! expect 0 env NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then
             echo "FAIL: $i should evaluate"
             fail=1
         elif ! diff lang/$i.out lang/$i.exp; then
@@ -57,7 +58,7 @@ for i in lang/eval-okay-*.nix; do
     fi
 
     if test -e lang/$i.exp.xml; then
-        if ! nix-instantiate --eval --xml --no-location --strict \
+        if ! expect 0 nix-instantiate --eval --xml --no-location --strict \
                 lang/$i.nix > lang/$i.out.xml; then
             echo "FAIL: $i should evaluate"
             fail=1