From 143b73f52dabb35cd56551c24caef95466bce488 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Fri, 29 Apr 2022 11:14:08 +0200 Subject: [PATCH] tests: Distinguish crashes from expected failures --- tests/lang.sh | 58 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/tests/lang.sh b/tests/lang.sh index 61bb444ba..6cb4b6e2b 100644 --- a/tests/lang.sh +++ b/tests/lang.sh @@ -10,32 +10,49 @@ nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw set +x fail=0 +just_failed=0 + +safe_expect () { + expected=$1 name=$2 msg=$3 cmd=$4 + just_failed=0 + res=0 + eval "$cmd" || res=$? + + if [ $res -ge 2 ]; then + [ $res -le 128 ] || { echo "FAIL: $name returned unexpected code '$res' (should be 0 or 1)"; just_failed=1; } + [ $res -gt 128 ] || { echo "FAIL: $name crashed with $res ($(kill -l $(($res - 128))))"; just_failed=1; } + elif [ $expected = "fail" ]; then + [ $res -ne 0 ] || { echo "FAIL: $name $msg"; just_failed=1; } + elif [ $expected = "pass" ]; then + [ $res -eq 0 ] || { echo "FAIL: $name $msg"; just_failed=1; } + else + echo "usage: safe_expect [pass|fail] name msg command" + exit 1 + fi + + [ $just_failed -eq 0 ] || fail=1 +} + for i in lang/parse-fail-*.nix; do echo "parsing $i (should fail)"; i=$(basename $i .nix) - if nix-instantiate --parse - < lang/$i.nix; then - echo "FAIL: $i shouldn't parse" - fail=1 - fi + safe_expect fail "$i" "shouldn't parse" \ + "nix-instantiate --parse - < lang/$i.nix" done for i in lang/parse-okay-*.nix; do echo "parsing $i (should succeed)"; i=$(basename $i .nix) - if ! nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then - echo "FAIL: $i should parse" - fail=1 - fi + safe_expect pass "$i" "should parse" \ + "nix-instantiate --parse - < lang/$i.nix > lang/$i.out" done for i in lang/eval-fail-*.nix; do echo "evaluating $i (should fail)"; i=$(basename $i .nix) - if nix-instantiate --eval lang/$i.nix; then - echo "FAIL: $i shouldn't evaluate" - fail=1 - fi + safe_expect fail "$i" "shouldn't evaluate" \ + "nix-instantiate --eval lang/$i.nix" done for i in lang/eval-okay-*.nix; do @@ -45,23 +62,20 @@ for i in lang/eval-okay-*.nix; do if test -e lang/$i.exp; then flags= if test -e lang/$i.flags; then - flags=$(cat lang/$i.flags) + flags='$'"(cat lang/$i.flags)" # because vi does not highlight "\$(...)" properly ;-). fi - if ! NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then - echo "FAIL: $i should evaluate" - fail=1 - elif ! diff lang/$i.out lang/$i.exp; then + safe_expect pass "$i" "should evaluate" \ + "NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out" + if [ $just_failed -eq 0 ] && ! diff lang/$i.out lang/$i.exp; then echo "FAIL: evaluation result of $i not as expected" fail=1 fi fi if test -e lang/$i.exp.xml; then - if ! nix-instantiate --eval --xml --no-location --strict \ - lang/$i.nix > lang/$i.out.xml; then - echo "FAIL: $i should evaluate" - fail=1 - elif ! cmp -s lang/$i.out.xml lang/$i.exp.xml; then + safe_expect pass "$i" "should evaluate" \ + "nix-instantiate --eval --xml --no-location --strict lang/$i.nix > lang/$i.out.xml" + if [ $just_failed -eq 0 ] && ! cmp -s lang/$i.out.xml lang/$i.exp.xml; then echo "FAIL: XML evaluation result of $i not as expected" fail=1 fi