Expanded test suite

* Lang now verifies errors and parse output

* Some new miscellaneous tests

* Easy way to update the tests

* Document workflow in manual

* Use `!` not `~` as separater char for sed

  It is confusing to use `~` when we are talking about paths and home
  directories!

* Test test suite itself (`test/lang-test/infra.sh`)

Additionally, run shellcheck on `tests/lang.sh` to help ensure it is
correct, now that is is more complex.

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>

.gitignore

@@ -95,6 +95,7 @@ perl/Makefile.config
 # /tests/lang/
 /tests/lang/*.out
 /tests/lang/*.out.xml
+/tests/lang/*.err
 /tests/lang/*.ast
 
 /perl/lib/Nix/Config.pm

doc/manual/src/contributing/testing.md

@@ -86,6 +86,31 @@ GNU gdb (GDB) 12.1
 One can debug the Nix invocation in all the usual ways.
 For example, enter `run` to start the Nix invocation.
 
+### Characterization testing
+
+Occasionally, Nix utilizes a technique called [Characterization Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests.
+This technique is to include the exact output/behavior of a former version of Nix in a test in order to check that Nix continues to produce the same behavior going forward.
+
+For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered.
+
+It is frequently useful to regenerate the expected output.
+To do that, rerun the failed test with `_NIX_TEST_ACCEPT=1`.
+(At least, this is the convention we've used for `tests/lang.sh`.
+If we add more characterization testing we should always strive to be consistent.)
+
+An interesting situation to document is the case when these tests are "overfitted".
+The language tests are, again, an example of this.
+The expected successful output of evaluation is supposed to be highly stable – we do not intend to make breaking changes to (the stable parts of) the Nix language.
+However, the errors and warnings during evaluation (successful or not) are not stable in this way.
+We are free to change how they are displayed at any time.
+
+It may be surprising that we would test non-normative behavior like diagnostic outputs.
+Diagnostic outputs are indeed not a stable interface, but they still are important to users.
+By recording the expected output, the test suite guards against accidental changes, and ensure the *result* (not just the code that implements it) of the diagnostic code paths are under code review.
+Regressions are caught, and improvements always show up in code review.
+
+To ensure that characterization testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`.
+
 ## Integration tests
 
 The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute.

tests/dependencies.sh

@@ -15,6 +15,9 @@ if test -n "$dot"; then
     $dot < $TEST_ROOT/graph
 fi
 
+# Test GraphML graph generation
+nix-store -q --graphml "$drvPath" > $TEST_ROOT/graphml
+
 outPath=$(nix-store -rvv "$drvPath") || fail "build failed"
 
 # Test Graphviz graph generation.

tests/lang-test-infra.sh

@@ -0,0 +1,86 @@
+# Test the function for lang.sh
+source common.sh
+
+source lang/framework.sh
+
+# We are testing this, so don't want outside world to affect us.
+unset _NIX_TEST_ACCEPT
+
+# We'll only modify this in subshells so we don't need to reset it.
+badDiff=0
+
+# matches non-empty
+echo Hi! > "$TEST_ROOT/got"
+cp "$TEST_ROOT/got" "$TEST_ROOT/expected"
+(
+  diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected"
+  (( "$badDiff" == 0 ))
+)
+
+# matches empty, non-existant file is the same as empty file
+echo -n > "$TEST_ROOT/got"
+(
+  diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist"
+  (( "$badDiff" == 0 ))
+)
+
+# doesn't matches non-empty, non-existant file is the same as empty file
+echo Hi! > "$TEST_ROOT/got"
+(
+  diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist"
+  (( "$badDiff" == 1 ))
+)
+
+# doesn't match, `badDiff` set, file unchanged
+echo Hi! > "$TEST_ROOT/got"
+echo Bye! > "$TEST_ROOT/expected"
+(
+  diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected"
+  (( "$badDiff" == 1 ))
+)
+[[ "$(echo Bye! )" == $(< "$TEST_ROOT/expected") ]]
+
+# _NIX_TEST_ACCEPT=1 matches non-empty
+echo Hi! > "$TEST_ROOT/got"
+cp "$TEST_ROOT/got" "$TEST_ROOT/expected"
+(
+  _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected"
+  (( "$badDiff" == 0 ))
+)
+
+# _NIX_TEST_ACCEPT doesn't match, `badDiff=1` set, file changed (was previously non-empty)
+echo Hi! > "$TEST_ROOT/got"
+echo Bye! > "$TEST_ROOT/expected"
+(
+  _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected"
+  (( "$badDiff" == 1 ))
+)
+[[ "$(echo Hi! )" == $(< "$TEST_ROOT/expected") ]]
+# second time succeeds
+(
+  diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected"
+  (( "$badDiff" == 0 ))
+)
+
+# _NIX_TEST_ACCEPT matches empty, non-existant file not created
+echo -n > "$TEST_ROOT/got"
+(
+  _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exists"
+  (( "$badDiff" == 0 ))
+)
+[[ ! -f "$TEST_ROOT/does-not-exist" ]]
+
+# _NIX_TEST_ACCEPT doesn't match, output empty, file deleted
+echo -n > "$TEST_ROOT/got"
+echo Bye! > "$TEST_ROOT/expected"
+badDiff=0
+(
+  _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected"
+  (( "$badDiff" == 1 ))
+)
+[[ ! -f "$TEST_ROOT/expected" ]]
+# second time succeeds
+(
+  diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected"
+  (( "$badDiff" == 0 ))
+)

tests/lang.sh

@@ -1,5 +1,17 @@
 source common.sh
 
+set -o pipefail
+
+source lang/framework.sh
+
+# specialize function a bit
+function diffAndAccept() {
+    local -r testName="$1"
+    local -r got="lang/$testName.$2"
+    local -r expected="lang/$testName.$3"
+    diffAndAcceptInner "$testName" "$got" "$expected"
+}
+
 export TEST_VAR=foo # for eval-okay-getenv.nix
 export NIX_REMOTE=dummy://
 export NIX_STORE_DIR=/nix/store
@@ -20,63 +32,114 @@ nix-instantiate --eval -E 'let x = { repeating = x; tracing = builtins.trace x t
 
 set +x
 
-fail=0
+badDiff=0
+badExitCode=0
 
 for i in lang/parse-fail-*.nix; do
     echo "parsing $i (should fail)";
-    i=$(basename $i .nix)
+    i=$(basename "$i" .nix)
-    if ! expect 1 nix-instantiate --parse - < lang/$i.nix; then
+    if expectStderr 1 nix-instantiate --parse - < "lang/$i.nix" > "lang/$i.err"
+    then
+        diffAndAccept "$i" err err.exp
+    else
         echo "FAIL: $i shouldn't parse"
-        fail=1
+        badExitCode=1
     fi
 done
 
 for i in lang/parse-okay-*.nix; do
     echo "parsing $i (should succeed)";
-    i=$(basename $i .nix)
+    i=$(basename "$i" .nix)
-    if ! expect 0 nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then
+    if
+        expect 0 nix-instantiate --parse - < "lang/$i.nix" \
+            1> >(sed "s!$(pwd)!/pwd!g" > "lang/$i.out") \
+            2> >(sed "s!$(pwd)!/pwd!g" > "lang/$i.err")
+    then
+        diffAndAccept "$i" out exp
+        diffAndAccept "$i" err err.exp
+    else
         echo "FAIL: $i should parse"
-        fail=1
+        badExitCode=1
     fi
 done
 
 for i in lang/eval-fail-*.nix; do
     echo "evaluating $i (should fail)";
-    i=$(basename $i .nix)
+    i=$(basename "$i" .nix)
-    if ! expect 1 nix-instantiate --eval lang/$i.nix; then
+    if
+        expectStderr 1 nix-instantiate --show-trace "lang/$i.nix" \
+            | sed "s!$(pwd)!/pwd!g" > "lang/$i.err"
+    then
+        diffAndAccept "$i" err err.exp
+    else
         echo "FAIL: $i shouldn't evaluate"
-        fail=1
+        badExitCode=1
     fi
 done
 
 for i in lang/eval-okay-*.nix; do
     echo "evaluating $i (should succeed)";
-    i=$(basename $i .nix)
+    i=$(basename "$i" .nix)
 
-    if test -e lang/$i.exp; then
+    if test -e "lang/$i.exp.xml"; then
-        flags=
+        if expect 0 nix-instantiate --eval --xml --no-location --strict \
-        if test -e lang/$i.flags; then
+                "lang/$i.nix" > "lang/$i.out.xml"
-            flags=$(cat lang/$i.flags)
+        then
-        fi
+            diffAndAccept "$i" out.xml exp.xml
-        if ! expect 0 env NIX_PATH=lang/dir3:lang/dir4 HOME=/fake-home nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then
+        else
             echo "FAIL: $i should evaluate"
-            fail=1
+            badExitCode=1
-        elif ! diff <(< lang/$i.out sed -e "s|$(pwd)|/pwd|g") lang/$i.exp; then
+        fi
-            echo "FAIL: evaluation result of $i not as expected"
+    elif test ! -e "lang/$i.exp-disabled"; then
-            fail=1
+        declare -a flags=()
+        if test -e "lang/$i.flags"; then
+            read -r -a flags < "lang/$i.flags"
         fi
-    fi
 
-    if test -e lang/$i.exp.xml; then
+        if
-        if ! expect 0 nix-instantiate --eval --xml --no-location --strict \
+            expect 0 env \
-                lang/$i.nix > lang/$i.out.xml; then
+                NIX_PATH=lang/dir3:lang/dir4 \
+                HOME=/fake-home \
+                nix-instantiate "${flags[@]}" --eval --strict "lang/$i.nix" \
+                1> "lang/$i.out" \
+                2> "lang/$i.err"
+        then
+            sed -i "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err"
+            diffAndAccept "$i" out exp
+            diffAndAccept "$i" err err.exp
+        else
             echo "FAIL: $i should evaluate"
-            fail=1
+            badExitCode=1
-        elif ! cmp -s lang/$i.out.xml lang/$i.exp.xml; then
-            echo "FAIL: XML evaluation result of $i not as expected"
-            fail=1
         fi
     fi
 done
 
-exit $fail
+if test -n "${_NIX_TEST_ACCEPT-}"; then
+    if (( "$badDiff" )); then
+        echo 'Output did mot match, but accepted output as the persisted expected output.'
+        echo 'That means the next time the tests are run, they should pass.'
+    else
+        echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,'
+        echo 'indicating the unexpected output should be accepted as the expected output going forward,'
+        echo 'but no tests had unexpected output so there was no expected output to update.'
+    fi
+    if (( "$badExitCode" )); then
+        exit "$badExitCode"
+    else
+        skipTest "regenerating golden masters"
+    fi
+else
+    if (( "$badDiff" )); then
+        echo ''
+        echo 'You can rerun this test with:'
+        echo ''
+        echo '    _NIX_TEST_ACCEPT=1 make tests/lang.sh.test'
+        echo ''
+        echo 'to regenerate the files containing the expected output,'
+        echo 'and then view the git diff to decide whether a change is'
+        echo 'good/intentional or bad/unintentional.'
+        echo 'If the diff contains arbitrary or impure information,'
+        echo 'please improve the normalization that the test applies to the output.'
+    fi
+    exit $(( "$badExitCode" + "$badDiff" ))
+fi

tests/lang/eval-fail-abort.err.exp

@@ -0,0 +1,10 @@
+error:
+       … while calling the 'abort' builtin
+
+         at /pwd/lang/eval-fail-abort.nix:1:14:
+
+            1| if true then abort "this should fail" else 1
+             |              ^
+            2|
+
+       error: evaluation aborted with the following error message: 'this should fail'

tests/lang/eval-fail-antiquoted-path.err.exp

@@ -0,0 +1 @@
+error: getting attributes of path ‘PWD/lang/fnord’: No such file or directory

tests/lang/eval-fail-assert.err.exp

@@ -0,0 +1,36 @@
+error:
+       … while evaluating the attribute 'body'
+
+         at /pwd/lang/eval-fail-assert.nix:4:3:
+
+            3|
+            4|   body = x "x";
+             |   ^
+            5| }
+
+       … from call site
+
+         at /pwd/lang/eval-fail-assert.nix:4:10:
+
+            3|
+            4|   body = x "x";
+             |          ^
+            5| }
+
+       … while calling 'x'
+
+         at /pwd/lang/eval-fail-assert.nix:2:7:
+
+            1| let {
+            2|   x = arg: assert arg == "y"; 123;
+             |       ^
+            3|
+
+       error: assertion '(arg == "y")' failed
+
+       at /pwd/lang/eval-fail-assert.nix:2:12:
+
+            1| let {
+            2|   x = arg: assert arg == "y"; 123;
+             |            ^
+            3|

tests/lang/eval-fail-bad-antiquote-1.err.exp

@@ -0,0 +1,10 @@
+error:
+       … while evaluating a path segment
+
+         at /pwd/lang/eval-fail-bad-antiquote-1.nix:1:2:
+
+            1| "${x: x}"
+             |  ^
+            2|
+
+       error: cannot coerce a function to a string

tests/lang/eval-fail-bad-antiquote-2.err.exp

@@ -0,0 +1 @@
+error: operation 'addToStoreFromDump' is not supported by store 'dummy'

tests/lang/eval-fail-bad-antiquote-3.err.exp

@@ -0,0 +1,10 @@
+error:
+       … while evaluating a path segment
+
+         at /pwd/lang/eval-fail-bad-antiquote-3.nix:1:3:
+
+            1| ''${x: x}''
+             |   ^
+            2|
+
+       error: cannot coerce a function to a string

tests/lang/eval-fail-bad-string-interpolation-1.err.exp

@@ -0,0 +1,10 @@
+error:
+       … while evaluating a path segment
+
+         at /pwd/lang/eval-fail-bad-string-interpolation-1.nix:1:2:
+
+            1| "${x: x}"
+             |  ^
+            2|
+
+       error: cannot coerce a function to a string

tests/lang/eval-fail-bad-string-interpolation-2.err.exp

@@ -0,0 +1 @@
+error: operation 'addToStoreFromDump' is not supported by store 'dummy'

tests/lang/eval-fail-bad-string-interpolation-3.err.exp

@@ -0,0 +1,10 @@
+error:
+       … while evaluating a path segment
+
+         at /pwd/lang/eval-fail-bad-string-interpolation-3.nix:1:3:
+
+            1| ''${x: x}''
+             |   ^
+            2|
+
+       error: cannot coerce a function to a string

tests/lang/eval-fail-blackhole.err.exp

@@ -0,0 +1,18 @@
+error:
+       … while evaluating the attribute 'body'
+
+         at /pwd/lang/eval-fail-blackhole.nix:2:3:
+
+            1| let {
+            2|   body = x;
+             |   ^
+            3|   x = y;
+
+       error: infinite recursion encountered
+
+       at /pwd/lang/eval-fail-blackhole.nix:3:7:
+
+            2|   body = x;
+            3|   x = y;
+             |       ^
+            4|   y = x;

tests/lang/eval-fail-deepseq.err.exp

@@ -0,0 +1,26 @@
+error:
+       … while calling the 'deepSeq' builtin
+
+         at /pwd/lang/eval-fail-deepseq.nix:1:1:
+
+            1| builtins.deepSeq { x = abort "foo"; } 456
+             | ^
+            2|
+
+       … while evaluating the attribute 'x'
+
+         at /pwd/lang/eval-fail-deepseq.nix:1:20:
+
+            1| builtins.deepSeq { x = abort "foo"; } 456
+             |                    ^
+            2|
+
+       … while calling the 'abort' builtin
+
+         at /pwd/lang/eval-fail-deepseq.nix:1:24:
+
+            1| builtins.deepSeq { x = abort "foo"; } 456
+             |                        ^
+            2|
+
+       error: evaluation aborted with the following error message: 'foo'

tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp

@@ -0,0 +1,38 @@
+error:
+       … while calling the 'foldl'' builtin
+
+         at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:2:1:
+
+            1| # Tests that the result of applying op is forced even if the value is never used
+            2| builtins.foldl'
+             | ^
+            3|   (_: f: f null)
+
+       … while calling anonymous lambda
+
+         at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:7:
+
+            2| builtins.foldl'
+            3|   (_: f: f null)
+             |       ^
+            4|   null
+
+       … from call site
+
+         at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:10:
+
+            2| builtins.foldl'
+            3|   (_: f: f null)
+             |          ^
+            4|   null
+
+       … while calling anonymous lambda
+
+         at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:5:6:
+
+            4|   null
+            5|   [ (_: throw "Not the final value, but is still forced!") (_: 23) ]
+             |      ^
+            6|
+
+       error: Not the final value, but is still forced!

tests/lang/eval-fail-fromTOML-timestamps.err.exp

@@ -0,0 +1,12 @@
+error:
+       … while calling the 'fromTOML' builtin
+
+         at /pwd/lang/eval-fail-fromTOML-timestamps.nix:1:1:
+
+            1| builtins.fromTOML ''
+             | ^
+            2|   key = "value"
+
+       error: while parsing a TOML string: Dates and times are not supported
+
+       at «none»:0: (source not available)

tests/lang/eval-fail-hashfile-missing.err.exp

@@ -0,0 +1,19 @@
+error:
+       … while calling the 'toString' builtin
+
+         at /pwd/lang/eval-fail-hashfile-missing.nix:4:3:
+
+            3| in
+            4|   toString (builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"]))
+             |   ^
+            5|
+
+       … while evaluating the first argument passed to builtins.toString
+
+         at «none»:0: (source not available)
+
+       … while calling the 'hashFile' builtin
+
+         at «none»:0: (source not available)
+
+       error: opening file '/pwd/lang/this-file-is-definitely-not-there-7392097': No such file or directory

tests/lang/eval-fail-list.err.exp

@@ -0,0 +1,10 @@
+error:
+       … while evaluating one of the elements to concatenate
+
+         at /pwd/lang/eval-fail-list.nix:1:2:
+
+            1| 8++1
+             |  ^
+            2|
+
+       error: value is an integer while a list was expected

tests/lang/eval-fail-list.nix

@@ -0,0 +1 @@
+8++1

tests/lang/eval-fail-missing-arg.err.exp

@@ -0,0 +1,16 @@
+error:
+       … from call site
+
+         at /pwd/lang/eval-fail-missing-arg.nix:1:1:
+
+            1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";}
+             | ^
+            2|
+
+       error: function 'anonymous lambda' called without required argument 'y'
+
+       at /pwd/lang/eval-fail-missing-arg.nix:1:2:
+
+            1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";}
+             |  ^
+            2|

tests/lang/eval-fail-nonexist-path.err.exp

@@ -0,0 +1 @@
+error: operation 'addToStoreFromDump' is not supported by store 'dummy'

tests/lang/eval-fail-path-slash.err.exp

@@ -0,0 +1,8 @@
+error: path has a trailing slash
+
+       at /pwd/lang/eval-fail-path-slash.nix:6:12:
+
+            5| # and https://nixos.org/nix-dev/2016-June/020829.html
+            6| /nix/store/
+             |            ^
+            7|

tests/lang/eval-fail-recursion.err.exp

@@ -0,0 +1,16 @@
+error:
+       … in the right operand of the update (//) operator
+
+         at /pwd/lang/eval-fail-recursion.nix:1:12:
+
+            1| let a = {} // a; in a.foo
+             |            ^
+            2|
+
+       error: infinite recursion encountered
+
+       at /pwd/lang/eval-fail-recursion.nix:1:15:
+
+            1| let a = {} // a; in a.foo
+             |               ^
+            2|

tests/lang/eval-fail-recursion.nix

@@ -0,0 +1 @@
+let a = {} // a; in a.foo

tests/lang/eval-fail-remove.err.exp

@@ -0,0 +1,19 @@
+error:
+       … while evaluating the attribute 'body'
+
+         at /pwd/lang/eval-fail-remove.nix:4:3:
+
+            3|
+            4|   body = (removeAttrs attrs ["x"]).x;
+             |   ^
+            5| }
+
+       error: attribute 'x' missing
+
+       at /pwd/lang/eval-fail-remove.nix:4:10:
+
+            3|
+            4|   body = (removeAttrs attrs ["x"]).x;
+             |          ^
+            5| }
+       Did you mean y?

tests/lang/eval-fail-scope-5.err.exp

@@ -0,0 +1,36 @@
+error:
+       … while evaluating the attribute 'body'
+
+         at /pwd/lang/eval-fail-scope-5.nix:8:3:
+
+            7|
+            8|   body = f {};
+             |   ^
+            9|
+
+       … from call site
+
+         at /pwd/lang/eval-fail-scope-5.nix:8:10:
+
+            7|
+            8|   body = f {};
+             |          ^
+            9|
+
+       … while calling 'f'
+
+         at /pwd/lang/eval-fail-scope-5.nix:6:7:
+
+            5|
+            6|   f = {x ? y, y ? x}: x + y;
+             |       ^
+            7|
+
+       error: infinite recursion encountered
+
+       at /pwd/lang/eval-fail-scope-5.nix:6:12:
+
+            5|
+            6|   f = {x ? y, y ? x}: x + y;
+             |            ^
+            7|

tests/lang/eval-fail-seq.err.exp

@@ -0,0 +1,18 @@
+error:
+       … while calling the 'seq' builtin
+
+         at /pwd/lang/eval-fail-seq.nix:1:1:
+
+            1| builtins.seq (abort "foo") 2
+             | ^
+            2|
+
+       … while calling the 'abort' builtin
+
+         at /pwd/lang/eval-fail-seq.nix:1:15:
+
+            1| builtins.seq (abort "foo") 2
+             |               ^
+            2|
+
+       error: evaluation aborted with the following error message: 'foo'

tests/lang/eval-fail-set-override.err.exp

@@ -0,0 +1,6 @@
+error:
+       … while evaluating the `__overrides` attribute
+
+         at «none»:0: (source not available)
+
+       error: value is an integer while a set was expected

tests/lang/eval-fail-set-override.nix

@@ -0,0 +1 @@
+rec { __overrides = 1; }

tests/lang/eval-fail-set.err.exp

@@ -0,0 +1,7 @@
+error: undefined variable 'x'
+
+       at /pwd/lang/eval-fail-set.nix:1:3:
+
+            1| 8.x
+             |   ^
+            2|

tests/lang/eval-fail-set.nix

@@ -0,0 +1 @@
+8.x

tests/lang/eval-fail-substring.err.exp

@@ -0,0 +1,12 @@
+error:
+       … while calling the 'substring' builtin
+
+         at /pwd/lang/eval-fail-substring.nix:1:1:
+
+            1| builtins.substring (builtins.sub 0 1) 1 "x"
+             | ^
+            2|
+
+       error: negative start position in 'substring'
+
+       at «none»:0: (source not available)

tests/lang/eval-fail-to-path.err.exp

@@ -0,0 +1,14 @@
+error:
+       … while calling the 'toPath' builtin
+
+         at /pwd/lang/eval-fail-to-path.nix:1:1:
+
+            1| builtins.toPath "foo/bar"
+             | ^
+            2|
+
+       … while evaluating the first argument passed to builtins.toPath
+
+         at «none»:0: (source not available)
+
+       error: string 'foo/bar' doesn't represent an absolute path

tests/lang/eval-fail-undeclared-arg.err.exp

@@ -0,0 +1,17 @@
+error:
+       … from call site
+
+         at /pwd/lang/eval-fail-undeclared-arg.nix:1:1:
+
+            1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";}
+             | ^
+            2|
+
+       error: function 'anonymous lambda' called with unexpected argument 'y'
+
+       at /pwd/lang/eval-fail-undeclared-arg.nix:1:2:
+
+            1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";}
+             |  ^
+            2|
+       Did you mean one of x or z?

tests/lang/eval-okay-fromjson.nix

@@ -11,9 +11,12 @@ builtins.fromJSON
               "Width":  200,
               "Height": 250
           },
+          "Animated" : false,
+          "IDs": [116, 943, 234, 38793, true  ,false,null, -100],
+          "Escapes": "\"\\\/\t\n\r\t",
           "Subtitle" : false,
-          "Latitude":  46.2051,
+          "Latitude":  37.7668,
-          "Longitude": 6.0723
+          "Longitude": -122.3959
         }
     }
   ''
@@ -28,8 +31,11 @@ builtins.fromJSON
           Width = 200;
           Height = 250;
         };
+      Animated = false;
+      IDs = [ 116 943 234 38793 true false null (0-100) ];
+      Escapes = "\"\\\/\t\n\r\t";  # supported in JSON but not Nix: \b\f
       Subtitle = false;
-      Latitude = 46.2051;
+      Latitude = 37.7668;
-      Longitude = 6.0723;
+      Longitude = -122.3959;
     };
   }

tests/lang/eval-okay-overrides.nix

@@ -1,6 +1,6 @@
 let
 
-  overrides = { a = 2; };
+  overrides = { a = 2; b = 3; };
 
 in (rec {
   __overrides = overrides;

tests/lang/eval-okay-print.err.exp

@@ -0,0 +1 @@
+trace: [ <CODE> ]

tests/lang/eval-okay-print.exp

@@ -0,0 +1 @@
+[ null <PRIMOP> <PRIMOP-APP> <LAMBDA> [ [ «repeated» ] ] ]

tests/lang/eval-okay-print.nix

@@ -0,0 +1 @@
+with builtins; trace [(1+1)] [ null toString (deepSeq "x") (a: a) (let x=[x]; in x) ]

tests/lang/framework.sh

@@ -0,0 +1,33 @@
+# Golden test support
+#
+# Test that the output of the given test matches what is expected. If
+# `_NIX_TEST_ACCEPT` is non-empty also update the expected output so
+# that next time the test succeeds.
+function diffAndAcceptInner() {
+    local -r testName=$1
+    local -r got="$2"
+    local -r expected="$3"
+
+    # Absence of expected file indicates empty output expected.
+    if test -e "$expected"; then
+        local -r expectedOrEmpty="$expected"
+    else
+        local -r expectedOrEmpty=lang/empty.exp
+    fi
+
+    # Diff so we get a nice message
+    if ! diff "$got" "$expectedOrEmpty"; then
+        echo "FAIL: evaluation result of $testName not as expected"
+        badDiff=1
+    fi
+
+    # Update expected if `_NIX_TEST_ACCEPT` is non-empty.
+    if test -n "${_NIX_TEST_ACCEPT-}"; then
+        cp "$got" "$expected"
+        # Delete empty expected files to avoid bloating the repo with
+        # empty files.
+        if ! test -s "$expected"; then
+            rm "$expected"
+        fi
+    fi
+}

tests/lang/parse-fail-dup-attrs-1.err.exp

@@ -0,0 +1,7 @@
+error: attribute 'x' already defined at «stdin»:1:3
+
+       at «stdin»:3:3:
+
+            2|   y = 456;
+            3|   x = 789;
+             |   ^

tests/lang/parse-fail-dup-attrs-2.err.exp

@@ -0,0 +1,7 @@
+error: attribute 'x' already defined at «stdin»:9:5
+
+       at «stdin»:10:17:
+
+            9|     x = 789;
+           10|     inherit (as) x;
+             |                 ^

tests/lang/parse-fail-dup-attrs-3.err.exp

@@ -0,0 +1,7 @@
+error: attribute 'x' already defined at «stdin»:9:5
+
+       at «stdin»:10:17:
+
+            9|     x = 789;
+           10|     inherit (as) x;
+             |                 ^

tests/lang/parse-fail-dup-attrs-4.err.exp

@@ -0,0 +1,7 @@
+error: attribute 'services.ssh.port' already defined at «stdin»:2:3
+
+       at «stdin»:3:3:
+
+            2|   services.ssh.port = 22;
+            3|   services.ssh.port = 23;
+             |   ^

tests/lang/parse-fail-dup-attrs-6.err.exp

@@ -0,0 +1 @@
+error: attribute ‘services.ssh’ at (string):3:3 already defined at (string):2:3

tests/lang/parse-fail-dup-attrs-7.err.exp

@@ -0,0 +1,7 @@
+error: attribute 'x' already defined at «stdin»:6:12
+
+       at «stdin»:7:12:
+
+            6|     inherit x;
+            7|     inherit x;
+             |            ^

tests/lang/parse-fail-dup-formals.err.exp

@@ -0,0 +1,6 @@
+error: duplicate formal function argument 'x'
+
+       at «stdin»:1:8:
+
+            1| {x, y, x}: x
+             |        ^

tests/lang/parse-fail-eof-in-string.err.exp

@@ -0,0 +1,7 @@
+error: syntax error, unexpected end of file, expecting '"'
+
+       at «stdin»:3:5:
+
+            2| # Note that this file must not end with a newline.
+            3| a 1"$
+             |     ^

tests/lang/parse-fail-mixed-nested-attrs1.err.exp

@@ -0,0 +1,8 @@
+error: attribute 'z' already defined at «stdin»:3:16
+
+       at «stdin»:2:3:
+
+            1| {
+            2|   x.z = 3;
+             |   ^
+            3|   x = { y = 3; z = 3; };

tests/lang/parse-fail-mixed-nested-attrs2.err.exp

@@ -0,0 +1,8 @@
+error: attribute 'y' already defined at «stdin»:3:9
+
+       at «stdin»:2:3:
+
+            1| {
+            2|   x.y.y = 3;
+             |   ^
+            3|   x = { y.y= 3; z = 3; };

tests/lang/parse-fail-patterns-1.err.exp

@@ -0,0 +1,7 @@
+error: duplicate formal function argument 'args'
+
+       at «stdin»:1:1:
+
+            1| args@{args, x, y, z}: x
+             | ^
+            2|

tests/lang/parse-fail-regression-20060610.err.exp

@@ -0,0 +1,8 @@
+error: undefined variable 'gcc'
+
+       at «stdin»:8:12:
+
+            7|
+            8|   body = ({
+             |            ^
+            9|     inherit gcc;

tests/lang/parse-fail-undef-var-2.err.exp

@@ -0,0 +1,7 @@
+error: syntax error, unexpected ':', expecting '}'
+
+       at «stdin»:3:13:
+
+            2|
+            3|   f = {x, y :
+             |             ^

tests/lang/parse-fail-undef-var.err.exp

@@ -0,0 +1,7 @@
+error: undefined variable 'y'
+
+       at «stdin»:1:4:
+
+            1| x: y
+             |    ^
+            2|

tests/lang/parse-fail-utf8.err.exp

@@ -0,0 +1,6 @@
+error: syntax error, unexpected invalid token, expecting end of file
+
+       at «stdin»:1:5:
+
+            1| 123 Ã
+             |     ^

tests/lang/parse-okay-1.exp

@@ -0,0 +1 @@
+({ x, y, z }: ((x + y) + z))

tests/lang/parse-okay-crlf.exp

@@ -0,0 +1 @@
+rec { foo = "multi\nline\n  string\n  test\r"; x = y; y = 123; z = 456; }

tests/lang/parse-okay-dup-attrs-5.exp

@@ -0,0 +1 @@
+{ services = { ssh = { enable = true; port = 23; }; }; }

tests/lang/parse-okay-dup-attrs-6.exp

@@ -0,0 +1 @@
+{ services = { ssh = { enable = true; port = 23; }; }; }

tests/lang/parse-okay-mixed-nested-attrs-1.exp

@@ -0,0 +1 @@
+{ x = { q = 3; y = 3; z = 3; }; }

tests/lang/parse-okay-mixed-nested-attrs-2.exp

@@ -0,0 +1 @@
+{ x = { q = 3; y = 3; z = 3; }; }

tests/lang/parse-okay-mixed-nested-attrs-3.exp

@@ -0,0 +1 @@
+{ services = { httpd = { enable = true; }; ssh = { enable = true; port = 123; }; }; }

tests/lang/parse-okay-regression-20041027.exp

@@ -0,0 +1 @@
+({ fetchurl, stdenv }: ((stdenv).mkDerivation { name = "libXi-6.0.1"; src = (fetchurl { md5 = "7e935a42428d63a387b3c048be0f2756"; url = "http://freedesktop.org/~xlibs/release/libXi-6.0.1.tar.bz2"; }); }))

tests/lang/parse-okay-regression-751.exp

@@ -0,0 +1 @@
+(let const = (a: "const"); in ((const { x = "q"; })))

tests/lang/parse-okay-subversion.exp

@@ -0,0 +1 @@
+({ fetchurl, localServer ? false, httpServer ? false, sslSupport ? false, pythonBindings ? false, javaSwigBindings ? false, javahlBindings ? false, stdenv, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null }: assert (expat != null); assert (localServer -> (db4 != null)); assert (httpServer -> ((httpd != null) && ((httpd).expat == expat))); assert (sslSupport -> ((openssl != null) && (httpServer -> ((httpd).openssl == openssl)))); assert (pythonBindings -> ((swig != null) && (swig).pythonSupport)); assert (javaSwigBindings -> ((swig != null) && (swig).javaSupport)); assert (javahlBindings -> (j2sdk != null)); ((stdenv).mkDerivation { builder = /foo/bar; db4 = (if localServer then db4 else null); inherit expat ; inherit httpServer ; httpd = (if httpServer then httpd else null); j2sdk = (if javaSwigBindings then (swig).j2sdk else (if javahlBindings then j2sdk else null)); inherit javaSwigBindings ; inherit javahlBindings ; inherit localServer ; name = "subversion-1.1.1"; openssl = (if sslSupport then openssl else null); patches = (if javahlBindings then [ (/javahl.patch) ] else [ ]); python = (if pythonBindings then (swig).python else null); inherit pythonBindings ; src = (fetchurl { md5 = "a180c3fe91680389c210c99def54d9e0"; url = "http://subversion.tigris.org/tarballs/subversion-1.1.1.tar.bz2"; }); inherit sslSupport ; swig = (if (pythonBindings || javaSwigBindings) then swig else null); }))

tests/lang/parse-okay-url.exp

@@ -0,0 +1 @@
+[ ("x:x") ("https://svn.cs.uu.nl:12443/repos/trace/trunk") ("http://www2.mplayerhq.hu/MPlayer/releases/fonts/font-arial-iso-8859-1.tar.bz2") ("http://losser.st-lab.cs.uu.nl/~armijn/.nix/gcc-3.3.4-static-nix.tar.gz") ("http://fpdownload.macromedia.com/get/shockwave/flash/english/linux/7.0r25/install_flash_player_7_linux.tar.gz") ("https://ftp5.gwdg.de/pub/linux/archlinux/extra/os/x86_64/unzip-6.0-14-x86_64.pkg.tar.zst") ("ftp://ftp.gtk.org/pub/gtk/v1.2/gtk+-1.2.10.tar.gz") ]

tests/local.mk

@@ -20,6 +20,7 @@ nix_tests = \
   remote-store.sh \
   legacy-ssh-store.sh \
   lang.sh \
+  lang-test-infra.sh \
   experimental-features.sh \
   fetchMercurial.sh \
   gc-auto.sh \

tests/misc.sh

@@ -24,3 +24,9 @@ eval_stdin_res=$(echo 'let a = {} // a; in a.foo' | nix-instantiate --eval -E -
 echo $eval_stdin_res | grep "at «stdin»:1:15:"
 echo $eval_stdin_res | grep "infinite recursion encountered"
 
+# Attribute path errors
+expectStderr 1 nix-instantiate --eval -E '{}' -A '"x' | grepQuiet "missing closing quote in selection path"
+expectStderr 1 nix-instantiate --eval -E '[]' -A 'x' | grepQuiet "should be a set"
+expectStderr 1 nix-instantiate --eval -E '{}' -A '1' | grepQuiet "should be a list"
+expectStderr 1 nix-instantiate --eval -E '{}' -A '.' | grepQuiet "empty attribute name"
+expectStderr 1 nix-instantiate --eval -E '[]' -A '1' | grepQuiet "out of range"

tests/restricted.sh

@@ -49,3 +49,5 @@ output="$(nix eval --raw --restrict-eval -I "$traverseDir" \
     2>&1 || :)"
 echo "$output" | grep "is forbidden"
 echo "$output" | grepInverse -F restricted-secret
+
+expectStderr 1 nix-instantiate --restrict-eval true ./dependencies.nix | grepQuiet "forbidden in restricted mode"