Merge pull request #7470 from obsidiansystems/simplify-tests-slightly

Make `./mk/run-test.sh` work by itself; add `mk/debug-test.sh`
This commit is contained in:
Eelco Dolstra 2023-01-02 14:14:30 +01:00 committed by GitHub
commit 80a0f77e49
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 111 additions and 11 deletions

View file

@ -99,8 +99,79 @@ You can run the whole testsuite with `make check`, or the tests for a specific c
### Functional tests ### Functional tests
The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. The functional tests reside under the `tests` directory and are listed in `tests/local.mk`.
The whole testsuite can be run with `make install && make installcheck`. Each test is a bash script.
Individual tests can be run with `make tests/{testName}.sh.test`.
The whole test suite can be run with:
```shell-session
$ make install && make installcheck
ran test tests/foo.sh... [PASS]
ran test tests/bar.sh... [PASS]
...
```
Individual tests can be run with `make`:
```shell-session
$ make tests/${testName}.sh.test
ran test tests/${testName}.sh... [PASS]
```
or without `make`:
```shell-session
$ ./mk/run-test.sh tests/${testName}.sh
ran test tests/${testName}.sh... [PASS]
```
To see the complete output, one can also run:
```shell-session
$ ./mk/debug-test.sh tests/${testName}.sh
+ foo
output from foo
+ bar
output from bar
...
```
The test script will then be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails.
#### Debugging failing functional tests
When a functional test fails, it usually does so somewhere in the middle of the script.
To figure out what's wrong, it is convenient to run the test regularly up to the failing `nix` command, and then run that command with a debugger like GDB.
For example, if the script looks like:
```bash
foo
nix blah blub
bar
```
edit it like so:
```diff
foo
-nix blah blub
+gdb --args nix blah blub
bar
```
Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point:
```shell-session
$ ./mk/debug-test.sh tests/${testName}.sh
...
+ gdb blash blub
GNU gdb (GDB) 12.1
...
(gdb)
```
One can debug the Nix invocation in all the usual ways.
For example, enter `run` to start the Nix invocation.
### Integration tests ### Integration tests

11
mk/common-test.sh Normal file
View file

@ -0,0 +1,11 @@
TESTS_ENVIRONMENT=("TEST_NAME=${test%.*}" 'NIX_REMOTE=')
: ${BASH:=/usr/bin/env bash}
init_test () {
cd tests && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null
}
run_test_proper () {
cd $(dirname $test) && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test)
}

11
mk/debug-test.sh Executable file
View file

@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -eu
test=$1
dir="$(dirname "${BASH_SOURCE[0]}")"
source "$dir/common-test.sh"
(init_test)
run_test_proper

View file

@ -1,4 +1,4 @@
#!/bin/sh #!/usr/bin/env bash
set -u set -u
@ -7,7 +7,12 @@ green=""
yellow="" yellow=""
normal="" normal=""
post_run_msg="ran test $1..." test=$1
dir="$(dirname "${BASH_SOURCE[0]}")"
source "$dir/common-test.sh"
post_run_msg="ran test $test..."
if [ -t 1 ]; then if [ -t 1 ]; then
red="" red=""
green="" green=""
@ -16,12 +21,12 @@ if [ -t 1 ]; then
fi fi
run_test () { run_test () {
(cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null) (init_test 2>/dev/null > /dev/null)
log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)" log="$(run_test_proper 2>&1)"
status=$? status=$?
} }
run_test "$1" run_test
# Hack: Retry the test if it fails with “unexpected EOF reading a line” as these # Hack: Retry the test if it fails with “unexpected EOF reading a line” as these
# appear randomly without anyone knowing why. # appear randomly without anyone knowing why.
@ -32,7 +37,7 @@ if [[ $status -ne 0 && $status -ne 99 && \
]]; then ]]; then
echo "$post_run_msg [${yellow}FAIL$normal] (possibly flaky, so will be retried)" echo "$post_run_msg [${yellow}FAIL$normal] (possibly flaky, so will be retried)"
echo "$log" | sed 's/^/ /' echo "$log" | sed 's/^/ /'
run_test "$1" run_test
fi fi
if [ $status -eq 0 ]; then if [ $status -eq 0 ]; then

View file

@ -8,7 +8,11 @@ define run-install-test
.PHONY: $1.test .PHONY: $1.test
$1.test: $1 $(test-deps) $1.test: $1 $(test-deps)
@env TEST_NAME=$(basename $1) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null @env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null
.PHONY: $1.test-debug
$1.test-debug: $1 $(test-deps)
@env BASH=$(bash) $(bash) mk/debug-test.sh $1 < /dev/null
endef endef

View file

@ -121,8 +121,6 @@ endif
install-tests += $(foreach x, $(nix_tests), tests/$(x)) install-tests += $(foreach x, $(nix_tests), tests/$(x))
tests-environment = NIX_REMOTE= $(bash) -e
clean-files += $(d)/common.sh $(d)/config.nix $(d)/ca/config.nix clean-files += $(d)/common.sh $(d)/config.nix $(d)/ca/config.nix
test-deps += tests/common.sh tests/config.nix tests/ca/config.nix test-deps += tests/common.sh tests/config.nix tests/ca/config.nix