From 0251d44cc2e991608ea39f2274ad2d197c668468 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 15 Dec 2022 20:17:08 -0500 Subject: [PATCH 1/2] Make `./mk/run-test.sh` work by itself; add `mk/debug-test.sh` First, logic is consolidated in the shell script instead of being spread between them and makefiles. That makes understanding what is going on a little easier. This would not be super interesting by itself, but it gives us a way to debug tests more easily. *That* in turn I hope is much more compelling. See the updated manual for details. Co-authored-by: Valentin Gagarin Co-authored-by: Eelco Dolstra Co-authored-by: Robert Hensing --- doc/manual/src/contributing/hacking.md | 75 +++++++++++++++++++++++++- mk/common-test.sh | 11 ++++ mk/debug-test.sh | 11 ++++ mk/{run_test.sh => run-test.sh} | 17 +++--- mk/tests.mk | 6 ++- tests/local.mk | 2 - 6 files changed, 111 insertions(+), 11 deletions(-) create mode 100644 mk/common-test.sh create mode 100755 mk/debug-test.sh rename mk/{run_test.sh => run-test.sh} (79%) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 9f7d5057b..b75d2554c 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -99,8 +99,79 @@ You can run the whole testsuite with `make check`, or the tests for a specific c ### Functional tests The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. -The whole testsuite can be run with `make install && make installcheck`. -Individual tests can be run with `make tests/{testName}.sh.test`. +Each test is a bash script. + +The whole testsuite can be run with: + +```shell-session +$ make install && make installcheck +ran test tests/foo.sh... [PASS] +ran test tests/bar.sh... [PASS] +... +``` + +Individual tests can be run with `make`: + +```shell-session +$ make tests/${testName}.sh.test +ran test tests/${testName}.sh... [PASS] +``` + +or without `make`: + +```shell-session +$ ./mk/run-test.sh tests/${testName}.sh +ran test tests/${testName}.sh... [PASS] +``` + +To see the complet eoutput, one can also run: + +```shell-session +$ ./mk/debug-test.sh tests/${testName}.sh ++ foo +output from foo ++ bar +output from bar +... +``` + +The test script will be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails. + +#### Debugging failing functional tests + +When a functional test fails, it usually does so somewhere in the middle of the script. + +To figure out what's wrong, it is convenient to run the test regularly up to the failing `nix` command, and then run that command with a debugger like GDB. + +For example, if the script looks like: + +```bash +foo +nix blah blub +bar +``` +one would could edit it like so: + +```diff + foo +-nix blah blub ++gdb --args nix blah blub + bar +``` + +Then, when one runs the script with `./mk/debug-test.sh`, it will drop them into GDB once the script reaches that point: + +```shell-session +$ ./mk/debug-test.sh tests/${testName}.sh +... ++ gdb blash blub +GNU gdb (GDB) 12.1 +... +(gdb) +``` + +One can debug the Nix invocation in all the usual ways. +(For exampling running `run` will run the Nix invocation.) ### Integration tests diff --git a/mk/common-test.sh b/mk/common-test.sh new file mode 100644 index 000000000..0a2e4c1c2 --- /dev/null +++ b/mk/common-test.sh @@ -0,0 +1,11 @@ +TESTS_ENVIRONMENT=("TEST_NAME=${test%.*}" 'NIX_REMOTE=') + +: ${BASH:=/usr/bin/env bash} + +init_test () { + cd tests && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null +} + +run_test_proper () { + cd $(dirname $test) && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test) +} diff --git a/mk/debug-test.sh b/mk/debug-test.sh new file mode 100755 index 000000000..6299e68a0 --- /dev/null +++ b/mk/debug-test.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -eu + +test=$1 + +dir="$(dirname "${BASH_SOURCE[0]}")" +source "$dir/common-test.sh" + +(init_test) +run_test_proper diff --git a/mk/run_test.sh b/mk/run-test.sh similarity index 79% rename from mk/run_test.sh rename to mk/run-test.sh index 7e95df2ac..219c8577f 100755 --- a/mk/run_test.sh +++ b/mk/run-test.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash set -u @@ -7,7 +7,12 @@ green="" yellow="" normal="" -post_run_msg="ran test $1..." +test=$1 + +dir="$(dirname "${BASH_SOURCE[0]}")" +source "$dir/common-test.sh" + +post_run_msg="ran test $test..." if [ -t 1 ]; then red="" green="" @@ -16,12 +21,12 @@ if [ -t 1 ]; then fi run_test () { - (cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null) - log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)" + (init_test 2>/dev/null > /dev/null) + log="$(run_test_proper 2>&1)" status=$? } -run_test "$1" +run_test # Hack: Retry the test if it fails with “unexpected EOF reading a line” as these # appear randomly without anyone knowing why. @@ -32,7 +37,7 @@ if [[ $status -ne 0 && $status -ne 99 && \ ]]; then echo "$post_run_msg [${yellow}FAIL$normal] (possibly flaky, so will be retried)" echo "$log" | sed 's/^/ /' - run_test "$1" + run_test fi if [ $status -eq 0 ]; then diff --git a/mk/tests.mk b/mk/tests.mk index a2e30a378..3ebbd86e3 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -8,7 +8,11 @@ define run-install-test .PHONY: $1.test $1.test: $1 $(test-deps) - @env TEST_NAME=$(basename $1) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null + @env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null + + .PHONY: $1.test-debug + $1.test-debug: $1 $(test-deps) + @env BASH=$(bash) $(bash) mk/debug-test.sh $1 < /dev/null endef diff --git a/tests/local.mk b/tests/local.mk index 2f7f76261..bba6ad9c9 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -121,8 +121,6 @@ endif install-tests += $(foreach x, $(nix_tests), tests/$(x)) -tests-environment = NIX_REMOTE= $(bash) -e - clean-files += $(d)/common.sh $(d)/config.nix $(d)/ca/config.nix test-deps += tests/common.sh tests/config.nix tests/ca/config.nix From efbd1d15c64a10068bdb8aed92b408a05f3b69a8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 23 Dec 2022 08:59:36 -0800 Subject: [PATCH 2/2] Apply suggestions from code review Co-authored-by: Valentin Gagarin --- doc/manual/src/contributing/hacking.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index b75d2554c..c9da1962f 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -101,7 +101,7 @@ You can run the whole testsuite with `make check`, or the tests for a specific c The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. Each test is a bash script. -The whole testsuite can be run with: +The whole test suite can be run with: ```shell-session $ make install && make installcheck @@ -124,7 +124,7 @@ $ ./mk/run-test.sh tests/${testName}.sh ran test tests/${testName}.sh... [PASS] ``` -To see the complet eoutput, one can also run: +To see the complete output, one can also run: ```shell-session $ ./mk/debug-test.sh tests/${testName}.sh @@ -135,7 +135,7 @@ output from bar ... ``` -The test script will be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails. +The test script will then be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails. #### Debugging failing functional tests @@ -150,7 +150,7 @@ foo nix blah blub bar ``` -one would could edit it like so: +edit it like so: ```diff foo @@ -159,7 +159,7 @@ one would could edit it like so: bar ``` -Then, when one runs the script with `./mk/debug-test.sh`, it will drop them into GDB once the script reaches that point: +Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: ```shell-session $ ./mk/debug-test.sh tests/${testName}.sh @@ -171,7 +171,7 @@ GNU gdb (GDB) 12.1 ``` One can debug the Nix invocation in all the usual ways. -(For exampling running `run` will run the Nix invocation.) +For example, enter `run` to start the Nix invocation. ### Integration tests