From ca49e13414b4d1d9b2cdd72e36920e28c4ef117e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 21 Jun 2023 23:33:33 -0400 Subject: [PATCH] Split testing into its own page in the contribution guide `hacking.md` has gotten really big! --- doc/manual/redirects.js | 8 ++ doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/contributing/hacking.md | 165 ------------------------ doc/manual/src/contributing/testing.md | 167 +++++++++++++++++++++++++ 4 files changed, 176 insertions(+), 165 deletions(-) create mode 100644 doc/manual/src/contributing/testing.md diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index f54f6ea6a..dcdb5d6e9 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -347,6 +347,14 @@ const redirects = { "contributing/hacking.html": { "nix-with-flakes": "#building-nix-with-flakes", "classic-nix": "#building-nix", + "running-tests": "testing.html#running-tests", + "unit-tests": "testing.html#unit-tests", + "functional-tests": "testing.html#functional-tests", + "debugging-failing-functional-tests": "testing.html#debugging-failing-functional-tests", + "integration-tests": "testing.html#integration-tests", + "installer-tests": "testing.html#installer-tests", + "one-time-setup": "testing.html#one-time-setup", + "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", } }; diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index bbb603713..1bd8fa774 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -104,6 +104,7 @@ - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) + - [Testing](contributing/testing.md) - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) - [C++ style guide](contributing/cxx.md) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 6c6f2eb52..7b2440971 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -190,171 +190,6 @@ Configure your editor to use the `clangd` from the shell, either by running it i > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. -## Running tests - -### Unit-tests - -The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined -under `src/{library_name}/tests` using the -[googletest](https://google.github.io/googletest/) and -[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks. - -You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option. - -### Functional tests - -The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. -Each test is a bash script. - -The whole test suite can be run with: - -```shell-session -$ make install && make installcheck -ran test tests/foo.sh... [PASS] -ran test tests/bar.sh... [PASS] -... -``` - -Individual tests can be run with `make`: - -```shell-session -$ make tests/${testName}.sh.test -ran test tests/${testName}.sh... [PASS] -``` - -or without `make`: - -```shell-session -$ ./mk/run-test.sh tests/${testName}.sh -ran test tests/${testName}.sh... [PASS] -``` - -To see the complete output, one can also run: - -```shell-session -$ ./mk/debug-test.sh tests/${testName}.sh -+ foo -output from foo -+ bar -output from bar -... -``` - -The test script will then be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails. - -#### Debugging failing functional tests - -When a functional test fails, it usually does so somewhere in the middle of the script. - -To figure out what's wrong, it is convenient to run the test regularly up to the failing `nix` command, and then run that command with a debugger like GDB. - -For example, if the script looks like: - -```bash -foo -nix blah blub -bar -``` -edit it like so: - -```diff - foo --nix blah blub -+gdb --args nix blah blub - bar -``` - -Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: - -```shell-session -$ ./mk/debug-test.sh tests/${testName}.sh -... -+ gdb blash blub -GNU gdb (GDB) 12.1 -... -(gdb) -``` - -One can debug the Nix invocation in all the usual ways. -For example, enter `run` to start the Nix invocation. - -### Integration tests - -The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. -These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. -Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). - -You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` - -### Installer tests - -After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. - -Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): - -- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: - - `x86_64-linux` - - `armv6l-linux` - - `armv7l-linux` - - `x86_64-darwin` - -- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. - -#### One-time setup - -1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix). -2. At cachix.org: - - Create or log in to an account. - - Create a Cachix cache using the format `-nix-install-tests`. - - Navigate to the new cache > Settings > Auth Tokens. - - Generate a new Cachix auth token and copy the generated value. -3. At github.com: - - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret. - - Name the secret `CACHIX_AUTH_TOKEN`. - - Paste the copied value of the Cachix cache auth token. - -#### Using the CI-generated installer for manual testing - -After the CI run completes, you can check the output to extract the installer URL: -1. Click into the detailed view of the CI run. -2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them). -3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`) -4. Copy the value of `install_url` -5. To generate an install command, plug this `install_url` and your GitHub username into this template: - - ```console - curl -L | sh -s -- --tarball-url-prefix https://-nix-install-tests.cachix.org/serve - ``` - - - ### Checking links in the manual The build checks for broken internal links. diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md new file mode 100644 index 000000000..e5f80a928 --- /dev/null +++ b/doc/manual/src/contributing/testing.md @@ -0,0 +1,167 @@ +# Running tests + +## Unit-tests + +The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined +under `src/{library_name}/tests` using the +[googletest](https://google.github.io/googletest/) and +[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks. + +You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option. + +## Functional tests + +The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. +Each test is a bash script. + +The whole test suite can be run with: + +```shell-session +$ make install && make installcheck +ran test tests/foo.sh... [PASS] +ran test tests/bar.sh... [PASS] +... +``` + +Individual tests can be run with `make`: + +```shell-session +$ make tests/${testName}.sh.test +ran test tests/${testName}.sh... [PASS] +``` + +or without `make`: + +```shell-session +$ ./mk/run-test.sh tests/${testName}.sh +ran test tests/${testName}.sh... [PASS] +``` + +To see the complete output, one can also run: + +```shell-session +$ ./mk/debug-test.sh tests/${testName}.sh ++ foo +output from foo ++ bar +output from bar +... +``` + +The test script will then be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails. + +### Debugging failing functional tests + +When a functional test fails, it usually does so somewhere in the middle of the script. + +To figure out what's wrong, it is convenient to run the test regularly up to the failing `nix` command, and then run that command with a debugger like GDB. + +For example, if the script looks like: + +```bash +foo +nix blah blub +bar +``` +edit it like so: + +```diff + foo +-nix blah blub ++gdb --args nix blah blub + bar +``` + +Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: + +```shell-session +$ ./mk/debug-test.sh tests/${testName}.sh +... ++ gdb blash blub +GNU gdb (GDB) 12.1 +... +(gdb) +``` + +One can debug the Nix invocation in all the usual ways. +For example, enter `run` to start the Nix invocation. + +## Integration tests + +The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. +These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. +Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). + +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` + +## Installer tests + +After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. + +Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): + +- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: + - `x86_64-linux` + - `armv6l-linux` + - `armv7l-linux` + - `x86_64-darwin` + +- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. + +### One-time setup + +1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix). +2. At cachix.org: + - Create or log in to an account. + - Create a Cachix cache using the format `-nix-install-tests`. + - Navigate to the new cache > Settings > Auth Tokens. + - Generate a new Cachix auth token and copy the generated value. +3. At github.com: + - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret. + - Name the secret `CACHIX_AUTH_TOKEN`. + - Paste the copied value of the Cachix cache auth token. + +## Working on documentation + +### Using the CI-generated installer for manual testing + +After the CI run completes, you can check the output to extract the installer URL: +1. Click into the detailed view of the CI run. +2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them). +3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`) +4. Copy the value of `install_url` +5. To generate an install command, plug this `install_url` and your GitHub username into this template: + + ```console + curl -L | sh -s -- --tarball-url-prefix https://-nix-install-tests.cachix.org/serve + ``` + + +