Merge changes I327db40f,If762efce into main

* changes:
  testsuite: use xdist for parallel test running
  testsuite: add a functional2 test suite based on pytest
This commit is contained in:
jade 2024-10-14 05:53:44 +00:00 committed by Gerrit Code Review
commit 326cbecb61
10 changed files with 205 additions and 0 deletions

View file

@ -0,0 +1,10 @@
---
synopsis: "The beginnings of a new pytest-based functional test suite"
category: Development
cls: [2036, 2037]
credits: jade
---
The existing integration/functional test suite is based on a large volume of shell scripts.
This often makes it somewhat challenging to debug at the best of times.
The goal of the pytest test suite is to make tests have more obvious dependencies on files and to make tests more concise and easier to write, as well as making new testing methods like snapshot testing easy.

View file

@ -607,6 +607,7 @@ endif
if enable_tests if enable_tests
subdir('tests/unit') subdir('tests/unit')
subdir('tests/functional') subdir('tests/functional')
subdir('tests/functional2')
endif endif
subdir('meson/clang-tidy') subdir('meson/clang-tidy')

View file

@ -170,6 +170,7 @@ let
functionalTestFiles = fileset.unions [ functionalTestFiles = fileset.unions [
./tests/functional ./tests/functional
./tests/functional2
./tests/unit ./tests/unit
(fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
]; ];
@ -243,6 +244,8 @@ stdenv.mkDerivation (finalAttrs: {
nativeBuildInputs = nativeBuildInputs =
[ [
python3 python3
python3.pkgs.pytest
python3.pkgs.pytest-xdist
meson meson
ninja ninja
cmake cmake
@ -474,6 +477,11 @@ stdenv.mkDerivation (finalAttrs: {
pythonPackages = ( pythonPackages = (
p: [ p: [
# FIXME: these have to be added twice due to the nix shell using a
# wrapped python instead of build inputs for its python inputs
p.pytest
p.pytest-xdist
p.yapf p.yapf
p.python-frontmatter p.python-frontmatter
p.requests p.requests

View file

@ -0,0 +1,24 @@
# functional2 tests
This uncreatively named test suite is a Pytest based replacement for the shell framework used to write traditional Nix integration tests.
Its primary goal is to make tests more concise, more self-contained, easier to write, and to produce better errors.
## Goals
- Eliminate implicit dependencies on files in the test directory as well as the requirement to copy the test files to the build directory as is currently hacked in the other functional test suite.
- You should be able to write a DirectoryTree of files for your test declaratively.
- Reduce the amount of global environment state being thrown around in the test suite.
- Make tests very concise and easy to reuse code for, and hopefully turn more of what is currently code into data.
- Provide rich ways of calling `nix` with pleasant syntax.
## TODO: Intended features
- [ ] Expect tests ([pytest-expect-test]) or snapshot tests ([pytest-insta]) or, likely, both!
We::jade prefer to have short output written in-line as it makes it greatly easier to read the tests, but pytest-expect doesn't allow for putting larger stuff in external files, so something else is necessary for those.
- [ ] Web server fixture: we don't test our network functionality because background processes are hard and this is simply goofy.
We could just test it.
- [ ] Nix daemon fixture.
- [x] Parallelism via pytest-xdist.
[pytest-expect-test]: https://pypi.org/project/pytest-expect-test/
[pytest-insta]: https://pypi.org/project/pytest-insta/

View file

View file

@ -0,0 +1,8 @@
import pytest
from pathlib import Path
from .testlib import fixtures
@pytest.fixture
def nix(tmp_path: Path):
return fixtures.Nix(tmp_path)

View file

@ -0,0 +1,29 @@
xdist_opts = [
# auto number of workers, max 12 jobs
'-n', 'auto', '--maxprocesses=12',
# group tests by module or class; ensures that any setup work occurs as little as possible
'--dist=loadscope',
]
# surprisingly, this actually works even if PATH is set to something before
# meson gets hold of it. neat!
functional2_env = environment()
functional2_env.prepend('PATH', bindir)
test(
'functional2',
python,
args : [
'-m', 'pytest',
'-v',
xdist_opts,
meson.current_source_dir()
],
env : functional2_env,
# FIXME: Although we can trivially use TAP here with pytest-tap, due to a meson bug, it is unusable.
# (failure output does not get displayed to the console. at all. someone should go fix it):
# https://github.com/mesonbuild/meson/issues/11185
# protocol : 'tap',
suite : 'installcheck',
timeout : 300,
)

View file

@ -0,0 +1,4 @@
from .testlib.fixtures import Nix
def test_trivial_addition(nix: Nix):
assert nix.eval('1 + 1').json() == 2

View file

View file

@ -0,0 +1,121 @@
import os
import json
import subprocess
from typing import Any
from pathlib import Path
import dataclasses
@dataclasses.dataclass
class CommandResult:
cmd: list[str]
rc: int
"""Return code"""
stderr: bytes
"""Outputted stderr"""
stdout: bytes
"""Outputted stdout"""
def ok(self):
if self.rc != 0:
raise subprocess.CalledProcessError(returncode=self.rc,
cmd=self.cmd,
stderr=self.stderr,
output=self.stdout)
return self
def json(self) -> Any:
self.ok()
return json.loads(self.stdout)
@dataclasses.dataclass
class NixSettings:
"""Settings for invoking Nix"""
experimental_features: set[str] | None = None
def feature(self, *names: str):
self.experimental_features = (self.experimental_features
or set()) | set(names)
return self
def to_config(self) -> str:
config = ''
def serialise(value):
if type(value) in {str, int}:
return str(value)
elif type(value) in {list, set}:
return ' '.join(str(e) for e in value)
else:
raise ValueError(
f'Value is unsupported in nix config: {value!r}')
def field_may(name, value, serialiser=serialise):
nonlocal config
if value is not None:
config += f'{name} = {serialiser(value)}\n'
field_may('experimental-features', self.experimental_features)
return config
@dataclasses.dataclass
class Nix:
test_root: Path
def hermetic_env(self):
# mirroring vars-and-functions.sh
home = self.test_root / 'test-home'
home.mkdir(parents=True, exist_ok=True)
return {
'NIX_STORE_DIR': self.test_root / 'store',
'NIX_LOCALSTATE_DIR': self.test_root / 'var',
'NIX_LOG_DIR': self.test_root / 'var/log/nix',
'NIX_STATE_DIR': self.test_root / 'var/nix',
'NIX_CONF_DIR': self.test_root / 'etc',
'NIX_DAEMON_SOCKET_PATH': self.test_root / 'daemon-socket',
'NIX_USER_CONF_FILES': '',
'HOME': home,
}
def make_env(self):
# We conservatively assume that people might want to successfully get
# some env through to the subprocess, so we override whatever is in the
# global env.
d = os.environ.copy()
d.update(self.hermetic_env())
return d
def call(self, cmd: list[str], extra_env: dict[str, str] = {}):
"""
Calls a process in the test environment.
"""
env = self.make_env()
env.update(extra_env)
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=self.test_root,
env=env,
)
(stdout, stderr) = proc.communicate()
rc = proc.returncode
return CommandResult(cmd=cmd, rc=rc, stdout=stdout, stderr=stderr)
def nix(self,
cmd: list[str],
settings: NixSettings = NixSettings(),
extra_env: dict[str, str] = {}):
extra_env = extra_env.copy()
extra_env.update({'NIX_CONFIG': settings.to_config()})
return self.call(['nix', *cmd], extra_env)
def eval(
self, expr: str,
settings: NixSettings = NixSettings()) -> CommandResult:
# clone due to reference-shenanigans
settings = dataclasses.replace(settings).feature('nix-command')
return self.nix(['eval', '--json', '--expr', expr], settings=settings)