meson: implement functional tests

Functional tests can be run with
`meson test -C build --suite installcheck`.

Notably, functional tests must be run *after* running `meson install`
(Lix's derivation runs the installcheck suite in installCheckPhase so it
does this correctly), due to some quirks between Meson and the testing
system.

As far as I can tell the functional tests are meant to be run after
installing anyway, but unfortunately I can't transparently make
`meson test --suite installcheck` depend on the install targets.

The script that runs the functional tests, meson/run-test.py, checks
that `meson install` has happened and fails fast with a (hopefully)
helpful error message if any of the functional tests are run before
installing.

TODO: this change needs reflection in developer documentation

Change-Id: I8dcb5fdfc0b6cb17580973d24ad930abd57018f6
This commit is contained in:
Qyriad 2024-03-25 12:12:56 -06:00
parent edba570664
commit 038daad218
11 changed files with 449 additions and 1 deletions

View file

@ -18,6 +18,12 @@
# Finally, src/nix/meson.build defines the Nix command itself, relying on all prior meson files. # Finally, src/nix/meson.build defines the Nix command itself, relying on all prior meson files.
# #
# Unit tests are setup in tests/unit/meson.build, under the test suite "check". # Unit tests are setup in tests/unit/meson.build, under the test suite "check".
#
# Functional tests are a bit more complicated. Generally they're defined in
# tests/functional/meson.build, and rely on helper scripts meson/setup-functional-tests.py
# and meson/run-test.py. Scattered around also are configure_file() invocations, which must
# be placed in specific directories' meson.build files to create the right directory tree
# in the build directory.
project('lix', 'cpp', project('lix', 'cpp',
version : run_command('bash', '-c', 'echo -n $(cat ./.version)$VERSION_SUFFIX', check : true).stdout().strip(), version : run_command('bash', '-c', 'echo -n $(cat ./.version)$VERSION_SUFFIX', check : true).stdout().strip(),
@ -27,6 +33,7 @@ project('lix', 'cpp',
'warning_level=1', 'warning_level=1',
'debug=true', 'debug=true',
'optimization=2', 'optimization=2',
'errorlogs=true', # Please print logs for tests that fail
], ],
) )
@ -48,6 +55,11 @@ path_opts = [
'state-dir', 'state-dir',
'log-dir', 'log-dir',
] ]
# For your grepping pleasure, this loop sets the following variables that aren't mentioned
# literally above:
# store_dir
# state_dir
# log_dir
foreach optname : path_opts foreach optname : path_opts
varname = optname.replace('-', '_') varname = optname.replace('-', '_')
path = get_option(optname) path = get_option(optname)
@ -203,6 +215,10 @@ deps += gtest
# Build-time tools # Build-time tools
# #
bash = find_program('bash') bash = find_program('bash')
coreutils = find_program('coreutils')
dot = find_program('dot', required : false)
pymod = import('python')
python = pymod.find_installation('python3')
# Used to workaround https://github.com/mesonbuild/meson/issues/2320 in src/nix/meson.build. # Used to workaround https://github.com/mesonbuild/meson/issues/2320 in src/nix/meson.build.
installcmd = find_program('install') installcmd = find_program('install')
@ -316,6 +332,13 @@ if get_option('profile-build').require(meson.get_compiler('cpp').get_id() == 'cl
endif endif
subdir('src') subdir('src')
if enable_tests if enable_tests
# Just configures `scripts/nix-profile.sh.in` (and copies the original to the build directory).
# Done as a subdirectory to convince Meson to put the configured files
# in `build/scripts` instead of just `build`.
subdir('scripts')
subdir('tests/unit') subdir('tests/unit')
subdir('tests/functional')
endif endif

87
meson/run-test.py Executable file
View file

@ -0,0 +1,87 @@
#!/usr/bin/env python3
"""
This script is a helper for this project's Meson buildsystem to run Lix's
functional tests. It is an analogue to mk/run-test.sh in the autoconf+Make
buildsystem.
These tests are run in the installCheckPhase in Lix's derivation, and as such
expect to be run after the project has already been "installed" to some extent.
Look at meson/setup-functional-tests.py for more details.
"""
import argparse
from pathlib import Path
import os
import shutil
import subprocess
import sys
name = 'run-test.py'
if 'MESON_BUILD_ROOT' not in os.environ:
raise ValueError(f'{name}: this script must be run from the Meson build system')
def main():
tests_dir = Path(os.path.join(os.environ['MESON_BUILD_ROOT'], 'tests/functional'))
parser = argparse.ArgumentParser(name)
parser.add_argument('target', help='the script path relative to tests/functional to run')
args = parser.parse_args()
target = Path(args.target)
# The test suite considers the test's name to be the path to the test relative to
# `tests/functional`, but without the file extension.
# e.g. for `tests/functional/flakes/develop.sh`, the test name is `flakes/develop`
test_name = target.with_suffix('').as_posix()
if not target.is_absolute():
target = tests_dir.joinpath(target).resolve()
assert target.exists(), f'{name}: test {target} does not exist; did you run `meson install`?'
bash = os.environ.get('BASH', shutil.which('bash'))
if bash is None:
raise ValueError(f'{name}: bash executable not found and BASH environment variable not set')
test_environment = os.environ | {
'TEST_NAME': test_name,
# mk/run-test.sh did this, but I don't know if it has any effect since it seems
# like the tests that interact with remote stores set it themselves?
'NIX_REMOTE': '',
}
# Initialize testing.
init_result = subprocess.run([bash, '-e', 'init.sh'], cwd=tests_dir, env=test_environment)
if init_result.returncode != 0:
print(f'{name}: internal error initializing {args.target}', file=sys.stderr)
print('[ERROR]')
# Meson interprets exit code 99 as indicating an *error* in the testing process.
return 99
# Run the test itself.
test_result = subprocess.run([bash, '-e', target.name], cwd=target.parent, env=test_environment)
if test_result.returncode == 0:
print('[PASS]')
elif test_result.returncode == 99:
print('[SKIP]')
# Meson interprets exit code 77 as indicating a skipped test.
return 77
else:
print('[FAIL]')
return test_result.returncode
try:
sys.exit(main())
except AssertionError as e:
# This should mean that this test was run not-from-Meson, probably without
# having run `meson install` first, which is not an bug in this script.
print(e, file=sys.stderr)
sys.exit(99)
except Exception as e:
print(f'{name}: INTERNAL ERROR running test ({sys.argv}): {e}', file=sys.stderr)
print(f'this is a bug in {name}')
sys.exit(99)

105
meson/setup-functional-tests.py Executable file
View file

@ -0,0 +1,105 @@
#!/usr/bin/env python3
"""
So like. This script is cursed.
It's a helper for this project's Meson buildsystem for Lix's functional tests.
The functional tests are a bunch of bash scripts, that each expect to be run from the
directory from the directory that that script is in, and also expect modifications to have
happened to the source tree, and even splork files around. The last is against the spirit
of Meson (and personally annoying), to have build processes that aren't self-contained in the
out-of-source build directory, but more problematically we need configured files in the test
tree.
So. We copy the tests tree into the build directory.
Meson doesn't have a good way of doing this natively -- the best you could do is subdir()
into every directory in the tests tree and configure_file(copy : true) on every file,
but this won't copy symlinks as symlinks, which we need since the test suite has, well,
tests on symlinks.
However, the functional tests are normally run during Lix's derivation's installCheckPhase,
after Lix has already been "installed" somewhere. So in Meson we setup add this file as an
install script and copy everything in tests/functional to the build directory, preserving
things like symlinks, even broken ones (which are intentional).
TODO(Qyriad): when we remove the old build system entirely, we can instead fix the tests.
"""
from pathlib import Path
import os, os.path
import shutil
import sys
import traceback
name = 'setup-functional-tests.py'
if 'MESON_SOURCE_ROOT' not in os.environ or 'MESON_BUILD_ROOT' not in os.environ:
raise ValueError(f'{name}: this script must be run from the Meson build system')
print(f'{name}: mirroring tests/functional to build directory')
tests_source = Path(os.environ['MESON_SOURCE_ROOT']) / 'tests/functional'
tests_build = Path(os.environ['MESON_BUILD_ROOT']) / 'tests/functional'
def main():
os.chdir(tests_build)
for src_dirpath, src_dirnames, src_filenames in os.walk(tests_source):
src_dirpath = Path(src_dirpath)
assert src_dirpath.is_absolute(), f'{src_dirpath=} is not absolute'
# os.walk() gives us the absolute path to the directory we're currently in as src_dirpath.
# We want to mirror from the perspective of `tests_source`.
rel_subdir = src_dirpath.relative_to(tests_source)
assert (not rel_subdir.is_absolute()), f'{rel_subdir=} is not relative'
# And then join that relative path on `tests_build` to get the absolute
# path in the build directory that corresponds to `src_dirpath`.
build_dirpath = tests_build / rel_subdir
assert build_dirpath.is_absolute(), f'{build_dirpath=} is not absolute'
# More concretely, for the test file tests/functional/ca/build.sh:
# - src_dirpath is `$MESON_SOURCE_ROOT/tests/functional/ca`
# - rel_subidr is `ca`
# - build_dirpath is `$MESON_BUILD_ROOT/tests/functional/ca`
# `src_dirname` are directories underneath `src_dirpath`, and will be relative
# to `src_dirpath`.
for src_dirname in src_dirnames:
# Take the name of the directory in the tests source and join it on `src_dirpath`
# to get the full path to this specific directory in the tests source.
src = src_dirpath / src_dirname
# If there isn't *something* here, then our logic is wrong.
# Path.exists(follow_symlinks=False) wasn't added until Python 3.12, so we use
# os.path.lexists() here.
assert os.path.lexists(src), f'{src=} does not exist'
# Take the name of this directory and join it on `build_dirpath` to get the full
# path to the directory in `build/tests/functional` that we need to create.
dest = build_dirpath / src_dirname
if src.is_symlink():
src_target = src.readlink()
dest.unlink(missing_ok=True)
dest.symlink_to(src_target)
else:
dest.mkdir(parents=True, exist_ok=True)
for src_filename in src_filenames:
# os.walk() should be giving us relative filenames.
# If it isn't, our path joins will be veeeery wrong.
assert (not Path(src_filename).is_absolute()), f'{src_filename=} is not relative'
src = src_dirpath / src_filename
dst = build_dirpath / src_filename
# Mildly misleading name -- unlink removes ordinary files as well as symlinks.
dst.unlink(missing_ok=True)
# shutil.copy2() best-effort preserves metadata.
shutil.copy2(src, dst, follow_symlinks=False)
try:
sys.exit(main())
except Exception as e:
# Any error is likely a bug in this script.
print(f'{name}: INTERNAL ERROR setting up functional tests: {e}', file=sys.stderr)
print(traceback.format_exc())
print(f'this is a bug in {name}')
sys.exit(1)

View file

@ -101,7 +101,8 @@
] ++ lib.optionals buildWithMeson [ ] ++ lib.optionals buildWithMeson [
./meson.build ./meson.build
./meson.options ./meson.options
./meson/cleanup-install.bash ./meson
./scripts/meson.build
]); ]);
functionalTestFiles = fileset.unions [ functionalTestFiles = fileset.unions [
@ -279,10 +280,21 @@ in stdenv.mkDerivation (finalAttrs: {
installCheckFlags = "sysconfdir=$(out)/etc"; installCheckFlags = "sysconfdir=$(out)/etc";
installCheckTarget = "installcheck"; # work around buggy detection in stdenv installCheckTarget = "installcheck"; # work around buggy detection in stdenv
mesonInstallCheckFlags = [
"--suite=installcheck"
];
preInstallCheck = lib.optionalString stdenv.hostPlatform.isDarwin '' preInstallCheck = lib.optionalString stdenv.hostPlatform.isDarwin ''
export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
''; '';
installCheckPhase = lib.optionalString buildWithMeson ''
runHook preInstallCheck
flagsArray=($mesonInstallCheckFlags "''${mesonInstallCheckFlagsArray[@]}")
meson test --no-rebuild "''${flagsArray[@]}"
runHook postInstallCheck
'';
separateDebugInfo = !stdenv.hostPlatform.isStatic && !finalAttrs.dontBuild; separateDebugInfo = !stdenv.hostPlatform.isStatic && !finalAttrs.dontBuild;
strictDeps = true; strictDeps = true;

14
scripts/meson.build Normal file
View file

@ -0,0 +1,14 @@
configure_file(
input : 'nix-profile.sh.in',
output : 'nix-profile.sh',
configuration : {
'localstatedir': state_dir,
}
)
# https://github.com/mesonbuild/meson/issues/860
configure_file(
input : 'nix-profile.sh.in',
output : 'nix-profile.sh.in',
copy : true,
)

View file

@ -63,3 +63,7 @@ nix2_commands_sources = [
# Finally, the nix command itself, which all of the other commands are implmented in terms of # Finally, the nix command itself, which all of the other commands are implmented in terms of
# as a multicall binary. # as a multicall binary.
subdir('nix') subdir('nix')
# Just copies nix-channel/unpack-channel.nix to the build directory.
# Done as a subdir to get Meson to respect the path hierarchy.
subdir('nix-channel')

View file

@ -0,0 +1,5 @@
configure_file(
input : 'unpack-channel.nix',
output : 'unpack-channel.nix',
copy : true,
)

View file

@ -0,0 +1,6 @@
# test_confdata set from tests/functional/meson.build
configure_file(
input : 'config.nix.in',
output : 'config.nix',
configuration : test_confdata,
)

View file

@ -0,0 +1,6 @@
# test_confdata set from tests/functional/meson.build
vars_and_functions = configure_file(
input : 'vars-and-functions.sh.in',
output : 'vars-and-functions.sh',
configuration : test_confdata,
)

View file

@ -0,0 +1,6 @@
# test_confdata set from tests/functional/meson.build
configure_file(
input : 'config.nix.in',
output : 'config.nix',
configuration : test_confdata,
)

View file

@ -0,0 +1,180 @@
test_confdata = {
'bindir': bindir,
'coreutils': fs.parent(coreutils.full_path()),
'lsof': lsof.full_path(),
'dot': dot.found() ? dot.full_path() : '',
'bash': bash.full_path(),
'sandbox_shell': busybox.found() ? busybox.full_path() : '',
'PACKAGE_VERSION': meson.project_version(),
'system': host_system,
'BUILD_SHARED_LIBS': '1', # XXX(Qyriad): detect this!
}
# Just configures `common/vars-and-functions.sh.in`.
# Done as a subdir() so Meson places it under `common` in the build directory as well.
subdir('common')
config_nix_in = configure_file(
input : 'config.nix.in',
output : 'config.nix',
configuration : test_confdata,
)
# Just configures `ca/config.nix.in`. Done as a subdir() for the same reason as above.
subdir('ca')
# Just configures `dyn-drv/config.nix.in`. Same as above.
subdir('dyn-drv')
functional_tests_scripts = [
'init.sh',
'test-infra.sh',
'flakes/flakes.sh',
'flakes/develop.sh',
'flakes/develop-r8854.sh',
'flakes/run.sh',
'flakes/mercurial.sh',
'flakes/circular.sh',
'flakes/init.sh',
'flakes/inputs.sh',
'flakes/follow-paths.sh',
'flakes/bundle.sh',
'flakes/check.sh',
'flakes/unlocked-override.sh',
'flakes/absolute-paths.sh',
'flakes/build-paths.sh',
'flakes/flake-in-submodule.sh',
'gc.sh',
'nix-collect-garbage-d.sh',
'remote-store.sh',
'legacy-ssh-store.sh',
'lang.sh',
'lang-test-infra.sh',
'experimental-features.sh',
'fetchMercurial.sh',
'gc-auto.sh',
'user-envs.sh',
'user-envs-migration.sh',
'binary-cache.sh',
'multiple-outputs.sh',
'nix-build.sh',
'gc-concurrent.sh',
'repair.sh',
'fixed.sh',
'export-graph.sh',
'timeout.sh',
'fetchGitRefs.sh',
'gc-runtime.sh',
'tarball.sh',
'fetchGit.sh',
'fetchurl.sh',
'fetchPath.sh',
'fetchTree-file.sh',
'simple.sh',
'referrers.sh',
'optimise-store.sh',
'substitute-with-invalid-ca.sh',
'signing.sh',
'hash.sh',
'gc-non-blocking.sh',
'check.sh',
'nix-shell.sh',
'check-refs.sh',
'build-remote-input-addressed.sh',
'secure-drv-outputs.sh',
'restricted.sh',
'fetchGitSubmodules.sh',
'flakes/search-root.sh',
'readfile-context.sh',
'nix-channel.sh',
'recursive.sh',
'dependencies.sh',
'check-reqs.sh',
'build-remote-content-addressed-fixed.sh',
'build-remote-content-addressed-floating.sh',
'build-remote-trustless-should-pass-0.sh',
'build-remote-trustless-should-pass-1.sh',
'build-remote-trustless-should-pass-2.sh',
'build-remote-trustless-should-pass-3.sh',
'build-remote-trustless-should-fail-0.sh',
'nar-access.sh',
'impure-eval.sh',
'pure-eval.sh',
'eval.sh',
'repl.sh',
'binary-cache-build-remote.sh',
'search.sh',
'logging.sh',
'export.sh',
'config.sh',
'add.sh',
'local-store.sh',
'filter-source.sh',
'misc.sh',
'dump-db.sh',
'linux-sandbox.sh',
'supplementary-groups.sh',
'build-dry.sh',
'structured-attrs.sh',
'shell.sh',
'brotli.sh',
'zstd.sh',
'compression-levels.sh',
'nix-copy-ssh.sh',
'nix-copy-ssh-ng.sh',
'post-hook.sh',
'function-trace.sh',
'flakes/config.sh',
'fmt.sh',
'eval-store.sh',
'why-depends.sh',
'derivation-json.sh',
'import-derivation.sh',
'nix_path.sh',
'case-hack.sh',
'placeholders.sh',
'ssh-relay.sh',
'build.sh',
'build-delete.sh',
'output-normalization.sh',
'selfref-gc.sh',
'db-migration.sh',
'bash-profile.sh',
'pass-as-file.sh',
'nix-profile.sh',
'suggestions.sh',
'store-ping.sh',
'fetchClosure.sh',
'completions.sh',
'flakes/show.sh',
'impure-derivations.sh',
'path-from-hash-part.sh',
'toString-path.sh',
'read-only-store.sh',
'nested-sandboxing.sh',
'debugger.sh',
]
# TODO(Qyriad): this will hopefully be able to be removed when we remove the autoconf+Make
# buildsystem. See the comments at the top of setup-functional-tests.py for why this is here.
meson.add_install_script(
python,
meson.project_source_root() / 'meson/setup-functional-tests.py',
)
foreach script : functional_tests_scripts
# Turns, e.g., `tests/functional/flakes/show.sh` into a Meson test target called
# `functional-flakes-show`.
name = 'functional-@0@'.format(fs.replace_suffix(script, '')).replace('/', '-')
test(
name,
python,
args: [
meson.project_source_root() / 'meson/run-test.py',
script,
],
suite : 'installcheck',
env : {
'MESON_BUILD_ROOT': meson.project_build_root(),
},
)
endforeach