meson: implement functional tests

Functional tests can be run with
`meson test -C build --suite installcheck`.

Notably, functional tests must be run *after* running `meson install`
(Lix's derivation runs the installcheck suite in installCheckPhase so it
does this correctly), due to some quirks between Meson and the testing
system.

As far as I can tell the functional tests are meant to be run after
installing anyway, but unfortunately I can't transparently make
`meson test --suite installcheck` depend on the install targets.

The script that runs the functional tests, meson/run-test.py, checks
that `meson install` has happened and fails fast with a (hopefully)
helpful error message if any of the functional tests are run before
installing.

TODO: this change needs reflection in developer documentation

Change-Id: I8dcb5fdfc0b6cb17580973d24ad930abd57018f6

meson.build

@@ -18,6 +18,12 @@
 # Finally, src/nix/meson.build defines the Nix command itself, relying on all prior meson files.
 #
 # Unit tests are setup in tests/unit/meson.build, under the test suite "check".
+#
+# Functional tests are a bit more complicated. Generally they're defined in
+# tests/functional/meson.build, and rely on helper scripts meson/setup-functional-tests.py
+# and meson/run-test.py. Scattered around also are configure_file() invocations, which must
+# be placed in specific directories' meson.build files to create the right directory tree
+# in the build directory.
 
 project('lix', 'cpp',
   version : run_command('bash', '-c', 'echo -n $(cat ./.version)$VERSION_SUFFIX', check : true).stdout().strip(),
@@ -27,6 +33,7 @@ project('lix', 'cpp',
     'warning_level=1',
     'debug=true',
     'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
   ],
 )
 
@@ -48,6 +55,11 @@ path_opts = [
   'state-dir',
   'log-dir',
 ]
+# For your grepping pleasure, this loop sets the following variables that aren't mentioned
+# literally above:
+# store_dir
+# state_dir
+# log_dir
 foreach optname : path_opts
   varname = optname.replace('-', '_')
   path = get_option(optname)
@@ -203,6 +215,10 @@ deps += gtest
 # Build-time tools
 #
 bash = find_program('bash')
+coreutils = find_program('coreutils')
+dot = find_program('dot', required : false)
+pymod = import('python')
+python = pymod.find_installation('python3')
 
 # Used to workaround https://github.com/mesonbuild/meson/issues/2320 in src/nix/meson.build.
 installcmd = find_program('install')
@@ -316,6 +332,13 @@ if get_option('profile-build').require(meson.get_compiler('cpp').get_id() == 'cl
 endif
 
 subdir('src')
+
 if enable_tests
+  # Just configures `scripts/nix-profile.sh.in` (and copies the original to the build directory).
+  # Done as a subdirectory to convince Meson to put the configured files
+  # in `build/scripts` instead of just `build`.
+  subdir('scripts')
+
   subdir('tests/unit')
+  subdir('tests/functional')
 endif

meson/run-test.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+
+"""
+This script is a helper for this project's Meson buildsystem to run Lix's
+functional tests. It is an analogue to mk/run-test.sh in the autoconf+Make
+buildsystem.
+
+These tests are run in the installCheckPhase in Lix's derivation, and as such
+expect to be run after the project has already been "installed" to some extent.
+Look at meson/setup-functional-tests.py for more details.
+"""
+
+import argparse
+from pathlib import Path
+import os
+import shutil
+import subprocess
+import sys
+
+name = 'run-test.py'
+
+if 'MESON_BUILD_ROOT' not in os.environ:
+    raise ValueError(f'{name}: this script must be run from the Meson build system')
+
+def main():
+
+    tests_dir = Path(os.path.join(os.environ['MESON_BUILD_ROOT'], 'tests/functional'))
+
+    parser = argparse.ArgumentParser(name)
+    parser.add_argument('target', help='the script path relative to tests/functional to run')
+    args = parser.parse_args()
+
+    target = Path(args.target)
+    # The test suite considers the test's name to be the path to the test relative to
+    # `tests/functional`, but without the file extension.
+    # e.g. for `tests/functional/flakes/develop.sh`, the test name is `flakes/develop`
+    test_name = target.with_suffix('').as_posix()
+    if not target.is_absolute():
+        target = tests_dir.joinpath(target).resolve()
+
+    assert target.exists(), f'{name}: test {target} does not exist; did you run `meson install`?'
+
+    bash = os.environ.get('BASH', shutil.which('bash'))
+    if bash is None:
+        raise ValueError(f'{name}: bash executable not found and BASH environment variable not set')
+
+    test_environment = os.environ | {
+        'TEST_NAME': test_name,
+        # mk/run-test.sh did this, but I don't know if it has any effect since it seems
+        # like the tests that interact with remote stores set it themselves?
+        'NIX_REMOTE': '',
+    }
+
+    # Initialize testing.
+    init_result = subprocess.run([bash, '-e', 'init.sh'], cwd=tests_dir, env=test_environment)
+    if init_result.returncode != 0:
+        print(f'{name}: internal error initializing {args.target}', file=sys.stderr)
+        print('[ERROR]')
+        # Meson interprets exit code 99 as indicating an *error* in the testing process.
+        return 99
+
+    # Run the test itself.
+    test_result = subprocess.run([bash, '-e', target.name], cwd=target.parent, env=test_environment)
+
+    if test_result.returncode == 0:
+        print('[PASS]')
+    elif test_result.returncode == 99:
+        print('[SKIP]')
+        # Meson interprets exit code 77 as indicating a skipped test.
+        return 77
+    else:
+        print('[FAIL]')
+
+    return test_result.returncode
+
+try:
+    sys.exit(main())
+except AssertionError as e:
+    # This should mean that this test was run not-from-Meson, probably without
+    # having run `meson install` first, which is not an bug in this script.
+    print(e, file=sys.stderr)
+    sys.exit(99)
+except Exception as e:
+    print(f'{name}: INTERNAL ERROR running test ({sys.argv}): {e}', file=sys.stderr)
+    print(f'this is a bug in {name}')
+    sys.exit(99)
+

meson/setup-functional-tests.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+
+"""
+So like. This script is cursed.
+It's a helper for this project's Meson buildsystem for Lix's functional tests.
+The functional tests are a bunch of bash scripts, that each expect to be run from the
+directory from the directory that that script is in, and also expect modifications to have
+happened to the source tree, and even splork files around. The last is against the spirit
+of Meson (and personally annoying), to have build processes that aren't self-contained in the
+out-of-source build directory, but more problematically we need configured files in the test
+tree.
+So. We copy the tests tree into the build directory.
+Meson doesn't have a good way of doing this natively -- the best you could do is subdir()
+into every directory in the tests tree and configure_file(copy : true) on every file,
+but this won't copy symlinks as symlinks, which we need since the test suite has, well,
+tests on symlinks.
+However, the functional tests are normally run during Lix's derivation's installCheckPhase,
+after Lix has already been "installed" somewhere. So in Meson we setup add this file as an
+install script and copy everything in tests/functional to the build directory, preserving
+things like symlinks, even broken ones (which are intentional).
+
+TODO(Qyriad): when we remove the old build system entirely, we can instead fix the tests.
+"""
+
+from pathlib import Path
+import os, os.path
+import shutil
+import sys
+import traceback
+
+name = 'setup-functional-tests.py'
+
+if 'MESON_SOURCE_ROOT' not in os.environ or 'MESON_BUILD_ROOT' not in os.environ:
+    raise ValueError(f'{name}: this script must be run from the Meson build system')
+
+print(f'{name}: mirroring tests/functional to build directory')
+
+tests_source = Path(os.environ['MESON_SOURCE_ROOT']) / 'tests/functional'
+tests_build = Path(os.environ['MESON_BUILD_ROOT']) / 'tests/functional'
+
+def main():
+
+    os.chdir(tests_build)
+
+    for src_dirpath, src_dirnames, src_filenames in os.walk(tests_source):
+        src_dirpath = Path(src_dirpath)
+        assert src_dirpath.is_absolute(), f'{src_dirpath=} is not absolute'
+
+        # os.walk() gives us the absolute path to the directory we're currently in as src_dirpath.
+        # We want to mirror from the perspective of `tests_source`.
+        rel_subdir = src_dirpath.relative_to(tests_source)
+        assert (not rel_subdir.is_absolute()), f'{rel_subdir=} is not relative'
+
+        # And then join that relative path on `tests_build` to get the absolute
+        # path in the build directory that corresponds to `src_dirpath`.
+        build_dirpath = tests_build / rel_subdir
+        assert build_dirpath.is_absolute(), f'{build_dirpath=} is not absolute'
+
+        # More concretely, for the test file tests/functional/ca/build.sh:
+        # - src_dirpath is `$MESON_SOURCE_ROOT/tests/functional/ca`
+        # - rel_subidr is `ca`
+        # - build_dirpath is `$MESON_BUILD_ROOT/tests/functional/ca`
+
+        # `src_dirname` are directories underneath `src_dirpath`, and will be relative
+        # to `src_dirpath`.
+        for src_dirname in src_dirnames:
+            # Take the name of the directory in the tests source and join it on `src_dirpath`
+            # to get the full path to this specific directory in the tests source.
+            src = src_dirpath / src_dirname
+            # If there isn't *something* here, then our logic is wrong.
+            # Path.exists(follow_symlinks=False) wasn't added until Python 3.12, so we use
+            # os.path.lexists() here.
+            assert os.path.lexists(src), f'{src=} does not exist'
+
+            # Take the name of this directory and join it on `build_dirpath` to get the full
+            # path to the directory in `build/tests/functional` that we need to create.
+            dest = build_dirpath / src_dirname
+            if src.is_symlink():
+                src_target = src.readlink()
+                dest.unlink(missing_ok=True)
+                dest.symlink_to(src_target)
+            else:
+                dest.mkdir(parents=True, exist_ok=True)
+
+        for src_filename in src_filenames:
+            # os.walk() should be giving us relative filenames.
+            # If it isn't, our path joins will be veeeery wrong.
+            assert (not Path(src_filename).is_absolute()), f'{src_filename=} is not relative'
+
+            src = src_dirpath / src_filename
+            dst = build_dirpath / src_filename
+            # Mildly misleading name -- unlink removes ordinary files as well as symlinks.
+            dst.unlink(missing_ok=True)
+            # shutil.copy2() best-effort preserves metadata.
+            shutil.copy2(src, dst, follow_symlinks=False)
+
+
+try:
+    sys.exit(main())
+except Exception as e:
+    # Any error is likely a bug in this script.
+    print(f'{name}: INTERNAL ERROR setting up functional tests: {e}', file=sys.stderr)
+    print(traceback.format_exc())
+    print(f'this is a bug in {name}')
+    sys.exit(1)

package.nix

@@ -101,7 +101,8 @@
   ] ++ lib.optionals buildWithMeson [
     ./meson.build
     ./meson.options
-    ./meson/cleanup-install.bash
+    ./meson
+    ./scripts/meson.build
   ]);
 
  functionalTestFiles = fileset.unions [
@@ -279,10 +280,21 @@ in stdenv.mkDerivation (finalAttrs: {
   installCheckFlags = "sysconfdir=$(out)/etc";
   installCheckTarget = "installcheck"; # work around buggy detection in stdenv
 
+  mesonInstallCheckFlags = [
+    "--suite=installcheck"
+  ];
+
   preInstallCheck = lib.optionalString stdenv.hostPlatform.isDarwin ''
     export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
   '';
 
+  installCheckPhase = lib.optionalString buildWithMeson ''
+    runHook preInstallCheck
+    flagsArray=($mesonInstallCheckFlags "''${mesonInstallCheckFlagsArray[@]}")
+    meson test --no-rebuild "''${flagsArray[@]}"
+    runHook postInstallCheck
+  '';
+
   separateDebugInfo = !stdenv.hostPlatform.isStatic && !finalAttrs.dontBuild;
 
   strictDeps = true;

scripts/meson.build

@@ -0,0 +1,14 @@
+configure_file(
+  input : 'nix-profile.sh.in',
+  output : 'nix-profile.sh',
+  configuration : {
+    'localstatedir': state_dir,
+  }
+)
+
+# https://github.com/mesonbuild/meson/issues/860
+configure_file(
+  input : 'nix-profile.sh.in',
+  output : 'nix-profile.sh.in',
+  copy : true,
+)

src/meson.build

@@ -63,3 +63,7 @@ nix2_commands_sources = [
 # Finally, the nix command itself, which all of the other commands are implmented in terms of
 # as a multicall binary.
 subdir('nix')
+
+# Just copies nix-channel/unpack-channel.nix to the build directory.
+# Done as a subdir to get Meson to respect the path hierarchy.
+subdir('nix-channel')

src/nix-channel/meson.build

@@ -0,0 +1,5 @@
+configure_file(
+  input : 'unpack-channel.nix',
+  output : 'unpack-channel.nix',
+  copy : true,
+)

tests/functional/ca/meson.build

@@ -0,0 +1,6 @@
+# test_confdata set from tests/functional/meson.build
+configure_file(
+  input : 'config.nix.in',
+  output : 'config.nix',
+  configuration : test_confdata,
+)

tests/functional/common/meson.build

@@ -0,0 +1,6 @@
+# test_confdata set from tests/functional/meson.build
+vars_and_functions = configure_file(
+  input : 'vars-and-functions.sh.in',
+  output : 'vars-and-functions.sh',
+  configuration : test_confdata,
+)

tests/functional/dyn-drv/meson.build

@@ -0,0 +1,6 @@
+# test_confdata set from tests/functional/meson.build
+configure_file(
+  input : 'config.nix.in',
+  output : 'config.nix',
+  configuration : test_confdata,
+)

tests/functional/meson.build

@@ -0,0 +1,180 @@
+test_confdata = {
+  'bindir': bindir,
+  'coreutils': fs.parent(coreutils.full_path()),
+  'lsof': lsof.full_path(),
+  'dot': dot.found() ? dot.full_path() : '',
+  'bash': bash.full_path(),
+  'sandbox_shell': busybox.found() ? busybox.full_path() : '',
+  'PACKAGE_VERSION': meson.project_version(),
+  'system': host_system,
+  'BUILD_SHARED_LIBS': '1', # XXX(Qyriad): detect this!
+}
+
+# Just configures `common/vars-and-functions.sh.in`.
+# Done as a subdir() so Meson places it under `common` in the build directory as well.
+subdir('common')
+
+config_nix_in = configure_file(
+  input : 'config.nix.in',
+  output : 'config.nix',
+  configuration : test_confdata,
+)
+
+# Just configures `ca/config.nix.in`. Done as a subdir() for the same reason as above.
+subdir('ca')
+# Just configures `dyn-drv/config.nix.in`. Same as above.
+subdir('dyn-drv')
+
+functional_tests_scripts = [
+  'init.sh',
+  'test-infra.sh',
+  'flakes/flakes.sh',
+  'flakes/develop.sh',
+  'flakes/develop-r8854.sh',
+  'flakes/run.sh',
+  'flakes/mercurial.sh',
+  'flakes/circular.sh',
+  'flakes/init.sh',
+  'flakes/inputs.sh',
+  'flakes/follow-paths.sh',
+  'flakes/bundle.sh',
+  'flakes/check.sh',
+  'flakes/unlocked-override.sh',
+  'flakes/absolute-paths.sh',
+  'flakes/build-paths.sh',
+  'flakes/flake-in-submodule.sh',
+  'gc.sh',
+  'nix-collect-garbage-d.sh',
+  'remote-store.sh',
+  'legacy-ssh-store.sh',
+  'lang.sh',
+  'lang-test-infra.sh',
+  'experimental-features.sh',
+  'fetchMercurial.sh',
+  'gc-auto.sh',
+  'user-envs.sh',
+  'user-envs-migration.sh',
+  'binary-cache.sh',
+  'multiple-outputs.sh',
+  'nix-build.sh',
+  'gc-concurrent.sh',
+  'repair.sh',
+  'fixed.sh',
+  'export-graph.sh',
+  'timeout.sh',
+  'fetchGitRefs.sh',
+  'gc-runtime.sh',
+  'tarball.sh',
+  'fetchGit.sh',
+  'fetchurl.sh',
+  'fetchPath.sh',
+  'fetchTree-file.sh',
+  'simple.sh',
+  'referrers.sh',
+  'optimise-store.sh',
+  'substitute-with-invalid-ca.sh',
+  'signing.sh',
+  'hash.sh',
+  'gc-non-blocking.sh',
+  'check.sh',
+  'nix-shell.sh',
+  'check-refs.sh',
+  'build-remote-input-addressed.sh',
+  'secure-drv-outputs.sh',
+  'restricted.sh',
+  'fetchGitSubmodules.sh',
+  'flakes/search-root.sh',
+  'readfile-context.sh',
+  'nix-channel.sh',
+  'recursive.sh',
+  'dependencies.sh',
+  'check-reqs.sh',
+  'build-remote-content-addressed-fixed.sh',
+  'build-remote-content-addressed-floating.sh',
+  'build-remote-trustless-should-pass-0.sh',
+  'build-remote-trustless-should-pass-1.sh',
+  'build-remote-trustless-should-pass-2.sh',
+  'build-remote-trustless-should-pass-3.sh',
+  'build-remote-trustless-should-fail-0.sh',
+  'nar-access.sh',
+  'impure-eval.sh',
+  'pure-eval.sh',
+  'eval.sh',
+  'repl.sh',
+  'binary-cache-build-remote.sh',
+  'search.sh',
+  'logging.sh',
+  'export.sh',
+  'config.sh',
+  'add.sh',
+  'local-store.sh',
+  'filter-source.sh',
+  'misc.sh',
+  'dump-db.sh',
+  'linux-sandbox.sh',
+  'supplementary-groups.sh',
+  'build-dry.sh',
+  'structured-attrs.sh',
+  'shell.sh',
+  'brotli.sh',
+  'zstd.sh',
+  'compression-levels.sh',
+  'nix-copy-ssh.sh',
+  'nix-copy-ssh-ng.sh',
+  'post-hook.sh',
+  'function-trace.sh',
+  'flakes/config.sh',
+  'fmt.sh',
+  'eval-store.sh',
+  'why-depends.sh',
+  'derivation-json.sh',
+  'import-derivation.sh',
+  'nix_path.sh',
+  'case-hack.sh',
+  'placeholders.sh',
+  'ssh-relay.sh',
+  'build.sh',
+  'build-delete.sh',
+  'output-normalization.sh',
+  'selfref-gc.sh',
+  'db-migration.sh',
+  'bash-profile.sh',
+  'pass-as-file.sh',
+  'nix-profile.sh',
+  'suggestions.sh',
+  'store-ping.sh',
+  'fetchClosure.sh',
+  'completions.sh',
+  'flakes/show.sh',
+  'impure-derivations.sh',
+  'path-from-hash-part.sh',
+  'toString-path.sh',
+  'read-only-store.sh',
+  'nested-sandboxing.sh',
+  'debugger.sh',
+]
+
+# TODO(Qyriad): this will hopefully be able to be removed when we remove the autoconf+Make
+# buildsystem. See the comments at the top of setup-functional-tests.py for why this is here.
+meson.add_install_script(
+  python,
+  meson.project_source_root() / 'meson/setup-functional-tests.py',
+)
+
+foreach script : functional_tests_scripts
+  # Turns, e.g., `tests/functional/flakes/show.sh` into a Meson test target called
+  # `functional-flakes-show`.
+  name = 'functional-@0@'.format(fs.replace_suffix(script, '')).replace('/', '-')
+  test(
+    name,
+    python,
+    args: [
+      meson.project_source_root() / 'meson/run-test.py',
+      script,
+    ],
+    suite : 'installcheck',
+    env : {
+      'MESON_BUILD_ROOT': meson.project_build_root(),
+    },
+  )
+endforeach