testsuite: add a functional2 test suite based on pytest

I am tired of bad shell scripts, let me write bad python quickly
instead. It's definitely, $100%, better.

This is not planned as an immediate replacement of the old test suite,
but we::jade would not oppose tests getting ported.

What is here is a mere starting point and there is a lot more
functionality that we need.

Fixes: #488

Change-Id: If762efce69030bb667491b263b874c36024bf7b6

meson.build

@@ -607,6 +607,7 @@ endif
 if enable_tests
   subdir('tests/unit')
   subdir('tests/functional')
+  subdir('tests/functional2')
 endif
 
 subdir('meson/clang-tidy')

package.nix

@@ -170,6 +170,7 @@ let
 
   functionalTestFiles = fileset.unions [
     ./tests/functional
+    ./tests/functional2
     ./tests/unit
     (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
   ];
@@ -243,6 +244,7 @@ stdenv.mkDerivation (finalAttrs: {
   nativeBuildInputs =
     [
       python3
+      python3.pkgs.pytest
       meson
       ninja
       cmake
@@ -474,6 +476,10 @@ stdenv.mkDerivation (finalAttrs: {
 
         pythonPackages = (
           p: [
+            # FIXME: these have to be added twice due to the nix shell using a
+            # wrapped python instead of build inputs for its python inputs
+            p.pytest
+
             p.yapf
             p.python-frontmatter
             p.requests

tests/functional2/README.md

@@ -0,0 +1,24 @@
+# functional2 tests
+
+This uncreatively named test suite is a Pytest based replacement for the shell framework used to write traditional Nix integration tests.
+Its primary goal is to make tests more concise, more self-contained, easier to write, and to produce better errors.
+
+## Goals
+
+- Eliminate implicit dependencies on files in the test directory as well as the requirement to copy the test files to the build directory as is currently hacked in the other functional test suite.
+  - You should be able to write a DirectoryTree of files for your test declaratively.
+- Reduce the amount of global environment state being thrown around in the test suite.
+- Make tests very concise and easy to reuse code for, and hopefully turn more of what is currently code into data.
+  - Provide rich ways of calling `nix` with pleasant syntax.
+
+## TODO: Intended features
+
+- [ ] Expect tests ([pytest-expect-test]) or snapshot tests ([pytest-insta]) or, likely, both!
+  We::jade prefer to have short output written in-line as it makes it greatly easier to read the tests, but pytest-expect doesn't allow for putting larger stuff in external files, so something else is necessary for those.
+- [ ] Web server fixture: we don't test our network functionality because background processes are hard and this is simply goofy.
+  We could just test it.
+- [ ] Nix daemon fixture.
+- [ ] Parallelism via pytest-xdist.
+
+[pytest-expect-test]: https://pypi.org/project/pytest-expect-test/
+[pytest-insta]: https://pypi.org/project/pytest-insta/

tests/functional2/conftest.py

@@ -0,0 +1,8 @@
+import pytest
+from pathlib import Path
+from .testlib import fixtures
+
+
+@pytest.fixture
+def nix(tmp_path: Path):
+    return fixtures.Nix(tmp_path)

tests/functional2/meson.build

@@ -0,0 +1,19 @@
+# surprisingly, this actually works even if PATH is set to something before
+# meson gets hold of it. neat!
+functional2_env = environment()
+functional2_env.prepend('PATH', bindir)
+
+test(
+  'functional2',
+  python,
+  args : [
+    '-m', 'pytest', meson.current_source_dir()
+  ],
+  env : functional2_env,
+  # FIXME: Although we can trivially use TAP here with pytest-tap, due to a meson bug, it is unusable.
+  # (failure output does not get displayed to the console. at all. someone should go fix it):
+  # https://github.com/mesonbuild/meson/issues/11185
+  # protocol : 'tap',
+  suite : 'installcheck',
+  timeout : 300,
+)

tests/functional2/test_eval_trivial.py

@@ -0,0 +1,4 @@
+from .testlib.fixtures import Nix
+
+def test_trivial_addition(nix: Nix):
+    assert nix.eval('1 + 1').json() == 2

tests/functional2/testlib/fixtures.py

@@ -0,0 +1,121 @@
+import os
+import json
+import subprocess
+from typing import Any
+from pathlib import Path
+import dataclasses
+
+
+@dataclasses.dataclass
+class CommandResult:
+    cmd: list[str]
+    rc: int
+    """Return code"""
+    stderr: bytes
+    """Outputted stderr"""
+    stdout: bytes
+    """Outputted stdout"""
+
+    def ok(self):
+        if self.rc != 0:
+            raise subprocess.CalledProcessError(returncode=self.rc,
+                                                cmd=self.cmd,
+                                                stderr=self.stderr,
+                                                output=self.stdout)
+        return self
+
+    def json(self) -> Any:
+        self.ok()
+        return json.loads(self.stdout)
+
+
+@dataclasses.dataclass
+class NixSettings:
+    """Settings for invoking Nix"""
+    experimental_features: set[str] | None = None
+
+    def feature(self, *names: str):
+        self.experimental_features = (self.experimental_features
+                                      or set()) | set(names)
+        return self
+
+    def to_config(self) -> str:
+        config = ''
+
+        def serialise(value):
+            if type(value) in {str, int}:
+                return str(value)
+            elif type(value) in {list, set}:
+                return ' '.join(str(e) for e in value)
+            else:
+                raise ValueError(
+                    f'Value is unsupported in nix config: {value!r}')
+
+        def field_may(name, value, serialiser=serialise):
+            nonlocal config
+            if value is not None:
+                config += f'{name} = {serialiser(value)}\n'
+
+        field_may('experimental-features', self.experimental_features)
+        return config
+
+
+@dataclasses.dataclass
+class Nix:
+    test_root: Path
+
+    def hermetic_env(self):
+        # mirroring vars-and-functions.sh
+        home = self.test_root / 'test-home'
+        home.mkdir(parents=True, exist_ok=True)
+        return {
+            'NIX_STORE_DIR': self.test_root / 'store',
+            'NIX_LOCALSTATE_DIR': self.test_root / 'var',
+            'NIX_LOG_DIR': self.test_root / 'var/log/nix',
+            'NIX_STATE_DIR': self.test_root / 'var/nix',
+            'NIX_CONF_DIR': self.test_root / 'etc',
+            'NIX_DAEMON_SOCKET_PATH': self.test_root / 'daemon-socket',
+            'NIX_USER_CONF_FILES': '',
+            'HOME': home,
+        }
+
+    def make_env(self):
+        # We conservatively assume that people might want to successfully get
+        # some env through to the subprocess, so we override whatever is in the
+        # global env.
+        d = os.environ.copy()
+        d.update(self.hermetic_env())
+        return d
+
+    def call(self, cmd: list[str], extra_env: dict[str, str] = {}):
+        """
+        Calls a process in the test environment.
+        """
+        env = self.make_env()
+        env.update(extra_env)
+        proc = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            cwd=self.test_root,
+            env=env,
+        )
+        (stdout, stderr) = proc.communicate()
+        rc = proc.returncode
+        return CommandResult(cmd=cmd, rc=rc, stdout=stdout, stderr=stderr)
+
+    def nix(self,
+            cmd: list[str],
+            settings: NixSettings = NixSettings(),
+            extra_env: dict[str, str] = {}):
+        extra_env = extra_env.copy()
+        extra_env.update({'NIX_CONFIG': settings.to_config()})
+        return self.call(['nix', *cmd], extra_env)
+
+    def eval(
+        self, expr: str,
+        settings: NixSettings = NixSettings()) -> CommandResult:
+        # clone due to reference-shenanigans
+        settings = dataclasses.replace(settings).feature('nix-command')
+
+        return self.nix(['eval', '--json', '--expr', expr], settings=settings)