initialize test suite for git fetchers

solves #9388

This utilizes nixos vm tests to allow:
- writing tests for fetchTree and fetchGit involving actual networking.
- writing small independent test cases by automating local and remote repository setup per test case.

This adds:
  - a gitea module setting up a gitea server
  - a setup module that simplifies writing test cases by automating the repo setup.
  - a simple git http test case

Other improvements:
For all nixos tests, add capability of overriding the nix version to test against.
This should make it easier to prevent regressions. If a new test is added it can simply be ran against any older nix version without having to backport the test.
For example, for running the container tests against nix 2.12.0:
`nix build "$(nix eval --raw .#hydraJobs.tests.containers --impure --apply 't: (t.forNix "2.12.0").drvPath')^*" -L`

tests/nixos/default.nix

@@ -5,13 +5,27 @@ let
   nixos-lib = import (nixpkgs + "/nixos/lib") { };
 
   # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests
-  runNixOSTestFor = system: test: nixos-lib.runTest {
+  runNixOSTestFor = system: test:
+    (nixos-lib.runTest {
       imports = [ test ];
       hostPkgs = nixpkgsFor.${system}.native;
       defaults = {
         nixpkgs.pkgs = nixpkgsFor.${system}.native;
+        nix.checkAllErrors = false;
       };
       _module.args.nixpkgs = nixpkgs;
+      _module.args.system = system;
+    })
+    // {
+      # allow running tests against older nix versions via `nix eval --apply`
+      # Example:
+      #   nix build "$(nix eval --raw --impure .#hydraJobs.tests.fetch-git --apply 't: (t.forNix "2.19.2").drvPath')^*"
+      forNix = nixVersion: runNixOSTestFor system {
+        imports = [test];
+        defaults.nixpkgs.overlays = [(curr: prev: {
+          nix = (builtins.getFlake "nix/${nixVersion}").packages.${system}.nix;
+        })];
+      };
     };
 
 in
@@ -40,4 +54,6 @@ in
   setuid = lib.genAttrs
     ["i686-linux" "x86_64-linux"]
     (system: runNixOSTestFor system ./setuid.nix);
+
+  fetch-git = runNixOSTestFor "x86_64-linux" ./fetch-git;
 }

tests/nixos/fetch-git/default.nix

@@ -0,0 +1,60 @@
+{ lib, config, ... }:
+{
+  name = "fetch-git";
+
+  imports = [
+    ./testsupport/gitea.nix
+  ];
+
+  /*
+  Test cases
+  The following is set up automatically for each test case:
+    - a repo with the {name} is created on the gitea server
+    - a repo with the {name} is created on the client
+    - the client repo is configured to push to the server repo
+  Python variables:
+    - repo.path: the path to the directory of the client repo
+    - repo.git: the git command with the client repo as the working directory
+    - repo.remote: the url to the server repo
+  */
+  testCases = [
+    {
+      name = "simple-http";
+      description = "can fetch a git repo via http";
+      script = ''
+        # add a file to the repo
+        client.succeed(f"""
+          echo chiang-mai > {repo.path}/thailand \
+          && {repo.git} add thailand \
+          && {repo.git} commit -m 'commit1'
+        """)
+
+        # memoize the revision
+        rev1 = client.succeed(f"""
+          {repo.git} rev-parse HEAD
+        """).strip()
+
+        # push to the server
+        client.succeed(f"""
+          {repo.git} push origin main
+        """)
+
+        # fetch the repo via nix
+        fetched1 = client.succeed(f"""
+          nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).outPath"
+        """)
+
+        # check if the committed file is there
+        client.succeed(f"""
+          test -f {fetched1}/thailand
+        """)
+
+        # check if the revision is the same
+        rev1_fetched = client.succeed(f"""
+          nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).rev"
+        """).strip()
+        assert rev1 == rev1_fetched
+      '';
+    }
+  ];
+}

tests/nixos/fetch-git/testsupport/gitea.nix

@@ -0,0 +1,63 @@
+{ lib, nixpkgs, system, ... }: {
+  imports = [
+    ../testsupport/setup.nix
+  ];
+  nodes = {
+    gitea = { pkgs, ... }: {
+      services.gitea.enable = true;
+      services.gitea.settings.service.DISABLE_REGISTRATION = true;
+      services.gitea.settings.log.LEVEL = "Info";
+      services.gitea.settings.database.LOG_SQL = false;
+      networking.firewall.allowedTCPPorts = [ 3000 ];
+      environment.systemPackages = [ pkgs.gitea ];
+
+      # TODO: remove this after updating to nixos-23.11
+      nixpkgs.pkgs = lib.mkForce (import nixpkgs {
+        inherit system;
+        config.permittedInsecurePackages = [
+          "gitea-1.19.4"
+        ];
+      });
+    };
+    client = { pkgs, ... }: {
+      environment.systemPackages = [ pkgs.git ];
+    };
+  };
+  defaults = { pkgs, ... }: {
+    environment.systemPackages = [ pkgs.jq ];
+  };
+
+  setupScript = ''
+    import shlex
+
+    gitea.wait_for_unit("gitea.service")
+
+    gitea_admin = "test"
+    gitea_admin_password = "test123test"
+
+    gitea.succeed(f"""
+      gitea --version >&2
+      su -l gitea -c 'GITEA_WORK_DIR=/var/lib/gitea gitea admin user create \
+        --username {gitea_admin} --password {gitea_admin_password} --email test@client'
+    """)
+
+    client.wait_for_unit("multi-user.target")
+    gitea.wait_for_open_port(3000)
+
+    gitea_admin_token = gitea.succeed(f"""
+      curl --fail -X POST http://{gitea_admin}:{gitea_admin_password}@gitea:3000/api/v1/users/test/tokens \
+        -H 'Accept: application/json' -H 'Content-Type: application/json' \
+        -d {shlex.quote( '{"name":"token", "scopes":["all"]}' )} \
+        | jq -r '.sha1'
+    """).strip()
+
+    client.succeed(f"""
+      echo "http://{gitea_admin}:{gitea_admin_password}@gitea:3000" >~/.git-credentials-admin
+      git config --global credential.helper 'store --file ~/.git-credentials-admin'
+      git config --global user.email "test@client"
+      git config --global user.name "Test User"
+      git config --global gc.autodetach 0
+      git config --global gc.auto 0
+    """)
+  '';
+}

tests/nixos/fetch-git/testsupport/setup.nix

@@ -0,0 +1,106 @@
+{ lib, config, extendModules, ... }:
+let
+  inherit (lib)
+    mkOption
+    types
+    ;
+
+  indent = lib.replaceStrings ["\n"] ["\n    "];
+
+  execTestCase = testCase: ''
+
+    ### TEST ${testCase.name}: ${testCase.description} ###
+
+    with subtest("${testCase.description}"):
+        repo = Repo("${testCase.name}")
+        ${indent testCase.script}
+  '';
+in
+{
+
+  options = {
+    setupScript = mkOption {
+      type = types.lines;
+      description = ''
+        Python code that runs before the main test.
+
+        Variables defined by this code will be available in the test.
+      '';
+      default = "";
+    };
+    testCases = mkOption {
+      description = ''
+        The test cases. See `testScript`.
+      '';
+      type = types.listOf (types.submodule {
+        options.name = mkOption {
+          type = types.str;
+          description = ''
+            The name of the test case.
+
+            A repository with that name will be set up on the gitea server and locally.
+
+            This name can also be used to execute only a single test case via:
+              `nix build .#hydraJobs.fetch-git.{test-case-name}`
+          '';
+        };
+        options.description = mkOption {
+          type = types.str;
+          description = ''
+            A description of the test case.
+          '';
+        };
+        options.script = mkOption {
+          type = types.lines;
+          description = ''
+            Python code that runs the test.
+
+            Variables defined by `setupScript` will be available here.
+          '';
+        };
+      });
+    };
+  };
+
+  config = {
+    nodes.client = {
+      environment.variables = {
+        _NIX_FORCE_HTTP = "1";
+      };
+      nix.settings.experimental-features = ["nix-command" "flakes"];
+    };
+    setupScript = ''
+      class Repo:
+        """
+        A class to create a git repository on the gitea server and locally.
+        """
+        def __init__(self, name):
+          self.name = name
+          self.path = "/tmp/repos/" + name
+          self.remote = "http://gitea:3000/test/" + name
+          self.git = f"git -C {self.path}"
+          self.create()
+
+        def create(self):
+          gitea.succeed(f"""
+            curl --fail -X POST http://{gitea_admin}:{gitea_admin_password}@gitea:3000/api/v1/user/repos \
+              -H 'Accept: application/json' -H 'Content-Type: application/json' \
+              -d {shlex.quote( f'{{"name":"{self.name}", "default_branch": "main"}}' )}
+          """)
+          client.succeed(f"""
+            mkdir -p {self.path} \
+            && git init -b main {self.path} \
+            && {self.git} remote add origin {self.remote}
+          """)
+    '';
+    testScript = ''
+      start_all();
+
+      ${config.setupScript}
+
+      ### SETUP COMPLETE ###
+
+      ${lib.concatStringsSep "\n" (map execTestCase config.testCases)}
+    '';
+  };
+}