From ca03f7cc28ba09cb4db59ca51c52d4e7783f11e0 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Thu, 7 Mar 2024 10:25:03 +0100 Subject: [PATCH] Merge pull request #9676 from DavHau/git-testsuite initialize test suite for git fetchers (cherry picked from commit 0bd9e10aea747df51c8a5af124864c722cbeafde) Change-Id: Idf94a47794190c3e1de07fc4e7848741c4e9ffed --- tests/nixos/default.nix | 30 ++++-- tests/nixos/fetch-git/default.nix | 32 ++++++ .../test-cases/http-simple/default.nix | 39 +++++++ .../test-cases/ssh-simple/default.nix | 43 ++++++++ .../fetch-git/testsupport/gitea-repo.nix | 51 +++++++++ tests/nixos/fetch-git/testsupport/gitea.nix | 102 ++++++++++++++++++ tests/nixos/fetch-git/testsupport/setup.nix | 91 ++++++++++++++++ 7 files changed, 381 insertions(+), 7 deletions(-) create mode 100644 tests/nixos/fetch-git/default.nix create mode 100644 tests/nixos/fetch-git/test-cases/http-simple/default.nix create mode 100644 tests/nixos/fetch-git/test-cases/ssh-simple/default.nix create mode 100644 tests/nixos/fetch-git/testsupport/gitea-repo.nix create mode 100644 tests/nixos/fetch-git/testsupport/gitea.nix create mode 100644 tests/nixos/fetch-git/testsupport/setup.nix diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index f2873f80e..4d4c61b70 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -5,14 +5,28 @@ let nixos-lib = import (nixpkgs + "/nixos/lib") { }; # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests - runNixOSTestFor = system: test: nixos-lib.runTest { - imports = [ test ]; - hostPkgs = nixpkgsFor.${system}.native; - defaults = { - nixpkgs.pkgs = nixpkgsFor.${system}.native; + runNixOSTestFor = system: test: + (nixos-lib.runTest { + imports = [ test ]; + hostPkgs = nixpkgsFor.${system}.native; + defaults = { + nixpkgs.pkgs = nixpkgsFor.${system}.native; + nix.checkAllErrors = false; + }; + _module.args.nixpkgs = nixpkgs; + _module.args.system = system; + }) + // { + # allow running tests against older nix versions via `nix eval --apply` + # Example: + # nix build "$(nix eval --raw --impure .#hydraJobs.tests.fetch-git --apply 't: (t.forNix "2.19.2").drvPath')^*" + forNix = nixVersion: runNixOSTestFor system { + imports = [test]; + defaults.nixpkgs.overlays = [(curr: prev: { + nix = (builtins.getFlake "nix/${nixVersion}").packages.${system}.nix; + })]; + }; }; - _module.args.nixpkgs = nixpkgs; - }; # Checks that a NixOS configuration does not contain any references to our # locally defined Nix version. @@ -142,4 +156,6 @@ in (system: runNixOSTestFor system ./setuid.nix); ca-fd-leak = runNixOSTestFor "x86_64-linux" ./ca-fd-leak; + + fetch-git = runNixOSTestFor "x86_64-linux" ./fetch-git; } diff --git a/tests/nixos/fetch-git/default.nix b/tests/nixos/fetch-git/default.nix new file mode 100644 index 000000000..1d6bcb637 --- /dev/null +++ b/tests/nixos/fetch-git/default.nix @@ -0,0 +1,32 @@ +{ lib, config, ... }: +{ + name = "fetch-git"; + + imports = [ + ./testsupport/gitea.nix + ]; + + /* + Test cases + + Test cases are automatically imported from ./test-cases/{name} + + The following is set up automatically for each test case: + - a repo with the {name} is created on the gitea server + - a repo with the {name} is created on the client + - the client repo is configured to push to the server repo + + Python variables: + - repo.path: the path to the directory of the client repo + - repo.git: the git command with the client repo as the working directory + - repo.remote: the url to the server repo + */ + testCases = + map + (testCaseName: {...}: { + imports = [ (./test-cases + "/${testCaseName}") ]; + # ensures tests are named like their directories they are defined in + name = testCaseName; + }) + (lib.attrNames (builtins.readDir ./test-cases)); +} diff --git a/tests/nixos/fetch-git/test-cases/http-simple/default.nix b/tests/nixos/fetch-git/test-cases/http-simple/default.nix new file mode 100644 index 000000000..dcab8067e --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/http-simple/default.nix @@ -0,0 +1,39 @@ +{ config, ... }: +{ + description = "can fetch a git repo via http"; + script = '' + # add a file to the repo + client.succeed(f""" + echo ${config.name /* to make the git tree and store path unique */} > {repo.path}/test-case \ + && echo chiang-mai > {repo.path}/thailand \ + && {repo.git} add test-case thailand \ + && {repo.git} commit -m 'commit1' + """) + + # save the revision + rev1 = client.succeed(f""" + {repo.git} rev-parse HEAD + """).strip() + + # push to the server + client.succeed(f""" + {repo.git} push origin main + """) + + # fetch the repo via nix + fetched1 = client.succeed(f""" + nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).outPath" + """) + + # check if the committed file is there + client.succeed(f""" + test -f {fetched1}/thailand + """) + + # check if the revision is the same + rev1_fetched = client.succeed(f""" + nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).rev" + """).strip() + assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}" + ''; +} diff --git a/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix b/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix new file mode 100644 index 000000000..f5fba1698 --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix @@ -0,0 +1,43 @@ +{ config, ... }: +{ + description = "can fetch a git repo via ssh"; + script = '' + # add a file to the repo + client.succeed(f""" + echo ${config.name /* to make the git tree and store path unique */} > {repo.path}/test-case \ + && echo chiang-mai > {repo.path}/thailand \ + && {repo.git} add test-case thailand \ + && {repo.git} commit -m 'commit1' + """) + + # save the revision + rev1 = client.succeed(f""" + {repo.git} rev-parse HEAD + """).strip() + + # push to the server + client.succeed(f""" + {repo.git} push origin-ssh main + """) + + # fetch the repo via nix + fetched1 = client.succeed(f""" + nix eval --impure --raw --expr ' + (builtins.fetchGit "{repo.remote_ssh}").outPath + ' + """) + + # check if the committed file is there + client.succeed(f""" + test -f {fetched1}/thailand + """) + + # check if the revision is the same + rev1_fetched = client.succeed(f""" + nix eval --impure --raw --expr ' + (builtins.fetchGit "{repo.remote_ssh}").rev + ' + """).strip() + assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}" + ''; +} diff --git a/tests/nixos/fetch-git/testsupport/gitea-repo.nix b/tests/nixos/fetch-git/testsupport/gitea-repo.nix new file mode 100644 index 000000000..916552bb2 --- /dev/null +++ b/tests/nixos/fetch-git/testsupport/gitea-repo.nix @@ -0,0 +1,51 @@ +{ lib, ... }: +let + inherit (lib) mkOption types; + + testCaseExtension = { config, ... }: { + setupScript = '' + repo = Repo("${config.name}") + ''; + }; +in +{ + options = { + testCases = mkOption { + type = types.listOf (types.submodule testCaseExtension); + }; + }; + config = { + setupScript = '' + class Repo: + """ + A class to create a git repository on the gitea server and locally. + """ + def __init__(self, name): + self.name = name + self.path = "/tmp/repos/" + name + self.remote = "http://gitea:3000/test/" + name + self.remote_ssh = "ssh://gitea/root/" + name + self.git = f"git -C {self.path}" + self.create() + + def create(self): + # create ssh remote repo + gitea.succeed(f""" + git init --bare -b main /root/{self.name} + """) + # create http remote repo + gitea.succeed(f""" + curl --fail -X POST http://{gitea_admin}:{gitea_admin_password}@gitea:3000/api/v1/user/repos \ + -H 'Accept: application/json' -H 'Content-Type: application/json' \ + -d {shlex.quote( f'{{"name":"{self.name}", "default_branch": "main"}}' )} + """) + # setup git remotes on client + client.succeed(f""" + mkdir -p {self.path} \ + && git init -b main {self.path} \ + && {self.git} remote add origin {self.remote} \ + && {self.git} remote add origin-ssh root@gitea:{self.name} + """) + ''; + }; +} \ No newline at end of file diff --git a/tests/nixos/fetch-git/testsupport/gitea.nix b/tests/nixos/fetch-git/testsupport/gitea.nix new file mode 100644 index 000000000..cf87bb466 --- /dev/null +++ b/tests/nixos/fetch-git/testsupport/gitea.nix @@ -0,0 +1,102 @@ +{ lib, nixpkgs, system, pkgs, ... }: let + clientPrivateKey = pkgs.writeText "id_ed25519" '' + -----BEGIN OPENSSH PRIVATE KEY----- + b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW + QyNTUxOQAAACBbeWvHh/AWGWI6EIc1xlSihyXtacNQ9KeztlW/VUy8wQAAAJAwVQ5VMFUO + VQAAAAtzc2gtZWQyNTUxOQAAACBbeWvHh/AWGWI6EIc1xlSihyXtacNQ9KeztlW/VUy8wQ + AAAEB7lbfkkdkJoE+4TKHPdPQWBKLSx+J54Eg8DaTr+3KoSlt5a8eH8BYZYjoQhzXGVKKH + Je1pw1D0p7O2Vb9VTLzBAAAACGJmb0BtaW5pAQIDBAU= + -----END OPENSSH PRIVATE KEY----- + ''; + + clientPublicKey = + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFt5a8eH8BYZYjoQhzXGVKKHJe1pw1D0p7O2Vb9VTLzB"; + +in { + imports = [ + ../testsupport/setup.nix + ../testsupport/gitea-repo.nix + ]; + nodes = { + gitea = { pkgs, ... }: { + services.gitea.enable = true; + services.gitea.settings.service.DISABLE_REGISTRATION = true; + services.gitea.settings.log.LEVEL = "Info"; + services.gitea.settings.database.LOG_SQL = false; + services.openssh.enable = true; + networking.firewall.allowedTCPPorts = [ 3000 ]; + environment.systemPackages = [ pkgs.git pkgs.gitea ]; + + users.users.root.openssh.authorizedKeys.keys = [clientPublicKey]; + + # TODO: remove this after updating to nixos-23.11 + nixpkgs.pkgs = lib.mkForce (import nixpkgs { + inherit system; + config.permittedInsecurePackages = [ + "gitea-1.19.4" + ]; + }); + }; + client = { pkgs, ... }: { + environment.systemPackages = [ pkgs.git ]; + }; + }; + defaults = { pkgs, ... }: { + environment.systemPackages = [ pkgs.jq ]; + }; + + setupScript = '' + import shlex + + gitea.wait_for_unit("gitea.service") + + gitea_admin = "test" + gitea_admin_password = "test123test" + + gitea.succeed(f""" + gitea --version >&2 + su -l gitea -c 'GITEA_WORK_DIR=/var/lib/gitea gitea admin user create \ + --username {gitea_admin} --password {gitea_admin_password} --email test@client' + """) + + client.wait_for_unit("multi-user.target") + gitea.wait_for_open_port(3000) + + gitea_admin_token = gitea.succeed(f""" + curl --fail -X POST http://{gitea_admin}:{gitea_admin_password}@gitea:3000/api/v1/users/test/tokens \ + -H 'Accept: application/json' -H 'Content-Type: application/json' \ + -d {shlex.quote( '{"name":"token", "scopes":["all"]}' )} \ + | jq -r '.sha1' + """).strip() + + client.succeed(f""" + echo "http://{gitea_admin}:{gitea_admin_password}@gitea:3000" >~/.git-credentials-admin + git config --global credential.helper 'store --file ~/.git-credentials-admin' + git config --global user.email "test@client" + git config --global user.name "Test User" + git config --global gc.autodetach 0 + git config --global gc.auto 0 + """) + + # add client's private key to ~/.ssh + client.succeed(""" + mkdir -p ~/.ssh + chmod 700 ~/.ssh + cat ${clientPrivateKey} >~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + """) + + client.succeed(""" + echo "Host gitea" >>~/.ssh/config + echo " StrictHostKeyChecking no" >>~/.ssh/config + echo " UserKnownHostsFile /dev/null" >>~/.ssh/config + echo " User root" >>~/.ssh/config + """) + + # ensure ssh from client to gitea works + client.succeed(""" + ssh root@gitea true + """) + + ''; +} diff --git a/tests/nixos/fetch-git/testsupport/setup.nix b/tests/nixos/fetch-git/testsupport/setup.nix new file mode 100644 index 000000000..a81d5614b --- /dev/null +++ b/tests/nixos/fetch-git/testsupport/setup.nix @@ -0,0 +1,91 @@ +{ lib, config, extendModules, ... }: +let + inherit (lib) + mkOption + types + ; + + indent = lib.replaceStrings ["\n"] ["\n "]; + + execTestCase = testCase: '' + + ### TEST ${testCase.name}: ${testCase.description} ### + + with subtest("${testCase.description}"): + # Setup + ${indent testCase.setupScript} + + # Test + ${indent testCase.script} + ''; +in +{ + + options = { + setupScript = mkOption { + type = types.lines; + description = '' + Python code that runs before the main test. + + Variables defined by this code will be available in the test. + ''; + default = ""; + }; + testCases = mkOption { + description = '' + The test cases. See `testScript`. + ''; + type = types.listOf (types.submodule { + options.name = mkOption { + type = types.str; + description = '' + The name of the test case. + + A repository with that name will be set up on the gitea server and locally. + ''; + }; + options.description = mkOption { + type = types.str; + description = '' + A description of the test case. + ''; + }; + options.setupScript = mkOption { + type = types.lines; + description = '' + Python code that runs before the test case. + ''; + default = ""; + }; + options.script = mkOption { + type = types.lines; + description = '' + Python code that runs the test. + + Variables defined by the global `setupScript`, as well as `testCases.*.setupScript` will be available here. + ''; + }; + }); + }; + }; + + config = { + nodes.client = { + environment.variables = { + _NIX_FORCE_HTTP = "1"; + }; + nix.settings.experimental-features = ["nix-command" "flakes"]; + }; + setupScript = '' + ''; + testScript = '' + start_all(); + + ${config.setupScript} + + ### SETUP COMPLETE ### + + ${lib.concatStringsSep "\n" (map execTestCase config.testCases)} + ''; + }; +}