From 4ae3ef7ca77c1ed6235e40501c876dd3957d04e6 Mon Sep 17 00:00:00 2001 From: phaer Date: Thu, 26 Oct 2023 09:25:50 +0200 Subject: [PATCH 01/10] make buildbot_nix a python module --- buildbot_nix/{buildbot_nix.py => __init__.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename buildbot_nix/{buildbot_nix.py => __init__.py} (99%) diff --git a/buildbot_nix/buildbot_nix.py b/buildbot_nix/__init__.py similarity index 99% rename from buildbot_nix/buildbot_nix.py rename to buildbot_nix/__init__.py index 1eb02bf..10b13a4 100644 --- a/buildbot_nix/buildbot_nix.py +++ b/buildbot_nix/__init__.py @@ -20,7 +20,7 @@ from buildbot.process.project import Project from buildbot.process.properties import Interpolate, Properties from buildbot.process.results import ALL_RESULTS, statusToString from buildbot.steps.trigger import Trigger -from github_projects import ( # noqa: E402 +from .github_projects import ( # noqa: E402 GithubProject, create_project_hook, load_projects, From f14619f0df9abfad165c1b938de72934e29012db Mon Sep 17 00:00:00 2001 From: phaer Date: Thu, 26 Oct 2023 10:17:30 +0200 Subject: [PATCH 02/10] strip read secrets --- buildbot_nix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 10b13a4..697ae4c 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -546,7 +546,7 @@ def read_secret_file(secret_name: str) -> str: if directory is None: print("directory not set", file=sys.stderr) sys.exit(1) - return Path(directory).joinpath(secret_name).read_text() + return Path(directory).joinpath(secret_name).read_text().strip() @dataclass From a758cb41ca6f6a7525d1b83b05831b70b2c8ace8 Mon Sep 17 00:00:00 2001 From: phaer Date: Thu, 26 Oct 2023 10:17:49 +0200 Subject: [PATCH 03/10] fix: also set repos if not cached --- buildbot_nix/github_projects.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 9f7756d..fb8f4b0 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -148,9 +148,8 @@ def refresh_projects(github_token: str, repo_cache_file: Path) -> None: def load_projects(github_token: str, repo_cache_file: Path) -> list[GithubProject]: - if repo_cache_file.exists(): - log.msg("fetching github repositories from cache") - repos: list[dict[str, Any]] = json.loads(repo_cache_file.read_text()) - else: + if not repo_cache_file.exists(): + log.msg("fetching github repositories") refresh_projects(github_token, repo_cache_file) + repos: list[dict[str, Any]] = json.loads(repo_cache_file.read_text()) return [GithubProject(repo) for repo in repos] From 44486f66336198c78f26810e5aeb6925ed9d873f Mon Sep 17 00:00:00 2001 From: phaer Date: Thu, 26 Oct 2023 11:08:07 +0200 Subject: [PATCH 04/10] skip github projects without sufficient perms --- buildbot_nix/github_projects.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index fb8f4b0..6dfa335 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -133,10 +133,18 @@ def create_project_hook( def refresh_projects(github_token: str, repo_cache_file: Path) -> None: - repos = paginated_github_request( - "https://api.github.com/user/repos?per_page=100", - github_token, - ) + repos = [] + + for repo in paginated_github_request( + "https://api.github.com/user/repos?per_page=100", + github_token, + ): + if not repo["permissions"]["admin"]: + name = repo['full_name'] + log.msg(f"skipping {name} because we do not have admin privileges, needed for hook management") + else: + repos.append(repo) + with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f: try: f.write(json.dumps(repos)) From 10db40a5a801cfd29c339c8e2a1e642070a4be72 Mon Sep 17 00:00:00 2001 From: phaer Date: Thu, 26 Oct 2023 12:34:56 +0200 Subject: [PATCH 05/10] worker: actually use cfg.masterUrl --- nix/worker.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/worker.nix b/nix/worker.nix index 73079a5..6b614bc 100644 --- a/nix/worker.nix +++ b/nix/worker.nix @@ -56,7 +56,7 @@ in pkgs.nix-eval-jobs ]; environment.PYTHONPATH = "${python.withPackages (_: [cfg.package])}/${python.sitePackages}"; - environment.MASTER_URL = ''tcp:host=localhost:port=9989''; + environment.MASTER_URL = cfg.masterUrl; environment.BUILDBOT_DIR = buildbotDir; serviceConfig = { From eac9acc0413a3d4e01783541e5b15135e4e666d3 Mon Sep 17 00:00:00 2001 From: phaer Date: Thu, 26 Oct 2023 13:01:08 +0200 Subject: [PATCH 06/10] only activate github auth if no other is active --- buildbot_nix/__init__.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 697ae4c..c1f6a27 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -761,20 +761,6 @@ class NixConfigurator(ConfiguratorBase): config["secretsProviders"] = config.get("secretsProviders", []) config["secretsProviders"].append(systemd_secrets) config["www"] = config.get("www", {}) - config["www"]["avatar_methods"] = config["www"].get("avatar_methods", []) - config["www"]["avatar_methods"].append(util.AvatarGitHub()) - config["www"]["auth"] = util.GitHubAuth( - self.github.oauth_id, read_secret_file(self.github.oauth_secret_name) - ) - config["www"]["authz"] = util.Authz( - roleMatchers=[ - util.RolesFromUsername(roles=["admin"], usernames=self.github.admins) - ], - allowRules=[ - util.AnyEndpointMatcher(role="admin", defaultDeny=False), - util.AnyControlEndpointMatcher(role="admins"), - ], - ) config["www"]["change_hook_dialects"] = config["www"].get( "change_hook_dialects", {} ) @@ -784,3 +770,19 @@ class NixConfigurator(ConfiguratorBase): "token": self.github.token(), "github_property_whitelist": "*", } + + if not config["www"].get("auth"): + config["www"]["avatar_methods"] = config["www"].get("avatar_methods", []) + config["www"]["avatar_methods"].append(util.AvatarGitHub()) + config["www"]["auth"] = util.GitHubAuth( + self.github.oauth_id, read_secret_file(self.github.oauth_secret_name) + ) + config["www"]["authz"] = util.Authz( + roleMatchers=[ + util.RolesFromUsername(roles=["admin"], usernames=self.github.admins) + ], + allowRules=[ + util.AnyEndpointMatcher(role="admin", defaultDeny=False), + util.AnyControlEndpointMatcher(role="admins"), + ], + ) From 2edf6860b0ee86e6b844a5e18b3849aea702ffa7 Mon Sep 17 00:00:00 2001 From: phaer Date: Thu, 26 Oct 2023 14:54:01 +0200 Subject: [PATCH 07/10] allow github projects with "." in their name --- buildbot_nix/__init__.py | 5 +++-- buildbot_nix/github_projects.py | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index c1f6a27..54fc72d 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -25,6 +25,7 @@ from .github_projects import ( # noqa: E402 create_project_hook, load_projects, refresh_projects, + slugify_project_name ) from twisted.internet import defer, threads from twisted.python.failure import Failure @@ -63,7 +64,7 @@ class BuildTrigger(Trigger): "github.base.repo.full_name", build_props.getProperty("github.repository.full_name"), ) - project_id = repo_name.replace("/", "-") + project_id = slugify_project_name(repo_name) source = f"nix-eval-{project_id}" sch = self.schedulerNames[0] @@ -151,7 +152,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): "github.base.repo.full_name", build_props.getProperty("github.repository.full_name"), ) - project_id = repo_name.replace("/", "-") + project_id = slugify_project_name(repo_name) scheduler = f"{project_id}-nix-build" filtered_jobs = [] for job in jobs: diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 6dfa335..ddfc0bd 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -69,6 +69,10 @@ def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]: return items +def slugify_project_name(name: str) -> str: + return name.replace(".", "-").replace("/", "-") + + class GithubProject: def __init__(self, data: dict[str, Any]) -> None: self.data = data @@ -91,8 +95,7 @@ class GithubProject: @property def id(self) -> str: - n = self.data["full_name"] - return n.replace("/", "-") + return slugify_project_name(self.data["full_name"]) @property def default_branch(self) -> str: From c7b360eb16f91f7f40576b9932cd61f4339a6d45 Mon Sep 17 00:00:00 2001 From: phaer Date: Fri, 27 Oct 2023 10:35:26 +0200 Subject: [PATCH 08/10] pass project name for forced builds --- buildbot_nix/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index 54fc72d..da7128c 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -311,7 +311,7 @@ def nix_update_flake_config( """ factory = util.BuildFactory() url_with_secret = util.Interpolate( - f"https://git:%(secret:{github_token_secret})s@github.com/{project.name}" + f"https://git:%(secret:{github_token_secret})s@github.com/%(prop:project)s" ) factory.addStep( steps.Git( @@ -615,7 +615,13 @@ def config_for_project( ), # allow to manually trigger a nix-build schedulers.ForceScheduler( - name=f"{project.id}-force", builderNames=[f"{project.name}/nix-eval"] + name=f"{project.id}-force", builderNames=[f"{project.name}/nix-eval"], + properties=[ + util.StringParameter( + name="project", + label="Name of the GitHub repository.", + default=project.name) + ] ), # allow to manually update flakes schedulers.ForceScheduler( From 9ad17c646b6c0357c0fe65227054c28a4d9f2e89 Mon Sep 17 00:00:00 2001 From: phaer Date: Fri, 27 Oct 2023 10:49:40 +0200 Subject: [PATCH 09/10] fmt --- buildbot_nix/__init__.py | 19 ++++++++++++------- buildbot_nix/github_projects.py | 10 ++++++---- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index da7128c..f221f08 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -20,15 +20,16 @@ from buildbot.process.project import Project from buildbot.process.properties import Interpolate, Properties from buildbot.process.results import ALL_RESULTS, statusToString from buildbot.steps.trigger import Trigger +from twisted.internet import defer, threads +from twisted.python.failure import Failure + from .github_projects import ( # noqa: E402 GithubProject, create_project_hook, load_projects, refresh_projects, - slugify_project_name + slugify_project_name, ) -from twisted.internet import defer, threads -from twisted.python.failure import Failure class BuildTrigger(Trigger): @@ -615,13 +616,15 @@ def config_for_project( ), # allow to manually trigger a nix-build schedulers.ForceScheduler( - name=f"{project.id}-force", builderNames=[f"{project.name}/nix-eval"], + name=f"{project.id}-force", + builderNames=[f"{project.name}/nix-eval"], properties=[ util.StringParameter( name="project", label="Name of the GitHub repository.", - default=project.name) - ] + default=project.name, + ) + ], ), # allow to manually update flakes schedulers.ForceScheduler( @@ -786,7 +789,9 @@ class NixConfigurator(ConfiguratorBase): ) config["www"]["authz"] = util.Authz( roleMatchers=[ - util.RolesFromUsername(roles=["admin"], usernames=self.github.admins) + util.RolesFromUsername( + roles=["admin"], usernames=self.github.admins + ) ], allowRules=[ util.AnyEndpointMatcher(role="admin", defaultDeny=False), diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index ddfc0bd..4be3861 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -139,12 +139,14 @@ def refresh_projects(github_token: str, repo_cache_file: Path) -> None: repos = [] for repo in paginated_github_request( - "https://api.github.com/user/repos?per_page=100", - github_token, + "https://api.github.com/user/repos?per_page=100", + github_token, ): if not repo["permissions"]["admin"]: - name = repo['full_name'] - log.msg(f"skipping {name} because we do not have admin privileges, needed for hook management") + name = repo["full_name"] + log.msg( + f"skipping {name} because we do not have admin privileges, needed for hook management" + ) else: repos.append(repo) From 9f2ec133e11463d0f71d89856f34a56829091d5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 27 Oct 2023 11:09:20 +0200 Subject: [PATCH 10/10] use rstrip to remove trailing newlines --- buildbot_nix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildbot_nix/__init__.py b/buildbot_nix/__init__.py index f221f08..a0f1d8b 100644 --- a/buildbot_nix/__init__.py +++ b/buildbot_nix/__init__.py @@ -548,7 +548,7 @@ def read_secret_file(secret_name: str) -> str: if directory is None: print("directory not set", file=sys.stderr) sys.exit(1) - return Path(directory).joinpath(secret_name).read_text().strip() + return Path(directory).joinpath(secret_name).read_text().rstrip() @dataclass