diff --git a/buildbot_nix/buildbot_nix.py b/buildbot_nix/__init__.py similarity index 94% rename from buildbot_nix/buildbot_nix.py rename to buildbot_nix/__init__.py index 1eb02bf..a0f1d8b 100644 --- a/buildbot_nix/buildbot_nix.py +++ b/buildbot_nix/__init__.py @@ -20,14 +20,16 @@ from buildbot.process.project import Project from buildbot.process.properties import Interpolate, Properties from buildbot.process.results import ALL_RESULTS, statusToString from buildbot.steps.trigger import Trigger -from github_projects import ( # noqa: E402 +from twisted.internet import defer, threads +from twisted.python.failure import Failure + +from .github_projects import ( # noqa: E402 GithubProject, create_project_hook, load_projects, refresh_projects, + slugify_project_name, ) -from twisted.internet import defer, threads -from twisted.python.failure import Failure class BuildTrigger(Trigger): @@ -63,7 +65,7 @@ class BuildTrigger(Trigger): "github.base.repo.full_name", build_props.getProperty("github.repository.full_name"), ) - project_id = repo_name.replace("/", "-") + project_id = slugify_project_name(repo_name) source = f"nix-eval-{project_id}" sch = self.schedulerNames[0] @@ -151,7 +153,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep): "github.base.repo.full_name", build_props.getProperty("github.repository.full_name"), ) - project_id = repo_name.replace("/", "-") + project_id = slugify_project_name(repo_name) scheduler = f"{project_id}-nix-build" filtered_jobs = [] for job in jobs: @@ -310,7 +312,7 @@ def nix_update_flake_config( """ factory = util.BuildFactory() url_with_secret = util.Interpolate( - f"https://git:%(secret:{github_token_secret})s@github.com/{project.name}" + f"https://git:%(secret:{github_token_secret})s@github.com/%(prop:project)s" ) factory.addStep( steps.Git( @@ -546,7 +548,7 @@ def read_secret_file(secret_name: str) -> str: if directory is None: print("directory not set", file=sys.stderr) sys.exit(1) - return Path(directory).joinpath(secret_name).read_text() + return Path(directory).joinpath(secret_name).read_text().rstrip() @dataclass @@ -614,7 +616,15 @@ def config_for_project( ), # allow to manually trigger a nix-build schedulers.ForceScheduler( - name=f"{project.id}-force", builderNames=[f"{project.name}/nix-eval"] + name=f"{project.id}-force", + builderNames=[f"{project.name}/nix-eval"], + properties=[ + util.StringParameter( + name="project", + label="Name of the GitHub repository.", + default=project.name, + ) + ], ), # allow to manually update flakes schedulers.ForceScheduler( @@ -761,20 +771,6 @@ class NixConfigurator(ConfiguratorBase): config["secretsProviders"] = config.get("secretsProviders", []) config["secretsProviders"].append(systemd_secrets) config["www"] = config.get("www", {}) - config["www"]["avatar_methods"] = config["www"].get("avatar_methods", []) - config["www"]["avatar_methods"].append(util.AvatarGitHub()) - config["www"]["auth"] = util.GitHubAuth( - self.github.oauth_id, read_secret_file(self.github.oauth_secret_name) - ) - config["www"]["authz"] = util.Authz( - roleMatchers=[ - util.RolesFromUsername(roles=["admin"], usernames=self.github.admins) - ], - allowRules=[ - util.AnyEndpointMatcher(role="admin", defaultDeny=False), - util.AnyControlEndpointMatcher(role="admins"), - ], - ) config["www"]["change_hook_dialects"] = config["www"].get( "change_hook_dialects", {} ) @@ -784,3 +780,21 @@ class NixConfigurator(ConfiguratorBase): "token": self.github.token(), "github_property_whitelist": "*", } + + if not config["www"].get("auth"): + config["www"]["avatar_methods"] = config["www"].get("avatar_methods", []) + config["www"]["avatar_methods"].append(util.AvatarGitHub()) + config["www"]["auth"] = util.GitHubAuth( + self.github.oauth_id, read_secret_file(self.github.oauth_secret_name) + ) + config["www"]["authz"] = util.Authz( + roleMatchers=[ + util.RolesFromUsername( + roles=["admin"], usernames=self.github.admins + ) + ], + allowRules=[ + util.AnyEndpointMatcher(role="admin", defaultDeny=False), + util.AnyControlEndpointMatcher(role="admins"), + ], + ) diff --git a/buildbot_nix/github_projects.py b/buildbot_nix/github_projects.py index 9f7756d..4be3861 100644 --- a/buildbot_nix/github_projects.py +++ b/buildbot_nix/github_projects.py @@ -69,6 +69,10 @@ def paginated_github_request(url: str, token: str) -> list[dict[str, Any]]: return items +def slugify_project_name(name: str) -> str: + return name.replace(".", "-").replace("/", "-") + + class GithubProject: def __init__(self, data: dict[str, Any]) -> None: self.data = data @@ -91,8 +95,7 @@ class GithubProject: @property def id(self) -> str: - n = self.data["full_name"] - return n.replace("/", "-") + return slugify_project_name(self.data["full_name"]) @property def default_branch(self) -> str: @@ -133,10 +136,20 @@ def create_project_hook( def refresh_projects(github_token: str, repo_cache_file: Path) -> None: - repos = paginated_github_request( + repos = [] + + for repo in paginated_github_request( "https://api.github.com/user/repos?per_page=100", github_token, - ) + ): + if not repo["permissions"]["admin"]: + name = repo["full_name"] + log.msg( + f"skipping {name} because we do not have admin privileges, needed for hook management" + ) + else: + repos.append(repo) + with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f: try: f.write(json.dumps(repos)) @@ -148,9 +161,8 @@ def refresh_projects(github_token: str, repo_cache_file: Path) -> None: def load_projects(github_token: str, repo_cache_file: Path) -> list[GithubProject]: - if repo_cache_file.exists(): - log.msg("fetching github repositories from cache") - repos: list[dict[str, Any]] = json.loads(repo_cache_file.read_text()) - else: + if not repo_cache_file.exists(): + log.msg("fetching github repositories") refresh_projects(github_token, repo_cache_file) + repos: list[dict[str, Any]] = json.loads(repo_cache_file.read_text()) return [GithubProject(repo) for repo in repos] diff --git a/nix/worker.nix b/nix/worker.nix index 73079a5..6b614bc 100644 --- a/nix/worker.nix +++ b/nix/worker.nix @@ -56,7 +56,7 @@ in pkgs.nix-eval-jobs ]; environment.PYTHONPATH = "${python.withPackages (_: [cfg.package])}/${python.sitePackages}"; - environment.MASTER_URL = ''tcp:host=localhost:port=9989''; + environment.MASTER_URL = cfg.masterUrl; environment.BUILDBOT_DIR = buildbotDir; serviceConfig = {