buildbot/__init__.py: rework the file for gerrit usecase, our oauth2 instance

Signed-off-by: Raito Bezarius <raito@lix.systems>

buildbot_nix/__init__.py

@@ -1,7 +1,6 @@
 import json
 import multiprocessing
 import os
-import signal
 import sys
 import uuid
 from collections import defaultdict
@@ -19,6 +18,8 @@ from buildbot.process.results import ALL_RESULTS, statusToString
 from buildbot.steps.trigger import Trigger
 from buildbot.util import asyncSleep
 from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match
+from buildbot.www.oauth2 import OAuth2Auth
+from buildbot.changes.gerritchangesource import GerritChangeSource
 
 if TYPE_CHECKING:
     from buildbot.process.log import Log
@@ -28,10 +29,6 @@ from twisted.logger import Logger
 from twisted.python.failure import Failure
 
 from .github_projects import (
-    GithubProject,
-    create_project_hook,
-    load_projects,
-    refresh_projects,
     slugify_project_name,
 )
 
@@ -39,10 +36,19 @@ SKIPPED_BUILDER_NAME = "skipped-builds"
 
 log = Logger()
 
+class LixSystemsOAuth2(OAuth2Auth):
+    name = 'identity-lix-systems'
+    faIcon = '...'
+    resourceEndpoint = ''
+    authUri = ''
+    tokenUri = ''
 
 class BuildbotNixError(Exception):
     pass
 
+@dataclass
+class GerritProject:
+    name: str
 
 class BuildTrigger(Trigger):
     """Dynamic trigger that creates a build for every attribute."""
@@ -274,58 +280,6 @@ class UpdateBuildOutput(steps.BuildStep):
         return util.SUCCESS
 
 
-class ReloadGithubProjects(steps.BuildStep):
-    name = "reload_github_projects"
-
-    def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None:
-        self.token = token
-        self.project_cache_file = project_cache_file
-        super().__init__(**kwargs)
-
-    def reload_projects(self) -> None:
-        refresh_projects(self.token, self.project_cache_file)
-
-    @defer.inlineCallbacks
-    def run(self) -> Generator[Any, object, Any]:
-        d = threads.deferToThread(self.reload_projects)  # type: ignore[no-untyped-call]
-
-        self.error_msg = ""
-
-        def error_cb(failure: Failure) -> int:
-            self.error_msg += failure.getTraceback()
-            return util.FAILURE
-
-        d.addCallbacks(lambda _: util.SUCCESS, error_cb)
-        res = yield d
-        if res == util.SUCCESS:
-            # reload the buildbot config
-            os.kill(os.getpid(), signal.SIGHUP)
-            return util.SUCCESS
-        else:
-            log: Log = yield self.addLog("log")
-            log.addStderr(f"Failed to reload project list: {self.error_msg}")
-            return util.FAILURE
-
-
-def reload_github_projects(
-    worker_names: list[str],
-    github_token_secret: str,
-    project_cache_file: Path,
-) -> util.BuilderConfig:
-    """Updates the flake an opens a PR for it."""
-    factory = util.BuildFactory()
-    factory.addStep(
-        ReloadGithubProjects(
-            github_token_secret, project_cache_file=project_cache_file
-        ),
-    )
-    return util.BuilderConfig(
-        name="reload-github-projects",
-        workernames=worker_names,
-        factory=factory,
-    )
-
-
 # The builtin retry mechanism doesn't seem to work for github,
 # since github is sometimes not delivering the pull request ref fast enough.
 class GitWithRetry(steps.Git):
@@ -353,9 +307,8 @@ class GitWithRetry(steps.Git):
 
 
 def nix_eval_config(
-    project: GithubProject,
+    project: GerritProject,
     worker_names: list[str],
-    github_token_secret: str,
     supported_systems: list[str],
     eval_lock: util.MasterLock,
     worker_count: int,
@@ -366,15 +319,12 @@ def nix_eval_config(
     """
     factory = util.BuildFactory()
     # check out the source
-    url_with_secret = util.Interpolate(
-        f"https://git:%(secret:{github_token_secret})s@github.com/%(prop:project)s",
-    )
     factory.addStep(
-        GitWithRetry(
+        steps.Gerrit(
-            repourl=url_with_secret,
+            repourl=project.url,
-            method="clean",
+            mode="full",
-            submodules=True,
+            retry=[60, 60],
-            haltOnFailure=True,
+            timeout=3600
         ),
     )
     drv_gcroots_dir = util.Interpolate(
@@ -443,7 +393,7 @@ class CachixConfig:
 
 
 def nix_build_config(
-    project: GithubProject,
+    project: GerritProject,
     worker_names: list[str],
     cachix: CachixConfig | None = None,
     outputs_path: Path | None = None,
@@ -531,7 +481,7 @@ def nix_build_config(
 
 
 def nix_skipped_build_config(
-    project: GithubProject,
+    project: GerritProject,
     worker_names: list[str],
 ) -> util.BuilderConfig:
     """Dummy builder that is triggered when a build is skipped."""
@@ -570,27 +520,11 @@ def read_secret_file(secret_name: str) -> str:
     return Path(directory).joinpath(secret_name).read_text().rstrip()
 
 
-@dataclass
-class GithubConfig:
-    oauth_id: str
-    admins: list[str]
-
-    buildbot_user: str
-    oauth_secret_name: str = "github-oauth-secret"
-    webhook_secret_name: str = "github-webhook-secret"
-    token_secret_name: str = "github-token"
-    project_cache_file: Path = Path("github-project-cache.json")
-    topic: str | None = "build-with-buildbot"
-
-    def token(self) -> str:
-        return read_secret_file(self.token_secret_name)
-
 
 def config_for_project(
     config: dict[str, Any],
-    project: GithubProject,
+    project: GerritProject,
     worker_names: list[str],
-    github: GithubConfig,
     nix_supported_systems: list[str],
     nix_eval_worker_count: int,
     nix_eval_max_memory_size: int,
@@ -601,25 +535,6 @@ def config_for_project(
     config["projects"].append(Project(project.name))
     config["schedulers"].extend(
         [
-            schedulers.SingleBranchScheduler(
-                name=f"{project.project_id}-default-branch",
-                change_filter=util.ChangeFilter(
-                    repository=project.url,
-                    filter_fn=lambda c: c.branch
-                    == c.properties.getProperty("github.repository.default_branch"),
-                ),
-                builderNames=[f"{project.name}/nix-eval"],
-                treeStableTimer=5,
-            ),
-            # this is compatible with bors or github's merge queue
-            schedulers.SingleBranchScheduler(
-                name=f"{project.project_id}-merge-queue",
-                change_filter=util.ChangeFilter(
-                    repository=project.url,
-                    branch_re="(gh-readonly-queue/.*|staging|trying)",
-                ),
-                builderNames=[f"{project.name}/nix-eval"],
-            ),
             # build all pull requests
             schedulers.SingleBranchScheduler(
                 name=f"{project.project_id}-prs",
@@ -660,7 +575,6 @@ def config_for_project(
             nix_eval_config(
                 project,
                 worker_names,
-                github_token_secret=github.token_secret_name,
                 supported_systems=nix_supported_systems,
                 worker_count=nix_eval_worker_count,
                 max_memory_size=nix_eval_max_memory_size,
@@ -677,97 +591,6 @@ def config_for_project(
     )
 
 
-class AnyProjectEndpointMatcher(EndpointMatcherBase):
-    def __init__(self, builders: set[str] | None = None, **kwargs: Any) -> None:
-        if builders is None:
-            builders = set()
-        self.builders = builders
-        super().__init__(**kwargs)
-
-    @defer.inlineCallbacks
-    def check_builder(
-        self,
-        endpoint_object: Any,
-        endpoint_dict: dict[str, Any],
-        object_type: str,
-    ) -> Generator[defer.Deferred[Match], Any, Any]:
-        res = yield endpoint_object.get({}, endpoint_dict)
-        if res is None:
-            return None
-
-        builder = yield self.master.data.get(("builders", res["builderid"]))
-        if builder["name"] in self.builders:
-            log.warn(
-                "Builder {builder} allowed by {role}: {builders}",
-                builder=builder["name"],
-                role=self.role,
-                builders=self.builders,
-            )
-            return Match(self.master, **{object_type: res})
-        else:
-            log.warn(
-                "Builder {builder} not allowed by {role}: {builders}",
-                builder=builder["name"],
-                role=self.role,
-                builders=self.builders,
-            )
-
-    def match_BuildEndpoint_rebuild(  # noqa: N802
-        self,
-        epobject: Any,
-        epdict: dict[str, Any],
-        options: dict[str, Any],
-    ) -> defer.Deferred[Match]:
-        return self.check_builder(epobject, epdict, "build")
-
-    def match_BuildEndpoint_stop(  # noqa: N802
-        self,
-        epobject: Any,
-        epdict: dict[str, Any],
-        options: dict[str, Any],
-    ) -> defer.Deferred[Match]:
-        return self.check_builder(epobject, epdict, "build")
-
-    def match_BuildRequestEndpoint_stop(  # noqa: N802
-        self,
-        epobject: Any,
-        epdict: dict[str, Any],
-        options: dict[str, Any],
-    ) -> defer.Deferred[Match]:
-        return self.check_builder(epobject, epdict, "buildrequest")
-
-
-def setup_authz(projects: list[GithubProject], admins: list[str]) -> util.Authz:
-    allow_rules = []
-    allowed_builders_by_org: defaultdict[str, set[str]] = defaultdict(
-        lambda: {"reload-github-projects"},
-    )
-
-    for project in projects:
-        if project.belongs_to_org:
-            for builder in ["nix-build", "nix-skipped-build", "nix-eval"]:
-                allowed_builders_by_org[project.owner].add(f"{project.name}/{builder}")
-
-    for org, allowed_builders in allowed_builders_by_org.items():
-        allow_rules.append(
-            AnyProjectEndpointMatcher(
-                builders=allowed_builders,
-                role=org,
-                defaultDeny=False,
-            ),
-        )
-
-    allow_rules.append(util.AnyEndpointMatcher(role="admin", defaultDeny=False))
-    allow_rules.append(util.AnyControlEndpointMatcher(role="admins"))
-    return util.Authz(
-        roleMatchers=[
-            util.RolesFromUsername(roles=["admin"], usernames=admins),
-            util.RolesFromGroups(groupPrefix=""),  # so we can match on ORG
-        ],
-        allowRules=allow_rules,
-    )
-
-
 class PeriodicWithStartup(schedulers.Periodic):
     def __init__(self, *args: Any, run_on_startup: bool = False, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
@@ -779,14 +602,72 @@ class PeriodicWithStartup(schedulers.Periodic):
             yield self.setState("last_build", None)
         yield super().activate()
 
+def gerritReviewCB(builderName, build, result, master, arg):
+    if result == util.RETRY:
+        return dict()
 
-class NixConfigurator(ConfiguratorBase):
+    message =  "Buildbot finished compiling your patchset\n"
+    message += "on configuration: %s\n" % builderName
+    message += "The result is: %s\n" % util.Results[result].upper()
+
+    if arg:
+        message += "\nFor more details visit:\n"
+        message += build['url'] + "\n"
+
+    if result == util.SUCCESS:
+        verified = 1
+    else:
+        verified = -1
+
+    return dict(message=message, labels={'Verified': verified})
+
+def gerritStartCB(builderName, build, arg):
+    message = "Buildbot started compiling your patchset\n"
+    message += "on configuration: %s\n" % builderName
+    message += "See your build here: %s" % build['url']
+
+    return dict(message=message)
+
+def gerritSummaryCB(buildInfoList, results, status, arg):
+    success = False
+    failure = False
+
+    msgs = []
+
+    for buildInfo in buildInfoList:
+        msg = "Builder %(name)s %(resultText)s (%(text)s)" % buildInfo
+        link = buildInfo.get('url', None)
+        if link:
+            msg += " - " + link
+        else:
+            msg += "."
+
+        msgs.append(msg)
+
+        if buildInfo['result'] == util.SUCCESS:
+            success = True
+        else:
+            failure = True
+
+    if success and not failure:
+        verified = 1
+    else:
+        verified = -1
+
+    return dict(message='\n\n'.join(msgs),
+                labels={
+                    'Verified': verified
+                })
+
+class GerritNixConfigurator(ConfiguratorBase):
     """Janitor is a configurator which create a Janitor Builder with all needed Janitor steps"""
 
     def __init__(
         self,
         # Shape of this file: [ { "name": "<worker-name>", "pass": "<worker-password>", "cores": "<cpu-cores>" } ]
-        github: GithubConfig,
+        gerrit_server: str,
+        gerrit_user: str,
+        gerrit_port: int,
         url: str,
         nix_supported_systems: list[str],
         nix_eval_worker_count: int | None,
@@ -796,11 +677,14 @@ class NixConfigurator(ConfiguratorBase):
         outputs_path: str | None = None,
     ) -> None:
         super().__init__()
+        self.gerrit_server = gerrit_server
+        self.gerrit_user = gerrit_user
+        self.gerrit_port = gerrit_port
         self.nix_workers_secret_name = nix_workers_secret_name
         self.nix_eval_max_memory_size = nix_eval_max_memory_size
         self.nix_eval_worker_count = nix_eval_worker_count
         self.nix_supported_systems = nix_supported_systems
-        self.github = github
+        self.gerrit_change_source = GerritChangeSource(gerrit_server, gerrit_user, gerritport=gerrit_port)
         self.url = url
         self.cachix = cachix
         if outputs_path is None:
@@ -809,9 +693,6 @@ class NixConfigurator(ConfiguratorBase):
             self.outputs_path = Path(outputs_path)
 
     def configure(self, config: dict[str, Any]) -> None:
-        projects = load_projects(self.github.token(), self.github.project_cache_file)
-        if self.github.topic is not None:
-            projects = [p for p in projects if self.github.topic in p.topics]
         worker_config = json.loads(read_secret_file(self.nix_workers_secret_name))
         worker_names = []
 
@@ -826,63 +707,30 @@ class NixConfigurator(ConfiguratorBase):
                 config["workers"].append(worker.Worker(worker_name, item["pass"]))
                 worker_names.append(worker_name)
 
-        webhook_secret = read_secret_file(self.github.webhook_secret_name)
         eval_lock = util.MasterLock("nix-eval")
 
-        for project in projects:
+       # TODO: initialize Lix
-            create_project_hook(
+       # config_for_project(
-                project.owner,
+       #     config,
-                project.repo,
+       #     project,
-                self.github.token(),
+       #     worker_names,
-                self.url + "change_hook/github",
+       #     self.nix_supported_systems,
-                webhook_secret,
+       #     self.nix_eval_worker_count or multiprocessing.cpu_count(),
-            )
+       #     self.nix_eval_max_memory_size,
-            config_for_project(
+       #     eval_lock,
-                config,
+       #     self.cachix,
-                project,
+       #     self.outputs_path,
-                worker_names,
+       # )
-                self.github,
-                self.nix_supported_systems,
-                self.nix_eval_worker_count or multiprocessing.cpu_count(),
-                self.nix_eval_max_memory_size,
-                eval_lock,
-                self.cachix,
-                self.outputs_path,
-            )
 
-        # Reload github projects
-        config["builders"].append(
-            reload_github_projects(
-                [worker_names[0]],
-                self.github.token(),
-                self.github.project_cache_file,
-            ),
-        )
         config["workers"].append(worker.LocalWorker(SKIPPED_BUILDER_NAME))
-        config["schedulers"].extend(
-            [
-                schedulers.ForceScheduler(
-                    name="reload-github-projects",
-                    builderNames=["reload-github-projects"],
-                    buttonName="Update projects",
-                ),
-                # project list twice a day and on startup
-                PeriodicWithStartup(
-                    name="reload-github-projects-bidaily",
-                    builderNames=["reload-github-projects"],
-                    periodicBuildTimer=12 * 60 * 60,
-                    run_on_startup=not self.github.project_cache_file.exists(),
-                ),
-            ],
-        )
         config["services"].append(
-            reporters.GitHubStatusPush(
+            reporters.GerritStatusPush(self.gerrit_server, self.gerrit_user,
-                token=self.github.token(),
+                                    reviewCB=gerritReviewCB,
-                # Since we dynamically create build steps,
+                                    reviewArg=self.url,
-                # we use `virtual_builder_name` in the webinterface
+                                    startCB=gerritStartCB,
-                # so that we distinguish what has beeing build
+                                    startArg=self.url,
-                context=Interpolate("buildbot/%(prop:status_name)s"),
+                                    summaryCB=gerritSummaryCB,
-            ),
+                                    summaryArg=self.url)
         )
 
         systemd_secrets = secrets.SecretInAFile(
@@ -893,26 +741,5 @@ class NixConfigurator(ConfiguratorBase):
         config["www"].setdefault("plugins", {})
         config["www"]["plugins"].update(dict(base_react={}))
 
-        config["www"].setdefault("change_hook_dialects", {})
-        config["www"]["change_hook_dialects"]["github"] = {
-            "secret": webhook_secret,
-            "strict": True,
-            "token": self.github.token(),
-            "github_property_whitelist": "*",
-        }
-
         if "auth" not in config["www"]:
-            config["www"].setdefault("avatar_methods", [])
+            config["www"]["auth"] = LixSystemsOAuth2()
-            config["www"]["avatar_methods"].append(
-                util.AvatarGitHub(token=self.github.token()),
-            )
-            config["www"]["auth"] = util.GitHubAuth(
-                self.github.oauth_id,
-                read_secret_file(self.github.oauth_secret_name),
-                apiVersion=4,
-            )
-
-            config["www"]["authz"] = setup_authz(
-                admins=self.github.admins,
-                projects=projects,
-            )