Merge pull request #10 from Mic92/fixes

add option to reload github projects

.mergify.yml

@@ -0,0 +1,16 @@
+queue_rules:
+  - name: default
+    merge_conditions:
+      - check-success=buildbot/nix-eval
+defaults:
+  actions:
+    queue:
+      allow_merging_configuration_change: true
+      method: rebase
+pull_request_rules:
+  - name: merge using the merge queue
+    conditions:
+      - base=main
+      - label~=merge-queue|dependencies
+    actions:
+      queue: {}

buildbot_nix/buildbot_nix.py

@@ -3,6 +3,7 @@
 import json
 import multiprocessing
 import os
+import signal
 import sys
 import uuid
 from collections import defaultdict
@@ -23,8 +24,10 @@ from github_projects import (  # noqa: E402
     GithubProject,
     create_project_hook,
     load_projects,
+    refresh_projects,
 )
-from twisted.internet import defer
+from twisted.internet import defer, threads
+from twisted.python.failure import Failure
 
 
 class BuildTrigger(Trigger):
@@ -51,10 +54,10 @@ class BuildTrigger(Trigger):
             **kwargs,
         )
 
-    def createTriggerProperties(self, props: Any) -> Any:
+    def createTriggerProperties(self, props: Any) -> Any:  # noqa: N802
         return props
 
-    def getSchedulersAndProperties(self) -> list[tuple[str, Properties]]:
+    def getSchedulersAndProperties(self) -> list[tuple[str, Properties]]:  # noqa: N802
         build_props = self.build.getProperties()
         repo_name = build_props.getProperty(
             "github.base.repo.full_name",
@@ -94,7 +97,7 @@ class BuildTrigger(Trigger):
             triggered_schedulers.append((sch, props))
         return triggered_schedulers
 
-    def getCurrentSummary(self) -> dict[str, str]:
+    def getCurrentSummary(self) -> dict[str, str]:  # noqa: N802
         """
         The original build trigger will the generic builder name `nix-build` in this case, which is not helpful
         """
@@ -250,6 +253,58 @@ class UpdateBuildOutput(steps.BuildStep):
         return util.SUCCESS
 
 
+class ReloadGithubProjects(steps.BuildStep):
+    name = "reload_github_projects"
+
+    def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None:
+        self.token = token
+        self.project_cache_file = project_cache_file
+        super().__init__(**kwargs)
+
+    def reload_projects(self) -> None:
+        refresh_projects(self.token, self.project_cache_file)
+
+    @defer.inlineCallbacks
+    def run(self) -> Generator[Any, object, Any]:
+        d = threads.deferToThread(self.reload_projects)
+
+        self.error_msg = ""
+
+        def error_cb(failure: Failure) -> int:
+            self.error_msg += failure.getTraceback()
+            return util.FAILURE
+
+        d.addCallbacks(lambda _: util.SUCCESS, error_cb)
+        res = yield d
+        if res == util.SUCCESS:
+            # reload the buildbot config
+            os.kill(os.getpid(), signal.SIGHUP)
+            return util.SUCCESS
+        else:
+            log: Log = yield self.addLog("log")
+            log.addStderr(f"Failed to reload project list: {self.error_msg}")
+            return util.FAILURE
+
+
+def reload_github_projects(
+    worker_names: list[str],
+    github_token_secret: str,
+    project_cache_file: Path,
+) -> util.BuilderConfig:
+    """
+    Updates the flake an opens a PR for it.
+    """
+    factory = util.BuildFactory()
+    factory.addStep(
+        ReloadGithubProjects(github_token_secret, project_cache_file=project_cache_file)
+    )
+    return util.BuilderConfig(
+        name="reload-github-projects",
+        workernames=worker_names,
+        factory=factory,
+    )
+
+
 def nix_update_flake_config(
     project: GithubProject,
     worker_names: list[str],
@@ -654,12 +709,15 @@ class NixConfigurator(ConfiguratorBase):
 
         config["projects"] = config.get("projects", [])
 
+        webhook_secret = read_secret_file(self.github.webhook_secret_name)
+
         for project in projects:
             create_project_hook(
                 project.owner,
                 project.repo,
                 self.github.token(),
                 f"{self.url}/change_hook/github",
+                webhook_secret,
             )
 
         for project in projects:
@@ -673,6 +731,21 @@ class NixConfigurator(ConfiguratorBase):
                 self.nix_eval_max_memory_size,
             )
 
+        # Reload github projects
+        config["builders"].append(
+            reload_github_projects(
+                [worker_names[0]],
+                self.github.token(),
+                self.github.project_cache_file,
+            )
+        )
+        config["schedulers"].append(
+            schedulers.ForceScheduler(
+                name="reload-github-projects",
+                builderNames=["reload-github-projects"],
+                buttonName="Update projects",
+            )
+        )
         config["services"] = config.get("services", [])
         config["services"].append(
             reporters.GitHubStatusPush(
@@ -707,7 +780,7 @@ class NixConfigurator(ConfiguratorBase):
             "change_hook_dialects", {}
         )
         config["www"]["change_hook_dialects"]["github"] = {
-            "secret": read_secret_file(self.github.webhook_secret_name),
+            "secret": webhook_secret,
             "strict": True,
             "token": self.github.token(),
             "github_property_whitelist": "*",

buildbot_nix/github_projects.py

@@ -1,7 +1,9 @@
 import http.client
 import json
+import os
 import urllib.request
 from pathlib import Path
+from tempfile import NamedTemporaryFile
 from typing import Any
 
 from twisted.python import log
@@ -33,13 +35,13 @@ def http_request(
     try:
         resp = urllib.request.urlopen(req)
     except urllib.request.HTTPError as e:
-        body = ""
+        resp_body = ""
         try:
-            body = e.fp.read()
+            resp_body = e.fp.read().decode("utf-8", "replace")
         except Exception:
             pass
         raise Exception(
-            f"Request for {method} {url} failed with {e.code} {e.reason}: {body}"
+            f"Request for {method} {url} failed with {e.code} {e.reason}: {resp_body}"
         ) from e
     return HttpResponse(resp)
 
@@ -101,11 +103,15 @@ class GithubProject:
         return self.data["topics"]
 
 
-def create_project_hook(owner: str, repo: str, token: str, webhook_url: str) -> None:
+def create_project_hook(
+    owner: str, repo: str, token: str, webhook_url: str, webhook_secret: str
+) -> None:
     hooks = paginated_github_request(
         f"https://api.github.com/repos/{owner}/{repo}/hooks?per_page=100", token
     )
-    config = dict(url=webhook_url, content_type="json", insecure_ssl="0")
+    config = dict(
+        url=webhook_url, content_type="json", insecure_ssl="0", secret=webhook_secret
+    )
     data = dict(name="web", active=True, events=["push", "pull_request"], config=config)
     headers = {
         "Authorization": f"Bearer {token}",
@@ -126,16 +132,25 @@ def create_project_hook(owner: str, repo: str, token: str, webhook_url: str) ->
     )
 
 
+def refresh_projects(github_token: str, repo_cache_file: Path) -> None:
+    repos = paginated_github_request(
+        "https://api.github.com/user/repos?per_page=100",
+        github_token,
+    )
+    with NamedTemporaryFile("w", delete=False, dir=repo_cache_file.parent) as f:
+        try:
+            f.write(json.dumps(repos))
+            f.flush()
+            os.rename(f.name, repo_cache_file)
+        except OSError:
+            os.unlink(f.name)
+            raise
+
+
 def load_projects(github_token: str, repo_cache_file: Path) -> list[GithubProject]:
     if repo_cache_file.exists():
         log.msg("fetching github repositories from cache")
         repos: list[dict[str, Any]] = json.loads(repo_cache_file.read_text())
     else:
-        log.msg("fetching github repositories from api")
-        repos = paginated_github_request(
-            "https://api.github.com/user/repos?per_page=100",
-            github_token,
-        )
-        repo_cache_file.write_text(json.dumps(repos, indent=2))
-
+        refresh_projects(github_token, repo_cache_file)
     return [GithubProject(repo) for repo in repos]