buildbot/__init__.py: rework the file for gerrit usecase, our oauth2 instance
Signed-off-by: Raito Bezarius <raito@lix.systems>
This commit is contained in:
parent
7a44c9f71c
commit
acfd225e6d
|
|
@ -1,7 +1,6 @@
|
|||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
|
|
@ -19,6 +18,8 @@ from buildbot.process.results import ALL_RESULTS, statusToString
|
|||
from buildbot.steps.trigger import Trigger
|
||||
from buildbot.util import asyncSleep
|
||||
from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match
|
||||
from buildbot.www.oauth2 import OAuth2Auth
|
||||
from buildbot.changes.gerritchangesource import GerritChangeSource
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from buildbot.process.log import Log
|
||||
|
|
@ -28,10 +29,6 @@ from twisted.logger import Logger
|
|||
from twisted.python.failure import Failure
|
||||
|
||||
from .github_projects import (
|
||||
GithubProject,
|
||||
create_project_hook,
|
||||
load_projects,
|
||||
refresh_projects,
|
||||
slugify_project_name,
|
||||
)
|
||||
|
||||
|
|
@ -39,10 +36,19 @@ SKIPPED_BUILDER_NAME = "skipped-builds"
|
|||
|
||||
log = Logger()
|
||||
|
||||
class LixSystemsOAuth2(OAuth2Auth):
|
||||
name = 'identity-lix-systems'
|
||||
faIcon = '...'
|
||||
resourceEndpoint = ''
|
||||
authUri = ''
|
||||
tokenUri = ''
|
||||
|
||||
class BuildbotNixError(Exception):
|
||||
pass
|
||||
|
||||
@dataclass
|
||||
class GerritProject:
|
||||
name: str
|
||||
|
||||
class BuildTrigger(Trigger):
|
||||
"""Dynamic trigger that creates a build for every attribute."""
|
||||
|
|
@ -274,58 +280,6 @@ class UpdateBuildOutput(steps.BuildStep):
|
|||
return util.SUCCESS
|
||||
|
||||
|
||||
class ReloadGithubProjects(steps.BuildStep):
|
||||
name = "reload_github_projects"
|
||||
|
||||
def __init__(self, token: str, project_cache_file: Path, **kwargs: Any) -> None:
|
||||
self.token = token
|
||||
self.project_cache_file = project_cache_file
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def reload_projects(self) -> None:
|
||||
refresh_projects(self.token, self.project_cache_file)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run(self) -> Generator[Any, object, Any]:
|
||||
d = threads.deferToThread(self.reload_projects) # type: ignore[no-untyped-call]
|
||||
|
||||
self.error_msg = ""
|
||||
|
||||
def error_cb(failure: Failure) -> int:
|
||||
self.error_msg += failure.getTraceback()
|
||||
return util.FAILURE
|
||||
|
||||
d.addCallbacks(lambda _: util.SUCCESS, error_cb)
|
||||
res = yield d
|
||||
if res == util.SUCCESS:
|
||||
# reload the buildbot config
|
||||
os.kill(os.getpid(), signal.SIGHUP)
|
||||
return util.SUCCESS
|
||||
else:
|
||||
log: Log = yield self.addLog("log")
|
||||
log.addStderr(f"Failed to reload project list: {self.error_msg}")
|
||||
return util.FAILURE
|
||||
|
||||
|
||||
def reload_github_projects(
|
||||
worker_names: list[str],
|
||||
github_token_secret: str,
|
||||
project_cache_file: Path,
|
||||
) -> util.BuilderConfig:
|
||||
"""Updates the flake an opens a PR for it."""
|
||||
factory = util.BuildFactory()
|
||||
factory.addStep(
|
||||
ReloadGithubProjects(
|
||||
github_token_secret, project_cache_file=project_cache_file
|
||||
),
|
||||
)
|
||||
return util.BuilderConfig(
|
||||
name="reload-github-projects",
|
||||
workernames=worker_names,
|
||||
factory=factory,
|
||||
)
|
||||
|
||||
|
||||
# The builtin retry mechanism doesn't seem to work for github,
|
||||
# since github is sometimes not delivering the pull request ref fast enough.
|
||||
class GitWithRetry(steps.Git):
|
||||
|
|
@ -353,9 +307,8 @@ class GitWithRetry(steps.Git):
|
|||
|
||||
|
||||
def nix_eval_config(
|
||||
project: GithubProject,
|
||||
project: GerritProject,
|
||||
worker_names: list[str],
|
||||
github_token_secret: str,
|
||||
supported_systems: list[str],
|
||||
eval_lock: util.MasterLock,
|
||||
worker_count: int,
|
||||
|
|
@ -366,15 +319,12 @@ def nix_eval_config(
|
|||
"""
|
||||
factory = util.BuildFactory()
|
||||
# check out the source
|
||||
url_with_secret = util.Interpolate(
|
||||
f"https://git:%(secret:{github_token_secret})s@github.com/%(prop:project)s",
|
||||
)
|
||||
factory.addStep(
|
||||
GitWithRetry(
|
||||
repourl=url_with_secret,
|
||||
method="clean",
|
||||
submodules=True,
|
||||
haltOnFailure=True,
|
||||
steps.Gerrit(
|
||||
repourl=project.url,
|
||||
mode="full",
|
||||
retry=[60, 60],
|
||||
timeout=3600
|
||||
),
|
||||
)
|
||||
drv_gcroots_dir = util.Interpolate(
|
||||
|
|
@ -443,7 +393,7 @@ class CachixConfig:
|
|||
|
||||
|
||||
def nix_build_config(
|
||||
project: GithubProject,
|
||||
project: GerritProject,
|
||||
worker_names: list[str],
|
||||
cachix: CachixConfig | None = None,
|
||||
outputs_path: Path | None = None,
|
||||
|
|
@ -531,7 +481,7 @@ def nix_build_config(
|
|||
|
||||
|
||||
def nix_skipped_build_config(
|
||||
project: GithubProject,
|
||||
project: GerritProject,
|
||||
worker_names: list[str],
|
||||
) -> util.BuilderConfig:
|
||||
"""Dummy builder that is triggered when a build is skipped."""
|
||||
|
|
@ -570,27 +520,11 @@ def read_secret_file(secret_name: str) -> str:
|
|||
return Path(directory).joinpath(secret_name).read_text().rstrip()
|
||||
|
||||
|
||||
@dataclass
|
||||
class GithubConfig:
|
||||
oauth_id: str
|
||||
admins: list[str]
|
||||
|
||||
buildbot_user: str
|
||||
oauth_secret_name: str = "github-oauth-secret"
|
||||
webhook_secret_name: str = "github-webhook-secret"
|
||||
token_secret_name: str = "github-token"
|
||||
project_cache_file: Path = Path("github-project-cache.json")
|
||||
topic: str | None = "build-with-buildbot"
|
||||
|
||||
def token(self) -> str:
|
||||
return read_secret_file(self.token_secret_name)
|
||||
|
||||
|
||||
def config_for_project(
|
||||
config: dict[str, Any],
|
||||
project: GithubProject,
|
||||
project: GerritProject,
|
||||
worker_names: list[str],
|
||||
github: GithubConfig,
|
||||
nix_supported_systems: list[str],
|
||||
nix_eval_worker_count: int,
|
||||
nix_eval_max_memory_size: int,
|
||||
|
|
@ -601,25 +535,6 @@ def config_for_project(
|
|||
config["projects"].append(Project(project.name))
|
||||
config["schedulers"].extend(
|
||||
[
|
||||
schedulers.SingleBranchScheduler(
|
||||
name=f"{project.project_id}-default-branch",
|
||||
change_filter=util.ChangeFilter(
|
||||
repository=project.url,
|
||||
filter_fn=lambda c: c.branch
|
||||
== c.properties.getProperty("github.repository.default_branch"),
|
||||
),
|
||||
builderNames=[f"{project.name}/nix-eval"],
|
||||
treeStableTimer=5,
|
||||
),
|
||||
# this is compatible with bors or github's merge queue
|
||||
schedulers.SingleBranchScheduler(
|
||||
name=f"{project.project_id}-merge-queue",
|
||||
change_filter=util.ChangeFilter(
|
||||
repository=project.url,
|
||||
branch_re="(gh-readonly-queue/.*|staging|trying)",
|
||||
),
|
||||
builderNames=[f"{project.name}/nix-eval"],
|
||||
),
|
||||
# build all pull requests
|
||||
schedulers.SingleBranchScheduler(
|
||||
name=f"{project.project_id}-prs",
|
||||
|
|
@ -660,7 +575,6 @@ def config_for_project(
|
|||
nix_eval_config(
|
||||
project,
|
||||
worker_names,
|
||||
github_token_secret=github.token_secret_name,
|
||||
supported_systems=nix_supported_systems,
|
||||
worker_count=nix_eval_worker_count,
|
||||
max_memory_size=nix_eval_max_memory_size,
|
||||
|
|
@ -677,97 +591,6 @@ def config_for_project(
|
|||
)
|
||||
|
||||
|
||||
class AnyProjectEndpointMatcher(EndpointMatcherBase):
|
||||
def __init__(self, builders: set[str] | None = None, **kwargs: Any) -> None:
|
||||
if builders is None:
|
||||
builders = set()
|
||||
self.builders = builders
|
||||
super().__init__(**kwargs)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def check_builder(
|
||||
self,
|
||||
endpoint_object: Any,
|
||||
endpoint_dict: dict[str, Any],
|
||||
object_type: str,
|
||||
) -> Generator[defer.Deferred[Match], Any, Any]:
|
||||
res = yield endpoint_object.get({}, endpoint_dict)
|
||||
if res is None:
|
||||
return None
|
||||
|
||||
builder = yield self.master.data.get(("builders", res["builderid"]))
|
||||
if builder["name"] in self.builders:
|
||||
log.warn(
|
||||
"Builder {builder} allowed by {role}: {builders}",
|
||||
builder=builder["name"],
|
||||
role=self.role,
|
||||
builders=self.builders,
|
||||
)
|
||||
return Match(self.master, **{object_type: res})
|
||||
else:
|
||||
log.warn(
|
||||
"Builder {builder} not allowed by {role}: {builders}",
|
||||
builder=builder["name"],
|
||||
role=self.role,
|
||||
builders=self.builders,
|
||||
)
|
||||
|
||||
def match_BuildEndpoint_rebuild( # noqa: N802
|
||||
self,
|
||||
epobject: Any,
|
||||
epdict: dict[str, Any],
|
||||
options: dict[str, Any],
|
||||
) -> defer.Deferred[Match]:
|
||||
return self.check_builder(epobject, epdict, "build")
|
||||
|
||||
def match_BuildEndpoint_stop( # noqa: N802
|
||||
self,
|
||||
epobject: Any,
|
||||
epdict: dict[str, Any],
|
||||
options: dict[str, Any],
|
||||
) -> defer.Deferred[Match]:
|
||||
return self.check_builder(epobject, epdict, "build")
|
||||
|
||||
def match_BuildRequestEndpoint_stop( # noqa: N802
|
||||
self,
|
||||
epobject: Any,
|
||||
epdict: dict[str, Any],
|
||||
options: dict[str, Any],
|
||||
) -> defer.Deferred[Match]:
|
||||
return self.check_builder(epobject, epdict, "buildrequest")
|
||||
|
||||
|
||||
def setup_authz(projects: list[GithubProject], admins: list[str]) -> util.Authz:
|
||||
allow_rules = []
|
||||
allowed_builders_by_org: defaultdict[str, set[str]] = defaultdict(
|
||||
lambda: {"reload-github-projects"},
|
||||
)
|
||||
|
||||
for project in projects:
|
||||
if project.belongs_to_org:
|
||||
for builder in ["nix-build", "nix-skipped-build", "nix-eval"]:
|
||||
allowed_builders_by_org[project.owner].add(f"{project.name}/{builder}")
|
||||
|
||||
for org, allowed_builders in allowed_builders_by_org.items():
|
||||
allow_rules.append(
|
||||
AnyProjectEndpointMatcher(
|
||||
builders=allowed_builders,
|
||||
role=org,
|
||||
defaultDeny=False,
|
||||
),
|
||||
)
|
||||
|
||||
allow_rules.append(util.AnyEndpointMatcher(role="admin", defaultDeny=False))
|
||||
allow_rules.append(util.AnyControlEndpointMatcher(role="admins"))
|
||||
return util.Authz(
|
||||
roleMatchers=[
|
||||
util.RolesFromUsername(roles=["admin"], usernames=admins),
|
||||
util.RolesFromGroups(groupPrefix=""), # so we can match on ORG
|
||||
],
|
||||
allowRules=allow_rules,
|
||||
)
|
||||
|
||||
|
||||
class PeriodicWithStartup(schedulers.Periodic):
|
||||
def __init__(self, *args: Any, run_on_startup: bool = False, **kwargs: Any) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
|
|
@ -779,14 +602,72 @@ class PeriodicWithStartup(schedulers.Periodic):
|
|||
yield self.setState("last_build", None)
|
||||
yield super().activate()
|
||||
|
||||
def gerritReviewCB(builderName, build, result, master, arg):
|
||||
if result == util.RETRY:
|
||||
return dict()
|
||||
|
||||
class NixConfigurator(ConfiguratorBase):
|
||||
message = "Buildbot finished compiling your patchset\n"
|
||||
message += "on configuration: %s\n" % builderName
|
||||
message += "The result is: %s\n" % util.Results[result].upper()
|
||||
|
||||
if arg:
|
||||
message += "\nFor more details visit:\n"
|
||||
message += build['url'] + "\n"
|
||||
|
||||
if result == util.SUCCESS:
|
||||
verified = 1
|
||||
else:
|
||||
verified = -1
|
||||
|
||||
return dict(message=message, labels={'Verified': verified})
|
||||
|
||||
def gerritStartCB(builderName, build, arg):
|
||||
message = "Buildbot started compiling your patchset\n"
|
||||
message += "on configuration: %s\n" % builderName
|
||||
message += "See your build here: %s" % build['url']
|
||||
|
||||
return dict(message=message)
|
||||
|
||||
def gerritSummaryCB(buildInfoList, results, status, arg):
|
||||
success = False
|
||||
failure = False
|
||||
|
||||
msgs = []
|
||||
|
||||
for buildInfo in buildInfoList:
|
||||
msg = "Builder %(name)s %(resultText)s (%(text)s)" % buildInfo
|
||||
link = buildInfo.get('url', None)
|
||||
if link:
|
||||
msg += " - " + link
|
||||
else:
|
||||
msg += "."
|
||||
|
||||
msgs.append(msg)
|
||||
|
||||
if buildInfo['result'] == util.SUCCESS:
|
||||
success = True
|
||||
else:
|
||||
failure = True
|
||||
|
||||
if success and not failure:
|
||||
verified = 1
|
||||
else:
|
||||
verified = -1
|
||||
|
||||
return dict(message='\n\n'.join(msgs),
|
||||
labels={
|
||||
'Verified': verified
|
||||
})
|
||||
|
||||
class GerritNixConfigurator(ConfiguratorBase):
|
||||
"""Janitor is a configurator which create a Janitor Builder with all needed Janitor steps"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
# Shape of this file: [ { "name": "<worker-name>", "pass": "<worker-password>", "cores": "<cpu-cores>" } ]
|
||||
github: GithubConfig,
|
||||
gerrit_server: str,
|
||||
gerrit_user: str,
|
||||
gerrit_port: int,
|
||||
url: str,
|
||||
nix_supported_systems: list[str],
|
||||
nix_eval_worker_count: int | None,
|
||||
|
|
@ -796,11 +677,14 @@ class NixConfigurator(ConfiguratorBase):
|
|||
outputs_path: str | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.gerrit_server = gerrit_server
|
||||
self.gerrit_user = gerrit_user
|
||||
self.gerrit_port = gerrit_port
|
||||
self.nix_workers_secret_name = nix_workers_secret_name
|
||||
self.nix_eval_max_memory_size = nix_eval_max_memory_size
|
||||
self.nix_eval_worker_count = nix_eval_worker_count
|
||||
self.nix_supported_systems = nix_supported_systems
|
||||
self.github = github
|
||||
self.gerrit_change_source = GerritChangeSource(gerrit_server, gerrit_user, gerritport=gerrit_port)
|
||||
self.url = url
|
||||
self.cachix = cachix
|
||||
if outputs_path is None:
|
||||
|
|
@ -809,9 +693,6 @@ class NixConfigurator(ConfiguratorBase):
|
|||
self.outputs_path = Path(outputs_path)
|
||||
|
||||
def configure(self, config: dict[str, Any]) -> None:
|
||||
projects = load_projects(self.github.token(), self.github.project_cache_file)
|
||||
if self.github.topic is not None:
|
||||
projects = [p for p in projects if self.github.topic in p.topics]
|
||||
worker_config = json.loads(read_secret_file(self.nix_workers_secret_name))
|
||||
worker_names = []
|
||||
|
||||
|
|
@ -826,63 +707,30 @@ class NixConfigurator(ConfiguratorBase):
|
|||
config["workers"].append(worker.Worker(worker_name, item["pass"]))
|
||||
worker_names.append(worker_name)
|
||||
|
||||
webhook_secret = read_secret_file(self.github.webhook_secret_name)
|
||||
eval_lock = util.MasterLock("nix-eval")
|
||||
|
||||
for project in projects:
|
||||
create_project_hook(
|
||||
project.owner,
|
||||
project.repo,
|
||||
self.github.token(),
|
||||
self.url + "change_hook/github",
|
||||
webhook_secret,
|
||||
)
|
||||
config_for_project(
|
||||
config,
|
||||
project,
|
||||
worker_names,
|
||||
self.github,
|
||||
self.nix_supported_systems,
|
||||
self.nix_eval_worker_count or multiprocessing.cpu_count(),
|
||||
self.nix_eval_max_memory_size,
|
||||
eval_lock,
|
||||
self.cachix,
|
||||
self.outputs_path,
|
||||
)
|
||||
# TODO: initialize Lix
|
||||
# config_for_project(
|
||||
# config,
|
||||
# project,
|
||||
# worker_names,
|
||||
# self.nix_supported_systems,
|
||||
# self.nix_eval_worker_count or multiprocessing.cpu_count(),
|
||||
# self.nix_eval_max_memory_size,
|
||||
# eval_lock,
|
||||
# self.cachix,
|
||||
# self.outputs_path,
|
||||
# )
|
||||
|
||||
# Reload github projects
|
||||
config["builders"].append(
|
||||
reload_github_projects(
|
||||
[worker_names[0]],
|
||||
self.github.token(),
|
||||
self.github.project_cache_file,
|
||||
),
|
||||
)
|
||||
config["workers"].append(worker.LocalWorker(SKIPPED_BUILDER_NAME))
|
||||
config["schedulers"].extend(
|
||||
[
|
||||
schedulers.ForceScheduler(
|
||||
name="reload-github-projects",
|
||||
builderNames=["reload-github-projects"],
|
||||
buttonName="Update projects",
|
||||
),
|
||||
# project list twice a day and on startup
|
||||
PeriodicWithStartup(
|
||||
name="reload-github-projects-bidaily",
|
||||
builderNames=["reload-github-projects"],
|
||||
periodicBuildTimer=12 * 60 * 60,
|
||||
run_on_startup=not self.github.project_cache_file.exists(),
|
||||
),
|
||||
],
|
||||
)
|
||||
config["services"].append(
|
||||
reporters.GitHubStatusPush(
|
||||
token=self.github.token(),
|
||||
# Since we dynamically create build steps,
|
||||
# we use `virtual_builder_name` in the webinterface
|
||||
# so that we distinguish what has beeing build
|
||||
context=Interpolate("buildbot/%(prop:status_name)s"),
|
||||
),
|
||||
reporters.GerritStatusPush(self.gerrit_server, self.gerrit_user,
|
||||
reviewCB=gerritReviewCB,
|
||||
reviewArg=self.url,
|
||||
startCB=gerritStartCB,
|
||||
startArg=self.url,
|
||||
summaryCB=gerritSummaryCB,
|
||||
summaryArg=self.url)
|
||||
)
|
||||
|
||||
systemd_secrets = secrets.SecretInAFile(
|
||||
|
|
@ -893,26 +741,5 @@ class NixConfigurator(ConfiguratorBase):
|
|||
config["www"].setdefault("plugins", {})
|
||||
config["www"]["plugins"].update(dict(base_react={}))
|
||||
|
||||
config["www"].setdefault("change_hook_dialects", {})
|
||||
config["www"]["change_hook_dialects"]["github"] = {
|
||||
"secret": webhook_secret,
|
||||
"strict": True,
|
||||
"token": self.github.token(),
|
||||
"github_property_whitelist": "*",
|
||||
}
|
||||
|
||||
if "auth" not in config["www"]:
|
||||
config["www"].setdefault("avatar_methods", [])
|
||||
config["www"]["avatar_methods"].append(
|
||||
util.AvatarGitHub(token=self.github.token()),
|
||||
)
|
||||
config["www"]["auth"] = util.GitHubAuth(
|
||||
self.github.oauth_id,
|
||||
read_secret_file(self.github.oauth_secret_name),
|
||||
apiVersion=4,
|
||||
)
|
||||
|
||||
config["www"]["authz"] = setup_authz(
|
||||
admins=self.github.admins,
|
||||
projects=projects,
|
||||
)
|
||||
config["www"]["auth"] = LixSystemsOAuth2()
|
||||
|
|
|
|||
Loading…
Reference in a new issue