lix-project/buildbot-nix
15
0
Fork 0
Code Issues 12 Pull requests 6 Packages Projects Releases Wiki Activity

Compare commits

base: lix-project:profiler
Branches Tags
lix-project:main
lix-project:profiler
lix-project:jade/report-eval-errors
lix-project:forkos
lix-project:bring-back-old-gerrit-reporting
lix-project:fix-reporting
lix-project:incoming-ref-for-non-flakes
lix-project:local-stores
lix-project:non-flakes-rerun
lix-project:prepare-for-non-flakes
lix-project:non-flakes
lix-project:jade/deny-flake-config
lix-project:fallable-binary-cache
lix-project:gerrit
lix-project:better-gerrit
lix-project:puck/gerrit-changes
..
compare: lix-project:main
Branches Tags
lix-project:profiler
lix-project:jade/report-eval-errors
lix-project:forkos
lix-project:bring-back-old-gerrit-reporting
lix-project:fix-reporting
lix-project:incoming-ref-for-non-flakes
lix-project:local-stores
lix-project:non-flakes-rerun
lix-project:prepare-for-non-flakes
lix-project:non-flakes
lix-project:main
lix-project:jade/deny-flake-config
lix-project:fallable-binary-cache
lix-project:gerrit
lix-project:better-gerrit
lix-project:puck/gerrit-changes

No commits in common. "profiler" and "main" have entirely different histories.
profiler ... main

8 changed files with 126 additions and 371 deletions
Show stats Expand all files Collapse all files

1
.envrc
View file

@ -1 +0,0 @@
use flake

385
buildbot_nix/__init__.py
View file

@ -7,47 +7,37 @@ import base64
from collections.abc import Generator from collections.abc import Generator
from dataclasses import dataclass, field from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
from typing import Any from typing import TYPE_CHECKING, Any
import buildbot import buildbot
from buildbot.configurators import ConfiguratorBase from buildbot.configurators import ConfiguratorBase
from buildbot.plugins import reporters, schedulers, secrets, steps, util, worker from buildbot.plugins import reporters, schedulers, secrets, steps, util, worker
from buildbot.process import buildstep, logobserver, remotecommand from buildbot.process import buildstep, logobserver, remotecommand
from buildbot.process.log import StreamLog
from buildbot.process.project import Project from buildbot.process.project import Project
from buildbot.process.properties import Properties from buildbot.process.properties import Properties
from buildbot.process.results import ALL_RESULTS, statusToString from buildbot.process.results import ALL_RESULTS, statusToString
from buildbot.www.auth import AuthBase from buildbot.www.auth import AuthBase
from buildbot.www.oauth2 import OAuth2Auth from buildbot.www.oauth2 import OAuth2Auth
from buildbot.changes.gerritchangesource import GerritChangeSource from buildbot.changes.gerritchangesource import GerritChangeSource
from buildbot.reporters.generators.build import BuildStatusGenerator
from buildbot.reporters.utils import getURLForBuildrequest from buildbot.reporters.utils import getURLForBuildrequest
from buildbot.reporters.generators.build import BuildStatusGenerator
from buildbot.reporters.message import MessageFormatterFunction
from buildbot.process.buildstep import EXCEPTION from buildbot.process.buildstep import EXCEPTION
from buildbot.process.buildstep import SUCCESS from buildbot.process.buildstep import SUCCESS
from buildbot.process.results import worst_status from buildbot.process.results import worst_status
import requests import requests
if TYPE_CHECKING:
from buildbot.process.log import Log
from twisted.internet import defer from twisted.internet import defer
from twisted.logger import Logger from twisted.logger import Logger
from .binary_cache import S3BinaryCacheConfig from .binary_cache import S3BinaryCacheConfig
from .message_formatter import ReasonableMessageFormatter, CallbackPayloadBuild, CallbackPayloadBuildSet, CallbackReturn
log = Logger() log = Logger()
# util is a plugin variable, not a module... FLAKE_TARGET_ATTRIBUTE_FOR_JOBS = "buildbotJobs"
BuilderConfig = util.BuilderConfig
MasterLock = util.MasterLock
FLAKE_TARGET_ATTRIBUTE_FOR_JOBS = "hydraJobs"
@dataclass
class EvaluatorSettings:
supported_systems: list[str]
worker_count: int
max_memory_size: int
gc_roots_dir: str
lock: MasterLock
@dataclass @dataclass
class NixBuilder: class NixBuilder:
@ -59,24 +49,14 @@ class NixBuilder:
publicHostKey: str | None = None publicHostKey: str | None = None
sshUser: str | None = None sshUser: str | None = None
sshKey: str | None = None sshKey: str | None = None
systems: list[str] = field(default_factory=lambda: []) systems: list[str] = field(default_factory=lambda: ["-"])
supportedFeatures: list[str] = field(default_factory=lambda: []) supportedFeatures: list[str] = field(default_factory=lambda: ["-"])
mandatoryFeatures: list[str] = field(default_factory=lambda: []) mandatoryFeatures: list[str] = field(default_factory=lambda: ["-"])
def to_nix_store(self): def to_nix_line(self):
fullConnection = f"{self.sshUser}@{self.hostName}" if self.sshUser is not None else self.hostName encoded_public_key = base64.b64encode(self.publicHostKey.encode('ascii')).decode('ascii') if self.publicHostKey is not None else "-"
fullConnection = f"{self.protocol}://{fullConnection}" fullConnection = f"{self.protocol}://{self.sshUser}@{self.hostName}" if self.sshUser is not None else self.hostName
params = [] return f"{fullConnection} {",".join(self.systems)} {self.sshKey or "-"} {self.maxJobs} {self.speedFactor} {",".join(self.supportedFeatures)} {",".join(self.mandatoryFeatures)} {encoded_public_key}"
if self.sshKey is not None:
params.append(f"ssh-key={self.sshKey}")
if self.publicHostKey is not None:
encoded_public_key = base64.b64encode(self.publicHostKey.encode('ascii')).decode('ascii')
params.append(f"base64-ssh-public-host-key={encoded_public_key}")
if params != []:
fullConnection += "?"
fullConnection += "&".join(params)
return fullConnection
@dataclass @dataclass
@ -91,10 +71,9 @@ class OAuth2Config:
debug: bool = False debug: bool = False
class KeycloakOAuth2Auth(OAuth2Auth): class KeycloakOAuth2Auth(OAuth2Auth):
userinfoUri: str def __init__(self, userinfoUri: str, *args, debug=False, **kwargs):
def __init__(self, *args, debug=False, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.userinfoUri = userinfoUri
self.debug = debug self.debug = debug
def createSessionFromToken(self, token): def createSessionFromToken(self, token):
@ -151,7 +130,7 @@ class GerritConfig:
""" """
Returns the prefix to build a repourl using that gerrit configuration. Returns the prefix to build a repourl using that gerrit configuration.
""" """
return f'ssh://{self.username}@{self.domain}:{self.port}/' return 'ssh://{self.username}@{self.domain}:{self.port}/'
class BuildTrigger(steps.BuildStep): class BuildTrigger(steps.BuildStep):
def __init__( def __init__(
@ -169,7 +148,7 @@ class BuildTrigger(steps.BuildStep):
self.ended = False self.ended = False
self.waitForFinishDeferred = None self.waitForFinishDeferred = None
self.brids = [] self.brids = []
self.description = f"building {len(jobs)} jobs" self.description = f"building {len(jobs)} hydra jobs"
super().__init__(**kwargs) super().__init__(**kwargs)
def interrupt(self, reason): def interrupt(self, reason):
@ -198,16 +177,15 @@ class BuildTrigger(steps.BuildStep):
return sch return sch
def schedule_one(self, build_props: Properties, job): def schedule_one(self, build_props: Properties, job):
project_name = build_props.getProperty("event.refUpdate.project") or build_props.getProperty("event.change.project") project_name = build_props.getProperty('event.project')
source = f"{project_name}-eval" source = f"{project_name}-eval-lix"
attr = job.get("attr", "eval-error") attr = job.get("attr", "eval-error")
# FIXME(raito): this was named this way for backward compatibility with Lix deployment. name = attr
# We should just parametrize this. name = f"{FLAKE_TARGET_ATTRIBUTE_FOR_JOBS}.{name}"
name = f"hydraJobs.{attr}"
error = job.get("error") error = job.get("error")
props = Properties() props = Properties()
props.setProperty("virtual_builder_name", name, source) props.setProperty("virtual_builder_name", name, source)
props.setProperty("status_name", f"building hydraJobs.{attr}", source) props.setProperty("status_name", f"nix-build .#{FLAKE_TARGET_ATTRIBUTE_FOR_JOBS}.{attr}", source)
props.setProperty("virtual_builder_tags", "", source) props.setProperty("virtual_builder_tags", "", source)
if error is not None: if error is not None:
@ -256,7 +234,7 @@ class BuildTrigger(steps.BuildStep):
def run(self): def run(self):
self.running = True self.running = True
build_props = self.build.getProperties() build_props = self.build.getProperties()
logs: StreamLog = yield self.addLog("build info") logs: Log = yield self.addLog("build info")
builds_to_schedule = list(self.jobs) builds_to_schedule = list(self.jobs)
build_schedule_order = [] build_schedule_order = []
@ -394,8 +372,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
# run nix-eval-jobs --flake .#$FLAKE_TARGET_ATTRIBUTE_FOR_JOBS to generate the dict of stages # run nix-eval-jobs --flake .#$FLAKE_TARGET_ATTRIBUTE_FOR_JOBS to generate the dict of stages
cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand() cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand()
build_props = self.build.getProperties() build_props = self.build.getProperties()
project_name = build_props.getProperty("event.refUpdate.project") or build_props.getProperty("event.change.project") project_name = build_props.get('event.project')
assert project_name is not None, "`event.refUpdate.project` or `event.change.project` is not available on the build properties, unexpected build type!"
yield self.runCommand(cmd) yield self.runCommand(cmd)
@ -419,11 +396,26 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
if not system or system in self.supported_systems: # report eval errors if not system or system in self.supported_systems: # report eval errors
filtered_jobs.append(job) filtered_jobs.append(job)
# Filter out failed evaluations drv_show_log: Log = yield self.getLog("stdio")
succeeded_jobs = [job for job in filtered_jobs if job.get('error') is None] drv_show_log.addStdout(f"getting derivation infos\n")
cmd = yield self.makeRemoteShellCommand(
drv_show_log: StreamLog = yield self.getLog("stdio") stdioLogName=None,
collectStdout=True,
command=(
["nix", "derivation", "show", "--recursive"]
+ [ drv for drv in (job.get("drvPath") for job in filtered_jobs) if drv ]
),
)
yield self.runCommand(cmd)
drv_show_log.addStdout(f"done\n")
try:
drv_info = json.loads(cmd.stdout)
except json.JSONDecodeError as e:
msg = f"Failed to parse `nix derivation show` output for {cmd.command}"
raise BuildbotNixError(msg) from e
all_deps = dict() all_deps = dict()
for drv, info in drv_info.items():
all_deps[drv] = set(info.get("inputDrvs").keys())
def closure_of(key, deps): def closure_of(key, deps):
r, size = set([key]), 0 r, size = set([key]), 0
@ -432,34 +424,14 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
r.update(*[ deps[k] for k in r ]) r.update(*[ deps[k] for k in r ])
return r.difference([key]) return r.difference([key])
if succeeded_jobs: job_set = set(( drv for drv in ( job.get("drvPath") for job in filtered_jobs ) if drv ))
drv_show_log.addStdout(f"getting derivation infos for valid derivations\n") all_deps = { k: list(closure_of(k, all_deps).intersection(job_set)) for k in job_set }
cmd = yield self.makeRemoteShellCommand(
stdioLogName=None,
collectStdout=True,
command=(
["nix", "derivation", "show", "--recursive"]
+ [ drv for drv in (job.get("drvPath") for job in succeeded_jobs) if drv ]
),
)
yield self.runCommand(cmd)
drv_show_log.addStdout(f"done\n")
try:
drv_info = json.loads(cmd.stdout)
except json.JSONDecodeError as e:
msg = f"Failed to parse `nix derivation show` output for {cmd.command}"
raise BuildbotNixError(msg) from e
for drv, info in drv_info.items():
all_deps[drv] = set(info.get("inputDrvs").keys())
job_set = set(( drv for drv in ( job.get("drvPath") for job in filtered_jobs ) if drv ))
all_deps = { k: list(closure_of(k, all_deps).intersection(job_set)) for k in job_set }
self.build.addStepsAfterCurrentStep( self.build.addStepsAfterCurrentStep(
[ [
BuildTrigger( BuildTrigger(
builds_scheduler_group=f"{project_name}-nix-build", builds_scheduler_group=f"{project_name}-nix-build",
name="build derivations", name="build flake",
jobs=filtered_jobs, jobs=filtered_jobs,
all_deps=all_deps, all_deps=all_deps,
), ),
@ -468,91 +440,6 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
return result return result
def make_job_evaluator(name: str, settings: EvaluatorSettings, flake: bool, incoming_ref_filename: str) -> NixEvalCommand:
actual_command = []
if flake:
actual_command += ["--flake", f".#{FLAKE_TARGET_ATTRIBUTE_FOR_JOBS}"]
else:
actual_command += ["--expr",
f"import ./.ci/buildbot.nix {{ incoming_ref_data = builtins.fromJSON (builtins.readFile {incoming_ref_filename}); }}"]
return NixEvalCommand(
env={},
name=name,
supported_systems=settings.supported_systems,
command=[
"nix-eval-jobs",
"--workers",
str(settings.worker_count),
"--max-memory-size",
str(settings.max_memory_size),
"--gc-roots-dir",
settings.gc_roots_dir,
"--force-recurse",
"--check-cache-status",
] + actual_command,
haltOnFailure=True,
locks=[settings.lock.access("exclusive")]
)
class NixConfigure(buildstep.CommandMixin, steps.BuildStep):
name = "determining jobs"
"""
Determine what `NixEvalCommand` step should be added after
based on the existence of:
- flake.nix
- .ci/buildbot.nix
"""
def __init__(self, eval_settings: EvaluatorSettings, **kwargs: Any) -> None:
self.evaluator_settings = eval_settings
super().__init__(**kwargs)
self.observer = logobserver.BufferLogObserver()
self.addLogObserver("stdio", self.observer)
@defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]:
try:
configure_log: StreamLog = yield self.getLog("stdio")
except Exception:
configure_log: StreamLog = yield self.addLog("stdio")
# Takes precedence.
configure_log.addStdout("checking if there's a .ci/buildbot.nix...\n")
ci_buildbot_defn_exists = yield self.pathExists('build/.ci/buildbot.nix')
if ci_buildbot_defn_exists:
configure_log.addStdout(".ci/buildbot.nix found, configured for non-flake CI\n")
self.build.addStepsAfterCurrentStep(
[
make_job_evaluator(
"evaluate `.ci/buildbot.nix` jobs",
self.evaluator_settings,
False,
"./incoming-ref.json"
)
]
)
return SUCCESS
flake_exists = yield self.pathExists('build/flake.nix')
if flake_exists:
configure_log.addStdout(f"flake.nix found")
self.build.addStepsAfterCurrentStep([
make_job_evaluator(
"evaluate `flake.nix` jobs",
self.evaluator_settings,
True,
"./incoming-ref.json"
)
]
)
return SUCCESS
configure_log.addStdout("neither flake.nix found neither .ci/buildbot.nix, no CI to run!")
return SUCCESS
class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep): class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep):
"""Builds a nix derivation.""" """Builds a nix derivation."""
@ -566,7 +453,7 @@ class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep):
if error := self.getProperty("error"): if error := self.getProperty("error"):
attr = self.getProperty("attr") attr = self.getProperty("attr")
# show eval error # show eval error
error_log: StreamLog = yield self.addLog("nix_error") error_log: Log = yield self.addLog("nix_error")
error_log.addStderr(f"{attr} failed to evaluate:\n{error}") error_log.addStderr(f"{attr} failed to evaluate:\n{error}")
return util.FAILURE return util.FAILURE
@ -590,23 +477,14 @@ def nix_eval_config(
project: GerritProject, project: GerritProject,
worker_names: list[str], worker_names: list[str],
supported_systems: list[str], supported_systems: list[str],
eval_lock: MasterLock, eval_lock: util.MasterLock,
worker_count: int, worker_count: int,
max_memory_size: int, max_memory_size: int,
) -> BuilderConfig: ) -> util.BuilderConfig:
""" """Uses nix-eval-jobs to evaluate $FLAKE_TARGET_ATTRIBUTE_FOR_JOBS (`.#hydraJobs` by default) from flake.nix in parallel.
Uses nix-eval-jobs to evaluate the entrypoint of this project.
For each evaluated attribute a new build pipeline is started. For each evaluated attribute a new build pipeline is started.
""" """
factory = util.BuildFactory() factory = util.BuildFactory()
gerrit_private_key = None
with open(project.private_sshkey_path, 'r') as f:
gerrit_private_key = f.read()
if gerrit_private_key is None:
raise RuntimeError('No gerrit private key to fetch the repositories')
# check out the source # check out the source
factory.addStep( factory.addStep(
steps.Gerrit( steps.Gerrit(
@ -614,10 +492,9 @@ def nix_eval_config(
mode="full", mode="full",
retry=[60, 60], retry=[60, 60],
timeout=3600, timeout=3600,
sshPrivateKey=gerrit_private_key sshPrivateKey=project.private_sshkey_path
), ),
) )
# use one gcroots directory per worker. this should be scoped to the largest unique resource # use one gcroots directory per worker. this should be scoped to the largest unique resource
# in charge of builds (ie, buildnumber is too narrow) to not litter the system with permanent # in charge of builds (ie, buildnumber is too narrow) to not litter the system with permanent
# gcroots in case of worker restarts. # gcroots in case of worker restarts.
@ -626,27 +503,27 @@ def nix_eval_config(
"/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/drvs/%(prop:workername)s/", "/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/drvs/%(prop:workername)s/",
) )
eval_settings = EvaluatorSettings(
supported_systems=supported_systems,
worker_count=worker_count,
max_memory_size=max_memory_size,
gc_roots_dir=drv_gcroots_dir,
lock=eval_lock
)
# This information can be passed at job evaluation time
# to skip some jobs, e.g. expensive jobs, etc.
# Transfer incoming ref data to the target.
factory.addStep(steps.JSONPropertiesDownload(workerdest="incoming-ref.json"))
# NixConfigure will choose
# how to add a NixEvalCommand job
# based on whether there's a flake.nix or
# a .ci/buildbot.nix.
factory.addStep( factory.addStep(
NixConfigure( NixEvalCommand(
eval_settings env={},
) name="evaluate flake",
supported_systems=supported_systems,
command=[
"nix-eval-jobs",
"--workers",
str(worker_count),
"--max-memory-size",
str(max_memory_size),
"--gc-roots-dir",
drv_gcroots_dir,
"--force-recurse",
"--check-cache-status",
"--flake",
f".#{FLAKE_TARGET_ATTRIBUTE_FOR_JOBS}"
],
haltOnFailure=True,
locks=[eval_lock.access("exclusive")],
),
) )
factory.addStep( factory.addStep(
@ -674,17 +551,12 @@ def nix_build_config(
project: GerritProject, project: GerritProject,
worker_arch: str, worker_arch: str,
worker_names: list[str], worker_names: list[str],
build_stores: list[str], builders_spec: str,
signing_keyfile: str | None = None, signing_keyfile: str | None = None,
binary_cache_config: S3BinaryCacheConfig | None = None binary_cache_config: S3BinaryCacheConfig | None = None
) -> BuilderConfig: ) -> util.BuilderConfig:
"""Builds one nix flake attribute.""" """Builds one nix flake attribute."""
factory = util.BuildFactory() factory = util.BuildFactory()
# pick a store to run the build on
# TODO proper scheduling instead of picking the first builder
build_store = build_stores[0]
factory.addStep( factory.addStep(
NixBuildCommand( NixBuildCommand(
env={}, env={},
@ -704,10 +576,8 @@ def nix_build_config(
# kill builds after two hours regardless of activity # kill builds after two hours regardless of activity
"--timeout", "--timeout",
"7200", "7200",
"--store", "--builders",
build_store, builders_spec,
"--eval-store",
"ssh-ng://localhost",
"--out-link", "--out-link",
util.Interpolate("result-%(prop:attr)s"), util.Interpolate("result-%(prop:attr)s"),
util.Interpolate("%(prop:drv_path)s^*"), util.Interpolate("%(prop:drv_path)s^*"),
@ -727,8 +597,6 @@ def nix_build_config(
"nix", "nix",
"store", "store",
"sign", "sign",
"--store",
build_store,
"--key-file", "--key-file",
signing_keyfile, signing_keyfile,
util.Interpolate( util.Interpolate(
@ -745,8 +613,6 @@ def nix_build_config(
command=[ command=[
"nix", "nix",
"copy", "copy",
"--store",
build_store,
"--to", "--to",
f"s3://{binary_cache_config.bucket}?profile={binary_cache_config.profile}&region={binary_cache_config.region}&endpoint={binary_cache_config.endpoint}", f"s3://{binary_cache_config.bucket}?profile={binary_cache_config.profile}&region={binary_cache_config.region}&endpoint={binary_cache_config.endpoint}",
util.Property( util.Property(
@ -807,11 +673,11 @@ def config_for_project(
nix_supported_systems: list[str], nix_supported_systems: list[str],
nix_eval_worker_count: int, nix_eval_worker_count: int,
nix_eval_max_memory_size: int, nix_eval_max_memory_size: int,
eval_lock: MasterLock, eval_lock: util.MasterLock,
nix_builders: list[NixBuilder], builders_spec: str,
signing_keyfile: str | None = None, signing_keyfile: str | None = None,
binary_cache_config: S3BinaryCacheConfig | None = None binary_cache_config: S3BinaryCacheConfig | None = None
) -> None: ) -> Project:
config["projects"].append(Project(project.name)) config["projects"].append(Project(project.name))
config["schedulers"].extend( config["schedulers"].extend(
[ [
@ -846,6 +712,12 @@ def config_for_project(
), ),
], ],
) )
gerrit_private_key = None
with open(project.private_sshkey_path, 'r') as f:
gerrit_private_key = f.read()
if gerrit_private_key is None:
raise RuntimeError('No gerrit private key to fetch the repositories')
config["builders"].extend( config["builders"].extend(
[ [
@ -865,7 +737,7 @@ def config_for_project(
project, project,
arch, arch,
[ f"{w}-{arch}" for w in worker_names ], [ f"{w}-{arch}" for w in worker_names ],
[b.to_nix_store() for b in nix_builders if arch in b.systems or arch == "other"], builders_spec,
signing_keyfile=signing_keyfile, signing_keyfile=signing_keyfile,
binary_cache_config=binary_cache_config binary_cache_config=binary_cache_config
) )
@ -886,25 +758,25 @@ class PeriodicWithStartup(schedulers.Periodic):
yield self.setState("last_build", None) yield self.setState("last_build", None)
yield super().activate() yield super().activate()
def gerritReviewFmt(url: str, payload: CallbackPayloadBuild | CallbackPayloadBuildSet) -> CallbackReturn: def gerritReviewFmt(url, data):
assert isinstance(payload, CallbackPayloadBuild), "BuildSet are not handled yet!" if 'build' not in data:
raise ValueError('`build` is supposed to be present to format a build')
build = payload.build build = data['build']
if 'builder' not in build and 'name' not in build['builder']: if 'builder' not in build and 'name' not in build['builder']:
raise ValueError('either `builder` or `builder.name` is not present in the build dictionary, unexpected format request') raise ValueError('either `builder` or `builder.name` is not present in the build dictionary, unexpected format request')
builderName = build['builder']['name'] builderName = build['builder']['name']
result = build['results'] if len(build['results']) != 1:
log.info("Formatting a message for a Gerrit build: {} -- result is {}".format(builderName, result)) raise ValueError('this review request contains more than one build results, unexpected format request')
result = build['results'][0]
if result == util.RETRY: if result == util.RETRY:
return CallbackReturn() return dict()
expectedBuilderName = f'{build["properties"].get("event.project")[0]}/nix-eval' if builderName != f'{build["properties"].get("event.project")}/nix-eval':
return dict()
if builderName != expectedBuilderName:
log.info("Passing {} builder which is not of the form '{}'".format(builderName, expectedBuilderName))
return CallbackReturn()
failed = build['properties'].get('failed_builds', [[]])[0] failed = build['properties'].get('failed_builds', [[]])[0]
@ -928,8 +800,7 @@ def gerritReviewFmt(url: str, payload: CallbackPayloadBuild | CallbackPayloadBui
message += "\nFor more details visit:\n" message += "\nFor more details visit:\n"
message += build['url'] + "\n" message += build['url'] + "\n"
log.info("Message formatted: {}, labels: Verified={}".format(message, labels['Verified'])) return dict(message=message, labels=labels)
return CallbackReturn(body=message, extra_info={'labels': labels})
class GerritNixConfigurator(ConfiguratorBase): class GerritNixConfigurator(ConfiguratorBase):
"""Janitor is a configurator which create a Janitor Builder with all needed Janitor steps""" """Janitor is a configurator which create a Janitor Builder with all needed Janitor steps"""
@ -953,16 +824,13 @@ class GerritNixConfigurator(ConfiguratorBase):
prometheus_config: dict[str, int | str] | None = None, prometheus_config: dict[str, int | str] | None = None,
binary_cache_config: dict[str, str] | None = None, binary_cache_config: dict[str, str] | None = None,
auth_method: AuthBase | None = None, auth_method: AuthBase | None = None,
manhole: Any = None,
enable_profiler: bool = False,
) -> None: ) -> None:
super().__init__() super().__init__()
self.manhole = manhole
self.allowed_origins = allowed_origins self.allowed_origins = allowed_origins
self.gerrit_server = gerrit_server self.gerrit_server = gerrit_server
self.gerrit_user = gerrit_user self.gerrit_user = gerrit_user
self.gerrit_port = gerrit_port self.gerrit_port = gerrit_port
self.gerrit_sshkey_path = str(gerrit_sshkey_path) self.gerrit_sshkey_path = gerrit_sshkey_path
self.gerrit_config = GerritConfig(domain=self.gerrit_server, self.gerrit_config = GerritConfig(domain=self.gerrit_server,
username=self.gerrit_user, username=self.gerrit_user,
port=self.gerrit_port) port=self.gerrit_port)
@ -978,7 +846,6 @@ class GerritNixConfigurator(ConfiguratorBase):
self.url = url self.url = url
self.prometheus_config = prometheus_config self.prometheus_config = prometheus_config
self.enable_profiler = enable_profiler
if binary_cache_config is not None: if binary_cache_config is not None:
self.binary_cache_config = S3BinaryCacheConfig(**binary_cache_config) self.binary_cache_config = S3BinaryCacheConfig(**binary_cache_config)
@ -989,32 +856,30 @@ class GerritNixConfigurator(ConfiguratorBase):
self.auth_method = auth_method self.auth_method = auth_method
def configure(self, config_dict: dict[str, Any]) -> None: def configure(self, config: dict[str, Any]) -> None:
worker_config_dict = json.loads(read_secret_file(self.nix_workers_secret_name)) worker_config = json.loads(read_secret_file(self.nix_workers_secret_name))
worker_names = [] worker_names = []
if self.manhole is not None: config.setdefault("projects", [])
config_dict["manhole"] = self.manhole config.setdefault("secretsProviders", [])
config.setdefault("www", {
'allowed_origins': self.allowed_origins
})
config_dict.setdefault("projects", []) for item in worker_config:
config_dict.setdefault("secretsProviders", [])
print('Default allowed origins for this Buildbot server: {}'.format(', '.join(self.allowed_origins)))
config_dict["www"]["allowed_origins"] = self.allowed_origins
for item in worker_config_dict:
cores = item.get("cores", 0) cores = item.get("cores", 0)
for i in range(cores): for i in range(cores):
for arch in self.nix_supported_systems + ["other"]: for arch in self.nix_supported_systems + ["other"]:
worker_name = f"{item['name']}-{i:03}" worker_name = f"{item['name']}-{i:03}"
config_dict["workers"].append(worker.Worker(f"{worker_name}-{arch}", item["pass"])) config["workers"].append(worker.Worker(f"{worker_name}-{arch}", item["pass"]))
worker_names.append(worker_name) worker_names.append(worker_name)
eval_lock = util.MasterLock("nix-eval") eval_lock = util.MasterLock("nix-eval")
builders_spec = " ; ".join(builder.to_nix_line() for builder in self.nix_builders)
for project in self.projects: for project in self.projects:
config_for_project( config_for_project(
config_dict, config,
self.gerrit_config, self.gerrit_config,
GerritProject(name=project, private_sshkey_path=self.gerrit_sshkey_path), GerritProject(name=project, private_sshkey_path=self.gerrit_sshkey_path),
worker_names, worker_names,
@ -1022,20 +887,20 @@ class GerritNixConfigurator(ConfiguratorBase):
self.nix_eval_worker_count or multiprocessing.cpu_count(), self.nix_eval_worker_count or multiprocessing.cpu_count(),
self.nix_eval_max_memory_size, self.nix_eval_max_memory_size,
eval_lock, eval_lock,
self.nix_builders, builders_spec,
signing_keyfile=self.signing_keyfile, signing_keyfile=self.signing_keyfile,
binary_cache_config=self.binary_cache_config binary_cache_config=self.binary_cache_config
) )
config_dict["change_source"] = self.gerrit_change_source config["change_source"] = self.gerrit_change_source
config_dict["services"].append( config["services"].append(
reporters.GerritStatusPush(self.gerrit_server, self.gerrit_user, reporters.GerritStatusPush(self.gerrit_server, self.gerrit_user,
port=self.gerrit_port, port=self.gerrit_port,
identity_file=self.gerrit_sshkey_path, identity_file=self.gerrit_sshkey_path,
generators=[ generators=[
# gerritReviewCB / self.url
BuildStatusGenerator( BuildStatusGenerator(
mode='all', message_formatter=MessageFormatterFunction(
message_formatter=ReasonableMessageFormatter(
lambda data: gerritReviewFmt(self.url, data), lambda data: gerritReviewFmt(self.url, data),
"plain", "plain",
want_properties=True, want_properties=True,
@ -1043,10 +908,11 @@ class GerritNixConfigurator(ConfiguratorBase):
), ),
), ),
]) ])
# startCB, summaryCB are too noisy, we won't use them.
) )
if self.prometheus_config is not None: if self.prometheus_config is not None:
config_dict['services'].append(reporters.Prometheus(port=self.prometheus_config.get('port', 9100), interface=self.prometheus_config.get('address', ''))) config['services'].append(reporters.Prometheus(port=self.prometheus_config.get('port', 9100), interface=self.prometheus_config.get('address', '')))
# Upstream defaults pretend they already do something similar # Upstream defaults pretend they already do something similar
# but they didn't work, hence the custom function. # but they didn't work, hence the custom function.
@ -1056,7 +922,7 @@ class GerritNixConfigurator(ConfiguratorBase):
return ref return ref
return ref.rsplit('/', 1)[0] return ref.rsplit('/', 1)[0]
config_dict["services"].append( config["services"].append(
util.OldBuildCanceller( util.OldBuildCanceller(
"build_canceller", "build_canceller",
filters=[ filters=[
@ -1079,15 +945,12 @@ class GerritNixConfigurator(ConfiguratorBase):
systemd_secrets = secrets.SecretInAFile( systemd_secrets = secrets.SecretInAFile(
dirname=os.environ["CREDENTIALS_DIRECTORY"], dirname=os.environ["CREDENTIALS_DIRECTORY"],
) )
config_dict["secretsProviders"].append(systemd_secrets) config["secretsProviders"].append(systemd_secrets)
config_dict["www"].setdefault("plugins", {}) config["www"].setdefault("plugins", {})
if self.enable_profiler: if "authz" not in config["www"]:
config_dict["www"]["plugins"]["profiler"] = True config["www"]["authz"] = util.Authz(
if "authz" not in config_dict["www"]:
config_dict["www"]["authz"] = util.Authz(
allowRules=[ allowRules=[
util.AnyEndpointMatcher(role="admin", defaultDeny=False), util.AnyEndpointMatcher(role="admin", defaultDeny=False),
util.StopBuildEndpointMatcher(role="owner"), util.StopBuildEndpointMatcher(role="owner"),
@ -1101,5 +964,5 @@ class GerritNixConfigurator(ConfiguratorBase):
], ],
) )
if "auth" not in config_dict["www"] and self.auth_method is not None: if "auth" not in config["www"] and self.auth_method is not None:
config_dict["www"]["auth"] = self.auth_method config["www"]["auth"] = self.auth_method

46
buildbot_nix/message_formatter.py
View file

@ -1,46 +0,0 @@
from typing import Any, Callable, Literal
from buildbot.reporters.message import MessageFormatterBase
import dataclasses
@dataclasses.dataclass
class CallbackPayloadBuild:
# buddy i have no idea what the fuck is in this
build: dict[str, Any]
@dataclasses.dataclass
class CallbackPayloadBuildSet:
buildset: dict[str, Any]
# i have no idea what the fuck is in this honestly
builds: Any
@dataclasses.dataclass
class CallbackReturn:
body: str | None = None
subject: str | None = None
type: Literal['plain'] | Literal['html'] | Literal['json'] = 'plain'
extra_info: dict[str, Any] | None = None
# FIXME: support other template types, if they actually become necessary
template_type: Literal['plain'] = 'plain'
class ReasonableMessageFormatter(MessageFormatterBase):
"""
Message formatter which uses strongly typed data classes to reduce suffering slightly.
"""
CallbackFunc = Callable[[CallbackPayloadBuild | CallbackPayloadBuildSet], CallbackReturn]
def __init__(self, function: CallbackFunc, template_type: str, **kwargs):
super().__init__(**kwargs)
self.template_type = template_type
self._function = function
def format_message_for_build(self, master, build, **kwargs):
return dataclasses.asdict(self._function(CallbackPayloadBuild(build=build)))
def format_message_for_buildset(self, master, buildset, builds, **kwargs):
return dataclasses.asdict(self._function(CallbackPayloadBuildSet(buildset=buildset, builds=builds)))
# These only exist as callbacks, the only one actually used is render_message_dict
def render_message_body(self, context):
return None
def render_message_subject(self, context):
return None

1
default.nix
View file

@ -1,5 +1,4 @@
{ setuptools, buildPythonPackage }: { setuptools, buildPythonPackage }:
# TODO: figure out how to make this depend upon buildbot without leaking it further.
buildPythonPackage { buildPythonPackage {
name = "buildbot-nix"; name = "buildbot-nix";
format = "pyproject"; format = "pyproject";

5
flake.nix
View file

@ -42,11 +42,6 @@
pkgs.ruff pkgs.ruff
]; ];
}; };
devShells.default = pkgs.mkShell {
packages = [
self'.packages.buildbot-nix
];
};
packages.buildbot-nix = pkgs.python3.pkgs.callPackage ./default.nix { }; packages.buildbot-nix = pkgs.python3.pkgs.callPackage ./default.nix { };
checks = checks =
let let

22
nix/coordinator.nix
View file

@ -7,9 +7,6 @@
let let
inherit (lib) filterAttrs; inherit (lib) filterAttrs;
cfg = config.services.buildbot-nix.coordinator; cfg = config.services.buildbot-nix.coordinator;
debuggingManhole = if cfg.debugging.enable then
"manhole.TelnetManhole(${toString cfg.debugging.port}, 'admin', 'admin')"
else "None";
in in
{ {
options = { options = {
@ -31,14 +28,6 @@ in
description = "List of local remote builders machines associated to that Buildbot instance"; description = "List of local remote builders machines associated to that Buildbot instance";
}; };
debugging = {
enable = lib.mkEnableOption "manhole's buildbot debugging on localhost using `admin:admin`";
port = lib.mkOption {
type = lib.types.port;
default = 15000;
};
};
oauth2 = { oauth2 = {
name = lib.mkOption { name = lib.mkOption {
type = lib.types.str; type = lib.types.str;
@ -140,8 +129,6 @@ in
}; };
}; };
profiler.enable = lib.mkEnableOption " the profiling of the buildbot instance";
gerrit = { gerrit = {
domain = lib.mkOption { domain = lib.mkOption {
type = lib.types.str; type = lib.types.str;
@ -229,7 +216,6 @@ in
extraImports = '' extraImports = ''
from datetime import timedelta from datetime import timedelta
from buildbot_nix import GerritNixConfigurator, read_secret_file, make_oauth2_method, OAuth2Config, assemble_secret_file_path from buildbot_nix import GerritNixConfigurator, read_secret_file, make_oauth2_method, OAuth2Config, assemble_secret_file_path
from buildbot import manhole
# TODO(raito): make me configurable from the NixOS module. # TODO(raito): make me configurable from the NixOS module.
# how? # how?
@ -271,9 +257,7 @@ in
auth_method=CustomOAuth2(${builtins.toJSON cfg.oauth2.clientId}, auth_method=CustomOAuth2(${builtins.toJSON cfg.oauth2.clientId},
read_secret_file('buildbot-oauth2-secret'), read_secret_file('buildbot-oauth2-secret'),
autologin=True autologin=True
), )
manhole=${debuggingManhole},
enable_profiler=${builtins.toJSON cfg.profiler.enable}
) )
'' ''
]; ];
@ -291,9 +275,7 @@ in
(ps.toPythonModule pkgs.buildbot-worker) (ps.toPythonModule pkgs.buildbot-worker)
pkgs.buildbot-plugins.www pkgs.buildbot-plugins.www
(pkgs.python3.pkgs.callPackage ../default.nix { }) (pkgs.python3.pkgs.callPackage ../default.nix { })
] ] ++ lib.optional cfg.prometheus.enable (pkgs.python3.pkgs.callPackage ./prometheus-plugin.nix { })
++ lib.optional cfg.prometheus.enable (pkgs.python3.pkgs.callPackage ./prometheus-plugin.nix { })
++ lib.optional cfg.profiler.enable (pkgs.python3.pkgs.callPackage ./profiler-plugin.nix { })
); );
}; };

36
nix/profiler-plugin.nix
View file

@ -1,36 +0,0 @@
{
lib,
buildPythonPackage,
fetchFromGitHub,
setuptools,
wheel,
}:
buildPythonPackage rec {
pname = "buildbot-profiler";
version = "1.2.2";
pyproject = true;
src = fetchFromGitHub {
owner = "tardyp";
repo = "buildbot_profiler";
rev = "v${version}";
hash = "sha256-r56Cze0v3gKWKJwRy0BUYz5z8d0g3jerCIu3KreNxUc=";
};
build-system = [
setuptools
wheel
];
pythonImportsCheck = [
"buildbot_profiler"
];
meta = {
description = "";
homepage = "https://github.com/tardyp/buildbot_profiler";
license = lib.licenses.unfree; # FIXME: nix-init did not find a license
maintainers = with lib.maintainers; [ raitobezarius ];
};
}

1
nix/worker.nix
View file

@ -62,7 +62,6 @@ in
pkgs.openssh pkgs.openssh
pkgs.nix pkgs.nix
pkgs.nix-eval-jobs pkgs.nix-eval-jobs
pkgs.bash
]; ];
environment.PYTHONPATH = "${python.withPackages (_: [cfg.package])}/${python.sitePackages}"; environment.PYTHONPATH = "${python.withPackages (_: [cfg.package])}/${python.sitePackages}";
environment.MASTER_URL = cfg.coordinatorUrl; environment.MASTER_URL = cfg.coordinatorUrl;