buildbot-nix/buildbot_nix/__init__.py

903 lines
33 KiB
Python
Raw Normal View History

2023-09-10 08:11:56 +00:00
import json
import multiprocessing
import os
2023-09-17 20:14:56 +00:00
import sys
2024-03-10 21:27:24 +00:00
import graphlib
2023-09-10 08:11:56 +00:00
from collections import defaultdict
2023-09-17 20:14:56 +00:00
from collections.abc import Generator
from dataclasses import dataclass
2023-09-10 08:11:56 +00:00
from pathlib import Path
2023-12-26 20:56:36 +00:00
from typing import TYPE_CHECKING, Any
2023-12-26 18:49:57 +00:00
2023-09-17 20:14:56 +00:00
from buildbot.configurators import ConfiguratorBase
2023-10-15 06:36:45 +00:00
from buildbot.plugins import reporters, schedulers, secrets, steps, util, worker
2023-09-10 08:11:56 +00:00
from buildbot.process import buildstep, logobserver, remotecommand
2023-09-17 20:14:56 +00:00
from buildbot.process.project import Project
from buildbot.process.properties import Interpolate, Properties
2023-09-10 08:11:56 +00:00
from buildbot.process.results import ALL_RESULTS, statusToString
from buildbot.steps.trigger import Trigger
2023-12-03 10:48:01 +00:00
from buildbot.util import asyncSleep
from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match
from buildbot.www.oauth2 import OAuth2Auth
from buildbot.changes.gerritchangesource import GerritChangeSource
2024-03-10 21:27:24 +00:00
from buildbot.reporters.utils import getURLForBuild
from buildbot.reporters.utils import getURLForBuildrequest
from buildbot.process.buildstep import CANCELLED
from buildbot.process.buildstep import EXCEPTION
from buildbot.process.buildstep import SUCCESS
from buildbot.process.results import worst_status
from buildbot_nix.binary_cache import LocalSigner
2023-10-27 08:49:40 +00:00
2023-12-26 20:56:36 +00:00
if TYPE_CHECKING:
from buildbot.process.log import Log
from twisted.internet import defer, threads
from twisted.logger import Logger
from twisted.python.failure import Failure
from .binary_cache import S3BinaryCacheConfig
2023-12-03 10:48:01 +00:00
from .github_projects import (
2023-10-27 08:49:40 +00:00
slugify_project_name,
2023-10-15 06:36:45 +00:00
)
2023-09-10 08:11:56 +00:00
log = Logger()
class LixSystemsOAuth2(OAuth2Auth):
name = 'Lix'
faIcon = 'fa-login'
resourceEndpoint = "https://identity.lix.systems"
# is passing scope necessary?
authUri = 'https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth'
tokenUri = 'https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token'
2023-09-10 08:11:56 +00:00
2023-12-26 20:56:36 +00:00
class BuildbotNixError(Exception):
pass
@dataclass
class GerritProject:
# `project` field.
name: str
2023-12-26 20:56:36 +00:00
2024-03-10 21:27:24 +00:00
class BuildTrigger(steps.BuildStep):
2023-09-10 08:11:56 +00:00
def __init__(
2023-11-18 07:18:46 +00:00
self,
builds_scheduler_group: str,
2023-11-18 07:18:46 +00:00
jobs: list[dict[str, Any]],
all_deps: dict[str, Any],
2023-11-18 07:18:46 +00:00
**kwargs: Any,
2023-09-10 08:11:56 +00:00
) -> None:
self.jobs = jobs
self.all_deps = all_deps
2023-09-10 08:11:56 +00:00
self.config = None
self.builds_scheduler_group = builds_scheduler_group
self._result_list = []
self.ended = False
self.waitForFinishDeferred = None
self.brids = []
self.description = f"building {len(jobs)} hydra jobs"
super().__init__(**kwargs)
def interrupt(self, reason):
# We cancel the buildrequests, as the data api handles
# both cases:
# - build started: stop is sent,
# - build not created yet: related buildrequests are set to CANCELLED.
# Note that there is an identified race condition though (more details
# are available at buildbot.data.buildrequests).
for brid in self.brids:
self.master.data.control(
"cancel", {'reason': 'parent build was interrupted'}, ("buildrequests", brid)
)
if self.running and not self.ended:
self.ended = True
# if we are interrupted because of a connection lost, we interrupt synchronously
if self.build.conn is None and self.waitForFinishDeferred is not None:
self.waitForFinishDeferred.cancel()
2024-03-10 21:27:24 +00:00
def getSchedulerByName(self, name):
schedulers = self.master.scheduler_manager.namedServices
if name not in schedulers:
raise ValueError(f"unknown triggered scheduler: {repr(name)}")
sch = schedulers[name]
# todo: check ITriggerableScheduler
return sch
def schedule_one(self, build_props, job):
source = f"nix-eval-lix"
2024-03-10 21:27:24 +00:00
attr = job.get("attr", "eval-error")
name = attr
name = f"hydraJobs.{name}"
error = job.get("error")
props = Properties()
props.setProperty("virtual_builder_name", name, source)
props.setProperty("status_name", f"nix-build .#hydraJobs.{attr}", source)
props.setProperty("virtual_builder_tags", "", source)
if error is not None:
props.setProperty("error", error, source)
return (f"{self.builds_scheduler_group}-other", props)
2024-03-10 21:27:24 +00:00
drv_path = job.get("drvPath")
system = job.get("system")
out_path = job.get("outputs", {}).get("out")
build_props.setProperty(f"{attr}-out_path", out_path, source)
build_props.setProperty(f"{attr}-drv_path", drv_path, source)
props.setProperty("attr", attr, source)
props.setProperty("system", system, source)
props.setProperty("drv_path", drv_path, source)
props.setProperty("out_path", out_path, source)
props.setProperty("isCached", job.get("isCached"), source)
return (f"{self.builds_scheduler_group}-{system}", props)
2024-03-10 21:27:24 +00:00
@defer.inlineCallbacks
def _add_results(self, brid):
2024-03-10 21:27:24 +00:00
@defer.inlineCallbacks
def _is_buildrequest_complete(brid):
buildrequest = yield self.master.db.buildrequests.getBuildRequest(brid)
return buildrequest['complete']
event = ('buildrequests', str(brid), 'complete')
yield self.master.mq.waitUntilEvent(event, lambda: _is_buildrequest_complete(brid))
builds = yield self.master.db.builds.getBuilds(buildrequestid=brid)
for build in builds:
self._result_list.append(build["results"])
self.updateSummary()
def prepareSourcestampListForTrigger(self):
ss_for_trigger = {}
objs_from_build = self.build.getAllSourceStamps()
for ss in objs_from_build:
ss_for_trigger[ss.codebase] = ss.asDict()
trigger_values = [ss_for_trigger[k] for k in sorted(ss_for_trigger.keys())]
return trigger_values
2024-03-10 21:27:24 +00:00
@defer.inlineCallbacks
def run(self):
2024-03-10 23:09:27 +00:00
self.running = True
2023-09-10 08:11:56 +00:00
build_props = self.build.getProperties()
source = f"nix-eval-lix"
logs: Log = yield self.addLog("build info")
2023-09-10 08:11:56 +00:00
2024-03-10 21:27:24 +00:00
builds_to_schedule = list(self.jobs)
build_schedule_order = []
sorter = graphlib.TopologicalSorter(self.all_deps)
2024-03-10 21:27:24 +00:00
for item in sorter.static_order():
i = 0
while i < len(builds_to_schedule):
2024-03-10 21:27:24 +00:00
if item == builds_to_schedule[i].get("drvPath"):
build_schedule_order.append(builds_to_schedule[i])
del builds_to_schedule[i]
else:
i += 1
done = []
scheduled = []
failed = []
all_results = SUCCESS
ss_for_trigger = self.prepareSourcestampListForTrigger()
2024-03-10 23:09:27 +00:00
while not self.ended and (len(build_schedule_order) > 0 or len(scheduled) > 0):
2024-03-10 21:27:24 +00:00
schedule_now = []
for build in list(build_schedule_order):
if self.all_deps.get(build.get("drvPath"), []) == []:
2024-03-10 21:27:24 +00:00
build_schedule_order.remove(build)
schedule_now.append(build)
for job in schedule_now:
if job.get('isCached'):
logs.addStdout(f"Cached {job.get('attr')} ({job.get('drvPath')}) - skipping\n")
for dep in self.all_deps:
if job.get("drvPath") in self.all_deps[dep]:
self.all_deps[dep].remove(job.get("drvPath"))
continue
logs.addStdout(f"Scheduling {job.get('attr')} ({job.get('drvPath')})\n")
2024-03-10 21:27:24 +00:00
(scheduler, props) = self.schedule_one(build_props, job)
scheduler = self.getSchedulerByName(scheduler)
idsDeferred, resultsDeferred = scheduler.trigger(
waited_for = True,
sourcestamps = ss_for_trigger,
set_props = props,
parent_buildid = self.build.buildid,
parent_relationship = "Triggered from",
)
brids = {}
try:
_, brids = yield idsDeferred
except Exception as e:
yield self.addLogWithException(e)
results = EXCEPTION
scheduled.append((job, brids, resultsDeferred))
for brid in brids.values():
url = getURLForBuildrequest(self.master, brid)
yield self.addURL(f"{scheduler.name} #{brid}", url)
2024-03-10 21:27:24 +00:00
self._add_results(brid)
self.brids.append(brid)
if len(scheduled) == 0:
if len(build_schedule_order) == 0:
logs.addStderr('Ran out of builds\n')
break
continue
2024-03-10 21:27:24 +00:00
wait_for_next = defer.DeferredList([results for _, _, results in scheduled], fireOnOneCallback = True, fireOnOneErrback=True)
self.waitForFinishDeferred = wait_for_next
2024-03-10 21:27:24 +00:00
results, index = yield wait_for_next
job, brids, _ = scheduled[index]
done.append((job, brids, results))
del scheduled[index]
result = results[0]
logs.addStdout(f'Build {job.get("attr")} ({job.get("drvPath")}) finished, result {util.Results[result].upper()}\n')
if result != SUCCESS:
failed_checks = []
2024-03-10 23:09:27 +00:00
failed_paths = [job.get('drvPath')]
removed = []
failed.append((
job.get("attr"),
"failed",
[ getURLForBuildrequest(self.master, brid) for brid in brids.values() ]
))
while True:
old_paths = list(failed_paths)
for build in list(build_schedule_order):
deps = self.all_deps.get(build.get("drvPath"), [])
for path in old_paths:
if path in deps:
failed_checks.append(build)
failed_paths.append(build.get("drvPath"))
build_schedule_order.remove(build)
removed.append(build.get("attr"))
failed.append((build.get("attr"), f"dependency {job.get('attr')} failed", []))
break
if old_paths == failed_paths:
break
if len(removed) > 3:
yield logs.addStdout(' Skipping jobs: ' + ', '.join(removed[:3]) + f', ... ({len(removed) - 3} more)\n')
else:
yield logs.addStdout(' Skipping jobs: ' + ', '.join(removed) + '\n')
all_results = worst_status(result, all_results)
for dep in self.all_deps:
if job.get("drvPath") in self.all_deps[dep]:
self.all_deps[dep].remove(job.get("drvPath"))
yield logs.addHeader('Done!\n')
yield logs.finish()
build_props.setProperty("failed_builds", failed, "nix-eval-lix")
2024-03-10 23:09:27 +00:00
if self.ended:
return util.CANCELLED
return all_results
2023-09-10 08:11:56 +00:00
def getCurrentSummary(self) -> dict[str, str]: # noqa: N802
2023-09-10 08:11:56 +00:00
summary = []
if self._result_list:
for status in ALL_RESULTS:
count = self._result_list.count(status)
if count:
summary.append(
2023-12-26 20:56:36 +00:00
f"{self._result_list.count(status)} {statusToString(status, count)}",
2023-09-10 08:11:56 +00:00
)
return {"step": f"({', '.join(summary)})"}
class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
2023-12-26 20:56:36 +00:00
"""Parses the output of `nix-eval-jobs` and triggers a `nix-build` build for
2023-09-10 08:11:56 +00:00
every attribute.
"""
def __init__(self, supported_systems: list[str], **kwargs: Any) -> None:
kwargs = self.setupShellMixin(kwargs)
super().__init__(**kwargs)
self.observer = logobserver.BufferLogObserver()
self.addLogObserver("stdio", self.observer)
self.supported_systems = supported_systems
@defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]:
2024-03-08 23:28:49 +00:00
# run nix-eval-jobs --flake .#hydraJobs to generate the dict of stages
2023-09-10 08:11:56 +00:00
cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand()
yield self.runCommand(cmd)
# if the command passes extract the list of stages
result = cmd.results()
if result == util.SUCCESS:
# create a ShellCommand for each stage and add them to the build
jobs = []
for line in self.observer.getStdout().split("\n"):
if line != "":
try:
job = json.loads(line)
except json.JSONDecodeError as e:
2023-12-26 20:56:36 +00:00
msg = f"Failed to parse line: {line}"
raise BuildbotNixError(msg) from e
2023-09-10 08:11:56 +00:00
jobs.append(job)
build_props = self.build.getProperties()
filtered_jobs = []
for job in jobs:
system = job.get("system")
2023-12-26 20:56:36 +00:00
if not system or system in self.supported_systems: # report eval errors
2023-09-10 08:11:56 +00:00
filtered_jobs.append(job)
drv_show_log: Log = yield self.getLog("stdio")
drv_show_log.addStdout(f"getting derivation infos\n")
cmd = yield self.makeRemoteShellCommand(
stdioLogName=None,
collectStdout=True,
command=(
["nix", "derivation", "show", "--recursive"]
+ [ drv for drv in (job.get("drvPath") for job in filtered_jobs) if drv ]
),
)
yield self.runCommand(cmd)
drv_show_log.addStdout(f"done\n")
try:
drv_info = json.loads(cmd.stdout)
except json.JSONDecodeError as e:
msg = f"Failed to parse `nix derivation show` output for {cmd.command}"
raise BuildbotNixError(msg) from e
all_deps = dict()
for drv, info in drv_info.items():
all_deps[drv] = set(info.get("inputDrvs").keys())
def closure_of(key, deps):
r, size = set([key]), 0
while len(r) != size:
size = len(r)
r.update(*[ deps[k] for k in r ])
return r.difference([key])
job_set = set(( drv for drv in ( job.get("drvPath") for job in filtered_jobs ) if drv ))
all_deps = { k: list(closure_of(k, all_deps).intersection(job_set)) for k in job_set }
2023-09-10 08:11:56 +00:00
self.build.addStepsAfterCurrentStep(
2023-10-27 09:39:11 +00:00
[
BuildTrigger(
builds_scheduler_group=f"lix-nix-build",
2023-11-18 07:18:46 +00:00
name="build flake",
jobs=filtered_jobs,
all_deps=all_deps,
2023-12-26 20:56:36 +00:00
),
],
2023-09-10 08:11:56 +00:00
)
return result
class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep):
2023-12-26 20:56:36 +00:00
"""Builds a nix derivation."""
2023-09-10 08:11:56 +00:00
def __init__(self, **kwargs: Any) -> None:
kwargs = self.setupShellMixin(kwargs)
super().__init__(**kwargs)
@defer.inlineCallbacks
def run(self) -> Generator[Any, object, Any]:
if error := self.getProperty("error"):
attr = self.getProperty("attr")
# show eval error
error_log: Log = yield self.addLog("nix_error")
error_log.addStderr(f"{attr} failed to evaluate:\n{error}")
return util.FAILURE
if self.getProperty("isCached"):
yield self.addCompleteLog(
"cached outpath from previous builds",
# buildbot apparently hides the first line in the ui?
f'\n{self.getProperty("out_path")}\n')
return util.SKIPPED
2023-09-10 08:11:56 +00:00
# run `nix build`
cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand()
yield self.runCommand(cmd)
return cmd.results()
2023-09-10 08:11:56 +00:00
class UpdateBuildOutput(steps.BuildStep):
2023-12-26 20:56:36 +00:00
"""Updates store paths in a public www directory.
2023-09-10 08:11:56 +00:00
This is useful to prefetch updates without having to evaluate
on the target machine.
"""
2023-11-04 08:50:23 +00:00
def __init__(self, path: Path, **kwargs: Any) -> None:
2023-09-10 08:11:56 +00:00
super().__init__(**kwargs)
2023-11-04 08:50:23 +00:00
self.path = path
2023-09-10 08:11:56 +00:00
def run(self) -> Generator[Any, object, Any]:
props = self.build.getProperties()
if props.getProperty("branch") != props.getProperty(
2023-12-26 20:56:36 +00:00
"github.repository.default_branch",
2023-09-10 08:11:56 +00:00
):
return util.SKIPPED
2023-11-17 15:27:55 +00:00
2023-12-26 20:56:36 +00:00
attr = Path(props.getProperty("attr")).name
2023-09-10 08:11:56 +00:00
out_path = props.getProperty("out_path")
# XXX don't hardcode this
2023-11-04 08:50:23 +00:00
self.path.mkdir(parents=True, exist_ok=True)
(self.path / attr).write_text(out_path)
2023-09-10 08:11:56 +00:00
return util.SUCCESS
def nix_eval_config(
project: GerritProject,
gerrit_private_key: str,
2023-09-10 08:11:56 +00:00
worker_names: list[str],
supported_systems: list[str],
eval_lock: util.MasterLock,
2023-11-12 06:07:30 +00:00
worker_count: int,
max_memory_size: int,
2023-09-10 08:11:56 +00:00
) -> util.BuilderConfig:
2023-12-26 20:56:36 +00:00
"""Uses nix-eval-jobs to evaluate hydraJobs from flake.nix in parallel.
2023-09-10 08:11:56 +00:00
For each evaluated attribute a new build pipeline is started.
"""
factory = util.BuildFactory()
# check out the source
factory.addStep(
steps.Gerrit(
repourl="ssh://buildbot@gerrit.lix.systems:2022/lix",
mode="full",
retry=[60, 60],
timeout=3600,
sshPrivateKey=gerrit_private_key
2023-12-26 20:56:36 +00:00
),
2023-09-10 08:11:56 +00:00
)
# use one gcroots directory per worker. this should be scoped to the largest unique resource
# in charge of builds (ie, buildnumber is too narrow) to not litter the system with permanent
# gcroots in case of worker restarts.
# TODO perhaps we should clean the entire /drvs/ directory up too during startup.
2024-01-15 12:41:48 +00:00
drv_gcroots_dir = util.Interpolate(
"/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/drvs/%(prop:workername)s/",
2024-01-15 12:41:48 +00:00
)
2023-09-10 08:11:56 +00:00
factory.addStep(
NixEvalCommand(
env={},
name="evaluate flake",
supported_systems=supported_systems,
command=[
2023-10-15 09:26:09 +00:00
"nix-eval-jobs",
2023-09-10 08:11:56 +00:00
"--workers",
2023-11-12 06:07:30 +00:00
str(worker_count),
2023-09-10 08:11:56 +00:00
"--max-memory-size",
str(max_memory_size),
"--option",
"accept-flake-config",
"true",
"--gc-roots-dir",
2024-01-15 12:41:48 +00:00
drv_gcroots_dir,
2023-09-10 08:11:56 +00:00
"--force-recurse",
2023-11-17 15:27:55 +00:00
"--check-cache-status",
2023-09-10 08:11:56 +00:00
"--flake",
2024-03-08 23:28:49 +00:00
".#hydraJobs",
2023-09-10 08:11:56 +00:00
],
haltOnFailure=True,
locks=[eval_lock.access("exclusive")],
2023-12-26 20:56:36 +00:00
),
2023-09-10 08:11:56 +00:00
)
2024-01-15 12:41:48 +00:00
factory.addStep(
steps.ShellCommand(
name="Cleanup drv paths",
command=[
"rm",
"-rf",
drv_gcroots_dir,
],
alwaysRun=True,
2024-01-15 12:41:48 +00:00
),
)
2023-09-10 08:11:56 +00:00
return util.BuilderConfig(
name=f"{project.name}/nix-eval",
workernames=worker_names,
project=project.name,
factory=factory,
properties=dict(status_name="nix-eval"),
)
def nix_build_config(
project: GerritProject,
worker_arch: str,
2023-09-10 08:11:56 +00:00
worker_names: list[str],
2023-11-04 08:50:23 +00:00
outputs_path: Path | None = None,
signing_keyfile: str | None = None,
binary_cache_config: S3BinaryCacheConfig | None = None
2023-09-10 08:11:56 +00:00
) -> util.BuilderConfig:
2023-12-26 20:56:36 +00:00
"""Builds one nix flake attribute."""
2023-09-10 08:11:56 +00:00
factory = util.BuildFactory()
factory.addStep(
NixBuildCommand(
env={},
name="Build flake attr",
command=[
"nix",
"build",
"-L",
"--option",
"keep-going",
"true",
# do not build directly on the coordinator
"--max-jobs", "0",
"--option",
# stop stuck builds after 20 minutes
"--max-silent-time",
str(60 * 20),
2023-09-10 08:11:56 +00:00
"--accept-flake-config",
"--out-link",
util.Interpolate("result-%(prop:attr)s"),
util.Interpolate("%(prop:drv_path)s^*"),
],
# 3 hours, defaults to 20 minutes
# We increase this over the default since the build output might end up in a different `nix build`.
timeout=60 * 60 * 3,
2023-09-10 08:11:56 +00:00
haltOnFailure=True,
2023-12-26 20:56:36 +00:00
),
2023-09-10 08:11:56 +00:00
)
2023-10-09 15:13:46 +00:00
if signing_keyfile is not None:
factory.addStep(
steps.ShellCommand(
name="Sign the store path",
command=[
"nix",
"store",
"sign",
"--key-file",
signing_keyfile,
util.Interpolate(
"%(prop:drv_path)s^*"
)
]
),
)
if binary_cache_config is not None:
factory.addStep(
steps.ShellCommand(
name="Upload the store path to the cache",
command=[
"nix",
"copy",
"--to",
f"s3://{binary_cache_config.bucket}?profile={binary_cache_config.profile}&region={binary_cache_config.region}&endpoint={binary_cache_config.endpoint}",
util.Property(
"out_path"
)
]
)
)
2023-09-10 08:11:56 +00:00
factory.addStep(
steps.ShellCommand(
name="Register gcroot",
command=[
"nix-store",
"--add-root",
# FIXME: cleanup old build attributes
util.Interpolate(
2023-12-26 20:56:36 +00:00
"/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/%(prop:attr)s",
2023-09-10 08:11:56 +00:00
),
"-r",
util.Property("out_path"),
],
2023-11-18 07:18:46 +00:00
doStepIf=lambda s: s.getProperty("branch")
2023-10-12 10:43:57 +00:00
== s.getProperty("github.repository.default_branch"),
2023-12-26 20:56:36 +00:00
),
2023-09-10 08:11:56 +00:00
)
factory.addStep(
steps.ShellCommand(
name="Delete temporary gcroots",
command=["rm", "-f", util.Interpolate("result-%(prop:attr)s")],
2023-12-26 20:56:36 +00:00
),
2023-09-10 08:11:56 +00:00
)
2023-11-04 08:50:23 +00:00
if outputs_path is not None:
factory.addStep(
UpdateBuildOutput(
name="Update build output",
path=outputs_path,
2023-12-26 20:56:36 +00:00
),
2023-11-04 08:50:23 +00:00
)
2023-09-10 08:11:56 +00:00
return util.BuilderConfig(
name=f"{project.name}/nix-build/{worker_arch}",
2023-09-10 08:11:56 +00:00
project=project.name,
workernames=worker_names,
collapseRequests=False,
env={},
factory=factory,
)
2023-09-17 20:14:56 +00:00
def read_secret_file(secret_name: str) -> str:
directory = os.environ.get("CREDENTIALS_DIRECTORY")
if directory is None:
print("directory not set", file=sys.stderr)
sys.exit(1)
2023-10-27 09:09:20 +00:00
return Path(directory).joinpath(secret_name).read_text().rstrip()
2023-09-17 20:14:56 +00:00
def config_for_project(
config: dict[str, Any],
project: GerritProject,
2023-09-17 20:14:56 +00:00
worker_names: list[str],
nix_supported_systems: list[str],
2023-11-12 06:07:30 +00:00
nix_eval_worker_count: int,
2023-09-17 20:14:56 +00:00
nix_eval_max_memory_size: int,
eval_lock: util.MasterLock,
2023-11-04 08:50:23 +00:00
outputs_path: Path | None = None,
signing_keyfile: str | None = None,
binary_cache_config: S3BinaryCacheConfig | None = None
2023-09-17 20:14:56 +00:00
) -> Project:
config["projects"].append(Project(project.name))
config["schedulers"].extend(
[
# build everything pertaining to a project
# TODO(raito): will this catch also post-merge? we don't really care about that… do we?
2023-09-17 20:14:56 +00:00
schedulers.SingleBranchScheduler(
name=f"{project.name}-changes",
2023-09-17 20:14:56 +00:00
change_filter=util.ChangeFilter(
project=project.name,
2023-09-17 20:14:56 +00:00
),
builderNames=[f"{project.name}/nix-eval"],
),
# this is triggered from `nix-eval`
*(
schedulers.Triggerable(
name=f"{project.name}-nix-build-{arch}",
builderNames=[f"{project.name}/nix-build/{arch}"],
)
for arch in nix_supported_systems + [ "other" ]
2023-09-17 20:14:56 +00:00
),
# allow to manually trigger a nix-build
schedulers.ForceScheduler(
name=f"{project.name}-force",
2023-10-27 08:49:40 +00:00
builderNames=[f"{project.name}/nix-eval"],
2023-10-27 08:35:26 +00:00
properties=[
util.StringParameter(
name="project",
label="Name of the Gerrit repository.",
2023-10-27 08:49:40 +00:00
default=project.name,
2023-12-26 20:56:36 +00:00
),
2023-10-27 08:49:40 +00:00
],
2023-09-17 20:14:56 +00:00
),
2023-12-26 20:56:36 +00:00
],
2023-09-17 20:14:56 +00:00
)
gerrit_private_key = None
with open('/var/lib/buildbot/master/id_gerrit', 'r') as f:
gerrit_private_key = f.read()
if gerrit_private_key is None:
raise RuntimeError('No gerrit private key to fetch the repositories')
2023-09-17 20:14:56 +00:00
config["builders"].extend(
[
# Since all workers run on the same machine, we only assign one of them to do the evaluation.
# This should prevent exessive memory usage.
nix_eval_config(
project,
gerrit_private_key,
[ f"{w}-other" for w in worker_names ],
2023-09-17 20:14:56 +00:00
supported_systems=nix_supported_systems,
2023-11-12 06:07:30 +00:00
worker_count=nix_eval_worker_count,
2023-09-17 20:14:56 +00:00
max_memory_size=nix_eval_max_memory_size,
eval_lock=eval_lock,
2023-09-17 20:14:56 +00:00
),
*(
nix_build_config(
project,
arch,
[ f"{w}-{arch}" for w in worker_names ],
outputs_path=outputs_path,
signing_keyfile=signing_keyfile,
binary_cache_config=binary_cache_config
)
for arch in nix_supported_systems + [ "other" ]
2023-09-17 20:14:56 +00:00
),
2023-12-26 20:56:36 +00:00
],
2023-09-17 20:14:56 +00:00
)
class PeriodicWithStartup(schedulers.Periodic):
def __init__(self, *args: Any, run_on_startup: bool = False, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.run_on_startup = run_on_startup
@defer.inlineCallbacks
def activate(self) -> Generator[Any, object, Any]:
if self.run_on_startup:
yield self.setState("last_build", None)
yield super().activate()
def gerritReviewCB(builderName, build, result, master, arg):
if result == util.RETRY:
return dict()
if builderName != 'lix/nix-eval':
return dict()
failed = build['properties'].get('failed_builds', [[]])[0]
labels = {
'Verified': -1 if result != util.SUCCESS else 1,
}
message = "Buildbot finished compiling your patchset!\n"
message += "The result is: %s\n" % util.Results[result].upper()
if result != util.SUCCESS:
message += "\nFailed checks:\n"
for check, how, urls in failed:
if not urls:
message += " "
message += f" - {check}: {how}"
if urls:
message += f" (see {', '.join(urls)})"
message += "\n"
if arg:
message += "\nFor more details visit:\n"
message += build['url'] + "\n"
return dict(message=message, labels=labels)
def gerritStartCB(builderName, build, arg):
message = "Buildbot started compiling your patchset\n"
message += "on configuration: %s\n" % builderName
message += "See your build here: %s" % build['url']
return dict(message=message)
def gerritSummaryCB(buildInfoList, results, status, arg):
success = False
failure = False
msgs = []
for buildInfo in buildInfoList:
msg = "Builder %(name)s %(resultText)s (%(text)s)" % buildInfo
link = buildInfo.get('url', None)
if link:
msg += " - " + link
else:
msg += "."
msgs.append(msg)
if buildInfo['result'] == util.SUCCESS:
success = True
else:
failure = True
if success and not failure:
verified = 1
else:
verified = -1
return dict(message='\n\n'.join(msgs),
labels={
'Verified': verified
})
class GerritNixConfigurator(ConfiguratorBase):
2023-09-17 20:14:56 +00:00
"""Janitor is a configurator which create a Janitor Builder with all needed Janitor steps"""
def __init__(
self,
2023-12-26 20:56:36 +00:00
# Shape of this file: [ { "name": "<worker-name>", "pass": "<worker-password>", "cores": "<cpu-cores>" } ]
gerrit_server: str,
gerrit_user: str,
gerrit_port: int,
gerrit_sshkey_path: str,
2023-10-12 13:59:26 +00:00
url: str,
2023-09-17 20:14:56 +00:00
nix_supported_systems: list[str],
2023-11-12 06:07:30 +00:00
nix_eval_worker_count: int | None,
nix_eval_max_memory_size: int,
2023-12-26 20:56:36 +00:00
nix_workers_secret_name: str = "buildbot-nix-workers", # noqa: S107
signing_keyfile: str | None = None,
binary_cache_config: dict[str, str] | None = None,
2023-11-04 08:50:23 +00:00
outputs_path: str | None = None,
2023-09-17 20:14:56 +00:00
) -> None:
super().__init__()
self.gerrit_server = gerrit_server
self.gerrit_user = gerrit_user
self.gerrit_port = gerrit_port
2023-09-17 20:14:56 +00:00
self.nix_workers_secret_name = nix_workers_secret_name
self.nix_eval_max_memory_size = nix_eval_max_memory_size
2023-11-12 06:07:30 +00:00
self.nix_eval_worker_count = nix_eval_worker_count
2023-09-17 20:14:56 +00:00
self.nix_supported_systems = nix_supported_systems
self.gerrit_change_source = GerritChangeSource(gerrit_server, gerrit_user, gerritport=gerrit_port, identity_file=gerrit_sshkey_path)
2023-10-12 13:59:26 +00:00
self.url = url
if binary_cache_config is not None:
self.binary_cache_config = S3BinaryCacheConfig(**binary_cache_config)
else:
self.binary_cache_config = None
self.signing_keyfile = signing_keyfile
2023-11-04 08:50:23 +00:00
if outputs_path is None:
self.outputs_path = None
else:
self.outputs_path = Path(outputs_path)
2023-09-17 20:14:56 +00:00
def configure(self, config: dict[str, Any]) -> None:
worker_config = json.loads(read_secret_file(self.nix_workers_secret_name))
worker_names = []
config.setdefault("projects", [])
config.setdefault("secretsProviders", [])
config.setdefault("www", {})
2023-09-17 20:14:56 +00:00
for item in worker_config:
cores = item.get("cores", 0)
for i in range(cores):
for arch in self.nix_supported_systems + ["other"]:
worker_name = f"{item['name']}-{i:03}"
config["workers"].append(worker.Worker(f"{worker_name}-{arch}", item["pass"]))
worker_names.append(worker_name)
2023-09-17 20:14:56 +00:00
eval_lock = util.MasterLock("nix-eval")
2023-10-12 13:59:26 +00:00
# Configure the Lix project.
config_for_project(
config,
GerritProject(name="lix"),
worker_names,
self.nix_supported_systems,
self.nix_eval_worker_count or multiprocessing.cpu_count(),
self.nix_eval_max_memory_size,
eval_lock,
self.outputs_path,
signing_keyfile=self.signing_keyfile,
binary_cache_config=self.binary_cache_config
)
2023-10-12 13:59:26 +00:00
config["change_source"] = self.gerrit_change_source
config["services"].append(
reporters.GerritStatusPush(self.gerrit_server, self.gerrit_user,
port=2022,
identity_file='/var/lib/buildbot/master/id_gerrit',
summaryCB=None,
startCB=None,
wantSteps=True,
reviewCB=gerritReviewCB,
reviewArg=self.url)
# startCB=gerritStartCB,
# startArg=self.url,
# summaryCB=gerritSummaryCB,
# summaryArg=self.url)
)
2024-03-28 02:52:13 +00:00
def gerritBranchKey(b):
ref = b['branch']
if not ref.startswith('refs/changes/'):
return ref
return ref.rsplit('/', 1)[0]
config["services"].append(
util.OldBuildCanceller(
"lix_build_canceller",
filters=[
(
[
f"lix/nix-{kind}"
for kind in [ "eval" ] + [
f"build/{arch}"
for arch in self.nix_supported_systems + [ "other" ]
]
],
util.SourceStampFilter(project_eq=["lix"])
)
],
branch_key=gerritBranchKey
)
)
2023-09-17 20:14:56 +00:00
systemd_secrets = secrets.SecretInAFile(
2023-12-26 20:56:36 +00:00
dirname=os.environ["CREDENTIALS_DIRECTORY"],
2023-09-17 20:14:56 +00:00
)
config["secretsProviders"].append(systemd_secrets)
config["www"].setdefault("plugins", {})
if "auth" not in config["www"]:
config["www"]["auth"] = LixSystemsOAuth2('buildbot', read_secret_file('buildbot-oauth2-secret'), autologin=True)