better gerrit integration #5
|
@ -2,7 +2,7 @@ import json
|
||||||
import multiprocessing
|
import multiprocessing
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import uuid
|
import graphlib
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from collections.abc import Generator
|
from collections.abc import Generator
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
@ -20,6 +20,12 @@ from buildbot.util import asyncSleep
|
||||||
from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match
|
from buildbot.www.authz.endpointmatchers import EndpointMatcherBase, Match
|
||||||
from buildbot.www.oauth2 import OAuth2Auth
|
from buildbot.www.oauth2 import OAuth2Auth
|
||||||
from buildbot.changes.gerritchangesource import GerritChangeSource
|
from buildbot.changes.gerritchangesource import GerritChangeSource
|
||||||
|
from buildbot.reporters.utils import getURLForBuild
|
||||||
|
from buildbot.reporters.utils import getURLForBuildrequest
|
||||||
|
from buildbot.process.buildstep import CANCELLED
|
||||||
|
from buildbot.process.buildstep import EXCEPTION
|
||||||
|
from buildbot.process.buildstep import SUCCESS
|
||||||
|
from buildbot.process.results import worst_status
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from buildbot.process.log import Log
|
from buildbot.process.log import Log
|
||||||
|
@ -32,8 +38,6 @@ from .github_projects import (
|
||||||
slugify_project_name,
|
slugify_project_name,
|
||||||
)
|
)
|
||||||
|
|
||||||
SKIPPED_BUILDER_NAME = "skipped-builds"
|
|
||||||
|
|
||||||
log = Logger()
|
log = Logger()
|
||||||
|
|
||||||
class LixSystemsOAuth2(OAuth2Auth):
|
class LixSystemsOAuth2(OAuth2Auth):
|
||||||
|
@ -52,82 +56,206 @@ class GerritProject:
|
||||||
# `project` field.
|
# `project` field.
|
||||||
name: str
|
name: str
|
||||||
|
|
||||||
class BuildTrigger(Trigger):
|
class BuildTrigger(steps.BuildStep):
|
||||||
"""Dynamic trigger that creates a build for every attribute."""
|
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
builds_scheduler: str,
|
builds_scheduler: str,
|
||||||
skipped_builds_scheduler: str,
|
|
||||||
jobs: list[dict[str, Any]],
|
jobs: list[dict[str, Any]],
|
||||||
|
all_deps: dict[str, Any],
|
||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
) -> None:
|
) -> None:
|
||||||
if "name" not in kwargs:
|
|
||||||
kwargs["name"] = "trigger"
|
|
||||||
self.jobs = jobs
|
self.jobs = jobs
|
||||||
|
self.all_deps = all_deps
|
||||||
self.config = None
|
self.config = None
|
||||||
self.builds_scheduler = builds_scheduler
|
self.builds_scheduler = builds_scheduler
|
||||||
self.skipped_builds_scheduler = skipped_builds_scheduler
|
self._result_list = []
|
||||||
Trigger.__init__(
|
self.ended = False
|
||||||
self,
|
self.waitForFinishDeferred = None
|
||||||
waitForFinish=True,
|
self.brids = []
|
||||||
schedulerNames=[builds_scheduler, skipped_builds_scheduler],
|
self.description = f"building {len(jobs)} hydra jobs"
|
||||||
haltOnFailure=True,
|
super().__init__(**kwargs)
|
||||||
flunkOnFailure=True,
|
|
||||||
sourceStamps=[],
|
|
||||||
alwaysUseLatest=False,
|
|
||||||
updateSourceStamp=False,
|
|
||||||
**kwargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
def createTriggerProperties(self, props: Any) -> Any: # noqa: N802
|
def interrupt(self, reason):
|
||||||
return props
|
# We cancel the buildrequests, as the data api handles
|
||||||
|
# both cases:
|
||||||
|
# - build started: stop is sent,
|
||||||
|
# - build not created yet: related buildrequests are set to CANCELLED.
|
||||||
|
# Note that there is an identified race condition though (more details
|
||||||
|
# are available at buildbot.data.buildrequests).
|
||||||
|
for brid in self.brids:
|
||||||
|
self.master.data.control(
|
||||||
|
"cancel", {'reason': 'parent build was interrupted'}, ("buildrequests", brid)
|
||||||
|
)
|
||||||
|
if self.running and not self.ended:
|
||||||
|
self.ended = True
|
||||||
|
# if we are interrupted because of a connection lost, we interrupt synchronously
|
||||||
|
if self.build.conn is None and self.waitForFinishDeferred is not None:
|
||||||
|
self.waitForFinishDeferred.cancel()
|
||||||
|
|
||||||
def getSchedulersAndProperties(self) -> list[tuple[str, Properties]]: # noqa: N802
|
def getSchedulerByName(self, name):
|
||||||
|
schedulers = self.master.scheduler_manager.namedServices
|
||||||
|
if name not in schedulers:
|
||||||
|
raise ValueError(f"unknown triggered scheduler: {repr(name)}")
|
||||||
|
sch = schedulers[name]
|
||||||
|
# todo: check ITriggerableScheduler
|
||||||
|
return sch
|
||||||
|
|
||||||
|
def schedule_one(self, build_props, job):
|
||||||
|
source = f"nix-eval-lix"
|
||||||
|
attr = job.get("attr", "eval-error")
|
||||||
|
name = attr
|
||||||
|
name = f"hydraJobs.{name}"
|
||||||
|
error = job.get("error")
|
||||||
|
props = Properties()
|
||||||
|
props.setProperty("virtual_builder_name", name, source)
|
||||||
|
props.setProperty("status_name", f"nix-build .#hydraJobs.{attr}", source)
|
||||||
|
props.setProperty("virtual_builder_tags", "", source)
|
||||||
|
|
||||||
|
if error is not None:
|
||||||
|
props.setProperty("error", error, source)
|
||||||
|
return (self.builds_scheduler, props)
|
||||||
|
|
||||||
|
drv_path = job.get("drvPath")
|
||||||
|
system = job.get("system")
|
||||||
|
out_path = job.get("outputs", {}).get("out")
|
||||||
|
|
||||||
|
build_props.setProperty(f"{attr}-out_path", out_path, source)
|
||||||
|
build_props.setProperty(f"{attr}-drv_path", drv_path, source)
|
||||||
|
|
||||||
|
props.setProperty("attr", attr, source)
|
||||||
|
props.setProperty("system", system, source)
|
||||||
|
props.setProperty("drv_path", drv_path, source)
|
||||||
|
props.setProperty("out_path", out_path, source)
|
||||||
|
props.setProperty("isCached", job.get("isCached"), source)
|
||||||
|
|
||||||
|
return (self.builds_scheduler, props)
|
||||||
|
|
||||||
|
@defer.inlineCallbacks
|
||||||
|
def _add_results(self, brid):
|
||||||
|
@defer.inlineCallbacks
|
||||||
|
def _is_buildrequest_complete(brid):
|
||||||
|
buildrequest = yield self.master.db.buildrequests.getBuildRequest(brid)
|
||||||
|
return buildrequest['complete']
|
||||||
|
|
||||||
|
event = ('buildrequests', str(brid), 'complete')
|
||||||
|
yield self.master.mq.waitUntilEvent(event, lambda: _is_buildrequest_complete(brid))
|
||||||
|
builds = yield self.master.db.builds.getBuilds(buildrequestid=brid)
|
||||||
|
for build in builds:
|
||||||
|
self._result_list.append(build["results"])
|
||||||
|
self.updateSummary()
|
||||||
|
|
||||||
|
def prepareSourcestampListForTrigger(self):
|
||||||
|
ss_for_trigger = {}
|
||||||
|
objs_from_build = self.build.getAllSourceStamps()
|
||||||
|
for ss in objs_from_build:
|
||||||
|
ss_for_trigger[ss.codebase] = ss.asDict()
|
||||||
|
|
||||||
|
trigger_values = [ss_for_trigger[k] for k in sorted(ss_for_trigger.keys())]
|
||||||
|
return trigger_values
|
||||||
|
|
||||||
|
@defer.inlineCallbacks
|
||||||
|
def run(self):
|
||||||
|
self.running = True
|
||||||
build_props = self.build.getProperties()
|
build_props = self.build.getProperties()
|
||||||
source = f"nix-eval-lix"
|
source = f"nix-eval-lix"
|
||||||
|
|
||||||
triggered_schedulers = []
|
builds_to_schedule = list(self.jobs)
|
||||||
for job in self.jobs:
|
build_schedule_order = []
|
||||||
attr = job.get("attr", "eval-error")
|
sorter = graphlib.TopologicalSorter(self.all_deps)
|
||||||
name = attr
|
for item in sorter.static_order():
|
||||||
name = f"hydraJobs.{name}"
|
i = 0
|
||||||
error = job.get("error")
|
while i < len(builds_to_schedule):
|
||||||
props = Properties()
|
if item == builds_to_schedule[i].get("drvPath"):
|
||||||
props.setProperty("virtual_builder_name", name, source)
|
build_schedule_order.append(builds_to_schedule[i])
|
||||||
props.setProperty("status_name", f"nix-build .#hydraJobs.{attr}", source)
|
del builds_to_schedule[i]
|
||||||
props.setProperty("virtual_builder_tags", "", source)
|
else:
|
||||||
|
i += 1
|
||||||
|
|
||||||
if error is not None:
|
done = []
|
||||||
props.setProperty("error", error, source)
|
scheduled = []
|
||||||
triggered_schedulers.append((self.skipped_builds_scheduler, props))
|
failed = {}
|
||||||
continue
|
all_results = SUCCESS
|
||||||
|
ss_for_trigger = self.prepareSourcestampListForTrigger()
|
||||||
|
while not self.ended and (len(build_schedule_order) > 0 or len(scheduled) > 0):
|
||||||
|
print('Scheduling..')
|
||||||
|
schedule_now = []
|
||||||
|
for build in list(build_schedule_order):
|
||||||
|
if self.all_deps.get(build.get("drvPath"), []) == []:
|
||||||
|
build_schedule_order.remove(build)
|
||||||
|
schedule_now.append(build)
|
||||||
|
if len(schedule_now) == 0:
|
||||||
|
print(' No builds to schedule found.')
|
||||||
|
for job in schedule_now:
|
||||||
|
print(f" - {job.get('attr')}")
|
||||||
|
(scheduler, props) = self.schedule_one(build_props, job)
|
||||||
|
scheduler = self.getSchedulerByName(scheduler)
|
||||||
|
|
||||||
if job.get("isCached"):
|
idsDeferred, resultsDeferred = scheduler.trigger(
|
||||||
triggered_schedulers.append((self.skipped_builds_scheduler, props))
|
waited_for = True,
|
||||||
continue
|
sourcestamps = ss_for_trigger,
|
||||||
|
set_props = props,
|
||||||
|
parent_buildid = self.build.buildid,
|
||||||
|
parent_relationship = "Triggered from",
|
||||||
|
)
|
||||||
|
|
||||||
drv_path = job.get("drvPath")
|
brids = {}
|
||||||
system = job.get("system")
|
try:
|
||||||
out_path = job.get("outputs", {}).get("out")
|
_, brids = yield idsDeferred
|
||||||
|
except Exception as e:
|
||||||
|
yield self.addLogWithException(e)
|
||||||
|
results = EXCEPTION
|
||||||
|
scheduled.append((job, brids, resultsDeferred))
|
||||||
|
|
||||||
build_props.setProperty(f"{attr}-out_path", out_path, source)
|
for brid in brids.values():
|
||||||
build_props.setProperty(f"{attr}-drv_path", drv_path, source)
|
url = getURLForBuildrequest(self.master, brid)
|
||||||
|
yield self.addURL(f"{scheduler.name} #{brid}", url)
|
||||||
|
self._add_results(brid)
|
||||||
|
self.brids.append(brid)
|
||||||
|
print('Waiting..')
|
||||||
|
wait_for_next = defer.DeferredList([results for _, _, results in scheduled], fireOnOneCallback = True, fireOnOneErrback=True)
|
||||||
|
self.waitForFinishDeferred = wait_for_next
|
||||||
|
results, index = yield wait_for_next
|
||||||
|
job, brids, _ = scheduled[index]
|
||||||
|
done.append((job, brids, results))
|
||||||
|
del scheduled[index]
|
||||||
|
result = results[0]
|
||||||
|
print(f' Found finished build {job.get("attr")}, result {util.Results[result].upper()}')
|
||||||
|
if result != SUCCESS:
|
||||||
|
failed_checks = []
|
||||||
|
failed_paths = [job.get('drvPath')]
|
||||||
|
removed = []
|
||||||
|
while True:
|
||||||
|
old_paths = list(failed_paths)
|
||||||
|
for build in list(build_schedule_order):
|
||||||
|
deps = self.all_deps.get(build.get("drvPath"), [])
|
||||||
|
for path in old_paths:
|
||||||
|
if path in deps:
|
||||||
|
failed_checks.append(build)
|
||||||
|
failed_paths.append(build.get("drvPath"))
|
||||||
|
build_schedule_order.remove(build)
|
||||||
|
removed.append(build.get("attr"))
|
||||||
|
failed[build.get("attr")] = (f"dependency {job.get('attr')} failed", [])
|
||||||
|
|
||||||
props.setProperty("attr", attr, source)
|
break
|
||||||
props.setProperty("system", system, source)
|
if old_paths == failed_paths:
|
||||||
props.setProperty("drv_path", drv_path, source)
|
break
|
||||||
props.setProperty("out_path", out_path, source)
|
failed[job.get("attr")] = (
|
||||||
# we use this to identify builds when running a retry
|
"failed",
|
||||||
props.setProperty("build_uuid", str(uuid.uuid4()), source)
|
[ getURLForBuildrequest(self.master, brid) for brid in brids.values() ]
|
||||||
|
)
|
||||||
triggered_schedulers.append((self.builds_scheduler, props))
|
print(' Removed jobs: ' + ', '.join(removed))
|
||||||
return triggered_schedulers
|
all_results = worst_status(result, all_results)
|
||||||
|
print(f' New result: {util.Results[all_results].upper()}')
|
||||||
|
for dep in self.all_deps:
|
||||||
|
if job.get("drvPath") in self.all_deps[dep]:
|
||||||
|
self.all_deps[dep].remove(job.get("drvPath"))
|
||||||
|
print('Done!')
|
||||||
|
build_props.setProperty("failed_builds", failed, "nix-eval-lix")
|
||||||
|
if self.ended:
|
||||||
|
return util.CANCELLED
|
||||||
|
return all_results
|
||||||
|
|
||||||
def getCurrentSummary(self) -> dict[str, str]: # noqa: N802
|
def getCurrentSummary(self) -> dict[str, str]: # noqa: N802
|
||||||
"""The original build trigger will the generic builder name `nix-build` in this case, which is not helpful"""
|
|
||||||
if not self.triggeredNames:
|
|
||||||
return {"step": "running"}
|
|
||||||
summary = []
|
summary = []
|
||||||
if self._result_list:
|
if self._result_list:
|
||||||
for status in ALL_RESULTS:
|
for status in ALL_RESULTS:
|
||||||
|
@ -178,13 +306,42 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
|
||||||
if not system or system in self.supported_systems: # report eval errors
|
if not system or system in self.supported_systems: # report eval errors
|
||||||
filtered_jobs.append(job)
|
filtered_jobs.append(job)
|
||||||
|
|
||||||
|
drv_show_log: Log = yield self.getLog("stdio")
|
||||||
|
drv_show_log.addStdout(f"getting derivation infos\n")
|
||||||
|
cmd = yield self.makeRemoteShellCommand(
|
||||||
|
stdioLogName=None,
|
||||||
|
collectStdout=True,
|
||||||
|
command=(
|
||||||
|
["nix", "derivation", "show", "--recursive"]
|
||||||
|
+ [ drv for drv in (job.get("drvPath") for job in filtered_jobs) if drv ]
|
||||||
|
),
|
||||||
|
)
|
||||||
|
yield self.runCommand(cmd)
|
||||||
|
drv_show_log.addStdout(f"done\n")
|
||||||
|
try:
|
||||||
|
drv_info = json.loads(cmd.stdout)
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
msg = f"Failed to parse `nix derivation show` output for {cmd.command}"
|
||||||
|
raise BuildbotNixError(msg) from e
|
||||||
|
all_deps = dict()
|
||||||
|
for drv, info in drv_info.items():
|
||||||
|
all_deps[drv] = set(info.get("inputDrvs").keys())
|
||||||
|
def closure_of(key, deps):
|
||||||
|
r, size = set([key]), 0
|
||||||
|
while len(r) != size:
|
||||||
|
size = len(r)
|
||||||
|
r.update(*[ deps[k] for k in r ])
|
||||||
|
return r.difference([key])
|
||||||
|
job_set = set(( drv for drv in ( job.get("drvPath") for job in filtered_jobs ) if drv ))
|
||||||
|
all_deps = { k: list(closure_of(k, all_deps).intersection(job_set)) for k in job_set }
|
||||||
|
|
||||||
self.build.addStepsAfterCurrentStep(
|
self.build.addStepsAfterCurrentStep(
|
||||||
[
|
[
|
||||||
BuildTrigger(
|
BuildTrigger(
|
||||||
builds_scheduler=f"lix-nix-build",
|
builds_scheduler=f"lix-nix-build",
|
||||||
skipped_builds_scheduler=f"lix-nix-skipped-build",
|
|
||||||
name="build flake",
|
name="build flake",
|
||||||
jobs=filtered_jobs,
|
jobs=filtered_jobs,
|
||||||
|
all_deps=all_deps,
|
||||||
),
|
),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -192,37 +349,6 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
# FIXME this leaks memory... but probably not enough that we care
|
|
||||||
class RetryCounter:
|
|
||||||
def __init__(self, retries: int) -> None:
|
|
||||||
self.builds: dict[uuid.UUID, int] = defaultdict(lambda: retries)
|
|
||||||
|
|
||||||
def retry_build(self, build_id: uuid.UUID) -> int:
|
|
||||||
retries = self.builds[build_id]
|
|
||||||
if retries > 1:
|
|
||||||
self.builds[build_id] = retries - 1
|
|
||||||
return retries
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
# For now we limit this to two. Often this allows us to make the error log
|
|
||||||
# shorter because we won't see the logs for all previous succeeded builds
|
|
||||||
RETRY_COUNTER = RetryCounter(retries=2)
|
|
||||||
|
|
||||||
|
|
||||||
class EvalErrorStep(steps.BuildStep):
|
|
||||||
"""Shows the error message of a failed evaluation."""
|
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
|
||||||
def run(self) -> Generator[Any, object, Any]:
|
|
||||||
error = self.getProperty("error")
|
|
||||||
attr = self.getProperty("attr")
|
|
||||||
# show eval error
|
|
||||||
error_log: Log = yield self.addLog("nix_error")
|
|
||||||
error_log.addStderr(f"{attr} failed to evaluate:\n{error}")
|
|
||||||
return util.FAILURE
|
|
||||||
|
|
||||||
|
|
||||||
class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep):
|
class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep):
|
||||||
"""Builds a nix derivation."""
|
"""Builds a nix derivation."""
|
||||||
|
|
||||||
|
@ -232,16 +358,25 @@ class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep):
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
def run(self) -> Generator[Any, object, Any]:
|
def run(self) -> Generator[Any, object, Any]:
|
||||||
|
if error := self.getProperty("error"):
|
||||||
|
attr = self.getProperty("attr")
|
||||||
|
# show eval error
|
||||||
|
error_log: Log = yield self.addLog("nix_error")
|
||||||
|
error_log.addStderr(f"{attr} failed to evaluate:\n{error}")
|
||||||
|
return util.FAILURE
|
||||||
|
|
||||||
|
if self.getProperty("isCached"):
|
||||||
|
yield self.addCompleteLog(
|
||||||
|
"cached outpath from previous builds",
|
||||||
|
# buildbot apparently hides the first line in the ui?
|
||||||
|
f'\n{self.getProperty("out_path")}\n')
|
||||||
|
return util.SKIPPED
|
||||||
|
|
||||||
# run `nix build`
|
# run `nix build`
|
||||||
cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand()
|
cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand()
|
||||||
yield self.runCommand(cmd)
|
yield self.runCommand(cmd)
|
||||||
|
|
||||||
res = cmd.results()
|
return cmd.results()
|
||||||
if res == util.FAILURE:
|
|
||||||
retries = RETRY_COUNTER.retry_build(self.getProperty("build_uuid"))
|
|
||||||
if retries > 0:
|
|
||||||
return util.RETRY
|
|
||||||
return res
|
|
||||||
|
|
||||||
|
|
||||||
class UpdateBuildOutput(steps.BuildStep):
|
class UpdateBuildOutput(steps.BuildStep):
|
||||||
|
@ -269,32 +404,6 @@ class UpdateBuildOutput(steps.BuildStep):
|
||||||
return util.SUCCESS
|
return util.SUCCESS
|
||||||
|
|
||||||
|
|
||||||
# The builtin retry mechanism doesn't seem to work for github,
|
|
||||||
# since github is sometimes not delivering the pull request ref fast enough.
|
|
||||||
class GitWithRetry(steps.Git):
|
|
||||||
@defer.inlineCallbacks
|
|
||||||
def run_vc(
|
|
||||||
self,
|
|
||||||
branch: str,
|
|
||||||
revision: str,
|
|
||||||
patch: str,
|
|
||||||
) -> Generator[Any, object, Any]:
|
|
||||||
retry_counter = 0
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
res = yield super().run_vc(branch, revision, patch)
|
|
||||||
except Exception as e: # noqa: BLE001
|
|
||||||
retry_counter += 1
|
|
||||||
if retry_counter == 3:
|
|
||||||
msg = "Failed to clone"
|
|
||||||
raise BuildbotNixError(msg) from e
|
|
||||||
log: Log = yield self.addLog("log")
|
|
||||||
yield log.addStderr(f"Retrying git clone (error: {e})\n")
|
|
||||||
yield asyncSleep(2 << retry_counter) # 2, 4, 8
|
|
||||||
else:
|
|
||||||
return res
|
|
||||||
|
|
||||||
|
|
||||||
def nix_eval_config(
|
def nix_eval_config(
|
||||||
project: GerritProject,
|
project: GerritProject,
|
||||||
gerrit_private_key: str,
|
gerrit_private_key: str,
|
||||||
|
@ -318,8 +427,12 @@ def nix_eval_config(
|
||||||
sshPrivateKey=gerrit_private_key
|
sshPrivateKey=gerrit_private_key
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
# use one gcroots directory per worker. this should be scoped to the largest unique resource
|
||||||
|
# in charge of builds (ie, buildnumber is too narrow) to not litter the system with permanent
|
||||||
|
# gcroots in case of worker restarts.
|
||||||
|
# TODO perhaps we should clean the entire /drvs/ directory up too during startup.
|
||||||
drv_gcroots_dir = util.Interpolate(
|
drv_gcroots_dir = util.Interpolate(
|
||||||
"/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/drvs/",
|
"/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/drvs/%(prop:workername)s/",
|
||||||
)
|
)
|
||||||
|
|
||||||
factory.addStep(
|
factory.addStep(
|
||||||
|
@ -356,6 +469,7 @@ def nix_eval_config(
|
||||||
"-rf",
|
"-rf",
|
||||||
drv_gcroots_dir,
|
drv_gcroots_dir,
|
||||||
],
|
],
|
||||||
|
alwaysRun=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -368,25 +482,9 @@ def nix_eval_config(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class CachixConfig:
|
|
||||||
name: str
|
|
||||||
signing_key_secret_name: str | None = None
|
|
||||||
auth_token_secret_name: str | None = None
|
|
||||||
|
|
||||||
def cachix_env(self) -> dict[str, str]:
|
|
||||||
env = {}
|
|
||||||
if self.signing_key_secret_name is not None:
|
|
||||||
env["CACHIX_SIGNING_KEY"] = util.Secret(self.signing_key_secret_name)
|
|
||||||
if self.auth_token_secret_name is not None:
|
|
||||||
env["CACHIX_AUTH_TOKEN"] = util.Secret(self.auth_token_secret_name)
|
|
||||||
return env
|
|
||||||
|
|
||||||
|
|
||||||
def nix_build_config(
|
def nix_build_config(
|
||||||
project: GerritProject,
|
project: GerritProject,
|
||||||
worker_names: list[str],
|
worker_names: list[str],
|
||||||
cachix: CachixConfig | None = None,
|
|
||||||
outputs_path: Path | None = None,
|
outputs_path: Path | None = None,
|
||||||
) -> util.BuilderConfig:
|
) -> util.BuilderConfig:
|
||||||
"""Builds one nix flake attribute."""
|
"""Builds one nix flake attribute."""
|
||||||
|
@ -417,19 +515,6 @@ def nix_build_config(
|
||||||
haltOnFailure=True,
|
haltOnFailure=True,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
if cachix:
|
|
||||||
factory.addStep(
|
|
||||||
steps.ShellCommand(
|
|
||||||
name="Upload cachix",
|
|
||||||
env=cachix.cachix_env(),
|
|
||||||
command=[
|
|
||||||
"cachix",
|
|
||||||
"push",
|
|
||||||
cachix.name,
|
|
||||||
util.Interpolate("result-%(prop:attr)s"),
|
|
||||||
],
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
factory.addStep(
|
factory.addStep(
|
||||||
steps.ShellCommand(
|
steps.ShellCommand(
|
||||||
|
@ -471,38 +556,6 @@ def nix_build_config(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def nix_skipped_build_config(
|
|
||||||
project: GerritProject,
|
|
||||||
worker_names: list[str],
|
|
||||||
) -> util.BuilderConfig:
|
|
||||||
"""Dummy builder that is triggered when a build is skipped."""
|
|
||||||
factory = util.BuildFactory()
|
|
||||||
factory.addStep(
|
|
||||||
EvalErrorStep(
|
|
||||||
name="Nix evaluation",
|
|
||||||
doStepIf=lambda s: s.getProperty("error"),
|
|
||||||
hideStepIf=lambda _, s: not s.getProperty("error"),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
# This is just a dummy step showing the cached build
|
|
||||||
factory.addStep(
|
|
||||||
steps.BuildStep(
|
|
||||||
name="Nix build (cached)",
|
|
||||||
doStepIf=lambda _: False,
|
|
||||||
hideStepIf=lambda _, s: s.getProperty("error"),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
return util.BuilderConfig(
|
|
||||||
name=f"{project.name}/nix-skipped-build",
|
|
||||||
project=project.name,
|
|
||||||
workernames=worker_names,
|
|
||||||
collapseRequests=False,
|
|
||||||
env={},
|
|
||||||
factory=factory,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def read_secret_file(secret_name: str) -> str:
|
def read_secret_file(secret_name: str) -> str:
|
||||||
directory = os.environ.get("CREDENTIALS_DIRECTORY")
|
directory = os.environ.get("CREDENTIALS_DIRECTORY")
|
||||||
if directory is None:
|
if directory is None:
|
||||||
|
@ -520,7 +573,6 @@ def config_for_project(
|
||||||
nix_eval_worker_count: int,
|
nix_eval_worker_count: int,
|
||||||
nix_eval_max_memory_size: int,
|
nix_eval_max_memory_size: int,
|
||||||
eval_lock: util.MasterLock,
|
eval_lock: util.MasterLock,
|
||||||
cachix: CachixConfig | None = None,
|
|
||||||
outputs_path: Path | None = None,
|
outputs_path: Path | None = None,
|
||||||
) -> Project:
|
) -> Project:
|
||||||
config["projects"].append(Project(project.name))
|
config["projects"].append(Project(project.name))
|
||||||
|
@ -540,11 +592,6 @@ def config_for_project(
|
||||||
name=f"{project.name}-nix-build",
|
name=f"{project.name}-nix-build",
|
||||||
builderNames=[f"{project.name}/nix-build"],
|
builderNames=[f"{project.name}/nix-build"],
|
||||||
),
|
),
|
||||||
# this is triggered from `nix-eval` when the build is skipped
|
|
||||||
schedulers.Triggerable(
|
|
||||||
name=f"{project.name}-nix-skipped-build",
|
|
||||||
builderNames=[f"{project.name}/nix-skipped-build"],
|
|
||||||
),
|
|
||||||
# allow to manually trigger a nix-build
|
# allow to manually trigger a nix-build
|
||||||
schedulers.ForceScheduler(
|
schedulers.ForceScheduler(
|
||||||
name=f"{project.name}-force",
|
name=f"{project.name}-force",
|
||||||
|
@ -582,10 +629,8 @@ def config_for_project(
|
||||||
nix_build_config(
|
nix_build_config(
|
||||||
project,
|
project,
|
||||||
worker_names,
|
worker_names,
|
||||||
cachix=cachix,
|
|
||||||
outputs_path=outputs_path,
|
outputs_path=outputs_path,
|
||||||
),
|
),
|
||||||
nix_skipped_build_config(project, [SKIPPED_BUILDER_NAME]),
|
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -608,62 +653,22 @@ def gerritReviewCB(builderName, build, result, master, arg):
|
||||||
if builderName != 'lix/nix-eval':
|
if builderName != 'lix/nix-eval':
|
||||||
return dict()
|
return dict()
|
||||||
|
|
||||||
all_checks = {}
|
failed = build['properties'].get('failed_builds', [{}])[0]
|
||||||
for step in build['steps']:
|
|
||||||
if step['name'] != 'build flake':
|
|
||||||
continue
|
|
||||||
|
|
||||||
for url in step['urls']:
|
labels = {
|
||||||
if url['name'].startswith('success: hydraJobs.'):
|
'Verified': -1 if failed else 1,
|
||||||
path = url['name'].split(' ')[1]
|
}
|
||||||
all_checks[path] = (True, url['url'])
|
|
||||||
elif url['name'].startswith('failure: hydraJobs.'):
|
|
||||||
path = url['name'].split(' ')[1]
|
|
||||||
all_checks[path] = (False, url['url'])
|
|
||||||
|
|
||||||
collected_oses = {}
|
|
||||||
for check in all_checks:
|
|
||||||
arch = check.split('.')[-1]
|
|
||||||
if not arch.endswith('-linux') and not arch.endswith('-darwin'):
|
|
||||||
# Not an architecture-specific job, just a test
|
|
||||||
os = "test"
|
|
||||||
else:
|
|
||||||
os = arch.split('-')[1]
|
|
||||||
(success, failure) = collected_oses.get(os, (0, 0))
|
|
||||||
if all_checks[check][0]:
|
|
||||||
success += 1
|
|
||||||
else:
|
|
||||||
failure += 1
|
|
||||||
|
|
||||||
collected_oses[os] = (success, failure)
|
|
||||||
labels = {}
|
|
||||||
|
|
||||||
if 'linux' in collected_oses:
|
|
||||||
(success, failure) = collected_oses['linux']
|
|
||||||
if success > 0 and failure == 0:
|
|
||||||
labels['Verified-On-Linux'] = 1
|
|
||||||
elif failure > 0:
|
|
||||||
labels['Verified-On-Linux'] = -1
|
|
||||||
|
|
||||||
if 'darwin' in collected_oses:
|
|
||||||
(success, failure) = collected_oses['darwin']
|
|
||||||
if success > 0 and failure == 0:
|
|
||||||
labels['Verified-On-Darwin'] = 1
|
|
||||||
elif failure > 0:
|
|
||||||
labels['Verified-On-Darwin'] = -1
|
|
||||||
|
|
||||||
message = "Buildbot finished compiling your patchset!\n"
|
message = "Buildbot finished compiling your patchset!\n"
|
||||||
message += "The result is: %s\n" % util.Results[result].upper()
|
message += "The result is: %s\n" % util.Results[result].upper()
|
||||||
if result != util.SUCCESS:
|
if result != util.SUCCESS:
|
||||||
successful_checks = []
|
message += "\nFailed checks:\n"
|
||||||
failed_checks = []
|
for check, context in sorted(failed.items()):
|
||||||
for check in all_checks:
|
how, urls = context
|
||||||
if not all_checks[check][0]:
|
message += f" - {check}: {how}"
|
||||||
failed_checks.append(f" - {check} (see {all_checks[check][1]})")
|
if urls:
|
||||||
|
message += f" (see {', '.join(urls)})"
|
||||||
if len(failed_checks) > 0:
|
message += "\n"
|
||||||
message += "Failed checks:\n" + "\n".join(failed_checks) + "\n"
|
|
||||||
|
|
||||||
|
|
||||||
if arg:
|
if arg:
|
||||||
message += "\nFor more details visit:\n"
|
message += "\nFor more details visit:\n"
|
||||||
|
@ -724,7 +729,6 @@ class GerritNixConfigurator(ConfiguratorBase):
|
||||||
nix_eval_worker_count: int | None,
|
nix_eval_worker_count: int | None,
|
||||||
nix_eval_max_memory_size: int,
|
nix_eval_max_memory_size: int,
|
||||||
nix_workers_secret_name: str = "buildbot-nix-workers", # noqa: S107
|
nix_workers_secret_name: str = "buildbot-nix-workers", # noqa: S107
|
||||||
cachix: CachixConfig | None = None,
|
|
||||||
outputs_path: str | None = None,
|
outputs_path: str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
@ -737,7 +741,6 @@ class GerritNixConfigurator(ConfiguratorBase):
|
||||||
self.nix_supported_systems = nix_supported_systems
|
self.nix_supported_systems = nix_supported_systems
|
||||||
self.gerrit_change_source = GerritChangeSource(gerrit_server, gerrit_user, gerritport=gerrit_port, identity_file=gerrit_sshkey_path)
|
self.gerrit_change_source = GerritChangeSource(gerrit_server, gerrit_user, gerritport=gerrit_port, identity_file=gerrit_sshkey_path)
|
||||||
self.url = url
|
self.url = url
|
||||||
self.cachix = cachix
|
|
||||||
if outputs_path is None:
|
if outputs_path is None:
|
||||||
self.outputs_path = None
|
self.outputs_path = None
|
||||||
else:
|
else:
|
||||||
|
@ -769,12 +772,10 @@ class GerritNixConfigurator(ConfiguratorBase):
|
||||||
self.nix_eval_worker_count or multiprocessing.cpu_count(),
|
self.nix_eval_worker_count or multiprocessing.cpu_count(),
|
||||||
self.nix_eval_max_memory_size,
|
self.nix_eval_max_memory_size,
|
||||||
eval_lock,
|
eval_lock,
|
||||||
self.cachix,
|
|
||||||
self.outputs_path,
|
self.outputs_path,
|
||||||
)
|
)
|
||||||
|
|
||||||
config["change_source"] = self.gerrit_change_source
|
config["change_source"] = self.gerrit_change_source
|
||||||
config["workers"].append(worker.LocalWorker(SKIPPED_BUILDER_NAME))
|
|
||||||
config["services"].append(
|
config["services"].append(
|
||||||
reporters.GerritStatusPush(self.gerrit_server, self.gerrit_user,
|
reporters.GerritStatusPush(self.gerrit_server, self.gerrit_user,
|
||||||
port=2022,
|
port=2022,
|
||||||
|
|
|
@ -46,14 +46,6 @@ in
|
||||||
# optional nix-eval-jobs settings
|
# optional nix-eval-jobs settings
|
||||||
# evalWorkerCount = 8; # limit number of concurrent evaluations
|
# evalWorkerCount = 8; # limit number of concurrent evaluations
|
||||||
# evalMaxMemorySize = "2048"; # limit memory usage per evaluation
|
# evalMaxMemorySize = "2048"; # limit memory usage per evaluation
|
||||||
|
|
||||||
# optional cachix
|
|
||||||
#cachix = {
|
|
||||||
# name = "my-cachix";
|
|
||||||
# # One of the following is required:
|
|
||||||
# signingKey = "/var/lib/secrets/cachix-key";
|
|
||||||
# authToken = "/var/lib/secrets/cachix-token";
|
|
||||||
#};
|
|
||||||
};
|
};
|
||||||
})
|
})
|
||||||
buildbot-nix.nixosModules.buildbot-master
|
buildbot-nix.nixosModules.buildbot-master
|
||||||
|
|
|
@ -15,25 +15,6 @@ in
|
||||||
default = "postgresql://@/buildbot";
|
default = "postgresql://@/buildbot";
|
||||||
description = "Postgresql database url";
|
description = "Postgresql database url";
|
||||||
};
|
};
|
||||||
cachix = {
|
|
||||||
name = lib.mkOption {
|
|
||||||
type = lib.types.nullOr lib.types.str;
|
|
||||||
default = null;
|
|
||||||
description = "Cachix name";
|
|
||||||
};
|
|
||||||
|
|
||||||
signingKeyFile = lib.mkOption {
|
|
||||||
type = lib.types.nullOr lib.types.path;
|
|
||||||
default = null;
|
|
||||||
description = "Cachix signing key";
|
|
||||||
};
|
|
||||||
|
|
||||||
authTokenFile = lib.mkOption {
|
|
||||||
type = lib.types.nullOr lib.types.str;
|
|
||||||
default = null;
|
|
||||||
description = "Cachix auth token";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
workersFile = lib.mkOption {
|
workersFile = lib.mkOption {
|
||||||
type = lib.types.path;
|
type = lib.types.path;
|
||||||
description = "File containing a list of nix workers";
|
description = "File containing a list of nix workers";
|
||||||
|
@ -88,13 +69,6 @@ in
|
||||||
isSystemUser = true;
|
isSystemUser = true;
|
||||||
};
|
};
|
||||||
|
|
||||||
assertions = [
|
|
||||||
{
|
|
||||||
assertion = cfg.cachix.name != null -> cfg.cachix.signingKeyFile != null || cfg.cachix.authTokenFile != null;
|
|
||||||
message = "if cachix.name is provided, then cachix.signingKeyFile and cachix.authTokenFile must be set";
|
|
||||||
}
|
|
||||||
];
|
|
||||||
|
|
||||||
services.buildbot-master = {
|
services.buildbot-master = {
|
||||||
enable = true;
|
enable = true;
|
||||||
|
|
||||||
|
@ -106,7 +80,7 @@ in
|
||||||
home = "/var/lib/buildbot";
|
home = "/var/lib/buildbot";
|
||||||
extraImports = ''
|
extraImports = ''
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
from buildbot_nix import GerritNixConfigurator, CachixConfig
|
from buildbot_nix import GerritNixConfigurator
|
||||||
'';
|
'';
|
||||||
configurators = [
|
configurators = [
|
||||||
''
|
''
|
||||||
|
@ -150,11 +124,7 @@ in
|
||||||
LoadCredential = [
|
LoadCredential = [
|
||||||
"buildbot-nix-workers:${cfg.workersFile}"
|
"buildbot-nix-workers:${cfg.workersFile}"
|
||||||
"buildbot-oauth2-secret:${cfg.oauth2SecretFile}"
|
"buildbot-oauth2-secret:${cfg.oauth2SecretFile}"
|
||||||
]
|
];
|
||||||
++ lib.optional (cfg.cachix.signingKeyFile != null)
|
|
||||||
"cachix-signing-key:${builtins.toString cfg.cachix.signingKeyFile}"
|
|
||||||
++ lib.optional (cfg.cachix.authTokenFile != null)
|
|
||||||
"cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}";
|
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue