Compare commits
No commits in common. "profiler" and "main" have entirely different histories.
|
@ -7,47 +7,37 @@ import base64
|
||||||
from collections.abc import Generator
|
from collections.abc import Generator
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import buildbot
|
import buildbot
|
||||||
from buildbot.configurators import ConfiguratorBase
|
from buildbot.configurators import ConfiguratorBase
|
||||||
from buildbot.plugins import reporters, schedulers, secrets, steps, util, worker
|
from buildbot.plugins import reporters, schedulers, secrets, steps, util, worker
|
||||||
from buildbot.process import buildstep, logobserver, remotecommand
|
from buildbot.process import buildstep, logobserver, remotecommand
|
||||||
from buildbot.process.log import StreamLog
|
|
||||||
from buildbot.process.project import Project
|
from buildbot.process.project import Project
|
||||||
from buildbot.process.properties import Properties
|
from buildbot.process.properties import Properties
|
||||||
from buildbot.process.results import ALL_RESULTS, statusToString
|
from buildbot.process.results import ALL_RESULTS, statusToString
|
||||||
from buildbot.www.auth import AuthBase
|
from buildbot.www.auth import AuthBase
|
||||||
from buildbot.www.oauth2 import OAuth2Auth
|
from buildbot.www.oauth2 import OAuth2Auth
|
||||||
from buildbot.changes.gerritchangesource import GerritChangeSource
|
from buildbot.changes.gerritchangesource import GerritChangeSource
|
||||||
from buildbot.reporters.generators.build import BuildStatusGenerator
|
|
||||||
from buildbot.reporters.utils import getURLForBuildrequest
|
from buildbot.reporters.utils import getURLForBuildrequest
|
||||||
|
from buildbot.reporters.generators.build import BuildStatusGenerator
|
||||||
|
from buildbot.reporters.message import MessageFormatterFunction
|
||||||
from buildbot.process.buildstep import EXCEPTION
|
from buildbot.process.buildstep import EXCEPTION
|
||||||
from buildbot.process.buildstep import SUCCESS
|
from buildbot.process.buildstep import SUCCESS
|
||||||
from buildbot.process.results import worst_status
|
from buildbot.process.results import worst_status
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from buildbot.process.log import Log
|
||||||
|
|
||||||
from twisted.internet import defer
|
from twisted.internet import defer
|
||||||
from twisted.logger import Logger
|
from twisted.logger import Logger
|
||||||
|
|
||||||
from .binary_cache import S3BinaryCacheConfig
|
from .binary_cache import S3BinaryCacheConfig
|
||||||
from .message_formatter import ReasonableMessageFormatter, CallbackPayloadBuild, CallbackPayloadBuildSet, CallbackReturn
|
|
||||||
|
|
||||||
log = Logger()
|
log = Logger()
|
||||||
|
|
||||||
# util is a plugin variable, not a module...
|
FLAKE_TARGET_ATTRIBUTE_FOR_JOBS = "buildbotJobs"
|
||||||
BuilderConfig = util.BuilderConfig
|
|
||||||
MasterLock = util.MasterLock
|
|
||||||
|
|
||||||
FLAKE_TARGET_ATTRIBUTE_FOR_JOBS = "hydraJobs"
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class EvaluatorSettings:
|
|
||||||
supported_systems: list[str]
|
|
||||||
worker_count: int
|
|
||||||
max_memory_size: int
|
|
||||||
gc_roots_dir: str
|
|
||||||
lock: MasterLock
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class NixBuilder:
|
class NixBuilder:
|
||||||
|
@ -59,24 +49,14 @@ class NixBuilder:
|
||||||
publicHostKey: str | None = None
|
publicHostKey: str | None = None
|
||||||
sshUser: str | None = None
|
sshUser: str | None = None
|
||||||
sshKey: str | None = None
|
sshKey: str | None = None
|
||||||
systems: list[str] = field(default_factory=lambda: [])
|
systems: list[str] = field(default_factory=lambda: ["-"])
|
||||||
supportedFeatures: list[str] = field(default_factory=lambda: [])
|
supportedFeatures: list[str] = field(default_factory=lambda: ["-"])
|
||||||
mandatoryFeatures: list[str] = field(default_factory=lambda: [])
|
mandatoryFeatures: list[str] = field(default_factory=lambda: ["-"])
|
||||||
|
|
||||||
def to_nix_store(self):
|
def to_nix_line(self):
|
||||||
fullConnection = f"{self.sshUser}@{self.hostName}" if self.sshUser is not None else self.hostName
|
encoded_public_key = base64.b64encode(self.publicHostKey.encode('ascii')).decode('ascii') if self.publicHostKey is not None else "-"
|
||||||
fullConnection = f"{self.protocol}://{fullConnection}"
|
fullConnection = f"{self.protocol}://{self.sshUser}@{self.hostName}" if self.sshUser is not None else self.hostName
|
||||||
params = []
|
return f"{fullConnection} {",".join(self.systems)} {self.sshKey or "-"} {self.maxJobs} {self.speedFactor} {",".join(self.supportedFeatures)} {",".join(self.mandatoryFeatures)} {encoded_public_key}"
|
||||||
if self.sshKey is not None:
|
|
||||||
params.append(f"ssh-key={self.sshKey}")
|
|
||||||
if self.publicHostKey is not None:
|
|
||||||
encoded_public_key = base64.b64encode(self.publicHostKey.encode('ascii')).decode('ascii')
|
|
||||||
params.append(f"base64-ssh-public-host-key={encoded_public_key}")
|
|
||||||
if params != []:
|
|
||||||
fullConnection += "?"
|
|
||||||
fullConnection += "&".join(params)
|
|
||||||
|
|
||||||
return fullConnection
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
@ -91,10 +71,9 @@ class OAuth2Config:
|
||||||
debug: bool = False
|
debug: bool = False
|
||||||
|
|
||||||
class KeycloakOAuth2Auth(OAuth2Auth):
|
class KeycloakOAuth2Auth(OAuth2Auth):
|
||||||
userinfoUri: str
|
def __init__(self, userinfoUri: str, *args, debug=False, **kwargs):
|
||||||
|
|
||||||
def __init__(self, *args, debug=False, **kwargs):
|
|
||||||
super().__init__(*args, **kwargs)
|
super().__init__(*args, **kwargs)
|
||||||
|
self.userinfoUri = userinfoUri
|
||||||
self.debug = debug
|
self.debug = debug
|
||||||
|
|
||||||
def createSessionFromToken(self, token):
|
def createSessionFromToken(self, token):
|
||||||
|
@ -151,7 +130,7 @@ class GerritConfig:
|
||||||
"""
|
"""
|
||||||
Returns the prefix to build a repourl using that gerrit configuration.
|
Returns the prefix to build a repourl using that gerrit configuration.
|
||||||
"""
|
"""
|
||||||
return f'ssh://{self.username}@{self.domain}:{self.port}/'
|
return 'ssh://{self.username}@{self.domain}:{self.port}/'
|
||||||
|
|
||||||
class BuildTrigger(steps.BuildStep):
|
class BuildTrigger(steps.BuildStep):
|
||||||
def __init__(
|
def __init__(
|
||||||
|
@ -169,7 +148,7 @@ class BuildTrigger(steps.BuildStep):
|
||||||
self.ended = False
|
self.ended = False
|
||||||
self.waitForFinishDeferred = None
|
self.waitForFinishDeferred = None
|
||||||
self.brids = []
|
self.brids = []
|
||||||
self.description = f"building {len(jobs)} jobs"
|
self.description = f"building {len(jobs)} hydra jobs"
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
|
|
||||||
def interrupt(self, reason):
|
def interrupt(self, reason):
|
||||||
|
@ -198,16 +177,15 @@ class BuildTrigger(steps.BuildStep):
|
||||||
return sch
|
return sch
|
||||||
|
|
||||||
def schedule_one(self, build_props: Properties, job):
|
def schedule_one(self, build_props: Properties, job):
|
||||||
project_name = build_props.getProperty("event.refUpdate.project") or build_props.getProperty("event.change.project")
|
project_name = build_props.getProperty('event.project')
|
||||||
source = f"{project_name}-eval"
|
source = f"{project_name}-eval-lix"
|
||||||
attr = job.get("attr", "eval-error")
|
attr = job.get("attr", "eval-error")
|
||||||
# FIXME(raito): this was named this way for backward compatibility with Lix deployment.
|
name = attr
|
||||||
# We should just parametrize this.
|
name = f"{FLAKE_TARGET_ATTRIBUTE_FOR_JOBS}.{name}"
|
||||||
name = f"hydraJobs.{attr}"
|
|
||||||
error = job.get("error")
|
error = job.get("error")
|
||||||
props = Properties()
|
props = Properties()
|
||||||
props.setProperty("virtual_builder_name", name, source)
|
props.setProperty("virtual_builder_name", name, source)
|
||||||
props.setProperty("status_name", f"building hydraJobs.{attr}", source)
|
props.setProperty("status_name", f"nix-build .#{FLAKE_TARGET_ATTRIBUTE_FOR_JOBS}.{attr}", source)
|
||||||
props.setProperty("virtual_builder_tags", "", source)
|
props.setProperty("virtual_builder_tags", "", source)
|
||||||
|
|
||||||
if error is not None:
|
if error is not None:
|
||||||
|
@ -256,7 +234,7 @@ class BuildTrigger(steps.BuildStep):
|
||||||
def run(self):
|
def run(self):
|
||||||
self.running = True
|
self.running = True
|
||||||
build_props = self.build.getProperties()
|
build_props = self.build.getProperties()
|
||||||
logs: StreamLog = yield self.addLog("build info")
|
logs: Log = yield self.addLog("build info")
|
||||||
|
|
||||||
builds_to_schedule = list(self.jobs)
|
builds_to_schedule = list(self.jobs)
|
||||||
build_schedule_order = []
|
build_schedule_order = []
|
||||||
|
@ -394,8 +372,7 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
|
||||||
# run nix-eval-jobs --flake .#$FLAKE_TARGET_ATTRIBUTE_FOR_JOBS to generate the dict of stages
|
# run nix-eval-jobs --flake .#$FLAKE_TARGET_ATTRIBUTE_FOR_JOBS to generate the dict of stages
|
||||||
cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand()
|
cmd: remotecommand.RemoteCommand = yield self.makeRemoteShellCommand()
|
||||||
build_props = self.build.getProperties()
|
build_props = self.build.getProperties()
|
||||||
project_name = build_props.getProperty("event.refUpdate.project") or build_props.getProperty("event.change.project")
|
project_name = build_props.get('event.project')
|
||||||
assert project_name is not None, "`event.refUpdate.project` or `event.change.project` is not available on the build properties, unexpected build type!"
|
|
||||||
|
|
||||||
yield self.runCommand(cmd)
|
yield self.runCommand(cmd)
|
||||||
|
|
||||||
|
@ -419,11 +396,26 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
|
||||||
if not system or system in self.supported_systems: # report eval errors
|
if not system or system in self.supported_systems: # report eval errors
|
||||||
filtered_jobs.append(job)
|
filtered_jobs.append(job)
|
||||||
|
|
||||||
# Filter out failed evaluations
|
drv_show_log: Log = yield self.getLog("stdio")
|
||||||
succeeded_jobs = [job for job in filtered_jobs if job.get('error') is None]
|
drv_show_log.addStdout(f"getting derivation infos\n")
|
||||||
|
cmd = yield self.makeRemoteShellCommand(
|
||||||
drv_show_log: StreamLog = yield self.getLog("stdio")
|
stdioLogName=None,
|
||||||
|
collectStdout=True,
|
||||||
|
command=(
|
||||||
|
["nix", "derivation", "show", "--recursive"]
|
||||||
|
+ [ drv for drv in (job.get("drvPath") for job in filtered_jobs) if drv ]
|
||||||
|
),
|
||||||
|
)
|
||||||
|
yield self.runCommand(cmd)
|
||||||
|
drv_show_log.addStdout(f"done\n")
|
||||||
|
try:
|
||||||
|
drv_info = json.loads(cmd.stdout)
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
msg = f"Failed to parse `nix derivation show` output for {cmd.command}"
|
||||||
|
raise BuildbotNixError(msg) from e
|
||||||
all_deps = dict()
|
all_deps = dict()
|
||||||
|
for drv, info in drv_info.items():
|
||||||
|
all_deps[drv] = set(info.get("inputDrvs").keys())
|
||||||
|
|
||||||
def closure_of(key, deps):
|
def closure_of(key, deps):
|
||||||
r, size = set([key]), 0
|
r, size = set([key]), 0
|
||||||
|
@ -432,34 +424,14 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
|
||||||
r.update(*[ deps[k] for k in r ])
|
r.update(*[ deps[k] for k in r ])
|
||||||
return r.difference([key])
|
return r.difference([key])
|
||||||
|
|
||||||
if succeeded_jobs:
|
job_set = set(( drv for drv in ( job.get("drvPath") for job in filtered_jobs ) if drv ))
|
||||||
drv_show_log.addStdout(f"getting derivation infos for valid derivations\n")
|
all_deps = { k: list(closure_of(k, all_deps).intersection(job_set)) for k in job_set }
|
||||||
cmd = yield self.makeRemoteShellCommand(
|
|
||||||
stdioLogName=None,
|
|
||||||
collectStdout=True,
|
|
||||||
command=(
|
|
||||||
["nix", "derivation", "show", "--recursive"]
|
|
||||||
+ [ drv for drv in (job.get("drvPath") for job in succeeded_jobs) if drv ]
|
|
||||||
),
|
|
||||||
)
|
|
||||||
yield self.runCommand(cmd)
|
|
||||||
drv_show_log.addStdout(f"done\n")
|
|
||||||
try:
|
|
||||||
drv_info = json.loads(cmd.stdout)
|
|
||||||
except json.JSONDecodeError as e:
|
|
||||||
msg = f"Failed to parse `nix derivation show` output for {cmd.command}"
|
|
||||||
raise BuildbotNixError(msg) from e
|
|
||||||
for drv, info in drv_info.items():
|
|
||||||
all_deps[drv] = set(info.get("inputDrvs").keys())
|
|
||||||
|
|
||||||
job_set = set(( drv for drv in ( job.get("drvPath") for job in filtered_jobs ) if drv ))
|
|
||||||
all_deps = { k: list(closure_of(k, all_deps).intersection(job_set)) for k in job_set }
|
|
||||||
|
|
||||||
self.build.addStepsAfterCurrentStep(
|
self.build.addStepsAfterCurrentStep(
|
||||||
[
|
[
|
||||||
BuildTrigger(
|
BuildTrigger(
|
||||||
builds_scheduler_group=f"{project_name}-nix-build",
|
builds_scheduler_group=f"{project_name}-nix-build",
|
||||||
name="build derivations",
|
name="build flake",
|
||||||
jobs=filtered_jobs,
|
jobs=filtered_jobs,
|
||||||
all_deps=all_deps,
|
all_deps=all_deps,
|
||||||
),
|
),
|
||||||
|
@ -468,91 +440,6 @@ class NixEvalCommand(buildstep.ShellMixin, steps.BuildStep):
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def make_job_evaluator(name: str, settings: EvaluatorSettings, flake: bool, incoming_ref_filename: str) -> NixEvalCommand:
|
|
||||||
actual_command = []
|
|
||||||
|
|
||||||
if flake:
|
|
||||||
actual_command += ["--flake", f".#{FLAKE_TARGET_ATTRIBUTE_FOR_JOBS}"]
|
|
||||||
else:
|
|
||||||
actual_command += ["--expr",
|
|
||||||
f"import ./.ci/buildbot.nix {{ incoming_ref_data = builtins.fromJSON (builtins.readFile {incoming_ref_filename}); }}"]
|
|
||||||
|
|
||||||
return NixEvalCommand(
|
|
||||||
env={},
|
|
||||||
name=name,
|
|
||||||
supported_systems=settings.supported_systems,
|
|
||||||
command=[
|
|
||||||
"nix-eval-jobs",
|
|
||||||
"--workers",
|
|
||||||
str(settings.worker_count),
|
|
||||||
"--max-memory-size",
|
|
||||||
str(settings.max_memory_size),
|
|
||||||
"--gc-roots-dir",
|
|
||||||
settings.gc_roots_dir,
|
|
||||||
"--force-recurse",
|
|
||||||
"--check-cache-status",
|
|
||||||
] + actual_command,
|
|
||||||
haltOnFailure=True,
|
|
||||||
locks=[settings.lock.access("exclusive")]
|
|
||||||
)
|
|
||||||
|
|
||||||
class NixConfigure(buildstep.CommandMixin, steps.BuildStep):
|
|
||||||
name = "determining jobs"
|
|
||||||
"""
|
|
||||||
Determine what `NixEvalCommand` step should be added after
|
|
||||||
based on the existence of:
|
|
||||||
|
|
||||||
- flake.nix
|
|
||||||
- .ci/buildbot.nix
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, eval_settings: EvaluatorSettings, **kwargs: Any) -> None:
|
|
||||||
self.evaluator_settings = eval_settings
|
|
||||||
super().__init__(**kwargs)
|
|
||||||
self.observer = logobserver.BufferLogObserver()
|
|
||||||
self.addLogObserver("stdio", self.observer)
|
|
||||||
|
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
|
||||||
def run(self) -> Generator[Any, object, Any]:
|
|
||||||
try:
|
|
||||||
configure_log: StreamLog = yield self.getLog("stdio")
|
|
||||||
except Exception:
|
|
||||||
configure_log: StreamLog = yield self.addLog("stdio")
|
|
||||||
|
|
||||||
# Takes precedence.
|
|
||||||
configure_log.addStdout("checking if there's a .ci/buildbot.nix...\n")
|
|
||||||
ci_buildbot_defn_exists = yield self.pathExists('build/.ci/buildbot.nix')
|
|
||||||
if ci_buildbot_defn_exists:
|
|
||||||
configure_log.addStdout(".ci/buildbot.nix found, configured for non-flake CI\n")
|
|
||||||
self.build.addStepsAfterCurrentStep(
|
|
||||||
[
|
|
||||||
make_job_evaluator(
|
|
||||||
"evaluate `.ci/buildbot.nix` jobs",
|
|
||||||
self.evaluator_settings,
|
|
||||||
False,
|
|
||||||
"./incoming-ref.json"
|
|
||||||
)
|
|
||||||
]
|
|
||||||
)
|
|
||||||
return SUCCESS
|
|
||||||
|
|
||||||
flake_exists = yield self.pathExists('build/flake.nix')
|
|
||||||
if flake_exists:
|
|
||||||
configure_log.addStdout(f"flake.nix found")
|
|
||||||
self.build.addStepsAfterCurrentStep([
|
|
||||||
make_job_evaluator(
|
|
||||||
"evaluate `flake.nix` jobs",
|
|
||||||
self.evaluator_settings,
|
|
||||||
True,
|
|
||||||
"./incoming-ref.json"
|
|
||||||
)
|
|
||||||
]
|
|
||||||
)
|
|
||||||
return SUCCESS
|
|
||||||
|
|
||||||
configure_log.addStdout("neither flake.nix found neither .ci/buildbot.nix, no CI to run!")
|
|
||||||
return SUCCESS
|
|
||||||
|
|
||||||
class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep):
|
class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep):
|
||||||
"""Builds a nix derivation."""
|
"""Builds a nix derivation."""
|
||||||
|
@ -566,7 +453,7 @@ class NixBuildCommand(buildstep.ShellMixin, steps.BuildStep):
|
||||||
if error := self.getProperty("error"):
|
if error := self.getProperty("error"):
|
||||||
attr = self.getProperty("attr")
|
attr = self.getProperty("attr")
|
||||||
# show eval error
|
# show eval error
|
||||||
error_log: StreamLog = yield self.addLog("nix_error")
|
error_log: Log = yield self.addLog("nix_error")
|
||||||
error_log.addStderr(f"{attr} failed to evaluate:\n{error}")
|
error_log.addStderr(f"{attr} failed to evaluate:\n{error}")
|
||||||
return util.FAILURE
|
return util.FAILURE
|
||||||
|
|
||||||
|
@ -590,23 +477,14 @@ def nix_eval_config(
|
||||||
project: GerritProject,
|
project: GerritProject,
|
||||||
worker_names: list[str],
|
worker_names: list[str],
|
||||||
supported_systems: list[str],
|
supported_systems: list[str],
|
||||||
eval_lock: MasterLock,
|
eval_lock: util.MasterLock,
|
||||||
worker_count: int,
|
worker_count: int,
|
||||||
max_memory_size: int,
|
max_memory_size: int,
|
||||||
) -> BuilderConfig:
|
) -> util.BuilderConfig:
|
||||||
"""
|
"""Uses nix-eval-jobs to evaluate $FLAKE_TARGET_ATTRIBUTE_FOR_JOBS (`.#hydraJobs` by default) from flake.nix in parallel.
|
||||||
Uses nix-eval-jobs to evaluate the entrypoint of this project.
|
|
||||||
For each evaluated attribute a new build pipeline is started.
|
For each evaluated attribute a new build pipeline is started.
|
||||||
"""
|
"""
|
||||||
factory = util.BuildFactory()
|
factory = util.BuildFactory()
|
||||||
|
|
||||||
gerrit_private_key = None
|
|
||||||
with open(project.private_sshkey_path, 'r') as f:
|
|
||||||
gerrit_private_key = f.read()
|
|
||||||
|
|
||||||
if gerrit_private_key is None:
|
|
||||||
raise RuntimeError('No gerrit private key to fetch the repositories')
|
|
||||||
|
|
||||||
# check out the source
|
# check out the source
|
||||||
factory.addStep(
|
factory.addStep(
|
||||||
steps.Gerrit(
|
steps.Gerrit(
|
||||||
|
@ -614,10 +492,9 @@ def nix_eval_config(
|
||||||
mode="full",
|
mode="full",
|
||||||
retry=[60, 60],
|
retry=[60, 60],
|
||||||
timeout=3600,
|
timeout=3600,
|
||||||
sshPrivateKey=gerrit_private_key
|
sshPrivateKey=project.private_sshkey_path
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
# use one gcroots directory per worker. this should be scoped to the largest unique resource
|
# use one gcroots directory per worker. this should be scoped to the largest unique resource
|
||||||
# in charge of builds (ie, buildnumber is too narrow) to not litter the system with permanent
|
# in charge of builds (ie, buildnumber is too narrow) to not litter the system with permanent
|
||||||
# gcroots in case of worker restarts.
|
# gcroots in case of worker restarts.
|
||||||
|
@ -626,27 +503,27 @@ def nix_eval_config(
|
||||||
"/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/drvs/%(prop:workername)s/",
|
"/nix/var/nix/gcroots/per-user/buildbot-worker/%(prop:project)s/drvs/%(prop:workername)s/",
|
||||||
)
|
)
|
||||||
|
|
||||||
eval_settings = EvaluatorSettings(
|
|
||||||
supported_systems=supported_systems,
|
|
||||||
worker_count=worker_count,
|
|
||||||
max_memory_size=max_memory_size,
|
|
||||||
gc_roots_dir=drv_gcroots_dir,
|
|
||||||
lock=eval_lock
|
|
||||||
)
|
|
||||||
|
|
||||||
# This information can be passed at job evaluation time
|
|
||||||
# to skip some jobs, e.g. expensive jobs, etc.
|
|
||||||
# Transfer incoming ref data to the target.
|
|
||||||
factory.addStep(steps.JSONPropertiesDownload(workerdest="incoming-ref.json"))
|
|
||||||
|
|
||||||
# NixConfigure will choose
|
|
||||||
# how to add a NixEvalCommand job
|
|
||||||
# based on whether there's a flake.nix or
|
|
||||||
# a .ci/buildbot.nix.
|
|
||||||
factory.addStep(
|
factory.addStep(
|
||||||
NixConfigure(
|
NixEvalCommand(
|
||||||
eval_settings
|
env={},
|
||||||
)
|
name="evaluate flake",
|
||||||
|
supported_systems=supported_systems,
|
||||||
|
command=[
|
||||||
|
"nix-eval-jobs",
|
||||||
|
"--workers",
|
||||||
|
str(worker_count),
|
||||||
|
"--max-memory-size",
|
||||||
|
str(max_memory_size),
|
||||||
|
"--gc-roots-dir",
|
||||||
|
drv_gcroots_dir,
|
||||||
|
"--force-recurse",
|
||||||
|
"--check-cache-status",
|
||||||
|
"--flake",
|
||||||
|
f".#{FLAKE_TARGET_ATTRIBUTE_FOR_JOBS}"
|
||||||
|
],
|
||||||
|
haltOnFailure=True,
|
||||||
|
locks=[eval_lock.access("exclusive")],
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
factory.addStep(
|
factory.addStep(
|
||||||
|
@ -674,17 +551,12 @@ def nix_build_config(
|
||||||
project: GerritProject,
|
project: GerritProject,
|
||||||
worker_arch: str,
|
worker_arch: str,
|
||||||
worker_names: list[str],
|
worker_names: list[str],
|
||||||
build_stores: list[str],
|
builders_spec: str,
|
||||||
signing_keyfile: str | None = None,
|
signing_keyfile: str | None = None,
|
||||||
binary_cache_config: S3BinaryCacheConfig | None = None
|
binary_cache_config: S3BinaryCacheConfig | None = None
|
||||||
) -> BuilderConfig:
|
) -> util.BuilderConfig:
|
||||||
"""Builds one nix flake attribute."""
|
"""Builds one nix flake attribute."""
|
||||||
factory = util.BuildFactory()
|
factory = util.BuildFactory()
|
||||||
|
|
||||||
# pick a store to run the build on
|
|
||||||
# TODO proper scheduling instead of picking the first builder
|
|
||||||
build_store = build_stores[0]
|
|
||||||
|
|
||||||
factory.addStep(
|
factory.addStep(
|
||||||
NixBuildCommand(
|
NixBuildCommand(
|
||||||
env={},
|
env={},
|
||||||
|
@ -704,10 +576,8 @@ def nix_build_config(
|
||||||
# kill builds after two hours regardless of activity
|
# kill builds after two hours regardless of activity
|
||||||
"--timeout",
|
"--timeout",
|
||||||
"7200",
|
"7200",
|
||||||
"--store",
|
"--builders",
|
||||||
build_store,
|
builders_spec,
|
||||||
"--eval-store",
|
|
||||||
"ssh-ng://localhost",
|
|
||||||
"--out-link",
|
"--out-link",
|
||||||
util.Interpolate("result-%(prop:attr)s"),
|
util.Interpolate("result-%(prop:attr)s"),
|
||||||
util.Interpolate("%(prop:drv_path)s^*"),
|
util.Interpolate("%(prop:drv_path)s^*"),
|
||||||
|
@ -727,8 +597,6 @@ def nix_build_config(
|
||||||
"nix",
|
"nix",
|
||||||
"store",
|
"store",
|
||||||
"sign",
|
"sign",
|
||||||
"--store",
|
|
||||||
build_store,
|
|
||||||
"--key-file",
|
"--key-file",
|
||||||
signing_keyfile,
|
signing_keyfile,
|
||||||
util.Interpolate(
|
util.Interpolate(
|
||||||
|
@ -745,8 +613,6 @@ def nix_build_config(
|
||||||
command=[
|
command=[
|
||||||
"nix",
|
"nix",
|
||||||
"copy",
|
"copy",
|
||||||
"--store",
|
|
||||||
build_store,
|
|
||||||
"--to",
|
"--to",
|
||||||
f"s3://{binary_cache_config.bucket}?profile={binary_cache_config.profile}®ion={binary_cache_config.region}&endpoint={binary_cache_config.endpoint}",
|
f"s3://{binary_cache_config.bucket}?profile={binary_cache_config.profile}®ion={binary_cache_config.region}&endpoint={binary_cache_config.endpoint}",
|
||||||
util.Property(
|
util.Property(
|
||||||
|
@ -807,11 +673,11 @@ def config_for_project(
|
||||||
nix_supported_systems: list[str],
|
nix_supported_systems: list[str],
|
||||||
nix_eval_worker_count: int,
|
nix_eval_worker_count: int,
|
||||||
nix_eval_max_memory_size: int,
|
nix_eval_max_memory_size: int,
|
||||||
eval_lock: MasterLock,
|
eval_lock: util.MasterLock,
|
||||||
nix_builders: list[NixBuilder],
|
builders_spec: str,
|
||||||
signing_keyfile: str | None = None,
|
signing_keyfile: str | None = None,
|
||||||
binary_cache_config: S3BinaryCacheConfig | None = None
|
binary_cache_config: S3BinaryCacheConfig | None = None
|
||||||
) -> None:
|
) -> Project:
|
||||||
config["projects"].append(Project(project.name))
|
config["projects"].append(Project(project.name))
|
||||||
config["schedulers"].extend(
|
config["schedulers"].extend(
|
||||||
[
|
[
|
||||||
|
@ -846,6 +712,12 @@ def config_for_project(
|
||||||
),
|
),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
gerrit_private_key = None
|
||||||
|
with open(project.private_sshkey_path, 'r') as f:
|
||||||
|
gerrit_private_key = f.read()
|
||||||
|
|
||||||
|
if gerrit_private_key is None:
|
||||||
|
raise RuntimeError('No gerrit private key to fetch the repositories')
|
||||||
|
|
||||||
config["builders"].extend(
|
config["builders"].extend(
|
||||||
[
|
[
|
||||||
|
@ -865,7 +737,7 @@ def config_for_project(
|
||||||
project,
|
project,
|
||||||
arch,
|
arch,
|
||||||
[ f"{w}-{arch}" for w in worker_names ],
|
[ f"{w}-{arch}" for w in worker_names ],
|
||||||
[b.to_nix_store() for b in nix_builders if arch in b.systems or arch == "other"],
|
builders_spec,
|
||||||
signing_keyfile=signing_keyfile,
|
signing_keyfile=signing_keyfile,
|
||||||
binary_cache_config=binary_cache_config
|
binary_cache_config=binary_cache_config
|
||||||
)
|
)
|
||||||
|
@ -886,25 +758,25 @@ class PeriodicWithStartup(schedulers.Periodic):
|
||||||
yield self.setState("last_build", None)
|
yield self.setState("last_build", None)
|
||||||
yield super().activate()
|
yield super().activate()
|
||||||
|
|
||||||
def gerritReviewFmt(url: str, payload: CallbackPayloadBuild | CallbackPayloadBuildSet) -> CallbackReturn:
|
def gerritReviewFmt(url, data):
|
||||||
assert isinstance(payload, CallbackPayloadBuild), "BuildSet are not handled yet!"
|
if 'build' not in data:
|
||||||
|
raise ValueError('`build` is supposed to be present to format a build')
|
||||||
|
|
||||||
build = payload.build
|
build = data['build']
|
||||||
if 'builder' not in build and 'name' not in build['builder']:
|
if 'builder' not in build and 'name' not in build['builder']:
|
||||||
raise ValueError('either `builder` or `builder.name` is not present in the build dictionary, unexpected format request')
|
raise ValueError('either `builder` or `builder.name` is not present in the build dictionary, unexpected format request')
|
||||||
|
|
||||||
builderName = build['builder']['name']
|
builderName = build['builder']['name']
|
||||||
|
|
||||||
result = build['results']
|
if len(build['results']) != 1:
|
||||||
log.info("Formatting a message for a Gerrit build: {} -- result is {}".format(builderName, result))
|
raise ValueError('this review request contains more than one build results, unexpected format request')
|
||||||
|
|
||||||
|
result = build['results'][0]
|
||||||
if result == util.RETRY:
|
if result == util.RETRY:
|
||||||
return CallbackReturn()
|
return dict()
|
||||||
|
|
||||||
expectedBuilderName = f'{build["properties"].get("event.project")[0]}/nix-eval'
|
if builderName != f'{build["properties"].get("event.project")}/nix-eval':
|
||||||
|
return dict()
|
||||||
if builderName != expectedBuilderName:
|
|
||||||
log.info("Passing {} builder which is not of the form '{}'".format(builderName, expectedBuilderName))
|
|
||||||
return CallbackReturn()
|
|
||||||
|
|
||||||
failed = build['properties'].get('failed_builds', [[]])[0]
|
failed = build['properties'].get('failed_builds', [[]])[0]
|
||||||
|
|
||||||
|
@ -928,8 +800,7 @@ def gerritReviewFmt(url: str, payload: CallbackPayloadBuild | CallbackPayloadBui
|
||||||
message += "\nFor more details visit:\n"
|
message += "\nFor more details visit:\n"
|
||||||
message += build['url'] + "\n"
|
message += build['url'] + "\n"
|
||||||
|
|
||||||
log.info("Message formatted: {}, labels: Verified={}".format(message, labels['Verified']))
|
return dict(message=message, labels=labels)
|
||||||
return CallbackReturn(body=message, extra_info={'labels': labels})
|
|
||||||
|
|
||||||
class GerritNixConfigurator(ConfiguratorBase):
|
class GerritNixConfigurator(ConfiguratorBase):
|
||||||
"""Janitor is a configurator which create a Janitor Builder with all needed Janitor steps"""
|
"""Janitor is a configurator which create a Janitor Builder with all needed Janitor steps"""
|
||||||
|
@ -953,16 +824,13 @@ class GerritNixConfigurator(ConfiguratorBase):
|
||||||
prometheus_config: dict[str, int | str] | None = None,
|
prometheus_config: dict[str, int | str] | None = None,
|
||||||
binary_cache_config: dict[str, str] | None = None,
|
binary_cache_config: dict[str, str] | None = None,
|
||||||
auth_method: AuthBase | None = None,
|
auth_method: AuthBase | None = None,
|
||||||
manhole: Any = None,
|
|
||||||
enable_profiler: bool = False,
|
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.manhole = manhole
|
|
||||||
self.allowed_origins = allowed_origins
|
self.allowed_origins = allowed_origins
|
||||||
self.gerrit_server = gerrit_server
|
self.gerrit_server = gerrit_server
|
||||||
self.gerrit_user = gerrit_user
|
self.gerrit_user = gerrit_user
|
||||||
self.gerrit_port = gerrit_port
|
self.gerrit_port = gerrit_port
|
||||||
self.gerrit_sshkey_path = str(gerrit_sshkey_path)
|
self.gerrit_sshkey_path = gerrit_sshkey_path
|
||||||
self.gerrit_config = GerritConfig(domain=self.gerrit_server,
|
self.gerrit_config = GerritConfig(domain=self.gerrit_server,
|
||||||
username=self.gerrit_user,
|
username=self.gerrit_user,
|
||||||
port=self.gerrit_port)
|
port=self.gerrit_port)
|
||||||
|
@ -978,7 +846,6 @@ class GerritNixConfigurator(ConfiguratorBase):
|
||||||
|
|
||||||
self.url = url
|
self.url = url
|
||||||
self.prometheus_config = prometheus_config
|
self.prometheus_config = prometheus_config
|
||||||
self.enable_profiler = enable_profiler
|
|
||||||
|
|
||||||
if binary_cache_config is not None:
|
if binary_cache_config is not None:
|
||||||
self.binary_cache_config = S3BinaryCacheConfig(**binary_cache_config)
|
self.binary_cache_config = S3BinaryCacheConfig(**binary_cache_config)
|
||||||
|
@ -989,32 +856,30 @@ class GerritNixConfigurator(ConfiguratorBase):
|
||||||
|
|
||||||
self.auth_method = auth_method
|
self.auth_method = auth_method
|
||||||
|
|
||||||
def configure(self, config_dict: dict[str, Any]) -> None:
|
def configure(self, config: dict[str, Any]) -> None:
|
||||||
worker_config_dict = json.loads(read_secret_file(self.nix_workers_secret_name))
|
worker_config = json.loads(read_secret_file(self.nix_workers_secret_name))
|
||||||
worker_names = []
|
worker_names = []
|
||||||
|
|
||||||
if self.manhole is not None:
|
config.setdefault("projects", [])
|
||||||
config_dict["manhole"] = self.manhole
|
config.setdefault("secretsProviders", [])
|
||||||
|
config.setdefault("www", {
|
||||||
|
'allowed_origins': self.allowed_origins
|
||||||
|
})
|
||||||
|
|
||||||
config_dict.setdefault("projects", [])
|
for item in worker_config:
|
||||||
config_dict.setdefault("secretsProviders", [])
|
|
||||||
|
|
||||||
print('Default allowed origins for this Buildbot server: {}'.format(', '.join(self.allowed_origins)))
|
|
||||||
config_dict["www"]["allowed_origins"] = self.allowed_origins
|
|
||||||
|
|
||||||
for item in worker_config_dict:
|
|
||||||
cores = item.get("cores", 0)
|
cores = item.get("cores", 0)
|
||||||
for i in range(cores):
|
for i in range(cores):
|
||||||
for arch in self.nix_supported_systems + ["other"]:
|
for arch in self.nix_supported_systems + ["other"]:
|
||||||
worker_name = f"{item['name']}-{i:03}"
|
worker_name = f"{item['name']}-{i:03}"
|
||||||
config_dict["workers"].append(worker.Worker(f"{worker_name}-{arch}", item["pass"]))
|
config["workers"].append(worker.Worker(f"{worker_name}-{arch}", item["pass"]))
|
||||||
worker_names.append(worker_name)
|
worker_names.append(worker_name)
|
||||||
|
|
||||||
eval_lock = util.MasterLock("nix-eval")
|
eval_lock = util.MasterLock("nix-eval")
|
||||||
|
|
||||||
|
builders_spec = " ; ".join(builder.to_nix_line() for builder in self.nix_builders)
|
||||||
for project in self.projects:
|
for project in self.projects:
|
||||||
config_for_project(
|
config_for_project(
|
||||||
config_dict,
|
config,
|
||||||
self.gerrit_config,
|
self.gerrit_config,
|
||||||
GerritProject(name=project, private_sshkey_path=self.gerrit_sshkey_path),
|
GerritProject(name=project, private_sshkey_path=self.gerrit_sshkey_path),
|
||||||
worker_names,
|
worker_names,
|
||||||
|
@ -1022,20 +887,20 @@ class GerritNixConfigurator(ConfiguratorBase):
|
||||||
self.nix_eval_worker_count or multiprocessing.cpu_count(),
|
self.nix_eval_worker_count or multiprocessing.cpu_count(),
|
||||||
self.nix_eval_max_memory_size,
|
self.nix_eval_max_memory_size,
|
||||||
eval_lock,
|
eval_lock,
|
||||||
self.nix_builders,
|
builders_spec,
|
||||||
signing_keyfile=self.signing_keyfile,
|
signing_keyfile=self.signing_keyfile,
|
||||||
binary_cache_config=self.binary_cache_config
|
binary_cache_config=self.binary_cache_config
|
||||||
)
|
)
|
||||||
|
|
||||||
config_dict["change_source"] = self.gerrit_change_source
|
config["change_source"] = self.gerrit_change_source
|
||||||
config_dict["services"].append(
|
config["services"].append(
|
||||||
reporters.GerritStatusPush(self.gerrit_server, self.gerrit_user,
|
reporters.GerritStatusPush(self.gerrit_server, self.gerrit_user,
|
||||||
port=self.gerrit_port,
|
port=self.gerrit_port,
|
||||||
identity_file=self.gerrit_sshkey_path,
|
identity_file=self.gerrit_sshkey_path,
|
||||||
generators=[
|
generators=[
|
||||||
|
# gerritReviewCB / self.url
|
||||||
BuildStatusGenerator(
|
BuildStatusGenerator(
|
||||||
mode='all',
|
message_formatter=MessageFormatterFunction(
|
||||||
message_formatter=ReasonableMessageFormatter(
|
|
||||||
lambda data: gerritReviewFmt(self.url, data),
|
lambda data: gerritReviewFmt(self.url, data),
|
||||||
"plain",
|
"plain",
|
||||||
want_properties=True,
|
want_properties=True,
|
||||||
|
@ -1043,10 +908,11 @@ class GerritNixConfigurator(ConfiguratorBase):
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
])
|
])
|
||||||
|
# startCB, summaryCB are too noisy, we won't use them.
|
||||||
)
|
)
|
||||||
|
|
||||||
if self.prometheus_config is not None:
|
if self.prometheus_config is not None:
|
||||||
config_dict['services'].append(reporters.Prometheus(port=self.prometheus_config.get('port', 9100), interface=self.prometheus_config.get('address', '')))
|
config['services'].append(reporters.Prometheus(port=self.prometheus_config.get('port', 9100), interface=self.prometheus_config.get('address', '')))
|
||||||
|
|
||||||
# Upstream defaults pretend they already do something similar
|
# Upstream defaults pretend they already do something similar
|
||||||
# but they didn't work, hence the custom function.
|
# but they didn't work, hence the custom function.
|
||||||
|
@ -1056,7 +922,7 @@ class GerritNixConfigurator(ConfiguratorBase):
|
||||||
return ref
|
return ref
|
||||||
return ref.rsplit('/', 1)[0]
|
return ref.rsplit('/', 1)[0]
|
||||||
|
|
||||||
config_dict["services"].append(
|
config["services"].append(
|
||||||
util.OldBuildCanceller(
|
util.OldBuildCanceller(
|
||||||
"build_canceller",
|
"build_canceller",
|
||||||
filters=[
|
filters=[
|
||||||
|
@ -1079,15 +945,12 @@ class GerritNixConfigurator(ConfiguratorBase):
|
||||||
systemd_secrets = secrets.SecretInAFile(
|
systemd_secrets = secrets.SecretInAFile(
|
||||||
dirname=os.environ["CREDENTIALS_DIRECTORY"],
|
dirname=os.environ["CREDENTIALS_DIRECTORY"],
|
||||||
)
|
)
|
||||||
config_dict["secretsProviders"].append(systemd_secrets)
|
config["secretsProviders"].append(systemd_secrets)
|
||||||
|
|
||||||
config_dict["www"].setdefault("plugins", {})
|
config["www"].setdefault("plugins", {})
|
||||||
|
|
||||||
if self.enable_profiler:
|
if "authz" not in config["www"]:
|
||||||
config_dict["www"]["plugins"]["profiler"] = True
|
config["www"]["authz"] = util.Authz(
|
||||||
|
|
||||||
if "authz" not in config_dict["www"]:
|
|
||||||
config_dict["www"]["authz"] = util.Authz(
|
|
||||||
allowRules=[
|
allowRules=[
|
||||||
util.AnyEndpointMatcher(role="admin", defaultDeny=False),
|
util.AnyEndpointMatcher(role="admin", defaultDeny=False),
|
||||||
util.StopBuildEndpointMatcher(role="owner"),
|
util.StopBuildEndpointMatcher(role="owner"),
|
||||||
|
@ -1101,5 +964,5 @@ class GerritNixConfigurator(ConfiguratorBase):
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
if "auth" not in config_dict["www"] and self.auth_method is not None:
|
if "auth" not in config["www"] and self.auth_method is not None:
|
||||||
config_dict["www"]["auth"] = self.auth_method
|
config["www"]["auth"] = self.auth_method
|
||||||
|
|
|
@ -1,46 +0,0 @@
|
||||||
from typing import Any, Callable, Literal
|
|
||||||
from buildbot.reporters.message import MessageFormatterBase
|
|
||||||
import dataclasses
|
|
||||||
|
|
||||||
@dataclasses.dataclass
|
|
||||||
class CallbackPayloadBuild:
|
|
||||||
# buddy i have no idea what the fuck is in this
|
|
||||||
build: dict[str, Any]
|
|
||||||
|
|
||||||
@dataclasses.dataclass
|
|
||||||
class CallbackPayloadBuildSet:
|
|
||||||
buildset: dict[str, Any]
|
|
||||||
# i have no idea what the fuck is in this honestly
|
|
||||||
builds: Any
|
|
||||||
|
|
||||||
@dataclasses.dataclass
|
|
||||||
class CallbackReturn:
|
|
||||||
body: str | None = None
|
|
||||||
subject: str | None = None
|
|
||||||
type: Literal['plain'] | Literal['html'] | Literal['json'] = 'plain'
|
|
||||||
extra_info: dict[str, Any] | None = None
|
|
||||||
# FIXME: support other template types, if they actually become necessary
|
|
||||||
template_type: Literal['plain'] = 'plain'
|
|
||||||
|
|
||||||
class ReasonableMessageFormatter(MessageFormatterBase):
|
|
||||||
"""
|
|
||||||
Message formatter which uses strongly typed data classes to reduce suffering slightly.
|
|
||||||
"""
|
|
||||||
CallbackFunc = Callable[[CallbackPayloadBuild | CallbackPayloadBuildSet], CallbackReturn]
|
|
||||||
def __init__(self, function: CallbackFunc, template_type: str, **kwargs):
|
|
||||||
super().__init__(**kwargs)
|
|
||||||
self.template_type = template_type
|
|
||||||
self._function = function
|
|
||||||
|
|
||||||
def format_message_for_build(self, master, build, **kwargs):
|
|
||||||
return dataclasses.asdict(self._function(CallbackPayloadBuild(build=build)))
|
|
||||||
|
|
||||||
def format_message_for_buildset(self, master, buildset, builds, **kwargs):
|
|
||||||
return dataclasses.asdict(self._function(CallbackPayloadBuildSet(buildset=buildset, builds=builds)))
|
|
||||||
|
|
||||||
# These only exist as callbacks, the only one actually used is render_message_dict
|
|
||||||
def render_message_body(self, context):
|
|
||||||
return None
|
|
||||||
|
|
||||||
def render_message_subject(self, context):
|
|
||||||
return None
|
|
|
@ -1,5 +1,4 @@
|
||||||
{ setuptools, buildPythonPackage }:
|
{ setuptools, buildPythonPackage }:
|
||||||
# TODO: figure out how to make this depend upon buildbot without leaking it further.
|
|
||||||
buildPythonPackage {
|
buildPythonPackage {
|
||||||
name = "buildbot-nix";
|
name = "buildbot-nix";
|
||||||
format = "pyproject";
|
format = "pyproject";
|
||||||
|
|
|
@ -42,11 +42,6 @@
|
||||||
pkgs.ruff
|
pkgs.ruff
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
devShells.default = pkgs.mkShell {
|
|
||||||
packages = [
|
|
||||||
self'.packages.buildbot-nix
|
|
||||||
];
|
|
||||||
};
|
|
||||||
packages.buildbot-nix = pkgs.python3.pkgs.callPackage ./default.nix { };
|
packages.buildbot-nix = pkgs.python3.pkgs.callPackage ./default.nix { };
|
||||||
checks =
|
checks =
|
||||||
let
|
let
|
||||||
|
|
|
@ -7,9 +7,6 @@
|
||||||
let
|
let
|
||||||
inherit (lib) filterAttrs;
|
inherit (lib) filterAttrs;
|
||||||
cfg = config.services.buildbot-nix.coordinator;
|
cfg = config.services.buildbot-nix.coordinator;
|
||||||
debuggingManhole = if cfg.debugging.enable then
|
|
||||||
"manhole.TelnetManhole(${toString cfg.debugging.port}, 'admin', 'admin')"
|
|
||||||
else "None";
|
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
options = {
|
options = {
|
||||||
|
@ -31,14 +28,6 @@ in
|
||||||
description = "List of local remote builders machines associated to that Buildbot instance";
|
description = "List of local remote builders machines associated to that Buildbot instance";
|
||||||
};
|
};
|
||||||
|
|
||||||
debugging = {
|
|
||||||
enable = lib.mkEnableOption "manhole's buildbot debugging on localhost using `admin:admin`";
|
|
||||||
port = lib.mkOption {
|
|
||||||
type = lib.types.port;
|
|
||||||
default = 15000;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
oauth2 = {
|
oauth2 = {
|
||||||
name = lib.mkOption {
|
name = lib.mkOption {
|
||||||
type = lib.types.str;
|
type = lib.types.str;
|
||||||
|
@ -140,8 +129,6 @@ in
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
profiler.enable = lib.mkEnableOption " the profiling of the buildbot instance";
|
|
||||||
|
|
||||||
gerrit = {
|
gerrit = {
|
||||||
domain = lib.mkOption {
|
domain = lib.mkOption {
|
||||||
type = lib.types.str;
|
type = lib.types.str;
|
||||||
|
@ -229,7 +216,6 @@ in
|
||||||
extraImports = ''
|
extraImports = ''
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
from buildbot_nix import GerritNixConfigurator, read_secret_file, make_oauth2_method, OAuth2Config, assemble_secret_file_path
|
from buildbot_nix import GerritNixConfigurator, read_secret_file, make_oauth2_method, OAuth2Config, assemble_secret_file_path
|
||||||
from buildbot import manhole
|
|
||||||
|
|
||||||
# TODO(raito): make me configurable from the NixOS module.
|
# TODO(raito): make me configurable from the NixOS module.
|
||||||
# how?
|
# how?
|
||||||
|
@ -271,9 +257,7 @@ in
|
||||||
auth_method=CustomOAuth2(${builtins.toJSON cfg.oauth2.clientId},
|
auth_method=CustomOAuth2(${builtins.toJSON cfg.oauth2.clientId},
|
||||||
read_secret_file('buildbot-oauth2-secret'),
|
read_secret_file('buildbot-oauth2-secret'),
|
||||||
autologin=True
|
autologin=True
|
||||||
),
|
)
|
||||||
manhole=${debuggingManhole},
|
|
||||||
enable_profiler=${builtins.toJSON cfg.profiler.enable}
|
|
||||||
)
|
)
|
||||||
''
|
''
|
||||||
];
|
];
|
||||||
|
@ -291,9 +275,7 @@ in
|
||||||
(ps.toPythonModule pkgs.buildbot-worker)
|
(ps.toPythonModule pkgs.buildbot-worker)
|
||||||
pkgs.buildbot-plugins.www
|
pkgs.buildbot-plugins.www
|
||||||
(pkgs.python3.pkgs.callPackage ../default.nix { })
|
(pkgs.python3.pkgs.callPackage ../default.nix { })
|
||||||
]
|
] ++ lib.optional cfg.prometheus.enable (pkgs.python3.pkgs.callPackage ./prometheus-plugin.nix { })
|
||||||
++ lib.optional cfg.prometheus.enable (pkgs.python3.pkgs.callPackage ./prometheus-plugin.nix { })
|
|
||||||
++ lib.optional cfg.profiler.enable (pkgs.python3.pkgs.callPackage ./profiler-plugin.nix { })
|
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -1,36 +0,0 @@
|
||||||
{
|
|
||||||
lib,
|
|
||||||
buildPythonPackage,
|
|
||||||
fetchFromGitHub,
|
|
||||||
setuptools,
|
|
||||||
wheel,
|
|
||||||
}:
|
|
||||||
|
|
||||||
buildPythonPackage rec {
|
|
||||||
pname = "buildbot-profiler";
|
|
||||||
version = "1.2.2";
|
|
||||||
pyproject = true;
|
|
||||||
|
|
||||||
src = fetchFromGitHub {
|
|
||||||
owner = "tardyp";
|
|
||||||
repo = "buildbot_profiler";
|
|
||||||
rev = "v${version}";
|
|
||||||
hash = "sha256-r56Cze0v3gKWKJwRy0BUYz5z8d0g3jerCIu3KreNxUc=";
|
|
||||||
};
|
|
||||||
|
|
||||||
build-system = [
|
|
||||||
setuptools
|
|
||||||
wheel
|
|
||||||
];
|
|
||||||
|
|
||||||
pythonImportsCheck = [
|
|
||||||
"buildbot_profiler"
|
|
||||||
];
|
|
||||||
|
|
||||||
meta = {
|
|
||||||
description = "";
|
|
||||||
homepage = "https://github.com/tardyp/buildbot_profiler";
|
|
||||||
license = lib.licenses.unfree; # FIXME: nix-init did not find a license
|
|
||||||
maintainers = with lib.maintainers; [ raitobezarius ];
|
|
||||||
};
|
|
||||||
}
|
|
|
@ -62,7 +62,6 @@ in
|
||||||
pkgs.openssh
|
pkgs.openssh
|
||||||
pkgs.nix
|
pkgs.nix
|
||||||
pkgs.nix-eval-jobs
|
pkgs.nix-eval-jobs
|
||||||
pkgs.bash
|
|
||||||
];
|
];
|
||||||
environment.PYTHONPATH = "${python.withPackages (_: [cfg.package])}/${python.sitePackages}";
|
environment.PYTHONPATH = "${python.withPackages (_: [cfg.package])}/${python.sitePackages}";
|
||||||
environment.MASTER_URL = cfg.coordinatorUrl;
|
environment.MASTER_URL = cfg.coordinatorUrl;
|
||||||
|
|
Loading…
Reference in a new issue