chore(builders): localize builders specification like Hydra does
Signed-off-by: Raito Bezarius <raito@lix.systems>
This commit is contained in:
parent
235ff9b138
commit
ea5e2c6b98
2 changed files with 42 additions and 2 deletions
|
|
@ -3,6 +3,7 @@ import multiprocessing
|
|||
import os
|
||||
import sys
|
||||
import graphlib
|
||||
import base64
|
||||
from collections.abc import Generator
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
|
@ -38,6 +39,26 @@ log = Logger()
|
|||
|
||||
FLAKE_TARGET_ATTRIBUTE_FOR_JOBS = "buildbotJobs"
|
||||
|
||||
@dataclass
|
||||
class NixBuilder:
|
||||
protocol: str
|
||||
hostName: str
|
||||
maxJobs: int
|
||||
speedFactor: int = 1
|
||||
# without base64
|
||||
publicHostKey: str | None = None
|
||||
sshUser: str | None = None
|
||||
sshKey: str | None = None
|
||||
systems: list[str] = ["-"]
|
||||
supportedFeatures: list[str] = ["-"]
|
||||
mandatoryFeatures: list[str] = ["-"]
|
||||
|
||||
def to_nix_line(self):
|
||||
encoded_public_key = base64.b64encode(self.publicHostKey.encode('ascii')).decode('ascii') if self.publicHostKey is not None else "-"
|
||||
fullConnection = f"{self.protocol}://{self.sshUser}@{self.hostName}" if self.sshUser is not None else self.hostName
|
||||
return f"{fullConnection} {",".join(self.systems)} {self.sshKey or "-"} {self.maxJobs} {self.speedFactor} {",".join(self.supportedFeatures)} {",".join(self.mandatoryFeatures)} {encoded_public_key}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class OAuth2Config:
|
||||
name: str
|
||||
|
|
@ -530,6 +551,7 @@ def nix_build_config(
|
|||
project: GerritProject,
|
||||
worker_arch: str,
|
||||
worker_names: list[str],
|
||||
builders_spec: str,
|
||||
signing_keyfile: str | None = None,
|
||||
binary_cache_config: S3BinaryCacheConfig | None = None
|
||||
) -> util.BuilderConfig:
|
||||
|
|
@ -549,9 +571,13 @@ def nix_build_config(
|
|||
# do not build directly on the coordinator
|
||||
"--max-jobs", "0",
|
||||
# stop stuck builds after 20 minutes
|
||||
"--max-silent-time", str(60 * 20),
|
||||
"--max-silent-time",
|
||||
str(60 * 20),
|
||||
# kill builds after two hours regardless of activity
|
||||
"--timeout", "7200",
|
||||
"--timeout",
|
||||
"7200",
|
||||
"--builders",
|
||||
builders_spec,
|
||||
"--out-link",
|
||||
util.Interpolate("result-%(prop:attr)s"),
|
||||
util.Interpolate("%(prop:drv_path)s^*"),
|
||||
|
|
@ -648,6 +674,7 @@ def config_for_project(
|
|||
nix_eval_worker_count: int,
|
||||
nix_eval_max_memory_size: int,
|
||||
eval_lock: util.MasterLock,
|
||||
builders_spec: str,
|
||||
signing_keyfile: str | None = None,
|
||||
binary_cache_config: S3BinaryCacheConfig | None = None
|
||||
) -> Project:
|
||||
|
|
@ -710,6 +737,7 @@ def config_for_project(
|
|||
project,
|
||||
arch,
|
||||
[ f"{w}-{arch}" for w in worker_names ],
|
||||
builders_spec,
|
||||
signing_keyfile=signing_keyfile,
|
||||
binary_cache_config=binary_cache_config
|
||||
)
|
||||
|
|
@ -787,6 +815,7 @@ class GerritNixConfigurator(ConfiguratorBase):
|
|||
projects: list[str],
|
||||
url: str,
|
||||
allowed_origins: list[str],
|
||||
nix_builders: list[dict[str, Any]],
|
||||
nix_supported_systems: list[str],
|
||||
nix_eval_worker_count: int | None,
|
||||
nix_eval_max_memory_size: int,
|
||||
|
|
@ -811,6 +840,7 @@ class GerritNixConfigurator(ConfiguratorBase):
|
|||
self.nix_eval_max_memory_size = nix_eval_max_memory_size
|
||||
self.nix_eval_worker_count = nix_eval_worker_count
|
||||
self.nix_supported_systems = nix_supported_systems
|
||||
self.nix_builders: list[NixBuilder] = [NixBuilder(**builder_cfg) for builder_cfg in nix_builders]
|
||||
|
||||
self.gerrit_change_source = GerritChangeSource(gerrit_server, gerrit_user, gerritport=gerrit_port, identity_file=gerrit_sshkey_path)
|
||||
|
||||
|
|
@ -846,6 +876,7 @@ class GerritNixConfigurator(ConfiguratorBase):
|
|||
|
||||
eval_lock = util.MasterLock("nix-eval")
|
||||
|
||||
builders_spec = " ; ".join(builder.to_nix_line() for builder in self.nix_builders)
|
||||
for project in self.projects:
|
||||
config_for_project(
|
||||
config,
|
||||
|
|
@ -856,6 +887,7 @@ class GerritNixConfigurator(ConfiguratorBase):
|
|||
self.nix_eval_worker_count or multiprocessing.cpu_count(),
|
||||
self.nix_eval_max_memory_size,
|
||||
eval_lock,
|
||||
builders_spec,
|
||||
signing_keyfile=self.signing_keyfile,
|
||||
binary_cache_config=self.binary_cache_config
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
{ config
|
||||
, options
|
||||
, pkgs
|
||||
, lib
|
||||
, ...
|
||||
|
|
@ -15,11 +16,17 @@ in
|
|||
default = "postgresql://@/buildbot";
|
||||
description = "Postgresql database url";
|
||||
};
|
||||
|
||||
workersFile = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
description = "File containing a list of nix workers";
|
||||
};
|
||||
|
||||
buildMachines = lib.mkOption {
|
||||
type = options.nix.buildMachines.type;
|
||||
description = "List of local remote builders machines associated to that Buildbot instance";
|
||||
};
|
||||
|
||||
oauth2 = {
|
||||
name = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
|
|
@ -239,6 +246,7 @@ in
|
|||
prometheus_config=${if (!cfg.prometheus.enable) then "None" else builtins.toJSON {
|
||||
inherit (cfg.prometheus) address port;
|
||||
}},
|
||||
nix_builders=${builtins.toJSON cfg.buildMachines},
|
||||
# Signing key file must be available on the workers and readable.
|
||||
signing_keyfile=${if cfg.signingKeyFile == null then "None" else builtins.toJSON cfg.signingKeyFile},
|
||||
binary_cache_config=${if (!cfg.binaryCache.enable) then "None" else builtins.toJSON {
|
||||
|
|
|
|||
Loading…
Reference in a new issue