lix-project/buildbot-nix
15
0
Fork 0
Code Issues 13 Pull requests 7 Projects Releases Packages Wiki Activity

allow worker counts to be set per arch

Browse source
This commit is contained in:
eldritch horrors 2024-04-05 15:13:11 +02:00
parent daa84f4169
commit 131fc792f7
2 changed files with 15 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
buildbot_nix/worker.py
View file

@ -22,11 +22,14 @@ class WorkerConfig:
.read_text() .read_text()
.rstrip("\r\n") .rstrip("\r\n")
) )
worker_count: int = int( worker_arch_list: dict[str, int] = field(
os.environ.get("WORKER_COUNT", str(multiprocessing.cpu_count())), default_factory=lambda: dict(other=1) | {
) arch: int(count)
worker_arch_list: list[str] = field( for arch, count in (
default_factory=lambda: os.environ.get("WORKER_ARCH_LIST", "").split(",") + ["other"], e.split("=")
for e in os.environ.get("WORKER_ARCH_LIST", "").split(",")
)
},
) )
buildbot_dir: Path = field( buildbot_dir: Path = field(
default_factory=lambda: Path(require_env("BUILDBOT_DIR")) default_factory=lambda: Path(require_env("BUILDBOT_DIR"))
@ -70,8 +73,8 @@ def setup_worker(
def setup_workers(application: components.Componentized, config: WorkerConfig) -> None: def setup_workers(application: components.Componentized, config: WorkerConfig) -> None:
for i in range(config.worker_count): for arch, jobs in config.worker_arch_list.items():
for arch in config.worker_arch_list: for i in range(jobs):
setup_worker(application, i, arch, config) setup_worker(application, i, arch, config)

8
nix/worker.nix
View file

@ -28,8 +28,8 @@ in
type = lib.types.path; type = lib.types.path;
description = "The buildbot worker password file."; description = "The buildbot worker password file.";
}; };
workerArchList = lib.mkOption { workerArchitectures = lib.mkOption {
type = lib.types.listOf lib.types.str; type = lib.types.attrsOf lib.types.int;
description = "Nix `system`s the worker should feel responsible for."; description = "Nix `system`s the worker should feel responsible for.";
}; };
}; };
@ -66,6 +66,9 @@ in
environment.PYTHONPATH = "${python.withPackages (_: [cfg.package])}/${python.sitePackages}"; environment.PYTHONPATH = "${python.withPackages (_: [cfg.package])}/${python.sitePackages}";
environment.MASTER_URL = cfg.coordinatorUrl; environment.MASTER_URL = cfg.coordinatorUrl;
environment.BUILDBOT_DIR = buildbotDir; environment.BUILDBOT_DIR = buildbotDir;
environment.WORKER_ARCH_LIST =
lib.concatStringsSep ","
(lib.mapAttrsToList (arch: jobs: "${arch}=${toString jobs}") cfg.workerArchitectures);
serviceConfig = { serviceConfig = {
# We rather want the CI job to fail on OOM than to have a broken buildbot worker. # We rather want the CI job to fail on OOM than to have a broken buildbot worker.
@ -75,7 +78,6 @@ in
LoadCredential = [ "worker-password-file:${cfg.workerPasswordFile}" ]; LoadCredential = [ "worker-password-file:${cfg.workerPasswordFile}" ];
Environment = [ Environment = [
"WORKER_PASSWORD_FILE=%d/worker-password-file" "WORKER_PASSWORD_FILE=%d/worker-password-file"
"WORKER_ARCH_LIST=${lib.concatStringsSep "," cfg.workerArchList}"
]; ];
Type = "simple"; Type = "simple";
User = "buildbot-worker"; User = "buildbot-worker";