lix-project/buildbot-nix
15
0
Fork 0
Code Issues 6 Pull requests 2 Packages Projects Releases Wiki Activity

allow worker counts to be set per arch

Browse source
This commit is contained in:
eldritch horrors 2024-04-05 15:13:11 +02:00
parent daa84f4169
commit 131fc792f7
2 changed files with 15 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
buildbot_nix/worker.py
View file

@ -22,11 +22,14 @@ class WorkerConfig:
.read_text()
.rstrip("\r\n")
)
worker_count: int = int(
os.environ.get("WORKER_COUNT", str(multiprocessing.cpu_count())),
)
worker_arch_list: list[str] = field(
default_factory=lambda: os.environ.get("WORKER_ARCH_LIST", "").split(",") + ["other"],
worker_arch_list: dict[str, int] = field(
default_factory=lambda: dict(other=1) | {
arch: int(count)
for arch, count in (
e.split("=")
for e in os.environ.get("WORKER_ARCH_LIST", "").split(",")
)
},
)
buildbot_dir: Path = field(
default_factory=lambda: Path(require_env("BUILDBOT_DIR"))
@ -70,8 +73,8 @@ def setup_worker(
def setup_workers(application: components.Componentized, config: WorkerConfig) -> None:
for i in range(config.worker_count):
for arch in config.worker_arch_list:
for arch, jobs in config.worker_arch_list.items():
for i in range(jobs):
setup_worker(application, i, arch, config)

8
nix/worker.nix
View file

@ -28,8 +28,8 @@ in
type = lib.types.path;
description = "The buildbot worker password file.";
};
workerArchList = lib.mkOption {
type = lib.types.listOf lib.types.str;
workerArchitectures = lib.mkOption {
type = lib.types.attrsOf lib.types.int;
description = "Nix `system`s the worker should feel responsible for.";
};
};
@ -66,6 +66,9 @@ in
environment.PYTHONPATH = "${python.withPackages (_: [cfg.package])}/${python.sitePackages}";
environment.MASTER_URL = cfg.coordinatorUrl;
environment.BUILDBOT_DIR = buildbotDir;
environment.WORKER_ARCH_LIST =
lib.concatStringsSep ","
(lib.mapAttrsToList (arch: jobs: "${arch}=${toString jobs}") cfg.workerArchitectures);
serviceConfig = {
# We rather want the CI job to fail on OOM than to have a broken buildbot worker.
@ -75,7 +78,6 @@ in
LoadCredential = [ "worker-password-file:${cfg.workerPasswordFile}" ];
Environment = [
"WORKER_PASSWORD_FILE=%d/worker-password-file"
"WORKER_ARCH_LIST=${lib.concatStringsSep "," cfg.workerArchList}"
];
Type = "simple";
User = "buildbot-worker";