eldritch horrors
d394f35f55
and an additional set for generic tasks like error reporting. this prevents hol blocking for underutilized arches when at least one arch is blocking, as usually happens to us with aarch64-linux.
82 lines
2.3 KiB
Python
82 lines
2.3 KiB
Python
import multiprocessing
|
|
import os
|
|
import socket
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
from buildbot_worker.bot import Worker
|
|
from twisted.application import service
|
|
from twisted.python import components
|
|
|
|
|
|
def require_env(key: str) -> str:
|
|
val = os.environ.get(key)
|
|
assert val is not None, "val is not set"
|
|
return val
|
|
|
|
|
|
@dataclass
|
|
class WorkerConfig:
|
|
password: str = field(
|
|
default_factory=lambda: Path(require_env("WORKER_PASSWORD_FILE"))
|
|
.read_text()
|
|
.rstrip("\r\n")
|
|
)
|
|
worker_count: int = int(
|
|
os.environ.get("WORKER_COUNT", str(multiprocessing.cpu_count())),
|
|
)
|
|
worker_arch_list: list[str] = field(
|
|
default_factory=lambda: os.environ.get("WORKER_ARCH_LIST", "").split(",") + ["other"],
|
|
)
|
|
buildbot_dir: Path = field(
|
|
default_factory=lambda: Path(require_env("BUILDBOT_DIR"))
|
|
)
|
|
master_url: str = field(default_factory=lambda: require_env("MASTER_URL"))
|
|
|
|
|
|
def setup_worker(
|
|
application: components.Componentized,
|
|
builder_id: int,
|
|
arch: str,
|
|
config: WorkerConfig,
|
|
) -> None:
|
|
basedir = config.buildbot_dir.parent / f"{config.buildbot_dir.name}-{builder_id:03}/{arch}"
|
|
basedir.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
|
|
hostname = socket.gethostname()
|
|
workername = f"{hostname}-{builder_id:03}-{arch}"
|
|
keepalive = 600
|
|
umask = None
|
|
maxdelay = 300
|
|
numcpus = None
|
|
allow_shutdown = None
|
|
|
|
s = Worker(
|
|
None,
|
|
None,
|
|
workername,
|
|
config.password,
|
|
str(basedir),
|
|
keepalive,
|
|
connection_string=config.master_url,
|
|
umask=umask,
|
|
maxdelay=maxdelay,
|
|
numcpus=numcpus,
|
|
allow_shutdown=allow_shutdown,
|
|
)
|
|
# defaults to 4096, bump to 10MB for nix-eval-jobs
|
|
s.bot.max_line_length = 10485760
|
|
s.setServiceParent(application)
|
|
|
|
|
|
def setup_workers(application: components.Componentized, config: WorkerConfig) -> None:
|
|
for i in range(config.worker_count):
|
|
for arch in config.worker_arch_list:
|
|
setup_worker(application, i, arch, config)
|
|
|
|
|
|
# note: this line is matched against to check that this is a worker
|
|
# directory; do not edit it.
|
|
application = service.Application("buildbot-worker") # type: ignore[no-untyped-call]
|
|
setup_workers(application, WorkerConfig())
|