lix-project/buildbot-nix
15
0
Fork 0
Code Issues 6 Pull requests 2 Packages Projects Releases Wiki Activity

Merge pull request 'feat: support Gerrit in Buildbot' (#1) from gerrit into main

Browse source 
Reviewed-on: #1
This commit is contained in:
Qyriad 2024-04-30 19:41:58 +00:00
parent 7a44c9f71c 131fc792f7
commit ea4b9ce350
7 changed files with 614 additions and 629 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

996
buildbot_nix/__init__.py
View file

File diff suppressed because it is too large Load diff

12
buildbot_nix/binary_cache.py Normal file
View file

@ -0,0 +1,12 @@
from dataclasses import dataclass
@dataclass
class S3BinaryCacheConfig:
region: str
bucket: str
endpoint: str
profile: str
class LocalSigner:
def __init__(self, keyfile: str):
self.keyfile = keyfile

20
buildbot_nix/worker.py
View file

@ -22,8 +22,14 @@ class WorkerConfig:
.read_text()
.rstrip("\r\n")
)
worker_count: int = int(
os.environ.get("WORKER_COUNT", str(multiprocessing.cpu_count())),
worker_arch_list: dict[str, int] = field(
default_factory=lambda: dict(other=1) | {
arch: int(count)
for arch, count in (
e.split("=")
for e in os.environ.get("WORKER_ARCH_LIST", "").split(",")
)
},
)
buildbot_dir: Path = field(
default_factory=lambda: Path(require_env("BUILDBOT_DIR"))
@ -34,13 +40,14 @@ class WorkerConfig:
def setup_worker(
application: components.Componentized,
builder_id: int,
arch: str,
config: WorkerConfig,
) -> None:
basedir = config.buildbot_dir.parent / f"{config.buildbot_dir.name}-{builder_id:03}"
basedir = config.buildbot_dir.parent / f"{config.buildbot_dir.name}-{builder_id:03}/{arch}"
basedir.mkdir(parents=True, exist_ok=True, mode=0o700)
hostname = socket.gethostname()
workername = f"{hostname}-{builder_id:03}"
workername = f"{hostname}-{builder_id:03}-{arch}"
keepalive = 600
umask = None
maxdelay = 300
@ -66,8 +73,9 @@ def setup_worker(
def setup_workers(application: components.Componentized, config: WorkerConfig) -> None:
for i in range(config.worker_count):
setup_worker(application, i, config)
for arch, jobs in config.worker_arch_list.items():
for i in range(jobs):
setup_worker(application, i, arch, config)
# note: this line is matched against to check that this is a worker

8
examples/default.nix
View file

@ -46,14 +46,6 @@ in
# optional nix-eval-jobs settings
# evalWorkerCount = 8; # limit number of concurrent evaluations
# evalMaxMemorySize = "2048"; # limit memory usage per evaluation
# optional cachix
#cachix = {
# name = "my-cachix";
# # One of the following is required:
# signingKey = "/var/lib/secrets/cachix-key";
# authToken = "/var/lib/secrets/cachix-token";
#};
};
})
buildbot-nix.nixosModules.buildbot-master

2
flake.nix
View file

@ -20,7 +20,7 @@
] ++ inputs.nixpkgs.lib.optional (inputs.treefmt-nix ? flakeModule) ./nix/treefmt/flake-module.nix;
systems = [ "x86_64-linux" ];
flake = {
nixosModules.buildbot-master = ./nix/master.nix;
nixosModules.buildbot-coordinator = ./nix/coordinator.nix;
nixosModules.buildbot-worker = ./nix/worker.nix;
nixosConfigurations =

187
nix/master.nix → nix/coordinator.nix
View file

@ -4,82 +4,25 @@
, ...
}:
let
cfg = config.services.buildbot-nix.master;
cfg = config.services.buildbot-nix.coordinator;
in
{
options = {
services.buildbot-nix.master = {
enable = lib.mkEnableOption "buildbot-master";
services.buildbot-nix.coordinator = {
enable = lib.mkEnableOption "buildbot-coordinator";
dbUrl = lib.mkOption {
type = lib.types.str;
default = "postgresql://@/buildbot";
description = "Postgresql database url";
};
cachix = {
name = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Cachix name";
};
signingKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = "Cachix signing key";
};
authTokenFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Cachix auth token";
};
};
github = {
tokenFile = lib.mkOption {
type = lib.types.path;
description = "Github token file";
};
webhookSecretFile = lib.mkOption {
type = lib.types.path;
description = "Github webhook secret file";
};
oauthSecretFile = lib.mkOption {
type = lib.types.path;
description = "Github oauth secret file";
};
# TODO: make this an option
# https://github.com/organizations/numtide/settings/applications
# Application name: BuildBot
# Homepage URL: https://buildbot.numtide.com
# Authorization callback URL: https://buildbot.numtide.com/auth/login
# oauth_token: 2516248ec6289e4d9818122cce0cbde39e4b788d
oauthId = lib.mkOption {
type = lib.types.str;
description = "Github oauth id. Used for the login button";
};
# Most likely you want to use the same user as for the buildbot
user = lib.mkOption {
type = lib.types.str;
description = "Github user that is used for the buildbot";
};
admins = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = "Users that are allowed to login to buildbot, trigger builds and change settings";
};
topic = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = "build-with-buildbot";
description = ''
Projects that have this topic will be built by buildbot.
If null, all projects that the buildbot github user has access to, are built.
'';
};
};
workersFile = lib.mkOption {
type = lib.types.path;
description = "File containing a list of nix workers";
};
oauth2SecretFile = lib.mkOption {
type = lib.types.path;
description = "File containing an OAuth 2 client secret";
};
buildSystems = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ pkgs.hostPlatform.system ];
@ -114,6 +57,41 @@ in
default = null;
example = "/var/www/buildbot/nix-outputs";
};
signingKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
description = "A path to a Nix signing key";
default = null;
example = "/run/agenix.d/signing-key";
};
binaryCache = {
enable = lib.mkEnableOption " binary cache upload to a S3 bucket";
profileCredentialsFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
description = "A path to the various AWS profile credentials related to the S3 bucket containing a profile named `default`";
default = null;
example = "/run/agenix.d/aws-profile";
};
bucket = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "Bucket where to store the data";
default = null;
example = "lix-cache";
};
endpoint = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "Endpoint for the S3 server";
default = null;
example = "s3.lix.systems";
};
region = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "Region for the S3 bucket";
default = null;
example = "garage";
};
};
};
};
config = lib.mkIf cfg.enable {
@ -126,13 +104,6 @@ in
isSystemUser = true;
};
assertions = [
{
assertion = cfg.cachix.name != null -> cfg.cachix.signingKeyFile != null || cfg.cachix.authTokenFile != null;
message = "if cachix.name is provided, then cachix.signingKeyFile and cachix.authTokenFile must be set";
}
];
services.buildbot-master = {
enable = true;
@ -144,30 +115,29 @@ in
home = "/var/lib/buildbot";
extraImports = ''
from datetime import timedelta
from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig
from buildbot_nix import GerritNixConfigurator
'';
configurators = [
''
util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6)
''
''
NixConfigurator(
github=GithubConfig(
oauth_id=${builtins.toJSON cfg.github.oauthId},
admins=${builtins.toJSON cfg.github.admins},
buildbot_user=${builtins.toJSON cfg.github.user},
topic=${builtins.toJSON cfg.github.topic},
),
cachix=${if cfg.cachix.name == null then "None" else "CachixConfig(
name=${builtins.toJSON cfg.cachix.name},
signing_key_secret_name=${if cfg.cachix.signingKeyFile != null then builtins.toJSON "cachix-signing-key" else "None"},
auth_token_secret_name=${if cfg.cachix.authTokenFile != null then builtins.toJSON "cachix-auth-token" else "None"},
)"},
GerritNixConfigurator(
"gerrit.lix.systems",
"buildbot",
2022,
"/var/lib/buildbot/master/id_gerrit",
url=${builtins.toJSON config.services.buildbot-master.buildbotUrl},
nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize},
nix_eval_worker_count=${if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount},
nix_supported_systems=${builtins.toJSON cfg.buildSystems},
outputs_path=${if cfg.outputsPath == null then "None" else builtins.toJSON cfg.outputsPath},
# Signing key file must be available on the workers and readable.
signing_keyfile=${if cfg.signingKeyFile == null then "None" else builtins.toJSON cfg.signingKeyFile},
binary_cache_config=${if (!cfg.binaryCache.enable) then "None" else builtins.toJSON {
inherit (cfg.binaryCache) bucket region endpoint;
profile = "default";
}}
)
''
];
@ -177,31 +147,41 @@ in
hasSSL = host.forceSSL || host.addSSL;
in
"${if hasSSL then "https" else "http"}://${cfg.domain}/";
dbUrl = config.services.buildbot-nix.master.dbUrl;
dbUrl = cfg.dbUrl;
pythonPackages = ps: [
ps.requests
ps.treq
ps.psycopg2
(ps.toPythonModule pkgs.buildbot-worker)
pkgs.buildbot-plugins.www-react
pkgs.buildbot-plugins.www
(pkgs.python3.pkgs.callPackage ../default.nix { })
];
};
# TODO(raito): we assume worker runs on coordinator. please clean up this later.
systemd.services.buildbot-worker.serviceConfig.Environment =
lib.mkIf cfg.binaryCache.enable (
let
awsConfigFile = pkgs.writeText "config.ini" ''
[default]
region = ${cfg.binaryCache.region}
endpoint_url = ${cfg.binaryCache.endpoint}
'';
in
[
"AWS_CONFIG_FILE=${awsConfigFile}"
"AWS_SHARED_CREDENTIALS_FILE=${cfg.binaryCache.profileCredentialsFile}"
]
);
systemd.services.buildbot-master = {
after = [ "postgresql.service" ];
serviceConfig = {
# in master.py we read secrets from $CREDENTIALS_DIRECTORY
LoadCredential = [
"github-token:${cfg.github.tokenFile}"
"github-webhook-secret:${cfg.github.webhookSecretFile}"
"github-oauth-secret:${cfg.github.oauthSecretFile}"
"buildbot-nix-workers:${cfg.workersFile}"
]
++ lib.optional (cfg.cachix.signingKeyFile != null)
"cachix-signing-key:${builtins.toString cfg.cachix.signingKeyFile}"
++ lib.optional (cfg.cachix.authTokenFile != null)
"cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}";
"buildbot-oauth2-secret:${cfg.oauth2SecretFile}"
];
};
};
@ -215,16 +195,20 @@ in
};
services.nginx.enable = true;
services.nginx.virtualHosts.${cfg.domain} = {
services.nginx.virtualHosts.${cfg.domain} =
let
port = config.services.buildbot-master.port;
in
{
locations = {
"/".proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/";
"/".proxyPass = "http://127.0.0.1:${builtins.toString port}/";
"/sse" = {
proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/sse";
proxyPass = "http://127.0.0.1:${builtins.toString port}/sse";
# proxy buffering will prevent sse to work
extraConfig = "proxy_buffering off;";
};
"/ws" = {
proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/ws";
proxyPass = "http://127.0.0.1:${builtins.toString port}/ws";
proxyWebsockets = true;
# raise the proxy timeout for the websocket
extraConfig = "proxy_read_timeout 6000s;";
@ -234,11 +218,8 @@ in
};
};
systemd.tmpfiles.rules = [
# delete legacy gcroot location, can be dropped after 2024-06-01
"R /var/lib/buildbot-worker/gcroot - - - - -"
] ++ lib.optional (cfg.outputsPath != null)
# Allow buildbot-master to write to this directory
systemd.tmpfiles.rules = lib.optional (cfg.outputsPath != null)
# Allow buildbot-coordinator to write to this directory
"d ${cfg.outputsPath} 0755 buildbot buildbot - -";
};
}

18
nix/worker.nix
View file

@ -19,15 +19,19 @@ in
defaultText = "pkgs.buildbot-worker";
description = "The buildbot-worker package to use.";
};
masterUrl = lib.mkOption {
coordinatorUrl = lib.mkOption {
type = lib.types.str;
default = "tcp:host=localhost:port=9989";
description = "The buildbot master url.";
description = "The buildbot coordinator url.";
};
workerPasswordFile = lib.mkOption {
type = lib.types.path;
description = "The buildbot worker password file.";
};
workerArchitectures = lib.mkOption {
type = lib.types.attrsOf lib.types.int;
description = "Nix `system`s the worker should feel responsible for.";
};
};
};
config = lib.mkIf cfg.enable {
@ -54,15 +58,17 @@ in
after = [ "network.target" "buildbot-master.service" ];
wantedBy = [ "multi-user.target" ];
path = [
pkgs.cachix
pkgs.git
pkgs.openssh
pkgs.nix
pkgs.nix-eval-jobs
];
environment.PYTHONPATH = "${python.withPackages (_: [cfg.package])}/${python.sitePackages}";
environment.MASTER_URL = cfg.masterUrl;
environment.MASTER_URL = cfg.coordinatorUrl;
environment.BUILDBOT_DIR = buildbotDir;
environment.WORKER_ARCH_LIST =
lib.concatStringsSep ","
(lib.mapAttrsToList (arch: jobs: "${arch}=${toString jobs}") cfg.workerArchitectures);
serviceConfig = {
# We rather want the CI job to fail on OOM than to have a broken buildbot worker.
@ -70,7 +76,9 @@ in
OOMPolicy = "continue";
LoadCredential = [ "worker-password-file:${cfg.workerPasswordFile}" ];
Environment = [ "WORKER_PASSWORD_FILE=%d/worker-password-file" ];
Environment = [
"WORKER_PASSWORD_FILE=%d/worker-password-file"
];
Type = "simple";
User = "buildbot-worker";
Group = "buildbot-worker";