feat: support Gerrit in Buildbot #1

Merged
qyriad merged 46 commits from gerrit into main 2024-04-30 19:42:02 +00:00
7 changed files with 614 additions and 629 deletions

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,12 @@
from dataclasses import dataclass
@dataclass
class S3BinaryCacheConfig:
region: str
bucket: str
endpoint: str
profile: str
class LocalSigner:
def __init__(self, keyfile: str):
self.keyfile = keyfile

View file

@ -22,8 +22,14 @@ class WorkerConfig:
.read_text() .read_text()
.rstrip("\r\n") .rstrip("\r\n")
) )
worker_count: int = int( worker_arch_list: dict[str, int] = field(
os.environ.get("WORKER_COUNT", str(multiprocessing.cpu_count())), default_factory=lambda: dict(other=1) | {
arch: int(count)
for arch, count in (
e.split("=")
for e in os.environ.get("WORKER_ARCH_LIST", "").split(",")
)
},
) )
buildbot_dir: Path = field( buildbot_dir: Path = field(
default_factory=lambda: Path(require_env("BUILDBOT_DIR")) default_factory=lambda: Path(require_env("BUILDBOT_DIR"))
@ -34,13 +40,14 @@ class WorkerConfig:
def setup_worker( def setup_worker(
application: components.Componentized, application: components.Componentized,
builder_id: int, builder_id: int,
arch: str,
config: WorkerConfig, config: WorkerConfig,
) -> None: ) -> None:
basedir = config.buildbot_dir.parent / f"{config.buildbot_dir.name}-{builder_id:03}" basedir = config.buildbot_dir.parent / f"{config.buildbot_dir.name}-{builder_id:03}/{arch}"
basedir.mkdir(parents=True, exist_ok=True, mode=0o700) basedir.mkdir(parents=True, exist_ok=True, mode=0o700)
hostname = socket.gethostname() hostname = socket.gethostname()
workername = f"{hostname}-{builder_id:03}" workername = f"{hostname}-{builder_id:03}-{arch}"
keepalive = 600 keepalive = 600
umask = None umask = None
maxdelay = 300 maxdelay = 300
@ -66,8 +73,9 @@ def setup_worker(
def setup_workers(application: components.Componentized, config: WorkerConfig) -> None: def setup_workers(application: components.Componentized, config: WorkerConfig) -> None:
for i in range(config.worker_count): for arch, jobs in config.worker_arch_list.items():
setup_worker(application, i, config) for i in range(jobs):
setup_worker(application, i, arch, config)
# note: this line is matched against to check that this is a worker # note: this line is matched against to check that this is a worker

View file

@ -46,14 +46,6 @@ in
# optional nix-eval-jobs settings # optional nix-eval-jobs settings
# evalWorkerCount = 8; # limit number of concurrent evaluations # evalWorkerCount = 8; # limit number of concurrent evaluations
# evalMaxMemorySize = "2048"; # limit memory usage per evaluation # evalMaxMemorySize = "2048"; # limit memory usage per evaluation
# optional cachix
#cachix = {
# name = "my-cachix";
# # One of the following is required:
# signingKey = "/var/lib/secrets/cachix-key";
# authToken = "/var/lib/secrets/cachix-token";
#};
}; };
}) })
buildbot-nix.nixosModules.buildbot-master buildbot-nix.nixosModules.buildbot-master

View file

@ -20,7 +20,7 @@
] ++ inputs.nixpkgs.lib.optional (inputs.treefmt-nix ? flakeModule) ./nix/treefmt/flake-module.nix; ] ++ inputs.nixpkgs.lib.optional (inputs.treefmt-nix ? flakeModule) ./nix/treefmt/flake-module.nix;
systems = [ "x86_64-linux" ]; systems = [ "x86_64-linux" ];
flake = { flake = {
nixosModules.buildbot-master = ./nix/master.nix; nixosModules.buildbot-coordinator = ./nix/coordinator.nix;
nixosModules.buildbot-worker = ./nix/worker.nix; nixosModules.buildbot-worker = ./nix/worker.nix;
nixosConfigurations = nixosConfigurations =

View file

@ -4,82 +4,25 @@
, ... , ...
}: }:
let let
cfg = config.services.buildbot-nix.master; cfg = config.services.buildbot-nix.coordinator;
in in
{ {
options = { options = {
services.buildbot-nix.master = { services.buildbot-nix.coordinator = {
enable = lib.mkEnableOption "buildbot-master"; enable = lib.mkEnableOption "buildbot-coordinator";
dbUrl = lib.mkOption { dbUrl = lib.mkOption {
type = lib.types.str; type = lib.types.str;
default = "postgresql://@/buildbot"; default = "postgresql://@/buildbot";
description = "Postgresql database url"; description = "Postgresql database url";
}; };
cachix = {
name = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Cachix name";
};
signingKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = "Cachix signing key";
};
authTokenFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = "Cachix auth token";
};
};
github = {
tokenFile = lib.mkOption {
type = lib.types.path;
description = "Github token file";
};
webhookSecretFile = lib.mkOption {
type = lib.types.path;
description = "Github webhook secret file";
};
oauthSecretFile = lib.mkOption {
type = lib.types.path;
description = "Github oauth secret file";
};
# TODO: make this an option
# https://github.com/organizations/numtide/settings/applications
# Application name: BuildBot
# Homepage URL: https://buildbot.numtide.com
# Authorization callback URL: https://buildbot.numtide.com/auth/login
# oauth_token: 2516248ec6289e4d9818122cce0cbde39e4b788d
oauthId = lib.mkOption {
type = lib.types.str;
description = "Github oauth id. Used for the login button";
};
# Most likely you want to use the same user as for the buildbot
user = lib.mkOption {
type = lib.types.str;
description = "Github user that is used for the buildbot";
};
admins = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = "Users that are allowed to login to buildbot, trigger builds and change settings";
};
topic = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = "build-with-buildbot";
description = ''
Projects that have this topic will be built by buildbot.
If null, all projects that the buildbot github user has access to, are built.
'';
};
};
workersFile = lib.mkOption { workersFile = lib.mkOption {
type = lib.types.path; type = lib.types.path;
description = "File containing a list of nix workers"; description = "File containing a list of nix workers";
}; };
oauth2SecretFile = lib.mkOption {
type = lib.types.path;
description = "File containing an OAuth 2 client secret";
};
buildSystems = lib.mkOption { buildSystems = lib.mkOption {
type = lib.types.listOf lib.types.str; type = lib.types.listOf lib.types.str;
default = [ pkgs.hostPlatform.system ]; default = [ pkgs.hostPlatform.system ];
@ -114,6 +57,41 @@ in
default = null; default = null;
example = "/var/www/buildbot/nix-outputs"; example = "/var/www/buildbot/nix-outputs";
}; };
signingKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
description = "A path to a Nix signing key";
default = null;
example = "/run/agenix.d/signing-key";
};
binaryCache = {
enable = lib.mkEnableOption " binary cache upload to a S3 bucket";
profileCredentialsFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
description = "A path to the various AWS profile credentials related to the S3 bucket containing a profile named `default`";
default = null;
example = "/run/agenix.d/aws-profile";
};
bucket = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "Bucket where to store the data";
default = null;
example = "lix-cache";
};
endpoint = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "Endpoint for the S3 server";
default = null;
example = "s3.lix.systems";
};
region = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "Region for the S3 bucket";
default = null;
example = "garage";
};
};
}; };
}; };
config = lib.mkIf cfg.enable { config = lib.mkIf cfg.enable {
@ -126,13 +104,6 @@ in
isSystemUser = true; isSystemUser = true;
}; };
assertions = [
{
assertion = cfg.cachix.name != null -> cfg.cachix.signingKeyFile != null || cfg.cachix.authTokenFile != null;
message = "if cachix.name is provided, then cachix.signingKeyFile and cachix.authTokenFile must be set";
}
];
services.buildbot-master = { services.buildbot-master = {
enable = true; enable = true;
@ -144,30 +115,29 @@ in
home = "/var/lib/buildbot"; home = "/var/lib/buildbot";
extraImports = '' extraImports = ''
from datetime import timedelta from datetime import timedelta
from buildbot_nix import GithubConfig, NixConfigurator, CachixConfig from buildbot_nix import GerritNixConfigurator
''; '';
configurators = [ configurators = [
'' ''
util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6) util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6)
'' ''
'' ''
NixConfigurator( GerritNixConfigurator(
github=GithubConfig( "gerrit.lix.systems",
oauth_id=${builtins.toJSON cfg.github.oauthId}, "buildbot",
admins=${builtins.toJSON cfg.github.admins}, 2022,
buildbot_user=${builtins.toJSON cfg.github.user}, "/var/lib/buildbot/master/id_gerrit",
topic=${builtins.toJSON cfg.github.topic},
),
cachix=${if cfg.cachix.name == null then "None" else "CachixConfig(
name=${builtins.toJSON cfg.cachix.name},
signing_key_secret_name=${if cfg.cachix.signingKeyFile != null then builtins.toJSON "cachix-signing-key" else "None"},
auth_token_secret_name=${if cfg.cachix.authTokenFile != null then builtins.toJSON "cachix-auth-token" else "None"},
)"},
url=${builtins.toJSON config.services.buildbot-master.buildbotUrl}, url=${builtins.toJSON config.services.buildbot-master.buildbotUrl},
nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize}, nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize},
nix_eval_worker_count=${if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount}, nix_eval_worker_count=${if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount},
nix_supported_systems=${builtins.toJSON cfg.buildSystems}, nix_supported_systems=${builtins.toJSON cfg.buildSystems},
outputs_path=${if cfg.outputsPath == null then "None" else builtins.toJSON cfg.outputsPath}, outputs_path=${if cfg.outputsPath == null then "None" else builtins.toJSON cfg.outputsPath},
# Signing key file must be available on the workers and readable.
signing_keyfile=${if cfg.signingKeyFile == null then "None" else builtins.toJSON cfg.signingKeyFile},
binary_cache_config=${if (!cfg.binaryCache.enable) then "None" else builtins.toJSON {
inherit (cfg.binaryCache) bucket region endpoint;
profile = "default";
}}
) )
'' ''
]; ];
@ -177,31 +147,41 @@ in
hasSSL = host.forceSSL || host.addSSL; hasSSL = host.forceSSL || host.addSSL;
in in
"${if hasSSL then "https" else "http"}://${cfg.domain}/"; "${if hasSSL then "https" else "http"}://${cfg.domain}/";
dbUrl = config.services.buildbot-nix.master.dbUrl; dbUrl = cfg.dbUrl;
pythonPackages = ps: [ pythonPackages = ps: [
ps.requests ps.requests
ps.treq ps.treq
ps.psycopg2 ps.psycopg2
(ps.toPythonModule pkgs.buildbot-worker) (ps.toPythonModule pkgs.buildbot-worker)
pkgs.buildbot-plugins.www-react pkgs.buildbot-plugins.www
(pkgs.python3.pkgs.callPackage ../default.nix { }) (pkgs.python3.pkgs.callPackage ../default.nix { })
]; ];
}; };
# TODO(raito): we assume worker runs on coordinator. please clean up this later.
systemd.services.buildbot-worker.serviceConfig.Environment =
lib.mkIf cfg.binaryCache.enable (
let
awsConfigFile = pkgs.writeText "config.ini" ''
[default]
region = ${cfg.binaryCache.region}
endpoint_url = ${cfg.binaryCache.endpoint}
'';
in
[
"AWS_CONFIG_FILE=${awsConfigFile}"
"AWS_SHARED_CREDENTIALS_FILE=${cfg.binaryCache.profileCredentialsFile}"
]
);
systemd.services.buildbot-master = { systemd.services.buildbot-master = {
after = [ "postgresql.service" ]; after = [ "postgresql.service" ];
serviceConfig = { serviceConfig = {
# in master.py we read secrets from $CREDENTIALS_DIRECTORY # in master.py we read secrets from $CREDENTIALS_DIRECTORY
LoadCredential = [ LoadCredential = [
"github-token:${cfg.github.tokenFile}"
"github-webhook-secret:${cfg.github.webhookSecretFile}"
"github-oauth-secret:${cfg.github.oauthSecretFile}"
"buildbot-nix-workers:${cfg.workersFile}" "buildbot-nix-workers:${cfg.workersFile}"
] "buildbot-oauth2-secret:${cfg.oauth2SecretFile}"
++ lib.optional (cfg.cachix.signingKeyFile != null) ];
"cachix-signing-key:${builtins.toString cfg.cachix.signingKeyFile}"
++ lib.optional (cfg.cachix.authTokenFile != null)
"cachix-auth-token:${builtins.toString cfg.cachix.authTokenFile}";
}; };
}; };
@ -215,16 +195,20 @@ in
}; };
services.nginx.enable = true; services.nginx.enable = true;
services.nginx.virtualHosts.${cfg.domain} = { services.nginx.virtualHosts.${cfg.domain} =
let
port = config.services.buildbot-master.port;
in
{
locations = { locations = {
"/".proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/"; "/".proxyPass = "http://127.0.0.1:${builtins.toString port}/";
"/sse" = { "/sse" = {
proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/sse"; proxyPass = "http://127.0.0.1:${builtins.toString port}/sse";
# proxy buffering will prevent sse to work # proxy buffering will prevent sse to work
extraConfig = "proxy_buffering off;"; extraConfig = "proxy_buffering off;";
}; };
"/ws" = { "/ws" = {
proxyPass = "http://127.0.0.1:${builtins.toString config.services.buildbot-master.port}/ws"; proxyPass = "http://127.0.0.1:${builtins.toString port}/ws";
proxyWebsockets = true; proxyWebsockets = true;
# raise the proxy timeout for the websocket # raise the proxy timeout for the websocket
extraConfig = "proxy_read_timeout 6000s;"; extraConfig = "proxy_read_timeout 6000s;";
@ -234,11 +218,8 @@ in
}; };
}; };
systemd.tmpfiles.rules = [ systemd.tmpfiles.rules = lib.optional (cfg.outputsPath != null)
# delete legacy gcroot location, can be dropped after 2024-06-01 # Allow buildbot-coordinator to write to this directory
"R /var/lib/buildbot-worker/gcroot - - - - -"
] ++ lib.optional (cfg.outputsPath != null)
# Allow buildbot-master to write to this directory
"d ${cfg.outputsPath} 0755 buildbot buildbot - -"; "d ${cfg.outputsPath} 0755 buildbot buildbot - -";
}; };
} }

View file

@ -19,15 +19,19 @@ in
defaultText = "pkgs.buildbot-worker"; defaultText = "pkgs.buildbot-worker";
description = "The buildbot-worker package to use."; description = "The buildbot-worker package to use.";
}; };
masterUrl = lib.mkOption { coordinatorUrl = lib.mkOption {
type = lib.types.str; type = lib.types.str;
default = "tcp:host=localhost:port=9989"; default = "tcp:host=localhost:port=9989";
description = "The buildbot master url."; description = "The buildbot coordinator url.";
}; };
workerPasswordFile = lib.mkOption { workerPasswordFile = lib.mkOption {
type = lib.types.path; type = lib.types.path;
description = "The buildbot worker password file."; description = "The buildbot worker password file.";
}; };
workerArchitectures = lib.mkOption {
type = lib.types.attrsOf lib.types.int;
description = "Nix `system`s the worker should feel responsible for.";
};
}; };
}; };
config = lib.mkIf cfg.enable { config = lib.mkIf cfg.enable {
@ -54,15 +58,17 @@ in
after = [ "network.target" "buildbot-master.service" ]; after = [ "network.target" "buildbot-master.service" ];
wantedBy = [ "multi-user.target" ]; wantedBy = [ "multi-user.target" ];
path = [ path = [
pkgs.cachix
pkgs.git pkgs.git
pkgs.openssh pkgs.openssh
pkgs.nix pkgs.nix
pkgs.nix-eval-jobs pkgs.nix-eval-jobs
]; ];
environment.PYTHONPATH = "${python.withPackages (_: [cfg.package])}/${python.sitePackages}"; environment.PYTHONPATH = "${python.withPackages (_: [cfg.package])}/${python.sitePackages}";
environment.MASTER_URL = cfg.masterUrl; environment.MASTER_URL = cfg.coordinatorUrl;
environment.BUILDBOT_DIR = buildbotDir; environment.BUILDBOT_DIR = buildbotDir;
environment.WORKER_ARCH_LIST =
lib.concatStringsSep ","
(lib.mapAttrsToList (arch: jobs: "${arch}=${toString jobs}") cfg.workerArchitectures);
serviceConfig = { serviceConfig = {
# We rather want the CI job to fail on OOM than to have a broken buildbot worker. # We rather want the CI job to fail on OOM than to have a broken buildbot worker.
@ -70,7 +76,9 @@ in
OOMPolicy = "continue"; OOMPolicy = "continue";
LoadCredential = [ "worker-password-file:${cfg.workerPasswordFile}" ]; LoadCredential = [ "worker-password-file:${cfg.workerPasswordFile}" ];
Environment = [ "WORKER_PASSWORD_FILE=%d/worker-password-file" ]; Environment = [
"WORKER_PASSWORD_FILE=%d/worker-password-file"
];
Type = "simple"; Type = "simple";
User = "buildbot-worker"; User = "buildbot-worker";
Group = "buildbot-worker"; Group = "buildbot-worker";