buildbot-nix/nix/coordinator.nix
raito fa83000f07 feat(debug): add manhole debugging
Signed-off-by: Raito Bezarius <raito@lix.systems>
2024-10-05 20:52:52 +02:00

356 lines
12 KiB
Nix

{ config
, options
, pkgs
, lib
, ...
}:
let
inherit (lib) filterAttrs;
cfg = config.services.buildbot-nix.coordinator;
debuggingManhole = if cfg.debugging.enable then
"manhole.TelnetManhole(${toString cfg.debugging.port}, 'admin', 'admin')"
else "None";
in
{
options = {
services.buildbot-nix.coordinator = {
enable = lib.mkEnableOption "buildbot-coordinator";
dbUrl = lib.mkOption {
type = lib.types.str;
default = "postgresql://@/buildbot";
description = "Postgresql database url";
};
workersFile = lib.mkOption {
type = lib.types.path;
description = "File containing a list of nix workers";
};
buildMachines = lib.mkOption {
type = options.nix.buildMachines.type;
description = "List of local remote builders machines associated to that Buildbot instance";
};
debugging = {
enable = lib.mkEnableOption "manhole's buildbot debugging on localhost using `admin:admin`";
port = lib.mkOption {
type = lib.types.port;
default = 15000;
};
};
oauth2 = {
name = lib.mkOption {
type = lib.types.str;
description = "Name of the OAuth2 login method";
};
icon = lib.mkOption {
type = lib.types.str;
description = "FontAwesome string for the icon associated to the OAuth2 login";
default = "fa-login";
example = "fa-login";
};
clientId = lib.mkOption {
type = lib.types.str;
description = "Client ID for the OAuth2 authentication";
};
clientSecretFile = lib.mkOption {
type = lib.types.path;
description = "Path to a file containing an OAuth 2 client secret";
};
resourceEndpoint = lib.mkOption {
type = lib.types.str;
description = "URL to the OAuth 2 resource";
example = "https://identity.lix.systems";
};
authUri = lib.mkOption {
type = lib.types.str;
description = "Authentication URI";
example = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth";
};
tokenUri = lib.mkOption {
type = lib.types.str;
description = "Token URI";
example = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
};
userinfoUri = lib.mkOption {
type = lib.types.str;
description = "User info URI";
example = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
};
};
buildSystems = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ pkgs.hostPlatform.system ];
description = "Systems that we will be build";
};
evalMaxMemorySize = lib.mkOption {
type = lib.types.str;
default = "2048";
description = ''
Maximum memory size for nix-eval-jobs (in MiB) per
worker. After the limit is reached, the worker is
restarted.
'';
};
evalWorkerCount = lib.mkOption {
type = lib.types.nullOr lib.types.int;
default = null;
description = ''
Number of nix-eval-jobs worker processes. If null, the number of cores is used.
If you experience memory issues (buildbot-workers going out-of-memory), you can reduce this number.
'';
};
domain = lib.mkOption {
type = lib.types.str;
description = "Buildbot domain";
example = "buildbot.numtide.com";
};
allowedOrigins = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "Allowed origins for buildbot";
example = [ "*.mydomain.com" ];
};
signingKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
description = "A path to a Nix signing key";
default = null;
example = "/run/agenix.d/signing-key";
};
prometheus = {
enable = lib.mkEnableOption " the export of metrics in Prometheus format";
address = lib.mkOption {
type = lib.types.str;
default = "";
description = "The local IPv4 or IPv6 address to which to bind; defaults to '' represents all IPv4 addresses.";
};
port = lib.mkOption {
type = lib.types.port;
default = 9100;
description = "A port on which the metrics endpoint will be available";
};
};
gerrit = {
domain = lib.mkOption {
type = lib.types.str;
description = "Domain to the Gerrit server";
example = "gerrit.lix.systems";
};
username = lib.mkOption {
type = lib.types.str;
description = "Username to log in to the Gerrit API";
example = "buildbot";
};
port = lib.mkOption {
type = lib.types.port;
description = "Port to log in to the Gerrit API";
example = 2022;
};
privateKeyFile = lib.mkOption {
type = lib.types.path;
description = ''
Path to the SSH private key to authenticate against the Gerrit API
'';
example = "/var/lib/buildbot/master/id_gerrit";
};
projects = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = ''
List of projects which are to check on Gerrit.
'';
example = [ "lix" ];
};
};
binaryCache = {
enable = lib.mkEnableOption " binary cache upload to a S3 bucket";
profileCredentialsFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
description = "A path to the various AWS profile credentials related to the S3 bucket containing a profile named `default`";
default = null;
example = "/run/agenix.d/aws-profile";
};
bucket = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "Bucket where to store the data";
default = null;
example = "lix-cache";
};
endpoint = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "Endpoint for the S3 server";
default = null;
example = "s3.lix.systems";
};
region = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = "Region for the S3 bucket";
default = null;
example = "garage";
};
};
};
};
config = lib.mkIf cfg.enable {
# By default buildbot uses a normal user, which is not a good default, because
# we grant normal users potentially access to other resources. Also
# we don't to be able to ssh into buildbot.
users.users.buildbot = {
isNormalUser = lib.mkForce false;
isSystemUser = true;
};
services.buildbot-master = {
enable = true;
# disable example workers from nixpkgs
builders = [ ];
schedulers = [ ];
workers = [ ];
home = "/var/lib/buildbot";
extraImports = ''
from datetime import timedelta
from buildbot_nix import GerritNixConfigurator, read_secret_file, make_oauth2_method, OAuth2Config, assemble_secret_file_path
from buildbot import manhole
# TODO(raito): make me configurable from the NixOS module.
# how?
CustomOAuth2 = make_oauth2_method(OAuth2Config(
name=${builtins.toJSON cfg.oauth2.name},
faIcon=${builtins.toJSON cfg.oauth2.icon},
resourceEndpoint=${builtins.toJSON cfg.oauth2.resourceEndpoint},
authUri=${builtins.toJSON cfg.oauth2.authUri},
tokenUri=${builtins.toJSON cfg.oauth2.tokenUri},
userinfoUri=${builtins.toJSON cfg.oauth2.userinfoUri}
))
'';
configurators = [
''
util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6)
''
''
GerritNixConfigurator(
"${cfg.gerrit.domain}",
"${cfg.gerrit.username}",
"${toString cfg.gerrit.port}",
assemble_secret_file_path('buildbot-service-private-key'),
projects=${builtins.toJSON cfg.gerrit.projects},
allowed_origins=${builtins.toJSON cfg.allowedOrigins},
url=${builtins.toJSON config.services.buildbot-master.buildbotUrl},
nix_eval_max_memory_size=${builtins.toJSON cfg.evalMaxMemorySize},
nix_eval_worker_count=${if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount},
nix_supported_systems=${builtins.toJSON cfg.buildSystems},
prometheus_config=${if (!cfg.prometheus.enable) then "None" else builtins.toJSON {
inherit (cfg.prometheus) address port;
}},
nix_builders=${builtins.toJSON (map (b: filterAttrs (n: _: n != "system") b) cfg.buildMachines)},
# Signing key file must be available on the workers and readable.
signing_keyfile=${if cfg.signingKeyFile == null then "None" else builtins.toJSON cfg.signingKeyFile},
binary_cache_config=${if (!cfg.binaryCache.enable) then "None" else builtins.toJSON {
inherit (cfg.binaryCache) bucket region endpoint;
profile = "default";
}},
auth_method=CustomOAuth2(${builtins.toJSON cfg.oauth2.clientId},
read_secret_file('buildbot-oauth2-secret'),
autologin=True
),
manhole=${debuggingManhole}
)
''
];
buildbotUrl =
let
host = config.services.nginx.virtualHosts.${cfg.domain};
hasSSL = host.forceSSL || host.addSSL;
in
"${if hasSSL then "https" else "http"}://${cfg.domain}/";
dbUrl = cfg.dbUrl;
pythonPackages = ps: ([
ps.requests
ps.treq
ps.psycopg2
(ps.toPythonModule pkgs.buildbot-worker)
pkgs.buildbot-plugins.www
(pkgs.python3.pkgs.callPackage ../default.nix { })
] ++ lib.optional cfg.prometheus.enable (pkgs.python3.pkgs.callPackage ./prometheus-plugin.nix { })
);
};
# TODO(raito): we assume worker runs on coordinator. please clean up this later.
systemd.services.buildbot-worker.serviceConfig.Environment =
lib.mkIf cfg.binaryCache.enable (
let
awsConfigFile = pkgs.writeText "config.ini" ''
[default]
region = ${cfg.binaryCache.region}
endpoint_url = ${cfg.binaryCache.endpoint}
'';
in
[
"AWS_CONFIG_FILE=${awsConfigFile}"
"AWS_SHARED_CREDENTIALS_FILE=${cfg.binaryCache.profileCredentialsFile}"
]
);
systemd.services.buildbot-master = {
after = [ "postgresql.service" ];
path = [ pkgs.openssh ];
serviceConfig = {
# in master.py we read secrets from $CREDENTIALS_DIRECTORY
LoadCredential = [
"buildbot-nix-workers:${cfg.workersFile}"
"buildbot-oauth2-secret:${cfg.oauth2.clientSecretFile}"
"buildbot-service-private-key:${cfg.gerrit.privateKeyFile}"
];
};
};
services.postgresql = {
enable = true;
ensureDatabases = [ "buildbot" ];
ensureUsers = [{
name = "buildbot";
ensureDBOwnership = true;
}];
};
services.nginx.enable = true;
services.nginx.virtualHosts.${cfg.domain} =
let
port = config.services.buildbot-master.port;
in
{
locations = {
"/".proxyPass = "http://127.0.0.1:${builtins.toString port}/";
"/sse" = {
proxyPass = "http://127.0.0.1:${builtins.toString port}/sse";
# proxy buffering will prevent sse to work
extraConfig = "proxy_buffering off;";
};
"/ws" = {
proxyPass = "http://127.0.0.1:${builtins.toString port}/ws";
proxyWebsockets = true;
# raise the proxy timeout for the websocket
extraConfig = "proxy_read_timeout 6000s;";
};
};
};
};
}