feat: support Prometheus exports

We package a quite old plugin for Buildbot: https://github.com/claws/buildbot-prometheus
Ideally, we should probably vendor it and maintain it ourselves.

There seems to be no protection against the metrics endpoint for
Buildbot, this is not a big deal given that the CI is public.

Signed-off-by: Raito Bezarius <raito@lix.systems>
This commit is contained in:
raito 2024-05-06 14:26:32 +02:00
parent ea4b9ce350
commit 3876a30117
3 changed files with 67 additions and 2 deletions

View file

@ -795,6 +795,7 @@ class GerritNixConfigurator(ConfiguratorBase):
nix_eval_max_memory_size: int, nix_eval_max_memory_size: int,
nix_workers_secret_name: str = "buildbot-nix-workers", # noqa: S107 nix_workers_secret_name: str = "buildbot-nix-workers", # noqa: S107
signing_keyfile: str | None = None, signing_keyfile: str | None = None,
prometheus_config: dict[str, int | str] | None = None,
binary_cache_config: dict[str, str] | None = None, binary_cache_config: dict[str, str] | None = None,
outputs_path: str | None = None, outputs_path: str | None = None,
) -> None: ) -> None:
@ -808,6 +809,7 @@ class GerritNixConfigurator(ConfiguratorBase):
self.nix_supported_systems = nix_supported_systems self.nix_supported_systems = nix_supported_systems
self.gerrit_change_source = GerritChangeSource(gerrit_server, gerrit_user, gerritport=gerrit_port, identity_file=gerrit_sshkey_path) self.gerrit_change_source = GerritChangeSource(gerrit_server, gerrit_user, gerritport=gerrit_port, identity_file=gerrit_sshkey_path)
self.url = url self.url = url
self.prometheus_config = prometheus_config
if binary_cache_config is not None: if binary_cache_config is not None:
self.binary_cache_config = S3BinaryCacheConfig(**binary_cache_config) self.binary_cache_config = S3BinaryCacheConfig(**binary_cache_config)
else: else:
@ -867,6 +869,9 @@ class GerritNixConfigurator(ConfiguratorBase):
) )
if self.prometheus_config is not None:
config['services'].append(reporters.Prometheus(port=self.prometheus_config.get('port', 9100), interface=self.prometheus_config.get('address', '')))
def gerritBranchKey(b): def gerritBranchKey(b):
ref = b['branch'] ref = b['branch']
if not ref.startswith('refs/changes/'): if not ref.startswith('refs/changes/'):

View file

@ -65,6 +65,20 @@ in
example = "/run/agenix.d/signing-key"; example = "/run/agenix.d/signing-key";
}; };
prometheus = {
enable = lib.mkEnableOption " the export of metrics in Prometheus format";
address = lib.mkOption {
type = lib.types.str;
default = "";
description = "The local IPv4 or IPv6 address to which to bind; defaults to '' represents all IPv4 addresses.";
};
port = lib.mkOption {
type = lib.types.port;
default = 9100;
description = "A port on which the metrics endpoint will be available";
};
};
binaryCache = { binaryCache = {
enable = lib.mkEnableOption " binary cache upload to a S3 bucket"; enable = lib.mkEnableOption " binary cache upload to a S3 bucket";
profileCredentialsFile = lib.mkOption { profileCredentialsFile = lib.mkOption {
@ -132,6 +146,9 @@ in
nix_eval_worker_count=${if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount}, nix_eval_worker_count=${if cfg.evalWorkerCount == null then "None" else builtins.toString cfg.evalWorkerCount},
nix_supported_systems=${builtins.toJSON cfg.buildSystems}, nix_supported_systems=${builtins.toJSON cfg.buildSystems},
outputs_path=${if cfg.outputsPath == null then "None" else builtins.toJSON cfg.outputsPath}, outputs_path=${if cfg.outputsPath == null then "None" else builtins.toJSON cfg.outputsPath},
prometheus_config=${if (!cfg.prometheus.enable) then "None" else builtins.toJSON {
inherit (cfg.prometheus) address port;
}}
# Signing key file must be available on the workers and readable. # Signing key file must be available on the workers and readable.
signing_keyfile=${if cfg.signingKeyFile == null then "None" else builtins.toJSON cfg.signingKeyFile}, signing_keyfile=${if cfg.signingKeyFile == null then "None" else builtins.toJSON cfg.signingKeyFile},
binary_cache_config=${if (!cfg.binaryCache.enable) then "None" else builtins.toJSON { binary_cache_config=${if (!cfg.binaryCache.enable) then "None" else builtins.toJSON {
@ -148,14 +165,15 @@ in
in in
"${if hasSSL then "https" else "http"}://${cfg.domain}/"; "${if hasSSL then "https" else "http"}://${cfg.domain}/";
dbUrl = cfg.dbUrl; dbUrl = cfg.dbUrl;
pythonPackages = ps: [ pythonPackages = ps: ([
ps.requests ps.requests
ps.treq ps.treq
ps.psycopg2 ps.psycopg2
(ps.toPythonModule pkgs.buildbot-worker) (ps.toPythonModule pkgs.buildbot-worker)
pkgs.buildbot-plugins.www pkgs.buildbot-plugins.www
(pkgs.python3.pkgs.callPackage ../default.nix { }) (pkgs.python3.pkgs.callPackage ../default.nix { })
]; ] ++ lib.optional cfg.prometheus.enable (pkgs.python3.pkgs.callPackage ./prometheus-plugin.nix { })
);
}; };
# TODO(raito): we assume worker runs on coordinator. please clean up this later. # TODO(raito): we assume worker runs on coordinator. please clean up this later.

42
nix/prometheus-plugin.nix Normal file
View file

@ -0,0 +1,42 @@
{ lib
, buildPythonPackage
, fetchFromGitHub
, setuptools
, wheel
, buildbot
, prometheus-client
, twisted
}:
buildPythonPackage rec {
pname = "buildbot-prometheus";
version = "unstable-2024-05-06";
pyproject = true;
src = fetchFromGitHub {
owner = "claws";
repo = "buildbot-prometheus";
rev = "0c81a89bbe34628362652fbea416610e215b5d1e";
hash = "sha256-bz2Nv2RZ44i1VoPvQ/XjGMfTT6TmW6jhEVwItPk23SM=";
};
nativeBuildInputs = [
setuptools
wheel
];
propagatedBuildInputs = [
buildbot
prometheus-client
twisted
];
pythonImportsCheck = [ "buildbot_prometheus" ];
meta = with lib; {
description = "";
homepage = "https://github.com/claws/buildbot-prometheus";
license = licenses.mit;
maintainers = with maintainers; [ raitobezarius ];
};
}