From 6d7ee27d25419a42f55e88b6bee1732067227e54 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 18 Aug 2021 15:09:54 -0400 Subject: [PATCH] hydra-notify: make the prometheus endpoint configurable, default-off --- doc/manual/src/configuration.md | 16 ++++++ doc/manual/src/monitoring/README.md | 8 +++ src/lib/Hydra/Helper/Nix.pm | 23 ++++++++ src/script/hydra-notify | 32 ++++++----- t/Config/hydra-notify.t | 87 +++++++++++++++++++++++++++++ 5 files changed, 152 insertions(+), 14 deletions(-) create mode 100644 t/Config/hydra-notify.t diff --git a/doc/manual/src/configuration.md b/doc/manual/src/configuration.md index 170287da..d051c00a 100644 --- a/doc/manual/src/configuration.md +++ b/doc/manual/src/configuration.md @@ -79,6 +79,22 @@ By default, Hydra will send stats to statsd at `localhost:8125`. Point Hydra to ``` +hydra-notify's Prometheus service +--------------------------------- + +hydra-notify supports running a Prometheus webserver for metrics. The +exporter does not run unless a listen address and port are specified +in the hydra configuration file, as below: + +```conf + + + listen_address = 127.0.0.1 + port = 9199 + + +``` + Using LDAP as authentication backend (optional) ----------------------------------------------- diff --git a/doc/manual/src/monitoring/README.md b/doc/manual/src/monitoring/README.md index 65872352..67fb3961 100644 --- a/doc/manual/src/monitoring/README.md +++ b/doc/manual/src/monitoring/README.md @@ -13,3 +13,11 @@ $ curl --header "Accept: application/json" http://localhost:63333/queue-runner-s ... JSON payload ... ``` +## Notification Daemon + +The `hydra-notify` process can expose Prometheus metrics for plugin execution. See +[hydra-notify's Prometheus service](../configuration.md#hydra-notifys-prometheus-service) +for details on enabling and configuring the exporter. + +The notification exporter exposes metrics on a per-plugin, per-event-type basis: execution +durations, frequency, successes, and failures. diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm index 7ab4ab60..ac0e54cf 100644 --- a/src/lib/Hydra/Helper/Nix.pm +++ b/src/lib/Hydra/Helper/Nix.pm @@ -70,6 +70,29 @@ sub getStatsdConfig { } } +sub getHydraNotifyPrometheusConfig { + my ($config) = @_; + my $cfg = $config->{hydra_notify}; + + if (!defined($cfg) || ref $cfg ne "HASH") { + return undef; + } + + my $cfg = $cfg->{prometheus}; + if (!defined($cfg) || ref $cfg ne "HASH") { + return undef; + } + + if (defined($cfg->{"listen_address"}) && defined($cfg->{"port"})) { + return { + "listen_address" => $cfg->{'listen_address'}, + "port" => $cfg->{'port'}, + }; + } + + return undef; +} + sub getBaseUrl { my ($config) = @_; diff --git a/src/script/hydra-notify b/src/script/hydra-notify index f64b5d72..9242d68f 100755 --- a/src/script/hydra-notify +++ b/src/script/hydra-notify @@ -14,30 +14,34 @@ use Parallel::ForkManager; use Prometheus::Tiny::Shared; use Time::HiRes qw( gettimeofday tv_interval ); -my $prom = Prometheus::Tiny::Shared->new; - -my $fork_manager = Parallel::ForkManager->new(1 ); -$fork_manager->start_child("metrics_exporter", sub { - my $server = HTTP::Server::PSGI->new( - host => "127.0.0.1", - port => 9091, - timeout => 5, - ); - - $server->run($prom->psgi); -}); - STDERR->autoflush(1); STDOUT->autoflush(1); binmode STDERR, ":encoding(utf8)"; +my $config = getHydraConfig(); + +my $prom = Prometheus::Tiny::Shared->new; + +my $promCfg = Hydra::Helper::Nix::getHydraNotifyPrometheusConfig($config); +if (defined($promCfg)) { + my $fork_manager = Parallel::ForkManager->new(1); + $fork_manager->start_child("metrics_exporter", sub { + my $server = HTTP::Server::PSGI->new( + host => $promCfg->{"listen_address"}, + port => $promCfg->{"port"}, + timeout => 1, + ); + + $server->run($prom->psgi); + }); +} + my $queued_only; GetOptions( "queued-only" => \$queued_only ) or exit 1; -my $config = getHydraConfig(); my $db = Hydra::Model::DB->new(); diff --git a/t/Config/hydra-notify.t b/t/Config/hydra-notify.t new file mode 100644 index 00000000..7fc8aa63 --- /dev/null +++ b/t/Config/hydra-notify.t @@ -0,0 +1,87 @@ +use strict; +use Setup; + +my %ctx = test_init(hydra_config => q| + + + listen_address = 127.0.0.1 + port = 9199 + + +|); + +require Hydra::Helper::Nix; +use Test2::V0; + +is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig(Hydra::Helper::Nix::getHydraConfig()), { + 'listen_address' => "127.0.0.1", + 'port' => 9199 +}, "Reading specific configuration from the hydra.conf works"); + + +is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({ + "hydra_notify" => ":)" +}), undef, "Invalid (hydra_notify is a string) configuration options are undef"); + +is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({ + "hydra_notify" => [] +}), undef, "Invalid (hydra_notify is a list) configuration options are undef"); + +is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({ + "hydra_notify" => {} +}), undef, "Invalid (hydra_notify is an empty hash) configuration options are undef"); + +is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({ + "hydra_notify" => { + "prometheus" => ":)" + } +}), undef, "Invalid (hydra_notify.prometheus is a string) configuration options are undef"); + +is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({ + "hydra_notify" => { + "prometheus" => {} + } +}), undef, "Invalid (hydra_notify.prometheus is an empty hash) configuration options are undef"); + +is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({ + "hydra_notify" => { + "prometheus" => { + "listen_address" => "0.0.0.0" + } + } +}), undef, "Invalid (hydra_notify.prometheus.port is missing) configuration options are undef"); + +is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({ + "hydra_notify" => { + "prometheus" => { + "port" => 1234 + } + } +}), undef, "Invalid (hydra_notify.prometheus.listen_address is missing) configuration options are undef"); + +is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({ + "hydra_notify" => { + "prometheus" => { + "listen_address" => "127.0.0.1", + "port" => 1234 + } + } +}), { + "listen_address" => "127.0.0.1", + "port" => 1234 +}, "Fully specified hydra_notify.prometheus config is valid and returned"); + +is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({ + "hydra_notify" => { + "prometheus" => { + "listen_address" => "127.0.0.1", + "port" => 1234, + "extra_keys" => "meh", + } + } +}), { + "listen_address" => "127.0.0.1", + "port" => 1234 +}, "extra configuration in hydra_notify.prometheus is not returned"); + +done_testing;