hydra-notify: make the prometheus endpoint configurable, default-off

doc/manual/src/configuration.md

@@ -79,6 +79,22 @@ By default, Hydra will send stats to statsd at `localhost:8125`. Point Hydra to
 </statsd>
 ```
 
+hydra-notify's Prometheus service
+---------------------------------
+
+hydra-notify supports running a Prometheus webserver for metrics. The
+exporter does not run unless a listen address and port are specified
+in the hydra configuration file, as below:
+
+```conf
+<hydra_notify>
+  <prometheus>
+    listen_address = 127.0.0.1
+    port = 9199
+  </prometheus>
+</hydra_notify>
+```
+
 Using LDAP as authentication backend (optional)
 -----------------------------------------------
 

doc/manual/src/monitoring/README.md

@@ -13,3 +13,11 @@ $ curl --header "Accept: application/json" http://localhost:63333/queue-runner-s
 ... JSON payload ...
 ```
 
+## Notification Daemon
+
+The `hydra-notify` process can expose Prometheus metrics for plugin execution. See
+[hydra-notify's Prometheus service](../configuration.md#hydra-notifys-prometheus-service)
+for details on enabling and configuring the exporter.
+
+The notification exporter exposes metrics on a per-plugin, per-event-type basis: execution
+durations, frequency, successes, and failures.

src/lib/Hydra/Helper/Nix.pm

@@ -70,6 +70,29 @@ sub getStatsdConfig {
     }
 }
 
+sub getHydraNotifyPrometheusConfig {
+    my ($config) = @_;
+    my $cfg = $config->{hydra_notify};
+
+    if (!defined($cfg) || ref $cfg ne "HASH") {
+        return undef;
+    }
+
+    my $cfg = $cfg->{prometheus};
+    if (!defined($cfg) || ref $cfg ne "HASH") {
+        return undef;
+    }
+
+    if (defined($cfg->{"listen_address"}) && defined($cfg->{"port"})) {
+        return {
+            "listen_address" => $cfg->{'listen_address'},
+            "port" => $cfg->{'port'},
+        };
+    }
+
+    return undef;
+}
+
 
 sub getBaseUrl {
     my ($config) = @_;

src/script/hydra-notify

@@ -14,22 +14,27 @@ use Parallel::ForkManager;
 use Prometheus::Tiny::Shared;
 use Time::HiRes qw( gettimeofday tv_interval );
 
+STDERR->autoflush(1);
+STDOUT->autoflush(1);
+binmode STDERR, ":encoding(utf8)";
+
+my $config = getHydraConfig();
+
 my $prom = Prometheus::Tiny::Shared->new;
 
+my $promCfg = Hydra::Helper::Nix::getHydraNotifyPrometheusConfig($config);
+if (defined($promCfg)) {
     my $fork_manager = Parallel::ForkManager->new(1);
     $fork_manager->start_child("metrics_exporter", sub {
         my $server = HTTP::Server::PSGI->new(
-        host => "127.0.0.1",
+            host => $promCfg->{"listen_address"},
-        port => 9091,
+            port => $promCfg->{"port"},
-        timeout => 5,
+            timeout => 1,
         );
 
         $server->run($prom->psgi);
     });
-
+}
-STDERR->autoflush(1);
-STDOUT->autoflush(1);
-binmode STDERR, ":encoding(utf8)";
 
 my $queued_only;
 
@@ -37,7 +42,6 @@ GetOptions(
     "queued-only" => \$queued_only
 ) or exit 1;
 
-my $config = getHydraConfig();
 
 my $db = Hydra::Model::DB->new();
 

t/Config/hydra-notify.t

@@ -0,0 +1,87 @@
+use strict;
+use Setup;
+
+my %ctx = test_init(hydra_config => q|
+<hydra_notify>
+  <prometheus>
+    listen_address = 127.0.0.1
+    port = 9199
+  </prometheus>
+</hydra_notify>
+|);
+
+require Hydra::Helper::Nix;
+use Test2::V0;
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig(Hydra::Helper::Nix::getHydraConfig()), {
+    'listen_address' => "127.0.0.1",
+    'port' => 9199
+}, "Reading specific configuration from the hydra.conf works");
+
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({
+    "hydra_notify" => ":)"
+}), undef, "Invalid (hydra_notify is a string) configuration options are undef");
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({
+    "hydra_notify" => []
+}), undef, "Invalid (hydra_notify is a list) configuration options are undef");
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({
+    "hydra_notify" => {}
+}), undef, "Invalid (hydra_notify is an empty hash) configuration options are undef");
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({
+    "hydra_notify" => {
+        "prometheus" => ":)"
+    }
+}), undef, "Invalid (hydra_notify.prometheus is a string) configuration options are undef");
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({
+    "hydra_notify" => {
+        "prometheus" => {}
+    }
+}), undef, "Invalid (hydra_notify.prometheus is an empty hash) configuration options are undef");
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({
+    "hydra_notify" => {
+        "prometheus" => {
+            "listen_address" => "0.0.0.0"
+        }
+    }
+}), undef, "Invalid (hydra_notify.prometheus.port is missing) configuration options are undef");
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({
+    "hydra_notify" => {
+        "prometheus" => {
+            "port" => 1234
+        }
+    }
+}), undef, "Invalid (hydra_notify.prometheus.listen_address is missing) configuration options are undef");
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({
+    "hydra_notify" => {
+        "prometheus" => {
+            "listen_address" => "127.0.0.1",
+            "port" => 1234
+        }
+    }
+}), {
+    "listen_address" => "127.0.0.1",
+    "port" => 1234
+}, "Fully specified hydra_notify.prometheus config is valid and returned");
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({
+    "hydra_notify" => {
+        "prometheus" => {
+            "listen_address" => "127.0.0.1",
+            "port" => 1234,
+            "extra_keys" => "meh",
+        }
+    }
+}), {
+    "listen_address" => "127.0.0.1",
+    "port" => 1234
+}, "extra configuration in hydra_notify.prometheus is not returned");
+
+done_testing;