From 5d0ad5f64916696987b2e19c7b3d60db7e068f2b Mon Sep 17 00:00:00 2001
From: Your Name <graham@grahamc.com>
Date: Wed, 18 Aug 2021 14:08:38 -0400
Subject: [PATCH 1/8] hydra-notify: initial scratch take of prometheus events

---
 flake.nix               |  1 +
 src/script/hydra-notify | 31 ++++++++++++++++++++++++++++++-
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/flake.nix b/flake.nix
index 616fa81c..44e5898c 100644
--- a/flake.nix
+++ b/flake.nix
@@ -470,6 +470,7 @@
                 NetPrometheus
                 NetStatsd
                 PadWalker
+                ParallelForkManager
                 PerlCriticCommunity
                 PrometheusTinyShared
                 Readonly
diff --git a/src/script/hydra-notify b/src/script/hydra-notify
index 181fba36..f64b5d72 100755
--- a/src/script/hydra-notify
+++ b/src/script/hydra-notify
@@ -3,12 +3,29 @@
 use strict;
 use utf8;
 use Getopt::Long;
+use HTTP::Server::PSGI;
 use Hydra::Event;
 use Hydra::Event::BuildFinished;
 use Hydra::Helper::AddBuilds;
 use Hydra::Helper::Nix;
 use Hydra::Plugin;
 use Hydra::PostgresListener;
+use Parallel::ForkManager;
+use Prometheus::Tiny::Shared;
+use Time::HiRes qw( gettimeofday tv_interval );
+
+my $prom = Prometheus::Tiny::Shared->new;
+
+my $fork_manager = Parallel::ForkManager->new(1 );
+$fork_manager->start_child("metrics_exporter", sub {
+    my $server = HTTP::Server::PSGI->new(
+        host => "127.0.0.1",
+        port => 9091,
+        timeout => 5,
+    );
+
+    $server->run($prom->psgi);
+});
 
 STDERR->autoflush(1);
 STDOUT->autoflush(1);
@@ -36,11 +53,19 @@ $listener->subscribe("step_finished");
 sub runPluginsForEvent {
     my ($event) = @_;
 
+    my $channelName = $event->{'channel_name'};
+
     foreach my $plugin (@plugins) {
+        $prom->inc("notify_plugin_executions", { channel => $channelName, plugin => ref $plugin });
         eval {
+            my $startTime = [gettimeofday()];
             $event->execute($db, $plugin);
+
+            $prom->histogram_observe("notify_plugin_runtime", tv_interval($startTime), { channel => $channelName, plugin => ref $plugin });
+            $prom->inc("notify_plugin_success", { channel => $channelName, plugin => ref $plugin });
             1;
         } or do {
+            $prom->inc("notify_plugin_error", { channel => $channelName, plugin => ref $plugin });
             print STDERR "error running $event->{'channel_name'} hooks: $@\n";
         }
     }
@@ -60,19 +85,23 @@ for my $build ($db->resultset('Builds')->search(
 
 # Process incoming notifications.
 while (!$queued_only) {
+    $prom->inc("event_loop_iterations");
     my $messages = $listener->block_for_messages();
     while (my $message = $messages->()) {
-
+        $prom->set("event_received", time());
         my $channelName = $message->{"channel"};
         my $pid = $message->{"pid"};
         my $payload = $message->{"payload"};
 
+        $prom->inc("notify_event", { channel => $channelName });
+
         eval {
             my $event = Hydra::Event->new_event($channelName, $message->{"payload"});
             runPluginsForEvent($event);
 
             1;
         } or do {
+            $prom->inc("notify_event_error", { channel => $channelName });
             print STDERR "error processing message '$payload' on channel '$channelName': $@\n";
         }
     }

From 6d7ee27d25419a42f55e88b6bee1732067227e54 Mon Sep 17 00:00:00 2001
From: Your Name <graham@grahamc.com>
Date: Wed, 18 Aug 2021 15:09:54 -0400
Subject: [PATCH 2/8] hydra-notify: make the prometheus endpoint configurable,
 default-off

---
 doc/manual/src/configuration.md     | 16 ++++++
 doc/manual/src/monitoring/README.md |  8 +++
 src/lib/Hydra/Helper/Nix.pm         | 23 ++++++++
 src/script/hydra-notify             | 32 ++++++-----
 t/Config/hydra-notify.t             | 87 +++++++++++++++++++++++++++++
 5 files changed, 152 insertions(+), 14 deletions(-)
 create mode 100644 t/Config/hydra-notify.t

diff --git a/doc/manual/src/configuration.md b/doc/manual/src/configuration.md
index 170287da..d051c00a 100644
--- a/doc/manual/src/configuration.md
+++ b/doc/manual/src/configuration.md
@@ -79,6 +79,22 @@ By default, Hydra will send stats to statsd at `localhost:8125`. Point Hydra to
 </statsd>
 ```
 
+hydra-notify's Prometheus service
+---------------------------------
+
+hydra-notify supports running a Prometheus webserver for metrics. The
+exporter does not run unless a listen address and port are specified
+in the hydra configuration file, as below:
+
+```conf
+<hydra_notify>
+  <prometheus>
+    listen_address = 127.0.0.1
+    port = 9199
+  </prometheus>
+</hydra_notify>
+```
+
 Using LDAP as authentication backend (optional)
 -----------------------------------------------
 
diff --git a/doc/manual/src/monitoring/README.md b/doc/manual/src/monitoring/README.md
index 65872352..67fb3961 100644
--- a/doc/manual/src/monitoring/README.md
+++ b/doc/manual/src/monitoring/README.md
@@ -13,3 +13,11 @@ $ curl --header "Accept: application/json" http://localhost:63333/queue-runner-s
 ... JSON payload ...
 ```
 
+## Notification Daemon
+
+The `hydra-notify` process can expose Prometheus metrics for plugin execution. See
+[hydra-notify's Prometheus service](../configuration.md#hydra-notifys-prometheus-service)
+for details on enabling and configuring the exporter.
+
+The notification exporter exposes metrics on a per-plugin, per-event-type basis: execution
+durations, frequency, successes, and failures.
diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm
index 7ab4ab60..ac0e54cf 100644
--- a/src/lib/Hydra/Helper/Nix.pm
+++ b/src/lib/Hydra/Helper/Nix.pm
@@ -70,6 +70,29 @@ sub getStatsdConfig {
     }
 }
 
+sub getHydraNotifyPrometheusConfig {
+    my ($config) = @_;
+    my $cfg = $config->{hydra_notify};
+
+    if (!defined($cfg) || ref $cfg ne "HASH") {
+        return undef;
+    }
+
+    my $cfg = $cfg->{prometheus};
+    if (!defined($cfg) || ref $cfg ne "HASH") {
+        return undef;
+    }
+
+    if (defined($cfg->{"listen_address"}) && defined($cfg->{"port"})) {
+        return {
+            "listen_address" => $cfg->{'listen_address'},
+            "port" => $cfg->{'port'},
+        };
+    }
+
+    return undef;
+}
+
 
 sub getBaseUrl {
     my ($config) = @_;
diff --git a/src/script/hydra-notify b/src/script/hydra-notify
index f64b5d72..9242d68f 100755
--- a/src/script/hydra-notify
+++ b/src/script/hydra-notify
@@ -14,30 +14,34 @@ use Parallel::ForkManager;
 use Prometheus::Tiny::Shared;
 use Time::HiRes qw( gettimeofday tv_interval );
 
-my $prom = Prometheus::Tiny::Shared->new;
-
-my $fork_manager = Parallel::ForkManager->new(1 );
-$fork_manager->start_child("metrics_exporter", sub {
-    my $server = HTTP::Server::PSGI->new(
-        host => "127.0.0.1",
-        port => 9091,
-        timeout => 5,
-    );
-
-    $server->run($prom->psgi);
-});
-
 STDERR->autoflush(1);
 STDOUT->autoflush(1);
 binmode STDERR, ":encoding(utf8)";
 
+my $config = getHydraConfig();
+
+my $prom = Prometheus::Tiny::Shared->new;
+
+my $promCfg = Hydra::Helper::Nix::getHydraNotifyPrometheusConfig($config);
+if (defined($promCfg)) {
+    my $fork_manager = Parallel::ForkManager->new(1);
+    $fork_manager->start_child("metrics_exporter", sub {
+        my $server = HTTP::Server::PSGI->new(
+            host => $promCfg->{"listen_address"},
+            port => $promCfg->{"port"},
+            timeout => 1,
+        );
+
+        $server->run($prom->psgi);
+    });
+}
+
 my $queued_only;
 
 GetOptions(
     "queued-only" => \$queued_only
 ) or exit 1;
 
-my $config = getHydraConfig();
 
 my $db = Hydra::Model::DB->new();
 
diff --git a/t/Config/hydra-notify.t b/t/Config/hydra-notify.t
new file mode 100644
index 00000000..7fc8aa63
--- /dev/null
+++ b/t/Config/hydra-notify.t
@@ -0,0 +1,87 @@
+use strict;
+use Setup;
+
+my %ctx = test_init(hydra_config => q|
+<hydra_notify>
+  <prometheus>
+    listen_address = 127.0.0.1
+    port = 9199
+  </prometheus>
+</hydra_notify>
+|);
+
+require Hydra::Helper::Nix;
+use Test2::V0;
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig(Hydra::Helper::Nix::getHydraConfig()), {
+    'listen_address' => "127.0.0.1",
+    'port' => 9199
+}, "Reading specific configuration from the hydra.conf works");
+
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({
+    "hydra_notify" => ":)"
+}), undef, "Invalid (hydra_notify is a string) configuration options are undef");
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({
+    "hydra_notify" => []
+}), undef, "Invalid (hydra_notify is a list) configuration options are undef");
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({
+    "hydra_notify" => {}
+}), undef, "Invalid (hydra_notify is an empty hash) configuration options are undef");
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({
+    "hydra_notify" => {
+        "prometheus" => ":)"
+    }
+}), undef, "Invalid (hydra_notify.prometheus is a string) configuration options are undef");
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({
+    "hydra_notify" => {
+        "prometheus" => {}
+    }
+}), undef, "Invalid (hydra_notify.prometheus is an empty hash) configuration options are undef");
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({
+    "hydra_notify" => {
+        "prometheus" => {
+            "listen_address" => "0.0.0.0"
+        }
+    }
+}), undef, "Invalid (hydra_notify.prometheus.port is missing) configuration options are undef");
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({
+    "hydra_notify" => {
+        "prometheus" => {
+            "port" => 1234
+        }
+    }
+}), undef, "Invalid (hydra_notify.prometheus.listen_address is missing) configuration options are undef");
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({
+    "hydra_notify" => {
+        "prometheus" => {
+            "listen_address" => "127.0.0.1",
+            "port" => 1234
+        }
+    }
+}), {
+    "listen_address" => "127.0.0.1",
+    "port" => 1234
+}, "Fully specified hydra_notify.prometheus config is valid and returned");
+
+is(Hydra::Helper::Nix::getHydraNotifyPrometheusConfig({
+    "hydra_notify" => {
+        "prometheus" => {
+            "listen_address" => "127.0.0.1",
+            "port" => 1234,
+            "extra_keys" => "meh",
+        }
+    }
+}), {
+    "listen_address" => "127.0.0.1",
+    "port" => 1234
+}, "extra configuration in hydra_notify.prometheus is not returned");
+
+done_testing;

From f4ad80527a5949ef27bbc52190a74a8af800d8fb Mon Sep 17 00:00:00 2001
From: Your Name <graham@grahamc.com>
Date: Wed, 18 Aug 2021 15:26:04 -0400
Subject: [PATCH 3/8] hydra-notify: Enable the prometheus exporter in
 development environments by default

---
 foreman/start-hydra.sh | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/foreman/start-hydra.sh b/foreman/start-hydra.sh
index 4e348266..cedcde6e 100755
--- a/foreman/start-hydra.sh
+++ b/foreman/start-hydra.sh
@@ -19,6 +19,13 @@ if [ ! -f ./.hydra-data/hydra.conf ]; then
     cat << EOF > .hydra-data/hydra.conf
 # test-time instances likely don't want to bootstrap nixpkgs from scratch
 use-substitutes = true
+
+<hydra_notify>
+  <prometheus>
+    listen_address = 127.0.0.1
+    port = 64445
+  </prometheus>
+</hydra_notify>
 EOF
 fi
 HYDRA_CONFIG=$(pwd)/.hydra-data/hydra.conf exec hydra-dev-server --port 63333

From 5c1228e141d9258492c5773fffe49a56ea4c5270 Mon Sep 17 00:00:00 2001
From: Your Name <graham@grahamc.com>
Date: Wed, 18 Aug 2021 15:28:01 -0400
Subject: [PATCH 4/8] hydra-notify: pre-declare metrics

---
 src/script/hydra-notify | 43 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/src/script/hydra-notify b/src/script/hydra-notify
index 9242d68f..b0bfcdc9 100755
--- a/src/script/hydra-notify
+++ b/src/script/hydra-notify
@@ -21,6 +21,49 @@ binmode STDERR, ":encoding(utf8)";
 my $config = getHydraConfig();
 
 my $prom = Prometheus::Tiny::Shared->new;
+# Note: It is very important to pre-declare any metrics before using them.
+# Add a new declaration for any new metrics you create. See:
+# https://metacpan.org/pod/Prometheus::Tiny#declare
+$prom->declare(
+    "notify_plugin_executions",
+    type => "counter",
+    help => "Number of times each plugin has been called by channel."
+);
+$prom->declare(
+    "notify_plugin_runtime",
+    type => "histogram",
+    help => "Number of seconds spent executing each plugin by channel."
+);
+$prom->declare(
+    "notify_plugin_success",
+    type => "counter",
+    help => "Number of successful executions of this plugin on this channel."
+);
+$prom->declare(
+    "notify_plugin_error",
+    type => "counter",
+    help => "Number of failed executions of this plugin on this channel."
+);
+$prom->declare(
+    "event_loop_iterations",
+    type => "counter",
+    help => "Number of iterations through the event loop. Incremented at the start of the event loop."
+);
+$prom->declare(
+    "event_received",
+    type => "counter",
+    help => "Timestamp of the last time a new event was received."
+);
+$prom->declare(
+    "notify_event",
+    type => "counter",
+    help => "Number of events received on the given channel."
+);
+$prom->declare(
+    "notify_event_error",
+    type => "counter",
+    help => "Number of events received that were unprocessable by channel."
+);
 
 my $promCfg = Hydra::Helper::Nix::getHydraNotifyPrometheusConfig($config);
 if (defined($promCfg)) {

From de2282bcf42742a810406100210bb97f2f88426e Mon Sep 17 00:00:00 2001
From: Your Name <graham@grahamc.com>
Date: Wed, 18 Aug 2021 15:30:35 -0400
Subject: [PATCH 5/8] hydra-notify: print out log lines indicating it is or is
 not launching the exporter

---
 src/script/hydra-notify | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/script/hydra-notify b/src/script/hydra-notify
index b0bfcdc9..fbadf7ee 100755
--- a/src/script/hydra-notify
+++ b/src/script/hydra-notify
@@ -67,6 +67,7 @@ $prom->declare(
 
 my $promCfg = Hydra::Helper::Nix::getHydraNotifyPrometheusConfig($config);
 if (defined($promCfg)) {
+    print STDERR "Starting the Prometheus exporter, listening on http://${\$promCfg->{'listen_address'}}:${\$promCfg->{'port'}}/metrics.\n";
     my $fork_manager = Parallel::ForkManager->new(1);
     $fork_manager->start_child("metrics_exporter", sub {
         my $server = HTTP::Server::PSGI->new(
@@ -77,6 +78,8 @@ if (defined($promCfg)) {
 
         $server->run($prom->psgi);
     });
+} else {
+    print STDERR "Not starting the hydra-notify Prometheus exporter.\n";
 }
 
 my $queued_only;

From 45e8fa53195f6328f6a7880753c3f7185b512bfa Mon Sep 17 00:00:00 2001
From: Your Name <graham@grahamc.com>
Date: Wed, 18 Aug 2021 15:42:03 -0400
Subject: [PATCH 6/8] hydra-notify: support sending diagnostic dumps to STDERR
 on request

---
 doc/manual/src/monitoring/README.md | 10 ++++++++++
 src/script/hydra-notify             |  6 ++++++
 2 files changed, 16 insertions(+)

diff --git a/doc/manual/src/monitoring/README.md b/doc/manual/src/monitoring/README.md
index 67fb3961..1f17a64d 100644
--- a/doc/manual/src/monitoring/README.md
+++ b/doc/manual/src/monitoring/README.md
@@ -21,3 +21,13 @@ for details on enabling and configuring the exporter.
 
 The notification exporter exposes metrics on a per-plugin, per-event-type basis: execution
 durations, frequency, successes, and failures.
+
+### Diagnostic Dump
+
+The notification daemon can also dump its metrics to stderr whether or not the exporter
+is configured. This is particularly useful for cases where metrics data is needed but the
+exporter was not enabled.
+
+To trigger this diagnostic dump, send a Postgres notification with the
+`hydra_notify_dump_metrics` channel and no payload. See
+[Re-sending a notification](../notifications.md#re-sending-a-notification).
diff --git a/src/script/hydra-notify b/src/script/hydra-notify
index fbadf7ee..92776eaf 100755
--- a/src/script/hydra-notify
+++ b/src/script/hydra-notify
@@ -99,6 +99,7 @@ my $listener = Hydra::PostgresListener->new($dbh);
 $listener->subscribe("build_started");
 $listener->subscribe("build_finished");
 $listener->subscribe("step_finished");
+$listener->subscribe("hydra_notify_dump_metrics");
 
 sub runPluginsForEvent {
     my ($event) = @_;
@@ -145,6 +146,11 @@ while (!$queued_only) {
 
         $prom->inc("notify_event", { channel => $channelName });
 
+        if ($channelName eq "hydra_notify_dump_metrics") {
+            print STDERR "Dumping prometheus metrics:\n${\$prom->format}\n";
+            next;
+        }
+
         eval {
             my $event = Hydra::Event->new_event($channelName, $message->{"payload"});
             runPluginsForEvent($event);

From 4ebdcc290e1fb713b2ab1ed7c4fee5214fc0bd38 Mon Sep 17 00:00:00 2001
From: Your Name <graham@grahamc.com>
Date: Tue, 24 Aug 2021 10:57:23 -0400
Subject: [PATCH 7/8] fixup! hydra-notify: pre-declare metrics

---
 src/script/hydra-notify | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/script/hydra-notify b/src/script/hydra-notify
index 92776eaf..770f0620 100755
--- a/src/script/hydra-notify
+++ b/src/script/hydra-notify
@@ -22,7 +22,8 @@ my $config = getHydraConfig();
 
 my $prom = Prometheus::Tiny::Shared->new;
 # Note: It is very important to pre-declare any metrics before using them.
-# Add a new declaration for any new metrics you create. See:
+# Add a new declaration for any new metrics you create. Metrics which are
+# not pre-declared disappear when their value is null. See:
 # https://metacpan.org/pod/Prometheus::Tiny#declare
 $prom->declare(
     "notify_plugin_executions",

From 24467a7bdede3945dcbec4a35454931a7571848d Mon Sep 17 00:00:00 2001
From: Your Name <graham@grahamc.com>
Date: Tue, 24 Aug 2021 11:35:38 -0400
Subject: [PATCH 8/8] Nix::getHydraNotifyPrometheusConfig: print errors if the
 configuration provided is invalid.

---
 src/lib/Hydra/Helper/Nix.pm | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm
index ac0e54cf..bd294b73 100644
--- a/src/lib/Hydra/Helper/Nix.pm
+++ b/src/lib/Hydra/Helper/Nix.pm
@@ -74,12 +74,22 @@ sub getHydraNotifyPrometheusConfig {
     my ($config) = @_;
     my $cfg = $config->{hydra_notify};
 
-    if (!defined($cfg) || ref $cfg ne "HASH") {
+    if (!defined($cfg)) {
+        return undef;
+    }
+
+    if (ref $cfg ne "HASH") {
+        print STDERR "Error reading Hydra's configuration file: hydra_notify should be a block.\n";
         return undef;
     }
 
     my $cfg = $cfg->{prometheus};
-    if (!defined($cfg) || ref $cfg ne "HASH") {
+    if (!defined($cfg)) {
+        return undef;
+    }
+
+    if (ref $cfg ne "HASH") {
+        print STDERR "Error reading Hydra's configuration file: hydra_notify.prometheus should be a block.\n";
         return undef;
     }
 
@@ -88,6 +98,9 @@ sub getHydraNotifyPrometheusConfig {
             "listen_address" => $cfg->{'listen_address'},
             "port" => $cfg->{'port'},
         };
+    } else {
+        print STDERR "Error reading Hydra's configuration file: hydra_notify.prometheus should include listen_address and port.\n";
+        return undef;
     }
 
     return undef;