From 45e8fa53195f6328f6a7880753c3f7185b512bfa Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 18 Aug 2021 15:42:03 -0400 Subject: [PATCH] hydra-notify: support sending diagnostic dumps to STDERR on request --- doc/manual/src/monitoring/README.md | 10 ++++++++++ src/script/hydra-notify | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/doc/manual/src/monitoring/README.md b/doc/manual/src/monitoring/README.md index 67fb3961..1f17a64d 100644 --- a/doc/manual/src/monitoring/README.md +++ b/doc/manual/src/monitoring/README.md @@ -21,3 +21,13 @@ for details on enabling and configuring the exporter. The notification exporter exposes metrics on a per-plugin, per-event-type basis: execution durations, frequency, successes, and failures. + +### Diagnostic Dump + +The notification daemon can also dump its metrics to stderr whether or not the exporter +is configured. This is particularly useful for cases where metrics data is needed but the +exporter was not enabled. + +To trigger this diagnostic dump, send a Postgres notification with the +`hydra_notify_dump_metrics` channel and no payload. See +[Re-sending a notification](../notifications.md#re-sending-a-notification). diff --git a/src/script/hydra-notify b/src/script/hydra-notify index fbadf7ee..92776eaf 100755 --- a/src/script/hydra-notify +++ b/src/script/hydra-notify @@ -99,6 +99,7 @@ my $listener = Hydra::PostgresListener->new($dbh); $listener->subscribe("build_started"); $listener->subscribe("build_finished"); $listener->subscribe("step_finished"); +$listener->subscribe("hydra_notify_dump_metrics"); sub runPluginsForEvent { my ($event) = @_; @@ -145,6 +146,11 @@ while (!$queued_only) { $prom->inc("notify_event", { channel => $channelName }); + if ($channelName eq "hydra_notify_dump_metrics") { + print STDERR "Dumping prometheus metrics:\n${\$prom->format}\n"; + next; + } + eval { my $event = Hydra::Event->new_event($channelName, $message->{"payload"}); runPluginsForEvent($event);