snaakey/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

hydra-notify: initial scratch take of prometheus events

Browse source
This commit is contained in:
Your Name 2021-08-18 14:08:38 -04:00
parent 4aa6a7b6ee
commit 5d0ad5f649
2 changed files with 31 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
flake.nix
View file

@ -470,6 +470,7 @@
NetPrometheus NetPrometheus
NetStatsd NetStatsd
PadWalker PadWalker
ParallelForkManager
PerlCriticCommunity PerlCriticCommunity
PrometheusTinyShared PrometheusTinyShared
Readonly Readonly

31
src/script/hydra-notify
View file

@ -3,12 +3,29 @@
use strict; use strict;
use utf8; use utf8;
use Getopt::Long; use Getopt::Long;
use HTTP::Server::PSGI;
use Hydra::Event; use Hydra::Event;
use Hydra::Event::BuildFinished; use Hydra::Event::BuildFinished;
use Hydra::Helper::AddBuilds; use Hydra::Helper::AddBuilds;
use Hydra::Helper::Nix; use Hydra::Helper::Nix;
use Hydra::Plugin; use Hydra::Plugin;
use Hydra::PostgresListener; use Hydra::PostgresListener;
use Parallel::ForkManager;
use Prometheus::Tiny::Shared;
use Time::HiRes qw( gettimeofday tv_interval );
my $prom = Prometheus::Tiny::Shared->new;
my $fork_manager = Parallel::ForkManager->new(1 );
$fork_manager->start_child("metrics_exporter", sub {
my $server = HTTP::Server::PSGI->new(
host => "127.0.0.1",
port => 9091,
timeout => 5,
);
$server->run($prom->psgi);
});
STDERR->autoflush(1); STDERR->autoflush(1);
STDOUT->autoflush(1); STDOUT->autoflush(1);
@ -36,11 +53,19 @@ $listener->subscribe("step_finished");
sub runPluginsForEvent { sub runPluginsForEvent {
my ($event) = @_; my ($event) = @_;
my $channelName = $event->{'channel_name'};
foreach my $plugin (@plugins) { foreach my $plugin (@plugins) {
$prom->inc("notify_plugin_executions", { channel => $channelName, plugin => ref $plugin });
eval { eval {
my $startTime = [gettimeofday()];
$event->execute($db, $plugin); $event->execute($db, $plugin);
$prom->histogram_observe("notify_plugin_runtime", tv_interval($startTime), { channel => $channelName, plugin => ref $plugin });
$prom->inc("notify_plugin_success", { channel => $channelName, plugin => ref $plugin });
1; 1;
} or do { } or do {
$prom->inc("notify_plugin_error", { channel => $channelName, plugin => ref $plugin });
print STDERR "error running $event->{'channel_name'} hooks: $@\n"; print STDERR "error running $event->{'channel_name'} hooks: $@\n";
} }
} }
@ -60,19 +85,23 @@ for my $build ($db->resultset('Builds')->search(
# Process incoming notifications. # Process incoming notifications.
while (!$queued_only) { while (!$queued_only) {
$prom->inc("event_loop_iterations");
my $messages = $listener->block_for_messages(); my $messages = $listener->block_for_messages();
while (my $message = $messages->()) { while (my $message = $messages->()) {
$prom->set("event_received", time());
my $channelName = $message->{"channel"}; my $channelName = $message->{"channel"};
my $pid = $message->{"pid"}; my $pid = $message->{"pid"};
my $payload = $message->{"payload"}; my $payload = $message->{"payload"};
$prom->inc("notify_event", { channel => $channelName });
eval { eval {
my $event = Hydra::Event->new_event($channelName, $message->{"payload"}); my $event = Hydra::Event->new_event($channelName, $message->{"payload"});
runPluginsForEvent($event); runPluginsForEvent($event);
1; 1;
} or do { } or do {
$prom->inc("notify_event_error", { channel => $channelName });
print STDERR "error processing message '$payload' on channel '$channelName': $@\n"; print STDERR "error processing message '$payload' on channel '$channelName': $@\n";
} }
} }