lix-project/hydra
3
0
Fork 6
Code Issues 2 Pull requests 1 Packages Projects Releases Wiki Activity

Merge pull request #1084 from DeterminateSystems/measure-per-event-duration

Browse source 
hydra-notify: track the duration of handling an event
This commit is contained in:
Graham Christensen 2021-12-21 20:00:30 -05:00 committed by GitHub
parent e1e5fafdff cf514baaf2
commit eb5f6c8490
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 10 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
src/script/hydra-notify
View file

@ -4,6 +4,7 @@ use strict;
use warnings; use warnings;
use utf8; use utf8;
use Getopt::Long; use Getopt::Long;
use Time::HiRes qw( gettimeofday tv_interval );
use HTTP::Server::PSGI; use HTTP::Server::PSGI;
use Hydra::Event; use Hydra::Event;
use Hydra::Event::BuildFinished; use Hydra::Event::BuildFinished;
@ -46,6 +47,11 @@ $prom->declare(
type => "counter", type => "counter",
help => "Number of events received that were unprocessable by channel." help => "Number of events received that were unprocessable by channel."
); );
$prom->declare(
"notify_event_runtime",
type => "histogram",
help => "Number of seconds spent executing events by channel."
);
my $promCfg = Hydra::Helper::Nix::getHydraNotifyPrometheusConfig($config); my $promCfg = Hydra::Helper::Nix::getHydraNotifyPrometheusConfig($config);
if (defined($promCfg)) { if (defined($promCfg)) {
@ -109,6 +115,7 @@ while (!$queued_only) {
$prom->inc("event_loop_iterations"); $prom->inc("event_loop_iterations");
my $messages = $listener->block_for_messages($taskretries->get_seconds_to_next_retry()); my $messages = $listener->block_for_messages($taskretries->get_seconds_to_next_retry());
while (my $message = $messages->()) { while (my $message = $messages->()) {
my $start_time = [gettimeofday()];
$prom->set("event_received", time()); $prom->set("event_received", time());
my $channelName = $message->{"channel"}; my $channelName = $message->{"channel"};
my $pid = $message->{"pid"}; my $pid = $message->{"pid"};
@ -129,7 +136,9 @@ while (!$queued_only) {
} or do { } or do {
$prom->inc("notify_event_error", { channel => $channelName }); $prom->inc("notify_event_error", { channel => $channelName });
print STDERR "error processing message '$payload' on channel '$channelName': $@\n"; print STDERR "error processing message '$payload' on channel '$channelName': $@\n";
} };
$prom->histogram_observe("notify_event_runtime", tv_interval($start_time), { channel => $channelName });
} }
my $task = $taskretries->get_retryable_task(); my $task = $taskretries->get_retryable_task();