From b1fcc0c25bc402c73f66269a14f915bd8dbc0994 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 7 Jun 2021 11:34:15 -0400 Subject: [PATCH 1/6] init DataRandom perl package for Prometheus stats --- flake.nix | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/flake.nix b/flake.nix index 017e0b20..1b748100 100644 --- a/flake.nix +++ b/flake.nix @@ -111,6 +111,20 @@ }; }; + DataRandom = final.buildPerlPackage { + pname = "Data-Random"; + version = "0.13"; + src = final.fetchurl { + url = "mirror://cpan/authors/id/B/BA/BAREFOOT/Data-Random-0.13.tar.gz"; + sha256 = "eb590184a8db28a7e49eab09e25f8650c33f1f668b6a472829de74a53256bfc0"; + }; + buildInputs = with final.perlPackages; [ FileShareDirInstall TestMockTime ]; + meta = { + description = "Perl module to generate random data"; + license = with final.lib.licenses; [ artistic1 gpl1Plus ]; + }; + }; + DirSelf = final.buildPerlPackage { pname = "Dir-Self"; version = "0.11"; From c6670b805fd05c643139fbd10a433cae4113170e Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 7 Jun 2021 11:34:30 -0400 Subject: [PATCH 2/6] init: HashSharedMem for Prometheus --- flake.nix | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/flake.nix b/flake.nix index 1b748100..73dbf94f 100644 --- a/flake.nix +++ b/flake.nix @@ -139,6 +139,19 @@ }; }; + HashSharedMem = final.perlPackages.buildPerlModule { + pname = "Hash-SharedMem"; + version = "0.005"; + src = final.fetchurl { + url = "mirror://cpan/authors/id/Z/ZE/ZEFRAM/Hash-SharedMem-0.005.tar.gz"; + sha256 = "324776808602f7bdc44adaa937895365454029a926fa611f321c9bf6b940bb5e"; + }; + buildInputs = with final.perlPackages; [ ScalarString ]; + meta = { + description = "Efficient shared mutable hash"; + license = with final.lib.licenses; [ artistic1 gpl1Plus ]; + }; + }; TieHashMethod = final.buildPerlPackage { pname = "Tie-Hash-Method"; version = "0.02"; From 1652631b213eb59c4b90d34a608e88caecd98cd5 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 7 Jun 2021 11:34:51 -0400 Subject: [PATCH 3/6] Prometheus: init Tiny, TinyShared --- flake.nix | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/flake.nix b/flake.nix index 73dbf94f..6bc04d2e 100644 --- a/flake.nix +++ b/flake.nix @@ -152,6 +152,38 @@ license = with final.lib.licenses; [ artistic1 gpl1Plus ]; }; }; + + PrometheusTiny = final.buildPerlPackage { + pname = "Prometheus-Tiny"; + version = "0.007"; + src = final.fetchurl { + url = "mirror://cpan/authors/id/R/RO/ROBN/Prometheus-Tiny-0.007.tar.gz"; + sha256 = "0ef8b226a2025cdde4df80129dd319aa29e884e653c17dc96f4823d985c028ec"; + }; + buildInputs = with final.perlPackages; [ HTTPMessage Plack TestException ]; + meta = { + homepage = "https://github.com/robn/Prometheus-Tiny"; + description = "A tiny Prometheus client"; + license = with final.lib.licenses; [ artistic1 gpl1Plus ]; + }; + }; + + PrometheusTinyShared = final.buildPerlPackage { + pname = "Prometheus-Tiny-Shared"; + version = "0.023"; + src = final.fetchurl { + url = "mirror://cpan/authors/id/R/RO/ROBN/Prometheus-Tiny-Shared-0.023.tar.gz"; + sha256 = "7c2c72397be5d8e4839d1bf4033c1800f467f2509689673c6419df48794f2abe"; + }; + buildInputs = with final.perlPackages; [ DataRandom HTTPMessage Plack TestDifferences TestException ]; + propagatedBuildInputs = with final.perlPackages; [ HashSharedMem JSONXS PrometheusTiny ]; + meta = { + homepage = "https://github.com/robn/Prometheus-Tiny-Shared"; + description = "A tiny Prometheus client with a shared database behind it"; + license = with final.lib.licenses; [ artistic1 gpl1Plus ]; + }; + }; + TieHashMethod = final.buildPerlPackage { pname = "Tie-Hash-Method"; version = "0.02"; From db6bad108bbf391f146cbbfe7bceb9fe965f9841 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 7 Jun 2021 11:35:12 -0400 Subject: [PATCH 4/6] Use PrometheusTinyShared --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 6bc04d2e..6231cc8a 100644 --- a/flake.nix +++ b/flake.nix @@ -391,6 +391,7 @@ NetPrometheus NetStatsd PadWalker + PrometheusTinyShared Readonly SQLSplitStatement SetScalar From 71453dd3418caaf0bd8fbc50d25e3ddf51ae1bc8 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 10 Jun 2021 17:45:45 -0400 Subject: [PATCH 5/6] Expose Prometheus metrics at /metrics Exposes metrics: * http_request_duration_seconds_bucket * http_request_size_bytes_bucket * http_response_size_bytes_bucket * http_requests_total with labels of action and controller to help identify popular endpoints and their performance characteristics. --- flake.nix | 16 ++++++++++++++++ src/lib/Hydra.pm | 6 +++++- t/Controller/metrics.t | 30 ++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 t/Controller/metrics.t diff --git a/flake.nix b/flake.nix index 6231cc8a..f53ed808 100644 --- a/flake.nix +++ b/flake.nix @@ -70,6 +70,21 @@ }; }; + CatalystPluginPrometheusTiny = final.buildPerlPackage { + pname = "Catalyst-Plugin-PrometheusTiny"; + version = "0.005"; + src = final.fetchurl { + url = "mirror://cpan/authors/id/S/SY/SYSPETE/Catalyst-Plugin-PrometheusTiny-0.005.tar.gz"; + sha256 = "a42ef09efdc3053899ae007c41220d3ed7207582cc86e491b4f534539c992c5a"; + }; + buildInputs = with final.perlPackages; [ HTTPMessage Plack SubOverride TestDeep ]; + propagatedBuildInputs = with final.perlPackages; [ CatalystRuntime Moose PrometheusTiny PrometheusTinyShared ]; + meta = { + description = "Prometheus metrics for Catalyst"; + license = with final.lib.licenses; [ artistic1 gpl1Plus ]; + }; + }; + CryptArgon2 = final.perlPackages.buildPerlModule { pname = "Crypt-Argon2"; version = "0.010"; @@ -358,6 +373,7 @@ CatalystPluginAccessLog CatalystPluginAuthorizationRoles CatalystPluginCaptcha + CatalystPluginPrometheusTiny CatalystPluginSessionStateCookie CatalystPluginSessionStoreFastMmap CatalystPluginStackTrace diff --git a/src/lib/Hydra.pm b/src/lib/Hydra.pm index 0ff86f11..d77c3dba 100644 --- a/src/lib/Hydra.pm +++ b/src/lib/Hydra.pm @@ -16,7 +16,8 @@ use Catalyst qw/ConfigLoader Session Session::Store::FastMmap Session::State::Cookie - Captcha/, + Captcha + PrometheusTiny/, '-Log=warn,fatal,error'; use CatalystX::RoleApplicator; use YAML qw(LoadFile); @@ -47,6 +48,9 @@ __PACKAGE__->config( file($ENV{'HYDRA_LDAP_CONFIG'}) ) : undef }, + 'Plugin::PrometheusTiny' => { + include_action_labels => 1, + }, 'Plugin::Static::Simple' => { send_etag => 1, expires => 3600 diff --git a/t/Controller/metrics.t b/t/Controller/metrics.t new file mode 100644 index 00000000..8915104d --- /dev/null +++ b/t/Controller/metrics.t @@ -0,0 +1,30 @@ +use feature 'unicode_strings'; +use strict; +use Setup; +use JSON qw(decode_json encode_json); + +my %ctx = test_init(); + +require Hydra::Schema; +require Hydra::Model::DB; +require Hydra::Helper::Nix; +use HTTP::Request::Common; + +use Test2::V0; +require Catalyst::Test; +Catalyst::Test->import('Hydra'); + +my $db = Hydra::Model::DB->new; +hydra_setup($db); + +request(GET '/'); +my $metrics = request(GET '/metrics'); +ok($metrics->is_success); + +like( + $metrics->content, + qr/http_requests_total\{action="index",code="200",controller="Hydra::Controller::Root",method="GET"\} 1/, + "Metrics are collected" +); + +done_testing; From 5700c351e8154d8ea1548e730ba21d2ceecce2e3 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 10 Jun 2021 20:13:08 -0400 Subject: [PATCH 6/6] Init a docs section for monitoring, document queue-runner-status and the prometheus metrics --- doc/manual/src/SUMMARY.md | 1 + doc/manual/src/monitoring/README.md | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 doc/manual/src/monitoring/README.md diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md index f0dc77a4..586c09aa 100644 --- a/doc/manual/src/SUMMARY.md +++ b/doc/manual/src/SUMMARY.md @@ -4,6 +4,7 @@ - [Installation](installation.md) - [Creating and Managing Projects](projects.md) - [Using the external API](api.md) +- [Monitoring Hydra](./monitoring/README.md) ----------- [About](about.md) [Hacking](hacking.md) diff --git a/doc/manual/src/monitoring/README.md b/doc/manual/src/monitoring/README.md new file mode 100644 index 00000000..65872352 --- /dev/null +++ b/doc/manual/src/monitoring/README.md @@ -0,0 +1,15 @@ +# Monitoring Hydra + +## Webserver + +The webserver exposes Prometheus metrics for the webserver itself at `/metrics`. + +## Queue Runner + +The queue runner's status is exposed at `/queue-runner-status`: + +```console +$ curl --header "Accept: application/json" http://localhost:63333/queue-runner-status +... JSON payload ... +``` +