services/monitoring: add scraping of Gerrit's internal metrics

.gitignore

@@ -4,3 +4,5 @@ config.tf.json
 .direnv
 .terraform
 .terraform.lock.hcl
+secrets/*
+!secrets/*.age

secrets.nix

@@ -11,6 +11,7 @@ let
     mimir-environment = [ machines.meta01 ];
     grafana-oauth-secret = [ machines.meta01 ];
     loki-environment = [ machines.meta01 ];
+    gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
 
     # These are the same password, but nginx wants it in htpasswd format
     metrics-push-htpasswd = [ machines.meta01 ];

secrets/gerrit-prometheus-bearer-token.age

@@ -0,0 +1,22 @@
+age-encryption.org/v1
+-> ssh-ed25519 2D+APA Vh/FrR9oyO8V1pEMQkmGbHCePB6RU+dPm+Z4bgKenEg
+2G5eLlYe8IS7fsEBorFljUwQZ9sEk/FEr25S4p5hWLk
+-> ssh-ed25519 j2r2qQ 9+NX0Guhux9QlAxx2MtSZH0OZpDk1CQZ4Blu1P9fpgQ
+PDUoAjBaIdKQAvRblvc0QEtrvp5MpE8HsCwKWwAn0uE
+-> ssh-ed25519 K3b7BA wuOc6LGnjsC4Rb9D9QX3YVgMqWPvBK27Q0vqADLpsk8
+wRnoNzkyaU9SGlOtpqY2pAeIwD9lGWKrqNn3D3W7U6Y
+-> ssh-ed25519 +qVung biXtZHmjJmsazEmp1iIGUqmuV1YP94bzrMjoZTmGPjg
+GDN4WZGTIP6b2nmjyhikHeOrZi9YEtiPOyaJLzUl138
+-> ssh-rsa krWCLQ
+UkNySvhS5o6v6/7xGvn43hgD5y2D91oH4pjU3Oa83CW6ha80dnE+JkSTpTdz7Og0
+vtZJuisNpcH254zTt8OAUpWN/tVXlD34RyV1xo1eHEWgUzKactrhlACpSbzYBdVJ
+8cUj7jiE+qjIOtrU2sHWo09NKpf0J2YEPwajuBy1/fPrivlgXAzdAAnP4gll02x1
+Et8lUn6HVfYDGtrDo/PUUdgcGudVeCOJbvvrKYkuqe8vsNYgnFHM8dkTJmObL8dz
+zp4MEuIQ3WrrXActSnTs+QAGIFSskOIr1DQlJRYzQcYtd8wkfx9a+6oxBECZyDAZ
+T4yso7ctflKlr6OqpJYzeA
+-> ssh-ed25519 /vwQcQ +jsCn0OlVpuyVA0XSvD3ZCDRTBq29UV9qsDvE4XaGk0
+p2qblImpl+G0pefJ0T/GjanIc7+bNuA0wRB4mUuFGXM
+-> ssh-ed25519 0R97PA /bE6+eVlzeJKOOMqz4QjFdsu+5XDv9L8cZ94cPZ5WQk
+Xco24ijeQnaT7jcsfXLQPzGr1FE/zy9+qVoQ20DLP+Q
+--- NDqgX11cTXR48vD9YmAIYx+og0n1OQj+bbkKwqv2BeE
+šÊ\”wÔ
ðä9Ì’7öcØšƒ’‹%}|k®?š×$9·lö&<13>=¸vñþܹ!Pã<50>Þ3b·<62>ù퀩

services/gerrit/default.nix

@@ -60,6 +60,7 @@ in
 
       plugins = with pkgs.gerritPlugins; [ 
         oauth
+        metrics-reporter-prometheus
       ];
 
       package = pkgs.gerrit;
@@ -220,5 +221,12 @@ in
       };
       environment.REVWALK_USE_PRIORITY_QUEUE = "true";
     };
+
+    age.secrets.gerrit-prometheus-bearer-token.file = ../../secrets/gerrit-prometheus-bearer-token.age;
+    bagel.monitoring.grafana-agent.exporters.gerrit = {
+      port = 4778;  # grrt
+      bearerTokenFile = config.age.secrets.gerrit-prometheus-bearer-token.path;
+      scrapeConfig.metrics_path = "/plugins/metrics-reporter-prometheus/metrics";
+    };
   };
 }

services/monitoring/agent.nix

@@ -20,12 +20,40 @@ in
         internally, which ends up exported as `job` label
         on all metrics of that exporter.
       '';
-      type = types.attrsOf (types.submodule {
+      type = types.attrsOf (types.submodule ({ config, name, ... }: {
         options.port = mkOption {
           description = "Exporter port";
           type = types.int;
         };
-      });
+        options.bearerTokenFile = mkOption {
+          description = "File containing a bearer token";
+          type = types.nullOr types.path;
+          default = null;
+        };
+
+        options.scrapeConfig = mkOption {
+          description = "Prometheus scrape config";
+          type = types.attrs;
+        };
+        config.scrapeConfig = lib.mkMerge [{
+          job_name = name;
+          static_configs = [
+            { targets = [ "localhost:${toString config.port}" ]; }
+          ];
+        } (lib.mkIf (config.bearerTokenFile != null) {
+          authorization.credentials_file = "\${CREDENTIALS_DIRECTORY}/${name}-bearer-token";
+        })];
+
+        options.secrets = mkOption {
+          description = "Secrets required for scrape config";
+          type = types.attrs;
+          internal = true;
+          default = {};
+        };
+        config.secrets = lib.mkIf (config.bearerTokenFile != null) {
+          "${name}-bearer-token" = config.bearerTokenFile;
+        };
+      }));
       default = {};
     };
   };
@@ -35,7 +63,8 @@ in
 
     services.grafana-agent = {
       enable = true;
-      credentials.password = config.age.secrets.grafana-agent-password.path;
+      credentials = lib.mkMerge ([{ password = config.age.secrets.grafana-agent-password.path; }] ++
+        lib.mapAttrsToList (name: value: value.secrets) config.bagel.monitoring.grafana-agent.exporters);
       settings = {
         metrics = {
           global.remote_write = [
@@ -51,12 +80,7 @@ in
           configs = [
             {
               name = config.networking.hostName;
-              scrape_configs = lib.mapAttrsToList (name: value: {
+              scrape_configs = lib.mapAttrsToList (name: value: value.scrapeConfig) config.bagel.monitoring.grafana-agent.exporters;
-                job_name = name;
-                static_configs = [
-                  { targets = [ "localhost:${toString value.port}" ]; }
-                ];
-              }) config.bagel.monitoring.grafana-agent.exporters;
             }
           ];
         };