Switch to push metrics with Grafana Agent

hosts/bagel-box/default.nix

@@ -40,7 +40,6 @@
     hydra.enable = true;
     hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
   };
-  bagel.meta.monitoring.address = "bagel-box.delroth.net";
 
   security.acme.acceptTerms = true;
   security.acme.defaults.email = "bagel@delroth.net";

hosts/cl.forkos.org/default.nix

@@ -24,7 +24,6 @@
       };
     };
   };
-  bagel.meta.monitoring.address = "gerrit01.infra.forkos.org";
 
   fileSystems."/gerrit-data" = {
     device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";

hosts/fodwatch.forkos.org/default.nix

@@ -24,8 +24,6 @@
     };
   };
 
-  bagel.meta.monitoring.address = "fodwatch.infra.forkos.org";
-
   i18n.defaultLocale = "en_US.UTF-8";
 
   system.stateVersion = "24.05";

hosts/meta01.nixpkgs.lahfa.xyz/default.nix

@@ -21,7 +21,6 @@
     enable = true;
     domain = "netbox.forkos.org";
   };
-  bagel.meta.monitoring.address = "meta01.infra.forkos.org";
   bagel.services.prometheus.enable = true;
   bagel.services.loki.enable = true;
   bagel.services.grafana.enable = true;

services/monitoring/agent.nix

@@ -0,0 +1,100 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.bagel.monitoring.grafana-agent;
+  inherit (lib) mkEnableOption mkOption mkIf types;
+  passwordAsCredential = "\${CREDENTIALS_DIRECTORY}/password";
+in
+{
+  options.bagel.monitoring.grafana-agent = {
+    enable = (mkEnableOption "Grafana Agent") // { default = true; };
+
+    exporters = mkOption {
+      description = "List of all exporters to scrape";
+      type = types.listOf (types.submodule {
+        options.port = mkOption {
+          description = "Exporter port";
+          type = types.int;
+        };
+      });
+      default = [];
+    };
+  };
+
+  config = mkIf cfg.enable {
+    age.secrets.grafana-agent-password.file = ../../secrets/metrics-push-password.age;
+
+    services.grafana-agent = {
+      enable = true;
+      credentials.password = config.age.secrets.grafana-agent-password.path;
+      settings = {
+        metrics = {
+          global.remote_write = [
+            {
+              url = "https://mimir.forkos.org/api/v1/push";
+              basic_auth = {
+                username = "promtail";
+                password_file = passwordAsCredential;
+              };
+            }
+          ];
+          configs = [
+            {
+              name = config.networking.hostName;
+              scrape_configs = [
+                {
+                  job_name = config.networking.hostName;
+                  static_configs = [
+                    { targets = map (e: "localhost:" + (toString e.port)) config.bagel.monitoring.grafana-agent.exporters; }
+                  ];
+                }
+              ];
+            }
+          ];
+        };
+        logs = {
+          global.clients = [
+            {
+              url = "https://loki.forkos.org/loki/api/v1/push";
+              basic_auth = {
+                username = "promtail";
+                password_file = passwordAsCredential;
+              };
+            }
+          ];
+          configs = [
+            {
+              name = "journald";
+              scrape_configs = [
+                {
+                  job_name = "system";
+                  journal = {
+                    max_age = "12h";
+                    labels = {
+                      job = "systemd-journal";
+                      host = config.networking.hostName;
+                    };
+                  };
+                  relabel_configs = [
+                    {
+                      source_labels = [ "__journal__systemd_unit" ];
+                      target_label = "unit";
+                    }
+                  ];
+                }
+              ];
+            }
+          ];
+          positions_directory = "\${STATE_DIRECTORY}/positions";
+        };
+        integrations.node_exporter.enable_collectors = [
+          "processes"
+          "systemd"
+        ];
+      };
+    };
+  };
+}

services/monitoring/default.nix

@@ -2,6 +2,6 @@
   imports = [
     ./exporters
     ./lgtm
-    ./promtail.nix
+    ./agent.nix
   ];
 }

services/monitoring/exporters/cadvisor.nix

@@ -17,6 +17,6 @@ in
       listenAddress = "0.0.0.0";
     };
 
-    bagel.meta.monitoring.exporters = [ { port = 9102; } ];
+    bagel.monitoring.grafana-agent.exporters = [ { port = 9102; } ];
   };
 }

services/monitoring/exporters/default.nix

@@ -1,37 +1,7 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  inherit (lib) mkOption types;
-in
 {
   imports = [
     ./cadvisor.nix
-    ./node.nix
     ./nginx.nix
     ./postgres.nix
   ];
-
-  options.bagel = {
-    meta.monitoring = {
-      address = mkOption {
-        description = "Node's public address";
-        type = types.str;
-      };
-      exporters = mkOption {
-        description = "List of all exporters to scrape";
-        type = types.listOf (types.submodule {
-          options.port = mkOption {
-            description = "Exporter port";
-            type = types.int;
-          };
-        });
-        default = [];
-      };
-    };
-  };
-
-  config.networking.firewall.allowedTCPPorts = map (e: e.port) config.bagel.meta.monitoring.exporters;
 }

services/monitoring/exporters/nginx.nix

@@ -30,7 +30,7 @@ in
       ];
     };
 
-    bagel.meta.monitoring.exporters = [
+    bagel.monitoring.grafana-agent.exporters = [
       { port = 9103; }
     ];
   };

services/monitoring/exporters/node.nix

@@ -1,25 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  cfg = config.bagel.monitoring.exporters.node;
-  inherit (lib) mkEnableOption mkIf;
-in
-{
-  options.bagel.monitoring.exporters.node.enable = (mkEnableOption "Standard node_exporter") // { default = true; };
-
-  config = mkIf cfg.enable {
-    services.prometheus.exporters.node = {
-      enable = true;
-      enabledCollectors = [
-        "processes"
-        "systemd"
-      ];
-      port = 9101;
-    };
-
-    bagel.meta.monitoring.exporters = [ { port = 9101; } ];
-  };
-}

services/monitoring/exporters/postgres.nix

@@ -24,7 +24,7 @@ in
 
     services.postgresql.settings.shared_preload_libraries = "pg_stat_statements";
 
-    bagel.meta.monitoring.exporters = [
+    bagel.monitoring.grafana-agent.exporters = [
       { port = 9104; }
     ];
   };

services/monitoring/lgtm/default.nix

@@ -2,6 +2,6 @@
   imports = [
     ./grafana.nix
     ./loki.nix
-    ./prometheus.nix
+    ./mimir.nix
   ];
 }

services/monitoring/lgtm/loki.nix

@@ -12,8 +12,8 @@ in
 
   config = mkIf cfg.enable {
     age.secrets = {
-      loki-htpasswd = {
+      metrics-push-htpasswd = {
-        file = ../../../secrets/loki-htpasswd.age;
+        file = ../../../secrets/metrics-push-htpasswd.age;
         owner = "nginx";
       };
       loki-environment.file = ../../../secrets/loki-environment.age;
@@ -93,7 +93,7 @@ in
       forceSSL = true;
       locations."/loki/api/v1/push" = {
         proxyPass = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}";
-        basicAuthFile = config.age.secrets.loki-htpasswd.path;
+        basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
       };
     };
   };

services/monitoring/lgtm/mimir.nix

@@ -1,7 +1,6 @@
 {
   config,
   lib,
-  nodes,
   pkgs,
   ...
 }:
@@ -9,43 +8,18 @@ let
   cfg = config.bagel.services.prometheus;
   inherit (lib) mkEnableOption mkIf;
 
-  forEachMachine = fn: map fn (builtins.attrValues nodes);
-
-  allMetas = forEachMachine (machine: {
-    name = machine.config.networking.hostName;
-    address = machine.config.bagel.meta.monitoring.address or null;
-    exporters = machine.config.bagel.meta.monitoring.exporters or [];
-  });
-
-  scrapableMetas = builtins.filter (m: m.address != null && m.exporters != []) allMetas;
-
-  toJobConfig = m: {
-    job_name = m.name;
-    static_configs = [
-      { targets = map (e: m.address + ":" + (toString e.port)) m.exporters; }
-    ];
-  };
-
-  jobConfigs = map toJobConfig scrapableMetas;
-
   mimirPort = config.services.mimir.configuration.server.http_listen_port;
 in
 {
   options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
 
   config = mkIf cfg.enable {
-    age.secrets.mimir-environment.file = ../../../secrets/mimir-environment.age;
+    age.secrets = {
-
+      metrics-push-htpasswd = {
-    services.prometheus = {
+        file = ../../../secrets/metrics-push-htpasswd.age;
-      enable = true;
+        owner = "nginx";
-      enableAgentMode = true;
+      };
-      listenAddress = "127.0.0.1";
+      mimir-environment.file = ../../../secrets/mimir-environment.age;
-      port = 9001;
-      globalConfig.scrape_interval = "15s";
-      scrapeConfigs = jobConfigs;
-      remoteWrite = [
-        { url = "http://localhost:${toString mimirPort}/api/v1/push"; }
-      ];
     };
 
     services.mimir = {
@@ -109,8 +83,9 @@ in
     services.nginx.virtualHosts."mimir.forkos.org" = {
       enableACME = true;
       forceSSL = true;
-      locations."/" = {
+      locations."/api/v1/push" = {
         proxyPass = "http://localhost:${toString mimirPort}";
+        basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
       };
     };
   };

services/monitoring/promtail.nix

@@ -1,53 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  cfg = config.bagel.monitoring.promtail;
-  inherit (lib) mkEnableOption mkIf;
-in
-{
-  options.bagel.monitoring.promtail.enable = (mkEnableOption "Promtail log export") // { default = true; };
-
-  config = mkIf cfg.enable {
-    age.secrets.promtail-password = {
-      file = ../../secrets/promtail-password.age;
-      owner = "promtail";
-    };
-
-    services.promtail = {
-      enable = true;
-      configuration = {
-        server.disable = true;
-        clients = [ 
-          {
-            url = "https://loki.forkos.org/loki/api/v1/push";
-            basic_auth = {
-              username = "promtail";
-              password_file = config.age.secrets.promtail-password.path;
-            };
-          }
-        ];
-        scrape_configs = [
-          {
-            job_name = "system";
-            journal = {
-              max_age = "12h";
-              labels = {
-                job = "systemd-journal";
-                host = config.networking.hostName;
-              };
-            };
-            relabel_configs = [
-              {
-                source_labels = [ "__journal__systemd_unit" ];
-                target_label = "unit";
-              }
-            ];
-          }
-        ];
-      };
-    };
-  };
-}