From e8e262c6a4d26032dc4724c22e0338d0e06c065b Mon Sep 17 00:00:00 2001
From: K900 <me@0upti.me>
Date: Sun, 7 Jul 2024 17:05:48 +0300
Subject: [PATCH 1/6] Enable Mimir Alertmanager, add example alert

Still TODO: actually connect it to Matrix
---
 services/monitoring/lgtm/alerts/demo.yaml |  5 +++
 services/monitoring/lgtm/prometheus.nix   | 38 +++++++++++++++++++++--
 2 files changed, 41 insertions(+), 2 deletions(-)
 create mode 100644 services/monitoring/lgtm/alerts/demo.yaml

diff --git a/services/monitoring/lgtm/alerts/demo.yaml b/services/monitoring/lgtm/alerts/demo.yaml
new file mode 100644
index 0000000..db38426
--- /dev/null
+++ b/services/monitoring/lgtm/alerts/demo.yaml
@@ -0,0 +1,5 @@
+groups:
+  - name: Demo alerts
+    rules:
+      - alert: Demo alert
+        expr: 1
diff --git a/services/monitoring/lgtm/prometheus.nix b/services/monitoring/lgtm/prometheus.nix
index 2f83b8f..ca07352 100644
--- a/services/monitoring/lgtm/prometheus.nix
+++ b/services/monitoring/lgtm/prometheus.nix
@@ -2,6 +2,7 @@
   config,
   lib,
   nodes,
+  pkgs,
   ...
 }:
 let
@@ -26,6 +27,8 @@ let
   };
 
   jobConfigs = map toJobConfig scrapableMetas;
+
+  mimirPort = config.services.mimir.configuration.server.http_listen_port;
 in
 {
   options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
@@ -41,7 +44,7 @@ in
       globalConfig.scrape_interval = "15s";
       scrapeConfigs = jobConfigs;
       remoteWrite = [
-        { url = "http://localhost:9009/api/v1/push"; }
+        { url = "http://localhost:${toString mimirPort}/api/v1/push"; }
       ];
     };
 
@@ -49,6 +52,8 @@ in
       enable = true;
       extraFlags = ["--config.expand-env=true"];
       configuration = {
+        target = "all,alertmanager";
+
         multitenancy_enabled = false;
 
         common.storage = {
@@ -73,11 +78,40 @@ in
         blocks_storage.backend = "s3";
         ruler_storage = {
           backend = "local";
-          local.directory = ./alerts;
+          local.directory = pkgs.runCommand "mimir-rules" {} ''
+            mkdir -p $out
+            ln -s ${./alerts} $out/anonymous
+          '';
         };
+
+        alertmanager = {
+          sharding_ring.replication_factor = 1;
+          fallback_config_file = pkgs.writers.writeYAML "alertmanager.yaml" {
+            route = {
+              group_by = ["alertname"];
+              receiver = "matrix";
+            };
+            receivers = [
+              {
+                name = "matrix";
+              }
+            ];
+          };
+        };
+        alertmanager_storage.backend = "filesystem";
+
+        ruler.alertmanager_url = "http://localhost:${toString mimirPort}/alertmanager";
       };
     };
 
     systemd.services.mimir.serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
+
+    services.nginx.virtualHosts."mimir.forkos.org" = {
+      enableACME = true;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://localhost:${toString mimirPort}";
+      };
+    };
   };
 }
-- 
2.44.1


From 346a74eabc93c47557bc08f3204957157c858077 Mon Sep 17 00:00:00 2001
From: K900 <me@0upti.me>
Date: Sun, 7 Jul 2024 17:13:12 +0300
Subject: [PATCH 2/6] Wire up Grafana to Alertmanager

---
 services/monitoring/lgtm/grafana.nix | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/services/monitoring/lgtm/grafana.nix b/services/monitoring/lgtm/grafana.nix
index 8fd2583..e567f65 100644
--- a/services/monitoring/lgtm/grafana.nix
+++ b/services/monitoring/lgtm/grafana.nix
@@ -92,6 +92,7 @@ in
                 uid = "mimir";
                 access = "proxy";
                 url = "http://127.0.0.1:9009/prometheus";
+                isDefault = true;
               }
               {
                 name = "Loki";
@@ -100,6 +101,17 @@ in
                 access = "proxy";
                 url = "http://127.0.0.1:9090/";
               }
+              {
+                name = "Mimir Alertmanager";
+                type = "alertmanager";
+                uid = "mimir-alertmanager";
+                access = "proxy";
+                url = "http://127.0.0.1:9009/";
+                jsonData = {
+                  handleGrafanaManagedAlerts = true;
+                  implementation = "mimir";
+                };
+              }
             ];
           };
         };
-- 
2.44.1


From 40ba3c4ae71521f8585e39743801adabb08cf18b Mon Sep 17 00:00:00 2001
From: K900 <me@0upti.me>
Date: Sun, 7 Jul 2024 17:58:51 +0300
Subject: [PATCH 3/6] Prepare for remote push metrics

---
 .../{loki-htpasswd.age => metrics-push-htpasswd.age}  |  0
 ...romtail-password.age => metrics-push-password.age} |  0
 services/monitoring/lgtm/loki.nix                     |  6 +++---
 services/monitoring/lgtm/prometheus.nix               | 11 +++++++++--
 services/monitoring/promtail.nix                      |  2 +-
 5 files changed, 13 insertions(+), 6 deletions(-)
 rename secrets/{loki-htpasswd.age => metrics-push-htpasswd.age} (100%)
 rename secrets/{promtail-password.age => metrics-push-password.age} (100%)

diff --git a/secrets/loki-htpasswd.age b/secrets/metrics-push-htpasswd.age
similarity index 100%
rename from secrets/loki-htpasswd.age
rename to secrets/metrics-push-htpasswd.age
diff --git a/secrets/promtail-password.age b/secrets/metrics-push-password.age
similarity index 100%
rename from secrets/promtail-password.age
rename to secrets/metrics-push-password.age
diff --git a/services/monitoring/lgtm/loki.nix b/services/monitoring/lgtm/loki.nix
index d862046..8168abd 100644
--- a/services/monitoring/lgtm/loki.nix
+++ b/services/monitoring/lgtm/loki.nix
@@ -12,8 +12,8 @@ in
 
   config = mkIf cfg.enable {
     age.secrets = {
-      loki-htpasswd = {
-        file = ../../../secrets/loki-htpasswd.age;
+      metrics-push-htpasswd = {
+        file = ../../../secrets/metrics-push-htpasswd.age;
         owner = "nginx";
       };
       loki-environment.file = ../../../secrets/loki-environment.age;
@@ -93,7 +93,7 @@ in
       forceSSL = true;
       locations."/loki/api/v1/push" = {
         proxyPass = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}";
-        basicAuthFile = config.age.secrets.loki-htpasswd.path;
+        basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
       };
     };
   };
diff --git a/services/monitoring/lgtm/prometheus.nix b/services/monitoring/lgtm/prometheus.nix
index ca07352..54667ce 100644
--- a/services/monitoring/lgtm/prometheus.nix
+++ b/services/monitoring/lgtm/prometheus.nix
@@ -34,7 +34,13 @@ in
   options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
 
   config = mkIf cfg.enable {
-    age.secrets.mimir-environment.file = ../../../secrets/mimir-environment.age;
+    age.secrets = {
+      metrics-push-htpasswd = {
+        file = ../../../secrets/metrics-push-htpasswd.age;
+        owner = "nginx";
+      };
+      mimir-environment.file = ../../../secrets/mimir-environment.age;
+    };
 
     services.prometheus = {
       enable = true;
@@ -109,8 +115,9 @@ in
     services.nginx.virtualHosts."mimir.forkos.org" = {
       enableACME = true;
       forceSSL = true;
-      locations."/" = {
+      locations."/api/v1/push" = {
         proxyPass = "http://localhost:${toString mimirPort}";
+        basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
       };
     };
   };
diff --git a/services/monitoring/promtail.nix b/services/monitoring/promtail.nix
index fe30173..c5e9c71 100644
--- a/services/monitoring/promtail.nix
+++ b/services/monitoring/promtail.nix
@@ -12,7 +12,7 @@ in
 
   config = mkIf cfg.enable {
     age.secrets.promtail-password = {
-      file = ../../secrets/promtail-password.age;
+      file = ../../secrets/metrics-push-password.age;
       owner = "promtail";
     };
 
-- 
2.44.1


From ba0d50624d2d1394422f113f9829597c4d5a94ae Mon Sep 17 00:00:00 2001
From: K900 <me@0upti.me>
Date: Sun, 7 Jul 2024 18:23:28 +0300
Subject: [PATCH 4/6] Switch to push metrics with Grafana Agent

---
 hosts/bagel-box/default.nix                   |   1 -
 hosts/cl.forkos.org/default.nix               |   1 -
 hosts/fodwatch.forkos.org/default.nix         |   2 -
 hosts/meta01.nixpkgs.lahfa.xyz/default.nix    |   1 -
 services/monitoring/agent.nix                 | 100 ++++++++++++++++++
 services/monitoring/default.nix               |   2 +-
 services/monitoring/exporters/cadvisor.nix    |   2 +-
 services/monitoring/exporters/default.nix     |  30 ------
 services/monitoring/exporters/nginx.nix       |   2 +-
 services/monitoring/exporters/node.nix        |  25 -----
 services/monitoring/exporters/postgres.nix    |   2 +-
 services/monitoring/lgtm/default.nix          |   2 +-
 .../lgtm/{prometheus.nix => mimir.nix}        |  32 ------
 services/monitoring/promtail.nix              |  53 ----------
 14 files changed, 105 insertions(+), 150 deletions(-)
 create mode 100644 services/monitoring/agent.nix
 delete mode 100644 services/monitoring/exporters/node.nix
 rename services/monitoring/lgtm/{prometheus.nix => mimir.nix} (73%)
 delete mode 100644 services/monitoring/promtail.nix

diff --git a/hosts/bagel-box/default.nix b/hosts/bagel-box/default.nix
index c7c52b2..8c45c61 100644
--- a/hosts/bagel-box/default.nix
+++ b/hosts/bagel-box/default.nix
@@ -40,7 +40,6 @@
     hydra.enable = true;
     hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
   };
-  bagel.meta.monitoring.address = "bagel-box.infra.forkos.org";
 
   security.acme.acceptTerms = true;
   security.acme.defaults.email = "infra@forkos.org";
diff --git a/hosts/cl.forkos.org/default.nix b/hosts/cl.forkos.org/default.nix
index a483271..a0f9f85 100755
--- a/hosts/cl.forkos.org/default.nix
+++ b/hosts/cl.forkos.org/default.nix
@@ -24,7 +24,6 @@
       };
     };
   };
-  bagel.meta.monitoring.address = "gerrit01.infra.forkos.org";
 
   fileSystems."/gerrit-data" = {
     device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";
diff --git a/hosts/fodwatch.forkos.org/default.nix b/hosts/fodwatch.forkos.org/default.nix
index 9eede50..93f4864 100755
--- a/hosts/fodwatch.forkos.org/default.nix
+++ b/hosts/fodwatch.forkos.org/default.nix
@@ -24,8 +24,6 @@
     };
   };
 
-  bagel.meta.monitoring.address = "fodwatch.infra.forkos.org";
-
   i18n.defaultLocale = "en_US.UTF-8";
 
   system.stateVersion = "24.05";
diff --git a/hosts/meta01.nixpkgs.lahfa.xyz/default.nix b/hosts/meta01.nixpkgs.lahfa.xyz/default.nix
index 6d38095..1654ebb 100755
--- a/hosts/meta01.nixpkgs.lahfa.xyz/default.nix
+++ b/hosts/meta01.nixpkgs.lahfa.xyz/default.nix
@@ -21,7 +21,6 @@
     enable = true;
     domain = "netbox.forkos.org";
   };
-  bagel.meta.monitoring.address = "meta01.infra.forkos.org";
   bagel.services.prometheus.enable = true;
   bagel.services.loki.enable = true;
   bagel.services.grafana.enable = true;
diff --git a/services/monitoring/agent.nix b/services/monitoring/agent.nix
new file mode 100644
index 0000000..dd1e95f
--- /dev/null
+++ b/services/monitoring/agent.nix
@@ -0,0 +1,100 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.bagel.monitoring.grafana-agent;
+  inherit (lib) mkEnableOption mkOption mkIf types;
+  passwordAsCredential = "\${CREDENTIALS_DIRECTORY}/password";
+in
+{
+  options.bagel.monitoring.grafana-agent = {
+    enable = (mkEnableOption "Grafana Agent") // { default = true; };
+
+    exporters = mkOption {
+      description = "List of all exporters to scrape";
+      type = types.listOf (types.submodule {
+        options.port = mkOption {
+          description = "Exporter port";
+          type = types.int;
+        };
+      });
+      default = [];
+    };
+  };
+
+  config = mkIf cfg.enable {
+    age.secrets.grafana-agent-password.file = ../../secrets/metrics-push-password.age;
+
+    services.grafana-agent = {
+      enable = true;
+      credentials.password = config.age.secrets.grafana-agent-password.path;
+      settings = {
+        metrics = {
+          global.remote_write = [
+            {
+              url = "https://mimir.forkos.org/api/v1/push";
+              basic_auth = {
+                username = "promtail";
+                password_file = passwordAsCredential;
+              };
+            }
+          ];
+          configs = [
+            {
+              name = config.networking.hostName;
+              scrape_configs = [
+                {
+                  job_name = config.networking.hostName;
+                  static_configs = [
+                    { targets = map (e: "localhost:" + (toString e.port)) config.bagel.monitoring.grafana-agent.exporters; }
+                  ];
+                }
+              ];
+            }
+          ];
+        };
+        logs = {
+          global.clients = [
+            {
+              url = "https://loki.forkos.org/loki/api/v1/push";
+              basic_auth = {
+                username = "promtail";
+                password_file = passwordAsCredential;
+              };
+            }
+          ];
+          configs = [
+            {
+              name = "journald";
+              scrape_configs = [
+                {
+                  job_name = "system";
+                  journal = {
+                    max_age = "12h";
+                    labels = {
+                      job = "systemd-journal";
+                      host = config.networking.hostName;
+                    };
+                  };
+                  relabel_configs = [
+                    {
+                      source_labels = [ "__journal__systemd_unit" ];
+                      target_label = "unit";
+                    }
+                  ];
+                }
+              ];
+            }
+          ];
+          positions_directory = "\${STATE_DIRECTORY}/positions";
+        };
+        integrations.node_exporter.enable_collectors = [
+          "processes"
+          "systemd"
+        ];
+      };
+    };
+  };
+}
diff --git a/services/monitoring/default.nix b/services/monitoring/default.nix
index 6c513f2..1431b36 100644
--- a/services/monitoring/default.nix
+++ b/services/monitoring/default.nix
@@ -2,6 +2,6 @@
   imports = [
     ./exporters
     ./lgtm
-    ./promtail.nix
+    ./agent.nix
   ];
 }
\ No newline at end of file
diff --git a/services/monitoring/exporters/cadvisor.nix b/services/monitoring/exporters/cadvisor.nix
index 34106f1..285eaae 100644
--- a/services/monitoring/exporters/cadvisor.nix
+++ b/services/monitoring/exporters/cadvisor.nix
@@ -17,6 +17,6 @@ in
       listenAddress = "0.0.0.0";
     };
 
-    bagel.meta.monitoring.exporters = [ { port = 9102; } ];
+    bagel.monitoring.grafana-agent.exporters = [ { port = 9102; } ];
   };
 }
diff --git a/services/monitoring/exporters/default.nix b/services/monitoring/exporters/default.nix
index 45d0ca7..457702a 100644
--- a/services/monitoring/exporters/default.nix
+++ b/services/monitoring/exporters/default.nix
@@ -1,37 +1,7 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  inherit (lib) mkOption types;
-in
 {
   imports = [
     ./cadvisor.nix
-    ./node.nix
     ./nginx.nix
     ./postgres.nix
   ];
-
-  options.bagel = {
-    meta.monitoring = {
-      address = mkOption {
-        description = "Node's public address";
-        type = types.str;
-      };
-      exporters = mkOption {
-        description = "List of all exporters to scrape";
-        type = types.listOf (types.submodule {
-          options.port = mkOption {
-            description = "Exporter port";
-            type = types.int;
-          };
-        });
-        default = [];
-      };
-    };
-  };
-
-  config.networking.firewall.allowedTCPPorts = map (e: e.port) config.bagel.meta.monitoring.exporters;
 }
\ No newline at end of file
diff --git a/services/monitoring/exporters/nginx.nix b/services/monitoring/exporters/nginx.nix
index d197868..1f190c7 100644
--- a/services/monitoring/exporters/nginx.nix
+++ b/services/monitoring/exporters/nginx.nix
@@ -30,7 +30,7 @@ in
       ];
     };
 
-    bagel.meta.monitoring.exporters = [
+    bagel.monitoring.grafana-agent.exporters = [
       { port = 9103; }
     ];
   };
diff --git a/services/monitoring/exporters/node.nix b/services/monitoring/exporters/node.nix
deleted file mode 100644
index 2ced93e..0000000
--- a/services/monitoring/exporters/node.nix
+++ /dev/null
@@ -1,25 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  cfg = config.bagel.monitoring.exporters.node;
-  inherit (lib) mkEnableOption mkIf;
-in
-{
-  options.bagel.monitoring.exporters.node.enable = (mkEnableOption "Standard node_exporter") // { default = true; };
-
-  config = mkIf cfg.enable {
-    services.prometheus.exporters.node = {
-      enable = true;
-      enabledCollectors = [
-        "processes"
-        "systemd"
-      ];
-      port = 9101;
-    };
-
-    bagel.meta.monitoring.exporters = [ { port = 9101; } ];
-  };
-}
diff --git a/services/monitoring/exporters/postgres.nix b/services/monitoring/exporters/postgres.nix
index 29b439f..8ce3f45 100644
--- a/services/monitoring/exporters/postgres.nix
+++ b/services/monitoring/exporters/postgres.nix
@@ -24,7 +24,7 @@ in
 
     services.postgresql.settings.shared_preload_libraries = "pg_stat_statements";
 
-    bagel.meta.monitoring.exporters = [
+    bagel.monitoring.grafana-agent.exporters = [
       { port = 9104; }
     ];
   };
diff --git a/services/monitoring/lgtm/default.nix b/services/monitoring/lgtm/default.nix
index 264118a..700428e 100644
--- a/services/monitoring/lgtm/default.nix
+++ b/services/monitoring/lgtm/default.nix
@@ -2,6 +2,6 @@
   imports = [
     ./grafana.nix
     ./loki.nix
-    ./prometheus.nix
+    ./mimir.nix
   ];
 }
\ No newline at end of file
diff --git a/services/monitoring/lgtm/prometheus.nix b/services/monitoring/lgtm/mimir.nix
similarity index 73%
rename from services/monitoring/lgtm/prometheus.nix
rename to services/monitoring/lgtm/mimir.nix
index 54667ce..de5f7e1 100644
--- a/services/monitoring/lgtm/prometheus.nix
+++ b/services/monitoring/lgtm/mimir.nix
@@ -1,7 +1,6 @@
 {
   config,
   lib,
-  nodes,
   pkgs,
   ...
 }:
@@ -9,25 +8,6 @@ let
   cfg = config.bagel.services.prometheus;
   inherit (lib) mkEnableOption mkIf;
 
-  forEachMachine = fn: map fn (builtins.attrValues nodes);
-
-  allMetas = forEachMachine (machine: {
-    name = machine.config.networking.hostName;
-    address = machine.config.bagel.meta.monitoring.address or null;
-    exporters = machine.config.bagel.meta.monitoring.exporters or [];
-  });
-
-  scrapableMetas = builtins.filter (m: m.address != null && m.exporters != []) allMetas;
-
-  toJobConfig = m: {
-    job_name = m.name;
-    static_configs = [
-      { targets = map (e: m.address + ":" + (toString e.port)) m.exporters; }
-    ];
-  };
-
-  jobConfigs = map toJobConfig scrapableMetas;
-
   mimirPort = config.services.mimir.configuration.server.http_listen_port;
 in
 {
@@ -42,18 +22,6 @@ in
       mimir-environment.file = ../../../secrets/mimir-environment.age;
     };
 
-    services.prometheus = {
-      enable = true;
-      enableAgentMode = true;
-      listenAddress = "127.0.0.1";
-      port = 9001;
-      globalConfig.scrape_interval = "15s";
-      scrapeConfigs = jobConfigs;
-      remoteWrite = [
-        { url = "http://localhost:${toString mimirPort}/api/v1/push"; }
-      ];
-    };
-
     services.mimir = {
       enable = true;
       extraFlags = ["--config.expand-env=true"];
diff --git a/services/monitoring/promtail.nix b/services/monitoring/promtail.nix
deleted file mode 100644
index c5e9c71..0000000
--- a/services/monitoring/promtail.nix
+++ /dev/null
@@ -1,53 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  cfg = config.bagel.monitoring.promtail;
-  inherit (lib) mkEnableOption mkIf;
-in
-{
-  options.bagel.monitoring.promtail.enable = (mkEnableOption "Promtail log export") // { default = true; };
-
-  config = mkIf cfg.enable {
-    age.secrets.promtail-password = {
-      file = ../../secrets/metrics-push-password.age;
-      owner = "promtail";
-    };
-
-    services.promtail = {
-      enable = true;
-      configuration = {
-        server.disable = true;
-        clients = [ 
-          {
-            url = "https://loki.forkos.org/loki/api/v1/push";
-            basic_auth = {
-              username = "promtail";
-              password_file = config.age.secrets.promtail-password.path;
-            };
-          }
-        ];
-        scrape_configs = [
-          {
-            job_name = "system";
-            journal = {
-              max_age = "12h";
-              labels = {
-                job = "systemd-journal";
-                host = config.networking.hostName;
-              };
-            };
-            relabel_configs = [
-              {
-                source_labels = [ "__journal__systemd_unit" ];
-                target_label = "unit";
-              }
-            ];
-          }
-        ];
-      };
-    };
-  };
-}
-- 
2.44.1


From db8c831c2f7ae9d9dcb70e255129e21d4b9df9d7 Mon Sep 17 00:00:00 2001
From: emilylange <git@emilylange.de>
Date: Mon, 8 Jul 2024 00:30:17 +0200
Subject: [PATCH 5/6] grafana-agent: set `hostname` label on all metrics

This is handy to quickly see all metrics exported by a node, without
having to mangle with the already existing `instance` label.

`hostname` is essentially a variant of `instance` but without ports.
---
 services/monitoring/agent.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/services/monitoring/agent.nix b/services/monitoring/agent.nix
index dd1e95f..3fc2e0d 100644
--- a/services/monitoring/agent.nix
+++ b/services/monitoring/agent.nix
@@ -41,6 +41,7 @@ in
               };
             }
           ];
+          global.external_labels.hostname = config.networking.hostName;
           configs = [
             {
               name = config.networking.hostName;
-- 
2.44.1


From 8d2a367e92bdba87ac61887d5b5e085ffc7b696b Mon Sep 17 00:00:00 2001
From: emilylange <git@emilylange.de>
Date: Mon, 8 Jul 2024 00:34:30 +0200
Subject: [PATCH 6/6] grafana-agent: make
 `bagel.monitoring.grafana-agent.exporters` an attrset

This allows us to use multiple jobs, one for each additional exporter,
and set their `job_name` accordingly.

`job_name` is exported as `job` label on the resulting metrics.
This allows us to quickly get an understanding what metrics of an
exporter are actually available by simply filtering all metrics by
`{job="$jobname"}`
---
 services/monitoring/agent.nix              | 26 +++++++++++++---------
 services/monitoring/exporters/cadvisor.nix |  2 +-
 services/monitoring/exporters/nginx.nix    |  4 +---
 services/monitoring/exporters/postgres.nix |  4 +---
 4 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/services/monitoring/agent.nix b/services/monitoring/agent.nix
index 3fc2e0d..17274ba 100644
--- a/services/monitoring/agent.nix
+++ b/services/monitoring/agent.nix
@@ -13,14 +13,20 @@ in
     enable = (mkEnableOption "Grafana Agent") // { default = true; };
 
     exporters = mkOption {
-      description = "List of all exporters to scrape";
-      type = types.listOf (types.submodule {
+      description = ''
+        Set of additional exporters to scrape.
+
+        The attribute name will be used as `job_name`
+        internally, which ends up exported as `job` label
+        on all metrics of that exporter.
+      '';
+      type = types.attrsOf (types.submodule {
         options.port = mkOption {
           description = "Exporter port";
           type = types.int;
         };
       });
-      default = [];
+      default = {};
     };
   };
 
@@ -45,14 +51,12 @@ in
           configs = [
             {
               name = config.networking.hostName;
-              scrape_configs = [
-                {
-                  job_name = config.networking.hostName;
-                  static_configs = [
-                    { targets = map (e: "localhost:" + (toString e.port)) config.bagel.monitoring.grafana-agent.exporters; }
-                  ];
-                }
-              ];
+              scrape_configs = lib.mapAttrsToList (name: value: {
+                job_name = name;
+                static_configs = [
+                  { targets = "localhost:" + (toString value.port); }
+                ];
+              }) config.bagel.monitoring.grafana-agent.exporters;
             }
           ];
         };
diff --git a/services/monitoring/exporters/cadvisor.nix b/services/monitoring/exporters/cadvisor.nix
index 285eaae..6f2bfba 100644
--- a/services/monitoring/exporters/cadvisor.nix
+++ b/services/monitoring/exporters/cadvisor.nix
@@ -17,6 +17,6 @@ in
       listenAddress = "0.0.0.0";
     };
 
-    bagel.monitoring.grafana-agent.exporters = [ { port = 9102; } ];
+    bagel.monitoring.grafana-agent.exporters.cadvisor.port = 9102;
   };
 }
diff --git a/services/monitoring/exporters/nginx.nix b/services/monitoring/exporters/nginx.nix
index 1f190c7..e066c79 100644
--- a/services/monitoring/exporters/nginx.nix
+++ b/services/monitoring/exporters/nginx.nix
@@ -30,8 +30,6 @@ in
       ];
     };
 
-    bagel.monitoring.grafana-agent.exporters = [
-      { port = 9103; }
-    ];
+    bagel.monitoring.grafana-agent.exporters.nginxlog.port = 9103;
   };
 }
\ No newline at end of file
diff --git a/services/monitoring/exporters/postgres.nix b/services/monitoring/exporters/postgres.nix
index 8ce3f45..c6a9469 100644
--- a/services/monitoring/exporters/postgres.nix
+++ b/services/monitoring/exporters/postgres.nix
@@ -24,8 +24,6 @@ in
 
     services.postgresql.settings.shared_preload_libraries = "pg_stat_statements";
 
-    bagel.monitoring.grafana-agent.exporters = [
-      { port = 9104; }
-    ];
+    bagel.monitoring.grafana-agent.exporters.postgres.port = 9104;
   };
 }
\ No newline at end of file
-- 
2.44.1