services/monitoring/lgtm/alerts/demo.yaml

@@ -0,0 +1,5 @@
+groups:
+  - name: Demo alerts
+    rules:
+      - alert: Demo alert
+        expr: 1

services/monitoring/lgtm/prometheus.nix

@@ -2,6 +2,7 @@
   config,
   lib,
   nodes,
+  pkgs,
   ...
 }:
 let
@@ -26,6 +27,8 @@ let
   };
 
   jobConfigs = map toJobConfig scrapableMetas;
+
+  mimirPort = config.services.mimir.configuration.server.http_listen_port;
 in
 {
   options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
@@ -41,7 +44,7 @@ in
       globalConfig.scrape_interval = "15s";
       scrapeConfigs = jobConfigs;
       remoteWrite = [
-        { url = "http://localhost:9009/api/v1/push"; }
+        { url = "http://localhost:${toString mimirPort}/api/v1/push"; }
       ];
     };
 
@@ -49,6 +52,8 @@ in
       enable = true;
       extraFlags = ["--config.expand-env=true"];
       configuration = {
+        target = "all,alertmanager";
+
         multitenancy_enabled = false;
 
         common.storage = {
@@ -73,11 +78,40 @@ in
         blocks_storage.backend = "s3";
         ruler_storage = {
           backend = "local";
-          local.directory = ./alerts;
+          local.directory = pkgs.runCommand "mimir-rules" {} ''
+            mkdir -p $out
+            ln -s ${./alerts} $out/anonymous
+          '';
         };
+
+        alertmanager = {
+          sharding_ring.replication_factor = 1;
+          fallback_config_file = pkgs.writers.writeYAML "alertmanager.yaml" {
+            route = {
+              group_by = ["alertname"];
+              receiver = "matrix";
+            };
+            receivers = [
+              {
+                name = "matrix";
+              }
+            ];
+          };
+        };
+        alertmanager_storage.backend = "filesystem";
+
+        ruler.alertmanager_url = "http://localhost:${toString mimirPort}/alertmanager";
       };
     };
 
     systemd.services.mimir.serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
+
+    services.nginx.virtualHosts."mimir.forkos.org" = {
+      enableACME = true;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://localhost:${toString mimirPort}";
+      };
+    };
   };
 }