diff --git a/services/monitoring/lgtm/alerts/demo.yaml b/services/monitoring/lgtm/alerts/demo.yaml new file mode 100644 index 0000000..db38426 --- /dev/null +++ b/services/monitoring/lgtm/alerts/demo.yaml @@ -0,0 +1,5 @@ +groups: + - name: Demo alerts + rules: + - alert: Demo alert + expr: 1 diff --git a/services/monitoring/lgtm/prometheus.nix b/services/monitoring/lgtm/prometheus.nix index 2f83b8f..ca07352 100644 --- a/services/monitoring/lgtm/prometheus.nix +++ b/services/monitoring/lgtm/prometheus.nix @@ -2,6 +2,7 @@ config, lib, nodes, + pkgs, ... }: let @@ -26,6 +27,8 @@ let }; jobConfigs = map toJobConfig scrapableMetas; + + mimirPort = config.services.mimir.configuration.server.http_listen_port; in { options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper"; @@ -41,7 +44,7 @@ in globalConfig.scrape_interval = "15s"; scrapeConfigs = jobConfigs; remoteWrite = [ - { url = "http://localhost:9009/api/v1/push"; } + { url = "http://localhost:${toString mimirPort}/api/v1/push"; } ]; }; @@ -49,6 +52,8 @@ in enable = true; extraFlags = ["--config.expand-env=true"]; configuration = { + target = "all,alertmanager"; + multitenancy_enabled = false; common.storage = { @@ -73,11 +78,40 @@ in blocks_storage.backend = "s3"; ruler_storage = { backend = "local"; - local.directory = ./alerts; + local.directory = pkgs.runCommand "mimir-rules" {} '' + mkdir -p $out + ln -s ${./alerts} $out/anonymous + ''; }; + + alertmanager = { + sharding_ring.replication_factor = 1; + fallback_config_file = pkgs.writers.writeYAML "alertmanager.yaml" { + route = { + group_by = ["alertname"]; + receiver = "matrix"; + }; + receivers = [ + { + name = "matrix"; + } + ]; + }; + }; + alertmanager_storage.backend = "filesystem"; + + ruler.alertmanager_url = "http://localhost:${toString mimirPort}/alertmanager"; }; }; systemd.services.mimir.serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ]; + + services.nginx.virtualHosts."mimir.forkos.org" = { + enableACME = true; + forceSSL = true; + locations."/" = { + proxyPass = "http://localhost:${toString mimirPort}"; + }; + }; }; }