infra/services/monitoring/lgtm/mimir.nix

{
  config,
  lib,
  pkgs,
  ...
}:
let
  cfg = config.bagel.services.prometheus;
  inherit (lib) mkEnableOption mkIf;

  mimirPort = config.services.mimir.configuration.server.http_listen_port;
in
{
  options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";

  config = mkIf cfg.enable {
    age.secrets = {
      metrics-push-htpasswd = {
        file = ../../../secrets/floral/metrics-push-htpasswd.age;
        owner = "nginx";
      };
      mimir-environment.file = ../../../secrets/floral/mimir-environment.age;
      mimir-webhook-url.file = ../../../secrets/floral/mimir-webhook-url.age;
    };

    services.mimir = {
      enable = true;
      extraFlags = ["--config.expand-env=true"];
      configuration = {
        target = "all,alertmanager";

        multitenancy_enabled = false;

        common.storage = {
          backend = "s3";
          s3 = {
            endpoint = "s3.delroth.net";
            bucket_name = "bagel-mimir";
            secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
            access_key_id = "\${S3_KEY_ID}";
          };
        };

        server = {
          http_listen_port = 9009;
          grpc_server_max_recv_msg_size = 104857600;
          grpc_server_max_send_msg_size = 104857600;
          grpc_server_max_concurrent_streams = 1000;
        };

        ingester.ring.replication_factor = 1;

        distributor.instance_limits.max_ingestion_rate = 0; # unlimited
        limits = {
          ingestion_rate = 1000000; # can't set to unlimited :(
          out_of_order_time_window = "12h";
          max_global_series_per_user = 0; # unlimited
        };

        blocks_storage.backend = "s3";
        ruler_storage = {
          backend = "local";
          local.directory = pkgs.runCommand "mimir-rules" {} ''
            mkdir -p $out
            ln -s ${./alerts} $out/anonymous
          '';
        };

        alertmanager = {
          sharding_ring.replication_factor = 1;
          fallback_config_file = pkgs.writers.writeYAML "alertmanager.yaml" {
            route = {
              group_by = ["alertname"];
              receiver = "matrix";
            };
            receivers = [
              {
                name = "matrix";
                webhook_configs = [{
                  # Mimir can't expand environment variables in external config files,
                  # so work around it.
                  url_file = "/run/credentials/mimir.service/webhook-url";
                }];
              }
            ];
          };
        };
        alertmanager_storage.backend = "filesystem";

        ruler.alertmanager_url = "http://localhost:${toString mimirPort}/alertmanager";
      };
    };

    systemd.services.mimir = {
      # Mimir tries to determine its own IP address for gossip purposes,
      # even when it's the only instance, and fails if it can't find one.
      # Avoid that by ensuring it starts after the network is set up.
      wants = [ "network-online.target" ];
      after = ["network-online.target"];
      serviceConfig = {
        EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
        LoadCredential = [ "webhook-url:${config.age.secrets.mimir-webhook-url.path}" ];
      };
    };

    services.nginx = {
      upstreams.mimir = {
        servers."127.0.0.1:${toString mimirPort}" = {};
        extraConfig = "keepalive 16;";
      };

      virtualHosts."mimir.forkos.org" = {
        enableACME = true;
        forceSSL = true;
        locations."/api/v1/push" = {
          proxyPass = "http://mimir";
          basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
        };
      };
    };

    bagel.monitoring.grafana-agent.exporters.mimir.port = 9009;
    bagel.services.alertmanager-hookshot-adapter.enable = true;
  };
}