infra/services/monitoring/agent.nix

{
  config,
  lib,
  ...
}:
let
  cfg = config.bagel.monitoring.grafana-agent;
  inherit (lib) mkEnableOption mkOption mkIf types;
  passwordAsCredential = "\${CREDENTIALS_DIRECTORY}/password";
in
{
  options.bagel.monitoring.grafana-agent = {
    enable = (mkEnableOption "Grafana Agent") // { default = true; };

    tenant = mkOption {
      description = ''
        Which tenant are we enabling Grafana Agent for.
      '';
      example = "lix";
      type = types.enum [ "lix" "floral" ];
    };

    exporters = mkOption {
      description = ''
        Set of additional exporters to scrape.

        The attribute name will be used as `job_name`
        internally, which ends up exported as `job` label
        on all metrics of that exporter.
      '';
      type = types.attrsOf (types.submodule ({ config, name, ... }: {
        options.port = mkOption {
          description = "Exporter port";
          type = types.int;
        };
        options.bearerTokenFile = mkOption {
          description = "File containing a bearer token";
          type = types.nullOr types.path;
          default = null;
        };

        options.scrapeConfig = mkOption {
          description = "Prometheus scrape config";
          type = types.attrs;
        };
        config.scrapeConfig = lib.mkMerge [{
          job_name = name;
          static_configs = [
            { targets = [ "localhost:${toString config.port}" ]; }
          ];
        } (lib.mkIf (config.bearerTokenFile != null) {
          authorization.credentials_file = "\${CREDENTIALS_DIRECTORY}/${name}-bearer-token";
        })];

        options.secrets = mkOption {
          description = "Secrets required for scrape config";
          type = types.attrs;
          internal = true;
          default = {};
        };
        config.secrets = lib.mkIf (config.bearerTokenFile != null) {
          "${name}-bearer-token" = config.bearerTokenFile;
        };
      }));
      default = {};
    };
  };

  config = mkIf cfg.enable {
    age.secrets.grafana-agent-password.file = ../../secrets/floral/metrics-push-password.age;

    services.grafana-agent = {
      enable = true;
      credentials = lib.mkMerge ([{ password = config.age.secrets.grafana-agent-password.path; }] ++
        lib.mapAttrsToList (name: value: value.secrets) config.bagel.monitoring.grafana-agent.exporters);
      settings = {
        metrics = {
          global.remote_write = [
            {
              url = "https://mimir.forkos.org/api/v1/push";
              basic_auth = {
                username = "promtail";
                password_file = passwordAsCredential;
              };
            }
          ];
          global.external_labels = {
            hostname = config.networking.hostName;
            inherit (cfg) tenant;
          };
          configs = [
            {
              name = config.networking.hostName;
              scrape_configs = lib.mapAttrsToList (name: value: value.scrapeConfig) config.bagel.monitoring.grafana-agent.exporters;
            }
          ];
        };
        logs = {
          global.clients = [
            {
              url = "https://loki.forkos.org/loki/api/v1/push";
              basic_auth = {
                username = "promtail";
                password_file = passwordAsCredential;
              };
            }
          ];
          configs = [
            {
              name = "journald";
              scrape_configs = [
                {
                  job_name = "system";
                  journal = {
                    max_age = "12h";
                    labels = {
                      job = "systemd-journal";
                      host = config.networking.hostName;
                    };
                  };
                  relabel_configs = [
                    {
                      source_labels = [ "__journal__systemd_unit" ];
                      target_label = "unit";
                    }
                  ];
                }
              ];
            }
          ];
          positions_directory = "\${STATE_DIRECTORY}/positions";
        };
        integrations.node_exporter.enable_collectors = [
          "processes"
          "systemd"
        ];
      };
    };
  };
}