the-infra/services/monitoring/agent.nix

{
  config,
  lib,
  ...
}:
let
  cfg = config.bagel.monitoring.grafana-agent;
  inherit (lib) mkEnableOption mkOption mkIf types;
  passwordAsCredential = "\${CREDENTIALS_DIRECTORY}/password";
in
{
  options.bagel.monitoring.grafana-agent = {
    enable = (mkEnableOption "Grafana Agent") // { default = true; };

    exporters = mkOption {
      description = ''
        Set of additional exporters to scrape.

        The attribute name will be used as `job_name`
        internally, which ends up exported as `job` label
        on all metrics of that exporter.
      '';
      type = types.attrsOf (types.submodule {
        options.port = mkOption {
          description = "Exporter port";
          type = types.int;
        };
      });
      default = {};
    };
  };

  config = mkIf cfg.enable {
    age.secrets.grafana-agent-password.file = ../../secrets/metrics-push-password.age;

    services.grafana-agent = {
      enable = true;
      credentials.password = config.age.secrets.grafana-agent-password.path;
      settings = {
        metrics = {
          global.remote_write = [
            {
              url = "https://mimir.forkos.org/api/v1/push";
              basic_auth = {
                username = "promtail";
                password_file = passwordAsCredential;
              };
            }
          ];
          global.external_labels.hostname = config.networking.hostName;
          configs = [
            {
              name = config.networking.hostName;
              scrape_configs = lib.mapAttrsToList (name: value: {
                job_name = name;
                static_configs = [
                  { targets = [ "localhost:${toString value.port}" ]; }
                ];
              }) config.bagel.monitoring.grafana-agent.exporters;
            }
          ];
        };
        logs = {
          global.clients = [
            {
              url = "https://loki.forkos.org/loki/api/v1/push";
              basic_auth = {
                username = "promtail";
                password_file = passwordAsCredential;
              };
            }
          ];
          configs = [
            {
              name = "journald";
              scrape_configs = [
                {
                  job_name = "system";
                  journal = {
                    max_age = "12h";
                    labels = {
                      job = "systemd-journal";
                      host = config.networking.hostName;
                    };
                  };
                  relabel_configs = [
                    {
                      source_labels = [ "__journal__systemd_unit" ];
                      target_label = "unit";
                    }
                  ];
                }
              ];
            }
          ];
          positions_directory = "\${STATE_DIRECTORY}/positions";
        };
        integrations.node_exporter.enable_collectors = [
          "processes"
          "systemd"
        ];
      };
    };
  };
}