From ddd0c365ae0ef06c35ca439bad7b9e021b1b4b36 Mon Sep 17 00:00:00 2001 From: K900 Date: Sun, 7 Jul 2024 18:23:28 +0300 Subject: [PATCH] Switch to push metrics with Grafana Agent --- hosts/bagel-box/default.nix | 1 - hosts/cl.forkos.org/default.nix | 1 - hosts/fodwatch.forkos.org/default.nix | 2 - hosts/meta01.nixpkgs.lahfa.xyz/default.nix | 1 - services/monitoring/agent.nix | 100 ++++++++++++++++++ services/monitoring/default.nix | 2 +- services/monitoring/exporters/cadvisor.nix | 2 +- services/monitoring/exporters/default.nix | 30 ------ services/monitoring/exporters/nginx.nix | 2 +- services/monitoring/exporters/node.nix | 25 ----- services/monitoring/exporters/postgres.nix | 2 +- services/monitoring/lgtm/default.nix | 2 +- .../lgtm/{prometheus.nix => mimir.nix} | 32 ------ services/monitoring/promtail.nix | 53 ---------- 14 files changed, 105 insertions(+), 150 deletions(-) create mode 100644 services/monitoring/agent.nix delete mode 100644 services/monitoring/exporters/node.nix rename services/monitoring/lgtm/{prometheus.nix => mimir.nix} (73%) delete mode 100644 services/monitoring/promtail.nix diff --git a/hosts/bagel-box/default.nix b/hosts/bagel-box/default.nix index f9a929a..eed24b1 100644 --- a/hosts/bagel-box/default.nix +++ b/hosts/bagel-box/default.nix @@ -40,7 +40,6 @@ hydra.enable = true; hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra"; }; - bagel.meta.monitoring.address = "bagel-box.delroth.net"; security.acme.acceptTerms = true; security.acme.defaults.email = "bagel@delroth.net"; diff --git a/hosts/cl.forkos.org/default.nix b/hosts/cl.forkos.org/default.nix index a483271..a0f9f85 100755 --- a/hosts/cl.forkos.org/default.nix +++ b/hosts/cl.forkos.org/default.nix @@ -24,7 +24,6 @@ }; }; }; - bagel.meta.monitoring.address = "gerrit01.infra.forkos.org"; fileSystems."/gerrit-data" = { device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4"; diff --git a/hosts/fodwatch.forkos.org/default.nix b/hosts/fodwatch.forkos.org/default.nix index 9eede50..93f4864 100755 --- a/hosts/fodwatch.forkos.org/default.nix +++ b/hosts/fodwatch.forkos.org/default.nix @@ -24,8 +24,6 @@ }; }; - bagel.meta.monitoring.address = "fodwatch.infra.forkos.org"; - i18n.defaultLocale = "en_US.UTF-8"; system.stateVersion = "24.05"; diff --git a/hosts/meta01.nixpkgs.lahfa.xyz/default.nix b/hosts/meta01.nixpkgs.lahfa.xyz/default.nix index 6d38095..1654ebb 100755 --- a/hosts/meta01.nixpkgs.lahfa.xyz/default.nix +++ b/hosts/meta01.nixpkgs.lahfa.xyz/default.nix @@ -21,7 +21,6 @@ enable = true; domain = "netbox.forkos.org"; }; - bagel.meta.monitoring.address = "meta01.infra.forkos.org"; bagel.services.prometheus.enable = true; bagel.services.loki.enable = true; bagel.services.grafana.enable = true; diff --git a/services/monitoring/agent.nix b/services/monitoring/agent.nix new file mode 100644 index 0000000..dd1e95f --- /dev/null +++ b/services/monitoring/agent.nix @@ -0,0 +1,100 @@ +{ + config, + lib, + ... +}: +let + cfg = config.bagel.monitoring.grafana-agent; + inherit (lib) mkEnableOption mkOption mkIf types; + passwordAsCredential = "\${CREDENTIALS_DIRECTORY}/password"; +in +{ + options.bagel.monitoring.grafana-agent = { + enable = (mkEnableOption "Grafana Agent") // { default = true; }; + + exporters = mkOption { + description = "List of all exporters to scrape"; + type = types.listOf (types.submodule { + options.port = mkOption { + description = "Exporter port"; + type = types.int; + }; + }); + default = []; + }; + }; + + config = mkIf cfg.enable { + age.secrets.grafana-agent-password.file = ../../secrets/metrics-push-password.age; + + services.grafana-agent = { + enable = true; + credentials.password = config.age.secrets.grafana-agent-password.path; + settings = { + metrics = { + global.remote_write = [ + { + url = "https://mimir.forkos.org/api/v1/push"; + basic_auth = { + username = "promtail"; + password_file = passwordAsCredential; + }; + } + ]; + configs = [ + { + name = config.networking.hostName; + scrape_configs = [ + { + job_name = config.networking.hostName; + static_configs = [ + { targets = map (e: "localhost:" + (toString e.port)) config.bagel.monitoring.grafana-agent.exporters; } + ]; + } + ]; + } + ]; + }; + logs = { + global.clients = [ + { + url = "https://loki.forkos.org/loki/api/v1/push"; + basic_auth = { + username = "promtail"; + password_file = passwordAsCredential; + }; + } + ]; + configs = [ + { + name = "journald"; + scrape_configs = [ + { + job_name = "system"; + journal = { + max_age = "12h"; + labels = { + job = "systemd-journal"; + host = config.networking.hostName; + }; + }; + relabel_configs = [ + { + source_labels = [ "__journal__systemd_unit" ]; + target_label = "unit"; + } + ]; + } + ]; + } + ]; + positions_directory = "\${STATE_DIRECTORY}/positions"; + }; + integrations.node_exporter.enable_collectors = [ + "processes" + "systemd" + ]; + }; + }; + }; +} diff --git a/services/monitoring/default.nix b/services/monitoring/default.nix index 6c513f2..1431b36 100644 --- a/services/monitoring/default.nix +++ b/services/monitoring/default.nix @@ -2,6 +2,6 @@ imports = [ ./exporters ./lgtm - ./promtail.nix + ./agent.nix ]; } \ No newline at end of file diff --git a/services/monitoring/exporters/cadvisor.nix b/services/monitoring/exporters/cadvisor.nix index 34106f1..285eaae 100644 --- a/services/monitoring/exporters/cadvisor.nix +++ b/services/monitoring/exporters/cadvisor.nix @@ -17,6 +17,6 @@ in listenAddress = "0.0.0.0"; }; - bagel.meta.monitoring.exporters = [ { port = 9102; } ]; + bagel.monitoring.grafana-agent.exporters = [ { port = 9102; } ]; }; } diff --git a/services/monitoring/exporters/default.nix b/services/monitoring/exporters/default.nix index 45d0ca7..457702a 100644 --- a/services/monitoring/exporters/default.nix +++ b/services/monitoring/exporters/default.nix @@ -1,37 +1,7 @@ -{ - config, - lib, - ... -}: -let - inherit (lib) mkOption types; -in { imports = [ ./cadvisor.nix - ./node.nix ./nginx.nix ./postgres.nix ]; - - options.bagel = { - meta.monitoring = { - address = mkOption { - description = "Node's public address"; - type = types.str; - }; - exporters = mkOption { - description = "List of all exporters to scrape"; - type = types.listOf (types.submodule { - options.port = mkOption { - description = "Exporter port"; - type = types.int; - }; - }); - default = []; - }; - }; - }; - - config.networking.firewall.allowedTCPPorts = map (e: e.port) config.bagel.meta.monitoring.exporters; } \ No newline at end of file diff --git a/services/monitoring/exporters/nginx.nix b/services/monitoring/exporters/nginx.nix index d197868..1f190c7 100644 --- a/services/monitoring/exporters/nginx.nix +++ b/services/monitoring/exporters/nginx.nix @@ -30,7 +30,7 @@ in ]; }; - bagel.meta.monitoring.exporters = [ + bagel.monitoring.grafana-agent.exporters = [ { port = 9103; } ]; }; diff --git a/services/monitoring/exporters/node.nix b/services/monitoring/exporters/node.nix deleted file mode 100644 index 2ced93e..0000000 --- a/services/monitoring/exporters/node.nix +++ /dev/null @@ -1,25 +0,0 @@ -{ - config, - lib, - ... -}: -let - cfg = config.bagel.monitoring.exporters.node; - inherit (lib) mkEnableOption mkIf; -in -{ - options.bagel.monitoring.exporters.node.enable = (mkEnableOption "Standard node_exporter") // { default = true; }; - - config = mkIf cfg.enable { - services.prometheus.exporters.node = { - enable = true; - enabledCollectors = [ - "processes" - "systemd" - ]; - port = 9101; - }; - - bagel.meta.monitoring.exporters = [ { port = 9101; } ]; - }; -} diff --git a/services/monitoring/exporters/postgres.nix b/services/monitoring/exporters/postgres.nix index 29b439f..8ce3f45 100644 --- a/services/monitoring/exporters/postgres.nix +++ b/services/monitoring/exporters/postgres.nix @@ -24,7 +24,7 @@ in services.postgresql.settings.shared_preload_libraries = "pg_stat_statements"; - bagel.meta.monitoring.exporters = [ + bagel.monitoring.grafana-agent.exporters = [ { port = 9104; } ]; }; diff --git a/services/monitoring/lgtm/default.nix b/services/monitoring/lgtm/default.nix index 264118a..700428e 100644 --- a/services/monitoring/lgtm/default.nix +++ b/services/monitoring/lgtm/default.nix @@ -2,6 +2,6 @@ imports = [ ./grafana.nix ./loki.nix - ./prometheus.nix + ./mimir.nix ]; } \ No newline at end of file diff --git a/services/monitoring/lgtm/prometheus.nix b/services/monitoring/lgtm/mimir.nix similarity index 73% rename from services/monitoring/lgtm/prometheus.nix rename to services/monitoring/lgtm/mimir.nix index 54667ce..de5f7e1 100644 --- a/services/monitoring/lgtm/prometheus.nix +++ b/services/monitoring/lgtm/mimir.nix @@ -1,7 +1,6 @@ { config, lib, - nodes, pkgs, ... }: @@ -9,25 +8,6 @@ let cfg = config.bagel.services.prometheus; inherit (lib) mkEnableOption mkIf; - forEachMachine = fn: map fn (builtins.attrValues nodes); - - allMetas = forEachMachine (machine: { - name = machine.config.networking.hostName; - address = machine.config.bagel.meta.monitoring.address or null; - exporters = machine.config.bagel.meta.monitoring.exporters or []; - }); - - scrapableMetas = builtins.filter (m: m.address != null && m.exporters != []) allMetas; - - toJobConfig = m: { - job_name = m.name; - static_configs = [ - { targets = map (e: m.address + ":" + (toString e.port)) m.exporters; } - ]; - }; - - jobConfigs = map toJobConfig scrapableMetas; - mimirPort = config.services.mimir.configuration.server.http_listen_port; in { @@ -42,18 +22,6 @@ in mimir-environment.file = ../../../secrets/mimir-environment.age; }; - services.prometheus = { - enable = true; - enableAgentMode = true; - listenAddress = "127.0.0.1"; - port = 9001; - globalConfig.scrape_interval = "15s"; - scrapeConfigs = jobConfigs; - remoteWrite = [ - { url = "http://localhost:${toString mimirPort}/api/v1/push"; } - ]; - }; - services.mimir = { enable = true; extraFlags = ["--config.expand-env=true"]; diff --git a/services/monitoring/promtail.nix b/services/monitoring/promtail.nix deleted file mode 100644 index c5e9c71..0000000 --- a/services/monitoring/promtail.nix +++ /dev/null @@ -1,53 +0,0 @@ -{ - config, - lib, - ... -}: -let - cfg = config.bagel.monitoring.promtail; - inherit (lib) mkEnableOption mkIf; -in -{ - options.bagel.monitoring.promtail.enable = (mkEnableOption "Promtail log export") // { default = true; }; - - config = mkIf cfg.enable { - age.secrets.promtail-password = { - file = ../../secrets/metrics-push-password.age; - owner = "promtail"; - }; - - services.promtail = { - enable = true; - configuration = { - server.disable = true; - clients = [ - { - url = "https://loki.forkos.org/loki/api/v1/push"; - basic_auth = { - username = "promtail"; - password_file = config.age.secrets.promtail-password.path; - }; - } - ]; - scrape_configs = [ - { - job_name = "system"; - journal = { - max_age = "12h"; - labels = { - job = "systemd-journal"; - host = config.networking.hostName; - }; - }; - relabel_configs = [ - { - source_labels = [ "__journal__systemd_unit" ]; - target_label = "unit"; - } - ]; - } - ]; - }; - }; - }; -}