From c3b4cc03dc7bae7e79bed2393913b7bbe15c5313 Mon Sep 17 00:00:00 2001 From: K900 Date: Fri, 5 Jul 2024 17:20:22 +0300 Subject: [PATCH] Add Loki + Promtail setup --- hosts/meta01.nixpkgs.lahfa.xyz/default.nix | 1 + secrets.nix | 5 ++ secrets/loki-environment.age | 9 ++ secrets/loki-htpasswd.age | 7 ++ secrets/promtail-password.age | 12 +++ services/monitoring/default.nix | 1 + services/monitoring/lgtm/default.nix | 1 + services/monitoring/lgtm/grafana.nix | 7 ++ services/monitoring/lgtm/loki.nix | 100 +++++++++++++++++++++ services/monitoring/promtail.nix | 53 +++++++++++ 10 files changed, 196 insertions(+) create mode 100644 secrets/loki-environment.age create mode 100644 secrets/loki-htpasswd.age create mode 100644 secrets/promtail-password.age create mode 100644 services/monitoring/lgtm/loki.nix create mode 100644 services/monitoring/promtail.nix diff --git a/hosts/meta01.nixpkgs.lahfa.xyz/default.nix b/hosts/meta01.nixpkgs.lahfa.xyz/default.nix index 5800aac..6d38095 100755 --- a/hosts/meta01.nixpkgs.lahfa.xyz/default.nix +++ b/hosts/meta01.nixpkgs.lahfa.xyz/default.nix @@ -23,6 +23,7 @@ }; bagel.meta.monitoring.address = "meta01.infra.forkos.org"; bagel.services.prometheus.enable = true; + bagel.services.loki.enable = true; bagel.services.grafana.enable = true; i18n.defaultLocale = "fr_FR.UTF-8"; diff --git a/secrets.nix b/secrets.nix index c3b1ab3..9e6f070 100644 --- a/secrets.nix +++ b/secrets.nix @@ -9,6 +9,11 @@ let netbox-environment = [ machines.meta01 ]; mimir-environment = [ machines.meta01 ]; grafana-oauth-secret = [ machines.meta01 ]; + loki-environment = [ machines.meta01 ]; + + # These are the same password, but nginx wants it in htpasswd format + loki-htpasswd = [ machines.meta01 ]; + promtail-password = builtins.attrValues machines; }; in builtins.listToAttrs ( diff --git a/secrets/loki-environment.age b/secrets/loki-environment.age new file mode 100644 index 0000000..af68245 --- /dev/null +++ b/secrets/loki-environment.age @@ -0,0 +1,9 @@ +age-encryption.org/v1 +-> ssh-ed25519 j2r2qQ w0lLquFUUcmEZ/Fh1YSt85tAJkBwavORQbwMr7gMqF4 +J4T+EHm1uHbCZkAUNoNcB9uGSz082mFL8+dkCnvYQnM +-> ssh-ed25519 K3b7BA 28bJZgBPPc2KIE5+b8LJuQ5L4YAiRAJzucEuOqXHdVM +7hKENFr8QX0jpwuuQEjGFrUywJuhL1Tdi2V4/gR8JWE +--- GSPZxz39TMMWv0qhotNgnXa5679Q7VK8JGjQjI7A8oM +J\@FN 2w!1VfOCwV̺.^݆7wn4dW-־"@0EϿck,]M}x̝y[J:! !螀c +BR +n^9M< \ No newline at end of file diff --git a/secrets/loki-htpasswd.age b/secrets/loki-htpasswd.age new file mode 100644 index 0000000..8958a2e --- /dev/null +++ b/secrets/loki-htpasswd.age @@ -0,0 +1,7 @@ +age-encryption.org/v1 +-> ssh-ed25519 j2r2qQ nLWy3WcVJWCl3rXkhcSbp1joqmkk06QnxhCZ4UtSvmw +iQ+Hx/vhiFgkWfbxHwGjxMBEqzyGww4/9do3W7V/y1Y +-> ssh-ed25519 K3b7BA RkF2ADcjOGtivl9MrhO/HFwxlTAkbFHWL3iinUldMiM +7q/zdVTMLevukZjkHtcN88iYzfTLvq2s3QdkgsFSO9M +--- 1b2HiK06vJPqBgHVDD0QELOtfkl7/rlgGS9uI1mSbus +uܧoL;" 4ۻZ@В3+93Q4 ow6M-DkJn;*g OY75S)ٰ \ No newline at end of file diff --git a/secrets/promtail-password.age b/secrets/promtail-password.age new file mode 100644 index 0000000..00835a9 --- /dev/null +++ b/secrets/promtail-password.age @@ -0,0 +1,12 @@ +age-encryption.org/v1 +-> ssh-ed25519 +HUDfA ZUM0ACC/NIekvX1PkCiXTHaTeE3ybudmY3piHw2iekQ +cHj94FIR6gNJ3Hw9FI7K15OYgxbjkajGtCftD+2Mr8c +-> ssh-ed25519 2D+APA tzlyOnAXnLxXO/47n45sFPiJF3FXd98UU5ajPhD2wSs +P8ZdUiBeME17SU2BpMgOq4plyAqgzLOQWHa1+Q7cjYo +-> ssh-ed25519 j2r2qQ 3OikD9JOmug7kdPAPz+JT/ryB6xBQhu2+cwS9h5sKGI +XiIuxOyey2I6hmqabUCPzLc85q/1r9OwVGjHWYNQsp0 +-> ssh-ed25519 K3b7BA Bdqcqt4GgLzuSiEnIyImDiOQGwyIhhozRXMmNrp7glI +65joZcnl0Hqe90Th2EdVgbcxUJFpy3fOgk6oPiSHh2A +--- 6x6BFNypc+u3DpsHX3SajwEy1TqsAtbFei0ddRpEoBg + +UGxj4b=Rd3sHY ԏ*Q96n34&w~h! ^[ \ No newline at end of file diff --git a/services/monitoring/default.nix b/services/monitoring/default.nix index 77289f2..6c513f2 100644 --- a/services/monitoring/default.nix +++ b/services/monitoring/default.nix @@ -2,5 +2,6 @@ imports = [ ./exporters ./lgtm + ./promtail.nix ]; } \ No newline at end of file diff --git a/services/monitoring/lgtm/default.nix b/services/monitoring/lgtm/default.nix index eccb275..264118a 100644 --- a/services/monitoring/lgtm/default.nix +++ b/services/monitoring/lgtm/default.nix @@ -1,6 +1,7 @@ { imports = [ ./grafana.nix + ./loki.nix ./prometheus.nix ]; } \ No newline at end of file diff --git a/services/monitoring/lgtm/grafana.nix b/services/monitoring/lgtm/grafana.nix index 64ed3d1..81ab6c7 100644 --- a/services/monitoring/lgtm/grafana.nix +++ b/services/monitoring/lgtm/grafana.nix @@ -93,6 +93,13 @@ in access = "proxy"; url = "http://127.0.0.1:9009/prometheus"; } + { + name = "Loki"; + type = "loki"; + uid = "loki"; + access = "proxy"; + url = "http://127.0.0.1:9090/"; + } ]; }; }; diff --git a/services/monitoring/lgtm/loki.nix b/services/monitoring/lgtm/loki.nix new file mode 100644 index 0000000..d862046 --- /dev/null +++ b/services/monitoring/lgtm/loki.nix @@ -0,0 +1,100 @@ +{ + config, + lib, + ... +}: +let + cfg = config.bagel.services.loki; + inherit (lib) mkEnableOption mkIf; +in +{ + options.bagel.services.loki.enable = mkEnableOption "Loki storage"; + + config = mkIf cfg.enable { + age.secrets = { + loki-htpasswd = { + file = ../../../secrets/loki-htpasswd.age; + owner = "nginx"; + }; + loki-environment.file = ../../../secrets/loki-environment.age; + }; + + services.loki = { + enable = true; + extraFlags = ["--config.expand-env"]; + + configuration = { + server = { + http_listen_port = 9090; + grpc_listen_port = 9096; + + # 16M + grpc_server_max_recv_msg_size = 16777216; + grpc_server_max_send_msg_size = 16777216; + }; + + auth_enabled = false; + + common = { + storage.s3 = { + endpoint = "s3.delroth.net"; + region = "garage"; + bucketnames = "bagel-loki"; + secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable + access_key_id = "\${S3_KEY_ID}"; + s3forcepathstyle = true; + }; + ring.kvstore.store = "memberlist"; + replication_factor = 1; + }; + + memberlist = { + bind_port = 7947; + advertise_port = 7947; + }; + + storage_config.tsdb_shipper = { + active_index_directory = "/var/lib/loki/index"; + cache_location = "/var/lib/loki/cache"; + }; + + compactor = { + working_directory = "/var/lib/loki/compactor"; + compaction_interval = "10m"; + retention_enabled = true; + retention_delete_delay = "1s"; + retention_delete_worker_count = 150; + delete_request_store = "filesystem"; + }; + + limits_config.retention_period = "1w"; + + schema_config = { + configs = [ + { + from = "2024-07-01"; + store = "tsdb"; + object_store = "s3"; + schema = "v13"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + }; + }; + }; + + systemd.services.loki.serviceConfig.EnvironmentFile = [ config.age.secrets.loki-environment.path ]; + + services.nginx.virtualHosts."loki.forkos.org" = { + enableACME = true; + forceSSL = true; + locations."/loki/api/v1/push" = { + proxyPass = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}"; + basicAuthFile = config.age.secrets.loki-htpasswd.path; + }; + }; + }; +} diff --git a/services/monitoring/promtail.nix b/services/monitoring/promtail.nix new file mode 100644 index 0000000..fe30173 --- /dev/null +++ b/services/monitoring/promtail.nix @@ -0,0 +1,53 @@ +{ + config, + lib, + ... +}: +let + cfg = config.bagel.monitoring.promtail; + inherit (lib) mkEnableOption mkIf; +in +{ + options.bagel.monitoring.promtail.enable = (mkEnableOption "Promtail log export") // { default = true; }; + + config = mkIf cfg.enable { + age.secrets.promtail-password = { + file = ../../secrets/promtail-password.age; + owner = "promtail"; + }; + + services.promtail = { + enable = true; + configuration = { + server.disable = true; + clients = [ + { + url = "https://loki.forkos.org/loki/api/v1/push"; + basic_auth = { + username = "promtail"; + password_file = config.age.secrets.promtail-password.path; + }; + } + ]; + scrape_configs = [ + { + job_name = "system"; + journal = { + max_age = "12h"; + labels = { + job = "systemd-journal"; + host = config.networking.hostName; + }; + }; + relabel_configs = [ + { + source_labels = [ "__journal__systemd_unit" ]; + target_label = "unit"; + } + ]; + } + ]; + }; + }; + }; +}