Add Loki + Promtail setup

hosts/meta01.nixpkgs.lahfa.xyz/default.nix

@@ -23,6 +23,7 @@
   };
   bagel.meta.monitoring.address = "meta01.infra.forkos.org";
   bagel.services.prometheus.enable = true;
+  bagel.services.loki.enable = true;
   bagel.services.grafana.enable = true;
 
   i18n.defaultLocale = "fr_FR.UTF-8";

secrets.nix

@@ -9,6 +9,11 @@ let
     netbox-environment = [ machines.meta01 ];
     mimir-environment = [ machines.meta01 ];
     grafana-oauth-secret = [ machines.meta01 ];
+    loki-environment = [ machines.meta01 ];
+
+    # These are the same password, but nginx wants it in htpasswd format
+    loki-htpasswd = [ machines.meta01 ];
+    promtail-password = builtins.attrValues machines;
   };
 in
   builtins.listToAttrs (

secrets/loki-environment.age

@@ -0,0 +1,9 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ w0lLquFUUcmEZ/Fh1YSt85tAJkBwavORQbwMr7gMqF4
+J4T+EHm1uHbCZkAUNoNcB9uGSz082mFL8+dkCnvYQnM
+-> ssh-ed25519 K3b7BA 28bJZgBPPc2KIE5+b8LJuQ5L4YAiRAJzucEuOqXHdVM
+7hKENFr8QX0jpwuuQEjGFrUywJuhL1Tdi2V4/gR8JWE
+--- GSPZxz39TMMWv0qhotNgnXa5679Q7VK8JGjQjI7A8oM
+J²\@F“N•	³å…2‹®ô¨w×!¯1V
f»§˜Ž·’ÞO²CÓw®®V°£šÌº.^݆ 7<C2A0>w‡n4äàdW-Ö¾"@0¨ú¹EÏ¿·°ck,]M}x<>øÌ<C3B8>¡Ûy°[×ÁJ:!è‘	!ø螀c¬
+Bë¹R
+nøê€þÀáÆ^9í¤–M<ú

secrets/loki-htpasswd.age

@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ nLWy3WcVJWCl3rXkhcSbp1joqmkk06QnxhCZ4UtSvmw
+iQ+Hx/vhiFgkWfbxHwGjxMBEqzyGww4/9do3W7V/y1Y
+-> ssh-ed25519 K3b7BA RkF2ADcjOGtivl9MrhO/HFwxlTAkbFHWL3iinUldMiM
+7q/zdVTMLevukZjkHtcN88iYzfTLvq2s3QdkgsFSO9M
+--- 1b2HiK06vJPqBgHVDD0QELOtfkl7/rlgGS9uI1mSbus
+„uܧoL;õå¬"
4¦Û»Z¼˜@§öãƒÐ’3+93Q4óÄ o•ŒØwé“„6<>M-²DkJn´;ñ*g
<0A>OÛYœ75ËSò)Ù°©

secrets/promtail-password.age

@@ -0,0 +1,12 @@
+age-encryption.org/v1
+-> ssh-ed25519 +HUDfA ZUM0ACC/NIekvX1PkCiXTHaTeE3ybudmY3piHw2iekQ
+cHj94FIR6gNJ3Hw9FI7K15OYgxbjkajGtCftD+2Mr8c
+-> ssh-ed25519 2D+APA tzlyOnAXnLxXO/47n45sFPiJF3FXd98UU5ajPhD2wSs
+P8ZdUiBeME17SU2BpMgOq4plyAqgzLOQWHa1+Q7cjYo
+-> ssh-ed25519 j2r2qQ 3OikD9JOmug7kdPAPz+JT/ryB6xBQhu2+cwS9h5sKGI
+XiIuxOyey2I6hmqabUCPzLc85q/1r9OwVGjHWYNQsp0
+-> ssh-ed25519 K3b7BA Bdqcqt4GgLzuSiEnIyImDiOQGwyIhhozRXMmNrp7glI
+65joZcnl0Hqe90Th2EdVgbcxUJFpy3fOgk6oPiSHh2A
+--- 6x6BFNypc+u3DpsHX3SajwEy1TqsAtbFei0ddRpEoBg
+
+äªUG¾xj4»®Îþ‡b=óžóñ¼Rd<52>3sHYÝÔ<>
*Qµ9Ã6‚n‹34&äw›»~h!§
^„[êš

services/monitoring/default.nix

@@ -2,5 +2,6 @@
   imports = [
     ./exporters
     ./lgtm
+    ./promtail.nix
   ];
 }

services/monitoring/lgtm/default.nix

@@ -1,6 +1,7 @@
 {
   imports = [
     ./grafana.nix
+    ./loki.nix
     ./prometheus.nix
   ];
 }

services/monitoring/lgtm/grafana.nix

@@ -93,6 +93,13 @@ in
                 access = "proxy";
                 url = "http://127.0.0.1:9009/prometheus";
               }
+              {
+                name = "Loki";
+                type = "loki";
+                uid = "loki";
+                access = "proxy";
+                url = "http://127.0.0.1:9090/";
+              }
             ];
           };
         };

services/monitoring/lgtm/loki.nix

@@ -0,0 +1,100 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.bagel.services.loki;
+  inherit (lib) mkEnableOption mkIf;
+in
+{
+  options.bagel.services.loki.enable = mkEnableOption "Loki storage";
+
+  config = mkIf cfg.enable {
+    age.secrets = {
+      loki-htpasswd = {
+        file = ../../../secrets/loki-htpasswd.age;
+        owner = "nginx";
+      };
+      loki-environment.file = ../../../secrets/loki-environment.age;
+    };
+
+    services.loki = {
+      enable = true;
+      extraFlags = ["--config.expand-env"];
+
+      configuration = {
+        server = {
+          http_listen_port = 9090;
+          grpc_listen_port = 9096;
+
+          # 16M
+          grpc_server_max_recv_msg_size = 16777216;
+          grpc_server_max_send_msg_size = 16777216;
+        };
+
+        auth_enabled = false;
+
+        common = {
+          storage.s3 = {
+            endpoint = "s3.delroth.net";
+            region = "garage";
+            bucketnames = "bagel-loki";
+            secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
+            access_key_id = "\${S3_KEY_ID}";
+            s3forcepathstyle = true;
+          };
+          ring.kvstore.store = "memberlist";
+          replication_factor = 1;
+        };
+
+        memberlist = {
+          bind_port = 7947;
+          advertise_port = 7947;
+        };
+
+        storage_config.tsdb_shipper = {
+          active_index_directory = "/var/lib/loki/index";
+          cache_location = "/var/lib/loki/cache";
+        };
+
+        compactor = {
+          working_directory = "/var/lib/loki/compactor";
+          compaction_interval = "10m";
+          retention_enabled = true;
+          retention_delete_delay = "1s";
+          retention_delete_worker_count = 150;
+          delete_request_store = "filesystem";
+        };
+
+        limits_config.retention_period = "1w";
+
+        schema_config = {
+          configs = [
+            {
+              from = "2024-07-01";
+              store = "tsdb";
+              object_store = "s3";
+              schema = "v13";
+              index = {
+                prefix = "index_";
+                period = "24h";
+              };
+            }
+          ];
+        };
+      };
+    };
+
+    systemd.services.loki.serviceConfig.EnvironmentFile = [ config.age.secrets.loki-environment.path ];
+
+    services.nginx.virtualHosts."loki.forkos.org" = {
+      enableACME = true;
+      forceSSL = true;
+      locations."/loki/api/v1/push" = {
+        proxyPass = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}";
+        basicAuthFile = config.age.secrets.loki-htpasswd.path;
+      };
+    };
+  };
+}

services/monitoring/promtail.nix

@@ -0,0 +1,53 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.bagel.monitoring.promtail;
+  inherit (lib) mkEnableOption mkIf;
+in
+{
+  options.bagel.monitoring.promtail.enable = (mkEnableOption "Promtail log export") // { default = true; };
+
+  config = mkIf cfg.enable {
+    age.secrets.promtail-password = {
+      file = ../../secrets/promtail-password.age;
+      owner = "promtail";
+    };
+
+    services.promtail = {
+      enable = true;
+      configuration = {
+        server.disable = true;
+        clients = [ 
+          {
+            url = "https://loki.forkos.org/loki/api/v1/push";
+            basic_auth = {
+              username = "promtail";
+              password_file = config.age.secrets.promtail-password.path;
+            };
+          }
+        ];
+        scrape_configs = [
+          {
+            job_name = "system";
+            journal = {
+              max_age = "12h";
+              labels = {
+                job = "systemd-journal";
+                host = config.networking.hostName;
+              };
+            };
+            relabel_configs = [
+              {
+                source_labels = [ "__journal__systemd_unit" ];
+                target_label = "unit";
+              }
+            ];
+          }
+        ];
+      };
+    };
+  };
+}