Add Loki + Promtail setup

More later, Loki also later.

.envrc

@@ -0,0 +1,2 @@
+# shellcheck shell=bash
+use flake

.gitignore

@@ -1,3 +1,4 @@
 result
 .gcroots
 config.tf.json
+.direnv

common/admins.nix

@@ -1,13 +1,16 @@
-{
+let
-  users.users.root.openssh.authorizedKeys.keys = [
+  keys = import ./ssh-keys.nix;
-    # delroth
+in {
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV"
+  users.users.root.openssh.authorizedKeys.keys = 
-    # raito
+    keys.users.delroth ++
+    keys.users.k900 ++ 
+    keys.users.raito ++
+    keys.users.maxine ++ 
+    [
+      # more raito
       "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
       "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
       "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
       "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFsZ7PMDt80tYXHyScQajNhqH4wuYg/o0OxfOHaZD4rXuT0VIKflKH1M9LslfHWIEH3XNeqhQOziH9r+Ny5JcM="
-    # k900
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun"
     ];
 }

common/ssh-keys.nix

@@ -2,10 +2,13 @@
   machines = {
     bagel-box = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsO4bNqY04uG13Pg3ubHfRDssTphDLzZ4YUniE5/p+M";
     meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT";
+    gerrit01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA+eSZu+u9sCynrMlsmFzQHLIELQAuVg0Cs1pBvwb4+A";
   };
 
   users = {
     delroth = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" ];
     raito = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp" ];
+    k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
+    maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
   };
 }

common/sysadmin/default.nix

@@ -17,6 +17,7 @@ in
         pv
         kitty.terminfo
         config.boot.kernelPackages.perf
+        bcc
         tcpdump
         ncdu
       ] ++ lib.optional (lib.hasAttr "pwru" pkgs) pkgs.pwru;

flake.nix

@@ -57,7 +57,15 @@
           '');
         };
     };
-    defaultApp.${system} = self.apps.${system}.apply;
+    apps.${system}.default = self.apps.${system}.apply;
+
+    devShells.${system}.default = pkgs.mkShell {
+      packages = [ 
+        inputs.agenix.packages.${system}.agenix
+        inputs.colmena.packages.${system}.colmena
+      ];
+    };
+
     colmena = {
       meta.nixpkgs = import nixpkgs {
         localSystem = system;

hosts/bagel-box/default.nix

@@ -40,6 +40,7 @@
     hydra.enable = true;
     hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
   };
+  bagel.meta.monitoring.address = "bagel-box.delroth.net";
 
   security.acme.acceptTerms = true;
   security.acme.defaults.email = "bagel@delroth.net";

hosts/cl.forkos.org/default.nix

@@ -24,6 +24,7 @@
       };
     };
   };
+  bagel.meta.monitoring.address = "gerrit01.infra.forkos.org";
 
   fileSystems."/gerrit-data" = {
     device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";

hosts/meta01.nixpkgs.lahfa.xyz/default.nix

@@ -21,6 +21,10 @@
     enable = true;
     domain = "netbox.forkos.org";
   };
+  bagel.meta.monitoring.address = "meta01.infra.forkos.org";
+  bagel.services.prometheus.enable = true;
+  bagel.services.loki.enable = true;
+  bagel.services.grafana.enable = true;
 
   i18n.defaultLocale = "fr_FR.UTF-8";
 

secrets.nix

@@ -7,6 +7,13 @@ let
     hydra-s3-credentials = [ machines.bagel-box ];
     hydra-ssh-key-priv = [ machines.bagel-box ];
     netbox-environment = [ machines.meta01 ];
+    mimir-environment = [ machines.meta01 ];
+    grafana-oauth-secret = [ machines.meta01 ];
+    loki-environment = [ machines.meta01 ];
+
+    # These are the same password, but nginx wants it in htpasswd format
+    loki-htpasswd = [ machines.meta01 ];
+    promtail-password = builtins.attrValues machines;
   };
 in
   builtins.listToAttrs (

secrets/grafana-oauth-secret.age

@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ Xl0fSOuF0xNTJrtVGdRLRIszd15LFrG5KCFNvSBK4Go
+qSEMBBw90jz4j8elpoUeyS4CTLBhZtNDhLNigesJq+0
+-> ssh-ed25519 K3b7BA cKI0twKiuuTKv1Js4jqt5v8cOqpxEMY9dmVghgJtbzw
+K5o31XP/nLsswsrMaxnIzCXVUtJqmJWoFglWFsV7+AQ
+--- X8pvqCHeCQ0LjzcjIHThkqp6YeOOT8dBMLuktgdgeY4
+sZÓ¸ŠíØ[þ²X<C2B2>“¡èÅ®Š5°=÷6)ÇT¿Q†‘N{•x³I1ƒ!Ó–ÜøB
ƒzš
*×íåL~K

secrets/loki-environment.age

@@ -0,0 +1,9 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ w0lLquFUUcmEZ/Fh1YSt85tAJkBwavORQbwMr7gMqF4
+J4T+EHm1uHbCZkAUNoNcB9uGSz082mFL8+dkCnvYQnM
+-> ssh-ed25519 K3b7BA 28bJZgBPPc2KIE5+b8LJuQ5L4YAiRAJzucEuOqXHdVM
+7hKENFr8QX0jpwuuQEjGFrUywJuhL1Tdi2V4/gR8JWE
+--- GSPZxz39TMMWv0qhotNgnXa5679Q7VK8JGjQjI7A8oM
+J²\@F“N•	³å…2‹®ô¨w×!¯1V
f»§˜Ž·’ÞO²CÓw®®V°£šÌº.^݆ 7<C2A0>w‡n4äàdW-Ö¾"@0¨ú¹EÏ¿·°ck,]M}x<>øÌ<C3B8>¡Ûy°[×ÁJ:!è‘	!ø螀c¬
+Bë¹R
+nøê€þÀáÆ^9í¤–M<ú

secrets/loki-htpasswd.age

@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 j2r2qQ nLWy3WcVJWCl3rXkhcSbp1joqmkk06QnxhCZ4UtSvmw
+iQ+Hx/vhiFgkWfbxHwGjxMBEqzyGww4/9do3W7V/y1Y
+-> ssh-ed25519 K3b7BA RkF2ADcjOGtivl9MrhO/HFwxlTAkbFHWL3iinUldMiM
+7q/zdVTMLevukZjkHtcN88iYzfTLvq2s3QdkgsFSO9M
+--- 1b2HiK06vJPqBgHVDD0QELOtfkl7/rlgGS9uI1mSbus
+„uܧoL;õå¬"
4¦Û»Z¼˜@§öãƒÐ’3+93Q4óÄ o•ŒØwé“„6<>M-²DkJn´;ñ*g
<0A>OÛYœ75ËSò)Ù°©

secrets/promtail-password.age

@@ -0,0 +1,12 @@
+age-encryption.org/v1
+-> ssh-ed25519 +HUDfA ZUM0ACC/NIekvX1PkCiXTHaTeE3ybudmY3piHw2iekQ
+cHj94FIR6gNJ3Hw9FI7K15OYgxbjkajGtCftD+2Mr8c
+-> ssh-ed25519 2D+APA tzlyOnAXnLxXO/47n45sFPiJF3FXd98UU5ajPhD2wSs
+P8ZdUiBeME17SU2BpMgOq4plyAqgzLOQWHa1+Q7cjYo
+-> ssh-ed25519 j2r2qQ 3OikD9JOmug7kdPAPz+JT/ryB6xBQhu2+cwS9h5sKGI
+XiIuxOyey2I6hmqabUCPzLc85q/1r9OwVGjHWYNQsp0
+-> ssh-ed25519 K3b7BA Bdqcqt4GgLzuSiEnIyImDiOQGwyIhhozRXMmNrp7glI
+65joZcnl0Hqe90Th2EdVgbcxUJFpy3fOgk6oPiSHh2A
+--- 6x6BFNypc+u3DpsHX3SajwEy1TqsAtbFei0ddRpEoBg
+
+äªUG¾xj4»®Îþ‡b=óžóñ¼Rd<52>3sHYÝÔ<>
*Qµ9Ã6‚n‹34&äw›»~h!§
^„[êš

services/default.nix

@@ -4,5 +4,6 @@
     ./postgres
     ./netbox
     ./gerrit
+    ./monitoring
   ];
 }

services/gerrit/default.nix

@@ -28,6 +28,8 @@ in
   config = mkIf cfg.enable {
     networking.firewall.allowedTCPPorts = [ 29418 ];
 
+    environment.systemPackages = [ pkgs.openjdk17_headless ];
+
     fileSystems."/var/lib/gerrit" = mkIf (cfg.data != "/var/lib/gerrit") {
       device = cfg.data;
       options = [ "bind" ];
@@ -70,18 +72,49 @@ in
       jvmPackage = pkgs.openjdk17_headless;
 
       settings = {
+        # Performance settings
         sshd.threads = 64;
         sshd.batchThreads = 8;
+
+        gc.aggressive = true;
         gc.interval = "1 day";
-        database.poolLimit = "250";
+
+        database.poolLimit = 250;
         database.poolMaxIdle = 16;
-        http.maxThreads = 100;
+
-        core.packedGitLimit = "4g";
+        httpd.maxThreads = 100;
-        core.packedGitWindowSize = "16k";
+
-        core.packedGitOpenFiles = "4096";
         receive.timeout = "4min";
-        transfer.timeout = "4min";
+        # Default is 0, infinite.
-        pack.threads = "8";
+        transfer.timeout = "30min";
+
+        # We may overshoot but it's OK.
+        core.packedGitWindowSize = "256k";
+        # Sum of all current packfiles is ~1.2G
+        # Largest packfile is 906MB.
+        # Average packfile is ~5-10MB.
+        core.packedGitLimit = "1g";
+        # We have plenty of memory, let's avoid file system cache → Gerrit needless copies.
+        core.packedGitUseStrongRefs = true;
+        core.packedGitOpenFiles = 4096;
+        # Big files in nixpkgs are usually lockfiles or machine-generated expressions
+        # containing a lot of hashes, they would weigh at most ~15MB.
+        core.streamFileThreshold = "20m";
+        # `mmap()` rather than `mmap()+read()` at the risk of running out of virtual address space.
+        core.packedGitMmap = true;
+
+        ## Takes more CPU but the transfer is smaller.
+        pack.deltacompression = false;
+        pack.threads = 8;
+
+        # FIXME(raito):
+        # Are we supposed to have private / hidden references?
+        # For a public server, that seems unlikely.
+        # But, we should be careful with this option.
+        # https://gerrit-documentation.storage.googleapis.com/Documentation/3.9.5/config-gerrit.html#receive.checkReferencedObjectsAreReachable
+        receive.checkReferencedObjectsAreReachable = false;
+
+        # Other settings
         log.jsonLogging = true;
         log.textLogging = false;
         sshd.advertisedAddress = "cl.forkos.org:29418";
@@ -90,11 +123,18 @@ in
         change.enableAttentionSet = true;
         change.enableAssignee = false;
 
+        user = {
+          name = "ForkOS Gerrit";
+          email = "gerrit@forkos.org";
+          anonymousCoward = "ForkOS contributor";
+        };
+
         # Configures gerrit for being reverse-proxied by nginx as per
         # https://gerrit-review.googlesource.com/Documentation/config-reverseproxy.html
         gerrit = {
           canonicalWebUrl = "https://cl.forkos.org";
           docUrl = "/Documentation";
+          defaultBranch = "refs/heads/main";
         };
 
         httpd.listenUrl = "proxy-https://${cfgGerrit.listenAddress}";

services/gerrit/www.nix

@@ -12,21 +12,30 @@ in
           add_header Permissions-Policy "interest-cohort=()";
         '';
         recommendedProxySettings = false;
+        commonHttpConfig = ''
+              log_format upstream_time '$remote_addr - $remote_user [$time_local] '
+                             '"$request" $status $body_bytes_sent '
+                             '"$http_referer" "$http_user_agent"'
+                             'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"';
+        '';
       };
       services.nginx.virtualHosts.gerrit = {
         serverName = builtins.head cfg.domains;
         serverAliases = builtins.tail cfg.domains;
         enableACME = true;
         forceSSL = true;
-
         extraConfig = ''
+          access_log /var/log/nginx/gerrit-access.log upstream_time;
+
           location / {
             proxy_pass http://localhost:4778;
             proxy_set_header  X-Forwarded-For $remote_addr;
             # The :443 suffix is a workaround for https://b.tvl.fyi/issues/88.
             proxy_set_header  Host $host:443;
             # Gerrit can throw a lot of data.
-            proxy_buffering off;
+            proxy_buffering on;
+            # NGINX should not give up super fast. Things can take time.
+            proxy_read_timeout 3600;
           }
 
           location = /robots.txt {

services/hydra/default.nix

@@ -53,7 +53,7 @@ in {
 
       buildMachinesFiles = [
         (pkgs.writeText "hydra-builders.conf" ''
-          ssh://bagel-builder@epyc.infra.newtype.fr i686-linux,x86_64-linux ${config.age.secrets.hydra-ssh-key-priv.path} 8 1 big-parallel,kvm,nixos-test - c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUJwcFBwKzhsdDFSTDNodW5aaGlXRUUvY1laaHJXYjFzaVhKVWpiU2l6Rzggcm9vdEBlcHljCg==
+          ssh://bagel-builder@epyc.infra.newtype.fr i686-linux,x86_64-linux ${config.age.secrets.hydra-ssh-key-priv.path} 8 1 big-parallel,kvm,nixos-test - c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSU9YVDlJbml0MU1oS3Q0cmpCQU5McTB0MGJQd3cvV1FaOTZ1QjRBRURybWwgcm9vdEBuaXhvcwo=
         '')
       ];
 

services/monitoring/default.nix

@@ -0,0 +1,7 @@
+{
+  imports = [
+    ./exporters
+    ./lgtm
+    ./promtail.nix
+  ];
+}

services/monitoring/exporters/baseline.nix

@@ -0,0 +1,34 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.bagel.monitoring.exporters.baseline;
+  inherit (lib) mkEnableOption mkIf;
+in
+{
+  options.bagel.monitoring.exporters.baseline.enable = (mkEnableOption "Standard set of exporters") // { default = true; };
+
+  config = mkIf cfg.enable {
+    services.prometheus.exporters.node = {
+      enable = true;
+      enabledCollectors = [
+        "processes"
+        "systemd"
+      ];
+      port = 9101;
+    };
+
+    services.cadvisor = {
+      enable = true;
+      port = 9102;
+      listenAddress = "0.0.0.0";
+    };
+
+    bagel.meta.monitoring.exporters = [
+      { port = 9101; }
+      { port = 9102; }
+    ];
+  };
+}

services/monitoring/exporters/default.nix

@@ -0,0 +1,36 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  inherit (lib) mkOption types;
+in
+{
+  imports = [
+    ./baseline.nix
+    ./nginx.nix
+    ./postgres.nix
+  ];
+
+  options.bagel = {
+    meta.monitoring = {
+      address = mkOption {
+        description = "Node's public address";
+        type = types.str;
+      };
+      exporters = mkOption {
+        description = "List of all exporters to scrape";
+        type = types.listOf (types.submodule {
+          options.port = mkOption {
+            description = "Exporter port";
+            type = types.int;
+          };
+        });
+        default = [];
+      };
+    };
+  };
+
+  config.networking.firewall.allowedTCPPorts = map (e: e.port) config.bagel.meta.monitoring.exporters;
+}

services/monitoring/exporters/nginx.nix

@@ -0,0 +1,37 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.bagel.monitoring.exporters.nginx;
+  inherit (lib) mkEnableOption mkIf;
+  logFormat = ''$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'';
+in
+{
+  options.bagel.monitoring.exporters.nginx.enable = (mkEnableOption "Nginx access.log exporter") // { default = config.services.nginx.enable; };
+
+  config = mkIf cfg.enable {
+    services.nginx.appendHttpConfig = ''
+      log_format ours '${logFormat}';
+      access_log /var/log/nginx/access.log ours;
+    '';
+    
+    services.prometheus.exporters.nginxlog = {
+      enable = true;
+      port = 9103;
+      group = "nginx";
+      settings.namespaces = [
+        {
+          name = "nginx";
+          format = logFormat;
+          source.files = ["/var/log/nginx/access.log"];
+        }
+      ];
+    };
+
+    bagel.meta.monitoring.exporters = [
+      { port = 9103; }
+    ];
+  };
+}

services/monitoring/exporters/postgres.nix

@@ -0,0 +1,31 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.bagel.monitoring.exporters.postgres;
+  inherit (lib) mkEnableOption mkIf;
+in
+{
+  options.bagel.monitoring.exporters.postgres.enable = (mkEnableOption "Postgres exporter") // { default = config.services.postgresql.enable; };
+
+  config = mkIf cfg.enable {
+    services.prometheus.exporters.postgres = {
+      enable = true;
+      port = 9104;
+      runAsLocalSuperUser = true;
+      extraFlags = [
+        "--collector.long_running_transactions"
+        "--collector.stat_activity_autovacuum"
+        "--collector.stat_statements"
+      ];
+    };
+
+    services.postgresql.settings.shared_preload_libraries = "pg_stat_statements";
+
+    bagel.meta.monitoring.exporters = [
+      { port = 9104; }
+    ];
+  };
+}

services/monitoring/lgtm/dashboards/nginx.json

@@ -0,0 +1,697 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "NGINX Log metrics with Prometheus based on https://github.com/martin-helmich/prometheus-nginxlog-exporter \r\nBased on namespace prefix 'nginx'. If different, you may need to adjust the metrics.\r\nDashboard based on 6482 dashboard",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "gnetId": 15947,
+  "graphTooltip": 0,
+  "id": 4,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "mimir"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "continuous-GrYlRd"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "percentage",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 5
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 11,
+        "x": 0,
+        "y": 0
+      },
+      "id": 12,
+      "options": {
+        "displayMode": "lcd",
+        "maxVizHeight": 300,
+        "minVizHeight": 16,
+        "minVizWidth": 8,
+        "namePlacement": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showUnfilled": true,
+        "sizing": "auto",
+        "text": {},
+        "valueMode": "color"
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "mimir"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "sum(rate(nginx_http_response_count_total{status=~\"^2..\",instance=\"$host\"}[1m])) / sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) * 100",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "2** status codes",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "mimir"
+          },
+          "exemplar": true,
+          "expr": "sum(rate(nginx_http_response_count_total{status=~\"^4..\",instance=\"$host\"}[1m])) / sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) * 100",
+          "interval": "",
+          "legendFormat": "4** status codes",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "mimir"
+          },
+          "exemplar": true,
+          "expr": "sum(rate(nginx_http_response_count_total{status=~\"^5..\",instance=\"$host\"}[1m])) / sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) * 100",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "5** status codes",
+          "refId": "B"
+        }
+      ],
+      "title": "Percentage Ratio of status codes to all status codes",
+      "transparent": true,
+      "type": "bargauge"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "mimir"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 4,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 11,
+        "x": 11,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "maxHeight": 600,
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "8.5.0-54880pre",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "mimir"
+          },
+          "exemplar": true,
+          "expr": "sum(rate(nginx_http_response_time_seconds_count{instance=\"$host\"}[1m])) ",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "$host",
+          "refId": "A"
+        }
+      ],
+      "title": "Requests per Second",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "mimir"
+      },
+      "description": "Response sizes in bytes",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 4,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 11,
+        "x": 11,
+        "y": 7
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "maxHeight": 600,
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "8.5.0-54880pre",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "mimir"
+          },
+          "exemplar": true,
+          "expr": "sum(rate(nginx_http_response_size_bytes{instance=\"$host\"}[5m])) ",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "$host",
+          "refId": "A"
+        }
+      ],
+      "title": "HTTP Traffic",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "mimir"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 21,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 4,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 11,
+        "x": 0,
+        "y": 8
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "maxHeight": 600,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "8.5.0-54880pre",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "mimir"
+          },
+          "exemplar": true,
+          "expr": "sum(rate(nginx_http_response_time_seconds_sum{instance=\"$host\"}[5m])) / sum(rate(nginx_http_response_time_seconds_count{instance=\"$host\"}[5m])) ",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "$host",
+          "refId": "A"
+        }
+      ],
+      "title": "Average Response Time [5m]",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "mimir"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 3,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 11,
+        "x": 11,
+        "y": 14
+      },
+      "id": 10,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "maxHeight": 600,
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "8.5.0-54880pre",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "mimir"
+          },
+          "exemplar": true,
+          "expr": "sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) by (status)",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Status codes per second",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "mimir"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 4,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 11,
+        "x": 0,
+        "y": 15
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "maxHeight": 600,
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "8.5.0-54880pre",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "mimir"
+          },
+          "exemplar": true,
+          "expr": "nginx_http_response_time_seconds{quantile=\"0.9\",method=\"GET\",status=~\"2[0-9]*\",instance=~\"$host\"}",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "$host",
+          "refId": "A"
+        }
+      ],
+      "title": "Response time (90% quantile)",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 39,
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "isNone": true,
+          "selected": false,
+          "text": "None",
+          "value": ""
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "mimir"
+        },
+        "definition": "label_values(nginx_http_response_count_total,instance)",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Host:",
+        "multi": false,
+        "name": "host",
+        "options": [],
+        "query": {
+          "query": "label_values(nginx_http_response_count_total,instance)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timeRangeUpdatedDuringEditOrView": false,
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "NGINX Log Metrics [M]",
+  "uid": "JfOTY2Pnk",
+  "version": 4,
+  "weekStart": ""
+}

services/monitoring/lgtm/default.nix

@@ -0,0 +1,7 @@
+{
+  imports = [
+    ./grafana.nix
+    ./loki.nix
+    ./prometheus.nix
+  ];
+}

services/monitoring/lgtm/grafana.nix

@@ -0,0 +1,133 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.bagel.services.grafana;
+  inherit (lib) mkEnableOption mkIf;
+in
+{
+  options.bagel.services.grafana.enable = mkEnableOption "Grafana frontend";
+
+  config = mkIf cfg.enable {
+    age.secrets.grafana-oauth-secret = {
+      file = ../../../secrets/grafana-oauth-secret.age;
+      owner = "grafana";
+    };
+
+    bagel.services.postgres.enable = true;
+
+    services = {
+      grafana = {
+        enable = true;
+
+        settings = {
+          server = {
+            domain = "grafana.forkos.org";
+            http_addr = "127.0.0.1";
+            http_port = 2342;
+            root_url = "https://grafana.forkos.org/";
+          };
+
+          database = {
+            type = "postgres";
+            user = "grafana";
+            host = "/run/postgresql";
+          };
+
+          "auth.generic_oauth" = {
+            enabled = true;
+
+            name = "Lix SSO";
+            client_id = "forkos-grafana";
+            client_secret = "$__file{${config.age.secrets.grafana-oauth-secret.path}}";
+
+            auth_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth";
+            token_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
+            api_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/userinfo";
+
+            login_attribute_path = "username";
+            email_attribute_path = "email";
+            name_attribute_path = "full_name";
+
+            scopes = [
+              "openid"
+              "profile"
+              "email"
+              "offline_access"
+              "roles"
+            ];
+
+            allow_sign_up = true;
+            auto_login = true;
+            allow_assign_grafana_admin = true;
+
+            role_attribute_path = "contains(grafana_roles[*], 'Admin') && 'GrafanaAdmin' || contains(grafana_roles[*], 'Editor') && 'Editor' || 'Viewer'";
+          };
+
+          dashboards.default_home_dashboard_path = "${./dashboards/node_exporter.json}";
+
+          feature_toggles.enable = "autoMigrateOldPanels newVizTooltips";
+          security.angular_support_enabled = false;
+        };
+
+        provision = {
+          dashboards.settings = {
+            apiVersion = 1;
+            providers = [
+              {
+                name = "default";
+                options.path = ./dashboards;
+              }
+            ];
+          };
+
+          datasources.settings = {
+            apiVersion = 1;
+            datasources = [
+              {
+                name = "Mimir";
+                type = "prometheus";
+                uid = "mimir";
+                access = "proxy";
+                url = "http://127.0.0.1:9009/prometheus";
+              }
+              {
+                name = "Loki";
+                type = "loki";
+                uid = "loki";
+                access = "proxy";
+                url = "http://127.0.0.1:9090/";
+              }
+            ];
+          };
+        };
+      };
+
+      postgresql = {
+        ensureDatabases = [ "grafana" ];
+        ensureUsers = [
+          {
+            name = "grafana";
+            ensureDBOwnership = true;
+          }
+        ];
+      };
+
+      nginx = let
+        scfg = config.services.grafana.settings.server;
+      in {
+        enable = true;
+        virtualHosts."${scfg.domain}" = {
+          enableACME = true;
+          forceSSL = true;
+          locations."/" = {
+            proxyPass = "http://${scfg.http_addr}:${toString scfg.http_port}";
+            proxyWebsockets = true;
+          };
+        };
+      };
+    };
+};
+}

services/monitoring/lgtm/loki.nix

@@ -0,0 +1,100 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.bagel.services.loki;
+  inherit (lib) mkEnableOption mkIf;
+in
+{
+  options.bagel.services.loki.enable = mkEnableOption "Loki storage";
+
+  config = mkIf cfg.enable {
+    age.secrets = {
+      loki-htpasswd = {
+        file = ../../../secrets/loki-htpasswd.age;
+        owner = "nginx";
+      };
+      loki-environment.file = ../../../secrets/loki-environment.age;
+    };
+
+    services.loki = {
+      enable = true;
+      extraFlags = ["--config.expand-env"];
+
+      configuration = {
+        server = {
+          http_listen_port = 9090;
+          grpc_listen_port = 9096;
+
+          # 16M
+          grpc_server_max_recv_msg_size = 16777216;
+          grpc_server_max_send_msg_size = 16777216;
+        };
+
+        auth_enabled = false;
+
+        common = {
+          storage.s3 = {
+            endpoint = "s3.delroth.net";
+            region = "garage";
+            bucketnames = "bagel-loki";
+            secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
+            access_key_id = "\${S3_KEY_ID}";
+            s3forcepathstyle = true;
+          };
+          ring.kvstore.store = "memberlist";
+          replication_factor = 1;
+        };
+
+        memberlist = {
+          bind_port = 7947;
+          advertise_port = 7947;
+        };
+
+        storage_config.tsdb_shipper = {
+          active_index_directory = "/var/lib/loki/index";
+          cache_location = "/var/lib/loki/cache";
+        };
+
+        compactor = {
+          working_directory = "/var/lib/loki/compactor";
+          compaction_interval = "10m";
+          retention_enabled = true;
+          retention_delete_delay = "1s";
+          retention_delete_worker_count = 150;
+          delete_request_store = "filesystem";
+        };
+
+        limits_config.retention_period = "1w";
+
+        schema_config = {
+          configs = [
+            {
+              from = "2024-07-01";
+              store = "tsdb";
+              object_store = "s3";
+              schema = "v13";
+              index = {
+                prefix = "index_";
+                period = "24h";
+              };
+            }
+          ];
+        };
+      };
+    };
+
+    systemd.services.loki.serviceConfig.EnvironmentFile = [ config.age.secrets.loki-environment.path ];
+
+    services.nginx.virtualHosts."loki.forkos.org" = {
+      enableACME = true;
+      forceSSL = true;
+      locations."/loki/api/v1/push" = {
+        proxyPass = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}";
+        basicAuthFile = config.age.secrets.loki-htpasswd.path;
+      };
+    };
+  };
+}

services/monitoring/lgtm/prometheus.nix

@@ -0,0 +1,83 @@
+{
+  config,
+  lib,
+  nodes,
+  ...
+}:
+let
+  cfg = config.bagel.services.prometheus;
+  inherit (lib) mkEnableOption mkIf;
+
+  forEachMachine = fn: map fn (builtins.attrValues nodes);
+
+  allMetas = forEachMachine (machine: {
+    name = machine.config.networking.hostName;
+    address = machine.config.bagel.meta.monitoring.address or null;
+    exporters = machine.config.bagel.meta.monitoring.exporters or [];
+  });
+
+  scrapableMetas = builtins.filter (m: m.address != null && m.exporters != []) allMetas;
+
+  toJobConfig = m: {
+    job_name = m.name;
+    static_configs = [
+      { targets = map (e: m.address + ":" + (toString e.port)) m.exporters; }
+    ];
+  };
+
+  jobConfigs = map toJobConfig scrapableMetas;
+in
+{
+  options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
+
+  config = mkIf cfg.enable {
+    age.secrets.mimir-environment.file = ../../../secrets/mimir-environment.age;
+
+    services.prometheus = {
+      enable = true;
+      enableAgentMode = true;
+      listenAddress = "127.0.0.1";
+      port = 9001;
+      globalConfig.scrape_interval = "15s";
+      scrapeConfigs = jobConfigs;
+      remoteWrite = [
+        { url = "http://localhost:9009/api/v1/push"; }
+      ];
+    };
+
+    services.mimir = {
+      enable = true;
+      extraFlags = ["--config.expand-env=true"];
+      configuration = {
+        multitenancy_enabled = false;
+
+        common.storage = {
+          backend = "s3";
+          s3 = {
+            endpoint = "s3.delroth.net";
+            bucket_name = "bagel-mimir";
+            secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
+            access_key_id = "\${S3_KEY_ID}";
+          };
+        };
+
+        server = {
+          http_listen_port = 9009;
+          grpc_server_max_recv_msg_size = 104857600;
+          grpc_server_max_send_msg_size = 104857600;
+          grpc_server_max_concurrent_streams = 1000;
+        };
+
+        ingester.ring.replication_factor = 1;
+
+        blocks_storage.backend = "s3";
+        ruler_storage = {
+          backend = "local";
+          local.directory = ./alerts;
+        };
+      };
+    };
+
+    systemd.services.mimir.serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
+  };
+}

services/monitoring/promtail.nix

@@ -0,0 +1,53 @@
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.bagel.monitoring.promtail;
+  inherit (lib) mkEnableOption mkIf;
+in
+{
+  options.bagel.monitoring.promtail.enable = (mkEnableOption "Promtail log export") // { default = true; };
+
+  config = mkIf cfg.enable {
+    age.secrets.promtail-password = {
+      file = ../../secrets/promtail-password.age;
+      owner = "promtail";
+    };
+
+    services.promtail = {
+      enable = true;
+      configuration = {
+        server.disable = true;
+        clients = [ 
+          {
+            url = "https://loki.forkos.org/loki/api/v1/push";
+            basic_auth = {
+              username = "promtail";
+              password_file = config.age.secrets.promtail-password.path;
+            };
+          }
+        ];
+        scrape_configs = [
+          {
+            job_name = "system";
+            journal = {
+              max_age = "12h";
+              labels = {
+                job = "systemd-journal";
+                host = config.networking.hostName;
+              };
+            };
+            relabel_configs = [
+              {
+                source_labels = [ "__journal__systemd_unit" ];
+                target_label = "unit";
+              }
+            ];
+          }
+        ];
+      };
+    };
+  };
+}