hydra: add declarative controls via terranix

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>

.envrc

@@ -1,2 +1 @@
-# shellcheck shell=bash
 use flake

.gitignore

@@ -1,5 +1,4 @@
 result
 .gcroots
 config.tf.json
-.direnv
 .terraform

common/admins.nix

@@ -1,17 +1,11 @@
-let
-  keys = import ./ssh-keys.nix;
-in {
-  users.users.root.openssh.authorizedKeys.keys = 
-    keys.users.delroth ++
-    keys.users.k900 ++ 
-    keys.users.raito ++
-    keys.users.maxine ++ 
-    keys.users.jade ++
-    [
-      # more raito
-      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
-      "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFsZ7PMDt80tYXHyScQajNhqH4wuYg/o0OxfOHaZD4rXuT0VIKflKH1M9LslfHWIEH3XNeqhQOziH9r+Ny5JcM="
-    ];
+{
+  users.users.root.openssh.authorizedKeys.keys = [
+    # delroth
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV"
+    # raito
+    "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
+    "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFsZ7PMDt80tYXHyScQajNhqH4wuYg/o0OxfOHaZD4rXuT0VIKflKH1M9LslfHWIEH3XNeqhQOziH9r+Ny5JcM="
+  ];
 }

common/base-server.nix

@@ -1,7 +1,7 @@
 { lib, pkgs, ... }: {
   nixpkgs.overlays = import ../overlays;
 
-  nix.package = lib.mkDefault pkgs.lix;
+  nix.package = pkgs.lix;
   services.openssh.enable = lib.mkForce true;
 
   networking.firewall.enable = true;

common/raito-vm.nix

@@ -32,7 +32,6 @@ in
     systemd.network.enable = true;
     security.acme.defaults.email = "bagel-acme@lahfa.xyz";
     security.acme.acceptTerms = true;
-    networking.useDHCP = lib.mkDefault false;
 
     systemd.network.networks."10-nat-lan" = {
       matchConfig.Name = "nat-lan";

common/ssh-keys.nix

@@ -1,20 +1,11 @@
 {
   machines = {
-    bagel-box = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsO4bNqY04uG13Pg3ubHfRDssTphDLzZ4YUniE5/p+M";
+    bagel-box = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJW7jmkJ73tx9lsrz9UhqJIJdoqZGuhsHti55xny5/yp";
     meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT";
-    gerrit01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA+eSZu+u9sCynrMlsmFzQHLIELQAuVg0Cs1pBvwb4+A";
-    fodwatch = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFRyTNfvKl5FcSyzGzw+h+bNFNOxdhvI67WdUZ2iIJ1L";
   };
 
   users = {
     delroth = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" ];
     raito = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp" ];
-    k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
-    maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
-    jade = [
-      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNldAg4t13/i69TD786The+U3wbiNUdW2Kc9KNWvEhgpf4y4x4Sft0oYfkPw5cjX4H3APqfD+b7ItAG0GCbwHw6KMYPoVMNK08zBMJUqt1XExbqGeFLqBaeqDsmEAYXJRbjMTAorpOCtgQdoCKK/DvZ51zUWXxT8UBNHSl19Ryv5Ry5VVdbAE35rqs57DQ9+ma6htXnsBEmmnC+1Zv1FE956m/OpBTId50mor7nS2FguAtPZnDPpTd5zl9kZmJEuWCrmy6iinw5V4Uy1mLeZkQv+/FtozbyifCRCvps9nHpv4mBSU5ABLgnRRvXs+D41Jx7xloNADr1nNgpsNrYaTh hed-bot-ssh-tpm-rsa"
-      "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIKYljH8iPMrH00lOb3ETxRrZimdKzPPEdsJQ5D5ovtOwAAAACnNzaDpzc2hrZXk= ssh:sshkey"
-      "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBO4idMfdJxDJuBNOid60d4I+qxj09RHt+YkCYV2eXt6tGrEXg+S8hTQusy/SqooiXUH9pt4tea2RuBPN9+UwrH0= type-a yubikey slot 9a"
-    ];
   };
 }

common/sysadmin/default.nix

@@ -17,7 +17,6 @@ in
         pv
         kitty.terminfo
         config.boot.kernelPackages.perf
-        bcc
         tcpdump
         ncdu
       ] ++ lib.optional (lib.hasAttr "pwru" pkgs) pkgs.pwru;

flake.nix

@@ -72,17 +72,13 @@
           '');
         };
     };
-    apps.${system}.default = self.apps.${system}.apply;
-
+    defaultApp.${system} = self.apps.${system}.apply;
     devShells.${system}.default = pkgs.mkShell {
-      packages = [
-        inputs.agenix.packages.${system}.agenix
-        inputs.colmena.packages.${system}.colmena
-
+      buildInputs = [
         pkgs.opentofu
+        pkgs.colmena
       ];
     };
-
     colmena = {
       meta.nixpkgs = import nixpkgs {
         localSystem = system;
@@ -99,7 +95,7 @@
           inputs.hydra.nixosModules.hydra
 
           ./services
-          ./common
+
           ./hosts/bagel-box
         ];
       };
@@ -125,17 +121,6 @@
           ./hosts/cl.forkos.org
         ];
       };
-
-      fodwatch = {
-        imports = [
-          inputs.agenix.nixosModules.default
-          inputs.hydra.nixosModules.hydra
-
-          ./services
-          ./common
-          ./hosts/fodwatch.forkos.org
-        ];
-      };
     };
   };
 }

hosts/bagel-box/default.nix

@@ -40,7 +40,6 @@
     hydra.enable = true;
     hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
   };
-  bagel.meta.monitoring.address = "bagel-box.delroth.net";
 
   security.acme.acceptTerms = true;
   security.acme.defaults.email = "bagel@delroth.net";

hosts/cl.forkos.org/default.nix

@@ -24,7 +24,6 @@
       };
     };
   };
-  bagel.meta.monitoring.address = "gerrit01.infra.forkos.org";
 
   fileSystems."/gerrit-data" = {
     device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";

hosts/fodwatch.forkos.org/default.nix

@@ -1,33 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  ...
-}:
-{
-  networking.hostName = "fodwatch";
-  networking.domain = "infra.forkos.org";
-
-  time.timeZone = "Europe/Paris";
-
-  bagel.sysadmin.enable = true;
-  # Fodwatch will be proxied.
-  bagel.raito.v6-proxy-awareness.enable = true;
-  bagel.hardware.raito-vm = {
-    enable = true;
-    networking = {
-      nat-lan-mac = "BC:24:11:F2:17:F8";
-      wan = {
-        address = "2001:bc8:38ee:100:1000::30/64";
-        mac = "BC:24:11:69:74:D0";
-      };
-    };
-  };
-
-  bagel.meta.monitoring.address = "fodwatch.infra.forkos.org";
-
-  i18n.defaultLocale = "en_US.UTF-8";
-
-  system.stateVersion = "24.05";
-  deployment.targetHost = "fodwatch.infra.forkos.org";
-}

hosts/meta01.nixpkgs.lahfa.xyz/default.nix

@@ -21,10 +21,6 @@
     enable = true;
     domain = "netbox.forkos.org";
   };
-  bagel.meta.monitoring.address = "meta01.infra.forkos.org";
-  bagel.services.prometheus.enable = true;
-  bagel.services.loki.enable = true;
-  bagel.services.grafana.enable = true;
 
   i18n.defaultLocale = "fr_FR.UTF-8";
 

secrets.nix

@@ -7,13 +7,6 @@ let
     hydra-s3-credentials = [ machines.bagel-box ];
     hydra-ssh-key-priv = [ machines.bagel-box ];
     netbox-environment = [ machines.meta01 ];
-    mimir-environment = [ machines.meta01 ];
-    grafana-oauth-secret = [ machines.meta01 ];
-    loki-environment = [ machines.meta01 ];
-
-    # These are the same password, but nginx wants it in htpasswd format
-    loki-htpasswd = [ machines.meta01 ];
-    promtail-password = builtins.attrValues machines;
   };
 in
   builtins.listToAttrs (

secrets/grafana-oauth-secret.age

@@ -1,7 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 j2r2qQ Xl0fSOuF0xNTJrtVGdRLRIszd15LFrG5KCFNvSBK4Go
-qSEMBBw90jz4j8elpoUeyS4CTLBhZtNDhLNigesJq+0
--> ssh-ed25519 K3b7BA cKI0twKiuuTKv1Js4jqt5v8cOqpxEMY9dmVghgJtbzw
-K5o31XP/nLsswsrMaxnIzCXVUtJqmJWoFglWFsV7+AQ
---- X8pvqCHeCQ0LjzcjIHThkqp6YeOOT8dBMLuktgdgeY4
-sZÓ¸ŠíØ[þ²X<C2B2>“¡èÅ®Š5°=÷6)ÇT¿Q†‘N{•x³I1ƒ!Ó–ÜøB
ƒzš
*×íåL~K

secrets/loki-environment.age

@@ -1,9 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 j2r2qQ w0lLquFUUcmEZ/Fh1YSt85tAJkBwavORQbwMr7gMqF4
-J4T+EHm1uHbCZkAUNoNcB9uGSz082mFL8+dkCnvYQnM
--> ssh-ed25519 K3b7BA 28bJZgBPPc2KIE5+b8LJuQ5L4YAiRAJzucEuOqXHdVM
-7hKENFr8QX0jpwuuQEjGFrUywJuhL1Tdi2V4/gR8JWE
---- GSPZxz39TMMWv0qhotNgnXa5679Q7VK8JGjQjI7A8oM
-J²\@F“N•	³å…2‹®ô¨w×!¯1V
f»§˜Ž·’ÞO²CÓw®®V°£šÌº.^݆ 7<C2A0>w‡n4äàdW-Ö¾"@0¨ú¹EÏ¿·°ck,]M}x<>øÌ<C3B8>¡Ûy°[×ÁJ:!è‘	!ø螀c¬
-Bë¹R
-nøê€þÀáÆ^9í¤–M<ú

secrets/loki-htpasswd.age

@@ -1,7 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 j2r2qQ nLWy3WcVJWCl3rXkhcSbp1joqmkk06QnxhCZ4UtSvmw
-iQ+Hx/vhiFgkWfbxHwGjxMBEqzyGww4/9do3W7V/y1Y
--> ssh-ed25519 K3b7BA RkF2ADcjOGtivl9MrhO/HFwxlTAkbFHWL3iinUldMiM
-7q/zdVTMLevukZjkHtcN88iYzfTLvq2s3QdkgsFSO9M
---- 1b2HiK06vJPqBgHVDD0QELOtfkl7/rlgGS9uI1mSbus
-„uܧoL;õå¬"
4¦Û»Z¼˜@§öãƒÐ’3+93Q4óÄ o•ŒØwé“„6<>M-²DkJn´;ñ*g
<0A>OÛYœ75ËSò)Ù°©

secrets/promtail-password.age

@@ -1,12 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 +HUDfA ZUM0ACC/NIekvX1PkCiXTHaTeE3ybudmY3piHw2iekQ
-cHj94FIR6gNJ3Hw9FI7K15OYgxbjkajGtCftD+2Mr8c
--> ssh-ed25519 2D+APA tzlyOnAXnLxXO/47n45sFPiJF3FXd98UU5ajPhD2wSs
-P8ZdUiBeME17SU2BpMgOq4plyAqgzLOQWHa1+Q7cjYo
--> ssh-ed25519 j2r2qQ 3OikD9JOmug7kdPAPz+JT/ryB6xBQhu2+cwS9h5sKGI
-XiIuxOyey2I6hmqabUCPzLc85q/1r9OwVGjHWYNQsp0
--> ssh-ed25519 K3b7BA Bdqcqt4GgLzuSiEnIyImDiOQGwyIhhozRXMmNrp7glI
-65joZcnl0Hqe90Th2EdVgbcxUJFpy3fOgk6oPiSHh2A
---- 6x6BFNypc+u3DpsHX3SajwEy1TqsAtbFei0ddRpEoBg
-
-äªUG¾xj4»®Îþ‡b=óžóñ¼Rd<52>3sHYÝÔ<>
*Qµ9Ã6‚n‹34&äw›»~h!§
^„[êš

services/default.nix

@@ -4,6 +4,5 @@
     ./postgres
     ./netbox
     ./gerrit
-    ./monitoring
   ];
 }

services/gerrit/default.nix

@@ -28,8 +28,6 @@ in
   config = mkIf cfg.enable {
     networking.firewall.allowedTCPPorts = [ 29418 ];
 
-    environment.systemPackages = [ pkgs.openjdk17_headless ];
-
     fileSystems."/var/lib/gerrit" = mkIf (cfg.data != "/var/lib/gerrit") {
       device = cfg.data;
       options = [ "bind" ];
@@ -72,49 +70,17 @@ in
       jvmPackage = pkgs.openjdk17_headless;
 
       settings = {
-        # Performance settings
         sshd.threads = 64;
         sshd.batchThreads = 8;
-
-        gc.aggressive = true;
         gc.interval = "1 day";
-
-        database.poolLimit = 250;
+        database.poolLimit = "250";
         database.poolMaxIdle = 16;
-
-        httpd.maxThreads = 100;
-
+        http.maxThreads = 100;
+        # core.packedGitLimit = "4g";
+        # core.packedGitWindowSize = "16k";
+        # core.packedGitOpenFiles = "4096";
         receive.timeout = "4min";
-        # Default is 0, infinite.
-        transfer.timeout = "30min";
-
-        # We may overshoot but it's OK.
-        core.packedGitWindowSize = "256k";
-        # Sum of all current packfiles is ~1.2G
-        # Largest packfile is 906MB.
-        # Average packfile is ~5-10MB.
-        core.packedGitLimit = "1g";
-        # We have plenty of memory, let's avoid file system cache → Gerrit needless copies.
-        core.packedGitUseStrongRefs = true;
-        core.packedGitOpenFiles = 4096;
-        # Big files in nixpkgs are usually lockfiles or machine-generated expressions
-        # containing a lot of hashes, they would weigh at most ~15MB.
-        core.streamFileThreshold = "20m";
-        # `mmap()` rather than `mmap()+read()` at the risk of running out of virtual address space.
-        core.packedGitMmap = true;
-
-        ## Takes more CPU but the transfer is smaller.
-        pack.deltacompression = false;
-        pack.threads = 8;
-
-        # FIXME(raito):
-        # Are we supposed to have private / hidden references?
-        # For a public server, that seems unlikely.
-        # But, we should be careful with this option.
-        # https://gerrit-documentation.storage.googleapis.com/Documentation/3.9.5/config-gerrit.html#receive.checkReferencedObjectsAreReachable
-        receive.checkReferencedObjectsAreReachable = false;
-
-        # Other settings
+        # pack.threads = "8";
         log.jsonLogging = true;
         log.textLogging = false;
         sshd.advertisedAddress = "cl.forkos.org:29418";
@@ -123,18 +89,11 @@ in
         change.enableAttentionSet = true;
         change.enableAssignee = false;
 
-        user = {
-          name = "ForkOS Gerrit";
-          email = "gerrit@forkos.org";
-          anonymousCoward = "ForkOS contributor";
-        };
-
         # Configures gerrit for being reverse-proxied by nginx as per
         # https://gerrit-review.googlesource.com/Documentation/config-reverseproxy.html
         gerrit = {
           canonicalWebUrl = "https://cl.forkos.org";
           docUrl = "/Documentation";
-          defaultBranch = "refs/heads/main";
         };
 
         httpd.listenUrl = "proxy-https://${cfgGerrit.listenAddress}";

services/gerrit/www.nix

@@ -11,23 +11,19 @@ in
         appendHttpConfig = ''
           add_header Permissions-Policy "interest-cohort=()";
         '';
-        recommendedProxySettings = false;
       };
       services.nginx.virtualHosts.gerrit = {
         serverName = builtins.head cfg.domains;
         serverAliases = builtins.tail cfg.domains;
         enableACME = true;
         forceSSL = true;
+
         extraConfig = ''
           location / {
             proxy_pass http://localhost:4778;
             proxy_set_header  X-Forwarded-For $remote_addr;
             # The :443 suffix is a workaround for https://b.tvl.fyi/issues/88.
             proxy_set_header  Host $host:443;
-            # Gerrit can throw a lot of data.
-            proxy_buffering on;
-            # NGINX should not give up super fast. Things can take time.
-            proxy_read_timeout 3600;
           }
 
           location = /robots.txt {

services/hydra/default.nix

@@ -53,7 +53,7 @@ in {
 
       buildMachinesFiles = [
         (pkgs.writeText "hydra-builders.conf" ''
-          ssh://bagel-builder@epyc.infra.newtype.fr i686-linux,x86_64-linux ${config.age.secrets.hydra-ssh-key-priv.path} 8 1 big-parallel,kvm,nixos-test - c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSU9YVDlJbml0MU1oS3Q0cmpCQU5McTB0MGJQd3cvV1FaOTZ1QjRBRURybWwgcm9vdEBuaXhvcwo=
+          ssh://bagel-builder@epyc.infra.newtype.fr i686-linux,x86_64-linux ${config.age.secrets.hydra-ssh-key-priv.path} 8 1 big-parallel,kvm,nixos-test - c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUJwcFBwKzhsdDFSTDNodW5aaGlXRUUvY1laaHJXYjFzaVhKVWpiU2l6Rzggcm9vdEBlcHljCg==
         '')
       ];
 

services/monitoring/default.nix

@@ -1,7 +0,0 @@
-{
-  imports = [
-    ./exporters
-    ./lgtm
-    ./promtail.nix
-  ];
-}

services/monitoring/exporters/cadvisor.nix

@@ -1,22 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  cfg = config.bagel.monitoring.exporters.cadvisor;
-  inherit (lib) mkEnableOption mkIf;
-in
-{
-  options.bagel.monitoring.exporters.cadvisor.enable = (mkEnableOption "Standard cAdvisor") // { default = !config.boot.isContainer; };
-
-  config = mkIf cfg.enable {
-    services.cadvisor = {
-      enable = true;
-      port = 9102;
-      listenAddress = "0.0.0.0";
-    };
-
-    bagel.meta.monitoring.exporters = [ { port = 9102; } ];
-  };
-}

services/monitoring/exporters/default.nix

@@ -1,37 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  inherit (lib) mkOption types;
-in
-{
-  imports = [
-    ./cadvisor.nix
-    ./node.nix
-    ./nginx.nix
-    ./postgres.nix
-  ];
-
-  options.bagel = {
-    meta.monitoring = {
-      address = mkOption {
-        description = "Node's public address";
-        type = types.str;
-      };
-      exporters = mkOption {
-        description = "List of all exporters to scrape";
-        type = types.listOf (types.submodule {
-          options.port = mkOption {
-            description = "Exporter port";
-            type = types.int;
-          };
-        });
-        default = [];
-      };
-    };
-  };
-
-  config.networking.firewall.allowedTCPPorts = map (e: e.port) config.bagel.meta.monitoring.exporters;
-}

services/monitoring/exporters/nginx.nix

@@ -1,37 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  cfg = config.bagel.monitoring.exporters.nginx;
-  inherit (lib) mkEnableOption mkIf;
-  logFormat = ''$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'';
-in
-{
-  options.bagel.monitoring.exporters.nginx.enable = (mkEnableOption "Nginx access.log exporter") // { default = config.services.nginx.enable; };
-
-  config = mkIf cfg.enable {
-    services.nginx.appendHttpConfig = ''
-      log_format ours '${logFormat}';
-      access_log /var/log/nginx/access.log ours;
-    '';
-    
-    services.prometheus.exporters.nginxlog = {
-      enable = true;
-      port = 9103;
-      group = "nginx";
-      settings.namespaces = [
-        {
-          name = "nginx";
-          format = logFormat;
-          source.files = ["/var/log/nginx/access.log"];
-        }
-      ];
-    };
-
-    bagel.meta.monitoring.exporters = [
-      { port = 9103; }
-    ];
-  };
-}

services/monitoring/exporters/node.nix

@@ -1,25 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  cfg = config.bagel.monitoring.exporters.node;
-  inherit (lib) mkEnableOption mkIf;
-in
-{
-  options.bagel.monitoring.exporters.node.enable = (mkEnableOption "Standard node_exporter") // { default = true; };
-
-  config = mkIf cfg.enable {
-    services.prometheus.exporters.node = {
-      enable = true;
-      enabledCollectors = [
-        "processes"
-        "systemd"
-      ];
-      port = 9101;
-    };
-
-    bagel.meta.monitoring.exporters = [ { port = 9101; } ];
-  };
-}

services/monitoring/exporters/postgres.nix

@@ -1,31 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  cfg = config.bagel.monitoring.exporters.postgres;
-  inherit (lib) mkEnableOption mkIf;
-in
-{
-  options.bagel.monitoring.exporters.postgres.enable = (mkEnableOption "Postgres exporter") // { default = config.services.postgresql.enable; };
-
-  config = mkIf cfg.enable {
-    services.prometheus.exporters.postgres = {
-      enable = true;
-      port = 9104;
-      runAsLocalSuperUser = true;
-      extraFlags = [
-        "--collector.long_running_transactions"
-        "--collector.stat_activity_autovacuum"
-        "--collector.stat_statements"
-      ];
-    };
-
-    services.postgresql.settings.shared_preload_libraries = "pg_stat_statements";
-
-    bagel.meta.monitoring.exporters = [
-      { port = 9104; }
-    ];
-  };
-}

services/monitoring/lgtm/dashboards/nginx.json

@@ -1,697 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "datasource",
-          "uid": "grafana"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "target": {
-          "limit": 100,
-          "matchAny": false,
-          "tags": [],
-          "type": "dashboard"
-        },
-        "type": "dashboard"
-      }
-    ]
-  },
-  "description": "NGINX Log metrics with Prometheus based on https://github.com/martin-helmich/prometheus-nginxlog-exporter \r\nBased on namespace prefix 'nginx'. If different, you may need to adjust the metrics.\r\nDashboard based on 6482 dashboard",
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "gnetId": 15947,
-  "graphTooltip": 0,
-  "id": 4,
-  "links": [],
-  "liveNow": false,
-  "panels": [
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "mimir"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "continuous-GrYlRd"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "percentage",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 5
-              }
-            ]
-          },
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 11,
-        "x": 0,
-        "y": 0
-      },
-      "id": 12,
-      "options": {
-        "displayMode": "lcd",
-        "maxVizHeight": 300,
-        "minVizHeight": 16,
-        "minVizWidth": 8,
-        "namePlacement": "auto",
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "showUnfilled": true,
-        "sizing": "auto",
-        "text": {},
-        "valueMode": "color"
-      },
-      "pluginVersion": "11.0.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "mimir"
-          },
-          "editorMode": "code",
-          "exemplar": true,
-          "expr": "sum(rate(nginx_http_response_count_total{status=~\"^2..\",instance=\"$host\"}[1m])) / sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) * 100",
-          "hide": false,
-          "interval": "",
-          "legendFormat": "2** status codes",
-          "range": true,
-          "refId": "C"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "mimir"
-          },
-          "exemplar": true,
-          "expr": "sum(rate(nginx_http_response_count_total{status=~\"^4..\",instance=\"$host\"}[1m])) / sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) * 100",
-          "interval": "",
-          "legendFormat": "4** status codes",
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "mimir"
-          },
-          "exemplar": true,
-          "expr": "sum(rate(nginx_http_response_count_total{status=~\"^5..\",instance=\"$host\"}[1m])) / sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) * 100",
-          "hide": false,
-          "interval": "",
-          "legendFormat": "5** status codes",
-          "refId": "B"
-        }
-      ],
-      "title": "Percentage Ratio of status codes to all status codes",
-      "transparent": true,
-      "type": "bargauge"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "mimir"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 4,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "short"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 11,
-        "x": 11,
-        "y": 0
-      },
-      "id": 4,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "8.5.0-54880pre",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "mimir"
-          },
-          "exemplar": true,
-          "expr": "sum(rate(nginx_http_response_time_seconds_count{instance=\"$host\"}[1m])) ",
-          "format": "time_series",
-          "interval": "",
-          "intervalFactor": 1,
-          "legendFormat": "$host",
-          "refId": "A"
-        }
-      ],
-      "title": "Requests per Second",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "mimir"
-      },
-      "description": "Response sizes in bytes",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 4,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "short"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 11,
-        "x": 11,
-        "y": 7
-      },
-      "id": 8,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "8.5.0-54880pre",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "mimir"
-          },
-          "exemplar": true,
-          "expr": "sum(rate(nginx_http_response_size_bytes{instance=\"$host\"}[5m])) ",
-          "format": "time_series",
-          "interval": "",
-          "intervalFactor": 1,
-          "legendFormat": "$host",
-          "refId": "A"
-        }
-      ],
-      "title": "HTTP Traffic",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "mimir"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 21,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 4,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 11,
-        "x": 0,
-        "y": 8
-      },
-      "id": 2,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "8.5.0-54880pre",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "mimir"
-          },
-          "exemplar": true,
-          "expr": "sum(rate(nginx_http_response_time_seconds_sum{instance=\"$host\"}[5m])) / sum(rate(nginx_http_response_time_seconds_count{instance=\"$host\"}[5m])) ",
-          "format": "time_series",
-          "interval": "",
-          "intervalFactor": 1,
-          "legendFormat": "$host",
-          "refId": "A"
-        }
-      ],
-      "title": "Average Response Time [5m]",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "mimir"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 3,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "short"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 11,
-        "x": 11,
-        "y": 14
-      },
-      "id": 10,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "8.5.0-54880pre",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "mimir"
-          },
-          "exemplar": true,
-          "expr": "sum(rate(nginx_http_response_count_total{instance=\"$host\"}[1m])) by (status)",
-          "format": "time_series",
-          "interval": "",
-          "intervalFactor": 1,
-          "legendFormat": "",
-          "refId": "A"
-        }
-      ],
-      "title": "Status codes per second",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "mimir"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 4,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "short"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 11,
-        "x": 0,
-        "y": 15
-      },
-      "id": 6,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "maxHeight": 600,
-          "mode": "multi",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "8.5.0-54880pre",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "mimir"
-          },
-          "exemplar": true,
-          "expr": "nginx_http_response_time_seconds{quantile=\"0.9\",method=\"GET\",status=~\"2[0-9]*\",instance=~\"$host\"}",
-          "format": "time_series",
-          "interval": "",
-          "intervalFactor": 1,
-          "legendFormat": "$host",
-          "refId": "A"
-        }
-      ],
-      "title": "Response time (90% quantile)",
-      "type": "timeseries"
-    }
-  ],
-  "refresh": "5s",
-  "schemaVersion": 39,
-  "tags": [],
-  "templating": {
-    "list": [
-      {
-        "current": {
-          "isNone": true,
-          "selected": false,
-          "text": "None",
-          "value": ""
-        },
-        "datasource": {
-          "type": "prometheus",
-          "uid": "mimir"
-        },
-        "definition": "label_values(nginx_http_response_count_total,instance)",
-        "hide": 0,
-        "includeAll": false,
-        "label": "Host:",
-        "multi": false,
-        "name": "host",
-        "options": [],
-        "query": {
-          "query": "label_values(nginx_http_response_count_total,instance)",
-          "refId": "StandardVariableQuery"
-        },
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 1,
-        "type": "query"
-      }
-    ]
-  },
-  "time": {
-    "from": "now-15m",
-    "to": "now"
-  },
-  "timeRangeUpdatedDuringEditOrView": false,
-  "timepicker": {
-    "refresh_intervals": [
-      "5s",
-      "10s",
-      "30s",
-      "1m",
-      "5m",
-      "15m",
-      "30m",
-      "1h",
-      "2h",
-      "1d"
-    ],
-    "time_options": [
-      "5m",
-      "15m",
-      "1h",
-      "6h",
-      "12h",
-      "24h",
-      "2d",
-      "7d",
-      "30d"
-    ]
-  },
-  "timezone": "",
-  "title": "NGINX Log Metrics [M]",
-  "uid": "JfOTY2Pnk",
-  "version": 4,
-  "weekStart": ""
-}

services/monitoring/lgtm/default.nix

@@ -1,7 +0,0 @@
-{
-  imports = [
-    ./grafana.nix
-    ./loki.nix
-    ./prometheus.nix
-  ];
-}

services/monitoring/lgtm/grafana.nix

@@ -1,133 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  cfg = config.bagel.services.grafana;
-  inherit (lib) mkEnableOption mkIf;
-in
-{
-  options.bagel.services.grafana.enable = mkEnableOption "Grafana frontend";
-
-  config = mkIf cfg.enable {
-    age.secrets.grafana-oauth-secret = {
-      file = ../../../secrets/grafana-oauth-secret.age;
-      owner = "grafana";
-    };
-
-    bagel.services.postgres.enable = true;
-
-    services = {
-      grafana = {
-        enable = true;
-
-        settings = {
-          server = {
-            domain = "grafana.forkos.org";
-            http_addr = "127.0.0.1";
-            http_port = 2342;
-            root_url = "https://grafana.forkos.org/";
-          };
-
-          database = {
-            type = "postgres";
-            user = "grafana";
-            host = "/run/postgresql";
-          };
-
-          "auth.generic_oauth" = {
-            enabled = true;
-
-            name = "Lix SSO";
-            client_id = "forkos-grafana";
-            client_secret = "$__file{${config.age.secrets.grafana-oauth-secret.path}}";
-
-            auth_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth";
-            token_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
-            api_url = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/userinfo";
-
-            login_attribute_path = "username";
-            email_attribute_path = "email";
-            name_attribute_path = "full_name";
-
-            scopes = [
-              "openid"
-              "profile"
-              "email"
-              "offline_access"
-              "roles"
-            ];
-
-            allow_sign_up = true;
-            auto_login = true;
-            allow_assign_grafana_admin = true;
-
-            role_attribute_path = "contains(grafana_roles[*], 'Admin') && 'GrafanaAdmin' || contains(grafana_roles[*], 'Editor') && 'Editor' || 'Viewer'";
-          };
-
-          dashboards.default_home_dashboard_path = "${./dashboards/node_exporter.json}";
-
-          feature_toggles.enable = "autoMigrateOldPanels newVizTooltips";
-          security.angular_support_enabled = false;
-        };
-
-        provision = {
-          dashboards.settings = {
-            apiVersion = 1;
-            providers = [
-              {
-                name = "default";
-                options.path = ./dashboards;
-              }
-            ];
-          };
-
-          datasources.settings = {
-            apiVersion = 1;
-            datasources = [
-              {
-                name = "Mimir";
-                type = "prometheus";
-                uid = "mimir";
-                access = "proxy";
-                url = "http://127.0.0.1:9009/prometheus";
-              }
-              {
-                name = "Loki";
-                type = "loki";
-                uid = "loki";
-                access = "proxy";
-                url = "http://127.0.0.1:9090/";
-              }
-            ];
-          };
-        };
-      };
-
-      postgresql = {
-        ensureDatabases = [ "grafana" ];
-        ensureUsers = [
-          {
-            name = "grafana";
-            ensureDBOwnership = true;
-          }
-        ];
-      };
-
-      nginx = let
-        scfg = config.services.grafana.settings.server;
-      in {
-        enable = true;
-        virtualHosts."${scfg.domain}" = {
-          enableACME = true;
-          forceSSL = true;
-          locations."/" = {
-            proxyPass = "http://${scfg.http_addr}:${toString scfg.http_port}";
-            proxyWebsockets = true;
-          };
-        };
-      };
-    };
-};
-}

services/monitoring/lgtm/loki.nix

@@ -1,100 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  cfg = config.bagel.services.loki;
-  inherit (lib) mkEnableOption mkIf;
-in
-{
-  options.bagel.services.loki.enable = mkEnableOption "Loki storage";
-
-  config = mkIf cfg.enable {
-    age.secrets = {
-      loki-htpasswd = {
-        file = ../../../secrets/loki-htpasswd.age;
-        owner = "nginx";
-      };
-      loki-environment.file = ../../../secrets/loki-environment.age;
-    };
-
-    services.loki = {
-      enable = true;
-      extraFlags = ["--config.expand-env"];
-
-      configuration = {
-        server = {
-          http_listen_port = 9090;
-          grpc_listen_port = 9096;
-
-          # 16M
-          grpc_server_max_recv_msg_size = 16777216;
-          grpc_server_max_send_msg_size = 16777216;
-        };
-
-        auth_enabled = false;
-
-        common = {
-          storage.s3 = {
-            endpoint = "s3.delroth.net";
-            region = "garage";
-            bucketnames = "bagel-loki";
-            secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
-            access_key_id = "\${S3_KEY_ID}";
-            s3forcepathstyle = true;
-          };
-          ring.kvstore.store = "memberlist";
-          replication_factor = 1;
-        };
-
-        memberlist = {
-          bind_port = 7947;
-          advertise_port = 7947;
-        };
-
-        storage_config.tsdb_shipper = {
-          active_index_directory = "/var/lib/loki/index";
-          cache_location = "/var/lib/loki/cache";
-        };
-
-        compactor = {
-          working_directory = "/var/lib/loki/compactor";
-          compaction_interval = "10m";
-          retention_enabled = true;
-          retention_delete_delay = "1s";
-          retention_delete_worker_count = 150;
-          delete_request_store = "filesystem";
-        };
-
-        limits_config.retention_period = "1w";
-
-        schema_config = {
-          configs = [
-            {
-              from = "2024-07-01";
-              store = "tsdb";
-              object_store = "s3";
-              schema = "v13";
-              index = {
-                prefix = "index_";
-                period = "24h";
-              };
-            }
-          ];
-        };
-      };
-    };
-
-    systemd.services.loki.serviceConfig.EnvironmentFile = [ config.age.secrets.loki-environment.path ];
-
-    services.nginx.virtualHosts."loki.forkos.org" = {
-      enableACME = true;
-      forceSSL = true;
-      locations."/loki/api/v1/push" = {
-        proxyPass = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}";
-        basicAuthFile = config.age.secrets.loki-htpasswd.path;
-      };
-    };
-  };
-}

services/monitoring/lgtm/prometheus.nix

@@ -1,83 +0,0 @@
-{
-  config,
-  lib,
-  nodes,
-  ...
-}:
-let
-  cfg = config.bagel.services.prometheus;
-  inherit (lib) mkEnableOption mkIf;
-
-  forEachMachine = fn: map fn (builtins.attrValues nodes);
-
-  allMetas = forEachMachine (machine: {
-    name = machine.config.networking.hostName;
-    address = machine.config.bagel.meta.monitoring.address or null;
-    exporters = machine.config.bagel.meta.monitoring.exporters or [];
-  });
-
-  scrapableMetas = builtins.filter (m: m.address != null && m.exporters != []) allMetas;
-
-  toJobConfig = m: {
-    job_name = m.name;
-    static_configs = [
-      { targets = map (e: m.address + ":" + (toString e.port)) m.exporters; }
-    ];
-  };
-
-  jobConfigs = map toJobConfig scrapableMetas;
-in
-{
-  options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
-
-  config = mkIf cfg.enable {
-    age.secrets.mimir-environment.file = ../../../secrets/mimir-environment.age;
-
-    services.prometheus = {
-      enable = true;
-      enableAgentMode = true;
-      listenAddress = "127.0.0.1";
-      port = 9001;
-      globalConfig.scrape_interval = "15s";
-      scrapeConfigs = jobConfigs;
-      remoteWrite = [
-        { url = "http://localhost:9009/api/v1/push"; }
-      ];
-    };
-
-    services.mimir = {
-      enable = true;
-      extraFlags = ["--config.expand-env=true"];
-      configuration = {
-        multitenancy_enabled = false;
-
-        common.storage = {
-          backend = "s3";
-          s3 = {
-            endpoint = "s3.delroth.net";
-            bucket_name = "bagel-mimir";
-            secret_access_key = "\${S3_KEY}"; # This is a secret injected via an environment variable
-            access_key_id = "\${S3_KEY_ID}";
-          };
-        };
-
-        server = {
-          http_listen_port = 9009;
-          grpc_server_max_recv_msg_size = 104857600;
-          grpc_server_max_send_msg_size = 104857600;
-          grpc_server_max_concurrent_streams = 1000;
-        };
-
-        ingester.ring.replication_factor = 1;
-
-        blocks_storage.backend = "s3";
-        ruler_storage = {
-          backend = "local";
-          local.directory = ./alerts;
-        };
-      };
-    };
-
-    systemd.services.mimir.serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
-  };
-}

services/monitoring/promtail.nix

@@ -1,53 +0,0 @@
-{
-  config,
-  lib,
-  ...
-}:
-let
-  cfg = config.bagel.monitoring.promtail;
-  inherit (lib) mkEnableOption mkIf;
-in
-{
-  options.bagel.monitoring.promtail.enable = (mkEnableOption "Promtail log export") // { default = true; };
-
-  config = mkIf cfg.enable {
-    age.secrets.promtail-password = {
-      file = ../../secrets/promtail-password.age;
-      owner = "promtail";
-    };
-
-    services.promtail = {
-      enable = true;
-      configuration = {
-        server.disable = true;
-        clients = [ 
-          {
-            url = "https://loki.forkos.org/loki/api/v1/push";
-            basic_auth = {
-              username = "promtail";
-              password_file = config.age.secrets.promtail-password.path;
-            };
-          }
-        ];
-        scrape_configs = [
-          {
-            job_name = "system";
-            journal = {
-              max_age = "12h";
-              labels = {
-                job = "systemd-journal";
-                host = config.networking.hostName;
-              };
-            };
-            relabel_configs = [
-              {
-                source_labels = [ "__journal__systemd_unit" ];
-                target_label = "unit";
-              }
-            ];
-          }
-        ];
-      };
-    };
-  };
-}