ofborg: enable nginx for certs

Disable nixos-option, it breaks
Update everything again
2024-11-17 14:48:08 +03:00 · 2024-11-17 14:39:25 +03:00 · 2024-11-17 14:39:19 +03:00 · 2024-11-12 23:42:51 +03:00 · 2024-11-12 23:21:30 +03:00 · 2024-11-12 23:16:46 +03:00
36 changed files with 1132 additions and 735 deletions
--- a/.envrc
+++ b/.envrc
@ -1,2 +1,11 @@
+#!/usr/bin/env bash
+# the shebang is ignored, but nice for editors
+
 # shellcheck shell=bash
-use flake
+if type -P lorri &>/dev/null; then
+  eval "$(lorri direnv --flake .)"
+else
+  echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]'
+  use flake
+fi
+
--- a/common/admins.nix
+++ b/common/admins.nix
@ -2,7 +2,7 @@
 let
  inherit (lib) genAttrs;
 in
-# Note: to add somefew in this list. 
+# Note: to add somefew in this list.
 # Ensure their SSH key is already in common/ssh-keys.nix with
 # the same username for here, so that the keys is automatically added.
 {
@ -19,12 +19,14 @@ in
      "thubrecht"
      "winter"
      "yuka"
+      "ckie"
    ];

    lix-infra.members = [
      "raito"
      "hexchen"
      "jade"
+      "pennae"
    ];
  };
  bagel.users = genAttrs [
@ -39,5 +41,7 @@ in
    "thubrecht"
    "winter"
    "yuka"
+    "ckie"
+    "pennae"
  ] (name: {});
 }
--- a/common/base-server.nix
+++ b/common/base-server.nix
@ -7,6 +7,7 @@
  nixpkgs.overlays = import ../overlays;

  nix.package = lib.mkDefault pkgs.lix;
+  system.tools.nixos-option.enable = false;
  services.openssh.enable = lib.mkForce true;

  networking.nftables.enable = true;
--- a/common/ssh-keys.nix
+++ b/common/ssh-keys.nix
@ -61,7 +61,12 @@
      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
    ];
    thubrecht = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPM1jpXR7BWQa7Sed7ii3SbvIPRRlKb3G91qC0vOwfJn" ];
-    yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKath4/fDnlv/4fzxkPrQN1ttmoPRNu/m9bEtdPJBDfY cardno:16_933_242" ];
+    yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIxQ3NYBi8v1f/vhxLKDcA6upmX0pctRDbnK6SER5OUR yureka" ];
    winter = [ "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIH/LDRUG+U+++UmlxvA2kspioTjktQZ8taDcHq8gVlkfAAAABHNzaDo=" ];
+    ckie = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH3uTwzSSMAPg84fwbNp2cq9+BdLFeA1VzDGth4zCAbz https://mei.puppycat.house" ];
+    pennae = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC5Wf5/IbyFpdziWfwxkQqxOf3r1L9pYn6xQBEKFwmMY"
+      "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIK8icXjHkb4XzbIVN3djH4CE7RvgGd+3xbG4cgh0Yls5AAAABHNzaDo="
+    ];
  };
 }
--- a/common/zsh.nix
+++ b/common/zsh.nix
@ -1,5 +1,4 @@
 { lib, pkgs, config, ... }: {
-  users.defaultUserShell = pkgs.zsh;
  programs.zsh = {
    enable = true;
    enableCompletion = true;
--- a/flake.lock
+++ b/flake.lock
@ -27,16 +27,17 @@
      "inputs": {
        "crane": "crane",
        "flake-compat": "flake-compat_2",
-        "flake-utils": "flake-utils_2",
+        "flake-parts": "flake-parts_2",
+        "nix-github-actions": "nix-github-actions_2",
        "nixpkgs": "nixpkgs",
        "nixpkgs-stable": "nixpkgs-stable"
      },
      "locked": {
-        "lastModified": 1711742460,
-        "narHash": "sha256-0O4v6e4a1toxXZ2gf5INhg4WPE5C5T+SVvsBt+45Mcc=",
+        "lastModified": 1730257295,
+        "narHash": "sha256-OQl+aAsKiyygvpzck1u0sZf/R4T9zM903CgNDFmmzA8=",
        "owner": "zhaofengli",
        "repo": "attic",
-        "rev": "4dbdbee45728d8ce5788db6461aaaa89d98081f0",
+        "rev": "48c8b395bfbc6b76c7eae74df6c74351255a095c",
        "type": "github"
      },
      "original": {
@ -49,11 +50,11 @@
    "bats-assert": {
      "flake": false,
      "locked": {
-        "lastModified": 1636059754,
-        "narHash": "sha256-ewME0l27ZqfmAwJO4h5biTALc9bDLv7Bl3ftBzBuZwk=",
+        "lastModified": 1692829535,
+        "narHash": "sha256-oDqhUQ6Xg7a3xx537SWLGRzqP3oKKeyY4UYGCdz9z/Y=",
        "owner": "bats-core",
        "repo": "bats-assert",
-        "rev": "34551b1d7f8c7b677c1a66fc0ac140d6223409e5",
+        "rev": "e2d855bc78619ee15b0c702b5c30fb074101159f",
        "type": "github"
      },
      "original": {
@ -65,11 +66,11 @@
    "bats-support": {
      "flake": false,
      "locked": {
-        "lastModified": 1548869839,
-        "narHash": "sha256-Gr4ntadr42F2Ks8Pte2D4wNDbijhujuoJi4OPZnTAZU=",
+        "lastModified": 1693050811,
+        "narHash": "sha256-PxJaH16+QrsfZqtkWVt5K6TwJB5gjIXnbGo+MB84WIU=",
        "owner": "bats-core",
        "repo": "bats-support",
-        "rev": "d140a65044b2d6810381935ae7f0c94c7023c8c3",
+        "rev": "9bf10e876dd6b624fe44423f0b35e064225f7556",
        "type": "github"
      },
      "original": {
@ -87,11 +88,11 @@
        "treefmt-nix": "treefmt-nix"
      },
      "locked": {
-        "lastModified": 1728162115,
-        "narHash": "sha256-O95Nv81XlYZjltq009951gwYHGo+Kjt9EBhh/6cYGDw=",
+        "lastModified": 1730064416,
+        "narHash": "sha256-Opbtu9hKijGkEx+GYbSu3MJms3lFxZmAGTFyckguWMM=",
        "ref": "refs/heads/forkos",
-        "rev": "ed8f940717724af853efcafcd82cf4d10f32ede5",
-        "revCount": 298,
+        "rev": "79137b14f3cb376204f739f44b05aebfc288ca89",
+        "revCount": 310,
        "type": "git",
        "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
      },
@ -125,17 +126,18 @@
      "inputs": {
        "flake-compat": "flake-compat",
        "flake-utils": "flake-utils",
+        "nix-github-actions": "nix-github-actions",
        "nixpkgs": [
          "nixpkgs"
        ],
        "stable": "stable"
      },
      "locked": {
-        "lastModified": 1711386353,
-        "narHash": "sha256-gWEpb8Hybnoqb4O4tmpohGZk6+aerAbJpywKcFIiMlg=",
+        "lastModified": 1731527002,
+        "narHash": "sha256-dI9I6suECoIAmbS4xcrqF8r2pbmed8WWm5LIF1yWPw8=",
        "owner": "zhaofengli",
        "repo": "colmena",
-        "rev": "cd65ef7a25cdc75052fbd04b120aeb066c3881db",
+        "rev": "e3ad42138015fcdf2524518dd564a13145c72ea1",
        "type": "github"
      },
      "original": {
@ -153,11 +155,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1702918879,
-        "narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=",
+        "lastModified": 1722960479,
+        "narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=",
        "owner": "ipetkov",
        "repo": "crane",
-        "rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb",
+        "rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4",
        "type": "github"
      },
      "original": {
@ -167,18 +169,12 @@
      }
    },
    "crane_2": {
-      "inputs": {
-        "nixpkgs": [
-          "grapevine",
-          "nixpkgs"
-        ]
-      },
      "locked": {
-        "lastModified": 1716569590,
-        "narHash": "sha256-5eDbq8TuXFGGO3mqJFzhUbt5zHVTf5zilQoyW5jnJwo=",
+        "lastModified": 1730060262,
+        "narHash": "sha256-RMgSVkZ9H03sxC+Vh4jxtLTCzSjPq18UWpiM0gq6shQ=",
        "owner": "ipetkov",
        "repo": "crane",
-        "rev": "109987da061a1bf452f435f1653c47511587d919",
+        "rev": "498d9f122c413ee1154e8131ace5a35a80d8fa76",
        "type": "github"
      },
      "original": {
@ -219,11 +215,11 @@
        "rust-analyzer-src": "rust-analyzer-src"
      },
      "locked": {
-        "lastModified": 1716359173,
-        "narHash": "sha256-pYcjP6Gy7i6jPWrjiWAVV0BCQp+DdmGaI/k65lBb/kM=",
+        "lastModified": 1730442928,
+        "narHash": "sha256-U1DWb5c3EfkA7pqx5V1H4AWRA+EaE6UJ0lIRvK1RxgM=",
        "owner": "nix-community",
        "repo": "fenix",
-        "rev": "b6fc5035b28e36a98370d0eac44f4ef3fd323df6",
+        "rev": "87b4d20f896c99018dde4702a9c6157b516f2a76",
        "type": "github"
      },
      "original": {
@ -252,11 +248,11 @@
    "flake-compat_2": {
      "flake": false,
      "locked": {
-        "lastModified": 1673956053,
-        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
        "owner": "edolstra",
        "repo": "flake-compat",
-        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
        "type": "github"
      },
      "original": {
@ -320,6 +316,28 @@
      }
    },
    "flake-parts_2": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "grapevine",
+          "attic",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1722555600,
+        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-parts_3": {
      "inputs": {
        "nixpkgs-lib": [
          "hydra",
@ -341,6 +359,23 @@
        "type": "github"
      }
    },
+    "flake-parts_4": {
+      "inputs": {
+        "nixpkgs-lib": "nixpkgs-lib"
+      },
+      "locked": {
+        "lastModified": 1727826117,
+        "narHash": "sha256-K5ZLCyfO/Zj9mPFldf3iwS6oZStJcU4tSpiXTMYaaL0=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "3d04084d54bedc3d6b8b736c70ef449225c361b1",
+        "type": "github"
+      },
+      "original": {
+        "id": "flake-parts",
+        "type": "indirect"
+      }
+    },
    "flake-utils": {
      "locked": {
        "lastModified": 1659877975,
@ -357,30 +392,15 @@
      }
    },
    "flake-utils_2": {
-      "locked": {
-        "lastModified": 1667395993,
-        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "flake-utils_3": {
      "inputs": {
        "systems": "systems_2"
      },
      "locked": {
-        "lastModified": 1710146030,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "lastModified": 1726560853,
+        "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
        "owner": "numtide",
        "repo": "flake-utils",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
        "type": "github"
      },
      "original": {
@ -390,21 +410,6 @@
        "type": "github"
      }
    },
-    "flake-utils_4": {
-      "locked": {
-        "lastModified": 1634851050,
-        "narHash": "sha256-N83GlSGPJJdcqhUxSCS/WwW5pksYf3VP1M13cDRTSVA=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "c91f3de5adaf1de973b797ef7485e441a65b8935",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
    "gerrit-dashboard": {
      "flake": false,
      "locked": {
@ -427,20 +432,21 @@
        "crane": "crane_2",
        "fenix": "fenix",
        "flake-compat": "flake-compat_3",
-        "flake-utils": "flake-utils_3",
+        "flake-utils": "flake-utils_2",
        "nix-filter": "nix-filter",
        "nixpkgs": [
          "nixpkgs"
        ],
+        "rocksdb": "rocksdb",
        "rust-manifest": "rust-manifest"
      },
      "locked": {
        "host": "gitlab.computer.surgery",
-        "lastModified": 1727994504,
-        "narHash": "sha256-FC6M1KKX58HbU9LG+cG6EJRr02J9lE/o0iiDi6m1gv8=",
+        "lastModified": 1731820516,
+        "narHash": "sha256-BJiFq5meQMwOLhVS9/m2HCZoNUIN5u5weJXr94dkIRU=",
        "owner": "matrix",
        "repo": "grapevine-fork",
-        "rev": "5a490a4397f0c6a36dab1cb631dadc67a849deab",
+        "rev": "887e59cf03efb70bfa68b56dda32c411ba6cd5fa",
        "type": "gitlab"
      },
      "original": {
@ -480,11 +486,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1728163965,
-        "narHash": "sha256-vHA2qZWoTY4DQUWFktq8DYfCOlm8rd1bNZpiIzpFBCk=",
+        "lastModified": 1728321752,
+        "narHash": "sha256-GbBAoBF7ZObz0IP+g0LZKxMafpMvNKjTEu9haiZbV54=",
        "ref": "refs/heads/main",
-        "rev": "e4d466ffcdc0e43f3bd005de3913a7e9ed4a58f5",
-        "revCount": 4216,
+        "rev": "ee1234c15cdcb427dbd4828e0add09d02cd606c9",
+        "revCount": 4220,
        "type": "git",
        "url": "https://git.lix.systems/lix-project/hydra.git"
      },
@ -505,11 +511,11 @@
        "pre-commit-hooks": "pre-commit-hooks"
      },
      "locked": {
-        "lastModified": 1728159440,
-        "narHash": "sha256-8DW+VfSIk5xYpYj1VDKMmUjXhMooIlAOwYG4J9PLbbg=",
+        "lastModified": 1728163191,
+        "narHash": "sha256-SW0IEBsPN1EysqzvfDT+8Kimtzy03O1BxQQm7ZB6fRY=",
        "ref": "refs/heads/main",
-        "rev": "7ff60b7445ccb599edcbf0078d181034f61a0859",
-        "revCount": 16318,
+        "rev": "ed9b7f4f84fd60ad8618645cc1bae2d686ff0db6",
+        "revCount": 16323,
        "type": "git",
        "url": "https://git.lix.systems/lix-project/lix"
      },
@ -520,12 +526,12 @@
    },
    "nix-eval-jobs": {
      "inputs": {
-        "flake-parts": "flake-parts_2",
+        "flake-parts": "flake-parts_3",
        "lix": [
          "hydra",
          "lix"
        ],
-        "nix-github-actions": "nix-github-actions",
+        "nix-github-actions": "nix-github-actions_3",
        "nixpkgs": [
          "hydra",
          "nixpkgs"
@ -548,11 +554,11 @@
    },
    "nix-filter": {
      "locked": {
-        "lastModified": 1710156097,
-        "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
+        "lastModified": 1730207686,
+        "narHash": "sha256-SCHiL+1f7q9TAnxpasriP6fMarWE5H43t25F5/9e28I=",
        "owner": "numtide",
        "repo": "nix-filter",
-        "rev": "3342559a24e85fc164b295c3444e8a139924675b",
+        "rev": "776e68c1d014c3adde193a18db9d738458cd2ba4",
        "type": "github"
      },
      "original": {
@ -583,6 +589,49 @@
      }
    },
    "nix-github-actions": {
+      "inputs": {
+        "nixpkgs": [
+          "colmena",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1729742964,
+        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
+    "nix-github-actions_2": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "attic",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1729742964,
+        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
+    "nix-github-actions_3": {
      "inputs": {
        "nixpkgs": [
          "hydra",
@ -622,11 +671,11 @@
    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1711401922,
-        "narHash": "sha256-QoQqXoj8ClGo0sqD/qWKFWezgEwUL0SUh37/vY2jNhc=",
+        "lastModified": 1726042813,
+        "narHash": "sha256-LnNKCCxnwgF+575y0pxUdlGZBO/ru1CtGHIqQVfvjlA=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "07262b18b97000d16a4bdb003418bd2fb067a932",
+        "rev": "159be5db480d1df880a0135ca0bfed84c2f88353",
        "type": "github"
      },
      "original": {
@ -636,6 +685,18 @@
        "type": "github"
      }
    },
+    "nixpkgs-lib": {
+      "locked": {
+        "lastModified": 1727825735,
+        "narHash": "sha256-0xHYkMkeLVQAMa7gvkddbPqpxph+hDzdu1XdGPJR+Os=",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
+      }
+    },
    "nixpkgs-regression": {
      "locked": {
        "lastModified": 1643052045,
@ -654,32 +715,32 @@
    },
    "nixpkgs-stable": {
      "locked": {
-        "lastModified": 1711460390,
-        "narHash": "sha256-akSgjDZL6pVHEfSE6sz1DNSXuYX6hq+P/1Z5IoYWs7E=",
+        "lastModified": 1724316499,
+        "narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "44733514b72e732bd49f5511bd0203dea9b9a434",
+        "rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
-        "ref": "nixos-23.11",
+        "ref": "nixos-24.05",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
    "nixpkgs_2": {
      "locked": {
-        "lastModified": 1728093190,
-        "narHash": "sha256-CAZF2NRuHmqTtRTNAruWpHA43Gg2UvuCNEIzabP0l6M=",
+        "lastModified": 1731676054,
+        "narHash": "sha256-OZiZ3m8SCMfh3B6bfGC/Bm4x3qc1m2SVEAlkV6iY7Yg=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "e2f08f4d8b3ecb5cf5c9fd9cb2d53bb3c71807da",
+        "rev": "5e4fbfb6b3de1aa2872b76d49fafc942626e2add",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
+        "ref": "nixos-unstable",
        "repo": "nixpkgs",
        "type": "github"
      }
@ -700,6 +761,23 @@
        "type": "github"
      }
    },
+    "rocksdb": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1730475155,
+        "narHash": "sha256-u5uuShM2SxHc9/zL4UU56IhCcR/ZQbzde0LgOYS44bM=",
+        "owner": "facebook",
+        "repo": "rocksdb",
+        "rev": "3c27a3dde0993210c5cc30d99717093f7537916f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "facebook",
+        "ref": "v9.7.4",
+        "repo": "rocksdb",
+        "type": "github"
+      }
+    },
    "root": {
      "inputs": {
        "agenix": "agenix",
@ -722,11 +800,11 @@
    "rust-analyzer-src": {
      "flake": false,
      "locked": {
-        "lastModified": 1716107283,
-        "narHash": "sha256-NJgrwLiLGHDrCia5AeIvZUHUY7xYGVryee0/9D3Ir1I=",
+        "lastModified": 1730386175,
+        "narHash": "sha256-0Uq+/B8eu7pw8B8pxuGdFYKjcVLwNMcHfDxU9sXh7rg=",
        "owner": "rust-lang",
        "repo": "rust-analyzer",
-        "rev": "21ec8f523812b88418b2bfc64240c62b3dd967bd",
+        "rev": "0ba893e1a00d92557ac91efb771d72eee36ca687",
        "type": "github"
      },
      "original": {
@ -750,16 +828,16 @@
    },
    "stable": {
      "locked": {
-        "lastModified": 1696039360,
-        "narHash": "sha256-g7nIUV4uq1TOVeVIDEZLb005suTWCUjSY0zYOlSBsyE=",
+        "lastModified": 1730883749,
+        "narHash": "sha256-mwrFF0vElHJP8X3pFCByJR365Q2463ATp2qGIrDUdlE=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "32dcb45f66c0487e92db8303a798ebc548cadedc",
+        "rev": "dba414932936fde69f0606b4f1d87c5bc0003ede",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
-        "ref": "nixos-23.05",
+        "ref": "nixos-24.05",
        "repo": "nixpkgs",
        "type": "github"
      }
@ -767,11 +845,11 @@
    "stateless-uptime-kuma": {
      "flake": false,
      "locked": {
-        "lastModified": 1727622797,
-        "narHash": "sha256-JRvy5zsPsk1ChmEB1UHKOlnzAKCPdtBLCcx+LXh3e4c=",
+        "lastModified": 1728243069,
+        "narHash": "sha256-l9fgwesnmFxasCaYUCD7L9bGGJXytLuwtx3CZMgpwJg=",
        "ref": "refs/heads/master",
-        "rev": "390363e6a977d71a96c53d7f8b252038dfee2e2e",
-        "revCount": 23,
+        "rev": "880f444ff7862d6127b051cf1a993ad1585b1652",
+        "revCount": 25,
        "type": "git",
        "url": "https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git"
      },
@ -810,22 +888,38 @@
        "type": "github"
      }
    },
+    "systems_3": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
    "terranix": {
      "inputs": {
        "bats-assert": "bats-assert",
        "bats-support": "bats-support",
-        "flake-utils": "flake-utils_4",
+        "flake-parts": "flake-parts_4",
        "nixpkgs": [
          "nixpkgs"
        ],
+        "systems": "systems_3",
        "terranix-examples": "terranix-examples"
      },
      "locked": {
-        "lastModified": 1695406838,
-        "narHash": "sha256-xiUfVD6rtsVWFotVtUW3Q1nQh4obKzgvpN1wqZuGXvM=",
+        "lastModified": 1728959489,
+        "narHash": "sha256-1Pu2j5xsBTuoyga08ZVf+rKp3FOMmJh/0fXen/idOrA=",
        "owner": "terranix",
        "repo": "terranix",
-        "rev": "fc9077ca02ab5681935dbf0ecd725c4d889b9275",
+        "rev": "7734e2ee6a1472807a33ce1e7da794bed2aaf91c",
        "type": "github"
      },
      "original": {
@ -836,11 +930,11 @@
    },
    "terranix-examples": {
      "locked": {
-        "lastModified": 1636300201,
-        "narHash": "sha256-0n1je1WpiR6XfCsvi8ZK7GrpEnMl+DpwhWaO1949Vbc=",
+        "lastModified": 1637156952,
+        "narHash": "sha256-KqvXIe1yiKOEP9BRYqNQN+LOWPCsWojh0WjEgv5jfEI=",
        "owner": "terranix",
        "repo": "terranix-examples",
-        "rev": "a934aa1cf88f6bd6c6ddb4c77b77ec6e1660bd5e",
+        "rev": "921680efb8af0f332d8ad73718d53907f9483e24",
        "type": "github"
      },
      "original": {
--- a/flake.nix
+++ b/flake.nix
@ -2,7 +2,7 @@
  description = "Bagel cooking infrastructure";

  inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";

    terranix.url = "github:terranix/terranix";
    terranix.inputs.nixpkgs.follows = "nixpkgs";
--- a/hosts/build-coord/hardware.nix
+++ b/hosts/build-coord/hardware.nix
@ -38,7 +38,7 @@

  zramSwap = {
    enable = true;
-    memoryPercent = 25;
+    memoryPercent = 100;
  };

  networking.useNetworkd = true;
--- a/hosts/buildbot-lix/default.nix
+++ b/hosts/buildbot-lix/default.nix
@ -38,7 +38,9 @@
      port = 2022;
      username = "buildbot";
    };
-    cors.allowedOrigin = "*.lix.systems";
+    cors.allowedOrigins = [
+      "https://*.lix.systems"
+    ];
    projects = [
      "lix"
      "lix-installer"
--- a/hosts/buildbot/default.nix
+++ b/hosts/buildbot/default.nix
@ -36,13 +36,15 @@
        port = cfgGerrit.port;
        username = "buildbot";
      };
-    cors.allowedOrigin = "*.forkos.org";
+    cors.allowedOrigins = [
+      "https://*.forkos.org"
+    ];
    projects = [
      "buildbot-test"
      "nixpkgs"
      "infra"
    ];
-    builders = [ "builder-10" ];
+    builders = [ "builder-9" ];
  };

  i18n.defaultLocale = "en_US.UTF-8";
--- a/hosts/gerrit01/default.nix
+++ b/hosts/gerrit01/default.nix
@ -23,6 +23,9 @@
    };
  };

+  # Block all these crawlers!!
+  bagel.services.nginx.crawler-blocker.enable = true;
+
  fileSystems."/gerrit-data" = {
    device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";
    fsType = "ext4";
@ -121,7 +124,7 @@
    };
  };

-  age.secrets.s3-channel-staging-keys.file = ../../secrets/s3-channel-staging-keys.age;
+  age.secrets.s3-channel-staging-keys.file = ../../secrets/floral/s3-channel-staging-keys.age;
  bagel.nixpkgs.channel-scripts = {
    enable = true;
    otlp.enable = true;
--- a/overlays/pyroscope/default.nix
+++ b/overlays/pyroscope/default.nix
@ -1,9 +1,10 @@
 { lib
-, buildGoModule
+, buildGo122Module
 , fetchFromGitHub
 }:

-buildGoModule rec {
+# FIXME: update, remove this pin
+buildGo122Module rec {
  pname = "pyroscope";
  version = "1.7.1";

--- a/services/block-crawlers/blocked-ua.txt
+++ b/services/block-crawlers/blocked-ua.txt
@ -0,0 +1,40 @@
+AI2Bot
+Ai2Bot-Dolma
+Amazonbot
+anthropic-ai
+Applebot
+Applebot-Extended
+Bytespider
+CCBot
+ChatGPT-User
+Claude-Web
+ClaudeBot
+cohere-ai
+Diffbot
+FacebookBot
+facebookexternalhit
+FriendlyCrawler
+Google-Extended
+GoogleOther
+GoogleOther-Image
+GoogleOther-Video
+GPTBot
+iaskspider/2.0
+ICC-Crawler
+ImagesiftBot
+img2dataset
+ISSCyberRiskCrawler
+Kangaroo Bot
+Meta-ExternalAgent
+Meta-ExternalFetcher
+OAI-SearchBot
+omgili
+omgilibot
+PerplexityBot
+PetalBot
+Scrapy
+Sidetrade indexer bot
+Timpibot
+VelenPublicWebCrawler
+Webzio-Extended
+YouBot
--- a/services/block-crawlers/default.nix
+++ b/services/block-crawlers/default.nix
@ -0,0 +1,32 @@
+{ pkgs, config, lib, ... }:
+let
+  inherit (lib) mkEnableOption mkIf mkOption types concatStringsSep mkDefault splitString;
+  cfg = config.bagel.services.nginx.crawler-blocker;
+  mkRobotsFile = blockedUAs: pkgs.writeText "robots.txt" ''
+    ${concatStringsSep "\n" (map (ua: "User-agent: ${ua}") blockedUAs)}
+    Disallow: /
+  '';
+in
+{
+  options = {
+    bagel.services.nginx.crawler-blocker = {
+      enable = mkEnableOption "the crawler blocker";
+
+      userAgents = mkOption {
+        type = types.listOf types.str;
+        default = splitString "\n" (builtins.readFile ./blocked-ua.txt);
+      };
+    };
+
+    services.nginx.virtualHosts = mkOption {
+      type = types.attrsOf (types.submodule {
+        config = {
+          locations."= /robots.txt" = mkIf cfg.enable (mkDefault {
+            alias = mkRobotsFile cfg.userAgents;
+          });
+        };
+      });
+    };
+  };
+}
+
--- a/services/buildbot/default.nix
+++ b/services/buildbot/default.nix
@ -53,9 +53,9 @@ in
      };
    };

-    cors.allowedOrigin = mkOption {
-      type = types.str;
-      example = "*.forkos.org";
+    cors.allowedOrigins = mkOption {
+      type = types.listOf types.str;
+      example = [ "*.forkos.org" ];
      description = "Allowed origin for Buildbot and NGINX for CORS without the protocol";
    };

@ -100,6 +100,7 @@ in
    };

    services.nginx = {
+      recommendedProxySettings = true;
      appendHttpConfig = ''
        # Our session stuff is too big with the TWISTED_COOKIE in addition.
        # Default is usually 4k or 8k.
@ -109,8 +110,8 @@ in
        forceSSL = true;
        enableACME = true;
        extraConfig = ''
+          # This is needed so that logged-in users in Buildbot can include their credentials in their requests.
          add_header Access-Control-Allow-Credentials 'true' always;
-          add_header Access-Control-Allow-Origin 'https://${cfg.cors.allowedOrigin}' always;
        '';
      };
    };
@ -155,9 +156,8 @@ in
      # we can replace all of this with automatic localworker generation on buildbot-nix side.
      workersFile = config.age.secrets.buildbot-workers.path;

-      allowedOrigins = [
-        cfg.cors.allowedOrigin
-      ];
+      # We rely on NGINX to do the CORS dance.
+      allowedOrigins = cfg.cors.allowedOrigins;

      buildMachines = map (n: {
          hostName = nodes.${n}.config.networking.fqdn;
@ -194,6 +194,32 @@ in
      RestartSteps = 10;
    };

+    services.postgresql.settings = {
+      # DB Version: 15
+      # OS Type: linux
+      # DB Type: web
+      # Total Memory (RAM): 64 GB
+      # CPUs num: 16
+      # Connections num: 100
+      # Data Storage: ssd
+      max_connections = 100;
+      shared_buffers = "16GB";
+      effective_cache_size = "48GB";
+      maintenance_work_mem = "2GB";
+      checkpoint_completion_target = 0.9;
+      wal_buffers = "16MB";
+      default_statistics_target = 100;
+      random_page_cost = 1.1;
+      effective_io_concurrency = 200;
+      work_mem = "41943kB";
+      huge_pages = "try";
+      min_wal_size = "1GB";
+      max_wal_size = "4GB";
+      max_worker_processes = 16;
+      max_parallel_workers_per_gather = 4;
+      max_parallel_workers = 16;
+      max_parallel_maintenance_workers = 4;
+    };

    nix.settings.keep-derivations = true;
    nix.gc = {
--- a/services/buildbot/lix.nix
+++ b/services/buildbot/lix.nix
@ -21,7 +21,8 @@ in
      sshUser = "nix";
      systems = [ "aarch64-linux" ];
      publicHostKey = ssh-keys.machines.build02-aarch64-lix;
-      supportedFeatures = nodes.build02-aarch64-lix.config.nix.settings.system-features;
+      # TODO: use build02 features.
+      supportedFeatures = nodes.build01-aarch64-lix.config.nix.settings.system-features;
    }
    {
      hostName = "build01.aarch64-darwin.lix.systems";
--- a/services/default.nix
+++ b/services/default.nix
@ -1,5 +1,6 @@
 {
  imports = [
+    ./block-crawlers
    ./gerrit
    ./channel-scripts
    ./hydra
--- a/services/gerrit/default.nix
+++ b/services/gerrit/default.nix
@ -129,7 +129,9 @@ in
      serverId = "9e5216ad-038d-4d74-a4e8-716515834a94";

      builtinPlugins = [
-        "gitiles"
+        # Disable gitiles as it generates too much traffic.
+        # Prefer git.forkos.org.
+        # "gitiles"
        "codemirror-editor"
        "reviewnotes"
        "download-commands"
@ -141,7 +143,9 @@ in
      plugins = with pkgs.gerritPlugins; [
        oauth
        metrics-reporter-prometheus
-        # Buildbot checks plugin (writeText because services.gerrit.plugins expects packages)
+        # Theme plugin
+        (pkgs.concatText "theme.js" [ ./theme.js ])
+        # Buildbot checks plugin
        (pkgs.runCommand "checks.js" {
          BASE_URI = builtins.toJSON "https://buildbot.forkos.org";
          SUPPORTED_PROJECTS = builtins.toJSON [
@ -326,7 +330,7 @@ in
      };
    };

-    age.secrets.gerrit-prometheus-bearer-token.file = ../../secrets/gerrit-prometheus-bearer-token.age;
+    age.secrets.gerrit-prometheus-bearer-token.file = ../../secrets/floral/gerrit-prometheus-bearer-token.age;
    bagel.monitoring.grafana-agent.exporters.gerrit = {
      port = 4778;  # grrt
      bearerTokenFile = config.age.secrets.gerrit-prometheus-bearer-token.path;
--- a/services/gerrit/git-gc-preserve.nix
+++ b/services/gerrit/git-gc-preserve.nix
@ -57,6 +57,7 @@ in
          name = "git-gc-preserve-${name}";
          value = {
            description = "Git-GC-Preserve Service - ${name}";
+            path = [ pkgs.util-linux ];
            serviceConfig = {
              WorkingDirectory = gcConfig.repoPath;
              Type = "oneshot";
--- a/services/gerrit/theme.js
+++ b/services/gerrit/theme.js
@ -0,0 +1,69 @@
+/* Set up theming for Floral.
+ * vim: set et ts=2 sw=2:
+ */
+Gerrit.install((plugin) => {
+  const stylesheet = new CSSStyleSheet();
+  stylesheet.replace(`
+    html {
+      --header-title-content: 'floral.systems';
+
+      --blue-50: #f3f4fb;
+      --blue-100: #e3e6f6;
+      --blue-200: #ced5ef;
+      --blue-300: #acb8e4;
+      --blue-400: #8495d6;
+      --blue-500: #6775ca;
+      --blue-600: #5158bb;
+      --blue-700: #494bac;
+      --blue-800: #41408d;
+      --blue-900: #383870;
+      --blue-950: #252546;
+
+      --coral-50: #fff1f1;
+      --coral-100: #ffe0e0;
+      --coral-200: #ffc5c5;
+      --coral-300: #ff9e9d;
+      --coral-400: #ff6665;
+      --coral-500: #fe4a49;
+      --coral-600: #ec1716;
+      --coral-700: #c70f0e;
+      --coral-800: #a41110;
+      --coral-900: #881514;
+      --coral-950: #4a0505;
+
+      --teal-50: #eefbf5;
+      --teal-100: #d6f5e5;
+      --teal-200: #b1e9d0;
+      --teal-300: #7ed7b5;
+      --teal-400: #49be95;
+      --teal-500: #27a27b;
+      --teal-600: #188162;
+      --teal-700: #136951;
+      --teal-800: #125342;
+      --teal-900: #104437;
+      --teal-950: #08261f;
+
+      --zinc-50: #fafafa;
+      --zinc-100: #f4f4f5;
+      --zinc-200: #e4e4e7;
+      --zinc-300: #d4d4d8;
+      --zinc-400: #a1a1aa;
+      --zinc-500: #71717a;
+      --zinc-600: #52525b;
+      --zinc-700: #3f3f46;
+      --zinc-800: #27272a;
+      --zinc-900: #18181b;
+      --zinc-950: #09090b;
+    }
+    html.lightTheme {
+      --header-background-color: var(--teal-700);
+      --header-text-color: var(--coral-50);
+    }
+    html.darkTheme {
+      --header-background-color: var(--teal-900);
+      --header-text-color: var(--coral-50);
+    }
+  `).then(() => {
+    document.adoptedStyleSheets = [...document.adoptedStyleSheets, stylesheet];
+  });
+});
--- a/services/gerrit/www.nix
+++ b/services/gerrit/www.nix
@ -29,10 +29,6 @@ in
            # NGINX should not give up super fast. Things can take time.
            proxy_read_timeout 3600;
          }
-
-          location = /robots.txt {
-            return 200 'User-agent: *\nAllow: /';
-          }
        '';
      };

--- a/services/matrix/default.nix
+++ b/services/matrix/default.nix
@ -32,6 +32,11 @@ in
          ];
          server_name = "forkos.org";
          database.backend = "rocksdb";
+
+          server_discovery = {
+            server.authority = "matrix.forkos.org:443";
+            client.base_url = "https://matrix.forkos.org";
+          };
        };
      };

@ -48,18 +53,7 @@ in
          "forkos.org" = {
            forceSSL = true;
            enableACME = true;
-            locations = {
-              "= /.well-known/matrix/server".extraConfig = ''
-                add_header Content-Type application/json;
-                add_header Access-Control-Allow-Origin *;
-                return 200 '{"m.server": "matrix.forkos.org:443"}';
-              '';
-              "= /.well-known/matrix/client".extraConfig = ''
-                add_header Content-Type application/json;
-                add_header Access-Control-Allow-Origin *;
-                return 200 '{"m.homeserver": {"base_url": "https://matrix.forkos.org/"}, "m.identity_server": {"base_url": "https://matrix.org/"}, "org.matrix.msc3575.proxy": {"url": "https://matrix.forkos.org"}}';
-              '';
-            };
+            locations."/.well-known/matrix".proxyPass = "http://grapevine";
          };
        };
      };
--- a/services/monitoring/exporters/postgres.nix
+++ b/services/monitoring/exporters/postgres.nix
@ -23,7 +23,10 @@ in
    };

    services.postgresql.settings.shared_preload_libraries = "pg_stat_statements";
+    systemd.services.postgresql.postStart = lib.mkAfter ''
+      ${config.services.postgresql.package}/bin/psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;";
+    '';

    bagel.monitoring.grafana-agent.exporters.postgres.port = 9104;
  };
-}
+}
--- a/services/monitoring/lgtm/alerts/forkos.yaml
+++ b/services/monitoring/lgtm/alerts/forkos.yaml
@ -1,11 +1,11 @@
 groups:
-  - name: ForkOS automation
-    rules:
-      - alert: SyncFailedTooOften
-        expr: 'changes(node_systemd_unit_state{name=~"ows.*.service",state="failed"}[24h]) > 2'
-        for: 30m
-        labels:
-          severity: critical
-        annotations:
-          summary: "Synchronization job {{ $labels.name }} has failed more than twice in the last 24 hours"
-          description: "On {{ $labels.instance }}, the synchronization job has failed more than twice in the last 24 hours, check if there's a conflict or a stdenv change."
+    - name: ForkOS automation
+      rules:
+        - alert: SyncFailedTooOften
+          expr: 'changes(node_systemd_unit_state{name=~"ows.*.service",state="failed"}[1d]) > 2'
+          for: 30m
+          labels:
+            severity: critical
+          annotations:
+            description: On {{ $labels.instance }}, the synchronization job has failed more than twice in the last 24 hours, check if there's a conflict or a stdenv change.
+            summary: Synchronization job {{ $labels.name }} has failed more than twice in the last 24 hours
--- a/services/monitoring/lgtm/alerts/postgres.yml
+++ b/services/monitoring/lgtm/alerts/postgres.yml
@ -1,102 +1,118 @@
 groups:
- name: PostgreSQL
-  rules:
-
-    - alert: PostgresqlTableNotAutoVacuumed
-      expr: '(pg_stat_user_tables_last_autovacuum > 0) and (time() - pg_stat_user_tables_last_autovacuum) > 60 * 60 * 24 * 10'
-      for: 0m
-      labels:
-        severity: warning
-      annotations:
-        summary: Postgresql table not auto vacuumed (instance {{ $labels.instance }})
-        description: "Table {{ $labels.relname }} has not been auto vacuumed for 10 days\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: PostgresqlTableNotAutoAnalyzed
-      expr: '(pg_stat_user_tables_last_autoanalyze > 0) and (time() - pg_stat_user_tables_last_autoanalyze) > 24 * 60 * 60 * 10'
-      for: 0m
-      labels:
-        severity: warning
-      annotations:
-        summary: Postgresql table not auto analyzed (instance {{ $labels.instance }})
-        description: "Table {{ $labels.relname }} has not been auto analyzed for 10 days\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: PostgresqlDeadLocks
-      expr: 'increase(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 5'
-      for: 0m
-      labels:
-        severity: warning
-      annotations:
-        summary: Postgresql dead locks (instance {{ $labels.instance }})
-        description: "PostgreSQL has dead-locks\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: PostgresqlHighRollbackRate
-      expr: 'sum by (namespace,datname) ((rate(pg_stat_database_xact_rollback{datname!~"template.*|postgres",datid!="0"}[3m])) / ((rate(pg_stat_database_xact_rollback{datname!~"template.*|postgres",datid!="0"}[3m])) + (rate(pg_stat_database_xact_commit{datname!~"template.*|postgres",datid!="0"}[3m])))) > 0.02'
-      for: 0m
-      labels:
-        severity: warning
-      annotations:
-        summary: Postgresql high rollback rate (instance {{ $labels.instance }})
-        description: "Ratio of transactions being aborted compared to committed is > 2 %\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: PostgresqlHighRateStatementTimeout
-      expr: 'rate(postgresql_errors_total{type="statement_timeout"}[1m]) > 3'
-      for: 0m
-      labels:
-        severity: critical
-      annotations:
-        summary: Postgresql high rate statement timeout (instance {{ $labels.instance }})
-        description: "Postgres transactions showing high rate of statement timeouts\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: PostgresqlHighRateDeadlock
-      expr: 'increase(postgresql_errors_total{type="deadlock_detected"}[1m]) > 1'
-      for: 0m
-      labels:
-        severity: critical
-      annotations:
-        summary: Postgresql high rate deadlock (instance {{ $labels.instance }})
-        description: "Postgres detected deadlocks\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: PostgresqlTooManyDeadTuples
-      expr: '((pg_stat_user_tables_n_dead_tup > 10000) / (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup)) >= 0.1'
-      for: 2m
-      labels:
-        severity: warning
-      annotations:
-        summary: Postgresql too many dead tuples (instance {{ $labels.instance }})
-        description: "PostgreSQL dead tuples is too large\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: PostgresqlTooManyLocksAcquired
-      expr: '((sum (pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.20'
-      for: 2m
-      labels:
-        severity: critical
-      annotations:
-        summary: Postgresql too many locks acquired (instance {{ $labels.instance }})
-        description: "Too many locks acquired on the database. If this alert happens frequently, we may need to increase the postgres setting max_locks_per_transaction.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: PostgresqlBloatIndexHigh(>80%)
-      expr: 'pg_bloat_btree_bloat_pct > 80 and on (idxname) (pg_bloat_btree_real_size > 100000000)'
-      for: 1h
-      labels:
-        severity: warning
-      annotations:
-        summary: Postgresql bloat index high (> 80%) (instance {{ $labels.instance }})
-        description: "The index {{ $labels.idxname }} is bloated. You should execute `REINDEX INDEX CONCURRENTLY {{ $labels.idxname }};`\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: PostgresqlBloatTableHigh(>80%)
-      expr: 'pg_bloat_table_bloat_pct > 80 and on (relname) (pg_bloat_table_real_size > 200000000)'
-      for: 1h
-      labels:
-        severity: warning
-      annotations:
-        summary: Postgresql bloat table high (> 80%) (instance {{ $labels.instance }})
-        description: "The table {{ $labels.relname }} is bloated. You should execute `VACUUM {{ $labels.relname }};`\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
-    - alert: PostgresqlInvalidIndex
-      expr: 'pg_genaral_index_info_pg_relation_size{indexrelname=~".*ccnew.*"}'
-      for: 6h
-      labels:
-        severity: warning
-      annotations:
-        summary: Postgresql invalid index (instance {{ $labels.instance }})
-        description: "The table {{ $labels.relname }} has an invalid index: {{ $labels.indexrelname }}. You should execute `DROP INDEX {{ $labels.indexrelname }};`\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+    - name: PostgreSQL
+      rules:
+        - alert: PostgresqlTableNotAutoVacuumed
+          expr: '(pg_stat_user_tables_last_autovacuum > 0) and (time() - pg_stat_user_tables_last_autovacuum) > 60 * 60 * 24 * 10'
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                Table {{ $labels.relname }} has not been auto vacuumed for 10 days
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Postgresql table not auto vacuumed (instance {{ $labels.instance }})
+        - alert: PostgresqlTableNotAutoAnalyzed
+          expr: '(pg_stat_user_tables_last_autoanalyze > 0) and (time() - pg_stat_user_tables_last_autoanalyze) > 24 * 60 * 60 * 10'
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                Table {{ $labels.relname }} has not been auto analyzed for 10 days
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Postgresql table not auto analyzed (instance {{ $labels.instance }})
+        - alert: PostgresqlDeadLocks
+          expr: 'increase(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 5'
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                PostgreSQL has dead-locks
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Postgresql dead locks (instance {{ $labels.instance }})
+        - alert: PostgresqlHighRollbackRate
+          expr: 'sum by (namespace, datname) ((rate(pg_stat_database_xact_rollback{datid!="0",datname!~"template.*|postgres"}[3m])) / ((rate(pg_stat_database_xact_rollback{datid!="0",datname!~"template.*|postgres"}[3m])) + (rate(pg_stat_database_xact_commit{datid!="0",datname!~"template.*|postgres"}[3m])))) > 0.02'
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                Ratio of transactions being aborted compared to committed is > 2 %
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Postgresql high rollback rate (instance {{ $labels.instance }})
+        - alert: PostgresqlHighRateStatementTimeout
+          expr: 'rate(postgresql_errors_total{type="statement_timeout"}[1m]) > 3'
+          labels:
+            severity: critical
+          annotations:
+            description: |-
+                Postgres transactions showing high rate of statement timeouts
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Postgresql high rate statement timeout (instance {{ $labels.instance }})
+        - alert: PostgresqlHighRateDeadlock
+          expr: 'increase(postgresql_errors_total{type="deadlock_detected"}[1m]) > 1'
+          labels:
+            severity: critical
+          annotations:
+            description: |-
+                Postgres detected deadlocks
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Postgresql high rate deadlock (instance {{ $labels.instance }})
+        - alert: PostgresqlTooManyDeadTuples
+          expr: '((pg_stat_user_tables_n_dead_tup > 10000) / (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup)) >= 0.1'
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                PostgreSQL dead tuples is too large
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Postgresql too many dead tuples (instance {{ $labels.instance }})
+        - alert: PostgresqlTooManyLocksAcquired
+          expr: '((sum(pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.2'
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            description: |-
+                Too many locks acquired on the database. If this alert happens frequently, we may need to increase the postgres setting max_locks_per_transaction.
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Postgresql too many locks acquired (instance {{ $labels.instance }})
+        - alert: PostgresqlBloatIndexHigh(>80%)
+          expr: 'pg_bloat_btree_bloat_pct > 80 and on (idxname) (pg_bloat_btree_real_size > 1e+08)'
+          for: 1h
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                The index {{ $labels.idxname }} is bloated. You should execute `REINDEX INDEX CONCURRENTLY {{ $labels.idxname }};`
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Postgresql bloat index high (> 80%) (instance {{ $labels.instance }})
+        - alert: PostgresqlBloatTableHigh(>80%)
+          expr: 'pg_bloat_table_bloat_pct > 80 and on (relname) (pg_bloat_table_real_size > 2e+08)'
+          for: 1h
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                The table {{ $labels.relname }} is bloated. You should execute `VACUUM {{ $labels.relname }};`
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Postgresql bloat table high (> 80%) (instance {{ $labels.instance }})
+        - alert: PostgresqlInvalidIndex
+          expr: 'pg_genaral_index_info_pg_relation_size{indexrelname=~".*ccnew.*"}'
+          for: 6h
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                The table {{ $labels.relname }} has an invalid index: {{ $labels.indexrelname }}. You should execute `DROP INDEX {{ $labels.indexrelname }};`
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Postgresql invalid index (instance {{ $labels.instance }})
--- a/services/monitoring/lgtm/alerts/resources.yaml
+++ b/services/monitoring/lgtm/alerts/resources.yaml
@ -1,76 +1,100 @@
 groups:
-  - name: Host & hardware
-    rules:
-      - alert: HostOutOfMemory
-        expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
-        for: 2m
-        labels:
-          severity: warning
-        annotations:
-          summary: Host out of memory (instance {{ $labels.instance }})
-          description: "Node memory is filling up (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-      - alert: HostMemoryUnderMemoryPressure
-        expr: (rate(node_vmstat_pgmajfault[1m]) > 1000) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
-        for: 2m
-        labels:
-          severity: warning
-        annotations:
-          summary: Host memory under memory pressure (instance {{ $labels.instance }})
-          description: "The node is under heavy memory pressure. High rate of major page faults\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-      - alert: HostMemoryIsUnderutilized
-        expr: (100 - (avg_over_time(node_memory_MemAvailable_bytes[30m]) / node_memory_MemTotal_bytes * 100) < 20) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
-        for: 1w
-        labels:
-          severity: info
-        annotations:
-          summary: Host Memory is underutilized (instance {{ $labels.instance }})
-          description: "Node memory is < 20% for 1 week. Consider reducing memory space. (instance {{ $labels.instance }})\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-      - alert: HostOutOfDiskSpace
-        expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
-        for: 2m
-        labels:
-          severity: warning
-        annotations:
-          summary: Host out of disk space (instance {{ $labels.instance }})
-          description: "Disk is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-      - alert: HostDiskWillFillIn24Hours
-        expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
-        for: 2m
-        labels:
-          severity: warning
-        annotations:
-          summary: Host disk will fill in 24 hours (instance {{ $labels.instance }})
-          description: "Filesystem is predicted to run out of space within the next 24 hours at current write rate\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-      - alert: HostCpuIsUnderutilized
-        expr: (100 - (rate(node_cpu_seconds_total{mode="idle"}[30m]) * 100) < 20) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
-        for: 1w
-        labels:
-          severity: info
-        annotations:
-          summary: Host CPU is underutilized (instance {{ $labels.instance }})
-          description: "CPU load is < 20% for 1 week. Consider reducing the number of CPUs.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-      - alert: HostCpuStealNoisyNeighbor
-        expr: (avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
-        for: 0m
-        labels:
-          severity: warning
-        annotations:
-          summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
-          description: "CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-      - alert: HostOomKillDetected
-        expr: (increase(node_vmstat_oom_kill[1m]) > 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
-        for: 0m
-        labels:
-          severity: warning
-        annotations:
-          summary: Host OOM kill detected (instance {{ $labels.instance }})
-          description: "OOM kill detected\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-      - alert: HostNetworkInterfaceSaturated
-        expr: ((rate(node_network_receive_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"} > 0.8 < 10000) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
-        for: 1m
-        labels:
-          severity: warning
-        annotations:
-          summary: Host Network Interface Saturated (instance {{ $labels.instance }})
-          description: "The network interface \"{{ $labels.device }}\" on \"{{ $labels.instance }}\" is getting overloaded.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
-
+    - name: Host & hardware
+      rules:
+        - alert: HostOutOfMemory
+          expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                Node memory is filling up (< 10% left)
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host out of memory (instance {{ $labels.instance }})
+        - alert: HostMemoryUnderMemoryPressure
+          expr: (rate(node_vmstat_pgmajfault[1m]) > 1000) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                The node is under heavy memory pressure. High rate of major page faults
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host memory under memory pressure (instance {{ $labels.instance }})
+        - alert: HostMemoryIsUnderutilized
+          expr: (100 - (avg_over_time(node_memory_MemAvailable_bytes[30m]) / node_memory_MemTotal_bytes * 100) < 20) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          for: 1w
+          labels:
+            severity: info
+          annotations:
+            description: |-
+                Node memory is < 20% for 1 week. Consider reducing memory space. (instance {{ $labels.instance }})
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host Memory is underutilized (instance {{ $labels.instance }})
+        - alert: HostOutOfDiskSpace
+          expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and on (instance, device, mountpoint) node_filesystem_readonly == 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                Disk is almost full (< 10% left)
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host out of disk space (instance {{ $labels.instance }})
+        - alert: HostDiskWillFillIn24Hours
+          expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and on (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and on (instance, device, mountpoint) node_filesystem_readonly == 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                Filesystem is predicted to run out of space within the next 24 hours at current write rate
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host disk will fill in 24 hours (instance {{ $labels.instance }})
+        - alert: HostCpuIsUnderutilized
+          expr: (100 - (rate(node_cpu_seconds_total{mode="idle"}[30m]) * 100) < 20) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          for: 1w
+          labels:
+            severity: info
+          annotations:
+            description: |-
+                CPU load is < 20% for 1 week. Consider reducing the number of CPUs.
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host CPU is underutilized (instance {{ $labels.instance }})
+        - alert: HostCpuStealNoisyNeighbor
+          expr: (avg by (instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
+        - alert: HostOomKillDetected
+          expr: (increase(node_vmstat_oom_kill[1m]) > 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                OOM kill detected
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host OOM kill detected (instance {{ $labels.instance }})
+        - alert: HostNetworkInterfaceSaturated
+          expr: ((rate(node_network_receive_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"} > 0.8 < 10000) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          for: 1m
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                The network interface "{{ $labels.device }}" on "{{ $labels.instance }}" is getting overloaded.
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host Network Interface Saturated (instance {{ $labels.instance }})
--- a/services/monitoring/lgtm/dashboards/node_exporter.json
+++ b/services/monitoring/lgtm/dashboards/node_exporter.json
--- a/services/monitoring/lgtm/dashboards/postgres2.json
+++ b/services/monitoring/lgtm/dashboards/postgres2.json
@ -20,7 +20,6 @@
  "fiscalYearStartMonth": 0,
  "gnetId": 9628,
  "graphTooltip": 0,
-  "id": 27,
  "links": [],
  "liveNow": false,
  "panels": [
@ -101,6 +100,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "mean"
@ -112,14 +112,14 @@
        "textMode": "name",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_static{hostname=\"$hostname\"}",
+          "expr": "pg_static{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "instant": true,
          "intervalFactor": 1,
@ -182,6 +182,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "mean"
@ -193,14 +194,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_postmaster_start_time_seconds{hostname=\"$hostname\"} * 1000",
+          "expr": "pg_postmaster_start_time_seconds{tenant=\"$tenant\",hostname=\"$hostname\"} * 1000",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "",
@ -261,6 +262,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -272,14 +274,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "SUM(pg_stat_database_tup_fetched{datname=~\"$datname\", hostname=~\"$hostname\"})",
+          "expr": "SUM(pg_stat_database_tup_fetched{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"})",
          "format": "time_series",
          "intervalFactor": 2,
          "refId": "A",
@ -340,6 +342,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -351,14 +354,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "SUM(pg_stat_database_tup_inserted{datname=~\"$datname\", hostname=~\"$hostname\"})",
+          "expr": "SUM(pg_stat_database_tup_inserted{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"})",
          "format": "time_series",
          "intervalFactor": 2,
          "refId": "A",
@ -419,6 +422,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -430,14 +434,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "SUM(pg_stat_database_tup_updated{datname=~\"$datname\", hostname=~\"$hostname\"})",
+          "expr": "SUM(pg_stat_database_tup_updated{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"})",
          "format": "time_series",
          "intervalFactor": 2,
          "refId": "A",
@ -498,6 +502,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "mean"
@ -509,14 +514,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_settings_max_connections{hostname=\"$hostname\"}",
+          "expr": "pg_settings_max_connections{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
@ -602,6 +607,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -613,14 +619,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_settings_shared_buffers_bytes{hostname=\"$hostname\"}",
+          "expr": "pg_settings_shared_buffers_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
@ -680,6 +686,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -691,14 +698,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_settings_effective_cache_size_bytes{hostname=\"$hostname\"}",
+          "expr": "pg_settings_effective_cache_size_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
@ -758,6 +765,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -769,14 +777,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_settings_maintenance_work_mem_bytes{hostname=\"$hostname\"}",
+          "expr": "pg_settings_maintenance_work_mem_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
@ -836,6 +844,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -847,14 +856,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_settings_work_mem_bytes{hostname=\"$hostname\"}",
+          "expr": "pg_settings_work_mem_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "",
@ -916,6 +925,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -927,14 +937,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_settings_max_wal_size_bytes{hostname=\"$hostname\"}",
+          "expr": "pg_settings_max_wal_size_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
@ -994,6 +1004,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -1005,14 +1016,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_settings_random_page_cost{hostname=\"$hostname\"}",
+          "expr": "pg_settings_random_page_cost{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
@ -1072,6 +1083,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -1083,7 +1095,7 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
@ -1150,6 +1162,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "mean"
@ -1161,14 +1174,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_settings_max_worker_processes{hostname=\"$hostname\"}",
+          "expr": "pg_settings_max_worker_processes{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
@ -1228,6 +1241,7 @@
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
@ -1239,14 +1253,14 @@
        "textMode": "auto",
        "wideLayout": true
      },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_settings_max_parallel_workers{hostname=\"$hostname\"}",
+          "expr": "pg_settings_max_parallel_workers{tenant=\"$tenant\",hostname=\"$hostname\"}",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
@ -1298,6 +1312,7 @@
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
+            "barWidthFactor": 0.6,
            "drawStyle": "points",
            "fillOpacity": 10,
            "gradientMode": "none",
@ -1372,7 +1387,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_stat_activity_count{datname=~\"$datname\", hostname=~\"$hostname\", state=\"active\"} !=0",
+          "expr": "pg_stat_activity_count{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\", state=\"active\"} !=0",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 2,
@ -1401,6 +1416,7 @@
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
+            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
@ -1474,7 +1490,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_database_xact_commit{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_xact_commit{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{datname}} commits",
@ -1485,7 +1501,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_database_xact_rollback{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_xact_rollback{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{datname}} rollbacks",
@ -1512,6 +1528,7 @@
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
+            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
@ -1585,7 +1602,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_stat_database_tup_updated{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_updated{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{datname}}",
@ -1613,6 +1630,7 @@
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
+            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
@ -1686,7 +1704,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_stat_database_tup_fetched{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_fetched{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{datname}}",
@ -1714,6 +1732,7 @@
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
+            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
@ -1787,7 +1806,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_stat_database_tup_inserted{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_inserted{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{datname}}",
@ -1815,6 +1834,7 @@
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
+            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
@ -1890,7 +1910,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_locks_count{datname=~\"$datname\", hostname=~\"$hostname\", mode=~\"$mode\"} != 0",
+          "expr": "pg_locks_count{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\", mode=~\"$mode\"} != 0",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{datname}},{{mode}}",
@ -1948,8 +1968,7 @@
            "mode": "absolute",
            "steps": [
              {
-                "color": "green",
-                "value": null
+                "color": "green"
              },
              {
                "color": "red",
@ -1991,7 +2010,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_stat_database_tup_returned{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_returned{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{datname}}",
@ -2050,8 +2069,7 @@
            "mode": "absolute",
            "steps": [
              {
-                "color": "green",
-                "value": null
+                "color": "green"
              },
              {
                "color": "red",
@ -2092,7 +2110,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_stat_activity_count{datname=~\"$datname\", hostname=~\"$hostname\", state=~\"idle|idle in transaction|idle in transaction (aborted)\"}",
+          "expr": "pg_stat_activity_count{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\", state=~\"idle|idle in transaction|idle in transaction (aborted)\"}",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{datname}}, s: {{state}}",
@ -2150,8 +2168,7 @@
            "mode": "absolute",
            "steps": [
              {
-                "color": "green",
-                "value": null
+                "color": "green"
              },
              {
                "color": "red",
@ -2193,7 +2210,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_stat_database_tup_deleted{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_deleted{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{datname}}",
@ -2252,8 +2269,7 @@
            "mode": "absolute",
            "steps": [
              {
-                "color": "green",
-                "value": null
+                "color": "green"
              },
              {
                "color": "red",
@ -2294,7 +2310,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "pg_stat_database_blks_hit{hostname=\"$hostname\", datname=~\"$datname\"} / (pg_stat_database_blks_read{hostname=\"$hostname\", datname=~\"$datname\"} + pg_stat_database_blks_hit{hostname=\"$hostname\", datname=~\"$datname\"})",
+          "expr": "pg_stat_database_blks_hit{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"} / (pg_stat_database_blks_read{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"} + pg_stat_database_blks_hit{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"})",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{ datname }}",
@ -2351,8 +2367,7 @@
            "mode": "absolute",
            "steps": [
              {
-                "color": "green",
-                "value": null
+                "color": "green"
              },
              {
                "color": "red",
@ -2395,7 +2410,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_bgwriter_buffers_backend{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_backend{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "buffers_backend",
@ -2406,7 +2421,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_bgwriter_buffers_alloc{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_alloc{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "buffers_alloc",
@ -2417,7 +2432,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "backend_fsync",
@ -2428,7 +2443,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "buffers_checkpoint",
@ -2439,7 +2454,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_bgwriter_buffers_clean{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_clean{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "buffers_clean",
@ -2497,8 +2512,7 @@
            "mode": "absolute",
            "steps": [
              {
-                "color": "green",
-                "value": null
+                "color": "green"
              },
              {
                "color": "red",
@ -2540,7 +2554,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_database_conflicts{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_conflicts{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{datname}} conflicts",
@ -2551,7 +2565,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_database_deadlocks{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_deadlocks{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{datname}} deadlocks",
@ -2610,8 +2624,7 @@
            "mode": "absolute",
            "steps": [
              {
-                "color": "green",
-                "value": null
+                "color": "green"
              },
              {
                "color": "red",
@ -2653,7 +2666,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_database_temp_bytes{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_temp_bytes{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "{{datname}}",
@ -2710,8 +2723,7 @@
            "mode": "absolute",
            "steps": [
              {
-                "color": "green",
-                "value": null
+                "color": "green"
              },
              {
                "color": "red",
@ -2754,7 +2766,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_bgwriter_checkpoint_write_time{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_checkpoint_write_time{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "write_time - Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk.",
@ -2765,7 +2777,7 @@
            "type": "prometheus",
            "uid": "mimir"
          },
-          "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "sync_time - Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk.",
@ -2866,8 +2878,32 @@
      {
        "current": {
          "selected": false,
-          "text": "bagel-box",
-          "value": "bagel-box"
+          "text": "fake",
+          "value": "fake"
+        },
+        "definition": "label_values(tenant)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Tenant",
+        "multi": true,
+        "name": "tenant",
+        "options": [],
+        "query": {
+          "qryType": 1,
+          "query": "label_values(tenant)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "current": {
+          "selected": true,
+          "text": "buildbot",
+          "value": "buildbot"
        },
        "datasource": {
          "type": "prometheus",
@ -2962,7 +2998,6 @@
    "from": "now-6h",
    "to": "now"
  },
-  "timeRangeUpdatedDuringEditOrView": false,
  "timepicker": {
    "refresh_intervals": [
      "5s",
@ -2993,4 +3028,4 @@
  "uid": "000000039",
  "version": 1,
  "weekStart": ""
-}
+}
--- a/services/monitoring/lgtm/grafana.nix
+++ b/services/monitoring/lgtm/grafana.nix
@ -17,7 +17,7 @@ in

  config = mkIf cfg.enable {
    age.secrets.grafana-oauth-secret = {
-      file = ../../../secrets/grafana-oauth-secret.age;
+      file = ../../../secrets/floral/grafana-oauth-secret.age;
      owner = "grafana";
    };

--- a/services/monitoring/lgtm/loki.nix
+++ b/services/monitoring/lgtm/loki.nix
@ -13,10 +13,10 @@ in
  config = mkIf cfg.enable {
    age.secrets = {
      metrics-push-htpasswd = {
-        file = ../../../secrets/metrics-push-htpasswd.age;
+        file = ../../../secrets/floral/metrics-push-htpasswd.age;
        owner = "nginx";
      };
-      loki-environment.file = ../../../secrets/loki-environment.age;
+      loki-environment.file = ../../../secrets/floral/loki-environment.age;
    };

    services.loki = {
--- a/services/monitoring/lgtm/mimir.nix
+++ b/services/monitoring/lgtm/mimir.nix
@ -9,6 +9,15 @@ let
  inherit (lib) mkEnableOption mkIf;

  mimirPort = config.services.mimir.configuration.server.http_listen_port;
+
+  alerts = pkgs.runCommand "mimir-alerts-checked" {
+    src = ./alerts;
+    nativeBuildInputs = with pkgs; [ prometheus.cli ];
+  } ''
+    promtool check rules $src/*
+    mkdir $out
+    cp -R $src $out/anonymous/
+  '';
 in
 {
  options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
@ -16,11 +25,11 @@ in
  config = mkIf cfg.enable {
    age.secrets = {
      metrics-push-htpasswd = {
-        file = ../../../secrets/metrics-push-htpasswd.age;
+        file = ../../../secrets/floral/metrics-push-htpasswd.age;
        owner = "nginx";
      };
-      mimir-environment.file = ../../../secrets/mimir-environment.age;
-      mimir-webhook-url.file = ../../../secrets/mimir-webhook-url.age;
+      mimir-environment.file = ../../../secrets/floral/mimir-environment.age;
+      mimir-webhook-url.file = ../../../secrets/floral/mimir-webhook-url.age;
    };

    services.mimir = {
@ -60,10 +69,7 @@ in
        blocks_storage.backend = "s3";
        ruler_storage = {
          backend = "local";
-          local.directory = pkgs.runCommand "mimir-rules" {} ''
-            mkdir -p $out
-            ln -s ${./alerts} $out/anonymous
-          '';
+          local.directory = alerts;
        };

        alertmanager = {
--- a/services/monitoring/lgtm/tempo.nix
+++ b/services/monitoring/lgtm/tempo.nix
@ -13,10 +13,10 @@ in
  config = mkIf cfg.enable {
    age.secrets = {
      metrics-push-htpasswd = {
-        file = ../../../secrets/metrics-push-htpasswd.age;
+        file = ../../../secrets/floral/metrics-push-htpasswd.age;
        owner = "nginx";
      };
-      tempo-environment.file = ../../../secrets/tempo-environment.age;
+      tempo-environment.file = ../../../secrets/floral/tempo-environment.age;
    };

    services.tempo = {
--- a/services/netbox/default.nix
+++ b/services/netbox/default.nix
@ -20,7 +20,10 @@ in
  };

  config = mkIf cfg.enable {
-    age.secrets.netbox-environment.file = ../../secrets/netbox-environment.age;
+    # FIXME: why
+    nixpkgs.config.permittedInsecurePackages = [ pkgs.netbox_3_7.name ];
+
+    age.secrets.netbox-environment.file = ../../secrets/floral/netbox-environment.age;
    services = {
      netbox = {
        enable = true;
--- a/services/ofborg/default.nix
+++ b/services/ofborg/default.nix
@ -26,10 +26,12 @@ in {
      webroot = "/var/lib/acme/.challenges";
      group = "rabbitmq";
    };
+
+    services.nginx.enable = true;
    services.nginx.virtualHosts.${amqpHost}.locations."/.well-known/acme-challenge".root =
      "/var/lib/acme/.challenges";
    systemd.services.rabbitmq.requires = ["acme-finished-${amqpHost}.target"];

-    networking.firewall.allowedTCPPorts = [ amqpPort ];
+    networking.firewall.allowedTCPPorts = [ 80 443 amqpPort ];
  };
 }
--- a/services/s3-revproxy/default.nix
+++ b/services/s3-revproxy/default.nix
@ -70,7 +70,7 @@ in
  ];

  config = mkIf cfg.enable {
-    age.secrets.s3-revproxy-api-keys.file = ../../secrets/s3-revproxy-api-keys.age;
+    age.secrets.s3-revproxy-api-keys.file = ../../secrets/floral/s3-revproxy-api-keys.age;
    # For each target, generate an entry that passes it to the s3-revproxy.
    services.nginx.virtualHosts = mapAttrs' (subdomain: _: nameValuePair "${subdomain}.${cfg.domain}" (mkProxiedSubdomain subdomain)) cfg.targets;
    # this solves garage supporting neither anonymous access nor automatic
--- a/services/uptime-kuma/default.nix
+++ b/services/uptime-kuma/default.nix
@ -59,7 +59,7 @@ in
      443
    ];

-    age.secrets.stateless-uptime-kuma-password.file = ../../secrets/stateless-uptime-kuma-password.age;
+    age.secrets.stateless-uptime-kuma-password.file = ../../secrets/floral/stateless-uptime-kuma-password.age;
    statelessUptimeKuma = {
      probesConfig = {
        monitors = lib.genAttrs subdomains (name: {
@ -83,6 +83,9 @@ in
            ];
          };
        };
+        settings = {
+          entryPage = "statusPage-forkos";
+        };
      };
      extraFlags = [ "-s" ];
      host = "http://localhost:${builtins.toString port}/";
Author	SHA1	Message	Date
Ilya K	cae763e8dd	ofborg: enable nginx for certs	2024-11-17 14:48:08 +03:00
Ilya K	4f1378937f	Disable nixos-option, it breaks	2024-11-17 14:39:25 +03:00
Ilya K	7e3074a769	Update everything again	2024-11-17 14:39:19 +03:00
Ilya K	3182a036c0	chore: bump everything	2024-11-12 23:42:51 +03:00
Ilya K	54e8282aac	fix: use promtool to verify rules, fix format	2024-11-12 23:21:30 +03:00
Ilya K	41be8dc170	fix: pin pyroscope to go 1.22	2024-11-12 23:16:46 +03:00
Ilya K	b08330c42b	fix: allow insecure netbox	2024-11-12 22:52:11 +03:00
Ilya K	268422f653	fix: update grapevine to build with current nixpkgs	2024-11-12 22:49:33 +03:00
Yureka	4e03cf2309	increase zram on build-coord	2024-11-04 08:48:40 +01:00
Yureka	00a5d373f3	flake.lock: Update Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/e2f08f4d8b3ecb5cf5c9fd9cb2d53bb3c71807da' (2024-10-05) → 'github:NixOS/nixpkgs/7ffd9ae656aec493492b44d0ddfb28e79a1ea25d' (2024-11-02)	2024-11-04 08:46:51 +01:00
Yureka	ec93c94e7e	revert default shell to bash zsh is unbearably slow on some machines	2024-10-30 13:29:27 +01:00
raito	f56576d644	fix: add util-linux in git-gc-preserve Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-28 12:07:54 +01:00
raito	1ae3d7c396	chore: move to forkos branch for buildbot Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-28 12:07:06 +01:00
raito	cca8156e52	Revert "chore: move to faster-depinfo branch for Buildbot" This reverts commit `7df7eaeb9b` because there was a missing intersection in the code and this broke Buildbot entirely by exhausting all the resources.	2024-10-27 22:01:34 +01:00
raito	7df7eaeb9b	chore: move to faster-depinfo branch for Buildbot Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-27 19:39:18 +01:00
Kiara Grouwstra	5ae71fff99	chore: add lorri to prevent direnv from blocking, closes #147	2024-10-27 09:42:11 +00:00
raito	02f8bc7ca4	chore(o11y): filter by tenancy on node_exporter Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-22 16:57:37 +02:00
raito	e0c029ba43	fix: make all buildbot postgres faster work_mem was effectively absurdly low. Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-22 16:57:24 +02:00
raito	3ed36f74fd	onboarding: add pennae keys on lix infra Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-22 16:57:06 +02:00
raito	b1f4674da0	chore: add tenancy in postgres Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-22 16:57:06 +02:00
raito	226eacdeec	chore: add tenancy in node_exporter Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-22 16:57:06 +02:00
Ilya K	14935c5e92	fix: update grapevine config	2024-10-21 16:31:26 +03:00
raito	bee402fecc	fix: ensure that pg_stat_statements is always created as an ext Otherwise, we will have issues with this exporter. Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-21 14:33:18 +02:00
raito	3efdd0f6c9	fix: disable gitiles on gerrit01 It is generating too much traffic and CPU load for no good reason. Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-20 11:24:58 +02:00
raito	8c0c7b517f	feat: block automatically crawlers if the blocker is enabled This help us getting rid of useless traffic by crawlers. It is enabled for gerrit01 which is suffering the most from this. Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-19 19:12:10 +02:00
raito	d5500d7c4e	fix(buildbot): bring back the old Gerrit reporting Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-18 23:22:51 +00:00
raito	eaf48a0cdd	fix(buildbot): use builder-9 as builder-10 is down Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-18 23:22:51 +00:00
raito	e3129fec51	fix(buildbot): fix CORS properly wildcards are not allowed in the headers. We need to include credentials as well. Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-18 23:22:51 +00:00
raito	437293bdaa	fix(buildbot): remove CORS wildcards for their precise Gerrit hosts wildcards are not supported in CORS headers, so this design was quite wrong actually. We can just use the actual Gerrit hostname for now. Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-18 23:22:51 +00:00
mei (ckie)	df8a57f91a	users: add ckie	2024-10-18 14:43:25 +03:00
Yureka	97bee26977	new ssh key for yureka	2024-10-10 13:42:29 +00:00
Luke Granger-Brown	84cfbdb050	feat: check formatting and validity of alerts Fixes #94.	2024-10-07 20:00:54 +00:00
Luke Granger-Brown	6a8f49f180	feat(gerrit): add some basic theming This is based on some of the preliminary colour work done by @ckie in the the-distro/floral.systems repo.	2024-10-07 19:27:13 +00:00
Yureka	06dd4d6e85	update hydra	2024-10-07 19:25:51 +02:00
Luke Granger-Brown	de085155a6	fix: update paths to floral secrets to secrets/floral/	2024-10-07 15:48:05 +00:00
Luke Granger-Brown	2001012325	feat(uptime-kuma): status.forkos.org should point at the ForkOS page	2024-10-07 15:47:33 +00:00
raito	fbf26302b6	hotfix(lix): use build01 features for build02 remote builder Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-07 15:26:27 +02:00
raito	1701a2b388	hotfix: bump buildbot-nix to restore backward compat with Lix deployments We oopsie dropped `hydraJobs` support to move to `buildbotJobs`. Signed-off-by: Raito Bezarius <masterancpp@gmail.com>	2024-10-07 15:26:19 +02:00