feat(uptime-kuma): status.forkos.org should point at the ForkOS page

.envrc

@@ -1,11 +1,2 @@
-#!/usr/bin/env bash
-# the shebang is ignored, but nice for editors
-
 # shellcheck shell=bash
-if type -P lorri &>/dev/null; then
-  eval "$(lorri direnv --flake .)"
-else
-  echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]'
 use flake
-fi
-

common/admins.nix

@@ -19,14 +19,12 @@ in
       "thubrecht"
       "winter"
       "yuka"
-      "ckie"
     ];
 
     lix-infra.members = [
       "raito"
       "hexchen"
       "jade"
-      "pennae"
     ];
   };
   bagel.users = genAttrs [
@@ -41,7 +39,5 @@ in
     "thubrecht"
     "winter"
     "yuka"
-    "ckie"
-    "pennae"
   ] (name: {});
 }

common/base-server.nix

@@ -7,7 +7,6 @@
   nixpkgs.overlays = import ../overlays;
 
   nix.package = lib.mkDefault pkgs.lix;
-  system.tools.nixos-option.enable = false;
   services.openssh.enable = lib.mkForce true;
 
   networking.nftables.enable = true;

common/ssh-keys.nix

@@ -61,12 +61,7 @@
       "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
     ];
     thubrecht = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPM1jpXR7BWQa7Sed7ii3SbvIPRRlKb3G91qC0vOwfJn" ];
-    yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIxQ3NYBi8v1f/vhxLKDcA6upmX0pctRDbnK6SER5OUR yureka" ];
+    yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKath4/fDnlv/4fzxkPrQN1ttmoPRNu/m9bEtdPJBDfY cardno:16_933_242" ];
     winter = [ "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIH/LDRUG+U+++UmlxvA2kspioTjktQZ8taDcHq8gVlkfAAAABHNzaDo=" ];
-    ckie = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH3uTwzSSMAPg84fwbNp2cq9+BdLFeA1VzDGth4zCAbz https://mei.puppycat.house" ];
-    pennae = [
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC5Wf5/IbyFpdziWfwxkQqxOf3r1L9pYn6xQBEKFwmMY"
-      "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIK8icXjHkb4XzbIVN3djH4CE7RvgGd+3xbG4cgh0Yls5AAAABHNzaDo="
-    ];
   };
 }

common/zsh.nix

@@ -1,4 +1,5 @@
 { lib, pkgs, config, ... }: {
+  users.defaultUserShell = pkgs.zsh;
   programs.zsh = {
     enable = true;
     enableCompletion = true;

flake.lock

@@ -27,17 +27,16 @@
       "inputs": {
         "crane": "crane",
         "flake-compat": "flake-compat_2",
-        "flake-parts": "flake-parts_2",
+        "flake-utils": "flake-utils_2",
-        "nix-github-actions": "nix-github-actions_2",
         "nixpkgs": "nixpkgs",
         "nixpkgs-stable": "nixpkgs-stable"
       },
       "locked": {
-        "lastModified": 1730257295,
+        "lastModified": 1711742460,
-        "narHash": "sha256-OQl+aAsKiyygvpzck1u0sZf/R4T9zM903CgNDFmmzA8=",
+        "narHash": "sha256-0O4v6e4a1toxXZ2gf5INhg4WPE5C5T+SVvsBt+45Mcc=",
         "owner": "zhaofengli",
         "repo": "attic",
-        "rev": "48c8b395bfbc6b76c7eae74df6c74351255a095c",
+        "rev": "4dbdbee45728d8ce5788db6461aaaa89d98081f0",
         "type": "github"
       },
       "original": {
@@ -50,11 +49,11 @@
     "bats-assert": {
       "flake": false,
       "locked": {
-        "lastModified": 1692829535,
+        "lastModified": 1636059754,
-        "narHash": "sha256-oDqhUQ6Xg7a3xx537SWLGRzqP3oKKeyY4UYGCdz9z/Y=",
+        "narHash": "sha256-ewME0l27ZqfmAwJO4h5biTALc9bDLv7Bl3ftBzBuZwk=",
         "owner": "bats-core",
         "repo": "bats-assert",
-        "rev": "e2d855bc78619ee15b0c702b5c30fb074101159f",
+        "rev": "34551b1d7f8c7b677c1a66fc0ac140d6223409e5",
         "type": "github"
       },
       "original": {
@@ -66,11 +65,11 @@
     "bats-support": {
       "flake": false,
       "locked": {
-        "lastModified": 1693050811,
+        "lastModified": 1548869839,
-        "narHash": "sha256-PxJaH16+QrsfZqtkWVt5K6TwJB5gjIXnbGo+MB84WIU=",
+        "narHash": "sha256-Gr4ntadr42F2Ks8Pte2D4wNDbijhujuoJi4OPZnTAZU=",
         "owner": "bats-core",
         "repo": "bats-support",
-        "rev": "9bf10e876dd6b624fe44423f0b35e064225f7556",
+        "rev": "d140a65044b2d6810381935ae7f0c94c7023c8c3",
         "type": "github"
       },
       "original": {
@@ -88,11 +87,11 @@
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1730064416,
+        "lastModified": 1728162115,
-        "narHash": "sha256-Opbtu9hKijGkEx+GYbSu3MJms3lFxZmAGTFyckguWMM=",
+        "narHash": "sha256-O95Nv81XlYZjltq009951gwYHGo+Kjt9EBhh/6cYGDw=",
         "ref": "refs/heads/forkos",
-        "rev": "79137b14f3cb376204f739f44b05aebfc288ca89",
+        "rev": "ed8f940717724af853efcafcd82cf4d10f32ede5",
-        "revCount": 310,
+        "revCount": 298,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
       },
@@ -126,18 +125,17 @@
       "inputs": {
         "flake-compat": "flake-compat",
         "flake-utils": "flake-utils",
-        "nix-github-actions": "nix-github-actions",
         "nixpkgs": [
           "nixpkgs"
         ],
         "stable": "stable"
       },
       "locked": {
-        "lastModified": 1731527002,
+        "lastModified": 1711386353,
-        "narHash": "sha256-dI9I6suECoIAmbS4xcrqF8r2pbmed8WWm5LIF1yWPw8=",
+        "narHash": "sha256-gWEpb8Hybnoqb4O4tmpohGZk6+aerAbJpywKcFIiMlg=",
         "owner": "zhaofengli",
         "repo": "colmena",
-        "rev": "e3ad42138015fcdf2524518dd564a13145c72ea1",
+        "rev": "cd65ef7a25cdc75052fbd04b120aeb066c3881db",
         "type": "github"
       },
       "original": {
@@ -155,11 +153,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1722960479,
+        "lastModified": 1702918879,
-        "narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=",
+        "narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4",
+        "rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb",
         "type": "github"
       },
       "original": {
@@ -169,12 +167,18 @@
       }
     },
     "crane_2": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "nixpkgs"
+        ]
+      },
       "locked": {
-        "lastModified": 1730060262,
+        "lastModified": 1716569590,
-        "narHash": "sha256-RMgSVkZ9H03sxC+Vh4jxtLTCzSjPq18UWpiM0gq6shQ=",
+        "narHash": "sha256-5eDbq8TuXFGGO3mqJFzhUbt5zHVTf5zilQoyW5jnJwo=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "498d9f122c413ee1154e8131ace5a35a80d8fa76",
+        "rev": "109987da061a1bf452f435f1653c47511587d919",
         "type": "github"
       },
       "original": {
@@ -215,11 +219,11 @@
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1730442928,
+        "lastModified": 1716359173,
-        "narHash": "sha256-U1DWb5c3EfkA7pqx5V1H4AWRA+EaE6UJ0lIRvK1RxgM=",
+        "narHash": "sha256-pYcjP6Gy7i6jPWrjiWAVV0BCQp+DdmGaI/k65lBb/kM=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "87b4d20f896c99018dde4702a9c6157b516f2a76",
+        "rev": "b6fc5035b28e36a98370d0eac44f4ef3fd323df6",
         "type": "github"
       },
       "original": {
@@ -248,11 +252,11 @@
     "flake-compat_2": {
       "flake": false,
       "locked": {
-        "lastModified": 1696426674,
+        "lastModified": 1673956053,
-        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
         "owner": "edolstra",
         "repo": "flake-compat",
-        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
         "type": "github"
       },
       "original": {
@@ -316,28 +320,6 @@
       }
     },
     "flake-parts_2": {
-      "inputs": {
-        "nixpkgs-lib": [
-          "grapevine",
-          "attic",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1722555600,
-        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
-        "type": "github"
-      },
-      "original": {
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "type": "github"
-      }
-    },
-    "flake-parts_3": {
       "inputs": {
         "nixpkgs-lib": [
           "hydra",
@@ -359,23 +341,6 @@
         "type": "github"
       }
     },
-    "flake-parts_4": {
-      "inputs": {
-        "nixpkgs-lib": "nixpkgs-lib"
-      },
-      "locked": {
-        "lastModified": 1727826117,
-        "narHash": "sha256-K5ZLCyfO/Zj9mPFldf3iwS6oZStJcU4tSpiXTMYaaL0=",
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "rev": "3d04084d54bedc3d6b8b736c70ef449225c361b1",
-        "type": "github"
-      },
-      "original": {
-        "id": "flake-parts",
-        "type": "indirect"
-      }
-    },
     "flake-utils": {
       "locked": {
         "lastModified": 1659877975,
@@ -392,15 +357,30 @@
       }
     },
     "flake-utils_2": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_3": {
       "inputs": {
         "systems": "systems_2"
       },
       "locked": {
-        "lastModified": 1726560853,
+        "lastModified": 1710146030,
-        "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
         "type": "github"
       },
       "original": {
@@ -410,6 +390,21 @@
         "type": "github"
       }
     },
+    "flake-utils_4": {
+      "locked": {
+        "lastModified": 1634851050,
+        "narHash": "sha256-N83GlSGPJJdcqhUxSCS/WwW5pksYf3VP1M13cDRTSVA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "c91f3de5adaf1de973b797ef7485e441a65b8935",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "gerrit-dashboard": {
       "flake": false,
       "locked": {
@@ -432,21 +427,20 @@
         "crane": "crane_2",
         "fenix": "fenix",
         "flake-compat": "flake-compat_3",
-        "flake-utils": "flake-utils_2",
+        "flake-utils": "flake-utils_3",
         "nix-filter": "nix-filter",
         "nixpkgs": [
           "nixpkgs"
         ],
-        "rocksdb": "rocksdb",
         "rust-manifest": "rust-manifest"
       },
       "locked": {
         "host": "gitlab.computer.surgery",
-        "lastModified": 1731820516,
+        "lastModified": 1727994504,
-        "narHash": "sha256-BJiFq5meQMwOLhVS9/m2HCZoNUIN5u5weJXr94dkIRU=",
+        "narHash": "sha256-FC6M1KKX58HbU9LG+cG6EJRr02J9lE/o0iiDi6m1gv8=",
         "owner": "matrix",
         "repo": "grapevine-fork",
-        "rev": "887e59cf03efb70bfa68b56dda32c411ba6cd5fa",
+        "rev": "5a490a4397f0c6a36dab1cb631dadc67a849deab",
         "type": "gitlab"
       },
       "original": {
@@ -486,11 +480,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1728321752,
+        "lastModified": 1728163965,
-        "narHash": "sha256-GbBAoBF7ZObz0IP+g0LZKxMafpMvNKjTEu9haiZbV54=",
+        "narHash": "sha256-vHA2qZWoTY4DQUWFktq8DYfCOlm8rd1bNZpiIzpFBCk=",
         "ref": "refs/heads/main",
-        "rev": "ee1234c15cdcb427dbd4828e0add09d02cd606c9",
+        "rev": "e4d466ffcdc0e43f3bd005de3913a7e9ed4a58f5",
-        "revCount": 4220,
+        "revCount": 4216,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/hydra.git"
       },
@@ -511,11 +505,11 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1728163191,
+        "lastModified": 1728159440,
-        "narHash": "sha256-SW0IEBsPN1EysqzvfDT+8Kimtzy03O1BxQQm7ZB6fRY=",
+        "narHash": "sha256-8DW+VfSIk5xYpYj1VDKMmUjXhMooIlAOwYG4J9PLbbg=",
         "ref": "refs/heads/main",
-        "rev": "ed9b7f4f84fd60ad8618645cc1bae2d686ff0db6",
+        "rev": "7ff60b7445ccb599edcbf0078d181034f61a0859",
-        "revCount": 16323,
+        "revCount": 16318,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/lix"
       },
@@ -526,12 +520,12 @@
     },
     "nix-eval-jobs": {
       "inputs": {
-        "flake-parts": "flake-parts_3",
+        "flake-parts": "flake-parts_2",
         "lix": [
           "hydra",
           "lix"
         ],
-        "nix-github-actions": "nix-github-actions_3",
+        "nix-github-actions": "nix-github-actions",
         "nixpkgs": [
           "hydra",
           "nixpkgs"
@@ -554,11 +548,11 @@
     },
     "nix-filter": {
       "locked": {
-        "lastModified": 1730207686,
+        "lastModified": 1710156097,
-        "narHash": "sha256-SCHiL+1f7q9TAnxpasriP6fMarWE5H43t25F5/9e28I=",
+        "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
         "owner": "numtide",
         "repo": "nix-filter",
-        "rev": "776e68c1d014c3adde193a18db9d738458cd2ba4",
+        "rev": "3342559a24e85fc164b295c3444e8a139924675b",
         "type": "github"
       },
       "original": {
@@ -589,49 +583,6 @@
       }
     },
     "nix-github-actions": {
-      "inputs": {
-        "nixpkgs": [
-          "colmena",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1729742964,
-        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "type": "github"
-      }
-    },
-    "nix-github-actions_2": {
-      "inputs": {
-        "nixpkgs": [
-          "grapevine",
-          "attic",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1729742964,
-        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "type": "github"
-      }
-    },
-    "nix-github-actions_3": {
       "inputs": {
         "nixpkgs": [
           "hydra",
@@ -671,11 +622,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1726042813,
+        "lastModified": 1711401922,
-        "narHash": "sha256-LnNKCCxnwgF+575y0pxUdlGZBO/ru1CtGHIqQVfvjlA=",
+        "narHash": "sha256-QoQqXoj8ClGo0sqD/qWKFWezgEwUL0SUh37/vY2jNhc=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "159be5db480d1df880a0135ca0bfed84c2f88353",
+        "rev": "07262b18b97000d16a4bdb003418bd2fb067a932",
         "type": "github"
       },
       "original": {
@@ -685,18 +636,6 @@
         "type": "github"
       }
     },
-    "nixpkgs-lib": {
-      "locked": {
-        "lastModified": 1727825735,
-        "narHash": "sha256-0xHYkMkeLVQAMa7gvkddbPqpxph+hDzdu1XdGPJR+Os=",
-        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
-      },
-      "original": {
-        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
-      }
-    },
     "nixpkgs-regression": {
       "locked": {
         "lastModified": 1643052045,
@@ -715,32 +654,32 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1724316499,
+        "lastModified": 1711460390,
-        "narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=",
+        "narHash": "sha256-akSgjDZL6pVHEfSE6sz1DNSXuYX6hq+P/1Z5IoYWs7E=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841",
+        "rev": "44733514b72e732bd49f5511bd0203dea9b9a434",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-24.05",
+        "ref": "nixos-23.11",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1731676054,
+        "lastModified": 1728093190,
-        "narHash": "sha256-OZiZ3m8SCMfh3B6bfGC/Bm4x3qc1m2SVEAlkV6iY7Yg=",
+        "narHash": "sha256-CAZF2NRuHmqTtRTNAruWpHA43Gg2UvuCNEIzabP0l6M=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "5e4fbfb6b3de1aa2872b76d49fafc942626e2add",
+        "rev": "e2f08f4d8b3ecb5cf5c9fd9cb2d53bb3c71807da",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-unstable",
+        "ref": "nixpkgs-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -761,23 +700,6 @@
         "type": "github"
       }
     },
-    "rocksdb": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1730475155,
-        "narHash": "sha256-u5uuShM2SxHc9/zL4UU56IhCcR/ZQbzde0LgOYS44bM=",
-        "owner": "facebook",
-        "repo": "rocksdb",
-        "rev": "3c27a3dde0993210c5cc30d99717093f7537916f",
-        "type": "github"
-      },
-      "original": {
-        "owner": "facebook",
-        "ref": "v9.7.4",
-        "repo": "rocksdb",
-        "type": "github"
-      }
-    },
     "root": {
       "inputs": {
         "agenix": "agenix",
@@ -800,11 +722,11 @@
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1730386175,
+        "lastModified": 1716107283,
-        "narHash": "sha256-0Uq+/B8eu7pw8B8pxuGdFYKjcVLwNMcHfDxU9sXh7rg=",
+        "narHash": "sha256-NJgrwLiLGHDrCia5AeIvZUHUY7xYGVryee0/9D3Ir1I=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "0ba893e1a00d92557ac91efb771d72eee36ca687",
+        "rev": "21ec8f523812b88418b2bfc64240c62b3dd967bd",
         "type": "github"
       },
       "original": {
@@ -828,16 +750,16 @@
     },
     "stable": {
       "locked": {
-        "lastModified": 1730883749,
+        "lastModified": 1696039360,
-        "narHash": "sha256-mwrFF0vElHJP8X3pFCByJR365Q2463ATp2qGIrDUdlE=",
+        "narHash": "sha256-g7nIUV4uq1TOVeVIDEZLb005suTWCUjSY0zYOlSBsyE=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "dba414932936fde69f0606b4f1d87c5bc0003ede",
+        "rev": "32dcb45f66c0487e92db8303a798ebc548cadedc",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-24.05",
+        "ref": "nixos-23.05",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -888,38 +810,22 @@
         "type": "github"
       }
     },
-    "systems_3": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
-    },
     "terranix": {
       "inputs": {
         "bats-assert": "bats-assert",
         "bats-support": "bats-support",
-        "flake-parts": "flake-parts_4",
+        "flake-utils": "flake-utils_4",
         "nixpkgs": [
           "nixpkgs"
         ],
-        "systems": "systems_3",
         "terranix-examples": "terranix-examples"
       },
       "locked": {
-        "lastModified": 1728959489,
+        "lastModified": 1695406838,
-        "narHash": "sha256-1Pu2j5xsBTuoyga08ZVf+rKp3FOMmJh/0fXen/idOrA=",
+        "narHash": "sha256-xiUfVD6rtsVWFotVtUW3Q1nQh4obKzgvpN1wqZuGXvM=",
         "owner": "terranix",
         "repo": "terranix",
-        "rev": "7734e2ee6a1472807a33ce1e7da794bed2aaf91c",
+        "rev": "fc9077ca02ab5681935dbf0ecd725c4d889b9275",
         "type": "github"
       },
       "original": {
@@ -930,11 +836,11 @@
     },
     "terranix-examples": {
       "locked": {
-        "lastModified": 1637156952,
+        "lastModified": 1636300201,
-        "narHash": "sha256-KqvXIe1yiKOEP9BRYqNQN+LOWPCsWojh0WjEgv5jfEI=",
+        "narHash": "sha256-0n1je1WpiR6XfCsvi8ZK7GrpEnMl+DpwhWaO1949Vbc=",
         "owner": "terranix",
         "repo": "terranix-examples",
-        "rev": "921680efb8af0f332d8ad73718d53907f9483e24",
+        "rev": "a934aa1cf88f6bd6c6ddb4c77b77ec6e1660bd5e",
         "type": "github"
       },
       "original": {

flake.nix

@@ -2,7 +2,7 @@
   description = "Bagel cooking infrastructure";
 
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
 
     terranix.url = "github:terranix/terranix";
     terranix.inputs.nixpkgs.follows = "nixpkgs";

hosts/build-coord/hardware.nix

@@ -38,7 +38,7 @@
 
   zramSwap = {
     enable = true;
-    memoryPercent = 100;
+    memoryPercent = 25;
   };
 
   networking.useNetworkd = true;

hosts/buildbot-lix/default.nix

@@ -38,9 +38,7 @@
       port = 2022;
       username = "buildbot";
     };
-    cors.allowedOrigins = [
+    cors.allowedOrigin = "*.lix.systems";
-      "https://*.lix.systems"
-    ];
     projects = [
       "lix"
       "lix-installer"

hosts/buildbot/default.nix

@@ -36,15 +36,13 @@
         port = cfgGerrit.port;
         username = "buildbot";
       };
-    cors.allowedOrigins = [
+    cors.allowedOrigin = "*.forkos.org";
-      "https://*.forkos.org"
-    ];
     projects = [
       "buildbot-test"
       "nixpkgs"
       "infra"
     ];
-    builders = [ "builder-9" ];
+    builders = [ "builder-10" ];
   };
 
   i18n.defaultLocale = "en_US.UTF-8";

hosts/gerrit01/default.nix

@@ -23,9 +23,6 @@
     };
   };
 
-  # Block all these crawlers!!
-  bagel.services.nginx.crawler-blocker.enable = true;
-
   fileSystems."/gerrit-data" = {
     device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";
     fsType = "ext4";
@@ -124,7 +121,7 @@
     };
   };
 
-  age.secrets.s3-channel-staging-keys.file = ../../secrets/floral/s3-channel-staging-keys.age;
+  age.secrets.s3-channel-staging-keys.file = ../../secrets/s3-channel-staging-keys.age;
   bagel.nixpkgs.channel-scripts = {
     enable = true;
     otlp.enable = true;

overlays/pyroscope/default.nix

@@ -1,10 +1,9 @@
 { lib
-, buildGo122Module
+, buildGoModule
 , fetchFromGitHub
 }:
 
-# FIXME: update, remove this pin
+buildGoModule rec {
-buildGo122Module rec {
   pname = "pyroscope";
   version = "1.7.1";
 

services/block-crawlers/blocked-ua.txt

@@ -1,40 +0,0 @@
-AI2Bot
-Ai2Bot-Dolma
-Amazonbot
-anthropic-ai
-Applebot
-Applebot-Extended
-Bytespider
-CCBot
-ChatGPT-User
-Claude-Web
-ClaudeBot
-cohere-ai
-Diffbot
-FacebookBot
-facebookexternalhit
-FriendlyCrawler
-Google-Extended
-GoogleOther
-GoogleOther-Image
-GoogleOther-Video
-GPTBot
-iaskspider/2.0
-ICC-Crawler
-ImagesiftBot
-img2dataset
-ISSCyberRiskCrawler
-Kangaroo Bot
-Meta-ExternalAgent
-Meta-ExternalFetcher
-OAI-SearchBot
-omgili
-omgilibot
-PerplexityBot
-PetalBot
-Scrapy
-Sidetrade indexer bot
-Timpibot
-VelenPublicWebCrawler
-Webzio-Extended
-YouBot

services/block-crawlers/default.nix

@@ -1,32 +0,0 @@
-{ pkgs, config, lib, ... }:
-let
-  inherit (lib) mkEnableOption mkIf mkOption types concatStringsSep mkDefault splitString;
-  cfg = config.bagel.services.nginx.crawler-blocker;
-  mkRobotsFile = blockedUAs: pkgs.writeText "robots.txt" ''
-    ${concatStringsSep "\n" (map (ua: "User-agent: ${ua}") blockedUAs)}
-    Disallow: /
-  '';
-in
-{
-  options = {
-    bagel.services.nginx.crawler-blocker = {
-      enable = mkEnableOption "the crawler blocker";
-
-      userAgents = mkOption {
-        type = types.listOf types.str;
-        default = splitString "\n" (builtins.readFile ./blocked-ua.txt);
-      };
-    };
-
-    services.nginx.virtualHosts = mkOption {
-      type = types.attrsOf (types.submodule {
-        config = {
-          locations."= /robots.txt" = mkIf cfg.enable (mkDefault {
-            alias = mkRobotsFile cfg.userAgents;
-          });
-        };
-      });
-    };
-  };
-}
-

services/buildbot/default.nix

@@ -53,9 +53,9 @@ in
       };
     };
 
-    cors.allowedOrigins = mkOption {
+    cors.allowedOrigin = mkOption {
-      type = types.listOf types.str;
+      type = types.str;
-      example = [ "*.forkos.org" ];
+      example = "*.forkos.org";
       description = "Allowed origin for Buildbot and NGINX for CORS without the protocol";
     };
 
@@ -100,7 +100,6 @@ in
     };
 
     services.nginx = {
-      recommendedProxySettings = true;
       appendHttpConfig = ''
         # Our session stuff is too big with the TWISTED_COOKIE in addition.
         # Default is usually 4k or 8k.
@@ -110,8 +109,8 @@ in
         forceSSL = true;
         enableACME = true;
         extraConfig = ''
-          # This is needed so that logged-in users in Buildbot can include their credentials in their requests.
           add_header Access-Control-Allow-Credentials 'true' always;
+          add_header Access-Control-Allow-Origin 'https://${cfg.cors.allowedOrigin}' always;
         '';
       };
     };
@@ -156,8 +155,9 @@ in
       # we can replace all of this with automatic localworker generation on buildbot-nix side.
       workersFile = config.age.secrets.buildbot-workers.path;
 
-      # We rely on NGINX to do the CORS dance.
+      allowedOrigins = [
-      allowedOrigins = cfg.cors.allowedOrigins;
+        cfg.cors.allowedOrigin
+      ];
 
       buildMachines = map (n: {
           hostName = nodes.${n}.config.networking.fqdn;
@@ -194,32 +194,6 @@ in
       RestartSteps = 10;
     };
 
-    services.postgresql.settings = {
-      # DB Version: 15
-      # OS Type: linux
-      # DB Type: web
-      # Total Memory (RAM): 64 GB
-      # CPUs num: 16
-      # Connections num: 100
-      # Data Storage: ssd
-      max_connections = 100;
-      shared_buffers = "16GB";
-      effective_cache_size = "48GB";
-      maintenance_work_mem = "2GB";
-      checkpoint_completion_target = 0.9;
-      wal_buffers = "16MB";
-      default_statistics_target = 100;
-      random_page_cost = 1.1;
-      effective_io_concurrency = 200;
-      work_mem = "41943kB";
-      huge_pages = "try";
-      min_wal_size = "1GB";
-      max_wal_size = "4GB";
-      max_worker_processes = 16;
-      max_parallel_workers_per_gather = 4;
-      max_parallel_workers = 16;
-      max_parallel_maintenance_workers = 4;
-    };
 
     nix.settings.keep-derivations = true;
     nix.gc = {

services/buildbot/lix.nix

@@ -21,8 +21,7 @@ in
       sshUser = "nix";
       systems = [ "aarch64-linux" ];
       publicHostKey = ssh-keys.machines.build02-aarch64-lix;
-      # TODO: use build02 features.
+      supportedFeatures = nodes.build02-aarch64-lix.config.nix.settings.system-features;
-      supportedFeatures = nodes.build01-aarch64-lix.config.nix.settings.system-features;
     }
     {
       hostName = "build01.aarch64-darwin.lix.systems";

services/default.nix

@@ -1,6 +1,5 @@
 {
   imports = [
-    ./block-crawlers
     ./gerrit
     ./channel-scripts
     ./hydra

services/gerrit/default.nix

@@ -129,9 +129,7 @@ in
       serverId = "9e5216ad-038d-4d74-a4e8-716515834a94";
 
       builtinPlugins = [
-        # Disable gitiles as it generates too much traffic.
+        "gitiles"
-        # Prefer git.forkos.org.
-        # "gitiles"
         "codemirror-editor"
         "reviewnotes"
         "download-commands"
@@ -143,9 +141,7 @@ in
       plugins = with pkgs.gerritPlugins; [
         oauth
         metrics-reporter-prometheus
-        # Theme plugin
+        # Buildbot checks plugin (writeText because services.gerrit.plugins expects packages)
-        (pkgs.concatText "theme.js" [ ./theme.js ])
-        # Buildbot checks plugin
         (pkgs.runCommand "checks.js" {
           BASE_URI = builtins.toJSON "https://buildbot.forkos.org";
           SUPPORTED_PROJECTS = builtins.toJSON [
@@ -330,7 +326,7 @@ in
       };
     };
 
-    age.secrets.gerrit-prometheus-bearer-token.file = ../../secrets/floral/gerrit-prometheus-bearer-token.age;
+    age.secrets.gerrit-prometheus-bearer-token.file = ../../secrets/gerrit-prometheus-bearer-token.age;
     bagel.monitoring.grafana-agent.exporters.gerrit = {
       port = 4778;  # grrt
       bearerTokenFile = config.age.secrets.gerrit-prometheus-bearer-token.path;

services/gerrit/git-gc-preserve.nix

@@ -57,7 +57,6 @@ in
           name = "git-gc-preserve-${name}";
           value = {
             description = "Git-GC-Preserve Service - ${name}";
-            path = [ pkgs.util-linux ];
             serviceConfig = {
               WorkingDirectory = gcConfig.repoPath;
               Type = "oneshot";

services/gerrit/theme.js

@@ -1,69 +0,0 @@
-/* Set up theming for Floral.
- * vim: set et ts=2 sw=2:
- */
-Gerrit.install((plugin) => {
-  const stylesheet = new CSSStyleSheet();
-  stylesheet.replace(`
-    html {
-      --header-title-content: 'floral.systems';
-
-      --blue-50: #f3f4fb;
-      --blue-100: #e3e6f6;
-      --blue-200: #ced5ef;
-      --blue-300: #acb8e4;
-      --blue-400: #8495d6;
-      --blue-500: #6775ca;
-      --blue-600: #5158bb;
-      --blue-700: #494bac;
-      --blue-800: #41408d;
-      --blue-900: #383870;
-      --blue-950: #252546;
-
-      --coral-50: #fff1f1;
-      --coral-100: #ffe0e0;
-      --coral-200: #ffc5c5;
-      --coral-300: #ff9e9d;
-      --coral-400: #ff6665;
-      --coral-500: #fe4a49;
-      --coral-600: #ec1716;
-      --coral-700: #c70f0e;
-      --coral-800: #a41110;
-      --coral-900: #881514;
-      --coral-950: #4a0505;
-
-      --teal-50: #eefbf5;
-      --teal-100: #d6f5e5;
-      --teal-200: #b1e9d0;
-      --teal-300: #7ed7b5;
-      --teal-400: #49be95;
-      --teal-500: #27a27b;
-      --teal-600: #188162;
-      --teal-700: #136951;
-      --teal-800: #125342;
-      --teal-900: #104437;
-      --teal-950: #08261f;
-
-      --zinc-50: #fafafa;
-      --zinc-100: #f4f4f5;
-      --zinc-200: #e4e4e7;
-      --zinc-300: #d4d4d8;
-      --zinc-400: #a1a1aa;
-      --zinc-500: #71717a;
-      --zinc-600: #52525b;
-      --zinc-700: #3f3f46;
-      --zinc-800: #27272a;
-      --zinc-900: #18181b;
-      --zinc-950: #09090b;
-    }
-    html.lightTheme {
-      --header-background-color: var(--teal-700);
-      --header-text-color: var(--coral-50);
-    }
-    html.darkTheme {
-      --header-background-color: var(--teal-900);
-      --header-text-color: var(--coral-50);
-    }
-  `).then(() => {
-    document.adoptedStyleSheets = [...document.adoptedStyleSheets, stylesheet];
-  });
-});

services/gerrit/www.nix

@@ -29,6 +29,10 @@ in
             # NGINX should not give up super fast. Things can take time.
             proxy_read_timeout 3600;
           }
+
+          location = /robots.txt {
+            return 200 'User-agent: *\nAllow: /';
+          }
         '';
       };
 

services/matrix/default.nix

@@ -32,11 +32,6 @@ in
           ];
           server_name = "forkos.org";
           database.backend = "rocksdb";
-
-          server_discovery = {
-            server.authority = "matrix.forkos.org:443";
-            client.base_url = "https://matrix.forkos.org";
-          };
         };
       };
 
@@ -53,7 +48,18 @@ in
           "forkos.org" = {
             forceSSL = true;
             enableACME = true;
-            locations."/.well-known/matrix".proxyPass = "http://grapevine";
+            locations = {
+              "= /.well-known/matrix/server".extraConfig = ''
+                add_header Content-Type application/json;
+                add_header Access-Control-Allow-Origin *;
+                return 200 '{"m.server": "matrix.forkos.org:443"}';
+              '';
+              "= /.well-known/matrix/client".extraConfig = ''
+                add_header Content-Type application/json;
+                add_header Access-Control-Allow-Origin *;
+                return 200 '{"m.homeserver": {"base_url": "https://matrix.forkos.org/"}, "m.identity_server": {"base_url": "https://matrix.org/"}, "org.matrix.msc3575.proxy": {"url": "https://matrix.forkos.org"}}';
+              '';
+            };
           };
         };
       };

services/monitoring/exporters/postgres.nix

@@ -23,9 +23,6 @@ in
     };
 
     services.postgresql.settings.shared_preload_libraries = "pg_stat_statements";
-    systemd.services.postgresql.postStart = lib.mkAfter ''
-      ${config.services.postgresql.package}/bin/psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;";
-    '';
 
     bagel.monitoring.grafana-agent.exporters.postgres.port = 9104;
   };

services/monitoring/lgtm/alerts/forkos.yaml

@@ -2,10 +2,10 @@ groups:
   - name: ForkOS automation
     rules:
       - alert: SyncFailedTooOften
-          expr: 'changes(node_systemd_unit_state{name=~"ows.*.service",state="failed"}[1d]) > 2'
+        expr: 'changes(node_systemd_unit_state{name=~"ows.*.service",state="failed"}[24h]) > 2'
         for: 30m
         labels:
           severity: critical
         annotations:
-            description: On {{ $labels.instance }}, the synchronization job has failed more than twice in the last 24 hours, check if there's a conflict or a stdenv change.
+          summary: "Synchronization job {{ $labels.name }} has failed more than twice in the last 24 hours"
-            summary: Synchronization job {{ $labels.name }} has failed more than twice in the last 24 hours
+          description: "On {{ $labels.instance }}, the synchronization job has failed more than twice in the last 24 hours, check if there's a conflict or a stdenv change."

services/monitoring/lgtm/alerts/postgres.yml

@@ -1,118 +1,102 @@
 groups:
 - name: PostgreSQL
   rules:
+
     - alert: PostgresqlTableNotAutoVacuumed
       expr: '(pg_stat_user_tables_last_autovacuum > 0) and (time() - pg_stat_user_tables_last_autovacuum) > 60 * 60 * 24 * 10'
+      for: 0m
       labels:
         severity: warning
       annotations:
-            description: |-
-                Table {{ $labels.relname }} has not been auto vacuumed for 10 days
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
         summary: Postgresql table not auto vacuumed (instance {{ $labels.instance }})
+        description: "Table {{ $labels.relname }} has not been auto vacuumed for 10 days\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
     - alert: PostgresqlTableNotAutoAnalyzed
       expr: '(pg_stat_user_tables_last_autoanalyze > 0) and (time() - pg_stat_user_tables_last_autoanalyze) > 24 * 60 * 60 * 10'
+      for: 0m
       labels:
         severity: warning
       annotations:
-            description: |-
-                Table {{ $labels.relname }} has not been auto analyzed for 10 days
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
         summary: Postgresql table not auto analyzed (instance {{ $labels.instance }})
+        description: "Table {{ $labels.relname }} has not been auto analyzed for 10 days\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
     - alert: PostgresqlDeadLocks
       expr: 'increase(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 5'
+      for: 0m
       labels:
         severity: warning
       annotations:
-            description: |-
-                PostgreSQL has dead-locks
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
         summary: Postgresql dead locks (instance {{ $labels.instance }})
+        description: "PostgreSQL has dead-locks\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
     - alert: PostgresqlHighRollbackRate
-          expr: 'sum by (namespace, datname) ((rate(pg_stat_database_xact_rollback{datid!="0",datname!~"template.*|postgres"}[3m])) / ((rate(pg_stat_database_xact_rollback{datid!="0",datname!~"template.*|postgres"}[3m])) + (rate(pg_stat_database_xact_commit{datid!="0",datname!~"template.*|postgres"}[3m])))) > 0.02'
+      expr: 'sum by (namespace,datname) ((rate(pg_stat_database_xact_rollback{datname!~"template.*|postgres",datid!="0"}[3m])) / ((rate(pg_stat_database_xact_rollback{datname!~"template.*|postgres",datid!="0"}[3m])) + (rate(pg_stat_database_xact_commit{datname!~"template.*|postgres",datid!="0"}[3m])))) > 0.02'
+      for: 0m
       labels:
         severity: warning
       annotations:
-            description: |-
-                Ratio of transactions being aborted compared to committed is > 2 %
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
         summary: Postgresql high rollback rate (instance {{ $labels.instance }})
+        description: "Ratio of transactions being aborted compared to committed is > 2 %\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
     - alert: PostgresqlHighRateStatementTimeout
       expr: 'rate(postgresql_errors_total{type="statement_timeout"}[1m]) > 3'
+      for: 0m
       labels:
         severity: critical
       annotations:
-            description: |-
-                Postgres transactions showing high rate of statement timeouts
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
         summary: Postgresql high rate statement timeout (instance {{ $labels.instance }})
+        description: "Postgres transactions showing high rate of statement timeouts\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
     - alert: PostgresqlHighRateDeadlock
       expr: 'increase(postgresql_errors_total{type="deadlock_detected"}[1m]) > 1'
+      for: 0m
       labels:
         severity: critical
       annotations:
-            description: |-
-                Postgres detected deadlocks
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
         summary: Postgresql high rate deadlock (instance {{ $labels.instance }})
+        description: "Postgres detected deadlocks\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
     - alert: PostgresqlTooManyDeadTuples
       expr: '((pg_stat_user_tables_n_dead_tup > 10000) / (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup)) >= 0.1'
       for: 2m
       labels:
         severity: warning
       annotations:
-            description: |-
-                PostgreSQL dead tuples is too large
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
         summary: Postgresql too many dead tuples (instance {{ $labels.instance }})
+        description: "PostgreSQL dead tuples is too large\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
     - alert: PostgresqlTooManyLocksAcquired
-          expr: '((sum(pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.2'
+      expr: '((sum (pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.20'
       for: 2m
       labels:
         severity: critical
       annotations:
-            description: |-
-                Too many locks acquired on the database. If this alert happens frequently, we may need to increase the postgres setting max_locks_per_transaction.
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
         summary: Postgresql too many locks acquired (instance {{ $labels.instance }})
+        description: "Too many locks acquired on the database. If this alert happens frequently, we may need to increase the postgres setting max_locks_per_transaction.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
     - alert: PostgresqlBloatIndexHigh(>80%)
-          expr: 'pg_bloat_btree_bloat_pct > 80 and on (idxname) (pg_bloat_btree_real_size > 1e+08)'
+      expr: 'pg_bloat_btree_bloat_pct > 80 and on (idxname) (pg_bloat_btree_real_size > 100000000)'
       for: 1h
       labels:
         severity: warning
       annotations:
-            description: |-
-                The index {{ $labels.idxname }} is bloated. You should execute `REINDEX INDEX CONCURRENTLY {{ $labels.idxname }};`
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
         summary: Postgresql bloat index high (> 80%) (instance {{ $labels.instance }})
+        description: "The index {{ $labels.idxname }} is bloated. You should execute `REINDEX INDEX CONCURRENTLY {{ $labels.idxname }};`\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
     - alert: PostgresqlBloatTableHigh(>80%)
-          expr: 'pg_bloat_table_bloat_pct > 80 and on (relname) (pg_bloat_table_real_size > 2e+08)'
+      expr: 'pg_bloat_table_bloat_pct > 80 and on (relname) (pg_bloat_table_real_size > 200000000)'
       for: 1h
       labels:
         severity: warning
       annotations:
-            description: |-
-                The table {{ $labels.relname }} is bloated. You should execute `VACUUM {{ $labels.relname }};`
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
         summary: Postgresql bloat table high (> 80%) (instance {{ $labels.instance }})
+        description: "The table {{ $labels.relname }} is bloated. You should execute `VACUUM {{ $labels.relname }};`\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
     - alert: PostgresqlInvalidIndex
       expr: 'pg_genaral_index_info_pg_relation_size{indexrelname=~".*ccnew.*"}'
       for: 6h
       labels:
         severity: warning
       annotations:
-            description: |-
-                The table {{ $labels.relname }} has an invalid index: {{ $labels.indexrelname }}. You should execute `DROP INDEX {{ $labels.indexrelname }};`
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
         summary: Postgresql invalid index (instance {{ $labels.instance }})
+        description: "The table {{ $labels.relname }} has an invalid index: {{ $labels.indexrelname }}. You should execute `DROP INDEX {{ $labels.indexrelname }};`\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

services/monitoring/lgtm/alerts/resources.yaml

@@ -7,94 +7,70 @@ groups:
         labels:
           severity: warning
         annotations:
-            description: |-
-                Node memory is filling up (< 10% left)
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
           summary: Host out of memory (instance {{ $labels.instance }})
+          description: "Node memory is filling up (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
       - alert: HostMemoryUnderMemoryPressure
         expr: (rate(node_vmstat_pgmajfault[1m]) > 1000) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
         for: 2m
         labels:
           severity: warning
         annotations:
-            description: |-
-                The node is under heavy memory pressure. High rate of major page faults
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
           summary: Host memory under memory pressure (instance {{ $labels.instance }})
+          description: "The node is under heavy memory pressure. High rate of major page faults\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
       - alert: HostMemoryIsUnderutilized
         expr: (100 - (avg_over_time(node_memory_MemAvailable_bytes[30m]) / node_memory_MemTotal_bytes * 100) < 20) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
         for: 1w
         labels:
           severity: info
         annotations:
-            description: |-
-                Node memory is < 20% for 1 week. Consider reducing memory space. (instance {{ $labels.instance }})
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
           summary: Host Memory is underutilized (instance {{ $labels.instance }})
+          description: "Node memory is < 20% for 1 week. Consider reducing memory space. (instance {{ $labels.instance }})\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
       - alert: HostOutOfDiskSpace
-          expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and on (instance, device, mountpoint) node_filesystem_readonly == 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+        expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
         for: 2m
         labels:
           severity: warning
         annotations:
-            description: |-
-                Disk is almost full (< 10% left)
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
           summary: Host out of disk space (instance {{ $labels.instance }})
+          description: "Disk is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
       - alert: HostDiskWillFillIn24Hours
-          expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and on (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and on (instance, device, mountpoint) node_filesystem_readonly == 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+        expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
         for: 2m
         labels:
           severity: warning
         annotations:
-            description: |-
-                Filesystem is predicted to run out of space within the next 24 hours at current write rate
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
           summary: Host disk will fill in 24 hours (instance {{ $labels.instance }})
+          description: "Filesystem is predicted to run out of space within the next 24 hours at current write rate\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
       - alert: HostCpuIsUnderutilized
         expr: (100 - (rate(node_cpu_seconds_total{mode="idle"}[30m]) * 100) < 20) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
         for: 1w
         labels:
           severity: info
         annotations:
-            description: |-
-                CPU load is < 20% for 1 week. Consider reducing the number of CPUs.
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
           summary: Host CPU is underutilized (instance {{ $labels.instance }})
+          description: "CPU load is < 20% for 1 week. Consider reducing the number of CPUs.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
       - alert: HostCpuStealNoisyNeighbor
         expr: (avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+        for: 0m
         labels:
           severity: warning
         annotations:
-            description: |-
-                CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
           summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
+          description: "CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
       - alert: HostOomKillDetected
         expr: (increase(node_vmstat_oom_kill[1m]) > 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+        for: 0m
         labels:
           severity: warning
         annotations:
-            description: |-
-                OOM kill detected
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
           summary: Host OOM kill detected (instance {{ $labels.instance }})
+          description: "OOM kill detected\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
       - alert: HostNetworkInterfaceSaturated
         expr: ((rate(node_network_receive_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"} > 0.8 < 10000) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
         for: 1m
         labels:
           severity: warning
         annotations:
-            description: |-
-                The network interface "{{ $labels.device }}" on "{{ $labels.instance }}" is getting overloaded.
-                  VALUE = {{ $value }}
-                  LABELS = {{ $labels }}
           summary: Host Network Interface Saturated (instance {{ $labels.instance }})
+          description: "The network interface \"{{ $labels.device }}\" on \"{{ $labels.instance }}\" is getting overloaded.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+

services/monitoring/lgtm/dashboards/postgres2.json

@@ -20,6 +20,7 @@
   "fiscalYearStartMonth": 0,
   "gnetId": 9628,
   "graphTooltip": 0,
+  "id": 27,
   "links": [],
   "liveNow": false,
   "panels": [
@@ -100,7 +101,6 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
-        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "mean"
@@ -112,14 +112,14 @@
         "textMode": "name",
         "wideLayout": true
       },
-      "pluginVersion": "11.2.1",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_static{tenant=\"$tenant\",hostname=\"$hostname\"}",
+          "expr": "pg_static{hostname=\"$hostname\"}",
           "format": "time_series",
           "instant": true,
           "intervalFactor": 1,
@@ -182,7 +182,6 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
-        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "mean"
@@ -194,14 +193,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.2.1",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_postmaster_start_time_seconds{tenant=\"$tenant\",hostname=\"$hostname\"} * 1000",
+          "expr": "pg_postmaster_start_time_seconds{hostname=\"$hostname\"} * 1000",
           "format": "time_series",
           "intervalFactor": 2,
           "legendFormat": "",
@@ -262,7 +261,6 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
-        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -274,14 +272,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.2.1",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "SUM(pg_stat_database_tup_fetched{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"})",
+          "expr": "SUM(pg_stat_database_tup_fetched{datname=~\"$datname\", hostname=~\"$hostname\"})",
           "format": "time_series",
           "intervalFactor": 2,
           "refId": "A",
@@ -342,7 +340,6 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
-        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -354,14 +351,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.2.1",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "SUM(pg_stat_database_tup_inserted{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"})",
+          "expr": "SUM(pg_stat_database_tup_inserted{datname=~\"$datname\", hostname=~\"$hostname\"})",
           "format": "time_series",
           "intervalFactor": 2,
           "refId": "A",
@@ -422,7 +419,6 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
-        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -434,14 +430,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.2.1",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "SUM(pg_stat_database_tup_updated{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"})",
+          "expr": "SUM(pg_stat_database_tup_updated{datname=~\"$datname\", hostname=~\"$hostname\"})",
           "format": "time_series",
           "intervalFactor": 2,
           "refId": "A",
@@ -502,7 +498,6 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
-        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "mean"
@@ -514,14 +509,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.2.1",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_settings_max_connections{tenant=\"$tenant\",hostname=\"$hostname\"}",
+          "expr": "pg_settings_max_connections{hostname=\"$hostname\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -607,7 +602,6 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
-        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -619,14 +613,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.2.1",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_settings_shared_buffers_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
+          "expr": "pg_settings_shared_buffers_bytes{hostname=\"$hostname\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -686,7 +680,6 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
-        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -698,14 +691,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.2.1",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_settings_effective_cache_size_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
+          "expr": "pg_settings_effective_cache_size_bytes{hostname=\"$hostname\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -765,7 +758,6 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
-        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -777,14 +769,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.2.1",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_settings_maintenance_work_mem_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
+          "expr": "pg_settings_maintenance_work_mem_bytes{hostname=\"$hostname\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -844,7 +836,6 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
-        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -856,14 +847,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.2.1",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_settings_work_mem_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
+          "expr": "pg_settings_work_mem_bytes{hostname=\"$hostname\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "",
@@ -925,7 +916,6 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
-        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -937,14 +927,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.2.1",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_settings_max_wal_size_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
+          "expr": "pg_settings_max_wal_size_bytes{hostname=\"$hostname\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -1004,7 +994,6 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
-        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -1016,14 +1005,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.2.1",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_settings_random_page_cost{tenant=\"$tenant\",hostname=\"$hostname\"}",
+          "expr": "pg_settings_random_page_cost{hostname=\"$hostname\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -1083,7 +1072,6 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
-        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -1095,7 +1083,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.2.1",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
@@ -1162,7 +1150,6 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
-        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "mean"
@@ -1174,14 +1161,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.2.1",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_settings_max_worker_processes{tenant=\"$tenant\",hostname=\"$hostname\"}",
+          "expr": "pg_settings_max_worker_processes{hostname=\"$hostname\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -1241,7 +1228,6 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
-        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -1253,14 +1239,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.2.1",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_settings_max_parallel_workers{tenant=\"$tenant\",hostname=\"$hostname\"}",
+          "expr": "pg_settings_max_parallel_workers{hostname=\"$hostname\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -1312,7 +1298,6 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
-            "barWidthFactor": 0.6,
             "drawStyle": "points",
             "fillOpacity": 10,
             "gradientMode": "none",
@@ -1387,7 +1372,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_stat_activity_count{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\", state=\"active\"} !=0",
+          "expr": "pg_stat_activity_count{datname=~\"$datname\", hostname=~\"$hostname\", state=\"active\"} !=0",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 2,
@@ -1416,7 +1401,6 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
-            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 10,
             "gradientMode": "none",
@@ -1490,7 +1474,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_database_xact_commit{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_xact_commit{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{datname}} commits",
@@ -1501,7 +1485,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_database_xact_rollback{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_xact_rollback{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{datname}} rollbacks",
@@ -1528,7 +1512,6 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
-            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 10,
             "gradientMode": "none",
@@ -1602,7 +1585,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_stat_database_tup_updated{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_updated{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
           "format": "time_series",
           "intervalFactor": 2,
           "legendFormat": "{{datname}}",
@@ -1630,7 +1613,6 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
-            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 10,
             "gradientMode": "none",
@@ -1704,7 +1686,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_stat_database_tup_fetched{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_fetched{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
           "format": "time_series",
           "intervalFactor": 2,
           "legendFormat": "{{datname}}",
@@ -1732,7 +1714,6 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
-            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 10,
             "gradientMode": "none",
@@ -1806,7 +1787,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_stat_database_tup_inserted{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_inserted{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
           "format": "time_series",
           "intervalFactor": 2,
           "legendFormat": "{{datname}}",
@@ -1834,7 +1815,6 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
-            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 10,
             "gradientMode": "none",
@@ -1910,7 +1890,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_locks_count{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\", mode=~\"$mode\"} != 0",
+          "expr": "pg_locks_count{datname=~\"$datname\", hostname=~\"$hostname\", mode=~\"$mode\"} != 0",
           "format": "time_series",
           "intervalFactor": 2,
           "legendFormat": "{{datname}},{{mode}}",
@@ -1968,7 +1948,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -2010,7 +1991,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_stat_database_tup_returned{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_returned{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
           "format": "time_series",
           "intervalFactor": 2,
           "legendFormat": "{{datname}}",
@@ -2069,7 +2050,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -2110,7 +2092,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_stat_activity_count{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\", state=~\"idle|idle in transaction|idle in transaction (aborted)\"}",
+          "expr": "pg_stat_activity_count{datname=~\"$datname\", hostname=~\"$hostname\", state=~\"idle|idle in transaction|idle in transaction (aborted)\"}",
           "format": "time_series",
           "intervalFactor": 2,
           "legendFormat": "{{datname}}, s: {{state}}",
@@ -2168,7 +2150,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -2210,7 +2193,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_stat_database_tup_deleted{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_deleted{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
           "format": "time_series",
           "intervalFactor": 2,
           "legendFormat": "{{datname}}",
@@ -2269,7 +2252,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -2310,7 +2294,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_stat_database_blks_hit{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"} / (pg_stat_database_blks_read{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"} + pg_stat_database_blks_hit{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"})",
+          "expr": "pg_stat_database_blks_hit{hostname=\"$hostname\", datname=~\"$datname\"} / (pg_stat_database_blks_read{hostname=\"$hostname\", datname=~\"$datname\"} + pg_stat_database_blks_hit{hostname=\"$hostname\", datname=~\"$datname\"})",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ datname }}",
@@ -2367,7 +2351,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -2410,7 +2395,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_bgwriter_buffers_backend{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_backend{hostname=\"$hostname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "buffers_backend",
@@ -2421,7 +2406,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_bgwriter_buffers_alloc{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_alloc{hostname=\"$hostname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "buffers_alloc",
@@ -2432,7 +2417,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{hostname=\"$hostname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "backend_fsync",
@@ -2443,7 +2428,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{hostname=\"$hostname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "buffers_checkpoint",
@@ -2454,7 +2439,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_bgwriter_buffers_clean{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_clean{hostname=\"$hostname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "buffers_clean",
@@ -2512,7 +2497,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -2554,7 +2540,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_database_conflicts{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_conflicts{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{datname}} conflicts",
@@ -2565,7 +2551,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_database_deadlocks{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_deadlocks{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{datname}} deadlocks",
@@ -2624,7 +2610,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -2666,7 +2653,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_database_temp_bytes{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_temp_bytes{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{datname}}",
@@ -2723,7 +2710,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -2766,7 +2754,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_bgwriter_checkpoint_write_time{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_checkpoint_write_time{hostname=\"$hostname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "write_time - Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk.",
@@ -2777,7 +2765,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time{hostname=\"$hostname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "sync_time - Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk.",
@@ -2878,32 +2866,8 @@
       {
         "current": {
           "selected": false,
-          "text": "fake",
+          "text": "bagel-box",
-          "value": "fake"
+          "value": "bagel-box"
-        },
-        "definition": "label_values(tenant)",
-        "hide": 0,
-        "includeAll": true,
-        "label": "Tenant",
-        "multi": true,
-        "name": "tenant",
-        "options": [],
-        "query": {
-          "qryType": 1,
-          "query": "label_values(tenant)",
-          "refId": "PrometheusVariableQueryEditor-VariableQuery"
-        },
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 0,
-        "type": "query"
-      },
-      {
-        "current": {
-          "selected": true,
-          "text": "buildbot",
-          "value": "buildbot"
         },
         "datasource": {
           "type": "prometheus",
@@ -2998,6 +2962,7 @@
     "from": "now-6h",
     "to": "now"
   },
+  "timeRangeUpdatedDuringEditOrView": false,
   "timepicker": {
     "refresh_intervals": [
       "5s",

services/monitoring/lgtm/grafana.nix

@@ -17,7 +17,7 @@ in
 
   config = mkIf cfg.enable {
     age.secrets.grafana-oauth-secret = {
-      file = ../../../secrets/floral/grafana-oauth-secret.age;
+      file = ../../../secrets/grafana-oauth-secret.age;
       owner = "grafana";
     };
 

services/monitoring/lgtm/loki.nix

@@ -13,10 +13,10 @@ in
   config = mkIf cfg.enable {
     age.secrets = {
       metrics-push-htpasswd = {
-        file = ../../../secrets/floral/metrics-push-htpasswd.age;
+        file = ../../../secrets/metrics-push-htpasswd.age;
         owner = "nginx";
       };
-      loki-environment.file = ../../../secrets/floral/loki-environment.age;
+      loki-environment.file = ../../../secrets/loki-environment.age;
     };
 
     services.loki = {

services/monitoring/lgtm/mimir.nix

@@ -9,15 +9,6 @@ let
   inherit (lib) mkEnableOption mkIf;
 
   mimirPort = config.services.mimir.configuration.server.http_listen_port;
-
-  alerts = pkgs.runCommand "mimir-alerts-checked" {
-    src = ./alerts;
-    nativeBuildInputs = with pkgs; [ prometheus.cli ];
-  } ''
-    promtool check rules $src/*
-    mkdir $out
-    cp -R $src $out/anonymous/
-  '';
 in
 {
   options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
@@ -25,11 +16,11 @@ in
   config = mkIf cfg.enable {
     age.secrets = {
       metrics-push-htpasswd = {
-        file = ../../../secrets/floral/metrics-push-htpasswd.age;
+        file = ../../../secrets/metrics-push-htpasswd.age;
         owner = "nginx";
       };
-      mimir-environment.file = ../../../secrets/floral/mimir-environment.age;
+      mimir-environment.file = ../../../secrets/mimir-environment.age;
-      mimir-webhook-url.file = ../../../secrets/floral/mimir-webhook-url.age;
+      mimir-webhook-url.file = ../../../secrets/mimir-webhook-url.age;
     };
 
     services.mimir = {
@@ -69,7 +60,10 @@ in
         blocks_storage.backend = "s3";
         ruler_storage = {
           backend = "local";
-          local.directory = alerts;
+          local.directory = pkgs.runCommand "mimir-rules" {} ''
+            mkdir -p $out
+            ln -s ${./alerts} $out/anonymous
+          '';
         };
 
         alertmanager = {

services/monitoring/lgtm/tempo.nix

@@ -13,10 +13,10 @@ in
   config = mkIf cfg.enable {
     age.secrets = {
       metrics-push-htpasswd = {
-        file = ../../../secrets/floral/metrics-push-htpasswd.age;
+        file = ../../../secrets/metrics-push-htpasswd.age;
         owner = "nginx";
       };
-      tempo-environment.file = ../../../secrets/floral/tempo-environment.age;
+      tempo-environment.file = ../../../secrets/tempo-environment.age;
     };
 
     services.tempo = {

services/netbox/default.nix

@@ -20,10 +20,7 @@ in
   };
 
   config = mkIf cfg.enable {
-    # FIXME: why
+    age.secrets.netbox-environment.file = ../../secrets/netbox-environment.age;
-    nixpkgs.config.permittedInsecurePackages = [ pkgs.netbox_3_7.name ];
-
-    age.secrets.netbox-environment.file = ../../secrets/floral/netbox-environment.age;
     services = {
       netbox = {
         enable = true;

services/ofborg/default.nix

@@ -26,12 +26,10 @@ in {
       webroot = "/var/lib/acme/.challenges";
       group = "rabbitmq";
     };
-
-    services.nginx.enable = true;
     services.nginx.virtualHosts.${amqpHost}.locations."/.well-known/acme-challenge".root =
       "/var/lib/acme/.challenges";
     systemd.services.rabbitmq.requires = ["acme-finished-${amqpHost}.target"];
 
-    networking.firewall.allowedTCPPorts = [ 80 443 amqpPort ];
+    networking.firewall.allowedTCPPorts = [ amqpPort ];
   };
 }

services/s3-revproxy/default.nix

@@ -70,7 +70,7 @@ in
   ];
 
   config = mkIf cfg.enable {
-    age.secrets.s3-revproxy-api-keys.file = ../../secrets/floral/s3-revproxy-api-keys.age;
+    age.secrets.s3-revproxy-api-keys.file = ../../secrets/s3-revproxy-api-keys.age;
     # For each target, generate an entry that passes it to the s3-revproxy.
     services.nginx.virtualHosts = mapAttrs' (subdomain: _: nameValuePair "${subdomain}.${cfg.domain}" (mkProxiedSubdomain subdomain)) cfg.targets;
     # this solves garage supporting neither anonymous access nor automatic

services/uptime-kuma/default.nix

@@ -59,7 +59,7 @@ in
       443
     ];
 
-    age.secrets.stateless-uptime-kuma-password.file = ../../secrets/floral/stateless-uptime-kuma-password.age;
+    age.secrets.stateless-uptime-kuma-password.file = ../../secrets/stateless-uptime-kuma-password.age;
     statelessUptimeKuma = {
       probesConfig = {
         monitors = lib.genAttrs subdomains (name: {