ofborg: enable nginx for certs

Flake lock file updates:

• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/e2f08f4d8b3ecb5cf5c9fd9cb2d53bb3c71807da' (2024-10-05)
  → 'github:NixOS/nixpkgs/7ffd9ae656aec493492b44d0ddfb28e79a1ea25d' (2024-11-02)

.envrc

@@ -1,2 +1,11 @@
+#!/usr/bin/env bash
+# the shebang is ignored, but nice for editors
+
 # shellcheck shell=bash
-use flake
+if type -P lorri &>/dev/null; then
+  eval "$(lorri direnv --flake .)"
+else
+  echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]'
+  use flake
+fi
+

README.md

@@ -1 +1,32 @@
-Infrastructure for the donut shaped thing that is absolutely not a donut.
+# Infrastructure for the donut shaped thing that is absolutely not a donut.
+
+## Quick start
+
+### Build the infrastructure
+
+```
+$ colmena build --on @localboot
+```
+
+Notice that `@localboot` is load-bearing as we have some machines that _cannot be_ deployed with vanilla Colmena. Fixing this is welcome.
+
+### Recommended deploy process
+
+```
+$ colmena apply dry-activate $machine # Verify that the nvd log is reasonable.
+$ colmena apply $machine
+```
+
+### Recommended upgrade process
+
+```
+$ nix flake update
+$ colmena apply dry-activate --on @localboot # Verify that the nvd log is reasonable. Run it twice to get only NVD logs shown.
+$ colmena apply --on @localboot
+```
+
+## Troubleshooting
+
+### I failed to deploy `gerrit01`
+
+Our Gerrit source build is known to have some hiccups sometimes, we are always interested in build logs, feel free to attach information in a new issue so we can make it more reliable.

common/admins.nix

@@ -1,17 +1,47 @@
+{ lib, ... }:
 let
-  keys = import ./ssh-keys.nix;
+  inherit (lib) genAttrs;
-in {
+in
-  users.users.root.openssh.authorizedKeys.keys =
+# Note: to add somefew in this list.
-    keys.users.delroth ++
+# Ensure their SSH key is already in common/ssh-keys.nix with
-    keys.users.emilylange ++
+# the same username for here, so that the keys is automatically added.
-    keys.users.hexchen ++
+{
-    keys.users.jade ++
+  bagel.groups = {
-    keys.users.janik ++
+    floral-infra.members = [
-    keys.users.k900 ++
+      "delroth"
-    keys.users.lukegb ++
+      "emilylange"
-    keys.users.maxine ++
+      "hexchen"
-    keys.users.raito ++
+      "jade"
-    keys.users.thubrecht ++
+      "janik"
-    keys.users.yuka ++
+      "k900"
-    keys.users.winter;
+      "maxine"
+      "raito"
+      "thubrecht"
+      "winter"
+      "yuka"
+      "ckie"
+    ];
+
+    lix-infra.members = [
+      "raito"
+      "hexchen"
+      "jade"
+      "pennae"
+    ];
+  };
+  bagel.users = genAttrs [
+    "delroth"
+    "emilylange"
+    "hexchen"
+    "jade"
+    "janik"
+    "k900"
+    "maxine"
+    "raito"
+    "thubrecht"
+    "winter"
+    "yuka"
+    "ckie"
+    "pennae"
+  ] (name: {});
 }

common/base-server.nix

@@ -1,11 +1,13 @@
 { lib, pkgs, ... }: {
   imports = [
     ./known-ssh-keys.nix
+    ./cgroups.nix
   ];
 
   nixpkgs.overlays = import ../overlays;
 
   nix.package = lib.mkDefault pkgs.lix;
+  system.tools.nixos-option.enable = false;
   services.openssh.enable = lib.mkForce true;
 
   networking.nftables.enable = true;
@@ -30,7 +32,7 @@
     automatic = true;
     persistent = true;
     dates = lib.mkDefault "daily";
-    options = "--delete-older-than 30d";
+    options = lib.mkDefault "--delete-older-than 30d";
   };
 
   services.journald.extraConfig = "SystemMaxUse=512M";

common/cgroups.nix

@@ -0,0 +1,83 @@
+# Relatively inspired by fbtax2:
+# https://facebookmicrosites.github.io/cgroup2/docs/fbtax-results.html
+#
+# See also the Chris Down talk at LISA'21:
+# https://www.usenix.org/conference/lisa21/presentation/down
+{ ... }:
+let
+  systemCriticalSliceConfig = {
+    ManagedOOMMemoryPressure = "kill";
+
+    # guarantee availability of memory
+    MemoryMin = "192M";
+    # default 100
+    IOWeight = 1000;
+    # default 100
+    CPUWeight = 1000;
+  };
+in
+{
+  systemd.oomd = {
+    enable = true;
+    # why not, we have cgroups at user level now so it'll just kill the
+    # terminal
+    enableRootSlice = true;
+    enableSystemSlice = true;
+    enableUserSlices = true;
+  };
+
+  systemd.enableCgroupAccounting = true;
+
+  systemd.services.nix-daemon = {
+    serviceConfig = {
+      # FIXME: how do i deprioritize this for memory
+      CPUWeight = 10;
+      IOWeight = 10;
+    };
+  };
+
+  systemd.slices.hostcritical = {
+    description = "Ensures that services to keep the system alive remain alive";
+
+    unitConfig = {
+      # required to avoid a dependency cycle on systemd-oomd. systemd will
+      # actually guess this right but we should fix it anyway.
+      DefaultDependencies = false;
+    };
+
+    sliceConfig = systemCriticalSliceConfig;
+  };
+
+  # make root logins higher priority for resources
+  systemd.slices."user-0" = {
+    sliceConfig = systemCriticalSliceConfig;
+  };
+
+
+  systemd.slices.system = {
+    sliceConfig = {
+      ManagedOOMMemoryPressure = "kill";
+      ManagedOOMMemoryPressureLimit = "50%";
+
+      IOWeight = 100;
+    };
+  };
+
+  systemd.services.sshd = {
+    serviceConfig = {
+      Slice = "hostcritical.slice";
+    };
+  };
+
+  systemd.services.systemd-oomd = {
+    serviceConfig = {
+      Slice = "hostcritical.slice";
+    };
+  };
+
+  systemd.services.systemd-journald = {
+    serviceConfig = {
+      Slice = "hostcritical.slice";
+    };
+  };
+}

common/default.nix

@@ -1,12 +1,14 @@
 {
   imports = [
     ./admins.nix
+    ./server-acl.nix
     ./base-server.nix
     ./hardening.nix
     ./nix.nix
     ./raito-proxy-aware-nginx.nix
-    ./raito-vm.nix
     ./sysadmin
+    ./hardware
     ./zsh.nix
+    ./secrets.nix
   ];
 }

common/hardware/default.nix

@@ -0,0 +1,7 @@
+{ ... }: {
+  imports = [
+    ./raito-vm.nix
+    ./oracle-vm.nix
+    ./hetzner.nix
+  ];
+}

common/hardware/hetzner.nix

@@ -0,0 +1,76 @@
+
+{ lib, config, ... }:
+let
+  cfg = config.bagel.hardware.hetzner;
+  inherit (lib) mkEnableOption mkIf mkOption types;
+in
+{
+  options.bagel.hardware.hetzner = {
+    enable = mkEnableOption "Hetzner's hardware defaults";
+
+    platformType = mkOption {
+      # Only VMs are supported.
+      type = types.enum [ "virtual-machine" ];
+    };
+
+    system = mkOption {
+      # Only the aarch64-linux VM Hetzner is supported.
+      type = types.enum [ "aarch64-linux" ];
+    };
+
+    networking.wan = {
+      mac = mkOption {
+        type = types.str;
+        description = "MAC address of the WAN interface in the Hetzner machine";
+      };
+      address = mkOption {
+        type = types.listOf types.str;
+        description = "List of static addresses attached to the WAN interface";
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    # A bunch of stuff is virtio.
+    boot.initrd.availableKernelModules = [
+      "xhci_pci"
+      "usbhid"
+      "sr_mod"
+      "virtio_gpu"
+      "virtio_scsi"
+      "virtio_rng"
+      "virtio_pci"
+    ];
+
+    boot.loader.systemd-boot.enable = true;
+    boot.loader.efi.canTouchEfiVariables = true;
+
+    networking.useDHCP = lib.mkDefault false;
+
+    # Stolen from the netplan provided by aarch64 Ubuntu images.
+    systemd.network.enable = true;
+    systemd.network.links."10-wan" = {
+      linkConfig.Name = "wan";
+      matchConfig.MACAddress = cfg.networking.mac;
+    };
+    systemd.network.networks."10-wan" = {
+      matchConfig.Name = "wan";
+      networkingConfig.Address = cfg.networking.address;
+      linkConfig.RequiredForOnline = true;
+      DHCP = "ipv4";
+      routes = [
+        {
+          routeConfig = {
+            Destination = "::/0";
+            GatewayOnLink = true;
+            Gateway = "fe80::1";
+          };
+        }
+      ];
+      dhcpV4Config = {
+        RouteMetric = 100;
+        UseMTU = true;
+      };
+    };
+  };
+}

common/hardware/oracle-vm.nix

@@ -0,0 +1,52 @@
+
+{ lib, config, modulesPath, ... }:
+let
+  cfg = config.bagel.hardware.oracle-vm;
+  inherit (lib) mkEnableOption mkIf mkOption types;
+in
+{
+  options.bagel.hardware.oracle-vm = {
+    enable = mkEnableOption "Oracle's VM hardware defaults";
+
+    system = mkOption {
+      # Only the free Oracle VMs are supported.
+      type = types.enum [ "aarch64-linux" ];
+    };
+  };
+
+  # Imports a bunch of virtio modules.
+  imports = [
+    "${modulesPath}/profiles/qemu-guest.nix"
+  ];
+
+  config = mkIf cfg.enable {
+    boot.loader.systemd-boot.enable = true;
+    boot.loader.efi.canTouchEfiVariables = true;
+    boot.initrd.systemd.enable = true;
+
+    boot.initrd.availableKernelModules = [
+      "xhci_pci" "virtio_pci" "usbhid" "sr_mod"
+    ];
+    boot.initrd.kernelModules = [ ];
+    boot.kernelModules = [ ];
+    boot.extraModulePackages = [ ];
+
+    nixpkgs.hostPlatform = cfg.system;
+
+    # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+    # (the default) this is the recommended approach. When using systemd-networkd it's
+    # still possible to use this option, but it's recommended to use it in conjunction
+    # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+    networking.useDHCP = lib.mkDefault false;
+    # Examples:
+    # 2: enp0s3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
+    #  link/ether 02:00:17:00:91:6e brd ff:ff:ff:ff:ff:ff
+    #  inet 10.0.0.94/24 brd 10.0.0.255 scope global dynamic noprefixroute enp0s3
+    #     valid_lft 44162sec preferred_lft 33362sec
+    #  inet6 fe80::17ff:fe00:916e/64 scope link
+    #     valid_lft forever preferred_lft forever
+    # [root@build02-aarch64-lahfa:~]# ip r
+    # default via 10.0.0.1 dev enp0s3 proto dhcp src 10.0.0.94 metric 1002 mtu 9000
+    networking.interfaces.enp0s3.useDHCP = lib.mkDefault true;
+  };
+}

common/known-ssh-keys.nix

@@ -2,5 +2,6 @@
 {
   programs.ssh.knownHosts = {
     "[cl.forkos.org]:29418".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM82mJ259C8Nc+BHHNBeRWXWhL3dfirQhmFbDAwHMle3";
+    "[gerrit.lix.systems]:2022".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICC/S6Z56uhv7zBMutkV0nU8eDuRcl3trykGWBch4L/l";
   };
 }

common/secrets.nix

@@ -0,0 +1,22 @@
+## This is a simple secret abstraction with multi-tenancy awareness.
+{ config, lib, ... }:
+let
+  cfg = config.bagel.secrets;
+  inherit (lib) mkOption types genAttrs;
+in
+{
+  options.bagel.secrets = {
+    tenant = mkOption {
+      type = types.enum [ "lix" "floral" ];
+    };
+
+    files = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+    };
+  };
+
+  config.age.secrets = genAttrs cfg.files (secretFile: {
+      file = ../secrets/${cfg.tenant}/${secretFile}.age;
+    });
+}

common/server-acl.nix

@@ -0,0 +1,69 @@
+{ lib, config, ... }:
+let
+  keys = import ./ssh-keys.nix;
+  inherit (lib) mkOption types length concatMap listToAttrs catAttrs attrValues;
+  cfgAdmins = config.bagel.admins;
+  cfgGroups = config.bagel.groups;
+  cfgUsers = config.bagel.users;
+
+  userOpts = { name, ... }: {
+    options = {
+      sshKeys = mkOption {
+        type = types.listOf types.str;
+        description = "List of SSH keys associated to this user, defaults to `ssh-keys.nix` entries.";
+        default = keys.users.${name} or [ ];
+      };
+    };
+  };
+  groupOpts = { name, ... }: {
+    options = {
+      members = mkOption {
+        type = types.listOf types.str;
+        description = "List of users member of this group";
+        example = [ "raito" ];
+        default = [ ];
+      };
+    };
+  };
+
+  # There might be duplicate in that list. We will turn it into an attribute set.
+  allowedMembers = listToAttrs (
+    map (member: {
+      name = member;
+      value = cfgUsers.${member};
+    }) (concatMap (allowedGroup: cfgGroups.${allowedGroup}.members) cfgAdmins.allowedGroups));
+
+  rootKeys = concatMap ({ sshKeys, ... }: sshKeys) (attrValues allowedMembers);
+in
+{
+  options.bagel.users = mkOption {
+    type = types.attrsOf (types.submodule userOpts);
+    description = "User configuration for server ACLs";
+  };
+
+  options.bagel.groups = mkOption {
+    type = types.attrsOf (types.submodule groupOpts);
+    description = "Group configuration for server ACLs";
+  };
+
+  options.bagel.admins = {
+    allowedGroups = mkOption {
+      type = types.listOf types.str;
+      default = [ "catch-all" ];
+      description = "List of groups which are allowed to admin this machine.";
+      example = [ "lix" "build-infra" ];
+    };
+  };
+
+  config = {
+    assertions = [
+      { assertion = length config.users.users.root.openssh.authorizedKeys.keys > 0;
+        # TODO: you can add printing of `concatStringsSep ", " cfg.allowedGroups` to diagnose
+        # which are the allowed groups and existing admins.
+        message = "root@${config.networking.fqdnOrHostName} has no SSH key attached, this machine will lose its access if you deploy it successfully! Set a valid `bagel.admins.allowedGroups` or ensure you have at least one administrator of the relevant group registered";
+      }
+    ];
+
+    users.users.root.openssh.authorizedKeys.keys = rootKeys;
+  };
+}

common/ssh-keys.nix

@@ -1,5 +1,6 @@
 {
   machines = {
+    # Floral
     bagel-box = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsO4bNqY04uG13Pg3ubHfRDssTphDLzZ4YUniE5/p+M";
     meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT";
     public01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPBy8G8rfLA6E9i+t5kjVafxU1c2NXATXKxoXTH4Kgtm";
@@ -20,6 +21,16 @@
     builder-9 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOhws9zGgocVY36dMtOL+CXadpvRMffxoWMkfEcTBJm7";
     builder-10 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE7sgIuTSqZiZhp8TvObSbIEhcHHsL5hcmYA22uzwxth";
     wob-vpn-gw = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINVytPPW8XnXf/rD5TFzsw//CZc2lBjQLmDzlVGPZsjh";
+
+    # Lix
+    build01-aarch64-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICC69NZD/zhIB/wUb5odg46bss5g8hH2fDl22bk4qeSW";
+    build02-aarch64-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGdJE375pe58RJbhKwXRp3D//+SJ3ssiVZrLsM9CLHn0";
+    build01-aarch64-darwin-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMVf1uO0lv5UBti/naW/+amqLxvWZg+StXk9aM+lJ7e4";
+
+    buildbot-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFoVSh35UqNQZ6ZZ1c6CzqERC40ovQ/KDXz8pC7nNlkR";
+
+    # Raito infrastructure
+    epyc-newtype-fr = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOXT9Init1MhKt4rjBANLq0t0bPww/WQZ96uB4AEDrml";
   };
 
   users = {
@@ -50,7 +61,12 @@
       "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
     ];
     thubrecht = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPM1jpXR7BWQa7Sed7ii3SbvIPRRlKb3G91qC0vOwfJn" ];
-    yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKath4/fDnlv/4fzxkPrQN1ttmoPRNu/m9bEtdPJBDfY cardno:16_933_242" ];
+    yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIxQ3NYBi8v1f/vhxLKDcA6upmX0pctRDbnK6SER5OUR yureka" ];
     winter = [ "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIH/LDRUG+U+++UmlxvA2kspioTjktQZ8taDcHq8gVlkfAAAABHNzaDo=" ];
+    ckie = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH3uTwzSSMAPg84fwbNp2cq9+BdLFeA1VzDGth4zCAbz https://mei.puppycat.house" ];
+    pennae = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC5Wf5/IbyFpdziWfwxkQqxOf3r1L9pYn6xQBEKFwmMY"
+      "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIK8icXjHkb4XzbIVN3djH4CE7RvgGd+3xbG4cgh0Yls5AAAABHNzaDo="
+    ];
   };
 }

common/sysadmin/default.nix

@@ -13,7 +13,11 @@ in
         tmux
         rsync
         fd
+        eza
+        grc
         ripgrep
+        delta
+        tshark
         pv
         kitty.terminfo
         config.boot.kernelPackages.perf

common/zsh.nix

@@ -1,5 +1,4 @@
 { lib, pkgs, config, ... }: {
-  users.defaultUserShell = pkgs.zsh;
   programs.zsh = {
     enable = true;
     enableCompletion = true;

flake.lock

@@ -27,16 +27,17 @@
       "inputs": {
         "crane": "crane",
         "flake-compat": "flake-compat_2",
-        "flake-utils": "flake-utils_2",
+        "flake-parts": "flake-parts_2",
+        "nix-github-actions": "nix-github-actions_2",
         "nixpkgs": "nixpkgs",
         "nixpkgs-stable": "nixpkgs-stable"
       },
       "locked": {
-        "lastModified": 1711742460,
+        "lastModified": 1730257295,
-        "narHash": "sha256-0O4v6e4a1toxXZ2gf5INhg4WPE5C5T+SVvsBt+45Mcc=",
+        "narHash": "sha256-OQl+aAsKiyygvpzck1u0sZf/R4T9zM903CgNDFmmzA8=",
         "owner": "zhaofengli",
         "repo": "attic",
-        "rev": "4dbdbee45728d8ce5788db6461aaaa89d98081f0",
+        "rev": "48c8b395bfbc6b76c7eae74df6c74351255a095c",
         "type": "github"
       },
       "original": {
@@ -49,11 +50,11 @@
     "bats-assert": {
       "flake": false,
       "locked": {
-        "lastModified": 1636059754,
+        "lastModified": 1692829535,
-        "narHash": "sha256-ewME0l27ZqfmAwJO4h5biTALc9bDLv7Bl3ftBzBuZwk=",
+        "narHash": "sha256-oDqhUQ6Xg7a3xx537SWLGRzqP3oKKeyY4UYGCdz9z/Y=",
         "owner": "bats-core",
         "repo": "bats-assert",
-        "rev": "34551b1d7f8c7b677c1a66fc0ac140d6223409e5",
+        "rev": "e2d855bc78619ee15b0c702b5c30fb074101159f",
         "type": "github"
       },
       "original": {
@@ -65,11 +66,11 @@
     "bats-support": {
       "flake": false,
       "locked": {
-        "lastModified": 1548869839,
+        "lastModified": 1693050811,
-        "narHash": "sha256-Gr4ntadr42F2Ks8Pte2D4wNDbijhujuoJi4OPZnTAZU=",
+        "narHash": "sha256-PxJaH16+QrsfZqtkWVt5K6TwJB5gjIXnbGo+MB84WIU=",
         "owner": "bats-core",
         "repo": "bats-support",
-        "rev": "d140a65044b2d6810381935ae7f0c94c7023c8c3",
+        "rev": "9bf10e876dd6b624fe44423f0b35e064225f7556",
         "type": "github"
       },
       "original": {
@@ -87,16 +88,16 @@
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1722939563,
+        "lastModified": 1730064416,
-        "narHash": "sha256-lMe8aXgF550iQLRaoU+yn8yYQ4x2qiyqANgsFyjfWwA=",
+        "narHash": "sha256-Opbtu9hKijGkEx+GYbSu3MJms3lFxZmAGTFyckguWMM=",
-        "ref": "refs/heads/non-flakes",
+        "ref": "refs/heads/forkos",
-        "rev": "4a162a8aa5dad6cecdb33bd8534e67e0bdaeb13f",
+        "rev": "79137b14f3cb376204f739f44b05aebfc288ca89",
-        "revCount": 295,
+        "revCount": 310,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
       },
       "original": {
-        "ref": "refs/heads/non-flakes",
+        "ref": "refs/heads/forkos",
         "type": "git",
         "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
       }
@@ -125,17 +126,18 @@
       "inputs": {
         "flake-compat": "flake-compat",
         "flake-utils": "flake-utils",
+        "nix-github-actions": "nix-github-actions",
         "nixpkgs": [
           "nixpkgs"
         ],
         "stable": "stable"
       },
       "locked": {
-        "lastModified": 1711386353,
+        "lastModified": 1731527002,
-        "narHash": "sha256-gWEpb8Hybnoqb4O4tmpohGZk6+aerAbJpywKcFIiMlg=",
+        "narHash": "sha256-dI9I6suECoIAmbS4xcrqF8r2pbmed8WWm5LIF1yWPw8=",
         "owner": "zhaofengli",
         "repo": "colmena",
-        "rev": "cd65ef7a25cdc75052fbd04b120aeb066c3881db",
+        "rev": "e3ad42138015fcdf2524518dd564a13145c72ea1",
         "type": "github"
       },
       "original": {
@@ -153,11 +155,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1702918879,
+        "lastModified": 1722960479,
-        "narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=",
+        "narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb",
+        "rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4",
         "type": "github"
       },
       "original": {
@@ -167,18 +169,12 @@
       }
     },
     "crane_2": {
-      "inputs": {
-        "nixpkgs": [
-          "grapevine",
-          "nixpkgs"
-        ]
-      },
       "locked": {
-        "lastModified": 1716569590,
+        "lastModified": 1730060262,
-        "narHash": "sha256-5eDbq8TuXFGGO3mqJFzhUbt5zHVTf5zilQoyW5jnJwo=",
+        "narHash": "sha256-RMgSVkZ9H03sxC+Vh4jxtLTCzSjPq18UWpiM0gq6shQ=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "109987da061a1bf452f435f1653c47511587d919",
+        "rev": "498d9f122c413ee1154e8131ace5a35a80d8fa76",
         "type": "github"
       },
       "original": {
@@ -219,11 +215,11 @@
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1716359173,
+        "lastModified": 1730442928,
-        "narHash": "sha256-pYcjP6Gy7i6jPWrjiWAVV0BCQp+DdmGaI/k65lBb/kM=",
+        "narHash": "sha256-U1DWb5c3EfkA7pqx5V1H4AWRA+EaE6UJ0lIRvK1RxgM=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "b6fc5035b28e36a98370d0eac44f4ef3fd323df6",
+        "rev": "87b4d20f896c99018dde4702a9c6157b516f2a76",
         "type": "github"
       },
       "original": {
@@ -252,11 +248,11 @@
     "flake-compat_2": {
       "flake": false,
       "locked": {
-        "lastModified": 1673956053,
+        "lastModified": 1696426674,
-        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
         "owner": "edolstra",
         "repo": "flake-compat",
-        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
         "type": "github"
       },
       "original": {
@@ -320,6 +316,28 @@
       }
     },
     "flake-parts_2": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "grapevine",
+          "attic",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1722555600,
+        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-parts_3": {
       "inputs": {
         "nixpkgs-lib": [
           "hydra",
@@ -341,6 +359,23 @@
         "type": "github"
       }
     },
+    "flake-parts_4": {
+      "inputs": {
+        "nixpkgs-lib": "nixpkgs-lib"
+      },
+      "locked": {
+        "lastModified": 1727826117,
+        "narHash": "sha256-K5ZLCyfO/Zj9mPFldf3iwS6oZStJcU4tSpiXTMYaaL0=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "3d04084d54bedc3d6b8b736c70ef449225c361b1",
+        "type": "github"
+      },
+      "original": {
+        "id": "flake-parts",
+        "type": "indirect"
+      }
+    },
     "flake-utils": {
       "locked": {
         "lastModified": 1659877975,
@@ -357,30 +392,15 @@
       }
     },
     "flake-utils_2": {
-      "locked": {
-        "lastModified": 1667395993,
-        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "flake-utils_3": {
       "inputs": {
         "systems": "systems_2"
       },
       "locked": {
-        "lastModified": 1710146030,
+        "lastModified": 1726560853,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
         "type": "github"
       },
       "original": {
@@ -390,21 +410,6 @@
         "type": "github"
       }
     },
-    "flake-utils_4": {
-      "locked": {
-        "lastModified": 1634851050,
-        "narHash": "sha256-N83GlSGPJJdcqhUxSCS/WwW5pksYf3VP1M13cDRTSVA=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "c91f3de5adaf1de973b797ef7485e441a65b8935",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
     "gerrit-dashboard": {
       "flake": false,
       "locked": {
@@ -427,20 +432,21 @@
         "crane": "crane_2",
         "fenix": "fenix",
         "flake-compat": "flake-compat_3",
-        "flake-utils": "flake-utils_3",
+        "flake-utils": "flake-utils_2",
         "nix-filter": "nix-filter",
         "nixpkgs": [
           "nixpkgs"
         ],
+        "rocksdb": "rocksdb",
         "rust-manifest": "rust-manifest"
       },
       "locked": {
         "host": "gitlab.computer.surgery",
-        "lastModified": 1723576377,
+        "lastModified": 1731820516,
-        "narHash": "sha256-sTa4XT5xMQkhhLknOfVd433YS1TvkMrE45qAsI1ZB6U=",
+        "narHash": "sha256-BJiFq5meQMwOLhVS9/m2HCZoNUIN5u5weJXr94dkIRU=",
         "owner": "matrix",
         "repo": "grapevine-fork",
-        "rev": "3b99032456700d06dd937db6a85976a8be9d4fa7",
+        "rev": "887e59cf03efb70bfa68b56dda32c411ba6cd5fa",
         "type": "gitlab"
       },
       "original": {
@@ -480,11 +486,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1724616313,
+        "lastModified": 1728321752,
-        "narHash": "sha256-9syppf9Gm/6F4wQQAbsf7rGY1DooMsprnsEY/0eaewg=",
+        "narHash": "sha256-GbBAoBF7ZObz0IP+g0LZKxMafpMvNKjTEu9haiZbV54=",
         "ref": "refs/heads/main",
-        "rev": "44b9a7b95d23e7a8587cb963f00382046707f2db",
+        "rev": "ee1234c15cdcb427dbd4828e0add09d02cd606c9",
-        "revCount": 4202,
+        "revCount": 4220,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/hydra.git"
       },
@@ -505,11 +511,11 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1723919517,
+        "lastModified": 1728163191,
-        "narHash": "sha256-D6+zmRXzr85p7riphuIrJQqangoJe70XM5jHhMWwXws=",
+        "narHash": "sha256-SW0IEBsPN1EysqzvfDT+8Kimtzy03O1BxQQm7ZB6fRY=",
         "ref": "refs/heads/main",
-        "rev": "278fddc317cf0cf4d3602d0ec0f24d1dd281fadb",
+        "rev": "ed9b7f4f84fd60ad8618645cc1bae2d686ff0db6",
-        "revCount": 16138,
+        "revCount": 16323,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/lix"
       },
@@ -520,12 +526,12 @@
     },
     "nix-eval-jobs": {
       "inputs": {
-        "flake-parts": "flake-parts_2",
+        "flake-parts": "flake-parts_3",
         "lix": [
           "hydra",
           "lix"
         ],
-        "nix-github-actions": "nix-github-actions",
+        "nix-github-actions": "nix-github-actions_3",
         "nixpkgs": [
           "hydra",
           "nixpkgs"
@@ -548,11 +554,11 @@
     },
     "nix-filter": {
       "locked": {
-        "lastModified": 1710156097,
+        "lastModified": 1730207686,
-        "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
+        "narHash": "sha256-SCHiL+1f7q9TAnxpasriP6fMarWE5H43t25F5/9e28I=",
         "owner": "numtide",
         "repo": "nix-filter",
-        "rev": "3342559a24e85fc164b295c3444e8a139924675b",
+        "rev": "776e68c1d014c3adde193a18db9d738458cd2ba4",
         "type": "github"
       },
       "original": {
@@ -583,6 +589,49 @@
       }
     },
     "nix-github-actions": {
+      "inputs": {
+        "nixpkgs": [
+          "colmena",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1729742964,
+        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
+    "nix-github-actions_2": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "attic",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1729742964,
+        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
+    "nix-github-actions_3": {
       "inputs": {
         "nixpkgs": [
           "hydra",
@@ -622,11 +671,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1711401922,
+        "lastModified": 1726042813,
-        "narHash": "sha256-QoQqXoj8ClGo0sqD/qWKFWezgEwUL0SUh37/vY2jNhc=",
+        "narHash": "sha256-LnNKCCxnwgF+575y0pxUdlGZBO/ru1CtGHIqQVfvjlA=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "07262b18b97000d16a4bdb003418bd2fb067a932",
+        "rev": "159be5db480d1df880a0135ca0bfed84c2f88353",
         "type": "github"
       },
       "original": {
@@ -636,6 +685,18 @@
         "type": "github"
       }
     },
+    "nixpkgs-lib": {
+      "locked": {
+        "lastModified": 1727825735,
+        "narHash": "sha256-0xHYkMkeLVQAMa7gvkddbPqpxph+hDzdu1XdGPJR+Os=",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
+      }
+    },
     "nixpkgs-regression": {
       "locked": {
         "lastModified": 1643052045,
@@ -654,32 +715,32 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1711460390,
+        "lastModified": 1724316499,
-        "narHash": "sha256-akSgjDZL6pVHEfSE6sz1DNSXuYX6hq+P/1Z5IoYWs7E=",
+        "narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "44733514b72e732bd49f5511bd0203dea9b9a434",
+        "rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.11",
+        "ref": "nixos-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1723221148,
+        "lastModified": 1731676054,
-        "narHash": "sha256-7pjpeQlZUNQ4eeVntytU3jkw9dFK3k1Htgk2iuXjaD8=",
+        "narHash": "sha256-OZiZ3m8SCMfh3B6bfGC/Bm4x3qc1m2SVEAlkV6iY7Yg=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "154bcb95ad51bc257c2ce4043a725de6ca700ef6",
+        "rev": "5e4fbfb6b3de1aa2872b76d49fafc942626e2add",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
+        "ref": "nixos-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -700,6 +761,23 @@
         "type": "github"
       }
     },
+    "rocksdb": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1730475155,
+        "narHash": "sha256-u5uuShM2SxHc9/zL4UU56IhCcR/ZQbzde0LgOYS44bM=",
+        "owner": "facebook",
+        "repo": "rocksdb",
+        "rev": "3c27a3dde0993210c5cc30d99717093f7537916f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "facebook",
+        "ref": "v9.7.4",
+        "repo": "rocksdb",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "agenix": "agenix",
@@ -715,17 +793,18 @@
         ],
         "nix-gerrit": "nix-gerrit",
         "nixpkgs": "nixpkgs_2",
+        "stateless-uptime-kuma": "stateless-uptime-kuma",
         "terranix": "terranix"
       }
     },
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1716107283,
+        "lastModified": 1730386175,
-        "narHash": "sha256-NJgrwLiLGHDrCia5AeIvZUHUY7xYGVryee0/9D3Ir1I=",
+        "narHash": "sha256-0Uq+/B8eu7pw8B8pxuGdFYKjcVLwNMcHfDxU9sXh7rg=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "21ec8f523812b88418b2bfc64240c62b3dd967bd",
+        "rev": "0ba893e1a00d92557ac91efb771d72eee36ca687",
         "type": "github"
       },
       "original": {
@@ -738,31 +817,47 @@
     "rust-manifest": {
       "flake": false,
       "locked": {
-        "narHash": "sha256-aZFye4UrtlcvLHrISldx4g9uGt3thDbVlLMK5keBSj0=",
+        "narHash": "sha256-tB9BZB6nRHDk5ELIVlGYlIjViLKBjQl52nC1avhcCwA=",
         "type": "file",
-        "url": "https://static.rust-lang.org/dist/channel-rust-1.78.0.toml"
+        "url": "https://static.rust-lang.org/dist/channel-rust-1.81.0.toml"
       },
       "original": {
         "type": "file",
-        "url": "https://static.rust-lang.org/dist/channel-rust-1.78.0.toml"
+        "url": "https://static.rust-lang.org/dist/channel-rust-1.81.0.toml"
       }
     },
     "stable": {
       "locked": {
-        "lastModified": 1696039360,
+        "lastModified": 1730883749,
-        "narHash": "sha256-g7nIUV4uq1TOVeVIDEZLb005suTWCUjSY0zYOlSBsyE=",
+        "narHash": "sha256-mwrFF0vElHJP8X3pFCByJR365Q2463ATp2qGIrDUdlE=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "32dcb45f66c0487e92db8303a798ebc548cadedc",
+        "rev": "dba414932936fde69f0606b4f1d87c5bc0003ede",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.05",
+        "ref": "nixos-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
+    "stateless-uptime-kuma": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1728243069,
+        "narHash": "sha256-l9fgwesnmFxasCaYUCD7L9bGGJXytLuwtx3CZMgpwJg=",
+        "ref": "refs/heads/master",
+        "rev": "880f444ff7862d6127b051cf1a993ad1585b1652",
+        "revCount": 25,
+        "type": "git",
+        "url": "https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git"
+      }
+    },
     "systems": {
       "locked": {
         "lastModified": 1681028828,
@@ -793,22 +888,38 @@
         "type": "github"
       }
     },
+    "systems_3": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
     "terranix": {
       "inputs": {
         "bats-assert": "bats-assert",
         "bats-support": "bats-support",
-        "flake-utils": "flake-utils_4",
+        "flake-parts": "flake-parts_4",
         "nixpkgs": [
           "nixpkgs"
         ],
+        "systems": "systems_3",
         "terranix-examples": "terranix-examples"
       },
       "locked": {
-        "lastModified": 1695406838,
+        "lastModified": 1728959489,
-        "narHash": "sha256-xiUfVD6rtsVWFotVtUW3Q1nQh4obKzgvpN1wqZuGXvM=",
+        "narHash": "sha256-1Pu2j5xsBTuoyga08ZVf+rKp3FOMmJh/0fXen/idOrA=",
         "owner": "terranix",
         "repo": "terranix",
-        "rev": "fc9077ca02ab5681935dbf0ecd725c4d889b9275",
+        "rev": "7734e2ee6a1472807a33ce1e7da794bed2aaf91c",
         "type": "github"
       },
       "original": {
@@ -819,11 +930,11 @@
     },
     "terranix-examples": {
       "locked": {
-        "lastModified": 1636300201,
+        "lastModified": 1637156952,
-        "narHash": "sha256-0n1je1WpiR6XfCsvi8ZK7GrpEnMl+DpwhWaO1949Vbc=",
+        "narHash": "sha256-KqvXIe1yiKOEP9BRYqNQN+LOWPCsWojh0WjEgv5jfEI=",
         "owner": "terranix",
         "repo": "terranix-examples",
-        "rev": "a934aa1cf88f6bd6c6ddb4c77b77ec6e1660bd5e",
+        "rev": "921680efb8af0f332d8ad73718d53907f9483e24",
         "type": "github"
       },
       "original": {

flake.nix

@@ -2,7 +2,7 @@
   description = "Bagel cooking infrastructure";
 
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
 
     terranix.url = "github:terranix/terranix";
     terranix.inputs.nixpkgs.follows = "nixpkgs";
@@ -22,12 +22,15 @@
     gerrit-dashboard.url = "git+https://git.lix.systems/the-distro/gerrit-monitoring.git";
     gerrit-dashboard.flake = false;
 
-    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/non-flakes";
+    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/forkos";
     buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";
 
     channel-scripts.url = "git+https://git.lix.systems/the-distro/channel-scripts.git";
     channel-scripts.inputs.nixpkgs.follows = "nixpkgs";
 
+    stateless-uptime-kuma.url = "git+https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git";
+    stateless-uptime-kuma.flake = false;
+
     lix.follows = "hydra/lix";
 
     grapevine = {
@@ -55,6 +58,7 @@
           inputs.lix.overlays.default
           inputs.nix-gerrit.overlays.default
           inputs.channel-scripts.overlays.default
+          (import "${inputs.stateless-uptime-kuma}/overlay.nix")
         ];
       };
       terraform = pkgs.opentofu;
@@ -64,7 +68,6 @@
           ./terraform
           {
             bagel.dnsimple.enable = true;
-            bagel.gandi.enable = true;
             bagel.hydra.enable = true;
           }
         ];
@@ -112,25 +115,90 @@
         ./common
       ];
 
-      makeBuilder = i: lib.nameValuePair "builder-${toString i}" {
+      floralInfraModules = commonModules ++ [
-        imports = commonModules;
+        ({ config, lib, ... }: {
-        bagel.baremetal.builders = { enable = true; num = i; netboot = i >= 6; };
+          # This means that anyone with @floral-infra permissions
+          # can ssh on root of every machines handled here.
+          bagel.admins.allowedGroups = [
+            "floral-infra"
+          ];
+
+          # Tag all machines which have local boot as local bootables.
+          deployment.tags = lib.mkMerge [
+            [ "floral" ]
+            (lib.mkIf (config.bagel.baremetal.builders.enable -> !config.bagel.baremetal.builders.netboot)
+              [ "localboot" ]
+            )
+          ];
+
+          bagel.monitoring.grafana-agent.tenant = "floral";
+          bagel.secrets.tenant = "floral";
+          bagel.builders.extra-build-capacity.provider.tenant = "floral";
+          bagel.services.buildbot.tenant = "floral";
+        })
+      ];
+
+      # These are Floral baremetal builders.
+      makeBuilder = i:
+      let
+        enableNetboot = i >= 6;
+      in
+      lib.nameValuePair "builder-${toString i}" {
+        imports = floralInfraModules;
+        bagel.baremetal.builders = { enable = true; num = i; netboot = enableNetboot; };
       };
 
+      lixInfraModules = commonModules ++ [
+        {
+          # This means that anyone with @lix-infra permissions
+          # can ssh on root of every machines handled here.
+          bagel.admins.allowedGroups = [
+            "lix-infra"
+          ];
+
+          # Tag all machines which have local boot as local bootables.
+          # Lix has no netbootable machine.
+          deployment.tags = [ "localboot" "lix" ];
+
+          bagel.monitoring.grafana-agent.tenant = "lix";
+          bagel.secrets.tenant = "lix";
+          bagel.builders.extra-build-capacity.provider = {
+            tenant = "lix";
+            buildfarmPublicKeys = [
+              # buildbot.lix.systems SSH key
+              "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDu4cEqZzAI/1vZjSQkTJ4ijIg9nuloOuSKUrnkJIOFn"
+            ];
+          };
+          bagel.services.buildbot.tenant = "lix";
+        }
+      ];
+
       builders = lib.listToAttrs (lib.genList makeBuilder 11);
     in {
       meta.nixpkgs = systemBits.x86_64-linux.pkgs;
+      # Add any non-x86_64 native systems here.
+      # Cross compilation is not supported yet.
+      meta.nodeNixpkgs =
+        let
+          aarch64-systems = systems: lib.genAttrs systems (system: systemBits.aarch64-linux.pkgs);
+        in
+        aarch64-systems [
+          "build01-aarch64-lix"
+        ];
       meta.specialArgs.inputs = inputs;
 
-      bagel-box.imports = commonModules ++ [ ./hosts/bagel-box ];
+      bagel-box.imports = floralInfraModules ++ [ ./hosts/bagel-box ];
-      meta01.imports = commonModules ++ [ ./hosts/meta01 ];
+      meta01.imports = floralInfraModules ++ [ ./hosts/meta01 ];
-      gerrit01.imports = commonModules ++ [ ./hosts/gerrit01 ];
+      gerrit01.imports = floralInfraModules ++ [ ./hosts/gerrit01 ];
-      fodwatch.imports = commonModules ++ [ ./hosts/fodwatch ];
+      fodwatch.imports = floralInfraModules ++ [ ./hosts/fodwatch ];
-      git.imports = commonModules ++ [ ./hosts/git ];
+      git.imports = floralInfraModules ++ [ ./hosts/git ];
-      wob-vpn-gw.imports = commonModules ++ [ ./hosts/wob-vpn-gw ];
+      wob-vpn-gw.imports = floralInfraModules ++ [ ./hosts/wob-vpn-gw ];
-      buildbot.imports = commonModules ++ [ ./hosts/buildbot ];
+      buildbot.imports = floralInfraModules ++ [ ./hosts/buildbot ];
-      public01.imports = commonModules ++ [ ./hosts/public01 ];
+      public01.imports = floralInfraModules ++ [ ./hosts/public01 ];
-      build-coord.imports = commonModules ++ [ ./hosts/build-coord ];
+      build-coord.imports = floralInfraModules ++ [ ./hosts/build-coord ];
+
+      build01-aarch64-lix.imports = lixInfraModules ++ [ ./hosts/build01-aarch64-lix ];
+      buildbot-lix.imports = lixInfraModules ++ [ ./hosts/buildbot-lix ];
     } // builders;
 
     hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.netbootDir or v.config.system.build.toplevel) self.nixosConfigurations;

hosts/build-coord/hardware.nix

@@ -38,7 +38,7 @@
 
   zramSwap = {
     enable = true;
-    memoryPercent = 25;
+    memoryPercent = 100;
   };
 
   networking.useNetworkd = true;

hosts/build01-aarch64-lix/default.nix

@@ -0,0 +1,27 @@
+{ ... }: {
+  networking.hostName = "build01";
+  networking.domain = "aarch64.lix.systems";
+
+  # Those free sweet VMs.
+  bagel.hardware.oracle-vm = {
+    enable = true;
+    system = "aarch64-linux";
+  };
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/a333323c-99f0-4258-8f68-496858d56f71";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/3E74-C937";
+      fsType = "vfat";
+    };
+
+  swapDevices = [ ];
+
+  bagel.builders.extra-build-capacity.provider.enable = true;
+  i18n.defaultLocale = "en_US.UTF-8";
+  system.stateVersion = "24.05";
+  deployment.targetHost = "build01.aarch64.lix.systems";
+}

hosts/buildbot-lix/default.nix

@@ -0,0 +1,71 @@
+# Configuration for a virtual machine in Raito's micro-DC basement.
+# 32 vCPU (2014 grade Xeon though)
+# 32GB RAM
+# 30GB SSD
+# 500GB HDD
+# All specifications can be upgraded to a certain extent, just ask Raito.
+# Hosts the coordinator for Buildbot.
+#
+# vim: et:ts=2:sw=2:
+#
+{ lib, modulesPath, ... }: {
+  networking.hostName = "buildbot";
+  networking.domain = "lix.systems";
+
+  zramSwap.enable = true;
+
+  bagel.sysadmin.enable = true;
+  # Buildbot is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.hardware.raito-vm = {
+    enable = true;
+    networking = {
+      nat-lan-mac = "BC:24:11:75:62:42";
+      wan = {
+        mac = "BC:24:11:B2:5F:2E";
+        address = "2001:bc8:38ee:100::200/56";
+      };
+    };
+  };
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  bagel.services.buildbot = {
+    enable = true;
+    domain = "buildbot.lix.systems";
+    gerrit =
+    {
+      domain = "gerrit.lix.systems";
+      port = 2022;
+      username = "buildbot";
+    };
+    cors.allowedOrigins = [
+      "https://*.lix.systems"
+    ];
+    projects = [
+      "lix"
+      "lix-installer"
+    ];
+    buildSystems = [
+      "x86_64-linux"
+      "aarch64-linux"
+      "aarch64-darwin"
+      # Too slow.
+      /* "x86_64-darwin" */
+    ];
+    # Lix is not allowed to use yet Floral's x86_64 builders for now.
+    builders = [ ];
+  };
+
+  # This machine does not use /nix from btrfs, and instead uses a store on a bigger disk.
+  fileSystems."/nix" =
+    lib.mkForce
+    { device = "/dev/disk/by-uuid/1815ca49-d0b0-4b99-8aec-0d790498ba6f";
+      fsType = "xfs";
+      neededForBoot = true;
+      options = [ "relatime" ];
+    };
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  system.stateVersion = "24.05";
+  deployment.targetHost = "buildbot.lix.systems";
+}

hosts/buildbot/default.nix

@@ -2,6 +2,7 @@
   config,
   lib,
   pkgs,
+  nodes,
   ...
 }:
 {
@@ -26,7 +27,24 @@
   bagel.services.buildbot = {
     enable = true;
     domain = "buildbot.forkos.org";
-    builders = [ "builder-10" ];
+    gerrit =
+      let
+        cfgGerrit = nodes.gerrit01.config.bagel.services.gerrit;
+      in
+      {
+        domain = cfgGerrit.canonicalDomain;
+        port = cfgGerrit.port;
+        username = "buildbot";
+      };
+    cors.allowedOrigins = [
+      "https://*.forkos.org"
+    ];
+    projects = [
+      "buildbot-test"
+      "nixpkgs"
+      "infra"
+    ];
+    builders = [ "builder-9" ];
   };
 
   i18n.defaultLocale = "en_US.UTF-8";

hosts/gerrit01/default.nix

@@ -23,6 +23,9 @@
     };
   };
 
+  # Block all these crawlers!!
+  bagel.services.nginx.crawler-blocker.enable = true;
+
   fileSystems."/gerrit-data" = {
     device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";
     fsType = "ext4";
@@ -39,7 +42,7 @@
   };
 
   age.secrets.ows-deploy-key = {
-    file = ../../secrets/ows-deploy-key.age;
+    file = ../../secrets/floral/ows-deploy-key.age;
     mode = "0600";
     owner = "git";
     group = "git";
@@ -121,7 +124,7 @@
     };
   };
 
-  age.secrets.s3-channel-staging-keys.file = ../../secrets/s3-channel-staging-keys.age;
+  age.secrets.s3-channel-staging-keys.file = ../../secrets/floral/s3-channel-staging-keys.age;
   bagel.nixpkgs.channel-scripts = {
     enable = true;
     otlp.enable = true;

hosts/public01/default.nix

@@ -9,6 +9,11 @@
   # TODO: make it the default
   networking.domain = "infra.forkos.org";
 
+  bagel.status = {
+    enable = true;
+    domain = "status.forkos.org";
+  };
+
   bagel.sysadmin.enable = true;
   # Newsletter is proxied.
   bagel.raito.v6-proxy-awareness.enable = true;

overlays/pyroscope/default.nix

@@ -1,9 +1,10 @@
 { lib
-, buildGoModule
+, buildGo122Module
 , fetchFromGitHub
 }:
 
-buildGoModule rec {
+# FIXME: update, remove this pin
+buildGo122Module rec {
   pname = "pyroscope";
   version = "1.7.1";
 

secrets.nix

@@ -1,52 +1,76 @@
 let
   keys = import common/ssh-keys.nix;
 
-  commonKeys = keys.users.delroth ++ keys.users.raito;
+  commonKeys = {
+    # WARNING: `keys.users.*` are *lists*, so you need concatenate them, don't put them into lists!
+    # Otherwise, agenix will be confused!
+    global = keys.users.raito;
+    lix = keys.users.hexchen ++ keys.users.jade;
+    floral = keys.users.delroth;
+  };
 
   secrets = with keys; {
-    hydra-postgres-key = [ machines.build-coord ];
+    floral = {
-    hydra-s3-credentials = [ machines.build-coord ];
+      hydra-postgres-key = [ machines.build-coord ];
-    hydra-signing-priv = [ machines.build-coord ];
+      hydra-s3-credentials = [ machines.build-coord ];
-    hydra-ssh-key-priv = [ machines.build-coord ];
+      hydra-signing-priv = [ machines.build-coord ];
+      hydra-ssh-key-priv = [ machines.build-coord ];
 
-    netbox-environment = [ machines.meta01 ];
+      netbox-environment = [ machines.meta01 ];
-    mimir-environment = [ machines.meta01 ];
+      mimir-environment = [ machines.meta01 ];
-    mimir-webhook-url = [ machines.meta01 ];
+      mimir-webhook-url = [ machines.meta01 ];
-    grafana-oauth-secret = [ machines.meta01 ];
+      grafana-oauth-secret = [ machines.meta01 ];
-    loki-environment = [ machines.meta01 ];
+      loki-environment = [ machines.meta01 ];
-    gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
+      gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
-    pyroscope-secrets = [ machines.meta01 ];
+      pyroscope-secrets = [ machines.meta01 ];
-    tempo-environment = [ machines.meta01 ];
+      tempo-environment = [ machines.meta01 ];
 
-    buildbot-worker-password = [ machines.buildbot ];
+      buildbot-worker-password = [ machines.buildbot ];
-    buildbot-oauth-secret = [ machines.buildbot ];
+      buildbot-oauth-secret = [ machines.buildbot ];
-    buildbot-workers = [ machines.buildbot ];
+      buildbot-workers = [ machines.buildbot ];
-    # Private SSH key to Gerrit
+      # Private SSH key to Gerrit
-    # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
+      # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
-    buildbot-service-key = [ machines.buildbot ];
+      buildbot-service-key = [ machines.buildbot ];
-    # Signing key for Buildbot's specific cache
+      # Signing key for Buildbot's specific cache
-    buildbot-signing-key = [ machines.buildbot ];
+      buildbot-signing-key = [ machines.buildbot ];
-    buildbot-remote-builder-key = [ machines.buildbot ];
+      buildbot-remote-builder-key = [ machines.buildbot ];
 
-    # These are the same password, but nginx wants it in htpasswd format
+      # These are the same password, but nginx wants it in htpasswd format
-    metrics-push-htpasswd = [ machines.meta01 ];
+      metrics-push-htpasswd = [ machines.meta01 ];
-    metrics-push-password = builtins.attrValues machines;
+      # Yes, even Lix machines are included in this monitoring infrastructure.
+      metrics-push-password = builtins.attrValues machines;
 
-    ows-deploy-key = [ machines.gerrit01 ];
+      ows-deploy-key = [ machines.gerrit01 ];
-    s3-channel-staging-keys = [ machines.gerrit01 ];
+      s3-channel-staging-keys = [ machines.gerrit01 ];
-    s3-channel-keys = [ machines.gerrit01 ];
+      s3-channel-keys = [ machines.gerrit01 ];
 
-    postgres-ca-priv = [ machines.bagel-box ];
+      postgres-ca-priv = [ machines.bagel-box ];
-    postgres-tls-priv = [ machines.bagel-box ];
+      postgres-tls-priv = [ machines.bagel-box ];
 
-    newsletter-secrets = [ machines.public01 ];
+      newsletter-secrets = [ machines.public01 ];
-    s3-revproxy-api-keys = [ machines.public01 ];
+      s3-revproxy-api-keys = [ machines.public01 ];
-    stateless-uptime-kuma-password = [ machines.public01 ];
+      stateless-uptime-kuma-password = [ machines.public01 ];
+    };
+
+    lix = {
+      buildbot-worker-password = [ machines.buildbot-lix ];
+      buildbot-oauth-secret = [ machines.buildbot-lix ];
+      buildbot-workers = [ machines.buildbot-lix ];
+      # Private SSH key to Gerrit
+      # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
+      buildbot-service-key = [ machines.buildbot-lix ];
+      # Signing key for Buildbot's specific cache
+      buildbot-signing-key = [ machines.buildbot-lix ];
+      buildbot-remote-builder-key = [ machines.buildbot-lix ];
+    };
   };
+
+  mkSecretListFor = tenant:
+    map (secretName: {
+      name = "secrets/${tenant}/${secretName}.age";
+      value.publicKeys = secrets.${tenant}."${secretName}" ++ commonKeys.global ++ commonKeys.${tenant};
+    }) (builtins.attrNames secrets.${tenant});
 in
   builtins.listToAttrs (
-    map (secretName: {
+    (mkSecretListFor "floral") ++ (mkSecretListFor "lix")
-      name = "secrets/${secretName}.age";
-      value.publicKeys = secrets."${secretName}" ++ commonKeys;
-    }) (builtins.attrNames secrets)
   )

secrets/floral/metrics-push-password.age

@@ -0,0 +1,68 @@
+age-encryption.org/v1
+-> ssh-ed25519 +HUDfA d5f2ESneC0wsoc9rwTjNfNXMBjCbjAQ7euthH2Buq1E
+5CynaQ8zhDRBvcmifhCsiDtllztCVAqs8rU36DOxgPw
+-> ssh-ed25519 +uvEmw EtYRis2LP0jv1W8mx8vFYNzkgi8OoqnA8cM2huS6NBk
+ll1csFIO+hVYk+I0uSVJmlDKj9aTWvf4kaYI5LJcm7w
+-> ssh-ed25519 DMaM1w ex4QJN8CG99J15i+yvqGEiEZn9OlGIC+cmLHL4u8ZEI
+VXnOv4CGK68q5t6hUV3oKAtxGZ+4FVbrmE1yMn16A0Q
+-> ssh-ed25519 sixKXw drXN6+q1y7L7ZU4chTfHfelu5GcTdff+i/UMFV0+3RQ
++8jmgnMh2OpQ3vhAuyQYWslfx7KO84a8KsCpoRD3Yl8
+-> ssh-ed25519 aHbF7w Af7NgjZ/Nvh5FHrX2VlF5riTIhJ+fdxTo6OR+8PcNwA
+ktKpm/HnOnw2Ym7xee3N1rneEX7+/xDhcp71N1NNHAA
+-> ssh-ed25519 87T2Ig 8mEUxJ/5NUvV+qQCDQH2Tm6Ryr5hf4xgsQlqXGf03Fw
+EavMcnsg/3EYBLQEBHX+0oTDKq5ZL4vj+mZntPM8UMU
+-> ssh-ed25519 Ao+7Wg UphWbatIaa+R1oZbfHazFhrawf0vax/3ZZS7YuX03Hs
+dwBbwoV0jpjiKr+nj+CRfUDgDl7ISpsCintVAzHnIFQ
+-> ssh-ed25519 wIR2ZA ZM58Nq7eJX9JVeYkoJf+mw8hxhYGoTx042ow1u3mJkw
+UtEaf7e4xsPO0ISlIF9LF+GcwTBqw4AXdMO4MASfgLQ
+-> ssh-ed25519 oGiV/Q G5KX/Eox+9md0yFRUZvGIsio2gWM17soHsL6H6zEX2g
+vI8jPjBAoFF0xhvRRLPzCMSiQOQ0fKuRb3CYVu3KUUo
+-> ssh-ed25519 gO3aog p9nZtjzoA0zJM+7Y6R16mpdub3dhu67yOYTUNKRytgI
+YL9vAp1+CK7jgmXkB47ufZMz+/swngkdUvEGR1zFZwc
+-> ssh-ed25519 N/+Clw 6LzFdtNsWewuJK2r97ZXJbRazvK3raN78UGanR/zWVU
+WT0y+sfDP3ffVwRcbYw51ArFR3OzXnoyi9IXwZZKEL8
+-> ssh-ed25519 CtkSZw CV0jQ5dIbgFtMxGK1X9b1qJOKmske8VgIPW5NW9mAwc
+clv7P3de61nZmXrvbOgL7Llw8ZqBMm2WFqgpznDwKv8
+-> ssh-ed25519 keg2lg 3Nk40ByQj8RThj4QDY2BdAkw55mXAJprXQRGjQqGvz0
+f8OFszJ8p90crFd+awEE12CNd7b22zgpH2XRKmH/Hf0
+-> ssh-ed25519 H885DA GDiJYH+gaC++TSqfoPDOTFcsCZRhEl0EeTeab7tgcWU
+kMILmwNMnMS7rgC3kKsAksu4Txn5owPU2y09h4aHKY8
+-> ssh-ed25519 Rq7K4Q VCNxGtCSCD2OYSWWwl0+yf189xV3QwRiwo80h4NPTkE
+hHkgYHLbISdl/RRdlInp9ub854M9ZKFSXpLgKW2YkmQ
+-> ssh-ed25519 vvyRpw XSCCrqEOtvzQRssI0U1DHirKoPgbOUKJxNKnioHhT2Y
+HGey1j0Kxae5Qs0aw6eqFziQGiRmNA+lEwbRdf5hhbM
+-> ssh-ed25519 aSEktQ mXY70Lgl76J4O5dPdDcIqmJ40EinigDuZrUghpGOq2I
+U2qeVFxGCYCEFWU+7vHc5Mu9EuzScowrjnwUyoqPj5U
+-> ssh-ed25519 cD6JxA at89poimBZyeeM8CQrxDxN0yCNDT2k04++py1fFycj8
+cQV/K5zc5x/oYnJ4N0MX3sTboT4G4ZNvVUVdHuJRzbA
+-> ssh-ed25519 ec6XRQ spJtb/xy4k4dmwKz8R2CPhC1WcuNV/rnDT978GkjHHk
+KrGEVGts/AhzbRNreqQ/CVanXL3l/9oMWxnpBLj23qU
+-> ssh-ed25519 1qYEfw KRkTYlvvnsCIExKQNmCyU7YxnGZsiI03kzecXNpLzUQ
+h2YagV7BzlsF7banzwXbOudTdlFzT7LC8PvtxAsX36U
+-> ssh-ed25519 2D+APA 4hdYlOnNIT9Q6tyKwXzy+u66Ti2EJopK43Sipebd0As
+tuesc9/QcEu4q9bTFJ5zJr0qvgLcmpn4at4cYtHrtbE
+-> ssh-ed25519 eTSU6g i1qT6PtepHXnoLCqDbhk86QG+SR9luQaw34a34gy5mw
+YE9VBAT5SLW2ECHRU+dMg9na6OQNVRVGuhY8vOdmE/Q
+-> ssh-ed25519 j2r2qQ TTTbSB/8UIDmmI3C9+u24PYZNfjl9jGADKHNWIwLfGE
+SNDforwii/GFp82TpyOcVIVrZWCe2QQKrjzPA6XA7Jc
+-> ssh-ed25519 CyxfgQ P5EiJ54v65Sz1gHuI0s170Z7c1WjcZLlb7NYigElfVs
+iYJUGpoE9LBIlv+O1navSSsy3EJ8tusXXX+/QAQvjNI
+-> ssh-ed25519 C/bBAQ hlBDpQRkcVCr3B6TCrBjxauhUX6Ndpm0s6x8W4hU6gM
+OFG3EuGJkSoEEXhbJ/Tp2DBdnBcs+hzxjNRdvcOSpQs
+-> ssh-ed25519 +qVung cGEGpO8NJfpj9ixAH9lhYkPKPDdQWryVxSOhMGQdnWM
++MycbIEab3P/AOS9i/YmPBDXB76hp3xUcWI4VMihV2w
+-> ssh-rsa krWCLQ
+Zv3dPYERlX1MaVaJTBDwIcjt1yLmu4Z7MovPgjGg01p+XsdBXeepTyOl+gRBwGgo
+AW5CIuaChYxtSNJ6nOgSaUpqzILycUF1xE1jROe3MIX2MZ4KGD1qoqcHbiCAng+a
+RqYrwAKnNea9FQMVfhYZBkRoYE6ne1R+0G6BoFM/okz24pAAFPBx+sMMhfTkt0uV
+kHVx0dgRw1pxa7Na98WH/7E0zp9VuBvVHGXfk1rfW/UQlbIO5RP3nldFoa6OmOWS
+JZ022UvjyC1re0KCurka4y+qmaiRKnTBmpIXxJFMwNCAQ8O8SeAQ3DHKHmXNMOIL
+ZVICtRRk0uX36AVU8DWDog
+-> ssh-ed25519 /vwQcQ kF8+hsA+0Msjd3q0SL52cae5RDqx4ls5kPKnc3UZyms
+Q33kIKJL3Vjxu7LQ5l4M3tlEuj+OW4uGh1x+JxthW8A
+-> ssh-ed25519 0R97PA gWBH71l6w9upTE0DwqOMSvWXc5VyJiKFAQLaSpWQ43E
+IrOrvzEa0bABw6UOpP8pM8WhuRNMaWJ2khljJIKwOS8
+-> ssh-ed25519 K3b7BA oS14iav9pSioLecMkOanJz89OJygLugvrnnTs5pKzz8
+akupMSiqXussXJyHwFm/f0imKALjowJVqd8/LFcC/58
+--- bCJXTEDaKg4FF2dQp2zFMU4Zm0zACekCuDvthcAyncM
+&­Ÿ
€Waïãà›BD R(¯¥Ñ”ufj<>úVÁ8nÆ>‚ß›øëæðZúâ{Idƒ„©,³*„%Ç“È‚z«

secrets/lix/buildbot-oauth-secret.age

@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 Ao+7Wg q7oRHUUlAvD8OUbpPT7d6eLMPWU0YS/verYTDE5BCkY
+/87/1uqOvRYeqel9IjnFmGyF9SXUQD8MTgfcj91b/Fs
+--- ulIeB91NJ7z/64h9BCLSD9/RW/zwv3m1Zo2ovNuInv8
+Îœç}³Óš#épÇ o>ä·*vµ÷ÄåŽs?[¦º´L
+<EFBFBD>þz™rý‰?R±Ñó7<Ê
+æi!€{X„¾òÓ

secrets/lix/buildbot-signing-key.age

@@ -0,0 +1,6 @@
+age-encryption.org/v1
+-> ssh-ed25519 Ao+7Wg EMpfs0EpWwaIKAoUBfEkyAHLIwi6JnGG6RvUWM5LjnU
+LKiwUBNc791U/GVRNlRPZE/TEMJjcFFrLruFJhiyiOI
+--- 0khp8u+4vHgGyQqP05m473Eo09eyOUZLI5+EK4olzoc
+N3(
+ª•û
xRq°<71>f<EFBFBD>Ó;ͼ3¬~RˆÓ
C^ñ+fœš1”®˜xˆ÷ÅëñSØ—hâ
£ÖË°GˆÓn–YIûµ:7¾!°u×Hþy/‰Øð‰™.¯¤á^¹lC™ôUÈËþ5cž:]ÿNž&'MÎè¶É-˜–ÆHF¦D0‘ cjô ‹Ð~

secrets/lix/buildbot-workers.age

@@ -0,0 +1,6 @@
+age-encryption.org/v1
+-> ssh-ed25519 Ao+7Wg RPKKoI5l5cYVdSvOxTHCUtwceac4vSi3+vlaqHr8kQg
+qbgTHCeQDNM30IJNZ/BU6wgryJwB316H5GWWaYB/wng
+--- GuFi3GSRdlBJ5YRjfAVyFDZ+4TH575kFJLnFp5847N0
+-èƒÞHÖÜ*x´M7¼t<,4ˆŠÑ^<5E>5@v°<>£€º,Z•MÒg=M
+» 3výJÄ«ÐÖê¿Nz­8'<^'4&WÂf"Êõ´À›ë\©º»ëêwmzúlAl|+„‘ÆKš~68ñE­Ýîk•8ø?S&òaM‹Ý~ž¹ê¿]Vfø ÝJxaõDù¥x

services/block-crawlers/blocked-ua.txt

@@ -0,0 +1,40 @@
+AI2Bot
+Ai2Bot-Dolma
+Amazonbot
+anthropic-ai
+Applebot
+Applebot-Extended
+Bytespider
+CCBot
+ChatGPT-User
+Claude-Web
+ClaudeBot
+cohere-ai
+Diffbot
+FacebookBot
+facebookexternalhit
+FriendlyCrawler
+Google-Extended
+GoogleOther
+GoogleOther-Image
+GoogleOther-Video
+GPTBot
+iaskspider/2.0
+ICC-Crawler
+ImagesiftBot
+img2dataset
+ISSCyberRiskCrawler
+Kangaroo Bot
+Meta-ExternalAgent
+Meta-ExternalFetcher
+OAI-SearchBot
+omgili
+omgilibot
+PerplexityBot
+PetalBot
+Scrapy
+Sidetrade indexer bot
+Timpibot
+VelenPublicWebCrawler
+Webzio-Extended
+YouBot

services/block-crawlers/default.nix

@@ -0,0 +1,32 @@
+{ pkgs, config, lib, ... }:
+let
+  inherit (lib) mkEnableOption mkIf mkOption types concatStringsSep mkDefault splitString;
+  cfg = config.bagel.services.nginx.crawler-blocker;
+  mkRobotsFile = blockedUAs: pkgs.writeText "robots.txt" ''
+    ${concatStringsSep "\n" (map (ua: "User-agent: ${ua}") blockedUAs)}
+    Disallow: /
+  '';
+in
+{
+  options = {
+    bagel.services.nginx.crawler-blocker = {
+      enable = mkEnableOption "the crawler blocker";
+
+      userAgents = mkOption {
+        type = types.listOf types.str;
+        default = splitString "\n" (builtins.readFile ./blocked-ua.txt);
+      };
+    };
+
+    services.nginx.virtualHosts = mkOption {
+      type = types.attrsOf (types.submodule {
+        config = {
+          locations."= /robots.txt" = mkIf cfg.enable (mkDefault {
+            alias = mkRobotsFile cfg.userAgents;
+          });
+        };
+      });
+    };
+  };
+}
+

services/buildbot/default.nix

@@ -7,15 +7,69 @@
 }:
 let
   cfg = config.bagel.services.buildbot;
-  cfgGerrit = nodes.gerrit01.config.bagel.services.gerrit;
   ssh-keys = import ../../common/ssh-keys.nix;
+  freeGbDiskSpace = 20;
+  extraTenantSpecificBuilders = {
+    lix = import ./lix.nix {
+      inherit config nodes;
+    };
+    floral = [ ];
+  }.${cfg.tenant or (throw "${cfg.tenant} is not a known tenant")};
+  clientId = {
+    lix = "buildbot";
+    floral = "forkos-buildbot";
+  }.${cfg.tenant or (throw "${cfg.tenant} is not a known tenant")};
   inherit (lib) mkEnableOption mkOption mkIf types;
 in
 {
   options.bagel.services.buildbot = {
     enable = mkEnableOption "Buildbot";
+
+    tenant = mkOption {
+      type = types.enum [ "lix" "floral" ];
+      description = "Which buildbot tenant to enable";
+    };
+
     domain = mkOption {
       type = types.str;
+      description = "Domain name for this Buildbot";
+    };
+
+    gerrit = {
+      domain = mkOption {
+        type = types.str;
+        description = "Canonical domain of the Gerrit associated to this Buildbot";
+        example = [ "cl.forkos.org" ];
+      };
+
+      port = mkOption {
+        type = types.port;
+        description = "Gerrit SSH port for this Buildbot";
+      };
+
+      username = mkOption {
+        type = types.str;
+        description = "Gerrit service username for this Buildbot";
+      };
+    };
+
+    cors.allowedOrigins = mkOption {
+      type = types.listOf types.str;
+      example = [ "*.forkos.org" ];
+      description = "Allowed origin for Buildbot and NGINX for CORS without the protocol";
+    };
+
+    buildSystems = mkOption {
+      type = types.listOf (types.enum [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]);
+      default = [ "x86_64-linux" ];
+      example = [ "x86_64-linux" "aarch64-linux" ];
+      description = "Supported build systems for this buildbot instance.";
+    };
+
+    projects = mkOption {
+      type = types.listOf types.str;
+      example = [ "nixpkgs" ];
+      description = "Static list of projects enabled for Buildbot CI";
     };
 
     builders = mkOption {
@@ -27,28 +81,39 @@ in
 
   config = mkIf cfg.enable {
     networking.firewall.allowedTCPPorts = [ 80 443 ];
-    age.secrets.buildbot-worker-password.file = ../../secrets/buildbot-worker-password.age;
+    bagel.secrets.files = [
-    age.secrets.buildbot-oauth-secret.file = ../../secrets/buildbot-oauth-secret.age;
+      "buildbot-worker-password"
-    age.secrets.buildbot-workers.file = ../../secrets/buildbot-workers.age;
+      "buildbot-oauth-secret"
-    age.secrets.buildbot-service-key.file = ../../secrets/buildbot-service-key.age;
+      "buildbot-workers"
+      "buildbot-service-key"
+      "buildbot-signing-key"
+      "buildbot-remote-builder-key"
+    ];
     age.secrets.buildbot-signing-key = {
-      file = ../../secrets/buildbot-signing-key.age;
       owner = "buildbot-worker";
       group = "buildbot-worker";
     };
     age.secrets.buildbot-remote-builder-key = {
-      file = ../../secrets/buildbot-remote-builder-key.age;
+      file = ../../secrets/${cfg.tenant}/buildbot-remote-builder-key.age;
       owner = "buildbot-worker";
       group = "buildbot-worker";
     };
 
-    services.nginx.virtualHosts.${cfg.domain} = {
+    services.nginx = {
-      forceSSL = true;
+      recommendedProxySettings = true;
-      enableACME = true;
+      appendHttpConfig = ''
-      extraConfig = ''
+        # Our session stuff is too big with the TWISTED_COOKIE in addition.
-        add_header Access-Control-Allow-Credentials 'true' always;
+        # Default is usually 4k or 8k.
-        add_header Access-Control-Allow-Origin 'https://cl.forkos.org' always;
+        large_client_header_buffers 4 16k;
       '';
+      virtualHosts.${cfg.domain} = {
+        forceSSL = true;
+        enableACME = true;
+        extraConfig = ''
+          # This is needed so that logged-in users in Buildbot can include their credentials in their requests.
+          add_header Access-Control-Allow-Credentials 'true' always;
+        '';
+      };
     };
 
     services.buildbot-nix.worker = {
@@ -74,30 +139,25 @@ in
       enable = true;
 
       inherit (cfg) domain;
-
+      # TODO(raito): is that really necessary when we can just collect buildMachines' systems?
-      debugging.enable = true;
+      inherit (cfg) buildSystems;
 
       oauth2 = {
         name = "Lix";
-        clientId = "forkos-buildbot";
+        inherit clientId;
         clientSecretFile = config.age.secrets.buildbot-oauth-secret.path;
         resourceEndpoint = "https://identity.lix.systems";
         authUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth";
         tokenUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
+        userinfoUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/userinfo";
       };
 
       # TODO(raito): this is not really necessary, we never have remote buildbot workers.
       # we can replace all of this with automatic localworker generation on buildbot-nix side.
       workersFile = config.age.secrets.buildbot-workers.path;
 
-      allowedOrigins = [
+      # We rely on NGINX to do the CORS dance.
-        "*.forkos.org"
+      allowedOrigins = cfg.cors.allowedOrigins;
-      ];
-
-      # TODO(raito): is that really necessary when we can just collect buildMachines' systems?
-      buildSystems = [
-        "x86_64-linux"
-      ];
 
       buildMachines = map (n: {
           hostName = nodes.${n}.config.networking.fqdn;
@@ -111,20 +171,14 @@ in
           # Contrary to how Nix works, here we can specify non-base64 public host keys.
           publicHostKey = ssh-keys.machines.${n};
         }
-      ) cfg.builders;
+      ) cfg.builders ++ extraTenantSpecificBuilders;
 
       gerrit = {
-        domain = cfgGerrit.canonicalDomain;
         # Manually managed account…
         # TODO: https://git.lix.systems/the-distro/infra/issues/69
-        username = "buildbot";
+        inherit (cfg.gerrit) domain port username;
-        port = cfgGerrit.port;
         privateKeyFile = config.age.secrets.buildbot-service-key.path;
-        projects = [
+        inherit (cfg) projects;
-          "buildbot-test"
-          "nixpkgs"
-          "infra"
-        ];
       };
 
       evalWorkerCount = 6;
@@ -133,10 +187,47 @@ in
       signingKeyFile = config.age.secrets.buildbot-signing-key.path;
     };
 
+    # Make PostgreSQL restart smoother.
+    systemd.services.postgresql.serviceConfig = {
+      Restart = "always";
+      RestartMaxDelaySec = "5m";
+      RestartSteps = 10;
+    };
+
+    services.postgresql.settings = {
+      # DB Version: 15
+      # OS Type: linux
+      # DB Type: web
+      # Total Memory (RAM): 64 GB
+      # CPUs num: 16
+      # Connections num: 100
+      # Data Storage: ssd
+      max_connections = 100;
+      shared_buffers = "16GB";
+      effective_cache_size = "48GB";
+      maintenance_work_mem = "2GB";
+      checkpoint_completion_target = 0.9;
+      wal_buffers = "16MB";
+      default_statistics_target = 100;
+      random_page_cost = 1.1;
+      effective_io_concurrency = 200;
+      work_mem = "41943kB";
+      huge_pages = "try";
+      min_wal_size = "1GB";
+      max_wal_size = "4GB";
+      max_worker_processes = 16;
+      max_parallel_workers_per_gather = 4;
+      max_parallel_workers = 16;
+      max_parallel_maintenance_workers = 4;
+    };
+
     nix.settings.keep-derivations = true;
     nix.gc = {
       automatic = true;
       dates = "hourly";
+      options = ''
+        --max-freed "$((${toString freeGbDiskSpace} * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"
+      '';
     };
   };
 }

services/buildbot/lix.nix

@@ -0,0 +1,50 @@
+{ config, nodes, ... }:
+let
+  ssh-keys = import ../../common/ssh-keys.nix;
+in
+  [
+    {
+      hostName = "build01.aarch64.lix.systems";
+      maxJobs = 2;
+      protocol = "ssh-ng";
+      sshKey = config.age.secrets.buildbot-remote-builder-key.path;
+      sshUser = "nix";
+      systems = [ "aarch64-linux" ];
+      publicHostKey = ssh-keys.machines.build01-aarch64-lix;
+      supportedFeatures = nodes.build01-aarch64-lix.config.nix.settings.system-features;
+    }
+    {
+      hostName = "build02.aarch64.lix.systems";
+      maxJobs = 4;
+      protocol = "ssh-ng";
+      sshKey = config.age.secrets.buildbot-remote-builder-key.path;
+      sshUser = "nix";
+      systems = [ "aarch64-linux" ];
+      publicHostKey = ssh-keys.machines.build02-aarch64-lix;
+      # TODO: use build02 features.
+      supportedFeatures = nodes.build01-aarch64-lix.config.nix.settings.system-features;
+    }
+    {
+      hostName = "build01.aarch64-darwin.lix.systems";
+      maxJobs = 2;
+      protocol = "ssh-ng";
+      sshKey = config.age.secrets.buildbot-remote-builder-key.path;
+      sshUser = "m1";
+      systems = [ "aarch64-darwin" "x86_64-darwin" ];
+      publicHostKey = ssh-keys.machines.build01-aarch64-darwin-lix;
+      supportedFeatures = [ "big-parallel" ];
+    }
+    # a.k.a. https://git.newtype.fr/newtype/newtype-org-configurations/src/branch/main/docs/epyc.md
+    {
+      hostName = "epyc.infra.newtype.fr";
+      # at 256G this could run 64 builds but the machine is shared
+      # (and historically we used no more than 16 concurrent jobs)
+      maxJobs = 16;
+      protocol = "ssh-ng";
+      sshKey = config.age.secrets.buildbot-remote-builder-key.path;
+      sshUser = "nix";
+      systems = [ "x86_64-linux" "i686-linux" ];
+      publicHostKey = ssh-keys.machines.epyc-newtype-fr;
+      supportedFeatures = [ "benchmark" "big-parallel" "nixos-test" "kvm" ];
+    }
+  ]

services/default.nix

@@ -1,10 +1,12 @@
 {
   imports = [
+    ./block-crawlers
     ./gerrit
     ./channel-scripts
     ./hydra
     ./matrix
     ./monitoring
+    ./uptime-kuma
     ./netbox
     ./ofborg
     ./postgres
@@ -13,5 +15,6 @@
     ./buildbot
     ./newsletter
     ./s3-revproxy
+    ./extra-builders
   ];
 }

services/extra-builders/default.nix

@@ -0,0 +1,6 @@
+{
+  imports = [
+    # Remote builders
+    ./provider.nix
+  ];
+}

services/extra-builders/provider.nix

@@ -0,0 +1,46 @@
+## Tenant-specific build capacity.
+## This can come from anywhere and is not hold to the same level of responsibility that our build-infra has.
+{ pkgs, config, lib, nodes, ... }:
+let
+  inherit (lib) mkIf types mkEnableOption mkOption;
+  freeGbDiskSpace = 10;
+  cfg = config.bagel.builders.extra-build-capacity.provider;
+in
+{
+  options.bagel.builders.extra-build-capacity.provider = {
+    enable = mkEnableOption "providing of extra build capacity to other systems";
+
+    buildfarmPublicKeys = mkOption {
+      type = types.listOf types.str;
+      description = "SSH public keys to allow to connect for remote builds";
+    };
+
+    # TODO: register tenant in some deployment wide module
+    # so that the consumer side can just automatically generate buildMachines entries.
+    tenant = mkOption {
+      type = types.enum [ "lix" ];
+    };
+  };
+
+  config = mkIf cfg.enable {
+    users.groups.builders = {};
+    users.users.nix = {
+      openssh.authorizedKeys.keys = cfg.buildfarmPublicKeys;
+      extraGroups = [ "builders" ];
+      isNormalUser = true;
+    };
+
+    nix.settings.allowed-users = [ "@wheel" "@builders" ];
+    nix.settings.trusted-users = [ "@builders" ];
+
+    nix.gc.automatic = true;
+    nix.gc.dates = "hourly";
+    nix.gc.options = ''
+      --max-freed "$((${toString freeGbDiskSpace} * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"
+    '';
+    # Bump the open files limit so that non-root users can run NixOS VM tests, if supported at all.
+    security.pam.loginLimits = [
+      { domain = "*"; item = "nofile"; type = "-"; value = "20480"; }
+    ];
+  };
+}

services/gerrit/default.nix

@@ -46,7 +46,7 @@ in
 
   config = mkIf cfg.enable {
     networking.firewall.allowedTCPPorts = [ cfg.port ];
-    age.secrets.alloy-push-password.file = ../../secrets/metrics-push-password.age;
+    age.secrets.alloy-push-password.file = ../../secrets/floral/metrics-push-password.age;
 
     environment.systemPackages = [ jdk
       pkgs.git
@@ -129,7 +129,9 @@ in
       serverId = "9e5216ad-038d-4d74-a4e8-716515834a94";
 
       builtinPlugins = [
-        "gitiles"
+        # Disable gitiles as it generates too much traffic.
+        # Prefer git.forkos.org.
+        # "gitiles"
         "codemirror-editor"
         "reviewnotes"
         "download-commands"
@@ -141,7 +143,9 @@ in
       plugins = with pkgs.gerritPlugins; [
         oauth
         metrics-reporter-prometheus
-        # Buildbot checks plugin (writeText because services.gerrit.plugins expects packages)
+        # Theme plugin
+        (pkgs.concatText "theme.js" [ ./theme.js ])
+        # Buildbot checks plugin
         (pkgs.runCommand "checks.js" {
           BASE_URI = builtins.toJSON "https://buildbot.forkos.org";
           SUPPORTED_PROJECTS = builtins.toJSON [
@@ -326,7 +330,7 @@ in
       };
     };
 
-    age.secrets.gerrit-prometheus-bearer-token.file = ../../secrets/gerrit-prometheus-bearer-token.age;
+    age.secrets.gerrit-prometheus-bearer-token.file = ../../secrets/floral/gerrit-prometheus-bearer-token.age;
     bagel.monitoring.grafana-agent.exporters.gerrit = {
       port = 4778;  # grrt
       bearerTokenFile = config.age.secrets.gerrit-prometheus-bearer-token.path;

services/gerrit/git-gc-preserve.nix

@@ -57,6 +57,7 @@ in
           name = "git-gc-preserve-${name}";
           value = {
             description = "Git-GC-Preserve Service - ${name}";
+            path = [ pkgs.util-linux ];
             serviceConfig = {
               WorkingDirectory = gcConfig.repoPath;
               Type = "oneshot";

services/gerrit/theme.js

@@ -0,0 +1,69 @@
+/* Set up theming for Floral.
+ * vim: set et ts=2 sw=2:
+ */
+Gerrit.install((plugin) => {
+  const stylesheet = new CSSStyleSheet();
+  stylesheet.replace(`
+    html {
+      --header-title-content: 'floral.systems';
+
+      --blue-50: #f3f4fb;
+      --blue-100: #e3e6f6;
+      --blue-200: #ced5ef;
+      --blue-300: #acb8e4;
+      --blue-400: #8495d6;
+      --blue-500: #6775ca;
+      --blue-600: #5158bb;
+      --blue-700: #494bac;
+      --blue-800: #41408d;
+      --blue-900: #383870;
+      --blue-950: #252546;
+
+      --coral-50: #fff1f1;
+      --coral-100: #ffe0e0;
+      --coral-200: #ffc5c5;
+      --coral-300: #ff9e9d;
+      --coral-400: #ff6665;
+      --coral-500: #fe4a49;
+      --coral-600: #ec1716;
+      --coral-700: #c70f0e;
+      --coral-800: #a41110;
+      --coral-900: #881514;
+      --coral-950: #4a0505;
+
+      --teal-50: #eefbf5;
+      --teal-100: #d6f5e5;
+      --teal-200: #b1e9d0;
+      --teal-300: #7ed7b5;
+      --teal-400: #49be95;
+      --teal-500: #27a27b;
+      --teal-600: #188162;
+      --teal-700: #136951;
+      --teal-800: #125342;
+      --teal-900: #104437;
+      --teal-950: #08261f;
+
+      --zinc-50: #fafafa;
+      --zinc-100: #f4f4f5;
+      --zinc-200: #e4e4e7;
+      --zinc-300: #d4d4d8;
+      --zinc-400: #a1a1aa;
+      --zinc-500: #71717a;
+      --zinc-600: #52525b;
+      --zinc-700: #3f3f46;
+      --zinc-800: #27272a;
+      --zinc-900: #18181b;
+      --zinc-950: #09090b;
+    }
+    html.lightTheme {
+      --header-background-color: var(--teal-700);
+      --header-text-color: var(--coral-50);
+    }
+    html.darkTheme {
+      --header-background-color: var(--teal-900);
+      --header-text-color: var(--coral-50);
+    }
+  `).then(() => {
+    document.adoptedStyleSheets = [...document.adoptedStyleSheets, stylesheet];
+  });
+});

services/gerrit/www.nix

@@ -29,10 +29,6 @@ in
             # NGINX should not give up super fast. Things can take time.
             proxy_read_timeout 3600;
           }
-
-          location = /robots.txt {
-            return 200 'User-agent: *\nAllow: /';
-          }
         '';
       };
 

services/hydra/default.nix

@@ -66,17 +66,19 @@ in {
     # does indeed have our public SSH key and are *builders*
     # as a simple evaluation preflight check.
 
-    age.secrets.hydra-s3-credentials.file = ../../secrets/hydra-s3-credentials.age;
+    bagel.secrets.files = [
+      "hydra-s3-credentials"
+      "hydra-postgres-key"
+      "hydra-signing-priv"
+      "hydra-ssh-key-priv"
+    ];
 
     age.secrets.hydra-postgres-key.group = "hydra";
     age.secrets.hydra-postgres-key.mode = "0440";
-    age.secrets.hydra-postgres-key.file = ../../secrets/hydra-postgres-key.age;
 
     age.secrets.hydra-signing-priv.owner = "hydra-queue-runner";
-    age.secrets.hydra-signing-priv.file = ../../secrets/hydra-signing-priv.age;
 
     age.secrets.hydra-ssh-key-priv.owner = "hydra-queue-runner";
-    age.secrets.hydra-ssh-key-priv.file = ../../secrets/hydra-ssh-key-priv.age;
 
     systemd.tmpfiles.rules = [
       "d /var/cache/hydra 0755 hydra hydra -  -"

services/matrix/default.nix

@@ -32,6 +32,11 @@ in
           ];
           server_name = "forkos.org";
           database.backend = "rocksdb";
+
+          server_discovery = {
+            server.authority = "matrix.forkos.org:443";
+            client.base_url = "https://matrix.forkos.org";
+          };
         };
       };
 
@@ -48,18 +53,7 @@ in
           "forkos.org" = {
             forceSSL = true;
             enableACME = true;
-            locations = {
+            locations."/.well-known/matrix".proxyPass = "http://grapevine";
-              "= /.well-known/matrix/server".extraConfig = ''
-                add_header Content-Type application/json;
-                add_header Access-Control-Allow-Origin *;
-                return 200 '{"m.server": "matrix.forkos.org:443"}';
-              '';
-              "= /.well-known/matrix/client".extraConfig = ''
-                add_header Content-Type application/json;
-                add_header Access-Control-Allow-Origin *;
-                return 200 '{"m.homeserver": {"base_url": "https://matrix.forkos.org/"}, "m.identity_server": {"base_url": "https://matrix.org/"}, "org.matrix.msc3575.proxy": {"url": "https://matrix.forkos.org"}}';
-              '';
-            };
           };
         };
       };

services/monitoring/agent.nix

@@ -12,6 +12,14 @@ in
   options.bagel.monitoring.grafana-agent = {
     enable = (mkEnableOption "Grafana Agent") // { default = true; };
 
+    tenant = mkOption {
+      description = ''
+        Which tenant are we enabling Grafana Agent for.
+      '';
+      example = "lix";
+      type = types.enum [ "lix" "floral" ];
+    };
+
     exporters = mkOption {
       description = ''
         Set of additional exporters to scrape.
@@ -59,7 +67,7 @@ in
   };
 
   config = mkIf cfg.enable {
-    age.secrets.grafana-agent-password.file = ../../secrets/metrics-push-password.age;
+    age.secrets.grafana-agent-password.file = ../../secrets/floral/metrics-push-password.age;
 
     services.grafana-agent = {
       enable = true;
@@ -76,7 +84,10 @@ in
               };
             }
           ];
-          global.external_labels.hostname = config.networking.hostName;
+          global.external_labels = {
+            hostname = config.networking.hostName;
+            inherit (cfg) tenant;
+          };
           configs = [
             {
               name = config.networking.hostName;

services/monitoring/exporters/hydra/hydra-exporter.py

@@ -22,6 +22,9 @@ steps_building = Gauge("hydra_steps_building", "Number of steps currently buildi
 steps_copying_to = Gauge(
     "hydra_steps_copying_to", "Number of steps copying inputs to a worker"
 )
+steps_waiting_for_download_slot = Gauge(
+    "hydra_steps_waiting_for_download_slot", "Number of steps waiting for download slot"
+)
 steps_copying_from = Gauge(
     "hydra_steps_copying_from", "Number of steps copying outputs from a worker"
 )
@@ -230,6 +233,7 @@ def update_metrics(status):
     steps_active.set({}, status["nrActiveSteps"])
     steps_building.set({}, status["nrStepsBuilding"])
     steps_copying_to.set({}, status["nrStepsCopyingTo"])
+    steps_waiting_for_download_slot.set({}, status["nrStepsWaitingForDownloadSlot"])
     steps_copying_from.set({}, status["nrStepsCopyingFrom"])
     steps_waiting.set({}, status["nrStepsWaiting"])
     steps_unsupported.set({}, status["nrUnsupportedSteps"])
@@ -334,11 +338,11 @@ async def update_metrics_loop(hydra_url, scrape_interval):
                 )
 
                 update_metrics(response.json())
-
-                await asyncio.sleep(scrape_interval)
             except Exception as ex:
                 logging.exception("Failed to update metrics", exc_info=ex)
 
+            await asyncio.sleep(scrape_interval)
+
 
 @click.command()
 @click.option("--hydra-url", default="https://hydra.forkos.org/")

services/monitoring/exporters/postgres.nix

@@ -23,7 +23,10 @@ in
     };
 
     services.postgresql.settings.shared_preload_libraries = "pg_stat_statements";
+    systemd.services.postgresql.postStart = lib.mkAfter ''
+      ${config.services.postgresql.package}/bin/psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;";
+    '';
 
     bagel.monitoring.grafana-agent.exporters.postgres.port = 9104;
   };
-}
+}

services/monitoring/lgtm/alerts/forkos.yaml

@@ -1,11 +1,11 @@
 groups:
-  - name: ForkOS automation
+    - name: ForkOS automation
-    rules:
+      rules:
-      - alert: SyncFailedTooOften
+        - alert: SyncFailedTooOften
-        expr: 'changes(node_systemd_unit_state{name=~"ows.*.service",state="failed"}[24h]) > 2'
+          expr: 'changes(node_systemd_unit_state{name=~"ows.*.service",state="failed"}[1d]) > 2'
-        for: 30m
+          for: 30m
-        labels:
+          labels:
-          severity: critical
+            severity: critical
-        annotations:
+          annotations:
-          summary: "Synchronization job {{ $labels.name }} has failed more than twice in the last 24 hours"
+            description: On {{ $labels.instance }}, the synchronization job has failed more than twice in the last 24 hours, check if there's a conflict or a stdenv change.
-          description: "On {{ $labels.instance }}, the synchronization job has failed more than twice in the last 24 hours, check if there's a conflict or a stdenv change."
+            summary: Synchronization job {{ $labels.name }} has failed more than twice in the last 24 hours

services/monitoring/lgtm/alerts/postgres.yml

@@ -1,102 +1,118 @@
 groups:
-- name: PostgreSQL
+    - name: PostgreSQL
-  rules:
+      rules:
-
+        - alert: PostgresqlTableNotAutoVacuumed
-    - alert: PostgresqlTableNotAutoVacuumed
+          expr: '(pg_stat_user_tables_last_autovacuum > 0) and (time() - pg_stat_user_tables_last_autovacuum) > 60 * 60 * 24 * 10'
-      expr: '(pg_stat_user_tables_last_autovacuum > 0) and (time() - pg_stat_user_tables_last_autovacuum) > 60 * 60 * 24 * 10'
+          labels:
-      for: 0m
+            severity: warning
-      labels:
+          annotations:
-        severity: warning
+            description: |-
-      annotations:
+                Table {{ $labels.relname }} has not been auto vacuumed for 10 days
-        summary: Postgresql table not auto vacuumed (instance {{ $labels.instance }})
+                  VALUE = {{ $value }}
-        description: "Table {{ $labels.relname }} has not been auto vacuumed for 10 days\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+                  LABELS = {{ $labels }}
-
+            summary: Postgresql table not auto vacuumed (instance {{ $labels.instance }})
-    - alert: PostgresqlTableNotAutoAnalyzed
+        - alert: PostgresqlTableNotAutoAnalyzed
-      expr: '(pg_stat_user_tables_last_autoanalyze > 0) and (time() - pg_stat_user_tables_last_autoanalyze) > 24 * 60 * 60 * 10'
+          expr: '(pg_stat_user_tables_last_autoanalyze > 0) and (time() - pg_stat_user_tables_last_autoanalyze) > 24 * 60 * 60 * 10'
-      for: 0m
+          labels:
-      labels:
+            severity: warning
-        severity: warning
+          annotations:
-      annotations:
+            description: |-
-        summary: Postgresql table not auto analyzed (instance {{ $labels.instance }})
+                Table {{ $labels.relname }} has not been auto analyzed for 10 days
-        description: "Table {{ $labels.relname }} has not been auto analyzed for 10 days\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+                  VALUE = {{ $value }}
-
+                  LABELS = {{ $labels }}
-    - alert: PostgresqlDeadLocks
+            summary: Postgresql table not auto analyzed (instance {{ $labels.instance }})
-      expr: 'increase(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 5'
+        - alert: PostgresqlDeadLocks
-      for: 0m
+          expr: 'increase(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 5'
-      labels:
+          labels:
-        severity: warning
+            severity: warning
-      annotations:
+          annotations:
-        summary: Postgresql dead locks (instance {{ $labels.instance }})
+            description: |-
-        description: "PostgreSQL has dead-locks\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+                PostgreSQL has dead-locks
-
+                  VALUE = {{ $value }}
-    - alert: PostgresqlHighRollbackRate
+                  LABELS = {{ $labels }}
-      expr: 'sum by (namespace,datname) ((rate(pg_stat_database_xact_rollback{datname!~"template.*|postgres",datid!="0"}[3m])) / ((rate(pg_stat_database_xact_rollback{datname!~"template.*|postgres",datid!="0"}[3m])) + (rate(pg_stat_database_xact_commit{datname!~"template.*|postgres",datid!="0"}[3m])))) > 0.02'
+            summary: Postgresql dead locks (instance {{ $labels.instance }})
-      for: 0m
+        - alert: PostgresqlHighRollbackRate
-      labels:
+          expr: 'sum by (namespace, datname) ((rate(pg_stat_database_xact_rollback{datid!="0",datname!~"template.*|postgres"}[3m])) / ((rate(pg_stat_database_xact_rollback{datid!="0",datname!~"template.*|postgres"}[3m])) + (rate(pg_stat_database_xact_commit{datid!="0",datname!~"template.*|postgres"}[3m])))) > 0.02'
-        severity: warning
+          labels:
-      annotations:
+            severity: warning
-        summary: Postgresql high rollback rate (instance {{ $labels.instance }})
+          annotations:
-        description: "Ratio of transactions being aborted compared to committed is > 2 %\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+            description: |-
-
+                Ratio of transactions being aborted compared to committed is > 2 %
-    - alert: PostgresqlHighRateStatementTimeout
+                  VALUE = {{ $value }}
-      expr: 'rate(postgresql_errors_total{type="statement_timeout"}[1m]) > 3'
+                  LABELS = {{ $labels }}
-      for: 0m
+            summary: Postgresql high rollback rate (instance {{ $labels.instance }})
-      labels:
+        - alert: PostgresqlHighRateStatementTimeout
-        severity: critical
+          expr: 'rate(postgresql_errors_total{type="statement_timeout"}[1m]) > 3'
-      annotations:
+          labels:
-        summary: Postgresql high rate statement timeout (instance {{ $labels.instance }})
+            severity: critical
-        description: "Postgres transactions showing high rate of statement timeouts\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+          annotations:
-
+            description: |-
-    - alert: PostgresqlHighRateDeadlock
+                Postgres transactions showing high rate of statement timeouts
-      expr: 'increase(postgresql_errors_total{type="deadlock_detected"}[1m]) > 1'
+                  VALUE = {{ $value }}
-      for: 0m
+                  LABELS = {{ $labels }}
-      labels:
+            summary: Postgresql high rate statement timeout (instance {{ $labels.instance }})
-        severity: critical
+        - alert: PostgresqlHighRateDeadlock
-      annotations:
+          expr: 'increase(postgresql_errors_total{type="deadlock_detected"}[1m]) > 1'
-        summary: Postgresql high rate deadlock (instance {{ $labels.instance }})
+          labels:
-        description: "Postgres detected deadlocks\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+            severity: critical
-
+          annotations:
-    - alert: PostgresqlTooManyDeadTuples
+            description: |-
-      expr: '((pg_stat_user_tables_n_dead_tup > 10000) / (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup)) >= 0.1'
+                Postgres detected deadlocks
-      for: 2m
+                  VALUE = {{ $value }}
-      labels:
+                  LABELS = {{ $labels }}
-        severity: warning
+            summary: Postgresql high rate deadlock (instance {{ $labels.instance }})
-      annotations:
+        - alert: PostgresqlTooManyDeadTuples
-        summary: Postgresql too many dead tuples (instance {{ $labels.instance }})
+          expr: '((pg_stat_user_tables_n_dead_tup > 10000) / (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup)) >= 0.1'
-        description: "PostgreSQL dead tuples is too large\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+          for: 2m
-
+          labels:
-    - alert: PostgresqlTooManyLocksAcquired
+            severity: warning
-      expr: '((sum (pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.20'
+          annotations:
-      for: 2m
+            description: |-
-      labels:
+                PostgreSQL dead tuples is too large
-        severity: critical
+                  VALUE = {{ $value }}
-      annotations:
+                  LABELS = {{ $labels }}
-        summary: Postgresql too many locks acquired (instance {{ $labels.instance }})
+            summary: Postgresql too many dead tuples (instance {{ $labels.instance }})
-        description: "Too many locks acquired on the database. If this alert happens frequently, we may need to increase the postgres setting max_locks_per_transaction.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: PostgresqlTooManyLocksAcquired
-
+          expr: '((sum(pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.2'
-    - alert: PostgresqlBloatIndexHigh(>80%)
+          for: 2m
-      expr: 'pg_bloat_btree_bloat_pct > 80 and on (idxname) (pg_bloat_btree_real_size > 100000000)'
+          labels:
-      for: 1h
+            severity: critical
-      labels:
+          annotations:
-        severity: warning
+            description: |-
-      annotations:
+                Too many locks acquired on the database. If this alert happens frequently, we may need to increase the postgres setting max_locks_per_transaction.
-        summary: Postgresql bloat index high (> 80%) (instance {{ $labels.instance }})
+                  VALUE = {{ $value }}
-        description: "The index {{ $labels.idxname }} is bloated. You should execute `REINDEX INDEX CONCURRENTLY {{ $labels.idxname }};`\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+                  LABELS = {{ $labels }}
-
+            summary: Postgresql too many locks acquired (instance {{ $labels.instance }})
-    - alert: PostgresqlBloatTableHigh(>80%)
+        - alert: PostgresqlBloatIndexHigh(>80%)
-      expr: 'pg_bloat_table_bloat_pct > 80 and on (relname) (pg_bloat_table_real_size > 200000000)'
+          expr: 'pg_bloat_btree_bloat_pct > 80 and on (idxname) (pg_bloat_btree_real_size > 1e+08)'
-      for: 1h
+          for: 1h
-      labels:
+          labels:
-        severity: warning
+            severity: warning
-      annotations:
+          annotations:
-        summary: Postgresql bloat table high (> 80%) (instance {{ $labels.instance }})
+            description: |-
-        description: "The table {{ $labels.relname }} is bloated. You should execute `VACUUM {{ $labels.relname }};`\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+                The index {{ $labels.idxname }} is bloated. You should execute `REINDEX INDEX CONCURRENTLY {{ $labels.idxname }};`
-
+                  VALUE = {{ $value }}
-    - alert: PostgresqlInvalidIndex
+                  LABELS = {{ $labels }}
-      expr: 'pg_genaral_index_info_pg_relation_size{indexrelname=~".*ccnew.*"}'
+            summary: Postgresql bloat index high (> 80%) (instance {{ $labels.instance }})
-      for: 6h
+        - alert: PostgresqlBloatTableHigh(>80%)
-      labels:
+          expr: 'pg_bloat_table_bloat_pct > 80 and on (relname) (pg_bloat_table_real_size > 2e+08)'
-        severity: warning
+          for: 1h
-      annotations:
+          labels:
-        summary: Postgresql invalid index (instance {{ $labels.instance }})
+            severity: warning
-        description: "The table {{ $labels.relname }} has an invalid index: {{ $labels.indexrelname }}. You should execute `DROP INDEX {{ $labels.indexrelname }};`\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+          annotations:
+            description: |-
+                The table {{ $labels.relname }} is bloated. You should execute `VACUUM {{ $labels.relname }};`
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Postgresql bloat table high (> 80%) (instance {{ $labels.instance }})
+        - alert: PostgresqlInvalidIndex
+          expr: 'pg_genaral_index_info_pg_relation_size{indexrelname=~".*ccnew.*"}'
+          for: 6h
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                The table {{ $labels.relname }} has an invalid index: {{ $labels.indexrelname }}. You should execute `DROP INDEX {{ $labels.indexrelname }};`
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Postgresql invalid index (instance {{ $labels.instance }})

services/monitoring/lgtm/alerts/resources.yaml

@@ -1,76 +1,100 @@
 groups:
-  - name: Host & hardware
+    - name: Host & hardware
-    rules:
+      rules:
-      - alert: HostOutOfMemory
+        - alert: HostOutOfMemory
-        expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
-        for: 2m
+          for: 2m
-        labels:
+          labels:
-          severity: warning
+            severity: warning
-        annotations:
+          annotations:
-          summary: Host out of memory (instance {{ $labels.instance }})
+            description: |-
-          description: "Node memory is filling up (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+                Node memory is filling up (< 10% left)
-      - alert: HostMemoryUnderMemoryPressure
+                  VALUE = {{ $value }}
-        expr: (rate(node_vmstat_pgmajfault[1m]) > 1000) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+                  LABELS = {{ $labels }}
-        for: 2m
+            summary: Host out of memory (instance {{ $labels.instance }})
-        labels:
+        - alert: HostMemoryUnderMemoryPressure
-          severity: warning
+          expr: (rate(node_vmstat_pgmajfault[1m]) > 1000) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
-        annotations:
+          for: 2m
-          summary: Host memory under memory pressure (instance {{ $labels.instance }})
+          labels:
-          description: "The node is under heavy memory pressure. High rate of major page faults\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+            severity: warning
-      - alert: HostMemoryIsUnderutilized
+          annotations:
-        expr: (100 - (avg_over_time(node_memory_MemAvailable_bytes[30m]) / node_memory_MemTotal_bytes * 100) < 20) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+            description: |-
-        for: 1w
+                The node is under heavy memory pressure. High rate of major page faults
-        labels:
+                  VALUE = {{ $value }}
-          severity: info
+                  LABELS = {{ $labels }}
-        annotations:
+            summary: Host memory under memory pressure (instance {{ $labels.instance }})
-          summary: Host Memory is underutilized (instance {{ $labels.instance }})
+        - alert: HostMemoryIsUnderutilized
-          description: "Node memory is < 20% for 1 week. Consider reducing memory space. (instance {{ $labels.instance }})\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+          expr: (100 - (avg_over_time(node_memory_MemAvailable_bytes[30m]) / node_memory_MemTotal_bytes * 100) < 20) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
-      - alert: HostOutOfDiskSpace
+          for: 1w
-        expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          labels:
-        for: 2m
+            severity: info
-        labels:
+          annotations:
-          severity: warning
+            description: |-
-        annotations:
+                Node memory is < 20% for 1 week. Consider reducing memory space. (instance {{ $labels.instance }})
-          summary: Host out of disk space (instance {{ $labels.instance }})
+                  VALUE = {{ $value }}
-          description: "Disk is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+                  LABELS = {{ $labels }}
-      - alert: HostDiskWillFillIn24Hours
+            summary: Host Memory is underutilized (instance {{ $labels.instance }})
-        expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+        - alert: HostOutOfDiskSpace
-        for: 2m
+          expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and on (instance, device, mountpoint) node_filesystem_readonly == 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
-        labels:
+          for: 2m
-          severity: warning
+          labels:
-        annotations:
+            severity: warning
-          summary: Host disk will fill in 24 hours (instance {{ $labels.instance }})
+          annotations:
-          description: "Filesystem is predicted to run out of space within the next 24 hours at current write rate\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+            description: |-
-      - alert: HostCpuIsUnderutilized
+                Disk is almost full (< 10% left)
-        expr: (100 - (rate(node_cpu_seconds_total{mode="idle"}[30m]) * 100) < 20) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+                  VALUE = {{ $value }}
-        for: 1w
+                  LABELS = {{ $labels }}
-        labels:
+            summary: Host out of disk space (instance {{ $labels.instance }})
-          severity: info
+        - alert: HostDiskWillFillIn24Hours
-        annotations:
+          expr: ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and on (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and on (instance, device, mountpoint) node_filesystem_readonly == 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
-          summary: Host CPU is underutilized (instance {{ $labels.instance }})
+          for: 2m
-          description: "CPU load is < 20% for 1 week. Consider reducing the number of CPUs.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+          labels:
-      - alert: HostCpuStealNoisyNeighbor
+            severity: warning
-        expr: (avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          annotations:
-        for: 0m
+            description: |-
-        labels:
+                Filesystem is predicted to run out of space within the next 24 hours at current write rate
-          severity: warning
+                  VALUE = {{ $value }}
-        annotations:
+                  LABELS = {{ $labels }}
-          summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
+            summary: Host disk will fill in 24 hours (instance {{ $labels.instance }})
-          description: "CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        - alert: HostCpuIsUnderutilized
-      - alert: HostOomKillDetected
+          expr: (100 - (rate(node_cpu_seconds_total{mode="idle"}[30m]) * 100) < 20) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
-        expr: (increase(node_vmstat_oom_kill[1m]) > 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          for: 1w
-        for: 0m
+          labels:
-        labels:
+            severity: info
-          severity: warning
+          annotations:
-        annotations:
+            description: |-
-          summary: Host OOM kill detected (instance {{ $labels.instance }})
+                CPU load is < 20% for 1 week. Consider reducing the number of CPUs.
-          description: "OOM kill detected\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+                  VALUE = {{ $value }}
-      - alert: HostNetworkInterfaceSaturated
+                  LABELS = {{ $labels }}
-        expr: ((rate(node_network_receive_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"} > 0.8 < 10000) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
+            summary: Host CPU is underutilized (instance {{ $labels.instance }})
-        for: 1m
+        - alert: HostCpuStealNoisyNeighbor
-        labels:
+          expr: (avg by (instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
-          severity: warning
+          labels:
-        annotations:
+            severity: warning
-          summary: Host Network Interface Saturated (instance {{ $labels.instance }})
+          annotations:
-          description: "The network interface \"{{ $labels.device }}\" on \"{{ $labels.instance }}\" is getting overloaded.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+            description: |-
-
+                CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
+        - alert: HostOomKillDetected
+          expr: (increase(node_vmstat_oom_kill[1m]) > 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                OOM kill detected
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host OOM kill detected (instance {{ $labels.instance }})
+        - alert: HostNetworkInterfaceSaturated
+          expr: ((rate(node_network_receive_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"} > 0.8 < 10000) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
+          for: 1m
+          labels:
+            severity: warning
+          annotations:
+            description: |-
+                The network interface "{{ $labels.device }}" on "{{ $labels.instance }}" is getting overloaded.
+                  VALUE = {{ $value }}
+                  LABELS = {{ $labels }}
+            summary: Host Network Interface Saturated (instance {{ $labels.instance }})

services/monitoring/lgtm/dashboards/postgres2.json

@@ -20,7 +20,6 @@
   "fiscalYearStartMonth": 0,
   "gnetId": 9628,
   "graphTooltip": 0,
-  "id": 27,
   "links": [],
   "liveNow": false,
   "panels": [
@@ -101,6 +100,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "mean"
@@ -112,14 +112,14 @@
         "textMode": "name",
         "wideLayout": true
       },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_static{hostname=\"$hostname\"}",
+          "expr": "pg_static{tenant=\"$tenant\",hostname=\"$hostname\"}",
           "format": "time_series",
           "instant": true,
           "intervalFactor": 1,
@@ -182,6 +182,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "mean"
@@ -193,14 +194,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_postmaster_start_time_seconds{hostname=\"$hostname\"} * 1000",
+          "expr": "pg_postmaster_start_time_seconds{tenant=\"$tenant\",hostname=\"$hostname\"} * 1000",
           "format": "time_series",
           "intervalFactor": 2,
           "legendFormat": "",
@@ -261,6 +262,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -272,14 +274,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "SUM(pg_stat_database_tup_fetched{datname=~\"$datname\", hostname=~\"$hostname\"})",
+          "expr": "SUM(pg_stat_database_tup_fetched{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"})",
           "format": "time_series",
           "intervalFactor": 2,
           "refId": "A",
@@ -340,6 +342,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -351,14 +354,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "SUM(pg_stat_database_tup_inserted{datname=~\"$datname\", hostname=~\"$hostname\"})",
+          "expr": "SUM(pg_stat_database_tup_inserted{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"})",
           "format": "time_series",
           "intervalFactor": 2,
           "refId": "A",
@@ -419,6 +422,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -430,14 +434,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "SUM(pg_stat_database_tup_updated{datname=~\"$datname\", hostname=~\"$hostname\"})",
+          "expr": "SUM(pg_stat_database_tup_updated{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"})",
           "format": "time_series",
           "intervalFactor": 2,
           "refId": "A",
@@ -498,6 +502,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "mean"
@@ -509,14 +514,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_settings_max_connections{hostname=\"$hostname\"}",
+          "expr": "pg_settings_max_connections{tenant=\"$tenant\",hostname=\"$hostname\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -602,6 +607,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -613,14 +619,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_settings_shared_buffers_bytes{hostname=\"$hostname\"}",
+          "expr": "pg_settings_shared_buffers_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -680,6 +686,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -691,14 +698,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_settings_effective_cache_size_bytes{hostname=\"$hostname\"}",
+          "expr": "pg_settings_effective_cache_size_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -758,6 +765,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -769,14 +777,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_settings_maintenance_work_mem_bytes{hostname=\"$hostname\"}",
+          "expr": "pg_settings_maintenance_work_mem_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -836,6 +844,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -847,14 +856,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_settings_work_mem_bytes{hostname=\"$hostname\"}",
+          "expr": "pg_settings_work_mem_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "",
@@ -916,6 +925,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -927,14 +937,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_settings_max_wal_size_bytes{hostname=\"$hostname\"}",
+          "expr": "pg_settings_max_wal_size_bytes{tenant=\"$tenant\",hostname=\"$hostname\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -994,6 +1004,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -1005,14 +1016,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_settings_random_page_cost{hostname=\"$hostname\"}",
+          "expr": "pg_settings_random_page_cost{tenant=\"$tenant\",hostname=\"$hostname\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -1072,6 +1083,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -1083,7 +1095,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
       "targets": [
         {
           "datasource": {
@@ -1150,6 +1162,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "mean"
@@ -1161,14 +1174,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_settings_max_worker_processes{hostname=\"$hostname\"}",
+          "expr": "pg_settings_max_worker_processes{tenant=\"$tenant\",hostname=\"$hostname\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -1228,6 +1241,7 @@
         "graphMode": "none",
         "justifyMode": "auto",
         "orientation": "horizontal",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -1239,14 +1253,14 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "11.0.0",
+      "pluginVersion": "11.2.1",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_settings_max_parallel_workers{hostname=\"$hostname\"}",
+          "expr": "pg_settings_max_parallel_workers{tenant=\"$tenant\",hostname=\"$hostname\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -1298,6 +1312,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "points",
             "fillOpacity": 10,
             "gradientMode": "none",
@@ -1372,7 +1387,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_stat_activity_count{datname=~\"$datname\", hostname=~\"$hostname\", state=\"active\"} !=0",
+          "expr": "pg_stat_activity_count{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\", state=\"active\"} !=0",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 2,
@@ -1401,6 +1416,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 10,
             "gradientMode": "none",
@@ -1474,7 +1490,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_database_xact_commit{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_xact_commit{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{datname}} commits",
@@ -1485,7 +1501,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_database_xact_rollback{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_xact_rollback{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{datname}} rollbacks",
@@ -1512,6 +1528,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 10,
             "gradientMode": "none",
@@ -1585,7 +1602,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_stat_database_tup_updated{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_updated{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
           "format": "time_series",
           "intervalFactor": 2,
           "legendFormat": "{{datname}}",
@@ -1613,6 +1630,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 10,
             "gradientMode": "none",
@@ -1686,7 +1704,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_stat_database_tup_fetched{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_fetched{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
           "format": "time_series",
           "intervalFactor": 2,
           "legendFormat": "{{datname}}",
@@ -1714,6 +1732,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 10,
             "gradientMode": "none",
@@ -1787,7 +1806,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_stat_database_tup_inserted{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_inserted{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
           "format": "time_series",
           "intervalFactor": 2,
           "legendFormat": "{{datname}}",
@@ -1815,6 +1834,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 10,
             "gradientMode": "none",
@@ -1890,7 +1910,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_locks_count{datname=~\"$datname\", hostname=~\"$hostname\", mode=~\"$mode\"} != 0",
+          "expr": "pg_locks_count{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\", mode=~\"$mode\"} != 0",
           "format": "time_series",
           "intervalFactor": 2,
           "legendFormat": "{{datname}},{{mode}}",
@@ -1948,8 +1968,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "green"
-                "value": null
               },
               {
                 "color": "red",
@@ -1991,7 +2010,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_stat_database_tup_returned{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_returned{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
           "format": "time_series",
           "intervalFactor": 2,
           "legendFormat": "{{datname}}",
@@ -2050,8 +2069,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "green"
-                "value": null
               },
               {
                 "color": "red",
@@ -2092,7 +2110,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_stat_activity_count{datname=~\"$datname\", hostname=~\"$hostname\", state=~\"idle|idle in transaction|idle in transaction (aborted)\"}",
+          "expr": "pg_stat_activity_count{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\", state=~\"idle|idle in transaction|idle in transaction (aborted)\"}",
           "format": "time_series",
           "intervalFactor": 2,
           "legendFormat": "{{datname}}, s: {{state}}",
@@ -2150,8 +2168,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "green"
-                "value": null
               },
               {
                 "color": "red",
@@ -2193,7 +2210,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_stat_database_tup_deleted{datname=~\"$datname\", hostname=~\"$hostname\"} != 0",
+          "expr": "pg_stat_database_tup_deleted{datname=~\"$datname\", tenant=\"$tenant\",hostname=~\"$hostname\"} != 0",
           "format": "time_series",
           "intervalFactor": 2,
           "legendFormat": "{{datname}}",
@@ -2252,8 +2269,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "green"
-                "value": null
               },
               {
                 "color": "red",
@@ -2294,7 +2310,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "pg_stat_database_blks_hit{hostname=\"$hostname\", datname=~\"$datname\"} / (pg_stat_database_blks_read{hostname=\"$hostname\", datname=~\"$datname\"} + pg_stat_database_blks_hit{hostname=\"$hostname\", datname=~\"$datname\"})",
+          "expr": "pg_stat_database_blks_hit{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"} / (pg_stat_database_blks_read{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"} + pg_stat_database_blks_hit{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"})",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{ datname }}",
@@ -2351,8 +2367,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "green"
-                "value": null
               },
               {
                 "color": "red",
@@ -2395,7 +2410,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_bgwriter_buffers_backend{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_backend{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "buffers_backend",
@@ -2406,7 +2421,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_bgwriter_buffers_alloc{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_alloc{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "buffers_alloc",
@@ -2417,7 +2432,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "backend_fsync",
@@ -2428,7 +2443,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "buffers_checkpoint",
@@ -2439,7 +2454,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_bgwriter_buffers_clean{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_clean{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "buffers_clean",
@@ -2497,8 +2512,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "green"
-                "value": null
               },
               {
                 "color": "red",
@@ -2540,7 +2554,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_database_conflicts{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_conflicts{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{datname}} conflicts",
@@ -2551,7 +2565,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_database_deadlocks{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_deadlocks{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{datname}} deadlocks",
@@ -2610,8 +2624,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "green"
-                "value": null
               },
               {
                 "color": "red",
@@ -2653,7 +2666,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_database_temp_bytes{hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
+          "expr": "irate(pg_stat_database_temp_bytes{tenant=\"$tenant\",hostname=\"$hostname\", datname=~\"$datname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{datname}}",
@@ -2710,8 +2723,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
+                "color": "green"
-                "value": null
               },
               {
                 "color": "red",
@@ -2754,7 +2766,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_bgwriter_checkpoint_write_time{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_checkpoint_write_time{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "write_time - Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk.",
@@ -2765,7 +2777,7 @@
             "type": "prometheus",
             "uid": "mimir"
           },
-          "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time{hostname=\"$hostname\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time{tenant=\"$tenant\",hostname=\"$hostname\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "sync_time - Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk.",
@@ -2866,8 +2878,32 @@
       {
         "current": {
           "selected": false,
-          "text": "bagel-box",
+          "text": "fake",
-          "value": "bagel-box"
+          "value": "fake"
+        },
+        "definition": "label_values(tenant)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Tenant",
+        "multi": true,
+        "name": "tenant",
+        "options": [],
+        "query": {
+          "qryType": 1,
+          "query": "label_values(tenant)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "current": {
+          "selected": true,
+          "text": "buildbot",
+          "value": "buildbot"
         },
         "datasource": {
           "type": "prometheus",
@@ -2962,7 +2998,6 @@
     "from": "now-6h",
     "to": "now"
   },
-  "timeRangeUpdatedDuringEditOrView": false,
   "timepicker": {
     "refresh_intervals": [
       "5s",
@@ -2993,4 +3028,4 @@
   "uid": "000000039",
   "version": 1,
   "weekStart": ""
-}
+}

services/monitoring/lgtm/grafana.nix

@@ -17,7 +17,7 @@ in
 
   config = mkIf cfg.enable {
     age.secrets.grafana-oauth-secret = {
-      file = ../../../secrets/grafana-oauth-secret.age;
+      file = ../../../secrets/floral/grafana-oauth-secret.age;
       owner = "grafana";
     };
 

services/monitoring/lgtm/loki.nix

@@ -13,10 +13,10 @@ in
   config = mkIf cfg.enable {
     age.secrets = {
       metrics-push-htpasswd = {
-        file = ../../../secrets/metrics-push-htpasswd.age;
+        file = ../../../secrets/floral/metrics-push-htpasswd.age;
         owner = "nginx";
       };
-      loki-environment.file = ../../../secrets/loki-environment.age;
+      loki-environment.file = ../../../secrets/floral/loki-environment.age;
     };
 
     services.loki = {

services/monitoring/lgtm/mimir.nix

@@ -9,6 +9,15 @@ let
   inherit (lib) mkEnableOption mkIf;
 
   mimirPort = config.services.mimir.configuration.server.http_listen_port;
+
+  alerts = pkgs.runCommand "mimir-alerts-checked" {
+    src = ./alerts;
+    nativeBuildInputs = with pkgs; [ prometheus.cli ];
+  } ''
+    promtool check rules $src/*
+    mkdir $out
+    cp -R $src $out/anonymous/
+  '';
 in
 {
   options.bagel.services.prometheus.enable = mkEnableOption "Prometheus scraper";
@@ -16,11 +25,11 @@ in
   config = mkIf cfg.enable {
     age.secrets = {
       metrics-push-htpasswd = {
-        file = ../../../secrets/metrics-push-htpasswd.age;
+        file = ../../../secrets/floral/metrics-push-htpasswd.age;
         owner = "nginx";
       };
-      mimir-environment.file = ../../../secrets/mimir-environment.age;
+      mimir-environment.file = ../../../secrets/floral/mimir-environment.age;
-      mimir-webhook-url.file = ../../../secrets/mimir-webhook-url.age;
+      mimir-webhook-url.file = ../../../secrets/floral/mimir-webhook-url.age;
     };
 
     services.mimir = {
@@ -60,10 +69,7 @@ in
         blocks_storage.backend = "s3";
         ruler_storage = {
           backend = "local";
-          local.directory = pkgs.runCommand "mimir-rules" {} ''
+          local.directory = alerts;
-            mkdir -p $out
-            ln -s ${./alerts} $out/anonymous
-          '';
         };
 
         alertmanager = {

services/monitoring/lgtm/tempo.nix

@@ -13,10 +13,10 @@ in
   config = mkIf cfg.enable {
     age.secrets = {
       metrics-push-htpasswd = {
-        file = ../../../secrets/metrics-push-htpasswd.age;
+        file = ../../../secrets/floral/metrics-push-htpasswd.age;
         owner = "nginx";
       };
-      tempo-environment.file = ../../../secrets/tempo-environment.age;
+      tempo-environment.file = ../../../secrets/floral/tempo-environment.age;
     };
 
     services.tempo = {

services/monitoring/pyroscope/default.nix

@@ -15,7 +15,7 @@ in
   ];
 
   config = mkIf cfg.enable {
-    age.secrets.pyroscope-secrets.file = ../../../secrets/pyroscope-secrets.age;
+    age.secrets.pyroscope-secrets.file = ../../../secrets/floral/pyroscope-secrets.age;
     services.nginx = {
       upstreams.pyroscope = {
         servers."127.0.0.1:${toString pyroscopePort}" = {};

services/netbox/default.nix

@@ -20,7 +20,10 @@ in
   };
 
   config = mkIf cfg.enable {
-    age.secrets.netbox-environment.file = ../../secrets/netbox-environment.age;
+    # FIXME: why
+    nixpkgs.config.permittedInsecurePackages = [ pkgs.netbox_3_7.name ];
+
+    age.secrets.netbox-environment.file = ../../secrets/floral/netbox-environment.age;
     services = {
       netbox = {
         enable = true;

services/newsletter/default.nix

@@ -14,7 +14,7 @@ in
   };
 
   config = mkIf cfg.enable {
-    age.secrets.newsletter-secrets.file = ../../secrets/newsletter-secrets.age;
+    age.secrets.newsletter-secrets.file = ../../secrets/floral/newsletter-secrets.age;
     services.listmonk = {
       enable = true;
       secretFile = config.age.secrets.newsletter-secrets.path;

services/ofborg/default.nix

@@ -26,10 +26,12 @@ in {
       webroot = "/var/lib/acme/.challenges";
       group = "rabbitmq";
     };
+
+    services.nginx.enable = true;
     services.nginx.virtualHosts.${amqpHost}.locations."/.well-known/acme-challenge".root =
       "/var/lib/acme/.challenges";
     systemd.services.rabbitmq.requires = ["acme-finished-${amqpHost}.target"];
 
-    networking.firewall.allowedTCPPorts = [ amqpPort ];
+    networking.firewall.allowedTCPPorts = [ 80 443 amqpPort ];
   };
 }

services/postgres/default.nix

@@ -11,7 +11,7 @@ in {
 
   config = lib.mkIf cfg.enable {
     age.secrets.postgresql-tls-priv.owner = "postgres";
-    age.secrets.postgresql-tls-priv.file = ../../secrets/postgres-tls-priv.age;
+    age.secrets.postgresql-tls-priv.file = ../../secrets/floral/postgres-tls-priv.age;
 
     systemd.tmpfiles.rules = [
       "d /var/db 0755 root root - -"
@@ -67,7 +67,7 @@ in {
 
     # Provisioned on the server so that CA operations can be done there.
     age.secrets.postgresql-ca-priv.owner = "postgres";
-    age.secrets.postgresql-ca-priv.file = ../../secrets/postgres-ca-priv.age;
+    age.secrets.postgresql-ca-priv.file = ../../secrets/floral/postgres-ca-priv.age;
 
     users.users.postgres.packages = [
       (pkgs.writeShellScriptBin "postgres-mint-new-client" ''

services/s3-revproxy/default.nix

@@ -70,7 +70,7 @@ in
   ];
 
   config = mkIf cfg.enable {
-    age.secrets.s3-revproxy-api-keys.file = ../../secrets/s3-revproxy-api-keys.age;
+    age.secrets.s3-revproxy-api-keys.file = ../../secrets/floral/s3-revproxy-api-keys.age;
     # For each target, generate an entry that passes it to the s3-revproxy.
     services.nginx.virtualHosts = mapAttrs' (subdomain: _: nameValuePair "${subdomain}.${cfg.domain}" (mkProxiedSubdomain subdomain)) cfg.targets;
     # this solves garage supporting neither anonymous access nor automatic

services/uptime-kuma/default.nix

@@ -0,0 +1,97 @@
+{
+  inputs,
+  lib,
+  config,
+  ...
+}:
+let
+  cfg = config.bagel.status;
+  # TODO: pull domains from a central place
+  subdomains = [
+    "cl"
+    "netbox"
+    "cache"
+    "grafana"
+    "hydra"
+    "loki"
+    "mimir"
+    "pyroscope"
+    "matrix"
+    "tempo"
+    "amqp"
+    "fodwatch"
+    "git"
+    "alerts"
+    "buildbot"
+    "b"
+    "postgres"
+    "news"
+  ];
+  port = 3001;
+in
+{
+  imports = [ "${inputs.stateless-uptime-kuma}/nixos/module.nix" ];
+
+  options.bagel.status = {
+    enable = lib.mkEnableOption "the status page service (uptime-kuma)";
+    domain = lib.mkOption {
+      type = lib.types.str;
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.uptime-kuma.enable = true;
+
+    services.nginx = {
+      enable = true;
+      virtualHosts.${cfg.domain} = {
+        enableACME = true;
+        forceSSL = true;
+        locations."/" = {
+          proxyPass = "http://127.0.0.1:${builtins.toString port}";
+          proxyWebsockets = true;
+        };
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = [
+      80
+      443
+    ];
+
+    age.secrets.stateless-uptime-kuma-password.file = ../../secrets/floral/stateless-uptime-kuma-password.age;
+    statelessUptimeKuma = {
+      probesConfig = {
+        monitors = lib.genAttrs subdomains (name: {
+          type = "http";
+          url = "https://${name}.forkos.org/";
+          tags = [];
+        });
+        status_pages = {
+          "forkos" = {
+            title = "ForkOS";
+            description = "health of the ForkOS infra";
+            showTags = true;
+            publicGroupList = [
+              {
+                name = "Services";
+                weight = 1;
+                monitorList = lib.genAttrs subdomains (id: {
+                  inherit id;
+                });
+              }
+            ];
+          };
+        };
+        settings = {
+          entryPage = "statusPage-forkos";
+        };
+      };
+      extraFlags = [ "-s" ];
+      host = "http://localhost:${builtins.toString port}/";
+      username = "forkos";
+      passwordFile = config.age.secrets."stateless-uptime-kuma-password".path;
+      enableService = true;
+    };
+  };
+}

terraform/default.nix

@@ -1,7 +1,6 @@
 {
   imports = [
     ./common.nix
-    ./gandi.nix
     ./dnsimple.nix
     ./hydra.nix
     ./state.nix

terraform/dnsimple.nix

@@ -114,6 +114,7 @@ in
           (record "b" 300 "CNAME" "public01.infra.p.forkos.org")
           (record "postgres" 300 "CNAME" "bagel-box.infra.p.forkos.org")
           (record "news" 3600 "CNAME" "public01.infra.p.forkos.org")
+          (record "status" 3600 "CNAME" "public01.infra.p.forkos.org")
 
           # S3 in delroth's basement
           (record "cache" 300 "AAAA" "2a02:168:6426::12") # smol.delroth.net

terraform/gandi.nix

@@ -1,115 +0,0 @@
-{ lib, config, ... }:
-let
-  inherit (lib) mkEnableOption mkIf tf genList;
-  cfg = config.bagel.gandi;
-in
-{
-  options.bagel.gandi = {
-    enable = mkEnableOption "the Gandi DNS configuration";
-  };
-
-  config = mkIf cfg.enable {
-    terraform.required_providers.gandi = {
-      version = "~> 2.3.0";
-      source = "go-gandi/gandi";
-    };
-
-    resource.secret_resource.gandi_pat.lifecycle.prevent_destroy = true;
-
-    provider.gandi = {
-      personal_access_token = tf.ref "resource.secret_resource.gandi_pat.value";
-    };
-
-    resource.gandi_livedns_domain.forkos_org = {
-      name = "forkos.org";
-    };
-
-    resource.gandi_livedns_record = let
-      record = name: ttl: type: values: {
-        inherit name ttl type values;
-      };
-
-      proxyRecords = name: ttl: type: values: [
-        # kurisu.lahfa.xyz running a sniproxy:
-        (record name ttl "A" ["163.172.69.160"])
-        (record name ttl type values)
-      ];
-
-      # Creates a extra *.p record pointing to the sniproxy
-      dualProxyRecords = name: ttl: type: values: lib.flatten [
-        (record name ttl type values)
-        (proxyRecords "${name}.p" ttl type values)
-      ];
-
-      # TODO: make less fragile and have actual unique and stable names
-      canonicalName = record: let
-        name = builtins.replaceStrings ["." "@"] ["_" "_root_"] record.name;
-      in
-        "forkos_org_${record.type}_${name}";
-
-      forkosRecords = records:
-        builtins.listToAttrs (map (record: {
-          name = canonicalName record;
-          value = record // {
-            zone = tf.ref "resource.gandi_livedns_domain.forkos_org.id";
-          };
-        }) (lib.flatten records));
-
-    in forkosRecords ([
-      # (record "@" 300 "A" ["163.172.69.160"])
-      (record "@" 300 "AAAA" ["2001:bc8:38ee:100:1000::20"])
-
-      (dualProxyRecords "bagel-box.infra" 300 "AAAA" ["2001:bc8:38ee:100:100::1"])
-      (dualProxyRecords "gerrit01.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::10"])
-      (dualProxyRecords "meta01.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::20"])
-      (dualProxyRecords "fodwatch.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::30"])
-      # git.infra.forkos.org exposes opensshd
-      (dualProxyRecords "git.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::41"])
-      # git.p.forkos.org exposes forgejo ssh server.
-      (proxyRecords "git.p" 300 "AAAA" ["2001:bc8:38ee:100:1000::40"])
-      (dualProxyRecords "buildbot.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::50"])
-      (dualProxyRecords "public01.infra" 300 "AAAA" ["2001:bc8:38ee:100:1000::60"])
-
-      (record "cl" 300 "CNAME" ["gerrit01.infra.p"])
-      (record "fodwatch" 300 "CNAME" ["fodwatch.infra.p"])
-      # git.p.forkos.org is the proxy variant of the Forgejo server.
-      (record "git" 300 "CNAME" ["git.p"])
-      (record "netbox" 300 "CNAME" ["meta01.infra.p"])
-      (record "amqp" 300 "CNAME" ["bagel-box.infra.p"])
-      (record "grafana" 300 "CNAME" ["meta01.infra.p"])
-      (record "hydra" 300 "CNAME" ["build-coord.wob01.infra.p"])
-      (record "loki" 300 "CNAME" ["meta01.infra.p"])
-      (record "mimir" 300 "CNAME" ["meta01.infra.p"])
-      (record "pyroscope" 300 "CNAME" ["meta01.infra.p"])
-      (record "tempo" 300 "CNAME" ["meta01.infra.p"])
-      (record "matrix" 300 "CNAME" ["meta01.infra.p"])
-      (record "alerts" 300 "CNAME" ["meta01.infra.p"])
-      (record "buildbot" 300 "CNAME" ["buildbot.infra.p"])
-      (record "b" 300 "CNAME" ["public01.infra.p"])
-      (record "postgres" 300 "CNAME" ["bagel-box.infra.p"])
-      (record "news" 3600 "CNAME" ["public01.infra.p"])
-
-      # S3 in delroth's basement
-      (record "cache" 300 "AAAA" ["2a02:168:6426::12"]) # smol.delroth.net
-      (record "cache" 300 "A" ["195.39.247.161"]) # sni proxy
-
-      (record "vpn-gw.wob01.infra" 300 "AAAA" [ "2a01:584:11::2" ])
-
-      (dualProxyRecords "build-coord.wob01.infra" 300 "AAAA" [ "2a01:584:11::1:11" ])
-      # TODO: do not hardcode, just reuse the Colmena hive module outputs to generate all the required details.
-    ]
-    ++ (map (index: record "builder-${toString index}.wob01.infra" 300 "AAAA" [ "2a01:584:11::1:${toString index}" ]) (genList lib.id 11))
-    ++ (
-      let
-        # FIXME: figure out a way to poke `config.services.s3-revproxy` and
-        # automate the DNS part away?
-        buckets = [
-          "channels"
-          "releases"
-          "channel-scripts-test"
-        ];
-      in
-      map (bucket: record "${bucket}" 300 "CNAME" [ "public01.infra.p" ]) buckets
-    ));
-  };
-}