fix(builders/netboot): make "normal" evaluation pass

Without this patch running `colmena build` will run into a few assertion errors for machines that have `config.bagel.baremetal.builders.netboot == true` set. This is due to an assertion check in the initrd module making sure there is a mount point for `/`. This can be trivially fixed by just setting the mount point to the real world value, which is a tmpfs with 64GB assigned. We also set `deployment.targetHost` to a domain that will never resolve in the public internet, to make sure nobody applies these machines by hand. It would have been nice to throw a error whenever `colmena apply` gets executed for one of these hosts, but doing so would defeat the purpose of this patch, because the colmena `build` and `apply` argument both evaluate the exact same code paths and thus colmena `build` would error again. The motivation behind this was, so we could run `colmena build` in CI in the future, and to not scare of new contributors with random build failures when they first try to build the machines. The proper solution would be to exclude all the network booted builders from the regular colmena hive that is exposed to the cli, but this is too many yaks to shave for now.
2024-09-29 03:43:53 +02:00
150 changed files with 1431 additions and 5063 deletions
--- a/.envrc
+++ b/.envrc
@ -1,11 +1,2 @@
-#!/usr/bin/env bash
-# the shebang is ignored, but nice for editors
-
 # shellcheck shell=bash
-if type -P lorri &>/dev/null; then
-  eval "$(lorri direnv --flake .)"
-else
-  echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]'
-  use flake
-fi
-
+use flake
--- a/README.md
+++ b/README.md
@ -1,32 +1 @@
-# Infrastructure for the donut shaped thing that is absolutely not a donut.
-
-## Quick start
-
-### Build the infrastructure
-
-```
-$ colmena build --on @localboot
-```
-
-Notice that `@localboot` is load-bearing as we have some machines that _cannot be_ deployed with vanilla Colmena. Fixing this is welcome.
-
-### Recommended deploy process
-
-```
-$ colmena apply dry-activate $machine # Verify that the nvd log is reasonable.
-$ colmena apply $machine
-```
-
-### Recommended upgrade process
-
-```
-$ nix flake update
-$ colmena apply dry-activate --on @localboot # Verify that the nvd log is reasonable. Run it twice to get only NVD logs shown.
-$ colmena apply --on @localboot
-```
-
-## Troubleshooting
-
-### I failed to deploy `gerrit01`
-
-Our Gerrit source build is known to have some hiccups sometimes, we are always interested in build logs, feel free to attach information in a new issue so we can make it more reliable.
+Infrastructure for the donut shaped thing that is absolutely not a donut.
--- a/common/admins.nix
+++ b/common/admins.nix
@ -1,47 +1,17 @@
-{ lib, ... }:
 let
-  inherit (lib) genAttrs;
-in
-# Note: to add somefew in this list.
-# Ensure their SSH key is already in common/ssh-keys.nix with
-# the same username for here, so that the keys is automatically added.
-{
-  bagel.groups = {
-    floral-infra.members = [
-      "delroth"
-      "emilylange"
-      "hexchen"
-      "jade"
-      "janik"
-      "k900"
-      "maxine"
-      "raito"
-      "thubrecht"
-      "winter"
-      "yuka"
-      "ckie"
-    ];
-
-    lix-infra.members = [
-      "raito"
-      "hexchen"
-      "jade"
-      "pennae"
-    ];
-  };
-  bagel.users = genAttrs [
-    "delroth"
-    "emilylange"
-    "hexchen"
-    "jade"
-    "janik"
-    "k900"
-    "maxine"
-    "raito"
-    "thubrecht"
-    "winter"
-    "yuka"
-    "ckie"
-    "pennae"
-  ] (name: {});
+  keys = import ./ssh-keys.nix;
+in {
+  users.users.root.openssh.authorizedKeys.keys =
+    keys.users.delroth ++
+    keys.users.emilylange ++
+    keys.users.hexchen ++
+    keys.users.jade ++
+    keys.users.janik ++
+    keys.users.k900 ++
+    keys.users.lukegb ++
+    keys.users.maxine ++
+    keys.users.raito ++
+    keys.users.thubrecht ++
+    keys.users.yuka ++
+    keys.users.winter;
 }
--- a/common/base-server.nix
+++ b/common/base-server.nix
@ -1,13 +1,11 @@
 { lib, pkgs, ... }: {
  imports = [
    ./known-ssh-keys.nix
-    ./cgroups.nix
  ];

  nixpkgs.overlays = import ../overlays;

  nix.package = lib.mkDefault pkgs.lix;
-  system.tools.nixos-option.enable = false;
  services.openssh.enable = lib.mkForce true;

  networking.nftables.enable = true;
@ -32,7 +30,7 @@
    automatic = true;
    persistent = true;
    dates = lib.mkDefault "daily";
-    options = lib.mkDefault "--delete-older-than 30d";
+    options = "--delete-older-than 30d";
  };

  services.journald.extraConfig = "SystemMaxUse=512M";
--- a/common/cgroups.nix
+++ b/common/cgroups.nix
@ -1,83 +0,0 @@
-# Relatively inspired by fbtax2:
-# https://facebookmicrosites.github.io/cgroup2/docs/fbtax-results.html
-#
-# See also the Chris Down talk at LISA'21:
-# https://www.usenix.org/conference/lisa21/presentation/down
-{ ... }:
-let
-  systemCriticalSliceConfig = {
-    ManagedOOMMemoryPressure = "kill";
-
-    # guarantee availability of memory
-    MemoryMin = "192M";
-    # default 100
-    IOWeight = 1000;
-    # default 100
-    CPUWeight = 1000;
-  };
-in
-{
-  systemd.oomd = {
-    enable = true;
-    # why not, we have cgroups at user level now so it'll just kill the
-    # terminal
-    enableRootSlice = true;
-    enableSystemSlice = true;
-    enableUserSlices = true;
-  };
-
-  systemd.enableCgroupAccounting = true;
-
-  systemd.services.nix-daemon = {
-    serviceConfig = {
-      # FIXME: how do i deprioritize this for memory
-      CPUWeight = 10;
-      IOWeight = 10;
-    };
-  };
-
-  systemd.slices.hostcritical = {
-    description = "Ensures that services to keep the system alive remain alive";
-
-    unitConfig = {
-      # required to avoid a dependency cycle on systemd-oomd. systemd will
-      # actually guess this right but we should fix it anyway.
-      DefaultDependencies = false;
-    };
-
-    sliceConfig = systemCriticalSliceConfig;
-  };
-
-  # make root logins higher priority for resources
-  systemd.slices."user-0" = {
-    sliceConfig = systemCriticalSliceConfig;
-  };
-
-
-  systemd.slices.system = {
-    sliceConfig = {
-      ManagedOOMMemoryPressure = "kill";
-      ManagedOOMMemoryPressureLimit = "50%";
-
-      IOWeight = 100;
-    };
-  };
-
-  systemd.services.sshd = {
-    serviceConfig = {
-      Slice = "hostcritical.slice";
-    };
-  };
-
-  systemd.services.systemd-oomd = {
-    serviceConfig = {
-      Slice = "hostcritical.slice";
-    };
-  };
-
-  systemd.services.systemd-journald = {
-    serviceConfig = {
-      Slice = "hostcritical.slice";
-    };
-  };
-}
--- a/common/default.nix
+++ b/common/default.nix
@ -1,15 +1,12 @@
 {
  imports = [
    ./admins.nix
-    ./server-acl.nix
    ./base-server.nix
    ./hardening.nix
    ./nix.nix
    ./raito-proxy-aware-nginx.nix
+    ./raito-vm.nix
    ./sysadmin
-    ./hardware
    ./zsh.nix
-    ./secrets.nix
-    ./pki.nix
  ];
 }
--- a/common/hardware/default.nix
+++ b/common/hardware/default.nix
@ -1,7 +0,0 @@
-{ ... }: {
-  imports = [
-    ./raito-vm.nix
-    ./oracle-vm.nix
-    ./hetzner.nix
-  ];
-}
--- a/common/hardware/hetzner.nix
+++ b/common/hardware/hetzner.nix
@ -1,76 +0,0 @@
-
-{ lib, config, ... }:
-let
-  cfg = config.bagel.hardware.hetzner;
-  inherit (lib) mkEnableOption mkIf mkOption types;
-in
-{
-  options.bagel.hardware.hetzner = {
-    enable = mkEnableOption "Hetzner's hardware defaults";
-
-    platformType = mkOption {
-      # Only VMs are supported.
-      type = types.enum [ "virtual-machine" ];
-    };
-
-    system = mkOption {
-      # Only the aarch64-linux VM Hetzner is supported.
-      type = types.enum [ "aarch64-linux" ];
-    };
-
-    networking.wan = {
-      mac = mkOption {
-        type = types.str;
-        description = "MAC address of the WAN interface in the Hetzner machine";
-      };
-      address = mkOption {
-        type = types.listOf types.str;
-        description = "List of static addresses attached to the WAN interface";
-      };
-    };
-  };
-
-  config = mkIf cfg.enable {
-    # A bunch of stuff is virtio.
-    boot.initrd.availableKernelModules = [
-      "xhci_pci"
-      "usbhid"
-      "sr_mod"
-      "virtio_gpu"
-      "virtio_scsi"
-      "virtio_rng"
-      "virtio_pci"
-    ];
-
-    boot.loader.systemd-boot.enable = true;
-    boot.loader.efi.canTouchEfiVariables = true;
-
-    networking.useDHCP = lib.mkDefault false;
-
-    # Stolen from the netplan provided by aarch64 Ubuntu images.
-    systemd.network.enable = true;
-    systemd.network.links."10-wan" = {
-      linkConfig.Name = "wan";
-      matchConfig.MACAddress = cfg.networking.mac;
-    };
-    systemd.network.networks."10-wan" = {
-      matchConfig.Name = "wan";
-      networkingConfig.Address = cfg.networking.address;
-      linkConfig.RequiredForOnline = true;
-      DHCP = "ipv4";
-      routes = [
-        {
-          routeConfig = {
-            Destination = "::/0";
-            GatewayOnLink = true;
-            Gateway = "fe80::1";
-          };
-        }
-      ];
-      dhcpV4Config = {
-        RouteMetric = 100;
-        UseMTU = true;
-      };
-    };
-  };
-}
--- a/common/hardware/oracle-vm.nix
+++ b/common/hardware/oracle-vm.nix
@ -1,52 +0,0 @@
-
-{ lib, config, modulesPath, ... }:
-let
-  cfg = config.bagel.hardware.oracle-vm;
-  inherit (lib) mkEnableOption mkIf mkOption types;
-in
-{
-  options.bagel.hardware.oracle-vm = {
-    enable = mkEnableOption "Oracle's VM hardware defaults";
-
-    system = mkOption {
-      # Only the free Oracle VMs are supported.
-      type = types.enum [ "aarch64-linux" ];
-    };
-  };
-
-  # Imports a bunch of virtio modules.
-  imports = [
-    "${modulesPath}/profiles/qemu-guest.nix"
-  ];
-
-  config = mkIf cfg.enable {
-    boot.loader.systemd-boot.enable = true;
-    boot.loader.efi.canTouchEfiVariables = true;
-    boot.initrd.systemd.enable = true;
-
-    boot.initrd.availableKernelModules = [
-      "xhci_pci" "virtio_pci" "usbhid" "sr_mod"
-    ];
-    boot.initrd.kernelModules = [ ];
-    boot.kernelModules = [ ];
-    boot.extraModulePackages = [ ];
-
-    nixpkgs.hostPlatform = cfg.system;
-
-    # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
-    # (the default) this is the recommended approach. When using systemd-networkd it's
-    # still possible to use this option, but it's recommended to use it in conjunction
-    # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
-    networking.useDHCP = lib.mkDefault false;
-    # Examples:
-    # 2: enp0s3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
-    #  link/ether 02:00:17:00:91:6e brd ff:ff:ff:ff:ff:ff
-    #  inet 10.0.0.94/24 brd 10.0.0.255 scope global dynamic noprefixroute enp0s3
-    #     valid_lft 44162sec preferred_lft 33362sec
-    #  inet6 fe80::17ff:fe00:916e/64 scope link
-    #     valid_lft forever preferred_lft forever
-    # [root@build02-aarch64-lahfa:~]# ip r
-    # default via 10.0.0.1 dev enp0s3 proto dhcp src 10.0.0.94 metric 1002 mtu 9000
-    networking.interfaces.enp0s3.useDHCP = lib.mkDefault true;
-  };
-}
--- a/common/known-ssh-keys.nix
+++ b/common/known-ssh-keys.nix
@ -2,6 +2,5 @@
 {
  programs.ssh.knownHosts = {
    "[cl.forkos.org]:29418".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM82mJ259C8Nc+BHHNBeRWXWhL3dfirQhmFbDAwHMle3";
-    "[gerrit.lix.systems]:2022".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICC/S6Z56uhv7zBMutkV0nU8eDuRcl3trykGWBch4L/l";
  };
 }
--- a/common/pki.nix
+++ b/common/pki.nix
@ -1,27 +0,0 @@
-{ config, lib, ... }:
-let
-  cfg = config.bagel.pki;
-  inherit (lib) mkOption types;
-in
-{
-  options.bagel.pki = {
-    rootPath = mkOption {
-      type = types.str;
-      default = "floral_systems/v1/infra/v1";
-      example = "floral_systems/v2/infra/v1";
-      description = "Root mountpoint for PKI issuing in the Vault cluster";
-    };
-    cacertFile = mkOption {
-      type = types.path;
-      # Trust our infrastructure CA chain certificate.
-      default = ../pki/cacerts/infra.crt;
-      description = "CA certificate file to put in our trust store";
-    };
-  };
-
-  config = {
-    security.pki.certificateFiles = [
-      cfg.cacertFile
-    ];
-  };
-}
--- a/common/hardware/raito-vm.nix
+++ b/common/hardware/raito-vm.nix
@ -1,7 +1,7 @@
-{ lib, config, ... }:
+{ lib, config, ... }: 
 let
  cfg = config.bagel.hardware.raito-vm;
-  inherit (lib) mkEnableOption mkIf mkOption types split toIntBase10;
+  inherit (lib) mkEnableOption mkIf mkOption types;
 in
 {
  options.bagel.hardware.raito-vm = {
@ -54,17 +54,6 @@ in
      linkConfig.Name = "wan";
    };

-    bagel.infra.self.wan =
-    let
-      parts = split "/" cfg.networking.wan.address;
-      address = builtins.elemAt parts 0;
-      prefixLength = toIntBase10 (builtins.elemAt 1 parts);
-    in
-    {
-      family = "inet6";
-      inherit address prefixLength;
-    };
-
    boot.loader.systemd-boot.enable = true;

    boot.initrd.kernelModules = [
--- a/common/secrets.nix
+++ b/common/secrets.nix
@ -1,22 +0,0 @@
-## This is a simple secret abstraction with multi-tenancy awareness.
-{ config, lib, ... }:
-let
-  cfg = config.bagel.secrets;
-  inherit (lib) mkOption types genAttrs;
-in
-{
-  options.bagel.secrets = {
-    tenant = mkOption {
-      type = types.enum [ "lix" "floral" ];
-    };
-
-    files = mkOption {
-      type = types.listOf types.str;
-      default = [ ];
-    };
-  };
-
-  config.age.secrets = genAttrs cfg.files (secretFile: {
-      file = ../secrets/${cfg.tenant}/${secretFile}.age;
-    });
-}
--- a/common/server-acl.nix
+++ b/common/server-acl.nix
@ -1,69 +0,0 @@
-{ lib, config, ... }:
-let
-  keys = import ./ssh-keys.nix;
-  inherit (lib) mkOption types length concatMap listToAttrs catAttrs attrValues;
-  cfgAdmins = config.bagel.admins;
-  cfgGroups = config.bagel.groups;
-  cfgUsers = config.bagel.users;
-
-  userOpts = { name, ... }: {
-    options = {
-      sshKeys = mkOption {
-        type = types.listOf types.str;
-        description = "List of SSH keys associated to this user, defaults to `ssh-keys.nix` entries.";
-        default = keys.users.${name} or [ ];
-      };
-    };
-  };
-  groupOpts = { name, ... }: {
-    options = {
-      members = mkOption {
-        type = types.listOf types.str;
-        description = "List of users member of this group";
-        example = [ "raito" ];
-        default = [ ];
-      };
-    };
-  };
-
-  # There might be duplicate in that list. We will turn it into an attribute set.
-  allowedMembers = listToAttrs (
-    map (member: {
-      name = member;
-      value = cfgUsers.${member};
-    }) (concatMap (allowedGroup: cfgGroups.${allowedGroup}.members) cfgAdmins.allowedGroups));
-
-  rootKeys = concatMap ({ sshKeys, ... }: sshKeys) (attrValues allowedMembers);
-in
-{
-  options.bagel.users = mkOption {
-    type = types.attrsOf (types.submodule userOpts);
-    description = "User configuration for server ACLs";
-  };
-
-  options.bagel.groups = mkOption {
-    type = types.attrsOf (types.submodule groupOpts);
-    description = "Group configuration for server ACLs";
-  };
-
-  options.bagel.admins = {
-    allowedGroups = mkOption {
-      type = types.listOf types.str;
-      default = [ "catch-all" ];
-      description = "List of groups which are allowed to admin this machine.";
-      example = [ "lix" "build-infra" ];
-    };
-  };
-
-  config = {
-    assertions = [
-      { assertion = length config.users.users.root.openssh.authorizedKeys.keys > 0;
-        # TODO: you can add printing of `concatStringsSep ", " cfg.allowedGroups` to diagnose
-        # which are the allowed groups and existing admins.
-        message = "root@${config.networking.fqdnOrHostName} has no SSH key attached, this machine will lose its access if you deploy it successfully! Set a valid `bagel.admins.allowedGroups` or ensure you have at least one administrator of the relevant group registered";
-      }
-    ];
-
-    users.users.root.openssh.authorizedKeys.keys = rootKeys;
-  };
-}
--- a/common/ssh-keys.nix
+++ b/common/ssh-keys.nix
@ -1,6 +1,5 @@
 {
  machines = {
-    # Floral
    bagel-box = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsO4bNqY04uG13Pg3ubHfRDssTphDLzZ4YUniE5/p+M";
    meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT";
    public01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPBy8G8rfLA6E9i+t5kjVafxU1c2NXATXKxoXTH4Kgtm";
@ -8,30 +7,19 @@
    fodwatch = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFRyTNfvKl5FcSyzGzw+h+bNFNOxdhvI67WdUZ2iIJ1L";
    buildbot = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJgIu6ouagYqBeMLfmn1CbaDJMuZcPH9bnUhkht8GfuB";
    git = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEQJcpkCUOx8+5oukMX6lxrYcIX8FyHu8Mc/3+ieKMUn";
-    bm-0 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBHSNcDGctvlG6BHcJuYIzW9WsBJsts2vpwSketsbXoL";
-    bm-1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIQOGUjERK7Mx8UPM/rbOdMqVyn1sbWqYOG6CbOzH2wm";
-    bm-2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMKzXIqCoYElEKIYgjbSpqEcDeOvV+Wo3Agq3jba83cB";
-    bm-3 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGq0A5233XGt34T097KaEKBUqFvaa7a6nYZRsSO0166l";
-    bm-4 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB9dVo2xZhgIMDgB1rUj5ApmppL39BtYu/+OFHeduvXr";
-    bm-5 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE7vZTBxrVHmHpv7slQ8A8XwjjbfN+ZJA0V5C3k0wNBD";
-    bm-6 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOt1qR/2BRtc6PABuSBulowwJVO6wBNDyEFzh0qsTeOF";
-    bm-7 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFinAAw1v8TJB8/wcmTVBbHHc4LCYh6z4TO6ViwUPkoh";
-    bm-8 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGSWHNeqT0kF/e4yVy2ieW98X5QMyCYIYZh9WTmQDs1";
-    bm-9 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOhws9zGgocVY36dMtOL+CXadpvRMffxoWMkfEcTBJm7";
-    bm-10 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE7sgIuTSqZiZhp8TvObSbIEhcHHsL5hcmYA22uzwxth";
-    # bm-11 actually?
    build-coord = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINpAEJP7F+XtJBpQP1jTzwXwQgJrFxwEJjPf/rnCXkJA";
+    builder-0 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBHSNcDGctvlG6BHcJuYIzW9WsBJsts2vpwSketsbXoL";
+    builder-1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIQOGUjERK7Mx8UPM/rbOdMqVyn1sbWqYOG6CbOzH2wm";
+    builder-2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMKzXIqCoYElEKIYgjbSpqEcDeOvV+Wo3Agq3jba83cB";
+    builder-3 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGq0A5233XGt34T097KaEKBUqFvaa7a6nYZRsSO0166l";
+    builder-4 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB9dVo2xZhgIMDgB1rUj5ApmppL39BtYu/+OFHeduvXr";
+    builder-5 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE7vZTBxrVHmHpv7slQ8A8XwjjbfN+ZJA0V5C3k0wNBD";
+    builder-6 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOt1qR/2BRtc6PABuSBulowwJVO6wBNDyEFzh0qsTeOF";
+    builder-7 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFinAAw1v8TJB8/wcmTVBbHHc4LCYh6z4TO6ViwUPkoh";
+    builder-8 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGSWHNeqT0kF/e4yVy2ieW98X5QMyCYIYZh9WTmQDs1";
+    builder-9 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOhws9zGgocVY36dMtOL+CXadpvRMffxoWMkfEcTBJm7";
+    builder-10 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE7sgIuTSqZiZhp8TvObSbIEhcHHsL5hcmYA22uzwxth";
    wob-vpn-gw = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINVytPPW8XnXf/rD5TFzsw//CZc2lBjQLmDzlVGPZsjh";
-
-    # Lix
-    build01-aarch64-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICC69NZD/zhIB/wUb5odg46bss5g8hH2fDl22bk4qeSW";
-    build02-aarch64-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGdJE375pe58RJbhKwXRp3D//+SJ3ssiVZrLsM9CLHn0";
-    build01-aarch64-darwin-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMVf1uO0lv5UBti/naW/+amqLxvWZg+StXk9aM+lJ7e4";
-
-    buildbot-lix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFoVSh35UqNQZ6ZZ1c6CzqERC40ovQ/KDXz8pC7nNlkR";
-
-    # Raito infrastructure
-    epyc-newtype-fr = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOXT9Init1MhKt4rjBANLq0t0bPww/WQZ96uB4AEDrml";
  };

  users = {
@ -62,12 +50,7 @@
      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
    ];
    thubrecht = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPM1jpXR7BWQa7Sed7ii3SbvIPRRlKb3G91qC0vOwfJn" ];
-    yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIxQ3NYBi8v1f/vhxLKDcA6upmX0pctRDbnK6SER5OUR yureka" ];
+    yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKath4/fDnlv/4fzxkPrQN1ttmoPRNu/m9bEtdPJBDfY cardno:16_933_242" ];
    winter = [ "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIH/LDRUG+U+++UmlxvA2kspioTjktQZ8taDcHq8gVlkfAAAABHNzaDo=" ];
-    ckie = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH3uTwzSSMAPg84fwbNp2cq9+BdLFeA1VzDGth4zCAbz https://mei.puppycat.house" ];
-    pennae = [
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC5Wf5/IbyFpdziWfwxkQqxOf3r1L9pYn6xQBEKFwmMY"
-      "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIK8icXjHkb4XzbIVN3djH4CE7RvgGd+3xbG4cgh0Yls5AAAABHNzaDo="
-    ];
  };
 }
--- a/common/sysadmin/default.nix
+++ b/common/sysadmin/default.nix
@ -13,11 +13,7 @@ in
        tmux
        rsync
        fd
-        eza
-        grc
        ripgrep
-        delta
-        tshark
        pv
        kitty.terminfo
        config.boot.kernelPackages.perf
--- a/common/zsh.nix
+++ b/common/zsh.nix
@ -1,4 +1,5 @@
 { lib, pkgs, config, ... }: {
+  users.defaultUserShell = pkgs.zsh;
  programs.zsh = {
    enable = true;
    enableCompletion = true;
--- a/default.nix
+++ b/default.nix
@ -1,13 +0,0 @@
-(import
-  (
-    let
-      lock = builtins.fromJSON (builtins.readFile ./flake.lock);
-      inherit (lock.nodes.flake-compat.locked) narHash rev url;
-    in
-    builtins.fetchTarball {
-      url = "${url}/archive/${rev}.tar.gz";
-      sha256 = narHash;
-    }
-  )
-  { src = ./.; }
-).defaultNix
--- a/flake.lock
+++ b/flake.lock
@ -26,20 +26,17 @@
    "attic": {
      "inputs": {
        "crane": "crane",
-        "flake-compat": [
-          "flake-compat"
-        ],
-        "flake-parts": "flake-parts_2",
-        "nix-github-actions": "nix-github-actions_2",
+        "flake-compat": "flake-compat_2",
+        "flake-utils": "flake-utils_2",
        "nixpkgs": "nixpkgs",
        "nixpkgs-stable": "nixpkgs-stable"
      },
      "locked": {
-        "lastModified": 1731270564,
-        "narHash": "sha256-6KMC/NH/VWP5Eb+hA56hz0urel3jP6Y6cF2PX6xaTkk=",
+        "lastModified": 1711742460,
+        "narHash": "sha256-0O4v6e4a1toxXZ2gf5INhg4WPE5C5T+SVvsBt+45Mcc=",
        "owner": "zhaofengli",
        "repo": "attic",
-        "rev": "47752427561f1c34debb16728a210d378f0ece36",
+        "rev": "4dbdbee45728d8ce5788db6461aaaa89d98081f0",
        "type": "github"
      },
      "original": {
@ -52,11 +49,11 @@
    "bats-assert": {
      "flake": false,
      "locked": {
-        "lastModified": 1692829535,
-        "narHash": "sha256-oDqhUQ6Xg7a3xx537SWLGRzqP3oKKeyY4UYGCdz9z/Y=",
+        "lastModified": 1636059754,
+        "narHash": "sha256-ewME0l27ZqfmAwJO4h5biTALc9bDLv7Bl3ftBzBuZwk=",
        "owner": "bats-core",
        "repo": "bats-assert",
-        "rev": "e2d855bc78619ee15b0c702b5c30fb074101159f",
+        "rev": "34551b1d7f8c7b677c1a66fc0ac140d6223409e5",
        "type": "github"
      },
      "original": {
@ -68,11 +65,11 @@
    "bats-support": {
      "flake": false,
      "locked": {
-        "lastModified": 1693050811,
-        "narHash": "sha256-PxJaH16+QrsfZqtkWVt5K6TwJB5gjIXnbGo+MB84WIU=",
+        "lastModified": 1548869839,
+        "narHash": "sha256-Gr4ntadr42F2Ks8Pte2D4wNDbijhujuoJi4OPZnTAZU=",
        "owner": "bats-core",
        "repo": "bats-support",
-        "rev": "9bf10e876dd6b624fe44423f0b35e064225f7556",
+        "rev": "d140a65044b2d6810381935ae7f0c94c7023c8c3",
        "type": "github"
      },
      "original": {
@ -90,16 +87,16 @@
        "treefmt-nix": "treefmt-nix"
      },
      "locked": {
-        "lastModified": 1730064416,
-        "narHash": "sha256-Opbtu9hKijGkEx+GYbSu3MJms3lFxZmAGTFyckguWMM=",
-        "ref": "refs/heads/forkos",
-        "rev": "79137b14f3cb376204f739f44b05aebfc288ca89",
-        "revCount": 310,
+        "lastModified": 1722939563,
+        "narHash": "sha256-lMe8aXgF550iQLRaoU+yn8yYQ4x2qiyqANgsFyjfWwA=",
+        "ref": "refs/heads/non-flakes",
+        "rev": "4a162a8aa5dad6cecdb33bd8534e67e0bdaeb13f",
+        "revCount": 295,
        "type": "git",
        "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
      },
      "original": {
-        "ref": "refs/heads/forkos",
+        "ref": "refs/heads/non-flakes",
        "type": "git",
        "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
      }
@ -111,11 +108,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1734197525,
-        "narHash": "sha256-rb/+iJBNsfXnz+PJSdlsCViodtEHrgfz/Fixq2NXUFI=",
+        "lastModified": 1725128016,
+        "narHash": "sha256-4TvaXELsl+1OcGNgqB/5HVXVxBvdIQkhJsY4FyiDcNU=",
        "ref": "refs/heads/main",
-        "rev": "6e4ae567a3f872bdb90a62d588bb5cc4b3596258",
-        "revCount": 265,
+        "rev": "23b6c38ed7e11417bf624f6e4fb6cde0d2be6400",
+        "revCount": 261,
        "type": "git",
        "url": "https://git.lix.systems/the-distro/channel-scripts.git"
      },
@ -126,22 +123,19 @@
    },
    "colmena": {
      "inputs": {
-        "flake-compat": [
-          "flake-compat"
-        ],
+        "flake-compat": "flake-compat",
        "flake-utils": "flake-utils",
-        "nix-github-actions": "nix-github-actions",
        "nixpkgs": [
          "nixpkgs"
        ],
        "stable": "stable"
      },
      "locked": {
-        "lastModified": 1731527002,
-        "narHash": "sha256-dI9I6suECoIAmbS4xcrqF8r2pbmed8WWm5LIF1yWPw8=",
+        "lastModified": 1711386353,
+        "narHash": "sha256-gWEpb8Hybnoqb4O4tmpohGZk6+aerAbJpywKcFIiMlg=",
        "owner": "zhaofengli",
        "repo": "colmena",
-        "rev": "e3ad42138015fcdf2524518dd564a13145c72ea1",
+        "rev": "cd65ef7a25cdc75052fbd04b120aeb066c3881db",
        "type": "github"
      },
      "original": {
@ -159,11 +153,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1722960479,
-        "narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=",
+        "lastModified": 1702918879,
+        "narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=",
        "owner": "ipetkov",
        "repo": "crane",
-        "rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4",
+        "rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb",
        "type": "github"
      },
      "original": {
@ -173,12 +167,18 @@
      }
    },
    "crane_2": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "nixpkgs"
+        ]
+      },
      "locked": {
-        "lastModified": 1731098351,
-        "narHash": "sha256-HQkYvKvaLQqNa10KEFGgWHfMAbWBfFp+4cAgkut+NNE=",
+        "lastModified": 1716569590,
+        "narHash": "sha256-5eDbq8TuXFGGO3mqJFzhUbt5zHVTf5zilQoyW5jnJwo=",
        "owner": "ipetkov",
        "repo": "crane",
-        "rev": "ef80ead953c1b28316cc3f8613904edc2eb90c28",
+        "rev": "109987da061a1bf452f435f1653c47511587d919",
        "type": "github"
      },
      "original": {
@ -219,11 +219,11 @@
        "rust-analyzer-src": "rust-analyzer-src"
      },
      "locked": {
-        "lastModified": 1731738660,
-        "narHash": "sha256-tIXhc9lX1b030v812yVJanSR37OnpTb/OY5rU3TbShA=",
+        "lastModified": 1716359173,
+        "narHash": "sha256-pYcjP6Gy7i6jPWrjiWAVV0BCQp+DdmGaI/k65lBb/kM=",
        "owner": "nix-community",
        "repo": "fenix",
-        "rev": "e10ba121773f754a30d31b6163919a3e404a434f",
+        "rev": "b6fc5035b28e36a98370d0eac44f4ef3fd323df6",
        "type": "github"
      },
      "original": {
@ -236,17 +236,66 @@
    "flake-compat": {
      "flake": false,
      "locked": {
-        "lastModified": 1732603366,
-        "narHash": "sha256-I1Z54H96iLmNjBtoAR8nONsj9HpagNvVZawOxn75nP0=",
-        "ref": "refs/heads/main",
-        "rev": "fe7f6ec62b50e6225406a0a4b339496530a019f8",
-        "revCount": 68,
-        "type": "git",
-        "url": "https://git.lix.systems/lix-project/flake-compat"
+        "lastModified": 1650374568,
+        "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "b4a34015c698c7793d592d66adbab377907a2be8",
+        "type": "github"
      },
      "original": {
-        "type": "git",
-        "url": "https://git.lix.systems/lix-project/flake-compat"
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_3": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "ref": "master",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_4": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
      }
    },
    "flake-parts": {
@ -273,8 +322,8 @@
    "flake-parts_2": {
      "inputs": {
        "nixpkgs-lib": [
-          "grapevine",
-          "attic",
+          "hydra",
+          "nix-eval-jobs",
          "nixpkgs"
        ]
      },
@ -292,45 +341,6 @@
        "type": "github"
      }
    },
-    "flake-parts_3": {
-      "inputs": {
-        "nixpkgs-lib": [
-          "hydra",
-          "nix-eval-jobs",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1730504689,
-        "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=",
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "rev": "506278e768c2a08bec68eb62932193e341f55c90",
-        "type": "github"
-      },
-      "original": {
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "type": "github"
-      }
-    },
-    "flake-parts_4": {
-      "inputs": {
-        "nixpkgs-lib": "nixpkgs-lib"
-      },
-      "locked": {
-        "lastModified": 1727826117,
-        "narHash": "sha256-K5ZLCyfO/Zj9mPFldf3iwS6oZStJcU4tSpiXTMYaaL0=",
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "rev": "3d04084d54bedc3d6b8b736c70ef449225c361b1",
-        "type": "github"
-      },
-      "original": {
-        "id": "flake-parts",
-        "type": "indirect"
-      }
-    },
    "flake-utils": {
      "locked": {
        "lastModified": 1659877975,
@ -347,15 +357,30 @@
      }
    },
    "flake-utils_2": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_3": {
      "inputs": {
        "systems": "systems_2"
      },
      "locked": {
-        "lastModified": 1731533236,
-        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
        "owner": "numtide",
        "repo": "flake-utils",
-        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
        "type": "github"
      },
      "original": {
@ -365,6 +390,21 @@
        "type": "github"
      }
    },
+    "flake-utils_4": {
+      "locked": {
+        "lastModified": 1634851050,
+        "narHash": "sha256-N83GlSGPJJdcqhUxSCS/WwW5pksYf3VP1M13cDRTSVA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "c91f3de5adaf1de973b797ef7485e441a65b8935",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
    "gerrit-dashboard": {
      "flake": false,
      "locked": {
@ -386,24 +426,21 @@
        "attic": "attic",
        "crane": "crane_2",
        "fenix": "fenix",
-        "flake-compat": [
-          "flake-compat"
-        ],
-        "flake-utils": "flake-utils_2",
+        "flake-compat": "flake-compat_3",
+        "flake-utils": "flake-utils_3",
        "nix-filter": "nix-filter",
        "nixpkgs": [
          "nixpkgs"
        ],
-        "rocksdb": "rocksdb",
        "rust-manifest": "rust-manifest"
      },
      "locked": {
        "host": "gitlab.computer.surgery",
-        "lastModified": 1734138037,
-        "narHash": "sha256-pN/nJ9tR6ewnpVUUzcF+Z9L/0R0WmtBVePJOqx9rzTk=",
+        "lastModified": 1723576377,
+        "narHash": "sha256-sTa4XT5xMQkhhLknOfVd433YS1TvkMrE45qAsI1ZB6U=",
        "owner": "matrix",
        "repo": "grapevine-fork",
-        "rev": "8537c0e8ac3eb388500587b035008e5f98204a4b",
+        "rev": "3b99032456700d06dd937db6a85976a8be9d4fa7",
        "type": "gitlab"
      },
      "original": {
@ -443,11 +480,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1733503045,
-        "narHash": "sha256-VoMam8Zzbk+X6dIYwH2f9NqItL6g9YDhQvGybzSl8xQ=",
+        "lastModified": 1724616313,
+        "narHash": "sha256-9syppf9Gm/6F4wQQAbsf7rGY1DooMsprnsEY/0eaewg=",
        "ref": "refs/heads/main",
-        "rev": "eccf01d4fef67f87b6383f96c73781bd08b686ac",
-        "revCount": 4230,
+        "rev": "44b9a7b95d23e7a8587cb963f00382046707f2db",
+        "revCount": 4202,
        "type": "git",
        "url": "https://git.lix.systems/lix-project/hydra.git"
      },
@ -458,9 +495,7 @@
    },
    "lix": {
      "inputs": {
-        "flake-compat": [
-          "flake-compat"
-        ],
+        "flake-compat": "flake-compat_4",
        "nix2container": "nix2container",
        "nixpkgs": [
          "hydra",
@ -470,11 +505,11 @@
        "pre-commit-hooks": "pre-commit-hooks"
      },
      "locked": {
-        "lastModified": 1732112222,
-        "narHash": "sha256-H7GN4++a4vE49SUNojZx+FSk4mmpb2ifJUtJMJHProI=",
+        "lastModified": 1723919517,
+        "narHash": "sha256-D6+zmRXzr85p7riphuIrJQqangoJe70XM5jHhMWwXws=",
        "ref": "refs/heads/main",
-        "rev": "66f6dbda32959dd5cf3a9aaba15af72d037ab7ff",
-        "revCount": 16513,
+        "rev": "278fddc317cf0cf4d3602d0ec0f24d1dd281fadb",
+        "revCount": 16138,
        "type": "git",
        "url": "https://git.lix.systems/lix-project/lix"
      },
@ -485,12 +520,12 @@
    },
    "nix-eval-jobs": {
      "inputs": {
-        "flake-parts": "flake-parts_3",
+        "flake-parts": "flake-parts_2",
        "lix": [
          "hydra",
          "lix"
        ],
-        "nix-github-actions": "nix-github-actions_3",
+        "nix-github-actions": "nix-github-actions",
        "nixpkgs": [
          "hydra",
          "nixpkgs"
@ -498,11 +533,11 @@
        "treefmt-nix": "treefmt-nix_2"
      },
      "locked": {
-        "lastModified": 1732351635,
-        "narHash": "sha256-H94CcQ3yamG5+RMxtxXllR02YIlxQ5WD/8PcolO9yEA=",
+        "lastModified": 1723579251,
+        "narHash": "sha256-xnHtfw0gRhV+2S9U7hQwvp2klTy1Iv7FlMMO0/WiMVc=",
        "ref": "refs/heads/main",
-        "rev": "dfc286ca3dc49118c30d8d6205d6d6af76c62b7a",
-        "revCount": 617,
+        "rev": "42a160bce2fd9ffebc3809746bc80cc7208f9b08",
+        "revCount": 609,
        "type": "git",
        "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
      },
@ -513,11 +548,11 @@
    },
    "nix-filter": {
      "locked": {
-        "lastModified": 1731533336,
-        "narHash": "sha256-oRam5PS1vcrr5UPgALW0eo1m/5/pls27Z/pabHNy2Ms=",
+        "lastModified": 1710156097,
+        "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
        "owner": "numtide",
        "repo": "nix-filter",
-        "rev": "f7653272fd234696ae94229839a99b73c9ab7de0",
+        "rev": "3342559a24e85fc164b295c3444e8a139924675b",
        "type": "github"
      },
      "original": {
@ -527,22 +562,6 @@
        "type": "github"
      }
    },
-    "nix-forgejo": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1734980732,
-        "narHash": "sha256-ToN/RwdfzvjAIL9n5HqLBOkupLn4emFvt6I7b5vN/+I=",
-        "ref": "refs/heads/main",
-        "rev": "404b26d8d40f36cf3953bbaa2ff602cdb8ca6acd",
-        "revCount": 4,
-        "type": "git",
-        "url": "https://git.lix.systems/the-distro/nix-forgejo.git"
-      },
-      "original": {
-        "type": "git",
-        "url": "https://git.lix.systems/the-distro/nix-forgejo.git"
-      }
-    },
    "nix-gerrit": {
      "inputs": {
        "nixpkgs": [
@ -550,11 +569,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1734192622,
-        "narHash": "sha256-AkT4QHHneyWBL9UDhvrmPnQUOfN9ETP295y6TtuW6rU=",
+        "lastModified": 1720891381,
+        "narHash": "sha256-bdZRPgnkROSejmwMOrlcqHMWmuPIVIzjk6r5FbS+fqU=",
        "ref": "refs/heads/main",
-        "rev": "c011f670b335b52150af5c75f21e987d166ecec2",
-        "revCount": 8,
+        "rev": "23dd318e6741ff686d3069c53ecf475eac8a0565",
+        "revCount": 5,
        "type": "git",
        "url": "https://git.lix.systems/the-distro/nix-gerrit.git"
      },
@ -564,49 +583,6 @@
      }
    },
    "nix-github-actions": {
-      "inputs": {
-        "nixpkgs": [
-          "colmena",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1729742964,
-        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "type": "github"
-      }
-    },
-    "nix-github-actions_2": {
-      "inputs": {
-        "nixpkgs": [
-          "grapevine",
-          "attic",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1729742964,
-        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "type": "github"
-      }
-    },
-    "nix-github-actions_3": {
      "inputs": {
        "nixpkgs": [
          "hydra",
@ -615,11 +591,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1731952509,
-        "narHash": "sha256-p4gB3Rhw8R6Ak4eMl8pqjCPOLCZRqaehZxdZ/mbFClM=",
+        "lastModified": 1720066371,
+        "narHash": "sha256-uPlLYH2S0ACj0IcgaK9Lsf4spmJoGejR9DotXiXSBZQ=",
        "owner": "nix-community",
        "repo": "nix-github-actions",
-        "rev": "7b5f051df789b6b20d259924d349a9ba3319b226",
+        "rev": "622f829f5fe69310a866c8a6cd07e747c44ef820",
        "type": "github"
      },
      "original": {
@ -631,11 +607,11 @@
    "nix2container": {
      "flake": false,
      "locked": {
-        "lastModified": 1724996935,
-        "narHash": "sha256-njRK9vvZ1JJsP8oV2OgkBrpJhgQezI03S7gzskCcHos=",
+        "lastModified": 1720642556,
+        "narHash": "sha256-qsnqk13UmREKmRT7c8hEnz26X3GFFyIQrqx4EaRc1Is=",
        "owner": "nlewo",
        "repo": "nix2container",
-        "rev": "fa6bb0a1159f55d071ba99331355955ae30b3401",
+        "rev": "3853e5caf9ad24103b13aa6e0e8bcebb47649fe4",
        "type": "github"
      },
      "original": {
@ -646,11 +622,11 @@
    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1726042813,
-        "narHash": "sha256-LnNKCCxnwgF+575y0pxUdlGZBO/ru1CtGHIqQVfvjlA=",
+        "lastModified": 1711401922,
+        "narHash": "sha256-QoQqXoj8ClGo0sqD/qWKFWezgEwUL0SUh37/vY2jNhc=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "159be5db480d1df880a0135ca0bfed84c2f88353",
+        "rev": "07262b18b97000d16a4bdb003418bd2fb067a932",
        "type": "github"
      },
      "original": {
@ -660,18 +636,6 @@
        "type": "github"
      }
    },
-    "nixpkgs-lib": {
-      "locked": {
-        "lastModified": 1727825735,
-        "narHash": "sha256-0xHYkMkeLVQAMa7gvkddbPqpxph+hDzdu1XdGPJR+Os=",
-        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
-      },
-      "original": {
-        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
-      }
-    },
    "nixpkgs-regression": {
      "locked": {
        "lastModified": 1643052045,
@ -690,62 +654,44 @@
    },
    "nixpkgs-stable": {
      "locked": {
-        "lastModified": 1724316499,
-        "narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=",
+        "lastModified": 1711460390,
+        "narHash": "sha256-akSgjDZL6pVHEfSE6sz1DNSXuYX6hq+P/1Z5IoYWs7E=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841",
+        "rev": "44733514b72e732bd49f5511bd0203dea9b9a434",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
-        "ref": "nixos-24.05",
+        "ref": "nixos-23.11",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
    "nixpkgs_2": {
      "locked": {
-        "lastModified": 1733940404,
-        "narHash": "sha256-Pj39hSoUA86ZePPF/UXiYHHM7hMIkios8TYG29kQT4g=",
+        "lastModified": 1723221148,
+        "narHash": "sha256-7pjpeQlZUNQ4eeVntytU3jkw9dFK3k1Htgk2iuXjaD8=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "5d67ea6b4b63378b9c13be21e2ec9d1afc921713",
+        "rev": "154bcb95ad51bc257c2ce4043a725de6ca700ef6",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
-        "ref": "nixos-unstable",
+        "ref": "nixpkgs-unstable",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
-    "ofborg": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1735749516,
-        "narHash": "sha256-QHI1DBDqDta447DE7zIc+HgKJ3mzmsP3G1kFjbe8mG4=",
-        "ref": "refs/heads/vcs-generalization",
-        "rev": "c741eb2dccee03b804bd7eaa36dfa6d3bd5e5308",
-        "revCount": 1500,
-        "type": "git",
-        "url": "https://git.lix.systems/the-distro/ofborg.git"
-      },
-      "original": {
-        "ref": "refs/heads/vcs-generalization",
-        "rev": "c741eb2dccee03b804bd7eaa36dfa6d3bd5e5308",
-        "type": "git",
-        "url": "https://git.lix.systems/the-distro/ofborg.git"
-      }
-    },
    "pre-commit-hooks": {
      "flake": false,
      "locked": {
-        "lastModified": 1726745158,
-        "narHash": "sha256-D5AegvGoEjt4rkKedmxlSEmC+nNLMBPWFxvmYnVLhjk=",
+        "lastModified": 1721042469,
+        "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=",
        "owner": "cachix",
        "repo": "git-hooks.nix",
-        "rev": "4e743a6920eab45e8ba0fbe49dc459f1423a4b74",
+        "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd",
        "type": "github"
      },
      "original": {
@ -754,30 +700,12 @@
        "type": "github"
      }
    },
-    "rocksdb": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1730475155,
-        "narHash": "sha256-u5uuShM2SxHc9/zL4UU56IhCcR/ZQbzde0LgOYS44bM=",
-        "owner": "facebook",
-        "repo": "rocksdb",
-        "rev": "3c27a3dde0993210c5cc30d99717093f7537916f",
-        "type": "github"
-      },
-      "original": {
-        "owner": "facebook",
-        "ref": "v9.7.4",
-        "repo": "rocksdb",
-        "type": "github"
-      }
-    },
    "root": {
      "inputs": {
        "agenix": "agenix",
        "buildbot-nix": "buildbot-nix",
        "channel-scripts": "channel-scripts",
        "colmena": "colmena",
-        "flake-compat": "flake-compat",
        "gerrit-dashboard": "gerrit-dashboard",
        "grapevine": "grapevine",
        "hydra": "hydra",
@ -785,23 +713,19 @@
          "hydra",
          "lix"
        ],
-        "nix-forgejo": "nix-forgejo",
        "nix-gerrit": "nix-gerrit",
        "nixpkgs": "nixpkgs_2",
-        "ofborg": "ofborg",
-        "stateless-uptime-kuma": "stateless-uptime-kuma",
-        "systemd-openbao": "systemd-openbao",
        "terranix": "terranix"
      }
    },
    "rust-analyzer-src": {
      "flake": false,
      "locked": {
-        "lastModified": 1731693936,
-        "narHash": "sha256-uHUUS1WPyW6ohp5Bt3dAZczUlQ22vOn7YZF8vaPKIEw=",
+        "lastModified": 1716107283,
+        "narHash": "sha256-NJgrwLiLGHDrCia5AeIvZUHUY7xYGVryee0/9D3Ir1I=",
        "owner": "rust-lang",
        "repo": "rust-analyzer",
-        "rev": "1b90e979aeee8d1db7fe14603a00834052505497",
+        "rev": "21ec8f523812b88418b2bfc64240c62b3dd967bd",
        "type": "github"
      },
      "original": {
@ -814,63 +738,31 @@
    "rust-manifest": {
      "flake": false,
      "locked": {
-        "narHash": "sha256-tB9BZB6nRHDk5ELIVlGYlIjViLKBjQl52nC1avhcCwA=",
+        "narHash": "sha256-aZFye4UrtlcvLHrISldx4g9uGt3thDbVlLMK5keBSj0=",
        "type": "file",
-        "url": "https://static.rust-lang.org/dist/channel-rust-1.81.0.toml"
+        "url": "https://static.rust-lang.org/dist/channel-rust-1.78.0.toml"
      },
      "original": {
        "type": "file",
-        "url": "https://static.rust-lang.org/dist/channel-rust-1.81.0.toml"
+        "url": "https://static.rust-lang.org/dist/channel-rust-1.78.0.toml"
      }
    },
    "stable": {
      "locked": {
-        "lastModified": 1730883749,
-        "narHash": "sha256-mwrFF0vElHJP8X3pFCByJR365Q2463ATp2qGIrDUdlE=",
+        "lastModified": 1696039360,
+        "narHash": "sha256-g7nIUV4uq1TOVeVIDEZLb005suTWCUjSY0zYOlSBsyE=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "dba414932936fde69f0606b4f1d87c5bc0003ede",
+        "rev": "32dcb45f66c0487e92db8303a798ebc548cadedc",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
-        "ref": "nixos-24.05",
+        "ref": "nixos-23.05",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
-    "stateless-uptime-kuma": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1728243069,
-        "narHash": "sha256-l9fgwesnmFxasCaYUCD7L9bGGJXytLuwtx3CZMgpwJg=",
-        "ref": "refs/heads/master",
-        "rev": "880f444ff7862d6127b051cf1a993ad1585b1652",
-        "revCount": 25,
-        "type": "git",
-        "url": "https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git"
-      },
-      "original": {
-        "type": "git",
-        "url": "https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git"
-      }
-    },
-    "systemd-openbao": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1735694158,
-        "narHash": "sha256-n8cyDX5qitjTNFQ2+nUeOpqSkXREir9p2bSqOZZ5sLs=",
-        "ref": "refs/heads/main",
-        "rev": "2479c46b0fa892c4fdcd3e315f0cdfe096b5e71a",
-        "revCount": 160,
-        "type": "git",
-        "url": "https://git.lix.systems/the-distro/systemd-openbao.git"
-      },
-      "original": {
-        "type": "git",
-        "url": "https://git.lix.systems/the-distro/systemd-openbao.git"
-      }
-    },
    "systems": {
      "locked": {
        "lastModified": 1681028828,
@ -901,38 +793,22 @@
        "type": "github"
      }
    },
-    "systems_3": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
-    },
    "terranix": {
      "inputs": {
        "bats-assert": "bats-assert",
        "bats-support": "bats-support",
-        "flake-parts": "flake-parts_4",
+        "flake-utils": "flake-utils_4",
        "nixpkgs": [
          "nixpkgs"
        ],
-        "systems": "systems_3",
        "terranix-examples": "terranix-examples"
      },
      "locked": {
-        "lastModified": 1728959489,
-        "narHash": "sha256-1Pu2j5xsBTuoyga08ZVf+rKp3FOMmJh/0fXen/idOrA=",
+        "lastModified": 1695406838,
+        "narHash": "sha256-xiUfVD6rtsVWFotVtUW3Q1nQh4obKzgvpN1wqZuGXvM=",
        "owner": "terranix",
        "repo": "terranix",
-        "rev": "7734e2ee6a1472807a33ce1e7da794bed2aaf91c",
+        "rev": "fc9077ca02ab5681935dbf0ecd725c4d889b9275",
        "type": "github"
      },
      "original": {
@ -943,11 +819,11 @@
    },
    "terranix-examples": {
      "locked": {
-        "lastModified": 1637156952,
-        "narHash": "sha256-KqvXIe1yiKOEP9BRYqNQN+LOWPCsWojh0WjEgv5jfEI=",
+        "lastModified": 1636300201,
+        "narHash": "sha256-0n1je1WpiR6XfCsvi8ZK7GrpEnMl+DpwhWaO1949Vbc=",
        "owner": "terranix",
        "repo": "terranix-examples",
-        "rev": "921680efb8af0f332d8ad73718d53907f9483e24",
+        "rev": "a934aa1cf88f6bd6c6ddb4c77b77ec6e1660bd5e",
        "type": "github"
      },
      "original": {
@ -986,11 +862,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1732292307,
-        "narHash": "sha256-5WSng844vXt8uytT5djmqBCkopyle6ciFgteuA9bJpw=",
+        "lastModified": 1723454642,
+        "narHash": "sha256-S0Gvsenh0II7EAaoc9158ZB4vYyuycvMGKGxIbERNAM=",
        "owner": "numtide",
        "repo": "treefmt-nix",
-        "rev": "705df92694af7093dfbb27109ce16d828a79155f",
+        "rev": "349de7bc435bdff37785c2466f054ed1766173be",
        "type": "github"
      },
      "original": {
--- a/flake.nix
+++ b/flake.nix
@ -2,7 +2,7 @@
  description = "Bagel cooking infrastructure";

  inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";

    terranix.url = "github:terranix/terranix";
    terranix.inputs.nixpkgs.follows = "nixpkgs";
@ -11,47 +11,22 @@
    agenix.inputs.nixpkgs.follows = "nixpkgs";

    colmena.url = "github:zhaofengli/colmena";
-    colmena.inputs = {
-      nixpkgs.follows = "nixpkgs";
-      flake-compat.follows = "flake-compat";
-    };
+    colmena.inputs.nixpkgs.follows = "nixpkgs";

    hydra.url = "git+https://git.lix.systems/lix-project/hydra.git";
-    hydra.inputs = {
-      nixpkgs.follows = "nixpkgs";
-      lix.inputs.flake-compat.follows = "flake-compat";
-    };
+    hydra.inputs.nixpkgs.follows = "nixpkgs";

    nix-gerrit.url = "git+https://git.lix.systems/the-distro/nix-gerrit.git";
    nix-gerrit.inputs.nixpkgs.follows = "nixpkgs";

-    nix-forgejo.url = "git+https://git.lix.systems/the-distro/nix-forgejo.git";
-    nix-forgejo.flake = false;
-
-    # This revision contains mTLS support.
-    ofborg.url = "git+https://git.lix.systems/the-distro/ofborg.git?ref=refs/heads/vcs-generalization&rev=c741eb2dccee03b804bd7eaa36dfa6d3bd5e5308";
-    ofborg.flake = false;
-
    gerrit-dashboard.url = "git+https://git.lix.systems/the-distro/gerrit-monitoring.git";
    gerrit-dashboard.flake = false;

-    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/forkos";
+    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/non-flakes";
    buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";

    channel-scripts.url = "git+https://git.lix.systems/the-distro/channel-scripts.git";
    channel-scripts.inputs.nixpkgs.follows = "nixpkgs";
-    channel-scripts.inputs.crane.inputs.attic.inputs.flake-compat.follows = "flake-compat";
-
-    systemd-openbao.url = "git+https://git.lix.systems/the-distro/systemd-openbao.git";
-    systemd-openbao.flake = false;
-
-    stateless-uptime-kuma.url = "git+https://git.dgnum.eu/DGNum/stateless-uptime-kuma.git";
-    stateless-uptime-kuma.flake = false;
-
-    flake-compat = {
-      url = "git+https://git.lix.systems/lix-project/flake-compat";
-      flake = false;
-    };

    lix.follows = "hydra/lix";

@ -60,15 +35,11 @@
      host = "gitlab.computer.surgery";
      owner = "matrix";
      repo = "grapevine-fork";
-      inputs = {
-        nixpkgs.follows = "nixpkgs";
-        flake-compat.follows = "flake-compat";
-        attic.inputs.flake-compat.follows = "flake-compat";
-      };
+      inputs.nixpkgs.follows = "nixpkgs";
    };
  };

-  outputs = { self, nixpkgs, terranix, colmena, ofborg, ... } @ inputs:
+  outputs = { self, nixpkgs, terranix, colmena, ... } @ inputs:
  let
    supportedSystems = [ "x86_64-linux" "aarch64-linux" ];
    forEachSystem = f: builtins.listToAttrs (map (system: {
@ -84,25 +55,16 @@
          inputs.lix.overlays.default
          inputs.nix-gerrit.overlays.default
          inputs.channel-scripts.overlays.default
-          (import inputs.ofborg {
-            pkgs = import nixpkgs { localSystem = system; };
-          }).overlay
-          (import "${inputs.stateless-uptime-kuma}/overlay.nix")
-          (self: super: {
-            openbao = super.callPackage ./services/vault/package.nix { };
-          })
        ];
      };
      terraform = pkgs.opentofu;
      terraformCfg = terranix.lib.terranixConfiguration {
        inherit system;
-        extraArgs = {
-          inherit (self) nixosConfigurations;
-        };
        modules = [
          ./terraform
          {
            bagel.dnsimple.enable = true;
+            bagel.gandi.enable = true;
            bagel.hydra.enable = true;
          }
        ];
@ -110,9 +72,6 @@
    });
    forEachSystem' = f: forEachSystem (system: (f systemBits.${system}));
    inherit (nixpkgs) lib;
-    # ForkOS' library functions.
-    flib = import ./lib { inherit (nixpkgs) lib; };
-    inherit (flib) singleton;
  in
  {
    apps = forEachSystem' ({ system, pkgs, terraformCfg, terraform, ... }: {
@ -134,26 +93,18 @@
          inputs.agenix.packages.${system}.agenix

          pkgs.opentofu
-          pkgs.openbao

          (pkgs.callPackage ./lib/colmena-wrapper.nix { })
        ];
-
-        BAO_ADDR = "https://vault.forkos.org";
      };
    });

    nixosConfigurations = (colmena.lib.makeHive self.outputs.colmena).nodes;
-    terraformConfiguration = forEachSystem' ({ terraformCfg, ... }: terraformCfg);

    colmena = let
-      systemd-openbao = import inputs.systemd-openbao { };
      commonModules = [
        inputs.agenix.nixosModules.default
        inputs.hydra.nixosModules.hydra
-        systemd-openbao.nixosModules.openbaoAgent
-        systemd-openbao.nixosModules.systemdOpenBaod
-        systemd-openbao.nixosModules.openbaoSecrets
        inputs.buildbot-nix.nixosModules.buildbot-coordinator
        inputs.buildbot-nix.nixosModules.buildbot-worker

@ -161,173 +112,26 @@
        ./common
      ];

-      floralInfraModules = commonModules ++ [
-        ({ config, lib, ... }: {
-          # This means that anyone with @floral-infra permissions
-          # can ssh on root of every machines handled here.
-          bagel.admins.allowedGroups = [
-            "floral-infra"
-          ];
-
-          # Tag all machines which have local boot as local bootables.
-          deployment.tags = lib.mkMerge [
-            [ "floral" ]
-            # All nodes that can be local booted, including baremetal nodes.
-            (lib.mkIf (config.bagel.baremetal.enable -> !config.bagel.baremetal.netboot)
-              [ "localboot" ]
-            )
-            # Only baremetal nodes that can be local booted.
-            (lib.mkIf (config.bagel.baremetal.enable && !config.bagel.baremetal.netboot)
-              [ "bm-localboot" ]
-            )
-          ];
-
-          bagel.monitoring.grafana-agent.tenant = "floral";
-          bagel.secrets.tenant = "floral";
-          bagel.builders.extra-build-capacity.provider.tenant = "floral";
-          bagel.services.buildbot.tenant = "floral";
-        })
-      ];
-
-      # These are Floral baremetal builders.
-      makeColoBaremetal = i:
-      let
-        enableNetboot = i >= 6;
-      in
-      # bm for baremetal.
-      lib.nameValuePair "bm-${toString i}" {
-        imports = floralInfraModules;
-        bagel.baremetal = { enable = true; num = i; netboot = enableNetboot; };
+      makeBuilder = i: lib.nameValuePair "builder-${toString i}" {
+        imports = commonModules;
+        bagel.baremetal.builders = { enable = true; num = i; netboot = i >= 6; };
      };

-      # Given the data of:
-      # - a selector function to filter NixOS nodes
-      # - a module factory function to extend a NixOS configuration
-      # this will return a function that will take a set of nodes and project it to the filtered
-      # nodes augmented with the module factory function.
-      # Composing twice the projector should have no effect.
-      # `mkSystem :: { renumberedIndex: int, node: NixOS configuration } → NixOS configuration`
-      mkProjector = { selector, mkSystem }: nodes:
-      let
-        # Select all the nodes using the selector.
-        selectedNodes = lib.filterAttrs (_: node: selector node.bagel.baremetal.num) nodes;
-      in
-        # Re-map selected nodes and renumber them in some iteration order
-        # and apply the module extension function.
-        flib.renumber
-          # Indexing function
-          (node: node.bagel.baremetal.num)
-          # Renumbering function
-          (renumberedIndex: node: mkSystem { inherit renumberedIndex node; })
-          selectedNodes;
-
-      # Current map:
-      # builders: [4, 10].
-      # storage: [5]
-      # build-coord: [11].
-
-      # Set of projectors that will take a generic baremetal node
-      # and reconfigure it for a specific role.
-      projectors = {
-        storage = {
-          # Selectors are just fancy functions that can filter based on the index information.
-          # It is possible to construct a range filter to express a collection of intervals,
-          # e.g. select 0→4 & 6→8 & 12→15.
-
-          # For now, we will only use pointwise as we have very few machines.
-          selector = flib.mkPointwiseFilter [ 5 ];
-          mkSystem = { renumberedIndex, node }:
-          {
-            imports = [ node ];
-            bagel.baremetal.storage = {
-              enable = true;
-              num = renumberedIndex;
-            };
-          };
-        };
-        builders = {
-          selector = flib.mkPointwiseFilter [ 4 10 ];
-          mkSystem = { renumberedIndex, node }: {
-            imports = [ node ];
-            bagel.baremetal.builders = {
-              enable = true;
-              num = renumberedIndex;
-            };
-          };
-        };
-      };
-      project = role: mkProjector projectors.${role};
-
-      lixInfraModules = commonModules ++ [
-        {
-          # This means that anyone with @lix-infra permissions
-          # can ssh on root of every machines handled here.
-          bagel.admins.allowedGroups = [
-            "lix-infra"
-          ];
-
-          # Tag all machines which have local boot as local bootables.
-          # Lix has no netbootable machine.
-          deployment.tags = [ "localboot" "lix" ];
-
-          bagel.monitoring.grafana-agent.tenant = "lix";
-          bagel.secrets.tenant = "lix";
-          bagel.builders.extra-build-capacity.provider = {
-            tenant = "lix";
-            buildfarmPublicKeys = [
-              # buildbot.lix.systems SSH key
-              "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDu4cEqZzAI/1vZjSQkTJ4ijIg9nuloOuSKUrnkJIOFn"
-            ];
-          };
-          bagel.services.buildbot.tenant = "lix";
-        }
-      ];
-
-      baremetalNodes =
-      let
-        # We consider all possible baremetal systems and we filter out a subset that is activated.
-        # To configure the set of used machines, configure the `setXYZ` role setter selectors.
-        allNodes = lib.listToAttrs (lib.genList makeColoBaremetal 11);
-        perRoles = {
-          # Project in the sense of linear algebra projectors.
-          # We are projecting allNodes on the set of storage nodes.
-          # (remember, a projector is a linear function such that p^2 = p).
-          storageNodes = project "storage" allNodes;
-          builderNodes = project "builders" allNodes;
-          # buildCoordinatorNodes = setBuildCoordinators allNodes;
-        };
-      in
-      # TODO: compute what are the offender nodes and their simultaneous roles.
-      assert (lib.assertMsg (flib.isValidPartition perRoles) "A baremetal node is simultaneously storage, builder and build coordinator, please review the ranges.");
-      # Merge all roles together into one big attribute set of nodes.
-      flib.chainAttrs perRoles;
-
+      builders = lib.listToAttrs (lib.genList makeBuilder 11);
    in {
      meta.nixpkgs = systemBits.x86_64-linux.pkgs;
-      # Add any non-x86_64 native systems here.
-      # Cross compilation is not supported yet.
-      meta.nodeNixpkgs =
-        let
-          aarch64-systems = systems: lib.genAttrs systems (system: systemBits.aarch64-linux.pkgs);
-        in
-        aarch64-systems [
-          "build01-aarch64-lix"
-        ];
      meta.specialArgs.inputs = inputs;

-      bagel-box.imports = floralInfraModules ++ [ ./hosts/bagel-box ];
-      meta01.imports = floralInfraModules ++ [ ./hosts/meta01 ];
-      gerrit01.imports = floralInfraModules ++ [ ./hosts/gerrit01 ];
-      fodwatch.imports = floralInfraModules ++ [ ./hosts/fodwatch ];
-      git.imports = floralInfraModules ++ [ ./hosts/git ];
-      wob-vpn-gw.imports = floralInfraModules ++ [ ./hosts/wob-vpn-gw ];
-      buildbot.imports = floralInfraModules ++ [ ./hosts/buildbot ];
-      public01.imports = floralInfraModules ++ [ ./hosts/public01 ];
-      build-coord.imports = floralInfraModules ++ [ ./hosts/build-coord ];
-
-      build01-aarch64-lix.imports = lixInfraModules ++ [ ./hosts/build01-aarch64-lix ];
-      buildbot-lix.imports = lixInfraModules ++ [ ./hosts/buildbot-lix ];
-    } // baremetalNodes;
+      bagel-box.imports = commonModules ++ [ ./hosts/bagel-box ];
+      meta01.imports = commonModules ++ [ ./hosts/meta01 ];
+      gerrit01.imports = commonModules ++ [ ./hosts/gerrit01 ];
+      fodwatch.imports = commonModules ++ [ ./hosts/fodwatch ];
+      git.imports = commonModules ++ [ ./hosts/git ];
+      wob-vpn-gw.imports = commonModules ++ [ ./hosts/wob-vpn-gw ];
+      buildbot.imports = commonModules ++ [ ./hosts/buildbot ];
+      public01.imports = commonModules ++ [ ./hosts/public01 ];
+      build-coord.imports = commonModules ++ [ ./hosts/build-coord ];
+    } // builders;

    hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.netbootDir or v.config.system.build.toplevel) self.nixosConfigurations;
    buildbotJobs = builtins.mapAttrs (_: v: v.config.system.build.toplevel) self.nixosConfigurations;
--- a/hosts/bagel-box/default.nix
+++ b/hosts/bagel-box/default.nix
@ -37,30 +37,7 @@

  bagel.services = {
    postgres.enable = true;
-    ofborg = {
-      rabbitmq.enable = true;
-      pastebin.enable = true;
-      # TODO: statcheck.enable = true;
-
-      mass-rebuilder.enable = true;
-      # TODO: enable once ready.
-      builder.enable = false;
-
-      gerrit-event-streamer.enable = true;
-      gerrit-generic-vcs-filter.enable = true;
-
-      # FIXME: plug into our prometheus stack.
-      stats.enable = true;
-    };
-
-    secrets-agent = {
-      enable = true;
-      methods.token = {
-        enable = true;
-        tenancy = "floral";
-        identifier = "bagel-box";
-      };
-    };
+    ofborg.enable = true;
  };

  bagel.sysadmin.enable = true;
--- a/hosts/build-coord/default.nix
+++ b/hosts/build-coord/default.nix
@ -9,19 +9,10 @@

  bagel.services = {
    hydra.enable = true;
-    # TODO: use the roles to avoid setting up builders which are not… builders!
-    hydra.builders = map (i: "bm-${builtins.toString i}") [4 10];
-
-    # Arguably, the build-coordinator is the most sensitive piece of our own infrastructure.
-    # Henceforth, it can run as well another sensitive piece of the system: the Vault.
-    vault = {
-      enable = true;
-      domain = "vault.forkos.org";
-    };
+    # Takes 10 builders (0 → 9).
+    hydra.builders = lib.genList (i: "builder-${builtins.toString i}") 10;
  };

-  bagel.monitoring.exporters.hydra.enable = true;
-
  # Hydra is proxied.
  bagel.raito.v6-proxy-awareness.enable = true;

--- a/hosts/build-coord/hardware.nix
+++ b/hosts/build-coord/hardware.nix
@ -38,7 +38,7 @@

  zramSwap = {
    enable = true;
-    memoryPercent = 100;
+    memoryPercent = 25;
  };

  networking.useNetworkd = true;
@ -72,12 +72,6 @@
  ];
  networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };

-  bagel.infra.self.wan = {
-    family = "inet6";
-    address = "2a01:584:11::1:11";
-    prefixLength = 64;
-  };
-
  services.coredns = {
    enable = true;
    config = ''
--- a/hosts/build01-aarch64-lix/default.nix
+++ b/hosts/build01-aarch64-lix/default.nix
@ -1,27 +0,0 @@
-{ ... }: {
-  networking.hostName = "build01";
-  networking.domain = "aarch64.lix.systems";
-
-  # Those free sweet VMs.
-  bagel.hardware.oracle-vm = {
-    enable = true;
-    system = "aarch64-linux";
-  };
-
-  fileSystems."/" =
-    { device = "/dev/disk/by-uuid/a333323c-99f0-4258-8f68-496858d56f71";
-      fsType = "ext4";
-    };
-
-  fileSystems."/boot" =
-    { device = "/dev/disk/by-uuid/3E74-C937";
-      fsType = "vfat";
-    };
-
-  swapDevices = [ ];
-
-  bagel.builders.extra-build-capacity.provider.enable = true;
-  i18n.defaultLocale = "en_US.UTF-8";
-  system.stateVersion = "24.05";
-  deployment.targetHost = "build01.aarch64.lix.systems";
-}
--- a/hosts/buildbot-lix/default.nix
+++ b/hosts/buildbot-lix/default.nix
@ -1,71 +0,0 @@
-# Configuration for a virtual machine in Raito's micro-DC basement.
-# 32 vCPU (2014 grade Xeon though)
-# 32GB RAM
-# 30GB SSD
-# 500GB HDD
-# All specifications can be upgraded to a certain extent, just ask Raito.
-# Hosts the coordinator for Buildbot.
-#
-# vim: et:ts=2:sw=2:
-#
-{ lib, modulesPath, ... }: {
-  networking.hostName = "buildbot";
-  networking.domain = "lix.systems";
-
-  zramSwap.enable = true;
-
-  bagel.sysadmin.enable = true;
-  # Buildbot is proxied.
-  bagel.raito.v6-proxy-awareness.enable = true;
-  bagel.hardware.raito-vm = {
-    enable = true;
-    networking = {
-      nat-lan-mac = "BC:24:11:75:62:42";
-      wan = {
-        mac = "BC:24:11:B2:5F:2E";
-        address = "2001:bc8:38ee:100::200/56";
-      };
-    };
-  };
-  i18n.defaultLocale = "en_US.UTF-8";
-
-  bagel.services.buildbot = {
-    enable = true;
-    domain = "buildbot.lix.systems";
-    gerrit =
-    {
-      domain = "gerrit.lix.systems";
-      port = 2022;
-      username = "buildbot";
-    };
-    cors.allowedOrigins = [
-      "https://*.lix.systems"
-    ];
-    projects = [
-      "lix"
-      "lix-installer"
-    ];
-    buildSystems = [
-      "x86_64-linux"
-      "aarch64-linux"
-      "aarch64-darwin"
-      # Too slow.
-      /* "x86_64-darwin" */
-    ];
-    # Lix is not allowed to use yet Floral's x86_64 builders for now.
-    builders = [ ];
-  };
-
-  # This machine does not use /nix from btrfs, and instead uses a store on a bigger disk.
-  fileSystems."/nix" =
-    lib.mkForce
-    { device = "/dev/disk/by-uuid/1815ca49-d0b0-4b99-8aec-0d790498ba6f";
-      fsType = "xfs";
-      neededForBoot = true;
-      options = [ "relatime" ];
-    };
-
-  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
-  system.stateVersion = "24.05";
-  deployment.targetHost = "buildbot.lix.systems";
-}
--- a/hosts/buildbot/default.nix
+++ b/hosts/buildbot/default.nix
@ -2,7 +2,6 @@
  config,
  lib,
  pkgs,
-  nodes,
  ...
 }:
 {
@ -27,24 +26,7 @@
  bagel.services.buildbot = {
    enable = true;
    domain = "buildbot.forkos.org";
-    gerrit =
-      let
-        cfgGerrit = nodes.gerrit01.config.bagel.services.gerrit;
-      in
-      {
-        domain = cfgGerrit.canonicalDomain;
-        port = cfgGerrit.port;
-        username = "buildbot";
-      };
-    cors.allowedOrigins = [
-      "https://*.forkos.org"
-    ];
-    projects = [
-      "buildbot-test"
-      "nixpkgs"
-      "infra"
-    ];
-    builders = [ "builder-4" ];
+    builders = [ "builder-10" ];
  };

  i18n.defaultLocale = "en_US.UTF-8";
--- a/hosts/gerrit01/default.nix
+++ b/hosts/gerrit01/default.nix
@ -23,9 +23,6 @@
    };
  };

-  # Block all these crawlers!!
-  bagel.services.nginx.crawler-blocker.enable = true;
-
  fileSystems."/gerrit-data" = {
    device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";
    fsType = "ext4";
@ -42,7 +39,7 @@
  };

  age.secrets.ows-deploy-key = {
-    file = ../../secrets/floral/ows-deploy-key.age;
+    file = ../../secrets/ows-deploy-key.age;
    mode = "0600";
    owner = "git";
    group = "git";
@ -61,8 +58,6 @@
  {
    enable = true;

-    stateDirectory = "/gerrit-data/ows";
-
    pushUrl = "ssh://ows_bot@cl.forkos.org:29418/nixpkgs";
    deployKeyPath = config.age.secrets.ows-deploy-key.path;

@ -109,9 +104,24 @@
      timer = "hourly";
      fromRefspec = "staging-23.11";
    };
+
+    # Testing jobs for personal sandbox branches
+    branches."raito-unstable-sync" = {
+      fromUri = "https://github.com/NixOS/nixpkgs";
+      fromRefspec = "nixos-unstable-small";
+      localRefspec = "sandbox/raito/raito-unstable-small";
+      timer = "*-*-* 12:00:00";
+    };
+
+    branches."raito-release-sync" = {
+      fromUri = "https://github.com/NixOS/nixpkgs";
+      fromRefspec = "nixos-24.05";
+      localRefspec = "sandbox/raito/raito-nixos-24.05";
+      timer = "daily";
+    };
  };

-  age.secrets.s3-channel-staging-keys.file = ../../secrets/floral/s3-channel-staging-keys.age;
+  age.secrets.s3-channel-staging-keys.file = ../../secrets/s3-channel-staging-keys.age;
  bagel.nixpkgs.channel-scripts = {
    enable = true;
    otlp.enable = true;
--- a/hosts/public01/default.nix
+++ b/hosts/public01/default.nix
@ -9,11 +9,6 @@
  # TODO: make it the default
  networking.domain = "infra.forkos.org";

-  bagel.status = {
-    enable = true;
-    domain = "status.forkos.org";
-  };
-
  bagel.sysadmin.enable = true;
  # Newsletter is proxied.
  bagel.raito.v6-proxy-awareness.enable = true;
--- a/lib/default.nix
+++ b/lib/default.nix
@ -1,65 +0,0 @@
-# Some useful utilities to do things that depends on the nixpkgs library.
-{ lib }:
-let
-  inherit (lib) listToAttrs zipListsWith nameValuePair length range foldl any mapAttrs;
-in
-rec {
-  closedOpenInterval = a: b: { start = a; end = b; };
-  interval = a: b: closedOpenInterval a b;
-  singleton = x: interval x (x + 1);
-
-  inRange = i: range: i >= range.start && i < range.end;
-
-  # Build a selector function that will filters point-by-point any index in xs.
-  # e.g. if you want to select specific indexes you can just use that.
-  # If you want to select contiguous interval of indexes, you are better served by
-  # `mkIntervalFilter`.
-  mkPointwiseFilter = xs: index: any (allowedIndex: index == allowedIndex) xs;
-
-  # Build a selector function that will filters interval-by-interval any index in intervals.
-  # It will check if the given index is present in any of the passed intervals according
-  # to `inRange`.
-  mkIntervalFilter = intervals: index: any (allowedRange: inRange index allowedRange) intervals;
-
-  # Build an attribute set map from values to indexes.
-  # e.g. reversedEnumerate [ "a" "b" ] == { "a" = 0; "b" = 1; }.
-  reversedEnumerate = list: listToAttrs
-    (zipListsWith
-      (index: value: nameValuePair value index)
-      (range 0 (length list - 1))
-    list);
-
-  # Collect a list of attribute sets into an attribute set.
-  # Merge order depends on attrValues iteration order and foldl.
-  chainAttrs = attrs: foldl (a: b: a // b) { } (builtins.attrValues attrs);
-
-  # Given an attribute set of an attribute set of items, does it describe a valid partition of some global set?
-  # This does not check for completeness.
-  # idFunction :: Attrs K V → List Identifier
-  isValidPartition = attrs:
-  let
-    values = builtins.attrValues attrs;
-  in
-  # TODO(performance?): this is the simple dumb idea.
-  # A better idea would use n(n - 1)/2 iterations over values to exploit symmetry of item equality.
-  # To do so, a strategy could be to consider all shifted toplevel identifiers lists and zip them.
-  # There's sum_k(n - k) such lists, and therefore: n(n - 1)/2 lists.
-  # For every list, we need to perform list intersection which is supposedly in O(n log n) in the size of the nodes identifiers.
-  # So, if we have N subsets in the partition and each subset has at most K items, we end up doing something like (K log K) * N(N - 1)/2
-  # In practice, K should be the biggest and N is quite small.
-  lib.all (subset:
-    lib.all (anotherSubset:
-      subset != anotherSubset -> lib.intersectAttrs subset anotherSubset == {}
-    ) values
-  ) values;
-
-  # Renumber an attribute set of items.
-  # For each item in the attribute set, we replace its value by a call to the renumbering function
-  # where we pass renumberedIndex and value.
-  # It's a form of imap for attribute sets.
-  renumber = indexFn: renumberingFn: attrs:
-  let
-    indexes = reversedEnumerate (map (n: toString (indexFn n)) (builtins.attrValues attrs));
-  in
-    mapAttrs (name: value: renumberingFn indexes.${toString (indexFn value)} value) attrs;
-}
--- a/overlays/pyroscope/default.nix
+++ b/overlays/pyroscope/default.nix
@ -1,10 +1,9 @@
 { lib
-, buildGo122Module
+, buildGoModule
 , fetchFromGitHub
 }:

-# FIXME: update, remove this pin
-buildGo122Module rec {
+buildGoModule rec {
  pname = "pyroscope";
  version = "1.7.1";

--- a/pkgs/forgejo/branch-view_remove-expensive-commit-divergence-metric.patch
+++ b/pkgs/forgejo/branch-view_remove-expensive-commit-divergence-metric.patch
@ -0,0 +1,59 @@
+diff --git a/services/repository/branch.go b/services/repository/branch.go
+index e1a313749f..5a8d823eef 100644
+--- a/services/repository/branch.go
+++ b/services/repository/branch.go
+@@ -26,7 +26,6 @@ import (
+ 	"code.gitea.io/gitea/modules/timeutil"
+ 	webhook_module "code.gitea.io/gitea/modules/webhook"
+ 	notify_service "code.gitea.io/gitea/services/notify"
+-	files_service "code.gitea.io/gitea/services/repository/files"
+ 
+ 	"xorm.io/builder"
+ )
+@@ -129,21 +128,7 @@ func loadOneBranch(ctx context.Context, repo *repo_model.Repository, dbBranch *g
+ 	p := protectedBranches.GetFirstMatched(branchName)
+ 	isProtected := p != nil
+ 
+-	var divergence *git.DivergeObject
+-
+-	// it's not default branch
+-	if repo.DefaultBranch != dbBranch.Name && !dbBranch.IsDeleted {
+-		var err error
+-		divergence, err = files_service.CountDivergingCommits(ctx, repo, git.BranchPrefix+branchName)
+-		if err != nil {
+-			return nil, fmt.Errorf("CountDivergingCommits: %v", err)
+-		}
+-	}
+-
+-	if divergence == nil {
+-		// tolerate the error that we cannot get divergence
+-		divergence = &git.DivergeObject{Ahead: -1, Behind: -1}
+-	}
+	divergence := &git.DivergeObject{Ahead: -1, Behind: -1}
+ 
+ 	pr, err := issues_model.GetLatestPullRequestByHeadInfo(ctx, repo.ID, branchName)
+ 	if err != nil {
+diff --git a/templates/repo/branch/list.tmpl b/templates/repo/branch/list.tmpl
+index a577fed450..e102796315 100644
+--- a/templates/repo/branch/list.tmpl
+++ b/templates/repo/branch/list.tmpl
+@@ -102,19 +102,6 @@
+ 							{{end}}
+ 							</td>
+ 							<td class="two wide ui">
+-								{{if and (not .DBBranch.IsDeleted) $.DefaultBranchBranch}}
+-								<div class="commit-divergence">
+-									<div class="bar-group">
+-										<div class="count count-behind">{{.CommitsBehind}}</div>
+-										{{/* old code bears 0/0.0 = NaN output, so it might output invalid "width: NaNpx", it just works and doesn't caues any problem. */}}
+-										<div class="bar bar-behind" style="width: {{Eval 100 "*" .CommitsBehind "/" "(" .CommitsBehind "+" .CommitsAhead "+" 0.0 ")"}}%"></div>
+-									</div>
+-									<div class="bar-group">
+-										<div class="count count-ahead">{{.CommitsAhead}}</div>
+-										<div class="bar bar-ahead" style="width: {{Eval 100 "*" .CommitsAhead "/" "(" .CommitsBehind "+" .CommitsAhead "+" 0.0 ")"}}%"></div>
+-									</div>
+-								</div>
+-								{{end}}
+ 							</td>
+ 							<td class="two wide right aligned">
+ 								{{if not .LatestPullRequest}}
--- a/pkgs/forgejo/commit-view_fix-broken-and-expensive-cherry-pick-default-branch-selection.patch
+++ b/pkgs/forgejo/commit-view_fix-broken-and-expensive-cherry-pick-default-branch-selection.patch
@ -0,0 +1,32 @@
+diff --git a/routers/web/repo/commit.go b/routers/web/repo/commit.go
+index 718454e063..8fa299710c 100644
+--- a/routers/web/repo/commit.go
+++ b/routers/web/repo/commit.go
+@@ -408,12 +408,6 @@ func Diff(ctx *context.Context) {
+ 		}
+ 	}
+ 
+-	ctx.Data["BranchName"], err = commit.GetBranchName()
+-	if err != nil {
+-		ctx.ServerError("commit.GetBranchName", err)
+-		return
+-	}
+-
+ 	ctx.HTML(http.StatusOK, tplCommitPage)
+ }
+ 
+diff --git a/templates/repo/commit_page.tmpl b/templates/repo/commit_page.tmpl
+index c37fb46975..18c9cf18f8 100644
+--- a/templates/repo/commit_page.tmpl
+++ b/templates/repo/commit_page.tmpl
+@@ -71,8 +71,8 @@
+ 												"branchForm" "branch-dropdown-form"
+ 												"branchURLPrefix" (printf "%s/_cherrypick/%s/" $.RepoLink .CommitID) "branchURLSuffix" ""
+ 												"setAction" true "submitForm" true}}
+-											<form method="get" action="{{$.RepoLink}}/_cherrypick/{{.CommitID}}/{{if $.BranchName}}{{PathEscapeSegments $.BranchName}}{{else}}{{PathEscapeSegments $.Repository.DefaultBranch}}{{end}}" id="branch-dropdown-form">
+-												<input type="hidden" name="ref" value="{{if $.BranchName}}{{$.BranchName}}{{else}}{{$.Repository.DefaultBranch}}{{end}}">
+											<form method="get" action="{{$.RepoLink}}/_cherrypick/{{.CommitID}}/{{PathEscapeSegments $.Repository.DefaultBranch}}" id="branch-dropdown-form">
+												<input type="hidden" name="ref" value="{{$.Repository.DefaultBranch}}">
+ 												<input type="hidden" name="refType" value="branch">
+ 												<input type="hidden" id="cherry-pick-type" name="cherry-pick-type"><br>
+ 												<button type="submit" id="cherry-pick-submit" class="ui primary button"></button>
--- a/pkgs/forgejo/default.nix
+++ b/pkgs/forgejo/default.nix
@ -0,0 +1,40 @@
+{ forgejo-lts }:
+
+forgejo-lts.overrideAttrs (prev: {
+  patches = [
+    # Branch divergence calculations for a single branch may take 100-200ms on something as big
+    # as nixpkgs. The branch view defaults to 20 branches for each page, taking roughtly 3s to
+    # calculate each branch sequentially and render, while consuming a single core at 100%.
+    # The idea is to look into making this less expensive or async.
+    # But for now, to get this going, we will simply drop that metric.
+    ./branch-view_remove-expensive-commit-divergence-metric.patch
+
+    # This is literally broken and eats resources for nothing of value.
+    # We should upstream this.
+    # The tl;dr is: It calculates the nearest branch for the requested commit at
+    # /:owner/:repo/commit/:commit to use it as the default cherry-pick target branch
+    # selection in a drop-down only users with commit perms can actually view and use.
+    # It's expensive to calculate and happens on every request to /commit/:commit.
+    # To add insult to injury, it's hardly of any use: The nearest branch of a commit
+    # will almost always be a branch that already carries the commit. The branch you
+    # most likely don't want to cherry-pick to.
+    ./commit-view_fix-broken-and-expensive-cherry-pick-default-branch-selection.patch
+
+    # Disable various /:owner/:repo/activity/ sub-views. They are expensive, which is
+    # totally fine and expected. There is even proper caching in place.
+    # However, on a scale of nixpkgs, those calculations take ages, while, of course,
+    # pinning a single CPU core at 100%.
+    # For now, we will simply disable this feature.
+    # Due to the 501 status code it returns, the frontend prints a "Not implemented"
+    # error, saving us from patching the frontend while still providing a helpful
+    # user-facing error text.
+    # It should be noted that this particular status code has the downside of being
+    # in the 5xx range, meaning it will show up as such in our prometheus metrics.
+    ./disable-expensive-repository-activity-stats.patch
+
+    # Migrations and pull-mirrors are something easily abused to bring a public instance to a complete halt.
+    # Both features can be disabled via repository.DISABLE_MIGRATIONS and mirror.ENABLE, but we want to keep
+    # this functionality for admins.
+    ./limit-migrations-and-pull-mirrors-to-admins.patch
+  ];
+})
--- a/pkgs/forgejo/disable-expensive-repository-activity-stats.patch
+++ b/pkgs/forgejo/disable-expensive-repository-activity-stats.patch
@ -0,0 +1,34 @@
+diff --git a/routers/web/web.go b/routers/web/web.go
+index ee9694f41c..f55b8d6f62 100644
+--- a/routers/web/web.go
+++ b/routers/web/web.go
+@@ -57,6 +57,10 @@ import (
+ 	"github.com/prometheus/client_golang/prometheus"
+ )
+ 
+func endpointNotImplemented(ctx *context.Context) {
+	ctx.JSON(http.StatusNotImplemented, "This endpoint has been removed due to performance issues with it and as such is not longer implemented.")
+}
+
+ // optionsCorsHandler return a http handler which sets CORS options if enabled by config, it blocks non-CORS OPTIONS requests.
+ func optionsCorsHandler() func(next http.Handler) http.Handler {
+ 	var corsHandler func(next http.Handler) http.Handler
+@@ -1425,15 +1429,15 @@ func registerRoutes(m *web.Route) {
+ 			m.Get("/{period}", repo.Activity)
+ 			m.Group("/contributors", func() {
+ 				m.Get("", repo.Contributors)
+-				m.Get("/data", repo.ContributorsData)
+				m.Get("/data", endpointNotImplemented)
+ 			}, repo.MustBeNotEmpty, context.RequireRepoReaderOr(unit.TypeCode))
+ 			m.Group("/code-frequency", func() {
+ 				m.Get("", repo.CodeFrequency)
+-				m.Get("/data", repo.CodeFrequencyData)
+				m.Get("/data", endpointNotImplemented)
+ 			}, repo.MustBeNotEmpty, context.RequireRepoReaderOr(unit.TypeCode))
+ 			m.Group("/recent-commits", func() {
+ 				m.Get("", repo.RecentCommits)
+-				m.Get("/data", repo.RecentCommitsData)
+				m.Get("/data", endpointNotImplemented)
+ 			}, repo.MustBeNotEmpty, context.RequireRepoReaderOr(unit.TypeCode))
+ 		}, context.RepoRef(), context.RequireRepoReaderOr(unit.TypeCode, unit.TypePullRequests, unit.TypeIssues, unit.TypeReleases))
+ 
--- a/pkgs/forgejo/limit-migrations-and-pull-mirrors-to-admins.patch
+++ b/pkgs/forgejo/limit-migrations-and-pull-mirrors-to-admins.patch
@ -0,0 +1,53 @@
+diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go
+index 2caaa130e8..455e89e93e 100644
+--- a/routers/api/v1/repo/migrate.go
+++ b/routers/api/v1/repo/migrate.go
+@@ -12,7 +12,6 @@ import (
+ 
+ 	"code.gitea.io/gitea/models"
+ 	"code.gitea.io/gitea/models/db"
+-	"code.gitea.io/gitea/models/organization"
+ 	"code.gitea.io/gitea/models/perm"
+ 	access_model "code.gitea.io/gitea/models/perm/access"
+ 	repo_model "code.gitea.io/gitea/models/repo"
+@@ -86,22 +85,7 @@ func Migrate(ctx *context.APIContext) {
+ 	}
+ 
+ 	if !ctx.Doer.IsAdmin {
+-		if !repoOwner.IsOrganization() && ctx.Doer.ID != repoOwner.ID {
+-			ctx.Error(http.StatusForbidden, "", "Given user is not an organization.")
+-			return
+-		}
+-
+-		if repoOwner.IsOrganization() {
+-			// Check ownership of organization.
+-			isOwner, err := organization.OrgFromUser(repoOwner).IsOwnedBy(ctx, ctx.Doer.ID)
+-			if err != nil {
+-				ctx.Error(http.StatusInternalServerError, "IsOwnedBy", err)
+-				return
+-			} else if !isOwner {
+-				ctx.Error(http.StatusForbidden, "", "Given user is not owner of organization.")
+-				return
+-			}
+-		}
+		ctx.Error(http.StatusForbidden, "", "You need to be administrator of this Forgejo instance to be able to create mirrors.")
+ 	}
+ 
+ 	remoteAddr, err := forms.ParseRemoteAddr(form.CloneAddr, form.AuthUsername, form.AuthPassword)
+diff --git a/routers/web/repo/migrate.go b/routers/web/repo/migrate.go
+index 97b0c425ea..554a470eab 100644
+--- a/routers/web/repo/migrate.go
+++ b/routers/web/repo/migrate.go
+@@ -150,6 +150,12 @@ func handleMigrateRemoteAddrError(ctx *context.Context, err error, tpl base.TplN
+ // MigratePost response for migrating from external git repository
+ func MigratePost(ctx *context.Context) {
+ 	form := web.GetForm(ctx).(*forms.MigrateRepoForm)
+
+	if !ctx.Doer.IsAdmin {
+		ctx.Error(http.StatusForbidden, "MigratePost: you need to be site administrator to use migrations and mirrors")
+		return
+	}
+
+ 	if setting.Repository.DisableMigrations {
+ 		ctx.Error(http.StatusForbidden, "MigratePost: the site administrator has disabled migrations")
+ 		return
--- a/pki/.envrc
+++ b/pki/.envrc
@ -1 +0,0 @@
-use nix
--- a/pki/README.md
+++ b/pki/README.md
@ -1,5 +0,0 @@
-# PKI management
-
-This is our expressions to generate and manage our PKI in the project.
-
-We are using NitroHSMs for the offline storage and OpenBao server for the online operations.
--- a/pki/cacerts/README.md
+++ b/pki/cacerts/README.md
@ -1,17 +0,0 @@
-# CA certificate chains
-
-## `ca.crt`
-
-The root CA.
-
-## `ica1.crt`
-
-The chain from ICA1 to root CA.
-
-## `ica2.crt`
-
-The chain from ICA2 to root CA (ICA2 → ICA1 → root CA), this is what you want to usually use to trust our PKI, assuming you send any intermediate CAs.
-
-## `infra.crt`
-
-The chain from the infra CA to root CA (infra → ICA2 → ICA1 → root CA), this is what you want to trust for the infrastructure.
--- a/pki/cacerts/ca.crt
+++ b/pki/cacerts/ca.crt
@ -1,19 +0,0 @@
-----BEGIN CERTIFICATE-----
-MIIDJDCCAoagAwIBAgIUHW9bhbgk6GXm5i+uamYWbInHDhkwCgYIKoZIzj0EAwQw
-gZsxCzAJBgNVBAYTAkRFMQ8wDQYDVQQIDAZIZXNzZW4xEjAQBgNVBAcMCURhcm1z
-dGFkdDEXMBUGA1UECgwORmxvcmFsIFN5c3RlbXMxLTArBgNVBAsMJEZsb3JhbCBT
-eXN0ZW1zIENlcnRpZmljYXRlIEF1dGhvcml0eTEfMB0GA1UEAwwWRmxvcmFsIFN5
-c3RlbXMgUm9vdCBDQTAeFw0yNDEyMzAxMzEwMDlaFw0zNDEyMjgxMzEwMDlaMIGb
-MQswCQYDVQQGEwJERTEPMA0GA1UECAwGSGVzc2VuMRIwEAYDVQQHDAlEYXJtc3Rh
-ZHQxFzAVBgNVBAoMDkZsb3JhbCBTeXN0ZW1zMS0wKwYDVQQLDCRGbG9yYWwgU3lz
-dGVtcyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxHzAdBgNVBAMMFkZsb3JhbCBTeXN0
-ZW1zIFJvb3QgQ0EwgZswEAYHKoZIzj0CAQYFK4EEACMDgYYABAD6xFA+QeHoUVZr
-WaDbfoUkELxnviEPLogl8+IgJ06ki+84yIAM3Zn+6IlmnJGoPaceoPIdYwHByWqf
-wvhvTobYRgB8T4l7vyt/KmMfkD2SU576syuR23PkJ6eImGklU3P1+H9CyU2BoPIg
-N21Kumx7GCvGAA8NsQyQVdZeLZ6lYjnCfaNjMGEwHQYDVR0OBBYEFGUVeaIqxFM/
-kmfTBNYf6sYyKyuBMB8GA1UdIwQYMBaAFGUVeaIqxFM/kmfTBNYf6sYyKyuBMA8G
-A1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMAoGCCqGSM49BAMEA4GLADCB
-hwJBLvw4lfu2efHxdkPZpddMe9wLrrOFwoeYMIJ4XN4qn8WwQCy4G0oXTKHzwm3y
-I82YwdK5r6tUtdoHhQ5BscrrnRsCQgGNejEZMet0lFgch1Dr2iunnsOEpdODtapD
-Jwp4PRUSTdlqk0C2GOWUtbcK2arZ/QexnqLAKhASuY/clqVZLLzHTw==
-----END CERTIFICATE-----
--- a/pki/cacerts/ica1.crt
+++ b/pki/cacerts/ica1.crt
@ -1,36 +0,0 @@
-----BEGIN CERTIFICATE-----
-MIICrzCCAhKgAwIBAgIUUfC3HiC4wWFjkavirLxjTpVrxkcwCgYIKoZIzj0EAwIw
-gZsxCzAJBgNVBAYTAkRFMQ8wDQYDVQQIDAZIZXNzZW4xEjAQBgNVBAcMCURhcm1z
-dGFkdDEXMBUGA1UECgwORmxvcmFsIFN5c3RlbXMxLTArBgNVBAsMJEZsb3JhbCBT
-eXN0ZW1zIENlcnRpZmljYXRlIEF1dGhvcml0eTEfMB0GA1UEAwwWRmxvcmFsIFN5
-c3RlbXMgUm9vdCBDQTAeFw0yNDEyMzExNjQ0MjJaFw0yNTAxMzAxNjQ0MjJaMIGZ
-MQswCQYDVQQGEwJERTEPMA0GA1UECBMGSGVzc2VuMRIwEAYDVQQHEwlEYXJtc3Rh
-ZHQxFzAVBgNVBAoTDkZsb3JhbCBTeXN0ZW1zMS0wKwYDVQQLEyRGbG9yYWwgU3lz
-dGVtcyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxHTAbBgNVBAMTFEludGVybWVkaWF0
-ZSBDQTEgdjEgMCowBQYDK2VwAyEA/SgktXV6oQ4Bk5X9P0uAtX08g4hgdyYY/q+z
-0C+D9OujYzBhMB0GA1UdDgQWBBRqxA1IFDZW0IULtTmjs6HdHnmL+zAfBgNVHSME
-GDAWgBRlFXmiKsRTP5Jn0wTWH+rGMisrgTAPBgNVHRMBAf8EBTADAQH/MA4GA1Ud
-DwEB/wQEAwIBhjAKBggqhkjOPQQDAgOBigAwgYYCQDgp6p7TvWOZmaC0WZHnVCeU
-AVJ1qSKjHRqnLUHAIBoPTvsEm1ActVcOYOyq5VxS7StirkULn7qWKzr2l67k5MYC
-QgG5sSKwP7vn+2B+/yNkBQTbHKyNZAQOg+tvPTwrmzmBzak3J1b2d4+qSkq9JEnZ
-uCAwXV3uHmNPlK4jgr4SHxwYKg==
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
-MIIDJDCCAoagAwIBAgIUHW9bhbgk6GXm5i+uamYWbInHDhkwCgYIKoZIzj0EAwQw
-gZsxCzAJBgNVBAYTAkRFMQ8wDQYDVQQIDAZIZXNzZW4xEjAQBgNVBAcMCURhcm1z
-dGFkdDEXMBUGA1UECgwORmxvcmFsIFN5c3RlbXMxLTArBgNVBAsMJEZsb3JhbCBT
-eXN0ZW1zIENlcnRpZmljYXRlIEF1dGhvcml0eTEfMB0GA1UEAwwWRmxvcmFsIFN5
-c3RlbXMgUm9vdCBDQTAeFw0yNDEyMzAxMzEwMDlaFw0zNDEyMjgxMzEwMDlaMIGb
-MQswCQYDVQQGEwJERTEPMA0GA1UECAwGSGVzc2VuMRIwEAYDVQQHDAlEYXJtc3Rh
-ZHQxFzAVBgNVBAoMDkZsb3JhbCBTeXN0ZW1zMS0wKwYDVQQLDCRGbG9yYWwgU3lz
-dGVtcyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxHzAdBgNVBAMMFkZsb3JhbCBTeXN0
-ZW1zIFJvb3QgQ0EwgZswEAYHKoZIzj0CAQYFK4EEACMDgYYABAD6xFA+QeHoUVZr
-WaDbfoUkELxnviEPLogl8+IgJ06ki+84yIAM3Zn+6IlmnJGoPaceoPIdYwHByWqf
-wvhvTobYRgB8T4l7vyt/KmMfkD2SU576syuR23PkJ6eImGklU3P1+H9CyU2BoPIg
-N21Kumx7GCvGAA8NsQyQVdZeLZ6lYjnCfaNjMGEwHQYDVR0OBBYEFGUVeaIqxFM/
-kmfTBNYf6sYyKyuBMB8GA1UdIwQYMBaAFGUVeaIqxFM/kmfTBNYf6sYyKyuBMA8G
-A1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMAoGCCqGSM49BAMEA4GLADCB
-hwJBLvw4lfu2efHxdkPZpddMe9wLrrOFwoeYMIJ4XN4qn8WwQCy4G0oXTKHzwm3y
-I82YwdK5r6tUtdoHhQ5BscrrnRsCQgGNejEZMet0lFgch1Dr2iunnsOEpdODtapD
-Jwp4PRUSTdlqk0C2GOWUtbcK2arZ/QexnqLAKhASuY/clqVZLLzHTw==
-----END CERTIFICATE-----
--- a/pki/cacerts/ica2.crt
+++ b/pki/cacerts/ica2.crt
@ -1,51 +0,0 @@
-----BEGIN CERTIFICATE-----
-MIICXTCCAg+gAwIBAgIUcLJmuRVLSn7NRVKOJnLIcIjAKy0wBQYDK2VwMIGZMQsw
-CQYDVQQGEwJERTEPMA0GA1UECBMGSGVzc2VuMRIwEAYDVQQHEwlEYXJtc3RhZHQx
-FzAVBgNVBAoTDkZsb3JhbCBTeXN0ZW1zMS0wKwYDVQQLEyRGbG9yYWwgU3lzdGVt
-cyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxHTAbBgNVBAMTFEludGVybWVkaWF0ZSBD
-QTEgdjEgMB4XDTI0MTIzMTE2NTA0OVoXDTI1MTIzMTE2NTExOVowgZoxCzAJBgNV
-BAYTAkRFMQ8wDQYDVQQIEwZIZXNzZW4xEjAQBgNVBAcTCURhcm1zdGFkdDEXMBUG
-A1UEChMORmxvcmFsIFN5c3RlbXMxLTArBgNVBAsTJEZsb3JhbCBTeXN0ZW1zIENl
-cnRpZmljYXRlIEF1dGhvcml0eTEeMBwGA1UEAxMVSW50ZXJtZWRpYXRlIENBMiB2
-MS4xMCowBQYDK2VwAyEAlMaf5T/o39ZZmieNszDxjsVP06xb3IIV7ds+01g2pQij
-ZjBkMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEBMB0GA1UdDgQW
-BBS8G8fUlv8s+7AvikDnIS4j8bp7HjAfBgNVHSMEGDAWgBRqxA1IFDZW0IULtTmj
-s6HdHnmL+zAFBgMrZXADQQDBLEUMedqJhNtRqEOY2NHsRdqhA5kvzDuYk+hUyCaQ
-VhLbW5+EfQL7vLkv8VihN7jlaRl+ngsfRBLK0LA4YJkB
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
-MIICrzCCAhKgAwIBAgIUUfC3HiC4wWFjkavirLxjTpVrxkcwCgYIKoZIzj0EAwIw
-gZsxCzAJBgNVBAYTAkRFMQ8wDQYDVQQIDAZIZXNzZW4xEjAQBgNVBAcMCURhcm1z
-dGFkdDEXMBUGA1UECgwORmxvcmFsIFN5c3RlbXMxLTArBgNVBAsMJEZsb3JhbCBT
-eXN0ZW1zIENlcnRpZmljYXRlIEF1dGhvcml0eTEfMB0GA1UEAwwWRmxvcmFsIFN5
-c3RlbXMgUm9vdCBDQTAeFw0yNDEyMzExNjQ0MjJaFw0yNTAxMzAxNjQ0MjJaMIGZ
-MQswCQYDVQQGEwJERTEPMA0GA1UECBMGSGVzc2VuMRIwEAYDVQQHEwlEYXJtc3Rh
-ZHQxFzAVBgNVBAoTDkZsb3JhbCBTeXN0ZW1zMS0wKwYDVQQLEyRGbG9yYWwgU3lz
-dGVtcyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxHTAbBgNVBAMTFEludGVybWVkaWF0
-ZSBDQTEgdjEgMCowBQYDK2VwAyEA/SgktXV6oQ4Bk5X9P0uAtX08g4hgdyYY/q+z
-0C+D9OujYzBhMB0GA1UdDgQWBBRqxA1IFDZW0IULtTmjs6HdHnmL+zAfBgNVHSME
-GDAWgBRlFXmiKsRTP5Jn0wTWH+rGMisrgTAPBgNVHRMBAf8EBTADAQH/MA4GA1Ud
-DwEB/wQEAwIBhjAKBggqhkjOPQQDAgOBigAwgYYCQDgp6p7TvWOZmaC0WZHnVCeU
-AVJ1qSKjHRqnLUHAIBoPTvsEm1ActVcOYOyq5VxS7StirkULn7qWKzr2l67k5MYC
-QgG5sSKwP7vn+2B+/yNkBQTbHKyNZAQOg+tvPTwrmzmBzak3J1b2d4+qSkq9JEnZ
-uCAwXV3uHmNPlK4jgr4SHxwYKg==
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
-MIIDJDCCAoagAwIBAgIUHW9bhbgk6GXm5i+uamYWbInHDhkwCgYIKoZIzj0EAwQw
-gZsxCzAJBgNVBAYTAkRFMQ8wDQYDVQQIDAZIZXNzZW4xEjAQBgNVBAcMCURhcm1z
-dGFkdDEXMBUGA1UECgwORmxvcmFsIFN5c3RlbXMxLTArBgNVBAsMJEZsb3JhbCBT
-eXN0ZW1zIENlcnRpZmljYXRlIEF1dGhvcml0eTEfMB0GA1UEAwwWRmxvcmFsIFN5
-c3RlbXMgUm9vdCBDQTAeFw0yNDEyMzAxMzEwMDlaFw0zNDEyMjgxMzEwMDlaMIGb
-MQswCQYDVQQGEwJERTEPMA0GA1UECAwGSGVzc2VuMRIwEAYDVQQHDAlEYXJtc3Rh
-ZHQxFzAVBgNVBAoMDkZsb3JhbCBTeXN0ZW1zMS0wKwYDVQQLDCRGbG9yYWwgU3lz
-dGVtcyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxHzAdBgNVBAMMFkZsb3JhbCBTeXN0
-ZW1zIFJvb3QgQ0EwgZswEAYHKoZIzj0CAQYFK4EEACMDgYYABAD6xFA+QeHoUVZr
-WaDbfoUkELxnviEPLogl8+IgJ06ki+84yIAM3Zn+6IlmnJGoPaceoPIdYwHByWqf
-wvhvTobYRgB8T4l7vyt/KmMfkD2SU576syuR23PkJ6eImGklU3P1+H9CyU2BoPIg
-N21Kumx7GCvGAA8NsQyQVdZeLZ6lYjnCfaNjMGEwHQYDVR0OBBYEFGUVeaIqxFM/
-kmfTBNYf6sYyKyuBMB8GA1UdIwQYMBaAFGUVeaIqxFM/kmfTBNYf6sYyKyuBMA8G
-A1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMAoGCCqGSM49BAMEA4GLADCB
-hwJBLvw4lfu2efHxdkPZpddMe9wLrrOFwoeYMIJ4XN4qn8WwQCy4G0oXTKHzwm3y
-I82YwdK5r6tUtdoHhQ5BscrrnRsCQgGNejEZMet0lFgch1Dr2iunnsOEpdODtapD
-Jwp4PRUSTdlqk0C2GOWUtbcK2arZ/QexnqLAKhASuY/clqVZLLzHTw==
-----END CERTIFICATE-----
--- a/pki/cacerts/infra.crt
+++ b/pki/cacerts/infra.crt
@ -1,66 +0,0 @@
-----BEGIN CERTIFICATE-----
-MIICbDCCAh6gAwIBAgIUfzoqAP1fiwDncDYJjtLvHQcjobQwBQYDK2VwMIGaMQsw
-CQYDVQQGEwJERTEPMA0GA1UECBMGSGVzc2VuMRIwEAYDVQQHEwlEYXJtc3RhZHQx
-FzAVBgNVBAoTDkZsb3JhbCBTeXN0ZW1zMS0wKwYDVQQLEyRGbG9yYWwgU3lzdGVt
-cyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxHjAcBgNVBAMTFUludGVybWVkaWF0ZSBD
-QTIgdjEuMTAeFw0yNTAxMDEwMTA3NDVaFw0yNjAxMDEwMTA4MTVaMIGoMQswCQYD
-VQQGEwJERTEPMA0GA1UECBMGSGVzc2VuMRIwEAYDVQQHEwlEYXJtc3RhZHQxFzAV
-BgNVBAoTDkZsb3JhbCBTeXN0ZW1zMS0wKwYDVQQLEyRGbG9yYWwgU3lzdGVtcyBD
-ZXJ0aWZpY2F0ZSBBdXRob3JpdHkxLDAqBgNVBAMTI0ludGVybWVkaWF0ZSBJbmZy
-YXN0cnVjdHVyZSBDQSB2MS4xMCowBQYDK2VwAyEAgE4pxQEoZ1nhYtBUoamCkJEZ
-VjnYABTm8iWSe4UPtdOjZjBkMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAG
-AQH/AgEAMB0GA1UdDgQWBBQyAkN71b8P9RTIIS8c8zpxmFpGaTAfBgNVHSMEGDAW
-gBS8G8fUlv8s+7AvikDnIS4j8bp7HjAFBgMrZXADQQC1mhYcFCc34g3Yu7I32Un1
-Ux60AnboO8eG+C8hGktxvBZNoGJ9uYjoyp+LwiAEa1NBLavPnOFFGATmCcCbGekA
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
-MIICXTCCAg+gAwIBAgIUcLJmuRVLSn7NRVKOJnLIcIjAKy0wBQYDK2VwMIGZMQsw
-CQYDVQQGEwJERTEPMA0GA1UECBMGSGVzc2VuMRIwEAYDVQQHEwlEYXJtc3RhZHQx
-FzAVBgNVBAoTDkZsb3JhbCBTeXN0ZW1zMS0wKwYDVQQLEyRGbG9yYWwgU3lzdGVt
-cyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxHTAbBgNVBAMTFEludGVybWVkaWF0ZSBD
-QTEgdjEgMB4XDTI0MTIzMTE2NTA0OVoXDTI1MTIzMTE2NTExOVowgZoxCzAJBgNV
-BAYTAkRFMQ8wDQYDVQQIEwZIZXNzZW4xEjAQBgNVBAcTCURhcm1zdGFkdDEXMBUG
-A1UEChMORmxvcmFsIFN5c3RlbXMxLTArBgNVBAsTJEZsb3JhbCBTeXN0ZW1zIENl
-cnRpZmljYXRlIEF1dGhvcml0eTEeMBwGA1UEAxMVSW50ZXJtZWRpYXRlIENBMiB2
-MS4xMCowBQYDK2VwAyEAlMaf5T/o39ZZmieNszDxjsVP06xb3IIV7ds+01g2pQij
-ZjBkMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEBMB0GA1UdDgQW
-BBS8G8fUlv8s+7AvikDnIS4j8bp7HjAfBgNVHSMEGDAWgBRqxA1IFDZW0IULtTmj
-s6HdHnmL+zAFBgMrZXADQQDBLEUMedqJhNtRqEOY2NHsRdqhA5kvzDuYk+hUyCaQ
-VhLbW5+EfQL7vLkv8VihN7jlaRl+ngsfRBLK0LA4YJkB
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
-MIICrzCCAhKgAwIBAgIUUfC3HiC4wWFjkavirLxjTpVrxkcwCgYIKoZIzj0EAwIw
-gZsxCzAJBgNVBAYTAkRFMQ8wDQYDVQQIDAZIZXNzZW4xEjAQBgNVBAcMCURhcm1z
-dGFkdDEXMBUGA1UECgwORmxvcmFsIFN5c3RlbXMxLTArBgNVBAsMJEZsb3JhbCBT
-eXN0ZW1zIENlcnRpZmljYXRlIEF1dGhvcml0eTEfMB0GA1UEAwwWRmxvcmFsIFN5
-c3RlbXMgUm9vdCBDQTAeFw0yNDEyMzExNjQ0MjJaFw0yNTAxMzAxNjQ0MjJaMIGZ
-MQswCQYDVQQGEwJERTEPMA0GA1UECBMGSGVzc2VuMRIwEAYDVQQHEwlEYXJtc3Rh
-ZHQxFzAVBgNVBAoTDkZsb3JhbCBTeXN0ZW1zMS0wKwYDVQQLEyRGbG9yYWwgU3lz
-dGVtcyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxHTAbBgNVBAMTFEludGVybWVkaWF0
-ZSBDQTEgdjEgMCowBQYDK2VwAyEA/SgktXV6oQ4Bk5X9P0uAtX08g4hgdyYY/q+z
-0C+D9OujYzBhMB0GA1UdDgQWBBRqxA1IFDZW0IULtTmjs6HdHnmL+zAfBgNVHSME
-GDAWgBRlFXmiKsRTP5Jn0wTWH+rGMisrgTAPBgNVHRMBAf8EBTADAQH/MA4GA1Ud
-DwEB/wQEAwIBhjAKBggqhkjOPQQDAgOBigAwgYYCQDgp6p7TvWOZmaC0WZHnVCeU
-AVJ1qSKjHRqnLUHAIBoPTvsEm1ActVcOYOyq5VxS7StirkULn7qWKzr2l67k5MYC
-QgG5sSKwP7vn+2B+/yNkBQTbHKyNZAQOg+tvPTwrmzmBzak3J1b2d4+qSkq9JEnZ
-uCAwXV3uHmNPlK4jgr4SHxwYKg==
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
-MIIDJDCCAoagAwIBAgIUHW9bhbgk6GXm5i+uamYWbInHDhkwCgYIKoZIzj0EAwQw
-gZsxCzAJBgNVBAYTAkRFMQ8wDQYDVQQIDAZIZXNzZW4xEjAQBgNVBAcMCURhcm1z
-dGFkdDEXMBUGA1UECgwORmxvcmFsIFN5c3RlbXMxLTArBgNVBAsMJEZsb3JhbCBT
-eXN0ZW1zIENlcnRpZmljYXRlIEF1dGhvcml0eTEfMB0GA1UEAwwWRmxvcmFsIFN5
-c3RlbXMgUm9vdCBDQTAeFw0yNDEyMzAxMzEwMDlaFw0zNDEyMjgxMzEwMDlaMIGb
-MQswCQYDVQQGEwJERTEPMA0GA1UECAwGSGVzc2VuMRIwEAYDVQQHDAlEYXJtc3Rh
-ZHQxFzAVBgNVBAoMDkZsb3JhbCBTeXN0ZW1zMS0wKwYDVQQLDCRGbG9yYWwgU3lz
-dGVtcyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxHzAdBgNVBAMMFkZsb3JhbCBTeXN0
-ZW1zIFJvb3QgQ0EwgZswEAYHKoZIzj0CAQYFK4EEACMDgYYABAD6xFA+QeHoUVZr
-WaDbfoUkELxnviEPLogl8+IgJ06ki+84yIAM3Zn+6IlmnJGoPaceoPIdYwHByWqf
-wvhvTobYRgB8T4l7vyt/KmMfkD2SU576syuR23PkJ6eImGklU3P1+H9CyU2BoPIg
-N21Kumx7GCvGAA8NsQyQVdZeLZ6lYjnCfaNjMGEwHQYDVR0OBBYEFGUVeaIqxFM/
-kmfTBNYf6sYyKyuBMB8GA1UdIwQYMBaAFGUVeaIqxFM/kmfTBNYf6sYyKyuBMA8G
-A1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMAoGCCqGSM49BAMEA4GLADCB
-hwJBLvw4lfu2efHxdkPZpddMe9wLrrOFwoeYMIJ4XN4qn8WwQCy4G0oXTKHzwm3y
-I82YwdK5r6tUtdoHhQ5BscrrnRsCQgGNejEZMet0lFgch1Dr2iunnsOEpdODtapD
-Jwp4PRUSTdlqk0C2GOWUtbcK2arZ/QexnqLAKhASuY/clqVZLLzHTw==
-----END CERTIFICATE-----
--- a/pki/csr/Floral_Systems_v1_ICA1_v1.csr
+++ b/pki/csr/Floral_Systems_v1_ICA1_v1.csr
@ -1,8 +0,0 @@
-----BEGIN CERTIFICATE REQUEST-----
-MIIBGjCBzQIBADCBmTELMAkGA1UEBhMCREUxDzANBgNVBAgTBkhlc3NlbjESMBAG
-A1UEBxMJRGFybXN0YWR0MRcwFQYDVQQKEw5GbG9yYWwgU3lzdGVtczEtMCsGA1UE
-CxMkRmxvcmFsIFN5c3RlbXMgQ2VydGlmaWNhdGUgQXV0aG9yaXR5MR0wGwYDVQQD
-ExRJbnRlcm1lZGlhdGUgQ0ExIHYxIDAqMAUGAytlcAMhAP0oJLV1eqEOAZOV/T9L
-gLV9PIOIYHcmGP6vs9Avg/TroAAwBQYDK2VwA0EATxwhMrur5dneuko3+Atpwt7V
-HIW1LrZKqbyo0DPVhs5mcQ9BXKFX1N+zhReR8Et/tx3ZIJ+OtjZslBQ71JESCA==
-----END CERTIFICATE REQUEST-----
--- a/pki/csr/README.md
+++ b/pki/csr/README.md
@ -1,3 +0,0 @@
-# A trace of our CSRs files
-
-This is a collection of the CSRs we built for our needs.
--- a/pki/default.nix
+++ b/pki/default.nix
@ -1,23 +0,0 @@
-{ flake ? import ../., nixpkgs ? flake.inputs.nixpkgs, pkgs ? import nixpkgs { } }:
-{
-  shell = pkgs.mkShell {
-    buildInputs = [
-      pkgs.openssl
-    ];
-
-    OPENSSL_CONF = pkgs.writeText "openssl-pkcs11.conf" ''
-      openssl_conf = openssl_def
-
-      [openssl_def]
-      engines = engine_section
-
-      [engine_section]
-      pkcs11 = pkcs11_section
-
-      [pkcs11_section]
-      engine_id = pkcs11
-      dynamic_path = ${pkgs.libp11}/lib/engines/libpkcs11.so
-      MODULE_PATH = ${pkgs.opensc}/lib/opensc-pkcs11.so
-    '';
-  };
-}
--- a/pki/policies/README.md
+++ b/pki/policies/README.md
@ -1 +0,0 @@
-# OpenSSL policies for extensions and CAs
--- a/pki/policies/ca.conf
+++ b/pki/policies/ca.conf
@ -1,56 +0,0 @@
-[ ca ]
-# `man ca`
-default_ca = CA_default
-
-[ CA_default ]
-# Directory and file locations.
-dir               = .
-certs             = $dir/certs
-crl_dir           = $dir/crl
-new_certs_dir     = $dir/newcerts
-database          = $dir/index.txt
-serial            = $dir/serial
-
-# SHA-1 is deprecated, so use SHA-2 instead.
-default_md        = sha512
-
-name_opt          = ca_default
-cert_opt          = ca_default
-default_days      = 375
-preserve          = no
-policy            = policy_strict
-
-[ policy_strict ]
-# The root CA should only sign intermediate certificates that match.
-# See the POLICY FORMAT section of `man ca`.
-countryName             = match
-stateOrProvinceName     = match
-organizationName        = match
-organizationalUnitName  = optional
-commonName              = supplied
-emailAddress            = optional
-
-[ req ]
-# Options for the `req` tool (`man req`).
-default_bits        = 4096
-distinguished_name  = req_distinguished_name
-string_mask         = utf8only
-prompt              = no
-
-# SHA-1 is deprecated, so use SHA-2 instead.
-default_md          = sha512
-
-[ req_distinguished_name ]
-C                   = DE
-ST                  = Hessen
-L                   = Darmstadt
-O                   = Floral Systems
-OU                  = Floral Systems Certificate Authority
-CN                  = Floral Systems Root CA
-
-[ v3_ca ]
-# Extensions for a typical CA (`man x509v3_config`).
-subjectKeyIdentifier = hash
-authorityKeyIdentifier = keyid:always,issuer
-basicConstraints = critical, CA:true
-keyUsage = critical, digitalSignature, cRLSign, keyCertSign
--- a/pki/policies/ica1_intermediate.conf
+++ b/pki/policies/ica1_intermediate.conf
@ -1,6 +0,0 @@
-[ v3_intermediate_ca ]
-# Extensions for a typical intermediate CA (`man x509v3_config`).
-subjectKeyIdentifier = hash
-authorityKeyIdentifier = keyid:always,issuer
-basicConstraints = critical, CA:true
-keyUsage = critical, digitalSignature, cRLSign, keyCertSign
--- a/pki/shell.nix
+++ b/pki/shell.nix
@ -1,2 +0,0 @@
-(import ./. { }).shell
-
--- a/secrets.nix
+++ b/secrets.nix
@ -1,80 +1,51 @@
 let
  keys = import common/ssh-keys.nix;

-  commonKeys = {
-    # WARNING: `keys.users.*` are *lists*, so you need concatenate them, don't put them into lists!
-    # Otherwise, agenix will be confused!
-    global = keys.users.raito;
-    lix = keys.users.hexchen ++ keys.users.jade;
-    floral = keys.users.delroth;
-  };
+  commonKeys = keys.users.delroth ++ keys.users.raito;

  secrets = with keys; {
-    floral = {
-      hydra-postgres-key = [ machines.build-coord ];
-      hydra-s3-credentials = [ machines.build-coord ];
-      hydra-signing-priv = [ machines.build-coord ];
-      hydra-ssh-key-priv = [ machines.build-coord ];
+    hydra-postgres-key = [ machines.build-coord ];
+    hydra-s3-credentials = [ machines.build-coord ];
+    hydra-signing-priv = [ machines.build-coord ];
+    hydra-ssh-key-priv = [ machines.build-coord ];

-      netbox-environment = [ machines.meta01 ];
-      mimir-environment = [ machines.meta01 ];
-      mimir-webhook-url = [ machines.meta01 ];
-      grafana-oauth-secret = [ machines.meta01 ];
-      loki-environment = [ machines.meta01 ];
-      gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
-      pyroscope-secrets = [ machines.meta01 ];
-      tempo-environment = [ machines.meta01 ];
+    netbox-environment = [ machines.meta01 ];
+    mimir-environment = [ machines.meta01 ];
+    mimir-webhook-url = [ machines.meta01 ];
+    grafana-oauth-secret = [ machines.meta01 ];
+    loki-environment = [ machines.meta01 ];
+    gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
+    pyroscope-secrets = [ machines.meta01 ];
+    tempo-environment = [ machines.meta01 ];

-      buildbot-worker-password = [ machines.buildbot ];
-      buildbot-oauth-secret = [ machines.buildbot ];
-      buildbot-workers = [ machines.buildbot ];
-      # Private SSH key to Gerrit
-      # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
-      buildbot-service-key = [ machines.buildbot ];
-      # Signing key for Buildbot's specific cache
-      buildbot-signing-key = [ machines.buildbot ];
-      buildbot-remote-builder-key = [ machines.buildbot ];
+    buildbot-worker-password = [ machines.buildbot ];
+    buildbot-oauth-secret = [ machines.buildbot ];
+    buildbot-workers = [ machines.buildbot ];
+    # Private SSH key to Gerrit
+    # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
+    buildbot-service-key = [ machines.buildbot ];
+    # Signing key for Buildbot's specific cache
+    buildbot-signing-key = [ machines.buildbot ];
+    buildbot-remote-builder-key = [ machines.buildbot ];

-      # These are the same password, but nginx wants it in htpasswd format
-      metrics-push-htpasswd = [ machines.meta01 ];
-      # Yes, even Lix machines are included in this monitoring infrastructure.
-      metrics-push-password = builtins.attrValues machines;
+    # These are the same password, but nginx wants it in htpasswd format
+    metrics-push-htpasswd = [ machines.meta01 ];
+    metrics-push-password = builtins.attrValues machines;

-      ows-deploy-key = [ machines.gerrit01 ];
-      s3-channel-staging-keys = [ machines.gerrit01 ];
-      s3-channel-keys = [ machines.gerrit01 ];
+    ows-deploy-key = [ machines.gerrit01 ];
+    s3-channel-staging-keys = [ machines.gerrit01 ];
+    s3-channel-keys = [ machines.gerrit01 ];

-      postgres-ca-priv = [ machines.bagel-box ];
-      postgres-tls-priv = [ machines.bagel-box ];
-      rabbitmq-password = [ machines.bagel-box ];
-      gerrit-event-listener-ssh-key = [ machines.bagel-box ];
+    postgres-ca-priv = [ machines.bagel-box ];
+    postgres-tls-priv = [ machines.bagel-box ];

-      newsletter-secrets = [ machines.public01 ];
-      s3-revproxy-api-keys = [ machines.public01 ];
-      stateless-uptime-kuma-password = [ machines.public01 ];
-
-      openbao-auth-token-bagel-box = [ machines.bagel-box ];
-    };
-
-    lix = {
-      buildbot-worker-password = [ machines.buildbot-lix ];
-      buildbot-oauth-secret = [ machines.buildbot-lix ];
-      buildbot-workers = [ machines.buildbot-lix ];
-      # Private SSH key to Gerrit
-      # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
-      buildbot-service-key = [ machines.buildbot-lix ];
-      # Signing key for Buildbot's specific cache
-      buildbot-signing-key = [ machines.buildbot-lix ];
-      buildbot-remote-builder-key = [ machines.buildbot-lix ];
-    };
+    newsletter-secrets = [ machines.public01 ];
+    s3-revproxy-api-keys = [ machines.public01 ];
  };
-
-  mkSecretListFor = tenant:
-    map (secretName: {
-      name = "secrets/${tenant}/${secretName}.age";
-      value.publicKeys = secrets.${tenant}."${secretName}" ++ commonKeys.global ++ commonKeys.${tenant};
-    }) (builtins.attrNames secrets.${tenant});
 in
  builtins.listToAttrs (
-    (mkSecretListFor "floral") ++ (mkSecretListFor "lix")
+    map (secretName: {
+      name = "secrets/${secretName}.age";
+      value.publicKeys = secrets."${secretName}" ++ commonKeys;
+    }) (builtins.attrNames secrets)
  )
--- a/secrets/floral/buildbot-oauth-secret.age
+++ b/secrets/floral/buildbot-oauth-secret.age
--- a/secrets/floral/buildbot-remote-builder-key.age
+++ b/secrets/floral/buildbot-remote-builder-key.age
--- a/secrets/floral/buildbot-service-key.age
+++ b/secrets/floral/buildbot-service-key.age
--- a/secrets/floral/buildbot-signing-key.age
+++ b/secrets/floral/buildbot-signing-key.age
--- a/secrets/floral/buildbot-worker-password.age
+++ b/secrets/floral/buildbot-worker-password.age
--- a/secrets/floral/buildbot-workers.age
+++ b/secrets/floral/buildbot-workers.age
--- a/secrets/floral/gerrit-event-listener-ssh-key.age
+++ b/secrets/floral/gerrit-event-listener-ssh-key.age
--- a/secrets/floral/metrics-push-password.age
+++ b/secrets/floral/metrics-push-password.age
@ -1,68 +0,0 @@
-age-encryption.org/v1
-> ssh-ed25519 +HUDfA d5f2ESneC0wsoc9rwTjNfNXMBjCbjAQ7euthH2Buq1E
-5CynaQ8zhDRBvcmifhCsiDtllztCVAqs8rU36DOxgPw
-> ssh-ed25519 +uvEmw EtYRis2LP0jv1W8mx8vFYNzkgi8OoqnA8cM2huS6NBk
-ll1csFIO+hVYk+I0uSVJmlDKj9aTWvf4kaYI5LJcm7w
-> ssh-ed25519 DMaM1w ex4QJN8CG99J15i+yvqGEiEZn9OlGIC+cmLHL4u8ZEI
-VXnOv4CGK68q5t6hUV3oKAtxGZ+4FVbrmE1yMn16A0Q
-> ssh-ed25519 sixKXw drXN6+q1y7L7ZU4chTfHfelu5GcTdff+i/UMFV0+3RQ
-+8jmgnMh2OpQ3vhAuyQYWslfx7KO84a8KsCpoRD3Yl8
-> ssh-ed25519 aHbF7w Af7NgjZ/Nvh5FHrX2VlF5riTIhJ+fdxTo6OR+8PcNwA
-ktKpm/HnOnw2Ym7xee3N1rneEX7+/xDhcp71N1NNHAA
-> ssh-ed25519 87T2Ig 8mEUxJ/5NUvV+qQCDQH2Tm6Ryr5hf4xgsQlqXGf03Fw
-EavMcnsg/3EYBLQEBHX+0oTDKq5ZL4vj+mZntPM8UMU
-> ssh-ed25519 Ao+7Wg UphWbatIaa+R1oZbfHazFhrawf0vax/3ZZS7YuX03Hs
-dwBbwoV0jpjiKr+nj+CRfUDgDl7ISpsCintVAzHnIFQ
-> ssh-ed25519 wIR2ZA ZM58Nq7eJX9JVeYkoJf+mw8hxhYGoTx042ow1u3mJkw
-UtEaf7e4xsPO0ISlIF9LF+GcwTBqw4AXdMO4MASfgLQ
-> ssh-ed25519 oGiV/Q G5KX/Eox+9md0yFRUZvGIsio2gWM17soHsL6H6zEX2g
-vI8jPjBAoFF0xhvRRLPzCMSiQOQ0fKuRb3CYVu3KUUo
-> ssh-ed25519 gO3aog p9nZtjzoA0zJM+7Y6R16mpdub3dhu67yOYTUNKRytgI
-YL9vAp1+CK7jgmXkB47ufZMz+/swngkdUvEGR1zFZwc
-> ssh-ed25519 N/+Clw 6LzFdtNsWewuJK2r97ZXJbRazvK3raN78UGanR/zWVU
-WT0y+sfDP3ffVwRcbYw51ArFR3OzXnoyi9IXwZZKEL8
-> ssh-ed25519 CtkSZw CV0jQ5dIbgFtMxGK1X9b1qJOKmske8VgIPW5NW9mAwc
-clv7P3de61nZmXrvbOgL7Llw8ZqBMm2WFqgpznDwKv8
-> ssh-ed25519 keg2lg 3Nk40ByQj8RThj4QDY2BdAkw55mXAJprXQRGjQqGvz0
-f8OFszJ8p90crFd+awEE12CNd7b22zgpH2XRKmH/Hf0
-> ssh-ed25519 H885DA GDiJYH+gaC++TSqfoPDOTFcsCZRhEl0EeTeab7tgcWU
-kMILmwNMnMS7rgC3kKsAksu4Txn5owPU2y09h4aHKY8
-> ssh-ed25519 Rq7K4Q VCNxGtCSCD2OYSWWwl0+yf189xV3QwRiwo80h4NPTkE
-hHkgYHLbISdl/RRdlInp9ub854M9ZKFSXpLgKW2YkmQ
-> ssh-ed25519 vvyRpw XSCCrqEOtvzQRssI0U1DHirKoPgbOUKJxNKnioHhT2Y
-HGey1j0Kxae5Qs0aw6eqFziQGiRmNA+lEwbRdf5hhbM
-> ssh-ed25519 aSEktQ mXY70Lgl76J4O5dPdDcIqmJ40EinigDuZrUghpGOq2I
-U2qeVFxGCYCEFWU+7vHc5Mu9EuzScowrjnwUyoqPj5U
-> ssh-ed25519 cD6JxA at89poimBZyeeM8CQrxDxN0yCNDT2k04++py1fFycj8
-cQV/K5zc5x/oYnJ4N0MX3sTboT4G4ZNvVUVdHuJRzbA
-> ssh-ed25519 ec6XRQ spJtb/xy4k4dmwKz8R2CPhC1WcuNV/rnDT978GkjHHk
-KrGEVGts/AhzbRNreqQ/CVanXL3l/9oMWxnpBLj23qU
-> ssh-ed25519 1qYEfw KRkTYlvvnsCIExKQNmCyU7YxnGZsiI03kzecXNpLzUQ
-h2YagV7BzlsF7banzwXbOudTdlFzT7LC8PvtxAsX36U
-> ssh-ed25519 2D+APA 4hdYlOnNIT9Q6tyKwXzy+u66Ti2EJopK43Sipebd0As
-tuesc9/QcEu4q9bTFJ5zJr0qvgLcmpn4at4cYtHrtbE
-> ssh-ed25519 eTSU6g i1qT6PtepHXnoLCqDbhk86QG+SR9luQaw34a34gy5mw
-YE9VBAT5SLW2ECHRU+dMg9na6OQNVRVGuhY8vOdmE/Q
-> ssh-ed25519 j2r2qQ TTTbSB/8UIDmmI3C9+u24PYZNfjl9jGADKHNWIwLfGE
-SNDforwii/GFp82TpyOcVIVrZWCe2QQKrjzPA6XA7Jc
-> ssh-ed25519 CyxfgQ P5EiJ54v65Sz1gHuI0s170Z7c1WjcZLlb7NYigElfVs
-iYJUGpoE9LBIlv+O1navSSsy3EJ8tusXXX+/QAQvjNI
-> ssh-ed25519 C/bBAQ hlBDpQRkcVCr3B6TCrBjxauhUX6Ndpm0s6x8W4hU6gM
-OFG3EuGJkSoEEXhbJ/Tp2DBdnBcs+hzxjNRdvcOSpQs
-> ssh-ed25519 +qVung cGEGpO8NJfpj9ixAH9lhYkPKPDdQWryVxSOhMGQdnWM
-+MycbIEab3P/AOS9i/YmPBDXB76hp3xUcWI4VMihV2w
-> ssh-rsa krWCLQ
-Zv3dPYERlX1MaVaJTBDwIcjt1yLmu4Z7MovPgjGg01p+XsdBXeepTyOl+gRBwGgo
-AW5CIuaChYxtSNJ6nOgSaUpqzILycUF1xE1jROe3MIX2MZ4KGD1qoqcHbiCAng+a
-RqYrwAKnNea9FQMVfhYZBkRoYE6ne1R+0G6BoFM/okz24pAAFPBx+sMMhfTkt0uV
-kHVx0dgRw1pxa7Na98WH/7E0zp9VuBvVHGXfk1rfW/UQlbIO5RP3nldFoa6OmOWS
-JZ022UvjyC1re0KCurka4y+qmaiRKnTBmpIXxJFMwNCAQ8O8SeAQ3DHKHmXNMOIL
-ZVICtRRk0uX36AVU8DWDog
-> ssh-ed25519 /vwQcQ kF8+hsA+0Msjd3q0SL52cae5RDqx4ls5kPKnc3UZyms
-Q33kIKJL3Vjxu7LQ5l4M3tlEuj+OW4uGh1x+JxthW8A
-> ssh-ed25519 0R97PA gWBH71l6w9upTE0DwqOMSvWXc5VyJiKFAQLaSpWQ43E
-IrOrvzEa0bABw6UOpP8pM8WhuRNMaWJ2khljJIKwOS8
-> ssh-ed25519 K3b7BA oS14iav9pSioLecMkOanJz89OJygLugvrnnTs5pKzz8
-akupMSiqXussXJyHwFm/f0imKALjowJVqd8/LFcC/58
--- bCJXTEDaKg4FF2dQp2zFMU4Zm0zACekCuDvthcAyncM
-&Ÿ€Waïãà›BD R(¯¥Ñ”ufj<>úVÁ8nÆ>‚ß›øëæðZúâ{Idƒ„©,³*„%Ç“È‚z«
--- a/secrets/floral/openbao-auth-token-bagel-box.age
+++ b/secrets/floral/openbao-auth-token-bagel-box.age
@ -1,20 +0,0 @@
-age-encryption.org/v1
-> ssh-ed25519 +HUDfA h3M3Sv+URNI4Bwyhtp4T/ongaF+PcccBItQySrm9LF4
-/N9DjQhFHXEWsi8+LMmE8fgAJqoc4kLIP9MNmQIAyPo
-> ssh-ed25519 +qVung lqAB+w68iIV5xb6XJW4XgSVlS6bKs+t3VIHV+pMqcTk
-QY+zLs7HlyhuVQLFjZUGUznRb4f18WMGt29yNNeoZAQ
-> ssh-rsa krWCLQ
-q90GbYnwXNyhuKo2b1fqr0t9KHVaX/5nLCTq9nm775/aE08ANAoqDZz1mx45BV3x
-o/YolJQK3KTUhfnBk84WpkbhbXQHsJ/3po5Tm5X4lLC1u65vE3NYUHAWzTvMHRl8
-761t/DArIOdKOF0a5aSJVc6Ctzkkrtr89JbHDQU7/1iGmVJEdyFMuqwCqCcxgEFc
-KSZ8X7A8u5kO+GBwzHjoW0PB79Y7zQSXl2+HeLBosg7hsgr0UvR4FCkQTNPh4BMb
-2Zd5vzXtpGCCHEwlHYLTwC4df0ZxRbGkIgOWMwLW+vXg0vLpp6JK1UAvtnEAmXHI
-pznGdso3v7jVZWoNVYj4VQ
-> ssh-ed25519 /vwQcQ vhf0oc9AHXuZFS9gycHJBPqDanBu6ccj8Sdj2twBRHo
-S2+UTHgbdqYT6mqPMOzxUOoRdQbcy24iFyzZlZS+cVc
-> ssh-ed25519 0R97PA Zdd0zeYNf54auL18VNZXFG7Pnt1dBTxy8Jd0BH0E0Cs
-XmAcUAlJw2oqt3npQ7TlUFHKkf02lhGbo4ZQF266GNg
-> ssh-ed25519 K3b7BA 3Nk8bfaks1xLHzD6fKVKApVON3PZHlphDnojrOESnl4
-Y6qrz0f0qdvkjj7wmmfC5NtYUWnmbW2Q2Eu/eGrXFHM
--- 3sa/7w1kNij8+Q0pzBWZVclsbJRM/PF03j36bbYFEZw
-Ïpª;ìÏhí¹Ãý™MÀœmŸ9d
j–@7¸U
Ÿ¸?#N9¯•ðB_™èõˆ‚l±Vµy<>*
--- a/secrets/floral/rabbitmq-password.age
+++ b/secrets/floral/rabbitmq-password.age
--- a/secrets/floral/stateless-uptime-kuma-password.age
+++ b/secrets/floral/stateless-uptime-kuma-password.age
@ -1,20 +0,0 @@
-age-encryption.org/v1
-> ssh-ed25519 CyxfgQ D2o8bUccO13DKF4COLBQ9mJbACsE2XsRa5S+N71WnTk
-ZaldT7HhQxbxf2ptIwdMYkC60eGtzihc7uwcAkq7s00
-> ssh-ed25519 K3b7BA AiUCG5CnNyv1DPu+iEwEgW9GqZ8zgpgxKJTAp350ADc
-cUVaDv7F1haQIF11/UhhDAR5DrfJlPttGfDjkv+z9vY
-> ssh-ed25519 +qVung 1JXeXyea+2Pcwoln/NLRiR8IPPIiB3gaFCP4imyv4DA
-JWmAY6ZnyU46KxzhRrQigGmUPba9lJDDyRQ2GjQShqc
-> ssh-rsa krWCLQ
-ciLu/+cXfQrB1ms8oTv+xi4eADyL4j0qwnY/6TE0wAXkQHuNXDmpF6ccWZoS2DqN
-NcnGXL6+WyWxmwlyBEq/rsBPvi1g0M6Md7Z4gXn2UvjJ+S7WyA8QEwkxoTDkJS7x
-k/NvtunmggVsWVK4Xdi5DKRw+f32qr/8GysDhIPrTt43iReBKNbyuYWmC5Ec85ep
-JU4JzCNZjJ07kixS5Y9BhaJbpEr47lCXE/KtJUvm3VAxS9IwfUn7KHHdFWynbExi
-F898j3zOR/kgYmeA0oTiexRD3Y2LCvjXIHQZ3MobbZ/PBrjWxe78Sw2vy2t5JLtB
-gFG0K8M1z8DT6a8TtvXEgg
-> ssh-ed25519 /vwQcQ kUM21TO9iSa8oVXMlNxR7Kc+8TV4C/uTzyQ+t3xnARA
-oXt+egWWONsKT48H4vZ2CPdy3Zfb2QeQVe9l7dDyO/w
-> ssh-ed25519 0R97PA e/piqf2RD5QgPaQs6jsJdzJgfZR9n1JDIWpbvLZErSs
-UTJH8POFdZ4+N9WkLoNESl1pvcVD0MS1qn7AdS/mg34
--- 9aYEP0eHDKMacIf09h+OJqIYw+N99+FrW/x/do8Lbo4
-$ ÖëWÛ\zú—¾=s/à@.Ç,?ƒW6n^ù#–i!§Ã–ï¶1]±Nvù±Ž'Ï¥¹6?‚'mµpPÒqýŸº
--- a/secrets/floral/gerrit-prometheus-bearer-token.age
+++ b/secrets/floral/gerrit-prometheus-bearer-token.age
--- a/secrets/floral/grafana-oauth-secret.age
+++ b/secrets/floral/grafana-oauth-secret.age
--- a/secrets/floral/hydra-postgres-key.age
+++ b/secrets/floral/hydra-postgres-key.age
--- a/secrets/floral/hydra-s3-credentials.age
+++ b/secrets/floral/hydra-s3-credentials.age
--- a/secrets/floral/hydra-signing-priv.age
+++ b/secrets/floral/hydra-signing-priv.age
--- a/secrets/floral/hydra-ssh-key-priv.age
+++ b/secrets/floral/hydra-ssh-key-priv.age
--- a/secrets/lix/buildbot-oauth-secret.age
+++ b/secrets/lix/buildbot-oauth-secret.age
@ -1,7 +0,0 @@
-age-encryption.org/v1
-> ssh-ed25519 Ao+7Wg q7oRHUUlAvD8OUbpPT7d6eLMPWU0YS/verYTDE5BCkY
-/87/1uqOvRYeqel9IjnFmGyF9SXUQD8MTgfcj91b/Fs
--- ulIeB91NJ7z/64h9BCLSD9/RW/zwv3m1Zo2ovNuInv8
-Îœç}³Óš#épÇ o>ä·*vµ÷ÄåŽs?[¦º´L
-<EFBFBD>þz™rý‰?R±Ñó7<Ê
-æi!€{X„¾òÓ
--- a/secrets/lix/buildbot-remote-builder-key.age
+++ b/secrets/lix/buildbot-remote-builder-key.age
--- a/secrets/lix/buildbot-service-key.age
+++ b/secrets/lix/buildbot-service-key.age
--- a/secrets/lix/buildbot-signing-key.age
+++ b/secrets/lix/buildbot-signing-key.age
@ -1,6 +0,0 @@
-age-encryption.org/v1
-> ssh-ed25519 Ao+7Wg EMpfs0EpWwaIKAoUBfEkyAHLIwi6JnGG6RvUWM5LjnU
-LKiwUBNc791U/GVRNlRPZE/TEMJjcFFrLruFJhiyiOI
--- 0khp8u+4vHgGyQqP05m473Eo09eyOUZLI5+EK4olzoc
-N3(
-ª•ûxRq°<71>f<EFBFBD>Ó;Í¼3¬~RˆÓC^ñ+fœš1”®˜xˆ÷ÅëñSØ—hâ
£ÖË°GˆÓn–YIûµ:7¾!°u×Hþy/‰Øð‰™.¯¤á^¹lC™ôUÈËþ5cž:]ÿNž&'MÎè¶É-˜–ÆHF¦D0‘ cjô ‹Ð~
--- a/secrets/lix/buildbot-worker-password.age
+++ b/secrets/lix/buildbot-worker-password.age
--- a/secrets/lix/buildbot-workers.age
+++ b/secrets/lix/buildbot-workers.age
@ -1,6 +0,0 @@
-age-encryption.org/v1
-> ssh-ed25519 Ao+7Wg RPKKoI5l5cYVdSvOxTHCUtwceac4vSi3+vlaqHr8kQg
-qbgTHCeQDNM30IJNZ/BU6wgryJwB316H5GWWaYB/wng
--- GuFi3GSRdlBJ5YRjfAVyFDZ+4TH575kFJLnFp5847N0
-èƒÞHÖÜ*x´M7¼t<,4ˆŠÑ^<5E>5@v°<>£€º,Z•MÒg=M
-» 3výJÄ«ÐÖê¿Nz8'<^'4&WÂf"Êõ´À›ë\©º»ëêwmzúlAl|+„‘ÆKš~68ñEÝîk•8ø?S&òaM‹Ý~ž¹ê¿]Vfø ÝJxaõDù¥x
--- a/secrets/floral/loki-environment.age
+++ b/secrets/floral/loki-environment.age
--- a/secrets/floral/metrics-push-htpasswd.age
+++ b/secrets/floral/metrics-push-htpasswd.age
--- a/secrets/metrics-push-password.age
+++ b/secrets/metrics-push-password.age
--- a/secrets/floral/mimir-environment.age
+++ b/secrets/floral/mimir-environment.age
--- a/secrets/floral/mimir-webhook-url.age
+++ b/secrets/floral/mimir-webhook-url.age
--- a/secrets/floral/netbox-environment.age
+++ b/secrets/floral/netbox-environment.age
--- a/secrets/floral/newsletter-secrets.age
+++ b/secrets/floral/newsletter-secrets.age
--- a/secrets/floral/ows-deploy-key.age
+++ b/secrets/floral/ows-deploy-key.age
--- a/secrets/floral/postgres-ca-priv.age
+++ b/secrets/floral/postgres-ca-priv.age
--- a/secrets/floral/postgres-tls-priv.age
+++ b/secrets/floral/postgres-tls-priv.age
--- a/secrets/floral/pyroscope-secrets.age
+++ b/secrets/floral/pyroscope-secrets.age
--- a/secrets/floral/s3-channel-keys.age
+++ b/secrets/floral/s3-channel-keys.age
--- a/secrets/floral/s3-channel-staging-keys.age
+++ b/secrets/floral/s3-channel-staging-keys.age
--- a/secrets/floral/s3-revproxy-api-keys.age
+++ b/secrets/floral/s3-revproxy-api-keys.age
--- a/secrets/floral/tempo-environment.age
+++ b/secrets/floral/tempo-environment.age
--- a/services/baremetal/builders/assignments.nix
+++ b/services/baremetal/builders/assignments.nix
@ -3,18 +3,27 @@
 let
  genBuilders = { offset ? 0, count, f }: builtins.genList (x: rec { name = "builder-${toString (offset + x)}"; value = f name; }) count;
 in builtins.listToAttrs (
-  genBuilders { offset = 0; count = 2; f = name: {
+  # The first 8 builders are general purpose hydra builders
+  genBuilders { count = 8; f = name: {
    cores = 8;
    max-jobs = 8;
    supported-features = [ "kvm" "nixos-test" ];
    required-features = [ ];
  }; }
  ++
-  # This builder is exclusively for big-parallel
-  genBuilders { offset = 2; count = 1; f = name: {
+  # The last 2 builders are exclusively for big-parallel
+  genBuilders { offset = 8; count = 2; f = name: {
    cores = 20;
    max-jobs = 1;
    supported-features = [ "kvm" "nixos-test" "big-parallel" ];
    required-features = [ "big-parallel" ];
  }; }
+  ++
+  # These are not currently used for hydra
+  genBuilders { offset = 10; count = 1; f = name: {
+    cores = 8;
+    max-jobs = 8;
+    supported-features = [ "kvm" "nixos-test" "big-parallel" ];
+    required-features = [ ];
+  }; }
 )
--- a/services/baremetal/builders/default.nix
+++ b/services/baremetal/builders/default.nix
@ -1,12 +1,15 @@
 { pkgs, lib, config, ... }:
 let
-  cfgParent = config.bagel.baremetal;
  cfg = config.bagel.baremetal.builders;
 in
 {
+  imports = [ ./netboot.nix ];
+
  options = {
+
    bagel.baremetal.builders = {
-      enable = lib.mkEnableOption "builder role";
+      enable = lib.mkEnableOption "baremetal bagel oven";
+      netboot = lib.mkEnableOption "netboot";
      num = lib.mkOption {
        type = lib.types.int;
      };
@ -14,6 +17,9 @@ in
  };

  config = lib.mkIf cfg.enable {
+    boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" ];
+    boot.initrd.kernelModules = [ "dm-snapshot" ];
+
    users.users.builder = {
      isSystemUser = true;
      group = "nogroup";
@ -42,21 +48,45 @@ in
      inherit ((import ./assignments.nix).${config.networking.hostName}) max-jobs cores;
    };

-    fileSystems = {
-      "/mnt" = {
-        device = "/dev/disk/by-label/hydra";
-        fsType = "xfs";
-        options = ["logbsize=256k"];
-      };
+    nixpkgs.hostPlatform = "x86_64-linux";
+    hardware.cpu.intel.updateMicrocode = true;

-      # We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
-      "/tmp" = {
-        device = "/mnt/tmp";
-        options = [ "bind" ];
-      };
-    };
+    boot.loader.systemd-boot.enable = true;
+    boot.loader.efi.canTouchEfiVariables = true;
+    boot.initrd.systemd.enable = true;

-    swapDevices = lib.optionals (!cfgParent.netboot) [
+    boot.initrd.services.lvm.enable = true;
+
+    boot.kernel.sysctl."fs.xfs.xfssyncd_centisecs" = "12000";
+    fileSystems = lib.mkMerge [
+      (lib.mkIf (!cfg.netboot) {
+        "/" = {
+          device = "/dev/disk/by-label/root";
+          fsType = "xfs";
+        };
+
+        "/boot" = {
+          device = "/dev/disk/by-label/BOOT";
+          fsType = "vfat";
+          options = [ "fmask=0022" "dmask=0022" ];
+        };
+      })
+      {
+        "/mnt" = {
+          device = "/dev/disk/by-label/hydra";
+          fsType = "xfs";
+          options = ["logbsize=256k"];
+        };
+
+        # We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
+        "/tmp" = {
+          device = "/mnt/tmp";
+          options = [ "bind" ];
+        };
+      }
+    ];
+
+    swapDevices = lib.optionals (!cfg.netboot) [
      {
        device = "/swapfile";
        size = 50 * 1024;  # 50GiB
@ -73,7 +103,39 @@ in
      "console=ttyS0,115200"
    ];

+    networking.useNetworkd = true;
    networking.hostName = "builder-${toString cfg.num}";
+    networking.domain = "wob01.infra.forkos.org";
+
+    systemd.network = {
+      netdevs = {
+        "40-uplink" = {
+          netdevConfig = {
+            Kind = "bond";
+            Name = "uplink";
+          };
+          bondConfig = {
+            Mode = "802.3ad";
+            TransmitHashPolicy = "layer3+4";
+          };
+        };
+      };
+      networks = {
+        "40-eno1" = {
+          name = "eno1";
+          bond = [ "uplink" ];
+        };
+        "40-eno2" = {
+          name = "eno2";
+          bond = [ "uplink" ];
+        };
+      };
+    };
+    networking.interfaces.uplink.ipv6.addresses = [
+      { address = "2a01:584:11::1:${toString cfg.num}"; prefixLength = 64; }
+    ];
+    networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
+    deployment.targetHost = lib.mkIf (!cfg.netboot) "2a01:584:11::1:${toString cfg.num}";
    deployment.tags = [ "builders" ];

    # Why can't we have nice things? https://bugs.openjdk.org/browse/JDK-8170568
@ -117,5 +179,11 @@ in
      wantedBy = [ "timers.target" ];
    };
    systemd.timers.hydra-gc.timerConfig.Persistent = true;
+
+    bagel.sysadmin.enable = true;
+
+    environment.systemPackages = [ pkgs.ipmitool ];
+
+    system.stateVersion = "24.05";
  };
 }
--- a/services/baremetal-builder/netboot.nix
+++ b/services/baremetal-builder/netboot.nix
@ -1,6 +1,6 @@
 { modulesPath, pkgs, lib, config, extendModules, ... }@node:
 let
-  cfg = config.bagel.baremetal;
+  cfg = config.bagel.baremetal.builders;
 in
 {
  config = lib.mkIf (cfg.enable && cfg.netboot) {
@ -21,13 +21,22 @@ in
      '';
    };

+    # machines with the netboot module enabled should only be updated by appliying wob-vpn-gw and rebooting
+    deployment.targetHost = "invalid.example.com";
+    # fixes initrd eval assertion error, and allows `colmena build` to succeed
+    fileSystems."/" = {
+      device = "none";
+      fsType = "tmpfs";
+      options = [ "defaults" "size=64G" "mode=755" ];
+    };
+
    system.build = {

      # Build a kernel and initramfs which will download the IPXE script from hydra using
      # u-root pxeboot tool and kexec into the final netbooted system.
      notipxe = import (modulesPath + "/..") {
        system = "x86_64-linux";
-        configuration = 
+        configuration =
          { pkgs, config, ... }:

          {
@ -57,7 +66,7 @@ in
              script = ''
                ln -sf /dev/console /dev/tty
                until ${pkgs.iputils}/bin/ping -c 1 hydra.forkos.org; do sleep 1; done
-                ${pkgs.u-root}/bin/pxeboot -v -ipv4=false -file https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe 
+                ${pkgs.u-root}/bin/pxeboot -v -ipv4=false -file https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe
              '';
            };
            boot.initrd.systemd.contents."/etc/ssl/certs/ca-certificates.crt".source = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
--- a/services/baremetal/default.nix
+++ b/services/baremetal/default.nix
@ -1,12 +0,0 @@
-{
-  imports = [
-    # Compute nodes
-    ./builders
-    # Storage nodes
-    ./storage
-
-    # Bases
-    ./netboot.nix
-    ./hardware.nix
-  ];
-}
--- a/services/baremetal/hardware.nix
+++ b/services/baremetal/hardware.nix
@ -1,96 +0,0 @@
-{ pkgs, lib, config, ... }:
-let
-  cfg = config.bagel.baremetal;
-in
-{
-  options = {
-    bagel.baremetal = {
-      enable = lib.mkEnableOption "baremetal bagel oven";
-      netboot = lib.mkEnableOption "netboot";
-      num = lib.mkOption {
-        type = lib.types.int;
-      };
-    };
-  };
-
-  config = lib.mkIf cfg.enable {
-    boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" ];
-    boot.initrd.kernelModules = [ "dm-snapshot" ];
-
-    nixpkgs.hostPlatform = "x86_64-linux";
-    hardware.cpu.intel.updateMicrocode = true;
-
-    boot.loader.systemd-boot.enable = true;
-    boot.loader.efi.canTouchEfiVariables = true;
-    boot.initrd.systemd.enable = true;
-    boot.initrd.services.lvm.enable = true;
-
-    boot.kernel.sysctl."fs.xfs.xfssyncd_centisecs" = "12000";
-    fileSystems = lib.mkIf (!cfg.netboot) {
-      "/" = {
-        device = "/dev/disk/by-label/root";
-        fsType = "xfs";
-      };
-
-      "/boot" = {
-        device = "/dev/disk/by-label/BOOT";
-        fsType = "vfat";
-        options = [ "fmask=0022" "dmask=0022" ];
-      };
-    };
-
-    zramSwap = {
-      enable = true;
-      memoryPercent = 25;
-    };
-
-    boot.kernelParams = [
-      "console=tty1"
-      "console=ttyS0,115200"
-    ];
-
-    networking.useNetworkd = true;
-    networking.domain = "wob01.infra.forkos.org";
-
-    systemd.network = {
-      netdevs = {
-        "40-uplink" = {
-          netdevConfig = {
-            Kind = "bond";
-            Name = "uplink";
-          };
-          bondConfig = {
-            Mode = "802.3ad";
-            TransmitHashPolicy = "layer3+4";
-          };
-        };
-      };
-      networks = {
-        "40-eno1" = {
-          name = "eno1";
-          bond = [ "uplink" ];
-        };
-        "40-eno2" = {
-          name = "eno2";
-          bond = [ "uplink" ];
-        };
-      };
-    };
-    networking.interfaces.uplink.ipv6.addresses = [
-      { address = "2a01:584:11::1:${toString cfg.num}"; prefixLength = 64; }
-    ];
-    networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
-    bagel.infra.self.wan = {
-      family = "inet6";
-      address = "2a01:584:11::1:${toString cfg.num}";
-      prefixLength = 64;
-    };
-    deployment.targetHost = "2a01:584:11::1:${toString cfg.num}";
-
-    bagel.sysadmin.enable = true;
-
-    environment.systemPackages = [ pkgs.ipmitool ];
-
-    system.stateVersion = "24.05";
-  };
-}
--- a/services/baremetal/storage/default.nix
+++ b/services/baremetal/storage/default.nix
@ -1,19 +0,0 @@
-{ lib, config, ... }:
-let
-  cfg = config.bagel.baremetal.storage;
-in
-{
-  options = {
-    bagel.baremetal.storage = {
-      enable = lib.mkEnableOption "storage role";
-      num = lib.mkOption {
-        type = lib.types.int;
-      };
-    };
-  };
-
-  config = lib.mkIf cfg.enable {
-    networking.hostName = "storage-${toString cfg.num}";
-    deployment.tags = [ "storage" ];
-  };
-}
--- a/services/block-crawlers/blocked-ua.txt
+++ b/services/block-crawlers/blocked-ua.txt
@ -1,40 +0,0 @@
-AI2Bot
-Ai2Bot-Dolma
-Amazonbot
-anthropic-ai
-Applebot
-Applebot-Extended
-Bytespider
-CCBot
-ChatGPT-User
-Claude-Web
-ClaudeBot
-cohere-ai
-Diffbot
-FacebookBot
-facebookexternalhit
-FriendlyCrawler
-Google-Extended
-GoogleOther
-GoogleOther-Image
-GoogleOther-Video
-GPTBot
-iaskspider/2.0
-ICC-Crawler
-ImagesiftBot
-img2dataset
-ISSCyberRiskCrawler
-Kangaroo Bot
-Meta-ExternalAgent
-Meta-ExternalFetcher
-OAI-SearchBot
-omgili
-omgilibot
-PerplexityBot
-PetalBot
-Scrapy
-Sidetrade indexer bot
-Timpibot
-VelenPublicWebCrawler
-Webzio-Extended
-YouBot
--- a/services/block-crawlers/default.nix
+++ b/services/block-crawlers/default.nix
@ -1,32 +0,0 @@
-{ pkgs, config, lib, ... }:
-let
-  inherit (lib) mkEnableOption mkIf mkOption types concatStringsSep mkDefault splitString;
-  cfg = config.bagel.services.nginx.crawler-blocker;
-  mkRobotsFile = blockedUAs: pkgs.writeText "robots.txt" ''
-    ${concatStringsSep "\n" (map (ua: "User-agent: ${ua}") blockedUAs)}
-    Disallow: /
-  '';
-in
-{
-  options = {
-    bagel.services.nginx.crawler-blocker = {
-      enable = mkEnableOption "the crawler blocker";
-
-      userAgents = mkOption {
-        type = types.listOf types.str;
-        default = splitString "\n" (builtins.readFile ./blocked-ua.txt);
-      };
-    };
-
-    services.nginx.virtualHosts = mkOption {
-      type = types.attrsOf (types.submodule {
-        config = {
-          locations."= /robots.txt" = mkIf cfg.enable (mkDefault {
-            alias = mkRobotsFile cfg.userAgents;
-          });
-        };
-      });
-    };
-  };
-}
-
--- a/services/buildbot/default.nix
+++ b/services/buildbot/default.nix
@ -7,69 +7,15 @@
 }:
 let
  cfg = config.bagel.services.buildbot;
+  cfgGerrit = nodes.gerrit01.config.bagel.services.gerrit;
  ssh-keys = import ../../common/ssh-keys.nix;
-  freeGbDiskSpace = 20;
-  extraTenantSpecificBuilders = {
-    lix = import ./lix.nix {
-      inherit config nodes;
-    };
-    floral = [ ];
-  }.${cfg.tenant or (throw "${cfg.tenant} is not a known tenant")};
-  clientId = {
-    lix = "buildbot";
-    floral = "forkos-buildbot";
-  }.${cfg.tenant or (throw "${cfg.tenant} is not a known tenant")};
  inherit (lib) mkEnableOption mkOption mkIf types;
 in
 {
  options.bagel.services.buildbot = {
    enable = mkEnableOption "Buildbot";
-
-    tenant = mkOption {
-      type = types.enum [ "lix" "floral" ];
-      description = "Which buildbot tenant to enable";
-    };
-
    domain = mkOption {
      type = types.str;
-      description = "Domain name for this Buildbot";
-    };
-
-    gerrit = {
-      domain = mkOption {
-        type = types.str;
-        description = "Canonical domain of the Gerrit associated to this Buildbot";
-        example = [ "cl.forkos.org" ];
-      };
-
-      port = mkOption {
-        type = types.port;
-        description = "Gerrit SSH port for this Buildbot";
-      };
-
-      username = mkOption {
-        type = types.str;
-        description = "Gerrit service username for this Buildbot";
-      };
-    };
-
-    cors.allowedOrigins = mkOption {
-      type = types.listOf types.str;
-      example = [ "*.forkos.org" ];
-      description = "Allowed origin for Buildbot and NGINX for CORS without the protocol";
-    };
-
-    buildSystems = mkOption {
-      type = types.listOf (types.enum [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]);
-      default = [ "x86_64-linux" ];
-      example = [ "x86_64-linux" "aarch64-linux" ];
-      description = "Supported build systems for this buildbot instance.";
-    };
-
-    projects = mkOption {
-      type = types.listOf types.str;
-      example = [ "nixpkgs" ];
-      description = "Static list of projects enabled for Buildbot CI";
    };

    builders = mkOption {
@ -81,39 +27,28 @@ in

  config = mkIf cfg.enable {
    networking.firewall.allowedTCPPorts = [ 80 443 ];
-    bagel.secrets.files = [
-      "buildbot-worker-password"
-      "buildbot-oauth-secret"
-      "buildbot-workers"
-      "buildbot-service-key"
-      "buildbot-signing-key"
-      "buildbot-remote-builder-key"
-    ];
+    age.secrets.buildbot-worker-password.file = ../../secrets/buildbot-worker-password.age;
+    age.secrets.buildbot-oauth-secret.file = ../../secrets/buildbot-oauth-secret.age;
+    age.secrets.buildbot-workers.file = ../../secrets/buildbot-workers.age;
+    age.secrets.buildbot-service-key.file = ../../secrets/buildbot-service-key.age;
    age.secrets.buildbot-signing-key = {
+      file = ../../secrets/buildbot-signing-key.age;
      owner = "buildbot-worker";
      group = "buildbot-worker";
    };
    age.secrets.buildbot-remote-builder-key = {
-      file = ../../secrets/${cfg.tenant}/buildbot-remote-builder-key.age;
+      file = ../../secrets/buildbot-remote-builder-key.age;
      owner = "buildbot-worker";
      group = "buildbot-worker";
    };

-    services.nginx = {
-      recommendedProxySettings = true;
-      appendHttpConfig = ''
-        # Our session stuff is too big with the TWISTED_COOKIE in addition.
-        # Default is usually 4k or 8k.
-        large_client_header_buffers 4 16k;
+    services.nginx.virtualHosts.${cfg.domain} = {
+      forceSSL = true;
+      enableACME = true;
+      extraConfig = ''
+        add_header Access-Control-Allow-Credentials 'true' always;
+        add_header Access-Control-Allow-Origin 'https://cl.forkos.org' always;
      '';
-      virtualHosts.${cfg.domain} = {
-        forceSSL = true;
-        enableACME = true;
-        extraConfig = ''
-          # This is needed so that logged-in users in Buildbot can include their credentials in their requests.
-          add_header Access-Control-Allow-Credentials 'true' always;
-        '';
-      };
    };

    services.buildbot-nix.worker = {
@ -139,25 +74,30 @@ in
      enable = true;

      inherit (cfg) domain;
-      # TODO(raito): is that really necessary when we can just collect buildMachines' systems?
-      inherit (cfg) buildSystems;
+
+      debugging.enable = true;

      oauth2 = {
        name = "Lix";
-        inherit clientId;
+        clientId = "forkos-buildbot";
        clientSecretFile = config.age.secrets.buildbot-oauth-secret.path;
        resourceEndpoint = "https://identity.lix.systems";
        authUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth";
        tokenUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
-        userinfoUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/userinfo";
      };

      # TODO(raito): this is not really necessary, we never have remote buildbot workers.
      # we can replace all of this with automatic localworker generation on buildbot-nix side.
      workersFile = config.age.secrets.buildbot-workers.path;

-      # We rely on NGINX to do the CORS dance.
-      allowedOrigins = cfg.cors.allowedOrigins;
+      allowedOrigins = [
+        "*.forkos.org"
+      ];
+
+      # TODO(raito): is that really necessary when we can just collect buildMachines' systems?
+      buildSystems = [
+        "x86_64-linux"
+      ];

      buildMachines = map (n: {
          hostName = nodes.${n}.config.networking.fqdn;
@ -171,14 +111,20 @@ in
          # Contrary to how Nix works, here we can specify non-base64 public host keys.
          publicHostKey = ssh-keys.machines.${n};
        }
-      ) cfg.builders ++ extraTenantSpecificBuilders;
+      ) cfg.builders;

      gerrit = {
+        domain = cfgGerrit.canonicalDomain;
        # Manually managed account…
        # TODO: https://git.lix.systems/the-distro/infra/issues/69
-        inherit (cfg.gerrit) domain port username;
+        username = "buildbot";
+        port = cfgGerrit.port;
        privateKeyFile = config.age.secrets.buildbot-service-key.path;
-        inherit (cfg) projects;
+        projects = [
+          "buildbot-test"
+          "nixpkgs"
+          "infra"
+        ];
      };

      evalWorkerCount = 6;
@ -187,47 +133,10 @@ in
      signingKeyFile = config.age.secrets.buildbot-signing-key.path;
    };

-    # Make PostgreSQL restart smoother.
-    systemd.services.postgresql.serviceConfig = {
-      Restart = "always";
-      RestartMaxDelaySec = "5m";
-      RestartSteps = 10;
-    };
-
-    services.postgresql.settings = {
-      # DB Version: 15
-      # OS Type: linux
-      # DB Type: web
-      # Total Memory (RAM): 64 GB
-      # CPUs num: 16
-      # Connections num: 100
-      # Data Storage: ssd
-      max_connections = 100;
-      shared_buffers = "16GB";
-      effective_cache_size = "48GB";
-      maintenance_work_mem = "2GB";
-      checkpoint_completion_target = 0.9;
-      wal_buffers = "16MB";
-      default_statistics_target = 100;
-      random_page_cost = 1.1;
-      effective_io_concurrency = 200;
-      work_mem = "41943kB";
-      huge_pages = "try";
-      min_wal_size = "1GB";
-      max_wal_size = "4GB";
-      max_worker_processes = 16;
-      max_parallel_workers_per_gather = 4;
-      max_parallel_workers = 16;
-      max_parallel_maintenance_workers = 4;
-    };
-
    nix.settings.keep-derivations = true;
    nix.gc = {
      automatic = true;
      dates = "hourly";
-      options = ''
-        --max-freed "$((${toString freeGbDiskSpace} * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"
-      '';
    };
  };
 }
--- a/services/buildbot/lix.nix
+++ b/services/buildbot/lix.nix
@ -1,50 +0,0 @@
-{ config, nodes, ... }:
-let
-  ssh-keys = import ../../common/ssh-keys.nix;
-in
-  [
-    {
-      hostName = "build01.aarch64.lix.systems";
-      maxJobs = 2;
-      protocol = "ssh-ng";
-      sshKey = config.age.secrets.buildbot-remote-builder-key.path;
-      sshUser = "nix";
-      systems = [ "aarch64-linux" ];
-      publicHostKey = ssh-keys.machines.build01-aarch64-lix;
-      supportedFeatures = nodes.build01-aarch64-lix.config.nix.settings.system-features;
-    }
-    {
-      hostName = "build02.aarch64.lix.systems";
-      maxJobs = 4;
-      protocol = "ssh-ng";
-      sshKey = config.age.secrets.buildbot-remote-builder-key.path;
-      sshUser = "nix";
-      systems = [ "aarch64-linux" ];
-      publicHostKey = ssh-keys.machines.build02-aarch64-lix;
-      # TODO: use build02 features.
-      supportedFeatures = nodes.build01-aarch64-lix.config.nix.settings.system-features;
-    }
-    {
-      hostName = "build01.aarch64-darwin.lix.systems";
-      maxJobs = 2;
-      protocol = "ssh-ng";
-      sshKey = config.age.secrets.buildbot-remote-builder-key.path;
-      sshUser = "m1";
-      systems = [ "aarch64-darwin" "x86_64-darwin" ];
-      publicHostKey = ssh-keys.machines.build01-aarch64-darwin-lix;
-      supportedFeatures = [ "big-parallel" ];
-    }
-    # a.k.a. https://git.newtype.fr/newtype/newtype-org-configurations/src/branch/main/docs/epyc.md
-    {
-      hostName = "epyc.infra.newtype.fr";
-      # at 256G this could run 64 builds but the machine is shared
-      # (and historically we used no more than 16 concurrent jobs)
-      maxJobs = 16;
-      protocol = "ssh-ng";
-      sshKey = config.age.secrets.buildbot-remote-builder-key.path;
-      sshUser = "nix";
-      systems = [ "x86_64-linux" "i686-linux" ];
-      publicHostKey = ssh-keys.machines.epyc-newtype-fr;
-      supportedFeatures = [ "benchmark" "big-parallel" "nixos-test" "kvm" ];
-    }
-  ]
--- a/services/channel-scripts/default.nix
+++ b/services/channel-scripts/default.nix
@ -152,9 +152,7 @@ in
      hydra_uri = cfg.hydraUrl;
      binary_cache_uri = cfg.binaryCacheUrl;
      base_git_uri_for_revision = cfg.baseUriForGitRevisions;
-      # TODO: this leaks information about where channel-scripts are hosted.
-      # Cleanup this later with a proper module option.
-      repo_dir = "/gerrit-data/channel-scripts/nixpkgs";
+      nixpkgs_dir = "/var/lib/channel-scripts/nixpkgs";
      s3_release_bucket_name = cfg.s3.release;
      s3_channel_bucket_name = cfg.s3.channel;
    };
@ -179,24 +177,6 @@ in
        };
        script = "true";
      };
-      "cleanup-failed-streaming-prefixes" = {
-        description = "Cleanup all failed streaming prefixes on the channel bucket (channel-scripts)";
-        conflicts = map (service: "${service.name}.service") updateJobs;
-        after = [ "networking.target" ];
-        serviceConfig = {
-          Type = "oneshot";
-          RemainAfterExit = false;
-          User = "channel-scripts";
-          DynamicUser = true;
-          StateDirectory = "channel-scripts";
-          EnvironmentFile = [
-            cfg.releaseBucketCredentialsFile
-          ];
-          Environment = cfg.extraEnvironment;
-          LoadCredential = [ "password:${config.age.secrets.alloy-push-password.path}" ];
-          ExecStart = "${cfg.package}/bin/mirror-forkos -c ${configFile} ${concatStringsSep " " cfg.extraArgs} cleanup-streamed-prefixes";
-        };
-      };
    };

    systemd.timers."update-all-channels" = {
@ -208,14 +188,5 @@ in
        AccuracySec = 300;
      };
    };
-
-    systemd.timers."cleanup-failed-streaming-prefixes" = {
-      description = "Cleanup failed streaming prefixes for channel-scripts";
-      wantedBy = [ "timers.target" ];
-      timerConfig = {
-        OnCalendar = "daily";
-        RandomizedDelaySec = "1h";
-      };
-    };
  };
 }
--- a/services/default.nix
+++ b/services/default.nix
@ -1,23 +1,17 @@
 {
  imports = [
-    ./block-crawlers
    ./gerrit
    ./channel-scripts
-    ./secrets-agent
    ./hydra
    ./matrix
    ./monitoring
-    ./uptime-kuma
-    ./self
    ./netbox
    ./ofborg
    ./postgres
    ./forgejo
-    ./baremetal
+    ./baremetal-builder
    ./buildbot
    ./newsletter
    ./s3-revproxy
-    ./vault
-    ./extra-builders
  ];
 }
--- a/Show more
+++ b/Show more