Set up alertmanager-hookshot-adapter

It doesn't want to work.

common/admins.nix

@@ -3,12 +3,14 @@ let
 in {
   users.users.root.openssh.authorizedKeys.keys = 
     keys.users.delroth ++
-    keys.users.k900 ++ 
+    keys.users.emilylange ++
-    keys.users.raito ++
+    keys.users.hexchen ++
-    keys.users.maxine ++ 
     keys.users.jade ++
     keys.users.janik ++
+    keys.users.k900 ++
     keys.users.lukegb ++
-    keys.users.emilylange ++
+    keys.users.maxine ++
+    keys.users.raito ++
+    keys.users.thubrecht ++
     keys.users.yuka;
 }

common/nix.nix

@@ -14,7 +14,7 @@
 
   # Use our cache and trust its signing key. Still use cache.nixos.org as
   # fallback.
-  nix.settings.substituters = [ "https://bagel-cache.s3-web.delroth.net/" ];
+  nix.settings.substituters = [ "https://cache.forkos.org/" ];
   nix.settings.trusted-public-keys = [
     "cache.forkos.org:xfXIUJO1yiEITJmYsVmNDa9BFSlgTh/YqZ+4ei1EhQg="
   ];

common/ssh-keys.nix

@@ -23,14 +23,12 @@
 
   users = {
     delroth = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" ];
-    raito = [
+    emilylange = [ "no-touch-required sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIL7jgq3i+N3gVJhs4shm7Kmw6dIocs2OuR0GBMG1RxfKAAAABHNzaDo=" ];
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp"
+    hexchen = [
-      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINJ0tCxsEilAzV6LaNpUpcjzyEn4ptw8kFz3R+Z3YjEF hexchen@backup"
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDI3T1eFS77URHZ/HVWkMOqx7W1U54zJtn9C7QWsHOtyH72i/4EVj8SxYqLllElh1kuKUXSUipPeEzVsipFVvfH0wEuTDgFffiSQ3a8lfUgdEBuoySwceEoPgc5deapkOmiDIDeeWlrRe3nqspLRrSWU1DirMxoFPbwqJXRvpl6qJPxRg+2IolDcXlZ6yxB4Vv48vzRfVzZNUz7Pjmy2ebU8PbDoFWL/S3m7yOzQpv3L7KYBz7+rkjuF3AU2vy6CAfIySkVpspZZLtkTGCIJF228ev0e8NvhuN6ZnjzXxVTQOy32HCdPdbBbicu0uHfZ5O7JX9DjGd8kk1r2dnZwwy/ hexchen@yubi5"
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC4CLJ+mFfq5XiBXROKewmN9WYmj+79bj/AoaR6Iud2pirulot3tkrrLe2cMjiNWFX8CGVqrsAELKUA8EyUTJfStlcTE0/QNESTRmdDaC+lZL41pWUO9KOiD6/0axAhHXrSJ0ScvbqtD0CtpnCKKxtuOflVPoUGZsH9cLKJNRKfEka0H0GgeKb5Tp618R/WNAQOwaCcXzg/nG4Bgv3gJW4Nm9IKy/MwRZqtILi8Mtd+2diTqpMwyNRmbenmRHCQ1vRw46joYkledVqrmSlfSMFgIHI1zRSBXb/JkG2IvIyB5TGbTkC4N2fqJNpH8wnCKuOvs46xmgdiRA26P48C2em3 hexchen@yubi5c"
     ];
-    k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
-    maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
     jade = [
       "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNldAg4t13/i69TD786The+U3wbiNUdW2Kc9KNWvEhgpf4y4x4Sft0oYfkPw5cjX4H3APqfD+b7ItAG0GCbwHw6KMYPoVMNK08zBMJUqt1XExbqGeFLqBaeqDsmEAYXJRbjMTAorpOCtgQdoCKK/DvZ51zUWXxT8UBNHSl19Ryv5Ry5VVdbAE35rqs57DQ9+ma6htXnsBEmmnC+1Zv1FE956m/OpBTId50mor7nS2FguAtPZnDPpTd5zl9kZmJEuWCrmy6iinw5V4Uy1mLeZkQv+/FtozbyifCRCvps9nHpv4mBSU5ABLgnRRvXs+D41Jx7xloNADr1nNgpsNrYaTh hed-bot-ssh-tpm-rsa"
       "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIKYljH8iPMrH00lOb3ETxRrZimdKzPPEdsJQ5D5ovtOwAAAACnNzaDpzc2hrZXk= ssh:sshkey"
@@ -41,8 +39,16 @@
       "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIOYg513QZsVzoyVycXZjg4F3T3+OwtcY3WAhrlfyLgLTAAAABHNzaDo="
       "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBLZxVITpJ8xbiCa/u2gjSSIupeiqOnRh+8tFIoVhCON"
     ];
+    k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
     lukegb = [ ''cert-authority,principals="lukegb" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEqNOwlR7Qa8cbGpDfSCOweDPbAGQOZIcoRgh6s/J8DR'' ];
-    emilylange = [ "no-touch-required sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIL7jgq3i+N3gVJhs4shm7Kmw6dIocs2OuR0GBMG1RxfKAAAABHNzaDo=" ];
+    maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
+    raito = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
+    ];
+    thubrecht = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPM1jpXR7BWQa7Sed7ii3SbvIPRRlKb3G91qC0vOwfJn" ];
     yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKath4/fDnlv/4fzxkPrQN1ttmoPRNu/m9bEtdPJBDfY cardno:16_933_242" ];
   };
 }

flake.lock

@@ -87,16 +87,16 @@
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1721409873,
+        "lastModified": 1722939563,
-        "narHash": "sha256-h0njWQRvtkjK0NJ/Kgj76sXBhWwq5HGJm7OMcigmNw4=",
+        "narHash": "sha256-lMe8aXgF550iQLRaoU+yn8yYQ4x2qiyqANgsFyjfWwA=",
-        "ref": "refs/heads/refactor",
+        "ref": "refs/heads/non-flakes",
-        "rev": "54bba654d4279dfd112345b6470547851feb1457",
+        "rev": "4a162a8aa5dad6cecdb33bd8534e67e0bdaeb13f",
-        "revCount": 267,
+        "revCount": 295,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
       },
       "original": {
-        "ref": "refs/heads/refactor",
+        "ref": "refs/heads/non-flakes",
         "type": "git",
         "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
       }
@@ -443,17 +443,17 @@
         ]
       },
       "locked": {
-        "lastModified": 1721682989,
+        "lastModified": 1722688238,
-        "narHash": "sha256-kjJiZ7m4HKqbZ2mxNQiB32/goKFb8BRi8OqC4wIU0OI=",
+        "narHash": "sha256-x6BnYtArF6IDs7bS8ExokgAQBOlrxXxD0EOBIlASmfM=",
         "ref": "refs/heads/main",
-        "rev": "4b107e6ff36bd89958fba36e0fe0340903e7cd13",
+        "rev": "9b5ac87de73ea4646dbb2af979db91f096d29960",
-        "revCount": 4190,
+        "revCount": 4191,
         "type": "git",
-        "url": "https://git.lix.systems/lix-project/hydra.git"
+        "url": "https://git.lix.systems/the-distro/hydra.git"
       },
       "original": {
         "type": "git",
-        "url": "https://git.lix.systems/lix-project/hydra.git"
+        "url": "https://git.lix.systems/the-distro/hydra.git"
       }
     },
     "lix": {

flake.nix

@@ -11,13 +11,13 @@
     colmena.url = "github:zhaofengli/colmena";
     colmena.inputs.nixpkgs.follows = "nixpkgs";
 
-    hydra.url = "git+https://git.lix.systems/lix-project/hydra.git";
+    hydra.url = "git+https://git.lix.systems/the-distro/hydra.git";
     hydra.inputs.nixpkgs.follows = "nixpkgs";
 
     nix-gerrit.url = "git+https://git.lix.systems/the-distro/nix-gerrit.git";
     nix-gerrit.inputs.nixpkgs.follows = "nixpkgs";
 
-    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/refactor";
+    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/non-flakes";
     buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";
 
     lix.follows = "hydra/lix";
@@ -33,30 +33,38 @@
 
   outputs = { self, nixpkgs, terranix, colmena, ... } @ inputs:
   let
-    system = "x86_64-linux";
+    supportedSystems = [ "x86_64-linux" "aarch64-linux" ];
-    pkgs = import nixpkgs {
+    forEachSystem = f: builtins.listToAttrs (map (system: {
-      localSystem = system;
+      name = system;
-      overlays = [
+      value = f system;
-        inputs.hydra.overlays.default
+    }) supportedSystems);
-        inputs.lix.overlays.default
+    systemBits = forEachSystem (system: rec {
-        inputs.nix-gerrit.overlays.default
-      ];
-    };
-    lib = pkgs.lib;
-    terraform = pkgs.opentofu;
-    terraformCfg = terranix.lib.terranixConfiguration {
       inherit system;
-      modules = [
+      pkgs = import nixpkgs {
-        ./terraform
+        localSystem = system;
-        {
+        overlays = [
-          bagel.gandi.enable = true;
+          inputs.hydra.overlays.default
-          bagel.hydra.enable = true;
+          inputs.lix.overlays.default
-        }
+          inputs.nix-gerrit.overlays.default
-      ];
+        ];
-    };
+      };
+      terraform = pkgs.opentofu;
+      terraformCfg = terranix.lib.terranixConfiguration {
+        inherit system;
+        modules = [
+          ./terraform
+          {
+            bagel.gandi.enable = true;
+            bagel.hydra.enable = true;
+          }
+        ];
+      };
+    });
+    forEachSystem' = f: forEachSystem (system: (f systemBits.${system}));
+    inherit (nixpkgs) lib;
   in
   {
-    apps.${system} = {
+    apps = forEachSystem' ({ system, pkgs, terraformCfg, terraform, ... }: {
       tf = {
         type = "app";
         program = toString (pkgs.writers.writeBash "tf" ''
@@ -67,16 +75,19 @@
       };
 
       default = self.apps.${system}.tf;
-    };
+    });
 
-    devShells.${system}.default = pkgs.mkShell {
+    devShells = forEachSystem' ({ system, pkgs, ... }: {
-      packages = [
+      default = pkgs.mkShell {
-        inputs.agenix.packages.${system}.agenix
+        packages = [
+          inputs.agenix.packages.${system}.agenix
 
-        pkgs.colmena
+          pkgs.opentofu
-        pkgs.opentofu
+
-      ];
+          (pkgs.callPackage ./lib/colmena-wrapper.nix { })
-    };
+        ];
+      };
+    });
 
     nixosConfigurations = (colmena.lib.makeHive self.outputs.colmena).nodes;
 
@@ -93,19 +104,12 @@
 
       makeBuilder = i: lib.nameValuePair "builder-${toString i}" {
         imports = commonModules;
-        bagel.baremetal.builders = { enable = true; num = i; };
+        bagel.baremetal.builders = { enable = true; num = i; netboot = i >= 6; };
       };
 
       builders = lib.listToAttrs (lib.genList makeBuilder 12);
     in {
-      meta.nixpkgs = import nixpkgs {
+      meta.nixpkgs = systemBits.x86_64-linux.pkgs;
-        localSystem = system;
-        overlays = [
-          inputs.hydra.overlays.default
-          inputs.lix.overlays.default
-          inputs.nix-gerrit.overlays.default
-        ];
-      };
       meta.specialArgs.inputs = inputs;
 
       bagel-box.imports = commonModules ++ [ ./hosts/bagel-box ];
@@ -115,9 +119,10 @@
       git.imports = commonModules ++ [ ./hosts/git ];
       wob-vpn-gw.imports = commonModules ++ [ ./hosts/wob-vpn-gw ];
       buildbot.imports = commonModules ++ [ ./hosts/buildbot ];
+      public01.imports = commonModules ++ [ ./hosts/public01 ];
     } // builders;
 
-    hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.toplevel) self.nixosConfigurations;
+    hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.netbootDir or v.config.system.build.toplevel) self.nixosConfigurations;
     buildbotJobs = builtins.mapAttrs (_: v: v.config.system.build.toplevel) self.nixosConfigurations;
   };
 }

hosts/buildbot/default.nix

@@ -28,7 +28,7 @@
   bagel.services.buildbot = {
     enable = true;
     domain = "buildbot.forkos.org";
-    builders = [ "builder-3" ];
+    builders = [ "builder-11" ];
   };
 
   i18n.defaultLocale = "en_US.UTF-8";

hosts/gerrit01/default.nix

@@ -47,12 +47,13 @@
   };
   bagel.nixpkgs.one-way-sync =
   let
-    mkNixpkgsJob = { timer, branchName }: {
+    mkNixpkgsJob = { timer, fromRefspec, localRefspec ? fromRefspec }: {
-      name = "nixpkgs-${branchName}";
       fromUri = "https://github.com/NixOS/nixpkgs";
-      fromRefspec = branchName;
+      inherit fromRefspec localRefspec timer;
-      localRefspec = branchName;
+    };
-      inherit timer;
+    mkLocalJob = { timer, fromRefspec, localRefspec }: {
+      fromUri = "https://cl.forkos.org/nixpkgs";
+      inherit fromRefspec localRefspec timer;
     };
     in
   {
@@ -61,47 +62,59 @@
     pushUrl = "ssh://ows_bot@cl.forkos.org:29418/nixpkgs";
     deployKeyPath = config.age.secrets.ows-deploy-key.path;
 
-    branches."refs/heads/main" = mkNixpkgsJob {
+    # Sync main -> staging-next -> staging
-      timer = "hourly";
+    branches."main-to-staging-next" = mkLocalJob {
-      branchName = "main";
+      timer = "00/8:20:00"; # every 8 hours, 20 minutes past the full hour
+      fromRefspec = "main";
+      localRefspec = "staging-next";
+    };
+    branches."staging-next-to-staging" = mkLocalJob {
+      timer = "00/8:40:00"; # every 8 hours, 40 minutes past the full hour
+      fromRefspec = "staging-next";
+      localRefspec = "staging";
     };
 
-    branches."refs/heads/staging" = mkNixpkgsJob {
+    # Sync nixpkgs -> fork
+    branches."nixpkgs-master" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "staging";
+      fromRefspec = "master";
+      localRefspec = "main";
     };
 
-    branches."refs/heads/release-24.05" = mkNixpkgsJob {
+    branches."nixpkgs-staging" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "release-24.05";
+      fromRefspec = "staging";
     };
 
-    branches."refs/heads/staging-24.05" = mkNixpkgsJob {
+    branches."nixpkgs-release-24.05" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "staging-24.05";
+      fromRefspec = "release-24.05";
     };
 
-    branches."refs/heads/release-23.11" = mkNixpkgsJob {
+    branches."nixpkgs-staging-24.05" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "release-23.11";
+      fromRefspec = "staging-24.05";
     };
 
-    branches."refs/heads/staging-23.11" = mkNixpkgsJob {
+    branches."nixpkgs-release-23.11" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "staging-23.11";
+      fromRefspec = "release-23.11";
+    };
+
+    branches."nixpkgs-staging-23.11" = mkNixpkgsJob {
+      timer = "hourly";
+      fromRefspec = "staging-23.11";
     };
 
     # Testing jobs for personal sandbox branches
-    branches."refs/heads/sandbox/raito/raito-unstable-small" = {
+    branches."raito-unstable-sync" = {
-      name = "raito-unstable-sync";
       fromUri = "https://github.com/NixOS/nixpkgs";
       fromRefspec = "nixos-unstable-small";
       localRefspec = "sandbox/raito/raito-unstable-small";
       timer = "*-*-* 12:00:00";
     };
 
-    branches."refs/heads/sandbox/raito/raito-nixos-24.05" = {
+    branches."raito-release-sync" = {
-      name = "raito-release-sync";
       fromUri = "https://github.com/NixOS/nixpkgs";
       fromRefspec = "nixos-24.05";
       localRefspec = "sandbox/raito/raito-nixos-24.05";

hosts/public01/default.nix

@@ -0,0 +1,32 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  networking.hostName = "public01";
+  # TODO: make it the default
+  networking.domain = "infra.forkos.org";
+
+  time.timeZone = "Europe/Paris";
+
+  bagel.sysadmin.enable = true;
+  # Buildbot is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.hardware.raito-vm = {
+    enable = true;
+    networking = {
+      nat-lan-mac = "BC:24:11:A4:F7:D3";
+      wan = {
+        address = "2001:bc8:38ee:100:1000::60/64";
+        mac = "BC:24:11:DB:B8:10";
+      };
+    };
+  };
+
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "public01.infra.forkos.org";
+}

hosts/wob-vpn-gw/default.nix

@@ -1,6 +1,10 @@
 { pkgs, lib, ... }:
 
 {
+  imports = [
+    ./netboot.nix
+  ];
+
   ###### Hardware ######
   boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "ehci_pci" "sd_mod" "sdhci_pci" ];
   boot.kernelModules = [ "kvm-amd" ];

hosts/wob-vpn-gw/netboot.nix

@@ -0,0 +1,63 @@
+{ config, lib, pkgs, nodes, modulesPath, ... }:
+
+# The way the connection is established is specific to the wob01 site and the Intel S2600KPR blades.
+# Proper netboot is not possible, because while the blades and the APU board (which is the netboot
+# server here) are in the same L2 network, the uplink connection of each blade is an LACP LAG,
+# meaning that the switch on the other side will only enable the port if it sees valid LACP packets.
+# We work around this by presenting a virtual floppy drive using the "IUSB" protocol of the BMC.
+# This virtual floppy drive contains an per-blade customized initramfs which will initialize the
+# network connection including IP configuration and load the actual image off hydra.
+
+let
+  netboot-server-ip = "2a01:584:11::2";
+  netbootNodes = lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes;
+in {
+
+  assertions = [
+    {
+      assertion = !(lib.elem 443 config.networking.firewall.allowedTCPPorts);
+      message = ''
+        Port 443 is in networking.firewalls.allowedTCPPorts, but should be only manually
+        allowed for specific IPs and source ports in ${builtins.toJSON __curPos}
+      '';
+    }
+  ];
+
+  systemd.services = lib.mapAttrs' (nodename: node: let
+    bmcIp = "192.168.1.${toString (node.config.bagel.baremetal.builders.num * 4 + 2)}";
+    notipxe = node.config.system.build.notipxe.config.system.build.usbImage;
+  in lib.nameValuePair "iusb-spoof-${nodename}" {
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Restart = "always";
+    };
+    script = ''
+      AUTH_TOKEN=$(${pkgs.iusb-spoof}/bin/make-token ${bmcIp})
+      exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${notipxe}
+    '';
+  }) netbootNodes;
+
+  # Since the builders are stateless, they can not store their ssh hostkeys
+  networking.firewall.allowedTCPPorts = [ 80 ]; # for ACME
+  networking.firewall.extraInputRules = ''
+    ip6 saddr 2a01:584:11::/64 tcp sport < 1024 tcp dport 443 accept;
+  '';
+  security.acme.acceptTerms = true;
+  security.acme.defaults.email = "infra@forkos.org";
+  services.nginx = {
+    enable = true;
+    virtualHosts."vpn-gw.wob01.infra.forkos.org" = {
+      enableACME = true;
+      forceSSL = true;
+      locations = lib.mapAttrs' (nodename: node: let
+        ip = "2a01:584:11::1:${toString node.config.bagel.baremetal.builders.num}";
+      in lib.nameValuePair "/${nodename}/" {
+        root = "/var/www";
+        extraConfig = ''
+          allow ${ip};
+          deny all;
+        '';
+      }) netbootNodes;
+    };
+  };
+}

lib/colmena-wrapper.nix

@@ -0,0 +1,14 @@
+# A wrapper for colmena that prevents accidentally deploying changes without
+# having pulled.
+{ colmena, runCommandNoCC }:
+runCommandNoCC "colmena-wrapper"
+{
+  env.colmena = "${colmena}/bin/colmena";
+} ''
+  mkdir -p $out
+  ln -s ${colmena}/share $out/share
+  mkdir $out/bin
+
+  substituteAll ${./colmena-wrapper.sh.in} $out/bin/colmena
+  chmod +x $out/bin/colmena
+''

lib/colmena-wrapper.sh.in

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+doChecks() {
+    # creates refs in the refs/prefetch/remotes/origin namespace
+    echo "Prefetching repo changes..." >&2
+    git fetch --quiet --prefetch --no-write-fetch-head origin
+
+    diffs=$(git rev-list --left-right --count HEAD...refs/prefetch/remotes/origin/main)
+    only_in_local=$(echo "$diffs" | cut -f1)
+    only_in_main=$(echo "$diffs" | cut -f2)
+
+    if [[ $only_in_main -gt 0 && ! -v $FOOTGUN_ME_UWU ]]; then
+        echo >&2
+        echo "Attempting to deploy when main has $only_in_main commits not in your branch!" >&2
+        echo "This will probably revert someone's changes. Consider merging them." >&2
+        echo "If you really mean it, set the environment variable FOOTGUN_ME_UWU" >&2
+        exit 1
+    fi
+
+    if [[ $only_in_local -gt 0 ]]; then
+        echo "You have $only_in_local commits not yet pushed to main. Reminder to push them after :)" >&2
+    fi
+}
+
+if [[ $1 == 'apply' ]]; then
+    doChecks
+fi
+
+exec @colmena@ "$@"

overlays/default.nix

@@ -1 +1,6 @@
-[]
+[
+  (final: prev: {
+    iusb-spoof = final.callPackage ./iusb-spoof.nix {};
+    u-root = final.callPackage ./u-root {};
+  })
+]

overlays/iusb-spoof.nix

@@ -0,0 +1,23 @@
+{ rustPlatform, python3, makeWrapper }:
+let
+  pythonEnv = python3.withPackages (p: with p; [ requests ]);
+in
+
+rustPlatform.buildRustPackage rec {
+  pname = "iusb-spoof";
+  version = "0.1.0";
+
+  src = builtins.fetchGit {
+    url = "https://git.lix.systems/the-distro/iusb-spoof/";
+    rev = "fafd47986239cc2f4dfbbae74b17555608806581";
+  };
+
+  cargoLock.lockFile = src + "/Cargo.lock";
+
+  nativeBuildInputs = [ makeWrapper ];
+
+  postInstall = ''
+    install -Dm644 $src/make-token.py $out/opt/make-token.py
+    makeWrapper ${pythonEnv.interpreter} $out/bin/make-token --add-flags "$out/opt/make-token.py"
+  '';
+}

overlays/u-root/default.nix

@@ -0,0 +1,20 @@
+{ buildGoModule, fetchFromGitHub }:
+
+buildGoModule rec {
+  pname = "u-root";
+  version = "0.14.0";
+
+  src = fetchFromGitHub {
+    owner = "u-root";
+    repo = "u-root";
+    rev = "v${version}";
+    hash = "sha256-8zA3pHf45MdUcq/MA/mf0KCTxB1viHieU/oigYwIPgo=";
+  };
+
+  patches = [
+    ./u-root-allow-https.patch
+  ];
+
+  vendorHash = null;
+  doCheck = false;
+}

overlays/u-root/u-root-allow-https.patch

@@ -0,0 +1,12 @@
+diff --git a/pkg/curl/schemes.go b/pkg/curl/schemes.go
+index 8bac3bc0..cd396cbc 100644
+--- a/pkg/curl/schemes.go
++++ b/pkg/curl/schemes.go
+@@ -81,6 +81,7 @@ var (
+ 	DefaultSchemes = Schemes{
+ 		"tftp": DefaultTFTPClient,
+ 		"http": DefaultHTTPClient,
++		"https": DefaultHTTPClient,
+ 		"file": &LocalFileClient{},
+ 	}
+ )

services/baremetal-builder/assignments.nix

@@ -0,0 +1,29 @@
+# This file contains information on which builder(s) are providing how many
+# job slots and providing which nix features
+let
+  genBuilders = { offset ? 0, count, f }: builtins.genList (x: rec { name = "builder-${toString (offset + x)}"; value = f name; }) count;
+in builtins.listToAttrs (
+  # The first 8 builders are general purpose hydra builders
+  genBuilders { count = 8; f = name: {
+    cores = 8;
+    max-jobs = 8;
+    supported-features = [ "kvm" "nixos-test" ];
+    required-features = [ ];
+  }; }
+  ++
+  # The last 2 builders are exclusively for big-parallel
+  genBuilders { offset = 8; count = 2; f = name: {
+    cores = 20;
+    max-jobs = 1;
+    supported-features = [ "kvm" "nixos-test" "big-parallel" ];
+    required-features = [ "big-parallel" ];
+  }; }
+  ++
+  # These are not currently used for hydra
+  genBuilders { offset = 10; count = 2; f = name: {
+    cores = 8;
+    max-jobs = 8;
+    supported-features = [ "kvm" "nixos-test" "big-parallel" ];
+    required-features = [ ];
+  }; }
+)

services/baremetal-builder/default.nix

@@ -3,10 +3,13 @@ let
   cfg = config.bagel.baremetal.builders;
 in
 {
+  imports = [ ./netboot.nix ];
+
   options = {
 
     bagel.baremetal.builders = {
       enable = lib.mkEnableOption "baremetal bagel oven";
+      netboot = lib.mkEnableOption "netboot";
       num = lib.mkOption {
         type = lib.types.int;
       };
@@ -40,8 +43,10 @@ in
         "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMnOLLX0vGTZbSJrUmF9ZFXt/NIId/MUrEpXmL2vxod"
       ];
     };
-    nix.settings.trusted-users = [ "builder" "buildbot" ];
+    nix.settings = {
-
+      trusted-users = [ "builder" "buildbot" ];
+      inherit ((import ./assignments.nix).${config.networking.hostName}) max-jobs cores;
+    };
 
     nixpkgs.hostPlatform = "x86_64-linux";
     hardware.cpu.intel.updateMicrocode = true;
@@ -52,28 +57,36 @@ in
 
     boot.initrd.services.lvm.enable = true;
 
-    fileSystems."/" = {
+    boot.kernel.sysctl."fs.xfs.xfssyncd_centisecs" = "12000";
-      device = "/dev/disk/by-label/root";
+    fileSystems = lib.mkMerge [
-      fsType = "xfs";
+      (lib.mkIf (!cfg.netboot) {
-    };
+        "/" = {
+          device = "/dev/disk/by-label/root";
+          fsType = "xfs";
+        };
 
-    fileSystems."/mnt" = {
+        "/boot" = {
-      device = "/dev/disk/by-label/hydra";
+          device = "/dev/disk/by-label/BOOT";
-      fsType = "xfs";
+          fsType = "vfat";
-    };
+          options = [ "fmask=0022" "dmask=0022" ];
-    # We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
+        };
-    fileSystems."/tmp" = {
+      })
-      device = "/mnt/tmp";
+      {
-      options = [ "bind" ];
+        "/mnt" = {
-    };
+          device = "/dev/disk/by-label/hydra";
+          fsType = "xfs";
+          options = ["logbsize=256k"];
+        };
 
-    fileSystems."/boot" = {
+        # We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
-      device = "/dev/disk/by-label/BOOT";
+        "/tmp" = {
-      fsType = "vfat";
+          device = "/mnt/tmp";
-      options = [ "fmask=0022" "dmask=0022" ];
+          options = [ "bind" ];
-    };
+        };
+      }
+    ];
 
-    swapDevices = [
+    swapDevices = lib.optionals (!cfg.netboot) [
       {
         device = "/swapfile";
         size = 50 * 1024;  # 50GiB
@@ -86,8 +99,8 @@ in
     };
 
     boot.kernelParams = [
-      "console=ttyS0,115200"
       "console=tty1"
+      "console=ttyS0,115200"
     ];
 
     networking.useNetworkd = true;
@@ -146,11 +159,24 @@ in
     };
 
     systemd.services.hydra-gc = {
+      wantedBy = [ "multi-user.target" ];
       description = "Nix Garbage Collector";
-      script = "exec ${config.nix.package.out}/bin/nix-store --gc --store /mnt";
+      script = ''
+        while : ; do
+          percent_filled=$(($(stat -f --format="100-(100*%a/%b)" /mnt)))
+          if [ "$percent_filled" -gt "54" ]; then
+            ${config.nix.package.out}/bin/nix-store --gc --max-freed 50G --store /mnt
+          else
+            break
+          fi
+        done
+      '';
       serviceConfig.Type = "oneshot";
       serviceConfig.User = "builder";
-      startAt = "*-*-* 00/8:00:00";
+    };
+    systemd.timers.hydra-gc = {
+      timerConfig.OnUnitInactiveSec = "10min";
+      wantedBy = [ "timers.target" ];
     };
     systemd.timers.hydra-gc.timerConfig.Persistent = true;
 

services/baremetal-builder/netboot.nix

@@ -0,0 +1,169 @@
+{ modulesPath, pkgs, lib, config, extendModules, ... }@node:
+let
+  cfg = config.bagel.baremetal.builders;
+in
+{
+  config = lib.mkIf (cfg.enable && cfg.netboot) {
+    systemd.services.sshd.after = [ "provision-ssh-hostkey.service" ];
+    systemd.services.provision-ssh-hostkey = {
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+      script = ''
+        mkdir -p /etc/ssh
+        umask 0077
+        until ${pkgs.iputils}/bin/ping -c 1 vpn-gw.wob01.infra.forkos.org; do sleep 1; done
+        ${pkgs.curl}/bin/curl --local-port 25-1024 https://vpn-gw.wob01.infra.forkos.org/${config.networking.hostName}/ssh_host_ed25519_key > /etc/ssh/ssh_host_ed25519_key
+        # Run the activation script again to trigger agenix decryption
+        /run/current-system/activate
+      '';
+    };
+
+    system.build = {
+
+      # Build a kernel and initramfs which will download the IPXE script from hydra using
+      # u-root pxeboot tool and kexec into the final netbooted system.
+      notipxe = import (modulesPath + "/..") {
+        system = "x86_64-linux";
+        configuration = 
+          { pkgs, config, ... }:
+
+          {
+            system.stateVersion = "24.11";
+            boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" "igb" "bonding" ];
+            boot.kernelParams = [ "console=ttyS0,115200" "panic=1" "boot.panic_on_fail" ];
+            #boot.initrd.systemd.emergencyAccess = true;
+            networking.hostName = "${node.config.networking.hostName}-boot";
+            nixpkgs.overlays = import ../../overlays;
+            boot.loader.grub.enable = false;
+            fileSystems."/".device = "bogus"; # this config will never be booted
+            boot.initrd.systemd.enable = true;
+            boot.initrd.systemd.network = {
+              enable = true;
+              networks = node.config.systemd.network.networks;
+              netdevs = node.config.systemd.network.netdevs;
+            };
+            boot.initrd.systemd.storePaths = [
+              "${pkgs.u-root}/bin/pxeboot"
+              "${pkgs.iputils}/bin/ping"
+            ];
+            boot.initrd.systemd.services.kexec = {
+              serviceConfig.Restart = "on-failure";
+              serviceConfig.Type = "oneshot";
+              wantedBy = [ "initrd-root-fs.target" ];
+              before = [ "sysroot.mount" ];
+              script = ''
+                ln -sf /dev/console /dev/tty
+                until ${pkgs.iputils}/bin/ping -c 1 hydra.forkos.org; do sleep 1; done
+                ${pkgs.u-root}/bin/pxeboot -v -ipv4=false -file https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe 
+              '';
+            };
+            boot.initrd.systemd.contents."/etc/ssl/certs/ca-certificates.crt".source = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+            boot.initrd.services.resolved.enable = false;
+            boot.initrd.systemd.contents."/etc/resolv.conf".text = ''
+              nameserver 2001:4860:4860::6464
+            '';
+            boot.initrd.systemd.contents."/etc/systemd/journald.conf".text = ''
+              [Journal]
+              ForwardToConsole=yes
+              MaxLevelConsole=debug
+            '';
+
+            # Provide a bootable USB drive image
+            system.build.usbImage = pkgs.callPackage ({ stdenv, runCommand, dosfstools, e2fsprogs, mtools, libfaketime, util-linux, nukeReferences }:
+              runCommand "boot-img-${node.config.networking.hostName}" {
+                nativeBuildInputs = [ dosfstools e2fsprogs libfaketime mtools util-linux ];
+                outputs = [ "out" "firmware_part" ];
+              } ''
+                export img=$out
+                truncate -s 40M $img
+
+                sfdisk $img <<EOF
+                    label: gpt
+                    label-id: F222513B-DED1-49FA-B591-20CE86A2FE7F
+
+                    type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B, bootable
+                EOF
+
+                # Create a FAT32 /boot/firmware partition of suitable size into firmware_part.img
+                eval $(partx $img -o START,SECTORS --nr 1 --pairs)
+                truncate -s $((2081 * 512 + SECTORS * 512)) firmware_part.img
+
+                mkfs.vfat --invariant -i 2e24ec82 -n BOOT firmware_part.img
+
+                # Populate the files intended for /boot/firmware
+                mkdir -p firmware/EFI/BOOT firmware/loader/entries
+                cp ${pkgs.systemd}/lib/systemd/boot/efi/systemd-boot*.efi firmware/EFI/BOOT/BOOT${lib.toUpper stdenv.hostPlatform.efiArch}.EFI
+
+                cat > firmware/loader/loader.conf << EOF
+                default foo
+                EOF
+                cat > firmware/loader/entries/default.conf << EOF
+                title Default
+                linux /EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target}
+                initrd /EFI/initrd
+                options init=${config.system.build.toplevel}/init ${toString config.boot.kernelParams}
+                EOF
+                cp ${config.system.build.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target} firmware/EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target}
+                cp ${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile} firmware/EFI/initrd
+
+                find firmware -exec touch --date=2000-01-01 {} +
+                # Copy the populated /boot/firmware into the SD image
+                cd firmware
+                # Force a fixed order in mcopy for better determinism, and avoid file globbing
+                for d in $(find . -type d -mindepth 1 | sort); do
+                  faketime "2000-01-01 00:00:00" mmd -i ../firmware_part.img "::/$d"
+                done
+                for f in $(find . -type f | sort); do
+                  mcopy -pvm -i ../firmware_part.img "$f" "::/$f"
+                done
+                cd ..
+
+                # Verify the FAT partition before copying it.
+                fsck.vfat -vn firmware_part.img
+                dd conv=notrunc if=firmware_part.img of=$img seek=$START count=$SECTORS
+
+                cp firmware_part.img $firmware_part
+              ''
+            ) {};
+          }
+        ;
+      };
+
+      # This is the config which will actually be booted
+      netbootVariant = extendModules {
+        modules = [
+          (
+            { modulesPath, ... }:
+
+            {
+              imports = [ (modulesPath + "/installer/netboot/netboot.nix") ];
+            }
+          )
+        ];
+      };
+      # A derivation combining all the artifacts required for netbooting for the hydra job
+      netbootDir = let
+        kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target;
+        build = config.system.build.netbootVariant.config.system.build;
+      in
+        pkgs.symlinkJoin {
+          name = "netboot";
+          paths = [
+            build.netbootRamdisk
+            build.kernel
+            build.netbootIpxeScript
+          ];
+          postBuild = ''
+            mkdir -p $out/nix-support
+            echo "file ${kernelTarget} $out/${kernelTarget}" >> $out/nix-support/hydra-build-products
+            echo "file initrd $out/initrd" >> $out/nix-support/hydra-build-products
+            echo "file ipxe $out/netboot.ipxe" >> $out/nix-support/hydra-build-products
+          '';
+          preferLocalBuild = true;
+        };
+    };
+  };
+}

services/buildbot/default.nix

@@ -31,8 +31,16 @@ in
     age.secrets.buildbot-oauth-secret.file = ../../secrets/buildbot-oauth-secret.age;
     age.secrets.buildbot-workers.file = ../../secrets/buildbot-workers.age;
     age.secrets.buildbot-service-key.file = ../../secrets/buildbot-service-key.age;
-    age.secrets.buildbot-signing-key.file = ../../secrets/buildbot-signing-key.age;
+    age.secrets.buildbot-signing-key = {
-    age.secrets.buildbot-remote-builder-key.file = ../../secrets/buildbot-remote-builder-key.age;
+      file = ../../secrets/buildbot-signing-key.age;
+      owner = "buildbot-worker";
+      group = "buildbot-worker";
+    };
+    age.secrets.buildbot-remote-builder-key = {
+      file = ../../secrets/buildbot-remote-builder-key.age;
+      owner = "buildbot-worker";
+      group = "buildbot-worker";
+    };
 
     services.nginx.virtualHosts.${cfg.domain} = {
       forceSSL = true;
@@ -58,7 +66,7 @@ in
             (_: lib.foldl' lib.add 0)
             (lib.concatMap
               (m: map (s: { ${s} = m.maxJobs; }) m.systems)
-              config.nix.buildMachines))
+              config.services.buildbot-nix.coordinator.buildMachines))
         );
     };
 
@@ -67,6 +75,8 @@ in
 
       inherit (cfg) domain;
 
+      debugging.enable = true;
+
       oauth2 = {
         name = "Lix";
         clientId = "forkos-buildbot";

services/forgejo/default.nix

@@ -55,6 +55,10 @@ in
           DEFAULT_KEEP_EMAIL_PRIVATE = true;
         };
 
+        "service.explore" = {
+          DISABLE_USERS_PAGE = true;
+        };
+
         oauth2_client = {
           REGISTER_EMAIL_CONFIRM = false;
           ENABLE_AUTO_REGISTRATION = true;

services/gerrit/one-way-sync.nix

@@ -3,7 +3,7 @@ let
   cfg = config.bagel.nixpkgs.one-way-sync;
   inherit (lib) mkIf mkOption mkEnableOption types mapAttrs';
 
-  mkSyncTimer = { name, timer, ... }: {
+  mkSyncTimer = name: { timer, ... }: {
     wantedBy = [ "timers.target" ];
 
     timerConfig = {
@@ -12,7 +12,7 @@ let
       Unit = "ows-${name}.service";
     };
   };
-  mkSyncService = targetRef: { name, fromUri, fromRefspec, localRefspec, ... }: {
+  mkSyncService = name: { fromUri, fromRefspec, localRefspec, ... }: {
     path = [ pkgs.gitFull pkgs.openssh pkgs.lix ];
     script = ''
       set -xe
@@ -25,11 +25,11 @@ let
       fi
 
       cd /var/lib/onewaysync/nixpkgs
-      echo "Syncing ${fromUri}:${fromRefspec} to /var/lib/onewaysync/nixpkgs:${targetRef}"
+      echo "Syncing ${fromUri}:${fromRefspec} to ${cfg.pushUrl}:refs/heads/${localRefspec}"
       echo "Current ref: $EXPECTED_REF"
       git worktree add -f "$RUNTIME_DIRECTORY"/${name} refs/remotes/origin/${localRefspec}
       cd "$RUNTIME_DIRECTORY"/${name}
-      git pull origin ${localRefspec}
+      git pull origin ${localRefspec} --no-rebase
       EXPECTED_REF=$(git rev-list refs/remotes/origin/${localRefspec} | head -1)
       git config user.name Fork-o-Tron
       git config user.email noreply@forkos.org
@@ -43,7 +43,7 @@ let
       # Do not allow auto-merging a staging iteration
       test "$OLD_STDENV" = "$NEW_STDENV"
     '' + ''
-      GIT_SSH_COMMAND='ssh -i ${cfg.deployKeyPath}' git push ${cfg.pushUrl} HEAD:${targetRef}
+      GIT_SSH_COMMAND='ssh -i ${cfg.deployKeyPath}' git push ${cfg.pushUrl} HEAD:refs/heads/${localRefspec}
     '';
     serviceConfig = {
       User = "git";
@@ -120,12 +120,12 @@ in
 
   config = mkIf cfg.enable {
     systemd.timers = mapAttrs' (name: value: {
-      name = "ows-${value.name}";
+      name = "ows-${name}";
-      value = mkSyncTimer value;
+      value = mkSyncTimer name value;
     }) cfg.branches;
 
     systemd.services = mapAttrs' (name: value: {
-      name = "ows-${value.name}";
+      name = "ows-${name}";
       value = mkSyncService name value;
     }) cfg.branches;
   };

services/hydra/default.nix

@@ -13,15 +13,36 @@ let
 
   # XXX: to support Nix's dumb public host key syntax (base64'd), this outputs
   # a string with shell-style command interpolations: $(...).
-  mkBaremetalBuilder = { parallelBuilds, publicHostKey, host, speedFactor ? 1, user ? "builder", supportedSystems ? [ "i686-linux" "x86_64-linux" ], supportedFeatures ? [ "big-parallel" "kvm" "nixos-test" ] }:
+  mkBaremetalBuilder = {
-  "ssh://${user}@${host}?remote-store=/mnt ${lib.concatStringsSep "," supportedSystems} ${config.age.secrets.hydra-ssh-key-priv.path} ${toString parallelBuilds} ${toString speedFactor} ${lib.concatStringsSep "," supportedFeatures} - $(echo -n '${publicHostKey}' | base64 -w0)";
+    parallelBuilds,
+    publicHostKey,
+    host,
+    speedFactor ? 1,
+    user ? "builder",
+    supportedSystems ? [ "i686-linux" "x86_64-linux" ],
+    supportedFeatures ? [ "big-parallel" "kvm" "nixos-test" ],
+    requiredFeatures ? [ ]
+  }:
+  let
+    supportedFeatures_ = if (supportedFeatures != []) then lib.concatStringsSep "," supportedFeatures else "-";
+    requiredFeatures_ = if (requiredFeatures != []) then lib.concatStringsSep "," requiredFeatures else "-";
+  in
+  "ssh://${user}@${host}?remote-store=/mnt ${lib.concatStringsSep "," supportedSystems} ${config.age.secrets.hydra-ssh-key-priv.path} ${toString parallelBuilds} ${toString speedFactor} ${supportedFeatures_} ${requiredFeatures_} $(echo -n '${publicHostKey}' | base64 -w0)";
 
   # TODO:
   # - generalize to new architectures
   # - generalize to new features
   baremetalBuilders = lib.concatStringsSep "\n"
-    (map (n: mkBaremetalBuilder {
+    (map (n: let
-      parallelBuilds = 8; # TODO: do not hardcode this, use the node's builder configuration.
+      assignments = (import ../baremetal-builder/assignments.nix).${n} or {
+        inherit (nodes.${n}.config.nix.settings) max-jobs;
+        supported-features = [ "big-parallel" "kvm" "nixos-test" ];
+        required-features = [];
+      };
+    in mkBaremetalBuilder {
+      parallelBuilds = assignments.max-jobs;
+      supportedFeatures = assignments.supported-features;
+      requiredFeatures = assignments.required-features;
       publicHostKey = ssh-keys.machines.${n};
       host = nodes.${n}.config.networking.fqdn;
     }) cfg.builders);

terraform/gandi.nix

@@ -68,6 +68,7 @@ in
       # git.p.forkos.org exposes forgejo ssh server.
       (proxyRecords "git.p" 3600 "AAAA" ["2001:bc8:38ee:100:1000::40"])
       (dualProxyRecords "buildbot.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::50"])
+      (dualProxyRecords "public01.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::60"])
 
       (record "cl" 3600 "CNAME" ["gerrit01.infra.p"])
       (record "fodwatch" 3600 "CNAME" ["fodwatch.infra.p"])
@@ -82,6 +83,10 @@ in
       (record "matrix" 3600 "CNAME" ["meta01.infra.p"])
       (record "alerts" 3600 "CNAME" ["meta01.infra.p"])
       (record "buildbot" 3600 "CNAME" ["buildbot.infra.p"])
+      (record "b" 3600 "CNAME" ["public01.infra.p"])
+
+      # S3 in delroth's basement
+      (record "cache" 3600 "CNAME" ["smol.delroth.net."])
 
       (record "vpn-gw.wob01.infra" 3600 "AAAA" [ "2a01:584:11::2" ])
       # TODO: do not hardcode, just reuse the Colmena hive module outputs to generate all the required details.

terraform/hydra.nix

@@ -205,13 +205,53 @@ in
       email_notifications = false;
     };
 
-    resource.hydra_jobset.yureka-staging-test = {
+    resource.hydra_jobset.nixos-staging-next-small = {
       project = config.resource.hydra_project.forkos.name;
       state = "enabled";
       visible = true;
-      name = "yureka-staging-test";
+      name = "nixos-staging-next-small";
       type = "legacy";
-      description = "staging branch for yureka-nixos";
+      description = "nixos jobset for the staging-next branch";
+
+      nix_expression = {
+        file = "nixos/release-small.nix";
+        input = "nixpkgs";
+      };
+
+      check_interval = 0;
+      scheduling_shares = 3000;
+      keep_evaluations = 3;
+
+      email_notifications = false;
+
+      input = [
+        {
+          name = "nixpkgs";
+          type = "git";
+          value = "https://cl.forkos.org/nixpkgs staging-next";
+          notify_committers = false;
+        }
+        {
+          name = "officialRelease";
+          type = "boolean";
+          value = "false";
+          notify_committers = false;
+        }
+        {
+          name = "supportedSystems";
+          type = "nix";
+          value = ''[ "x86_64-linux" ]'';
+        }
+      ];
+    };
+
+    resource.hydra_jobset.nixpkgs-staging-next = {
+      project = config.resource.hydra_project.forkos.name;
+      state = "enabled";
+      visible = true;
+      name = "nixpkgs-staging-next";
+      type = "legacy";
+      description = "nixpkgs jobset for the staging-next branch";
 
       nix_expression = {
         file = "pkgs/top-level/release.nix";
@@ -228,7 +268,47 @@ in
         {
           name = "nixpkgs";
           type = "git";
-          value = "https://cl.forkos.org/nixpkgs sandbox/yureka/staging-test";
+          value = "https://cl.forkos.org/nixpkgs staging-next";
+          notify_committers = false;
+        }
+        {
+          name = "officialRelease";
+          type = "boolean";
+          value = "false";
+          notify_committers = false;
+        }
+        {
+          name = "supportedSystems";
+          type = "nix";
+          value = ''[ "x86_64-linux" ]'';
+        }
+      ];
+    };
+
+    resource.hydra_jobset.nixos-main = {
+      project = config.resource.hydra_project.forkos.name;
+      state = "enabled";
+      visible = true;
+      name = "nixos-main";
+      type = "legacy";
+      description = "nixos jobset for the main branch";
+
+      nix_expression = {
+        file = "nixos/release-combined.nix";
+        input = "nixpkgs";
+      };
+
+      check_interval = 0;
+      scheduling_shares = 3000;
+      keep_evaluations = 3;
+
+      email_notifications = false;
+
+      input = [
+        {
+          name = "nixpkgs";
+          type = "git";
+          value = "https://cl.forkos.org/nixpkgs main";
           notify_committers = false;
         }
         {