Set up alertmanager-hookshot-adapter

It doesn't want to work.

common/admins.nix

@@ -3,12 +3,14 @@ let
 in {
   users.users.root.openssh.authorizedKeys.keys = 
     keys.users.delroth ++
-    keys.users.k900 ++ 
+    keys.users.emilylange ++
-    keys.users.raito ++
+    keys.users.hexchen ++
-    keys.users.maxine ++ 
     keys.users.jade ++
     keys.users.janik ++
+    keys.users.k900 ++
     keys.users.lukegb ++
-    keys.users.emilylange ++
+    keys.users.maxine ++
+    keys.users.raito ++
+    keys.users.thubrecht ++
     keys.users.yuka;
 }

common/nix.nix

@@ -14,7 +14,7 @@
 
   # Use our cache and trust its signing key. Still use cache.nixos.org as
   # fallback.
-  nix.settings.substituters = [ "https://bagel-cache.s3-web.delroth.net/" ];
+  nix.settings.substituters = [ "https://cache.forkos.org/" ];
   nix.settings.trusted-public-keys = [
     "cache.forkos.org:xfXIUJO1yiEITJmYsVmNDa9BFSlgTh/YqZ+4ei1EhQg="
   ];

common/ssh-keys.nix

@@ -23,14 +23,12 @@
 
   users = {
     delroth = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" ];
-    raito = [
+    emilylange = [ "no-touch-required sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIL7jgq3i+N3gVJhs4shm7Kmw6dIocs2OuR0GBMG1RxfKAAAABHNzaDo=" ];
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp"
+    hexchen = [
-      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINJ0tCxsEilAzV6LaNpUpcjzyEn4ptw8kFz3R+Z3YjEF hexchen@backup"
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDI3T1eFS77URHZ/HVWkMOqx7W1U54zJtn9C7QWsHOtyH72i/4EVj8SxYqLllElh1kuKUXSUipPeEzVsipFVvfH0wEuTDgFffiSQ3a8lfUgdEBuoySwceEoPgc5deapkOmiDIDeeWlrRe3nqspLRrSWU1DirMxoFPbwqJXRvpl6qJPxRg+2IolDcXlZ6yxB4Vv48vzRfVzZNUz7Pjmy2ebU8PbDoFWL/S3m7yOzQpv3L7KYBz7+rkjuF3AU2vy6CAfIySkVpspZZLtkTGCIJF228ev0e8NvhuN6ZnjzXxVTQOy32HCdPdbBbicu0uHfZ5O7JX9DjGd8kk1r2dnZwwy/ hexchen@yubi5"
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC4CLJ+mFfq5XiBXROKewmN9WYmj+79bj/AoaR6Iud2pirulot3tkrrLe2cMjiNWFX8CGVqrsAELKUA8EyUTJfStlcTE0/QNESTRmdDaC+lZL41pWUO9KOiD6/0axAhHXrSJ0ScvbqtD0CtpnCKKxtuOflVPoUGZsH9cLKJNRKfEka0H0GgeKb5Tp618R/WNAQOwaCcXzg/nG4Bgv3gJW4Nm9IKy/MwRZqtILi8Mtd+2diTqpMwyNRmbenmRHCQ1vRw46joYkledVqrmSlfSMFgIHI1zRSBXb/JkG2IvIyB5TGbTkC4N2fqJNpH8wnCKuOvs46xmgdiRA26P48C2em3 hexchen@yubi5c"
     ];
-    k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
-    maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
     jade = [
       "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNldAg4t13/i69TD786The+U3wbiNUdW2Kc9KNWvEhgpf4y4x4Sft0oYfkPw5cjX4H3APqfD+b7ItAG0GCbwHw6KMYPoVMNK08zBMJUqt1XExbqGeFLqBaeqDsmEAYXJRbjMTAorpOCtgQdoCKK/DvZ51zUWXxT8UBNHSl19Ryv5Ry5VVdbAE35rqs57DQ9+ma6htXnsBEmmnC+1Zv1FE956m/OpBTId50mor7nS2FguAtPZnDPpTd5zl9kZmJEuWCrmy6iinw5V4Uy1mLeZkQv+/FtozbyifCRCvps9nHpv4mBSU5ABLgnRRvXs+D41Jx7xloNADr1nNgpsNrYaTh hed-bot-ssh-tpm-rsa"
       "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIKYljH8iPMrH00lOb3ETxRrZimdKzPPEdsJQ5D5ovtOwAAAACnNzaDpzc2hrZXk= ssh:sshkey"
@@ -41,8 +39,16 @@
       "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIOYg513QZsVzoyVycXZjg4F3T3+OwtcY3WAhrlfyLgLTAAAABHNzaDo="
       "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBLZxVITpJ8xbiCa/u2gjSSIupeiqOnRh+8tFIoVhCON"
     ];
+    k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
     lukegb = [ ''cert-authority,principals="lukegb" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEqNOwlR7Qa8cbGpDfSCOweDPbAGQOZIcoRgh6s/J8DR'' ];
-    emilylange = [ "no-touch-required sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIL7jgq3i+N3gVJhs4shm7Kmw6dIocs2OuR0GBMG1RxfKAAAABHNzaDo=" ];
+    maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
+    raito = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
+    ];
+    thubrecht = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPM1jpXR7BWQa7Sed7ii3SbvIPRRlKb3G91qC0vOwfJn" ];
     yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKath4/fDnlv/4fzxkPrQN1ttmoPRNu/m9bEtdPJBDfY cardno:16_933_242" ];
   };
 }

flake.lock

@@ -10,11 +10,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1720546205,
+        "lastModified": 1722339003,
-        "narHash": "sha256-boCXsjYVxDviyzoEyAk624600f3ZBo/DKtUdvMTpbGY=",
+        "narHash": "sha256-ZeS51uJI30ehNkcZ4uKqT4ZDARPyqrHADSKAwv5vVCU=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "de96bd907d5fbc3b14fc33ad37d1b9a3cb15edc6",
+        "rev": "3f1dae074a12feb7327b4bf43cbac0d124488bb7",
         "type": "github"
       },
       "original": {
@@ -23,6 +23,29 @@
         "type": "github"
       }
     },
+    "attic": {
+      "inputs": {
+        "crane": "crane",
+        "flake-compat": "flake-compat_2",
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": "nixpkgs",
+        "nixpkgs-stable": "nixpkgs-stable"
+      },
+      "locked": {
+        "lastModified": 1711742460,
+        "narHash": "sha256-0O4v6e4a1toxXZ2gf5INhg4WPE5C5T+SVvsBt+45Mcc=",
+        "owner": "zhaofengli",
+        "repo": "attic",
+        "rev": "4dbdbee45728d8ce5788db6461aaaa89d98081f0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "zhaofengli",
+        "ref": "main",
+        "repo": "attic",
+        "type": "github"
+      }
+    },
     "bats-assert": {
       "flake": false,
       "locked": {
@@ -64,16 +87,16 @@
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1721409873,
+        "lastModified": 1722939563,
-        "narHash": "sha256-h0njWQRvtkjK0NJ/Kgj76sXBhWwq5HGJm7OMcigmNw4=",
+        "narHash": "sha256-lMe8aXgF550iQLRaoU+yn8yYQ4x2qiyqANgsFyjfWwA=",
-        "ref": "refs/heads/refactor",
+        "ref": "refs/heads/non-flakes",
-        "rev": "54bba654d4279dfd112345b6470547851feb1457",
+        "rev": "4a162a8aa5dad6cecdb33bd8534e67e0bdaeb13f",
-        "revCount": 267,
+        "revCount": 295,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
       },
       "original": {
-        "ref": "refs/heads/refactor",
+        "ref": "refs/heads/non-flakes",
         "type": "git",
         "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
       }
@@ -101,6 +124,50 @@
         "type": "github"
       }
     },
+    "crane": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "attic",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1702918879,
+        "narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
+    "crane_2": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1716569590,
+        "narHash": "sha256-5eDbq8TuXFGGO3mqJFzhUbt5zHVTf5zilQoyW5jnJwo=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "109987da061a1bf452f435f1653c47511587d919",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "ref": "master",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
     "darwin": {
       "inputs": {
         "nixpkgs": [
@@ -123,6 +190,29 @@
         "type": "github"
       }
     },
+    "fenix": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "nixpkgs"
+        ],
+        "rust-analyzer-src": "rust-analyzer-src"
+      },
+      "locked": {
+        "lastModified": 1716359173,
+        "narHash": "sha256-pYcjP6Gy7i6jPWrjiWAVV0BCQp+DdmGaI/k65lBb/kM=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "b6fc5035b28e36a98370d0eac44f4ef3fd323df6",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "ref": "main",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
     "flake-compat": {
       "flake": false,
       "locked": {
@@ -140,6 +230,39 @@
       }
     },
     "flake-compat_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_3": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "ref": "master",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_4": {
       "flake": false,
       "locked": {
         "lastModified": 1696426674,
@@ -214,6 +337,40 @@
       }
     },
     "flake-utils_2": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_3": {
+      "inputs": {
+        "systems": "systems_2"
+      },
+      "locked": {
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "ref": "main",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_4": {
       "locked": {
         "lastModified": 1634851050,
         "narHash": "sha256-N83GlSGPJJdcqhUxSCS/WwW5pksYf3VP1M13cDRTSVA=",
@@ -228,6 +385,34 @@
         "type": "github"
       }
     },
+    "grapevine": {
+      "inputs": {
+        "attic": "attic",
+        "crane": "crane_2",
+        "fenix": "fenix",
+        "flake-compat": "flake-compat_3",
+        "flake-utils": "flake-utils_3",
+        "nix-filter": "nix-filter",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "host": "gitlab.computer.surgery",
+        "lastModified": 1721671623,
+        "narHash": "sha256-ELE+AD83jG3zIbYITbSfo6Ykn+R1gVjMHoS5rhDccuY=",
+        "owner": "matrix",
+        "repo": "grapevine-fork",
+        "rev": "dd24a441121b94d389fb46f08c7ec51886d5aa32",
+        "type": "gitlab"
+      },
+      "original": {
+        "host": "gitlab.computer.surgery",
+        "owner": "matrix",
+        "repo": "grapevine-fork",
+        "type": "gitlab"
+      }
+    },
     "home-manager": {
       "inputs": {
         "nixpkgs": [
@@ -258,22 +443,22 @@
         ]
       },
       "locked": {
-        "lastModified": 1721682989,
+        "lastModified": 1722688238,
-        "narHash": "sha256-kjJiZ7m4HKqbZ2mxNQiB32/goKFb8BRi8OqC4wIU0OI=",
+        "narHash": "sha256-x6BnYtArF6IDs7bS8ExokgAQBOlrxXxD0EOBIlASmfM=",
         "ref": "refs/heads/main",
-        "rev": "4b107e6ff36bd89958fba36e0fe0340903e7cd13",
+        "rev": "9b5ac87de73ea4646dbb2af979db91f096d29960",
-        "revCount": 4190,
+        "revCount": 4191,
         "type": "git",
-        "url": "https://git.lix.systems/lix-project/hydra.git"
+        "url": "https://git.lix.systems/the-distro/hydra.git"
       },
       "original": {
         "type": "git",
-        "url": "https://git.lix.systems/lix-project/hydra.git"
+        "url": "https://git.lix.systems/the-distro/hydra.git"
       }
     },
     "lix": {
       "inputs": {
-        "flake-compat": "flake-compat_2",
+        "flake-compat": "flake-compat_4",
         "nix2container": "nix2container",
         "nixpkgs": [
           "hydra",
@@ -324,6 +509,22 @@
         "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
       }
     },
+    "nix-filter": {
+      "locked": {
+        "lastModified": 1710156097,
+        "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "rev": "3342559a24e85fc164b295c3444e8a139924675b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "ref": "main",
+        "repo": "nix-filter",
+        "type": "github"
+      }
+    },
     "nix-gerrit": {
       "inputs": {
         "nixpkgs": [
@@ -384,11 +585,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1721116560,
+        "lastModified": 1711401922,
-        "narHash": "sha256-++TYlGMAJM1Q+0nMVaWBSEvEUjRs7ZGiNQOpqbQApCU=",
+        "narHash": "sha256-QoQqXoj8ClGo0sqD/qWKFWezgEwUL0SUh37/vY2jNhc=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "9355fa86e6f27422963132c2c9aeedb0fb963d93",
+        "rev": "07262b18b97000d16a4bdb003418bd2fb067a932",
         "type": "github"
       },
       "original": {
@@ -414,7 +615,39 @@
         "type": "github"
       }
     },
+    "nixpkgs-stable": {
+      "locked": {
+        "lastModified": 1711460390,
+        "narHash": "sha256-akSgjDZL6pVHEfSE6sz1DNSXuYX6hq+P/1Z5IoYWs7E=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "44733514b72e732bd49f5511bd0203dea9b9a434",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-23.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
     "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1723151389,
+        "narHash": "sha256-9AVY0ReCmSGXHrlx78+1RrqcDgVSRhHUKDVV1LLBy28=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "13fe00cb6c75461901f072ae62b5805baef9f8b2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_3": {
       "locked": {
         "lastModified": 1636823747,
         "narHash": "sha256-oWo1nElRAOZqEf90Yek2ixdHyjD+gqtS/pAgwaQ9UhQ=",
@@ -450,16 +683,34 @@
         "agenix": "agenix",
         "buildbot-nix": "buildbot-nix",
         "colmena": "colmena",
+        "grapevine": "grapevine",
         "hydra": "hydra",
         "lix": [
           "hydra",
           "lix"
         ],
         "nix-gerrit": "nix-gerrit",
-        "nixpkgs": "nixpkgs",
+        "nixpkgs": "nixpkgs_2",
         "terranix": "terranix"
       }
     },
+    "rust-analyzer-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1716107283,
+        "narHash": "sha256-NJgrwLiLGHDrCia5AeIvZUHUY7xYGVryee0/9D3Ir1I=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "21ec8f523812b88418b2bfc64240c62b3dd967bd",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
+        "type": "github"
+      }
+    },
     "stable": {
       "locked": {
         "lastModified": 1696039360,
@@ -491,12 +742,27 @@
         "type": "github"
       }
     },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
     "terranix": {
       "inputs": {
         "bats-assert": "bats-assert",
         "bats-support": "bats-support",
-        "flake-utils": "flake-utils_2",
+        "flake-utils": "flake-utils_4",
-        "nixpkgs": "nixpkgs_2",
+        "nixpkgs": "nixpkgs_3",
         "terranix-examples": "terranix-examples"
       },
       "locked": {

flake.nix

@@ -11,21 +11,35 @@
     colmena.url = "github:zhaofengli/colmena";
     colmena.inputs.nixpkgs.follows = "nixpkgs";
 
-    hydra.url = "git+https://git.lix.systems/lix-project/hydra.git";
+    hydra.url = "git+https://git.lix.systems/the-distro/hydra.git";
     hydra.inputs.nixpkgs.follows = "nixpkgs";
 
     nix-gerrit.url = "git+https://git.lix.systems/the-distro/nix-gerrit.git";
     nix-gerrit.inputs.nixpkgs.follows = "nixpkgs";
 
-    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/refactor";
+    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/non-flakes";
     buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";
 
     lix.follows = "hydra/lix";
+
+    grapevine = {
+      type = "gitlab";
+      host = "gitlab.computer.surgery";
+      owner = "matrix";
+      repo = "grapevine-fork";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
   };
 
   outputs = { self, nixpkgs, terranix, colmena, ... } @ inputs:
   let
-    system = "x86_64-linux";
+    supportedSystems = [ "x86_64-linux" "aarch64-linux" ];
+    forEachSystem = f: builtins.listToAttrs (map (system: {
+      name = system;
+      value = f system;
+    }) supportedSystems);
+    systemBits = forEachSystem (system: rec {
+      inherit system;
       pkgs = import nixpkgs {
         localSystem = system;
         overlays = [
@@ -34,7 +48,6 @@
           inputs.nix-gerrit.overlays.default
         ];
       };
-    lib = pkgs.lib;
       terraform = pkgs.opentofu;
       terraformCfg = terranix.lib.terranixConfiguration {
         inherit system;
@@ -46,9 +59,12 @@
           }
         ];
       };
+    });
+    forEachSystem' = f: forEachSystem (system: (f systemBits.${system}));
+    inherit (nixpkgs) lib;
   in
   {
-    apps.${system} = {
+    apps = forEachSystem' ({ system, pkgs, terraformCfg, terraform, ... }: {
       tf = {
         type = "app";
         program = toString (pkgs.writers.writeBash "tf" ''
@@ -59,16 +75,19 @@
       };
 
       default = self.apps.${system}.tf;
-    };
+    });
 
-    devShells.${system}.default = pkgs.mkShell {
+    devShells = forEachSystem' ({ system, pkgs, ... }: {
+      default = pkgs.mkShell {
         packages = [
           inputs.agenix.packages.${system}.agenix
 
-        pkgs.colmena
           pkgs.opentofu
+
+          (pkgs.callPackage ./lib/colmena-wrapper.nix { })
         ];
       };
+    });
 
     nixosConfigurations = (colmena.lib.makeHive self.outputs.colmena).nodes;
 
@@ -85,19 +104,12 @@
 
       makeBuilder = i: lib.nameValuePair "builder-${toString i}" {
         imports = commonModules;
-        bagel.baremetal.builders = { enable = true; num = i; };
+        bagel.baremetal.builders = { enable = true; num = i; netboot = i >= 6; };
       };
 
       builders = lib.listToAttrs (lib.genList makeBuilder 12);
     in {
-      meta.nixpkgs = import nixpkgs {
+      meta.nixpkgs = systemBits.x86_64-linux.pkgs;
-        localSystem = system;
-        overlays = [
-          inputs.hydra.overlays.default
-          inputs.lix.overlays.default
-          inputs.nix-gerrit.overlays.default
-        ];
-      };
       meta.specialArgs.inputs = inputs;
 
       bagel-box.imports = commonModules ++ [ ./hosts/bagel-box ];
@@ -107,9 +119,10 @@
       git.imports = commonModules ++ [ ./hosts/git ];
       wob-vpn-gw.imports = commonModules ++ [ ./hosts/wob-vpn-gw ];
       buildbot.imports = commonModules ++ [ ./hosts/buildbot ];
+      public01.imports = commonModules ++ [ ./hosts/public01 ];
     } // builders;
 
-    hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.toplevel) self.nixosConfigurations;
+    hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.netbootDir or v.config.system.build.toplevel) self.nixosConfigurations;
     buildbotJobs = builtins.mapAttrs (_: v: v.config.system.build.toplevel) self.nixosConfigurations;
   };
 }

hosts/buildbot/default.nix

@@ -28,7 +28,7 @@
   bagel.services.buildbot = {
     enable = true;
     domain = "buildbot.forkos.org";
-    builders = [ "builder-3" ];
+    builders = [ "builder-11" ];
   };
 
   i18n.defaultLocale = "en_US.UTF-8";

hosts/gerrit01/default.nix

@@ -47,12 +47,13 @@
   };
   bagel.nixpkgs.one-way-sync =
   let
-    mkNixpkgsJob = { timer, branchName }: {
+    mkNixpkgsJob = { timer, fromRefspec, localRefspec ? fromRefspec }: {
-      name = "nixpkgs-${branchName}";
       fromUri = "https://github.com/NixOS/nixpkgs";
-      fromRefspec = branchName;
+      inherit fromRefspec localRefspec timer;
-      localRefspec = branchName;
+    };
-      inherit timer;
+    mkLocalJob = { timer, fromRefspec, localRefspec }: {
+      fromUri = "https://cl.forkos.org/nixpkgs";
+      inherit fromRefspec localRefspec timer;
     };
     in
   {
@@ -61,47 +62,59 @@
     pushUrl = "ssh://ows_bot@cl.forkos.org:29418/nixpkgs";
     deployKeyPath = config.age.secrets.ows-deploy-key.path;
 
-    branches."refs/heads/main" = mkNixpkgsJob {
+    # Sync main -> staging-next -> staging
-      timer = "hourly";
+    branches."main-to-staging-next" = mkLocalJob {
-      branchName = "main";
+      timer = "00/8:20:00"; # every 8 hours, 20 minutes past the full hour
+      fromRefspec = "main";
+      localRefspec = "staging-next";
+    };
+    branches."staging-next-to-staging" = mkLocalJob {
+      timer = "00/8:40:00"; # every 8 hours, 40 minutes past the full hour
+      fromRefspec = "staging-next";
+      localRefspec = "staging";
     };
 
-    branches."refs/heads/staging" = mkNixpkgsJob {
+    # Sync nixpkgs -> fork
+    branches."nixpkgs-master" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "staging";
+      fromRefspec = "master";
+      localRefspec = "main";
     };
 
-    branches."refs/heads/release-24.05" = mkNixpkgsJob {
+    branches."nixpkgs-staging" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "release-24.05";
+      fromRefspec = "staging";
     };
 
-    branches."refs/heads/staging-24.05" = mkNixpkgsJob {
+    branches."nixpkgs-release-24.05" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "staging-24.05";
+      fromRefspec = "release-24.05";
     };
 
-    branches."refs/heads/release-23.11" = mkNixpkgsJob {
+    branches."nixpkgs-staging-24.05" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "release-23.11";
+      fromRefspec = "staging-24.05";
     };
 
-    branches."refs/heads/staging-23.11" = mkNixpkgsJob {
+    branches."nixpkgs-release-23.11" = mkNixpkgsJob {
       timer = "hourly";
-      branchName = "staging-23.11";
+      fromRefspec = "release-23.11";
+    };
+
+    branches."nixpkgs-staging-23.11" = mkNixpkgsJob {
+      timer = "hourly";
+      fromRefspec = "staging-23.11";
     };
 
     # Testing jobs for personal sandbox branches
-    branches."refs/heads/sandbox/raito/raito-unstable-small" = {
+    branches."raito-unstable-sync" = {
-      name = "raito-unstable-sync";
       fromUri = "https://github.com/NixOS/nixpkgs";
       fromRefspec = "nixos-unstable-small";
       localRefspec = "sandbox/raito/raito-unstable-small";
       timer = "*-*-* 12:00:00";
     };
 
-    branches."refs/heads/sandbox/raito/raito-nixos-24.05" = {
+    branches."raito-release-sync" = {
-      name = "raito-release-sync";
       fromUri = "https://github.com/NixOS/nixpkgs";
       fromRefspec = "nixos-24.05";
       localRefspec = "sandbox/raito/raito-nixos-24.05";

hosts/meta01/default.nix

@@ -24,6 +24,13 @@
   bagel.services.prometheus.enable = true;
   bagel.services.loki.enable = true;
   bagel.services.grafana.enable = true;
+  bagel.services.grapevine.enable = true;
+  bagel.services.hookshot = {
+    enable = true;
+    admins = [
+      "@k900:0upti.me"
+    ];
+  };
 
   i18n.defaultLocale = "fr_FR.UTF-8";
 

hosts/public01/default.nix

@@ -0,0 +1,32 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  networking.hostName = "public01";
+  # TODO: make it the default
+  networking.domain = "infra.forkos.org";
+
+  time.timeZone = "Europe/Paris";
+
+  bagel.sysadmin.enable = true;
+  # Buildbot is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.hardware.raito-vm = {
+    enable = true;
+    networking = {
+      nat-lan-mac = "BC:24:11:A4:F7:D3";
+      wan = {
+        address = "2001:bc8:38ee:100:1000::60/64";
+        mac = "BC:24:11:DB:B8:10";
+      };
+    };
+  };
+
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "public01.infra.forkos.org";
+}

hosts/wob-vpn-gw/default.nix

@@ -1,6 +1,10 @@
 { pkgs, lib, ... }:
 
 {
+  imports = [
+    ./netboot.nix
+  ];
+
   ###### Hardware ######
   boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "ehci_pci" "sd_mod" "sdhci_pci" ];
   boot.kernelModules = [ "kvm-amd" ];

hosts/wob-vpn-gw/netboot.nix

@@ -0,0 +1,63 @@
+{ config, lib, pkgs, nodes, modulesPath, ... }:
+
+# The way the connection is established is specific to the wob01 site and the Intel S2600KPR blades.
+# Proper netboot is not possible, because while the blades and the APU board (which is the netboot
+# server here) are in the same L2 network, the uplink connection of each blade is an LACP LAG,
+# meaning that the switch on the other side will only enable the port if it sees valid LACP packets.
+# We work around this by presenting a virtual floppy drive using the "IUSB" protocol of the BMC.
+# This virtual floppy drive contains an per-blade customized initramfs which will initialize the
+# network connection including IP configuration and load the actual image off hydra.
+
+let
+  netboot-server-ip = "2a01:584:11::2";
+  netbootNodes = lib.filterAttrs (_: node: node.config.bagel.baremetal.builders.enable && node.config.bagel.baremetal.builders.netboot) nodes;
+in {
+
+  assertions = [
+    {
+      assertion = !(lib.elem 443 config.networking.firewall.allowedTCPPorts);
+      message = ''
+        Port 443 is in networking.firewalls.allowedTCPPorts, but should be only manually
+        allowed for specific IPs and source ports in ${builtins.toJSON __curPos}
+      '';
+    }
+  ];
+
+  systemd.services = lib.mapAttrs' (nodename: node: let
+    bmcIp = "192.168.1.${toString (node.config.bagel.baremetal.builders.num * 4 + 2)}";
+    notipxe = node.config.system.build.notipxe.config.system.build.usbImage;
+  in lib.nameValuePair "iusb-spoof-${nodename}" {
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Restart = "always";
+    };
+    script = ''
+      AUTH_TOKEN=$(${pkgs.iusb-spoof}/bin/make-token ${bmcIp})
+      exec ${pkgs.iusb-spoof}/bin/iusb-spoof -r ${bmcIp} 5123 $AUTH_TOKEN ${notipxe}
+    '';
+  }) netbootNodes;
+
+  # Since the builders are stateless, they can not store their ssh hostkeys
+  networking.firewall.allowedTCPPorts = [ 80 ]; # for ACME
+  networking.firewall.extraInputRules = ''
+    ip6 saddr 2a01:584:11::/64 tcp sport < 1024 tcp dport 443 accept;
+  '';
+  security.acme.acceptTerms = true;
+  security.acme.defaults.email = "infra@forkos.org";
+  services.nginx = {
+    enable = true;
+    virtualHosts."vpn-gw.wob01.infra.forkos.org" = {
+      enableACME = true;
+      forceSSL = true;
+      locations = lib.mapAttrs' (nodename: node: let
+        ip = "2a01:584:11::1:${toString node.config.bagel.baremetal.builders.num}";
+      in lib.nameValuePair "/${nodename}/" {
+        root = "/var/www";
+        extraConfig = ''
+          allow ${ip};
+          deny all;
+        '';
+      }) netbootNodes;
+    };
+  };
+}

lib/colmena-wrapper.nix

@@ -0,0 +1,14 @@
+# A wrapper for colmena that prevents accidentally deploying changes without
+# having pulled.
+{ colmena, runCommandNoCC }:
+runCommandNoCC "colmena-wrapper"
+{
+  env.colmena = "${colmena}/bin/colmena";
+} ''
+  mkdir -p $out
+  ln -s ${colmena}/share $out/share
+  mkdir $out/bin
+
+  substituteAll ${./colmena-wrapper.sh.in} $out/bin/colmena
+  chmod +x $out/bin/colmena
+''

lib/colmena-wrapper.sh.in

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+doChecks() {
+    # creates refs in the refs/prefetch/remotes/origin namespace
+    echo "Prefetching repo changes..." >&2
+    git fetch --quiet --prefetch --no-write-fetch-head origin
+
+    diffs=$(git rev-list --left-right --count HEAD...refs/prefetch/remotes/origin/main)
+    only_in_local=$(echo "$diffs" | cut -f1)
+    only_in_main=$(echo "$diffs" | cut -f2)
+
+    if [[ $only_in_main -gt 0 && ! -v $FOOTGUN_ME_UWU ]]; then
+        echo >&2
+        echo "Attempting to deploy when main has $only_in_main commits not in your branch!" >&2
+        echo "This will probably revert someone's changes. Consider merging them." >&2
+        echo "If you really mean it, set the environment variable FOOTGUN_ME_UWU" >&2
+        exit 1
+    fi
+
+    if [[ $only_in_local -gt 0 ]]; then
+        echo "You have $only_in_local commits not yet pushed to main. Reminder to push them after :)" >&2
+    fi
+}
+
+if [[ $1 == 'apply' ]]; then
+    doChecks
+fi
+
+exec @colmena@ "$@"

overlays/default.nix

@@ -1 +1,6 @@
-[]
+[
+  (final: prev: {
+    iusb-spoof = final.callPackage ./iusb-spoof.nix {};
+    u-root = final.callPackage ./u-root {};
+  })
+]

overlays/iusb-spoof.nix

@@ -0,0 +1,23 @@
+{ rustPlatform, python3, makeWrapper }:
+let
+  pythonEnv = python3.withPackages (p: with p; [ requests ]);
+in
+
+rustPlatform.buildRustPackage rec {
+  pname = "iusb-spoof";
+  version = "0.1.0";
+
+  src = builtins.fetchGit {
+    url = "https://git.lix.systems/the-distro/iusb-spoof/";
+    rev = "fafd47986239cc2f4dfbbae74b17555608806581";
+  };
+
+  cargoLock.lockFile = src + "/Cargo.lock";
+
+  nativeBuildInputs = [ makeWrapper ];
+
+  postInstall = ''
+    install -Dm644 $src/make-token.py $out/opt/make-token.py
+    makeWrapper ${pythonEnv.interpreter} $out/bin/make-token --add-flags "$out/opt/make-token.py"
+  '';
+}

overlays/u-root/default.nix

@@ -0,0 +1,20 @@
+{ buildGoModule, fetchFromGitHub }:
+
+buildGoModule rec {
+  pname = "u-root";
+  version = "0.14.0";
+
+  src = fetchFromGitHub {
+    owner = "u-root";
+    repo = "u-root";
+    rev = "v${version}";
+    hash = "sha256-8zA3pHf45MdUcq/MA/mf0KCTxB1viHieU/oigYwIPgo=";
+  };
+
+  patches = [
+    ./u-root-allow-https.patch
+  ];
+
+  vendorHash = null;
+  doCheck = false;
+}

overlays/u-root/u-root-allow-https.patch

@@ -0,0 +1,12 @@
+diff --git a/pkg/curl/schemes.go b/pkg/curl/schemes.go
+index 8bac3bc0..cd396cbc 100644
+--- a/pkg/curl/schemes.go
++++ b/pkg/curl/schemes.go
+@@ -81,6 +81,7 @@ var (
+ 	DefaultSchemes = Schemes{
+ 		"tftp": DefaultTFTPClient,
+ 		"http": DefaultHTTPClient,
++		"https": DefaultHTTPClient,
+ 		"file": &LocalFileClient{},
+ 	}
+ )

secrets.nix

@@ -9,6 +9,7 @@ let
     hydra-ssh-key-priv = [ machines.bagel-box ];
     netbox-environment = [ machines.meta01 ];
     mimir-environment = [ machines.meta01 ];
+    mimir-webhook-url = [ machines.meta01 ];
     grafana-oauth-secret = [ machines.meta01 ];
     loki-environment = [ machines.meta01 ];
     gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];

services/baremetal-builder/assignments.nix

@@ -0,0 +1,29 @@
+# This file contains information on which builder(s) are providing how many
+# job slots and providing which nix features
+let
+  genBuilders = { offset ? 0, count, f }: builtins.genList (x: rec { name = "builder-${toString (offset + x)}"; value = f name; }) count;
+in builtins.listToAttrs (
+  # The first 8 builders are general purpose hydra builders
+  genBuilders { count = 8; f = name: {
+    cores = 8;
+    max-jobs = 8;
+    supported-features = [ "kvm" "nixos-test" ];
+    required-features = [ ];
+  }; }
+  ++
+  # The last 2 builders are exclusively for big-parallel
+  genBuilders { offset = 8; count = 2; f = name: {
+    cores = 20;
+    max-jobs = 1;
+    supported-features = [ "kvm" "nixos-test" "big-parallel" ];
+    required-features = [ "big-parallel" ];
+  }; }
+  ++
+  # These are not currently used for hydra
+  genBuilders { offset = 10; count = 2; f = name: {
+    cores = 8;
+    max-jobs = 8;
+    supported-features = [ "kvm" "nixos-test" "big-parallel" ];
+    required-features = [ ];
+  }; }
+)

services/baremetal-builder/default.nix

@@ -3,10 +3,13 @@ let
   cfg = config.bagel.baremetal.builders;
 in
 {
+  imports = [ ./netboot.nix ];
+
   options = {
 
     bagel.baremetal.builders = {
       enable = lib.mkEnableOption "baremetal bagel oven";
+      netboot = lib.mkEnableOption "netboot";
       num = lib.mkOption {
         type = lib.types.int;
       };
@@ -40,8 +43,10 @@ in
         "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMnOLLX0vGTZbSJrUmF9ZFXt/NIId/MUrEpXmL2vxod"
       ];
     };
-    nix.settings.trusted-users = [ "builder" "buildbot" ];
+    nix.settings = {
-
+      trusted-users = [ "builder" "buildbot" ];
+      inherit ((import ./assignments.nix).${config.networking.hostName}) max-jobs cores;
+    };
 
     nixpkgs.hostPlatform = "x86_64-linux";
     hardware.cpu.intel.updateMicrocode = true;
@@ -52,28 +57,36 @@ in
 
     boot.initrd.services.lvm.enable = true;
 
-    fileSystems."/" = {
+    boot.kernel.sysctl."fs.xfs.xfssyncd_centisecs" = "12000";
+    fileSystems = lib.mkMerge [
+      (lib.mkIf (!cfg.netboot) {
+        "/" = {
           device = "/dev/disk/by-label/root";
           fsType = "xfs";
         };
 
-    fileSystems."/mnt" = {
+        "/boot" = {
-      device = "/dev/disk/by-label/hydra";
-      fsType = "xfs";
-    };
-    # We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
-    fileSystems."/tmp" = {
-      device = "/mnt/tmp";
-      options = [ "bind" ];
-    };
-
-    fileSystems."/boot" = {
           device = "/dev/disk/by-label/BOOT";
           fsType = "vfat";
           options = [ "fmask=0022" "dmask=0022" ];
         };
+      })
+      {
+        "/mnt" = {
+          device = "/dev/disk/by-label/hydra";
+          fsType = "xfs";
+          options = ["logbsize=256k"];
+        };
 
-    swapDevices = [
+        # We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
+        "/tmp" = {
+          device = "/mnt/tmp";
+          options = [ "bind" ];
+        };
+      }
+    ];
+
+    swapDevices = lib.optionals (!cfg.netboot) [
       {
         device = "/swapfile";
         size = 50 * 1024;  # 50GiB
@@ -86,8 +99,8 @@ in
     };
 
     boot.kernelParams = [
-      "console=ttyS0,115200"
       "console=tty1"
+      "console=ttyS0,115200"
     ];
 
     networking.useNetworkd = true;
@@ -146,11 +159,24 @@ in
     };
 
     systemd.services.hydra-gc = {
+      wantedBy = [ "multi-user.target" ];
       description = "Nix Garbage Collector";
-      script = "exec ${config.nix.package.out}/bin/nix-store --gc --store /mnt";
+      script = ''
+        while : ; do
+          percent_filled=$(($(stat -f --format="100-(100*%a/%b)" /mnt)))
+          if [ "$percent_filled" -gt "54" ]; then
+            ${config.nix.package.out}/bin/nix-store --gc --max-freed 50G --store /mnt
+          else
+            break
+          fi
+        done
+      '';
       serviceConfig.Type = "oneshot";
       serviceConfig.User = "builder";
-      startAt = "*-*-* 00/8:00:00";
+    };
+    systemd.timers.hydra-gc = {
+      timerConfig.OnUnitInactiveSec = "10min";
+      wantedBy = [ "timers.target" ];
     };
     systemd.timers.hydra-gc.timerConfig.Persistent = true;
 

services/baremetal-builder/netboot.nix

@@ -0,0 +1,169 @@
+{ modulesPath, pkgs, lib, config, extendModules, ... }@node:
+let
+  cfg = config.bagel.baremetal.builders;
+in
+{
+  config = lib.mkIf (cfg.enable && cfg.netboot) {
+    systemd.services.sshd.after = [ "provision-ssh-hostkey.service" ];
+    systemd.services.provision-ssh-hostkey = {
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+      script = ''
+        mkdir -p /etc/ssh
+        umask 0077
+        until ${pkgs.iputils}/bin/ping -c 1 vpn-gw.wob01.infra.forkos.org; do sleep 1; done
+        ${pkgs.curl}/bin/curl --local-port 25-1024 https://vpn-gw.wob01.infra.forkos.org/${config.networking.hostName}/ssh_host_ed25519_key > /etc/ssh/ssh_host_ed25519_key
+        # Run the activation script again to trigger agenix decryption
+        /run/current-system/activate
+      '';
+    };
+
+    system.build = {
+
+      # Build a kernel and initramfs which will download the IPXE script from hydra using
+      # u-root pxeboot tool and kexec into the final netbooted system.
+      notipxe = import (modulesPath + "/..") {
+        system = "x86_64-linux";
+        configuration = 
+          { pkgs, config, ... }:
+
+          {
+            system.stateVersion = "24.11";
+            boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" "igb" "bonding" ];
+            boot.kernelParams = [ "console=ttyS0,115200" "panic=1" "boot.panic_on_fail" ];
+            #boot.initrd.systemd.emergencyAccess = true;
+            networking.hostName = "${node.config.networking.hostName}-boot";
+            nixpkgs.overlays = import ../../overlays;
+            boot.loader.grub.enable = false;
+            fileSystems."/".device = "bogus"; # this config will never be booted
+            boot.initrd.systemd.enable = true;
+            boot.initrd.systemd.network = {
+              enable = true;
+              networks = node.config.systemd.network.networks;
+              netdevs = node.config.systemd.network.netdevs;
+            };
+            boot.initrd.systemd.storePaths = [
+              "${pkgs.u-root}/bin/pxeboot"
+              "${pkgs.iputils}/bin/ping"
+            ];
+            boot.initrd.systemd.services.kexec = {
+              serviceConfig.Restart = "on-failure";
+              serviceConfig.Type = "oneshot";
+              wantedBy = [ "initrd-root-fs.target" ];
+              before = [ "sysroot.mount" ];
+              script = ''
+                ln -sf /dev/console /dev/tty
+                until ${pkgs.iputils}/bin/ping -c 1 hydra.forkos.org; do sleep 1; done
+                ${pkgs.u-root}/bin/pxeboot -v -ipv4=false -file https://hydra.forkos.org/job/infra/main/${node.config.networking.hostName}/latest/download-by-type/file/ipxe 
+              '';
+            };
+            boot.initrd.systemd.contents."/etc/ssl/certs/ca-certificates.crt".source = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+            boot.initrd.services.resolved.enable = false;
+            boot.initrd.systemd.contents."/etc/resolv.conf".text = ''
+              nameserver 2001:4860:4860::6464
+            '';
+            boot.initrd.systemd.contents."/etc/systemd/journald.conf".text = ''
+              [Journal]
+              ForwardToConsole=yes
+              MaxLevelConsole=debug
+            '';
+
+            # Provide a bootable USB drive image
+            system.build.usbImage = pkgs.callPackage ({ stdenv, runCommand, dosfstools, e2fsprogs, mtools, libfaketime, util-linux, nukeReferences }:
+              runCommand "boot-img-${node.config.networking.hostName}" {
+                nativeBuildInputs = [ dosfstools e2fsprogs libfaketime mtools util-linux ];
+                outputs = [ "out" "firmware_part" ];
+              } ''
+                export img=$out
+                truncate -s 40M $img
+
+                sfdisk $img <<EOF
+                    label: gpt
+                    label-id: F222513B-DED1-49FA-B591-20CE86A2FE7F
+
+                    type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B, bootable
+                EOF
+
+                # Create a FAT32 /boot/firmware partition of suitable size into firmware_part.img
+                eval $(partx $img -o START,SECTORS --nr 1 --pairs)
+                truncate -s $((2081 * 512 + SECTORS * 512)) firmware_part.img
+
+                mkfs.vfat --invariant -i 2e24ec82 -n BOOT firmware_part.img
+
+                # Populate the files intended for /boot/firmware
+                mkdir -p firmware/EFI/BOOT firmware/loader/entries
+                cp ${pkgs.systemd}/lib/systemd/boot/efi/systemd-boot*.efi firmware/EFI/BOOT/BOOT${lib.toUpper stdenv.hostPlatform.efiArch}.EFI
+
+                cat > firmware/loader/loader.conf << EOF
+                default foo
+                EOF
+                cat > firmware/loader/entries/default.conf << EOF
+                title Default
+                linux /EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target}
+                initrd /EFI/initrd
+                options init=${config.system.build.toplevel}/init ${toString config.boot.kernelParams}
+                EOF
+                cp ${config.system.build.kernel}/${pkgs.stdenv.hostPlatform.linux-kernel.target} firmware/EFI/${pkgs.stdenv.hostPlatform.linux-kernel.target}
+                cp ${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile} firmware/EFI/initrd
+
+                find firmware -exec touch --date=2000-01-01 {} +
+                # Copy the populated /boot/firmware into the SD image
+                cd firmware
+                # Force a fixed order in mcopy for better determinism, and avoid file globbing
+                for d in $(find . -type d -mindepth 1 | sort); do
+                  faketime "2000-01-01 00:00:00" mmd -i ../firmware_part.img "::/$d"
+                done
+                for f in $(find . -type f | sort); do
+                  mcopy -pvm -i ../firmware_part.img "$f" "::/$f"
+                done
+                cd ..
+
+                # Verify the FAT partition before copying it.
+                fsck.vfat -vn firmware_part.img
+                dd conv=notrunc if=firmware_part.img of=$img seek=$START count=$SECTORS
+
+                cp firmware_part.img $firmware_part
+              ''
+            ) {};
+          }
+        ;
+      };
+
+      # This is the config which will actually be booted
+      netbootVariant = extendModules {
+        modules = [
+          (
+            { modulesPath, ... }:
+
+            {
+              imports = [ (modulesPath + "/installer/netboot/netboot.nix") ];
+            }
+          )
+        ];
+      };
+      # A derivation combining all the artifacts required for netbooting for the hydra job
+      netbootDir = let
+        kernelTarget = pkgs.stdenv.hostPlatform.linux-kernel.target;
+        build = config.system.build.netbootVariant.config.system.build;
+      in
+        pkgs.symlinkJoin {
+          name = "netboot";
+          paths = [
+            build.netbootRamdisk
+            build.kernel
+            build.netbootIpxeScript
+          ];
+          postBuild = ''
+            mkdir -p $out/nix-support
+            echo "file ${kernelTarget} $out/${kernelTarget}" >> $out/nix-support/hydra-build-products
+            echo "file initrd $out/initrd" >> $out/nix-support/hydra-build-products
+            echo "file ipxe $out/netboot.ipxe" >> $out/nix-support/hydra-build-products
+          '';
+          preferLocalBuild = true;
+        };
+    };
+  };
+}

services/buildbot/default.nix

@@ -31,8 +31,16 @@ in
     age.secrets.buildbot-oauth-secret.file = ../../secrets/buildbot-oauth-secret.age;
     age.secrets.buildbot-workers.file = ../../secrets/buildbot-workers.age;
     age.secrets.buildbot-service-key.file = ../../secrets/buildbot-service-key.age;
-    age.secrets.buildbot-signing-key.file = ../../secrets/buildbot-signing-key.age;
+    age.secrets.buildbot-signing-key = {
-    age.secrets.buildbot-remote-builder-key.file = ../../secrets/buildbot-remote-builder-key.age;
+      file = ../../secrets/buildbot-signing-key.age;
+      owner = "buildbot-worker";
+      group = "buildbot-worker";
+    };
+    age.secrets.buildbot-remote-builder-key = {
+      file = ../../secrets/buildbot-remote-builder-key.age;
+      owner = "buildbot-worker";
+      group = "buildbot-worker";
+    };
 
     services.nginx.virtualHosts.${cfg.domain} = {
       forceSSL = true;
@@ -58,7 +66,7 @@ in
             (_: lib.foldl' lib.add 0)
             (lib.concatMap
               (m: map (s: { ${s} = m.maxJobs; }) m.systems)
-              config.nix.buildMachines))
+              config.services.buildbot-nix.coordinator.buildMachines))
         );
     };
 
@@ -67,6 +75,8 @@ in
 
       inherit (cfg) domain;
 
+      debugging.enable = true;
+
       oauth2 = {
         name = "Lix";
         clientId = "forkos-buildbot";

services/default.nix

@@ -2,6 +2,7 @@
   imports = [
     ./gerrit
     ./hydra
+    ./matrix
     ./monitoring
     ./netbox
     ./ofborg

services/forgejo/default.nix

@@ -55,6 +55,10 @@ in
           DEFAULT_KEEP_EMAIL_PRIVATE = true;
         };
 
+        "service.explore" = {
+          DISABLE_USERS_PAGE = true;
+        };
+
         oauth2_client = {
           REGISTER_EMAIL_CONFIRM = false;
           ENABLE_AUTO_REGISTRATION = true;

services/gerrit/one-way-sync.nix

@@ -3,7 +3,7 @@ let
   cfg = config.bagel.nixpkgs.one-way-sync;
   inherit (lib) mkIf mkOption mkEnableOption types mapAttrs';
 
-  mkSyncTimer = { name, timer, ... }: {
+  mkSyncTimer = name: { timer, ... }: {
     wantedBy = [ "timers.target" ];
 
     timerConfig = {
@@ -12,7 +12,7 @@ let
       Unit = "ows-${name}.service";
     };
   };
-  mkSyncService = targetRef: { name, fromUri, fromRefspec, localRefspec, ... }: {
+  mkSyncService = name: { fromUri, fromRefspec, localRefspec, ... }: {
     path = [ pkgs.gitFull pkgs.openssh pkgs.lix ];
     script = ''
       set -xe
@@ -25,11 +25,11 @@ let
       fi
 
       cd /var/lib/onewaysync/nixpkgs
-      echo "Syncing ${fromUri}:${fromRefspec} to /var/lib/onewaysync/nixpkgs:${targetRef}"
+      echo "Syncing ${fromUri}:${fromRefspec} to ${cfg.pushUrl}:refs/heads/${localRefspec}"
       echo "Current ref: $EXPECTED_REF"
       git worktree add -f "$RUNTIME_DIRECTORY"/${name} refs/remotes/origin/${localRefspec}
       cd "$RUNTIME_DIRECTORY"/${name}
-      git pull origin ${localRefspec}
+      git pull origin ${localRefspec} --no-rebase
       EXPECTED_REF=$(git rev-list refs/remotes/origin/${localRefspec} | head -1)
       git config user.name Fork-o-Tron
       git config user.email noreply@forkos.org
@@ -43,7 +43,7 @@ let
       # Do not allow auto-merging a staging iteration
       test "$OLD_STDENV" = "$NEW_STDENV"
     '' + ''
-      GIT_SSH_COMMAND='ssh -i ${cfg.deployKeyPath}' git push ${cfg.pushUrl} HEAD:${targetRef}
+      GIT_SSH_COMMAND='ssh -i ${cfg.deployKeyPath}' git push ${cfg.pushUrl} HEAD:refs/heads/${localRefspec}
     '';
     serviceConfig = {
       User = "git";
@@ -120,12 +120,12 @@ in
 
   config = mkIf cfg.enable {
     systemd.timers = mapAttrs' (name: value: {
-      name = "ows-${value.name}";
+      name = "ows-${name}";
-      value = mkSyncTimer value;
+      value = mkSyncTimer name value;
     }) cfg.branches;
 
     systemd.services = mapAttrs' (name: value: {
-      name = "ows-${value.name}";
+      name = "ows-${name}";
       value = mkSyncService name value;
     }) cfg.branches;
   };

services/hydra/default.nix

@@ -13,15 +13,36 @@ let
 
   # XXX: to support Nix's dumb public host key syntax (base64'd), this outputs
   # a string with shell-style command interpolations: $(...).
-  mkBaremetalBuilder = { parallelBuilds, publicHostKey, host, speedFactor ? 1, user ? "builder", supportedSystems ? [ "i686-linux" "x86_64-linux" ], supportedFeatures ? [ "big-parallel" "kvm" "nixos-test" ] }:
+  mkBaremetalBuilder = {
-  "ssh://${user}@${host}?remote-store=/mnt ${lib.concatStringsSep "," supportedSystems} ${config.age.secrets.hydra-ssh-key-priv.path} ${toString parallelBuilds} ${toString speedFactor} ${lib.concatStringsSep "," supportedFeatures} - $(echo -n '${publicHostKey}' | base64 -w0)";
+    parallelBuilds,
+    publicHostKey,
+    host,
+    speedFactor ? 1,
+    user ? "builder",
+    supportedSystems ? [ "i686-linux" "x86_64-linux" ],
+    supportedFeatures ? [ "big-parallel" "kvm" "nixos-test" ],
+    requiredFeatures ? [ ]
+  }:
+  let
+    supportedFeatures_ = if (supportedFeatures != []) then lib.concatStringsSep "," supportedFeatures else "-";
+    requiredFeatures_ = if (requiredFeatures != []) then lib.concatStringsSep "," requiredFeatures else "-";
+  in
+  "ssh://${user}@${host}?remote-store=/mnt ${lib.concatStringsSep "," supportedSystems} ${config.age.secrets.hydra-ssh-key-priv.path} ${toString parallelBuilds} ${toString speedFactor} ${supportedFeatures_} ${requiredFeatures_} $(echo -n '${publicHostKey}' | base64 -w0)";
 
   # TODO:
   # - generalize to new architectures
   # - generalize to new features
   baremetalBuilders = lib.concatStringsSep "\n"
-    (map (n: mkBaremetalBuilder {
+    (map (n: let
-      parallelBuilds = 8; # TODO: do not hardcode this, use the node's builder configuration.
+      assignments = (import ../baremetal-builder/assignments.nix).${n} or {
+        inherit (nodes.${n}.config.nix.settings) max-jobs;
+        supported-features = [ "big-parallel" "kvm" "nixos-test" ];
+        required-features = [];
+      };
+    in mkBaremetalBuilder {
+      parallelBuilds = assignments.max-jobs;
+      supportedFeatures = assignments.supported-features;
+      requiredFeatures = assignments.required-features;
       publicHostKey = ssh-keys.machines.${n};
       host = nodes.${n}.config.networking.fqdn;
     }) cfg.builders);

services/matrix/default.nix

@@ -0,0 +1,68 @@
+{
+  config,
+  lib,
+  inputs,
+  ...
+}:
+
+let
+  cfg = config.bagel.services.grapevine;
+  inherit (lib) mkEnableOption mkIf;
+in
+
+{
+  imports = [
+    inputs.grapevine.nixosModules.default
+    ./hookshot.nix
+  ];
+
+  options.bagel.services.grapevine.enable = mkEnableOption "Grapevine";
+
+  config = mkIf cfg.enable {
+    services = {
+      grapevine = {
+        enable = true;
+        settings = {
+          listen = [
+            {
+              type = "tcp";
+              address = "127.0.0.1";
+              port = 6167;
+            }
+          ];
+          server_name = "forkos.org";
+          database.backend = "rocksdb";
+        };
+      };
+
+      nginx = {
+        upstreams.grapevine.servers."127.0.0.1:6167" = { };
+
+        virtualHosts = {
+          "matrix.forkos.org" = {
+            forceSSL = true;
+            enableACME = true;
+            locations."/".proxyPass = "http://grapevine";
+          };
+
+          "forkos.org" = {
+            forceSSL = true;
+            enableACME = true;
+            locations = {
+              "= /.well-known/matrix/server".extraConfig = ''
+                add_header Content-Type application/json;
+                add_header Access-Control-Allow-Origin *;
+                return 200 '{"m.server": "matrix.forkos.org:443"}';
+              '';
+              "= /.well-known/matrix/client".extraConfig = ''
+                add_header Content-Type application/json;
+                add_header Access-Control-Allow-Origin *;
+                return 200 '{"m.homeserver": {"base_url": "https://matrix.forkos.org/"}, "m.identity_server": {"base_url": "https://matrix.org/"}, "org.matrix.msc3575.proxy": {"url": "https://matrix.forkos.org"}}';
+              '';
+            };
+          };
+        };
+      };
+    };
+  };
+}

services/matrix/hookshot.nix

@@ -0,0 +1,77 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.bagel.services.hookshot;
+  inherit (lib) mkEnableOption mkIf mkOption types;
+  keyPath = "/var/lib/matrix-hookshot/key.pem";
+in
+{
+  options.bagel.services.hookshot = {
+    enable = mkEnableOption "matrix-hookshot";
+    settings = mkOption {
+      description = "Settings";
+      type = (pkgs.formats.yaml { }).type;
+    };
+    admins = mkOption {
+      description = "List of admin MXIDs";
+      type = types.listOf types.str;
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.matrix-hookshot = {
+      wantedBy = ["multi-user.target"];
+      wants = ["network-online.target"];
+      after = ["network-online.target"];
+      serviceConfig = {
+        ExecStart = "${lib.getExe pkgs.matrix-hookshot} ${pkgs.writers.writeYAML "config.yaml" cfg.settings}";
+        ExecStartPre = pkgs.writeShellScript "hookshot-generate-key" ''
+          if [ ! -f ${keyPath} ]; then
+            mkdir -p $(dirname ${keyPath})
+            ${lib.getExe pkgs.openssl} genpkey -out ${keyPath} -outform PEM -algorithm RSA -pkeyopt rsa_keygen_bits:4096
+          fi
+        '';
+        DynamicUser = true;
+        StateDirectory = "matrix-hookshot";
+        WorkingDirectory = "/var/lib/matrix-hookshot";
+      };
+    };
+
+    bagel.services.hookshot.settings = {
+      bridge = {
+        domain = "forkos.org";
+        url = "https://matrix.forkos.org";
+        mediaUrl = "https://forkos.org";
+        port = 9993;
+        bindAddress = "127.0.0.1";
+      };
+      passFile = keyPath;
+      listeners = [{
+        port = 9994;
+        bindAddress = "127.0.0.1";
+        resources = [ "webhooks" ];
+      }];
+      generic = {
+        enabled = true;
+        urlPrefix = "https://alerts.forkos.org/webhook";
+      };
+      permissions = map (mxid: {
+        actor = mxid;
+        services = [{
+          service = "*";
+          level = "admin";
+        }];
+      }) cfg.admins;
+    };
+
+    services.nginx.virtualHosts."alerts.forkos.org" = {
+      forceSSL = true;
+      enableACME = true;
+      locations."/".proxyPass = "http://127.0.0.1:9994";
+    };
+  };
+}

services/monitoring/default.nix

@@ -3,5 +3,6 @@
     ./exporters
     ./lgtm
     ./agent.nix
+    ./hookshot-adapter
   ];
 }

services/monitoring/hookshot-adapter/default.nix

@@ -0,0 +1,30 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.bagel.services.alertmanager-hookshot-adapter;
+  inherit (lib) mkEnableOption mkIf;
+  package = pkgs.callPackage ./package.nix {};
+in
+{
+  options.bagel.services.alertmanager-hookshot-adapter.enable = mkEnableOption "alertmanager to matrix-hookshot adapter";
+
+  config = mkIf cfg.enable {
+    systemd.services.alertmanager-hookshot-adapter = {
+      wantedBy = ["multi-user.target"];
+      wants = ["network-online.target"];
+      after = ["network-online.target"];
+      environment = {
+        PORT = "9100";
+        UPSTREAM = "https://alerts.forkos.org/webhook";
+      };
+      serviceConfig = {
+        ExecStart = lib.getExe package;
+        DynamicUser = true;
+      };
+    };
+  };
+}

services/monitoring/hookshot-adapter/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "alertmanager-hookshot-adapter",
+  "version": "1.0.0",
+  "description": "Adapter between alertmanager webhooks and the Matrix Hookshot Apapter",
+  "main": "index.ts",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/hm-edu/alertmanager-hookshot-adapter"
+  },
+  "dependencies": {
+    "@types/express": "^4.17.21",
+    "@types/node": "^20.11.20",
+    "dotenv": "^16.4.5",
+    "express": "^4.18.2",
+    "node-fetch": "^3.3.2",
+    "typescript": "^5.3.3",
+    "winston": "^3.13.0"
+  },
+  "scripts": {
+    "build": "npx tsc"
+  }
+}

services/monitoring/hookshot-adapter/package.nix

@@ -0,0 +1,40 @@
+{
+  lib,
+  mkYarnPackage,
+  fetchFromGitHub,
+  fetchYarnDeps,
+  makeWrapper,
+  nodejs,
+}:
+
+mkYarnPackage rec {
+  pname = "alertmanager-hookshot-adapter";
+  version = "1.9.1";
+
+  src = fetchFromGitHub {
+    owner = "hm-edu";
+    repo = "alertmanager-hookshot-adapter";
+    rev = "v${version}";
+    hash = "sha256-KTk70zFA1tymmR8AYrAl2XIyA+SPs5Uksd6Z3kvUb+o=";
+  };
+
+  packageJSON = ./package.json;
+
+  offlineCache = fetchYarnDeps {
+    yarnLock = "${src}/yarn.lock";
+    hash = "sha256-LU25cXB+0DdcHRzKQ1hjQIVntarqPOUXZTgcw6lvLRM=";
+  };
+
+  buildPhase = ''
+    yarn build
+  '';
+
+  nativeBuildInputs = [ makeWrapper ];
+
+  postInstall = ''
+    makeWrapper ${lib.getExe nodejs} $out/bin/alertmanager-hookshot-adapter \
+      --add-flags $out/libexec/alertmanager-hookshot-adapter/deps/alertmanager-hookshot-adapter/dist/index.js
+  '';
+
+  meta.mainProgram = "alertmanager-hookshot-adapter";
+}

services/monitoring/lgtm/mimir.nix

@@ -20,6 +20,7 @@ in
         owner = "nginx";
       };
       mimir-environment.file = ../../../secrets/mimir-environment.age;
+      mimir-webhook-url.file = ../../../secrets/mimir-webhook-url.age;
     };
 
     services.mimir = {
@@ -75,6 +76,11 @@ in
             receivers = [
               {
                 name = "matrix";
+                webhook_configs = [{
+                  # Mimir can't expand environment variables in external config files,
+                  # so work around it.
+                  url_file = "/run/credentials/mimir.service/webhook-url";
+                }];
               }
             ];
           };
@@ -91,7 +97,10 @@ in
       # Avoid that by ensuring it starts after the network is set up.
       wants = [ "network-online.target" ];
       after = ["network-online.target"];
-      serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
+      serviceConfig = {
+        EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
+        LoadCredential = [ "webhook-url:${config.age.secrets.mimir-webhook-url.path}" ];
+      };
     };
 
     services.nginx = {
@@ -111,5 +120,6 @@ in
     };
 
     bagel.monitoring.grafana-agent.exporters.mimir.port = 9009;
+    bagel.services.alertmanager-hookshot-adapter.enable = true;
   };
 }

terraform/gandi.nix

@@ -68,6 +68,7 @@ in
       # git.p.forkos.org exposes forgejo ssh server.
       (proxyRecords "git.p" 3600 "AAAA" ["2001:bc8:38ee:100:1000::40"])
       (dualProxyRecords "buildbot.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::50"])
+      (dualProxyRecords "public01.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::60"])
 
       (record "cl" 3600 "CNAME" ["gerrit01.infra.p"])
       (record "fodwatch" 3600 "CNAME" ["fodwatch.infra.p"])
@@ -80,7 +81,12 @@ in
       (record "loki" 3600 "CNAME" ["meta01.infra.p"])
       (record "mimir" 3600 "CNAME" ["meta01.infra.p"])
       (record "matrix" 3600 "CNAME" ["meta01.infra.p"])
+      (record "alerts" 3600 "CNAME" ["meta01.infra.p"])
       (record "buildbot" 3600 "CNAME" ["buildbot.infra.p"])
+      (record "b" 3600 "CNAME" ["public01.infra.p"])
+
+      # S3 in delroth's basement
+      (record "cache" 3600 "CNAME" ["smol.delroth.net."])
 
       (record "vpn-gw.wob01.infra" 3600 "AAAA" [ "2a01:584:11::2" ])
       # TODO: do not hardcode, just reuse the Colmena hive module outputs to generate all the required details.

terraform/hydra.nix

@@ -205,13 +205,53 @@ in
       email_notifications = false;
     };
 
-    resource.hydra_jobset.yureka-staging-test = {
+    resource.hydra_jobset.nixos-staging-next-small = {
       project = config.resource.hydra_project.forkos.name;
       state = "enabled";
       visible = true;
-      name = "yureka-staging-test";
+      name = "nixos-staging-next-small";
       type = "legacy";
-      description = "staging branch for yureka-nixos";
+      description = "nixos jobset for the staging-next branch";
+
+      nix_expression = {
+        file = "nixos/release-small.nix";
+        input = "nixpkgs";
+      };
+
+      check_interval = 0;
+      scheduling_shares = 3000;
+      keep_evaluations = 3;
+
+      email_notifications = false;
+
+      input = [
+        {
+          name = "nixpkgs";
+          type = "git";
+          value = "https://cl.forkos.org/nixpkgs staging-next";
+          notify_committers = false;
+        }
+        {
+          name = "officialRelease";
+          type = "boolean";
+          value = "false";
+          notify_committers = false;
+        }
+        {
+          name = "supportedSystems";
+          type = "nix";
+          value = ''[ "x86_64-linux" ]'';
+        }
+      ];
+    };
+
+    resource.hydra_jobset.nixpkgs-staging-next = {
+      project = config.resource.hydra_project.forkos.name;
+      state = "enabled";
+      visible = true;
+      name = "nixpkgs-staging-next";
+      type = "legacy";
+      description = "nixpkgs jobset for the staging-next branch";
 
       nix_expression = {
         file = "pkgs/top-level/release.nix";
@@ -228,7 +268,47 @@ in
         {
           name = "nixpkgs";
           type = "git";
-          value = "https://cl.forkos.org/nixpkgs sandbox/yureka/staging-test";
+          value = "https://cl.forkos.org/nixpkgs staging-next";
+          notify_committers = false;
+        }
+        {
+          name = "officialRelease";
+          type = "boolean";
+          value = "false";
+          notify_committers = false;
+        }
+        {
+          name = "supportedSystems";
+          type = "nix";
+          value = ''[ "x86_64-linux" ]'';
+        }
+      ];
+    };
+
+    resource.hydra_jobset.nixos-main = {
+      project = config.resource.hydra_project.forkos.name;
+      state = "enabled";
+      visible = true;
+      name = "nixos-main";
+      type = "legacy";
+      description = "nixos jobset for the main branch";
+
+      nix_expression = {
+        file = "nixos/release-combined.nix";
+        input = "nixpkgs";
+      };
+
+      check_interval = 0;
+      scheduling_shares = 3000;
+      keep_evaluations = 3;
+
+      email_notifications = false;
+
+      input = [
+        {
+          name = "nixpkgs";
+          type = "git";
+          value = "https://cl.forkos.org/nixpkgs main";
           notify_committers = false;
         }
         {