Set up alertmanager-hookshot-adapter

It doesn't want to work.

.editorconfig

@@ -0,0 +1,7 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+charset = utf-8

.gitignore

@@ -4,3 +4,5 @@ config.tf.json
 .direnv
 .terraform
 .terraform.lock.hcl
+secrets/*
+!secrets/*.age

common/admins.nix

@@ -7,12 +7,8 @@ in {
     keys.users.raito ++
     keys.users.maxine ++ 
     keys.users.jade ++
+    keys.users.janik ++
     keys.users.lukegb ++
-    [
+    keys.users.emilylange ++
-      # more raito
+    keys.users.yuka;
-      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
-      "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJFsZ7PMDt80tYXHyScQajNhqH4wuYg/o0OxfOHaZD4rXuT0VIKflKH1M9LslfHWIEH3XNeqhQOziH9r+Ny5JcM="
-    ];
 }

common/base-server.nix

@@ -1,9 +1,14 @@
 { lib, pkgs, ... }: {
+  imports = [
+    ./known-ssh-keys.nix
+  ];
+
   nixpkgs.overlays = import ../overlays;
 
   nix.package = lib.mkDefault pkgs.lix;
   services.openssh.enable = lib.mkForce true;
 
+  networking.nftables.enable = true;
   networking.firewall.enable = true;
   networking.firewall.logRefusedConnections = false;
   networking.firewall.logReversePathDrops = true;
@@ -13,12 +18,41 @@
     recommendedTlsSettings = lib.mkDefault true;
     recommendedProxySettings = lib.mkDefault true;
     recommendedGzipSettings = lib.mkDefault true;
+    eventsConfig = ''
+      worker_connections 8192;
+    '';
+    appendConfig = ''
+      worker_rlimit_nofile 16384;
+    '';
   };
 
   nix.gc = {
     automatic = true;
     persistent = true;
-    dates = "daily";
+    dates = lib.mkDefault "daily";
     options = "--delete-older-than 30d";
   };
+
+  services.journald.extraConfig = "SystemMaxUse=512M";
+
+  boot.kernelParams = [
+    "panic=30" "boot.panic_on_fail"
+  ];
+
+  boot.kernel.sysctl = {
+    # Set default TCP congestion control algorithm
+    "net.ipv4.tcp_congestion_control" = "bbr";
+
+    # Enable ECN
+    "net.ipv4.tcp_ecn" = 1;
+
+    # Enable TCP fast open
+    "net.ipv4.tcp_fastopen" = 3;
+  };
+
+  # reduce closure size, feel free to add your locale here
+  i18n.supportedLocales = [
+    "en_US.UTF-8/UTF-8"
+    "fr_FR.UTF-8/UTF-8"
+  ];
 }

common/default.nix

@@ -1,9 +1,12 @@
 {
   imports = [
     ./admins.nix
-    ./raito-vm.nix
-    ./raito-proxy-aware-nginx.nix
     ./base-server.nix
+    ./hardening.nix
+    ./nix.nix
+    ./raito-proxy-aware-nginx.nix
+    ./raito-vm.nix
     ./sysadmin
+    ./zsh.nix
   ];
 }

common/hardening.nix

@@ -0,0 +1,23 @@
+{ config, lib, ... }:
+
+{
+  nix.settings.allowed-users = [ "root" ];
+
+  boot.specialFileSystems = lib.mkIf (!config.security.rtkit.enable && !config.security.polkit.enable) {
+    "/proc".options = [ "hidepid=2" ];
+  };
+
+  boot.kernel.sysctl."kernel.dmesg_restrict" = 1;
+
+  services.openssh = {
+    settings.PasswordAuthentication = false;
+    settings.KbdInteractiveAuthentication = false;
+
+    # prevents mutable /home/$user/.ssh/authorized_keys from being loaded to ensure that all user keys are config managed
+    authorizedKeysFiles = lib.mkForce [
+      "/etc/ssh/authorized_keys.d/%u"
+    ];
+  };
+
+  users.mutableUsers = false;
+}

common/known-ssh-keys.nix

@@ -0,0 +1,6 @@
+{ ... }:
+{
+  programs.ssh.knownHosts = {
+    "[cl.forkos.org]:29418".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM82mJ259C8Nc+BHHNBeRWXWhL3dfirQhmFbDAwHMle3";
+  };
+}

common/nix.nix

@@ -0,0 +1,21 @@
+{ lib, pkgs, ... }:
+{
+  nix.extraOptions = ''
+    experimental-features = nix-command flakes
+  '';
+
+  # Provision a useful nixpkgs in NIX_PATH and flake registry on infra
+  # machines.
+  nixpkgs.flake = {
+    source = lib.cleanSource pkgs.path;
+    setNixPath = true;
+    setFlakeRegistry = true;
+  };
+
+  # Use our cache and trust its signing key. Still use cache.nixos.org as
+  # fallback.
+  nix.settings.substituters = [ "https://bagel-cache.s3-web.delroth.net/" ];
+  nix.settings.trusted-public-keys = [
+    "cache.forkos.org:xfXIUJO1yiEITJmYsVmNDa9BFSlgTh/YqZ+4ei1EhQg="
+  ];
+}

common/ssh-keys.nix

@@ -4,11 +4,31 @@
     meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT";
     gerrit01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA+eSZu+u9sCynrMlsmFzQHLIELQAuVg0Cs1pBvwb4+A";
     fodwatch = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFRyTNfvKl5FcSyzGzw+h+bNFNOxdhvI67WdUZ2iIJ1L";
+    buildbot = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJgIu6ouagYqBeMLfmn1CbaDJMuZcPH9bnUhkht8GfuB";
+    git = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEQJcpkCUOx8+5oukMX6lxrYcIX8FyHu8Mc/3+ieKMUn";
+    builder-0 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBHSNcDGctvlG6BHcJuYIzW9WsBJsts2vpwSketsbXoL";
+    builder-1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIQOGUjERK7Mx8UPM/rbOdMqVyn1sbWqYOG6CbOzH2wm";
+    builder-2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMKzXIqCoYElEKIYgjbSpqEcDeOvV+Wo3Agq3jba83cB";
+    builder-3 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGq0A5233XGt34T097KaEKBUqFvaa7a6nYZRsSO0166l";
+    builder-4 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB9dVo2xZhgIMDgB1rUj5ApmppL39BtYu/+OFHeduvXr";
+    builder-5 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE7vZTBxrVHmHpv7slQ8A8XwjjbfN+ZJA0V5C3k0wNBD";
+    builder-6 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOt1qR/2BRtc6PABuSBulowwJVO6wBNDyEFzh0qsTeOF";
+    builder-7 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFinAAw1v8TJB8/wcmTVBbHHc4LCYh6z4TO6ViwUPkoh";
+    builder-8 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGSWHNeqT0kF/e4yVy2ieW98X5QMyCYIYZh9WTmQDs1";
+    builder-9 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOhws9zGgocVY36dMtOL+CXadpvRMffxoWMkfEcTBJm7";
+    builder-10 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE7sgIuTSqZiZhp8TvObSbIEhcHHsL5hcmYA22uzwxth";
+    builder-11 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEAqFo1qJY7MSUkfB+zxXB8Lpt/Iqz/RR5A+zwhpRWhr";
+    wob-vpn-gw = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINVytPPW8XnXf/rD5TFzsw//CZc2lBjQLmDzlVGPZsjh";
   };
 
   users = {
     delroth = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII3tjB4KYDok3KlWxdBp/yEmqhhmybd+w0VO4xUwLKKV" ];
-    raito = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp" ];
+    raito = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICaw9ihTG7ucB8P38XdalEWev8+q96e2yNm4B+/I9IJp"
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcEkYM1r8QVNM/G5CxJInEdoBCWjEHHDdHlzDYNSUIdHHsn04QY+XI67AdMCm8w30GZnLUIj5RiJEWXREUApby0GrfxGGcy8otforygfgtmuUKAUEHdU2MMwrQI7RtTZ8oQ0USRGuqvmegxz3l5caVU7qGvBllJ4NUHXrkZSja2/51vq80RF4MKkDGiz7xUTixI2UcBwQBCA/kQedKV9G28EH+1XfvePqmMivZjl+7VyHsgUVj9eRGA1XWFw59UPZG8a7VkxO/Eb3K9NF297HUAcFMcbY6cPFi9AaBgu3VC4eetDnoN/+xT1owiHi7BReQhGAy/6cdf7C/my5ehZwD"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE0xMwWedkKosax9+7D2OlnMxFL/eV4CvFZLsbLptpXr"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKiXXYkhRh+s7ixZ8rvG8ntIqd6FELQ9hh7HoaHQJRPU"
+    ];
     k900 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOi9vgVGs+S5kEsUqHPvyMMh1Q9gqL4TcbHoe5d73tun" ];
     maxine = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpWQfhNFdrxMTP/1DwBVuk49f3df9iH7Tbdu8ltIKjr" ];
     jade = [
@@ -16,6 +36,13 @@
       "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIKYljH8iPMrH00lOb3ETxRrZimdKzPPEdsJQ5D5ovtOwAAAACnNzaDpzc2hrZXk= ssh:sshkey"
       "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBO4idMfdJxDJuBNOid60d4I+qxj09RHt+YkCYV2eXt6tGrEXg+S8hTQusy/SqooiXUH9pt4tea2RuBPN9+UwrH0= type-a yubikey slot 9a"
     ];
+    janik = [
+      "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIJ4yq7oHBO2iPs4xj797a//0ypnBr27sSadKUeL2NsK6AAAABHNzaDo="
+      "sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIOYg513QZsVzoyVycXZjg4F3T3+OwtcY3WAhrlfyLgLTAAAABHNzaDo="
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBLZxVITpJ8xbiCa/u2gjSSIupeiqOnRh+8tFIoVhCON"
+    ];
     lukegb = [ ''cert-authority,principals="lukegb" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEqNOwlR7Qa8cbGpDfSCOweDPbAGQOZIcoRgh6s/J8DR'' ];
+    emilylange = [ "no-touch-required sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIL7jgq3i+N3gVJhs4shm7Kmw6dIocs2OuR0GBMG1RxfKAAAABHNzaDo=" ];
+    yuka = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKath4/fDnlv/4fzxkPrQN1ttmoPRNu/m9bEtdPJBDfY cardno:16_933_242" ];
   };
 }

common/sysadmin/default.nix

@@ -22,6 +22,8 @@ in
         ncdu
         # Useful to invoke `coredumpctl gdb`
         gdb
-      ] ++ lib.optional (lib.hasAttr "pwru" pkgs) pkgs.pwru;
+        htop
+        btop
+      ];
     };
   }

common/zsh.nix

@@ -0,0 +1,16 @@
+{ lib, pkgs, config, ... }: {
+  users.defaultUserShell = pkgs.zsh;
+  programs.zsh = {
+    enable = true;
+    enableCompletion = true;
+    autosuggestions.enable = true;
+    interactiveShellInit = ''
+      ${lib.getExe pkgs.nix-your-shell} zsh | source /dev/stdin
+    '';
+    promptInit = ''
+      # https://grml.org/zsh/grml-zsh-refcard.pdf
+      source ${pkgs.grml-zsh-config}/etc/zsh/zshrc
+      PS1='%n@${config.networking.fqdn} %/ \$ '
+    '';
+  };
+}

flake.lock

@@ -10,11 +10,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1718371084,
+        "lastModified": 1722339003,
-        "narHash": "sha256-abpBi61mg0g+lFFU0zY4C6oP6fBwPzbHPKBGw676xsA=",
+        "narHash": "sha256-ZeS51uJI30ehNkcZ4uKqT4ZDARPyqrHADSKAwv5vVCU=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "3a56735779db467538fb2e577eda28a9daacaca6",
+        "rev": "3f1dae074a12feb7327b4bf43cbac0d124488bb7",
         "type": "github"
       },
       "original": {
@@ -23,6 +23,29 @@
         "type": "github"
       }
     },
+    "attic": {
+      "inputs": {
+        "crane": "crane",
+        "flake-compat": "flake-compat_2",
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": "nixpkgs",
+        "nixpkgs-stable": "nixpkgs-stable"
+      },
+      "locked": {
+        "lastModified": 1711742460,
+        "narHash": "sha256-0O4v6e4a1toxXZ2gf5INhg4WPE5C5T+SVvsBt+45Mcc=",
+        "owner": "zhaofengli",
+        "repo": "attic",
+        "rev": "4dbdbee45728d8ce5788db6461aaaa89d98081f0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "zhaofengli",
+        "ref": "main",
+        "repo": "attic",
+        "type": "github"
+      }
+    },
     "bats-assert": {
       "flake": false,
       "locked": {
@@ -55,6 +78,29 @@
         "type": "github"
       }
     },
+    "buildbot-nix": {
+      "inputs": {
+        "flake-parts": "flake-parts",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "treefmt-nix": "treefmt-nix"
+      },
+      "locked": {
+        "lastModified": 1721409873,
+        "narHash": "sha256-h0njWQRvtkjK0NJ/Kgj76sXBhWwq5HGJm7OMcigmNw4=",
+        "ref": "refs/heads/refactor",
+        "rev": "54bba654d4279dfd112345b6470547851feb1457",
+        "revCount": 267,
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
+      },
+      "original": {
+        "ref": "refs/heads/refactor",
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
+      }
+    },
     "colmena": {
       "inputs": {
         "flake-compat": "flake-compat",
@@ -78,6 +124,50 @@
         "type": "github"
       }
     },
+    "crane": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "attic",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1702918879,
+        "narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
+    "crane_2": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1716569590,
+        "narHash": "sha256-5eDbq8TuXFGGO3mqJFzhUbt5zHVTf5zilQoyW5jnJwo=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "109987da061a1bf452f435f1653c47511587d919",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "ref": "master",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
     "darwin": {
       "inputs": {
         "nixpkgs": [
@@ -100,6 +190,29 @@
         "type": "github"
       }
     },
+    "fenix": {
+      "inputs": {
+        "nixpkgs": [
+          "grapevine",
+          "nixpkgs"
+        ],
+        "rust-analyzer-src": "rust-analyzer-src"
+      },
+      "locked": {
+        "lastModified": 1716359173,
+        "narHash": "sha256-pYcjP6Gy7i6jPWrjiWAVV0BCQp+DdmGaI/k65lBb/kM=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "b6fc5035b28e36a98370d0eac44f4ef3fd323df6",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "ref": "main",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
     "flake-compat": {
       "flake": false,
       "locked": {
@@ -117,6 +230,39 @@
       }
     },
     "flake-compat_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_3": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "ref": "master",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_4": {
       "flake": false,
       "locked": {
         "lastModified": 1696426674,
@@ -132,6 +278,49 @@
         "type": "github"
       }
     },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "buildbot-nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1706830856,
+        "narHash": "sha256-a0NYyp+h9hlb7ddVz4LUn1vT/PLwqfrWYcHMvFB1xYg=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "b253292d9c0a5ead9bc98c4e9a26c6312e27d69f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-parts_2": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "hydra",
+          "nix-eval-jobs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1719994518,
+        "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
     "flake-utils": {
       "locked": {
         "lastModified": 1659877975,
@@ -148,6 +337,40 @@
       }
     },
     "flake-utils_2": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_3": {
+      "inputs": {
+        "systems": "systems_2"
+      },
+      "locked": {
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "ref": "main",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_4": {
       "locked": {
         "lastModified": 1634851050,
         "narHash": "sha256-N83GlSGPJJdcqhUxSCS/WwW5pksYf3VP1M13cDRTSVA=",
@@ -162,6 +385,34 @@
         "type": "github"
       }
     },
+    "grapevine": {
+      "inputs": {
+        "attic": "attic",
+        "crane": "crane_2",
+        "fenix": "fenix",
+        "flake-compat": "flake-compat_3",
+        "flake-utils": "flake-utils_3",
+        "nix-filter": "nix-filter",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "host": "gitlab.computer.surgery",
+        "lastModified": 1721671623,
+        "narHash": "sha256-ELE+AD83jG3zIbYITbSfo6Ykn+R1gVjMHoS5rhDccuY=",
+        "owner": "matrix",
+        "repo": "grapevine-fork",
+        "rev": "dd24a441121b94d389fb46f08c7ec51886d5aa32",
+        "type": "gitlab"
+      },
+      "original": {
+        "host": "gitlab.computer.surgery",
+        "owner": "matrix",
+        "repo": "grapevine-fork",
+        "type": "gitlab"
+      }
+    },
     "home-manager": {
       "inputs": {
         "nixpkgs": [
@@ -185,17 +436,18 @@
     },
     "hydra": {
       "inputs": {
-        "nix": "nix",
+        "lix": "lix",
+        "nix-eval-jobs": "nix-eval-jobs",
         "nixpkgs": [
           "nixpkgs"
         ]
       },
       "locked": {
-        "lastModified": 1719258100,
+        "lastModified": 1721682989,
-        "narHash": "sha256-Eu8ausj0RsXV5MraCPezwX+j51iZD0ukif110Yj2+6k=",
+        "narHash": "sha256-kjJiZ7m4HKqbZ2mxNQiB32/goKFb8BRi8OqC4wIU0OI=",
         "ref": "refs/heads/main",
-        "rev": "a9a2679793a17325c966dec4cbb27d44b0531694",
+        "rev": "4b107e6ff36bd89958fba36e0fe0340903e7cd13",
-        "revCount": 4172,
+        "revCount": 4190,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/hydra.git"
       },
@@ -204,9 +456,9 @@
         "url": "https://git.lix.systems/lix-project/hydra.git"
       }
     },
-    "nix": {
+    "lix": {
       "inputs": {
-        "flake-compat": "flake-compat_2",
+        "flake-compat": "flake-compat_4",
         "nix2container": "nix2container",
         "nixpkgs": [
           "hydra",
@@ -216,17 +468,61 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1719211568,
+        "lastModified": 1721091462,
-        "narHash": "sha256-oIgmvhe3CV/36LC0KXgqWnKXma39wabks8U9JBMDfO4=",
+        "narHash": "sha256-0cmEeoOiB91BviTJHzIyxkY+Gxv3O8ZnnExVAoXEFGI=",
         "ref": "refs/heads/main",
-        "rev": "4c3d93611f2848c56ebc69c85f2b1e18001ed3c7",
+        "rev": "6b4d46e9e0e1dd80e0977684ab20d14bcd1a6bc3",
-        "revCount": 15877,
+        "revCount": 15967,
         "type": "git",
-        "url": "https://git@git.lix.systems/lix-project/lix"
+        "url": "https://git.lix.systems/lix-project/lix"
       },
       "original": {
         "type": "git",
-        "url": "https://git@git.lix.systems/lix-project/lix"
+        "url": "https://git.lix.systems/lix-project/lix"
+      }
+    },
+    "nix-eval-jobs": {
+      "inputs": {
+        "flake-parts": "flake-parts_2",
+        "lix": [
+          "hydra",
+          "lix"
+        ],
+        "nix-github-actions": "nix-github-actions",
+        "nixpkgs": [
+          "hydra",
+          "nixpkgs"
+        ],
+        "treefmt-nix": "treefmt-nix_2"
+      },
+      "locked": {
+        "lastModified": 1721195872,
+        "narHash": "sha256-TlvRq634MSl22BWLmpTy2vdtKntbZlsUwdMq8Mp9AWs=",
+        "ref": "refs/heads/main",
+        "rev": "c057494450f2d1420726ddb0bab145a5ff4ddfdd",
+        "revCount": 608,
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/nix-eval-jobs"
+      }
+    },
+    "nix-filter": {
+      "locked": {
+        "lastModified": 1710156097,
+        "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "rev": "3342559a24e85fc164b295c3444e8a139924675b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "ref": "main",
+        "repo": "nix-filter",
+        "type": "github"
       }
     },
     "nix-gerrit": {
@@ -236,11 +532,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1720472191,
+        "lastModified": 1720891381,
-        "narHash": "sha256-v42zXC8syDpAuF3cdpwCxU9DsbcDmYVKCfkDq4ZBufU=",
+        "narHash": "sha256-bdZRPgnkROSejmwMOrlcqHMWmuPIVIzjk6r5FbS+fqU=",
         "ref": "refs/heads/main",
-        "rev": "eb589c659e494e5fcb5b47b75be9984ae87a500a",
+        "rev": "23dd318e6741ff686d3069c53ecf475eac8a0565",
-        "revCount": 4,
+        "revCount": 5,
         "type": "git",
         "url": "https://git.lix.systems/the-distro/nix-gerrit.git"
       },
@@ -249,6 +545,28 @@
         "url": "https://git.lix.systems/the-distro/nix-gerrit.git"
       }
     },
+    "nix-github-actions": {
+      "inputs": {
+        "nixpkgs": [
+          "hydra",
+          "nix-eval-jobs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1720066371,
+        "narHash": "sha256-uPlLYH2S0ACj0IcgaK9Lsf4spmJoGejR9DotXiXSBZQ=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "622f829f5fe69310a866c8a6cd07e747c44ef820",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
     "nix2container": {
       "flake": false,
       "locked": {
@@ -267,11 +585,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1719082008,
+        "lastModified": 1711401922,
-        "narHash": "sha256-jHJSUH619zBQ6WdC21fFAlDxHErKVDJ5fpN0Hgx4sjs=",
+        "narHash": "sha256-QoQqXoj8ClGo0sqD/qWKFWezgEwUL0SUh37/vY2jNhc=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "9693852a2070b398ee123a329e68f0dab5526681",
+        "rev": "07262b18b97000d16a4bdb003418bd2fb067a932",
         "type": "github"
       },
       "original": {
@@ -297,7 +615,39 @@
         "type": "github"
       }
     },
+    "nixpkgs-stable": {
+      "locked": {
+        "lastModified": 1711460390,
+        "narHash": "sha256-akSgjDZL6pVHEfSE6sz1DNSXuYX6hq+P/1Z5IoYWs7E=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "44733514b72e732bd49f5511bd0203dea9b9a434",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-23.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
     "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1723151389,
+        "narHash": "sha256-9AVY0ReCmSGXHrlx78+1RrqcDgVSRhHUKDVV1LLBy28=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "13fe00cb6c75461901f072ae62b5805baef9f8b2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_3": {
       "locked": {
         "lastModified": 1636823747,
         "narHash": "sha256-oWo1nElRAOZqEf90Yek2ixdHyjD+gqtS/pAgwaQ9UhQ=",
@@ -331,17 +681,36 @@
     "root": {
       "inputs": {
         "agenix": "agenix",
+        "buildbot-nix": "buildbot-nix",
         "colmena": "colmena",
+        "grapevine": "grapevine",
         "hydra": "hydra",
         "lix": [
           "hydra",
-          "nix"
+          "lix"
         ],
         "nix-gerrit": "nix-gerrit",
-        "nixpkgs": "nixpkgs",
+        "nixpkgs": "nixpkgs_2",
         "terranix": "terranix"
       }
     },
+    "rust-analyzer-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1716107283,
+        "narHash": "sha256-NJgrwLiLGHDrCia5AeIvZUHUY7xYGVryee0/9D3Ir1I=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "21ec8f523812b88418b2bfc64240c62b3dd967bd",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
+        "type": "github"
+      }
+    },
     "stable": {
       "locked": {
         "lastModified": 1696039360,
@@ -373,12 +742,27 @@
         "type": "github"
       }
     },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
     "terranix": {
       "inputs": {
         "bats-assert": "bats-assert",
         "bats-support": "bats-support",
-        "flake-utils": "flake-utils_2",
+        "flake-utils": "flake-utils_4",
-        "nixpkgs": "nixpkgs_2",
+        "nixpkgs": "nixpkgs_3",
         "terranix-examples": "terranix-examples"
       },
       "locked": {
@@ -409,6 +793,49 @@
         "repo": "terranix-examples",
         "type": "github"
       }
+    },
+    "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "buildbot-nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1708897213,
+        "narHash": "sha256-QECZB+Hgz/2F/8lWvHNk05N6NU/rD9bWzuNn6Cv8oUk=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "e497a9ddecff769c2a7cbab51e1ed7a8501e7a3a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
+    },
+    "treefmt-nix_2": {
+      "inputs": {
+        "nixpkgs": [
+          "hydra",
+          "nix-eval-jobs",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1721059077,
+        "narHash": "sha256-gCICMMX7VMSKKt99giDDtRLkHJ0cwSgBtDijJAqTlto=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "0fb28f237f83295b4dd05e342f333b447c097398",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -17,10 +17,21 @@
     nix-gerrit.url = "git+https://git.lix.systems/the-distro/nix-gerrit.git";
     nix-gerrit.inputs.nixpkgs.follows = "nixpkgs";
 
-    lix.follows = "hydra/nix";
+    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/refactor";
+    buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";
+
+    lix.follows = "hydra/lix";
+
+    grapevine = {
+      type = "gitlab";
+      host = "gitlab.computer.surgery";
+      owner = "matrix";
+      repo = "grapevine-fork";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
   };
 
-  outputs = { self, nixpkgs, terranix, ... } @ inputs:
+  outputs = { self, nixpkgs, terranix, colmena, ... } @ inputs:
   let
     system = "x86_64-linux";
     pkgs = import nixpkgs {
@@ -61,13 +72,32 @@
     devShells.${system}.default = pkgs.mkShell {
       packages = [
         inputs.agenix.packages.${system}.agenix
-        inputs.colmena.packages.${system}.colmena
 
+        pkgs.colmena
         pkgs.opentofu
       ];
     };
 
-    colmena = {
+    nixosConfigurations = (colmena.lib.makeHive self.outputs.colmena).nodes;
+
+    colmena = let
+      commonModules = [
+        inputs.agenix.nixosModules.default
+        inputs.hydra.nixosModules.hydra
+        inputs.buildbot-nix.nixosModules.buildbot-coordinator
+        inputs.buildbot-nix.nixosModules.buildbot-worker
+
+        ./services
+        ./common
+      ];
+
+      makeBuilder = i: lib.nameValuePair "builder-${toString i}" {
+        imports = commonModules;
+        bagel.baremetal.builders = { enable = true; num = i; };
+      };
+
+      builders = lib.listToAttrs (lib.genList makeBuilder 12);
+    in {
       meta.nixpkgs = import nixpkgs {
         localSystem = system;
         overlays = [
@@ -78,49 +108,16 @@
       };
       meta.specialArgs.inputs = inputs;
 
-      bagel-box = {
+      bagel-box.imports = commonModules ++ [ ./hosts/bagel-box ];
-        imports = [
+      meta01.imports = commonModules ++ [ ./hosts/meta01 ];
-          inputs.agenix.nixosModules.default
+      gerrit01.imports = commonModules ++ [ ./hosts/gerrit01 ];
-          inputs.hydra.nixosModules.hydra
+      fodwatch.imports = commonModules ++ [ ./hosts/fodwatch ];
+      git.imports = commonModules ++ [ ./hosts/git ];
+      wob-vpn-gw.imports = commonModules ++ [ ./hosts/wob-vpn-gw ];
+      buildbot.imports = commonModules ++ [ ./hosts/buildbot ];
+    } // builders;
 
-          ./services
+    hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.toplevel) self.nixosConfigurations;
-          ./common
+    buildbotJobs = builtins.mapAttrs (_: v: v.config.system.build.toplevel) self.nixosConfigurations;
-          ./hosts/bagel-box
-        ];
-      };
-
-      meta01 = {
-        imports = [
-          inputs.agenix.nixosModules.default
-          inputs.hydra.nixosModules.hydra
-
-          ./services
-          ./common
-          ./hosts/meta01.nixpkgs.lahfa.xyz
-        ];
-      };
-
-      gerrit01 = {
-        imports = [
-          inputs.agenix.nixosModules.default
-          inputs.hydra.nixosModules.hydra
-
-          ./services
-          ./common
-          ./hosts/cl.forkos.org
-        ];
-      };
-
-      fodwatch = {
-        imports = [
-          inputs.agenix.nixosModules.default
-          inputs.hydra.nixosModules.hydra
-
-          ./services
-          ./common
-          ./hosts/fodwatch.forkos.org
-        ];
-      };
-    };
   };
 }

hosts/bagel-box/default.nix

@@ -20,6 +20,7 @@
     useHostResolvConf = false;
 
     hostName = "bagel-box";
+    domain = "infra.forkos.org";
     nameservers = [ "2001:4860:4860::8844" ];
 
     interfaces.host0.ipv6.addresses = [
@@ -39,6 +40,8 @@
 
     hydra.enable = true;
     hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
+    # Takes 10 builders (0 → 9).
+    hydra.builders = lib.genList (i: "builder-${builtins.toString i}") 10;
 
     ofborg.enable = true;
   };
@@ -50,5 +53,6 @@
 
   services.openssh.enable = true;
 
+  system.stateVersion = "24.11";
   deployment.targetHost = "bagel-box.infra.forkos.org";
 }

hosts/buildbot/default.nix

@@ -0,0 +1,38 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  networking.hostName = "buildbot";
+  # TODO: make it the default
+  networking.domain = "infra.forkos.org";
+
+  time.timeZone = "Europe/Paris";
+
+  bagel.sysadmin.enable = true;
+  # Buildbot is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.hardware.raito-vm = {
+    enable = true;
+    networking = {
+      nat-lan-mac = "BC:24:11:E7:42:8B";
+      wan = {
+        address = "2001:bc8:38ee:100:1000::50/64";
+        mac = "BC:24:11:C9:BA:6C";
+      };
+    };
+  };
+
+  bagel.services.buildbot = {
+    enable = true;
+    domain = "buildbot.forkos.org";
+    builders = [ "builder-3" ];
+  };
+
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "buildbot.infra.forkos.org";
+}

hosts/cl.forkos.org/default.nix

@@ -1,45 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  ...
-}:
-{
-  networking.hostName = "gerrit01";
-  # TODO: make it the default
-  networking.domain = "infra.forkos.org";
-
-  time.timeZone = "Europe/Paris";
-
-  bagel.sysadmin.enable = true;
-  # Gerrit is proxied.
-  bagel.raito.v6-proxy-awareness.enable = true;
-  bagel.hardware.raito-vm = {
-    enable = true;
-    networking = {
-      nat-lan-mac = "bc:24:11:f7:29:6c";
-      wan = {
-        address = "2001:bc8:38ee:100:1000::10/64";
-        mac = "bc:24:11:4a:9d:32";
-      };
-    };
-  };
-
-  fileSystems."/gerrit-data" = {
-    device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";
-    fsType = "ext4";
-  };
-
-  bagel.services.gerrit = {
-    enable = true;
-    domains = [
-      "cl.forkos.org"
-    ];
-    data = "/gerrit-data";
-  };
-
-  i18n.defaultLocale = "fr_FR.UTF-8";
-
-  system.stateVersion = "24.05";
-  deployment.targetHost = "gerrit01.infra.forkos.org";
-}

hosts/gerrit01/default.nix

@@ -0,0 +1,116 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  networking.hostName = "gerrit01";
+  # TODO: make it the default
+  networking.domain = "infra.forkos.org";
+
+  time.timeZone = "Europe/Paris";
+
+  bagel.sysadmin.enable = true;
+  # Gerrit is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.hardware.raito-vm = {
+    enable = true;
+    networking = {
+      nat-lan-mac = "bc:24:11:f7:29:6c";
+      wan = {
+        address = "2001:bc8:38ee:100:1000::10/64";
+        mac = "bc:24:11:4a:9d:32";
+      };
+    };
+  };
+
+  fileSystems."/gerrit-data" = {
+    device = "/dev/disk/by-uuid/d1062305-0dea-4740-9a27-b6b1691862a4";
+    fsType = "ext4";
+  };
+
+  bagel.services.gerrit = {
+    enable = true;
+    domains = [
+      "cl.forkos.org"
+    ];
+    canonicalDomain = "cl.forkos.org";
+    data = "/gerrit-data";
+  };
+
+  age.secrets.ows-deploy-key = {
+    file = ../../secrets/ows-deploy-key.age;
+    mode = "0600";
+    owner = "git";
+    group = "git";
+  };
+  bagel.nixpkgs.one-way-sync =
+  let
+    mkNixpkgsJob = { timer, branchName }: {
+      name = "nixpkgs-${branchName}";
+      fromUri = "https://github.com/NixOS/nixpkgs";
+      fromRefspec = branchName;
+      localRefspec = branchName;
+      inherit timer;
+    };
+    in
+  {
+    enable = true;
+
+    pushUrl = "ssh://ows_bot@cl.forkos.org:29418/nixpkgs";
+    deployKeyPath = config.age.secrets.ows-deploy-key.path;
+
+    branches."refs/heads/main" = mkNixpkgsJob {
+      timer = "hourly";
+      branchName = "main";
+    };
+
+    branches."refs/heads/staging" = mkNixpkgsJob {
+      timer = "hourly";
+      branchName = "staging";
+    };
+
+    branches."refs/heads/release-24.05" = mkNixpkgsJob {
+      timer = "hourly";
+      branchName = "release-24.05";
+    };
+
+    branches."refs/heads/staging-24.05" = mkNixpkgsJob {
+      timer = "hourly";
+      branchName = "staging-24.05";
+    };
+
+    branches."refs/heads/release-23.11" = mkNixpkgsJob {
+      timer = "hourly";
+      branchName = "release-23.11";
+    };
+
+    branches."refs/heads/staging-23.11" = mkNixpkgsJob {
+      timer = "hourly";
+      branchName = "staging-23.11";
+    };
+
+    # Testing jobs for personal sandbox branches
+    branches."refs/heads/sandbox/raito/raito-unstable-small" = {
+      name = "raito-unstable-sync";
+      fromUri = "https://github.com/NixOS/nixpkgs";
+      fromRefspec = "nixos-unstable-small";
+      localRefspec = "sandbox/raito/raito-unstable-small";
+      timer = "*-*-* 12:00:00";
+    };
+
+    branches."refs/heads/sandbox/raito/raito-nixos-24.05" = {
+      name = "raito-release-sync";
+      fromUri = "https://github.com/NixOS/nixpkgs";
+      fromRefspec = "nixos-24.05";
+      localRefspec = "sandbox/raito/raito-nixos-24.05";
+      timer = "daily";
+    };
+  };
+
+  i18n.defaultLocale = "fr_FR.UTF-8";
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "gerrit01.infra.forkos.org";
+}

hosts/git/default.nix

@@ -0,0 +1,49 @@
+let
+  ipv6 = {
+    openssh ="2001:bc8:38ee:100:1000::41";
+    forgejo = "2001:bc8:38ee:100:1000::40";
+  };
+in
+{
+  networking.hostName = "git";
+  networking.domain = "infra.forkos.org";
+
+  time.timeZone = "Europe/Paris";
+
+  bagel.sysadmin.enable = true;
+  # Forgejo will be proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.hardware.raito-vm = {
+    enable = true;
+    networking = {
+      nat-lan-mac = "BC:24:11:83:71:56";
+      wan = {
+        address = "${ipv6.forgejo}/64";
+        mac = "BC:24:11:0B:8A:81";
+      };
+    };
+  };
+
+  # Add one additional IPv6, so we can have both OpenSSH and
+  # Forgejo's built-in server bind on port :22.
+  systemd.network.networks."10-wan".networkConfig.Address = [ "${ipv6.openssh}/64" ];
+  services.openssh.listenAddresses = [{
+    addr = "[${ipv6.openssh}]";
+  }];
+  # Defaults to network.target, but networkd may take a while to settle and set up
+  # the required (additional) IPv6 address, leading to sshd to not being able to
+  # bind to the requested IP, crashing 5 times and running into the default
+  # restart counter limit (5).
+  systemd.services.sshd.wants = [ "network-online.target" ];
+  systemd.services.sshd.after = [ "network-online.target" ];
+
+  bagel.services.forgejo = {
+    enable = true;
+    sshBindAddr = ipv6.forgejo;
+  };
+
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "git.infra.forkos.org";
+}

hosts/meta01/default.nix

@@ -24,6 +24,13 @@
   bagel.services.prometheus.enable = true;
   bagel.services.loki.enable = true;
   bagel.services.grafana.enable = true;
+  bagel.services.grapevine.enable = true;
+  bagel.services.hookshot = {
+    enable = true;
+    admins = [
+      "@k900:0upti.me"
+    ];
+  };
 
   i18n.defaultLocale = "fr_FR.UTF-8";
 

hosts/wob-vpn-gw/default.nix

@@ -0,0 +1,122 @@
+{ pkgs, lib, ... }:
+
+{
+  ###### Hardware ######
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "ehci_pci" "sd_mod" "sdhci_pci" ];
+  boot.kernelModules = [ "kvm-amd" ];
+
+  boot.loader.grub.device = "/dev/sda";
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/58688a5c-e3ce-4868-804b-4e34d1370f36";
+      fsType = "f2fs";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/38caa628-3b6d-4fb4-8767-beee09a196a6";
+      fsType = "ext2";
+    };
+
+  nixpkgs.hostPlatform = "x86_64-linux";
+
+  hardware.cpu.amd.updateMicrocode = true;
+
+  # Enable serial output
+  boot.loader.grub.extraConfig = ''
+    serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1
+    terminal_input serial
+    terminal_output serial
+  '';
+  boot.kernelParams = [
+    "console=ttyS0,115200"
+    "console=tty1"
+  ];
+
+  ###### Config #######
+
+  boot.initrd.systemd.enable = true;
+
+  networking.useNetworkd = true;
+
+  systemd.network = {
+    netdevs = {
+      "40-uplink" = {
+        netdevConfig = {
+          Kind = "bond";
+          Name = "uplink";
+        };
+        bondConfig = {
+          Mode = "802.3ad";
+          TransmitHashPolicy = "layer3+4";
+        };
+      };
+      "40-oob" = {
+        netdevConfig = {
+          Kind = "bond";
+          Name = "oob";
+        };
+        bondConfig = {
+          Mode = "802.3ad";
+          TransmitHashPolicy = "layer3+4";
+        };
+      };
+    };
+    networks = {
+      "40-enp1s0" = {
+        name = "enp1s0";
+        bond = [ "uplink" ];
+      };
+      "40-enp2s0" = {
+        name = "enp2s0";
+        bond = [ "uplink" ];
+      };
+      "40-enp3s0" = {
+        name = "enp3s0";
+        bond = [ "oob" ];
+      };
+      "40-enp4s0" = {
+        name = "enp4s0";
+        bond = [ "oob" ];
+      };
+    } // lib.listToAttrs (map (x: lib.nameValuePair "40-bmc${toString x}" {
+      name = "bmc${toString x}";
+      address = [ "192.168.1.${toString (x*4 + 1)}/30" ];
+      #address = [ "192.168.${toString x}.1/24" ];
+      networkConfig.DHCPServer = true;
+    }) (lib.genList lib.id 12));
+  };
+
+  networking.nftables.enable = true;
+  networking.firewall.extraInputRules = ''
+    iifname { "bmc*" } meta nfproto ipv4 udp dport 67 accept comment "DHCP server"
+  '';
+
+  networking.vlans = lib.listToAttrs (map (x: lib.nameValuePair "bmc${toString x}" {
+    interface = "oob";
+    id = 101 + x;
+  }) (lib.genList lib.id 12));
+
+  networking.interfaces = {
+    uplink = {
+      ipv6.addresses = [
+        {
+          address = "2a01:584:11::2";
+          prefixLength = 64;
+        }
+      ];
+    };
+  };
+
+  networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
+
+  networking.hostName = "vpn-gw";
+  networking.domain = "wob01.infra.forkos.org";
+
+  deployment.targetHost = "2a01:584:11::2";
+
+  bagel.sysadmin.enable = true;
+
+  environment.systemPackages = [ pkgs.ipmitool ];
+
+  system.stateVersion = "24.05";
+}

pkgs/forgejo/branch-view_remove-expensive-commit-divergence-metric.patch

@@ -0,0 +1,59 @@
+diff --git a/services/repository/branch.go b/services/repository/branch.go
+index e1a313749f..5a8d823eef 100644
+--- a/services/repository/branch.go
++++ b/services/repository/branch.go
+@@ -26,7 +26,6 @@ import (
+ 	"code.gitea.io/gitea/modules/timeutil"
+ 	webhook_module "code.gitea.io/gitea/modules/webhook"
+ 	notify_service "code.gitea.io/gitea/services/notify"
+-	files_service "code.gitea.io/gitea/services/repository/files"
+ 
+ 	"xorm.io/builder"
+ )
+@@ -129,21 +128,7 @@ func loadOneBranch(ctx context.Context, repo *repo_model.Repository, dbBranch *g
+ 	p := protectedBranches.GetFirstMatched(branchName)
+ 	isProtected := p != nil
+ 
+-	var divergence *git.DivergeObject
+-
+-	// it's not default branch
+-	if repo.DefaultBranch != dbBranch.Name && !dbBranch.IsDeleted {
+-		var err error
+-		divergence, err = files_service.CountDivergingCommits(ctx, repo, git.BranchPrefix+branchName)
+-		if err != nil {
+-			return nil, fmt.Errorf("CountDivergingCommits: %v", err)
+-		}
+-	}
+-
+-	if divergence == nil {
+-		// tolerate the error that we cannot get divergence
+-		divergence = &git.DivergeObject{Ahead: -1, Behind: -1}
+-	}
++	divergence := &git.DivergeObject{Ahead: -1, Behind: -1}
+ 
+ 	pr, err := issues_model.GetLatestPullRequestByHeadInfo(ctx, repo.ID, branchName)
+ 	if err != nil {
+diff --git a/templates/repo/branch/list.tmpl b/templates/repo/branch/list.tmpl
+index a577fed450..e102796315 100644
+--- a/templates/repo/branch/list.tmpl
++++ b/templates/repo/branch/list.tmpl
+@@ -102,19 +102,6 @@
+ 							{{end}}
+ 							</td>
+ 							<td class="two wide ui">
+-								{{if and (not .DBBranch.IsDeleted) $.DefaultBranchBranch}}
+-								<div class="commit-divergence">
+-									<div class="bar-group">
+-										<div class="count count-behind">{{.CommitsBehind}}</div>
+-										{{/* old code bears 0/0.0 = NaN output, so it might output invalid "width: NaNpx", it just works and doesn't caues any problem. */}}
+-										<div class="bar bar-behind" style="width: {{Eval 100 "*" .CommitsBehind "/" "(" .CommitsBehind "+" .CommitsAhead "+" 0.0 ")"}}%"></div>
+-									</div>
+-									<div class="bar-group">
+-										<div class="count count-ahead">{{.CommitsAhead}}</div>
+-										<div class="bar bar-ahead" style="width: {{Eval 100 "*" .CommitsAhead "/" "(" .CommitsBehind "+" .CommitsAhead "+" 0.0 ")"}}%"></div>
+-									</div>
+-								</div>
+-								{{end}}
+ 							</td>
+ 							<td class="two wide right aligned">
+ 								{{if not .LatestPullRequest}}

pkgs/forgejo/commit-view_fix-broken-and-expensive-cherry-pick-default-branch-selection.patch

@@ -0,0 +1,32 @@
+diff --git a/routers/web/repo/commit.go b/routers/web/repo/commit.go
+index 718454e063..8fa299710c 100644
+--- a/routers/web/repo/commit.go
++++ b/routers/web/repo/commit.go
+@@ -408,12 +408,6 @@ func Diff(ctx *context.Context) {
+ 		}
+ 	}
+ 
+-	ctx.Data["BranchName"], err = commit.GetBranchName()
+-	if err != nil {
+-		ctx.ServerError("commit.GetBranchName", err)
+-		return
+-	}
+-
+ 	ctx.HTML(http.StatusOK, tplCommitPage)
+ }
+ 
+diff --git a/templates/repo/commit_page.tmpl b/templates/repo/commit_page.tmpl
+index c37fb46975..18c9cf18f8 100644
+--- a/templates/repo/commit_page.tmpl
++++ b/templates/repo/commit_page.tmpl
+@@ -71,8 +71,8 @@
+ 												"branchForm" "branch-dropdown-form"
+ 												"branchURLPrefix" (printf "%s/_cherrypick/%s/" $.RepoLink .CommitID) "branchURLSuffix" ""
+ 												"setAction" true "submitForm" true}}
+-											<form method="get" action="{{$.RepoLink}}/_cherrypick/{{.CommitID}}/{{if $.BranchName}}{{PathEscapeSegments $.BranchName}}{{else}}{{PathEscapeSegments $.Repository.DefaultBranch}}{{end}}" id="branch-dropdown-form">
+-												<input type="hidden" name="ref" value="{{if $.BranchName}}{{$.BranchName}}{{else}}{{$.Repository.DefaultBranch}}{{end}}">
++											<form method="get" action="{{$.RepoLink}}/_cherrypick/{{.CommitID}}/{{PathEscapeSegments $.Repository.DefaultBranch}}" id="branch-dropdown-form">
++												<input type="hidden" name="ref" value="{{$.Repository.DefaultBranch}}">
+ 												<input type="hidden" name="refType" value="branch">
+ 												<input type="hidden" id="cherry-pick-type" name="cherry-pick-type"><br>
+ 												<button type="submit" id="cherry-pick-submit" class="ui primary button"></button>

pkgs/forgejo/default.nix

@@ -0,0 +1,40 @@
+{ forgejo }:
+
+forgejo.overrideAttrs (prev: {
+  patches = [
+    # Branch divergence calculations for a single branch may take 100-200ms on something as big
+    # as nixpkgs. The branch view defaults to 20 branches for each page, taking roughtly 3s to
+    # calculate each branch sequentially and render, while consuming a single core at 100%.
+    # The idea is to look into making this less expensive or async.
+    # But for now, to get this going, we will simply drop that metric.
+    ./branch-view_remove-expensive-commit-divergence-metric.patch
+
+    # This is literally broken and eats resources for nothing of value.
+    # We should upstream this.
+    # The tl;dr is: It calculates the nearest branch for the requested commit at
+    # /:owner/:repo/commit/:commit to use it as the default cherry-pick target branch
+    # selection in a drop-down only users with commit perms can actually view and use.
+    # It's expensive to calculate and happens on every request to /commit/:commit.
+    # To add insult to injury, it's hardly of any use: The nearest branch of a commit
+    # will almost always be a branch that already carries the commit. The branch you
+    # most likely don't want to cherry-pick to.
+    ./commit-view_fix-broken-and-expensive-cherry-pick-default-branch-selection.patch
+
+    # Disable various /:owner/:repo/activity/ sub-views. They are expensive, which is
+    # totally fine and expected. There is even proper caching in place.
+    # However, on a scale of nixpkgs, those calculations take ages, while, of course,
+    # pinning a single CPU core at 100%.
+    # For now, we will simply disable this feature.
+    # Due to the 501 status code it returns, the frontend prints a "Not implemented"
+    # error, saving us from patching the frontend while still providing a helpful
+    # user-facing error text.
+    # It should be noted that this particular status code has the downside of being
+    # in the 5xx range, meaning it will show up as such in our prometheus metrics.
+    ./disable-expensive-repository-activity-stats.patch
+
+    # Migrations and pull-mirrors are something easily abused to bring a public instance to a complete halt.
+    # Both features can be disabled via repository.DISABLE_MIGRATIONS and mirror.ENABLE, but we want to keep
+    # this functionality for admins.
+    ./limit-migrations-and-pull-mirrors-to-admins.patch
+  ];
+})

pkgs/forgejo/disable-expensive-repository-activity-stats.patch

@@ -0,0 +1,34 @@
+diff --git a/routers/web/web.go b/routers/web/web.go
+index ee9694f41c..f55b8d6f62 100644
+--- a/routers/web/web.go
++++ b/routers/web/web.go
+@@ -57,6 +57,10 @@ import (
+ 	"github.com/prometheus/client_golang/prometheus"
+ )
+ 
++func endpointNotImplemented(ctx *context.Context) {
++	ctx.JSON(http.StatusNotImplemented, "This endpoint has been removed due to performance issues with it and as such is not longer implemented.")
++}
++
+ // optionsCorsHandler return a http handler which sets CORS options if enabled by config, it blocks non-CORS OPTIONS requests.
+ func optionsCorsHandler() func(next http.Handler) http.Handler {
+ 	var corsHandler func(next http.Handler) http.Handler
+@@ -1425,15 +1429,15 @@ func registerRoutes(m *web.Route) {
+ 			m.Get("/{period}", repo.Activity)
+ 			m.Group("/contributors", func() {
+ 				m.Get("", repo.Contributors)
+-				m.Get("/data", repo.ContributorsData)
++				m.Get("/data", endpointNotImplemented)
+ 			}, repo.MustBeNotEmpty, context.RequireRepoReaderOr(unit.TypeCode))
+ 			m.Group("/code-frequency", func() {
+ 				m.Get("", repo.CodeFrequency)
+-				m.Get("/data", repo.CodeFrequencyData)
++				m.Get("/data", endpointNotImplemented)
+ 			}, repo.MustBeNotEmpty, context.RequireRepoReaderOr(unit.TypeCode))
+ 			m.Group("/recent-commits", func() {
+ 				m.Get("", repo.RecentCommits)
+-				m.Get("/data", repo.RecentCommitsData)
++				m.Get("/data", endpointNotImplemented)
+ 			}, repo.MustBeNotEmpty, context.RequireRepoReaderOr(unit.TypeCode))
+ 		}, context.RepoRef(), context.RequireRepoReaderOr(unit.TypeCode, unit.TypePullRequests, unit.TypeIssues, unit.TypeReleases))
+ 

pkgs/forgejo/limit-migrations-and-pull-mirrors-to-admins.patch

@@ -0,0 +1,53 @@
+diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go
+index 2caaa130e8..455e89e93e 100644
+--- a/routers/api/v1/repo/migrate.go
++++ b/routers/api/v1/repo/migrate.go
+@@ -12,7 +12,6 @@ import (
+ 
+ 	"code.gitea.io/gitea/models"
+ 	"code.gitea.io/gitea/models/db"
+-	"code.gitea.io/gitea/models/organization"
+ 	"code.gitea.io/gitea/models/perm"
+ 	access_model "code.gitea.io/gitea/models/perm/access"
+ 	repo_model "code.gitea.io/gitea/models/repo"
+@@ -86,22 +85,7 @@ func Migrate(ctx *context.APIContext) {
+ 	}
+ 
+ 	if !ctx.Doer.IsAdmin {
+-		if !repoOwner.IsOrganization() && ctx.Doer.ID != repoOwner.ID {
+-			ctx.Error(http.StatusForbidden, "", "Given user is not an organization.")
+-			return
+-		}
+-
+-		if repoOwner.IsOrganization() {
+-			// Check ownership of organization.
+-			isOwner, err := organization.OrgFromUser(repoOwner).IsOwnedBy(ctx, ctx.Doer.ID)
+-			if err != nil {
+-				ctx.Error(http.StatusInternalServerError, "IsOwnedBy", err)
+-				return
+-			} else if !isOwner {
+-				ctx.Error(http.StatusForbidden, "", "Given user is not owner of organization.")
+-				return
+-			}
+-		}
++		ctx.Error(http.StatusForbidden, "", "You need to be administrator of this Forgejo instance to be able to create mirrors.")
+ 	}
+ 
+ 	remoteAddr, err := forms.ParseRemoteAddr(form.CloneAddr, form.AuthUsername, form.AuthPassword)
+diff --git a/routers/web/repo/migrate.go b/routers/web/repo/migrate.go
+index 97b0c425ea..554a470eab 100644
+--- a/routers/web/repo/migrate.go
++++ b/routers/web/repo/migrate.go
+@@ -150,6 +150,12 @@ func handleMigrateRemoteAddrError(ctx *context.Context, err error, tpl base.TplN
+ // MigratePost response for migrating from external git repository
+ func MigratePost(ctx *context.Context) {
+ 	form := web.GetForm(ctx).(*forms.MigrateRepoForm)
++
++	if !ctx.Doer.IsAdmin {
++		ctx.Error(http.StatusForbidden, "MigratePost: you need to be site administrator to use migrations and mirrors")
++		return
++	}
++
+ 	if setting.Repository.DisableMigrations {
+ 		ctx.Error(http.StatusForbidden, "MigratePost: the site administrator has disabled migrations")
+ 		return

secrets.nix

@@ -1,19 +1,34 @@
 let
   keys = import common/ssh-keys.nix;
 
-  commonKeys = keys.users.delroth;
+  commonKeys = keys.users.delroth ++ keys.users.raito;
 
   secrets = with keys; {
     hydra-s3-credentials = [ machines.bagel-box ];
+    hydra-signing-priv = [ machines.bagel-box ];
     hydra-ssh-key-priv = [ machines.bagel-box ];
     netbox-environment = [ machines.meta01 ];
     mimir-environment = [ machines.meta01 ];
+    mimir-webhook-url = [ machines.meta01 ];
     grafana-oauth-secret = [ machines.meta01 ];
     loki-environment = [ machines.meta01 ];
+    gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
+
+    buildbot-worker-password = [ machines.buildbot ];
+    buildbot-oauth-secret = [ machines.buildbot ];
+    buildbot-workers = [ machines.buildbot ];
+    # Private SSH key to Gerrit
+    # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
+    buildbot-service-key = [ machines.buildbot ];
+    # Signing key for Buildbot's specific cache
+    buildbot-signing-key = [ machines.buildbot ];
+    buildbot-remote-builder-key = [ machines.buildbot ];
 
     # These are the same password, but nginx wants it in htpasswd format
     metrics-push-htpasswd = [ machines.meta01 ];
     metrics-push-password = builtins.attrValues machines;
+
+    ows-deploy-key = [ machines.gerrit01 ];
   };
 in
   builtins.listToAttrs (

secrets/buildbot-oauth-secret.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 87T2Ig g15A5EWi9IhaxPFS6SD6YYm/aFnC0Dum7zK8/ZUtW0s
+791D6C8mAy2dhDAlqRQ+q41FlQTJX2WfZQPjuwetP2A
+-> ssh-ed25519 K3b7BA cJY9qIFVmucmMJLTFffkRCNYeudZl+8Yrm5SkxQ4eSI
+97nXyKffZGoGJ6252UKUEJHiFgdk8XUkAAkXy2PLepM
+-> ssh-ed25519 +qVung HMBSUjfmaFLVx64epj0djkqNMe3CdKN1fxAVuu+Dtmg
+AxT62n2p/pP9WZmmuHClSKKgXhr4FjEQpEs0HfdNGfw
+-> ssh-rsa krWCLQ
+N0Duz2bONcCUZ76QhPsCJ4BHHWqzFdZLqFdl+6GeW+tgIp2Nb4la8eNfgzYGSwTy
+53bRePNMIBTkChXFYt/4fUdqaiiVYg25swMeVLQBJnjJkcAks0Gf44FXLIaoPr1M
+56rtixpSX31WDKwHbUF/40G6Xut8KNlI8BdwiOl9ibgnuEf4mYQbwFbRQbLMK5IK
+Rf/7SEmAqqfY/HG1RqqgCs4kEpvFTKqEEDpgjOoyS2tyKN2351jya91YzotLja4I
+sLoMg/G3UNtxfdaCgK7TP4IxV9blkVMDPAbyR622VbS0sEa7uJGzb86jDDsZXaKX
+9iWK9n4hMKZDv9gBbhTIWg
+-> ssh-ed25519 /vwQcQ hMkCrUcLGxdZMYgi1D1Kr5qUdGNfza2UTvRJKiHObgM
+7Lz70zSMPk/tsU1CZGOk/BPA7NSSnSJgFbG5TjyOXvA
+-> ssh-ed25519 0R97PA OQjDTknVmrYVclcqlT31YjZx+3a/0GxfjuVQFmPJ7UQ
+KMGTMfO/mO5EAYacyz1hmHnQgzunRqkDeglhbGVNWe4
+--- ScDZvSiVSjNXm8TSoLSAM+KpcFORnCXiemYbCBcz2jQ
+™ŸÄhÜ}E¹ÊœËíUÌùᢌƒÿ…<C3BF>é™k¢ág[<5B>ñCƒ"<22>–NÛj•u5«<0C>ÄCXÕöÈGt¡TOmñ

secrets/buildbot-worker-password.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 87T2Ig df+IMqWM/HNjaY74zibFQIdUdC3K7uQlm3U9R9NUtFY
+hPSbCuWvqy/7FEj7YScYztyt5GVx4Y7tgGuKKkSKoRg
+-> ssh-ed25519 K3b7BA xN8wzUKHqjOb/tqA+EI+0H0MSQRihRfydchwVqYWAVU
+maLMpZe8orvTT6Av+YkhT8FcG4dc7bzDgOW339nSw1g
+-> ssh-ed25519 +qVung oM1uphTbjI54t4U9jNd1zORqpjBG17MwDf2eNDmOlkg
+oUHVuQt2SHIwtV82pgnKJ7g2jcVBAHWOzPK46otoh34
+-> ssh-rsa krWCLQ
+eYspf5hUKdFQl1RxPaNTj0viAPd+kzp8Xbwn+q6fSITMacmyTY5J8FckLx2YXDxy
+Qm/OsEK0ZOvxnHMrL0oAJjKSy/MamE+9heT3QO+LUN30QxbOIOqHMrl3waadWZdx
+ZGOWK+r+dKGYNsxFv+t1Y/4DBKKzlXFWhJ0aL7nMOqq9+Ca+UZuE41j7eWGGPPLy
+fuW/iOVVxQ+EEeCDpatQSrFPKaeWCCVP9oIDFtE4dsKxubMa4EpUoag0UvEIW182
+UGS8BvMqYgx+obqJDkhXXBK9apmJS2ojcfdtCbNOCV9Ett72Nm/iY5NjLprFMLde
+8wWGA6s3hBOP39lq0eiSxw
+-> ssh-ed25519 /vwQcQ 3zLcLDaDVhIn2knezexYM5Fqu/O9wwORnJIhsXHqgj0
+HchGikQMgkDj0qQgtDdsdKokV+nMjdv6t0uVISeU7Q8
+-> ssh-ed25519 0R97PA 6lm6B6B3dzSdhdcf5rjyTu+7cCtWRxVpWeapJX3nbQo
+x/w4dEfFyxPi4lbNEqgjEblPVfQyj+q1JjeQHiVFhDw
+--- oo5BK1pG+43amUg803Uv511RNtdQ/PDwlXUrV/AbOAA
+…ÙUqÆçïµ[f7ƒêŒë¼¨FìˆY<13>™Ùm¶ØLS?Úℶ‡÷ƒöæ<Kø©F¤z¥V^³U¨N»¯ôƒ)zÔ<7A>¥ž@<40>SÀF€Y‡ËG2^žƒ˜à„»N|

secrets/gerrit-prometheus-bearer-token.age

@@ -0,0 +1,22 @@
+age-encryption.org/v1
+-> ssh-ed25519 2D+APA jiLDQ8JlYhaivXQQhjEfZrGWn7o6Wd2OMrLorEVSPns
+qRzHYcBhtGSm4RW7C4oW+VWSzHiDXkCN6bGeej2Gcpo
+-> ssh-ed25519 j2r2qQ OcnIHB/vJoKuvhsT9dx1B+5lXguARtB9wSquW2KBB3M
+pgzC2KOFi3Yj1gCPemVK3a9Grv2SkwZ6AI1EFdh4hoc
+-> ssh-ed25519 K3b7BA ibHY8wN3rNit1mO2dJZ44rwLylMaR39a7Oz3CGV561o
+4ElWORF/4lVEz33CJiuFG4rwUSIIOyi2L/W7Td7MX5M
+-> ssh-ed25519 +qVung q4DDHS3M24kke2NCcpHEaUbUgoQB6QwnmDiwmdIOuBw
+Yfa6v23oezdDICE8I0UaVCShKlx9lN3DnBnSb63LU64
+-> ssh-rsa krWCLQ
+gLBHP4Z8EBW1y7Yf9sfWMU+/fJ4WWp+NGRR7ebO5GwUeYobDYm/eYQ7rD3Q9k0rF
+kU51GYBaO7m5gLqc2Tq4+YjE2/EXDvjqkDSoyNrjQaaGTLqzvPYlCvKWyROjqJjX
+UwzPbQx5XVIKNgpsR9e6/hoJiJbDpavM+HQo+1zwoKAg5FvZZkE5UnIiSjuAxMgR
++tmrhBfHEYkpbCCrXVE0jLCup8gPIci1PyXWkdhJy+HyHVkbYowGwNawNobNr1cF
+dJ5IU8P/DSSqZ1qWSl6ju7JKjzXU2Xq87/g7wJyrKGpe37pJmPIT86nCJTut+AK9
+iFED/y/p5NCtohyhztosgA
+-> ssh-ed25519 /vwQcQ rzEjV56G+USMdpWklrGQSHuzG8d+S0zWhhwrmuyTyiA
+y+uMRG8NdAD0H4ipRN+sJPn1P0CGs4bk+U4qtetP3O0
+-> ssh-ed25519 0R97PA ULWdDUjDg9oTEOqzCKUJl8yN+qwwmlSi1PFwRvr7aWM
+YWaE+STxKfQzxYMtP/cA20q0atXLdsjeA5nJyl2f8iI
+--- Avs8hTgLwcBy8hyYWjR/Jbs5YaKozv2oBmGs51ckquA
+Wœ·Ü<C2B7>dŸ›ÝàÕò`@½Óµ3ž ‚¼½5è½b›Y%³A†Z=KiÐÑ76,¢w,1žŒèáÎôkØåRšAÄ‚FuÎÎ

secrets/grafana-oauth-secret.age

@@ -1,7 +1,20 @@
 age-encryption.org/v1
--> ssh-ed25519 j2r2qQ Xl0fSOuF0xNTJrtVGdRLRIszd15LFrG5KCFNvSBK4Go
+-> ssh-ed25519 j2r2qQ JSveX4zYEjb4jJH4eg4oXA6r3oc0jBx8NgjhN9JrjlQ
-qSEMBBw90jz4j8elpoUeyS4CTLBhZtNDhLNigesJq+0
+1ZIr/XFClbwJHn0ppJnolpb4QlgZOA8JX5OjjY4x6pU
--> ssh-ed25519 K3b7BA cKI0twKiuuTKv1Js4jqt5v8cOqpxEMY9dmVghgJtbzw
+-> ssh-ed25519 K3b7BA sXUjuZFK0PL/KndxRCJCM5Kg8OmVseRZNWG8mL1alRc
-K5o31XP/nLsswsrMaxnIzCXVUtJqmJWoFglWFsV7+AQ
+U9MMgDtqtmsS1W5i04Pa/b4JBTSjK6FffZxgYI3phtg
---- X8pvqCHeCQ0LjzcjIHThkqp6YeOOT8dBMLuktgdgeY4
+-> ssh-ed25519 +qVung FNSElbiw0frYcsO0xoyPQgRGqAe/aVX21dTB6yk+GQg
-sZÓ¸ŠíØ[þ²X<C2B2>“¡èÅ®Š5°=÷6)ÇT¿Q†‘N{•x³I1ƒ!Ó–ÜøB
ƒzš
*×íåL~K
+zHT/xU+yfXYSBO2HLwoHrGf5ns6BDVb8MlhVVQCBlOc
+-> ssh-rsa krWCLQ
+ye0mLiYeyvlp4EZX7mZ3F7B9V9JSeoiCodzccS+5qIEd6gr+RTHSnKYqwf/nwf8F
+qKLwbxWjpmkIzBWeswy8AJ8159aucGEmB+3/tTSwd+QlRkru4Z/7jtfU64KQttgt
+vaRfc9J/85AJJ2V6Sw/xG8SgxyLBbp/XIN2+tmb0g3kAWiuLcrLk3H/MsfmxDVXg
+RQjugP5K2+fEZc77dHQTrMI58K9TrSw1zYA1ee8J/fl9IJ7J77qi5UgizY+YfX8T
+SmR9DeYUe+hKgCB2k/KgAxp4WOQNgUOFBTsE5FW+kQQpfGx5aqR6vCYU+CPsA3Zb
+FwV0l+g4FUVy+xAtqaGSAQ
+-> ssh-ed25519 /vwQcQ fbnK1jYiUwUsgD8sSTboJCBfcuwJXKNCaJaWYuIfmVk
+Uj2+uBABMTxq1MBsiHXgkdFMOpIN7gfxoJVKOQff1Pw
+-> ssh-ed25519 0R97PA yYOb6AYAFWvm7W2KYT5v9zznkF4Di/vatH48Xgx0x2E
+yUm+MKj9496BkdX2FpLyhML7budUyqT1hL9hpghxSnI
+--- ogCPBrmdbeDorj3t5BL05ge6VngXBpUEDW4qaaKIa0U
+%¨šÚlD]Ϫ?©ßŠÑ(ÿ†E/Wu穉T¶îç[}ž$ÁÍS„Šˆ^[:¸]
he0XUœp¸äq<C3A4>`0A

secrets/hydra-signing-priv.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 +HUDfA NMyTM3c++HKU2klLjAbUUFS81k21LUwEoqR1OUBuLjU
+OrKxpksxoay93URtmN9HhnK43QrM/Gs0qRuENZvHWJI
+-> ssh-ed25519 K3b7BA LloEGN8cbVvGraHs5cPIZRJJyTPFrmmeGwZyqov9m2U
+XPvNpQT3aFVoidOhmePGgiTyytIWtd4rs59Qq9xl/I4
+-> ssh-ed25519 +qVung 3hat0gKIl1WjXnkP6p+/8RyTxZkaVnLgV9B8plICPlY
+jmRKWCUCDpDExmq4SEq8WpqQheBSRD4uqrTgxy2u6PM
+-> ssh-rsa krWCLQ
+QPOl96dmoxY5YtMmL68+6MQpGwZc68ajaRkcEKmYYu4/XB+mffRKsNtyiKJQwEi7
+szvAced8C4RMNrCf3xyF77Sm1UV8YCyaHyplb2/yjv5YDvCDwTp2GnadDoAaLrXU
+jf6ocI8409XWQHEEEofHZRjmfmIBUx1lTwbGFMt48V7MZdadFjXmSmUMvxsu/Rj5
+NLjoPNRBzqPIw6U7nTSmkG2HOeHlA9Z5a33MsXYs8NPH22Spjjy+VvxrLv8VAjnf
+7kGjviW4ZcdEQ7Aox+9V+6qArrIy7lJ9lOIZA2LueCtKhQAmKnInFxRyyN0Nk4ls
+tjlBFJQEG2v14iaHENwRAg
+-> ssh-ed25519 /vwQcQ o7qseMFb4ViV7ylSl2ug7xFZn7GZGqCapWRCq2vyVVs
+hKqzk9BcK5l0VhLfPONKKv6SRnDCw2n+RoaeQbOnT8Y
+-> ssh-ed25519 0R97PA DTEowwoCXTMGxfQIXOnwn5fjlih0UmQJCKs2II4gCVY
+8BpVhUV8qg3zcCJe7OwHoJrfYIpBtOquqhFyfZx7mRQ
+--- r2NmEyV0/Goas5lXMHeFoafcrbMHvs4ob0Zg4cVil3w
+=‚TÏ`Ô}%xÖ¸œêFtÆêlãµ)•ò1Ð]Œ<DvwFøè:Qf@nÇÀU=~FžË-Ò!©ò÷þÌ«i6wXtü'k®ç`)A›äûÌtI1Tî£U+Ú~×’aÒÉПONƒÄèLþI£p@œT)¤'_ÞoDö0»,JfGVó­k’/

secrets/loki-environment.age

@@ -1,9 +1,21 @@
 age-encryption.org/v1
--> ssh-ed25519 j2r2qQ w0lLquFUUcmEZ/Fh1YSt85tAJkBwavORQbwMr7gMqF4
+-> ssh-ed25519 j2r2qQ 6qyr94uky6B36UOY0jd5NXgF2rJ3RWBUzZ32c5iOTmY
-J4T+EHm1uHbCZkAUNoNcB9uGSz082mFL8+dkCnvYQnM
+fjlI3fjYjwyNQBs4K4pq/5c7oBkf5XUXoGlBOBpmPu4
--> ssh-ed25519 K3b7BA 28bJZgBPPc2KIE5+b8LJuQ5L4YAiRAJzucEuOqXHdVM
+-> ssh-ed25519 K3b7BA N9VYT/ZslG07KldzO8sPE5TiYYwxJqpYU87ED4PuBXw
-7hKENFr8QX0jpwuuQEjGFrUywJuhL1Tdi2V4/gR8JWE
+P1s9L57prPqM4fjcYHv+g0rgP/NvFr13CgCxthVHZ4c
---- GSPZxz39TMMWv0qhotNgnXa5679Q7VK8JGjQjI7A8oM
+-> ssh-ed25519 +qVung Ry8uUFsmYmP+Urw46lhAsCc3S+QiWu1mn8J3rIy+KFQ
-J˛\@F“N•	łĺ…2‹®ô¨w×!Ż1V
f»§<C2BB>Ž·’ŢO˛CÓw®®V°ŁšĚş.^݆ 7Ťw‡n4äŕdW-Öľ"@0¨úąEĎż·°ck,]M}xŤřĚťˇŰy°[×ÁJ:!č‘	!ř螀c¬
+iB7xAfdpHwOzAnLvosJb+F50QKsOYWr7CHC3srsS6ME
-BëąR
+-> ssh-rsa krWCLQ
-nřę€ţŔáĆ^9í¤–M<ú
+w0xIVFtUghdAO7SxZD10rBMtdQESEvYUEKxnWzLh0cjcRhaVT/BXSZQsKV2Rupoo
+nDL5uy0k+tPXm0HroZ6VkZ0fH/lOpeUR69ZvJmClKql3Fnf1385+5BvT719cbbaq
+yll49gx0+ms/oB9jS3SPwbOg+UJgnkZCeu9138h3MG7yWNtVuA9l5hsJioVvOVlS
+Z5EXbjdQR9xYjSwR+b8MYZ97ej5fXpuULEopbx2wXt84u1e67vTETqflitR7lrzy
+A6F65g35aagPJZGHzfrKVToy3pfXm9ky/30DolWLD0DpG7G6o/8afy8O4yBAGlv3
+ZLTaUbrdILSz2ff1Njx4Nw
+-> ssh-ed25519 /vwQcQ YqqmX/f4whOk97kCgSPo6oj/274eYlBWtS+OahAAQ34
+hoCbhupzSTx+wNIorzYGHyGvU/L8unKEyD7Bqq23YP0
+-> ssh-ed25519 0R97PA 17SDtfT9GzAsIsQB24AmYXpW8v4+LEakup+tdFroHTk
+HIvBhAGA2GMVWFBP3OTFEn+XpPFBJDOJDK3SQ94mNKM
+--- CD1QrxYGAhhy+l7U5kOXn1shCwz8pYJNuGRugPxmzJw
+ñY¾Æ‹N	Ï<>x
™êÿrR^z[¤ã¸è…•ªa”z
+óæÔÉ¿Ïžu0c¯c;y<>Ÿ¢›&{ñèxA]‚þ†¨Q¨¼_:̱ í€öUoiDl (‹Å
ëwÝKi,j.oFyÌ°$}•Y§@1”È™„Y£²è¶u Ò*¡ÏþÅ<C3BE>¥™0…

secrets/metrics-push-htpasswd.age

@@ -1,7 +1,21 @@
 age-encryption.org/v1
--> ssh-ed25519 j2r2qQ nLWy3WcVJWCl3rXkhcSbp1joqmkk06QnxhCZ4UtSvmw
+-> ssh-ed25519 j2r2qQ sIYTVOTWNToDSNa4qiIaSoac7zka54g/opQ70q1SAA8
-iQ+Hx/vhiFgkWfbxHwGjxMBEqzyGww4/9do3W7V/y1Y
+2Z1mlCWxjakHqRbArU2BkT7B/Dx0XKH7kCnBa+OYI+s
--> ssh-ed25519 K3b7BA RkF2ADcjOGtivl9MrhO/HFwxlTAkbFHWL3iinUldMiM
+-> ssh-ed25519 K3b7BA PGyd27M/Hmk6qpRf8bcI4QWrS0vrPgjiZzaXvKQkJDQ
-7q/zdVTMLevukZjkHtcN88iYzfTLvq2s3QdkgsFSO9M
+ixrciiNR/th0FM9MxVx/omHdI61EmAhTA465SjxECF8
---- 1b2HiK06vJPqBgHVDD0QELOtfkl7/rlgGS9uI1mSbus
+-> ssh-ed25519 +qVung Q7k74fDLKwCdzobz0b6ByS2LrhMOIC58Ofto0gpBLFE
-„uܧoL;őĺ¬"
4¦Ű»ZĽ<5A>@§öă<C3B6>Đ’3+93Q4óÄ o•ŚŘwé“„6ŤM-˛DkJn´;ń*g
<0A>OŰYś75ËSň)Ů°©
+p4CIje+sO/nOaO1lzAY9n2HYLUKxEvKDbxeR6dOyM00
+-> ssh-rsa krWCLQ
+ezrZTitn0/BRD0K7e2K53qz9AZCa0aHlzFSuyzqyVJLdAZUxBUnfBwmGuJgKTa4Q
+fWsXBs+L65hkcL6/VKS7oSGGyoEHmoPFKbb08B6FKLHt9V1td5xbHIoTYbvSavUA
+g3wpTUa4eG3ivcu96VjyyBKTAc7LN7h7dSMbvvP5tpWT5vL+WstCdFf7zzUL9HBS
+yI8dzEbCQIgAAaHj90MREgIIgIB27Dn1PvkEBGYky5ybBRa3DXVyqnX0dDtsXWpK
+ipRPDV7HC1+x2TlqQjD5ED737r/AP573IXbnRLSEWnGDjtd/JWQmfOO3JACoRjU6
+qfb5SSDT9QriuWSow7CDhQ
+-> ssh-ed25519 /vwQcQ duuo3BGe4Q1MHMljgzmtpzvtiOvAHqKu2HS9SBxLuhE
+GCwccbE5lX5uPIri/7Vn6hzpfL7ouJBFU14bKjl6yTM
+-> ssh-ed25519 0R97PA WIFf8tbMlmNrNFF5tRcL+mOJ40SvIdppAtItWtxzCk8
+miU7Z4poEVMZCeAEef1VS0jouCDxGro2xLEE3hnRJEQ
+--- Iaff5rxl9r1qEnlpkOpGyBGtAvGMLyBlJQ45iInuAnw
+cýI±C«¤2ˆ7µ	½³Ú“nZMþ`œ{7È`¨½V@ñyzÀÅžª€)ÛY‰DÄßÇX—o“óä	~<PÙ›òš5Tpúx
+ÓRÏÜö

services/baremetal-builder/default.nix

@@ -0,0 +1,163 @@
+{ pkgs, lib, config, ... }:
+let
+  cfg = config.bagel.baremetal.builders;
+in
+{
+  options = {
+
+    bagel.baremetal.builders = {
+      enable = lib.mkEnableOption "baremetal bagel oven";
+      num = lib.mkOption {
+        type = lib.types.int;
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    boot.initrd.availableKernelModules = [ "ahci" "ehci_pci" "usb_storage" "usbhid" "sd_mod" ];
+    boot.initrd.kernelModules = [ "dm-snapshot" ];
+
+    users.users.builder = {
+      isSystemUser = true;
+      group = "nogroup";
+      home = "/var/empty";
+      shell = "/bin/sh";
+      openssh.authorizedKeys.keys = [
+        # Do not hardcode Hydra's public key, selectively
+        # add the keys of the coordinators that require us.
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAvUT9YBig9LQPHgypIBHQuC32XqDKxlFZ2CfgDi0ZKx"
+      ];
+    };
+
+    users.users.buildbot = {
+      isSystemUser = true;
+      group = "nogroup";
+      home = "/var/empty";
+      shell = "/bin/sh";
+      openssh.authorizedKeys.keys = [
+        # Do not hardcode Buildbot's public key, selectively
+        # add the keys of the coordinators that require us.
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMnOLLX0vGTZbSJrUmF9ZFXt/NIId/MUrEpXmL2vxod"
+      ];
+    };
+    nix.settings.trusted-users = [ "builder" "buildbot" ];
+
+
+    nixpkgs.hostPlatform = "x86_64-linux";
+    hardware.cpu.intel.updateMicrocode = true;
+
+    boot.loader.systemd-boot.enable = true;
+    boot.loader.efi.canTouchEfiVariables = true;
+    boot.initrd.systemd.enable = true;
+
+    boot.initrd.services.lvm.enable = true;
+
+    fileSystems."/" = {
+      device = "/dev/disk/by-label/root";
+      fsType = "xfs";
+    };
+
+    fileSystems."/mnt" = {
+      device = "/dev/disk/by-label/hydra";
+      fsType = "xfs";
+    };
+    # We want the tmp filesystem on the same filesystem as the hydra store, so that builds can use reflinks
+    fileSystems."/tmp" = {
+      device = "/mnt/tmp";
+      options = [ "bind" ];
+    };
+
+    fileSystems."/boot" = {
+      device = "/dev/disk/by-label/BOOT";
+      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
+    };
+
+    swapDevices = [
+      {
+        device = "/swapfile";
+        size = 50 * 1024;  # 50GiB
+      }
+    ];
+
+    zramSwap = {
+      enable = true;
+      memoryPercent = 25;
+    };
+
+    boot.kernelParams = [
+      "console=ttyS0,115200"
+      "console=tty1"
+    ];
+
+    networking.useNetworkd = true;
+    networking.hostName = "builder-${toString cfg.num}";
+    networking.domain = "wob01.infra.forkos.org";
+
+    systemd.network = {
+      netdevs = {
+        "40-uplink" = {
+          netdevConfig = {
+            Kind = "bond";
+            Name = "uplink";
+          };
+          bondConfig = {
+            Mode = "802.3ad";
+            TransmitHashPolicy = "layer3+4";
+          };
+        };
+      };
+      networks = {
+        "40-eno1" = {
+          name = "eno1";
+          bond = [ "uplink" ];
+        };
+        "40-eno2" = {
+          name = "eno2";
+          bond = [ "uplink" ];
+        };
+      };
+    };
+    networking.interfaces.uplink.ipv6.addresses = [
+      { address = "2a01:584:11::1:${toString cfg.num}"; prefixLength = 64; }
+    ];
+    networking.defaultGateway6 = { interface = "uplink"; address = "2a01:584:11::1"; };
+    deployment.targetHost = "2a01:584:11::1:${toString cfg.num}";
+    deployment.tags = [ "builders" ];
+
+    # Why can't we have nice things? https://bugs.openjdk.org/browse/JDK-8170568
+    services.coredns = {
+      enable = true;
+      config = ''
+        . {
+          bind lo
+          forward . 2001:4860:4860::6464
+          template ANY A { rcode NOERROR }
+        }
+      '';
+    };
+    services.resolved.enable = false;
+    networking.resolvconf.useLocalResolver = true;
+
+    # Hydra blasts ssh connections and does not multiplex. Loosen some of the
+    # rate limiting.
+    services.openssh.settings = {
+      MaxStartups = "500:30:1000";
+    };
+
+    systemd.services.hydra-gc = {
+      description = "Nix Garbage Collector";
+      script = "exec ${config.nix.package.out}/bin/nix-store --gc --store /mnt";
+      serviceConfig.Type = "oneshot";
+      serviceConfig.User = "builder";
+      startAt = "*-*-* 00/8:00:00";
+    };
+    systemd.timers.hydra-gc.timerConfig.Persistent = true;
+
+    bagel.sysadmin.enable = true;
+
+    environment.systemPackages = [ pkgs.ipmitool ];
+
+    system.stateVersion = "24.05";
+  };
+}

services/buildbot/default.nix

@@ -0,0 +1,132 @@
+{
+  nodes,
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.bagel.services.buildbot;
+  cfgGerrit = nodes.gerrit01.config.bagel.services.gerrit;
+  ssh-keys = import ../../common/ssh-keys.nix;
+  inherit (lib) mkEnableOption mkOption mkIf types;
+in
+{
+  options.bagel.services.buildbot = {
+    enable = mkEnableOption "Buildbot";
+    domain = mkOption {
+      type = types.str;
+    };
+
+    builders = mkOption {
+      type = types.listOf types.str;
+      description = "List of builders to configure for Buildbot";
+      example = [ "builder-2" "builder-3" ];
+    };
+  };
+
+  config = mkIf cfg.enable {
+    networking.firewall.allowedTCPPorts = [ 80 443 ];
+    age.secrets.buildbot-worker-password.file = ../../secrets/buildbot-worker-password.age;
+    age.secrets.buildbot-oauth-secret.file = ../../secrets/buildbot-oauth-secret.age;
+    age.secrets.buildbot-workers.file = ../../secrets/buildbot-workers.age;
+    age.secrets.buildbot-service-key.file = ../../secrets/buildbot-service-key.age;
+    age.secrets.buildbot-signing-key.file = ../../secrets/buildbot-signing-key.age;
+    age.secrets.buildbot-remote-builder-key.file = ../../secrets/buildbot-remote-builder-key.age;
+
+    services.nginx.virtualHosts.${cfg.domain} = {
+      forceSSL = true;
+      enableACME = true;
+      extraConfig = ''
+        add_header Access-Control-Allow-Credentials 'true' always;
+        add_header Access-Control-Allow-Origin 'https://cl.forkos.org' always;
+      '';
+    };
+
+    services.buildbot-nix.worker = {
+      enable = true;
+      workerPasswordFile = config.age.secrets.buildbot-worker-password.path;
+      # All credits to eldritch horrors for this beauty.
+      workerArchitectures =
+      {
+        # nix-eval-jobs runs under a lock, error reports do not (but are cheap)
+        other = 8;
+      } // (
+        lib.filterAttrs
+          (n: v: lib.elem n config.services.buildbot-nix.coordinator.buildSystems)
+          (lib.zipAttrsWith
+            (_: lib.foldl' lib.add 0)
+            (lib.concatMap
+              (m: map (s: { ${s} = m.maxJobs; }) m.systems)
+              config.nix.buildMachines))
+        );
+    };
+
+    services.buildbot-nix.coordinator = {
+      enable = true;
+
+      inherit (cfg) domain;
+
+      oauth2 = {
+        name = "Lix";
+        clientId = "forkos-buildbot";
+        clientSecretFile = config.age.secrets.buildbot-oauth-secret.path;
+        resourceEndpoint = "https://identity.lix.systems";
+        authUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth";
+        tokenUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
+      };
+
+      # TODO(raito): this is not really necessary, we never have remote buildbot workers.
+      # we can replace all of this with automatic localworker generation on buildbot-nix side.
+      workersFile = config.age.secrets.buildbot-workers.path;
+
+      allowedOrigins = [
+        "*.forkos.org"
+      ];
+
+      # TODO(raito): is that really necessary when we can just collect buildMachines' systems?
+      buildSystems = [
+        "x86_64-linux"
+      ];
+
+      buildMachines = map (n: {
+          hostName = nodes.${n}.config.networking.fqdn;
+          protocol = "ssh-ng";
+          # Follows Hydra.
+          maxJobs = 8;
+          sshKey = config.age.secrets.buildbot-remote-builder-key.path;
+          sshUser = "buildbot";
+          systems = [ "x86_64-linux" ];
+          supportedFeatures = nodes.${n}.config.nix.settings.system-features;
+          # Contrary to how Nix works, here we can specify non-base64 public host keys.
+          publicHostKey = ssh-keys.machines.${n};
+        }
+      ) cfg.builders;
+
+      gerrit = {
+        domain = cfgGerrit.canonicalDomain;
+        # Manually managed account…
+        # TODO: https://git.lix.systems/the-distro/infra/issues/69
+        username = "buildbot";
+        port = cfgGerrit.port;
+        privateKeyFile = config.age.secrets.buildbot-service-key.path;
+        projects = [
+          "buildbot-test"
+          "nixpkgs"
+          "infra"
+        ];
+      };
+
+      evalWorkerCount = 6;
+      evalMaxMemorySize = "4096";
+
+      signingKeyFile = config.age.secrets.buildbot-signing-key.path;
+    };
+
+    nix.settings.keep-derivations = true;
+    nix.gc = {
+      automatic = true;
+      dates = "hourly";
+    };
+  };
+}

services/default.nix

@@ -2,9 +2,13 @@
   imports = [
     ./gerrit
     ./hydra
+    ./matrix
     ./monitoring
     ./netbox
     ./ofborg
     ./postgres
+    ./forgejo
+    ./baremetal-builder
+    ./buildbot
   ];
 }

services/forgejo/default.nix

@@ -0,0 +1,139 @@
+{ pkgs, lib, config, ... }:
+
+
+let
+  cfg = config.bagel.services.forgejo;
+  inherit (lib) mkIf mkEnableOption mkOption types;
+
+  domain = "git.forkos.org";
+in
+{
+  options.bagel.services.forgejo = {
+    enable = mkEnableOption "Forgejo";
+    sshBindAddr = mkOption {
+      type = types.str;
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.forgejo = {
+      enable = true;
+
+      package = pkgs.callPackage ../../pkgs/forgejo { };
+
+      database = {
+        type = "postgres";
+        createDatabase = true;
+      };
+
+      lfs.enable = true;
+
+      settings = {
+        DEFAULT = {
+          APP_NAME = "ForkOS";
+        };
+
+        server = {
+          PROTOCOL = "http+unix";
+          ROOT_URL = "https://${domain}/";
+          DOMAIN = "${domain}";
+
+          BUILTIN_SSH_SERVER_USER = "git";
+          SSH_PORT = 22;
+          SSH_LISTEN_HOST = cfg.sshBindAddr;
+          START_SSH_SERVER = true;
+        };
+
+        session = {
+          PROVIDER = "redis";
+          PROVIDER_CONFIG = "network=unix,addr=${config.services.redis.servers.forgejo.unixSocket},db=0";
+          COOKIE_NAME = "session";
+        };
+
+        service = {
+          DISABLE_REGISTRATION = true;
+          DEFAULT_KEEP_EMAIL_PRIVATE = true;
+        };
+
+        oauth2_client = {
+          REGISTER_EMAIL_CONFIRM = false;
+          ENABLE_AUTO_REGISTRATION = true;
+        };
+
+        # TODO: transactional mails
+
+        cache = {
+          ADAPTER = "redis";
+          HOST = "network=unix,addr=${config.services.redis.servers.forgejo.unixSocket},db=1";
+          ITEM_TTL = "72h"; # increased from default 16h
+        };
+
+        ui = {
+          SHOW_USER_EMAIL = false;
+        };
+
+        repository = {
+          # Forks in forgejo are suprisingly expensive because they are full git clones.
+          # If we do want to enable forks, we can write a small patch that disables
+          # only for repositories that are as large as nixpkgs.
+          DISABLE_FORKS = true;
+        };
+
+        packages = {
+          # Forgejo's various package registries can easily take up a lot of space.
+          # We could either store the blobs on some slower disks but larger, or even
+          # better, use an s3 bucket for it. But until we actually have a use-case for
+          # this feature, we will simply keep it disabled for now.
+          ENABLED = false;
+        };
+
+        indexer = {
+          REPO_INDEXER_REPO_TYPES = "sources,mirrors,templates"; # skip forks
+          REPO_INDEXER_ENABLED = true;
+          ISSUE_INDEXER_TYPE = "bleve";
+        };
+
+        "git.timeout" = {
+          MIGRATE = 3600; # increase from default 600 (seconds) for something as large as nixpkgs on a slow uplink
+        };
+
+        log = {
+          LEVEL = "Warn";
+        };
+      };
+    };
+
+    systemd.services.forgejo = {
+      serviceConfig = lib.optionalAttrs (config.services.forgejo.settings.server.SSH_PORT < 1024) {
+        AmbientCapabilities = lib.mkForce "CAP_NET_BIND_SERVICE";
+        CapabilityBoundingSet = lib.mkForce "CAP_NET_BIND_SERVICE";
+        PrivateUsers = lib.mkForce false;
+      };
+
+      # start Forgejo *after* sshd.service, so in case Forgejo tries to wildcard bind :22 due to
+      # a bug or whatever, we don't lose OpenSSH in a race.
+      wants = [ "sshd.service" "redis-forgejo.service" ];
+      requires = [ "sshd.service" "redis-forgejo.service" ];
+    };
+
+    services.redis.servers.forgejo = {
+      enable = true;
+      user = "forgejo";
+    };
+
+    services.nginx = {
+      enable = true;
+      virtualHosts.${domain} = {
+        enableACME = true;
+        forceSSL = true;
+        locations."/".proxyPass = "http://unix:${config.services.forgejo.settings.server.HTTP_ADDR}";
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = [
+      80
+      443
+      config.services.forgejo.settings.server.SSH_PORT
+    ];
+  };
+}

services/gerrit/checks.js

@@ -0,0 +1,113 @@
+/* Inspired from the Lix setup.
+ * Original-Author: puckipedia
+ */
+Gerrit.install((plugin) => {
+  // TODO: can we just use `plugin.serverInfo().plugin` and control the settings over there.
+  const configuration = {
+    baseUri: @BASE_URI@,
+    supportedProjects: @SUPPORTED_PROJECTS@,
+  };
+
+  function makeBuildbotUri(suffix) {
+    return `${configuration.baseUri}/${suffix}`;
+  }
+
+  let builders = [];
+  let fetchBuilders = async () => {
+    if (builders.length > 0) return;
+      let data = await (await fetch(makeBuildbotUri(`api/v2/builders`), { credentials: 'include' })).json();
+      builders = data.builders;
+  };
+
+
+  let checksProvider;
+  checksProvider = {
+    async fetch({ repo, patchsetSha, changeNumber, patchsetNumber }, runBefore = false) {
+      if (!configuration.supportedProjects.includes(repo)) {
+        return { responseCode: 'OK' };
+      }
+
+      let num = changeNumber.toString(10);
+
+      let branch = `refs/changes/${num.substr(-2)}/${num}/${patchsetNumber}`;
+
+      let changeFetch = await fetch(makeBuildbotUri(`api/v2/changes?limit=1&order=-changeid&revision=${patchsetSha}&branch=${branch}`), { credentials: 'include' });
+      if (changeFetch.status == 400) {
+        if ((await changeFetch.json()).error === 'invalid origin' && !runBefore) {
+          return await checksProvider.fetch({ repo, patchsetSha, changeNumber, patchsetNumber }, true);
+        }
+
+        return { responseCode: 'OK' };
+      } else if (changeFetch.status === 403) {
+        return { responseCode: 'NOT_LOGGED_IN', loginCallback() {
+          window.open(configuration.baseUri);
+        } };
+      }
+
+      let changes = await changeFetch.json();
+      if (changes.meta.total === 0) {
+        return { responseCode: 'OK' };
+      }
+
+      let { changeid } = changes.changes[0];
+      let { builds } = await (await fetch(makeBuildbotUri(`api/v2/changes/${changeid}/builds?property=owners&property=workername`), { credentials: 'include' })).json();
+      await fetchBuilders();
+      let links = [];
+      let runs = [];
+      for (let build of builds) {
+        let name = `unknown builder ${build.builderid}`;
+        for (let builder of builders) {
+          if (builder.builderid === build.builderid) {
+            name = builder.name;
+            break;
+          }
+        }
+
+        if (name === `${repo}/nix-eval`) {
+            links.push({
+              url: makeBuildbotUri(`#/builders/${build.builderid}/builds/${build.number}`),
+              primary: true,
+              icon: 'external',
+            });
+        }
+
+        let checkrun = {
+          attempt: build.buildrequestid,
+          // FIXME: generalize this accordingly once auto-discovery is available.
+          checkName: name.replace(/^hydraJobs\./, ''),
+          externalId: build.buildrequestid.toString(),
+          status: build.complete ? 'COMPLETED' : (typeof build.started_at !== 'number' ? 'SCHEDULED' : 'RUNNING'),
+          checkLink: makeBuildbotUri(`#/builders/${build.builderid}/builds/${build.number}`),
+          labelName: 'Verified',
+          results: [],
+          links: [{
+            url: makeBuildbotUri(`#/builders/${build.builderid}/builds/${build.number}`),
+            primary: true,
+            icon: 'external',
+          }],
+        };
+
+        if (build.started_at !== null) {
+          checkrun.startedTimestamp = new Date(build.started_at * 1000);
+        }
+
+        if (build.complete_at !== null) {
+          checkrun.finishedTimestamp = new Date(build.complete_at * 1000);
+        }
+
+        if (build.results !== null) {
+          checkrun.results = [{
+            category: build.results < 2 ? 'SUCCESS' : 'ERROR',
+            summary: build.state_string,
+          }];
+        }
+
+        runs.push(checkrun);
+      }
+
+      return { responseCode: 'OK', runs, links };
+    }
+  };
+
+  plugin.checks().register(checksProvider);
+});

services/gerrit/default.nix

@@ -3,7 +3,7 @@
 { pkgs, config, lib, ... }:
 
 let
-  inherit (lib) mkEnableOption mkIf mkOption types;
+  inherit (lib) mkEnableOption mkIf mkOption types head;
   cfgGerrit = config.services.gerrit;
   cfg = config.bagel.services.gerrit;
 
@@ -16,19 +16,31 @@ in
       type = types.listOf types.str;
       description = "List of domains that Gerrit will answer to";
     };
+    canonicalDomain = mkOption {
+      type = types.str;
+      description = "Canonical domain for this Gerrit instance";
+      default = head cfg.domains;
+    };
     data = mkOption {
       type = types.path;
       default = "/var/lib/gerrit";
       description = "Root of data directory for the Gerrit";
     };
+    port = mkOption {
+      type = types.port;
+      default = 29418;
+      readOnly = true;
+      description = "Port for the Gerrit SSH server";
+    };
   };
 
   imports = [
     ./www.nix
+    ./one-way-sync.nix
   ];
 
   config = mkIf cfg.enable {
-    networking.firewall.allowedTCPPorts = [ 29418 ];
+    networking.firewall.allowedTCPPorts = [ cfg.port ];
 
     environment.systemPackages = [ jdk ];
 
@@ -60,6 +72,22 @@ in
 
       plugins = with pkgs.gerritPlugins; [
         oauth
+        metrics-reporter-prometheus
+        # Buildbot checks plugin (writeText because services.gerrit.plugins expects packages)
+        (pkgs.runCommand "checks.js" {
+          BASE_URI = builtins.toJSON "https://buildbot.forkos.org";
+          SUPPORTED_PROJECTS = builtins.toJSON [
+            "infra"
+            "nixpkgs"
+            "buildbot-test"
+          ];
+        }
+        ''
+          echo "configuring buildbot checks plugin for $BASE_URI with $SUPPORTED_PROJECTS project list"
+          substitute ${./checks.js} $out \
+            --replace-fail "@BASE_URI@" "$BASE_URI" \
+            --replace-fail "@SUPPORTED_PROJECTS@" "$SUPPORTED_PROJECTS"
+        '')
       ];
 
       package = pkgs.gerrit;
@@ -101,7 +129,7 @@ in
         core.packedGitMmap = true;
 
         ## Takes more CPU but the transfer is smaller.
-        pack.deltacompression = false;
+        pack.deltacompression = true;
         pack.threads = 8;
 
         # FIXME(raito):
@@ -114,7 +142,7 @@ in
         # Other settings
         log.jsonLogging = true;
         log.textLogging = false;
-        sshd.advertisedAddress = "cl.forkos.org:29418";
+        sshd.advertisedAddress = "${cfg.canonicalDomain}:${toString cfg.port}";
         cache.web_sessions.maxAge = "3 months";
         plugins.allowRemoteAdmin = false;
         change.enableAttentionSet = true;
@@ -129,7 +157,7 @@ in
         # Configures gerrit for being reverse-proxied by nginx as per
         # https://gerrit-review.googlesource.com/Documentation/config-reverseproxy.html
         gerrit = {
-          canonicalWebUrl = "https://cl.forkos.org";
+          canonicalWebUrl = "https://${cfg.canonicalDomain}";
           docUrl = "/Documentation";
           defaultBranch = "refs/heads/main";
         };
@@ -146,7 +174,7 @@ in
         # Auto-link other CLs
         commentlink.gerrit = {
           match = "cl/(\\d+)";
-          link = "https://cl.forkos.org/$1";
+          link = "https://${cfg.canonicalDomain}/$1";
         };
 
         # Configures integration with Keycloak, which then integrates with a
@@ -218,6 +246,14 @@ in
         User = "git";
         Group = "git";
       };
+      environment.REVWALK_USE_PRIORITY_QUEUE = "true";
+    };
+
+    age.secrets.gerrit-prometheus-bearer-token.file = ../../secrets/gerrit-prometheus-bearer-token.age;
+    bagel.monitoring.grafana-agent.exporters.gerrit = {
+      port = 4778;  # grrt
+      bearerTokenFile = config.age.secrets.gerrit-prometheus-bearer-token.path;
+      scrapeConfig.metrics_path = "/plugins/metrics-reporter-prometheus/metrics";
     };
   };
 }

services/gerrit/one-way-sync.nix

@@ -0,0 +1,132 @@
+{ lib, config, pkgs, ... }:
+let
+  cfg = config.bagel.nixpkgs.one-way-sync;
+  inherit (lib) mkIf mkOption mkEnableOption types mapAttrs';
+
+  mkSyncTimer = { name, timer, ... }: {
+    wantedBy = [ "timers.target" ];
+
+    timerConfig = {
+      OnCalendar = timer;
+      Persistent = true;
+      Unit = "ows-${name}.service";
+    };
+  };
+  mkSyncService = targetRef: { name, fromUri, fromRefspec, localRefspec, ... }: {
+    path = [ pkgs.gitFull pkgs.openssh pkgs.lix ];
+    script = ''
+      set -xe
+      RUNTIME_DIRECTORY="/run/onewaysync-${name}"
+      trap "git worktree remove -f "$RUNTIME_DIRECTORY"/${name}" EXIT
+
+      if [ ! -d "/var/lib/onewaysync/nixpkgs" ]; then
+        echo "First run, synchronizing nixpkgs..."
+        git clone https://cl.forkos.org/nixpkgs /var/lib/onewaysync/nixpkgs
+      fi
+
+      cd /var/lib/onewaysync/nixpkgs
+      echo "Syncing ${fromUri}:${fromRefspec} to /var/lib/onewaysync/nixpkgs:${targetRef}"
+      echo "Current ref: $EXPECTED_REF"
+      git worktree add -f "$RUNTIME_DIRECTORY"/${name} refs/remotes/origin/${localRefspec}
+      cd "$RUNTIME_DIRECTORY"/${name}
+      git pull origin ${localRefspec}
+      EXPECTED_REF=$(git rev-list refs/remotes/origin/${localRefspec} | head -1)
+      git config user.name Fork-o-Tron
+      git config user.email noreply@forkos.org
+      git fetch ${fromUri} ${fromRefspec}
+    '' + lib.optionalString (!(lib.hasInfix "staging" localRefspec)) ''
+      OLD_STDENV=$(nix eval -f . stdenv.outPath --store "$RUNTIME_DIRECTORY")
+    '' + ''
+      git merge FETCH_HEAD
+    '' + lib.optionalString (!(lib.hasInfix "staging" localRefspec)) ''
+      NEW_STDENV=$(nix eval -f . stdenv.outPath --store "$RUNTIME_DIRECTORY")
+      # Do not allow auto-merging a staging iteration
+      test "$OLD_STDENV" = "$NEW_STDENV"
+    '' + ''
+      GIT_SSH_COMMAND='ssh -i ${cfg.deployKeyPath}' git push ${cfg.pushUrl} HEAD:${targetRef}
+    '';
+    serviceConfig = {
+      User = "git";
+      Group = "git";
+      Type = "oneshot";
+      RuntimeDirectory = "onewaysync-${name}";
+      WorkingDirectory = "/run/onewaysync-${name}";
+      StateDirectory = "onewaysync";
+    };
+  };
+in
+{
+  options.bagel.nixpkgs.one-way-sync = {
+    enable = mkEnableOption "the one-way sync from GitHub repositories";
+
+    referenceDir = mkOption {
+      type = types.str;
+      default = "/var/lib/gerrit/git/nixpkgs.git";
+      description = "Local repository reference";
+    };
+
+    workingDir = mkOption {
+      type = types.str;
+      default = "/run/onewaysync/";
+      description = "Working directory for the service";
+    };
+
+    pushUrl = mkOption {
+      type = types.str;
+      example = "ssh://...";
+      description = "Push URL for the target repository";
+    };
+
+    deployKeyPath = mkOption {
+      type = types.path;
+      example = "/run/agenix.d/ows-priv-key";
+      description = "Deployment private SSH key to push to the repository";
+    };
+
+    branches = mkOption {
+      type = types.attrsOf (types.submodule ({ ... }:
+      {
+        options = {
+          name = mkOption {
+            type = types.str;
+            description = "User-friendly name";
+          };
+
+          fromUri = mkOption {
+            type = types.str;
+            description = "Git URI from which we need to sync";
+          };
+
+          fromRefspec = mkOption {
+            type = types.str;
+            description = "refspec for the fetch";
+          };
+
+          localRefspec = mkOption {
+            type = types.str;
+            default = "local refspec in the local repository to get the expected reference and avoid stale info";
+          };
+
+          timer = mkOption {
+            type = types.str;
+            description = "Calendar format everytime we need to run the sync";
+          };
+        };
+      }));
+
+      description = "Set of branches mapping from cl.forkos.org to other Git repositories";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.timers = mapAttrs' (name: value: {
+      name = "ows-${value.name}";
+      value = mkSyncTimer value;
+    }) cfg.branches;
+
+    systemd.services = mapAttrs' (name: value: {
+      name = "ows-${value.name}";
+      value = mkSyncService name value;
+    }) cfg.branches;
+  };
+}

services/gerrit/www.nix

@@ -25,7 +25,7 @@ in
             # The :443 suffix is a workaround for https://b.tvl.fyi/issues/88.
             proxy_set_header  Host $host:443;
             # Gerrit can throw a lot of data.
-            proxy_buffering on;
+            proxy_buffering off;
             # NGINX should not give up super fast. Things can take time.
             proxy_read_timeout 3600;
           }

services/hydra/default.nix

@@ -1,14 +1,30 @@
-{ config, lib, pkgs, ... }:
+{ nodes, config, lib, pkgs, ... }:
 
 let
   cfg = config.bagel.services.hydra;
+  ssh-keys = import ../../common/ssh-keys.nix;
 
   narCacheDir = "/var/cache/hydra/nar-cache";
   port = 3000;
 
   mkCacheSettings = settings: builtins.concatStringsSep "&" (
     lib.mapAttrsToList (k: v: "${k}=${v}") settings
-  );
+    );
+
+  # XXX: to support Nix's dumb public host key syntax (base64'd), this outputs
+  # a string with shell-style command interpolations: $(...).
+  mkBaremetalBuilder = { parallelBuilds, publicHostKey, host, speedFactor ? 1, user ? "builder", supportedSystems ? [ "i686-linux" "x86_64-linux" ], supportedFeatures ? [ "big-parallel" "kvm" "nixos-test" ] }:
+  "ssh://${user}@${host}?remote-store=/mnt ${lib.concatStringsSep "," supportedSystems} ${config.age.secrets.hydra-ssh-key-priv.path} ${toString parallelBuilds} ${toString speedFactor} ${lib.concatStringsSep "," supportedFeatures} - $(echo -n '${publicHostKey}' | base64 -w0)";
+
+  # TODO:
+  # - generalize to new architectures
+  # - generalize to new features
+  baremetalBuilders = lib.concatStringsSep "\n"
+    (map (n: mkBaremetalBuilder {
+      parallelBuilds = 8; # TODO: do not hardcode this, use the node's builder configuration.
+      publicHostKey = ssh-keys.machines.${n};
+      host = nodes.${n}.config.networking.fqdn;
+    }) cfg.builders);
 in {
   options.bagel.services.hydra = with lib; {
     enable = mkEnableOption "Hydra coordinator";
@@ -17,11 +33,24 @@ in {
       type = types.str;
       description = "DBI connection string for the Hydra postgres database";
     };
+
+    builders = mkOption {
+      type = types.listOf types.str;
+      description = "List of builders to configure for Hydra";
+      example = [ "builder-0" "builder-1" ];
+    };
   };
 
   config = lib.mkIf cfg.enable {
+    # TODO: we should assert or warn that the builders
+    # does indeed have our public SSH key and are *builders*
+    # as a simple evaluation preflight check.
+
     age.secrets.hydra-s3-credentials.file = ../../secrets/hydra-s3-credentials.age;
 
+    age.secrets.hydra-signing-priv.owner = "hydra-queue-runner";
+    age.secrets.hydra-signing-priv.file = ../../secrets/hydra-signing-priv.age;
+
     age.secrets.hydra-ssh-key-priv.owner = "hydra-queue-runner";
     age.secrets.hydra-ssh-key-priv.file = ../../secrets/hydra-ssh-key-priv.age;
 
@@ -33,7 +62,16 @@ in {
     # XXX: Otherwise services.hydra-dev overwrites it to only hydra-queue-runner...
     #
     # Can be removed once this is added to some common config template.
-    nix.settings.trusted-users = [ "root" "@wheel" ];
+    nix.settings.trusted-users = [ "root" "hydra" "hydra-www" "@wheel" ];
+
+    # Because Hydra can't fetch flake inputs otherwise... also yes, this
+    # prefix-based matching is absurdly bad.
+    nix.settings.allowed-uris = [
+      "github:"
+      "https://github.com/"
+      "https://git.lix.systems/"
+      "https://git@git.lix.systems/"
+    ];
 
     services.hydra-dev = {
       enable = true;
@@ -52,8 +90,10 @@ in {
       package = pkgs.hydra;
 
       buildMachinesFiles = [
-        (pkgs.writeText "hydra-builders.conf" ''
+        (pkgs.runCommandNoCC "hydra-builders.conf" {} ''
-          ssh://bagel-builder@epyc.infra.newtype.fr i686-linux,x86_64-linux ${config.age.secrets.hydra-ssh-key-priv.path} 8 1 big-parallel,kvm,nixos-test - c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSU9YVDlJbml0MU1oS3Q0cmpCQU5McTB0MGJQd3cvV1FaOTZ1QjRBRURybWwgcm9vdEBuaXhvcwo=
+          cat >$out <<EOF
+          ${baremetalBuilders}
+          EOF
         '')
       ];
 
@@ -62,7 +102,7 @@ in {
           endpoint = "s3.delroth.net";
           region = "garage";
 
-          #secret-key = "TODO";
+          secret-key = config.age.secrets.hydra-signing-priv.path;
 
           compression = "zstd";
           log-compression = "br";
@@ -77,7 +117,7 @@ in {
 
         upload_logs_to_binary_cache = true
 
-        evaluator_workers = 4
+        evaluator_workers = 16
         evaluator_max_memory_size = 4096
         max_concurrent_evals = 1
 
@@ -88,8 +128,11 @@ in {
       '';
     };
 
-    systemd.services.hydra-queue-runner.serviceConfig.EnvironmentFile =
+    systemd.services.hydra-queue-runner = {
-      config.age.secrets.hydra-s3-credentials.path;
+      # FIXME: should probably be set in the upstream Hydra module?
+      wants = [ "network-online.target" ];
+      serviceConfig.EnvironmentFile = config.age.secrets.hydra-s3-credentials.path;
+    };
 
     services.nginx = {
       enable = true;

services/matrix/default.nix

@@ -0,0 +1,68 @@
+{
+  config,
+  lib,
+  inputs,
+  ...
+}:
+
+let
+  cfg = config.bagel.services.grapevine;
+  inherit (lib) mkEnableOption mkIf;
+in
+
+{
+  imports = [
+    inputs.grapevine.nixosModules.default
+    ./hookshot.nix
+  ];
+
+  options.bagel.services.grapevine.enable = mkEnableOption "Grapevine";
+
+  config = mkIf cfg.enable {
+    services = {
+      grapevine = {
+        enable = true;
+        settings = {
+          listen = [
+            {
+              type = "tcp";
+              address = "127.0.0.1";
+              port = 6167;
+            }
+          ];
+          server_name = "forkos.org";
+          database.backend = "rocksdb";
+        };
+      };
+
+      nginx = {
+        upstreams.grapevine.servers."127.0.0.1:6167" = { };
+
+        virtualHosts = {
+          "matrix.forkos.org" = {
+            forceSSL = true;
+            enableACME = true;
+            locations."/".proxyPass = "http://grapevine";
+          };
+
+          "forkos.org" = {
+            forceSSL = true;
+            enableACME = true;
+            locations = {
+              "= /.well-known/matrix/server".extraConfig = ''
+                add_header Content-Type application/json;
+                add_header Access-Control-Allow-Origin *;
+                return 200 '{"m.server": "matrix.forkos.org:443"}';
+              '';
+              "= /.well-known/matrix/client".extraConfig = ''
+                add_header Content-Type application/json;
+                add_header Access-Control-Allow-Origin *;
+                return 200 '{"m.homeserver": {"base_url": "https://matrix.forkos.org/"}, "m.identity_server": {"base_url": "https://matrix.org/"}, "org.matrix.msc3575.proxy": {"url": "https://matrix.forkos.org"}}';
+              '';
+            };
+          };
+        };
+      };
+    };
+  };
+}

services/matrix/hookshot.nix

@@ -0,0 +1,77 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.bagel.services.hookshot;
+  inherit (lib) mkEnableOption mkIf mkOption types;
+  keyPath = "/var/lib/matrix-hookshot/key.pem";
+in
+{
+  options.bagel.services.hookshot = {
+    enable = mkEnableOption "matrix-hookshot";
+    settings = mkOption {
+      description = "Settings";
+      type = (pkgs.formats.yaml { }).type;
+    };
+    admins = mkOption {
+      description = "List of admin MXIDs";
+      type = types.listOf types.str;
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.matrix-hookshot = {
+      wantedBy = ["multi-user.target"];
+      wants = ["network-online.target"];
+      after = ["network-online.target"];
+      serviceConfig = {
+        ExecStart = "${lib.getExe pkgs.matrix-hookshot} ${pkgs.writers.writeYAML "config.yaml" cfg.settings}";
+        ExecStartPre = pkgs.writeShellScript "hookshot-generate-key" ''
+          if [ ! -f ${keyPath} ]; then
+            mkdir -p $(dirname ${keyPath})
+            ${lib.getExe pkgs.openssl} genpkey -out ${keyPath} -outform PEM -algorithm RSA -pkeyopt rsa_keygen_bits:4096
+          fi
+        '';
+        DynamicUser = true;
+        StateDirectory = "matrix-hookshot";
+        WorkingDirectory = "/var/lib/matrix-hookshot";
+      };
+    };
+
+    bagel.services.hookshot.settings = {
+      bridge = {
+        domain = "forkos.org";
+        url = "https://matrix.forkos.org";
+        mediaUrl = "https://forkos.org";
+        port = 9993;
+        bindAddress = "127.0.0.1";
+      };
+      passFile = keyPath;
+      listeners = [{
+        port = 9994;
+        bindAddress = "127.0.0.1";
+        resources = [ "webhooks" ];
+      }];
+      generic = {
+        enabled = true;
+        urlPrefix = "https://alerts.forkos.org/webhook";
+      };
+      permissions = map (mxid: {
+        actor = mxid;
+        services = [{
+          service = "*";
+          level = "admin";
+        }];
+      }) cfg.admins;
+    };
+
+    services.nginx.virtualHosts."alerts.forkos.org" = {
+      forceSSL = true;
+      enableACME = true;
+      locations."/".proxyPass = "http://127.0.0.1:9994";
+    };
+  };
+}

services/monitoring/agent.nix

@@ -20,12 +20,40 @@ in
         internally, which ends up exported as `job` label
         on all metrics of that exporter.
       '';
-      type = types.attrsOf (types.submodule {
+      type = types.attrsOf (types.submodule ({ config, name, ... }: {
         options.port = mkOption {
           description = "Exporter port";
           type = types.int;
         };
-      });
+        options.bearerTokenFile = mkOption {
+          description = "File containing a bearer token";
+          type = types.nullOr types.path;
+          default = null;
+        };
+
+        options.scrapeConfig = mkOption {
+          description = "Prometheus scrape config";
+          type = types.attrs;
+        };
+        config.scrapeConfig = lib.mkMerge [{
+          job_name = name;
+          static_configs = [
+            { targets = [ "localhost:${toString config.port}" ]; }
+          ];
+        } (lib.mkIf (config.bearerTokenFile != null) {
+          authorization.credentials_file = "\${CREDENTIALS_DIRECTORY}/${name}-bearer-token";
+        })];
+
+        options.secrets = mkOption {
+          description = "Secrets required for scrape config";
+          type = types.attrs;
+          internal = true;
+          default = {};
+        };
+        config.secrets = lib.mkIf (config.bearerTokenFile != null) {
+          "${name}-bearer-token" = config.bearerTokenFile;
+        };
+      }));
       default = {};
     };
   };
@@ -35,7 +63,8 @@ in
 
     services.grafana-agent = {
       enable = true;
-      credentials.password = config.age.secrets.grafana-agent-password.path;
+      credentials = lib.mkMerge ([{ password = config.age.secrets.grafana-agent-password.path; }] ++
+        lib.mapAttrsToList (name: value: value.secrets) config.bagel.monitoring.grafana-agent.exporters);
       settings = {
         metrics = {
           global.remote_write = [
@@ -51,12 +80,7 @@ in
           configs = [
             {
               name = config.networking.hostName;
-              scrape_configs = lib.mapAttrsToList (name: value: {
+              scrape_configs = lib.mapAttrsToList (name: value: value.scrapeConfig) config.bagel.monitoring.grafana-agent.exporters;
-                job_name = name;
-                static_configs = [
-                  { targets = [ "localhost:${toString value.port}" ]; }
-                ];
-              }) config.bagel.monitoring.grafana-agent.exporters;
             }
           ];
         };

services/monitoring/default.nix

@@ -3,5 +3,6 @@
     ./exporters
     ./lgtm
     ./agent.nix
+    ./hookshot-adapter
   ];
 }

services/monitoring/hookshot-adapter/default.nix

@@ -0,0 +1,30 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.bagel.services.alertmanager-hookshot-adapter;
+  inherit (lib) mkEnableOption mkIf;
+  package = pkgs.callPackage ./package.nix {};
+in
+{
+  options.bagel.services.alertmanager-hookshot-adapter.enable = mkEnableOption "alertmanager to matrix-hookshot adapter";
+
+  config = mkIf cfg.enable {
+    systemd.services.alertmanager-hookshot-adapter = {
+      wantedBy = ["multi-user.target"];
+      wants = ["network-online.target"];
+      after = ["network-online.target"];
+      environment = {
+        PORT = "9100";
+        UPSTREAM = "https://alerts.forkos.org/webhook";
+      };
+      serviceConfig = {
+        ExecStart = lib.getExe package;
+        DynamicUser = true;
+      };
+    };
+  };
+}

services/monitoring/hookshot-adapter/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "alertmanager-hookshot-adapter",
+  "version": "1.0.0",
+  "description": "Adapter between alertmanager webhooks and the Matrix Hookshot Apapter",
+  "main": "index.ts",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/hm-edu/alertmanager-hookshot-adapter"
+  },
+  "dependencies": {
+    "@types/express": "^4.17.21",
+    "@types/node": "^20.11.20",
+    "dotenv": "^16.4.5",
+    "express": "^4.18.2",
+    "node-fetch": "^3.3.2",
+    "typescript": "^5.3.3",
+    "winston": "^3.13.0"
+  },
+  "scripts": {
+    "build": "npx tsc"
+  }
+}

services/monitoring/hookshot-adapter/package.nix

@@ -0,0 +1,40 @@
+{
+  lib,
+  mkYarnPackage,
+  fetchFromGitHub,
+  fetchYarnDeps,
+  makeWrapper,
+  nodejs,
+}:
+
+mkYarnPackage rec {
+  pname = "alertmanager-hookshot-adapter";
+  version = "1.9.1";
+
+  src = fetchFromGitHub {
+    owner = "hm-edu";
+    repo = "alertmanager-hookshot-adapter";
+    rev = "v${version}";
+    hash = "sha256-KTk70zFA1tymmR8AYrAl2XIyA+SPs5Uksd6Z3kvUb+o=";
+  };
+
+  packageJSON = ./package.json;
+
+  offlineCache = fetchYarnDeps {
+    yarnLock = "${src}/yarn.lock";
+    hash = "sha256-LU25cXB+0DdcHRzKQ1hjQIVntarqPOUXZTgcw6lvLRM=";
+  };
+
+  buildPhase = ''
+    yarn build
+  '';
+
+  nativeBuildInputs = [ makeWrapper ];
+
+  postInstall = ''
+    makeWrapper ${lib.getExe nodejs} $out/bin/alertmanager-hookshot-adapter \
+      --add-flags $out/libexec/alertmanager-hookshot-adapter/deps/alertmanager-hookshot-adapter/dist/index.js
+  '';
+
+  meta.mainProgram = "alertmanager-hookshot-adapter";
+}

services/monitoring/lgtm/grafana.nix

@@ -36,6 +36,12 @@ in
             host = "/run/postgresql";
           };
 
+          "auth.anonymous" = {
+            enabled = true;
+            org_name = "Main Org.";
+            org_role = "Viewer";
+          };
+
           "auth.generic_oauth" = {
             enabled = true;
 

services/monitoring/lgtm/loki.nix

@@ -88,12 +88,19 @@ in
 
     systemd.services.loki.serviceConfig.EnvironmentFile = [ config.age.secrets.loki-environment.path ];
 
-    services.nginx.virtualHosts."loki.forkos.org" = {
+    services.nginx = {
-      enableACME = true;
+      upstreams.loki = {
-      forceSSL = true;
+        servers."127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}" = {};
-      locations."/loki/api/v1/push" = {
+        extraConfig = "keepalive 16;";
-        proxyPass = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}";
+      };
-        basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
+
+      virtualHosts."loki.forkos.org" = {
+        enableACME = true;
+        forceSSL = true;
+        locations."/loki/api/v1/push" = {
+          proxyPass = "http://loki";
+          basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
+        };
       };
     };
 

services/monitoring/lgtm/mimir.nix

@@ -20,6 +20,7 @@ in
         owner = "nginx";
       };
       mimir-environment.file = ../../../secrets/mimir-environment.age;
+      mimir-webhook-url.file = ../../../secrets/mimir-webhook-url.age;
     };
 
     services.mimir = {
@@ -49,6 +50,13 @@ in
 
         ingester.ring.replication_factor = 1;
 
+        distributor.instance_limits.max_ingestion_rate = 0; # unlimited
+        limits = {
+          ingestion_rate = 1000000; # can't set to unlimited :(
+          out_of_order_time_window = "12h";
+          max_global_series_per_user = 0; # unlimited
+        };
+
         blocks_storage.backend = "s3";
         ruler_storage = {
           backend = "local";
@@ -68,6 +76,11 @@ in
             receivers = [
               {
                 name = "matrix";
+                webhook_configs = [{
+                  # Mimir can't expand environment variables in external config files,
+                  # so work around it.
+                  url_file = "/run/credentials/mimir.service/webhook-url";
+                }];
               }
             ];
           };
@@ -78,17 +91,35 @@ in
       };
     };
 
-    systemd.services.mimir.serviceConfig.EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
+    systemd.services.mimir = {
+      # Mimir tries to determine its own IP address for gossip purposes,
+      # even when it's the only instance, and fails if it can't find one.
+      # Avoid that by ensuring it starts after the network is set up.
+      wants = [ "network-online.target" ];
+      after = ["network-online.target"];
+      serviceConfig = {
+        EnvironmentFile = [ config.age.secrets.mimir-environment.path ];
+        LoadCredential = [ "webhook-url:${config.age.secrets.mimir-webhook-url.path}" ];
+      };
+    };
 
-    services.nginx.virtualHosts."mimir.forkos.org" = {
+    services.nginx = {
-      enableACME = true;
+      upstreams.mimir = {
-      forceSSL = true;
+        servers."127.0.0.1:${toString mimirPort}" = {};
-      locations."/api/v1/push" = {
+        extraConfig = "keepalive 16;";
-        proxyPass = "http://localhost:${toString mimirPort}";
+      };
-        basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
+
+      virtualHosts."mimir.forkos.org" = {
+        enableACME = true;
+        forceSSL = true;
+        locations."/api/v1/push" = {
+          proxyPass = "http://mimir";
+          basicAuthFile = config.age.secrets.metrics-push-htpasswd.path;
+        };
       };
     };
 
     bagel.monitoring.grafana-agent.exporters.mimir.port = 9009;
+    bagel.services.alertmanager-hookshot-adapter.enable = true;
   };
 }

terraform/gandi.nix

@@ -1,6 +1,6 @@
 { lib, config, ... }:
 let
-  inherit (lib) mkEnableOption mkIf tf;
+  inherit (lib) mkEnableOption mkIf tf genList;
   cfg = config.bagel.gandi;
 in
 {
@@ -29,6 +29,18 @@ in
         inherit name ttl type values;
       };
 
+      proxyRecords = name: ttl: type: values: [
+        # kurisu.lahfa.xyz running a sniproxy:
+        (record name ttl "A" ["163.172.69.160"])
+        (record name ttl type values)
+      ];
+
+      # Creates a extra *.p record pointing to the sniproxy
+      dualProxyRecords = name: ttl: type: values: lib.flatten [
+        (record name ttl type values)
+        (proxyRecords "${name}.p" ttl type values)
+      ];
+
       # TODO: make less fragile and have actual unique and stable names
       canonicalName = record: let
         name = builtins.replaceStrings ["." "@"] ["_" "_root_"] record.name;
@@ -41,32 +53,38 @@ in
           value = record // {
             zone = tf.ref "resource.gandi_livedns_domain.forkos_org.id";
           };
-        }) records);
+        }) (lib.flatten records));
 
-    in forkosRecords [
+    in forkosRecords ([
       # (record "@" 3600 "A" ["163.172.69.160"])
       (record "@" 3600 "AAAA" ["2001:bc8:38ee:100:1000::20"])
 
-      (record "cl" 3600 "A" ["163.172.69.160"])
+      (dualProxyRecords "bagel-box.infra" 3600 "AAAA" ["2001:bc8:38ee:100:100::1"])
-      (record "cl" 3600 "AAAA" ["2001:bc8:38ee:100:1000::10"])
+      (dualProxyRecords "gerrit01.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::10"])
+      (dualProxyRecords "meta01.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::20"])
+      (dualProxyRecords "fodwatch.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::30"])
+      # git.infra.forkos.org exposes opensshd
+      (dualProxyRecords "git.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::41"])
+      # git.p.forkos.org exposes forgejo ssh server.
+      (proxyRecords "git.p" 3600 "AAAA" ["2001:bc8:38ee:100:1000::40"])
+      (dualProxyRecords "buildbot.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::50"])
 
-      (record "fodwatch" 3600 "A" ["163.172.69.160"])
+      (record "cl" 3600 "CNAME" ["gerrit01.infra.p"])
-      (record "fodwatch" 3600 "AAAA" ["2001:bc8:38ee:100:1000::30"])
+      (record "fodwatch" 3600 "CNAME" ["fodwatch.infra.p"])
+      # git.p.forkos.org is the proxy variant of the Forgejo server.
+      (record "git" 3600 "CNAME" ["git.p"])
+      (record "netbox" 3600 "CNAME" ["meta01.infra.p"])
+      (record "amqp" 3600 "CNAME" ["bagel-box.infra.p"])
+      (record "grafana" 3600 "CNAME" ["meta01.infra.p"])
+      (record "hydra" 3600 "CNAME" ["bagel-box.infra.p"])
+      (record "loki" 3600 "CNAME" ["meta01.infra.p"])
+      (record "mimir" 3600 "CNAME" ["meta01.infra.p"])
+      (record "matrix" 3600 "CNAME" ["meta01.infra.p"])
+      (record "alerts" 3600 "CNAME" ["meta01.infra.p"])
+      (record "buildbot" 3600 "CNAME" ["buildbot.infra.p"])
 
-      (record "netbox" 3600 "A" ["163.172.69.160"])
+      (record "vpn-gw.wob01.infra" 3600 "AAAA" [ "2a01:584:11::2" ])
-      (record "netbox" 3600 "AAAA" ["2001:bc8:38ee:100:1000::20"])
+      # TODO: do not hardcode, just reuse the Colmena hive module outputs to generate all the required details.
-
+    ] ++ map (index: record "builder-${toString index}.wob01.infra" 3600 "AAAA" [ "2a01:584:11::1:${toString index}" ]) (genList lib.id 12));
-      (record "bagel-box.infra" 3600 "AAAA" ["2001:bc8:38ee:100:100::1"])
-      (record "gerrit01.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::10"])
-      (record "fodwatch.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::30"])
-      (record "meta01.infra" 3600 "AAAA" ["2001:bc8:38ee:100:1000::20"])
-
-      (record "amqp" 3600 "CNAME" ["bagel-box.infra"])
-      (record "grafana" 3600 "CNAME" ["meta01.infra"])
-      (record "hydra" 3600 "CNAME" ["bagel-box.infra"])
-      (record "loki" 3600 "CNAME" ["meta01.infra"])
-      (record "mimir" 3600 "CNAME" ["meta01.infra"])
-      (record "matrix" 3600 "CNAME" ["meta01.infra"])
-    ];
   };
 }

terraform/hydra.nix

@@ -32,6 +32,45 @@ in
       visible = true;
     };
 
+    resource.hydra_jobset.k900-experiments = {
+      project = config.resource.hydra_project.forkos.name;
+      state = "enabled";
+      visible = true;
+      name = "nixpkgs-experiments";
+      type = "legacy";
+      description = "experiments branch to test things for K900";
+
+      nix_expression = {
+        file = "nixos/release.nix";
+        input = "nixpkgs";
+      };
+
+      check_interval = 0;
+      scheduling_shares = 3000;
+      keep_evaluations = 3;
+
+      email_notifications = false;
+
+      input = [
+        {
+          name = "nixpkgs";
+          type = "git";
+          value = "https://github.com/nixos/nixpkgs 03ff49192b044786362c8c94d8501eac5c6eada4";
+          notify_committers = false;
+        }
+        {
+          name = "officialRelease";
+          type = "boolean";
+          value = false;
+        }
+        {
+          name = "supportedSystems";
+          type = "nix";
+          value = ''[ "x86_64-linux" ]'';
+        }
+      ];
+    };
+
     resource.hydra_jobset.raito-nixos-rolling-small = {
       project = config.resource.hydra_project.forkos.name;
       state = "enabled";
@@ -71,5 +110,139 @@ in
         }
       ];
     };
+
+    resource.hydra_jobset.delroth-nixpkgs-staging-small = {
+      project = config.resource.hydra_project.forkos.name;
+      state = "enabled";
+      visible = true;
+      name = "delroth-nixpkgs-staging-small";
+      type = "legacy";
+      description = "small eval of nixpkgs staging for testing";
+
+      nix_expression = {
+        file = "pkgs/top-level/release-small.nix";
+        input = "nixpkgs";
+      };
+
+      check_interval = 0;
+      scheduling_shares = 3000;
+      keep_evaluations = 3;
+
+      email_notifications = false;
+
+      input = [
+        {
+          name = "nixpkgs";
+          type = "git";
+          value = "https://github.com/nixos/nixpkgs staging";
+          notify_committers = false;
+        }
+        {
+          name = "officialRelease";
+          type = "boolean";
+          value = "false";
+          notify_committers = false;
+        }
+        {
+          name = "supportedSystems";
+          type = "nix";
+          value = ''[ "x86_64-linux" ]'';
+        }
+      ];
+    };
+
+    resource.hydra_project.infra = {
+      name = "infra";
+      display_name = "ForkOS Infra";
+      description = "ForkOS infra repository";
+      homepage = "https://git.lix.system/the-distro/infra";
+      owner = "terraform";
+      enabled = true;
+      visible = true;
+    };
+
+    resource.hydra_jobset.infra_main = {
+      project = config.resource.hydra_project.infra.name;
+      state = "enabled";
+      visible = true;
+      name = "main";
+      type = "flake";
+      description = "main branch for the infra repo";
+
+      flake_uri = "git+https://git.lix.systems/the-distro/infra";
+
+      check_interval = 600;
+      scheduling_shares = 3000;
+      keep_evaluations = 5;
+
+      email_notifications = false;
+    };
+
+    resource.hydra_project.hydra = {
+      name = "hydra";
+      display_name = "ForkOS Hydra";
+      description = "ForkOS hydra fork";
+      homepage = "https://git.lix.system/lix-project/hydra";
+      owner = "terraform";
+      enabled = true;
+      visible = true;
+    };
+
+    resource.hydra_jobset.hydra_main = {
+      project = config.resource.hydra_project.hydra.name;
+      state = "enabled";
+      visible = true;
+      name = "main";
+      type = "flake";
+      description = "main branch for the hydra repo";
+
+      flake_uri = "git+https://git.lix.systems/lix-project/hydra";
+
+      check_interval = 600;
+      scheduling_shares = 3000;
+      keep_evaluations = 5;
+
+      email_notifications = false;
+    };
+
+    resource.hydra_jobset.yureka-staging-test = {
+      project = config.resource.hydra_project.forkos.name;
+      state = "enabled";
+      visible = true;
+      name = "yureka-staging-test";
+      type = "legacy";
+      description = "staging branch for yureka-nixos";
+
+      nix_expression = {
+        file = "pkgs/top-level/release.nix";
+        input = "nixpkgs";
+      };
+
+      check_interval = 0;
+      scheduling_shares = 3000;
+      keep_evaluations = 3;
+
+      email_notifications = false;
+
+      input = [
+        {
+          name = "nixpkgs";
+          type = "git";
+          value = "https://cl.forkos.org/nixpkgs sandbox/yureka/staging-test";
+          notify_committers = false;
+        }
+        {
+          name = "officialRelease";
+          type = "boolean";
+          value = "false";
+          notify_committers = false;
+        }
+        {
+          name = "supportedSystems";
+          type = "nix";
+          value = ''[ "x86_64-linux" ]'';
+        }
+      ];
+    };
   };
 }