buildbot: init

Reviewed-on: the-distro/infra#68

common/base-server.nix

@@ -1,4 +1,8 @@
 { lib, pkgs, ... }: {
+  imports = [
+    ./known-ssh-keys.nix
+  ];
+
   nixpkgs.overlays = import ../overlays;
 
   nix.package = lib.mkDefault pkgs.lix;
@@ -25,7 +29,7 @@
   nix.gc = {
     automatic = true;
     persistent = true;
-    dates = "daily";
+    dates = lib.mkDefault "daily";
     options = "--delete-older-than 30d";
   };
 

common/known-ssh-keys.nix

@@ -0,0 +1,6 @@
+{ ... }:
+{
+  programs.ssh.knownHosts = {
+    "[cl.forkos.org]:29418".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM82mJ259C8Nc+BHHNBeRWXWhL3dfirQhmFbDAwHMle3";
+  };
+}

common/ssh-keys.nix

@@ -4,6 +4,7 @@
     meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT";
     gerrit01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA+eSZu+u9sCynrMlsmFzQHLIELQAuVg0Cs1pBvwb4+A";
     fodwatch = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFRyTNfvKl5FcSyzGzw+h+bNFNOxdhvI67WdUZ2iIJ1L";
+    buildbot = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJgIu6ouagYqBeMLfmn1CbaDJMuZcPH9bnUhkht8GfuB";
     git = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEQJcpkCUOx8+5oukMX6lxrYcIX8FyHu8Mc/3+ieKMUn";
     builder-0 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBHSNcDGctvlG6BHcJuYIzW9WsBJsts2vpwSketsbXoL";
     builder-1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIQOGUjERK7Mx8UPM/rbOdMqVyn1sbWqYOG6CbOzH2wm";

flake.lock

@@ -55,6 +55,29 @@
         "type": "github"
       }
     },
+    "buildbot-nix": {
+      "inputs": {
+        "flake-parts": "flake-parts",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "treefmt-nix": "treefmt-nix"
+      },
+      "locked": {
+        "lastModified": 1721229951,
+        "narHash": "sha256-RO7jlz2T0h9l7Hmij6Iy3qdYps33wDuAoBMQ21ROvyw=",
+        "ref": "refs/heads/refactor",
+        "rev": "8286c1028b2a69ee72680dc06d26bd80665ce02a",
+        "revCount": 262,
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
+      },
+      "original": {
+        "ref": "refs/heads/refactor",
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
+      }
+    },
     "colmena": {
       "inputs": {
         "flake-compat": "flake-compat",
@@ -133,6 +156,27 @@
       }
     },
     "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "buildbot-nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1706830856,
+        "narHash": "sha256-a0NYyp+h9hlb7ddVz4LUn1vT/PLwqfrWYcHMvFB1xYg=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "b253292d9c0a5ead9bc98c4e9a26c6312e27d69f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-parts_2": {
       "inputs": {
         "nixpkgs-lib": [
           "hydra",
@@ -254,7 +298,7 @@
     },
     "nix-eval-jobs": {
       "inputs": {
-        "flake-parts": "flake-parts",
+        "flake-parts": "flake-parts_2",
         "lix": [
           "hydra",
           "lix"
@@ -264,7 +308,7 @@
           "hydra",
           "nixpkgs"
         ],
-        "treefmt-nix": "treefmt-nix"
+        "treefmt-nix": "treefmt-nix_2"
       },
       "locked": {
         "lastModified": 1721195872,
@@ -404,6 +448,7 @@
     "root": {
       "inputs": {
         "agenix": "agenix",
+        "buildbot-nix": "buildbot-nix",
         "colmena": "colmena",
         "hydra": "hydra",
         "lix": [
@@ -484,6 +529,27 @@
       }
     },
     "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "buildbot-nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1708897213,
+        "narHash": "sha256-QECZB+Hgz/2F/8lWvHNk05N6NU/rD9bWzuNn6Cv8oUk=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "e497a9ddecff769c2a7cbab51e1ed7a8501e7a3a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
+    },
+    "treefmt-nix_2": {
       "inputs": {
         "nixpkgs": [
           "hydra",

flake.nix

@@ -17,6 +17,9 @@
     nix-gerrit.url = "git+https://git.lix.systems/the-distro/nix-gerrit.git";
     nix-gerrit.inputs.nixpkgs.follows = "nixpkgs";
 
+    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/refactor";
+    buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";
+
     lix.follows = "hydra/lix";
   };
 
@@ -73,6 +76,8 @@
       commonModules = [
         inputs.agenix.nixosModules.default
         inputs.hydra.nixosModules.hydra
+        inputs.buildbot-nix.nixosModules.buildbot-coordinator
+        inputs.buildbot-nix.nixosModules.buildbot-worker
 
         ./services
         ./common
@@ -101,6 +106,7 @@
       fodwatch.imports = commonModules ++ [ ./hosts/fodwatch ];
       git.imports = commonModules ++ [ ./hosts/git ];
       wob-vpn-gw.imports = commonModules ++ [ ./hosts/wob-vpn-gw ];
+      buildbot.imports = commonModules ++ [ ./hosts/buildbot ];
     } // builders;
 
     hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.toplevel) self.nixosConfigurations;

hosts/buildbot/default.nix

@@ -0,0 +1,38 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  networking.hostName = "buildbot";
+  # TODO: make it the default
+  networking.domain = "infra.forkos.org";
+
+  time.timeZone = "Europe/Paris";
+
+  bagel.sysadmin.enable = true;
+  # Buildbot is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.hardware.raito-vm = {
+    enable = true;
+    networking = {
+      nat-lan-mac = "BC:24:11:E7:42:8B";
+      wan = {
+        address = "2001:bc8:38ee:100:1000::50/64";
+        mac = "BC:24:11:C9:BA:6C";
+      };
+    };
+  };
+
+  bagel.services.buildbot = {
+    enable = true;
+    domain = "buildbot.forkos.org";
+    builders = [ "builder-3" ];
+  };
+
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "buildbot.infra.forkos.org";
+}

hosts/gerrit01/default.nix

@@ -35,6 +35,7 @@
     domains = [
       "cl.forkos.org"
     ];
+    canonicalDomain = "cl.forkos.org";
     data = "/gerrit-data";
   };
 

secrets.nix

@@ -13,6 +13,16 @@ let
     loki-environment = [ machines.meta01 ];
     gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
 
+    buildbot-worker-password = [ machines.buildbot ];
+    buildbot-oauth-secret = [ machines.buildbot ];
+    buildbot-workers = [ machines.buildbot ];
+    # Private SSH key to Gerrit
+    # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
+    buildbot-service-key = [ machines.buildbot ];
+    # Signing key for Buildbot's specific cache
+    buildbot-signing-key = [ machines.buildbot ];
+    buildbot-remote-builder-key = [ machines.buildbot ];
+
     # These are the same password, but nginx wants it in htpasswd format
     metrics-push-htpasswd = [ machines.meta01 ];
     metrics-push-password = builtins.attrValues machines;

secrets/buildbot-oauth-secret.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 87T2Ig g15A5EWi9IhaxPFS6SD6YYm/aFnC0Dum7zK8/ZUtW0s
+791D6C8mAy2dhDAlqRQ+q41FlQTJX2WfZQPjuwetP2A
+-> ssh-ed25519 K3b7BA cJY9qIFVmucmMJLTFffkRCNYeudZl+8Yrm5SkxQ4eSI
+97nXyKffZGoGJ6252UKUEJHiFgdk8XUkAAkXy2PLepM
+-> ssh-ed25519 +qVung HMBSUjfmaFLVx64epj0djkqNMe3CdKN1fxAVuu+Dtmg
+AxT62n2p/pP9WZmmuHClSKKgXhr4FjEQpEs0HfdNGfw
+-> ssh-rsa krWCLQ
+N0Duz2bONcCUZ76QhPsCJ4BHHWqzFdZLqFdl+6GeW+tgIp2Nb4la8eNfgzYGSwTy
+53bRePNMIBTkChXFYt/4fUdqaiiVYg25swMeVLQBJnjJkcAks0Gf44FXLIaoPr1M
+56rtixpSX31WDKwHbUF/40G6Xut8KNlI8BdwiOl9ibgnuEf4mYQbwFbRQbLMK5IK
+Rf/7SEmAqqfY/HG1RqqgCs4kEpvFTKqEEDpgjOoyS2tyKN2351jya91YzotLja4I
+sLoMg/G3UNtxfdaCgK7TP4IxV9blkVMDPAbyR622VbS0sEa7uJGzb86jDDsZXaKX
+9iWK9n4hMKZDv9gBbhTIWg
+-> ssh-ed25519 /vwQcQ hMkCrUcLGxdZMYgi1D1Kr5qUdGNfza2UTvRJKiHObgM
+7Lz70zSMPk/tsU1CZGOk/BPA7NSSnSJgFbG5TjyOXvA
+-> ssh-ed25519 0R97PA OQjDTknVmrYVclcqlT31YjZx+3a/0GxfjuVQFmPJ7UQ
+KMGTMfO/mO5EAYacyz1hmHnQgzunRqkDeglhbGVNWe4
+--- ScDZvSiVSjNXm8TSoLSAM+KpcFORnCXiemYbCBcz2jQ
+™ŸÄhÜ}E¹ÊœËíUÌùᢌƒÿ…<C3BF>é™k¢ág[<5B>ñCƒ"<22>–NÛj•u5«<0C>ÄCXÕöÈGt¡TOmñ

secrets/buildbot-worker-password.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 87T2Ig df+IMqWM/HNjaY74zibFQIdUdC3K7uQlm3U9R9NUtFY
+hPSbCuWvqy/7FEj7YScYztyt5GVx4Y7tgGuKKkSKoRg
+-> ssh-ed25519 K3b7BA xN8wzUKHqjOb/tqA+EI+0H0MSQRihRfydchwVqYWAVU
+maLMpZe8orvTT6Av+YkhT8FcG4dc7bzDgOW339nSw1g
+-> ssh-ed25519 +qVung oM1uphTbjI54t4U9jNd1zORqpjBG17MwDf2eNDmOlkg
+oUHVuQt2SHIwtV82pgnKJ7g2jcVBAHWOzPK46otoh34
+-> ssh-rsa krWCLQ
+eYspf5hUKdFQl1RxPaNTj0viAPd+kzp8Xbwn+q6fSITMacmyTY5J8FckLx2YXDxy
+Qm/OsEK0ZOvxnHMrL0oAJjKSy/MamE+9heT3QO+LUN30QxbOIOqHMrl3waadWZdx
+ZGOWK+r+dKGYNsxFv+t1Y/4DBKKzlXFWhJ0aL7nMOqq9+Ca+UZuE41j7eWGGPPLy
+fuW/iOVVxQ+EEeCDpatQSrFPKaeWCCVP9oIDFtE4dsKxubMa4EpUoag0UvEIW182
+UGS8BvMqYgx+obqJDkhXXBK9apmJS2ojcfdtCbNOCV9Ett72Nm/iY5NjLprFMLde
+8wWGA6s3hBOP39lq0eiSxw
+-> ssh-ed25519 /vwQcQ 3zLcLDaDVhIn2knezexYM5Fqu/O9wwORnJIhsXHqgj0
+HchGikQMgkDj0qQgtDdsdKokV+nMjdv6t0uVISeU7Q8
+-> ssh-ed25519 0R97PA 6lm6B6B3dzSdhdcf5rjyTu+7cCtWRxVpWeapJX3nbQo
+x/w4dEfFyxPi4lbNEqgjEblPVfQyj+q1JjeQHiVFhDw
+--- oo5BK1pG+43amUg803Uv511RNtdQ/PDwlXUrV/AbOAA
+…ÙUqÆçïµ[f7ƒêŒë¼¨FìˆY<13>™Ùm¶ØLS?Úℶ‡÷ƒöæ<Kø©F¤z¥V^³U¨N»¯ôƒ)zÔ<7A>¥ž@<40>SÀF€Y‡ËG2^žƒ˜à„»N|

secrets/gerrit-prometheus-bearer-token.age

@@ -1,22 +1,22 @@
 age-encryption.org/v1
--> ssh-ed25519 2D+APA Vh/FrR9oyO8V1pEMQkmGbHCePB6RU+dPm+Z4bgKenEg
+-> ssh-ed25519 2D+APA jiLDQ8JlYhaivXQQhjEfZrGWn7o6Wd2OMrLorEVSPns
-2G5eLlYe8IS7fsEBorFljUwQZ9sEk/FEr25S4p5hWLk
+qRzHYcBhtGSm4RW7C4oW+VWSzHiDXkCN6bGeej2Gcpo
--> ssh-ed25519 j2r2qQ 9+NX0Guhux9QlAxx2MtSZH0OZpDk1CQZ4Blu1P9fpgQ
+-> ssh-ed25519 j2r2qQ OcnIHB/vJoKuvhsT9dx1B+5lXguARtB9wSquW2KBB3M
-PDUoAjBaIdKQAvRblvc0QEtrvp5MpE8HsCwKWwAn0uE
+pgzC2KOFi3Yj1gCPemVK3a9Grv2SkwZ6AI1EFdh4hoc
--> ssh-ed25519 K3b7BA wuOc6LGnjsC4Rb9D9QX3YVgMqWPvBK27Q0vqADLpsk8
+-> ssh-ed25519 K3b7BA ibHY8wN3rNit1mO2dJZ44rwLylMaR39a7Oz3CGV561o
-wRnoNzkyaU9SGlOtpqY2pAeIwD9lGWKrqNn3D3W7U6Y
+4ElWORF/4lVEz33CJiuFG4rwUSIIOyi2L/W7Td7MX5M
--> ssh-ed25519 +qVung biXtZHmjJmsazEmp1iIGUqmuV1YP94bzrMjoZTmGPjg
+-> ssh-ed25519 +qVung q4DDHS3M24kke2NCcpHEaUbUgoQB6QwnmDiwmdIOuBw
-GDN4WZGTIP6b2nmjyhikHeOrZi9YEtiPOyaJLzUl138
+Yfa6v23oezdDICE8I0UaVCShKlx9lN3DnBnSb63LU64
 -> ssh-rsa krWCLQ
-UkNySvhS5o6v6/7xGvn43hgD5y2D91oH4pjU3Oa83CW6ha80dnE+JkSTpTdz7Og0
+gLBHP4Z8EBW1y7Yf9sfWMU+/fJ4WWp+NGRR7ebO5GwUeYobDYm/eYQ7rD3Q9k0rF
-vtZJuisNpcH254zTt8OAUpWN/tVXlD34RyV1xo1eHEWgUzKactrhlACpSbzYBdVJ
+kU51GYBaO7m5gLqc2Tq4+YjE2/EXDvjqkDSoyNrjQaaGTLqzvPYlCvKWyROjqJjX
-8cUj7jiE+qjIOtrU2sHWo09NKpf0J2YEPwajuBy1/fPrivlgXAzdAAnP4gll02x1
+UwzPbQx5XVIKNgpsR9e6/hoJiJbDpavM+HQo+1zwoKAg5FvZZkE5UnIiSjuAxMgR
-Et8lUn6HVfYDGtrDo/PUUdgcGudVeCOJbvvrKYkuqe8vsNYgnFHM8dkTJmObL8dz
++tmrhBfHEYkpbCCrXVE0jLCup8gPIci1PyXWkdhJy+HyHVkbYowGwNawNobNr1cF
-zp4MEuIQ3WrrXActSnTs+QAGIFSskOIr1DQlJRYzQcYtd8wkfx9a+6oxBECZyDAZ
+dJ5IU8P/DSSqZ1qWSl6ju7JKjzXU2Xq87/g7wJyrKGpe37pJmPIT86nCJTut+AK9
-T4yso7ctflKlr6OqpJYzeA
+iFED/y/p5NCtohyhztosgA
--> ssh-ed25519 /vwQcQ +jsCn0OlVpuyVA0XSvD3ZCDRTBq29UV9qsDvE4XaGk0
+-> ssh-ed25519 /vwQcQ rzEjV56G+USMdpWklrGQSHuzG8d+S0zWhhwrmuyTyiA
-p2qblImpl+G0pefJ0T/GjanIc7+bNuA0wRB4mUuFGXM
+y+uMRG8NdAD0H4ipRN+sJPn1P0CGs4bk+U4qtetP3O0
--> ssh-ed25519 0R97PA /bE6+eVlzeJKOOMqz4QjFdsu+5XDv9L8cZ94cPZ5WQk
+-> ssh-ed25519 0R97PA ULWdDUjDg9oTEOqzCKUJl8yN+qwwmlSi1PFwRvr7aWM
-Xco24ijeQnaT7jcsfXLQPzGr1FE/zy9+qVoQ20DLP+Q
+YWaE+STxKfQzxYMtP/cA20q0atXLdsjeA5nJyl2f8iI
---- NDqgX11cTXR48vD9YmAIYx+og0n1OQj+bbkKwqv2BeE
+--- Avs8hTgLwcBy8hyYWjR/Jbs5YaKozv2oBmGs51ckquA
-šÊ\”wÔ
ðä9Ì’7öcØšƒ’‹%}|k®?š×$9·lö&<13>=¸vñþܹ!Pã<50>Þ3b·<62>ù퀩
+Wœ·Ü<C2B7>dŸ›ÝàÕò`@½Óµ3ž ‚¼½5è½b›Y%³A†Z=KiÐÑ76,¢w,1žŒèáÎôkØåRšAÄ‚FuÎÎ

secrets/grafana-oauth-secret.age

@@ -1,20 +1,20 @@
 age-encryption.org/v1
--> ssh-ed25519 j2r2qQ qI/dlkHZYcNkCVgZbxpw5Ps2anl8pofaFPi4p6kOHAo
+-> ssh-ed25519 j2r2qQ JSveX4zYEjb4jJH4eg4oXA6r3oc0jBx8NgjhN9JrjlQ
-KWL+H9at/p/AfCjfO8+SgMhn97F+DqLO2ymYUOHkWjQ
+1ZIr/XFClbwJHn0ppJnolpb4QlgZOA8JX5OjjY4x6pU
--> ssh-ed25519 K3b7BA URYQ0jFY5yHS+dodR1RqodNWrrXkMnzTp5OCSv1gbWI
+-> ssh-ed25519 K3b7BA sXUjuZFK0PL/KndxRCJCM5Kg8OmVseRZNWG8mL1alRc
-bnyrPvWnzDRNh4mI5HBPkNl3NSZE1ycMK3LLExMEYbo
+U9MMgDtqtmsS1W5i04Pa/b4JBTSjK6FffZxgYI3phtg
--> ssh-ed25519 +qVung z8e56tCZ4TLkrX7BfH+5RrGxGoT3q9V1FB/ySsH3tg4
+-> ssh-ed25519 +qVung FNSElbiw0frYcsO0xoyPQgRGqAe/aVX21dTB6yk+GQg
-jIpEEVF8jCp/ks5eYXh3O7+TLidvzYsnBRFd3LkgLXw
+zHT/xU+yfXYSBO2HLwoHrGf5ns6BDVb8MlhVVQCBlOc
 -> ssh-rsa krWCLQ
-XG8KKBT/hEvB+c1RDGUrDR4HrfAertfOIzQTquMQ+Z3Nde3Ybxf8W+rWGQDErbq4
+ye0mLiYeyvlp4EZX7mZ3F7B9V9JSeoiCodzccS+5qIEd6gr+RTHSnKYqwf/nwf8F
-VlvC/wVVnGnqgE/tJMQP41sCMKSH61MPyiNZC63g4RW9e2H9YQfWWrnuBh668G+3
+qKLwbxWjpmkIzBWeswy8AJ8159aucGEmB+3/tTSwd+QlRkru4Z/7jtfU64KQttgt
-3sE0FSdIAB+UlI2jlbMiG60QaT6zV0XyOrugLX/G2R+D4aXYIVvMtcwYq2oIHy58
+vaRfc9J/85AJJ2V6Sw/xG8SgxyLBbp/XIN2+tmb0g3kAWiuLcrLk3H/MsfmxDVXg
-1DE5llUZHGsQ8APXZle7ZGyO48ELOQkVn8ozPlPFhvz2y9srgBZvNL/wadjvLstv
+RQjugP5K2+fEZc77dHQTrMI58K9TrSw1zYA1ee8J/fl9IJ7J77qi5UgizY+YfX8T
-2vBTBoRk8HnTLOiybAnGtOfK6kWUMdfSYMvhu0IM8UBSoxwxOHTfIttKDu2ZMB8g
+SmR9DeYUe+hKgCB2k/KgAxp4WOQNgUOFBTsE5FW+kQQpfGx5aqR6vCYU+CPsA3Zb
-c/RnKbV2z0PBdXVrYuijPg
+FwV0l+g4FUVy+xAtqaGSAQ
--> ssh-ed25519 /vwQcQ qinzScNz0IFoHUaCeGXne6ddllQ0dA/TJr5Z/nbfvTQ
+-> ssh-ed25519 /vwQcQ fbnK1jYiUwUsgD8sSTboJCBfcuwJXKNCaJaWYuIfmVk
-0YpTZ2Z2WwN0sJ1CIV8voPS298u9uHbRQMlV0GMrvFI
+Uj2+uBABMTxq1MBsiHXgkdFMOpIN7gfxoJVKOQff1Pw
--> ssh-ed25519 0R97PA en5iGTQoH0/QJKl38HNe4xun/FxVBIun7Z23mBW+4XE
+-> ssh-ed25519 0R97PA yYOb6AYAFWvm7W2KYT5v9zznkF4Di/vatH48Xgx0x2E
-Sjshx8hLyP4iY40y/Fehc0wZTBH0d1Lu+auX8L5n28s
+yUm+MKj9496BkdX2FpLyhML7budUyqT1hL9hpghxSnI
---- i5+vCeWbFTRR2YbIX4lwbEORRhaI5NkCwqaMEJqrPEs
+--- ogCPBrmdbeDorj3t5BL05ge6VngXBpUEDW4qaaKIa0U
-ÿ\ìƒF·Ri±ñXa,.øÝoªâr›çhE0=$Ç‚uGa/oÑÑÆÂiíf¥•x¦Óš?Ðg¹CiÉ
+%¨šÚlD]Ϫ?©ßŠÑ(ÿ†E/Wu穉T¶îç[}ž$ÁÍS„Šˆ^[:¸]
he0XUœp¸äq<C3A4>`0A

secrets/loki-environment.age

@@ -1,20 +1,21 @@
 age-encryption.org/v1
--> ssh-ed25519 j2r2qQ JzVKQt25f18L96aJWsJtFAR4mvMVCgYMKu/xtJ1BeDw
+-> ssh-ed25519 j2r2qQ 6qyr94uky6B36UOY0jd5NXgF2rJ3RWBUzZ32c5iOTmY
-vj+HpNQCNNxDRA+7HgjiD0XlGG/Yy+tk8KmszMkxdag
+fjlI3fjYjwyNQBs4K4pq/5c7oBkf5XUXoGlBOBpmPu4
--> ssh-ed25519 K3b7BA judlH57lGOGmaTEG19gYiORJT9uXiAlxZrP+ISTHDT4
+-> ssh-ed25519 K3b7BA N9VYT/ZslG07KldzO8sPE5TiYYwxJqpYU87ED4PuBXw
-MS7e24A6rEMUtUUl8DlYXPy9NhqAq4buOWT0iYKvbSY
+P1s9L57prPqM4fjcYHv+g0rgP/NvFr13CgCxthVHZ4c
--> ssh-ed25519 +qVung vglRR5LYFZw8v6zRhybGPBctwDgYoskbpGYiLNW9qxM
+-> ssh-ed25519 +qVung Ry8uUFsmYmP+Urw46lhAsCc3S+QiWu1mn8J3rIy+KFQ
-VdjQTykQSVWubGimCHiekQX7EQdgOB3PYsRHiFnpPkg
+iB7xAfdpHwOzAnLvosJb+F50QKsOYWr7CHC3srsS6ME
 -> ssh-rsa krWCLQ
-hLYT6U+dUVuicVO8hSw4KcfkM9bay4JR3TEWGlmmIxcQ67LNggzuyRvV6U2yfucg
+w0xIVFtUghdAO7SxZD10rBMtdQESEvYUEKxnWzLh0cjcRhaVT/BXSZQsKV2Rupoo
-Xyxezdd9LArf8z1eV/y3iwsY0PvK9qwtgpgH/NxaF7djhTA8+c3c3a6w4sqdHn0m
+nDL5uy0k+tPXm0HroZ6VkZ0fH/lOpeUR69ZvJmClKql3Fnf1385+5BvT719cbbaq
-/RZU+eKSFeDWII7fn6o7JxzITFhF1FYH6PJYA2cb3PvbPw/JSja8EVZ7192ShqGW
+yll49gx0+ms/oB9jS3SPwbOg+UJgnkZCeu9138h3MG7yWNtVuA9l5hsJioVvOVlS
-22TThbZmmKoOPbmDxmQIygZTxqyaXkoFOnTWqqTzOfNtBOBFXT+cIFh3ctGWLw79
+Z5EXbjdQR9xYjSwR+b8MYZ97ej5fXpuULEopbx2wXt84u1e67vTETqflitR7lrzy
-u7O5c2dmpXoE0bdndQ7GUSPrgRzOYHQ5hLg8WtC56EYjE11Bxj88fktzw4hZTbYQ
+A6F65g35aagPJZGHzfrKVToy3pfXm9ky/30DolWLD0DpG7G6o/8afy8O4yBAGlv3
-jrS8Pa68UPhUmSfutlpd4A
+ZLTaUbrdILSz2ff1Njx4Nw
--> ssh-ed25519 /vwQcQ MqdVxRlS+EMA3f6B0D6m2ylvCE7WVq1av/CvsNVAB24
+-> ssh-ed25519 /vwQcQ YqqmX/f4whOk97kCgSPo6oj/274eYlBWtS+OahAAQ34
-KX8RJ1bzUUhsYW6qN06FTzis5i13IIoIpUb5FkW9wkw
+hoCbhupzSTx+wNIorzYGHyGvU/L8unKEyD7Bqq23YP0
--> ssh-ed25519 0R97PA RHUvc9XQIxOW0GCyt0vRxPHyVXlpqM9gaUps4q/Grx8
+-> ssh-ed25519 0R97PA 17SDtfT9GzAsIsQB24AmYXpW8v4+LEakup+tdFroHTk
-bxgFxtbtbvDi9knzasdR7u33Mb7x7LcBzqEB/g4Oc4A
+HIvBhAGA2GMVWFBP3OTFEn+XpPFBJDOJDK3SQ94mNKM
---- Z175YCdbPBBSItxomyXPSo6xILLV4GT4gpA4Oxz9qgo
+--- CD1QrxYGAhhy+l7U5kOXn1shCwz8pYJNuGRugPxmzJw
-EìVÀõ±ž™êÞ<EFBFBD>Ú¾¾Ó¦xYÊqšÑ84™6¦¯&Ö‘ï<13>·”ž„Ý!óZmëû°¤Ãd.à™46ÅÈ·ØËòø/<2F>´<EFBFBD>=°ß܈'hM³_ü£j
>ªÑ6ãR<>&Ú
·u²þŸøEùÜ^8c;×Ä›¶:Q1Ü)ú1L¹_~,<2C>K¥ÞÃîôµB¤7–
+ñY¾Æ‹N	Ï<>x
™êÿrR^z[¤ã¸è…•ªa”z
+óæÔÉ¿Ïžu0c¯c;y<>Ÿ¢›&{ñèxA]‚þ†¨Q¨¼_:̱ í€öUoiDl (‹Å
ëwÝKi,j.oFyÌ°$}•Y§@1”È™„Y£²è¶u Ò*¡ÏþÅ<C3BE>¥™0…

secrets/metrics-push-htpasswd.age

@@ -1,20 +1,21 @@
 age-encryption.org/v1
--> ssh-ed25519 j2r2qQ n1lfxDP73nfF/CYtE4gpUH6YgjAQbx/2TTuyfFUBiHQ
+-> ssh-ed25519 j2r2qQ sIYTVOTWNToDSNa4qiIaSoac7zka54g/opQ70q1SAA8
-LGzudpjsYA92pM0UpUT9CWZD+e+rzGFP4ndxPE0MByo
+2Z1mlCWxjakHqRbArU2BkT7B/Dx0XKH7kCnBa+OYI+s
--> ssh-ed25519 K3b7BA NRnnKaOtdtIjkRdam5vAA9Yj1RUJRReugWKRglWAoQ4
+-> ssh-ed25519 K3b7BA PGyd27M/Hmk6qpRf8bcI4QWrS0vrPgjiZzaXvKQkJDQ
-Xprx5TSU1rNH7NMl0X07K1KexCVXMEu7BFxbiPwxvBY
+ixrciiNR/th0FM9MxVx/omHdI61EmAhTA465SjxECF8
--> ssh-ed25519 +qVung qZsGi4JqgpHrjlg2VdY+OhXb0BzYTytBBqY3jNsrSgU
+-> ssh-ed25519 +qVung Q7k74fDLKwCdzobz0b6ByS2LrhMOIC58Ofto0gpBLFE
-GgvQG5iMd6XTZRCC3EBBvqF7nhkqAJmxdIkCFRV46Ok
+p4CIje+sO/nOaO1lzAY9n2HYLUKxEvKDbxeR6dOyM00
 -> ssh-rsa krWCLQ
-EkmY8uc79xWfKjlIozS4Yigorz9IdK8T8VjMnVcJN6+rhoRctQNVCj4JgogY4wa0
+ezrZTitn0/BRD0K7e2K53qz9AZCa0aHlzFSuyzqyVJLdAZUxBUnfBwmGuJgKTa4Q
-V3ObjoRPZgVU3qPmkPgIKVa2Mvf6MrCMwvvE4j2Yyy6lmQEwFdvk4s2c6AD6T8Bf
+fWsXBs+L65hkcL6/VKS7oSGGyoEHmoPFKbb08B6FKLHt9V1td5xbHIoTYbvSavUA
-rktRYqOcFavuDr348e0ZzKniFTRcPMcY49mqBR/mWIfSEtLxBgpFUCn6f40PLndT
+g3wpTUa4eG3ivcu96VjyyBKTAc7LN7h7dSMbvvP5tpWT5vL+WstCdFf7zzUL9HBS
-3dse7kgRBlrKbzmf6JIsITHejqwDRq2bZqHWAmZhb6+ske7oDicAt90FDoDbrwvd
+yI8dzEbCQIgAAaHj90MREgIIgIB27Dn1PvkEBGYky5ybBRa3DXVyqnX0dDtsXWpK
-YwXPRDCxgATlNz8n/xFUxd35X+zEftUUtANSGtihIE4LcdsO7IOwv/FCjdEn/3YW
+ipRPDV7HC1+x2TlqQjD5ED737r/AP573IXbnRLSEWnGDjtd/JWQmfOO3JACoRjU6
-ZtQjphnxgDsY61PEFCMnYg
+qfb5SSDT9QriuWSow7CDhQ
--> ssh-ed25519 /vwQcQ DKQuo5jVunUFTCbOxVV57Xl6q+DDOVDWXdon/lZlLi0
+-> ssh-ed25519 /vwQcQ duuo3BGe4Q1MHMljgzmtpzvtiOvAHqKu2HS9SBxLuhE
-doN6en8IK4Ju0uATp+IZAhYl1tvdnfyxHziSobb1ER4
+GCwccbE5lX5uPIri/7Vn6hzpfL7ouJBFU14bKjl6yTM
--> ssh-ed25519 0R97PA I1GECXSPagJ5kD7CeVA21TQmpMEgLeaiB7XYEomUl2U
+-> ssh-ed25519 0R97PA WIFf8tbMlmNrNFF5tRcL+mOJ40SvIdppAtItWtxzCk8
-d0kO+4SkAPC/ois39SZafEhTqvmDpCZbWTUU1aUZ47o
+miU7Z4poEVMZCeAEef1VS0jouCDxGro2xLEE3hnRJEQ
---- 555iE+C2kDLIdAJ5KARyKcBQZSDRWASuzcNiKZ9IbRI
+--- Iaff5rxl9r1qEnlpkOpGyBGtAvGMLyBlJQ45iInuAnw
-òeÕceV&˜ßà‰g˜óáÔÄýæ›6•=6!õC<C3B5>Cˆû^»âÕèí€zÕ§®(Ó<>!ÄB•B|ô<>ï°Ú'¿Rªîž†_a	UtI³3
+cýI±C«¤2ˆ7µ	½³Ú“nZMþ`œ{7È`¨½V@ñyzÀÅžª€)ÛY‰DÄßÇX—o“óä	~<PÙ›òš5Tpúx
+ÓRÏÜö

secrets/metrics-push-password.age

@@ -1,52 +1,57 @@
 age-encryption.org/v1
--> ssh-ed25519 +HUDfA FOqd+I9DzoloOMK2InPz8yAGsk+ZgMKy0n542DmF5ig
+-> ssh-ed25519 +HUDfA SrjyocQ2U/mcmsVX3bhTDPiNfnRepZ+J//d4JkVrQ0w
-sui4rdOQcvjL6H9rPSbSAyIggaSbsIVrontrkFpPPC0
+MELfJrKcLlC3rWKHdMZKZyXB0ztzmZUjWUcT8ibP8vE
--> ssh-ed25519 wIR2ZA V4KPrGw2NKeOBWpjsRbhUJ/eLR8/hvExNMpcBvC7gCY
+-> ssh-ed25519 87T2Ig IN9MMxRNzgKHBmGwidVWIvq2xpNVkbioWjG0lf+B5zM
-Zjc+HtALqZbp+L8tUUgaFe9LR4NKptpFq/L7xhTItXM
+sXIXfrTak7E8isigDDnrzvjJli5ma5f9fOJnWCdDRpU
--> ssh-ed25519 oGiV/Q kJS4DAPBTOgADY7LCZnIfORMM1RJez/5XGoKDfErHjM
+-> ssh-ed25519 wIR2ZA 4DD/V3Xq1B2t8Zb11MnvtSZ3Oq5Glvka93g313dVSyU
-LN3XE7qM2SHqQwb+JjIq5tMvt77NI4+YOxYnZh82udA
+TrQiCJGOtitCCfNy0PdaRaPnk2mYCEPKtnOtdAzGolg
--> ssh-ed25519 gO3aog gJFIrngWZp4ypA2IZwr+c0JkWgUu9VN5AzoyyhozlDE
+-> ssh-ed25519 oGiV/Q W67zxBlGYg3PhUbwBiGE2vVoIl455R+4g3EClZKwulI
-lezfokY1lgABSKNO+Fr+tTlIjC3gzc4Bw2YlGLy+WvI
+2sldkyyBUGxhXRCoa/vW5LrxbI0TqerOeOqrTtzY3Mo
--> ssh-ed25519 r/iJSw VzO6pblztwci/TMfha+dOc6Vg4DC/1oSNEt0aFaCYRE
+-> ssh-ed25519 gO3aog YVF4hdjNYxOPE8v95BENIb6khsu0+tztaPNNCsXoWDE
-Mf0LjSjWJA2lMt1M1z+tGJ+9NVMxd8J5CSMvaLK8zB4
+LLX/uofYt5/HQ7q5L35UK2t05rOlhCDnC4SIJx0bNtM
--> ssh-ed25519 N/+Clw uNBuYGWU+LLY856o15jLkJNk6pu42FnX55CoE98/ukA
+-> ssh-ed25519 r/iJSw RMwg0xLCOVA+wc08f67kkUVIgy6W3Ypd3jRkRHFA+l4
-zh+sZ0nskVPUKd3Ajg1FHng7caKhkEHiRFcm8c53siw
+KR5RElZHGzzLU9hjr3Qg3NwudDxMtHqcf2t6xjDMz+U
--> ssh-ed25519 CtkSZw YP79uyNelg7+nbeois1vu64anUC0lhUhIie6EqUz2i0
+-> ssh-ed25519 N/+Clw BBYMWbIT8dXcD7SU+LrIuFeM+2RodGF2rW1ubx/W9mU
-rb9zte3dN0+uwjyJLGaUfeEQcVtMerKEOVAocLGXUYs
+yANEUWhFtNkx3VArOTTW+rREcxwzkN47CD2kK6JsMns
--> ssh-ed25519 keg2lg +g5uYkOOyQABVmL+9t08aaMklNEbBO2j6vqKyrwYrhA
+-> ssh-ed25519 CtkSZw wy5ZfWI6tqN3OZDqRZvb6lhj8Pt+GrP3YryqhjH0ugo
-U4FzATeou9spmYchqHPR/WR79Y+ILWpwhLwxjYQd7d4
+OtY/WsGkJJghGGAh4cfZOxkg/WcYJ4w2gu4Hu9VHntc
--> ssh-ed25519 H885DA tAx+W9kfJkvERw9KPKZInC0s44QqQIu71MPUosasHy4
+-> ssh-ed25519 keg2lg lzE0HqDHBwDyuc5m5T9YSxxTgEk4mOQWY3l7a1+QKD0
-5ks2qkZfkMLK4meVHTfWpR8qCeU3vKdPiWVRTyD6OhI
+cn07YAocsIrSeWo1ZGyFzq3un8kdpEuS6zYpKs7G/iI
--> ssh-ed25519 Rq7K4Q xwSlrqIh+rZFv6w1iDcPyD0nEmESlmHleUHsVPrG2Bg
+-> ssh-ed25519 H885DA eZJW1T2VPMhDs/ygauDFdd1Md3D830ysel1yUZkZoSI
-OgrWCBqb7SAtQQSUnTQ1l9JRyDGS2DgzKRRbMCtKK7g
+wpq1+ndzQWUUN2yYMKnEZrOcgCuqKIrDjaeX+XpkQgk
--> ssh-ed25519 vvyRpw wQB8wg6bGvb68pvEp+7khrNpZTUxSVzLIfubbYsX+34
+-> ssh-ed25519 Rq7K4Q CQ+Y2k5F8Q79GF5PQh8qDmxWgrKcqJHjAodVBqKqQkc
-KZ2/Vnxg7Gpazc26lYddjNnMxpoteb5ysuTZUg00ZvE
+SkcUl6dFoBQmPOOjTEopgcn5vzLH2oHICymAAS7nsAQ
--> ssh-ed25519 aSEktQ KdKSZuVH/v+gkZkL07YdUJ5vvH2+mcUR4x+mXHylhys
+-> ssh-ed25519 vvyRpw nW2eCEqQ6uCT9RgIJyCSpP4JHwQtKDSiBBp1wdVFtTE
-MRGd8l+0X6XVq1KpLqYqUZD/4EkOKz3mpHsdQepc6kc
+DQcHIBTNqvFVYV1fXbGhu0pCwa++knjLpCVFC3npaS0
--> ssh-ed25519 cD6JxA FesXIZs/X+fWefYjP0sfkwz6bYLxOkuIzQppwZYXNTU
+-> ssh-ed25519 aSEktQ 7SEG8F8UyH0gR9uT+mFfBIXsAIUFnNd2bZgyJ8C/gVQ
-hg+ZTdCGuQ66FIc+NZI023Aunnhz+Ds5cFKUwNj+MGU
+JTlr5eIhpepOoCxi54nrG7Wjxq9CXZYkb33kd2urdak
--> ssh-ed25519 1qYEfw HRQdZ4u1UWpzwIF/0lbJ1NVDQ+/Rl913jk+BwLM0KCE
+-> ssh-ed25519 cD6JxA QKVkY0MS3LeJf+YfwJT2yysuseg8tSAEGHOBgHFsVkc
-CHlDCaov7TWme5YMBiV6Tby0IReB8pER/RbDkpI3TWM
+IpAAWCWxHNg1MOBjG+JNXcTE/xNrDW8+5Cz/hNWVYvU
--> ssh-ed25519 2D+APA BTVVWo3G0tZj/hUMH5cwByYf3LjAg2RNVMhYrkXxXjQ
+-> ssh-ed25519 1qYEfw pA2G6CxFosIcXsBnTUfN1wsPs3Ue5aMzo7wameAacXM
-iKghO+M6xpp95xVrmydz9GJJIOK5JrIsoL+CSFD77uM
+av7xGnRkh57JtgF37QtaF//eYS/pHqznHY4DJewRp5s
--> ssh-ed25519 j2r2qQ RC/2vV5yr1af4iyeouQwIBK/r8b4nD51WwxgbuMEgG0
+-> ssh-ed25519 2D+APA SOSVjgiiugDWg9HeFIlaLa+mo3q8AHhntl1tHEB6QUQ
-L+uqV7eeCNqnMTqCNmvLPZFNTdmlYu/i7+3NVwmpIxA
+QINZr847DASGM32Si6t1mHH6fCkKnq/sa1+3IXhaSlE
--> ssh-ed25519 C/bBAQ KO1owoeb7pbuXtDS+f/TziotgffL0Eg6qnjJ9W8Yp2c
+-> ssh-ed25519 eTSU6g NuV8gm/Ijo6BpZptiYua2bnYNoxuHcOtce9zGNyi0yo
-af4IhSiXlMPiNuM473dIeWQqNbRgb3ciHyoa6buolyU
+E4zAIpZN5eTWJanPEwS7B6RfnnMRLDaOj+5l5L4GdCk
--> ssh-ed25519 K3b7BA h4mC/hZ10ToaaYDRyBOyPpcvA28sY5FPCQPuaTTRIws
+-> ssh-ed25519 j2r2qQ PpKKKAJikQKWAaYvDhIoiPeTkWtE1chw8lCpZ4O+LHs
-VG4QtmEOnubhhjV3CS49aYOyVl/Dq+ryxfZENgFJZTo
+4kR0ZNRMt0fljaOu3UgqVrUFnc6v916IyKdYkvz/zfA
--> ssh-ed25519 +qVung 6gs9DdduYx2twVsFED7HJnGFfKZynUctQIO4F3MXfj8
+-> ssh-ed25519 C/bBAQ m7XsRBwlHgWXifCif/8H9TcSqs0so5hha2T4tCq6qn4
-gMmU2tXwR9K8Nb5gMKPbTexE58FOAK6QlVYzGvaX3hw
+QltQrR6Y3Im4xo8DtpzN5kMsHNfkpG0FE6Y2GnkrH5Y
+-> ssh-ed25519 K3b7BA x91SNkgN6NSlw2FZnliA+c6zoTYyeuZh2iT+Rl+qtT4
+nKU6GcX4WLTRncStiW6BS7iK7zlCVhn55FPjRNniqSc
+-> ssh-ed25519 +qVung opSEU5VaLZcm4GhcKlNtG/Ut0jU6oTYQuqvnDkuSGT4
+ny6Wfsi/PIj5A9q/fwL3vwnkft/yH6fqlPIXo0cklfY
 -> ssh-rsa krWCLQ
-vjNcmgDmmaNUSXIUgKf1digOgbohvyKkYSUalTOskvPo+9NRZbp0IJ7DoYLRrSBB
+p5Y5fVwyG2s7m9ClsgbcVz/fSF2lJvbXxuN8O4b6sp+QiABmSGs0R3pZuf1v9xBr
-DobCBM078iKOvIGGJCIbMS86/z/7lz6SSPcbfM1EG+hknVJLZaj+K3PYYSX6QTUC
+Jc0JWhl4vvvb9F9WUbJR50hIpdWo6iX4vrz3TnSvPFmnpUpRfe+a29ZJhp0vCA4a
-6rWSC+yg0gKehAhnYO3q+8mnismk7SERdyCZDNtPwHOhTAt6NZ6e+33VFxnbJPTz
+HVaOJGlnGZ5BdSkvPslGVCPu684OmO/veL5G1H7xmN6yg2b3n7SaGF7A4+rpVqgI
-IvoNU/RTUhV+XuKbtosm55PqDkOuTM27jesZ0/SARYL+gVgaltacqt4kzbEMOP/W
+6GZiFpnM6LpyKyoTyXRL0ghzjhwggQCCnBaN7GIUhvPacPdilAJWmnagQzx8aZpT
-tv2kU6f1eNaX71c57DGI7rfcvLrPRAjTxUhsuKJPGQeaHtfiWz832gUMIJOEjoo0
+LRe1WAeKH2Lbar4UNeot3MzWkZxUXyyWszTMe1ca94N3jY7MG8adzX3guMykP5qA
-mvrAfyoykJRbPGNFl5pMmg
+eya7UOphIwkQKlVB3N5bfQ
--> ssh-ed25519 /vwQcQ gpPktkJ57USbj7kn1qbeUQDbHHSCuzWM5OcmNooBMi8
+-> ssh-ed25519 /vwQcQ xQFghc3LzwG82u+h80e3NdfbCh85OKdai32pwvS3uzs
-6JPXUJYQ1IjRVv90r1EJx3EUMDPmU9X1FK6j/6vT5hE
+MdUPg9BHvPX85jWnV7evkNekPrzoJuT8FP0l/mhfZDk
--> ssh-ed25519 0R97PA vzT774La7rcOMz7/KYjSUsY+D6V5bi5j3ghdDBLBoAU
+-> ssh-ed25519 0R97PA 8cDQRKrujysaUiD5OxdrpmWn7ZZCJ9SNbLYtWuTSmXg
-HAXfMmFuj3YJGCBR1U0btPlr9MdIBYnwT1ufbHaAxVk
+HFa/6WbK4aMK3cKEMEycyiclTu8jOcCMcr1R7Ebh73c
---- /0DCLjy0dwjRGPnkNk/a9fZ1ox9+LVkwh9Y5jiyA8x4
+--- wZAdkwtibHAVLCqtfmZ54ZtPwDPogkRwfKREBR2xOeY
-·1ë³KëB³|†Ü\<5C>öST¾¦i¸ð¦/<2F>ØÕ
hž9}%ä\÷Ÿþ,"g<>ì°Z³<5A>ÊšwÍþ0»ä5¸´
Rm’'
+[‹‰Ã×÷ÂûȹÂÃökR‘î\Äub<75>çГRö
+‚†qþà«‘ÎHÈbe{Y
gÿ<67>mÀ¿¥Ûs®Æ¬„[-p¸

services/baremetal-builder/default.nix

@@ -28,7 +28,19 @@ in
         "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAvUT9YBig9LQPHgypIBHQuC32XqDKxlFZ2CfgDi0ZKx"
       ];
     };
-    nix.settings.trusted-users = [ "builder" ];
+
+    users.users.buildbot = {
+      isSystemUser = true;
+      group = "nogroup";
+      home = "/var/empty";
+      shell = "/bin/sh";
+      openssh.authorizedKeys.keys = [
+        # Do not hardcode Buildbot's public key, selectively
+        # add the keys of the coordinators that require us.
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMnOLLX0vGTZbSJrUmF9ZFXt/NIId/MUrEpXmL2vxod"
+      ];
+    };
+    nix.settings.trusted-users = [ "builder" "buildbot" ];
 
 
     nixpkgs.hostPlatform = "x86_64-linux";

services/buildbot/default.nix

@@ -0,0 +1,137 @@
+{
+  nodes,
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.bagel.services.buildbot;
+  cfgGerrit = nodes.gerrit01.config.bagel.services.gerrit;
+  ssh-keys = import ../../common/ssh-keys.nix;
+  inherit (lib) mkEnableOption mkOption mkIf types;
+in
+{
+  options.bagel.services.buildbot = {
+    enable = mkEnableOption "Buildbot";
+    domain = mkOption {
+      type = types.str;
+    };
+
+    builders = mkOption {
+      type = types.listOf types.str;
+      description = "List of builders to configure for Buildbot";
+      example = [ "builder-2" "builder-3" ];
+    };
+  };
+
+  config = mkIf cfg.enable {
+    networking.firewall.allowedTCPPorts = [ 80 443 ];
+    age.secrets.buildbot-worker-password.file = ../../secrets/buildbot-worker-password.age;
+    age.secrets.buildbot-oauth-secret.file = ../../secrets/buildbot-oauth-secret.age;
+    age.secrets.buildbot-workers.file = ../../secrets/buildbot-workers.age;
+    age.secrets.buildbot-service-key.file = ../../secrets/buildbot-service-key.age;
+    age.secrets.buildbot-signing-key.file = ../../secrets/buildbot-signing-key.age;
+    age.secrets.buildbot-remote-builder-key.file = ../../secrets/buildbot-remote-builder-key.age;
+
+    services.nginx.virtualHosts.${cfg.domain} = {
+      forceSSL = true;
+      enableACME = true;
+      extraConfig = ''
+        add_header Access-Control-Allow-Credentials 'true' always;
+        add_header Access-Control-Allow-Origin 'https://cl.forkos.org' always;
+      '';
+    };
+
+    services.buildbot-nix.worker = {
+      enable = true;
+      workerPasswordFile = config.age.secrets.buildbot-worker-password.path;
+      # All credits to eldritch horrors for this beauty.
+      workerArchitectures =
+      {
+        # nix-eval-jobs runs under a lock, error reports do not (but are cheap)
+        other = 8;
+      } // (
+        lib.filterAttrs
+          (n: v: lib.elem n config.services.buildbot-nix.coordinator.buildSystems)
+          (lib.zipAttrsWith
+            (_: lib.foldl' lib.add 0)
+            (lib.concatMap
+              (m: map (s: { ${s} = m.maxJobs; }) m.systems)
+              config.nix.buildMachines))
+        );
+    };
+
+    services.buildbot-nix.coordinator = {
+      enable = true;
+
+      inherit (cfg) domain;
+
+      oauth2 = {
+        name = "Lix";
+        clientId = "forkos-buildbot";
+        clientSecretFile = config.age.secrets.buildbot-oauth-secret.path;
+        resourceEndpoint = "https://identity.lix.systems";
+        authUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth";
+        tokenUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
+      };
+
+      workersFile = config.age.secrets.buildbot-workers.path;
+
+      allowedOrigins = [
+        "*.forkos.org"
+      ];
+
+      buildSystems = [
+        "x86_64-linux"
+      ];
+
+      gerrit = {
+        domain = cfgGerrit.canonicalDomain;
+        # Manually managed account…
+        # TODO: https://git.lix.systems/the-distro/infra/issues/69
+        username = "buildbot";
+        port = cfgGerrit.port;
+        privateKeyFile = config.age.secrets.buildbot-service-key.path;
+        projects = [
+          "buildbot-test"
+          "nixpkgs"
+          "infra"
+        ];
+      };
+
+      evalWorkerCount = 6;
+      evalMaxMemorySize = "4096";
+
+      signingKeyFile = config.age.secrets.buildbot-signing-key.path;
+    };
+
+    nix.distributedBuilds = true;
+    nix.buildMachines = map (n: {
+        hostName = nodes.${n}.config.networking.fqdn;
+        protocol = "ssh-ng";
+        # Follows Hydra.
+        maxJobs = 8;
+        sshKey = config.age.secrets.buildbot-remote-builder-key.path;
+        sshUser = "buildbot";
+        systems = [ "x86_64-linux" ];
+        supportedFeatures = nodes.${n}.config.nix.settings.system-features;
+        # TODO: fix it, see the Hydra file about it.
+        # IFD already exist in NixOS, so it's fine, I guess.
+        publicHostKey = builtins.readFile (pkgs.runCommandLocal "in-the-right-form" {
+          buildInputs = [
+            pkgs.coreutils
+          ];
+        } ''
+          echo -n '${ssh-keys.machines.${n}}' | base64 -w0 > $out
+        '');
+      }
+    ) cfg.builders;
+
+    nix.settings.keep-derivations = true;
+    nix.gc = {
+      automatic = true;
+      dates = "hourly";
+    };
+  };
+}

services/default.nix

@@ -8,5 +8,6 @@
     ./postgres
     ./forgejo
     ./baremetal-builder
+    ./buildbot
   ];
 }

services/gerrit/checks.js

@@ -0,0 +1,113 @@
+/* Inspired from the Lix setup.
+ * Original-Author: puckipedia
+ */
+Gerrit.install((plugin) => {
+  // TODO: can we just use `plugin.serverInfo().plugin` and control the settings over there.
+  const configuration = {
+    baseUri: @BASE_URI@,
+    supportedProjects: @SUPPORTED_PROJECTS@,
+  };
+
+  function makeBuildbotUri(suffix) {
+    return `${configuration.baseUri}/${suffix}`;
+  }
+
+  let builders = [];
+  let fetchBuilders = async () => {
+    if (builders.length > 0) return;
+      let data = await (await fetch(makeBuildbotUri(`api/v2/builders`), { credentials: 'include' })).json();
+      builders = data.builders;
+  };
+
+
+  let checksProvider;
+  checksProvider = {
+    async fetch({ repo, patchsetSha, changeNumber, patchsetNumber }, runBefore = false) {
+      if (!configuration.supportedProjects.includes(repo)) {
+        return { responseCode: 'OK' };
+      }
+
+      let num = changeNumber.toString(10);
+
+      let branch = `refs/changes/${num.substr(-2)}/${num}/${patchsetNumber}`;
+
+      let changeFetch = await fetch(makeBuildbotUri(`api/v2/changes?limit=1&order=-changeid&revision=${patchsetSha}&branch=${branch}`), { credentials: 'include' });
+      if (changeFetch.status == 400) {
+        if ((await changeFetch.json()).error === 'invalid origin' && !runBefore) {
+          return await checksProvider.fetch({ repo, patchsetSha, changeNumber, patchsetNumber }, true);
+        }
+
+        return { responseCode: 'OK' };
+      } else if (changeFetch.status === 403) {
+        return { responseCode: 'NOT_LOGGED_IN', loginCallback() {
+          window.open(configuration.baseUri);
+        } };
+      }
+
+      let changes = await changeFetch.json();
+      if (changes.meta.total === 0) {
+        return { responseCode: 'OK' };
+      }
+
+      let { changeid } = changes.changes[0];
+      let { builds } = await (await fetch(makeBuildbotUri(`api/v2/changes/${changeid}/builds?property=owners&property=workername`), { credentials: 'include' })).json();
+      await fetchBuilders();
+      let links = [];
+      let runs = [];
+      for (let build of builds) {
+        let name = `unknown builder ${build.builderid}`;
+        for (let builder of builders) {
+          if (builder.builderid === build.builderid) {
+            name = builder.name;
+            break;
+          }
+        }
+
+        if (name === `${repo}/nix-eval`) {
+            links.push({
+              url: makeBuildbotUri(`#/builders/${build.builderid}/builds/${build.number}`),
+              primary: true,
+              icon: 'external',
+            });
+        }
+
+        let checkrun = {
+          attempt: build.buildrequestid,
+          // FIXME: generalize this accordingly once auto-discovery is available.
+          checkName: name.replace(/^hydraJobs\./, ''),
+          externalId: build.buildrequestid.toString(),
+          status: build.complete ? 'COMPLETED' : (typeof build.started_at !== 'number' ? 'SCHEDULED' : 'RUNNING'),
+          checkLink: makeBuildbotUri(`#/builders/${build.builderid}/builds/${build.number}`),
+          labelName: 'Verified',
+          results: [],
+          links: [{
+            url: makeBuildbotUri(`#/builders/${build.builderid}/builds/${build.number}`),
+            primary: true,
+            icon: 'external',
+          }],
+        };
+
+        if (build.started_at !== null) {
+          checkrun.startedTimestamp = new Date(build.started_at * 1000);
+        }
+
+        if (build.complete_at !== null) {
+          checkrun.finishedTimestamp = new Date(build.complete_at * 1000);
+        }
+
+        if (build.results !== null) {
+          checkrun.results = [{
+            category: build.results < 2 ? 'SUCCESS' : 'ERROR',
+            summary: build.state_string,
+          }];
+        }
+
+        runs.push(checkrun);
+      }
+
+      return { responseCode: 'OK', runs, links };
+    }
+  };
+
+  plugin.checks().register(checksProvider);
+});

services/gerrit/default.nix

@@ -3,7 +3,7 @@
 { pkgs, config, lib, ... }:
 
 let
-  inherit (lib) mkEnableOption mkIf mkOption types;
+  inherit (lib) mkEnableOption mkIf mkOption types head;
   cfgGerrit = config.services.gerrit;
   cfg = config.bagel.services.gerrit;
 
@@ -16,11 +16,22 @@ in
       type = types.listOf types.str;
       description = "List of domains that Gerrit will answer to";
     };
+    canonicalDomain = mkOption {
+      type = types.str;
+      description = "Canonical domain for this Gerrit instance";
+      default = head cfg.domains;
+    };
     data = mkOption {
       type = types.path;
       default = "/var/lib/gerrit";
       description = "Root of data directory for the Gerrit";
     };
+    port = mkOption {
+      type = types.port;
+      default = 29418;
+      readOnly = true;
+      description = "Port for the Gerrit SSH server";
+    };
   };
 
   imports = [
@@ -28,7 +39,7 @@ in
   ];
 
   config = mkIf cfg.enable {
-    networking.firewall.allowedTCPPorts = [ 29418 ];
+    networking.firewall.allowedTCPPorts = [ cfg.port ];
 
     environment.systemPackages = [ jdk ];
 
@@ -58,9 +69,24 @@ in
         "webhooks"
       ];
 
-      plugins = with pkgs.gerritPlugins; [ 
+      plugins = with pkgs.gerritPlugins; [
         oauth
         metrics-reporter-prometheus
+        # Buildbot checks plugin (writeText because services.gerrit.plugins expects packages)
+        (pkgs.runCommand "checks.js" {
+          BASE_URI = builtins.toJSON "https://buildbot.forkos.org";
+          SUPPORTED_PROJECTS = builtins.toJSON [
+            "infra"
+            "nixpkgs"
+            "buildbot-test"
+          ];
+        }
+        ''
+          echo "configuring buildbot checks plugin for $BASE_URI with $SUPPORTED_PROJECTS project list"
+          substitute ${./checks.js} $out \
+            --replace-fail "@BASE_URI@" "$BASE_URI" \
+            --replace-fail "@SUPPORTED_PROJECTS@" "$SUPPORTED_PROJECTS"
+        '')
       ];
 
       package = pkgs.gerrit;
@@ -115,7 +141,7 @@ in
         # Other settings
         log.jsonLogging = true;
         log.textLogging = false;
-        sshd.advertisedAddress = "cl.forkos.org:29418";
+        sshd.advertisedAddress = "${cfg.canonicalDomain}:${toString cfg.port}";
         cache.web_sessions.maxAge = "3 months";
         plugins.allowRemoteAdmin = false;
         change.enableAttentionSet = true;
@@ -130,7 +156,7 @@ in
         # Configures gerrit for being reverse-proxied by nginx as per
         # https://gerrit-review.googlesource.com/Documentation/config-reverseproxy.html
         gerrit = {
-          canonicalWebUrl = "https://cl.forkos.org";
+          canonicalWebUrl = "https://${cfg.canonicalDomain}";
           docUrl = "/Documentation";
           defaultBranch = "refs/heads/main";
         };
@@ -147,7 +173,7 @@ in
         # Auto-link other CLs
         commentlink.gerrit = {
           match = "cl/(\\d+)";
-          link = "https://cl.forkos.org/$1";
+          link = "https://${cfg.canonicalDomain}/$1";
         };
 
         # Configures integration with Keycloak, which then integrates with a