From 68d956f1ba8889292c45bf6562639575a8026d47 Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Wed, 17 Jul 2024 14:47:38 +0200
Subject: [PATCH 1/8] flake: add buildbot-nix on the refactor branch

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
---
 flake.lock | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
 flake.nix  |  5 ++++
 2 files changed, 73 insertions(+), 2 deletions(-)

diff --git a/flake.lock b/flake.lock
index 066cd89..0f3747d 100644
--- a/flake.lock
+++ b/flake.lock
@@ -55,6 +55,29 @@
         "type": "github"
       }
     },
+    "buildbot-nix": {
+      "inputs": {
+        "flake-parts": "flake-parts",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "treefmt-nix": "treefmt-nix"
+      },
+      "locked": {
+        "lastModified": 1715022238,
+        "narHash": "sha256-sDD6WWJXJ/1j07aQE0RAUlrQBekXABtEKm7gtaTN45w=",
+        "ref": "refs/heads/refactor",
+        "rev": "d5e3345097cdda5c74bccddb27abb5b5c84eff5b",
+        "revCount": 257,
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
+      },
+      "original": {
+        "ref": "refs/heads/refactor",
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
+      }
+    },
     "colmena": {
       "inputs": {
         "flake-compat": "flake-compat",
@@ -133,6 +156,27 @@
       }
     },
     "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "buildbot-nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1706830856,
+        "narHash": "sha256-a0NYyp+h9hlb7ddVz4LUn1vT/PLwqfrWYcHMvFB1xYg=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "b253292d9c0a5ead9bc98c4e9a26c6312e27d69f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-parts_2": {
       "inputs": {
         "nixpkgs-lib": [
           "hydra",
@@ -254,7 +298,7 @@
     },
     "nix-eval-jobs": {
       "inputs": {
-        "flake-parts": "flake-parts",
+        "flake-parts": "flake-parts_2",
         "lix": [
           "hydra",
           "lix"
@@ -264,7 +308,7 @@
           "hydra",
           "nixpkgs"
         ],
-        "treefmt-nix": "treefmt-nix"
+        "treefmt-nix": "treefmt-nix_2"
       },
       "locked": {
         "lastModified": 1721195872,
@@ -404,6 +448,7 @@
     "root": {
       "inputs": {
         "agenix": "agenix",
+        "buildbot-nix": "buildbot-nix",
         "colmena": "colmena",
         "hydra": "hydra",
         "lix": [
@@ -484,6 +529,27 @@
       }
     },
     "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "buildbot-nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1708897213,
+        "narHash": "sha256-QECZB+Hgz/2F/8lWvHNk05N6NU/rD9bWzuNn6Cv8oUk=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "e497a9ddecff769c2a7cbab51e1ed7a8501e7a3a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
+    },
+    "treefmt-nix_2": {
       "inputs": {
         "nixpkgs": [
           "hydra",
diff --git a/flake.nix b/flake.nix
index 1a929e8..c8eef8f 100644
--- a/flake.nix
+++ b/flake.nix
@@ -17,6 +17,9 @@
     nix-gerrit.url = "git+https://git.lix.systems/the-distro/nix-gerrit.git";
     nix-gerrit.inputs.nixpkgs.follows = "nixpkgs";
 
+    buildbot-nix.url = "git+https://git.lix.systems/lix-project/buildbot-nix.git?ref=refs/heads/refactor";
+    buildbot-nix.inputs.nixpkgs.follows = "nixpkgs";
+
     lix.follows = "hydra/lix";
   };
 
@@ -73,6 +76,8 @@
       commonModules = [
         inputs.agenix.nixosModules.default
         inputs.hydra.nixosModules.hydra
+        inputs.buildbot-nix.nixosModules.buildbot-coordinator
+        inputs.buildbot-nix.nixosModules.buildbot-worker
 
         ./services
         ./common

From fda59ee6c0310cc363367d57e0dc3768526222b9 Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Wed, 17 Jul 2024 15:43:22 +0200
Subject: [PATCH 2/8] gerrit: factor more configuration in the NixOS module for
 external consumption

Other modules may require information to configure themselves from the
Gerrit module.

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
---
 hosts/gerrit01/default.nix  |  1 +
 services/gerrit/default.nix | 23 +++++++++++++++++------
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/hosts/gerrit01/default.nix b/hosts/gerrit01/default.nix
index a0f9f85..49e93ae 100755
--- a/hosts/gerrit01/default.nix
+++ b/hosts/gerrit01/default.nix
@@ -35,6 +35,7 @@
     domains = [
       "cl.forkos.org"
     ];
+    canonicalDomain = "cl.forkos.org";
     data = "/gerrit-data";
   };
 
diff --git a/services/gerrit/default.nix b/services/gerrit/default.nix
index 5b4f8c5..3796dc1 100644
--- a/services/gerrit/default.nix
+++ b/services/gerrit/default.nix
@@ -3,7 +3,7 @@
 { pkgs, config, lib, ... }:
 
 let
-  inherit (lib) mkEnableOption mkIf mkOption types;
+  inherit (lib) mkEnableOption mkIf mkOption types head;
   cfgGerrit = config.services.gerrit;
   cfg = config.bagel.services.gerrit;
 
@@ -16,11 +16,22 @@ in
       type = types.listOf types.str;
       description = "List of domains that Gerrit will answer to";
     };
+    canonicalDomain = mkOption {
+      type = types.str;
+      description = "Canonical domain for this Gerrit instance";
+      default = head cfg.domains;
+    };
     data = mkOption {
       type = types.path;
       default = "/var/lib/gerrit";
       description = "Root of data directory for the Gerrit";
     };
+    port = mkOption {
+      type = types.port;
+      default = 29418;
+      readOnly = true;
+      description = "Port for the Gerrit SSH server";
+    };
   };
 
   imports = [
@@ -28,7 +39,7 @@ in
   ];
 
   config = mkIf cfg.enable {
-    networking.firewall.allowedTCPPorts = [ 29418 ];
+    networking.firewall.allowedTCPPorts = [ cfg.port ];
 
     environment.systemPackages = [ jdk ];
 
@@ -58,7 +69,7 @@ in
         "webhooks"
       ];
 
-      plugins = with pkgs.gerritPlugins; [ 
+      plugins = with pkgs.gerritPlugins; [
         oauth
         metrics-reporter-prometheus
       ];
@@ -115,7 +126,7 @@ in
         # Other settings
         log.jsonLogging = true;
         log.textLogging = false;
-        sshd.advertisedAddress = "cl.forkos.org:29418";
+        sshd.advertisedAddress = "${cfg.canonicalDomain}:${cfg.port}";
         cache.web_sessions.maxAge = "3 months";
         plugins.allowRemoteAdmin = false;
         change.enableAttentionSet = true;
@@ -130,7 +141,7 @@ in
         # Configures gerrit for being reverse-proxied by nginx as per
         # https://gerrit-review.googlesource.com/Documentation/config-reverseproxy.html
         gerrit = {
-          canonicalWebUrl = "https://cl.forkos.org";
+          canonicalWebUrl = "https://${cfg.canonicalDomain}";
           docUrl = "/Documentation";
           defaultBranch = "refs/heads/main";
         };
@@ -147,7 +158,7 @@ in
         # Auto-link other CLs
         commentlink.gerrit = {
           match = "cl/(\\d+)";
-          link = "https://cl.forkos.org/$1";
+          link = "https://${cfg.canonicalDomain}/$1";
         };
 
         # Configures integration with Keycloak, which then integrates with a

From 7789e9ce75c656bcfcaf60720b544232f55df5e9 Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Wed, 17 Jul 2024 15:43:29 +0200
Subject: [PATCH 3/8] services/buildbot: init

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
---
 common/base-server.nix               |   2 +-
 flake.lock                           |   8 +--
 secrets.nix                          |   9 +++
 secrets/buildbot-oauth-secret.age    |  20 ++++++
 secrets/buildbot-service-key.age     | Bin 0 -> 1429 bytes
 secrets/buildbot-signing-key.age     | Bin 0 -> 1133 bytes
 secrets/buildbot-worker-password.age |  20 ++++++
 secrets/buildbot-workers.age         |  22 ++++++
 services/buildbot/default.nix        | 103 +++++++++++++++++++++++++++
 services/default.nix                 |   1 +
 10 files changed, 180 insertions(+), 5 deletions(-)
 create mode 100644 secrets/buildbot-oauth-secret.age
 create mode 100644 secrets/buildbot-service-key.age
 create mode 100644 secrets/buildbot-signing-key.age
 create mode 100644 secrets/buildbot-worker-password.age
 create mode 100644 secrets/buildbot-workers.age
 create mode 100644 services/buildbot/default.nix

diff --git a/common/base-server.nix b/common/base-server.nix
index 57d6eab..3f5616e 100644
--- a/common/base-server.nix
+++ b/common/base-server.nix
@@ -25,7 +25,7 @@
   nix.gc = {
     automatic = true;
     persistent = true;
-    dates = "daily";
+    dates = lib.mkDefault "daily";
     options = "--delete-older-than 30d";
   };
 
diff --git a/flake.lock b/flake.lock
index 0f3747d..38d1859 100644
--- a/flake.lock
+++ b/flake.lock
@@ -64,11 +64,11 @@
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1715022238,
-        "narHash": "sha256-sDD6WWJXJ/1j07aQE0RAUlrQBekXABtEKm7gtaTN45w=",
+        "lastModified": 1721229951,
+        "narHash": "sha256-RO7jlz2T0h9l7Hmij6Iy3qdYps33wDuAoBMQ21ROvyw=",
         "ref": "refs/heads/refactor",
-        "rev": "d5e3345097cdda5c74bccddb27abb5b5c84eff5b",
-        "revCount": 257,
+        "rev": "8286c1028b2a69ee72680dc06d26bd80665ce02a",
+        "revCount": 262,
         "type": "git",
         "url": "https://git.lix.systems/lix-project/buildbot-nix.git"
       },
diff --git a/secrets.nix b/secrets.nix
index e4b3446..6bc7773 100644
--- a/secrets.nix
+++ b/secrets.nix
@@ -13,6 +13,15 @@ let
     loki-environment = [ machines.meta01 ];
     gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
 
+    buildbot-worker-password = [ machines.buildbot ];
+    buildbot-oauth-secret = [ machines.buildbot ];
+    buildbot-workers = [ machines.buildbot ];
+    # Private SSH key to Gerrit
+    # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
+    buildbot-service-key = [ machines.buildbot ];
+    # Signing key for Buildbot's specific cache
+    buildbot-signing-key = [ machines.buildbot ];
+
     # These are the same password, but nginx wants it in htpasswd format
     metrics-push-htpasswd = [ machines.meta01 ];
     metrics-push-password = builtins.attrValues machines;
diff --git a/secrets/buildbot-oauth-secret.age b/secrets/buildbot-oauth-secret.age
new file mode 100644
index 0000000..94e62b4
--- /dev/null
+++ b/secrets/buildbot-oauth-secret.age
@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 87T2Ig vfLpqc38U9RwGG1QmSSl5YTXcOU0eoTrpmBjVpP+9xE
+XbCUtuC9G9zSyVIgUmH0TO2sdH/3YjAf1erstVAUnHQ
+-> ssh-ed25519 K3b7BA zk89m8PXhx59Jf7ovoSvASaaOZqMQxiGMEB/ZF2iFFs
+pCfQv3PRw0IMjjXnjTxasVaAZVdfrRhmiRDVK3Pr2GI
+-> ssh-ed25519 +qVung ry8P1mOJwSHAXk9XaNGOLRLH2Q6QIxTueoBz+IcS/0M
+q9JsGjlS7HQqscAvOO2aSWlH3ruQC5ozDCkDBwp7g0o
+-> ssh-rsa krWCLQ
+DG2BpVdLziPUuo2HJfzDg/+aqugaOTfmVV+hEFjRV/B9pX90WnLCxp0lNpeNpTdU
+v889q7ojKs6jHuJGsUwUPy29Jn9PHOecE/gpcRTt6BI4/2JiwF2brLV+dVbWSOEv
+6lf9ecjmbJ/vbHnh94Aqa6kfBREazsZSYPGTAwNdcOdHRsoiK1PKCJmxPvZnfGuY
+o6144GTqTIGnxvbdlJ7XPzS8KEoP0SfPb2PFhfq6+z4JPdm116rhXIErPZNcQynP
+y0f/TRJPSu5QZ2YzZmwyBTpUqSQx1MWrY/5T3e0cCLY6d2E6evbnPb8eauJl3XHd
+I/kqqFKigixDBUPNlwW19Q
+-> ssh-ed25519 /vwQcQ Q1589zmSRC/Wvgi1TUfsr6itT7QvBpqsNteNmPhHtHs
+Gt3/5u8NW8dcJubLZuiBQjwPIfLNbFQNIAk5+MIoSo0
+-> ssh-ed25519 0R97PA j2DEcmdRz8hOGvkwn6r/6vqPTdNo2AtZKSAjBdQ2n1Y
++w7ky1+gP0O93DXeADjMdBu43Dxno1meh7idgjNdojg
+--- 2exgH3r1FIdc2mrQEC0XQmqO3r1bfKZdjWZttrilThE
+�]��,A`������`ㅊ'&�T�����}q1�\K���7�K�'K���`lx���F
i��#�
\ No newline at end of file
diff --git a/secrets/buildbot-service-key.age b/secrets/buildbot-service-key.age
new file mode 100644
index 0000000000000000000000000000000000000000..d4bad7ff2456b74dea2a7ffb54c58ce17555dd39
GIT binary patch
literal 1429
zcmZ9KYmCzb0LKp@Jcj0sfZ|DHMoqwRtlgt)IS<*ocIzJ9tL+|u+I3sob<cL)+IgOE
z5Jd%_yo^GONVsshQxYBt^5rxJ1OY)#E)oq#c-;{X0|JT(dSCd$_kaGse}4J1e6~GX
z;$*#26U(K=WtlhEK|}jeqoN4y*^I?*2N5IepezJ$ECpQIyxBwrqfUDk2uQhDIB({p
zSXtqMa2UjGx-CgK2nTQB4DNKU>0+4}S*hY`-BGJ}Ql%!R_<;;XP&P#m*ex-NPPhe&
z@*vG`O*>ppW1J-s&Z|=#1ZW{QE4xj&1@kBs4i<2&0q7QxV9AO*8f9RECE{<0njo>N
z>`roSw@T0+J0E3GsZ6PSg|Nalo1f6_7J++^W~wr3i8tawpA*+>;dD$;Xxe5^W@*5p
zH)#|o;g*1nREtosXaaP$K^Ou|NGfAu0Rc%>!Afy7Kv_*3;1pOWt4i{Oh@{sWPYQvE
zbe|?GtWlCDk`aRh!-yCFs|}`#X22jDFXx*n*vbf!Dphh7dl(f7B;=)%-gw1Tv*H+l
zC^-;@YYZLc5^3B}XHd2f&v2y>=@21RBAt9f4-tM`=OMcbEtFWKCODHCnr1Xa@?v#b
zNH|F@U2_C6x8#Qng*xSkhpm{}WSS%;7O;k0Cq#@aatb3i3r$#UREbIvX_!N?T*L|S
zCQXfrB$Be|b+c7t4Uu#zDJJXTbi<tDTtw6A&1R!0S@2;=bYfk}R-+Ubs7838R8P@q
zXHldbV73@G)tYiCY=sRj*ky~05ERX2pa~BO6cKR;8UW;!Vpd2nP=I_0%ZQ;O*+>>N
zC}$0Fnl)4A1SICNp*6Pw6=_cnF?*q^9WTpbnBz;HsPn$<`@c2U8xbyI^wrG3L>(9s
zA-$OQ#G;LOga8<Y<Xmo`0Z>>XUeP5?PV=<E=AlA>I~j|Y6Sx<4IIAF$Y06Yl4^~ya
zp$1}-Mh8rE4haSyqybUH5q28kfYl!syr{=*@#TwR+^;r#!3sr#L94w{DligEst{XM
zlss?n+7d}?qr!2r=9Do>K%s)>$w!f}PZomyq~;?7nxfU^m`)qo+uMx=uRl}dA|)+D
z#w05)AvSw33i=`{t)r+S+wzp(MgkU+PVt5#i6b4GwvNQy|LmGQb6;QY_&e7YcCJnj
zIcLp1{lTujO#iQ-eMWxr{3$bsF1-Hut&PNwwMT}J&%QG!cYwZd{fVVB=Z(hBr_Ft5
zx3;dhbnw&<8=yPOFRs}!Y1)VoqClm;3wH0EgYRxJBzvYVHgj3W*tfmsKI=NMlIp(t
zP%CuyQLt^>8|m8DYxaLPmCs(d{L>46x7ir6<CcEtt=sL}E}zRU8k_uue{FLu`R79m
zH*A@{x#g+cqAp?crE_d8Y<~|u4qYQ=eLnwU@0r<qjy`uM*!l}Lxbgc#-KT2zS{y^O
zOP%?Fft8zX6wb_dQPrCtc~*4{z4Y_-={?7uIoZ}(e7DDKI`4@-xp!dq!1@ukWjy`t
z;x`}LE^J-UbNEK$#4(E9Iq&)Y^{4yRjmA5-zH)c!zphvJ7bpF(h1syE^=T>h<;$;0
z9e@3{#CwJNc+7QsopA7z;mWAja(Mh|wK8MX-PIVL+B@2N=HA2o@56frSH!+qKYXAo
z><E4M-=)mkW^AD1Jmr~tc|mj6oF)4v4PC(|ue>>ObX|APjBfG8du`g3mht6B_fH!f
Y{%Wij*uLY_-%pBo%kII@GId$uf9~oQ0{{R3

literal 0
HcmV?d00001

diff --git a/secrets/buildbot-signing-key.age b/secrets/buildbot-signing-key.age
new file mode 100644
index 0000000000000000000000000000000000000000..2028dbfc54f37ea425a00c1cbfdf58a4d7cf398b
GIT binary patch
literal 1133
zcmZ9~NvPul0LF1eQHVOYpbUdTPl9G@nlxz|MWxxAv`y0{X*NY@(llF}q-l1Tn+GqC
zaZqtVamIrph{_Cj5CjELTn6+c3c@=$GN1=}g9kn7<z3$WgCBhSm#))xB0H^<%#WkT
z;?#u-7-&D$+V8!#vqCToCR%-5L3p?I|B49QkerW}i{0LeENs)6QNgSL(PWziG&w<B
zg>bEoF922IR`s5rHT_JJOn*Q(!w{(;u9vg@E~RCJD8t=}vz~T^liIjz;pCuYLzppF
zl@ZV9mQh3VWI?i&sMP$(<v?oCsUX2yA+)+IOThJ;z$<%+O<{#aAWUxT_r9eUJVTmL
z>dR8P=@?y9%<9fb4cw`D<hDd9s0=*vOivI8u<7{XV(&>JgU2|IDyi*{wAilQo)N}C
z3h#x2OFfEOF<CW@Tx`RsEKhU{Vrb|^ILK6)lx4ePhsRpFU$(+jqb75J5$biV=vtSl
z=cz`<RI#fvvk8rm)A3}*%m>+6FX<USf%#tFApImlc9?4T)N04|nkm3>C7mi`WUL5P
z0O1uXXj>|TkvK%Pj!6_*&Ma9rC)of=d1!4Z8%-)P$|9%LcDK{E09mjY#P1|DwO6D!
z!g4E8SGiB1h0;f8*>%iVQ$k#oBQWqOg+dmAP1Vs_Px8V-5*<L-eaB;Tp-W|Q!mvi<
za`S}L*bbu!7PYQvf~l4hln4rYslz*C+ch!5!(??OGAY010f^xm7LQhZMuK6fTOl~d
zS%^c?Ndo6?#xv6><5HANvXv!}aaxQpxl5a}H=9*p+$2Cg6e)bCDQRzN75*+YJYyi2
z$8R5Z3l-(up10<tJ}_c`WM*A1Q8sw($*>(YbSY^r=IESjIoxt8**!`CAQs$I!A$hH
zAYXG8jf_CKf#5aRW6g4;`irqW$qR|6=k7_ZBN6y)&=MOp9Pd-lT`d|^Gj*mxXdFyr
zESdTpdm$<^jr)RDuf09cEC`vD#9U+{#n%`q#1Ui7+JlKD>r1VpaLcfC;6N;#7-+ZK
zt*%{8R(Va+L#Ez<&2F%n1=E3QGb5`s5R&LG9Ew1-@i5o}(sxgM@bbGi-mkbTUtd4`
z;iqSxx%<ohd*C~-|9kBlwSDn{Gj~4xBL2|xpC6#7yTAQ$>g}UTuibm}=U4G%>6Hs0
zT_Xq6^TV_M2#cqyi`U<{{gYqL9G!Zkx$xv|XU{!HUA^Zc<mQ9(f4%VN-w!(9-Z9~R
uj6apHT)Y3{H&5U7;p49!His{Le+z7W@#_ud*57aX`LlEK)q_j-z4aewf`-cg

literal 0
HcmV?d00001

diff --git a/secrets/buildbot-worker-password.age b/secrets/buildbot-worker-password.age
new file mode 100644
index 0000000..6422c56
--- /dev/null
+++ b/secrets/buildbot-worker-password.age
@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 87T2Ig y4P08L2yYSjVcWdbRCqWSCM+WcgqXpxOwr1Ip2Ipd3Q
+7C/3MXVbAX0HIdEULKu0bc9q2U+4mPDiDb2l5rRwBI4
+-> ssh-ed25519 K3b7BA wl46ZMqLHMOTG3RojLVgwC2hskjUJWUGZ4h9dwBYaws
+xxrJQ8Ws1evKgfKej8WwbucuArULWNtCdMlSDdVNe6E
+-> ssh-ed25519 +qVung 4fix0OAAyW/34W1HVfc5ivIr8ijqNz0Vz8oWaSY2lyk
+8ZAguZR31I0hysn265ELYeYwrLiDx07BepG0w1R8uhU
+-> ssh-rsa krWCLQ
+vRU5uF64cQZwJrGr0oBRBJFo2mr30pz6yhXwEm4BJjKt/yCCikggPUFTW/KOjnqZ
+JcUoLpeDVIk3+FBJl4p3PVRn1pjRUve4vEcNAEjmkVgBwiZWtpfE6vVLn5pIvm+A
+nwybTTwMJomDTLDsMOq0Ur+S3rw4Nb6ADqDKhmjlmlaSlTqxUmZoznQduoSSINI/
+VJw/+VjwFxsMxdD5swxEAcrDk2rKoQLrfO83PO3HNMX5SmYHHYEaWB0/YeLgvi8a
+4OBueRKLWOiy2WUCqtxiQG5XYGYNdgOKIeNLnPNH6RRwFoBz7Zmn2uuQjmysY9h8
+lryoR6quxdOTRTL2WwGPAw
+-> ssh-ed25519 /vwQcQ 8sOHrthroDrjuL14hij7sPiK9BGlOLzKG1pBe5+HMFw
+vQqm96T/H5tINHJxnfi6DYm9YO9UAaj8etmk7K0GJ7U
+-> ssh-ed25519 0R97PA Dd3db0zh0/ZUsm3UgsWRbGz9mVvm8s3W2HQkjTM6L3k
+/+IRsPs2KoqEYnxmFoKmNc/00jOesKXv33rO4Yx+l68
+--- jPrqv7h6AGoqNl1LCOtzXvU4dKK2PnGsj/FqhstbSGw
+��f+`ϙ+�]&�w=��:�$UQ�7���h��K��U���1_Y�z��0�%\�N�L�0o��
�޼��5~��_
�Z��7x��[�\���v[�o
\ No newline at end of file
diff --git a/secrets/buildbot-workers.age b/secrets/buildbot-workers.age
new file mode 100644
index 0000000..dcd9c20
--- /dev/null
+++ b/secrets/buildbot-workers.age
@@ -0,0 +1,22 @@
+age-encryption.org/v1
+-> ssh-ed25519 87T2Ig arwhM8DLVpft4PdPw4A6ZoPk5KqXORhE9iDG6etDOzk
+ZVNgF/J3YiCTj2lq2280vU95pX36cpH+sT/wRjmExHk
+-> ssh-ed25519 K3b7BA fBr1rUtTQVs0LLSR6RVX1eJBEpYs3COyJITpGm4ngi0
+jfYyrD/0gh1QCAq8SnsWjUQin3g21NEgCQAlCc6uQ9g
+-> ssh-ed25519 +qVung cJEfk9HdCsdVmuhI7OAgWsly4P5o/n9JbPRtsDZ2FVY
+MJvfsbd9+pbhG1BwF4xVafqu+LvPy3geN7n9MALFP68
+-> ssh-rsa krWCLQ
+PuiiAwETSr4SDb4XOtn6AECDJedzd3KfTAsjrq3giwCrjfSqYeTpBaH8mhf4t5D5
+fAXHtIoChcZNb1dhxQtP0r4A4cy1faf87XGkOwAeikFv9S8cMjjgZ71sX8g8Srp/
+Mjla0+5CVGRsUMcev/t9uMj04qHDtr7swbjLoOPwvCQBUWHZrOA/Fq/T2g9qU32g
+YQgxtR3zzseb/vOFHzpWc6fkR8UO0j1H1hyFkJ1XkipeQ5UIwg0g57lsPkNXuZfI
+BbKzzg521HChK5ssibITLdtp6piwIpxHUxwSNpLXG8vbT33e24kFEeTZ0QX4NStl
+r6U4j3NL1lPChpdSIhy/2Q
+-> ssh-ed25519 /vwQcQ Q8Hxbxto0EN1odEFt/dNfeK1l4xSIO9lY/ewYpa1DgY
+4jeNmuwK4tvJzX62/x/1aq+L4R6dD61akUmo0+GCICc
+-> ssh-ed25519 0R97PA of4aEATYi3ad7nYvexirIErAWbsLOW1ijGPc/IETSCU
+qT/O8DIYaMm0MlvS9eVBSe2th16yDHODlT1VgF9iLDI
+--- rWScSs0yVovPOWI2zmDTIyLJdBIRlKIPu6jivzty7p8
+����d�}Emi�KC�y5�L`G���T�Z^Q?g2�|��S
+g2���F`��_��j�l
��1�G��f��W��ƃ0����� ���H�)�
+{����\�í<^�#J�g��JJ���G�J�h>�2����G%���Tra�B
\ No newline at end of file
diff --git a/services/buildbot/default.nix b/services/buildbot/default.nix
new file mode 100644
index 0000000..970b288
--- /dev/null
+++ b/services/buildbot/default.nix
@@ -0,0 +1,103 @@
+{
+  nodes,
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.bagel.services.buildbot;
+  cfgGerrit = nodes.gerrit01.config.bagel.services.gerrit;
+  inherit (lib) mkEnableOption mkOption mkIf types;
+in
+{
+  options.bagel.services.buildbot = {
+    enable = mkEnableOption "Buildbot";
+    domain = mkOption {
+      type = types.str;
+    };
+  };
+
+  config = mkIf cfg.enable {
+    networking.firewall.allowedTCPPorts = [ 80 443 ];
+    age.secrets.buildbot-worker-password.file = ../../secrets/buildbot-worker-password.age;
+    age.secrets.buildbot-oauth-secret.file = ../../secrets/buildbot-oauth-secret.age;
+    age.secrets.buildbot-workers.file = ../../secrets/buildbot-workers.age;
+    age.secrets.buildbot-service-key.file = ../../secrets/buildbot-service-key.age;
+    age.secrets.buildbot-signing-key.file = ../../secrets/buildbot-signing-key.age;
+
+    services.nginx.virtualHosts.${cfg.domain} = {
+      forceSSL = true;
+      enableACME = true;
+    };
+
+    services.buildbot-nix.worker = {
+      enable = true;
+      workerPasswordFile = config.age.secrets.buildbot-worker-password.path;
+      # All credits to eldritch horrors for this beauty.
+      workerArchitectures =
+      {
+        # nix-eval-jobs runs under a lock, error reports do not (but are cheap)
+        other = 8;
+      } // (
+        lib.filterAttrs
+          (n: v: lib.elem n config.services.buildbot-nix.coordinator.buildSystems)
+          (lib.zipAttrsWith
+            (_: lib.foldl' lib.add 0)
+            (lib.concatMap
+              (m: map (s: { ${s} = m.maxJobs; }) m.systems)
+              config.nix.buildMachines))
+        );
+    };
+
+    services.buildbot-nix.coordinator = {
+      enable = true;
+
+      inherit (cfg) domain;
+
+      oauth2 = {
+        name = "Lix";
+        clientId = "forkos-buildbot";
+        clientSecretFile = config.age.secrets.buildbot-oauth-secret.path;
+        resourceEndpoint = "https://identity.lix.systems";
+        authUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/auth";
+        tokenUri = "https://identity.lix.systems/realms/lix-project/protocol/openid-connect/token";
+      };
+
+      workersFile = config.age.secrets.buildbot-workers.path;
+
+      allowedOrigins = [
+        "*.forkos.org"
+      ];
+
+      buildSystems = [
+        "x86_64-linux"
+      ];
+
+      gerrit = {
+        domain = cfgGerrit.canonicalDomain;
+        # Manually managed account…
+        # TODO: https://git.lix.systems/the-distro/infra/issues/69
+        username = "buildbot";
+        port = cfgGerrit.port;
+        privateKeyFile = config.age.secrets.buildbot-service-key.path;
+        projects = [
+          "buildbot-test"
+          "nixpkgs"
+          "infra"
+        ];
+      };
+
+      evalWorkerCount = 6;
+      evalMaxMemorySize = "4096";
+
+      signingKeyFile = config.age.secrets.buildbot-signing-key.path;
+    };
+
+    nix.settings.keep-derivations = true;
+    nix.gc = {
+      automatic = true;
+      dates = "hourly";
+    };
+  };
+}
diff --git a/services/default.nix b/services/default.nix
index 0599eb5..27dacfd 100644
--- a/services/default.nix
+++ b/services/default.nix
@@ -8,5 +8,6 @@
     ./postgres
     ./forgejo
     ./baremetal-builder
+    ./buildbot
   ];
 }

From c3394264ba464faf24eeb7ac220cf98917c77154 Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Wed, 17 Jul 2024 14:47:52 +0200
Subject: [PATCH 4/8] hosts/buildbot: init

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
---
 common/ssh-keys.nix        |  1 +
 flake.nix                  |  1 +
 hosts/buildbot/default.nix | 37 +++++++++++++++++++++++++++++++++++++
 3 files changed, 39 insertions(+)
 create mode 100755 hosts/buildbot/default.nix

diff --git a/common/ssh-keys.nix b/common/ssh-keys.nix
index eb1afb8..7d06233 100644
--- a/common/ssh-keys.nix
+++ b/common/ssh-keys.nix
@@ -4,6 +4,7 @@
     meta01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM5t9gYorOWgpCFDJgb24pyCKIabGpeI2H/UfdvXODcT";
     gerrit01 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA+eSZu+u9sCynrMlsmFzQHLIELQAuVg0Cs1pBvwb4+A";
     fodwatch = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFRyTNfvKl5FcSyzGzw+h+bNFNOxdhvI67WdUZ2iIJ1L";
+    buildbot = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJgIu6ouagYqBeMLfmn1CbaDJMuZcPH9bnUhkht8GfuB";
     git = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEQJcpkCUOx8+5oukMX6lxrYcIX8FyHu8Mc/3+ieKMUn";
     builder-0 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBHSNcDGctvlG6BHcJuYIzW9WsBJsts2vpwSketsbXoL";
     builder-1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIQOGUjERK7Mx8UPM/rbOdMqVyn1sbWqYOG6CbOzH2wm";
diff --git a/flake.nix b/flake.nix
index c8eef8f..a5a662a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -106,6 +106,7 @@
       fodwatch.imports = commonModules ++ [ ./hosts/fodwatch ];
       git.imports = commonModules ++ [ ./hosts/git ];
       wob-vpn-gw.imports = commonModules ++ [ ./hosts/wob-vpn-gw ];
+      buildbot.imports = commonModules ++ [ ./hosts/buildbot ];
     } // builders;
 
     hydraJobs = builtins.mapAttrs (n: v: v.config.system.build.toplevel) self.nixosConfigurations;
diff --git a/hosts/buildbot/default.nix b/hosts/buildbot/default.nix
new file mode 100755
index 0000000..a06689e
--- /dev/null
+++ b/hosts/buildbot/default.nix
@@ -0,0 +1,37 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  networking.hostName = "buildbot";
+  # TODO: make it the default
+  networking.domain = "infra.forkos.org";
+
+  time.timeZone = "Europe/Paris";
+
+  bagel.sysadmin.enable = true;
+  # Buildbot is proxied.
+  bagel.raito.v6-proxy-awareness.enable = true;
+  bagel.hardware.raito-vm = {
+    enable = true;
+    networking = {
+      nat-lan-mac = "BC:24:11:E7:42:8B";
+      wan = {
+        address = "2001:bc8:38ee:100:1000::50/64";
+        mac = "BC:24:11:C9:BA:6C";
+      };
+    };
+  };
+
+  bagel.services.buildbot = {
+    enable = true;
+    domain = "buildbot.forkos.org";
+  };
+
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  system.stateVersion = "24.05";
+  deployment.targetHost = "buildbot.infra.forkos.org";
+}

From a56426e6c9379c20566f8f0abe4d3b285aea65e3 Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Wed, 17 Jul 2024 16:41:32 +0200
Subject: [PATCH 5/8] secrets: rekey for new machine (buildbot)

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
---
 secrets/buildbot-oauth-secret.age          |  36 +++----
 secrets/buildbot-service-key.age           | Bin 1429 -> 1429 bytes
 secrets/buildbot-signing-key.age           | Bin 1133 -> 1133 bytes
 secrets/buildbot-worker-password.age       |  36 +++----
 secrets/buildbot-workers.age               | Bin 1134 -> 1134 bytes
 secrets/gerrit-prometheus-bearer-token.age |  40 ++++----
 secrets/grafana-oauth-secret.age           |  36 +++----
 secrets/hydra-s3-credentials.age           | Bin 1150 -> 1150 bytes
 secrets/hydra-signing-priv.age             | Bin 1124 -> 1124 bytes
 secrets/hydra-ssh-key-priv.age             | Bin 1429 -> 1429 bytes
 secrets/loki-environment.age               |  37 ++++----
 secrets/metrics-push-htpasswd.age          |  37 ++++----
 secrets/metrics-push-password.age          | 105 +++++++++++----------
 secrets/mimir-environment.age              | Bin 1127 -> 1127 bytes
 secrets/netbox-environment.age             | Bin 1226 -> 1226 bytes
 15 files changed, 167 insertions(+), 160 deletions(-)

diff --git a/secrets/buildbot-oauth-secret.age b/secrets/buildbot-oauth-secret.age
index 94e62b4..f9525da 100644
--- a/secrets/buildbot-oauth-secret.age
+++ b/secrets/buildbot-oauth-secret.age
@@ -1,20 +1,20 @@
 age-encryption.org/v1
--> ssh-ed25519 87T2Ig vfLpqc38U9RwGG1QmSSl5YTXcOU0eoTrpmBjVpP+9xE
-XbCUtuC9G9zSyVIgUmH0TO2sdH/3YjAf1erstVAUnHQ
--> ssh-ed25519 K3b7BA zk89m8PXhx59Jf7ovoSvASaaOZqMQxiGMEB/ZF2iFFs
-pCfQv3PRw0IMjjXnjTxasVaAZVdfrRhmiRDVK3Pr2GI
--> ssh-ed25519 +qVung ry8P1mOJwSHAXk9XaNGOLRLH2Q6QIxTueoBz+IcS/0M
-q9JsGjlS7HQqscAvOO2aSWlH3ruQC5ozDCkDBwp7g0o
+-> ssh-ed25519 87T2Ig g15A5EWi9IhaxPFS6SD6YYm/aFnC0Dum7zK8/ZUtW0s
+791D6C8mAy2dhDAlqRQ+q41FlQTJX2WfZQPjuwetP2A
+-> ssh-ed25519 K3b7BA cJY9qIFVmucmMJLTFffkRCNYeudZl+8Yrm5SkxQ4eSI
+97nXyKffZGoGJ6252UKUEJHiFgdk8XUkAAkXy2PLepM
+-> ssh-ed25519 +qVung HMBSUjfmaFLVx64epj0djkqNMe3CdKN1fxAVuu+Dtmg
+AxT62n2p/pP9WZmmuHClSKKgXhr4FjEQpEs0HfdNGfw
 -> ssh-rsa krWCLQ
-DG2BpVdLziPUuo2HJfzDg/+aqugaOTfmVV+hEFjRV/B9pX90WnLCxp0lNpeNpTdU
-v889q7ojKs6jHuJGsUwUPy29Jn9PHOecE/gpcRTt6BI4/2JiwF2brLV+dVbWSOEv
-6lf9ecjmbJ/vbHnh94Aqa6kfBREazsZSYPGTAwNdcOdHRsoiK1PKCJmxPvZnfGuY
-o6144GTqTIGnxvbdlJ7XPzS8KEoP0SfPb2PFhfq6+z4JPdm116rhXIErPZNcQynP
-y0f/TRJPSu5QZ2YzZmwyBTpUqSQx1MWrY/5T3e0cCLY6d2E6evbnPb8eauJl3XHd
-I/kqqFKigixDBUPNlwW19Q
--> ssh-ed25519 /vwQcQ Q1589zmSRC/Wvgi1TUfsr6itT7QvBpqsNteNmPhHtHs
-Gt3/5u8NW8dcJubLZuiBQjwPIfLNbFQNIAk5+MIoSo0
--> ssh-ed25519 0R97PA j2DEcmdRz8hOGvkwn6r/6vqPTdNo2AtZKSAjBdQ2n1Y
-+w7ky1+gP0O93DXeADjMdBu43Dxno1meh7idgjNdojg
---- 2exgH3r1FIdc2mrQEC0XQmqO3r1bfKZdjWZttrilThE
-�]��,A`������`ㅊ'&�T�����}q1�\K���7�K�'K���`lx���F
i��#�
\ No newline at end of file
+N0Duz2bONcCUZ76QhPsCJ4BHHWqzFdZLqFdl+6GeW+tgIp2Nb4la8eNfgzYGSwTy
+53bRePNMIBTkChXFYt/4fUdqaiiVYg25swMeVLQBJnjJkcAks0Gf44FXLIaoPr1M
+56rtixpSX31WDKwHbUF/40G6Xut8KNlI8BdwiOl9ibgnuEf4mYQbwFbRQbLMK5IK
+Rf/7SEmAqqfY/HG1RqqgCs4kEpvFTKqEEDpgjOoyS2tyKN2351jya91YzotLja4I
+sLoMg/G3UNtxfdaCgK7TP4IxV9blkVMDPAbyR622VbS0sEa7uJGzb86jDDsZXaKX
+9iWK9n4hMKZDv9gBbhTIWg
+-> ssh-ed25519 /vwQcQ hMkCrUcLGxdZMYgi1D1Kr5qUdGNfza2UTvRJKiHObgM
+7Lz70zSMPk/tsU1CZGOk/BPA7NSSnSJgFbG5TjyOXvA
+-> ssh-ed25519 0R97PA OQjDTknVmrYVclcqlT31YjZx+3a/0GxfjuVQFmPJ7UQ
+KMGTMfO/mO5EAYacyz1hmHnQgzunRqkDeglhbGVNWe4
+--- ScDZvSiVSjNXm8TSoLSAM+KpcFORnCXiemYbCBcz2jQ
+���h�}E�ʜ��U��ᢌ�����k��g[��C�"��N�j�u5���CX���Gt�TOm�
\ No newline at end of file
diff --git a/secrets/buildbot-service-key.age b/secrets/buildbot-service-key.age
index d4bad7ff2456b74dea2a7ffb54c58ce17555dd39..bcdb7ceb09d9130c01c1da1aaa48be6b126eb789 100644
GIT binary patch
literal 1429
zcmZ9~`;XHE0KjobJQJ26Mq@+{k|jtal%ws|Jt}b6J-SEt>e{u}flAk{?Vj!0uIqLX
zQ6Zd<7zl!1z+gb(z<@vi14u+bk|RfgfgA`yMLc;K<f%MF{E1)q3qHv=UtTJ76>5Uk
zQjMZgo1thDXu`IxZo96_U4^{E<$_VW*F!rZlHI2enxmyKhFk?jfz%PdsCvwB+y`bD
z)8j8^G)l$7IaqZFws^Ja7khBtWt3#5Z6#oD+#PY_p-P)e5DW{)EA?O~&K1jC<VlTi
z<~%;kZk2=rNt-<q>ni43a5F3>QMW0^n_!-$<p#{7En1TbY)dZNz(~c+f^lB7%6S26
zA++Gb48&quKA8v;x(HQqJYr^}uAbnN8c=5%HOUS#4PS(G1Ttl(Qcjb&=E@)uDke7#
zy(dh2@=79{g*XXIvVyH1(4w^<7f~pJf*g1o=V_v(q75~bLLtQzwFW9uTAXBp<YP_K
zd3#x7{V~!;p=!G3W9y;1;%D<t8boT%n(Rz+mJz|cH8Up&8t;(Vs*7kLDKiXakxU41
zyWO<Z3OJ)tTZ!|Q4J%8>0+?u~<0egD^@_(Ki6WCmc~OWZd@3)uAib0eQfRuCM?z>z
zD{^^f3yfJ_&DO;I(NrN}$Oz&pvTYU#L9S{cD#RMD3~s^UHjfap*MYeiG@FMEpK3Mo
z$wV<5Hk@S5#PT*u4A5=4Ni?yB(u8R+C$UkT<@u(GS_Y93!DhSy*0E;OovM3FnuWs^
zw^a0!kx~MRHUN)JKxhDGszEP|HxLpEm~y0^2hEmW(vfmB2q-;iTxVSATnd3r&fNfn
zcq&FJp;Vqzl>`Uc5JoLRXvOe~33o;>O9pFJ0qk+@$KQfxlN3mM8e`Ff%Todbwb)G6
z7)~N{V3Hv~Hzm7-SPm(b5e4w6lvA<+Mj&ei0izhO{IplaXh6!sl)t4^H5J4fd@KS1
z9)%|&5ci}8rBP1`v-hC5F468-u9bu0^#m>wn4(wW%_5koLQLJkM=UF)i>wzHY=np5
z*>;HmNSP8b#S3{eJ$btAN>Y>@;4M$EOrT^u5<-%at*fict}}u_l|e8@sziw*0|i*|
zNpwYt1GP5hM*?{v7c<phm9LXF?)HDcJ>Th{6MsWMySs0%lYFpTo@eZoxLqd(ysz(J
zw1Rx(cK7^E9}cWs(f|Is^VgO?c=y1z&eiFcI?i>N+wo(_=qq!2_p1ZrreAn&LSI+u
zZvUEbzA-Bfr2g4As=G9H#^OJX%}ZkU*G~QSSm^3+iEE`^Vdhe*G;5Rf4JO=KwDjoF
z^uisJ=ky=$OZV>m;@QN}oA_*L&&fFh7hl*<4y?RzWYOuvpML&*--b(<OZ!G4-PgHW
z_g;O}v8b!Ovwz9(didU8>c>^n4iYnGE$JH>ym9J=04>`!(s}7__OkvItZ(g|q6JQU
zedWZKceYMGGUw3o&K(`kCpOL+E*Jwdmfu;H-M{j#ZGPvP;`K?vrF%CXhsPh9i|+2d
z{PxZBL?s3}*LFVIy!MS#uO0iu-0K+H-f{NCp`nBK`qzz&HPc@%nEJ=?nQ<?UY`wbS
z;#c*ba?@UJj4q$1o__Pp{cQ`-Uz3HcT5@Rh#3{)2?SnHWOg=n5KmPag$SUT$lMnye
zJ^1j&c^}<*=G*BL5A2$}a47rn!!eVtyrSJkm!^Li{(zgHRp)MSY<h2T-)B9C`wvc^
WwEyguZOhg}`ng-9-}>R~?8g66%_mp@

literal 1429
zcmZ9KYmCzb0LKp@Jcj0sfZ|DHMoqwRtlgt)IS<*ocIzJ9tL+|u+I3sob<cL)+IgOE
z5Jd%_yo^GONVsshQxYBt^5rxJ1OY)#E)oq#c-;{X0|JT(dSCd$_kaGse}4J1e6~GX
z;$*#26U(K=WtlhEK|}jeqoN4y*^I?*2N5IepezJ$ECpQIyxBwrqfUDk2uQhDIB({p
zSXtqMa2UjGx-CgK2nTQB4DNKU>0+4}S*hY`-BGJ}Ql%!R_<;;XP&P#m*ex-NPPhe&
z@*vG`O*>ppW1J-s&Z|=#1ZW{QE4xj&1@kBs4i<2&0q7QxV9AO*8f9RECE{<0njo>N
z>`roSw@T0+J0E3GsZ6PSg|Nalo1f6_7J++^W~wr3i8tawpA*+>;dD$;Xxe5^W@*5p
zH)#|o;g*1nREtosXaaP$K^Ou|NGfAu0Rc%>!Afy7Kv_*3;1pOWt4i{Oh@{sWPYQvE
zbe|?GtWlCDk`aRh!-yCFs|}`#X22jDFXx*n*vbf!Dphh7dl(f7B;=)%-gw1Tv*H+l
zC^-;@YYZLc5^3B}XHd2f&v2y>=@21RBAt9f4-tM`=OMcbEtFWKCODHCnr1Xa@?v#b
zNH|F@U2_C6x8#Qng*xSkhpm{}WSS%;7O;k0Cq#@aatb3i3r$#UREbIvX_!N?T*L|S
zCQXfrB$Be|b+c7t4Uu#zDJJXTbi<tDTtw6A&1R!0S@2;=bYfk}R-+Ubs7838R8P@q
zXHldbV73@G)tYiCY=sRj*ky~05ERX2pa~BO6cKR;8UW;!Vpd2nP=I_0%ZQ;O*+>>N
zC}$0Fnl)4A1SICNp*6Pw6=_cnF?*q^9WTpbnBz;HsPn$<`@c2U8xbyI^wrG3L>(9s
zA-$OQ#G;LOga8<Y<Xmo`0Z>>XUeP5?PV=<E=AlA>I~j|Y6Sx<4IIAF$Y06Yl4^~ya
zp$1}-Mh8rE4haSyqybUH5q28kfYl!syr{=*@#TwR+^;r#!3sr#L94w{DligEst{XM
zlss?n+7d}?qr!2r=9Do>K%s)>$w!f}PZomyq~;?7nxfU^m`)qo+uMx=uRl}dA|)+D
z#w05)AvSw33i=`{t)r+S+wzp(MgkU+PVt5#i6b4GwvNQy|LmGQb6;QY_&e7YcCJnj
zIcLp1{lTujO#iQ-eMWxr{3$bsF1-Hut&PNwwMT}J&%QG!cYwZd{fVVB=Z(hBr_Ft5
zx3;dhbnw&<8=yPOFRs}!Y1)VoqClm;3wH0EgYRxJBzvYVHgj3W*tfmsKI=NMlIp(t
zP%CuyQLt^>8|m8DYxaLPmCs(d{L>46x7ir6<CcEtt=sL}E}zRU8k_uue{FLu`R79m
zH*A@{x#g+cqAp?crE_d8Y<~|u4qYQ=eLnwU@0r<qjy`uM*!l}Lxbgc#-KT2zS{y^O
zOP%?Fft8zX6wb_dQPrCtc~*4{z4Y_-={?7uIoZ}(e7DDKI`4@-xp!dq!1@ukWjy`t
z;x`}LE^J-UbNEK$#4(E9Iq&)Y^{4yRjmA5-zH)c!zphvJ7bpF(h1syE^=T>h<;$;0
z9e@3{#CwJNc+7QsopA7z;mWAja(Mh|wK8MX-PIVL+B@2N=HA2o@56frSH!+qKYXAo
z><E4M-=)mkW^AD1Jmr~tc|mj6oF)4v4PC(|ue>>ObX|APjBfG8du`g3mht6B_fH!f
Y{%Wij*uLY_-%pBo%kII@GId$uf9~oQ0{{R3

diff --git a/secrets/buildbot-signing-key.age b/secrets/buildbot-signing-key.age
index 2028dbfc54f37ea425a00c1cbfdf58a4d7cf398b..143eb53f7a17af5f94ec7ead9f64f4f63abd6420 100644
GIT binary patch
literal 1133
zcmZXT&5Pr70ERuR9x?|-LGiM5`~b1rNt*8<is>}pZPHAWrU{5NZTf4PHc9iHu%1N3
zgR;Y_huxzf2qOsV!h+%=-n_0}MD#M~VNn)bkW~=p<YD{^p67+<odK`oh4ZA2Gd~Is
zq6EN27wWvyO4C)xTc8BdWm+_4pb}`InY1A?hcS5G33wb<<}nXbi>eway>e7DW0o|K
zph#@rgzQD1M~>t!y%j9k3wd>=_Xkc%=BQo9IKyV4w)J7{NZx?+GzD|19@~m)S+c{8
zOzYiQvppnYI$G6?SQ%qc)R3mEXwq1*3r^-zXoD`3V2kWT#49&d+fwaZ95Y)Hn@I+u
zqgs$y1-!)jdBB~gwRe*|1g(@{oB``=eBXD~Y+yCJHZ`rxjW&_p!yN80C0`W{%E(gy
z1zHGD-iNzU!1{JcdUJE7$gy6_JING^wN#0mV1h8${-Y)7taT=FIM5-#;M|z7+S%EH
zBGHPeRBOgg9gYTgpn^c-42vyvPcQuyJM@&om0_|@MSRV<e4vdjh)WWO&&d_BE?IOw
zEUc!t2t0S$k2efG>mJ5|BiFDIOxkmH3aEIo%TT|WFtXhQKx{<F*=A$4l36ts5EmaH
zh_oXHOC>~OCp0IuT`gQ%!H_`6V=D@09!274IL1j0ZE)3*I4Gr2EFTsyoFpN};4J7y
zeYqL0#nmn}u*4qHM^4n}1Y04b!UbbO=_ebtVaW&vT-_c+sKkq~&S~Sp!p2F`k!Z_S
z6R8cRy<IoVh!w+;?23|d(i{X~Y8%4D?uk<tQG~8)6fRU3!B*%|sEfd}bW`|^r5M8h
zN`i}0pX;q)HJfR<uVQ1BQ<Ap8498_Xug^MVM$)wEM&1U-LenO;EM!gr?^_cg1W`=W
z7`izG!yWK)YO*jq!w|uUtG0~>ud(!eNyrgHjo6mUav)NSkw-Zot@K2V0FrQhYq-tH
z2%nAIY#PUL9i3eY;spmp05kEXCd>ju%AU*z@suJRS>pZ7(R!guC`Qt(bg0wmw2GOU
z@p+|8NRg-cU|PfZGHf#fkz!7lTH_!A+XS=PvB5$oe|`P!!SlZz|7_}?T&B|Bhy1H|
zAM9WK;ESslxj);-w@+{FU7|j|^~^(GUHJalA7A|Ywtf0q{`4Q$80z8X{^af>Z&#0B
zyFS9Nzcty{-gIt0c5+beJ-WF4p>gH%OFt!hpZ&9Vt~!3leNTD)<>>uo{`tL6lW&fG
lK|Z?q2K>~6d+*-A^Mfe8@WgkQPWGR?a^uE@(=RXn`Y%yyg%<z-

literal 1133
zcmZ9~NvPul0LF1eQHVOYpbUdTPl9G@nlxz|MWxxAv`y0{X*NY@(llF}q-l1Tn+GqC
zaZqtVamIrph{_Cj5CjELTn6+c3c@=$GN1=}g9kn7<z3$WgCBhSm#))xB0H^<%#WkT
z;?#u-7-&D$+V8!#vqCToCR%-5L3p?I|B49QkerW}i{0LeENs)6QNgSL(PWziG&w<B
zg>bEoF922IR`s5rHT_JJOn*Q(!w{(;u9vg@E~RCJD8t=}vz~T^liIjz;pCuYLzppF
zl@ZV9mQh3VWI?i&sMP$(<v?oCsUX2yA+)+IOThJ;z$<%+O<{#aAWUxT_r9eUJVTmL
z>dR8P=@?y9%<9fb4cw`D<hDd9s0=*vOivI8u<7{XV(&>JgU2|IDyi*{wAilQo)N}C
z3h#x2OFfEOF<CW@Tx`RsEKhU{Vrb|^ILK6)lx4ePhsRpFU$(+jqb75J5$biV=vtSl
z=cz`<RI#fvvk8rm)A3}*%m>+6FX<USf%#tFApImlc9?4T)N04|nkm3>C7mi`WUL5P
z0O1uXXj>|TkvK%Pj!6_*&Ma9rC)of=d1!4Z8%-)P$|9%LcDK{E09mjY#P1|DwO6D!
z!g4E8SGiB1h0;f8*>%iVQ$k#oBQWqOg+dmAP1Vs_Px8V-5*<L-eaB;Tp-W|Q!mvi<
za`S}L*bbu!7PYQvf~l4hln4rYslz*C+ch!5!(??OGAY010f^xm7LQhZMuK6fTOl~d
zS%^c?Ndo6?#xv6><5HANvXv!}aaxQpxl5a}H=9*p+$2Cg6e)bCDQRzN75*+YJYyi2
z$8R5Z3l-(up10<tJ}_c`WM*A1Q8sw($*>(YbSY^r=IESjIoxt8**!`CAQs$I!A$hH
zAYXG8jf_CKf#5aRW6g4;`irqW$qR|6=k7_ZBN6y)&=MOp9Pd-lT`d|^Gj*mxXdFyr
zESdTpdm$<^jr)RDuf09cEC`vD#9U+{#n%`q#1Ui7+JlKD>r1VpaLcfC;6N;#7-+ZK
zt*%{8R(Va+L#Ez<&2F%n1=E3QGb5`s5R&LG9Ew1-@i5o}(sxgM@bbGi-mkbTUtd4`
z;iqSxx%<ohd*C~-|9kBlwSDn{Gj~4xBL2|xpC6#7yTAQ$>g}UTuibm}=U4G%>6Hs0
zT_Xq6^TV_M2#cqyi`U<{{gYqL9G!Zkx$xv|XU{!HUA^Zc<mQ9(f4%VN-w!(9-Z9~R
uj6apHT)Y3{H&5U7;p49!His{Le+z7W@#_ud*57aX`LlEK)q_j-z4aewf`-cg

diff --git a/secrets/buildbot-worker-password.age b/secrets/buildbot-worker-password.age
index 6422c56..382ac61 100644
--- a/secrets/buildbot-worker-password.age
+++ b/secrets/buildbot-worker-password.age
@@ -1,20 +1,20 @@
 age-encryption.org/v1
--> ssh-ed25519 87T2Ig y4P08L2yYSjVcWdbRCqWSCM+WcgqXpxOwr1Ip2Ipd3Q
-7C/3MXVbAX0HIdEULKu0bc9q2U+4mPDiDb2l5rRwBI4
--> ssh-ed25519 K3b7BA wl46ZMqLHMOTG3RojLVgwC2hskjUJWUGZ4h9dwBYaws
-xxrJQ8Ws1evKgfKej8WwbucuArULWNtCdMlSDdVNe6E
--> ssh-ed25519 +qVung 4fix0OAAyW/34W1HVfc5ivIr8ijqNz0Vz8oWaSY2lyk
-8ZAguZR31I0hysn265ELYeYwrLiDx07BepG0w1R8uhU
+-> ssh-ed25519 87T2Ig df+IMqWM/HNjaY74zibFQIdUdC3K7uQlm3U9R9NUtFY
+hPSbCuWvqy/7FEj7YScYztyt5GVx4Y7tgGuKKkSKoRg
+-> ssh-ed25519 K3b7BA xN8wzUKHqjOb/tqA+EI+0H0MSQRihRfydchwVqYWAVU
+maLMpZe8orvTT6Av+YkhT8FcG4dc7bzDgOW339nSw1g
+-> ssh-ed25519 +qVung oM1uphTbjI54t4U9jNd1zORqpjBG17MwDf2eNDmOlkg
+oUHVuQt2SHIwtV82pgnKJ7g2jcVBAHWOzPK46otoh34
 -> ssh-rsa krWCLQ
-vRU5uF64cQZwJrGr0oBRBJFo2mr30pz6yhXwEm4BJjKt/yCCikggPUFTW/KOjnqZ
-JcUoLpeDVIk3+FBJl4p3PVRn1pjRUve4vEcNAEjmkVgBwiZWtpfE6vVLn5pIvm+A
-nwybTTwMJomDTLDsMOq0Ur+S3rw4Nb6ADqDKhmjlmlaSlTqxUmZoznQduoSSINI/
-VJw/+VjwFxsMxdD5swxEAcrDk2rKoQLrfO83PO3HNMX5SmYHHYEaWB0/YeLgvi8a
-4OBueRKLWOiy2WUCqtxiQG5XYGYNdgOKIeNLnPNH6RRwFoBz7Zmn2uuQjmysY9h8
-lryoR6quxdOTRTL2WwGPAw
--> ssh-ed25519 /vwQcQ 8sOHrthroDrjuL14hij7sPiK9BGlOLzKG1pBe5+HMFw
-vQqm96T/H5tINHJxnfi6DYm9YO9UAaj8etmk7K0GJ7U
--> ssh-ed25519 0R97PA Dd3db0zh0/ZUsm3UgsWRbGz9mVvm8s3W2HQkjTM6L3k
-/+IRsPs2KoqEYnxmFoKmNc/00jOesKXv33rO4Yx+l68
---- jPrqv7h6AGoqNl1LCOtzXvU4dKK2PnGsj/FqhstbSGw
-��f+`ϙ+�]&�w=��:�$UQ�7���h��K��U���1_Y�z��0�%\�N�L�0o��
�޼��5~��_
�Z��7x��[�\���v[�o
\ No newline at end of file
+eYspf5hUKdFQl1RxPaNTj0viAPd+kzp8Xbwn+q6fSITMacmyTY5J8FckLx2YXDxy
+Qm/OsEK0ZOvxnHMrL0oAJjKSy/MamE+9heT3QO+LUN30QxbOIOqHMrl3waadWZdx
+ZGOWK+r+dKGYNsxFv+t1Y/4DBKKzlXFWhJ0aL7nMOqq9+Ca+UZuE41j7eWGGPPLy
+fuW/iOVVxQ+EEeCDpatQSrFPKaeWCCVP9oIDFtE4dsKxubMa4EpUoag0UvEIW182
+UGS8BvMqYgx+obqJDkhXXBK9apmJS2ojcfdtCbNOCV9Ett72Nm/iY5NjLprFMLde
+8wWGA6s3hBOP39lq0eiSxw
+-> ssh-ed25519 /vwQcQ 3zLcLDaDVhIn2knezexYM5Fqu/O9wwORnJIhsXHqgj0
+HchGikQMgkDj0qQgtDdsdKokV+nMjdv6t0uVISeU7Q8
+-> ssh-ed25519 0R97PA 6lm6B6B3dzSdhdcf5rjyTu+7cCtWRxVpWeapJX3nbQo
+x/w4dEfFyxPi4lbNEqgjEblPVfQyj+q1JjeQHiVFhDw
+--- oo5BK1pG+43amUg803Uv511RNtdQ/PDwlXUrV/AbOAA
+��Uq���[f7��뼨F�Y���m��LS?�ℶ�����<K��F�z�V^�U�N���)zԐ��@�S�F�Y��G2^������N|
\ No newline at end of file
diff --git a/secrets/buildbot-workers.age b/secrets/buildbot-workers.age
index dcd9c20b06b1b4316f7aa8ab8be175024ffb6723..037f37b9bc7a3777c884fb1e803ff0f8716e8b4c 100644
GIT binary patch
literal 1134
zcmZ9~$*bH10LSqbdWs7uNWor4Q7VLIGTCFHl6{%XWHOV?CQ>rlCz)iD%w!pQu_8!s
zN*A<9g?A7<ilP-k&=zkM@#2CgA_yuXc&O+>y}Zl&3qJ7u{p`Ry@KUE}H`^#npU#Q^
zs=C0z`A%7e2cC=Ic$e-_q=_g&XO`5XY~0k*X6y$&-^6DUCGmc;W5rg`b_<skN7|Uy
z?KB3M@!Xx2F>XX&qaxyJuoahS4Z@8XlyiEwUL@P@UQU>DXdTyj=#pfaj=fs#Xc~7C
zmbdw-y3(6XutE$QcT8Gg8jV_;NW=^3sAd3awbI&ArV+Ghz;(VH5G9>8&XA|=q2=qS
z2r3h5lk?S_I<A5FWS0h=?kMV6Tni0sYNT*+YRDC$CZ3&V3Q8b!rV}u%=iMO^#(+-k
zx_fer2vp)bR7w^)Sky#fY;pr~$qg}BWvR4gR$OEpX+>#w;$q5a2GHAZ6Le`(>nX*7
z0X)`Kp{FSRFeyiMT1ZQH;;N+AmV)KUUSDz(Tr%eC^|aRA&?|5Pm^qMOX#EL2wa^mU
zliaXr1KuZzrfwNm?(+n=#9Lv@YF^GzHa8IcZePq(un*;|k$ZrYjKoydGv43pT|HeY
zLWprtNVeU~rs`c(mzZi8i7BFl2hp@IuN}Om2s=yyse#QHNQ_yP&tX<XW|%gRSi4s_
z);!dT9ujBRVwlHj><YCV$`#}<hNz9gY%-fyS<EbSYN`OT=I;pHoLRaxwG|#jWo@We
z7VFMInxJ~J)HRbTCG@uH2Ca=jtG*$KBry=SQ)Zcndti~wmVs{w#+1VwlB0c79<jku
z?W5m9Rc$y%$CqNpMacg*qMEsx?w}>7^+z*o&qr%jsiQSk!X%0jWo!YFWo#k24NZJl
zDN2cgdvxy*cs<{3#!#Rcv=BG0-K$xJIj+GYnjA8n5?PZzU-o7=vl;AlCW}onofPU=
zEop5fy6HU7)^%h~!vMgO!s)X;qc3Tw&|hgtn!xUq>=zhc8Ign=TTZPicjqY?aBy(Y
z3Bt_fI8Aq;aSr<+)(r#E9H(5lt7s=u3Lj>`sMUz3w15{bHaGu)v3u<|pI-ZL^4;Z!
z-+1JUXRcg-_f_=$6Sv7zd~oT^1BWMHyY+be)MqaZU%qqt{3-B{C*B*Jv+td~|Ce)b
zKl{_kyPwN<reEFn;`6Uyzdd-%yz=Lp<mpQv|9kx!_T=vuPP~5k#^L^Ic=*tdkKH|U
s;o?VseQ@h=ufSK=KYvp6-qFC#&F}xdVSWAO4{xc(qxP@Q>EE9H54`7tMgRZ+

literal 1134
zcmZ9~z3bd`0LSs6ijXR(AP!bI5C=Ja<~f&uRC0OF<u1u3_lVTy@tdc+%O$zwaw@b+
zwTn17xH)wb2TLm`B8n&u1?^NE<foJPn`$SeV0G#8`wL$1{(Kw|90hByX!9!GtY6*~
z0IpH!=zOOvqocsbND^f_bkD{V&~b_;8q&Nr&BKZ?e4`;)YPm5|a<HB2<49m)UZR4E
z*AFSQoQ{A9tN0@3>;%iU7)B%Ygdy`ePI<Z9Ep50NCo5qur>C_6?)Lg@C+rt!F{x~G
zS|XZejj8c;js|L9$QLEf>8+~RxwInIAV#1hT(kubBOpRej@{DMx|}5wGhX8WLq`JO
zOg7^@YG*RwwAS5DcWcn`R3S_m+2=}sy4pobPiMib%v!=AH}INKU1MC8{1OwV3uvg;
zVd?rzH+Lfm?KdK^pE}`o*VSsH#X&IYt(hUKi3UYK)QZySq{WQWOvu>9G1~~XRS*{M
z5_7#;Q>?)8YTzGyd=T1fnG{<b#0^&@p|xEEcHVbndKHC4Me-yRvU9moHeBQ_M=t6|
zd$Tf-f?x@+MMEc~dvht(8#ahjv1TmV8zu=@_Rw-p1KKKb7#b#-gLFx5Dvis@&<kp~
zVs^s>A+|DK6}_@?6K$g#P0gA8$xL1rItz<i*v0^}o!}UN7ADwNWBhO^0~fA!Q9k4|
zj|$T<J<*W_mC>jbQx%=3aUPiDL}>s5$X-?&>1e)Nh6>bo2ZsY7F;wQF0a=!@tJs=f
z<y0OwO1_sT`^FmO+FYV**T(SxBT`Wa>?LB(iIG)hP(e+I1RrT=W^hsNTT0ZznE9~w
z!{5SnV|r%Cq~*Q4uQrG<LN~r3R<J(`g8`ZmdrQ%oYyk(&B6m<8EFdBYMyp*jAgWqD
z%qa}s!>F_EYQ&iGc?xyX$!ddiB~IbI(;6~ndIsCsgoGoo_9Dg|zqel0!9Fe&p<rij
zsp&H`P9(#FmH+qWCeYS~b(&We&Txecv)W>UY2OO4DnhB2mvug~(J2s_SmPDw=;)|Z
z%q-6;k#@SN4Sl9yhZS!tt)}{YWt<HZV@D-%eW==APD5AjpL}rJKYmMC#V5}WxTl}L
z-;y6|XD&V^-8sH(|FXPbo_iHwx8A&S?-L6G*n`iniH~1=^yIhKZ#+ouWl#M0Gy1D^
z_uL~_!%Kh7E`R;QJIGJ}Jo(LIcW-sR{OS+++n0X71if+d`r9{OJNfR@S1x??g8DDG
o_NV%s`o-1vq+|8(==^o;nJbsi-1ty>@%_)fw+rX&yZy8O0V&soga7~l

diff --git a/secrets/gerrit-prometheus-bearer-token.age b/secrets/gerrit-prometheus-bearer-token.age
index 425573e..5d850ab 100644
--- a/secrets/gerrit-prometheus-bearer-token.age
+++ b/secrets/gerrit-prometheus-bearer-token.age
@@ -1,22 +1,22 @@
 age-encryption.org/v1
--> ssh-ed25519 2D+APA Vh/FrR9oyO8V1pEMQkmGbHCePB6RU+dPm+Z4bgKenEg
-2G5eLlYe8IS7fsEBorFljUwQZ9sEk/FEr25S4p5hWLk
--> ssh-ed25519 j2r2qQ 9+NX0Guhux9QlAxx2MtSZH0OZpDk1CQZ4Blu1P9fpgQ
-PDUoAjBaIdKQAvRblvc0QEtrvp5MpE8HsCwKWwAn0uE
--> ssh-ed25519 K3b7BA wuOc6LGnjsC4Rb9D9QX3YVgMqWPvBK27Q0vqADLpsk8
-wRnoNzkyaU9SGlOtpqY2pAeIwD9lGWKrqNn3D3W7U6Y
--> ssh-ed25519 +qVung biXtZHmjJmsazEmp1iIGUqmuV1YP94bzrMjoZTmGPjg
-GDN4WZGTIP6b2nmjyhikHeOrZi9YEtiPOyaJLzUl138
+-> ssh-ed25519 2D+APA jiLDQ8JlYhaivXQQhjEfZrGWn7o6Wd2OMrLorEVSPns
+qRzHYcBhtGSm4RW7C4oW+VWSzHiDXkCN6bGeej2Gcpo
+-> ssh-ed25519 j2r2qQ OcnIHB/vJoKuvhsT9dx1B+5lXguARtB9wSquW2KBB3M
+pgzC2KOFi3Yj1gCPemVK3a9Grv2SkwZ6AI1EFdh4hoc
+-> ssh-ed25519 K3b7BA ibHY8wN3rNit1mO2dJZ44rwLylMaR39a7Oz3CGV561o
+4ElWORF/4lVEz33CJiuFG4rwUSIIOyi2L/W7Td7MX5M
+-> ssh-ed25519 +qVung q4DDHS3M24kke2NCcpHEaUbUgoQB6QwnmDiwmdIOuBw
+Yfa6v23oezdDICE8I0UaVCShKlx9lN3DnBnSb63LU64
 -> ssh-rsa krWCLQ
-UkNySvhS5o6v6/7xGvn43hgD5y2D91oH4pjU3Oa83CW6ha80dnE+JkSTpTdz7Og0
-vtZJuisNpcH254zTt8OAUpWN/tVXlD34RyV1xo1eHEWgUzKactrhlACpSbzYBdVJ
-8cUj7jiE+qjIOtrU2sHWo09NKpf0J2YEPwajuBy1/fPrivlgXAzdAAnP4gll02x1
-Et8lUn6HVfYDGtrDo/PUUdgcGudVeCOJbvvrKYkuqe8vsNYgnFHM8dkTJmObL8dz
-zp4MEuIQ3WrrXActSnTs+QAGIFSskOIr1DQlJRYzQcYtd8wkfx9a+6oxBECZyDAZ
-T4yso7ctflKlr6OqpJYzeA
--> ssh-ed25519 /vwQcQ +jsCn0OlVpuyVA0XSvD3ZCDRTBq29UV9qsDvE4XaGk0
-p2qblImpl+G0pefJ0T/GjanIc7+bNuA0wRB4mUuFGXM
--> ssh-ed25519 0R97PA /bE6+eVlzeJKOOMqz4QjFdsu+5XDv9L8cZ94cPZ5WQk
-Xco24ijeQnaT7jcsfXLQPzGr1FE/zy9+qVoQ20DLP+Q
---- NDqgX11cTXR48vD9YmAIYx+og0n1OQj+bbkKwqv2BeE
-��\�w�
��9̒7�cؚ���%}|k�?��$9�l�&�=�v��ܹ!P��3b���퀩
\ No newline at end of file
+gLBHP4Z8EBW1y7Yf9sfWMU+/fJ4WWp+NGRR7ebO5GwUeYobDYm/eYQ7rD3Q9k0rF
+kU51GYBaO7m5gLqc2Tq4+YjE2/EXDvjqkDSoyNrjQaaGTLqzvPYlCvKWyROjqJjX
+UwzPbQx5XVIKNgpsR9e6/hoJiJbDpavM+HQo+1zwoKAg5FvZZkE5UnIiSjuAxMgR
++tmrhBfHEYkpbCCrXVE0jLCup8gPIci1PyXWkdhJy+HyHVkbYowGwNawNobNr1cF
+dJ5IU8P/DSSqZ1qWSl6ju7JKjzXU2Xq87/g7wJyrKGpe37pJmPIT86nCJTut+AK9
+iFED/y/p5NCtohyhztosgA
+-> ssh-ed25519 /vwQcQ rzEjV56G+USMdpWklrGQSHuzG8d+S0zWhhwrmuyTyiA
+y+uMRG8NdAD0H4ipRN+sJPn1P0CGs4bk+U4qtetP3O0
+-> ssh-ed25519 0R97PA ULWdDUjDg9oTEOqzCKUJl8yN+qwwmlSi1PFwRvr7aWM
+YWaE+STxKfQzxYMtP/cA20q0atXLdsjeA5nJyl2f8iI
+--- Avs8hTgLwcBy8hyYWjR/Jbs5YaKozv2oBmGs51ckquA
+W��ܐd������`@�ӵ3�����5�b�Y%�A�Z=Ki��76,�w,1������k��R�AĂFu��
\ No newline at end of file
diff --git a/secrets/grafana-oauth-secret.age b/secrets/grafana-oauth-secret.age
index 28213bc..c389cf6 100644
--- a/secrets/grafana-oauth-secret.age
+++ b/secrets/grafana-oauth-secret.age
@@ -1,20 +1,20 @@
 age-encryption.org/v1
--> ssh-ed25519 j2r2qQ qI/dlkHZYcNkCVgZbxpw5Ps2anl8pofaFPi4p6kOHAo
-KWL+H9at/p/AfCjfO8+SgMhn97F+DqLO2ymYUOHkWjQ
--> ssh-ed25519 K3b7BA URYQ0jFY5yHS+dodR1RqodNWrrXkMnzTp5OCSv1gbWI
-bnyrPvWnzDRNh4mI5HBPkNl3NSZE1ycMK3LLExMEYbo
--> ssh-ed25519 +qVung z8e56tCZ4TLkrX7BfH+5RrGxGoT3q9V1FB/ySsH3tg4
-jIpEEVF8jCp/ks5eYXh3O7+TLidvzYsnBRFd3LkgLXw
+-> ssh-ed25519 j2r2qQ JSveX4zYEjb4jJH4eg4oXA6r3oc0jBx8NgjhN9JrjlQ
+1ZIr/XFClbwJHn0ppJnolpb4QlgZOA8JX5OjjY4x6pU
+-> ssh-ed25519 K3b7BA sXUjuZFK0PL/KndxRCJCM5Kg8OmVseRZNWG8mL1alRc
+U9MMgDtqtmsS1W5i04Pa/b4JBTSjK6FffZxgYI3phtg
+-> ssh-ed25519 +qVung FNSElbiw0frYcsO0xoyPQgRGqAe/aVX21dTB6yk+GQg
+zHT/xU+yfXYSBO2HLwoHrGf5ns6BDVb8MlhVVQCBlOc
 -> ssh-rsa krWCLQ
-XG8KKBT/hEvB+c1RDGUrDR4HrfAertfOIzQTquMQ+Z3Nde3Ybxf8W+rWGQDErbq4
-VlvC/wVVnGnqgE/tJMQP41sCMKSH61MPyiNZC63g4RW9e2H9YQfWWrnuBh668G+3
-3sE0FSdIAB+UlI2jlbMiG60QaT6zV0XyOrugLX/G2R+D4aXYIVvMtcwYq2oIHy58
-1DE5llUZHGsQ8APXZle7ZGyO48ELOQkVn8ozPlPFhvz2y9srgBZvNL/wadjvLstv
-2vBTBoRk8HnTLOiybAnGtOfK6kWUMdfSYMvhu0IM8UBSoxwxOHTfIttKDu2ZMB8g
-c/RnKbV2z0PBdXVrYuijPg
--> ssh-ed25519 /vwQcQ qinzScNz0IFoHUaCeGXne6ddllQ0dA/TJr5Z/nbfvTQ
-0YpTZ2Z2WwN0sJ1CIV8voPS298u9uHbRQMlV0GMrvFI
--> ssh-ed25519 0R97PA en5iGTQoH0/QJKl38HNe4xun/FxVBIun7Z23mBW+4XE
-Sjshx8hLyP4iY40y/Fehc0wZTBH0d1Lu+auX8L5n28s
---- i5+vCeWbFTRR2YbIX4lwbEORRhaI5NkCwqaMEJqrPEs
-�\�F�Ri��Xa,.��o��r��hE0=$ǂuGa/o����i�f��x�Ӛ?�g�Ci�
\ No newline at end of file
+ye0mLiYeyvlp4EZX7mZ3F7B9V9JSeoiCodzccS+5qIEd6gr+RTHSnKYqwf/nwf8F
+qKLwbxWjpmkIzBWeswy8AJ8159aucGEmB+3/tTSwd+QlRkru4Z/7jtfU64KQttgt
+vaRfc9J/85AJJ2V6Sw/xG8SgxyLBbp/XIN2+tmb0g3kAWiuLcrLk3H/MsfmxDVXg
+RQjugP5K2+fEZc77dHQTrMI58K9TrSw1zYA1ee8J/fl9IJ7J77qi5UgizY+YfX8T
+SmR9DeYUe+hKgCB2k/KgAxp4WOQNgUOFBTsE5FW+kQQpfGx5aqR6vCYU+CPsA3Zb
+FwV0l+g4FUVy+xAtqaGSAQ
+-> ssh-ed25519 /vwQcQ fbnK1jYiUwUsgD8sSTboJCBfcuwJXKNCaJaWYuIfmVk
+Uj2+uBABMTxq1MBsiHXgkdFMOpIN7gfxoJVKOQff1Pw
+-> ssh-ed25519 0R97PA yYOb6AYAFWvm7W2KYT5v9zznkF4Di/vatH48Xgx0x2E
+yUm+MKj9496BkdX2FpLyhML7budUyqT1hL9hpghxSnI
+--- ogCPBrmdbeDorj3t5BL05ge6VngXBpUEDW4qaaKIa0U
+%���lD]Ϫ?����(��E/Wu穉T���[}�$��S���^[:�]
he0XU�p��q�`0A
\ No newline at end of file
diff --git a/secrets/hydra-s3-credentials.age b/secrets/hydra-s3-credentials.age
index 9fb3d81a54684b32089a42bcc8e284e3fbb04fe2..804f08830645e982029289293650b01cb80c6f7c 100644
GIT binary patch
literal 1150
zcmZA0&FkBA0LO7RL9i;$Iqy{u%Ia@vnl=w0gEVQ<Hc68<&jH1xNxn_eG!Jd_<eZne
zi$nAz;tun&9h@HYB8n3);^BADU7QC6J&7O+Qyd;PciCls!3#dTKQ{0Vyv;JN(_N5k
zP9-^j3b=J}y1m`54m`Iv7{ElkGgfJzY)^T=Fd09Ci7GBcjzuc6kY*|2rBz^!iOfjw
zsIa`DBCD-@y4ATRoBB1g99j9iXLTxos@;tXd255KH4I$<6eDbEd%wmZ4mKp)VpGie
z<&K;6ms6*jp`|!386D4P*VXL(w4;&6VljXxKuKB!FwH$jGhr+Xq#ZDe)TA@LWEX<K
zmAOf%5=Qu{X9BRiU+ZMregoPNHnPJRl&|2#uTsdVGzPNP60@=<=mdjoz*<xjPSgZ3
zTR4y?hAon6o^T;NM-(ZvBQGPUK;c8KYzn@VYbj#|P~Ow>t=*1to#N*$%k59L{lLjp
zhSnrko+VTRQ3I_jj2%BPlnw9WFs>9mG>n9i&i95g7A%s`S{HR3)=f)+2!4PAYr`gW
zp;>G%FNV8GT~gdq)M8VfcO?rO6_j4rtk5k`g$Q|TW!9*yjVM&4S}+O2yi={grsQkP
zT`d)D(2bm;X475-MQgRGO4OLqF<M6<&D51xRQr@$d%B!%>zUL7u}cm^GmFaH8YknK
zuofhMpzb!py;(1#h)VM&>?kd*&d!3c705My-lW_8R+Hx5E@|OL5oDat7@Pi7rilfE
zh?5e6vQF1s7ZS^sYB8Q+f#p!^V9AWRWXYh;2y>7)<O~GVt%4$p)nK3IvR`{tX(Wt3
z;=#SO?{y0m<$O7B6S~fK4Rlx_Y3=#ph;#$DH*|ZZuZNK_*+8y~4XJL$Hr*Ddj<O6N
zm{U(*1TpR8sI!;VO3HVAnus=@GV^d~Cy{l8->-ES1SXN~hH{k!A*Q%6=Q&uzWf$rB
z%~)U*AEMP905Ak&gpB?5-EM2iE}7vZ+<9Vex|>R!I`rlpaWjck+)&b0#trd+gsbh{
zmmD0lH&d!+@BtDpAUP&zpAn4SSc1C|zp7Lc(7N7WL@Z@F=(jE(|CQ~8FP*$_=cs7w
zH?yBEefH6zNPacN&mOu@KW?tipE+{zftT%1wa?AV7n)bk{{jBGa`u;pPkj5ce8c|w
z*tO)`kLtNw&d1psHy-=o-A~@Tbn1F|`1dzw-nn}7`jO+mpML7rYmeUePZa)Kyzt;F
zM{oc81&%oD6U*P6=Wg3m<8uGl$#ZXB`SOv@GjBg1+<6gu|H$iCgAc#I_$2bgLqAA!
K>$`uRed~W7N02!H

literal 1150
zcmZY8&#T)60KoBG+`}mTJj{bar#KI_=691^22I+g`Pnv2lcp$|q)D2jX?`_n8+D?H
zBB;9vf;!OCgduEiq7HPz4igj+Hnx*YZ;E&k<xS){JiW{O3qFT0U(<6M&eG1d$;ywH
zPsN!BmN3vbUd!`&!<ivC4pX&yZ`^iUwcP77pbCrFN+WQ!wCRpNDLt45^-@$q8Rx-h
z79*n(hhTt2B^W2-HYcy>VnDNc-mAK>1<4~^i-+SvFu`n`)}fYV+^z|zMbNDp+RHPn
zDya4f%k}MOG3j;aOq>~@E{C|rzy@0=Jk?W@jt!tqNnEW3nPWYSSY->|*RUR%z!nm)
zZA9pHkxBvK5<%PR+qHVC6-%#X$dj<5rCHo$Rtiljrh*9KVlLY}ue4oJ?q||^1kV>%
z)B(n7I+r>_s;^l-$NL^WlS&aZtNF~V&j(xIPEcqmOd*Jk540>dYhk7{;t<f3Wh;=J
zIcsq$HIHbOTsyrk9M%_lI*xoZUgDc#>T3dwXk67SVb$#sc&<*Wz$ORt2@qvGX_mUz
zb&G_~Fq-bWWL7L74@$+B$xTL3>b6bU>BsDZ;!}z0=DC~jF%ej!%_Lxe$Xv$|%?t;g
zE~PWrU%<*5Bbtkrm2a8;)TJ{R3%J;HEIbRe9v--;Q;1tM519(wdMjX~Hbssfvq~F;
zz2>GEf_U87m~*s};fjg~QXcX!vKoxLQ(owkI@uD@r34LhV+LhX#|4`42$vCs9n-OF
zm;w?LZd%PMSabQA;d@<cxfyT*DfGS6)!M0wE10^%OA?6<o$dB5pzZagNzuleB7>zq
zGPP~Hi~g@ISgwcmu%@intSJ+an(&ZXc3~!xIfT=lVBWF%8^d6lgtM8&38C14%aCI-
z_I#x>HE$!K39+eraF3U8k&SwOi*cmPq7r8rK?iLQE`bz^7_@36iwTv{p~qn|tmIB<
zbueNcZ3zl?#F40t1k0eC6yhOX3qZr7KK7gi??77$l{4EcY;cs*cm(GVJPoP(GzzCn
zS{`<QMx#+vt0E9wRjN-Vbh3?F&}OLU+d#<&qqQ@Y$#GcdMo3^N9s;$TzxU~zyGO5n
zd-v(To_qE6Ti@&*KYZg6^3E?$Japyeh24+8dX&Am``LGszdwJIe(BWy&bv?Cb?k*B
zZ~VCb{6CMs_KuwV`Q)Md-uv~)^*er7E<Jne%BP2{6V^v(c7n^VTwgGIXa7yly;yqB
zye%Jo|F=`Z-oqFExOVwvsQ!WZ)??>C`RVEdKQCP)T--A`ncd&qKYi`YgTtfWAN%Et
TOJDymJ=y&5Y`Amr#<~9hp}CE5

diff --git a/secrets/hydra-signing-priv.age b/secrets/hydra-signing-priv.age
index 23cd618cf5e693566f066bff527ff9229ca504b2..3e6ca1a4ca081726b3e40b67492281a71a82c2a3 100644
GIT binary patch
literal 1124
zcmZ9~yX)fw00(ez`Vkxi#o<(_gB(9%ZJVZfh$olk-MrHzZ3E(^X_7YYSM&IRC+O;Q
z;N+phxrpc{2rdqyhzA~rC^)z%c--J{f(pvvEx+aW7kv0WoWL7;say7WeaO<6vN9Mo
z6JYr2psMymZwJk06MO)2Cgl?YS?UdG>VhCAm{1gpA+a@0W{FKAXd;>|ofY~-G3Q4y
zGy$a)x;(17%<1u|sn>~&bNf_)r&=kE*lJwpM_RTUZnfYqYQl7j&WV8-XEZBg?$&D9
z(&4I^tMazKSoDU<lu6>Tt=$({gC~#)tkp)&HDl^9R(9Aa`@~b(;9&H9&9CYmTvjt&
zEATEJ@$f|rEUaT145oXh9tQ#+PgdbNjZ`EDN0{ClogFz=Dh3ul$MTpb)p!Ghi7u>2
zlcaKr?vest(gn843vKVsmpTO>i_!?X9y%hbbgq?^Gl<F+DQbYGDlv}ilC0azHno(9
z2nHgOmOsklBA638dgRhVve_Kr(RvOhzOM_FT=xfIp|$h@1*)@#aJ?Ny!n!1vq(AL?
z21S?2#I&#t**jdH$6ILBhs|t5H6+y<ImD*Uj>-r}EY~Iw`bg&T7(B+;D~+RRI`cVO
z4--G(=VmvFjdsg2QVnAkD%m?{$As%-nV3@3RmYJY&Hz~qGga2-XNAhQh>51gGKpA-
zQ#>vvaC_!Vr&6`vRG6%C5VXw0um`QCE5!!0h%og5G-AW%u!3!OM{6jNbs%nmNI{e(
z-=pyYj}_h~Ao1vtwXoNSL`eCR>GZyr$8rS66;OxqoLOiz3pEq?;Bd6u>O^qf_W5r|
zO{=-uAVZ7Flb9`R9W1numoan<(2lWFl(pG{bn=x>7Ese$Rt4;Z;w-}Ft}u(o)?V~g
zMHYkxS?Vc8wonS{nd50&B{i_kMQyC(s7eedgU;GEBiDw+cD}~1m9$M~ei$G2V>8hf
zq)<UT4+o^R0kC;)?H|on1k*E@TzII`a@nHvH+(*?mEu_JnclZMG|@F+I2;a2NKWY9
z8jsk_sb=w7;yfmEe7D?6TvNf>c5DZ*YaQZg3tV~2`2N}tm)>~xh3>H*Kfn3peWrf=
znXm81zkYV|^y#|~CU4$(?VVSsre*$l>fXzme>MH)OY_Qg=Kb%6cc1_0!QT&We|$im
zXiwJ9)&E_NKK=dLDe>NK|K6@I@e|{ZTPFDP^`CAycfY;!fg->C)jjbce=EOw(>OW(
X@bc@wWYk~dFJ4^uto7$7(T&mn$}EIs

literal 1124
zcmZ9~&FkBA00wX`!VKy~1W&VvA!7acZId)<m8p4YljgN~X_`corD<N8v`LyYNt2UK
zc2U7O*+rQX1yN98I1okgx*Z&N@CD{YJ&wUZy!*TC^7{)u&-3s&fp_F3Zr)`3b(%bv
z<^ibs(9w&Xq6m+?6+9U9N1ZM^r+uQcVRmdhFgLmfYqX}wY(3ZI{t)9W#-xJvrq&ys
z$PNWPsi7G~G=|ObN<?@#=?elinH&yIZE$kmh3hpxXd<~uDhEMNYCN*UNutwA^$Ho{
zHUJ}Q>dd!9WTi@x@MT>})yB51-73adwcV(5s4)%JVq;NSESM?BY{hk%8(d(KF{f2w
zt=KRf*=3O@<@lu5-5F&PbZCb&s36G216mKUVC$^h!GdIR8ZIluG%HE+v<R~uSwkUI
zu6qa)xwV)GiUtxTi&q>MiNx5I97~emGTJQX<KQ^;eJwAXPMj|&L4jzhn=#7)V8g>S
z(S665>j#vZ!$Iz1Lp`x;K4sjIOxRn*S;#wKk*BUm+SnSdW}0pep}Z@j4o3I3IIQyx
z?eANV$+G14c574hOm7lSy8e)-GS{Vr@qs+pIp0n*ln`eAW-;L)BQ?8>LuCt@>WN#H
ztpt!{pp9fzc1wOE5}@ZrD+YLsx@#4;G>H)okWF3i{as7g3AFZ>QtHduDv6!pu`e!*
zff#P_319==nNka?so-jlG)glm8M&eR8tFE?QU+m812cvesm-LV-roS(#J6_hgDAq%
z0B$(~sENF1v)I*_g*&e2=<%Aclrdzc#Y=&nF-?#p(C+vHq#dJy7M6`M>?xcYdBOd)
z@BbE5wc;urR%vV=Z_~gC$}qGu^`O)U;mww`uZUZ0=i<4b_l!1c6sW9ir(h3r*oy&7
zRw=d;NAMV(6?@P0(>4%=)SkKFvYc5bwVpb{XGF(wrO@2KZnsMjJ333#h*9;BtW-+n
zX?C|QOii0Gz8M(`Bv)>!*}hrtr_p2$yTYM~{BcFIdomp%pv6-Ko8k!Oo5$UbjykF@
z0jn5Ns6oQBG8=?&AgzbQ*c+ojCnCgB1W6e7=V~;D{yG1$b@`KD@X!BjpS<+aneM;e
ze)iEFA^WMk^2p6=?>&3>$9umj(3dXw{f!5%e)DSgySJ}ifX+UD>a|zSo_p-w+uY^f
zZr`|g>(=>e|NX#R1oz%QRo(sTg|9uLbLY)jdgf;R;2+no8>hejp!@pY=brlcnLc#m
g!V@2!zQSDj;%VsNyKjBUKlJfem#lXl)gO2M2ZV5hBLDyZ

diff --git a/secrets/hydra-ssh-key-priv.age b/secrets/hydra-ssh-key-priv.age
index e4ae755cebd3128ae83a3510ad162a964264d1bc..20deb4b476668c83055bb972cb0b47f68d41de4f 100644
GIT binary patch
literal 1429
zcmZ9~`;XHE0KjqOp->;^De@MM_(JBnb&sw^@NVlKy*;|N+q$k8aBbIa>(=hkuG<bc
z;k-o7pos?wo-q-KU=$LGo;(aMBLap%1O(&=9;XB@J`fSbi-7*ZFZ>0c<eP6wO1GvJ
zzFaSASyg#KElcJq2(-S`QmJHG(}LCE0Ff3G7Was*7D_AWXsN+BQuZVPTdZk6<|%a4
z2*W^mpQ05tg@bg)A!U3dfYMB(%rH1_;<I7NnhV-+kH^M%aM2+kAmnuy>UdgaIa|qo
zw-&T<PWS(sh}n6I(4m(@(In5+oCV#@mSRrX%z5foJMHy0Di#9N4XOiBVrU|VWQu{Z
zf#hAHEzcHZlF5K2yEB+z0<a5B=b=0y({RV#nyEzViqs;&2<`~h;u)MLycmmsHlL>)
zF1bX%2}X2#4YitTl&KsHOJSf^GW=d7=&txJ;aoP2^MWZ48)-v>!6b^M%viO+!Wm13
zpyGG5awXLwmlJ4+1manu&XS>m7or(Xwux4iwt0er0;@T(9(H--H5kOn2&B?^0*V*n
zT!M>{Lbl2mGXiK=F(5!#RMG;|6*%Q_c&)OPqeP-sHbs0{hsxC4EG-vJHfM}1IGv6p
zoX<v-de*6;Dvw|qtAznQk@b0;A#1d%yAq0-pfy&KNXSpd-AIl$kcgOIFwPAtd{Spv
zMiw9`!`ED4b0zOEfdmc^9aS9m$61_o7rlIl<D+QAT7^m}d!>>zIdn7^<ejKJT+c=7
zQ9@%vI2Z)Ey4edtu__!T0ZO&x-99A6<Z^n1L!&7H1rs2P6&=Zh9Mlyf=apS3D7wNC
zrD{>DNQOm9YB`)r*gP<uF#%;?EC|6ATdx*EY&@VeDmF?a@6^8Yt+{HDJlP^49?q}W
zDmBLED?#2kC2?l6q6u<=@DcGq!K|k+xsbJBbP_<4agEnx#gKC;k#iL5g1M1oA%m^i
zsak`fJq0fs3)8+B4&SX=D8xy)TB7-CE?#6j_IkY%b9h4Oq9qoJHNc=NP($2h+(`(w
zN{3($C?rtEeU74{hD8fc#W95R<}3}A^d{{A-Yyp<pWp4RGLp_P1P`>fwzgzUAjO~*
zN$7&%(;93(&S()sbcOORb5s(g4mDUJF|I*5?SSPZ*84~MBV*pYyg1)Ed)IW^trG!P
zCz3k+Y3IIqTba}^e{Xblem*1^tI{9+`*5;(Xxd0Oest|yN6&4THG1Ryu7gV+%C+qo
zzWCeQ;}<yIxQD$seO&XE=LTP0J8<Uqv~#@mz22Ug<Dl_J#3v?fjrKQf=;_&ZV)U;`
z{U`neZjrU+nYocQ+%qpeUKqBm#%9u!JPV{51HW%9_l#kB+p*0f>|B$!bKvF|O?x{x
zeYHQdZvKVQ=hMwAy0&bPe?4_`@=(*E?LD`@{eIrb(9%<{{CuVeymsLH*c;@8L#;=k
z-VdKHJ-OuSqp#06c<J`DsCE4!b<uQf-F0@DGY%|2t)Bht+Qhc`D<<?&dmfvcnYyQK
zr0<j2-+j=&ci$9O_)!1AecQ~tg{Ov>SGzK8JD6klH%*x{2Dx>9;mpFS1)Gun$kCDz
zTm8@A>8m>)xY&IhNS=lF4~{Hb`As+UwUM40WtYucc&wdWbHVq+y{{hMd44=pO)~vU
zXLVoiS~WEJ()C?UePijDulzXYGBCJh<;kBmeN6X@c-LYquxZWqo7vp1Q;}fXJL@OP
d%MS<k@qN^T%dTz4tW!tdZ62wtT=b=X_kVP-7s&ts

literal 1429
zcmZ9~`)?Bk0Kjpc6RRPTsGtO>f*`50y|&l(AWZK1ZtwcKyS6L8*k13hy|(vi@2*#2
z>d;8$LS#fVus}dz0biLgP~@3STwn+~h=fEX0?Gz#LBJ>?>M#7lU+_u3`DR(VndT{3
z7i&d<e^HQGvkDlRTa49ep_$HGK@fn9rU2<-V55ifqjW77g|ko!_f-=Put9}s+1zT!
zH4@I`g6W_O4_Z_Q0vj~VYC_CqkrN0FiNt7|h?pC>vNvDOcu~&hO*I&_1miVFoqSLW
z+j33>Hi|Zfzl;P6_6SNzmQtwVgHRp?Ni<VXBML^tNG6a@Wqo0|Y7i8T@P*u@SP%OB
zCa_3QGLVX}ZcEJ@BM5<DAeE8PhTo?P4{9bUsqn0^L^oV8=b}Suk*t_PxMWjVPSr&Y
z$E}qZ&-e&WxE2tKh{bL&6DbMv`NQV0HE)SmEszs5({d%oF-odHl-w8^_oPceT`C6e
zYjQPfEXgT18aL<?W-@^uQIC_LBNI<H+^M?YD3WOv<m7^`DfS#*0v&oHgZLFMk+RwK
zsG`vV6|_@D9uy5Y2Whz?7{fHWZpq~7bjGU3otUHozA}S(lO~yBI0T?X)`cZWyQji7
zB1FW4y5zK^VoAur+F}eOVtU?Luh}pP^h5~K5)wGPCaEEN9E5VUg1=FTN)j2ex>O`0
z##2F<BGZ7%v#=r?Vs!uwWx%pEk<TVPW)`WjP>>=e$?eBEpNkCo5{YQ3T#V(a3>Bqw
z4wuzN*kzN#=mAQo=myjgrct(4CvgTO0nwW!2@thg?DjIulsr*|tg~(&aOP`dCX_CD
z0<A(61ES7cm}IIUr$1rvS0D|jm@{FEh-ie8DClfXhVR$D|E*co;#Axy1c-P@fkS0V
ztG6;-yo@;Uq=OQyNCIo51cZ0-CZJ&@QUq!6dSfQL$B8166Ax$9La|lw1{}q@E9*wU
zx*5yWd<`AYYCLa$P_rZ;XACx0gt!NW6?Y_COZg&YSu{flNU>EbRY?(I&MNLERTKzm
zo_N*3rcLEqEU#L9m5|S2E7#?ysPbu2t!LGQIb#mMWp6gEKoQUXCp9-4Jz)+r$zlaS
zgJqN7UjZ;%teCe)12HX~R!A!YH7o#JcPFs8VOjJ_cgI~ETKDRfFrQcUO<J<?1K;D3
zo#WG#ecvgiL$h|xp1=C=0`B&VS(ghd+J~C@7Ug@^_KvOLHxynT_+zO&JX~FU>b*mQ
zb9Z%389374SD3zIB{p(BJ!Y5_9hHV!J85M?*Qr~B3p-v9-&?S1ck27;3#m=>+0Ks*
zZD;o%J+b{N+)j#p`5(!a<2_@~@BQ_)ADWB^yl3JWesufQzh3BmfA75I>ih!-+riko
zD@>Qv3-|Wy8`Q6VyJPBN^SPH=zM2aS^$s=NJ$Pg1g~GP2?-)&=Cx<UC?b-D9b*47(
z#nXQ`j}3p;J$`%KdvYZ+yZ7{)iJ$wfj{kD{+L~oYCog&IYv+-+(eQ~6JAZq!cdEPh
z*2;gD->Lp7-Wi{M(R^+uI|)C(^pl$hs>uGX2_2zp>!(fFI?_2~q-XUgGI2iOKelS>
zr7K*n@1rHhAJ+6oip$T<aZPGEd5o`5d1&G{gGc?|jRWguf^Xf}Q&5Wwy88P&_zky*
z#O}q5v;&>}^vIg$60Of}zIfqsTicggdhX5peZ#4FYo9r55n9q`{+-fbAKg6gDgKJK
cE!wfZczE|uPrNcY_{RD6<V~vQsT-^Q1LOxJivR!s

diff --git a/secrets/loki-environment.age b/secrets/loki-environment.age
index 4d2a214..c58d72d 100644
--- a/secrets/loki-environment.age
+++ b/secrets/loki-environment.age
@@ -1,20 +1,21 @@
 age-encryption.org/v1
--> ssh-ed25519 j2r2qQ JzVKQt25f18L96aJWsJtFAR4mvMVCgYMKu/xtJ1BeDw
-vj+HpNQCNNxDRA+7HgjiD0XlGG/Yy+tk8KmszMkxdag
--> ssh-ed25519 K3b7BA judlH57lGOGmaTEG19gYiORJT9uXiAlxZrP+ISTHDT4
-MS7e24A6rEMUtUUl8DlYXPy9NhqAq4buOWT0iYKvbSY
--> ssh-ed25519 +qVung vglRR5LYFZw8v6zRhybGPBctwDgYoskbpGYiLNW9qxM
-VdjQTykQSVWubGimCHiekQX7EQdgOB3PYsRHiFnpPkg
+-> ssh-ed25519 j2r2qQ 6qyr94uky6B36UOY0jd5NXgF2rJ3RWBUzZ32c5iOTmY
+fjlI3fjYjwyNQBs4K4pq/5c7oBkf5XUXoGlBOBpmPu4
+-> ssh-ed25519 K3b7BA N9VYT/ZslG07KldzO8sPE5TiYYwxJqpYU87ED4PuBXw
+P1s9L57prPqM4fjcYHv+g0rgP/NvFr13CgCxthVHZ4c
+-> ssh-ed25519 +qVung Ry8uUFsmYmP+Urw46lhAsCc3S+QiWu1mn8J3rIy+KFQ
+iB7xAfdpHwOzAnLvosJb+F50QKsOYWr7CHC3srsS6ME
 -> ssh-rsa krWCLQ
-hLYT6U+dUVuicVO8hSw4KcfkM9bay4JR3TEWGlmmIxcQ67LNggzuyRvV6U2yfucg
-Xyxezdd9LArf8z1eV/y3iwsY0PvK9qwtgpgH/NxaF7djhTA8+c3c3a6w4sqdHn0m
-/RZU+eKSFeDWII7fn6o7JxzITFhF1FYH6PJYA2cb3PvbPw/JSja8EVZ7192ShqGW
-22TThbZmmKoOPbmDxmQIygZTxqyaXkoFOnTWqqTzOfNtBOBFXT+cIFh3ctGWLw79
-u7O5c2dmpXoE0bdndQ7GUSPrgRzOYHQ5hLg8WtC56EYjE11Bxj88fktzw4hZTbYQ
-jrS8Pa68UPhUmSfutlpd4A
--> ssh-ed25519 /vwQcQ MqdVxRlS+EMA3f6B0D6m2ylvCE7WVq1av/CvsNVAB24
-KX8RJ1bzUUhsYW6qN06FTzis5i13IIoIpUb5FkW9wkw
--> ssh-ed25519 0R97PA RHUvc9XQIxOW0GCyt0vRxPHyVXlpqM9gaUps4q/Grx8
-bxgFxtbtbvDi9knzasdR7u33Mb7x7LcBzqEB/g4Oc4A
---- Z175YCdbPBBSItxomyXPSo6xILLV4GT4gpA4Oxz9qgo
-E�V������ށ���ӦxY�q��84�6��&֑�������!�Zm�����d.��46�ȷ����/���=��܈'hM�_��j
>��6�R�&�
�u����E��^8c;�ě�:Q1�)�1L�_~,�K������B�7�
\ No newline at end of file
+w0xIVFtUghdAO7SxZD10rBMtdQESEvYUEKxnWzLh0cjcRhaVT/BXSZQsKV2Rupoo
+nDL5uy0k+tPXm0HroZ6VkZ0fH/lOpeUR69ZvJmClKql3Fnf1385+5BvT719cbbaq
+yll49gx0+ms/oB9jS3SPwbOg+UJgnkZCeu9138h3MG7yWNtVuA9l5hsJioVvOVlS
+Z5EXbjdQR9xYjSwR+b8MYZ97ej5fXpuULEopbx2wXt84u1e67vTETqflitR7lrzy
+A6F65g35aagPJZGHzfrKVToy3pfXm9ky/30DolWLD0DpG7G6o/8afy8O4yBAGlv3
+ZLTaUbrdILSz2ff1Njx4Nw
+-> ssh-ed25519 /vwQcQ YqqmX/f4whOk97kCgSPo6oj/274eYlBWtS+OahAAQ34
+hoCbhupzSTx+wNIorzYGHyGvU/L8unKEyD7Bqq23YP0
+-> ssh-ed25519 0R97PA 17SDtfT9GzAsIsQB24AmYXpW8v4+LEakup+tdFroHTk
+HIvBhAGA2GMVWFBP3OTFEn+XpPFBJDOJDK3SQ94mNKM
+--- CD1QrxYGAhhy+l7U5kOXn1shCwz8pYJNuGRugPxmzJw
+�Y���N	Ϗx
���rR^z[��腕�a�z
+���ɿϞu0c�c;y����&{��xA]����Q��_:̱���UoiDl�(��
�w�Ki,j.oFy̰$}�Y�@1�ș�Y����u �*���ŏ��0�
\ No newline at end of file
diff --git a/secrets/metrics-push-htpasswd.age b/secrets/metrics-push-htpasswd.age
index 9892486..9347cff 100644
--- a/secrets/metrics-push-htpasswd.age
+++ b/secrets/metrics-push-htpasswd.age
@@ -1,20 +1,21 @@
 age-encryption.org/v1
--> ssh-ed25519 j2r2qQ n1lfxDP73nfF/CYtE4gpUH6YgjAQbx/2TTuyfFUBiHQ
-LGzudpjsYA92pM0UpUT9CWZD+e+rzGFP4ndxPE0MByo
--> ssh-ed25519 K3b7BA NRnnKaOtdtIjkRdam5vAA9Yj1RUJRReugWKRglWAoQ4
-Xprx5TSU1rNH7NMl0X07K1KexCVXMEu7BFxbiPwxvBY
--> ssh-ed25519 +qVung qZsGi4JqgpHrjlg2VdY+OhXb0BzYTytBBqY3jNsrSgU
-GgvQG5iMd6XTZRCC3EBBvqF7nhkqAJmxdIkCFRV46Ok
+-> ssh-ed25519 j2r2qQ sIYTVOTWNToDSNa4qiIaSoac7zka54g/opQ70q1SAA8
+2Z1mlCWxjakHqRbArU2BkT7B/Dx0XKH7kCnBa+OYI+s
+-> ssh-ed25519 K3b7BA PGyd27M/Hmk6qpRf8bcI4QWrS0vrPgjiZzaXvKQkJDQ
+ixrciiNR/th0FM9MxVx/omHdI61EmAhTA465SjxECF8
+-> ssh-ed25519 +qVung Q7k74fDLKwCdzobz0b6ByS2LrhMOIC58Ofto0gpBLFE
+p4CIje+sO/nOaO1lzAY9n2HYLUKxEvKDbxeR6dOyM00
 -> ssh-rsa krWCLQ
-EkmY8uc79xWfKjlIozS4Yigorz9IdK8T8VjMnVcJN6+rhoRctQNVCj4JgogY4wa0
-V3ObjoRPZgVU3qPmkPgIKVa2Mvf6MrCMwvvE4j2Yyy6lmQEwFdvk4s2c6AD6T8Bf
-rktRYqOcFavuDr348e0ZzKniFTRcPMcY49mqBR/mWIfSEtLxBgpFUCn6f40PLndT
-3dse7kgRBlrKbzmf6JIsITHejqwDRq2bZqHWAmZhb6+ske7oDicAt90FDoDbrwvd
-YwXPRDCxgATlNz8n/xFUxd35X+zEftUUtANSGtihIE4LcdsO7IOwv/FCjdEn/3YW
-ZtQjphnxgDsY61PEFCMnYg
--> ssh-ed25519 /vwQcQ DKQuo5jVunUFTCbOxVV57Xl6q+DDOVDWXdon/lZlLi0
-doN6en8IK4Ju0uATp+IZAhYl1tvdnfyxHziSobb1ER4
--> ssh-ed25519 0R97PA I1GECXSPagJ5kD7CeVA21TQmpMEgLeaiB7XYEomUl2U
-d0kO+4SkAPC/ois39SZafEhTqvmDpCZbWTUU1aUZ47o
---- 555iE+C2kDLIdAJ5KARyKcBQZSDRWASuzcNiKZ9IbRI
-�e�ceV&����g��������6�=6!�C�C��^�����zէ�(Ӂ!�B�B|���'�R�_a	UtI�3
\ No newline at end of file
+ezrZTitn0/BRD0K7e2K53qz9AZCa0aHlzFSuyzqyVJLdAZUxBUnfBwmGuJgKTa4Q
+fWsXBs+L65hkcL6/VKS7oSGGyoEHmoPFKbb08B6FKLHt9V1td5xbHIoTYbvSavUA
+g3wpTUa4eG3ivcu96VjyyBKTAc7LN7h7dSMbvvP5tpWT5vL+WstCdFf7zzUL9HBS
+yI8dzEbCQIgAAaHj90MREgIIgIB27Dn1PvkEBGYky5ybBRa3DXVyqnX0dDtsXWpK
+ipRPDV7HC1+x2TlqQjD5ED737r/AP573IXbnRLSEWnGDjtd/JWQmfOO3JACoRjU6
+qfb5SSDT9QriuWSow7CDhQ
+-> ssh-ed25519 /vwQcQ duuo3BGe4Q1MHMljgzmtpzvtiOvAHqKu2HS9SBxLuhE
+GCwccbE5lX5uPIri/7Vn6hzpfL7ouJBFU14bKjl6yTM
+-> ssh-ed25519 0R97PA WIFf8tbMlmNrNFF5tRcL+mOJ40SvIdppAtItWtxzCk8
+miU7Z4poEVMZCeAEef1VS0jouCDxGro2xLEE3hnRJEQ
+--- Iaff5rxl9r1qEnlpkOpGyBGtAvGMLyBlJQ45iInuAnw
+c�I�C«�2�7�	��ړnZM�`�{7�`¨�V@�yz�Ş��)�Y�D���X�o���	~<Pٛ�5Tp�x
+�R���
\ No newline at end of file
diff --git a/secrets/metrics-push-password.age b/secrets/metrics-push-password.age
index 27be01f..5815275 100644
--- a/secrets/metrics-push-password.age
+++ b/secrets/metrics-push-password.age
@@ -1,52 +1,57 @@
 age-encryption.org/v1
--> ssh-ed25519 +HUDfA FOqd+I9DzoloOMK2InPz8yAGsk+ZgMKy0n542DmF5ig
-sui4rdOQcvjL6H9rPSbSAyIggaSbsIVrontrkFpPPC0
--> ssh-ed25519 wIR2ZA V4KPrGw2NKeOBWpjsRbhUJ/eLR8/hvExNMpcBvC7gCY
-Zjc+HtALqZbp+L8tUUgaFe9LR4NKptpFq/L7xhTItXM
--> ssh-ed25519 oGiV/Q kJS4DAPBTOgADY7LCZnIfORMM1RJez/5XGoKDfErHjM
-LN3XE7qM2SHqQwb+JjIq5tMvt77NI4+YOxYnZh82udA
--> ssh-ed25519 gO3aog gJFIrngWZp4ypA2IZwr+c0JkWgUu9VN5AzoyyhozlDE
-lezfokY1lgABSKNO+Fr+tTlIjC3gzc4Bw2YlGLy+WvI
--> ssh-ed25519 r/iJSw VzO6pblztwci/TMfha+dOc6Vg4DC/1oSNEt0aFaCYRE
-Mf0LjSjWJA2lMt1M1z+tGJ+9NVMxd8J5CSMvaLK8zB4
--> ssh-ed25519 N/+Clw uNBuYGWU+LLY856o15jLkJNk6pu42FnX55CoE98/ukA
-zh+sZ0nskVPUKd3Ajg1FHng7caKhkEHiRFcm8c53siw
--> ssh-ed25519 CtkSZw YP79uyNelg7+nbeois1vu64anUC0lhUhIie6EqUz2i0
-rb9zte3dN0+uwjyJLGaUfeEQcVtMerKEOVAocLGXUYs
--> ssh-ed25519 keg2lg +g5uYkOOyQABVmL+9t08aaMklNEbBO2j6vqKyrwYrhA
-U4FzATeou9spmYchqHPR/WR79Y+ILWpwhLwxjYQd7d4
--> ssh-ed25519 H885DA tAx+W9kfJkvERw9KPKZInC0s44QqQIu71MPUosasHy4
-5ks2qkZfkMLK4meVHTfWpR8qCeU3vKdPiWVRTyD6OhI
--> ssh-ed25519 Rq7K4Q xwSlrqIh+rZFv6w1iDcPyD0nEmESlmHleUHsVPrG2Bg
-OgrWCBqb7SAtQQSUnTQ1l9JRyDGS2DgzKRRbMCtKK7g
--> ssh-ed25519 vvyRpw wQB8wg6bGvb68pvEp+7khrNpZTUxSVzLIfubbYsX+34
-KZ2/Vnxg7Gpazc26lYddjNnMxpoteb5ysuTZUg00ZvE
--> ssh-ed25519 aSEktQ KdKSZuVH/v+gkZkL07YdUJ5vvH2+mcUR4x+mXHylhys
-MRGd8l+0X6XVq1KpLqYqUZD/4EkOKz3mpHsdQepc6kc
--> ssh-ed25519 cD6JxA FesXIZs/X+fWefYjP0sfkwz6bYLxOkuIzQppwZYXNTU
-hg+ZTdCGuQ66FIc+NZI023Aunnhz+Ds5cFKUwNj+MGU
--> ssh-ed25519 1qYEfw HRQdZ4u1UWpzwIF/0lbJ1NVDQ+/Rl913jk+BwLM0KCE
-CHlDCaov7TWme5YMBiV6Tby0IReB8pER/RbDkpI3TWM
--> ssh-ed25519 2D+APA BTVVWo3G0tZj/hUMH5cwByYf3LjAg2RNVMhYrkXxXjQ
-iKghO+M6xpp95xVrmydz9GJJIOK5JrIsoL+CSFD77uM
--> ssh-ed25519 j2r2qQ RC/2vV5yr1af4iyeouQwIBK/r8b4nD51WwxgbuMEgG0
-L+uqV7eeCNqnMTqCNmvLPZFNTdmlYu/i7+3NVwmpIxA
--> ssh-ed25519 C/bBAQ KO1owoeb7pbuXtDS+f/TziotgffL0Eg6qnjJ9W8Yp2c
-af4IhSiXlMPiNuM473dIeWQqNbRgb3ciHyoa6buolyU
--> ssh-ed25519 K3b7BA h4mC/hZ10ToaaYDRyBOyPpcvA28sY5FPCQPuaTTRIws
-VG4QtmEOnubhhjV3CS49aYOyVl/Dq+ryxfZENgFJZTo
--> ssh-ed25519 +qVung 6gs9DdduYx2twVsFED7HJnGFfKZynUctQIO4F3MXfj8
-gMmU2tXwR9K8Nb5gMKPbTexE58FOAK6QlVYzGvaX3hw
+-> ssh-ed25519 +HUDfA SrjyocQ2U/mcmsVX3bhTDPiNfnRepZ+J//d4JkVrQ0w
+MELfJrKcLlC3rWKHdMZKZyXB0ztzmZUjWUcT8ibP8vE
+-> ssh-ed25519 87T2Ig IN9MMxRNzgKHBmGwidVWIvq2xpNVkbioWjG0lf+B5zM
+sXIXfrTak7E8isigDDnrzvjJli5ma5f9fOJnWCdDRpU
+-> ssh-ed25519 wIR2ZA 4DD/V3Xq1B2t8Zb11MnvtSZ3Oq5Glvka93g313dVSyU
+TrQiCJGOtitCCfNy0PdaRaPnk2mYCEPKtnOtdAzGolg
+-> ssh-ed25519 oGiV/Q W67zxBlGYg3PhUbwBiGE2vVoIl455R+4g3EClZKwulI
+2sldkyyBUGxhXRCoa/vW5LrxbI0TqerOeOqrTtzY3Mo
+-> ssh-ed25519 gO3aog YVF4hdjNYxOPE8v95BENIb6khsu0+tztaPNNCsXoWDE
+LLX/uofYt5/HQ7q5L35UK2t05rOlhCDnC4SIJx0bNtM
+-> ssh-ed25519 r/iJSw RMwg0xLCOVA+wc08f67kkUVIgy6W3Ypd3jRkRHFA+l4
+KR5RElZHGzzLU9hjr3Qg3NwudDxMtHqcf2t6xjDMz+U
+-> ssh-ed25519 N/+Clw BBYMWbIT8dXcD7SU+LrIuFeM+2RodGF2rW1ubx/W9mU
+yANEUWhFtNkx3VArOTTW+rREcxwzkN47CD2kK6JsMns
+-> ssh-ed25519 CtkSZw wy5ZfWI6tqN3OZDqRZvb6lhj8Pt+GrP3YryqhjH0ugo
+OtY/WsGkJJghGGAh4cfZOxkg/WcYJ4w2gu4Hu9VHntc
+-> ssh-ed25519 keg2lg lzE0HqDHBwDyuc5m5T9YSxxTgEk4mOQWY3l7a1+QKD0
+cn07YAocsIrSeWo1ZGyFzq3un8kdpEuS6zYpKs7G/iI
+-> ssh-ed25519 H885DA eZJW1T2VPMhDs/ygauDFdd1Md3D830ysel1yUZkZoSI
+wpq1+ndzQWUUN2yYMKnEZrOcgCuqKIrDjaeX+XpkQgk
+-> ssh-ed25519 Rq7K4Q CQ+Y2k5F8Q79GF5PQh8qDmxWgrKcqJHjAodVBqKqQkc
+SkcUl6dFoBQmPOOjTEopgcn5vzLH2oHICymAAS7nsAQ
+-> ssh-ed25519 vvyRpw nW2eCEqQ6uCT9RgIJyCSpP4JHwQtKDSiBBp1wdVFtTE
+DQcHIBTNqvFVYV1fXbGhu0pCwa++knjLpCVFC3npaS0
+-> ssh-ed25519 aSEktQ 7SEG8F8UyH0gR9uT+mFfBIXsAIUFnNd2bZgyJ8C/gVQ
+JTlr5eIhpepOoCxi54nrG7Wjxq9CXZYkb33kd2urdak
+-> ssh-ed25519 cD6JxA QKVkY0MS3LeJf+YfwJT2yysuseg8tSAEGHOBgHFsVkc
+IpAAWCWxHNg1MOBjG+JNXcTE/xNrDW8+5Cz/hNWVYvU
+-> ssh-ed25519 1qYEfw pA2G6CxFosIcXsBnTUfN1wsPs3Ue5aMzo7wameAacXM
+av7xGnRkh57JtgF37QtaF//eYS/pHqznHY4DJewRp5s
+-> ssh-ed25519 2D+APA SOSVjgiiugDWg9HeFIlaLa+mo3q8AHhntl1tHEB6QUQ
+QINZr847DASGM32Si6t1mHH6fCkKnq/sa1+3IXhaSlE
+-> ssh-ed25519 eTSU6g NuV8gm/Ijo6BpZptiYua2bnYNoxuHcOtce9zGNyi0yo
+E4zAIpZN5eTWJanPEwS7B6RfnnMRLDaOj+5l5L4GdCk
+-> ssh-ed25519 j2r2qQ PpKKKAJikQKWAaYvDhIoiPeTkWtE1chw8lCpZ4O+LHs
+4kR0ZNRMt0fljaOu3UgqVrUFnc6v916IyKdYkvz/zfA
+-> ssh-ed25519 C/bBAQ m7XsRBwlHgWXifCif/8H9TcSqs0so5hha2T4tCq6qn4
+QltQrR6Y3Im4xo8DtpzN5kMsHNfkpG0FE6Y2GnkrH5Y
+-> ssh-ed25519 K3b7BA x91SNkgN6NSlw2FZnliA+c6zoTYyeuZh2iT+Rl+qtT4
+nKU6GcX4WLTRncStiW6BS7iK7zlCVhn55FPjRNniqSc
+-> ssh-ed25519 +qVung opSEU5VaLZcm4GhcKlNtG/Ut0jU6oTYQuqvnDkuSGT4
+ny6Wfsi/PIj5A9q/fwL3vwnkft/yH6fqlPIXo0cklfY
 -> ssh-rsa krWCLQ
-vjNcmgDmmaNUSXIUgKf1digOgbohvyKkYSUalTOskvPo+9NRZbp0IJ7DoYLRrSBB
-DobCBM078iKOvIGGJCIbMS86/z/7lz6SSPcbfM1EG+hknVJLZaj+K3PYYSX6QTUC
-6rWSC+yg0gKehAhnYO3q+8mnismk7SERdyCZDNtPwHOhTAt6NZ6e+33VFxnbJPTz
-IvoNU/RTUhV+XuKbtosm55PqDkOuTM27jesZ0/SARYL+gVgaltacqt4kzbEMOP/W
-tv2kU6f1eNaX71c57DGI7rfcvLrPRAjTxUhsuKJPGQeaHtfiWz832gUMIJOEjoo0
-mvrAfyoykJRbPGNFl5pMmg
--> ssh-ed25519 /vwQcQ gpPktkJ57USbj7kn1qbeUQDbHHSCuzWM5OcmNooBMi8
-6JPXUJYQ1IjRVv90r1EJx3EUMDPmU9X1FK6j/6vT5hE
--> ssh-ed25519 0R97PA vzT774La7rcOMz7/KYjSUsY+D6V5bi5j3ghdDBLBoAU
-HAXfMmFuj3YJGCBR1U0btPlr9MdIBYnwT1ufbHaAxVk
---- /0DCLjy0dwjRGPnkNk/a9fZ1ox9+LVkwh9Y5jiyA8x4
-�1�K�B�|��\��ST��i��/���
h�9}%�\�Ÿ�,"g��Z��ʚw��0��5��
Rm�'
\ No newline at end of file
+p5Y5fVwyG2s7m9ClsgbcVz/fSF2lJvbXxuN8O4b6sp+QiABmSGs0R3pZuf1v9xBr
+Jc0JWhl4vvvb9F9WUbJR50hIpdWo6iX4vrz3TnSvPFmnpUpRfe+a29ZJhp0vCA4a
+HVaOJGlnGZ5BdSkvPslGVCPu684OmO/veL5G1H7xmN6yg2b3n7SaGF7A4+rpVqgI
+6GZiFpnM6LpyKyoTyXRL0ghzjhwggQCCnBaN7GIUhvPacPdilAJWmnagQzx8aZpT
+LRe1WAeKH2Lbar4UNeot3MzWkZxUXyyWszTMe1ca94N3jY7MG8adzX3guMykP5qA
+eya7UOphIwkQKlVB3N5bfQ
+-> ssh-ed25519 /vwQcQ xQFghc3LzwG82u+h80e3NdfbCh85OKdai32pwvS3uzs
+MdUPg9BHvPX85jWnV7evkNekPrzoJuT8FP0l/mhfZDk
+-> ssh-ed25519 0R97PA 8cDQRKrujysaUiD5OxdrpmWn7ZZCJ9SNbLYtWuTSmXg
+HFa/6WbK4aMK3cKEMEycyiclTu8jOcCMcr1R7Ebh73c
+--- wZAdkwtibHAVLCqtfmZ54ZtPwDPogkRwfKREBR2xOeY
+[�������ȹ���kR��\�ub��ГR�
+��q�૑�H�be{Y
g��m����s�Ƭ�[-p�
\ No newline at end of file
diff --git a/secrets/mimir-environment.age b/secrets/mimir-environment.age
index a7a14ad9f3a50efb8e318a336708428e820bd58d..c5787cf270537969c27a1e05fb229bfb769d0d6f 100644
GIT binary patch
literal 1127
zcmZY7&CA;a0LO7q1j#NFFXB8D@noq<ljiL*%zK)qN%PV+2~(fEH1BDeG)>cIXTghz
z;G^s?1b0$~j(POtVImAgc@z&D!bI6coG|es1HC+#`4@cn{rI}kYOq@S^<mQ_<@%Mf
zj=**Z3@-Mz+jy`FU=$say%etDO6$$jh9=Rx=+=FUCNi+KNNpVE1mx%Oz7XB6GDj10
zlS$AXF!u_>;?zM=#F^VJta(m`GFO?c7_4{DtO)BN%)&OL3f$w`1o3d1>e;0}X-tn|
zMUWaVIzg{JE4By_v(r^UZVv)Jb#!OLGFk>?3LO(&8gQT)Qhr7DeWUDBz2f#&I!TJ-
zqzvOsSV#P1^tjfq7Q1!SvrT<RV&l!!n=f2=<MMDxgv)9H$-cy{MVw)MY-!C@Fo}SD
z$!0oakJN(qz%)w(BxWUdrS9d-sX&}6$9P%u)Os3fXIi~=ds)3?WDVGAk-LXnm!KMH
zL`JNV&`_C6veBq0WL0%ZFBnsWT)b)MzJpoixRTQXa~wV#OBB2WR*IBobA5)D6jAXp
z1LiH(nJgb^d4xk0>!CB98@rB#8CE9=QkjrzOU+v)#&JhAMt~&chX@?5FhvxsAZMva
zSCIC2PxB*X*7AZ-MJ*WHUMK7ci;!r8orcQBK=Rs=;Sdo3R}kdTAVF0Omi&~kDK+Ly
z$(#52xjJj+ZQ%QTakl~$Mw`<$M9#;C4k{Xn9NQ?fgpUC;%ZM${ZDC$4aNomiS>_Mg
z5DhEO9Imz+x501_M^ly<fi1KMb5xxLt`loCQX^Av3lt_pNQl84CR?tsIU3sWQA3@*
zebz15?loWQjZ_v-N1I{?nJu)|Th`4Ww^Pzg^SP!}U2}<;<0)tiVGe+uev=iOAk4Ri
zWYwVZa7q?(x+IXI19ABv`3@%rMLlu<*Da)z_>Afut5zQg+XDS>d|$|SzJaq!Mk=rp
z!dZm}a3^;|*qN}_9O!$=!V48jZ`++?ia|1;qKqju<xn=*g>_gtv#vhWDH4r=!C=ti
zv1IOTqe!HK*!P_v5gEsuX-;~O4Yb;KT{ltkA(qF<7`QK<-nx4K+SQYnpSXPPXmBfd
z^v9F$_{Yz#?|uH<$DiD7p8osF`EQ>3uzmN%<!kXx<i;C!&I^~Y-!DHpP5Pg{@G34{
z|Ld2d+u7jzuTI~4aPzI--hck}D;I3+!KKfB?vL(Vd-IED++WXqH?Uv2e)8>~7r=Aq
h?T7#TbmJk1d^x#S{<!ylFh647KKa^?KR}Nz{tGN5fzki~

literal 1127
zcmZ9~%j??&0Kjp76A^<>hKkIKMM3x(YTmyj3<r6%Y0^BJG)dFo!6s=QZIiZnHOWO#
z4>BBh@!)h94^t2O2Lum0=@`gRkU89Gu!Hyj@gO+C(=NZi-~(U2i)h(jW`5r8%4D0p
zvCSi>Mu7gsUQxvTWq=X{Libi^j_&o|jD+$HHnvrmADuLc;2u`OaIkgr^ssjgN|U*j
zv!FzJ!5n~%G={4zwySpK7c_*6;(|9fvLWN#DsnhzW)z0Wjx{f^PHSV#BUz@GjtDHt
zW6g1}kS(2ATaux=ZabC_mAD;N>6(m6X5yt;0c5Jm3Bk^XB8-8{YRTB-*e>*`Cr%X@
z<$RRZe5K9_nWau^;NGsXsJB$Cq9Y|A?_B95HXAWReH(3OSjCLI&hg5P#Z4BS%9$eI
zFJaM4k!mjU8y&BSuBj2tj-ff+MvwuDpfP0U0)uUF|B;p#i{3h)ak37Kloi2+Q03^8
zsv7W3m?;A`s1PgBaszLcd}?h|Fo=_occMiZfh+}LNW_5NJX+%@3N&#Blfr0?7A7*+
zp{6~cDN{yU*NwF0Zl+M`Y|h&P;dmp8S{KC-UTa$(wB|Uh;`^Bi*w!dqRx*y*%~0lo
zniz2gDd>tB4e32TH4RI{z13Ei5{cZjNJUR{ax3tHhKz-kzrlbmQbBBkM($j(s+HLc
zDQUGJ%A(Hdc9WJ9K|Ja346&zfV1YTTz!^pdj<)8Zom5P$7T71K$R`YEo<vorXTD*l
za|EM{z1R&W(OjzVh__6h7If@{G9VKtsK8IxQjVY#)*KxGio;<#@ppm1T8J{;xdSkp
z>W^wa`WC7Wy07=rdXRNxP(gOQOUG(TqlJtWYcU){2B8Zz98AZELD1v$0EnBF6=>>U
zxz#asU+v@u6-qDX8*Mx!8#E&@HznjW%(um94K`?U!t}sok7z+Mri9w6u@U3S14mj#
zhiJ*Mqe4UDsAA*HQ`B}1$Q{#(YTE=YS`7!d=%j8iSx7A&*|_e3>;#({B;>$E0t5Yi
zzc-wUJ0)dznYYO+m!3NLK2=i{B`jV;7%T)?31q`<go-+F@7xc!?gHV<pZtDQ|9kG|
z-i_XWe@b@-&pdwng);f$D81r-bL+WxW?y{v)bUTheQ@vEcORZn?q5F>zyIsi*Z-Nu
zSGxWyM=yTmet+To{Nl&gAG>hprMEAiJ^#enzvd5qdG@k&^~+)S{LP1N-FO=Od~@kN
l`^`)5(hitZ-`;=nZ)E-2wU2IAcW)mb#}6L9`t_$j{ttQKd&>X-

diff --git a/secrets/netbox-environment.age b/secrets/netbox-environment.age
index 11e7c18823b5bf32c5c628764b4f6e624b32a3f6..19317d64d35c2a3c0ebb34eaedda5bc6553c6364 100644
GIT binary patch
literal 1226
zcmZ9~?d#M80KoAAC0JldB)(8M%NOPKYrAcC+ajdh9&V4f+wNhzGofvFx7%*F?RMMl
zwoFn8dLc@Y5K$SGLPQ{mRv={dricuSwCEQ>51<)_K@ZTIdgB}af)9N8n$Bo#6xeYd
zC2klz9L5fqRe-hKrNv^hHX6d!YK16ya177Y5|I^*#n8B$n6zun(s4}786(j($msJP
z;@6T%!Sq>CHM+o1hJ38=l2W>=XuU|M@v5e~i3~NxNgrts`#9aqO?sKB>aCiHSXiAb
zjkK5@GhV7w!a{_#k<_SbXxvQ)DU3O7CZ%Rl4KpUlR8s+)i@^ERhIltjS&NY~&1ZRH
zoL3Vn;o+ra!7a;@dk!UbZ`I24UK%*1X{=?D3G)?x+H&~?0ehk9BWaybrYn8KiIr|q
z>-Lr`&+&l7T4bvhQ%cgo&^F$5C9gtt`XEowR?9YI8kE%L{K?QRmN&I{Vg66(l%xXF
zC^SWMY$an|Q8-g+G%A&dv&_Jo>KrLKhS-cj17jJy%X4u%hi5V!_B(MQLR>gNNq|K<
zohgZEEM_T*&oxn)t}|WIb27RIr7X_sBec`#$et%yLMNQGs~PF%k(F_{iZB=o3}i{1
z`3Q*&Y=~TPEyx`AZ3nZRaV`-;)d;vSZzcQ!EEJ|zLj)ca>JxC5rbwKO!BqhmR9K%Y
z;h>_`>s*<uwiYB;9?C+J8bp8-m;=fRX|1qC=%@zA`x!4pO{Xho1185bX%66M&A~~d
ztw|C!gv5H0MXB7vLJKEkShohF2&q=5gcADWPN-M1y5H$C<(lM$Ri8#-7oeq}fay%f
z2N*tSkS4n{#;9|%_RVj>Y^mC6X;L<ag)8)2N^7``9oqaNi7alF<qI`{6fPYV0%;+7
zlSP5to{LJhB%76>LD|i@2^I~E>B_o{M_@({m$N(ov#z8E<gFT{5m<{X4Vtv2GqRYH
z5i}W6URBB)=^)WuSeW#np|>myY|%t2PM`v-fOZ>gIqsCrx=T(%O~wYIfLgUdJc_GY
z$5;KlWzR{W#{+9?Yb8|9tYI7VJ*C`ir!a_%LApqrNfm65+!k7l)Vv+VP%&k?K;v=m
zNqjFuf4%wj`@X>Ue_VU}+so|XAKy9r!|C_Gs_%Mcy|riOwL?39`(UH}&qoi~Zz=~5
z{CVKQwul06JauH2<R7kY3b)+$6E0qtt~3bi{s&Jv&#gA@*uMSw^WnKiuidrd@9Qt@
z6K>xzF7}?<_u}@FJs6M9?%6i^<)pFRdWd-M(hc$8)}MEjwbjw@u0H$8iTKs^W1rpl
z{pCkCeRJptadh{Y{nVZ7+})qn%STThkbLLzy~VMs#A{Dqtem^@<!ozji*n)knGLU;
wJh9=t{@CZg{;}&N_R>911g*yAvwvOLbo%?Z=u5kgYb>}=voHSp?j3LZ2hlpPYybcN

literal 1226
zcmZ9}+pF6I0Dy7O;Uz=Qm@<aq)`9N9+9heyv_%n`UeY#6b8C_|DB4_4PTJfrZC<7m
z*$xgF6XBpJA_LKhcVyxeWtRsx+#G{3(1AW|YzSkUmj`9|=EM05etdir%P1MH7B2kQ
z_S{`wXn`pVl%6g`(Q?Vq%asaTFRYZqa=;a80nN|dK2`CnbhT~8_6(#IZa9aON<}AO
ziBx%54?7OcJ3vrv$ek9NS*X$Od2G&(5o^W{;(98uX+466y<VAzCMf4M)@vl9R`Et5
zR$^adF|U=5%e8buW%-oKrYXpAlh#a4GM(kqsJwzK<}_freOA?}K24$YRQLMBEaJgB
zVtY(auOPiN5nP|kg4x&`uh)u!l(<&G950BdWw~ZKW6*(8%o)R$<pjwn1A^dr*HR|&
zD3x>!$^dd6)f2d_s4O^96WmClq;A$6Bb<=7ntD>J8;sLbRr<{TZDBMiIHBC6I6x=t
z6^BqYUk(OV7>$Jy(H>h=f1+Dv*`7&!OYv!+FAIZ!G;k~J@j_^494ClP*Nqh%@UvuE
z(Q~ElIU$VALu;ADu()aGxSXYCH(oTFjgHoX=8I<AEJI^`2K(wH6l|5O^DPwt`dMF3
zMK46EP&9NBd1B!?jTAN0H<(&?dEQP?EFKUr&z9S9+tGy)SA!~;RUL+c$i*Nq>LIm?
zPyyv631_%8RT624kEAFi#v;n#jovgI+lv`G^(F#C^l7n4)@X$`ts;^Y`H>8mf)*)|
zB)|raww#9DZ~Jk7YDA&mWf{W?irs~l&xwW)<<*(SK&GkSOi1Q9hgZ!=mw?EHc-SKS
zd3{kIVp1DS`smPF{}xO$PU8wQv8xO1(PY>Ro$??68P@Ya#ICK36q8a#Trn~W3$s)w
zIG~!%MRv*sjxri}SiT(Splfq2QISJZ$}L%TL{^i*#L%Pl8pPMDY@^^NbQy;sDPv>1
z*u)pAlH&xGdsTt1PrDY`py8&nf{HbJ3?Q+oVS^^vDo$C}sdOAHZpo?!S7vD21EU@)
z6T>Eg<Xl4oN~Kc4FawPUx=u8TW(3n=vStwywQ(#u#ZF=qG0JwGZYSqZh6DD!gZz5<
z!*f3fzpZ@n<jbwg?_WNCC-&{9{2KMk+Do(J8;iG(X<P3;uy#rOQa*83`1h^F^&N}v
zH=LSgCkjW`k|pHKpSZf}PQ9D=29Ml#?ac6}i#K*&eSOyl{u}7(yQ@cb+<NiUkLR~-
zx%9%BjTiSE+q(Pa!F|o&H|*&gIrQj;haWib-s<k1z|vEjdh^X+QSt)0vhDnhO-n(l
zZn-~t$iH^>)zN3vMNi%PL}~NS{nIx-CJvta{5$N*)myK9@(+A8dM-S9+uC1!?4E5G
vp8fml{lVLNFTC0KW4m<d@$1h&czEfRYtNiKeaAO3<^OfpcKf64m(2eF2GY7$


From e00d0331ec261c4789dd93d4bcc83f2e2ae7286e Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Wed, 17 Jul 2024 17:32:34 +0200
Subject: [PATCH 6/8] common/known-ssh-keys: init
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let's ensure that all our servers are aware of all host keys to avoid
host key verification issues when needed.

(example: buildbot → gerrit)

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
---
 common/base-server.nix    | 4 ++++
 common/known-ssh-keys.nix | 6 ++++++
 2 files changed, 10 insertions(+)
 create mode 100644 common/known-ssh-keys.nix

diff --git a/common/base-server.nix b/common/base-server.nix
index 3f5616e..c3f0251 100644
--- a/common/base-server.nix
+++ b/common/base-server.nix
@@ -1,4 +1,8 @@
 { lib, pkgs, ... }: {
+  imports = [
+    ./known-ssh-keys.nix
+  ];
+
   nixpkgs.overlays = import ../overlays;
 
   nix.package = lib.mkDefault pkgs.lix;
diff --git a/common/known-ssh-keys.nix b/common/known-ssh-keys.nix
new file mode 100644
index 0000000..9fca84d
--- /dev/null
+++ b/common/known-ssh-keys.nix
@@ -0,0 +1,6 @@
+{ ... }:
+{
+  programs.ssh.knownHosts = {
+    "[cl.forkos.org]:29418".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM82mJ259C8Nc+BHHNBeRWXWhL3dfirQhmFbDAwHMle3";
+  };
+}

From da7175303c5309da4645383c974acebed9d30c8b Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Wed, 17 Jul 2024 18:18:59 +0200
Subject: [PATCH 7/8] buildbot: add support for remote builders via baremetal
 machines

For now, only builder-3 is used.

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
---
 hosts/buildbot/default.nix              |   1 +
 secrets.nix                             |   1 +
 secrets/buildbot-remote-builder-key.age | Bin 0 -> 1417 bytes
 services/baremetal-builder/default.nix  |  14 ++++++++++-
 services/buildbot/default.nix           |  30 ++++++++++++++++++++++++
 5 files changed, 45 insertions(+), 1 deletion(-)
 create mode 100644 secrets/buildbot-remote-builder-key.age

diff --git a/hosts/buildbot/default.nix b/hosts/buildbot/default.nix
index a06689e..5748069 100755
--- a/hosts/buildbot/default.nix
+++ b/hosts/buildbot/default.nix
@@ -28,6 +28,7 @@
   bagel.services.buildbot = {
     enable = true;
     domain = "buildbot.forkos.org";
+    builders = [ "builder-3" ];
   };
 
   i18n.defaultLocale = "en_US.UTF-8";
diff --git a/secrets.nix b/secrets.nix
index 6bc7773..badeab9 100644
--- a/secrets.nix
+++ b/secrets.nix
@@ -21,6 +21,7 @@ let
     buildbot-service-key = [ machines.buildbot ];
     # Signing key for Buildbot's specific cache
     buildbot-signing-key = [ machines.buildbot ];
+    buildbot-remote-builder-key = [ machines.buildbot ];
 
     # These are the same password, but nginx wants it in htpasswd format
     metrics-push-htpasswd = [ machines.meta01 ];
diff --git a/secrets/buildbot-remote-builder-key.age b/secrets/buildbot-remote-builder-key.age
new file mode 100644
index 0000000000000000000000000000000000000000..f3bc083f6d09623a65c7955a72fc173ed49bfca5
GIT binary patch
literal 1417
zcmZ9~`;XHE0KoAB<l%r5$OVl^TtK6wjILX^b&8L6uXf$Kc3t-<B&K_{+q!Pu*0n1_
zlw1J83+P26$w`O=d>tsLgeVx494IKSi1EZ+f^rugCjz2`h>xH6g}>mFeDf8=f~}zB
z8l8HpR8yAJ8Y0++EVgH@&8B24<Q*;-<h3H6go6;Ro>CJJi@YMTy1Nx)2*Jq#WL67G
z!IT5@2Xa77fpiV@y1Aw$Qmd&>I0Z;ef3YS5e2L}Trk2eC0+dWcj9LN^Q*er@q>kLH
zk<P3KL#<{7$jH$a3%J^;NC!z~8(IjbtFjzN?Kmc2CWY0qa)S;A?H16WeVl4Uxwa6B
zQ<YS?s?x<O=gkQ&G}aJY^>R$9i*yTw#CtVBO=^m0<>EvdihE(0L3NR=E3KrIq&%u7
zNF~0^6p9H5lM%5Ar{hgaI*b+F-Ub0PWe#;fBw6+7xd?@7L@`T+%MP$zbwFkfjkC?W
zTB9jgD~*&d!dPNnXQZYvESB_neKn3SBeZF-g6d@*Kqv@R%5@}6;(5K04{2P7YY2pr
z)=PSr463v<Lj)|1HtZx*Ed{_qjo_<6)kqc-aS8{3xB!s4M?pw4p7X<DLeT28!=2>v
zm@Xygde+~NoLbmIOLST`Koo%tDqREvB{Ap<g-Kr#V-QLO%~mK;^f&69=+^LfnqmSp
z(v;1Z;N$URqsVwH2pN{5f<N6b6OE7`$P)rqaiDye)6#*8PH~h5@`fCitA%J2;qsJX
z#skS_2h3qS;c0p(3t+TeGVCnF3@f6nmaL}hy4{fNY^{L1nG#iSaP2fI+l>w^#oPHv
zQfkXNEbYNy##hQlYnE^#R{*kgJOO8za4is1m6#%<cWdAM7HsQGj<LG<R0^iELcT6|
zaHJ&}DCmnu$bf?dglaUPE1{~CVc}FXO<2rMERi8I!K9hWaAZmj0TDf_$GNZ*^c#VI
z!>=KLGl|Dw2Xn7xk9$2dYIR2{?oP|62Y9_x379ZXxOpm?p(P0~!<AOd%(ol_0R=e&
zv7`z<Jj}a|fG<&pI!RUHSTxp=#YmzIVt`VzvpR<6X`c_zS!_0&Rp#Ac&QDhqf#^VD
zP^r}GWeQIyUc94b?GA6y!*Mz#s2Q_jp>3Hd_0>atzm4}9XD)4B*SD(c_sg4O$0kaj
zU%Ku~o#`3-^v~D%KaLxbujd%E*DpH1z3a`(15cB#%Fzou*o7x{Pgp%X)Q=zL7p~g-
z*Nm4I{ZN=Y`sB>SD8dd;=y{RWBBKNSNA~SKxldeP8ywqtbnDg~{r}{rEk1SMmCW$V
zylV43-g)np@Xia<uC2V|`L<{1`nEa0bWVTx>VkA;!=kIt+#FxO;l_99e?y<#?%vsb
zwrkU&HT;g=Cm#8CPkHz)+nwFB_MF@?P#~7yS{hq_d|qMG+S^l`TVC^@5B=JIZOd)2
z`^Uh!kqwirgZJCE<JV8^x_aj7nzds?+i#9+Gb6nwH2L7<9`c}n_Y_c_y7PlJzD0<h
zy2!q}fBY;v_`t}G1uNzqI-x)OD17`Rw98gn`P`l>vuD0A3A0Y@nsVt{-;i}~V*F!7
zniVv^|4H11mf!wt{^7BAmcH+F6*nLI=G@#KPib}J`4#ZMGUdy~M`l!)96)-xzt5+~
z-te3rSr%JAe-m@#^7JLs7hRn3(5On!B#u4!^SX^6Z9K59_XKha?cTrtsq$Cw(n;0F
MH0$9*`@b0c9~WE}ZU6uP

literal 0
HcmV?d00001

diff --git a/services/baremetal-builder/default.nix b/services/baremetal-builder/default.nix
index af661fa..5ae0820 100644
--- a/services/baremetal-builder/default.nix
+++ b/services/baremetal-builder/default.nix
@@ -28,7 +28,19 @@ in
         "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAvUT9YBig9LQPHgypIBHQuC32XqDKxlFZ2CfgDi0ZKx"
       ];
     };
-    nix.settings.trusted-users = [ "builder" ];
+
+    users.users.buildbot = {
+      isSystemUser = true;
+      group = "nogroup";
+      home = "/var/empty";
+      shell = "/bin/sh";
+      openssh.authorizedKeys.keys = [
+        # Do not hardcode Buildbot's public key, selectively
+        # add the keys of the coordinators that require us.
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMnOLLX0vGTZbSJrUmF9ZFXt/NIId/MUrEpXmL2vxod"
+      ];
+    };
+    nix.settings.trusted-users = [ "builder" "buildbot" ];
 
 
     nixpkgs.hostPlatform = "x86_64-linux";
diff --git a/services/buildbot/default.nix b/services/buildbot/default.nix
index 970b288..5e2565e 100644
--- a/services/buildbot/default.nix
+++ b/services/buildbot/default.nix
@@ -8,6 +8,7 @@
 let
   cfg = config.bagel.services.buildbot;
   cfgGerrit = nodes.gerrit01.config.bagel.services.gerrit;
+  ssh-keys = import ../../common/ssh-keys.nix;
   inherit (lib) mkEnableOption mkOption mkIf types;
 in
 {
@@ -16,6 +17,12 @@ in
     domain = mkOption {
       type = types.str;
     };
+
+    builders = mkOption {
+      type = types.listOf types.str;
+      description = "List of builders to configure for Buildbot";
+      example = [ "builder-2" "builder-3" ];
+    };
   };
 
   config = mkIf cfg.enable {
@@ -25,6 +32,7 @@ in
     age.secrets.buildbot-workers.file = ../../secrets/buildbot-workers.age;
     age.secrets.buildbot-service-key.file = ../../secrets/buildbot-service-key.age;
     age.secrets.buildbot-signing-key.file = ../../secrets/buildbot-signing-key.age;
+    age.secrets.buildbot-remote-builder-key.file = ../../secrets/buildbot-remote-builder-key.age;
 
     services.nginx.virtualHosts.${cfg.domain} = {
       forceSSL = true;
@@ -94,6 +102,28 @@ in
       signingKeyFile = config.age.secrets.buildbot-signing-key.path;
     };
 
+    nix.distributedBuilds = true;
+    nix.buildMachines = map (n: {
+        hostName = nodes.${n}.config.networking.fqdn;
+        protocol = "ssh-ng";
+        # Follows Hydra.
+        maxJobs = 8;
+        sshKey = config.age.secrets.buildbot-remote-builder-key.path;
+        sshUser = "buildbot";
+        systems = [ "x86_64-linux" ];
+        supportedFeatures = nodes.${n}.config.nix.settings.system-features;
+        # TODO: fix it, see the Hydra file about it.
+        # IFD already exist in NixOS, so it's fine, I guess.
+        publicHostKey = builtins.readFile (pkgs.runCommandLocal "in-the-right-form" {
+          buildInputs = [
+            pkgs.coreutils
+          ];
+        } ''
+          echo -n '${ssh-keys.machines.${n}}' | base64 -w0 > $out
+        '');
+      }
+    ) cfg.builders;
+
     nix.settings.keep-derivations = true;
     nix.gc = {
       automatic = true;

From 4473717e9f2297bc9698826d47aad744180a3562 Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Wed, 17 Jul 2024 18:42:54 +0200
Subject: [PATCH 8/8] gerrit: introduce buildbot checks plugin

It's a modified version of @puck's Lix buildbot checks for
gerrit.lix.systems with a slight generalization in the configuration for
many repositories.

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
---
 services/buildbot/default.nix |   4 ++
 services/gerrit/checks.js     | 113 ++++++++++++++++++++++++++++++++++
 services/gerrit/default.nix   |  17 ++++-
 3 files changed, 133 insertions(+), 1 deletion(-)
 create mode 100644 services/gerrit/checks.js

diff --git a/services/buildbot/default.nix b/services/buildbot/default.nix
index 5e2565e..5b3187f 100644
--- a/services/buildbot/default.nix
+++ b/services/buildbot/default.nix
@@ -37,6 +37,10 @@ in
     services.nginx.virtualHosts.${cfg.domain} = {
       forceSSL = true;
       enableACME = true;
+      extraConfig = ''
+        add_header Access-Control-Allow-Credentials 'true' always;
+        add_header Access-Control-Allow-Origin 'https://cl.forkos.org' always;
+      '';
     };
 
     services.buildbot-nix.worker = {
diff --git a/services/gerrit/checks.js b/services/gerrit/checks.js
new file mode 100644
index 0000000..dd6a814
--- /dev/null
+++ b/services/gerrit/checks.js
@@ -0,0 +1,113 @@
+/* Inspired from the Lix setup.
+ * Original-Author: puckipedia
+ */
+Gerrit.install((plugin) => {
+  // TODO: can we just use `plugin.serverInfo().plugin` and control the settings over there.
+  const configuration = {
+    baseUri: @BASE_URI@,
+    supportedProjects: @SUPPORTED_PROJECTS@,
+  };
+
+  function makeBuildbotUri(suffix) {
+    return `${configuration.baseUri}/${suffix}`;
+  }
+
+  let builders = [];
+  let fetchBuilders = async () => {
+    if (builders.length > 0) return;
+      let data = await (await fetch(makeBuildbotUri(`api/v2/builders`), { credentials: 'include' })).json();
+      builders = data.builders;
+  };
+
+
+  let checksProvider;
+  checksProvider = {
+    async fetch({ repo, patchsetSha, changeNumber, patchsetNumber }, runBefore = false) {
+      if (!configuration.supportedProjects.includes(repo)) {
+        return { responseCode: 'OK' };
+      }
+
+      let num = changeNumber.toString(10);
+
+      let branch = `refs/changes/${num.substr(-2)}/${num}/${patchsetNumber}`;
+
+      let changeFetch = await fetch(makeBuildbotUri(`api/v2/changes?limit=1&order=-changeid&revision=${patchsetSha}&branch=${branch}`), { credentials: 'include' });
+      if (changeFetch.status == 400) {
+        if ((await changeFetch.json()).error === 'invalid origin' && !runBefore) {
+          return await checksProvider.fetch({ repo, patchsetSha, changeNumber, patchsetNumber }, true);
+        }
+
+        return { responseCode: 'OK' };
+      } else if (changeFetch.status === 403) {
+        return { responseCode: 'NOT_LOGGED_IN', loginCallback() {
+          window.open(configuration.baseUri);
+        } };
+      }
+
+      let changes = await changeFetch.json();
+      if (changes.meta.total === 0) {
+        return { responseCode: 'OK' };
+      }
+
+      let { changeid } = changes.changes[0];
+      let { builds } = await (await fetch(makeBuildbotUri(`api/v2/changes/${changeid}/builds?property=owners&property=workername`), { credentials: 'include' })).json();
+      await fetchBuilders();
+      let links = [];
+      let runs = [];
+      for (let build of builds) {
+        let name = `unknown builder ${build.builderid}`;
+        for (let builder of builders) {
+          if (builder.builderid === build.builderid) {
+            name = builder.name;
+            break;
+          }
+        }
+
+        if (name === `${repo}/nix-eval`) {
+            links.push({
+              url: makeBuildbotUri(`#/builders/${build.builderid}/builds/${build.number}`),
+              primary: true,
+              icon: 'external',
+            });
+        }
+
+        let checkrun = {
+          attempt: build.buildrequestid,
+          // FIXME: generalize this accordingly once auto-discovery is available.
+          checkName: name.replace(/^hydraJobs\./, ''),
+          externalId: build.buildrequestid.toString(),
+          status: build.complete ? 'COMPLETED' : (typeof build.started_at !== 'number' ? 'SCHEDULED' : 'RUNNING'),
+          checkLink: makeBuildbotUri(`#/builders/${build.builderid}/builds/${build.number}`),
+          labelName: 'Verified',
+          results: [],
+          links: [{
+            url: makeBuildbotUri(`#/builders/${build.builderid}/builds/${build.number}`),
+            primary: true,
+            icon: 'external',
+          }],
+        };
+
+        if (build.started_at !== null) {
+          checkrun.startedTimestamp = new Date(build.started_at * 1000);
+        }
+
+        if (build.complete_at !== null) {
+          checkrun.finishedTimestamp = new Date(build.complete_at * 1000);
+        }
+
+        if (build.results !== null) {
+          checkrun.results = [{
+            category: build.results < 2 ? 'SUCCESS' : 'ERROR',
+            summary: build.state_string,
+          }];
+        }
+
+        runs.push(checkrun);
+      }
+
+      return { responseCode: 'OK', runs, links };
+    }
+  };
+
+  plugin.checks().register(checksProvider);
+});
diff --git a/services/gerrit/default.nix b/services/gerrit/default.nix
index 3796dc1..61e7ca8 100644
--- a/services/gerrit/default.nix
+++ b/services/gerrit/default.nix
@@ -72,6 +72,21 @@ in
       plugins = with pkgs.gerritPlugins; [
         oauth
         metrics-reporter-prometheus
+        # Buildbot checks plugin (writeText because services.gerrit.plugins expects packages)
+        (pkgs.runCommand "checks.js" {
+          BASE_URI = builtins.toJSON "https://buildbot.forkos.org";
+          SUPPORTED_PROJECTS = builtins.toJSON [
+            "infra"
+            "nixpkgs"
+            "buildbot-test"
+          ];
+        }
+        ''
+          echo "configuring buildbot checks plugin for $BASE_URI with $SUPPORTED_PROJECTS project list"
+          substitute ${./checks.js} $out \
+            --replace-fail "@BASE_URI@" "$BASE_URI" \
+            --replace-fail "@SUPPORTED_PROJECTS@" "$SUPPORTED_PROJECTS"
+        '')
       ];
 
       package = pkgs.gerrit;
@@ -126,7 +141,7 @@ in
         # Other settings
         log.jsonLogging = true;
         log.textLogging = false;
-        sshd.advertisedAddress = "${cfg.canonicalDomain}:${cfg.port}";
+        sshd.advertisedAddress = "${cfg.canonicalDomain}:${toString cfg.port}";
         cache.web_sessions.maxAge = "3 months";
         plugins.allowRemoteAdmin = false;
         change.enableAttentionSet = true;