feat(services/ofborg): add client mTLS for RabbitMQ via Vault

This adds the counterpart of the mTLS for RabbitMQ connections.

This required:

- an update in OfBorg
- some trick to have a PKCS#12 container
- move to a binary-specific runtime directory

And this is not even done due to OfBorg sending auth mechanism = PLAIN
instead of EXTERNAL. I did not figure out yet how to send the right
thing.

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>

flake.lock

@@ -723,16 +723,17 @@
     "ofborg": {
       "flake": false,
       "locked": {
-        "lastModified": 1734308727,
+        "lastModified": 1735749516,
-        "narHash": "sha256-/bJhMZQ5VSblvgqAR9hSLwdm5pxenn/UMY8pDDVSquI=",
+        "narHash": "sha256-QHI1DBDqDta447DE7zIc+HgKJ3mzmsP3G1kFjbe8mG4=",
         "ref": "refs/heads/vcs-generalization",
-        "rev": "7bcc8fa584c66f317923337658974c0525e5779f",
+        "rev": "c741eb2dccee03b804bd7eaa36dfa6d3bd5e5308",
-        "revCount": 1495,
+        "revCount": 1500,
         "type": "git",
         "url": "https://git.lix.systems/the-distro/ofborg.git"
       },
       "original": {
         "ref": "refs/heads/vcs-generalization",
+        "rev": "c741eb2dccee03b804bd7eaa36dfa6d3bd5e5308",
         "type": "git",
         "url": "https://git.lix.systems/the-distro/ofborg.git"
       }

flake.nix

@@ -28,7 +28,8 @@
     nix-forgejo.url = "git+https://git.lix.systems/the-distro/nix-forgejo.git";
     nix-forgejo.flake = false;
 
-    ofborg.url = "git+https://git.lix.systems/the-distro/ofborg.git?ref=refs/heads/vcs-generalization";
+    # This revision contains mTLS support.
+    ofborg.url = "git+https://git.lix.systems/the-distro/ofborg.git?ref=refs/heads/vcs-generalization&rev=c741eb2dccee03b804bd7eaa36dfa6d3bd5e5308";
     ofborg.flake = false;
 
     gerrit-dashboard.url = "git+https://git.lix.systems/the-distro/gerrit-monitoring.git";

services/ofborg/default.nix

@@ -9,21 +9,41 @@ let
   generators = pkgs.formats.json { };
   configFile = generators.generate "ofborg-config.json" config.bagel.services.ofborg.settings;
   mkOfborgWorker = binaryName: extra: extra // {
+    vault = {
+      template = ''
+        {{ with pkiCert "${cfg.pki.rootPath}/issue/rabbitmq-client" "common_name=ofborg-${binaryName}" }}
+        {{ scratch.MapSet "secrets" "client.pem" .Data.Cert }}
+        {{ scratch.MapSet "secrets" "client.key" .Data.Key }}
+        {{ end }}
+        {{ scratch.Get "secrets" | explodeMap | toJSON }}
+      '';
+
+      secrets = {
+        "client.key" = {};
+        "client.pem" = {};
+      };
+    };
     wantedBy = [ "multi-user.target" ];
     description = "ofborg CI service - ${binaryName} worker";
     after = [ "rabbitmq.service" ];
+    path = [ pkgs.openssl ];
     serviceConfig = {
       DynamicUser = true;
+      # TODO: https://github.com/amqp-rs/tcp-stream/pull/7
+      ExecStartPre = pkgs.writeShellScript "convert-private-key-into-pkcs12.sh" ''
+        openssl pkcs12 -export -inkey $CREDENTIALS_DIRECTORY/client.key -in $CREDENTIALS_DIRECTORY/client.pem -out $RUNTIME_DIRECTORY/client.pfx -password pass:""
+        echo "Converted mTLS certificates to PKCS#12 format"
+      '';
       ExecStart = "${cfg.package}/bin/${binaryName} ${configFile}";
       # TODO: more hardening.
       StateDirectory = "ofborg";
       LogsDirectory = "ofborg";
-      RuntimeDirectory = "ofborg";
+      RuntimeDirectory = "ofborg-${binaryName}";
       WorkingDirectory = "/var/lib/ofborg";
       LoadCredential =
         optional (hasAttr "gerrit-event-listener-ssh-key" config.age.secrets) "gerrit-ssh-key:${config.age.secrets.gerrit-event-listener-ssh-key.path}";
       Environment = [
-        "XDG_STATE_HOME=/run/ofborg"
+        "XDG_STATE_HOME=/run/ofborg-${binaryName}"
       ];
     };
   };
@@ -68,8 +88,12 @@ in {
           ssl = true;
           host = "amqp.forkos.org";
           virtualhost = "/";
+          # TODO: remove.
           username = "ofborg";
-          password_file = "$CREDENTIALS_DIRECTORY/rabbitmq-password";
+          ssl_cacert_file = cfg.pki.cacertFile;
+          ssl_client_key_file = "$RUNTIME_DIRECTORY/client.pfx";
+          # Username is extracted from the CN of our certificate.
+          auth_mechanism = "external";
         };
         feedback.full_logs = lib.mkDefault true;
         log_storage.path = lib.mkDefault "/var/log/ofborg";
@@ -126,6 +150,7 @@ in {
       };
       services.rabbitmq = {
         enable = true;
+        plugins = [ "rabbitmq_auth_mechanism_ssl" ];
         configItems = {
           "listeners.tcp" = "none";
           "listeners.ssl.default" = builtins.toString amqpPort;

terraform/vault/default.nix

@@ -75,13 +75,15 @@
           };
 
           roles = {
-            ci = {
+            # We make clients expire very quickly so we can disconnect any OfBorg node fairly quickly by revoking their Vault token.
-              ttl = "6h";
+            rabbitmq-client = {
-              max_ttl = "15d";
+              ttl = "24h";
-              allowed_domains = [ "*.ofborg.infra.forkos" ];
+              max_ttl = "1d";
-              allow_subdomains = true;
+              allowed_domains = [ "ofborg-*" ];
               allow_glob_domains = true;
+              allow_subdomains = false;
               allow_wildcard_certificates = false;
+              allow_bare_domains = false;
               ou = [ "Floral Systems Continuous Integration Systems" ];
             };