hydra: switch to using mTLS instead of local peer auth

New client certs can be minted via the provided script, which is meant
to be run on the postgres server (where the CA private key is
conveniently deployed).

hosts/bagel-box/default.nix

@@ -39,7 +39,6 @@
     postgres.enable = true;
 
     hydra.enable = true;
-    hydra.dbi = "dbi:Pg:dbname=hydra;user=hydra";
     # Takes 10 builders (0 → 9).
     hydra.builders = lib.genList (i: "builder-${builtins.toString i}") 10;
 

secrets.nix

@@ -4,9 +4,11 @@ let
   commonKeys = keys.users.delroth ++ keys.users.raito;
 
   secrets = with keys; {
+    hydra-postgres-key = [ machines.bagel-box ];
     hydra-s3-credentials = [ machines.bagel-box ];
     hydra-signing-priv = [ machines.bagel-box ];
     hydra-ssh-key-priv = [ machines.bagel-box ];
+
     netbox-environment = [ machines.meta01 ];
     mimir-environment = [ machines.meta01 ];
     mimir-webhook-url = [ machines.meta01 ];
@@ -29,6 +31,9 @@ let
     metrics-push-password = builtins.attrValues machines;
 
     ows-deploy-key = [ machines.gerrit01 ];
+
+    postgres-ca-priv = [ machines.bagel-box ];
+    postgres-tls-priv = [ machines.bagel-box ];
   };
 in
   builtins.listToAttrs (

secrets/hydra-postgres-key.age

@@ -0,0 +1,20 @@
+age-encryption.org/v1
+-> ssh-ed25519 +HUDfA PrYITxe8VIsLOsP3oP7jMtammV3emIQBvI4fid5CoxE
+rBD0meQCiOc/WBHtfWws24Byequp9vAZqzKQLXE6HsQ
+-> ssh-ed25519 K3b7BA LK7uLle6TYr2MAmOkGflQDAkO1vcbk+Fr61QbOEAvRU
+Nb8JX7v/kCIF64ItKYiACHefHRrJM583X8jRcF2jdjI
+-> ssh-ed25519 +qVung DiA/3kVlwDqmU++K13kV5lWhWofJHQyuPoc7UTBGTEI
+k9zuwi7WvZxxuX4SWMp/41xK4EcepJvk3iPCNplCSzY
+-> ssh-rsa krWCLQ
+WMNLx1M3XAQ4yVyudEU4GQg6nBUPvDbv+U8Br+D46KeI/FcYZPsdseieT/QVrNRB
+fkPnrNNgCWDcNaT6bgri8kGGzG3ytRJrUFPTCDlobKWKNZMGWSfEQB8xKb7GBwa1
+LwE237DiuSHiYcxNzwfza0ygi7RnIhGq42IPJkiNUtZvYyzs+d/eJ5KU4Z9/us1k
+kqi2Wku69y4QBsZRgZQaH1gkCaQbVfjhq10eTnxuWXLyvtX3E+1YRWSmLKKSNU1/
+wEMPuEEXyf9MYFa+oNDtDO0VgqAebepTjIOedEyyX+QVhPt3/VoLPPfTdFtPkAtd
+MeVQcLmlpRzgn+KENtS/Rg
+-> ssh-ed25519 /vwQcQ vI3LkaJKygKzAeacTOxQV12kqpBphZxd5t0Iz5tTb1U
++TR67BdaqzmUL0P23KvYR/zqbszup6yBw2WO13nwx80
+-> ssh-ed25519 0R97PA Hq35zNwe2YXd7BQL9rt8u9wPo8eCPUmPfgMjaE2OpjE
+b5X5G182EJ6sGCZlAa96GY2sRSF8YH3WGJjFekIqpFA
+--- 9kwuc/FdoWYejJjN2OqUL+BUll4yaChv8s2zv0Kclcw
+µòm¤fŒ‚,ýM¥ÌóÕ5`…•ÊwVÄ\Šž¶TÓ[ŸÿÕ^'b5ÙWþà€	z?ñ<>å_1ú)<29>®b²î'Ys¥äÙëQr<51>ƒ]\nWE³`y³™¦î,Øý7:ÖêHŠ[ÉZM—µá—<A Ò¿ÞKbÄTâLúÜ¢^üehÜØ,u#­ñÏ´úšÍõ
¡Mxȼá~@ªo|—ÍJÖ[dx]3¡/ièSÔ!¼ÆZ‘D™½GoÿBXXZÃq—(!ÞÁ0!˜šûWï2É‚Àf<œÙOØ*ˆãÄ;>)Ï@˜–E¯Ä<C2AF>õL=DÛÄ·.ë&½ÜmKEQÎfÊ'V ›ÌŠ‰p«lqƒ™

secrets/postgres-ca-priv.age

@@ -0,0 +1,21 @@
+age-encryption.org/v1
+-> ssh-ed25519 +HUDfA 6hHADgvaridGcsj+P4VRcDoVp2AV2YujDL+qcoThhAU
+K8ByxbOspbAEIHEeiyWBS9ovXYygexeX3rDz3O3XJGY
+-> ssh-ed25519 K3b7BA FBpeG2yPrHaLzL+P6ijii50O9x+JWObHDNbFRbaS01k
+2IBxYuDS/ioAHy2LM/TDDSxQBIvdwNYb2w0klhnDvmE
+-> ssh-ed25519 +qVung 3SvoxonymVeGxJ/eohwArOLjs8mvOxDOt58ZJnYuvA8
+VCXPMrhOvCL+A04fnxwWznkD8kGHX5NdRj+N0AKI1bo
+-> ssh-rsa krWCLQ
+nRZZUmHY1FgYzbG7bwgH8dhThLxuavE2z78+mvVEsVz3LsAPlli3Ry1Ks9mnqWDA
+PWTpk99dg6aFZpFEkt47dS2qh2ooVGhonaA04xIzetdNAiOOvonUWBzpBLVQ+gVo
+VpixPPkFJij7FOUaeEVLbBbWlButCr21iW3UJWspcuta7ezc90MBTrasstahgi8u
+NtaACxZACZyW0t47BKvhNjcrU//dxQKiZeq1BeSeyHsD7iqiKsXSYYL0441HUq2I
+um9FZWHPYghPrMAkLGsRxsgRP/w0nHZdDt91vVkIrPtDFQr/kFUP6LUXaquOiD8z
+Z1eI3IWGDn6+Frtux6zMrA
+-> ssh-ed25519 /vwQcQ TWZs4l1Ka2dRBdgY0B4uOKP9sXxyq9Hav7Yu7zuWklY
+0gs/D7dwKOFCIVQ+UAHAi6SY786AAWfwfBpB86d84Sc
+-> ssh-ed25519 0R97PA 31zCzKOqK8ZNy8rwLm35uVlyQnyrbXBNw66BOs/Hln4
+yc/1lRbCOmEM8WVtnDdn+Tc9pGGDHy+L59RSs1qbr6g
+--- 8z5ZE3gPQIJ7Vo99Ys7wa0p5rNzwyHCJ3tL+FXx231A
+ûÂ=ÕØÐHaI,ãÜ#¨ý6)>.u×´»Fa=õ¢ãvwÎ×wÜ!òHHOt­cÖ‡!cÏý/%Dè‡×qÎ_ P2Ü
+ÉýLë´E»]Æ­K¿mÔ0ŠP<7F>æÏgg/<0F>K!‚OÝ¢7ªâIëç[<5B>(â@ži%†ëÔBWÀø¸þ
œ8T‡êU_Ül""»4ǹ#–Ÿúí"öÅ<}Ö|ÿ¬u‚×–WÉd÷nF¢Ø—².Qiö®)]}_O_øPÈ9%¡]ÃìÖ¹€Ï—b?yd°w9ˆ¯Œ.ç	Ì”«Ýë/܎⩹ƒ¡¶Žãî´ÃŸ¸ä=¿’ õ-Óºƒ<C2BA>3›

secrets/postgres-tls-priv.age

@@ -0,0 +1,21 @@
+age-encryption.org/v1
+-> ssh-ed25519 +HUDfA fp3At2qJ9LIC+rm21fuita6OrL38aWyV3wnBtLeLATc
+bxl+1d6ZmZpTKQyJxooO/Mf9fenWUrEQjmeKqgGYa4I
+-> ssh-ed25519 K3b7BA Zz9JP0JSNQvIA9/4uUw0+n7RmfnraUcGKmP/IhDdqgI
+ywsd9bjsrZ/9Q/fwQBktj77rvtPUiUSJvGGKybKWA1c
+-> ssh-ed25519 +qVung nByDqgLfajgIncdJ/+nQhLd3LoTmueUqvXmmZVc8gCg
+07Ox246AoHxlKlpoS/KP3us7ea9vRdWsiiPBdOHiU30
+-> ssh-rsa krWCLQ
+fnf6GcgenHfFTwnz4FHXgITnc5O/MKYSVS1I175Mjgm4q+AvcfRG+YI9t3T8+iYf
+1KOVMnb5r4Gi3h9rI7EYCpxt8iurAqikz+Ro1mAtzLun9cMwwyboDfxWSvX6iUKe
+SgYVPdAn3hBpsrV4yNXwkD9XwwrutcW658bzRD61+0DlYbJxP+ZUIB7ZZfxvPXkS
+Psauhp2WSSPo2SZ4TWryjaSUzwYuDWieL7ChnA45IozjJl2vzjKnlMJGlfN0dRJr
+UI3tDqp2XuXwTfuYmqhfK+KsFJn3sYnJoeURcmik/QAlqgFPlEiY7CAsrBsV6T51
+Ff+EyizcUbBsIWtFsDn7aQ
+-> ssh-ed25519 /vwQcQ kJ5JUOx7hWG3Y6zxu6Tf4xmqutCnSQBp5N0wT1Nv1jM
+08Q+bhCt0ZL9Ca+dlWOwkyWJQLiG9kzue3Dun+USK0I
+-> ssh-ed25519 0R97PA fZTdFXtIHRarTFpX90cw3Pxr417MkXjDg7/7R/DAMxc
+eGy2cfm5YuVZZ68zAJrG0xEhTGOfLWqO6w0G1wzUMPs
+--- yobB9cab+40Yd0CSbOu4F5B+BeNT5+C0QH5574pDbz0
+O‘GBÖOÎ&´Œ•|ÙåQòªêb,¬–\Þ}¯+ÓòÏÑëBˆm<CB86>3vãé†eÛNOg!i³q<C2B3>”Ñ_Au» oºšG˜?;QyðrÈ-Yxþ±ˆ7ä8µÈ&XPônâ.Ü<>„}Á²š×£5ŸSål¼|§`¨RûuOóADÔÜ|•vé-ÍA1ÚØ?ùøÜÝmi¥ÉÂ,pûÒ‘O3);ÔÏÁ8\NL.§>ö`Є=	^dhdn_Ž&ÂcŸ~RÂôîp˜øªâe“`#:úe ÛŒ&®‚~¼jÃõgt˜eu
+‹eµlO€ñÑ®YšŒ5Ñà›ìQ,דmâ	

services/hydra/default.nix

@@ -9,7 +9,11 @@ let
 
   mkCacheSettings = settings: builtins.concatStringsSep "&" (
     lib.mapAttrsToList (k: v: "${k}=${v}") settings
-    );
+  );
+
+  mkPgConnString = options: builtins.concatStringsSep ";" (
+    lib.mapAttrsToList (k: v: "${k}=${v}") options
+  );
 
   # XXX: to support Nix's dumb public host key syntax (base64'd), this outputs
   # a string with shell-style command interpolations: $(...).
@@ -50,11 +54,6 @@ in {
   options.bagel.services.hydra = with lib; {
     enable = mkEnableOption "Hydra coordinator";
 
-    dbi = mkOption {
-      type = types.str;
-      description = "DBI connection string for the Hydra postgres database";
-    };
-
     builders = mkOption {
       type = types.listOf types.str;
       description = "List of builders to configure for Hydra";
@@ -69,6 +68,10 @@ in {
 
     age.secrets.hydra-s3-credentials.file = ../../secrets/hydra-s3-credentials.age;
 
+    age.secrets.hydra-postgres-key.group = "hydra";
+    age.secrets.hydra-postgres-key.mode = "0440";
+    age.secrets.hydra-postgres-key.file = ../../secrets/hydra-postgres-key.age;
+
     age.secrets.hydra-signing-priv.owner = "hydra-queue-runner";
     age.secrets.hydra-signing-priv.file = ../../secrets/hydra-signing-priv.age;
 
@@ -99,7 +102,16 @@ in {
 
       listenHost = "localhost";
       port = port;
-      dbi = cfg.dbi;
+
+      dbi = "dbi:Pg:${mkPgConnString {
+        host = "postgres.forkos.org";
+        dbname = "hydra";
+        user = "hydra";
+        sslmode = "verify-full";
+        sslcert = "${./postgres.crt}";
+        sslkey = config.age.secrets.hydra-postgres-key.path;
+        sslrootcert = "${../postgres/ca.crt}";
+      }}";
 
       hydraURL = "https://hydra.forkos.org";
       useSubstitutes = false;

services/hydra/postgres.crt

@@ -0,0 +1,12 @@
+-----BEGIN CERTIFICATE-----
+MIIBtDCCAVugAwIBAgIQTU55o4gtG8EFZArwsuj4NjAKBggqhkjOPQQDAjAiMSAw
+HgYDVQQDExdGb3JrT1MgUG9zdGdyZXMgUm9vdCBDQTAeFw0yNDA4MTYwNTU0MTda
+Fw0zNDA4MTYxNzU0MTdaMBAxDjAMBgNVBAMTBWh5ZHJhMFkwEwYHKoZIzj0CAQYI
+KoZIzj0DAQcDQgAEnTgiFZOXBrcPlWDxJPXUFgxIi7/T7LmwLtpGPK/G6R8KA9cS
+4UXF5Ifz2dCgozTlhqLROKb81yhNsSy1tOcFyKOBhDCBgTAOBgNVHQ8BAf8EBAMC
+B4AwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMB0GA1UdDgQWBBQGh6HM
+jl41qw/F0vpBdmOWQ2IfGzAfBgNVHSMEGDAWgBQy4WX/hUExQ/i1h7MvF6Ow2irN
+izAQBgNVHREECTAHggVoeWRyYTAKBggqhkjOPQQDAgNHADBEAiAEypqfyMOGbEJv
+dKI1tyj890uq5Osr5+9wxGBvJDMJNwIgefyOdFcvJTzbfHgLmORpBOVtnpbkwj5y
+rMnjT8gYjEA=
+-----END CERTIFICATE-----

services/postgres/ca.crt

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBiDCCAS6gAwIBAgIRAIX2H5zZS7SY7/uXfA59emswCgYIKoZIzj0EAwIwIjEg
+MB4GA1UEAxMXRm9ya09TIFBvc3RncmVzIFJvb3QgQ0EwHhcNMjQwODE2MDUyNzIx
+WhcNMzQwODE0MDUyNzIxWjAiMSAwHgYDVQQDExdGb3JrT1MgUG9zdGdyZXMgUm9v
+dCBDQTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABP7WPT1ooa4Irv1V58Bgqg7t
+S4hXymbps7CyFMwy1gILOazDqh1YmgacofWST1gf0qm9Uo4YgKtWyCdZndWjLqmj
+RTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEBMB0GA1UdDgQW
+BBQy4WX/hUExQ/i1h7MvF6Ow2irNizAKBggqhkjOPQQDAgNIADBFAiBnpvX9a4N9
+f0pnG58f8GG/Yu91N2s0eESiPcMjzRB3vwIhAPo6YMFzNrB6IWxiUGtlOni1eY06
+iCsMoQ7B0zTfwBGW
+-----END CERTIFICATE-----

services/postgres/default.nix

@@ -10,6 +10,9 @@ in {
   };
 
   config = lib.mkIf cfg.enable {
+    age.secrets.postgresql-tls-priv.owner = "postgres";
+    age.secrets.postgresql-tls-priv.file = ../../secrets/postgres-tls-priv.age;
+
     systemd.tmpfiles.rules = [
       "d /var/db 0755 root root - -"
       "d /var/db/postgresql 0750 postgres postgres - -"
@@ -21,6 +24,8 @@ in {
       package = pkgs.postgresql_16;
       dataDir = dataDir;
 
+      enableTCPIP = true;
+
       # TODO: Where to put this to properly couple things? It doesn't belong
       # here, but using it in services/hydra would require running on
       # localhost. Probably needs to be replaced with some different way of
@@ -41,12 +46,60 @@ in {
         hydra-users postgres postgres
       '';
       authentication = ''
-        local hydra all ident map=hydra-users
+        local hydra all peer map=hydra-users
+
+        # Allow any connection over TLS with a valid client certificate: signed
+        # by our CA, and with username = cert CN.
+        hostssl all all all cert clientcert=verify-full
       '';
 
       settings = {
         max_connections = 500;
+
+        ssl = true;
+        ssl_ca_file = "${./ca.crt}";
+        ssl_cert_file = "${./server.crt}";
+        ssl_key_file = config.age.secrets.postgresql-tls-priv.path;
       };
     };
+
+    networking.firewall.allowedTCPPorts = [ config.services.postgresql.settings.port ];
+
+    # Provisioned on the server so that CA operations can be done there.
+    age.secrets.postgresql-ca-priv.owner = "postgres";
+    age.secrets.postgresql-ca-priv.file = ../../secrets/postgres-ca-priv.age;
+
+    users.users.postgres.packages = [
+      (pkgs.writeShellScriptBin "postgres-mint-new-client" ''
+        #! ${pkgs.runtimeShell}
+
+        set -eo pipefail
+
+        if [ $# -eq 0 ]; then
+          echo "usage: $0 <pg-username>" >&2
+          exit 1
+        fi
+
+        user=$1
+        step=${pkgs.lib.getExe pkgs.step-cli}
+
+        tmpdir=$(mktemp -d)
+        trap "rm -rf $tmpdir" EXIT
+
+        $step certificate create "$user" "$tmpdir/client.crt" "$tmpdir/client.key" \
+          --profile leaf \
+          --ca ${./ca.crt} \
+          --ca-key ${config.age.secrets.postgresql-ca-priv.path} \
+          --not-after 87660h \
+          --no-password --insecure
+
+        echo "Client certificate:"
+        cat "$tmpdir/client.crt"
+        echo
+
+        echo "Client private key:"
+        cat "$tmpdir/client.key"
+      '')
+    ];
   };
 }

services/postgres/server.crt

@@ -0,0 +1,12 @@
+-----BEGIN CERTIFICATE-----
+MIIB0DCCAXegAwIBAgIQTayv82V9dp0ptyMbDcN70zAKBggqhkjOPQQDAjAiMSAw
+HgYDVQQDExdGb3JrT1MgUG9zdGdyZXMgUm9vdCBDQTAeFw0yNDA4MTYwNTMwMjBa
+Fw0zNDA4MTYxNzMwMjBaMB4xHDAaBgNVBAMTE3Bvc3RncmVzLmZvcmtvcy5vcmcw
+WTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAR7St4dklWkvYbCi+3xPY5YFfBwgErz
+8TLtT5F2l5aKN0+I0sBo+ktiTNl8BzaVzXJmLa2xzRt2jQgB2R0IZgyko4GSMIGP
+MA4GA1UdDwEB/wQEAwIHgDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIw
+HQYDVR0OBBYEFCELKmGcbzVZf6Qx1cWCeXVC3XXJMB8GA1UdIwQYMBaAFDLhZf+F
+QTFD+LWHsy8Xo7DaKs2LMB4GA1UdEQQXMBWCE3Bvc3RncmVzLmZvcmtvcy5vcmcw
+CgYIKoZIzj0EAwIDRwAwRAIgJTM0nO2UaJnlGNIOJ1oT7HClNBxmH5oQu2DwVMuS
+MB8CIDbg/nrYjnRmFGGtbWvLbdvHIZ7t0A4kjkLwJ2QCr6Ni
+-----END CERTIFICATE-----

terraform/gandi.nix

@@ -84,6 +84,7 @@ in
       (record "alerts" 3600 "CNAME" ["meta01.infra.p"])
       (record "buildbot" 3600 "CNAME" ["buildbot.infra.p"])
       (record "b" 3600 "CNAME" ["public01.infra.p"])
+      (record "postgres" 3600 "CNAME" ["bagel-box.infra.p"])
 
       # S3 in delroth's basement
       (record "cache" 3600 "CNAME" ["smol.delroth.net."])