postgres: add mTLS support

New client certs can be minted via the provided script, which is meant
to be run on the postgres server (where the CA private key is
conveniently deployed).

secrets.nix

@@ -7,6 +7,7 @@ let
     hydra-s3-credentials = [ machines.bagel-box ];
     hydra-signing-priv = [ machines.bagel-box ];
     hydra-ssh-key-priv = [ machines.bagel-box ];
+
     netbox-environment = [ machines.meta01 ];
     mimir-environment = [ machines.meta01 ];
     mimir-webhook-url = [ machines.meta01 ];
@@ -29,6 +30,9 @@ let
     metrics-push-password = builtins.attrValues machines;
 
     ows-deploy-key = [ machines.gerrit01 ];
+
+    postgres-ca-priv = [ machines.bagel-box ];
+    postgres-tls-priv = [ machines.bagel-box ];
   };
 in
   builtins.listToAttrs (

secrets/postgres-ca-priv.age

@@ -0,0 +1,21 @@
+age-encryption.org/v1
+-> ssh-ed25519 +HUDfA 6hHADgvaridGcsj+P4VRcDoVp2AV2YujDL+qcoThhAU
+K8ByxbOspbAEIHEeiyWBS9ovXYygexeX3rDz3O3XJGY
+-> ssh-ed25519 K3b7BA FBpeG2yPrHaLzL+P6ijii50O9x+JWObHDNbFRbaS01k
+2IBxYuDS/ioAHy2LM/TDDSxQBIvdwNYb2w0klhnDvmE
+-> ssh-ed25519 +qVung 3SvoxonymVeGxJ/eohwArOLjs8mvOxDOt58ZJnYuvA8
+VCXPMrhOvCL+A04fnxwWznkD8kGHX5NdRj+N0AKI1bo
+-> ssh-rsa krWCLQ
+nRZZUmHY1FgYzbG7bwgH8dhThLxuavE2z78+mvVEsVz3LsAPlli3Ry1Ks9mnqWDA
+PWTpk99dg6aFZpFEkt47dS2qh2ooVGhonaA04xIzetdNAiOOvonUWBzpBLVQ+gVo
+VpixPPkFJij7FOUaeEVLbBbWlButCr21iW3UJWspcuta7ezc90MBTrasstahgi8u
+NtaACxZACZyW0t47BKvhNjcrU//dxQKiZeq1BeSeyHsD7iqiKsXSYYL0441HUq2I
+um9FZWHPYghPrMAkLGsRxsgRP/w0nHZdDt91vVkIrPtDFQr/kFUP6LUXaquOiD8z
+Z1eI3IWGDn6+Frtux6zMrA
+-> ssh-ed25519 /vwQcQ TWZs4l1Ka2dRBdgY0B4uOKP9sXxyq9Hav7Yu7zuWklY
+0gs/D7dwKOFCIVQ+UAHAi6SY786AAWfwfBpB86d84Sc
+-> ssh-ed25519 0R97PA 31zCzKOqK8ZNy8rwLm35uVlyQnyrbXBNw66BOs/Hln4
+yc/1lRbCOmEM8WVtnDdn+Tc9pGGDHy+L59RSs1qbr6g
+--- 8z5ZE3gPQIJ7Vo99Ys7wa0p5rNzwyHCJ3tL+FXx231A
+ûÂ=ÕØÐHaI,ãÜ#¨ý6)>.u×´»Fa=õ¢ãvwÎ×wÜ!òHHOt­cÖ‡!cÏý/%Dè‡×qÎ_ P2Ü
+ÉýLë´E»]Æ­K¿mÔ0ŠP<7F>æÏgg/<0F>K!‚OÝ¢7ªâIëç[<5B>(â@ži%†ëÔBWÀø¸þ
œ8T‡êU_Ül""»4ǹ#–Ÿúí"öÅ<}Ö|ÿ¬u‚×–WÉd÷nF¢Ø—².Qiö®)]}_O_øPÈ9%¡]ÃìÖ¹€Ï—b?yd°w9ˆ¯Œ.ç	Ì”«Ýë/܎⩹ƒ¡¶Žãî´ÃŸ¸ä=¿’ õ-Óºƒ<C2BA>3›

secrets/postgres-tls-priv.age

@@ -0,0 +1,21 @@
+age-encryption.org/v1
+-> ssh-ed25519 +HUDfA fp3At2qJ9LIC+rm21fuita6OrL38aWyV3wnBtLeLATc
+bxl+1d6ZmZpTKQyJxooO/Mf9fenWUrEQjmeKqgGYa4I
+-> ssh-ed25519 K3b7BA Zz9JP0JSNQvIA9/4uUw0+n7RmfnraUcGKmP/IhDdqgI
+ywsd9bjsrZ/9Q/fwQBktj77rvtPUiUSJvGGKybKWA1c
+-> ssh-ed25519 +qVung nByDqgLfajgIncdJ/+nQhLd3LoTmueUqvXmmZVc8gCg
+07Ox246AoHxlKlpoS/KP3us7ea9vRdWsiiPBdOHiU30
+-> ssh-rsa krWCLQ
+fnf6GcgenHfFTwnz4FHXgITnc5O/MKYSVS1I175Mjgm4q+AvcfRG+YI9t3T8+iYf
+1KOVMnb5r4Gi3h9rI7EYCpxt8iurAqikz+Ro1mAtzLun9cMwwyboDfxWSvX6iUKe
+SgYVPdAn3hBpsrV4yNXwkD9XwwrutcW658bzRD61+0DlYbJxP+ZUIB7ZZfxvPXkS
+Psauhp2WSSPo2SZ4TWryjaSUzwYuDWieL7ChnA45IozjJl2vzjKnlMJGlfN0dRJr
+UI3tDqp2XuXwTfuYmqhfK+KsFJn3sYnJoeURcmik/QAlqgFPlEiY7CAsrBsV6T51
+Ff+EyizcUbBsIWtFsDn7aQ
+-> ssh-ed25519 /vwQcQ kJ5JUOx7hWG3Y6zxu6Tf4xmqutCnSQBp5N0wT1Nv1jM
+08Q+bhCt0ZL9Ca+dlWOwkyWJQLiG9kzue3Dun+USK0I
+-> ssh-ed25519 0R97PA fZTdFXtIHRarTFpX90cw3Pxr417MkXjDg7/7R/DAMxc
+eGy2cfm5YuVZZ68zAJrG0xEhTGOfLWqO6w0G1wzUMPs
+--- yobB9cab+40Yd0CSbOu4F5B+BeNT5+C0QH5574pDbz0
+O‘GBÖOÎ&´Œ•|ÙåQòªêb,¬–\Þ}¯+ÓòÏÑëBˆm<CB86>3vãé†eÛNOg!i³q<C2B3>”Ñ_Au» oºšG˜?;QyðrÈ-Yxþ±ˆ7ä8µÈ&XPônâ.Ü<>„}Á²š×£5ŸSål¼|§`¨RûuOóADÔÜ|•vé-ÍA1ÚØ?ùøÜÝmi¥ÉÂ,pûÒ‘O3);ÔÏÁ8\NL.§>ö`Є=	^dhdn_Ž&ÂcŸ~RÂôîp˜øªâe“`#:úe ÛŒ&®‚~¼jÃõgt˜eu
+‹eµlO€ñÑ®YšŒ5Ñà›ìQ,דmâ	

services/postgres/ca.crt

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBiDCCAS6gAwIBAgIRAIX2H5zZS7SY7/uXfA59emswCgYIKoZIzj0EAwIwIjEg
+MB4GA1UEAxMXRm9ya09TIFBvc3RncmVzIFJvb3QgQ0EwHhcNMjQwODE2MDUyNzIx
+WhcNMzQwODE0MDUyNzIxWjAiMSAwHgYDVQQDExdGb3JrT1MgUG9zdGdyZXMgUm9v
+dCBDQTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABP7WPT1ooa4Irv1V58Bgqg7t
+S4hXymbps7CyFMwy1gILOazDqh1YmgacofWST1gf0qm9Uo4YgKtWyCdZndWjLqmj
+RTBDMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEBMB0GA1UdDgQW
+BBQy4WX/hUExQ/i1h7MvF6Ow2irNizAKBggqhkjOPQQDAgNIADBFAiBnpvX9a4N9
+f0pnG58f8GG/Yu91N2s0eESiPcMjzRB3vwIhAPo6YMFzNrB6IWxiUGtlOni1eY06
+iCsMoQ7B0zTfwBGW
+-----END CERTIFICATE-----

services/postgres/default.nix

@@ -10,6 +10,9 @@ in {
   };
 
   config = lib.mkIf cfg.enable {
+    age.secrets.postgresql-tls-priv.owner = "postgres";
+    age.secrets.postgresql-tls-priv.file = ../../secrets/postgres-tls-priv.age;
+
     systemd.tmpfiles.rules = [
       "d /var/db 0755 root root - -"
       "d /var/db/postgresql 0750 postgres postgres - -"
@@ -21,6 +24,8 @@ in {
       package = pkgs.postgresql_16;
       dataDir = dataDir;
 
+      enableTCPIP = true;
+
       # TODO: Where to put this to properly couple things? It doesn't belong
       # here, but using it in services/hydra would require running on
       # localhost. Probably needs to be replaced with some different way of
@@ -41,12 +46,60 @@ in {
         hydra-users postgres postgres
       '';
       authentication = ''
-        local hydra all ident map=hydra-users
+        local hydra all peer map=hydra-users
+
+        # Allow any connection over TLS with a valid client certificate: signed
+        # by our CA, and with username = cert CN.
+        hostssl all all all cert clientcert=verify-full
       '';
 
       settings = {
         max_connections = 500;
+
+        ssl = true;
+        ssl_ca_file = "${./ca.crt}";
+        ssl_cert_file = "${./server.crt}";
+        ssl_key_file = config.age.secrets.postgresql-tls-priv.path;
       };
     };
+
+    networking.firewall.allowedTCPPorts = [ config.services.postgresql.settings.port ];
+
+    # Provisioned on the server so that CA operations can be done there.
+    age.secrets.postgresql-ca-priv.owner = "postgres";
+    age.secrets.postgresql-ca-priv.file = ../../secrets/postgres-ca-priv.age;
+
+    users.users.postgres.packages = [
+      (pkgs.writeShellScriptBin "postgres-mint-new-client" ''
+        #! ${pkgs.runtimeShell}
+
+        set -eo pipefail
+
+        if [ $# -eq 0 ]; then
+          echo "usage: $0 <pg-username>" >&2
+          exit 1
+        fi
+
+        user=$1
+        step=${pkgs.lib.getExe pkgs.step-cli}
+
+        tmpdir=$(mktemp -d)
+        trap "rm -rf $tmpdir" EXIT
+
+        $step certificate create "$user" "$tmpdir/client.crt" "$tmpdir/client.key" \
+          --profile leaf \
+          --ca ${./ca.crt} \
+          --ca-key ${config.age.secrets.postgresql-ca-priv.path} \
+          --not-after 87660h \
+          --no-password --insecure
+
+        echo "Client certificate:"
+        cat "$tmpdir/client.crt"
+        echo
+
+        echo "Client private key:"
+        cat "$tmpdir/client.key"
+      '')
+    ];
   };
 }

services/postgres/server.crt

@@ -0,0 +1,12 @@
+-----BEGIN CERTIFICATE-----
+MIIB0DCCAXegAwIBAgIQTayv82V9dp0ptyMbDcN70zAKBggqhkjOPQQDAjAiMSAw
+HgYDVQQDExdGb3JrT1MgUG9zdGdyZXMgUm9vdCBDQTAeFw0yNDA4MTYwNTMwMjBa
+Fw0zNDA4MTYxNzMwMjBaMB4xHDAaBgNVBAMTE3Bvc3RncmVzLmZvcmtvcy5vcmcw
+WTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAR7St4dklWkvYbCi+3xPY5YFfBwgErz
+8TLtT5F2l5aKN0+I0sBo+ktiTNl8BzaVzXJmLa2xzRt2jQgB2R0IZgyko4GSMIGP
+MA4GA1UdDwEB/wQEAwIHgDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIw
+HQYDVR0OBBYEFCELKmGcbzVZf6Qx1cWCeXVC3XXJMB8GA1UdIwQYMBaAFDLhZf+F
+QTFD+LWHsy8Xo7DaKs2LMB4GA1UdEQQXMBWCE3Bvc3RncmVzLmZvcmtvcy5vcmcw
+CgYIKoZIzj0EAwIDRwAwRAIgJTM0nO2UaJnlGNIOJ1oT7HClNBxmH5oQu2DwVMuS
+MB8CIDbg/nrYjnRmFGGtbWvLbdvHIZ7t0A4kjkLwJ2QCr6Ni
+-----END CERTIFICATE-----