feat: multi-tenant secrets

Lix may have its own secrets and we want to maintain a certain
generalization level on the NixOS modules, so we can decorrelate which
secret we select dynamically by having a simple tenancy hierarchy
system.

This unfortunately requires to rewrite all call sites with a floral
prefix until we migrate them to the simple internal secret module which
is aware of this.

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>

common/default.nix

@@ -9,5 +9,6 @@
     ./sysadmin
     ./hardware
     ./zsh.nix
+    ./secrets.nix
   ];
 }

common/secrets.nix

@@ -0,0 +1,22 @@
+## This is a simple secret abstraction with multi-tenancy awareness.
+{ config, lib, ... }:
+let
+  cfg = config.bagel.secrets;
+  inherit (lib) mkOption types genAttrs;
+in
+{
+  options.bagel.secrets = {
+    tenant = mkOption {
+      type = types.enum [ "lix" "floral" ];
+    };
+
+    files = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+    };
+  };
+
+  config.age.secrets = genAttrs cfg.files (secretFile: {
+      file = ../secrets/${cfg.tenant}/${secretFile}.age;
+    });
+}

hosts/gerrit01/default.nix

@@ -39,7 +39,7 @@
   };
 
   age.secrets.ows-deploy-key = {
-    file = ../../secrets/ows-deploy-key.age;
+    file = ../../secrets/floral/ows-deploy-key.age;
     mode = "0600";
     owner = "git";
     group = "git";

secrets.nix

@@ -1,52 +1,66 @@
 let
   keys = import common/ssh-keys.nix;
 
-  commonKeys = keys.users.delroth ++ keys.users.raito;
+  commonKeys = {
+    global = [ keys.users.raito ];
+    lix = [ keys.users.jade ];
+    floral = [ keys.users.delroth ];
+  };
 
   secrets = with keys; {
-    hydra-postgres-key = [ machines.build-coord ];
-    hydra-s3-credentials = [ machines.build-coord ];
-    hydra-signing-priv = [ machines.build-coord ];
-    hydra-ssh-key-priv = [ machines.build-coord ];
+    floral = {
+      hydra-postgres-key = [ machines.build-coord ];
+      hydra-s3-credentials = [ machines.build-coord ];
+      hydra-signing-priv = [ machines.build-coord ];
+      hydra-ssh-key-priv = [ machines.build-coord ];
 
-    netbox-environment = [ machines.meta01 ];
-    mimir-environment = [ machines.meta01 ];
-    mimir-webhook-url = [ machines.meta01 ];
-    grafana-oauth-secret = [ machines.meta01 ];
-    loki-environment = [ machines.meta01 ];
-    gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
-    pyroscope-secrets = [ machines.meta01 ];
-    tempo-environment = [ machines.meta01 ];
+      netbox-environment = [ machines.meta01 ];
+      mimir-environment = [ machines.meta01 ];
+      mimir-webhook-url = [ machines.meta01 ];
+      grafana-oauth-secret = [ machines.meta01 ];
+      loki-environment = [ machines.meta01 ];
+      gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
+      pyroscope-secrets = [ machines.meta01 ];
+      tempo-environment = [ machines.meta01 ];
 
-    buildbot-worker-password = [ machines.buildbot ];
-    buildbot-oauth-secret = [ machines.buildbot ];
-    buildbot-workers = [ machines.buildbot ];
-    # Private SSH key to Gerrit
-    # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
-    buildbot-service-key = [ machines.buildbot ];
-    # Signing key for Buildbot's specific cache
-    buildbot-signing-key = [ machines.buildbot ];
-    buildbot-remote-builder-key = [ machines.buildbot ];
+      buildbot-worker-password = [ machines.buildbot ];
+      buildbot-oauth-secret = [ machines.buildbot ];
+      buildbot-workers = [ machines.buildbot ];
+      # Private SSH key to Gerrit
+      # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
+      buildbot-service-key = [ machines.buildbot ];
+      # Signing key for Buildbot's specific cache
+      buildbot-signing-key = [ machines.buildbot ];
+      buildbot-remote-builder-key = [ machines.buildbot ];
 
-    # These are the same password, but nginx wants it in htpasswd format
-    metrics-push-htpasswd = [ machines.meta01 ];
-    metrics-push-password = builtins.attrValues machines;
+      # These are the same password, but nginx wants it in htpasswd format
+      metrics-push-htpasswd = [ machines.meta01 ];
+      # Yes, even Lix machines are included in this monitoring infrastructure.
+      metrics-push-password = builtins.attrValues machines;
 
-    ows-deploy-key = [ machines.gerrit01 ];
-    s3-channel-staging-keys = [ machines.gerrit01 ];
-    s3-channel-keys = [ machines.gerrit01 ];
+      ows-deploy-key = [ machines.gerrit01 ];
+      s3-channel-staging-keys = [ machines.gerrit01 ];
+      s3-channel-keys = [ machines.gerrit01 ];
 
-    postgres-ca-priv = [ machines.bagel-box ];
-    postgres-tls-priv = [ machines.bagel-box ];
+      postgres-ca-priv = [ machines.bagel-box ];
+      postgres-tls-priv = [ machines.bagel-box ];
 
-    newsletter-secrets = [ machines.public01 ];
-    s3-revproxy-api-keys = [ machines.public01 ];
-    stateless-uptime-kuma-password = [ machines.public01 ];
+      newsletter-secrets = [ machines.public01 ];
+      s3-revproxy-api-keys = [ machines.public01 ];
+      stateless-uptime-kuma-password = [ machines.public01 ];
+    };
+
+    lix = {
+      buildbot-remote-builder-key = [ machines.buildbot-lix ];
+    };
   };
+
+  mkSecretListFor = tenant:
+    map (secretName: {
+      name = "secrets/${tenant}/${secretName}.age";
+      value.publicKeys = secrets.${tenant}."${secretName}" ++ commonKeys.${tenant};
+    }) (builtins.attrNames secrets.${tenant});
 in
   builtins.listToAttrs (
-    map (secretName: {
-      name = "secrets/${secretName}.age";
-      value.publicKeys = secrets."${secretName}" ++ commonKeys;
-    }) (builtins.attrNames secrets)
+    (mkSecretListFor "floral") ++ (mkSecretListFor "lix")
   )

services/gerrit/default.nix

@@ -46,7 +46,7 @@ in
 
   config = mkIf cfg.enable {
     networking.firewall.allowedTCPPorts = [ cfg.port ];
-    age.secrets.alloy-push-password.file = ../../secrets/metrics-push-password.age;
+    age.secrets.alloy-push-password.file = ../../secrets/floral/metrics-push-password.age;
 
     environment.systemPackages = [ jdk
       pkgs.git

services/hydra/default.nix

@@ -66,17 +66,19 @@ in {
     # does indeed have our public SSH key and are *builders*
     # as a simple evaluation preflight check.
 
-    age.secrets.hydra-s3-credentials.file = ../../secrets/hydra-s3-credentials.age;
+    bagel.secrets.files = [
+      "hydra-s3-credentials"
+      "hydra-postgres-key"
+      "hydra-signing-priv"
+      "hydra-ssh-key-priv"
+    ];
 
     age.secrets.hydra-postgres-key.group = "hydra";
     age.secrets.hydra-postgres-key.mode = "0440";
-    age.secrets.hydra-postgres-key.file = ../../secrets/hydra-postgres-key.age;
 
     age.secrets.hydra-signing-priv.owner = "hydra-queue-runner";
-    age.secrets.hydra-signing-priv.file = ../../secrets/hydra-signing-priv.age;
 
     age.secrets.hydra-ssh-key-priv.owner = "hydra-queue-runner";
-    age.secrets.hydra-ssh-key-priv.file = ../../secrets/hydra-ssh-key-priv.age;
 
     systemd.tmpfiles.rules = [
       "d /var/cache/hydra 0755 hydra hydra -  -"

services/monitoring/agent.nix

@@ -59,7 +59,7 @@ in
   };
 
   config = mkIf cfg.enable {
-    age.secrets.grafana-agent-password.file = ../../secrets/metrics-push-password.age;
+    age.secrets.grafana-agent-password.file = ../../secrets/floral/metrics-push-password.age;
 
     services.grafana-agent = {
       enable = true;

services/monitoring/pyroscope/default.nix

@@ -15,7 +15,7 @@ in
   ];
 
   config = mkIf cfg.enable {
-    age.secrets.pyroscope-secrets.file = ../../../secrets/pyroscope-secrets.age;
+    age.secrets.pyroscope-secrets.file = ../../../secrets/floral/pyroscope-secrets.age;
     services.nginx = {
       upstreams.pyroscope = {
         servers."127.0.0.1:${toString pyroscopePort}" = {};

services/newsletter/default.nix

@@ -14,7 +14,7 @@ in
   };
 
   config = mkIf cfg.enable {
-    age.secrets.newsletter-secrets.file = ../../secrets/newsletter-secrets.age;
+    age.secrets.newsletter-secrets.file = ../../secrets/floral/newsletter-secrets.age;
     services.listmonk = {
       enable = true;
       secretFile = config.age.secrets.newsletter-secrets.path;

services/postgres/default.nix

@@ -11,7 +11,7 @@ in {
 
   config = lib.mkIf cfg.enable {
     age.secrets.postgresql-tls-priv.owner = "postgres";
-    age.secrets.postgresql-tls-priv.file = ../../secrets/postgres-tls-priv.age;
+    age.secrets.postgresql-tls-priv.file = ../../secrets/floral/postgres-tls-priv.age;
 
     systemd.tmpfiles.rules = [
       "d /var/db 0755 root root - -"
@@ -67,7 +67,7 @@ in {
 
     # Provisioned on the server so that CA operations can be done there.
     age.secrets.postgresql-ca-priv.owner = "postgres";
-    age.secrets.postgresql-ca-priv.file = ../../secrets/postgres-ca-priv.age;
+    age.secrets.postgresql-ca-priv.file = ../../secrets/floral/postgres-ca-priv.age;
 
     users.users.postgres.packages = [
       (pkgs.writeShellScriptBin "postgres-mint-new-client" ''