From 92560708b8cfcbbb1bad2ebc3703800f6003ecd0 Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Sat, 5 Oct 2024 18:03:41 +0200
Subject: [PATCH] feat: multi-tenant secrets

Lix may have its own secrets and we want to maintain a certain
generalization level on the NixOS modules, so we can decorrelate which
secret we select dynamically by having a simple tenancy hierarchy
system.

This unfortunately requires to rewrite all call sites with a floral
prefix until we migrate them to the simple internal secret module which
is aware of this.

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
---
 common/default.nix                            |   1 +
 common/secrets.nix                            |  22 +++++
 hosts/gerrit01/default.nix                    |   2 +-
 secrets.nix                                   |  88 ++++++++++--------
 .../{ => floral}/buildbot-oauth-secret.age    |   0
 .../buildbot-remote-builder-key.age           | Bin
 secrets/{ => floral}/buildbot-service-key.age | Bin
 secrets/{ => floral}/buildbot-signing-key.age | Bin
 .../{ => floral}/buildbot-worker-password.age |   0
 secrets/{ => floral}/buildbot-workers.age     | Bin
 .../gerrit-prometheus-bearer-token.age        | Bin
 secrets/{ => floral}/grafana-oauth-secret.age |   0
 secrets/{ => floral}/hydra-postgres-key.age   | Bin
 secrets/{ => floral}/hydra-s3-credentials.age |   0
 secrets/{ => floral}/hydra-signing-priv.age   |   0
 secrets/{ => floral}/hydra-ssh-key-priv.age   |   0
 secrets/{ => floral}/loki-environment.age     |   0
 .../{ => floral}/metrics-push-htpasswd.age    | Bin
 .../{ => floral}/metrics-push-password.age    | Bin
 secrets/{ => floral}/mimir-environment.age    | Bin
 secrets/{ => floral}/mimir-webhook-url.age    |   0
 secrets/{ => floral}/netbox-environment.age   | Bin
 secrets/{ => floral}/newsletter-secrets.age   |   0
 secrets/{ => floral}/ows-deploy-key.age       | Bin
 secrets/{ => floral}/postgres-ca-priv.age     | Bin
 secrets/{ => floral}/postgres-tls-priv.age    | Bin
 secrets/{ => floral}/pyroscope-secrets.age    | Bin
 secrets/{ => floral}/s3-channel-keys.age      | Bin
 .../{ => floral}/s3-channel-staging-keys.age  | Bin
 secrets/{ => floral}/s3-revproxy-api-keys.age | Bin
 .../stateless-uptime-kuma-password.age        |   0
 secrets/{ => floral}/tempo-environment.age    |   0
 services/gerrit/default.nix                   |   2 +-
 services/hydra/default.nix                    |  10 +-
 services/monitoring/agent.nix                 |   2 +-
 services/monitoring/pyroscope/default.nix     |   2 +-
 services/newsletter/default.nix               |   2 +-
 services/postgres/default.nix                 |   4 +-
 38 files changed, 87 insertions(+), 48 deletions(-)
 create mode 100644 common/secrets.nix
 rename secrets/{ => floral}/buildbot-oauth-secret.age (100%)
 rename secrets/{ => floral}/buildbot-remote-builder-key.age (100%)
 rename secrets/{ => floral}/buildbot-service-key.age (100%)
 rename secrets/{ => floral}/buildbot-signing-key.age (100%)
 rename secrets/{ => floral}/buildbot-worker-password.age (100%)
 rename secrets/{ => floral}/buildbot-workers.age (100%)
 rename secrets/{ => floral}/gerrit-prometheus-bearer-token.age (100%)
 rename secrets/{ => floral}/grafana-oauth-secret.age (100%)
 rename secrets/{ => floral}/hydra-postgres-key.age (100%)
 rename secrets/{ => floral}/hydra-s3-credentials.age (100%)
 rename secrets/{ => floral}/hydra-signing-priv.age (100%)
 rename secrets/{ => floral}/hydra-ssh-key-priv.age (100%)
 rename secrets/{ => floral}/loki-environment.age (100%)
 rename secrets/{ => floral}/metrics-push-htpasswd.age (100%)
 rename secrets/{ => floral}/metrics-push-password.age (100%)
 rename secrets/{ => floral}/mimir-environment.age (100%)
 rename secrets/{ => floral}/mimir-webhook-url.age (100%)
 rename secrets/{ => floral}/netbox-environment.age (100%)
 rename secrets/{ => floral}/newsletter-secrets.age (100%)
 rename secrets/{ => floral}/ows-deploy-key.age (100%)
 rename secrets/{ => floral}/postgres-ca-priv.age (100%)
 rename secrets/{ => floral}/postgres-tls-priv.age (100%)
 rename secrets/{ => floral}/pyroscope-secrets.age (100%)
 rename secrets/{ => floral}/s3-channel-keys.age (100%)
 rename secrets/{ => floral}/s3-channel-staging-keys.age (100%)
 rename secrets/{ => floral}/s3-revproxy-api-keys.age (100%)
 rename secrets/{ => floral}/stateless-uptime-kuma-password.age (100%)
 rename secrets/{ => floral}/tempo-environment.age (100%)

diff --git a/common/default.nix b/common/default.nix
index e41b089..09b0f24 100644
--- a/common/default.nix
+++ b/common/default.nix
@@ -9,5 +9,6 @@
     ./sysadmin
     ./hardware
     ./zsh.nix
+    ./secrets.nix
   ];
 }
diff --git a/common/secrets.nix b/common/secrets.nix
new file mode 100644
index 0000000..1bd9983
--- /dev/null
+++ b/common/secrets.nix
@@ -0,0 +1,22 @@
+## This is a simple secret abstraction with multi-tenancy awareness.
+{ config, lib, ... }:
+let
+  cfg = config.bagel.secrets;
+  inherit (lib) mkOption types genAttrs;
+in
+{
+  options.bagel.secrets = {
+    tenant = mkOption {
+      type = types.enum [ "lix" "floral" ];
+    };
+
+    files = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+    };
+  };
+
+  config.age.secrets = genAttrs cfg.files (secretFile: {
+      file = ../secrets/${cfg.tenant}/${secretFile}.age;
+    });
+}
diff --git a/hosts/gerrit01/default.nix b/hosts/gerrit01/default.nix
index a532697..fed45ab 100755
--- a/hosts/gerrit01/default.nix
+++ b/hosts/gerrit01/default.nix
@@ -39,7 +39,7 @@
   };
 
   age.secrets.ows-deploy-key = {
-    file = ../../secrets/ows-deploy-key.age;
+    file = ../../secrets/floral/ows-deploy-key.age;
     mode = "0600";
     owner = "git";
     group = "git";
diff --git a/secrets.nix b/secrets.nix
index de24612..417e9ba 100644
--- a/secrets.nix
+++ b/secrets.nix
@@ -1,52 +1,66 @@
 let
   keys = import common/ssh-keys.nix;
 
-  commonKeys = keys.users.delroth ++ keys.users.raito;
+  commonKeys = {
+    global = [ keys.users.raito ];
+    lix = [ keys.users.jade ];
+    floral = [ keys.users.delroth ];
+  };
 
   secrets = with keys; {
-    hydra-postgres-key = [ machines.build-coord ];
-    hydra-s3-credentials = [ machines.build-coord ];
-    hydra-signing-priv = [ machines.build-coord ];
-    hydra-ssh-key-priv = [ machines.build-coord ];
+    floral = {
+      hydra-postgres-key = [ machines.build-coord ];
+      hydra-s3-credentials = [ machines.build-coord ];
+      hydra-signing-priv = [ machines.build-coord ];
+      hydra-ssh-key-priv = [ machines.build-coord ];
 
-    netbox-environment = [ machines.meta01 ];
-    mimir-environment = [ machines.meta01 ];
-    mimir-webhook-url = [ machines.meta01 ];
-    grafana-oauth-secret = [ machines.meta01 ];
-    loki-environment = [ machines.meta01 ];
-    gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
-    pyroscope-secrets = [ machines.meta01 ];
-    tempo-environment = [ machines.meta01 ];
+      netbox-environment = [ machines.meta01 ];
+      mimir-environment = [ machines.meta01 ];
+      mimir-webhook-url = [ machines.meta01 ];
+      grafana-oauth-secret = [ machines.meta01 ];
+      loki-environment = [ machines.meta01 ];
+      gerrit-prometheus-bearer-token = [ machines.gerrit01 machines.meta01 ];
+      pyroscope-secrets = [ machines.meta01 ];
+      tempo-environment = [ machines.meta01 ];
 
-    buildbot-worker-password = [ machines.buildbot ];
-    buildbot-oauth-secret = [ machines.buildbot ];
-    buildbot-workers = [ machines.buildbot ];
-    # Private SSH key to Gerrit
-    # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
-    buildbot-service-key = [ machines.buildbot ];
-    # Signing key for Buildbot's specific cache
-    buildbot-signing-key = [ machines.buildbot ];
-    buildbot-remote-builder-key = [ machines.buildbot ];
+      buildbot-worker-password = [ machines.buildbot ];
+      buildbot-oauth-secret = [ machines.buildbot ];
+      buildbot-workers = [ machines.buildbot ];
+      # Private SSH key to Gerrit
+      # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx52RUPWzTa2rBA96xcnGjjzAboNN/hm6gW+Q6JiSos
+      buildbot-service-key = [ machines.buildbot ];
+      # Signing key for Buildbot's specific cache
+      buildbot-signing-key = [ machines.buildbot ];
+      buildbot-remote-builder-key = [ machines.buildbot ];
 
-    # These are the same password, but nginx wants it in htpasswd format
-    metrics-push-htpasswd = [ machines.meta01 ];
-    metrics-push-password = builtins.attrValues machines;
+      # These are the same password, but nginx wants it in htpasswd format
+      metrics-push-htpasswd = [ machines.meta01 ];
+      # Yes, even Lix machines are included in this monitoring infrastructure.
+      metrics-push-password = builtins.attrValues machines;
 
-    ows-deploy-key = [ machines.gerrit01 ];
-    s3-channel-staging-keys = [ machines.gerrit01 ];
-    s3-channel-keys = [ machines.gerrit01 ];
+      ows-deploy-key = [ machines.gerrit01 ];
+      s3-channel-staging-keys = [ machines.gerrit01 ];
+      s3-channel-keys = [ machines.gerrit01 ];
 
-    postgres-ca-priv = [ machines.bagel-box ];
-    postgres-tls-priv = [ machines.bagel-box ];
+      postgres-ca-priv = [ machines.bagel-box ];
+      postgres-tls-priv = [ machines.bagel-box ];
 
-    newsletter-secrets = [ machines.public01 ];
-    s3-revproxy-api-keys = [ machines.public01 ];
-    stateless-uptime-kuma-password = [ machines.public01 ];
+      newsletter-secrets = [ machines.public01 ];
+      s3-revproxy-api-keys = [ machines.public01 ];
+      stateless-uptime-kuma-password = [ machines.public01 ];
+    };
+
+    lix = {
+      buildbot-remote-builder-key = [ machines.buildbot-lix ];
+    };
   };
+
+  mkSecretListFor = tenant:
+    map (secretName: {
+      name = "secrets/${tenant}/${secretName}.age";
+      value.publicKeys = secrets.${tenant}."${secretName}" ++ commonKeys.${tenant};
+    }) (builtins.attrNames secrets.${tenant});
 in
   builtins.listToAttrs (
-    map (secretName: {
-      name = "secrets/${secretName}.age";
-      value.publicKeys = secrets."${secretName}" ++ commonKeys;
-    }) (builtins.attrNames secrets)
+    (mkSecretListFor "floral") ++ (mkSecretListFor "lix")
   )
diff --git a/secrets/buildbot-oauth-secret.age b/secrets/floral/buildbot-oauth-secret.age
similarity index 100%
rename from secrets/buildbot-oauth-secret.age
rename to secrets/floral/buildbot-oauth-secret.age
diff --git a/secrets/buildbot-remote-builder-key.age b/secrets/floral/buildbot-remote-builder-key.age
similarity index 100%
rename from secrets/buildbot-remote-builder-key.age
rename to secrets/floral/buildbot-remote-builder-key.age
diff --git a/secrets/buildbot-service-key.age b/secrets/floral/buildbot-service-key.age
similarity index 100%
rename from secrets/buildbot-service-key.age
rename to secrets/floral/buildbot-service-key.age
diff --git a/secrets/buildbot-signing-key.age b/secrets/floral/buildbot-signing-key.age
similarity index 100%
rename from secrets/buildbot-signing-key.age
rename to secrets/floral/buildbot-signing-key.age
diff --git a/secrets/buildbot-worker-password.age b/secrets/floral/buildbot-worker-password.age
similarity index 100%
rename from secrets/buildbot-worker-password.age
rename to secrets/floral/buildbot-worker-password.age
diff --git a/secrets/buildbot-workers.age b/secrets/floral/buildbot-workers.age
similarity index 100%
rename from secrets/buildbot-workers.age
rename to secrets/floral/buildbot-workers.age
diff --git a/secrets/gerrit-prometheus-bearer-token.age b/secrets/floral/gerrit-prometheus-bearer-token.age
similarity index 100%
rename from secrets/gerrit-prometheus-bearer-token.age
rename to secrets/floral/gerrit-prometheus-bearer-token.age
diff --git a/secrets/grafana-oauth-secret.age b/secrets/floral/grafana-oauth-secret.age
similarity index 100%
rename from secrets/grafana-oauth-secret.age
rename to secrets/floral/grafana-oauth-secret.age
diff --git a/secrets/hydra-postgres-key.age b/secrets/floral/hydra-postgres-key.age
similarity index 100%
rename from secrets/hydra-postgres-key.age
rename to secrets/floral/hydra-postgres-key.age
diff --git a/secrets/hydra-s3-credentials.age b/secrets/floral/hydra-s3-credentials.age
similarity index 100%
rename from secrets/hydra-s3-credentials.age
rename to secrets/floral/hydra-s3-credentials.age
diff --git a/secrets/hydra-signing-priv.age b/secrets/floral/hydra-signing-priv.age
similarity index 100%
rename from secrets/hydra-signing-priv.age
rename to secrets/floral/hydra-signing-priv.age
diff --git a/secrets/hydra-ssh-key-priv.age b/secrets/floral/hydra-ssh-key-priv.age
similarity index 100%
rename from secrets/hydra-ssh-key-priv.age
rename to secrets/floral/hydra-ssh-key-priv.age
diff --git a/secrets/loki-environment.age b/secrets/floral/loki-environment.age
similarity index 100%
rename from secrets/loki-environment.age
rename to secrets/floral/loki-environment.age
diff --git a/secrets/metrics-push-htpasswd.age b/secrets/floral/metrics-push-htpasswd.age
similarity index 100%
rename from secrets/metrics-push-htpasswd.age
rename to secrets/floral/metrics-push-htpasswd.age
diff --git a/secrets/metrics-push-password.age b/secrets/floral/metrics-push-password.age
similarity index 100%
rename from secrets/metrics-push-password.age
rename to secrets/floral/metrics-push-password.age
diff --git a/secrets/mimir-environment.age b/secrets/floral/mimir-environment.age
similarity index 100%
rename from secrets/mimir-environment.age
rename to secrets/floral/mimir-environment.age
diff --git a/secrets/mimir-webhook-url.age b/secrets/floral/mimir-webhook-url.age
similarity index 100%
rename from secrets/mimir-webhook-url.age
rename to secrets/floral/mimir-webhook-url.age
diff --git a/secrets/netbox-environment.age b/secrets/floral/netbox-environment.age
similarity index 100%
rename from secrets/netbox-environment.age
rename to secrets/floral/netbox-environment.age
diff --git a/secrets/newsletter-secrets.age b/secrets/floral/newsletter-secrets.age
similarity index 100%
rename from secrets/newsletter-secrets.age
rename to secrets/floral/newsletter-secrets.age
diff --git a/secrets/ows-deploy-key.age b/secrets/floral/ows-deploy-key.age
similarity index 100%
rename from secrets/ows-deploy-key.age
rename to secrets/floral/ows-deploy-key.age
diff --git a/secrets/postgres-ca-priv.age b/secrets/floral/postgres-ca-priv.age
similarity index 100%
rename from secrets/postgres-ca-priv.age
rename to secrets/floral/postgres-ca-priv.age
diff --git a/secrets/postgres-tls-priv.age b/secrets/floral/postgres-tls-priv.age
similarity index 100%
rename from secrets/postgres-tls-priv.age
rename to secrets/floral/postgres-tls-priv.age
diff --git a/secrets/pyroscope-secrets.age b/secrets/floral/pyroscope-secrets.age
similarity index 100%
rename from secrets/pyroscope-secrets.age
rename to secrets/floral/pyroscope-secrets.age
diff --git a/secrets/s3-channel-keys.age b/secrets/floral/s3-channel-keys.age
similarity index 100%
rename from secrets/s3-channel-keys.age
rename to secrets/floral/s3-channel-keys.age
diff --git a/secrets/s3-channel-staging-keys.age b/secrets/floral/s3-channel-staging-keys.age
similarity index 100%
rename from secrets/s3-channel-staging-keys.age
rename to secrets/floral/s3-channel-staging-keys.age
diff --git a/secrets/s3-revproxy-api-keys.age b/secrets/floral/s3-revproxy-api-keys.age
similarity index 100%
rename from secrets/s3-revproxy-api-keys.age
rename to secrets/floral/s3-revproxy-api-keys.age
diff --git a/secrets/stateless-uptime-kuma-password.age b/secrets/floral/stateless-uptime-kuma-password.age
similarity index 100%
rename from secrets/stateless-uptime-kuma-password.age
rename to secrets/floral/stateless-uptime-kuma-password.age
diff --git a/secrets/tempo-environment.age b/secrets/floral/tempo-environment.age
similarity index 100%
rename from secrets/tempo-environment.age
rename to secrets/floral/tempo-environment.age
diff --git a/services/gerrit/default.nix b/services/gerrit/default.nix
index 2ba0d42..d450566 100644
--- a/services/gerrit/default.nix
+++ b/services/gerrit/default.nix
@@ -46,7 +46,7 @@ in
 
   config = mkIf cfg.enable {
     networking.firewall.allowedTCPPorts = [ cfg.port ];
-    age.secrets.alloy-push-password.file = ../../secrets/metrics-push-password.age;
+    age.secrets.alloy-push-password.file = ../../secrets/floral/metrics-push-password.age;
 
     environment.systemPackages = [ jdk
       pkgs.git
diff --git a/services/hydra/default.nix b/services/hydra/default.nix
index 22111e4..5f318eb 100644
--- a/services/hydra/default.nix
+++ b/services/hydra/default.nix
@@ -66,17 +66,19 @@ in {
     # does indeed have our public SSH key and are *builders*
     # as a simple evaluation preflight check.
 
-    age.secrets.hydra-s3-credentials.file = ../../secrets/hydra-s3-credentials.age;
+    bagel.secrets.files = [
+      "hydra-s3-credentials"
+      "hydra-postgres-key"
+      "hydra-signing-priv"
+      "hydra-ssh-key-priv"
+    ];
 
     age.secrets.hydra-postgres-key.group = "hydra";
     age.secrets.hydra-postgres-key.mode = "0440";
-    age.secrets.hydra-postgres-key.file = ../../secrets/hydra-postgres-key.age;
 
     age.secrets.hydra-signing-priv.owner = "hydra-queue-runner";
-    age.secrets.hydra-signing-priv.file = ../../secrets/hydra-signing-priv.age;
 
     age.secrets.hydra-ssh-key-priv.owner = "hydra-queue-runner";
-    age.secrets.hydra-ssh-key-priv.file = ../../secrets/hydra-ssh-key-priv.age;
 
     systemd.tmpfiles.rules = [
       "d /var/cache/hydra 0755 hydra hydra -  -"
diff --git a/services/monitoring/agent.nix b/services/monitoring/agent.nix
index b7aa3d7..9bb29a1 100644
--- a/services/monitoring/agent.nix
+++ b/services/monitoring/agent.nix
@@ -59,7 +59,7 @@ in
   };
 
   config = mkIf cfg.enable {
-    age.secrets.grafana-agent-password.file = ../../secrets/metrics-push-password.age;
+    age.secrets.grafana-agent-password.file = ../../secrets/floral/metrics-push-password.age;
 
     services.grafana-agent = {
       enable = true;
diff --git a/services/monitoring/pyroscope/default.nix b/services/monitoring/pyroscope/default.nix
index 00c79ee..ac7f754 100644
--- a/services/monitoring/pyroscope/default.nix
+++ b/services/monitoring/pyroscope/default.nix
@@ -15,7 +15,7 @@ in
   ];
 
   config = mkIf cfg.enable {
-    age.secrets.pyroscope-secrets.file = ../../../secrets/pyroscope-secrets.age;
+    age.secrets.pyroscope-secrets.file = ../../../secrets/floral/pyroscope-secrets.age;
     services.nginx = {
       upstreams.pyroscope = {
         servers."127.0.0.1:${toString pyroscopePort}" = {};
diff --git a/services/newsletter/default.nix b/services/newsletter/default.nix
index 627238a..2773d3f 100644
--- a/services/newsletter/default.nix
+++ b/services/newsletter/default.nix
@@ -14,7 +14,7 @@ in
   };
 
   config = mkIf cfg.enable {
-    age.secrets.newsletter-secrets.file = ../../secrets/newsletter-secrets.age;
+    age.secrets.newsletter-secrets.file = ../../secrets/floral/newsletter-secrets.age;
     services.listmonk = {
       enable = true;
       secretFile = config.age.secrets.newsletter-secrets.path;
diff --git a/services/postgres/default.nix b/services/postgres/default.nix
index 1b316e4..61b0c63 100644
--- a/services/postgres/default.nix
+++ b/services/postgres/default.nix
@@ -11,7 +11,7 @@ in {
 
   config = lib.mkIf cfg.enable {
     age.secrets.postgresql-tls-priv.owner = "postgres";
-    age.secrets.postgresql-tls-priv.file = ../../secrets/postgres-tls-priv.age;
+    age.secrets.postgresql-tls-priv.file = ../../secrets/floral/postgres-tls-priv.age;
 
     systemd.tmpfiles.rules = [
       "d /var/db 0755 root root - -"
@@ -67,7 +67,7 @@ in {
 
     # Provisioned on the server so that CA operations can be done there.
     age.secrets.postgresql-ca-priv.owner = "postgres";
-    age.secrets.postgresql-ca-priv.file = ../../secrets/postgres-ca-priv.age;
+    age.secrets.postgresql-ca-priv.file = ../../secrets/floral/postgres-ca-priv.age;
 
     users.users.postgres.packages = [
       (pkgs.writeShellScriptBin "postgres-mint-new-client" ''