diff --git a/common/admins.nix b/common/admins.nix
index 84f9231..b15df11 100644
--- a/common/admins.nix
+++ b/common/admins.nix
@@ -1,16 +1,21 @@
+{ lib, ... }:
 let
-  keys = import ./ssh-keys.nix;
-in {
-  users.users.root.openssh.authorizedKeys.keys = 
-    keys.users.delroth ++
-    keys.users.emilylange ++
-    keys.users.hexchen ++
-    keys.users.jade ++
-    keys.users.janik ++
-    keys.users.k900 ++
-    keys.users.lukegb ++
-    keys.users.maxine ++
-    keys.users.raito ++
-    keys.users.thubrecht ++
-    keys.users.yuka;
-}
+  inherit (lib) genAttrs;
+  buildInfraMembers = [
+    "delroth"
+    "emilylange"
+    "hexchen"
+    "jade"
+    "janik"
+    "k900"
+    "maxine"
+    "raito"
+    "thubrecht"
+    "yuka"
+  ];
+in
+  {
+    bagel.admins.users = genAttrs buildInfraMembers (username: {
+      groups = [ "build-infra" ];
+    });
+  }
diff --git a/common/default.nix b/common/default.nix
index b329e4a..ac1ac79 100644
--- a/common/default.nix
+++ b/common/default.nix
@@ -1,6 +1,7 @@
 {
   imports = [
     ./admins.nix
+    ./server-acl.nix
     ./base-server.nix
     ./hardening.nix
     ./nix.nix
diff --git a/common/server-acl.nix b/common/server-acl.nix
new file mode 100644
index 0000000..c968044
--- /dev/null
+++ b/common/server-acl.nix
@@ -0,0 +1,51 @@
+{ lib, config, ... }:
+let
+  keys = import ./ssh-keys.nix;
+  inherit (lib) mkOption types length filterAttrs any catAttrs concatLists attrValues;
+  cfg = config.bagel.admins;
+  userOpts = { name, ... }: {
+    options = {
+      groups = mkOption {
+        type = types.listOf types.str;
+        description = "List of groups this user is part of";
+        example = [ "build-infra" ];
+        default = [ ];
+      };
+
+      sshKeys = mkOption {
+        type = types.listOf types.str;
+        description = "List of SSH keys associated to this user, defaults to `ssh-keys.nix` entries.";
+        default = keys.users.${name} or [ ];
+      };
+    };
+  };
+  isAllowedGroup = group: any (allowedGroup: group == allowedGroup) cfg.allowedGroups;
+  rootKeys = concatLists (catAttrs "sshKeys" (attrValues (filterAttrs (username: { groups, ... }: any isAllowedGroup groups) cfg.users)));
+in
+{
+  options.bagel.admins = {
+    allowedGroups = mkOption {
+      type = types.listOf types.str;
+      default = [ "catch-all" ];
+      description = "List of groups which are allowed to admin this machine.";
+      example = [ "lix" "build-infra" ];
+    };
+
+    users = mkOption {
+      type = types.attrsOf (types.submodule userOpts);
+      description = "Attribute set of admins with their groups and credentials, the username is the key of the attrset";
+    };
+  };
+
+  config = {
+    assertions = [
+      { assertion = length config.users.users.root.openssh.authorizedKeys.keys > 0;
+        # TODO: you can add printing of `concatStringsSep ", " cfg.allowedGroups` to diagnose
+        # which are the allowed groups and existing admins.
+        message = "root@${config.networking.fqdnOrHostName} has no SSH key attached, this machine will lose its access if you deploy it successfully! Set a valid `bagel.admins.allowedGroups` or ensure you have at least one administrator of the relevant group registered";
+      }
+    ];
+
+    users.users.root.openssh.authorizedKeys.keys = rootKeys;
+  };
+}
diff --git a/flake.nix b/flake.nix
index ce9ca5c..dba4785 100644
--- a/flake.nix
+++ b/flake.nix
@@ -92,6 +92,13 @@
 
         ./services
         ./common
+        {
+          # This means that anyone with @build-infra permissions
+          # can ssh on root of every machines handled here.
+          bagel.admins.allowedGroups = [
+            "build-infra"
+          ];
+        }
       ];
 
       makeBuilder = i: lib.nameValuePair "builder-${toString i}" {