diff --git a/common/admins.nix b/common/admins.nix
index 0e82e92..f66e100 100644
--- a/common/admins.nix
+++ b/common/admins.nix
@@ -1,17 +1,43 @@
+{ lib, ... }:
 let
-  keys = import ./ssh-keys.nix;
-in {
-  users.users.root.openssh.authorizedKeys.keys =
-    keys.users.delroth ++
-    keys.users.emilylange ++
-    keys.users.hexchen ++
-    keys.users.jade ++
-    keys.users.janik ++
-    keys.users.k900 ++
-    keys.users.lukegb ++
-    keys.users.maxine ++
-    keys.users.raito ++
-    keys.users.thubrecht ++
-    keys.users.yuka ++
-    keys.users.winter;
+  inherit (lib) genAttrs;
+in
+# Note: to add somefew in this list. 
+# Ensure their SSH key is already in common/ssh-keys.nix with
+# the same username for here, so that the keys is automatically added.
+{
+  bagel.groups = {
+    floral-infra.members = [
+      "delroth"
+      "emilylange"
+      "hexchen"
+      "jade"
+      "janik"
+      "k900"
+      "maxine"
+      "raito"
+      "thubrecht"
+      "winter"
+      "yuka"
+    ];
+
+    lix-infra.members = [
+      "raito"
+      "hexchen"
+      "jade"
+    ];
+  };
+  bagel.users = genAttrs [
+    "delroth"
+    "emilylange"
+    "hexchen"
+    "jade"
+    "janik"
+    "k900"
+    "maxine"
+    "raito"
+    "thubrecht"
+    "winter"
+    "yuka"
+  ] (name: {});
 }
diff --git a/common/default.nix b/common/default.nix
index b329e4a..ac1ac79 100644
--- a/common/default.nix
+++ b/common/default.nix
@@ -1,6 +1,7 @@
 {
   imports = [
     ./admins.nix
+    ./server-acl.nix
     ./base-server.nix
     ./hardening.nix
     ./nix.nix
diff --git a/common/server-acl.nix b/common/server-acl.nix
new file mode 100644
index 0000000..f54521a
--- /dev/null
+++ b/common/server-acl.nix
@@ -0,0 +1,69 @@
+{ lib, config, ... }:
+let
+  keys = import ./ssh-keys.nix;
+  inherit (lib) mkOption types length concatMap listToAttrs catAttrs attrValues;
+  cfgAdmins = config.bagel.admins;
+  cfgGroups = config.bagel.groups;
+  cfgUsers = config.bagel.users;
+
+  userOpts = { name, ... }: {
+    options = {
+      sshKeys = mkOption {
+        type = types.listOf types.str;
+        description = "List of SSH keys associated to this user, defaults to `ssh-keys.nix` entries.";
+        default = keys.users.${name} or [ ];
+      };
+    };
+  };
+  groupOpts = { name, ... }: {
+    options = {
+      members = mkOption {
+        type = types.listOf types.str;
+        description = "List of users member of this group";
+        example = [ "raito" ];
+        default = [ ];
+      };
+    };
+  };
+
+  # There might be duplicate in that list. We will turn it into an attribute set.
+  allowedMembers = listToAttrs (
+    map (member: {
+      name = member;
+      value = cfgUsers.${member};
+    }) (concatMap (allowedGroup: cfgGroups.${allowedGroup}.members) cfgAdmins.allowedGroups));
+
+  rootKeys = concatMap ({ sshKeys, ... }: sshKeys) (attrValues allowedMembers);
+in
+{
+  options.bagel.users = mkOption {
+    type = types.attrsOf (types.submodule userOpts);
+    description = "User configuration for server ACLs";
+  };
+
+  options.bagel.groups = mkOption {
+    type = types.attrsOf (types.submodule groupOpts);
+    description = "Group configuration for server ACLs";
+  };
+
+  options.bagel.admins = {
+    allowedGroups = mkOption {
+      type = types.listOf types.str;
+      default = [ "catch-all" ];
+      description = "List of groups which are allowed to admin this machine.";
+      example = [ "lix" "build-infra" ];
+    };
+  };
+
+  config = {
+    assertions = [
+      { assertion = length config.users.users.root.openssh.authorizedKeys.keys > 0;
+        # TODO: you can add printing of `concatStringsSep ", " cfg.allowedGroups` to diagnose
+        # which are the allowed groups and existing admins.
+        message = "root@${config.networking.fqdnOrHostName} has no SSH key attached, this machine will lose its access if you deploy it successfully! Set a valid `bagel.admins.allowedGroups` or ensure you have at least one administrator of the relevant group registered";
+      }
+    ];
+
+    users.users.root.openssh.authorizedKeys.keys = rootKeys;
+  };
+}
diff --git a/flake.nix b/flake.nix
index 48fbfb5..d2a52df 100644
--- a/flake.nix
+++ b/flake.nix
@@ -114,6 +114,13 @@
 
         ./services
         ./common
+        {
+          # This means that anyone with @floral-infra permissions
+          # can ssh on root of every machines handled here.
+          bagel.admins.allowedGroups = [
+            "floral-infra"
+          ];
+        }
       ];
 
       makeBuilder = i: lib.nameValuePair "builder-${toString i}" {