refactor(modules): use store composition

modules/default.nix

@@ -6,6 +6,7 @@
 }:
 let
   cfg = config.services.tvix-binary-cache;
+  settingsFormat = pkgs.formats.toml { };
   systemdHardening = {
     PrivateDevices = true;
     PrivateTmp = true;
@@ -13,7 +14,7 @@ let
     ProtectKernelTunables = true;
     RestrictSUIDSGID = true;
 
-    ProtectSystem = "strict";
+    #ProtectSystem = "strict";
     ProtectKernelLogs = true;
     ProtectProc = "invisible";
     PrivateUsers = true;
@@ -28,61 +29,78 @@ in
   options = {
     services.tvix-binary-cache = {
       enable = lib.mkEnableOption "BinaryCache using tvix ca-store";
-      blob-service-addr = lib.mkOption {
+      castoreDir = lib.mkOption {
-        type = lib.types.str;
+        type = lib.types.nullOr lib.types.str;
-        default = "objectstore+file://%S/tvix-castore/blobs.object-store";
+        default = "/var/lib/castore";
-        description = ''
-          `blob-service-addr` option for the mutualized content addressed storage.
-        '';
-      };
-      directory-service-addr = lib.mkOption {
-        type = lib.types.str;
-        default = "sled://%S/tvix-castore/directories.sled";
-        description = ''
-          `directory-service-addr` option for the mutualized content addressed storage.
-        '';
       };
       caches = lib.mkOption {
         type = lib.types.attrsOf (
           lib.types.submodule (
-            { name, ... }:
+            { name, ... }@cacheAttrs:
             {
-              options = {
+              config =
-                port = lib.mkOption {
+                let
-                  type = lib.types.port;
+                  common-composition = {
-                  default = 9000;
+                    blobservices.default = {
+                      type = "objectstore";
+                      object_store_url = "file://${cfg.castoreDir}/blobs.object-store";
+                      object_store_options = { };
+                    };
+                    directoryservices = {
+                      objectstore = {
+                        type = "objectstore";
+                        object_store_url = "file://${cfg.castoreDir}/directories.object-store";
+                        object_store_options = { };
+                      };
+                      memory = {
+                        type = "memory";
+                      };
+                      cache = {
+                        type = "cache";
+                        near = "memory";
+                        far = "objectstore";
+                      };
+                      default = {
+                        type = "router";
+                        writes = "objectstore";
+                        reads = "cache";
+                      };
+                    };
+                  };
+                in
+                {
+                  nar-bridge-composition = lib.recursiveUpdate common-composition {
+                    pathinfoservices.default = {
+                      type = "grpc";
+                      url = "grpc+http://${cacheAttrs.config.grpcListenAddress}";
+                    };
+                  };
+                  tvix-daemon-composition = lib.recursiveUpdate common-composition {
+                    pathinfoservices.default = {
+                      type = "sled";
+                      is_temporary = false;
+                      path = "/var/lib/tvix-daemon-${name}/pathinfos.sled";
+                    };
+                  };
                 };
+
+              options = {
+                grpcListenAddress = lib.mkOption { type = lib.types.str; };
+                narBridgeListenAddress = lib.mkOption { type = lib.types.str; };
+                nar-bridge-composition = lib.mkOption { inherit (settingsFormat) type; };
+                tvix-daemon-composition = lib.mkOption { inherit (settingsFormat) type; };
                 name = lib.mkOption {
                   type = lib.types.str;
                   description = "Name of the cache";
                   default = name;
                   defaultText = lib.literalMD "Defaults to attribute name in services.tvix-binary-cache.caches";
                 };
-                path-info-service-addr = lib.mkOption {
-                  type = with lib.types; str;
-                  description = "Path info service path";
-                  default = "sled://%S/%N/pathinfo.sled";
-                };
                 remote-path-info-service-addr = lib.mkOption {
                   type = with lib.types; nullOr str;
                   description = "Upstream cache to substitute from if nothing in ";
                   example = "nix+https://cache.nixos.org?trusted-public-keys=cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=";
                   default = null;
                 };
-                blob-service-addr = lib.mkOption {
-                  type = with lib.types; nullOr str;
-                  default = null;
-                  description = ''
-                    Use a specific blob service and do not use the mutualized one.
-                  '';
-                };
-                directory-service-addr = lib.mkOption {
-                  type = with lib.types; nullOr str;
-                  default = null;
-                  description = ''
-                    Use a specific directory address and do not use the mutualized one.
-                  '';
-                };
               };
             }
           )
@@ -93,63 +111,48 @@ in
 
   config = lib.mkIf cfg.enable {
 
-    environment.systemPackages = [ pkgs.tvix-store ];
+    environment.systemPackages = [ pkgs.tvix ];
+    users.users.tvix-castore = {
+      isSystemUser = true;
+      group = "tvix-castore";
+    };
+    users.groups.tvix-castore = { };
+
+    systemd.tmpfiles.rules = [ "d ${cfg.castoreDir} 770 root tvix-castore -" ];
 
     systemd.services = lib.mkMerge (
-      (lib.singleton {
+      (lib.singleton { })
-        tvix-castore = {
+      ++ (lib.mapAttrsToList (name: cache: {
+        "tvix-daemon-${cache.name}" = {
           environment = {
-            BLOB_SERVICE_ADDR = cfg.blob-service-addr;
+            EXPERIMENTAL_STORE_COMPOSITION = settingsFormat.generate "Config.toml" cache.tvix-daemon-composition;
-            DIRECTORY_SERVICE_ADDR = cfg.directory-service-addr;
-            PATH_INFO_SERVICE_ADDR = "sled://%S/tvix-castore/pathinfo.sled"; # Unused but probably needed
           };
           serviceConfig = {
-            ExecStart = "${pkgs.tvix-store}/bin/tvix-store --otlp=false daemon --listen-address=\"%t/tvix-castore/socket\"";
+            UMask = "007";
-            DynamicUser = true;
+            #ExecStart = "${pkgs.tvix-store}/bin/tvix-store --otlp=false daemon --listen-address=\"%t/tvix-castore/socket\"";
-            User = "tvix-binary-cache";
+            ExecStart = "${pkgs.tvix}/bin/tvix-store --otlp=false daemon --listen-address=\"${cache.grpcListenAddress}\"";
-            StateDirectory = "tvix-castore";
+            StateDirectory = "tvix-daemon-${cache.name}";
-            RuntimeDirectory = "tvix-castore";
+            RuntimeDirectory = "tvix-daemon-${cache.name}";
+            User = "tvix-castore";
+            Group = "tvix-castore";
           } // systemdHardening;
-
         };
-      })
+        "narbridge-${cache.name}" = {
-      ++ (lib.mapAttrsToList (
+          wantedBy = [ "multi-user.target" ];
-        name: cfg:
+          wants = [ "tvix-daemon-${cache.name}.service" ];
-        let
+          after = [ "tvix-daemon-${cache.name}.service" ];
-          unitName = "tvix-store-${cfg.name}";
+          environment = {
-        in
+            EXPERIMENTAL_STORE_COMPOSITION = settingsFormat.generate "Config.toml" cache.nar-bridge-composition;
-        {
-          ${unitName} = {
-            wantedBy = [ "multi-user.target" ];
-            wants = [ "tvix-castore.service" ];
-            after = [ "tvix-castore.service" ];
-            environment = {
-              BLOB_SERVICE_ADDR =
-                if cfg.blob-service-addr != null then
-                  cfg.blob-service-addr
-                else
-                  "grpc+unix://%t/tvix-castore/socket";
-              DIRECTORY_SERVICE_ADDR =
-                if cfg.directory-service-addr != null then
-                  cfg.directory-service-addr
-                else
-                  "grpc+unix://%t/tvix-castore/socket";
-              PATH_INFO_SERVICE_ADDR = cfg.path-info-service-addr;
-              REMOTE_PATH_INFO_SERVICE_ADDR = lib.mkIf (
-                cfg.remote-path-info-service-addr != null
-              ) cfg.remote-path-info-service-addr;
-            };
-            serviceConfig = {
-              ExecStart = "${pkgs.nar-bridge}/bin/nar-bridge --otlp=false --listen-address=\"[::1]:${builtins.toString cfg.port}\"";
-              DynamicUser = true;
-              User = "tvix-binary-cache";
-              StateDirectory = unitName;
-              RuntimeDirectory = unitName;
-            } // systemdHardening;
           };
+          serviceConfig = {
+            ExecStart = "${pkgs.tvix}/bin/nar-bridge --otlp=false --listen-address=\"${cache.narBridgeListenAddress}\"";
+            User = "tvix-castore";
+            Group = "tvix-castore";
+            RuntimeDirectory = "narbridge-${cache.name}";
+          } // systemdHardening;
+        };
 
-        }
+      }) cfg.caches)
-      ) cfg.caches)
     );
 
   };

modules/nginx.nix

@@ -25,7 +25,7 @@
           lib.mapAttrsToList (name: cfg: {
             "/${name}".return = "302 /${name}/";
             "/${name}/" = {
-              proxyPass = "http://localhost:${toString cfg.port}/";
+              proxyPass = "http://${toString cfg.narBridgeListenAddress}/";
             };
           }) config.services.tvix-binary-cache.caches
         );