From 3350646db9d2139ad4a5d1b051ec2ad4afc8fe43 Mon Sep 17 00:00:00 2001 From: Yureka Date: Mon, 22 Jul 2024 01:13:19 +0200 Subject: [PATCH] refactor(modules): use store composition --- modules/default.nix | 175 ++++++++++++++++++++++---------------------- modules/nginx.nix | 2 +- 2 files changed, 90 insertions(+), 87 deletions(-) diff --git a/modules/default.nix b/modules/default.nix index e340cc1..c2d00df 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -6,6 +6,7 @@ }: let cfg = config.services.tvix-binary-cache; + settingsFormat = pkgs.formats.toml { }; systemdHardening = { PrivateDevices = true; PrivateTmp = true; @@ -13,7 +14,7 @@ let ProtectKernelTunables = true; RestrictSUIDSGID = true; - ProtectSystem = "strict"; + #ProtectSystem = "strict"; ProtectKernelLogs = true; ProtectProc = "invisible"; PrivateUsers = true; @@ -28,61 +29,78 @@ in options = { services.tvix-binary-cache = { enable = lib.mkEnableOption "BinaryCache using tvix ca-store"; - blob-service-addr = lib.mkOption { - type = lib.types.str; - default = "objectstore+file://%S/tvix-castore/blobs.object-store"; - description = '' - `blob-service-addr` option for the mutualized content addressed storage. - ''; - }; - directory-service-addr = lib.mkOption { - type = lib.types.str; - default = "sled://%S/tvix-castore/directories.sled"; - description = '' - `directory-service-addr` option for the mutualized content addressed storage. - ''; + castoreDir = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = "/var/lib/castore"; }; caches = lib.mkOption { type = lib.types.attrsOf ( lib.types.submodule ( - { name, ... }: + { name, ... }@cacheAttrs: { - options = { - port = lib.mkOption { - type = lib.types.port; - default = 9000; + config = + let + common-composition = { + blobservices.default = { + type = "objectstore"; + object_store_url = "file://${cfg.castoreDir}/blobs.object-store"; + object_store_options = { }; + }; + directoryservices = { + objectstore = { + type = "objectstore"; + object_store_url = "file://${cfg.castoreDir}/directories.object-store"; + object_store_options = { }; + }; + memory = { + type = "memory"; + }; + cache = { + type = "cache"; + near = "memory"; + far = "objectstore"; + }; + default = { + type = "router"; + writes = "objectstore"; + reads = "cache"; + }; + }; + }; + in + { + nar-bridge-composition = lib.recursiveUpdate common-composition { + pathinfoservices.default = { + type = "grpc"; + url = "grpc+http://${cacheAttrs.config.grpcListenAddress}"; + }; + }; + tvix-daemon-composition = lib.recursiveUpdate common-composition { + pathinfoservices.default = { + type = "sled"; + is_temporary = false; + path = "/var/lib/tvix-daemon-${name}/pathinfos.sled"; + }; + }; }; + + options = { + grpcListenAddress = lib.mkOption { type = lib.types.str; }; + narBridgeListenAddress = lib.mkOption { type = lib.types.str; }; + nar-bridge-composition = lib.mkOption { inherit (settingsFormat) type; }; + tvix-daemon-composition = lib.mkOption { inherit (settingsFormat) type; }; name = lib.mkOption { type = lib.types.str; description = "Name of the cache"; default = name; defaultText = lib.literalMD "Defaults to attribute name in services.tvix-binary-cache.caches"; }; - path-info-service-addr = lib.mkOption { - type = with lib.types; str; - description = "Path info service path"; - default = "sled://%S/%N/pathinfo.sled"; - }; remote-path-info-service-addr = lib.mkOption { type = with lib.types; nullOr str; description = "Upstream cache to substitute from if nothing in "; example = "nix+https://cache.nixos.org?trusted-public-keys=cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; default = null; }; - blob-service-addr = lib.mkOption { - type = with lib.types; nullOr str; - default = null; - description = '' - Use a specific blob service and do not use the mutualized one. - ''; - }; - directory-service-addr = lib.mkOption { - type = with lib.types; nullOr str; - default = null; - description = '' - Use a specific directory address and do not use the mutualized one. - ''; - }; }; } ) @@ -93,63 +111,48 @@ in config = lib.mkIf cfg.enable { - environment.systemPackages = [ pkgs.tvix-store ]; + environment.systemPackages = [ pkgs.tvix ]; + users.users.tvix-castore = { + isSystemUser = true; + group = "tvix-castore"; + }; + users.groups.tvix-castore = { }; + + systemd.tmpfiles.rules = [ "d ${cfg.castoreDir} 770 root tvix-castore -" ]; systemd.services = lib.mkMerge ( - (lib.singleton { - tvix-castore = { + (lib.singleton { }) + ++ (lib.mapAttrsToList (name: cache: { + "tvix-daemon-${cache.name}" = { environment = { - BLOB_SERVICE_ADDR = cfg.blob-service-addr; - DIRECTORY_SERVICE_ADDR = cfg.directory-service-addr; - PATH_INFO_SERVICE_ADDR = "sled://%S/tvix-castore/pathinfo.sled"; # Unused but probably needed + EXPERIMENTAL_STORE_COMPOSITION = settingsFormat.generate "Config.toml" cache.tvix-daemon-composition; }; serviceConfig = { - ExecStart = "${pkgs.tvix-store}/bin/tvix-store --otlp=false daemon --listen-address=\"%t/tvix-castore/socket\""; - DynamicUser = true; - User = "tvix-binary-cache"; - StateDirectory = "tvix-castore"; - RuntimeDirectory = "tvix-castore"; + UMask = "007"; + #ExecStart = "${pkgs.tvix-store}/bin/tvix-store --otlp=false daemon --listen-address=\"%t/tvix-castore/socket\""; + ExecStart = "${pkgs.tvix}/bin/tvix-store --otlp=false daemon --listen-address=\"${cache.grpcListenAddress}\""; + StateDirectory = "tvix-daemon-${cache.name}"; + RuntimeDirectory = "tvix-daemon-${cache.name}"; + User = "tvix-castore"; + Group = "tvix-castore"; } // systemdHardening; - }; - }) - ++ (lib.mapAttrsToList ( - name: cfg: - let - unitName = "tvix-store-${cfg.name}"; - in - { - ${unitName} = { - wantedBy = [ "multi-user.target" ]; - wants = [ "tvix-castore.service" ]; - after = [ "tvix-castore.service" ]; - environment = { - BLOB_SERVICE_ADDR = - if cfg.blob-service-addr != null then - cfg.blob-service-addr - else - "grpc+unix://%t/tvix-castore/socket"; - DIRECTORY_SERVICE_ADDR = - if cfg.directory-service-addr != null then - cfg.directory-service-addr - else - "grpc+unix://%t/tvix-castore/socket"; - PATH_INFO_SERVICE_ADDR = cfg.path-info-service-addr; - REMOTE_PATH_INFO_SERVICE_ADDR = lib.mkIf ( - cfg.remote-path-info-service-addr != null - ) cfg.remote-path-info-service-addr; - }; - serviceConfig = { - ExecStart = "${pkgs.nar-bridge}/bin/nar-bridge --otlp=false --listen-address=\"[::1]:${builtins.toString cfg.port}\""; - DynamicUser = true; - User = "tvix-binary-cache"; - StateDirectory = unitName; - RuntimeDirectory = unitName; - } // systemdHardening; + "narbridge-${cache.name}" = { + wantedBy = [ "multi-user.target" ]; + wants = [ "tvix-daemon-${cache.name}.service" ]; + after = [ "tvix-daemon-${cache.name}.service" ]; + environment = { + EXPERIMENTAL_STORE_COMPOSITION = settingsFormat.generate "Config.toml" cache.nar-bridge-composition; }; + serviceConfig = { + ExecStart = "${pkgs.tvix}/bin/nar-bridge --otlp=false --listen-address=\"${cache.narBridgeListenAddress}\""; + User = "tvix-castore"; + Group = "tvix-castore"; + RuntimeDirectory = "narbridge-${cache.name}"; + } // systemdHardening; + }; - } - ) cfg.caches) + }) cfg.caches) ); }; diff --git a/modules/nginx.nix b/modules/nginx.nix index 67b9f97..3f943c0 100644 --- a/modules/nginx.nix +++ b/modules/nginx.nix @@ -25,7 +25,7 @@ lib.mapAttrsToList (name: cfg: { "/${name}".return = "302 /${name}/"; "/${name}/" = { - proxyPass = "http://localhost:${toString cfg.port}/"; + proxyPass = "http://${toString cfg.narBridgeListenAddress}/"; }; }) config.services.tvix-binary-cache.caches );