refactor(modules): use store composition

This commit is contained in:
Yureka 2024-07-22 01:13:19 +02:00 committed by sinavir
parent 066248d032
commit 3350646db9
2 changed files with 90 additions and 87 deletions

View file

@ -6,6 +6,7 @@
}: }:
let let
cfg = config.services.tvix-binary-cache; cfg = config.services.tvix-binary-cache;
settingsFormat = pkgs.formats.toml { };
systemdHardening = { systemdHardening = {
PrivateDevices = true; PrivateDevices = true;
PrivateTmp = true; PrivateTmp = true;
@ -13,7 +14,7 @@ let
ProtectKernelTunables = true; ProtectKernelTunables = true;
RestrictSUIDSGID = true; RestrictSUIDSGID = true;
ProtectSystem = "strict"; #ProtectSystem = "strict";
ProtectKernelLogs = true; ProtectKernelLogs = true;
ProtectProc = "invisible"; ProtectProc = "invisible";
PrivateUsers = true; PrivateUsers = true;
@ -28,61 +29,78 @@ in
options = { options = {
services.tvix-binary-cache = { services.tvix-binary-cache = {
enable = lib.mkEnableOption "BinaryCache using tvix ca-store"; enable = lib.mkEnableOption "BinaryCache using tvix ca-store";
blob-service-addr = lib.mkOption { castoreDir = lib.mkOption {
type = lib.types.str; type = lib.types.nullOr lib.types.str;
default = "objectstore+file://%S/tvix-castore/blobs.object-store"; default = "/var/lib/castore";
description = ''
`blob-service-addr` option for the mutualized content addressed storage.
'';
};
directory-service-addr = lib.mkOption {
type = lib.types.str;
default = "sled://%S/tvix-castore/directories.sled";
description = ''
`directory-service-addr` option for the mutualized content addressed storage.
'';
}; };
caches = lib.mkOption { caches = lib.mkOption {
type = lib.types.attrsOf ( type = lib.types.attrsOf (
lib.types.submodule ( lib.types.submodule (
{ name, ... }: { name, ... }@cacheAttrs:
{ {
options = { config =
port = lib.mkOption { let
type = lib.types.port; common-composition = {
default = 9000; blobservices.default = {
type = "objectstore";
object_store_url = "file://${cfg.castoreDir}/blobs.object-store";
object_store_options = { };
};
directoryservices = {
objectstore = {
type = "objectstore";
object_store_url = "file://${cfg.castoreDir}/directories.object-store";
object_store_options = { };
};
memory = {
type = "memory";
};
cache = {
type = "cache";
near = "memory";
far = "objectstore";
};
default = {
type = "router";
writes = "objectstore";
reads = "cache";
};
};
};
in
{
nar-bridge-composition = lib.recursiveUpdate common-composition {
pathinfoservices.default = {
type = "grpc";
url = "grpc+http://${cacheAttrs.config.grpcListenAddress}";
};
};
tvix-daemon-composition = lib.recursiveUpdate common-composition {
pathinfoservices.default = {
type = "sled";
is_temporary = false;
path = "/var/lib/tvix-daemon-${name}/pathinfos.sled";
};
};
}; };
options = {
grpcListenAddress = lib.mkOption { type = lib.types.str; };
narBridgeListenAddress = lib.mkOption { type = lib.types.str; };
nar-bridge-composition = lib.mkOption { inherit (settingsFormat) type; };
tvix-daemon-composition = lib.mkOption { inherit (settingsFormat) type; };
name = lib.mkOption { name = lib.mkOption {
type = lib.types.str; type = lib.types.str;
description = "Name of the cache"; description = "Name of the cache";
default = name; default = name;
defaultText = lib.literalMD "Defaults to attribute name in services.tvix-binary-cache.caches"; defaultText = lib.literalMD "Defaults to attribute name in services.tvix-binary-cache.caches";
}; };
path-info-service-addr = lib.mkOption {
type = with lib.types; str;
description = "Path info service path";
default = "sled://%S/%N/pathinfo.sled";
};
remote-path-info-service-addr = lib.mkOption { remote-path-info-service-addr = lib.mkOption {
type = with lib.types; nullOr str; type = with lib.types; nullOr str;
description = "Upstream cache to substitute from if nothing in "; description = "Upstream cache to substitute from if nothing in ";
example = "nix+https://cache.nixos.org?trusted-public-keys=cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; example = "nix+https://cache.nixos.org?trusted-public-keys=cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=";
default = null; default = null;
}; };
blob-service-addr = lib.mkOption {
type = with lib.types; nullOr str;
default = null;
description = ''
Use a specific blob service and do not use the mutualized one.
'';
};
directory-service-addr = lib.mkOption {
type = with lib.types; nullOr str;
default = null;
description = ''
Use a specific directory address and do not use the mutualized one.
'';
};
}; };
} }
) )
@ -93,63 +111,48 @@ in
config = lib.mkIf cfg.enable { config = lib.mkIf cfg.enable {
environment.systemPackages = [ pkgs.tvix-store ]; environment.systemPackages = [ pkgs.tvix ];
users.users.tvix-castore = {
isSystemUser = true;
group = "tvix-castore";
};
users.groups.tvix-castore = { };
systemd.tmpfiles.rules = [ "d ${cfg.castoreDir} 770 root tvix-castore -" ];
systemd.services = lib.mkMerge ( systemd.services = lib.mkMerge (
(lib.singleton { (lib.singleton { })
tvix-castore = { ++ (lib.mapAttrsToList (name: cache: {
"tvix-daemon-${cache.name}" = {
environment = { environment = {
BLOB_SERVICE_ADDR = cfg.blob-service-addr; EXPERIMENTAL_STORE_COMPOSITION = settingsFormat.generate "Config.toml" cache.tvix-daemon-composition;
DIRECTORY_SERVICE_ADDR = cfg.directory-service-addr;
PATH_INFO_SERVICE_ADDR = "sled://%S/tvix-castore/pathinfo.sled"; # Unused but probably needed
}; };
serviceConfig = { serviceConfig = {
ExecStart = "${pkgs.tvix-store}/bin/tvix-store --otlp=false daemon --listen-address=\"%t/tvix-castore/socket\""; UMask = "007";
DynamicUser = true; #ExecStart = "${pkgs.tvix-store}/bin/tvix-store --otlp=false daemon --listen-address=\"%t/tvix-castore/socket\"";
User = "tvix-binary-cache"; ExecStart = "${pkgs.tvix}/bin/tvix-store --otlp=false daemon --listen-address=\"${cache.grpcListenAddress}\"";
StateDirectory = "tvix-castore"; StateDirectory = "tvix-daemon-${cache.name}";
RuntimeDirectory = "tvix-castore"; RuntimeDirectory = "tvix-daemon-${cache.name}";
User = "tvix-castore";
Group = "tvix-castore";
} // systemdHardening; } // systemdHardening;
}; };
}) "narbridge-${cache.name}" = {
++ (lib.mapAttrsToList ( wantedBy = [ "multi-user.target" ];
name: cfg: wants = [ "tvix-daemon-${cache.name}.service" ];
let after = [ "tvix-daemon-${cache.name}.service" ];
unitName = "tvix-store-${cfg.name}"; environment = {
in EXPERIMENTAL_STORE_COMPOSITION = settingsFormat.generate "Config.toml" cache.nar-bridge-composition;
{
${unitName} = {
wantedBy = [ "multi-user.target" ];
wants = [ "tvix-castore.service" ];
after = [ "tvix-castore.service" ];
environment = {
BLOB_SERVICE_ADDR =
if cfg.blob-service-addr != null then
cfg.blob-service-addr
else
"grpc+unix://%t/tvix-castore/socket";
DIRECTORY_SERVICE_ADDR =
if cfg.directory-service-addr != null then
cfg.directory-service-addr
else
"grpc+unix://%t/tvix-castore/socket";
PATH_INFO_SERVICE_ADDR = cfg.path-info-service-addr;
REMOTE_PATH_INFO_SERVICE_ADDR = lib.mkIf (
cfg.remote-path-info-service-addr != null
) cfg.remote-path-info-service-addr;
};
serviceConfig = {
ExecStart = "${pkgs.nar-bridge}/bin/nar-bridge --otlp=false --listen-address=\"[::1]:${builtins.toString cfg.port}\"";
DynamicUser = true;
User = "tvix-binary-cache";
StateDirectory = unitName;
RuntimeDirectory = unitName;
} // systemdHardening;
}; };
serviceConfig = {
ExecStart = "${pkgs.tvix}/bin/nar-bridge --otlp=false --listen-address=\"${cache.narBridgeListenAddress}\"";
User = "tvix-castore";
Group = "tvix-castore";
RuntimeDirectory = "narbridge-${cache.name}";
} // systemdHardening;
};
} }) cfg.caches)
) cfg.caches)
); );
}; };

View file

@ -25,7 +25,7 @@
lib.mapAttrsToList (name: cfg: { lib.mapAttrsToList (name: cfg: {
"/${name}".return = "302 /${name}/"; "/${name}".return = "302 /${name}/";
"/${name}/" = { "/${name}/" = {
proxyPass = "http://localhost:${toString cfg.port}/"; proxyPass = "http://${toString cfg.narBridgeListenAddress}/";
}; };
}) config.services.tvix-binary-cache.caches }) config.services.tvix-binary-cache.caches
); );