feat: Add options for multiple decoupled caches with same storage backend

This commit is contained in:
sinavir 2024-07-08 19:35:59 +02:00
parent e7f078780a
commit 279a3d67e5
4 changed files with 172 additions and 114 deletions

View file

@ -6,85 +6,126 @@
}: }:
let let
cfg = config.services.tvix-binary-cache; cfg = config.services.tvix-binary-cache;
systemdHardening = {
PrivateDevices = true;
PrivateTmp = true;
ProtectControlGroups = true;
ProtectKernelTunables = true;
RestrictSUIDSGID = true;
ProtectSystem = "strict";
ProtectKernelLogs = true;
ProtectProc = "invisible";
PrivateUsers = true;
ProtectHome = true;
UMask = "0077";
RuntimeDirectoryMode = "0750";
StateDirectoryMode = "0750";
};
in in
{ {
options = { options = {
services.tvix-binary-cache = { services.tvix-binary-cache = {
enable = lib.mkEnableOption "BinaryCache using tvix ca-store"; enable = lib.mkEnableOption "BinaryCache using tvix ca-store";
port = lib.mkOption { blob-service-addr = lib.mkOption {
type = lib.types.port; type = lib.types.str;
default = 9000; default = "objectstore+file://%S/tvix-castore/blobs.object-store";
description = ''
`blob-service-addr` option for the castore (please read tvix source/docs to learn more).
'';
};
directory-service-addr = lib.mkOption {
type = lib.types.str;
default = "sled://%S/tvix-castore/directories.sled";
description = ''
`directory-service-addr` option for the castore (please read tvix source/docs to learn more).
'';
};
caches = lib.mkOption {
type = lib.types.attrsOf (
lib.types.submodule (
{ name, ... }:
{
options = {
port = lib.mkOption {
type = lib.types.port;
default = 9000;
};
name = lib.mkOption {
type = lib.types.str;
description = "Name of the cache";
default = name;
defaultText = lib.literalMD "Defaults to attribute name in services.tvix-binary-cache.caches";
};
};
}
)
);
}; };
}; };
}; };
config = { config = lib.mkIf cfg.enable {
systemd.services = systemd.services = lib.mkMerge (
let (lib.singleton {
stateDir = "tvix-binary-cache"; tvix-castore = {
in wants = [ "tvix-castore.service" ];
lib.mkIf cfg.enable { after = [ "tvix-castore.service" ];
nar-bridge = {
wants = [ "tvix-store.service" ];
wantedBy = [ "multi-user.target" ];
after = [ "tvix-store.service" ];
serviceConfig = rec {
ExecStart = "${lib.getExe pkgs.nar-bridge-go} --otlp=false --listen-addr=\"[::1]:${builtins.toString cfg.port}\" --store-addr=\"unix://%t/${stateDir}/socket\"";
DynamicUser = true;
User = "tvix-binary-cache";
Group = "nginx";
PrivateDevices = true;
PrivateTmp = true;
ProtectControlGroups = true;
ProtectKernelTunables = true;
RestrictSUIDSGID = true;
ProtectSystem = "strict";
ProtectKernelLogs = true;
ProtectProc = "invisible";
PrivateUsers = true;
ProtectHome = true;
UMask = "0077";
RuntimeDirectoryMode = "0750";
StateDirectoryMode = "0750";
};
};
tvix-store = {
environment = { environment = {
BLOB_SERVICE_ADDR = "objectstore+file://%S/${stateDir}/blobs.object_store"; BLOB_SERVICE_ADDR = cfg.blob-service-addr;
DIRECTORY_SERVICE_ADDR = "sled://%S/${stateDir}/directories.sled"; DIRECTORY_SERVICE_ADDR = cfg.directory-service-addr;
PATH_INFO_SERVICE_ADDR = "sled://%S/${stateDir}/pathinfo.sled"; PATH_INFO_SERVICE_ADDR = "sled://%S/tvix-castore/pathinfo.sled"; # Unused but probably needed
}; };
serviceConfig = { serviceConfig = {
ExecStart = "${pkgs.tvix-store}/bin/tvix-store --otlp=false daemon --listen-address=\"%t/${stateDir}/socket\""; ExecStart = "${pkgs.tvix-store}/bin/tvix-store --otlp=false daemon --listen-address=\"%t/tvix-castore/socket\"";
DynamicUser = true; DynamicUser = true;
User = "tvix-binary-cache"; User = "tvix-binary-cache";
Group = "nginx"; StateDirectory = "tvix-castore";
StateDirectory = stateDir; RuntimeDirectory = "tvix-castore";
RuntimeDirectory = stateDir; } // systemdHardening;
PrivateDevices = true;
PrivateTmp = true;
ProtectControlGroups = true;
ProtectKernelTunables = true;
RestrictSUIDSGID = true;
ProtectSystem = "strict";
ProtectKernelLogs = true;
ProtectProc = "invisible";
PrivateUsers = true;
ProtectHome = true;
UMask = "0077";
RuntimeDirectoryMode = "0750";
StateDirectoryMode = "0750";
};
}; };
})
++ (lib.mapAttrsToList (
name: cfg:
let
stateDir = "tvix-binary-cache-${cfg.name}";
in
{
"nar-bridge-${cfg.name}" = {
wants = [ "tvix-store-${cfg.name}.service" ];
wantedBy = [ "multi-user.target" ];
after = [ "tvix-store-${cfg.name}.service" ];
serviceConfig = rec {
ExecStart = "${lib.getExe pkgs.nar-bridge-go} --otlp=false --listen-addr=\"[::1]:${builtins.toString cfg.port}\" --store-addr=\"unix://%t/${stateDir}/socket\"";
}; DynamicUser = true;
User = "tvix-binary-cache";
} // systemdHardening;
};
"tvix-store-${cfg.name}" = {
wants = [ "tvix-castore.service" ];
after = [ "tvix-castore.service" ];
environment = {
BLOB_SERVICE_ADDR = "grpc+unix://%t/tvix-castore/socket";
DIRECTORY_SERVICE_ADDR = "grpc+unix://%t/tvix-castore/socket";
PATH_INFO_SERVICE_ADDR = "sled://%S/${stateDir}/pathinfo.sled";
};
serviceConfig = {
ExecStart = "${pkgs.tvix-store}/bin/tvix-store --otlp=false daemon --listen-address=\"%t/${stateDir}/socket\"";
DynamicUser = true;
User = "tvix-binary-cache";
StateDirectory = stateDir;
RuntimeDirectory = stateDir;
} // systemdHardening;
};
}
) cfg.caches)
);
}; };
} }

View file

@ -1,53 +0,0 @@
let
sources = import ./npins;
inherit (sources) nixpkgs;
pkgs = import nixpkgs { overlays = [ (import ./pkgs/overlay.nix) ]; };
inherit (pkgs) hello;
in
pkgs.testers.runNixOSTest (_: {
name = "cache smoke test";
nodes.machine =
{ config, ... }:
{
imports = [ ./modules ];
system.extraDependencies = [ hello ];
services.tvix-binary-cache.enable = true;
services.nginx = {
enable = true;
recommendedProxySettings = true;
virtualHosts.cachix = {
default = true;
locations."/" = {
proxyPass = "http://localhost:${toString config.services.tvix-binary-cache.port}";
extraConfig = ''
client_max_body_size 10G;
'';
};
};
};
};
testScript = ''
import sys
import time
start_all()
machine.wait_for_unit("tvix-store.service")
machine.wait_for_unit("nginx.service")
machine.wait_for_unit("nar-bridge.service")
time.sleep(1)
with subtest("Nar bridge home"):
out = machine.succeed("curl http://127.0.0.1/")
print(repr(out))
if out != "nar-bridge":
sys.exit(1)
with subtest("Nar upload"):
machine.succeed("nix copy --extra-experimental-features nix-command --to 'http://127.0.0.1/?compression=none' ${hello}")
with subtest("narinfo retrieve"):
narHash = "${hello}"[11:11+32]
machine.succeed("curl 'http://127.0.0.1/{narHash}.narinfo'")
'';
})

62
tests/basic.nix Normal file
View file

@ -0,0 +1,62 @@
{ pkgs }:
let
inherit (pkgs) hello;
in
pkgs.testers.runNixOSTest (_: {
name = "cache smoke test";
nodes.machine =
{ config, ... }:
{
imports = [ ../modules ];
system.extraDependencies = [ hello ];
services.tvix-binary-cache = {
enable = true;
caches = {
one.port = 8000;
two.port = 8001;
};
};
services.nginx = {
enable = true;
recommendedProxySettings = true;
virtualHosts.cachix = {
default = true;
locations = {
"/one".return = "302 /one/";
"/one/" = {
proxyPass = "http://localhost:${toString config.services.tvix-binary-cache.caches.one.port}/";
};
"/two".return = "302 /two/";
"/two/" = {
proxyPass = "http://localhost:${toString config.services.tvix-binary-cache.caches.two.port}/";
};
};
extraConfig = "client_max_body_size 1G;";
};
};
};
testScript = ''
import sys
import time
start_all()
machine.wait_for_unit("nginx.service")
machine.wait_for_unit("nar-bridge-one.service")
machine.wait_for_unit("nar-bridge-two.service")
time.sleep(1)
with subtest("Nar bridge home"):
out = machine.succeed("curl -L http://127.0.0.1/one")
if out != "nar-bridge":
sys.exit(1)
with subtest("Nar upload"):
machine.succeed("nix copy --extra-experimental-features nix-command --to 'http://127.0.0.1/one/?compression=none' ${hello}")
with subtest("narinfo retrieve"):
narHash = "${hello}"[11:11+32]
machine.succeed(f"curl -f 'http://127.0.0.1/one/{narHash}.narinfo'")
machine.fail(f"curl -f 'http://127.0.0.1/two/{narHash}.narinfo'")
'';
})

8
tests/default.nix Normal file
View file

@ -0,0 +1,8 @@
let
sources = import ../npins;
inherit (sources) nixpkgs;
pkgs = import nixpkgs { overlays = [ (import ../pkgs/overlay.nix) ]; };
in
{
basic = pkgs.callPackage ./basic.nix { };
}