yu-re-ka/binary-cache
0
0
Fork 1
Code Issues Pull requests Packages Projects Releases Wiki Activity

feat(module): Switch to rust nar bridge

Browse source
This commit is contained in:
sinavir 2024-07-20 18:22:43 +02:00
parent 5b0e841351
commit b9f8a02e3b
8 changed files with 262 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
modules/default.nix
View file

@ -98,8 +98,6 @@ in
systemd.services = lib.mkMerge (
(lib.singleton {
tvix-castore = {
wants = [ "tvix-castore.service" ];
after = [ "tvix-castore.service" ];
environment = {
BLOB_SERVICE_ADDR = cfg.blob-service-addr;
DIRECTORY_SERVICE_ADDR = cfg.directory-service-addr;
@ -121,18 +119,8 @@ in
unitName = "tvix-store-${cfg.name}";
in
{
"nar-bridge-${cfg.name}" = {
wants = [ "tvix-store-${cfg.name}.service" ];
wantedBy = [ "multi-user.target" ];
after = [ "tvix-store-${cfg.name}.service" ];
serviceConfig = rec {
ExecStart = "${lib.getExe pkgs.nar-bridge-go} --otlp=false --listen-addr=\"[::1]:${builtins.toString cfg.port}\" --store-addr=\"unix://%t/${unitName}/socket\"";
DynamicUser = true;
User = "tvix-binary-cache";
} // systemdHardening;
};
${unitName} = {
wantedBy = [ "multi-user.target" ];
wants = [ "tvix-castore.service" ];
after = [ "tvix-castore.service" ];
environment = {
@ -152,7 +140,7 @@ in
) cfg.remote-path-info-service-addr;
};
serviceConfig = {
ExecStart = "${pkgs.tvix-store}/bin/tvix-store --otlp=false daemon --listen-address=\"%t/${unitName}/socket\"";
ExecStart = "${pkgs.nar-bridge}/bin/nar-bridge --otlp=false --listen-address=\"[::1]:${builtins.toString cfg.port}\"";
DynamicUser = true;
User = "tvix-binary-cache";
StateDirectory = unitName;

8
tests/default.nix
View file

@ -4,7 +4,13 @@
pkgs ? import nixpkgs { overlays = [ (import ../pkgs/overlay.nix) ]; },
}:
{
# Disabled since nar-bridge doesn't expose gRPC
# ingest = pkgs.callPackage ./ingest.nix { };
multi-cache = pkgs.callPackage ./multi-cache.nix { };
ingest = pkgs.callPackage ./ingest.nix { };
substitution = pkgs.callPackage ./substitution.nix { };
upstream-cache = {
tvix = pkgs.callPackage ./upstream-cache-tvix.nix { };
http-directory = pkgs.callPackage ./upstream-cache-http-directory.nix { };
nix-serve = pkgs.callPackage ./upstream-cache-nix-serve.nix { };
};
}

2
tests/ingest.nix
View file

@ -41,7 +41,7 @@ pkgs.testers.runNixOSTest (_: {
import time
start_all()
cache.wait_for_unit("nginx.service")
cache.wait_for_unit("nar-bridge-cache.service")
cache.wait_for_unit("tvix-store-cache.service")
time.sleep(1)
socket_addr = "grpc+unix:///run/tvix-store-cache/socket"
cache.succeed(f"BLOB_SERVICE_ADDR={socket_addr} DIRECTORY_SERVICE_ADDR={socket_addr} PATH_INFO_SERVICE_ADDR={socket_addr} tvix-store copy ${builtins.toString references}")

10
tests/multi-cache.nix
View file

@ -35,23 +35,19 @@ pkgs.testers.runNixOSTest (_: {
};
};
testScript = ''
import sys
import time
start_all()
machine.wait_for_unit("nginx.service")
machine.wait_for_unit("nar-bridge-one.service")
machine.wait_for_unit("nar-bridge-two.service")
machine.wait_for_unit("tvix-store-one.service")
machine.wait_for_unit("tvix-store-two.service")
time.sleep(1)
with subtest("Nar bridge home"):
out = machine.succeed("curl -L http://127.0.0.1/one")
if out != "nar-bridge":
sys.exit(1)
machine.succeed("curl -L http://127.0.0.1/one/nix-cache-info")
with subtest("Nar upload"):
machine.succeed("nix copy --to 'http://127.0.0.1/one/?compression=none' ${hello}")
with subtest("narinfo retrieve"):
narHash = "${hello}"[11:11+32]
machine.succeed(f"curl -f 'http://127.0.0.1/one/{narHash}.narinfo'")
machine.fail(f"curl -f 'http://127.0.0.1/two/{narHash}.narinfo'")
'';
})

7
tests/substitution.nix
View file

@ -42,16 +42,13 @@ pkgs.testers.runNixOSTest (_: {
};
};
testScript = ''
import sys
import time
start_all()
cache.wait_for_unit("nginx.service")
cache.wait_for_unit("nar-bridge-cache.service")
cache.wait_for_unit("tvix-store-cache.service")
time.sleep(1)
with subtest("Nar bridge home"):
out = cache.succeed("curl -L http://127.0.0.1/cache")
if out != "nar-bridge":
sys.exit(1)
cache.succeed("curl -f http://127.0.0.1/cache/nix-cache-info")
with subtest("Path signature and copy"):
# Sign
cache.succeed("nix store sign -k ${./cache-keys/privkey} ${hello}")

81
tests/upstream-cache-http-directory.nix Normal file
View file

@ -0,0 +1,81 @@
{ pkgs }:
let
#hello = pkgs.hello.overrideAttrs { pname = "custom-hello"; };
inherit (pkgs) hello;
in
pkgs.testers.runNixOSTest (_: {
name = "caching of upstream nar-store";
nodes = {
cache = {
imports = [
./common
../modules
];
system.extraDependencies = [ hello ];
services = {
tvix-binary-cache = {
enable = true;
enableNginx = true;
nginx = {
clientMaxBodySize = "50G";
host = "cache";
};
caches = {
cache = {
port = 8000;
remote-path-info-service-addr = "nix+http://localhost/upstream/"; # ?trusted-public-keys=${lib.escapeURL (builtins.readFile ./cache-keys/pubkey)}";
};
};
};
nginx = {
virtualHosts.cache = {
default = true;
locations = {
"/upstream".return = "302 /upstream/";
"/upstream/".alias = "/srv/";
};
};
};
};
networking.firewall.allowedTCPPorts = [ 80 ];
};
client =
{ lib, ... }:
{
imports = [ ./common ];
nix.settings = {
substituters = lib.mkForce [ "http://cache/cache" ];
trusted-public-keys = lib.mkForce [ (builtins.readFile ./cache-keys/pubkey) ];
};
};
};
testScript = ''
import time
start_all()
cache.wait_for_unit("nginx.service")
cache.wait_for_unit("tvix-store-cache.service")
time.sleep(1)
def delete_and_substitute():
client.succeed("nix-store --delete ${hello}")
client.fail("stat ${hello}")
client.succeed("nix-store -r ${hello}")
client.succeed("stat ${hello}")
with subtest("Upload"):
cache.succeed("nix store sign -k ${./cache-keys/privkey} ${hello}")
cache.succeed("nix copy --to file:///srv ${hello}")
narHash = "${hello}"[11:11+32]
out = cache.succeed(f"curl -f 'http://127.0.0.1/upstream/{narHash}.narinfo'")
print(out)
cache.succeed(f"curl -f 'http://127.0.0.1/cache/{narHash}.narinfo'")
with subtest("Try to substitute from cache"):
delete_and_substitute()
with subtest("Check effective caching"):
cache.succeed("rm -rf /srv")
delete_and_substitute()
'';
})

87
tests/upstream-cache-nix-serve.nix Normal file
View file

@ -0,0 +1,87 @@
{ pkgs }:
let
#hello = pkgs.hello.overrideAttrs { pname = "custom-hello"; };
inherit (pkgs) hello;
in
pkgs.testers.runNixOSTest (_: {
name = "caching of upstream nar-store";
nodes = {
cache = {
imports = [
./common
../modules
];
system.extraDependencies = [ hello ];
services = {
tvix-binary-cache = {
enable = true;
enableNginx = true;
nginx = {
clientMaxBodySize = "50G";
host = "cache";
};
caches = {
cache = {
port = 8000;
remote-path-info-service-addr = "nix+http://localhost/upstream/"; # ?trusted-public-keys=${lib.escapeURL (builtins.readFile ./cache-keys/pubkey)}";
};
};
};
nix-serve = {
enable = true;
port = 8001;
secretKeyFile = "${./cache-keys/privkey}";
};
nginx = {
virtualHosts.cache = {
default = true;
locations = {
"/upstream".return = "302 /upstream/";
"/upstream/".proxyPass = "http://localhost:8001/";
};
};
};
};
networking.firewall.allowedTCPPorts = [ 80 ];
};
client =
{ lib, ... }:
{
imports = [ ./common ];
nix.settings = {
substituters = lib.mkForce [ "http://cache/cache" ];
trusted-public-keys = lib.mkForce [ (builtins.readFile ./cache-keys/pubkey) ];
};
};
};
testScript = ''
import time
start_all()
cache.wait_for_unit("nginx.service")
cache.wait_for_unit("tvix-store-cache.service")
cache.wait_for_unit("nix-serve.service")
cache.wait_for_open_port(8001)
time.sleep(1)
def delete_and_substitute():
client.succeed("nix-store --delete ${hello}")
client.fail("stat ${hello}")
client.succeed("nix-store -r ${hello}")
client.succeed("stat ${hello}")
with subtest("Upload"):
narHash = "${hello}"[11:11+32]
cache.succeed(f"curl -f 'http://127.0.0.1/upstream/{narHash}.narinfo'")
cache.succeed(f"curl -f 'http://127.0.0.1/cache/{narHash}.narinfo'")
with subtest("Try to substitute from cache"):
delete_and_substitute()
with subtest("Check effective caching"):
cache.succeed("systemctl stop nix-serve.service")
delete_and_substitute()
'';
})

79
tests/upstream-cache-tvix.nix Normal file
View file

@ -0,0 +1,79 @@
{ pkgs }:
let
inherit (pkgs) hello;
in
pkgs.testers.runNixOSTest (_: {
name = "caching of upstream nar-store";
nodes = {
cache = {
imports = [
./common
../modules
];
system.extraDependencies = [ hello ];
services.tvix-binary-cache = {
enable = true;
enableNginx = true;
nginx = {
clientMaxBodySize = "50G";
host = "cache";
};
caches = {
cache = {
port = 8000;
path-info-service-addr = "nix+http://localhost/upstream/"; # ?trusted-public-keys=${lib.escapeURL (builtins.readFile ./cache-keys/pubkey)}";
};
upstream = {
port = 8001;
blob-service-addr = "objectstore+file://%S/tvix-store-upstream/blobs.object-store";
directory-service-addr = "sled://%S/tvix-store-upstream/directories.sled";
};
};
};
services.nginx = {
virtualHosts.cache = {
default = true;
};
};
networking.firewall.allowedTCPPorts = [ 80 ];
};
client =
{ lib, ... }:
{
imports = [ ./common ];
nix.settings = {
substituters = lib.mkForce [ "http://cache/cache" ];
trusted-public-keys = lib.mkForce [ (builtins.readFile ./cache-keys/pubkey) ];
};
};
};
testScript = ''
import time
start_all()
cache.wait_for_unit("nginx.service")
cache.wait_for_unit("tvix-store-cache.service")
cache.wait_for_unit("tvix-store-upstream.service")
time.sleep(1)
def delete_and_substitute():
client.succeed("nix-store --delete ${hello}")
client.fail("stat ${hello}")
client.succeed("nix-store -r ${hello}")
client.succeed("stat ${hello}")
with subtest("Upload"):
cache.succeed("nix store sign -k ${./cache-keys/privkey} ${hello}")
cache.succeed("nix copy --to 'http://127.0.0.1/upstream/?compression=none' ${hello}")
narHash = "${hello}"[11:11+32]
cache.succeed(f"curl -f 'http://127.0.0.1/upstream/{narHash}.narinfo'")
cache.succeed(f"curl -f 'http://127.0.0.1/cache/{narHash}.narinfo'")
with subtest("Try to substitute from cache"):
delete_and_substitute()
with subtest("Check effective caching"):
cache.succeed("systemctl stop tvix-store-upstream.service")
delete_and_substitute()
'';
})