forked from yu-re-ka/binary-cache
feat(module): Switch to rust nar bridge
This commit is contained in:
parent
5b0e841351
commit
b9f8a02e3b
|
@ -98,8 +98,6 @@ in
|
|||
systemd.services = lib.mkMerge (
|
||||
(lib.singleton {
|
||||
tvix-castore = {
|
||||
wants = [ "tvix-castore.service" ];
|
||||
after = [ "tvix-castore.service" ];
|
||||
environment = {
|
||||
BLOB_SERVICE_ADDR = cfg.blob-service-addr;
|
||||
DIRECTORY_SERVICE_ADDR = cfg.directory-service-addr;
|
||||
|
@ -121,18 +119,8 @@ in
|
|||
unitName = "tvix-store-${cfg.name}";
|
||||
in
|
||||
{
|
||||
"nar-bridge-${cfg.name}" = {
|
||||
wants = [ "tvix-store-${cfg.name}.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "tvix-store-${cfg.name}.service" ];
|
||||
serviceConfig = rec {
|
||||
ExecStart = "${lib.getExe pkgs.nar-bridge-go} --otlp=false --listen-addr=\"[::1]:${builtins.toString cfg.port}\" --store-addr=\"unix://%t/${unitName}/socket\"";
|
||||
|
||||
DynamicUser = true;
|
||||
User = "tvix-binary-cache";
|
||||
} // systemdHardening;
|
||||
};
|
||||
${unitName} = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
wants = [ "tvix-castore.service" ];
|
||||
after = [ "tvix-castore.service" ];
|
||||
environment = {
|
||||
|
@ -152,7 +140,7 @@ in
|
|||
) cfg.remote-path-info-service-addr;
|
||||
};
|
||||
serviceConfig = {
|
||||
ExecStart = "${pkgs.tvix-store}/bin/tvix-store --otlp=false daemon --listen-address=\"%t/${unitName}/socket\"";
|
||||
ExecStart = "${pkgs.nar-bridge}/bin/nar-bridge --otlp=false --listen-address=\"[::1]:${builtins.toString cfg.port}\"";
|
||||
DynamicUser = true;
|
||||
User = "tvix-binary-cache";
|
||||
StateDirectory = unitName;
|
||||
|
|
|
@ -4,7 +4,13 @@
|
|||
pkgs ? import nixpkgs { overlays = [ (import ../pkgs/overlay.nix) ]; },
|
||||
}:
|
||||
{
|
||||
# Disabled since nar-bridge doesn't expose gRPC
|
||||
# ingest = pkgs.callPackage ./ingest.nix { };
|
||||
multi-cache = pkgs.callPackage ./multi-cache.nix { };
|
||||
ingest = pkgs.callPackage ./ingest.nix { };
|
||||
substitution = pkgs.callPackage ./substitution.nix { };
|
||||
upstream-cache = {
|
||||
tvix = pkgs.callPackage ./upstream-cache-tvix.nix { };
|
||||
http-directory = pkgs.callPackage ./upstream-cache-http-directory.nix { };
|
||||
nix-serve = pkgs.callPackage ./upstream-cache-nix-serve.nix { };
|
||||
};
|
||||
}
|
||||
|
|
|
@ -41,7 +41,7 @@ pkgs.testers.runNixOSTest (_: {
|
|||
import time
|
||||
start_all()
|
||||
cache.wait_for_unit("nginx.service")
|
||||
cache.wait_for_unit("nar-bridge-cache.service")
|
||||
cache.wait_for_unit("tvix-store-cache.service")
|
||||
time.sleep(1)
|
||||
socket_addr = "grpc+unix:///run/tvix-store-cache/socket"
|
||||
cache.succeed(f"BLOB_SERVICE_ADDR={socket_addr} DIRECTORY_SERVICE_ADDR={socket_addr} PATH_INFO_SERVICE_ADDR={socket_addr} tvix-store copy ${builtins.toString references}")
|
||||
|
|
|
@ -35,23 +35,19 @@ pkgs.testers.runNixOSTest (_: {
|
|||
};
|
||||
};
|
||||
testScript = ''
|
||||
import sys
|
||||
import time
|
||||
start_all()
|
||||
machine.wait_for_unit("nginx.service")
|
||||
machine.wait_for_unit("nar-bridge-one.service")
|
||||
machine.wait_for_unit("nar-bridge-two.service")
|
||||
machine.wait_for_unit("tvix-store-one.service")
|
||||
machine.wait_for_unit("tvix-store-two.service")
|
||||
time.sleep(1)
|
||||
with subtest("Nar bridge home"):
|
||||
out = machine.succeed("curl -L http://127.0.0.1/one")
|
||||
if out != "nar-bridge":
|
||||
sys.exit(1)
|
||||
machine.succeed("curl -L http://127.0.0.1/one/nix-cache-info")
|
||||
with subtest("Nar upload"):
|
||||
machine.succeed("nix copy --to 'http://127.0.0.1/one/?compression=none' ${hello}")
|
||||
with subtest("narinfo retrieve"):
|
||||
narHash = "${hello}"[11:11+32]
|
||||
machine.succeed(f"curl -f 'http://127.0.0.1/one/{narHash}.narinfo'")
|
||||
machine.fail(f"curl -f 'http://127.0.0.1/two/{narHash}.narinfo'")
|
||||
|
||||
'';
|
||||
})
|
||||
|
|
|
@ -42,16 +42,13 @@ pkgs.testers.runNixOSTest (_: {
|
|||
};
|
||||
};
|
||||
testScript = ''
|
||||
import sys
|
||||
import time
|
||||
start_all()
|
||||
cache.wait_for_unit("nginx.service")
|
||||
cache.wait_for_unit("nar-bridge-cache.service")
|
||||
cache.wait_for_unit("tvix-store-cache.service")
|
||||
time.sleep(1)
|
||||
with subtest("Nar bridge home"):
|
||||
out = cache.succeed("curl -L http://127.0.0.1/cache")
|
||||
if out != "nar-bridge":
|
||||
sys.exit(1)
|
||||
cache.succeed("curl -f http://127.0.0.1/cache/nix-cache-info")
|
||||
with subtest("Path signature and copy"):
|
||||
# Sign
|
||||
cache.succeed("nix store sign -k ${./cache-keys/privkey} ${hello}")
|
||||
|
|
81
tests/upstream-cache-http-directory.nix
Normal file
81
tests/upstream-cache-http-directory.nix
Normal file
|
@ -0,0 +1,81 @@
|
|||
{ pkgs }:
|
||||
let
|
||||
#hello = pkgs.hello.overrideAttrs { pname = "custom-hello"; };
|
||||
inherit (pkgs) hello;
|
||||
in
|
||||
pkgs.testers.runNixOSTest (_: {
|
||||
name = "caching of upstream nar-store";
|
||||
nodes = {
|
||||
cache = {
|
||||
imports = [
|
||||
./common
|
||||
../modules
|
||||
];
|
||||
|
||||
system.extraDependencies = [ hello ];
|
||||
|
||||
services = {
|
||||
tvix-binary-cache = {
|
||||
enable = true;
|
||||
enableNginx = true;
|
||||
nginx = {
|
||||
clientMaxBodySize = "50G";
|
||||
host = "cache";
|
||||
};
|
||||
caches = {
|
||||
cache = {
|
||||
port = 8000;
|
||||
remote-path-info-service-addr = "nix+http://localhost/upstream/"; # ?trusted-public-keys=${lib.escapeURL (builtins.readFile ./cache-keys/pubkey)}";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
nginx = {
|
||||
virtualHosts.cache = {
|
||||
default = true;
|
||||
locations = {
|
||||
"/upstream".return = "302 /upstream/";
|
||||
"/upstream/".alias = "/srv/";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
networking.firewall.allowedTCPPorts = [ 80 ];
|
||||
};
|
||||
client =
|
||||
{ lib, ... }:
|
||||
{
|
||||
imports = [ ./common ];
|
||||
nix.settings = {
|
||||
substituters = lib.mkForce [ "http://cache/cache" ];
|
||||
trusted-public-keys = lib.mkForce [ (builtins.readFile ./cache-keys/pubkey) ];
|
||||
};
|
||||
};
|
||||
};
|
||||
testScript = ''
|
||||
import time
|
||||
start_all()
|
||||
cache.wait_for_unit("nginx.service")
|
||||
cache.wait_for_unit("tvix-store-cache.service")
|
||||
time.sleep(1)
|
||||
|
||||
def delete_and_substitute():
|
||||
client.succeed("nix-store --delete ${hello}")
|
||||
client.fail("stat ${hello}")
|
||||
client.succeed("nix-store -r ${hello}")
|
||||
client.succeed("stat ${hello}")
|
||||
|
||||
with subtest("Upload"):
|
||||
cache.succeed("nix store sign -k ${./cache-keys/privkey} ${hello}")
|
||||
cache.succeed("nix copy --to file:///srv ${hello}")
|
||||
narHash = "${hello}"[11:11+32]
|
||||
out = cache.succeed(f"curl -f 'http://127.0.0.1/upstream/{narHash}.narinfo'")
|
||||
print(out)
|
||||
cache.succeed(f"curl -f 'http://127.0.0.1/cache/{narHash}.narinfo'")
|
||||
with subtest("Try to substitute from cache"):
|
||||
delete_and_substitute()
|
||||
with subtest("Check effective caching"):
|
||||
cache.succeed("rm -rf /srv")
|
||||
delete_and_substitute()
|
||||
'';
|
||||
})
|
87
tests/upstream-cache-nix-serve.nix
Normal file
87
tests/upstream-cache-nix-serve.nix
Normal file
|
@ -0,0 +1,87 @@
|
|||
{ pkgs }:
|
||||
let
|
||||
#hello = pkgs.hello.overrideAttrs { pname = "custom-hello"; };
|
||||
inherit (pkgs) hello;
|
||||
in
|
||||
pkgs.testers.runNixOSTest (_: {
|
||||
name = "caching of upstream nar-store";
|
||||
nodes = {
|
||||
cache = {
|
||||
imports = [
|
||||
./common
|
||||
../modules
|
||||
];
|
||||
|
||||
system.extraDependencies = [ hello ];
|
||||
|
||||
services = {
|
||||
tvix-binary-cache = {
|
||||
enable = true;
|
||||
enableNginx = true;
|
||||
nginx = {
|
||||
clientMaxBodySize = "50G";
|
||||
host = "cache";
|
||||
};
|
||||
caches = {
|
||||
cache = {
|
||||
port = 8000;
|
||||
remote-path-info-service-addr = "nix+http://localhost/upstream/"; # ?trusted-public-keys=${lib.escapeURL (builtins.readFile ./cache-keys/pubkey)}";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
nix-serve = {
|
||||
enable = true;
|
||||
port = 8001;
|
||||
secretKeyFile = "${./cache-keys/privkey}";
|
||||
};
|
||||
|
||||
nginx = {
|
||||
virtualHosts.cache = {
|
||||
default = true;
|
||||
locations = {
|
||||
"/upstream".return = "302 /upstream/";
|
||||
"/upstream/".proxyPass = "http://localhost:8001/";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
};
|
||||
networking.firewall.allowedTCPPorts = [ 80 ];
|
||||
};
|
||||
client =
|
||||
{ lib, ... }:
|
||||
{
|
||||
imports = [ ./common ];
|
||||
nix.settings = {
|
||||
substituters = lib.mkForce [ "http://cache/cache" ];
|
||||
trusted-public-keys = lib.mkForce [ (builtins.readFile ./cache-keys/pubkey) ];
|
||||
};
|
||||
};
|
||||
};
|
||||
testScript = ''
|
||||
import time
|
||||
start_all()
|
||||
cache.wait_for_unit("nginx.service")
|
||||
cache.wait_for_unit("tvix-store-cache.service")
|
||||
cache.wait_for_unit("nix-serve.service")
|
||||
cache.wait_for_open_port(8001)
|
||||
time.sleep(1)
|
||||
|
||||
def delete_and_substitute():
|
||||
client.succeed("nix-store --delete ${hello}")
|
||||
client.fail("stat ${hello}")
|
||||
client.succeed("nix-store -r ${hello}")
|
||||
client.succeed("stat ${hello}")
|
||||
|
||||
with subtest("Upload"):
|
||||
narHash = "${hello}"[11:11+32]
|
||||
cache.succeed(f"curl -f 'http://127.0.0.1/upstream/{narHash}.narinfo'")
|
||||
cache.succeed(f"curl -f 'http://127.0.0.1/cache/{narHash}.narinfo'")
|
||||
with subtest("Try to substitute from cache"):
|
||||
delete_and_substitute()
|
||||
with subtest("Check effective caching"):
|
||||
cache.succeed("systemctl stop nix-serve.service")
|
||||
delete_and_substitute()
|
||||
'';
|
||||
})
|
79
tests/upstream-cache-tvix.nix
Normal file
79
tests/upstream-cache-tvix.nix
Normal file
|
@ -0,0 +1,79 @@
|
|||
{ pkgs }:
|
||||
let
|
||||
inherit (pkgs) hello;
|
||||
in
|
||||
pkgs.testers.runNixOSTest (_: {
|
||||
name = "caching of upstream nar-store";
|
||||
nodes = {
|
||||
cache = {
|
||||
imports = [
|
||||
./common
|
||||
../modules
|
||||
];
|
||||
|
||||
system.extraDependencies = [ hello ];
|
||||
|
||||
services.tvix-binary-cache = {
|
||||
enable = true;
|
||||
enableNginx = true;
|
||||
nginx = {
|
||||
clientMaxBodySize = "50G";
|
||||
host = "cache";
|
||||
};
|
||||
caches = {
|
||||
cache = {
|
||||
port = 8000;
|
||||
path-info-service-addr = "nix+http://localhost/upstream/"; # ?trusted-public-keys=${lib.escapeURL (builtins.readFile ./cache-keys/pubkey)}";
|
||||
};
|
||||
upstream = {
|
||||
port = 8001;
|
||||
blob-service-addr = "objectstore+file://%S/tvix-store-upstream/blobs.object-store";
|
||||
directory-service-addr = "sled://%S/tvix-store-upstream/directories.sled";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
services.nginx = {
|
||||
virtualHosts.cache = {
|
||||
default = true;
|
||||
};
|
||||
};
|
||||
networking.firewall.allowedTCPPorts = [ 80 ];
|
||||
};
|
||||
client =
|
||||
{ lib, ... }:
|
||||
{
|
||||
imports = [ ./common ];
|
||||
nix.settings = {
|
||||
substituters = lib.mkForce [ "http://cache/cache" ];
|
||||
trusted-public-keys = lib.mkForce [ (builtins.readFile ./cache-keys/pubkey) ];
|
||||
};
|
||||
};
|
||||
};
|
||||
testScript = ''
|
||||
import time
|
||||
start_all()
|
||||
cache.wait_for_unit("nginx.service")
|
||||
cache.wait_for_unit("tvix-store-cache.service")
|
||||
cache.wait_for_unit("tvix-store-upstream.service")
|
||||
time.sleep(1)
|
||||
|
||||
def delete_and_substitute():
|
||||
client.succeed("nix-store --delete ${hello}")
|
||||
client.fail("stat ${hello}")
|
||||
client.succeed("nix-store -r ${hello}")
|
||||
client.succeed("stat ${hello}")
|
||||
|
||||
with subtest("Upload"):
|
||||
cache.succeed("nix store sign -k ${./cache-keys/privkey} ${hello}")
|
||||
cache.succeed("nix copy --to 'http://127.0.0.1/upstream/?compression=none' ${hello}")
|
||||
narHash = "${hello}"[11:11+32]
|
||||
cache.succeed(f"curl -f 'http://127.0.0.1/upstream/{narHash}.narinfo'")
|
||||
cache.succeed(f"curl -f 'http://127.0.0.1/cache/{narHash}.narinfo'")
|
||||
with subtest("Try to substitute from cache"):
|
||||
delete_and_substitute()
|
||||
with subtest("Check effective caching"):
|
||||
cache.succeed("systemctl stop tvix-store-upstream.service")
|
||||
delete_and_substitute()
|
||||
'';
|
||||
})
|
Loading…
Reference in a new issue