diff --git a/modules/default.nix b/modules/default.nix index e166f11..e340cc1 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -98,8 +98,6 @@ in systemd.services = lib.mkMerge ( (lib.singleton { tvix-castore = { - wants = [ "tvix-castore.service" ]; - after = [ "tvix-castore.service" ]; environment = { BLOB_SERVICE_ADDR = cfg.blob-service-addr; DIRECTORY_SERVICE_ADDR = cfg.directory-service-addr; @@ -121,18 +119,8 @@ in unitName = "tvix-store-${cfg.name}"; in { - "nar-bridge-${cfg.name}" = { - wants = [ "tvix-store-${cfg.name}.service" ]; - wantedBy = [ "multi-user.target" ]; - after = [ "tvix-store-${cfg.name}.service" ]; - serviceConfig = rec { - ExecStart = "${lib.getExe pkgs.nar-bridge-go} --otlp=false --listen-addr=\"[::1]:${builtins.toString cfg.port}\" --store-addr=\"unix://%t/${unitName}/socket\""; - - DynamicUser = true; - User = "tvix-binary-cache"; - } // systemdHardening; - }; ${unitName} = { + wantedBy = [ "multi-user.target" ]; wants = [ "tvix-castore.service" ]; after = [ "tvix-castore.service" ]; environment = { @@ -152,7 +140,7 @@ in ) cfg.remote-path-info-service-addr; }; serviceConfig = { - ExecStart = "${pkgs.tvix-store}/bin/tvix-store --otlp=false daemon --listen-address=\"%t/${unitName}/socket\""; + ExecStart = "${pkgs.nar-bridge}/bin/nar-bridge --otlp=false --listen-address=\"[::1]:${builtins.toString cfg.port}\""; DynamicUser = true; User = "tvix-binary-cache"; StateDirectory = unitName; diff --git a/tests/default.nix b/tests/default.nix index 54f18a0..b133aec 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -4,7 +4,13 @@ pkgs ? import nixpkgs { overlays = [ (import ../pkgs/overlay.nix) ]; }, }: { + # Disabled since nar-bridge doesn't expose gRPC + # ingest = pkgs.callPackage ./ingest.nix { }; multi-cache = pkgs.callPackage ./multi-cache.nix { }; - ingest = pkgs.callPackage ./ingest.nix { }; substitution = pkgs.callPackage ./substitution.nix { }; + upstream-cache = { + tvix = pkgs.callPackage ./upstream-cache-tvix.nix { }; + http-directory = pkgs.callPackage ./upstream-cache-http-directory.nix { }; + nix-serve = pkgs.callPackage ./upstream-cache-nix-serve.nix { }; + }; } diff --git a/tests/ingest.nix b/tests/ingest.nix index 2888637..df20451 100644 --- a/tests/ingest.nix +++ b/tests/ingest.nix @@ -41,7 +41,7 @@ pkgs.testers.runNixOSTest (_: { import time start_all() cache.wait_for_unit("nginx.service") - cache.wait_for_unit("nar-bridge-cache.service") + cache.wait_for_unit("tvix-store-cache.service") time.sleep(1) socket_addr = "grpc+unix:///run/tvix-store-cache/socket" cache.succeed(f"BLOB_SERVICE_ADDR={socket_addr} DIRECTORY_SERVICE_ADDR={socket_addr} PATH_INFO_SERVICE_ADDR={socket_addr} tvix-store copy ${builtins.toString references}") diff --git a/tests/multi-cache.nix b/tests/multi-cache.nix index 90b632d..7f086c9 100644 --- a/tests/multi-cache.nix +++ b/tests/multi-cache.nix @@ -35,23 +35,19 @@ pkgs.testers.runNixOSTest (_: { }; }; testScript = '' - import sys import time start_all() machine.wait_for_unit("nginx.service") - machine.wait_for_unit("nar-bridge-one.service") - machine.wait_for_unit("nar-bridge-two.service") + machine.wait_for_unit("tvix-store-one.service") + machine.wait_for_unit("tvix-store-two.service") time.sleep(1) with subtest("Nar bridge home"): - out = machine.succeed("curl -L http://127.0.0.1/one") - if out != "nar-bridge": - sys.exit(1) + machine.succeed("curl -L http://127.0.0.1/one/nix-cache-info") with subtest("Nar upload"): machine.succeed("nix copy --to 'http://127.0.0.1/one/?compression=none' ${hello}") with subtest("narinfo retrieve"): narHash = "${hello}"[11:11+32] machine.succeed(f"curl -f 'http://127.0.0.1/one/{narHash}.narinfo'") machine.fail(f"curl -f 'http://127.0.0.1/two/{narHash}.narinfo'") - ''; }) diff --git a/tests/substitution.nix b/tests/substitution.nix index dd3b8ff..0cffbd5 100644 --- a/tests/substitution.nix +++ b/tests/substitution.nix @@ -42,16 +42,13 @@ pkgs.testers.runNixOSTest (_: { }; }; testScript = '' - import sys import time start_all() cache.wait_for_unit("nginx.service") - cache.wait_for_unit("nar-bridge-cache.service") + cache.wait_for_unit("tvix-store-cache.service") time.sleep(1) with subtest("Nar bridge home"): - out = cache.succeed("curl -L http://127.0.0.1/cache") - if out != "nar-bridge": - sys.exit(1) + cache.succeed("curl -f http://127.0.0.1/cache/nix-cache-info") with subtest("Path signature and copy"): # Sign cache.succeed("nix store sign -k ${./cache-keys/privkey} ${hello}") diff --git a/tests/upstream-cache-http-directory.nix b/tests/upstream-cache-http-directory.nix new file mode 100644 index 0000000..0b72719 --- /dev/null +++ b/tests/upstream-cache-http-directory.nix @@ -0,0 +1,81 @@ +{ pkgs }: +let + #hello = pkgs.hello.overrideAttrs { pname = "custom-hello"; }; + inherit (pkgs) hello; +in +pkgs.testers.runNixOSTest (_: { + name = "caching of upstream nar-store"; + nodes = { + cache = { + imports = [ + ./common + ../modules + ]; + + system.extraDependencies = [ hello ]; + + services = { + tvix-binary-cache = { + enable = true; + enableNginx = true; + nginx = { + clientMaxBodySize = "50G"; + host = "cache"; + }; + caches = { + cache = { + port = 8000; + remote-path-info-service-addr = "nix+http://localhost/upstream/"; # ?trusted-public-keys=${lib.escapeURL (builtins.readFile ./cache-keys/pubkey)}"; + }; + }; + }; + + nginx = { + virtualHosts.cache = { + default = true; + locations = { + "/upstream".return = "302 /upstream/"; + "/upstream/".alias = "/srv/"; + }; + }; + }; + }; + networking.firewall.allowedTCPPorts = [ 80 ]; + }; + client = + { lib, ... }: + { + imports = [ ./common ]; + nix.settings = { + substituters = lib.mkForce [ "http://cache/cache" ]; + trusted-public-keys = lib.mkForce [ (builtins.readFile ./cache-keys/pubkey) ]; + }; + }; + }; + testScript = '' + import time + start_all() + cache.wait_for_unit("nginx.service") + cache.wait_for_unit("tvix-store-cache.service") + time.sleep(1) + + def delete_and_substitute(): + client.succeed("nix-store --delete ${hello}") + client.fail("stat ${hello}") + client.succeed("nix-store -r ${hello}") + client.succeed("stat ${hello}") + + with subtest("Upload"): + cache.succeed("nix store sign -k ${./cache-keys/privkey} ${hello}") + cache.succeed("nix copy --to file:///srv ${hello}") + narHash = "${hello}"[11:11+32] + out = cache.succeed(f"curl -f 'http://127.0.0.1/upstream/{narHash}.narinfo'") + print(out) + cache.succeed(f"curl -f 'http://127.0.0.1/cache/{narHash}.narinfo'") + with subtest("Try to substitute from cache"): + delete_and_substitute() + with subtest("Check effective caching"): + cache.succeed("rm -rf /srv") + delete_and_substitute() + ''; +}) diff --git a/tests/upstream-cache-nix-serve.nix b/tests/upstream-cache-nix-serve.nix new file mode 100644 index 0000000..a9f78fb --- /dev/null +++ b/tests/upstream-cache-nix-serve.nix @@ -0,0 +1,87 @@ +{ pkgs }: +let + #hello = pkgs.hello.overrideAttrs { pname = "custom-hello"; }; + inherit (pkgs) hello; +in +pkgs.testers.runNixOSTest (_: { + name = "caching of upstream nar-store"; + nodes = { + cache = { + imports = [ + ./common + ../modules + ]; + + system.extraDependencies = [ hello ]; + + services = { + tvix-binary-cache = { + enable = true; + enableNginx = true; + nginx = { + clientMaxBodySize = "50G"; + host = "cache"; + }; + caches = { + cache = { + port = 8000; + remote-path-info-service-addr = "nix+http://localhost/upstream/"; # ?trusted-public-keys=${lib.escapeURL (builtins.readFile ./cache-keys/pubkey)}"; + }; + }; + }; + + nix-serve = { + enable = true; + port = 8001; + secretKeyFile = "${./cache-keys/privkey}"; + }; + + nginx = { + virtualHosts.cache = { + default = true; + locations = { + "/upstream".return = "302 /upstream/"; + "/upstream/".proxyPass = "http://localhost:8001/"; + }; + }; + }; + + }; + networking.firewall.allowedTCPPorts = [ 80 ]; + }; + client = + { lib, ... }: + { + imports = [ ./common ]; + nix.settings = { + substituters = lib.mkForce [ "http://cache/cache" ]; + trusted-public-keys = lib.mkForce [ (builtins.readFile ./cache-keys/pubkey) ]; + }; + }; + }; + testScript = '' + import time + start_all() + cache.wait_for_unit("nginx.service") + cache.wait_for_unit("tvix-store-cache.service") + cache.wait_for_unit("nix-serve.service") + cache.wait_for_open_port(8001) + time.sleep(1) + + def delete_and_substitute(): + client.succeed("nix-store --delete ${hello}") + client.fail("stat ${hello}") + client.succeed("nix-store -r ${hello}") + client.succeed("stat ${hello}") + + with subtest("Upload"): + narHash = "${hello}"[11:11+32] + cache.succeed(f"curl -f 'http://127.0.0.1/upstream/{narHash}.narinfo'") + cache.succeed(f"curl -f 'http://127.0.0.1/cache/{narHash}.narinfo'") + with subtest("Try to substitute from cache"): + delete_and_substitute() + with subtest("Check effective caching"): + cache.succeed("systemctl stop nix-serve.service") + delete_and_substitute() + ''; +}) diff --git a/tests/upstream-cache-tvix.nix b/tests/upstream-cache-tvix.nix new file mode 100644 index 0000000..6988b46 --- /dev/null +++ b/tests/upstream-cache-tvix.nix @@ -0,0 +1,79 @@ +{ pkgs }: +let + inherit (pkgs) hello; +in +pkgs.testers.runNixOSTest (_: { + name = "caching of upstream nar-store"; + nodes = { + cache = { + imports = [ + ./common + ../modules + ]; + + system.extraDependencies = [ hello ]; + + services.tvix-binary-cache = { + enable = true; + enableNginx = true; + nginx = { + clientMaxBodySize = "50G"; + host = "cache"; + }; + caches = { + cache = { + port = 8000; + path-info-service-addr = "nix+http://localhost/upstream/"; # ?trusted-public-keys=${lib.escapeURL (builtins.readFile ./cache-keys/pubkey)}"; + }; + upstream = { + port = 8001; + blob-service-addr = "objectstore+file://%S/tvix-store-upstream/blobs.object-store"; + directory-service-addr = "sled://%S/tvix-store-upstream/directories.sled"; + }; + }; + }; + + services.nginx = { + virtualHosts.cache = { + default = true; + }; + }; + networking.firewall.allowedTCPPorts = [ 80 ]; + }; + client = + { lib, ... }: + { + imports = [ ./common ]; + nix.settings = { + substituters = lib.mkForce [ "http://cache/cache" ]; + trusted-public-keys = lib.mkForce [ (builtins.readFile ./cache-keys/pubkey) ]; + }; + }; + }; + testScript = '' + import time + start_all() + cache.wait_for_unit("nginx.service") + cache.wait_for_unit("tvix-store-cache.service") + cache.wait_for_unit("tvix-store-upstream.service") + time.sleep(1) + + def delete_and_substitute(): + client.succeed("nix-store --delete ${hello}") + client.fail("stat ${hello}") + client.succeed("nix-store -r ${hello}") + client.succeed("stat ${hello}") + + with subtest("Upload"): + cache.succeed("nix store sign -k ${./cache-keys/privkey} ${hello}") + cache.succeed("nix copy --to 'http://127.0.0.1/upstream/?compression=none' ${hello}") + narHash = "${hello}"[11:11+32] + cache.succeed(f"curl -f 'http://127.0.0.1/upstream/{narHash}.narinfo'") + cache.succeed(f"curl -f 'http://127.0.0.1/cache/{narHash}.narinfo'") + with subtest("Try to substitute from cache"): + delete_and_substitute() + with subtest("Check effective caching"): + cache.succeed("systemctl stop tvix-store-upstream.service") + delete_and_substitute() + ''; +})