feat(module): Switch to rust nar bridge

modules/default.nix

@@ -98,8 +98,6 @@ in
     systemd.services = lib.mkMerge (
       (lib.singleton {
         tvix-castore = {
-          wants = [ "tvix-castore.service" ];
-          after = [ "tvix-castore.service" ];
           environment = {
             BLOB_SERVICE_ADDR = cfg.blob-service-addr;
             DIRECTORY_SERVICE_ADDR = cfg.directory-service-addr;
@@ -121,18 +119,8 @@ in
           unitName = "tvix-store-${cfg.name}";
         in
         {
-          "nar-bridge-${cfg.name}" = {
-            wants = [ "tvix-store-${cfg.name}.service" ];
-            wantedBy = [ "multi-user.target" ];
-            after = [ "tvix-store-${cfg.name}.service" ];
-            serviceConfig = rec {
-              ExecStart = "${lib.getExe pkgs.nar-bridge-go} --otlp=false --listen-addr=\"[::1]:${builtins.toString cfg.port}\" --store-addr=\"unix://%t/${unitName}/socket\"";
-
-              DynamicUser = true;
-              User = "tvix-binary-cache";
-            } // systemdHardening;
-          };
           ${unitName} = {
+            wantedBy = [ "multi-user.target" ];
             wants = [ "tvix-castore.service" ];
             after = [ "tvix-castore.service" ];
             environment = {
@@ -152,7 +140,7 @@ in
               ) cfg.remote-path-info-service-addr;
             };
             serviceConfig = {
-              ExecStart = "${pkgs.tvix-store}/bin/tvix-store --otlp=false daemon --listen-address=\"%t/${unitName}/socket\"";
+              ExecStart = "${pkgs.nar-bridge}/bin/nar-bridge --otlp=false --listen-address=\"[::1]:${builtins.toString cfg.port}\"";
               DynamicUser = true;
               User = "tvix-binary-cache";
               StateDirectory = unitName;

tests/default.nix

@@ -4,7 +4,13 @@
   pkgs ? import nixpkgs { overlays = [ (import ../pkgs/overlay.nix) ]; },
 }:
 {
+  # Disabled since nar-bridge doesn't expose gRPC
+  # ingest = pkgs.callPackage ./ingest.nix { };
   multi-cache = pkgs.callPackage ./multi-cache.nix { };
-  ingest = pkgs.callPackage ./ingest.nix { };
   substitution = pkgs.callPackage ./substitution.nix { };
+  upstream-cache = {
+    tvix = pkgs.callPackage ./upstream-cache-tvix.nix { };
+    http-directory = pkgs.callPackage ./upstream-cache-http-directory.nix { };
+    nix-serve = pkgs.callPackage ./upstream-cache-nix-serve.nix { };
+  };
 }

tests/ingest.nix

@@ -41,7 +41,7 @@ pkgs.testers.runNixOSTest (_: {
     import time
     start_all()
     cache.wait_for_unit("nginx.service")
-    cache.wait_for_unit("nar-bridge-cache.service")
+    cache.wait_for_unit("tvix-store-cache.service")
     time.sleep(1)
     socket_addr = "grpc+unix:///run/tvix-store-cache/socket"
     cache.succeed(f"BLOB_SERVICE_ADDR={socket_addr} DIRECTORY_SERVICE_ADDR={socket_addr} PATH_INFO_SERVICE_ADDR={socket_addr} tvix-store copy ${builtins.toString references}")

tests/multi-cache.nix

@@ -35,23 +35,19 @@ pkgs.testers.runNixOSTest (_: {
     };
   };
   testScript = ''
-    import sys
     import time
     start_all()
     machine.wait_for_unit("nginx.service")
-    machine.wait_for_unit("nar-bridge-one.service")
+    machine.wait_for_unit("tvix-store-one.service")
-    machine.wait_for_unit("nar-bridge-two.service")
+    machine.wait_for_unit("tvix-store-two.service")
     time.sleep(1)
     with subtest("Nar bridge home"):
-        out = machine.succeed("curl -L http://127.0.0.1/one")
+        machine.succeed("curl -L http://127.0.0.1/one/nix-cache-info")
-        if out != "nar-bridge":
-            sys.exit(1)
     with subtest("Nar upload"):
         machine.succeed("nix copy --to 'http://127.0.0.1/one/?compression=none' ${hello}")
     with subtest("narinfo retrieve"):
         narHash = "${hello}"[11:11+32]
         machine.succeed(f"curl -f 'http://127.0.0.1/one/{narHash}.narinfo'")
         machine.fail(f"curl -f 'http://127.0.0.1/two/{narHash}.narinfo'")
-
   '';
 })

tests/substitution.nix

@@ -42,16 +42,13 @@ pkgs.testers.runNixOSTest (_: {
       };
   };
   testScript = ''
-    import sys
     import time
     start_all()
     cache.wait_for_unit("nginx.service")
-    cache.wait_for_unit("nar-bridge-cache.service")
+    cache.wait_for_unit("tvix-store-cache.service")
     time.sleep(1)
     with subtest("Nar bridge home"):
-        out = cache.succeed("curl -L http://127.0.0.1/cache")
+        cache.succeed("curl -f http://127.0.0.1/cache/nix-cache-info")
-        if out != "nar-bridge":
-            sys.exit(1)
     with subtest("Path signature and copy"):
         # Sign
         cache.succeed("nix store sign -k ${./cache-keys/privkey} ${hello}")

tests/upstream-cache-http-directory.nix

@@ -0,0 +1,81 @@
+{ pkgs }:
+let
+  #hello = pkgs.hello.overrideAttrs { pname = "custom-hello"; };
+  inherit (pkgs) hello;
+in
+pkgs.testers.runNixOSTest (_: {
+  name = "caching of upstream nar-store";
+  nodes = {
+    cache = {
+      imports = [
+        ./common
+        ../modules
+      ];
+
+      system.extraDependencies = [ hello ];
+
+      services = {
+        tvix-binary-cache = {
+          enable = true;
+          enableNginx = true;
+          nginx = {
+            clientMaxBodySize = "50G";
+            host = "cache";
+          };
+          caches = {
+            cache = {
+              port = 8000;
+              remote-path-info-service-addr = "nix+http://localhost/upstream/"; # ?trusted-public-keys=${lib.escapeURL (builtins.readFile ./cache-keys/pubkey)}";
+            };
+          };
+        };
+
+        nginx = {
+          virtualHosts.cache = {
+            default = true;
+            locations = {
+              "/upstream".return = "302 /upstream/";
+              "/upstream/".alias = "/srv/";
+            };
+          };
+        };
+      };
+      networking.firewall.allowedTCPPorts = [ 80 ];
+    };
+    client =
+      { lib, ... }:
+      {
+        imports = [ ./common ];
+        nix.settings = {
+          substituters = lib.mkForce [ "http://cache/cache" ];
+          trusted-public-keys = lib.mkForce [ (builtins.readFile ./cache-keys/pubkey) ];
+        };
+      };
+  };
+  testScript = ''
+    import time
+    start_all()
+    cache.wait_for_unit("nginx.service")
+    cache.wait_for_unit("tvix-store-cache.service")
+    time.sleep(1)
+
+    def delete_and_substitute():
+        client.succeed("nix-store --delete ${hello}")
+        client.fail("stat ${hello}")
+        client.succeed("nix-store -r ${hello}")
+        client.succeed("stat ${hello}")
+
+    with subtest("Upload"):
+        cache.succeed("nix store sign -k ${./cache-keys/privkey} ${hello}")
+        cache.succeed("nix copy --to file:///srv ${hello}")
+        narHash = "${hello}"[11:11+32]
+        out = cache.succeed(f"curl -f 'http://127.0.0.1/upstream/{narHash}.narinfo'")
+        print(out)
+        cache.succeed(f"curl -f 'http://127.0.0.1/cache/{narHash}.narinfo'")
+    with subtest("Try to substitute from cache"):
+        delete_and_substitute()
+    with subtest("Check effective caching"):
+        cache.succeed("rm -rf /srv")
+        delete_and_substitute()
+  '';
+})

tests/upstream-cache-nix-serve.nix

@@ -0,0 +1,87 @@
+{ pkgs }:
+let
+  #hello = pkgs.hello.overrideAttrs { pname = "custom-hello"; };
+  inherit (pkgs) hello;
+in
+pkgs.testers.runNixOSTest (_: {
+  name = "caching of upstream nar-store";
+  nodes = {
+    cache = {
+      imports = [
+        ./common
+        ../modules
+      ];
+
+      system.extraDependencies = [ hello ];
+
+      services = {
+        tvix-binary-cache = {
+          enable = true;
+          enableNginx = true;
+          nginx = {
+            clientMaxBodySize = "50G";
+            host = "cache";
+          };
+          caches = {
+            cache = {
+              port = 8000;
+              remote-path-info-service-addr = "nix+http://localhost/upstream/"; # ?trusted-public-keys=${lib.escapeURL (builtins.readFile ./cache-keys/pubkey)}";
+            };
+          };
+        };
+
+        nix-serve = {
+          enable = true;
+          port = 8001;
+          secretKeyFile = "${./cache-keys/privkey}";
+        };
+
+        nginx = {
+          virtualHosts.cache = {
+            default = true;
+            locations = {
+              "/upstream".return = "302 /upstream/";
+              "/upstream/".proxyPass = "http://localhost:8001/";
+            };
+          };
+        };
+
+      };
+      networking.firewall.allowedTCPPorts = [ 80 ];
+    };
+    client =
+      { lib, ... }:
+      {
+        imports = [ ./common ];
+        nix.settings = {
+          substituters = lib.mkForce [ "http://cache/cache" ];
+          trusted-public-keys = lib.mkForce [ (builtins.readFile ./cache-keys/pubkey) ];
+        };
+      };
+  };
+  testScript = ''
+    import time
+    start_all()
+    cache.wait_for_unit("nginx.service")
+    cache.wait_for_unit("tvix-store-cache.service")
+    cache.wait_for_unit("nix-serve.service")
+    cache.wait_for_open_port(8001)
+    time.sleep(1)
+
+    def delete_and_substitute():
+        client.succeed("nix-store --delete ${hello}")
+        client.fail("stat ${hello}")
+        client.succeed("nix-store -r ${hello}")
+        client.succeed("stat ${hello}")
+
+    with subtest("Upload"):
+        narHash = "${hello}"[11:11+32]
+        cache.succeed(f"curl -f 'http://127.0.0.1/upstream/{narHash}.narinfo'")
+        cache.succeed(f"curl -f 'http://127.0.0.1/cache/{narHash}.narinfo'")
+    with subtest("Try to substitute from cache"):
+        delete_and_substitute()
+    with subtest("Check effective caching"):
+        cache.succeed("systemctl stop nix-serve.service")
+        delete_and_substitute()
+  '';
+})

tests/upstream-cache-tvix.nix

@@ -0,0 +1,79 @@
+{ pkgs }:
+let
+  inherit (pkgs) hello;
+in
+pkgs.testers.runNixOSTest (_: {
+  name = "caching of upstream nar-store";
+  nodes = {
+    cache = {
+      imports = [
+        ./common
+        ../modules
+      ];
+
+      system.extraDependencies = [ hello ];
+
+      services.tvix-binary-cache = {
+        enable = true;
+        enableNginx = true;
+        nginx = {
+          clientMaxBodySize = "50G";
+          host = "cache";
+        };
+        caches = {
+          cache = {
+            port = 8000;
+            path-info-service-addr = "nix+http://localhost/upstream/"; # ?trusted-public-keys=${lib.escapeURL (builtins.readFile ./cache-keys/pubkey)}";
+          };
+          upstream = {
+            port = 8001;
+            blob-service-addr = "objectstore+file://%S/tvix-store-upstream/blobs.object-store";
+            directory-service-addr = "sled://%S/tvix-store-upstream/directories.sled";
+          };
+        };
+      };
+
+      services.nginx = {
+        virtualHosts.cache = {
+          default = true;
+        };
+      };
+      networking.firewall.allowedTCPPorts = [ 80 ];
+    };
+    client =
+      { lib, ... }:
+      {
+        imports = [ ./common ];
+        nix.settings = {
+          substituters = lib.mkForce [ "http://cache/cache" ];
+          trusted-public-keys = lib.mkForce [ (builtins.readFile ./cache-keys/pubkey) ];
+        };
+      };
+  };
+  testScript = ''
+    import time
+    start_all()
+    cache.wait_for_unit("nginx.service")
+    cache.wait_for_unit("tvix-store-cache.service")
+    cache.wait_for_unit("tvix-store-upstream.service")
+    time.sleep(1)
+
+    def delete_and_substitute():
+        client.succeed("nix-store --delete ${hello}")
+        client.fail("stat ${hello}")
+        client.succeed("nix-store -r ${hello}")
+        client.succeed("stat ${hello}")
+
+    with subtest("Upload"):
+        cache.succeed("nix store sign -k ${./cache-keys/privkey} ${hello}")
+        cache.succeed("nix copy --to 'http://127.0.0.1/upstream/?compression=none' ${hello}")
+        narHash = "${hello}"[11:11+32]
+        cache.succeed(f"curl -f 'http://127.0.0.1/upstream/{narHash}.narinfo'")
+        cache.succeed(f"curl -f 'http://127.0.0.1/cache/{narHash}.narinfo'")
+    with subtest("Try to substitute from cache"):
+        delete_and_substitute()
+    with subtest("Check effective caching"):
+        cache.succeed("systemctl stop tvix-store-upstream.service")
+        delete_and_substitute()
+  '';
+})