diff --git a/Cargo.lock b/Cargo.lock index d12be74..ac228da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -156,7 +156,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -177,7 +177,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -194,7 +194,7 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -291,7 +291,7 @@ checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -530,7 +530,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -669,7 +669,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -693,7 +693,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -704,7 +704,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -752,7 +752,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -829,7 +829,7 @@ checksum = "ba7795da175654fe16979af73f81f26a8ea27638d8d9823d317016888a63dc4c" dependencies = [ "num-traits", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -1006,7 +1006,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -1455,7 +1455,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -1795,8 +1795,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" [[package]] -name = "multitenant-tvix-cache" -version = "0.3.0" +name = "multitenant-tvix-binary-cache" +version = "0.4.0" dependencies = [ "axum", "clap", @@ -1923,7 +1923,7 @@ source = "git+https://git.dgnum.eu/mdebray/tvl-depot?rev=f3fdb02e6b56ed164b476a2 dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -1989,7 +1989,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -2196,7 +2196,7 @@ checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -2255,7 +2255,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -2292,8 +2292,8 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0f3e5beed80eb580c68e2c600937ac2c4eedabdfd5ef1e5b7ea4f3fba84497b" dependencies = [ - "heck 0.4.1", - "itertools 0.12.1", + "heck 0.5.0", + "itertools 0.13.0", "log", "multimap", "once_cell", @@ -2304,7 +2304,7 @@ dependencies = [ "pulldown-cmark", "pulldown-cmark-to-cmark", "regex", - "syn 2.0.91", + "syn 2.0.92", "tempfile", ] @@ -2315,10 +2315,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "157c5a9d7ea5c2ed2d9fb8f495b64759f7816c7eaea54ba3978f0d63000162e3" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -2351,7 +2351,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a669d5acbe719010c6f62a64e6d7d88fdedc1fe46e419747949ecb6312e9b14" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "prost", "prost-build", "prost-types", @@ -2853,7 +2853,7 @@ checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -2947,7 +2947,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -3099,9 +3099,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.91" +version = "2.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" +checksum = "70ae51629bf965c5c098cc9e87908a3df5301051a9e087d6f9bef5c9771ed126" dependencies = [ "proc-macro2", "quote", @@ -3125,7 +3125,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -3167,7 +3167,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -3178,7 +3178,7 @@ checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -3308,7 +3308,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -3439,7 +3439,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -3589,7 +3589,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -3841,7 +3841,7 @@ checksum = "d9d30226ac9cbd2d1ff775f74e8febdab985dab14fb14aa2582c29a92d5555dc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -4023,7 +4023,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", "wasm-bindgen-shared", ] @@ -4058,7 +4058,7 @@ checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4124,7 +4124,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -4386,7 +4386,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", "synstructure", ] @@ -4408,7 +4408,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] @@ -4428,7 +4428,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", "synstructure", ] @@ -4457,7 +4457,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.92", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 77043f5..c6fbb54 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "multitenant-tvix-cache" -version = "0.3.0" +name = "multitenant-tvix-binary-cache" +version = "0.4.0" edition = "2021" [dependencies] diff --git a/default.nix b/default.nix index a370621..4c2534d 100644 --- a/default.nix +++ b/default.nix @@ -1,6 +1,6 @@ { sources ? import ./npins, - pkgs ? import sources.nixpkgs { }, + pkgs ? import sources.nixpkgs {}, }: let check = (import sources.git-hooks).run { @@ -29,6 +29,7 @@ let in { inherit pkgs; + package = pkgs.callPackage ./pkgs {}; shell = pkgs.mkShell { name = "multitenant-tvix-binary-cache"; buildInputs = check.enabledPackages ++ [ diff --git a/module.nix b/module.nix new file mode 100644 index 0000000..b14185a --- /dev/null +++ b/module.nix @@ -0,0 +1,138 @@ +# SPDX-FileCopyrightText: 2024 Maurice Debray +# +# SPDX-License-Identifier: EUPL-1.2 + +{ pkgs, ... }: +let + + # How to add a cache: + # - Add the relevant services (likely only a pathinfoservice) to the + # composition config (store-config.composition). + # - Add an endpoint (store-config.endpoints). + # - Append a proxy configuration to nginx in order to make the store + # accessible. + store-config = { + composition = { + blobservices.default = { + type = "objectstore"; + object_store_url = "file://${dataDir}/blob.objectstore"; + object_store_options = { }; + }; + directoryservices = { + redb = { + type = "redb"; + is_temporary = false; + path = "${dataDir}/directory.redb"; + }; + }; + pathinfoservices = { + bare = { + type = "redb"; + is_temporary = false; + path = "${dataDir}/pathinfo.redb"; + }; + signing = { + type = "keyfile-signing"; + inner = "bare"; + keyfile = ./tests/cache-keys/privkey; + }; + }; + }; + + endpoints = { + "127.0.0.1:8056" = { + endpoint_type = "Http"; + blob_service = "default"; + directory_service = "redb"; + path_info_service = "bare"; + }; + "127.0.0.1:8058" = { + endpoint_type = "Http"; + blob_service = "default"; + directory_service = "redb"; + path_info_service = "signing"; + }; + # Add grpc for management and because it is nice + "127.0.0.1:8057" = { + endpoint_type = "Grpc"; + blob_service = "default"; + directory_service = "redb"; + path_info_service = "bare"; + }; + }; + }; + + settingsFormat = pkgs.formats.toml { }; + + dataDir = "/var/lib/tvix-store"; + + systemdHardening = { + PrivateDevices = true; + PrivateTmp = true; + ProtectControlGroups = true; + ProtectKernelTunables = true; + RestrictSUIDSGID = true; + + ProtectSystem = "strict"; + ProtectKernelLogs = true; + ProtectProc = "invisible"; + PrivateUsers = true; + ProtectHome = true; + UMask = "0077"; + RuntimeDirectoryMode = "0750"; + StateDirectoryMode = "0750"; + }; + + toml = { + composition = settingsFormat.generate "composition.toml" store-config.composition; + endpoints = settingsFormat.generate "endpoints.toml" store-config.endpoints; + }; +in +{ + services.nginx.enable = true; + services.nginx.virtualHosts."localhost" = { + default = true; + locations = { + "/" = { + proxyPass = "http://127.0.0.1:8056/"; + extraConfig = '' + client_max_body_size 50G; + ''; + }; + "/signing/" = { + proxyPass = "http://127.0.0.1:8058/"; + extraConfig = '' + client_max_body_size 50G; + ''; + }; + }; + }; + + users.users.tvix-store = { + isSystemUser = true; + group = "tvix-store"; + }; + users.groups.tvix-store = { }; + + #systemd.tmpfiles.rules = [ "d ${dataDir} 770 tvix-castore tvix-castore -" ]; + + systemd.services."tvix-store" = { + wantedBy = [ "multi-user.target" ]; + environment = { + RUST_LOG = "debug"; + }; + serviceConfig = { + UMask = "007"; + ExecStart = "${pkgs.multitenant-tvix-binary-cache}/bin/multitenant-tvix-binary-cache --endpoints-config ${toml.endpoints} --store-composition ${toml.composition}"; + StateDirectory = "tvix-store"; + RuntimeDirectory = "tvix-store"; + User = "tvix-store"; + Group = "tvix-store"; + ReadWritePaths = [ dataDir ]; + } // systemdHardening; + }; + networking.firewall.allowedTCPPorts = [ + 80 + 443 + ]; +} diff --git a/pkgs/default.nix b/pkgs/default.nix new file mode 100644 index 0000000..d3aa354 --- /dev/null +++ b/pkgs/default.nix @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: 2024 Maurice Debray +# +# SPDX-License-Identifier: EUPL-1.2 + +{ + lib, + fetchgit, + rustPlatform, + protobuf, + runCommand, +}: +let + + tvix-hash = "sha256-66bj0CK0vaW+kwqkOpTWjEaKoP2ka69WLuMoYeh5IWg="; + tvix-src = fetchgit { + url = "https://git.dgnum.eu/mdebray/tvl-depot.git"; + rev = "f3fdb02e6b56ed164b476a202bac27b7f6eda6cc"; + hash = tvix-hash; + }; + protos = runCommand "tvix-protos" { } '' + mkdir $out + cd ${tvix-src}/tvix + find . -name '*.proto' -exec install -D {} $out/{} \; + ''; +in + +rustPlatform.buildRustPackage rec { + pname = "multitenant-binary-cache"; + version = "0.1.0"; + + src = + let + inherit (lib.fileset) + difference + fileFilter + fileset + gitTracked + intersection + toSource + ; + + removeFilesets = lib.foldl difference; + in + toSource { + root = ../.; + fileset = + removeFilesets (intersection (gitTracked ../.) (fileFilter (file: !file.hasExt "nix") ../.)) + [ + ../tests + ../pkgs + ../npins + ]; + }; + + PROTO_ROOT = protos; + + nativeBuildInputs = [ protobuf ]; + + cargoLock = { + lockFile = ../Cargo.lock; + outputHashes = { + "bigtable_rs-0.2.10" = "sha256-2NC3rHbS2rdD0Rnovymn1xaR22KaR6yzWr298wOPxlY="; + "nar-bridge-0.1.0" = tvix-hash; + "reqwest-middleware-0.4.0" = "sha256-atWD4FFTcXPp28Y19w+VVbS1sB5pWlAtdw01irfu6oE="; + "wu-manber-0.1.0" = "sha256-7YIttaQLfFC/32utojh2DyOHVsZiw8ul/z0lvOhAE/4="; + }; + }; + + meta = { }; +} diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix new file mode 100644 index 0000000..4fad101 --- /dev/null +++ b/pkgs/overlay.nix @@ -0,0 +1,3 @@ +final: _: { + multitenant-tvix-binary-cache = final.callPackage ./. { }; +} diff --git a/src/main.rs b/src/main.rs index ecabb58..b058218 100644 --- a/src/main.rs +++ b/src/main.rs @@ -53,7 +53,7 @@ async fn main() -> Result<(), Box> { let endpoints_text = tokio::fs::read_to_string(cli.endpoints_config).await?; let endpoints: EndpointsConfig = toml::from_str(&endpoints_text)?; - info!("Starting multitenant-tvix-cache..."); + info!("Starting multitenant-tvix-binary-cache..."); let mut comp = Composition::new(®); diff --git a/tests/cache-keys/privkey b/tests/cache-keys/privkey new file mode 100644 index 0000000..ed213db --- /dev/null +++ b/tests/cache-keys/privkey @@ -0,0 +1 @@ +do.not.use-1:+Vtfnroj2hvBFPf5Vf6EXNn1kY9kJmgmrkOG23Qi451PUUA6e0PhhwoUpMybaMjNn2BhED781Jf9+NjPgUs1Lw== \ No newline at end of file diff --git a/tests/cache-keys/pubkey b/tests/cache-keys/pubkey new file mode 100644 index 0000000..417ee4b --- /dev/null +++ b/tests/cache-keys/pubkey @@ -0,0 +1 @@ +do.not.use-1:T1FAOntD4YcKFKTMm2jIzZ9gYRA+/NSX/fjYz4FLNS8= \ No newline at end of file diff --git a/tests/common/default.nix b/tests/common/default.nix new file mode 100644 index 0000000..2c42c4c --- /dev/null +++ b/tests/common/default.nix @@ -0,0 +1 @@ +{ imports = [ ./nix.nix ]; } diff --git a/tests/common/nix.nix b/tests/common/nix.nix new file mode 100644 index 0000000..6dc6032 --- /dev/null +++ b/tests/common/nix.nix @@ -0,0 +1,8 @@ +{ + nix.settings = { + extra-experimental-features = [ + "nix-command" + "flakes" + ]; + }; +} diff --git a/tests/default.nix b/tests/default.nix new file mode 100644 index 0000000..9d81613 --- /dev/null +++ b/tests/default.nix @@ -0,0 +1,10 @@ +{ + sources ? import ../npins, + nixpkgs ? sources.nixpkgs, + pkgs ? import nixpkgs { overlays = [ (import ../pkgs/overlay.nix) ]; }, +}: +{ + substitution = pkgs.callPackage ./substitution.nix { }; + # Disable. We have to customize store config + #upstream-cache-http-directory = pkgs.callPackage ./upstream-cache-http-directory.nix { }; +} diff --git a/tests/substitution.nix b/tests/substitution.nix new file mode 100644 index 0000000..6574303 --- /dev/null +++ b/tests/substitution.nix @@ -0,0 +1,47 @@ +{ pkgs }: +let + #hello = pkgs.hello.overrideAttrs { pname = "custom-hello"; }; + inherit (pkgs) hello; +in +pkgs.testers.runNixOSTest (_: { + name = "cache substitution test"; + nodes = { + cache = { + imports = [ + ./common + ../module.nix + ]; + + system.extraDependencies = [ hello ]; + + networking.firewall.allowedTCPPorts = [ 80 ]; + }; + client = + { lib, ... }: + { + imports = [ ./common ]; + nix.settings = { + substituters = lib.mkForce [ "http://cache" ]; + trusted-public-keys = lib.mkForce [ (builtins.readFile ./cache-keys/pubkey) ]; + }; + }; + }; + testScript = '' + import time + start_all() + cache.wait_for_unit("nginx.service") + cache.wait_for_unit("tvix-store.service") + time.sleep(1) + with subtest("Nar bridge home"): + cache.succeed("curl -f http://127.0.0.1/nix-cache-info") + with subtest("Path signature and copy"): + # Sign + cache.succeed("nix store sign -k ${./cache-keys/privkey} ${hello}") + cache.succeed("nix copy --to 'http://127.0.0.1/?compression=none' ${hello}") + with subtest("Substitution"): + client.succeed("nix-store --delete ${hello}") + client.fail("stat ${hello}") + client.succeed("nix-store -r ${hello}") + client.succeed("stat ${hello}") + ''; +}) diff --git a/tests/upstream-cache-http-directory.nix b/tests/upstream-cache-http-directory.nix new file mode 100644 index 0000000..fdd7a25 --- /dev/null +++ b/tests/upstream-cache-http-directory.nix @@ -0,0 +1,66 @@ +{ pkgs }: +let + #hello = pkgs.hello.overrideAttrs { pname = "custom-hello"; }; + inherit (pkgs) hello; +in +pkgs.testers.runNixOSTest (_: { + name = "caching of upstream nar-store"; + nodes = { + cache = { + imports = [ + ./common + ../module.nix + ]; + + system.extraDependencies = [ hello ]; + + services = { + nginx = { + virtualHosts."localhost" = { + default = true; + locations = { + "/upstream".return = "302 /upstream/"; + "/upstream/".alias = "/srv/"; + }; + }; + }; + }; + networking.firewall.allowedTCPPorts = [ 80 ]; + }; + client = + { lib, ... }: + { + imports = [ ./common ]; + nix.settings = { + substituters = lib.mkForce [ "http://cache/" ]; + trusted-public-keys = lib.mkForce [ (builtins.readFile ./cache-keys/pubkey) ]; + }; + }; + }; + testScript = '' + import time + start_all() + cache.wait_for_unit("nginx.service") + cache.wait_for_unit("tvix-store.service") + time.sleep(1) + + def delete_and_substitute(): + client.succeed("nix-store --delete ${hello}") + client.fail("stat ${hello}") + client.succeed("nix-store -r ${hello}") + client.succeed("stat ${hello}") + + with subtest("Upload"): + cache.succeed("nix store sign -k ${./cache-keys/privkey} ${hello}") + cache.succeed("nix copy --to file:///srv ${hello}") + narHash = "${hello}"[11:11+32] + out = cache.succeed(f"curl -f 'http://localhost/upstream/{narHash}.narinfo'") + print(out) + cache.succeed(f"curl -f 'http://localhost/{narHash}.narinfo'") + with subtest("Try to substitute from cache"): + delete_and_substitute() + with subtest("Check effective caching"): + cache.succeed("rm -rf /srv") + delete_and_substitute() + ''; +})