From 12cfba0147dbb41e9c63a81233db3bad1a0a91be Mon Sep 17 00:00:00 2001 From: Artemis Tosini Date: Fri, 19 Jul 2024 20:03:10 +0000 Subject: [PATCH] Support downloading nix from substituters Currently lix-installer requires a lix distribution tarball downloaded over HTTP or on the system, meaning releases require manual effort to place files in the correct place. Add an option to download lix instead from standard nix substituters, such as cache.nixos.org and cache.lix.systems. Priority and nar parsing code do not exactly match lix, but are sufficient to securely download and install nars. Co-authored-by: Skye Change-Id: Ia8a771ad2a99ac461cf5839a52e45f9dca65f3c8 --- Cargo.lock | 575 +++++++++++--- Cargo.toml | 7 +- flake.lock | 10 +- nix/eval-versions.nix | 17 + nix/tests/vm-test/default.nix | 107 ++- set_version.py | 68 +- .../base/fetch_and_unpack_nix_substituter.rs | 709 ++++++++++++++++++ src/action/base/mod.rs | 2 + src/action/common/place_nix_configuration.rs | 6 +- src/action/common/provision_nix.rs | 68 +- src/settings.rs | 137 +++- tests/fixtures/linux/linux.json | 31 +- tests/fixtures/linux/steam-deck.json | 31 +- tests/fixtures/macos/macos.json | 31 +- 14 files changed, 1592 insertions(+), 207 deletions(-) create mode 100644 nix/eval-versions.nix create mode 100644 src/action/base/fetch_and_unpack_nix_substituter.rs diff --git a/Cargo.lock b/Cargo.lock index 0d953e0..04c4ceb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,6 +90,20 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "async-compression" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" +dependencies = [ + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "zstd", + "zstd-safe", +] + [[package]] name = "async-trait" version = "0.1.80" @@ -134,6 +148,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bitflags" version = "1.3.2" @@ -146,6 +166,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -177,11 +206,22 @@ dependencies = [ "serde", ] +[[package]] +name = "camino" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0ec6b951b160caa93cc0c7b209e5a3bff7aae9062213451ac99493cd844c239" + [[package]] name = "cc" version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "065a29261d53ba54260972629f9ca6bffa69bac13cd1fed61420f7fa68b9f8bd" +dependencies = [ + "jobserver", + "libc", + "once_cell", +] [[package]] name = "cfg-if" @@ -282,6 +322,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "core-foundation" version = "0.9.4" @@ -298,6 +344,52 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "darling" version = "0.20.8" @@ -333,6 +425,16 @@ dependencies = [ "syn", ] +[[package]] +name = "der" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "deranged" version = "0.3.11" @@ -343,6 +445,16 @@ dependencies = [ "serde", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dirs" version = "5.0.1" @@ -391,21 +503,37 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "serde", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" -[[package]] -name = "encoding_rs" -version = "0.8.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" -dependencies = [ - "cfg-if", -] - [[package]] name = "enum-as-inner" version = "0.6.0" @@ -459,6 +587,12 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "filetime" version = "0.2.23" @@ -553,6 +687,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.14" @@ -576,25 +720,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" -[[package]] -name = "h2" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" -dependencies = [ - "bytes 1.6.0", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap 2.2.6", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -642,9 +767,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes 1.6.0", "fnv", @@ -653,12 +778,24 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.6" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes 1.6.0", "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes 1.6.0", + "futures-util", + "http", + "http-body", "pin-project-lite", ] @@ -668,48 +805,61 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - [[package]] name = "hyper" -version = "0.14.28" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes 1.6.0", "futures-channel", - "futures-core", "futures-util", - "h2", "http", "http-body", "httparse", - "httpdate", "itoa", "pin-project-lite", - "socket2", + "smallvec", "tokio", - "tower-service", - "tracing", "want", ] [[package]] name = "hyper-rustls" -version = "0.24.2" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", "http", "hyper", + "hyper-util", "rustls", + "rustls-native-certs", + "rustls-pki-types", "tokio", "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +dependencies = [ + "bytes 1.6.0", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -817,6 +967,15 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" +[[package]] +name = "is_executable" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa9acdc6d67b75e626ad644734e8bc6df893d9cd2a834129065d3dd6158ea9c8" +dependencies = [ + "winapi", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.0" @@ -829,6 +988,15 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "jobserver" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.69" @@ -877,17 +1045,20 @@ name = "lix-installer" version = "0.17.1" dependencies = [ "async-trait", + "base64 0.22.1", "bytes 1.6.0", "clap", "color-eyre", "dirs", "dyn-clone", + "ed25519-dalek", "eyre", "glob", "indexmap 2.2.6", "is_ci", "nix", "nix-config-parser", + "nix-nar", "os-release", "owo-colors 4.0.0", "plist", @@ -897,6 +1068,7 @@ dependencies = [ "serde", "serde_json", "serde_with", + "sha2", "strum", "sysctl", "tar", @@ -914,6 +1086,7 @@ dependencies = [ "walkdir", "which", "xz2", + "zstd", ] [[package]] @@ -1007,6 +1180,18 @@ dependencies = [ "thiserror", ] +[[package]] +name = "nix-nar" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5549158a8b179c4fcd06a19f4bcc557db60c9cbd6771add9563f46c8d0325b5" +dependencies = [ + "camino", + "is_executable", + "symlink", + "thiserror", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1128,6 +1313,26 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -1140,6 +1345,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.30" @@ -1190,6 +1405,53 @@ dependencies = [ "memchr", ] +[[package]] +name = "quinn" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad" +dependencies = [ + "bytes 1.6.0", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddf517c03a109db8100448a4be38d498df8a210a99fe0e1b9eaf39e78c640efe" +dependencies = [ + "bytes 1.6.0", + "rand", + "ring", + "rustc-hash", + "rustls", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + +[[package]] +name = "quinn-udp" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9096629c45860fc7fb143e125eb826b5e721e10be3263160c7d60ca832cf8c46" +dependencies = [ + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.36" @@ -1304,20 +1566,21 @@ checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "reqwest" -version = "0.11.27" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" dependencies = [ - "base64 0.21.7", + "async-compression", + "base64 0.22.1", "bytes 1.6.0", - "encoding_rs", "futures-core", "futures-util", - "h2", "http", "http-body", + "http-body-util", "hyper", "hyper-rustls", + "hyper-util", "ipnet", "js-sys", "log", @@ -1325,14 +1588,15 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "quinn", "rustls", "rustls-native-certs", "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", - "system-configuration", "tokio", "tokio-rustls", "tokio-socks", @@ -1367,6 +1631,21 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.38.34" @@ -1382,44 +1661,55 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.12" +version = "0.23.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0" dependencies = [ - "log", + "once_cell", "ring", + "rustls-pki-types", "rustls-webpki", - "sct", + "subtle", + "zeroize", ] [[package]] name = "rustls-native-certs" -version = "0.6.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" dependencies = [ "openssl-probe", "rustls-pemfile", + "rustls-pki-types", "schannel", "security-framework", ] [[package]] name = "rustls-pemfile" -version = "1.0.4" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", + "rustls-pki-types", ] [[package]] -name = "rustls-webpki" -version = "0.101.7" +name = "rustls-pki-types" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" + +[[package]] +name = "rustls-webpki" +version = "0.102.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" dependencies = [ "ring", + "rustls-pki-types", "untrusted", ] @@ -1459,16 +1749,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "security-framework" version = "2.11.0" @@ -1574,6 +1854,17 @@ dependencies = [ "syn", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1592,6 +1883,15 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core", +] + [[package]] name = "slab" version = "0.4.9" @@ -1623,6 +1923,16 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "strsim" version = "0.10.0" @@ -1657,6 +1967,12 @@ dependencies = [ "syn", ] +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "supports-color" version = "2.1.0" @@ -1667,6 +1983,12 @@ dependencies = [ "is_ci", ] +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "2.0.60" @@ -1680,9 +2002,9 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "0.1.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" [[package]] name = "sysctl" @@ -1698,27 +2020,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "system-configuration" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "tar" version = "0.4.40" @@ -1879,11 +2180,12 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.24.1" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ "rustls", + "rustls-pki-types", "tokio", ] @@ -1912,6 +2214,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -1995,6 +2318,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "typetag" version = "0.2.16" @@ -2079,6 +2408,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "walkdir" version = "2.5.0" @@ -2386,9 +2721,9 @@ checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "winreg" -version = "0.50.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" dependencies = [ "cfg-if", "windows-sys 0.48.0", @@ -2421,3 +2756,37 @@ dependencies = [ "lzma-sys", "tokio-io", ] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zstd" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.12+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 5a510a0..5d2e1b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ eyre = { version = "0.6.8", default-features = false, features = [ "track-caller glob = { version = "0.3.0", default-features = false } nix = { version = "0.28.0", default-features = false, features = ["user", "fs", "process", "term"] } owo-colors = { version = "4.0.0", default-features = false, features = [ "supports-colors" ] } -reqwest = { version = "0.11.11", default-features = false, features = ["rustls-tls-native-roots", "stream", "socks"] } +reqwest = { version = "0.12.5", default-features = false, features = ["rustls-tls-native-roots", "stream", "socks", "zstd"] } serde = { version = "1.0.144", default-features = false, features = [ "std", "derive" ] } serde_json = { version = "1.0.85", default-features = false, features = [ "std" ] } serde_with = { version = "3", default-features = false, features = [ "std", "macros" ] } @@ -60,6 +60,11 @@ which = "6.0.0" sysctl = "0.5.4" walkdir = "2.3.3" indexmap = { version = "2.0.2", features = ["serde"] } +nix-nar = "0.3.0" +zstd = { version = "0.13.2", default-features = false } +sha2 = "0.10.8" +ed25519-dalek = { version = "2.1.1", features = ["serde"] } +base64 = "0.22.1" [dev-dependencies] eyre = { version = "0.6.8", default-features = false, features = [ "track-caller" ] } diff --git a/flake.lock b/flake.lock index a27a16a..033ca18 100644 --- a/flake.lock +++ b/flake.lock @@ -46,15 +46,15 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1718419213, - "narHash": "sha256-WY7BGnu5PnbK4O8cKKv9kvxwzZIGbIQUQLGPHFXitI0=", - "rev": "253546d5fbf8a5aa60ac8164c1b4f5794dc4e9d1", + "lastModified": 1720626042, + "narHash": "sha256-f8k+BezKdJfmE+k7zgBJiohtS3VkkriycdXYsKOm3sc=", + "rev": "2a4376be20d70feaa2b0e640c5041fb66ddc67ed", "type": "tarball", - "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/253546d5fbf8a5aa60ac8164c1b4f5794dc4e9d1.tar.gz" + "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/2a4376be20d70feaa2b0e640c5041fb66ddc67ed.tar.gz" }, "original": { "type": "tarball", - "url": "https://git.lix.systems/lix-project/lix/archive/2.90.0-rc1.tar.gz" + "url": "https://git.lix.systems/lix-project/lix/archive/2.90.0.tar.gz" } }, "naersk": { diff --git a/nix/eval-versions.nix b/nix/eval-versions.nix new file mode 100644 index 0000000..e1a662e --- /dev/null +++ b/nix/eval-versions.nix @@ -0,0 +1,17 @@ +let + args = builtins.fromJSON (builtins.getEnv "LIX_EVAL_VERSIONS_ARGS"); + flake = builtins.getFlake args.flakeUrl; + nixpkgs = flake.inputs.nixpkgs; + lix = system: flake.outputs.packages.${system}.nix.outPath; + cacert = system: nixpkgs.outputs.legacyPackages.${system}.cacert.outPath; + targets = system: [ + (lix system) + (cacert system) + ]; +in +builtins.listToAttrs ( + builtins.map (system: { + name = system; + value = targets system; + }) args.systems +) diff --git a/nix/tests/vm-test/default.nix b/nix/tests/vm-test/default.nix index f317bb5..bb9d0d1 100644 --- a/nix/tests/vm-test/default.nix +++ b/nix/tests/vm-test/default.nix @@ -132,6 +132,37 @@ let fi ''; }; + install-substituter = pkgs: let + rootPaths = binaryTarball.${pkgs.system}.passthru.rootPaths; + binaryCache = pkgs.mkBinaryCache { inherit rootPaths; }; + targetArg = pkg: "--substitution-targets ${pkg.outPath}"; + targetArgs = lib.concatMapStringsSep " " targetArg rootPaths; + in { + # We need -O for the old scp protocol because RHEL 8 has old SSH with broken SFTP + # Might be deprecated soon, so remove when we remove RHEL 8 + copyTarball = '' + scp -O -P 20022 $ssh_opts -r ${binaryCache} vagrant@localhost:binaryCache + ''; + install = '' + pushd binaryCache + python3 -m http.server 8000 & + server_pid=$! + trap "kill $server_pid" EXIT + popd + sleep 1 + RUST_BACKTRACE="full" ./nix-installer install \ + --use-substituters \ + --substituters http://127.0.0.1:8000 \ + --no-require-sigs \ + ${targetArgs} \ + --no-confirm \ + --logger pretty \ + --log-directive lix_installer=info + ''; + check = installCases.install-default.check; + uninstall = installCases.install-default.uninstall; + uninstallCheck = installCases.install-default.uninstallCheck; + }; install-no-start-daemon = { install = '' NIX_PATH=$(readlink -f nix.tar.xz) @@ -400,25 +431,10 @@ let disableSELinux = "sudo setenforce 0"; images = { - - # End of standard support https://wiki.ubuntu.com/Releases - # No systemd - /* - "ubuntu-v14_04" = { + "ubuntu-v20_04" = { image = import { - url = "https://app.vagrantup.com/ubuntu/boxes/trusty64/versions/20190514.0.0/providers/virtualbox.box"; - hash = "sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="; - }; - rootDisk = "box-disk1.vmdk"; - system = "x86_64-linux"; - }; - */ - - # End of standard support https://wiki.ubuntu.com/Releases - "ubuntu-v16_04" = { - image = import { - url = "https://app.vagrantup.com/generic/boxes/ubuntu1604/versions/4.1.12/providers/libvirt.box"; - hash = "sha256-lO4oYQR2tCh5auxAYe6bPOgEqOgv3Y3GC1QM1tEEEU8="; + url = "https://app.vagrantup.com/generic/boxes/ubuntu2004/versions/4.3.12/providers/libvirt.box"; + hash = "sha256-lo6fkz6N/Q9mdD+RWoUssak9TVod0F7QSgZvxnMj9IQ="; }; rootDisk = "box.img"; system = "x86_64-linux"; @@ -433,50 +449,16 @@ let system = "x86_64-linux"; }; - "fedora-v36" = { + "fedora-v39" = { image = import { - url = "https://app.vagrantup.com/generic/boxes/fedora36/versions/4.1.12/providers/libvirt.box"; - hash = "sha256-rxPgnDnFkTDwvdqn2CV3ZUo3re9AdPtSZ9SvOHNvaks="; + url = "https://app.vagrantup.com/generic/boxes/fedora39/versions/4.3.12/providers/libvirt.box"; + hash = "sha256-VJbWmcy3XiEm7cUAXtod8VlFwsIwnVYlZ/LYTuoj9WI="; }; rootDisk = "box.img"; system = "x86_64-linux"; upstreamScriptsWork = false; # SELinux! }; - "fedora-v37" = { - image = import { - url = "https://app.vagrantup.com/generic/boxes/fedora37/versions/4.2.14/providers/libvirt.box"; - hash = "sha256-rxPgnDnFkTDwvdqn2CV3ZUo3re9AdPtSZ9SvOHNvaks="; - }; - rootDisk = "box.img"; - system = "x86_64-linux"; - upstreamScriptsWork = false; # SELinux! - }; - - # Currently fails with 'error while loading shared libraries: - # libsodium.so.23: cannot stat shared object: Invalid argument'. - /* - "rhel-v6" = { - image = import { - url = "https://app.vagrantup.com/generic/boxes/rhel6/versions/4.1.12/providers/libvirt.box"; - hash = "sha256-QwzbvRoRRGqUCQptM7X/InRWFSP2sqwRt2HaaO6zBGM="; - }; - rootDisk = "box.img"; - upstreamScriptsWork = false; # SELinux! - system = "x86_64-linux"; - }; - */ - - "rhel-v7" = { - image = import { - url = "https://app.vagrantup.com/generic/boxes/rhel7/versions/4.1.12/providers/libvirt.box"; - hash = "sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="; - }; - rootDisk = "box.img"; - upstreamScriptsWork = false; # SELinux! - system = "x86_64-linux"; - }; - "rhel-v8" = { image = import { url = "https://app.vagrantup.com/generic/boxes/rhel8/versions/4.1.12/providers/libvirt.box"; @@ -500,9 +482,13 @@ let }; - makeTest = imageName: testName: test: - let image = images.${imageName}; in - with (forSystem image.system ({ system, pkgs, ... }: pkgs)); + makeTest = imageName: testName: testFunc: + let + image = images.${imageName}; + pkgs = (forSystem image.system ({ system, pkgs, ... }: pkgs)); + test = if builtins.isFunction testFunc then (testFunc pkgs) else testFunc; + in + with pkgs; runCommand "installer-test-${imageName}-${testName}" { @@ -517,6 +503,8 @@ let uninstallCheckScript = test.uninstallCheck; installer = lix-installer-static; binaryTarball = binaryTarball.${system}; + copyTarball = test.copyTarball + or "scp -P 20022 $ssh_opts $binaryTarball/lix-*.tar.xz vagrant@localhost:nix.tar.xz"; } '' shopt -s nullglob @@ -575,8 +563,9 @@ let echo "Copying installer..." scp -P 20022 $ssh_opts $installer/bin/lix-installer vagrant@localhost:nix-installer + ls $binaryTarball echo "Copying nix tarball..." - scp -P 20022 $ssh_opts $binaryTarball/lix-*.tar.xz vagrant@localhost:nix.tar.xz + eval "$copyTarball" echo "Running preinstall..." $ssh "set -eux; $preinstallScript" diff --git a/set_version.py b/set_version.py index 05562f2..d96316c 100755 --- a/set_version.py +++ b/set_version.py @@ -3,6 +3,9 @@ from pathlib import Path import textwrap import dataclasses import requests +import os +import json +import subprocess SYSTEMS = ['x86_64-linux', 'x86_64-darwin', 'aarch64-linux', 'aarch64-darwin'] @@ -29,6 +32,58 @@ def make_urls_section(packages: list[Package]): return '\n'.join(one_item(package) for package in packages) +@dataclasses.dataclass +class SubstitutionTargets: + system: str + targets: list[str] + + def variable_name(self): + system = self.system.replace('-', '_').upper() + return f'LIX_{system}_SUBSTITUTION_TARGETS' + +def make_substitution_targets_section(packages: list[SubstitutionTargets]): + def one_item(package: SubstitutionTargets): + targets = "\n".join([f' "{target}",' for target in package.targets]) + + return textwrap.dedent("""\ + /// Default [`substitution_targets`](CommonSettings::substitution_targets) for {system}. + pub const {variable_name}: &[&str; {target_len}] = &[ + {targets} + ]; + """).format( + system=package.system, + variable_name=package.variable_name(), + target_len=len(package.targets), + targets=targets + ) + + return '\n'.join(one_item(package) for package in packages) + +def eval_substitution_targets(version: str) -> list[SubstitutionTargets]: + options = { + "systems": SYSTEMS, + "flakeUrl": lix_flake_url(version) + } + + env = os.environb | { + b"LIX_EVAL_VERSIONS_ARGS": json.dumps(options) + } + + completed = subprocess.run([ + "nix", + "--extra-experimental-features", + "nix-command flakes", + "eval", + "--impure", + "--json", + "--file", + Path("nix/eval-versions.nix") + ], env=env, check=True, capture_output=True) + + result = json.loads(completed.stdout) + + return [SubstitutionTargets(system, targets) for (system, targets) in result.items()] + def replace_section(old: str, section: str) -> str: lines = [] eat = False @@ -54,8 +109,11 @@ def replace_in_file(file: Path, section: str): file.write_text(new_file) +def lix_flake_url(version: str) -> str: + return f"https://git.lix.systems/lix-project/lix/archive/{version}.tar.gz" + def make_flake_url_section(version: str) -> str: - return f' url = "https://git.lix.systems/lix-project/lix/archive/{version}.tar.gz";' + return f' url = "{lix_flake_url(version)}";' def main(): @@ -69,12 +127,18 @@ def main(): settings_rs = Path('src/settings.rs') packages = [Package(system, args.new_version) for system in SYSTEMS] + substitution_targets = eval_substitution_targets(args.new_version) + for package in packages: resp = requests.head(package.url()) if resp.status_code != 200: print(f'Warning: broken URL {package.url()} returns HTTP {resp.status_code}') - replace_in_file(settings_rs, make_urls_section(packages)) + replace_in_file(settings_rs, + make_urls_section(packages) + + "\n" + + make_substitution_targets_section(substitution_targets) + ) flake_nix = Path('flake.nix') replace_in_file(flake_nix, make_flake_url_section(args.new_version)) diff --git a/src/action/base/fetch_and_unpack_nix_substituter.rs b/src/action/base/fetch_and_unpack_nix_substituter.rs new file mode 100644 index 0000000..be593e1 --- /dev/null +++ b/src/action/base/fetch_and_unpack_nix_substituter.rs @@ -0,0 +1,709 @@ +use std::{ + collections::{HashMap, HashSet}, + fmt::Write, + io::{BufRead, Read}, + path::{Path, PathBuf}, +}; + +use base64::Engine; +use bytes::Buf; +use ed25519_dalek::VerifyingKey; +use reqwest::Url; +use serde::{Deserialize, Serialize}; +use sha2::Digest; +use tokio::io::AsyncWriteExt; +use tracing::{span, Span}; + +use crate::{ + action::{Action, ActionDescription, ActionError, ActionErrorKind, ActionTag, StatefulAction}, + parse_ssl_cert, + settings::UrlOrPathError, +}; + +/// Fetch an output and its dependencies from a set of substituters, +/// given an output path, subsititer URLs, and trusted keys. +/// Also generates a ".reginfo" compatible with `nix-store --load-db` +/// Only implements a subset of nix substitution features: +/// * Substituter priorites are highest to lowest as given to [`plan`], +/// instead of priority from nix-cache-info +/// * narinfo signatures are always required +/// * ca-derivations are not supported +/// * NarHash must be sha256 +#[derive(Debug, serde::Deserialize, serde::Serialize, Clone)] +pub struct FetchAndUnpackNixSubstituter { + /// Whether to require valid signatures when checking narinfo + require_sigs: bool, + /// Map from key name (e.g. cache.nixos.org-1) to parsed ed25519 key + trusted_keys: HashMap, + /// Base URLs for substituters, e.g. https://cache.nixos.org/ + substituters: Vec, + /// Desired derivation output, e.g. + /// `/nix/store/n50jk09x9hshwx1lh6k3qaiygc7yxbv9-lix-2.90.0-rc1` + targets: Vec, + /// Destination directory, normally temporary. + /// For compatibility with tarballs, files will be placed in + /// the nix-/store subdirectory of the destination + dest: PathBuf, + /// Proxy used for all requests from substituters + proxy: Option, + /// Extra SSL certificates trusted for all requests + ssl_cert_file: Option, +} + +/// Root directory of the nix store. +/// Technically this could be something other than /nix/store, +/// but that is rarely done in production +const STORE_DIR: &str = "/nix/store/"; + +impl FetchAndUnpackNixSubstituter { + #[tracing::instrument(level = "debug", skip_all)] + pub async fn plan( + targets: Vec, + dest: PathBuf, + trusted_keys: Vec, + require_sigs: bool, + substituters: Vec, + proxy: Option, + ssl_cert_file: Option, + ) -> Result, ActionError> { + let trusted_keys_parsed = trusted_keys + .iter() + .map(|key| parse_key(key)) + .collect::, _>>() + .map_err(Self::error)?; + + if let Some(proxy) = &proxy { + match proxy.scheme() { + "https" | "http" | "socks5" => (), + _ => return Err(Self::error(SubstitutionError::UnknownProxyScheme)), + }; + } + + if let Some(ssl_cert_file) = &ssl_cert_file { + parse_ssl_cert(ssl_cert_file).await.map_err(Self::error)?; + } + + let targets = targets + .iter() + .map(|p| StorePath::from_path(p)) + .collect::, _>>() + .map_err(Self::error)?; + + if !require_sigs { + tracing::warn!("Signatures are not required during substitution. This is insecure."); + } + + Ok(Self { + require_sigs, + targets, + trusted_keys: trusted_keys_parsed, + dest, + proxy, + substituters, + ssl_cert_file, + } + .into()) + } + + #[tracing::instrument(level = "trace", skip(self, client))] + async fn fetch_narinfo( + &self, + client: &reqwest::Client, + output: &StorePath, + ) -> Result { + for substituter in &self.substituters { + let narinfo_url = substituter + .join(&format!("{}.narinfo", &output.digest())) + .map_err(|err| UrlOrPathError::Url("".to_string(), err)) + .map_err(ActionErrorKind::UrlOrPathError) + .map_err(Self::error)?; + + let response = client + .get(narinfo_url) + .send() + .await + .map_err(ActionErrorKind::Reqwest) + .map_err(Self::error)?; + + if !response.status().is_success() { + continue; + } + + let narinfo = NarInfo::parse_and_verify( + self.require_sigs, + &self.trusted_keys, + substituter, + output, + &response + .bytes() + .await + .map_err(ActionErrorKind::Reqwest) + .map_err(Self::error)?, + ) + .map_err(Self::error)?; + + return Ok(narinfo); + } + + Err(Self::error(SubstitutionError::NonexistantNarInfo( + output.full_path().clone(), + ))) + } +} + +#[async_trait::async_trait] +#[typetag::serde(name = "fetch_and_unpack_nix")] +impl Action for FetchAndUnpackNixSubstituter { + fn action_tag() -> ActionTag { + ActionTag("fetch_and_unpack_nix_substituter") + } + fn tracing_synopsis(&self) -> String { + format!( + "Fetch {} from substituters to `{}`", + self.targets + .iter() + .map(|t| format!("`{}`", t.full_path())) + .collect::>() + .join(", "), + self.dest.display() + ) + } + + fn tracing_span(&self) -> Span { + let span = span!( + tracing::Level::DEBUG, + "fetch_and_unpack_nix_substituter", + targets = tracing::field::debug(&self.targets), + proxy = tracing::field::Empty, + ssl_cert_file = tracing::field::Empty, + dest = tracing::field::display(self.dest.display()), + ); + if let Some(proxy) = &self.proxy { + span.record("proxy", tracing::field::display(&proxy)); + } + if let Some(ssl_cert_file) = &self.ssl_cert_file { + span.record( + "ssl_cert_file", + tracing::field::display(&ssl_cert_file.display()), + ); + } + span + } + + fn execute_description(&self) -> Vec { + vec![ActionDescription::new(self.tracing_synopsis(), vec![])] + } + + #[tracing::instrument(level = "debug", skip_all)] + async fn execute(&mut self) -> Result<(), ActionError> { + let mut client_builder = reqwest::Client::builder(); + if let Some(proxy) = &self.proxy { + client_builder = client_builder.proxy( + reqwest::Proxy::all(proxy.clone()) + .map_err(ActionErrorKind::Reqwest) + .map_err(Self::error)?, + ) + } + if let Some(ssl_cert_file) = &self.ssl_cert_file { + let ssl_cert = parse_ssl_cert(ssl_cert_file).await.map_err(Self::error)?; + client_builder = client_builder.add_root_certificate(ssl_cert); + } + let client = client_builder + .build() + .map_err(ActionErrorKind::Reqwest) + .map_err(Self::error)?; + + let nix_store_dir = self.dest.join("nix-/store"); + tokio::fs::create_dir_all(&nix_store_dir) + .await + .map_err(|e| ActionErrorKind::CreateDirectory(nix_store_dir.clone(), e)) + .map_err(Self::error)?; + + let mut outputs_remaining = self.targets.clone(); + let mut outputs_done = HashSet::new(); + let mut reginfo = String::new(); + + loop { + let Some(output) = outputs_remaining.pop() else { + break; + }; + // Make sure we don't download the same output twice + if !outputs_done.insert(output.clone()) { + continue; + } + + let narinfo = self.fetch_narinfo(&client, &output).await?; + + for reference in &narinfo.references { + outputs_remaining.push(reference.clone()); + } + + let compressed_nar = client + .get(narinfo.url.clone()) + .send() + .await + .map_err(ActionErrorKind::Reqwest) + .map_err(Self::error)? + .error_for_status() + .map_err(ActionErrorKind::Reqwest) + .map_err(Self::error)? + .bytes() + .await + .map_err(ActionErrorKind::Reqwest) + .map_err(Self::error)?; + + let nar_size: usize = narinfo + .nar_size + .try_into() + .map_err(|_| Self::error(SubstitutionError::BadNarInfo))?; + + // Decompress to a vec since we need to go through the data twice + // (once for hashing, one for unpacking). + // Otherwise we'd need to decompress twice + let decompressed_nar = match narinfo.compression { + NarCompression::Zstd => zstd::bulk::decompress(&compressed_nar, nar_size) + .map_err(|e| SubstitutionError::Decompress(narinfo.url.clone(), e)) + .map_err(Self::error)?, + NarCompression::Xz => { + let mut decompressor = xz2::read::XzDecoder::new(compressed_nar.reader()); + let mut result = Vec::with_capacity(nar_size); + decompressor + .read_to_end(&mut result) + .map_err(|e| SubstitutionError::Decompress(narinfo.url.clone(), e)) + .map_err(Self::error)?; + + result + }, + }; + + if decompressed_nar.len() != nar_size { + return Err(Self::error(SubstitutionError::BadNar(narinfo.url.clone()))); + } + + let found_hash = { + let mut hasher = sha2::Sha256::new(); + hasher.update(&decompressed_nar); + hasher.finalize() + }; + + if encode_nix32(&found_hash) != narinfo.nar_hash { + return Err(Self::error(SubstitutionError::BadNar(narinfo.url.clone()))); + } + + let out_dir = self.dest.join("nix-/store").join(output.full_name()); + + let decoder = nix_nar::Decoder::new(decompressed_nar.reader()) + .map_err(|e| SubstitutionError::Unpack(narinfo.url.clone(), e)) + .map_err(Self::error)?; + + decoder + .unpack(out_dir) + .map_err(|e| SubstitutionError::Unpack(narinfo.url.clone(), e)) + .map_err(Self::error)?; + + // File format isn't documented anywhere but implementation is simple: + // https://git.lix.systems/lix-project/lix/src/commit/d461cc1d7b2f489c3886f147166ba5b5e0e37541/src/libstore/store-api.cc#L846 + // Unwrapping because string can't fail methods in std::fmt::Write + writeln!(reginfo, "{}", output.full_path()).unwrap(); + writeln!(reginfo, "sha256:{}", narinfo.nar_hash).unwrap(); + writeln!(reginfo, "{}", narinfo.nar_size).unwrap(); + // Leave deriver empty, same as lix binary tarballs + reginfo.push('\n'); + writeln!(reginfo, "{}", narinfo.references.len()).unwrap(); + for reference in &narinfo.references { + writeln!(reginfo, "{}", reference.full_path()).unwrap(); + } + } + + let reginfo_path = self.dest.join("nix-/.reginfo"); + let mut reginfo_file = tokio::fs::File::create(®info_path) + .await + .map_err(|e| ActionErrorKind::Write(reginfo_path.clone(), e)) + .map_err(Self::error)?; + + reginfo_file + .write_all(reginfo.as_bytes()) + .await + .map_err(|e| ActionErrorKind::Write(reginfo_path.clone(), e)) + .map_err(Self::error)?; + + Ok(()) + } + + fn revert_description(&self) -> Vec { + vec![/* Deliberately empty -- this is a noop */] + } + + #[tracing::instrument(level = "debug", skip_all)] + async fn revert(&mut self) -> Result<(), ActionError> { + Ok(()) + } +} + +/// Parse a nix trusted key into name and ed25519 +fn parse_key(key: &str) -> Result<(String, VerifyingKey), SubstitutionError> { + let (name, key_base64) = key + .split_once(':') + .ok_or_else(|| SubstitutionError::PublicKey)?; + + // seems to be the best way to handle keys both with and without padding + let key_bytes = base64::engine::general_purpose::STANDARD_NO_PAD + .decode(key_base64.trim_end_matches('=').as_bytes()) + .map_err(|_| SubstitutionError::PublicKey)?; + + let key_array: [u8; ed25519_dalek::PUBLIC_KEY_LENGTH] = key_bytes + .try_into() + .map_err(|_| SubstitutionError::PublicKey)?; + + let verifying_key = ed25519_dalek::VerifyingKey::from_bytes(&key_array) + .map_err(|_| SubstitutionError::PublicKey)?; + + Ok((name.to_string(), verifying_key)) +} + +/// Utility struct representing a store path +#[derive(Clone, PartialEq, Eq, Hash, Debug, Serialize, Deserialize)] +#[serde(transparent)] +struct StorePath(String); + +impl StorePath { + /// The full name of a path, not including STORE_DIR, + /// e.g. `n50jk09x9hshwx1lh6k3qaiygc7yxbv9-lix-2.90.0-rc1` + pub fn full_name(&self) -> &str { + &self.0 + } + + /// The base32 hash part of a store path, + /// e.g. `n50jk09x9hshwx1lh6k3qaiygc7yxbv9` + pub fn digest(&self) -> &str { + self.0.split_once('-').unwrap().0 + } + + /// Full path of the output including STORE_DIR, + /// as seen in StorePath in narinfo + /// e.g. `/nix/store/n50jk09x9hshwx1lh6k3qaiygc7yxbv9-lix-2.90.0-rc1` + pub fn full_path(&self) -> String { + format!("{}{}", STORE_DIR, &self.0) + } + + pub fn from_full_path(full_path: &str) -> Option { + if !full_path.starts_with(STORE_DIR) { + return None; + } + + let (_, full_name) = full_path.split_at(STORE_DIR.len()); + + Self::from_full_name(full_name) + } + + pub fn from_path(path: &Path) -> Result { + let path_str = path + .to_str() + .ok_or_else(|| SubstitutionError::InvalidStorePath(path.to_owned()))?; + Self::from_full_path(path_str) + .ok_or_else(|| SubstitutionError::InvalidStorePath(path.to_owned())) + } + + pub fn from_full_name(full_name: &str) -> Option { + let (digest, name) = full_name.split_once('-')?; + + if digest.len() != 32 + || digest.contains(|c: char| !c.is_ascii_lowercase() && !c.is_ascii_digit()) + { + return None; + } + + if name.contains(|c: char| { + !c.is_ascii_alphanumeric() + && c != '+' + && c != '-' + && c != '.' + && c != '_' + && c != '?' + && c != '=' + }) { + return None; + } + + Some(Self(full_name.to_string())) + } +} + +/// Compression types for nar files. +/// Not exhaustive, this is just what +/// I've seen in the real world +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +enum NarCompression { + Zstd, + Xz, +} + +impl NarCompression { + pub fn from_name(name: &str) -> Option { + match name { + "zstd" => Some(Self::Zstd), + "xz" => Some(Self::Xz), + _ => None, + } + } +} + +/// Extracted data from a narinfo. +/// May only be constructed by verifying signature +/// Missing some fields, like Deriver and FileHash because they aren't signed +#[derive(Debug, Clone, PartialEq, Eq)] +struct NarInfo { + /// Store path represented by the nar + pub store_path: StorePath, + /// Full URL to download the nar + pub url: Url, + /// Method used to compress the nar + pub compression: NarCompression, + /// sha256 hash of the nar after decompression + /// encoded in nix base32 format + pub nar_hash: String, + /// Size of the decompressed nar in bytes + pub nar_size: u64, + /// Other store paths referenced by the nar + pub references: Vec, + /// Signature of the nar, used to sign other items + pub sig: Option<(String, Vec)>, +} + +impl NarInfo { + fn parse(substituter_url: &Url, contents: &bytes::Bytes) -> Result { + fn assign_unique_or_error( + field: &mut Option, + value: Result, + ) -> Result<(), SubstitutionError> { + if field.replace(value?).is_some() { + Err(SubstitutionError::BadNarInfo) + } else { + Ok(()) + } + } + + let mut store_path = None; + let mut url = None; + let mut compression = None; + let mut nar_hash = None; + let mut nar_size = None; + let mut references = None; + let mut sig = None; + + for maybe_line in contents.lines() { + // Error if contents cannot be split into a list of line strings, probably if the NAR isn't valid utf-8 + let line = maybe_line.map_err(SubstitutionError::UndecodableNarInfo)?; + let (tag, rest) = line + .split_once(':') + .ok_or_else(|| SubstitutionError::BadNarInfo)?; + let value = rest.trim_start_matches(' '); + + match tag { + "StorePath" => { + let path = StorePath::from_full_path(value) + .ok_or_else(|| SubstitutionError::InvalidStorePath(value.into())); + assign_unique_or_error(&mut store_path, path)? + }, + "URL" => assign_unique_or_error( + &mut url, + substituter_url + .join(value) + .map_err(SubstitutionError::UrlParseError), + )?, + "Compression" => assign_unique_or_error( + &mut compression, + NarCompression::from_name(value).ok_or(SubstitutionError::BadNarInfo), + )?, + "NarHash" => { + let (algorithm, digest) = value + .split_once(':') + .ok_or_else(|| SubstitutionError::BadNarInfo)?; + if algorithm != "sha256" { + Err(SubstitutionError::BadNarInfo)? + } + assign_unique_or_error(&mut nar_hash, Ok(digest.to_string()))? + }, + "NarSize" => assign_unique_or_error( + &mut nar_size, + value.parse().map_err(|_| SubstitutionError::BadNarInfo), + )?, + "References" => { + let refs = if value.is_empty() { + // split on empty strings still returns one value + Ok(Vec::new()) + } else { + value + .split(' ') + .map(StorePath::from_full_name) + .map(|value| value.ok_or_else(|| SubstitutionError::BadNarInfo)) + .collect::, _>>() + }; + assign_unique_or_error(&mut references, refs)? + }, + "Sig" => { + let (signer, base64_signature) = value + .split_once(':') + .ok_or_else(|| SubstitutionError::BadNarInfo)?; + let signature = base64::engine::general_purpose::STANDARD + .decode(base64_signature) + .map_err(|_| SubstitutionError::BadNarInfo)?; + + assign_unique_or_error(&mut sig, Ok((signer.to_string(), signature)))? + }, + // Ignore any unmatched tags instead of erroring because there are some valid tags we are not parsing + _ => {}, + } + } + Ok(Self { + store_path: store_path.ok_or_else(|| SubstitutionError::BadNarInfo)?, + url: url.ok_or_else(|| SubstitutionError::BadNarInfo)?, + compression: compression.ok_or_else(|| SubstitutionError::BadNarInfo)?, + nar_hash: nar_hash.ok_or_else(|| SubstitutionError::BadNarInfo)?, + nar_size: nar_size.ok_or_else(|| SubstitutionError::BadNarInfo)?, + references: references.ok_or_else(|| SubstitutionError::BadNarInfo)?, + sig, + }) + } + + fn verify( + &self, + trusted_keys: &HashMap, + ) -> Result<(), SubstitutionError> { + let Some(sig) = &self.sig else { + return Err(SubstitutionError::BadSignature); + }; + + let Some(key) = trusted_keys.get(&sig.0) else { + return Err(SubstitutionError::BadSignature); + }; + + // Fingerprint format not documented, but implemented in lix: + // https://git.lix.systems/lix-project/lix/src/commit/d461cc1d7b2f489c3886f147166ba5b5e0e37541/src/libstore/path-info.cc#L25 + let fingerprint = format!( + "1;{};sha256:{};{};{}", + self.store_path.full_path(), + self.nar_hash, + self.nar_size, + self.references + .iter() + .map(|reference| reference.full_path()) + .collect::>() + .join(",") + ); + + key.verify_strict( + fingerprint.as_bytes(), + &ed25519_dalek::Signature::from_bytes( + sig.1 + .as_slice() + .try_into() + .map_err(|_| SubstitutionError::BadSignature)?, + ), + ) + .map_err(|_| SubstitutionError::BadSignature) + } + + pub fn parse_and_verify( + require_sigs: bool, + trusted_keys: &HashMap, + substituter_url: &Url, + expected_store_path: &StorePath, + contents: &bytes::Bytes, + ) -> Result { + let parsed = Self::parse(substituter_url, contents)?; + + if &parsed.store_path != expected_store_path { + return Err(SubstitutionError::BadSignature); + } + + if require_sigs { + parsed.verify(trusted_keys)?; + } + + Ok(parsed) + } +} + +static NIX32_CHARS: &[u8; 32] = b"0123456789abcdfghijklmnpqrsvwxyz"; + +fn encode_nix32(input: &[u8]) -> String { + // ceil(input.len() * 8 / 5) + let length = (input.len() * 8 + 4) / 5; + let mut output = String::with_capacity(length); + + // nix32 hashes feel like they're a bug that stuck + // The output is backwards and bits are grouped + // from the least significant bit in each byte + // instead of the most significant bit. + // e.g. for a 4-byte input, bits are mapped from input to output like this: + // Out No.: 5 5 5 6 6 6 6 6 | 3 4 4 4 4 4 5 5 | 2 2 2 2 3 3 3 3 | 0 0 1 1 1 1 1 2 + // Out Bit: 2 1 0 4 3 2 1 0 | 0 4 3 2 1 0 4 3 | 3 2 1 0 4 3 2 1 | 1 0 4 3 2 1 0 4 + // + // where "Out No." is the index of the output charater responsible for a given bit; + // and "Out Bit" is the value of a given bit within the output character, + // with 4 being most significant and 0 being least significant. + // + // for "Meow" we'd get: + // Char: M (0x4d) e (0x65) o (0x6f) w (0x77) + // Value: 0 1 0 0 1 1 0 1 | 0 1 1 0 0 1 0 1 | 0 1 1 0 1 1 1 1 | 0 1 1 1 0 1 1 1 + // Out No.: 5 5 5 6 6 6 6 6 | 3 4 4 4 4 4 5 5 | 2 2 2 2 3 3 3 3 | 0 0 1 1 1 1 1 2 + // Out Bit: 2 1 0 4 3 2 1 0 | 0 4 3 2 1 0 4 3 | 3 2 1 0 4 3 2 1 | 1 0 4 3 2 1 0 4 + // 6: 01101 (0x0d) + // 5: 01010 (0x0a) + // 4: 11001 (0x19) + // 3: 11110 (0x1e) + // 2: 10110 (0x16) + // 1: 11011 (0x1b) + // 0: 00001 (0x01) + // Indexing into the alphabet gives us "1vnyrad" + + for char_no in 0..length { + let bit_no = (length - char_no - 1) * 5; + let byte_no = bit_no / 8; + let bit_offset = bit_no % 8; + + let higher_order_byte = *input.get(byte_no + 1).unwrap_or(&0); + let lower_order_byte = input[byte_no]; + let window = ((higher_order_byte as u16) << 8) | (lower_order_byte as u16); + let value = (window >> bit_offset) & 0b11111; + output.push(NIX32_CHARS[value as usize] as char); + } + + output +} + +#[non_exhaustive] +#[derive(Debug, thiserror::Error)] +pub enum SubstitutionError { + #[error("Decompression error for nar from {0}")] + Decompress(Url, #[source] std::io::Error), + #[error("Unpacking error for nar from {0}")] + Unpack(Url, #[source] nix_nar::NarError), + #[error("Unknown proxy scheme, `https://`, `socks5://`, and `http://` supported")] + UnknownProxyScheme, + #[error("Invalid public key")] + /// Normally an ed25519_dalek::SignatureError, + /// but that comes with no extra information so no need to include it + PublicKey, + #[error("Undecodable narinfo")] + UndecodableNarInfo(#[source] std::io::Error), + #[error("Bad narinfo contents")] + BadNarInfo, + #[error("Bad narinfo signature")] + BadSignature, + #[error("Incorrect nar size or hash for {0}")] + BadNar(Url), + #[error("No substituter has path {0}")] + NonexistantNarInfo(String), + #[error("Invalid nix store path {0}")] + InvalidStorePath(PathBuf), + #[error("Url in narinfo could not be parsed \"{0}\"")] + UrlParseError(url::ParseError), +} + +impl From for ActionErrorKind { + fn from(val: SubstitutionError) -> Self { + ActionErrorKind::Custom(Box::new(val)) + } +} diff --git a/src/action/base/mod.rs b/src/action/base/mod.rs index 71a4c02..376d0f6 100644 --- a/src/action/base/mod.rs +++ b/src/action/base/mod.rs @@ -9,6 +9,7 @@ pub(crate) mod create_or_merge_nix_config; pub(crate) mod create_user; pub(crate) mod delete_user; pub(crate) mod fetch_and_unpack_nix; +pub(crate) mod fetch_and_unpack_nix_substituter; pub(crate) mod move_unpacked_nix; pub(crate) mod remove_directory; pub(crate) mod setup_default_profile; @@ -22,6 +23,7 @@ pub use create_or_merge_nix_config::CreateOrMergeNixConfig; pub use create_user::CreateUser; pub use delete_user::DeleteUser; pub use fetch_and_unpack_nix::{FetchAndUnpackNix, FetchUrlError}; +pub use fetch_and_unpack_nix_substituter::{FetchAndUnpackNixSubstituter, SubstitutionError}; pub use move_unpacked_nix::{MoveUnpackedNix, MoveUnpackedNixError}; pub use remove_directory::RemoveDirectory; pub use setup_default_profile::{SetupDefaultProfile, SetupDefaultProfileError}; diff --git a/src/action/common/place_nix_configuration.rs b/src/action/common/place_nix_configuration.rs index 0e2ffdc..76f4ab2 100644 --- a/src/action/common/place_nix_configuration.rs +++ b/src/action/common/place_nix_configuration.rs @@ -7,7 +7,7 @@ use crate::action::{ Action, ActionDescription, ActionError, ActionErrorKind, ActionTag, StatefulAction, }; use crate::parse_ssl_cert; -use crate::settings::UrlOrPathOrString; +use crate::settings::{UrlOrPathOrString, LIX_DEFAULT_SUBSTITUTERS, LIX_DEFAULT_SUBSTITUTER_KEYS}; use indexmap::map::Entry; use std::path::PathBuf; @@ -136,11 +136,11 @@ impl PlaceNixConfiguration { // Set up our substituters. settings.insert( "substituters".to_string(), - "https://cache.nixos.org https://cache.lix.systems".to_string(), + LIX_DEFAULT_SUBSTITUTERS.join(" "), ); settings.insert( "trusted-public-keys".to_string(), - "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o=".to_string() + LIX_DEFAULT_SUBSTITUTER_KEYS.join(" "), ); if enable_flakes { diff --git a/src/action/common/provision_nix.rs b/src/action/common/provision_nix.rs index e05a209..b048ce0 100644 --- a/src/action/common/provision_nix.rs +++ b/src/action/common/provision_nix.rs @@ -3,19 +3,52 @@ use tracing::{span, Span}; use super::CreateNixTree; use crate::{ action::{ - base::{FetchAndUnpackNix, MoveUnpackedNix}, + base::{FetchAndUnpackNix, FetchAndUnpackNixSubstituter, MoveUnpackedNix}, Action, ActionDescription, ActionError, ActionErrorKind, ActionTag, StatefulAction, }, settings::{CommonSettings, SCRATCH_DIR}, }; use std::path::PathBuf; +#[derive(Debug, serde::Deserialize, serde::Serialize, Clone)] +pub enum FetchNix { + FromTarball(StatefulAction), + FromSubstituter(StatefulAction), +} + +impl FetchNix { + pub async fn try_execute(&mut self) -> Result<(), ActionError> { + match self { + FetchNix::FromTarball(action) => action.try_execute().await, + FetchNix::FromSubstituter(action) => action.try_execute().await, + } + } + pub fn describe_execute(&self) -> Vec { + match self { + FetchNix::FromTarball(action) => action.describe_execute(), + FetchNix::FromSubstituter(action) => action.describe_execute(), + } + } + pub async fn try_revert(&mut self) -> Result<(), ActionError> { + match self { + FetchNix::FromTarball(action) => action.try_revert().await, + FetchNix::FromSubstituter(action) => action.try_revert().await, + } + } + pub fn describe_revert(&self) -> Vec { + match self { + FetchNix::FromTarball(action) => action.describe_revert(), + FetchNix::FromSubstituter(action) => action.describe_revert(), + } + } +} + /** Place Nix and it's requirements onto the target */ #[derive(Debug, serde::Deserialize, serde::Serialize, Clone)] pub struct ProvisionNix { - fetch_nix: StatefulAction, + fetch_nix: FetchNix, create_nix_tree: StatefulAction, move_unpacked_nix: StatefulAction, } @@ -23,13 +56,30 @@ pub struct ProvisionNix { impl ProvisionNix { #[tracing::instrument(level = "debug", skip_all)] pub async fn plan(settings: &CommonSettings) -> Result, ActionError> { - let fetch_nix = FetchAndUnpackNix::plan( - settings.nix_package_url.clone(), - PathBuf::from(SCRATCH_DIR), - settings.proxy.clone(), - settings.ssl_cert_file.clone(), - ) - .await?; + let fetch_nix = if settings.use_substituters { + FetchNix::FromSubstituter( + FetchAndUnpackNixSubstituter::plan( + settings.substitution_targets.clone(), + PathBuf::from(SCRATCH_DIR), + settings.substituter_trusted_keys.clone(), + settings.substituter_require_sigs, + settings.substituters.clone(), + settings.proxy.clone(), + settings.ssl_cert_file.clone(), + ) + .await?, + ) + } else { + FetchNix::FromTarball( + FetchAndUnpackNix::plan( + settings.nix_package_url.clone(), + PathBuf::from(SCRATCH_DIR), + settings.proxy.clone(), + settings.ssl_cert_file.clone(), + ) + .await?, + ) + }; let create_nix_tree = CreateNixTree::plan().await.map_err(Self::error)?; let move_unpacked_nix = MoveUnpackedNix::plan(PathBuf::from(SCRATCH_DIR)) diff --git a/src/settings.rs b/src/settings.rs index 51e10b9..9a6f377 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -29,8 +29,40 @@ pub const LIX_AARCH64_LINUX_URL: &str = pub const LIX_AARCH64_DARWIN_URL: &str = "https://releases.lix.systems/lix/lix-2.90.0/lix-2.90.0-aarch64-darwin.tar.xz"; +/// Default [`substitution_targets`](CommonSettings::substitution_targets) for aarch64-darwin. +pub const LIX_AARCH64_DARWIN_SUBSTITUTION_TARGETS: &[&str; 2] = &[ + "/nix/store/7hhkbhcqj9rii306yv83g6k26rcflssh-lix-2.90.0", + "/nix/store/dnzgnky19lq1gnyx4qrzrxi2qs2yvjin-nss-cacert-3.98", +]; + +/// Default [`substitution_targets`](CommonSettings::substitution_targets) for aarch64-linux. +pub const LIX_AARCH64_LINUX_SUBSTITUTION_TARGETS: &[&str; 2] = &[ + "/nix/store/l2ykng7d4pjiwz0791xnxy7br5261dxg-lix-2.90.0", + "/nix/store/k78zmyfjzaas4ryaaigbdsbfqj3myzdr-nss-cacert-3.98", +]; + +/// Default [`substitution_targets`](CommonSettings::substitution_targets) for x86_64-darwin. +pub const LIX_X86_64_DARWIN_SUBSTITUTION_TARGETS: &[&str; 2] = &[ + "/nix/store/ylqvqp34kyvzvwshqs738k8l8saxwy16-lix-2.90.0", + "/nix/store/hc8xagapf38y3mvfarhi7jcwnfa5w3n9-nss-cacert-3.98", +]; + +/// Default [`substitution_targets`](CommonSettings::substitution_targets) for x86_64-linux. +pub const LIX_X86_64_LINUX_SUBSTITUTION_TARGETS: &[&str; 2] = &[ + "/nix/store/rp7y16q2py2n9y19jvxkjr83lp77bh7y-lix-2.90.0", + "/nix/store/26n4d7n6bm3d1kvai6zmvzx929z9q5c9-nss-cacert-3.98", +]; + // END GENERATE-URLS +pub const LIX_DEFAULT_SUBSTITUTERS: &[&str; 2] = + &["https://cache.nixos.org", "https://cache.lix.systems"]; + +pub const LIX_DEFAULT_SUBSTITUTER_KEYS: &[&str; 2] = &[ + "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=", + "cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o=", +]; + #[derive(Debug, serde::Deserialize, serde::Serialize, Clone, Copy, PartialEq, Eq)] #[cfg_attr(feature = "cli", derive(clap::ValueEnum))] pub enum InitSystem { @@ -146,6 +178,68 @@ pub struct CommonSettings { )] pub nix_build_user_id_base: u32, + /// Substituters used to download Lix, when enabled + #[cfg_attr( + feature = "cli", + clap(long, env = "NIX_INSTALLER_SUBSTITUTERS", default_values = LIX_DEFAULT_SUBSTITUTERS) + )] + pub substituters: Vec, + + /// Trusted signing keys for substituters + #[cfg_attr( + feature = "cli", + clap(long, env = "NIX_INSTALLER_TRUSTED_KEYS", default_values = LIX_DEFAULT_SUBSTITUTER_KEYS) + )] + pub substituter_trusted_keys: Vec, + + /// Store paths to download when use_substituters is set. Should include lix and cacert + #[cfg_attr( + feature = "cli", + clap(long, env = "NIX_INSTALLER_SUBSTITUTION_TARGETS",) + )] + #[cfg_attr( + all(target_os = "macos", target_arch = "x86_64", feature = "cli"), + clap( + default_values = LIX_X86_64_DARWIN_SUBSTITUTION_TARGETS, + ) + )] + #[cfg_attr( + all(target_os = "macos", target_arch = "aarch64", feature = "cli"), + clap( + default_values = LIX_AARCH64_DARWIN_SUBSTITUTION_TARGETS, + ) + )] + #[cfg_attr( + all(target_os = "linux", target_arch = "x86_64", feature = "cli"), + clap( + default_values = LIX_X86_64_LINUX_SUBSTITUTION_TARGETS, + ) + )] + #[cfg_attr( + all(target_os = "linux", target_arch = "aarch64", feature = "cli"), + clap( + default_values = LIX_AARCH64_LINUX_SUBSTITUTION_TARGETS, + ) + )] + pub substitution_targets: Vec, + + /// Require trusted signatures when downloading from substituters + #[cfg_attr( + feature = "cli", + clap( + action(ArgAction::SetFalse), + default_value = "true", + global = true, + env = "NIX_INSTALLER_REQUIRE_SIGS", + long = "no-require-sigs" + ) + )] + pub substituter_require_sigs: bool, + + /// Download Lix from a substituter instead of an install tarball + #[cfg_attr(feature = "cli", clap(long, env = "NIX_INSTALLER_USE_SUBSTITUTER"))] + pub use_substituters: bool, + /// The Nix package URL #[cfg_attr( feature = "cli", @@ -228,6 +322,7 @@ pub struct CommonSettings { impl CommonSettings { /// The default settings for the given Architecture & Operating System pub async fn default() -> Result { + let substitution_targets; let url; let nix_build_user_prefix; let nix_build_user_id_base; @@ -237,6 +332,7 @@ impl CommonSettings { match (Architecture::host(), OperatingSystem::host()) { #[cfg(target_os = "linux")] (Architecture::X86_64, OperatingSystem::Linux) => { + substitution_targets = LIX_X86_64_LINUX_SUBSTITUTION_TARGETS; url = LIX_X86_64_LINUX_URL; nix_build_user_prefix = "nixbld"; nix_build_user_id_base = 30000; @@ -254,6 +350,7 @@ impl CommonSettings { */ #[cfg(target_os = "linux")] (Architecture::Aarch64(_), OperatingSystem::Linux) => { + substitution_targets = LIX_AARCH64_LINUX_SUBSTITUTION_TARGETS; url = LIX_AARCH64_LINUX_URL; nix_build_user_prefix = "nixbld"; nix_build_user_id_base = 30000; @@ -262,6 +359,7 @@ impl CommonSettings { #[cfg(target_os = "macos")] (Architecture::X86_64, OperatingSystem::MacOSX { .. }) | (Architecture::X86_64, OperatingSystem::Darwin) => { + substitution_targets = LIX_X86_64_DARWIN_SUBSTITUTION_TARGETS; url = LIX_X86_64_DARWIN_URL; nix_build_user_prefix = "_nixbld"; nix_build_user_id_base = 300; @@ -270,6 +368,7 @@ impl CommonSettings { #[cfg(target_os = "macos")] (Architecture::Aarch64(_), OperatingSystem::MacOSX { .. }) | (Architecture::Aarch64(_), OperatingSystem::Darwin) => { + substitution_targets = LIX_AARCH64_DARWIN_SUBSTITUTION_TARGETS; url = LIX_AARCH64_DARWIN_URL; nix_build_user_prefix = "_nixbld"; nix_build_user_id_base = 300; @@ -289,6 +388,20 @@ impl CommonSettings { nix_build_user_id_base, nix_build_user_count, nix_build_user_prefix: nix_build_user_prefix.to_string(), + substituters: LIX_DEFAULT_SUBSTITUTERS + .iter() + .map(|s| s.parse()) + .collect::, _>>()?, + substituter_trusted_keys: LIX_DEFAULT_SUBSTITUTER_KEYS + .iter() + .map(|s| s.to_string()) + .collect(), + substitution_targets: substitution_targets + .iter() + .map(|s| PathBuf::from(*s)) + .collect(), + substituter_require_sigs: true, + use_substituters: false, nix_package_url: url.parse()?, proxy: Default::default(), extra_conf: Default::default(), @@ -307,6 +420,11 @@ impl CommonSettings { nix_build_user_prefix, nix_build_user_id_base, nix_build_user_count, + substituters, + substituter_trusted_keys, + substitution_targets, + substituter_require_sigs, + use_substituters, nix_package_url, proxy, extra_conf, @@ -340,10 +458,27 @@ impl CommonSettings { "nix_build_user_count".into(), serde_json::to_value(nix_build_user_count)?, ); + map.insert( + "substituter_trusted_keys".into(), + serde_json::to_value(substituter_trusted_keys)?, + ); + map.insert( + "substitution_targets".into(), + serde_json::to_value(substitution_targets)?, + ); + map.insert( + "substituter_require_sigs".into(), + serde_json::to_value(substituter_require_sigs)?, + ); + map.insert( + "use_substituters".into(), + serde_json::to_value(use_substituters)?, + ); map.insert( "nix_package_url".into(), serde_json::to_value(nix_package_url)?, ); + map.insert("substituters".into(), serde_json::to_value(substituters)?); map.insert("proxy".into(), serde_json::to_value(proxy)?); map.insert("ssl_cert_file".into(), serde_json::to_value(ssl_cert_file)?); map.insert("extra_conf".into(), serde_json::to_value(extra_conf)?); @@ -677,4 +812,4 @@ mod tests { ); Ok(()) } -} \ No newline at end of file +} diff --git a/tests/fixtures/linux/linux.json b/tests/fixtures/linux/linux.json index c670668..5eeef64 100644 --- a/tests/fixtures/linux/linux.json +++ b/tests/fixtures/linux/linux.json @@ -17,15 +17,17 @@ "action": { "action": "provision_nix", "fetch_nix": { - "action": { - "url_or_path": { - "Url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-linux.tar.xz" + "FromTarball": { + "action": { + "url_or_path": { + "Url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-linux.tar.xz" + }, + "dest": "/nix/temp-install-dir", + "proxy": null, + "ssl_cert_file": null }, - "dest": "/nix/temp-install-dir", - "proxy": null, - "ssl_cert_file": null - }, - "state": "Uncompleted" + "state": "Uncompleted" + } }, "delete_users": [], "create_group": { @@ -409,6 +411,19 @@ "nix_build_user_count": 0, "nix_build_user_prefix": "nixbld", "nix_build_user_id_base": 30000, + "substituters": [ + "https://cache.nixos.org/", + "https://cache.lix.systems/" + ], + "substituter_trusted_keys": [ + "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=", + "cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o=" + ], + "substitution_targets": [ + "/nix/store/rp7y16q2py2n9y19jvxkjr83lp77bh7y-lix-2.90.0" + ], + "substituter_require_sigs": true, + "use_substituters": false, "nix_package_url": { "Url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-linux.tar.xz" }, diff --git a/tests/fixtures/linux/steam-deck.json b/tests/fixtures/linux/steam-deck.json index 39525f4..43bdfe3 100644 --- a/tests/fixtures/linux/steam-deck.json +++ b/tests/fixtures/linux/steam-deck.json @@ -61,15 +61,17 @@ "action": { "action": "provision_nix", "fetch_nix": { - "action": { - "url_or_path": { - "Url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-linux.tar.xz" + "FromTarball": { + "action": { + "url_or_path": { + "Url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-linux.tar.xz" + }, + "dest": "/nix/temp-install-dir", + "proxy": null, + "ssl_cert_file": null }, - "dest": "/nix/temp-install-dir", - "proxy": null, - "ssl_cert_file": null - }, - "state": "Uncompleted" + "state": "Uncompleted" + } }, "delete_users": [], "create_group": { @@ -393,6 +395,19 @@ "nix_build_user_count": 0, "nix_build_user_prefix": "nixbld", "nix_build_user_id_base": 30000, + "substituters": [ + "https://cache.nixos.org/", + "https://cache.lix.systems/" + ], + "substituter_trusted_keys": [ + "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=", + "cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o=" + ], + "substitution_targets": [ + "/nix/store/rp7y16q2py2n9y19jvxkjr83lp77bh7y-lix-2.90.0" + ], + "substituter_require_sigs": true, + "use_substituters": false, "nix_package_url": { "Url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-linux.tar.xz" }, diff --git a/tests/fixtures/macos/macos.json b/tests/fixtures/macos/macos.json index a8e6058..aaec8f0 100644 --- a/tests/fixtures/macos/macos.json +++ b/tests/fixtures/macos/macos.json @@ -87,15 +87,17 @@ "action": { "action": "provision_nix", "fetch_nix": { - "action": { - "url_or_path": { - "Url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-darwin.tar.xz" + "FromTarball": { + "action": { + "url_or_path": { + "Url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-darwin.tar.xz" + }, + "dest": "/nix/temp-install-dir", + "proxy": null, + "ssl_cert_file": null }, - "dest": "/nix/temp-install-dir", - "proxy": null, - "ssl_cert_file": null - }, - "state": "Uncompleted" + "state": "Uncompleted" + } }, "delete_users_in_group": null, "create_group": { @@ -420,6 +422,19 @@ "nix_build_user_count": 32, "nix_build_user_prefix": "_nixbld", "nix_build_user_id_base": 300, + "substituters": [ + "https://cache.nixos.org/", + "https://cache.lix.systems/" + ], + "substituter_trusted_keys": [ + "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=", + "cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o=" + ], + "substitution_target": [ + "/nix/store/rp7y16q2py2n9y19jvxkjr83lp77bh7y-lix-2.90.0" + ], + "substituter_require_sigs": true, + "use_substituters": false, "nix_package_url": { "Url": "https://releases.nixos.org/nix/nix-2.17.0/nix-2.17.0-x86_64-darwin.tar.xz" },