diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index dbe2692f9..0e3d8b462 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,3 +1,6 @@ # Release X.Y (202?-??-??) -- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set. \ No newline at end of file +- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set. + +- Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md). + This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem. diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 105d92f25..be9b0b0d3 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -657,6 +657,21 @@ static void performOp(TunnelLogger * logger, ref store, break; } + case WorkerProto::Op::AddPermRoot: { + if (!trusted) + throw Error( + "you are not privileged to create perm roots\n\n" + "hint: you can just do this client-side without special privileges, and probably want to do that instead."); + auto storePath = WorkerProto::Serialise::read(*store, rconn); + Path gcRoot = absPath(readString(from)); + logger->startWork(); + auto & localFSStore = require(*store); + localFSStore.addPermRoot(storePath, gcRoot); + logger->stopWork(); + to << gcRoot; + break; + } + case WorkerProto::Op::AddIndirectRoot: { Path path = absPath(readString(from)); diff --git a/src/libstore/indirect-root-store.hh b/src/libstore/indirect-root-store.hh index 59e45af45..c11679fe8 100644 --- a/src/libstore/indirect-root-store.hh +++ b/src/libstore/indirect-root-store.hh @@ -11,6 +11,30 @@ namespace nix { * reference. * * See methods for details on the operations it represents. + * + * @note + * To understand the purpose of this class, it might help to do some + * "closed-world" rather than "open-world" reasoning, and consider the + * problem it solved for us. This class was factored out from + * `LocalFSStore` in order to support the following table, which + * contains 4 concrete store types (non-abstract classes, exposed to the + * user), and how they implemented the two GC root methods: + * + * @note + * | | `addPermRoot()` | `addIndirectRoot()` | + * |-------------------|-----------------|---------------------| + * | `LocalStore` | local | local | + * | `UDSRemoteStore` | local | remote | + * | `SSHStore` | doesn't have | doesn't have | + * | `MountedSSHStore` | remote | doesn't have | + * + * @note + * Note how only the local implementations of `addPermRoot()` need + * `addIndirectRoot()`; that is what this class enforces. Without it, + * and with `addPermRoot()` and `addIndirectRoot()` both `virtual`, we + * would accidentally be allowing for a combinatorial explosion of + * possible implementations many of which make no sense. Having this and + * that invariant enforced cuts down that space. */ struct IndirectRootStore : public virtual LocalFSStore { diff --git a/src/libstore/mounted-ssh-store.md b/src/libstore/mounted-ssh-store.md new file mode 100644 index 000000000..1ebfe3081 --- /dev/null +++ b/src/libstore/mounted-ssh-store.md @@ -0,0 +1,18 @@ +R"( + +**Store URL format**: `mounted-ssh-ng://[username@]hostname` + +Experimental store type that allows full access to a Nix store on a remote machine, +and additionally requires that store be mounted in the local file system. + +The mounting of that store is not managed by Nix, and must by managed manually. +It could be accomplished with SSHFS or NFS, for example. + +The local file system is used to optimize certain operations. +For example, rather than serializing Nix archives and sending over the Nix channel, +we can directly access the file system data via the mount-point. + +The local file system is also used to make certain operations possible that wouldn't otherwise be. +For example, persistent GC roots can be created if they reside on the same file system as the remote store: +the remote side will create the symlinks necessary to avoid race conditions. +)" diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 4a6aad449..d4c8ab5b2 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -3,9 +3,10 @@ #include "local-fs-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" -#include "remote-fs-accessor.hh" +#include "source-accessor.hh" #include "archive.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "pool.hh" #include "ssh.hh" @@ -78,6 +79,8 @@ protected: std::string host; + std::vector extraRemoteProgramArgs; + SSHMaster master; void setOptions(RemoteStore::Connection & conn) override @@ -91,6 +94,121 @@ protected: }; }; +struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig +{ + using SSHStoreConfig::SSHStoreConfig; + using LocalFSStoreConfig::LocalFSStoreConfig; + + MountedSSHStoreConfig(StringMap params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) + { + } + + const std::string name() override { return "Experimental SSH Store with filesytem mounted"; } + + std::string doc() override + { + return + #include "mounted-ssh-store.md" + ; + } + + std::optional experimentalFeature() const override + { + return ExperimentalFeature::MountedSSHStore; + } +}; + +/** + * The mounted ssh store assumes that filesystems on the remote host are + * shared with the local host. This means that the remote nix store is + * available locally and is therefore treated as a local filesystem + * store. + * + * MountedSSHStore is very similar to UDSRemoteStore --- ignoring the + * superficial differnce of SSH vs Unix domain sockets, they both are + * accessing remote stores, and they both assume the store will be + * mounted in the local filesystem. + * + * The difference lies in how they manage GC roots. See addPermRoot + * below for details. + */ +class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSHStore, public virtual LocalFSStore +{ +public: + + MountedSSHStore(const std::string & scheme, const std::string & host, const Params & params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) + , MountedSSHStoreConfig(params) + , Store(params) + , RemoteStore(params) + , SSHStore(scheme, host, params) + , LocalFSStore(params) + { + extraRemoteProgramArgs = { + "--process-ops", + }; + } + + static std::set uriSchemes() + { + return {"mounted-ssh-ng"}; + } + + std::string getUri() override + { + return *uriSchemes().begin() + "://" + host; + } + + void narFromPath(const StorePath & path, Sink & sink) override + { + return LocalFSStore::narFromPath(path, sink); + } + + ref getFSAccessor(bool requireValidPath) override + { + return LocalFSStore::getFSAccessor(requireValidPath); + } + + std::optional getBuildLogExact(const StorePath & path) override + { + return LocalFSStore::getBuildLogExact(path); + } + + /** + * This is the key difference from UDSRemoteStore: UDSRemote store + * has the client create the direct root, and the remote side create + * the indirect root. + * + * We could also do that, but the race conditions (will the remote + * side see the direct root the client made?) seems bigger. + * + * In addition, the remote-side will have a process associated with + * the authenticating user handling the connection (even if there + * is a system-wide daemon or similar). This process can safely make + * the direct and indirect roots without there being such a risk of + * privilege escalation / symlinks in directories owned by the + * originating requester that they cannot delete. + */ + Path addPermRoot(const StorePath & path, const Path & gcRoot) override + { + auto conn(getConnection()); + conn->to << WorkerProto::Op::AddPermRoot; + WorkerProto::write(*this, *conn, path); + WorkerProto::write(*this, *conn, gcRoot); + conn.processStderr(); + return readString(conn->from); + } +}; + ref SSHStore::openConnection() { auto conn = make_ref(); @@ -98,6 +216,8 @@ ref SSHStore::openConnection() std::string command = remoteProgram + " --stdio"; if (remoteStore.get() != "") command += " --store " + shellEscape(remoteStore.get()); + for (auto & arg : extraRemoteProgramArgs) + command += " " + shellEscape(arg); conn->sshConn = master.startCommand(command); conn->to = FdSink(conn->sshConn->in.get()); @@ -106,5 +226,6 @@ ref SSHStore::openConnection() } static RegisterStoreImplementation regSSHStore; +static RegisterStoreImplementation regMountedSSHStore; } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 25d544ba7..8a26c09c5 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -9,7 +9,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 35) +#define PROTOCOL_VERSION (1 << 8 | 36) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -161,6 +161,7 @@ enum struct WorkerProto::Op : uint64_t AddMultipleToStore = 44, AddBuildLog = 45, BuildPathsWithResults = 46, + AddPermRoot = 47, }; /** diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index ac4d189e1..2418e3f4c 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -262,6 +262,13 @@ constexpr std::array xpFeatureDetails Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting. )", }, + { + .tag = Xp::MountedSSHStore, + .name = "mounted-ssh-store", + .description = R"( + Allow the use of the [`mounted SSH store`](@docroot@/command-ref/new-cli/nix3-help-stores.html#experimental-ssh-store-with-filesytem-mounted). + )", + }, { .tag = Xp::VerifiedFetches, .name = "verified-fetches", diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index c355b8081..eae4fa9b8 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -34,6 +34,7 @@ enum struct ExperimentalFeature ParseTomlTimestamps, ReadOnlyLocalStore, ConfigurableImpureEnv, + MountedSSHStore, VerifiedFetches, }; diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 373dedf7c..4dada8e0e 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -443,16 +443,23 @@ static void processStdioConnection(ref store, TrustedFlag trustClient) * * @param forceTrustClientOpt See `daemonLoop()` and the parameter with * the same name over there for details. + * + * @param procesOps Whether to force processing ops even if the next + * store also is a remote store and could process it directly. */ -static void runDaemon(bool stdio, std::optional forceTrustClientOpt) +static void runDaemon(bool stdio, std::optional forceTrustClientOpt, bool processOps) { if (stdio) { auto store = openUncachedStore(); + std::shared_ptr remoteStore; + // If --force-untrusted is passed, we cannot forward the connection and // must process it ourselves (before delegating to the next store) to // force untrusting the client. - if (auto remoteStore = store.dynamic_pointer_cast(); remoteStore && (!forceTrustClientOpt || *forceTrustClientOpt != NotTrusted)) + processOps |= !forceTrustClientOpt || *forceTrustClientOpt != NotTrusted; + + if (!processOps && (remoteStore = store.dynamic_pointer_cast())) forwardStdioConnection(*remoteStore); else // `Trusted` is passed in the auto (no override case) because we @@ -468,6 +475,7 @@ static int main_nix_daemon(int argc, char * * argv) { auto stdio = false; std::optional isTrustedOpt = std::nullopt; + auto processOps = false; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--daemon") @@ -487,11 +495,14 @@ static int main_nix_daemon(int argc, char * * argv) } else if (*arg == "--default-trust") { experimentalFeatureSettings.require(Xp::DaemonTrustOverride); isTrustedOpt = std::nullopt; + } else if (*arg == "--process-ops") { + experimentalFeatureSettings.require(Xp::MountedSSHStore); + processOps = true; } else return false; return true; }); - runDaemon(stdio, isTrustedOpt); + runDaemon(stdio, isTrustedOpt, processOps); return 0; } @@ -503,6 +514,7 @@ struct CmdDaemon : StoreCommand { bool stdio = false; std::optional isTrustedOpt = std::nullopt; + bool processOps = false; CmdDaemon() { @@ -538,6 +550,19 @@ struct CmdDaemon : StoreCommand }}, .experimentalFeature = Xp::DaemonTrustOverride, }); + + addFlag({ + .longName = "process-ops", + .description = R"( + Forces the daemon to process received commands itself rather than forwarding the commands straight to the remote store. + + This is useful for the `mounted-ssh://` store where some actions need to be performed on the remote end but as connected user, and not as the user of the underlying daemon on the remote end. + )", + .handler = {[&]() { + processOps = true; + }}, + .experimentalFeature = Xp::MountedSSHStore, + }); } std::string description() override @@ -556,7 +581,7 @@ struct CmdDaemon : StoreCommand void run(ref store) override { - runDaemon(stdio, isTrustedOpt); + runDaemon(stdio, isTrustedOpt, processOps); } }; diff --git a/tests/functional/build-remote-with-mounted-ssh-ng.sh b/tests/functional/build-remote-with-mounted-ssh-ng.sh new file mode 100644 index 000000000..443acb6ca --- /dev/null +++ b/tests/functional/build-remote-with-mounted-ssh-ng.sh @@ -0,0 +1,22 @@ +source common.sh + +requireSandboxSupport +[[ $busybox =~ busybox ]] || skipTest "no busybox" + +enableFeatures mounted-ssh-store + +nix build -Lvf simple.nix \ + --arg busybox $busybox \ + --out-link $TEST_ROOT/result-from-remote \ + --store mounted-ssh-ng://localhost + +nix build -Lvf simple.nix \ + --arg busybox $busybox \ + --out-link $TEST_ROOT/result-from-remote-new-cli \ + --store 'mounted-ssh-ng://localhost?remote-program=nix daemon' + +# This verifies that the out link was actually created and valid. The ability +# to create out links (permanent gc roots) is the distinguishing feature of +# the mounted-ssh-ng store. +cat $TEST_ROOT/result-from-remote/hello | grepQuiet 'Hello World!' +cat $TEST_ROOT/result-from-remote-new-cli/hello | grepQuiet 'Hello World!' diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 21dabca88..8d584142a 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -69,6 +69,7 @@ nix_tests = \ build-remote-trustless-should-pass-2.sh \ build-remote-trustless-should-pass-3.sh \ build-remote-trustless-should-fail-0.sh \ + build-remote-with-mounted-ssh-ng.sh \ nar-access.sh \ pure-eval.sh \ eval.sh \