MountedSSHStore: stores on shared filesystems

This commit is contained in:
Matej Urbas 2023-04-15 11:02:41 +01:00 committed by John Ericson
parent 226b0f3956
commit 06b8902562
3 changed files with 164 additions and 1 deletions

View file

@ -11,6 +11,30 @@ namespace nix {
* reference. * reference.
* *
* See methods for details on the operations it represents. * See methods for details on the operations it represents.
*
* @note
* To understand the purpose of this class, it might help to do some
* "closed-world" rather than "open-world" reasoning, and consider the
* problem it solved for us. This class was factored out from
* `LocalFSStore` in order to support the following table, which
* contains 4 concrete store types (non-abstract classes, exposed to the
* user), and how they implemented the two GC root methods:
*
* @note
* | | `addPermRoot()` | `addIndirectRoot()` |
* |-------------------|-----------------|---------------------|
* | `LocalStore` | local | local |
* | `UDSRemoteStore` | local | remote |
* | `SSHStore` | doesn't have | doesn't have |
* | `MountedSSHStore` | remote | doesn't have |
*
* @note
* Note how only the local implementations of `addPermRoot()` need
* `addIndirectRoot()`; that is what this class enforces. Without it,
* and with `addPermRoot()` and `addIndirectRoot()` both `virtual`, we
* would accidentally be allowing for a combinatorial explosion of
* possible implementations many of which make no sense. Having this and
* that invariant enforced cuts down that space.
*/ */
struct IndirectRootStore : public virtual LocalFSStore struct IndirectRootStore : public virtual LocalFSStore
{ {

View file

@ -0,0 +1,18 @@
R"(
**Store URL format**: `mounted-ssh-ng://[username@]hostname`
Experimental store type that allows full access to a Nix store on a remote machine,
and additionally requires that store be mounted in the local file system.
The mounting of that store is not managed by Nix, and must by managed manually.
It could be accomplished with SSHFS or NFS, for example.
The local file system is used to optimize certain operations.
For example, rather than serializing Nix archives and sending over the Nix channel,
we can directly access the file system data via the mount-point.
The local file system is also used to make certain operations possible that wouldn't otherwise be.
For example, persistent GC roots can be created if they reside on the same file system as the remote store:
the remote side will create the symlinks necessary to avoid race conditions.
)"

View file

@ -3,9 +3,10 @@
#include "local-fs-store.hh" #include "local-fs-store.hh"
#include "remote-store.hh" #include "remote-store.hh"
#include "remote-store-connection.hh" #include "remote-store-connection.hh"
#include "remote-fs-accessor.hh" #include "source-accessor.hh"
#include "archive.hh" #include "archive.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "worker-protocol-impl.hh"
#include "pool.hh" #include "pool.hh"
#include "ssh.hh" #include "ssh.hh"
@ -78,6 +79,8 @@ protected:
std::string host; std::string host;
std::vector<std::string> extraRemoteProgramArgs;
SSHMaster master; SSHMaster master;
void setOptions(RemoteStore::Connection & conn) override void setOptions(RemoteStore::Connection & conn) override
@ -91,6 +94,121 @@ protected:
}; };
}; };
struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig
{
using SSHStoreConfig::SSHStoreConfig;
using LocalFSStoreConfig::LocalFSStoreConfig;
MountedSSHStoreConfig(StringMap params)
: StoreConfig(params)
, RemoteStoreConfig(params)
, CommonSSHStoreConfig(params)
, SSHStoreConfig(params)
, LocalFSStoreConfig(params)
{
}
const std::string name() override { return "Experimental SSH Store with filesytem mounted"; }
std::string doc() override
{
return
#include "mounted-ssh-store.md"
;
}
std::optional<ExperimentalFeature> experimentalFeature() const override
{
return ExperimentalFeature::MountedSSHStore;
}
};
/**
* The mounted ssh store assumes that filesystems on the remote host are
* shared with the local host. This means that the remote nix store is
* available locally and is therefore treated as a local filesystem
* store.
*
* MountedSSHStore is very similar to UDSRemoteStore --- ignoring the
* superficial differnce of SSH vs Unix domain sockets, they both are
* accessing remote stores, and they both assume the store will be
* mounted in the local filesystem.
*
* The difference lies in how they manage GC roots. See addPermRoot
* below for details.
*/
class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSHStore, public virtual LocalFSStore
{
public:
MountedSSHStore(const std::string & scheme, const std::string & host, const Params & params)
: StoreConfig(params)
, RemoteStoreConfig(params)
, CommonSSHStoreConfig(params)
, SSHStoreConfig(params)
, LocalFSStoreConfig(params)
, MountedSSHStoreConfig(params)
, Store(params)
, RemoteStore(params)
, SSHStore(scheme, host, params)
, LocalFSStore(params)
{
extraRemoteProgramArgs = {
"--process-ops",
};
}
static std::set<std::string> uriSchemes()
{
return {"mounted-ssh-ng"};
}
std::string getUri() override
{
return *uriSchemes().begin() + "://" + host;
}
void narFromPath(const StorePath & path, Sink & sink) override
{
return LocalFSStore::narFromPath(path, sink);
}
ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
{
return LocalFSStore::getFSAccessor(requireValidPath);
}
std::optional<std::string> getBuildLogExact(const StorePath & path) override
{
return LocalFSStore::getBuildLogExact(path);
}
/**
* This is the key difference from UDSRemoteStore: UDSRemote store
* has the client create the direct root, and the remote side create
* the indirect root.
*
* We could also do that, but the race conditions (will the remote
* side see the direct root the client made?) seems bigger.
*
* In addition, the remote-side will have a process associated with
* the authenticating user handling the connection (even if there
* is a system-wide daemon or similar). This process can safely make
* the direct and indirect roots without there being such a risk of
* privilege escalation / symlinks in directories owned by the
* originating requester that they cannot delete.
*/
Path addPermRoot(const StorePath & path, const Path & gcRoot) override
{
auto conn(getConnection());
conn->to << WorkerProto::Op::AddPermRoot;
WorkerProto::write(*this, *conn, path);
WorkerProto::write(*this, *conn, gcRoot);
conn.processStderr();
return readString(conn->from);
}
};
ref<RemoteStore::Connection> SSHStore::openConnection() ref<RemoteStore::Connection> SSHStore::openConnection()
{ {
auto conn = make_ref<Connection>(); auto conn = make_ref<Connection>();
@ -98,6 +216,8 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
std::string command = remoteProgram + " --stdio"; std::string command = remoteProgram + " --stdio";
if (remoteStore.get() != "") if (remoteStore.get() != "")
command += " --store " + shellEscape(remoteStore.get()); command += " --store " + shellEscape(remoteStore.get());
for (auto & arg : extraRemoteProgramArgs)
command += " " + shellEscape(arg);
conn->sshConn = master.startCommand(command); conn->sshConn = master.startCommand(command);
conn->to = FdSink(conn->sshConn->in.get()); conn->to = FdSink(conn->sshConn->in.get());
@ -106,5 +226,6 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
} }
static RegisterStoreImplementation<SSHStore, SSHStoreConfig> regSSHStore; static RegisterStoreImplementation<SSHStore, SSHStoreConfig> regSSHStore;
static RegisterStoreImplementation<MountedSSHStore, MountedSSHStoreConfig> regMountedSSHStore;
} }