From 0aa79dcc6f4e67891cdd9500ae4ce85e4189a951 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Fri, 17 Jul 2020 17:24:02 -0400 Subject: [PATCH 1/3] Remove StoreType abstraction and delegate regStore to each Store implementation. The generic regStore implementation will only be for the ambiguous shorthands, like "" and "auto". This also could get us close to simplifying the daemon command. --- src/libstore/local-store.cc | 16 ++++++++++++ src/libstore/remote-store.cc | 10 +++++--- src/libstore/store-api.cc | 47 ++++++++++-------------------------- src/libstore/store-api.hh | 10 -------- src/nix-daemon/nix-daemon.cc | 5 +++- src/nix/doctor.cc | 4 +-- 6 files changed, 41 insertions(+), 51 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d49d00d6d..9752f3c5f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1580,4 +1580,20 @@ void LocalStore::createUser(const std::string & userName, uid_t userId) } +static RegisterStoreImplementation regStore([]( + const std::string & uri, const Store::Params & params) + -> std::shared_ptr +{ + Store::Params params2 = params; + if (uri == "local") { + } else if (hasPrefix(uri, "/")) { + params2["root"] = uri; + } else if (hasPrefix(uri, "./")) { + params2["root"] = absPath(uri); + } else { + return nullptr; + } + return std::shared_ptr(std::make_shared(params2)); +}); + } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 9af4364b7..a9fbf9f82 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -843,14 +843,18 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } -static std::string uriScheme = "unix://"; +static std::string_view uriScheme = "unix://"; static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr { - if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; - return std::make_shared(std::string(uri, uriScheme.size()), params); + if (hasPrefix(uri, uriScheme)) + return std::make_shared(std::string(uri, uriScheme.size()), params); + else if (uri == "daemon") + return std::make_shared(params); + else + return nullptr; }); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index d37e970df..7a380f127 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -915,44 +915,23 @@ ref openStore(const std::string & uri_, } -StoreType getStoreType(const std::string & uri, const std::string & stateDir) -{ - if (uri == "daemon") { - return tDaemon; - } else if (uri == "local" || hasPrefix(uri, "/") || hasPrefix(uri, "./")) { - return tLocal; - } else if (uri == "" || uri == "auto") { - if (access(stateDir.c_str(), R_OK | W_OK) == 0) - return tLocal; - else if (pathExists(settings.nixDaemonSocketFile)) - return tDaemon; - else - return tLocal; - } else { - return tOther; - } -} - - +// Specific prefixes are handled by the specific types of store, while here we +// handle the general cases not covered by the other ones. static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr { - switch (getStoreType(uri, get(params, "state").value_or(settings.nixStateDir))) { - case tDaemon: - return std::shared_ptr(std::make_shared(params)); - case tLocal: { - Store::Params params2 = params; - if (hasPrefix(uri, "/")) { - params2["root"] = uri; - } else if (hasPrefix(uri, "./")) { - params2["root"] = absPath(uri); - } - return std::shared_ptr(std::make_shared(params2)); - } - default: - return nullptr; - } + auto stateDir = get(params, "state").value_or(settings.nixStateDir); + if (uri == "" || uri == "auto") { + if (access(stateDir.c_str(), R_OK | W_OK) == 0) + return std::make_shared(params); + else if (pathExists(settings.nixDaemonSocketFile)) + return std::make_shared(params); + else + return std::make_shared(params); + } else { + return nullptr; + } }); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a4be0411e..c38290add 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -796,16 +796,6 @@ ref openStore(const std::string & uri = settings.storeUri.get(), const Store::Params & extraParams = Store::Params()); -enum StoreType { - tDaemon, - tLocal, - tOther -}; - - -StoreType getStoreType(const std::string & uri = settings.storeUri.get(), - const std::string & stateDir = settings.nixStateDir); - /* Return the default substituter stores, defined by the ‘substituters’ option and various legacy options. */ std::list> getDefaultSubstituters(); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index bcb86cbce..b52cd7989 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -1,5 +1,6 @@ #include "shared.hh" #include "local-store.hh" +#include "remote-store.hh" #include "util.hh" #include "serialise.hh" #include "archive.hh" @@ -277,7 +278,9 @@ static int _main(int argc, char * * argv) initPlugins(); if (stdio) { - if (getStoreType() == tDaemon) { + if (openUncachedStore().dynamic_pointer_cast()) { + // FIXME Use the connection the UDSRemoteStore opened + // Forward on this connection to the real daemon auto socketPath = settings.nixDaemonSocketFile; auto s = socket(PF_UNIX, SOCK_STREAM, 0); diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 82e92cdd0..683e91446 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -49,9 +49,7 @@ struct CmdDoctor : StoreCommand { logger->log("Running checks against store uri: " + store->getUri()); - auto type = getStoreType(); - - if (type < tOther) { + if (store.dynamic_pointer_cast()) { success &= checkNixInPath(); success &= checkProfileRoots(store); } From a83694c7a1212567ec2f032f84c0d72722bcf5ea Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 19 Aug 2020 19:34:47 +0000 Subject: [PATCH 2/3] Use `RemoteStore` to open connection for proxying daemon Removes duplicate websocket opening code, and also means we should be able to to ssh-ssh-... daemon relays, not just uds-uds-... ones. --- src/libstore/remote-store.cc | 13 ++++++++++--- src/libstore/remote-store.hh | 4 ++-- src/libstore/ssh-store.cc | 1 - src/nix-daemon/nix-daemon.cc | 36 +++++++++--------------------------- 4 files changed, 21 insertions(+), 33 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 553069b89..ff7149643 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -86,7 +86,16 @@ RemoteStore::RemoteStore(const Params & params) : Store(params) , connections(make_ref>( std::max(1, (int) maxConnections), - [this]() { return openConnectionWrapper(); }, + [this]() { + auto conn = openConnectionWrapper(); + try { + initConnection(*conn); + } catch (...) { + failed = true; + throw; + } + return conn; + }, [this](const ref & r) { return r->to.good() @@ -169,8 +178,6 @@ ref UDSRemoteStore::openConnection() conn->startTime = std::chrono::steady_clock::now(); - initConnection(*conn); - return conn; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 72d2a6689..5ad2dc75b 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -102,8 +102,6 @@ public: void flushBadConnections(); -protected: - struct Connection { AutoCloseFD fd; @@ -119,6 +117,8 @@ protected: ref openConnectionWrapper(); +protected: + virtual ref openConnection() = 0; void initConnection(Connection & conn); diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index caae6b596..7e9b44afe 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -89,7 +89,6 @@ ref SSHStore::openConnection() + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); - initConnection(*conn); return conn; } diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 9613cb7d3..bdf56d2e2 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -286,46 +286,28 @@ static int _main(int argc, char * * argv) initPlugins(); if (stdio) { - if (openUncachedStore().dynamic_pointer_cast()) { - // FIXME Use the connection the UDSRemoteStore opened + if (auto store = openUncachedStore().dynamic_pointer_cast()) { + auto conn = store->openConnectionWrapper(); + int from = conn->from.fd; + int to = conn->to.fd; - // Forward on this connection to the real daemon - auto socketPath = settings.nixDaemonSocketFile; - auto s = socket(PF_UNIX, SOCK_STREAM, 0); - if (s == -1) - throw SysError("creating Unix domain socket"); - - auto socketDir = dirOf(socketPath); - if (chdir(socketDir.c_str()) == -1) - throw SysError("changing to socket directory '%1%'", socketDir); - - auto socketName = std::string(baseNameOf(socketPath)); - auto addr = sockaddr_un{}; - addr.sun_family = AF_UNIX; - if (socketName.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket name %1% is too long", socketName); - strcpy(addr.sun_path, socketName.c_str()); - - if (connect(s, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to daemon at %1%", socketPath); - - auto nfds = (s > STDIN_FILENO ? s : STDIN_FILENO) + 1; + auto nfds = std::max(from, to) + 1; while (true) { fd_set fds; FD_ZERO(&fds); - FD_SET(s, &fds); + FD_SET(from, &fds); FD_SET(STDIN_FILENO, &fds); if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) throw SysError("waiting for data from client or server"); - if (FD_ISSET(s, &fds)) { - auto res = splice(s, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (FD_ISSET(from, &fds)) { + auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); if (res == -1) throw SysError("splicing data from daemon socket to stdout"); else if (res == 0) throw EndOfFile("unexpected EOF from daemon socket"); } if (FD_ISSET(STDIN_FILENO, &fds)) { - auto res = splice(STDIN_FILENO, nullptr, s, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); if (res == -1) throw SysError("splicing data from stdin to daemon socket"); else if (res == 0) From 3df78858f2ad91a80e30ba910119a0c16c05c66a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 20 Aug 2020 05:08:50 +0000 Subject: [PATCH 3/3] Fix max fd calc and add test --- src/nix-daemon/nix-daemon.cc | 2 +- tests/local.mk | 1 + tests/ssh-relay.sh | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 tests/ssh-relay.sh diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index bdf56d2e2..6e652ccbf 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -291,7 +291,7 @@ static int _main(int argc, char * * argv) int from = conn->from.fd; int to = conn->to.fd; - auto nfds = std::max(from, to) + 1; + auto nfds = std::max(from, STDIN_FILENO) + 1; while (true) { fd_set fds; FD_ZERO(&fds); diff --git a/tests/local.mk b/tests/local.mk index 53035da41..fd9438386 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -15,6 +15,7 @@ nix_tests = \ linux-sandbox.sh \ build-dry.sh \ build-remote-input-addressed.sh \ + ssh-relay.sh \ nar-access.sh \ structured-attrs.sh \ fetchGit.sh \ diff --git a/tests/ssh-relay.sh b/tests/ssh-relay.sh new file mode 100644 index 000000000..dce50974b --- /dev/null +++ b/tests/ssh-relay.sh @@ -0,0 +1,16 @@ +source common.sh + +echo foo > $TEST_ROOT/hello.sh + +ssh_localhost=ssh://localhost +remote_store=?remote-store=$ssh_localhost + +store=$ssh_localhost + +store+=$remote_store +store+=$remote_store +store+=$remote_store + +out=$(nix add-to-store --store "$store" $TEST_ROOT/hello.sh) + +[ foo = $(< $out) ]