From 3a63fc6cd515fb009b17d864fede23a356832a5e Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Wed, 21 Oct 2020 21:31:19 +0200 Subject: [PATCH] Allow substituting paths when building remotely using `ssh-ng://` Until now, it was not possible to substitute missing paths from e.g. `https://cache.nixos.org` on a remote server when building on it using the new `ssh-ng` protocol. This is because every store implementation except legacy `ssh://` ignores the substitution flag passed to `Store::queryValidPaths` while the `legacy-ssh-store` substitutes the remote store using `cmdQueryValidPaths` when the remote store is opened with `nix-store --serve`. This patch slightly modifies the daemon protocol to allow passing an integer value suggesting whether to substitute missing paths during `wopQueryValidPaths`. To implement this on the daemon-side, the substitution logic from `nix-store --serve` has been moved into a protected method named `Store::substitutePaths` which gets currently called from `LocalStore::queryValidPaths` and `Store::queryValidPaths` if `maybeSubstitute` is `true`. Fixes #2770 --- flake.nix | 3 ++- src/libstore/daemon.cc | 11 ++++++++++- src/libstore/remote-store.cc | 3 +++ src/libstore/store-api.cc | 22 ++++++++++++++++++++++ src/libstore/store-api.hh | 5 +++++ src/libstore/worker-protocol.hh | 2 +- src/nix-store/nix-store.cc | 23 +---------------------- 7 files changed, 44 insertions(+), 25 deletions(-) diff --git a/flake.nix b/flake.nix index 2abbdff53..8ff048be3 100644 --- a/flake.nix +++ b/flake.nix @@ -12,7 +12,7 @@ versionSuffix = if officialRelease then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified)}_${self.shortRev or "dirty"}"; + else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; officialRelease = false; @@ -117,6 +117,7 @@ nix = with final; with commonDeps pkgs; (stdenv.mkDerivation { name = "nix-${version}"; + inherit version; src = self; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 4dbc7ba38..60cca4fda 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -274,8 +274,17 @@ static void performOp(TunnelLogger * logger, ref store, case wopQueryValidPaths: { auto paths = worker_proto::read(*store, from, Phantom {}); + + SubstituteFlag substitute = NoSubstitute; + if (GET_PROTOCOL_MINOR(clientVersion) >= 27) { + substitute = readInt(from) ? Substitute : NoSubstitute; + } + logger->startWork(); - auto res = store->queryValidPaths(paths); + if (substitute) { + store->substitutePaths(paths); + } + auto res = store->queryValidPaths(paths, substitute); logger->stopWork(); worker_proto::write(*store, to, res); break; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b6f70057d..c27a0f278 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -259,6 +259,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute } else { conn->to << wopQueryValidPaths; worker_proto::write(*this, conn->to, paths); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) { + conn->to << (settings.buildersUseSubstitutes ? 1 : 0); + } conn.processStderr(); return worker_proto::read(*this, conn->from, Phantom {}); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 83d3a1fa1..3129d6637 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -522,6 +522,28 @@ void Store::queryPathInfo(const StorePath & storePath, } +void Store::substitutePaths(const StorePathSet & paths) +{ + std::vector paths2; + for (auto & path : paths) + if (!path.isDerivation()) + paths2.push_back({path}); + uint64_t downloadSize, narSize; + StorePathSet willBuild, willSubstitute, unknown; + queryMissing(paths2, + willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (!willSubstitute.empty()) + try { + std::vector subs; + for (auto & p : willSubstitute) subs.push_back({p}); + buildPaths(subs); + } catch (Error & e) { + logWarning(e.info()); + } +} + + StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { struct State diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f77bc21d1..9a373b561 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -360,6 +360,11 @@ protected: public: + /* If requested, substitute missing paths. This + implements nix-copy-closure's --use-substitutes + flag. */ + void substitutePaths(const StorePathSet & paths); + /* Query which of the given paths is valid. Optionally, try to substitute missing paths. */ virtual StorePathSet queryValidPaths(const StorePathSet & paths, diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index b3705578e..63bd6ea49 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x11a +#define PROTOCOL_VERSION 0x11b #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 6c2702bfe..54394e921 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -830,29 +830,8 @@ static void opServe(Strings opFlags, Strings opArgs) for (auto & path : paths) store->addTempRoot(path); - /* If requested, substitute missing paths. This - implements nix-copy-closure's --use-substitutes - flag. */ if (substitute && writeAllowed) { - /* Filter out .drv files (we don't want to build anything). */ - std::vector paths2; - for (auto & path : paths) - if (!path.isDerivation()) - paths2.push_back({path}); - uint64_t downloadSize, narSize; - StorePathSet willBuild, willSubstitute, unknown; - store->queryMissing(paths2, - willBuild, willSubstitute, unknown, downloadSize, narSize); - /* FIXME: should use ensurePath(), but it only - does one path at a time. */ - if (!willSubstitute.empty()) - try { - std::vector subs; - for (auto & p : willSubstitute) subs.push_back({p}); - store->buildPaths(subs); - } catch (Error & e) { - logWarning(e.info()); - } + store->substitutePaths(paths); } worker_proto::write(*store, out, store->queryValidPaths(paths));