Allow substituting paths when building remotely using ssh-ng://

Until now, it was not possible to substitute missing paths from e.g.
`https://cache.nixos.org` on a remote server when building on it using
the new `ssh-ng` protocol.

This is because every store implementation except legacy `ssh://`
ignores the substitution flag passed to `Store::queryValidPaths` while
the `legacy-ssh-store` substitutes the remote store using
`cmdQueryValidPaths` when the remote store is opened with `nix-store
--serve`.

This patch slightly modifies the daemon protocol to allow passing an
integer value suggesting whether to substitute missing paths during
`wopQueryValidPaths`. To implement this on the daemon-side, the
substitution logic from `nix-store --serve` has been moved into a
protected method named `Store::substitutePaths` which gets currently
called from `LocalStore::queryValidPaths` and `Store::queryValidPaths`
if `maybeSubstitute` is `true`.

Fixes #2770
This commit is contained in:
Maximilian Bosch 2020-10-21 21:31:19 +02:00
parent 387f824cab
commit 3a63fc6cd5
No known key found for this signature in database
GPG key ID: 091DBF4D1FC46B8E
7 changed files with 44 additions and 25 deletions

View file

@ -12,7 +12,7 @@
versionSuffix = versionSuffix =
if officialRelease if officialRelease
then "" then ""
else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified)}_${self.shortRev or "dirty"}"; else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
officialRelease = false; officialRelease = false;
@ -117,6 +117,7 @@
nix = with final; with commonDeps pkgs; (stdenv.mkDerivation { nix = with final; with commonDeps pkgs; (stdenv.mkDerivation {
name = "nix-${version}"; name = "nix-${version}";
inherit version;
src = self; src = self;

View file

@ -274,8 +274,17 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopQueryValidPaths: { case wopQueryValidPaths: {
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {}); auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
SubstituteFlag substitute = NoSubstitute;
if (GET_PROTOCOL_MINOR(clientVersion) >= 27) {
substitute = readInt(from) ? Substitute : NoSubstitute;
}
logger->startWork(); logger->startWork();
auto res = store->queryValidPaths(paths); if (substitute) {
store->substitutePaths(paths);
}
auto res = store->queryValidPaths(paths, substitute);
logger->stopWork(); logger->stopWork();
worker_proto::write(*store, to, res); worker_proto::write(*store, to, res);
break; break;

View file

@ -259,6 +259,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
} else { } else {
conn->to << wopQueryValidPaths; conn->to << wopQueryValidPaths;
worker_proto::write(*this, conn->to, paths); worker_proto::write(*this, conn->to, paths);
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) {
conn->to << (settings.buildersUseSubstitutes ? 1 : 0);
}
conn.processStderr(); conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
} }

View file

@ -522,6 +522,28 @@ void Store::queryPathInfo(const StorePath & storePath,
} }
void Store::substitutePaths(const StorePathSet & paths)
{
std::vector<StorePathWithOutputs> paths2;
for (auto & path : paths)
if (!path.isDerivation())
paths2.push_back({path});
uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
queryMissing(paths2,
willBuild, willSubstitute, unknown, downloadSize, narSize);
if (!willSubstitute.empty())
try {
std::vector<StorePathWithOutputs> subs;
for (auto & p : willSubstitute) subs.push_back({p});
buildPaths(subs);
} catch (Error & e) {
logWarning(e.info());
}
}
StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute)
{ {
struct State struct State

View file

@ -360,6 +360,11 @@ protected:
public: public:
/* If requested, substitute missing paths. This
implements nix-copy-closure's --use-substitutes
flag. */
void substitutePaths(const StorePathSet & paths);
/* Query which of the given paths is valid. Optionally, try to /* Query which of the given paths is valid. Optionally, try to
substitute missing paths. */ substitute missing paths. */
virtual StorePathSet queryValidPaths(const StorePathSet & paths, virtual StorePathSet queryValidPaths(const StorePathSet & paths,

View file

@ -6,7 +6,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f #define WORKER_MAGIC_2 0x6478696f
#define PROTOCOL_VERSION 0x11a #define PROTOCOL_VERSION 0x11b
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)

View file

@ -830,29 +830,8 @@ static void opServe(Strings opFlags, Strings opArgs)
for (auto & path : paths) for (auto & path : paths)
store->addTempRoot(path); store->addTempRoot(path);
/* If requested, substitute missing paths. This
implements nix-copy-closure's --use-substitutes
flag. */
if (substitute && writeAllowed) { if (substitute && writeAllowed) {
/* Filter out .drv files (we don't want to build anything). */ store->substitutePaths(paths);
std::vector<StorePathWithOutputs> paths2;
for (auto & path : paths)
if (!path.isDerivation())
paths2.push_back({path});
uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths2,
willBuild, willSubstitute, unknown, downloadSize, narSize);
/* FIXME: should use ensurePath(), but it only
does one path at a time. */
if (!willSubstitute.empty())
try {
std::vector<StorePathWithOutputs> subs;
for (auto & p : willSubstitute) subs.push_back({p});
store->buildPaths(subs);
} catch (Error & e) {
logWarning(e.info());
}
} }
worker_proto::write(*store, out, store->queryValidPaths(paths)); worker_proto::write(*store, out, store->queryValidPaths(paths));