From ba20730b3f7ad6b09aa86b66748df575e56d442e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 6 Apr 2017 18:40:19 +0200 Subject: [PATCH] Implement RemoteStore::queryMissing() This provides a significant speedup, e.g. 64 s -> 12 s for nix-build --dry-run -I nixpkgs=channel:nixos-16.03 '' -A test on a cold local and CloudFront cache. The alternative is to use lots of concurrent daemon connections but that seems wasteful. --- src/libstore/remote-store.cc | 25 +++++++++++++++++++++++++ src/libstore/remote-store.hh | 4 ++++ src/libstore/store-api.hh | 2 +- src/libstore/worker-protocol.hh | 5 +++-- src/nix-daemon/nix-daemon.cc | 11 +++++++++++ 5 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index a1f2db5b0..c9c590787 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -588,6 +588,31 @@ void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs) } +void RemoteStore::queryMissing(const PathSet & targets, + PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize) +{ + { + auto conn(connections->get()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) + // Don't hold the connection handle in the fallback case + // to prevent a deadlock. + goto fallback; + conn->to << wopQueryMissing << targets; + conn->processStderr(); + willBuild = readStorePaths(*this, conn->from); + willSubstitute = readStorePaths(*this, conn->from); + unknown = readStorePaths(*this, conn->from); + conn->from >> downloadSize >> narSize; + return; + } + + fallback: + return Store::queryMissing(targets, willBuild, willSubstitute, + unknown, downloadSize, narSize); +} + + RemoteStore::Connection::~Connection() { try { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index a08bd3056..db8da7eaa 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -85,6 +85,10 @@ public: void addSignatures(const Path & storePath, const StringSet & sigs) override; + void queryMissing(const PathSet & targets, + PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize) override; + protected: struct Connection diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e07dec495..68c59a9f2 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -524,7 +524,7 @@ public: /* Given a set of paths that are to be built, return the set of derivations that will be built, and the set of output paths that will be substituted. */ - void queryMissing(const PathSet & targets, + virtual void queryMissing(const PathSet & targets, PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, unsigned long long & downloadSize, unsigned long long & narSize); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 6a4ed47cc..6c6766b36 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x112 +#define PROTOCOL_VERSION 0x113 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -47,7 +47,8 @@ typedef enum { wopBuildDerivation = 36, wopAddSignatures = 37, wopNarFromPath = 38, - wopAddToStoreNar = 39 + wopAddToStoreNar = 39, + wopQueryMissing = 40, } WorkerOp; diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index b6a46642c..8786e2561 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -592,6 +592,17 @@ static void performOp(ref store, bool trusted, unsigned int clientVe break; } + case wopQueryMissing: { + PathSet targets = readStorePaths(*store, from); + startWork(); + PathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); + stopWork(); + to << willBuild << willSubstitute << unknown << downloadSize << narSize; + break; + } + default: throw Error(format("invalid operation %1%") % op); }