From 09a6321aeb7393cdb4b5af62d2e4106d83124fdf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Jul 2012 17:52:18 -0400 Subject: [PATCH] Replace hasSubstitutes() with querySubstitutablePaths() querySubstitutablePaths() takes a set of paths, so this greatly reduces daemon <-> client latency. --- src/libstore/local-store.cc | 18 +++++++++++++----- src/libstore/local-store.hh | 4 +--- src/libstore/remote-store.cc | 24 +++++++++++++++++------- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.hh | 4 ++-- src/libstore/worker-protocol.hh | 1 + src/nix-env/nix-env.cc | 10 ++++++---- src/nix-worker/nix-worker.cc | 13 +++++++++++-- 8 files changed, 52 insertions(+), 24 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e3d23fdfb..6e4cd053c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -932,16 +932,24 @@ template T getIntLine(int fd) } -bool LocalStore::hasSubstitutes(const Path & path) +PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) { + PathSet res; foreach (Paths::iterator, i, substituters) { + if (res.size() == paths.size()) break; RunningSubstituter & run(runningSubstituters[*i]); startSubstituter(*i, run); - writeLine(run.to, "have\n" + path); - if (getIntLine(run.from)) return true; + string s = "have "; + foreach (PathSet::const_iterator, i, paths) + if (res.find(*i) == res.end()) { s += *i; s += " "; } + writeLine(run.to, s); + while (true) { + Path path = readLine(run.from); + if (path == "") break; + res.insert(path); + } } - - return false; + return res; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 7398c1b9e..3281a9106 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -123,9 +123,7 @@ public: StringSet queryDerivationOutputNames(const Path & path); - PathSet querySubstitutablePaths(); - - bool hasSubstitutes(const Path & path); + PathSet querySubstitutablePaths(const PathSet & paths); void querySubstitutablePathInfos(const Path & substituter, PathSet & paths, SubstitutablePathInfos & infos); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 0cd29c575..2232720c2 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -219,13 +219,13 @@ bool RemoteStore::isValidPath(const Path & path) PathSet RemoteStore::queryValidPaths(const PathSet & paths) { + openConnection(); if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { PathSet res; foreach (PathSet::const_iterator, i, paths) if (isValidPath(*i)) res.insert(*i); return res; } else { - openConnection(); writeInt(wopQueryValidPaths, to); writeStrings(paths, to); processStderr(); @@ -243,14 +243,24 @@ PathSet RemoteStore::queryAllValidPaths() } -bool RemoteStore::hasSubstitutes(const Path & path) +PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths) { openConnection(); - writeInt(wopHasSubstitutes, to); - writeString(path, to); - processStderr(); - unsigned int reply = readInt(from); - return reply != 0; + if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + PathSet res; + foreach (PathSet::const_iterator, i, paths) { + writeInt(wopHasSubstitutes, to); + writeString(*i, to); + processStderr(); + if (readInt(from)) res.insert(*i); + } + return res; + } else { + writeInt(wopQuerySubstitutablePaths, to); + writeStrings(paths, to); + processStderr(); + return readStorePaths(from); + } } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 2668fe256..5b007be48 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -45,7 +45,7 @@ public: StringSet queryDerivationOutputNames(const Path & path); - bool hasSubstitutes(const Path & path); + PathSet querySubstitutablePaths(const PathSet & paths); void querySubstitutablePathInfos(const PathSet & paths, SubstitutablePathInfos & infos); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 13dcd9269..e7963d053 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -145,8 +145,8 @@ public: /* Query the output names of the derivation denoted by `path'. */ virtual StringSet queryDerivationOutputNames(const Path & path) = 0; - /* Query whether a path has substitutes. */ - virtual bool hasSubstitutes(const Path & path) = 0; + /* Query which of the given paths have substitutes. */ + virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0; /* Query substitute info (i.e. references, derivers and download sizes) of a set of paths. If a path does not have substitute diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index b34ad4846..6011ec211 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -42,6 +42,7 @@ typedef enum { wopQueryDerivationOutputNames = 28, wopQuerySubstitutablePathInfos = 29, wopQueryValidPaths = 30, + wopQuerySubstitutablePaths = 31, } WorkerOp; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 91b82c0d0..f06f23dad 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -211,9 +211,12 @@ static int comparePriorities(EvalState & state, static bool isPrebuilt(EvalState & state, const DrvInfo & elem) { + assert(false); +#if 0 return store->isValidPath(elem.queryOutPath(state)) || store->hasSubstitutes(elem.queryOutPath(state)); +#endif } @@ -931,8 +934,7 @@ static void opQuery(Globals & globals, /* Query which paths have substitutes. */ - SubstitutablePathInfos subs; - PathSet validPaths; + PathSet validPaths, substitutablePaths; if (printStatus) { PathSet paths; foreach (vector::iterator, i, elems2) @@ -943,7 +945,7 @@ static void opQuery(Globals & globals, i->setFailed(); } validPaths = store->queryValidPaths(paths); - store->querySubstitutablePathInfos(paths, subs); + substitutablePaths = store->querySubstitutablePaths(paths); } @@ -969,7 +971,7 @@ static void opQuery(Globals & globals, if (printStatus) { Path outPath = i->queryOutPath(globals.state); - bool hasSubs = subs.find(outPath) != subs.end(); + bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end(); bool isInstalled = installed.find(outPath) != installed.end(); bool isValid = validPaths.find(outPath) != validPaths.end(); if (xmlOutput) { diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 4d22f7885..d3907fa8f 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -309,12 +309,21 @@ static void performOp(unsigned int clientVersion, case wopHasSubstitutes: { Path path = readStorePath(from); startWork(); - bool result = store->hasSubstitutes(path); + PathSet res = store->querySubstitutablePaths(singleton(path)); stopWork(); - writeInt(result, to); + writeInt(res.find(path) != res.end(), to); break; } + case wopQuerySubstitutablePaths: { + PathSet paths = readStorePaths(from); + startWork(); + PathSet res = store->querySubstitutablePaths(paths); + stopWork(); + writeStrings(res, to); + break; + } + case wopQueryPathHash: { Path path = readStorePath(from); startWork();