nix search: Add a cache

The package list is now cached in
~/.cache/nix/package-search.json. This gives a substantial speedup to
"nix search" queries. For example (on an SSD):

First run: (no package search cache, cold page cache)

  $ time nix search blender
  Attribute name: nixpkgs.blender
  Package name: blender
  Version: 2.78c
  Description: 3D Creation/Animation/Publishing System

  real    0m6.516s

Second run: (package search cache populated)

  $ time nix search blender
  Attribute name: nixpkgs.blender
  Package name: blender
  Version: 2.78c
  Description: 3D Creation/Animation/Publishing System

  real    0m0.143s
This commit is contained in:
Eelco Dolstra 2017-07-26 17:21:46 +02:00
parent 4c9ff89c26
commit 57b9505731
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
3 changed files with 92 additions and 20 deletions

View file

@ -50,20 +50,22 @@ template<> void toJSON<std::nullptr_t>(std::ostream & str, const std::nullptr_t
JSONWriter::JSONWriter(std::ostream & str, bool indent) JSONWriter::JSONWriter(std::ostream & str, bool indent)
: state(new JSONState(str, indent)) : state(new JSONState(str, indent))
{ {
state->stack.push_back(this); state->stack++;
} }
JSONWriter::JSONWriter(JSONState * state) JSONWriter::JSONWriter(JSONState * state)
: state(state) : state(state)
{ {
state->stack.push_back(this); state->stack++;
} }
JSONWriter::~JSONWriter() JSONWriter::~JSONWriter()
{ {
assertActive(); if (state) {
state->stack.pop_back(); assertActive();
if (state->stack.empty()) delete state; state->stack--;
if (state->stack == 0) delete state;
}
} }
void JSONWriter::comma() void JSONWriter::comma()
@ -121,9 +123,11 @@ void JSONObject::open()
JSONObject::~JSONObject() JSONObject::~JSONObject()
{ {
state->depth--; if (state) {
if (state->indent && !first) indent(); state->depth--;
state->str << "}"; if (state->indent && !first) indent();
state->str << "}";
}
} }
void JSONObject::attr(const std::string & s) void JSONObject::attr(const std::string & s)

View file

@ -21,11 +21,11 @@ protected:
std::ostream & str; std::ostream & str;
bool indent; bool indent;
size_t depth = 0; size_t depth = 0;
std::vector<JSONWriter *> stack; size_t stack = 0;
JSONState(std::ostream & str, bool indent) : str(str), indent(indent) { } JSONState(std::ostream & str, bool indent) : str(str), indent(indent) { }
~JSONState() ~JSONState()
{ {
assert(stack.empty()); assert(stack == 0);
} }
}; };
@ -41,7 +41,7 @@ protected:
void assertActive() void assertActive()
{ {
assert(!state->stack.empty() && state->stack.back() == this); assert(state->stack != 0);
} }
void comma(); void comma();
@ -117,6 +117,14 @@ public:
open(); open();
} }
JSONObject(const JSONObject & obj) = delete;
JSONObject(JSONObject && obj)
: JSONWriter(obj.state)
{
obj.state = 0;
}
~JSONObject(); ~JSONObject();
template<typename T> template<typename T>

View file

@ -6,8 +6,10 @@
#include "get-drvs.hh" #include "get-drvs.hh"
#include "common-args.hh" #include "common-args.hh"
#include "json.hh" #include "json.hh"
#include "json-to-value.hh"
#include <regex> #include <regex>
#include <fstream>
using namespace nix; using namespace nix;
@ -25,9 +27,23 @@ struct CmdSearch : SourceExprCommand, MixJSON
{ {
std::string re; std::string re;
bool writeCache = true;
bool useCache = true;
CmdSearch() CmdSearch()
{ {
expectArg("regex", &re, true); expectArg("regex", &re, true);
mkFlag()
.longName("update-cache")
.shortName('u')
.description("update the package search cache")
.handler([&](Strings ss) { writeCache = true; useCache = false; });
mkFlag()
.longName("no-cache")
.description("do not use or update the package search cache")
.handler([&](Strings ss) { writeCache = false; useCache = false; });
} }
std::string name() override std::string name() override
@ -48,15 +64,18 @@ struct CmdSearch : SourceExprCommand, MixJSON
auto state = getEvalState(); auto state = getEvalState();
std::function<void(Value *, std::string, bool)> doExpr;
bool first = true; bool first = true;
auto jsonOut = json ? std::make_unique<JSONObject>(std::cout, true) : nullptr; auto jsonOut = json ? std::make_unique<JSONObject>(std::cout, true) : nullptr;
auto sToplevel = state->symbols.create("_toplevel"); auto sToplevel = state->symbols.create("_toplevel");
auto sRecurse = state->symbols.create("recurseForDerivations");
doExpr = [&](Value * v, std::string attrPath, bool toplevel) { bool fromCache = false;
std::function<void(Value *, std::string, bool, JSONObject *)> doExpr;
doExpr = [&](Value * v, std::string attrPath, bool toplevel, JSONObject * cache) {
debug("at attribute %s", attrPath); debug("at attribute %s", attrPath);
try { try {
@ -115,23 +134,41 @@ struct CmdSearch : SourceExprCommand, MixJSON
hilite(description, descriptionMatch)); hilite(description, descriptionMatch));
} }
} }
if (cache) {
cache->attr("type", "derivation");
cache->attr("name", drv.queryName());
cache->attr("system", drv.querySystem());
if (description != "") {
auto meta(cache->object("meta"));
meta.attr("description", description);
}
}
} }
else if (v->type == tAttrs) { else if (v->type == tAttrs) {
if (!toplevel) { if (!toplevel) {
auto attrs = v->attrs; auto attrs = v->attrs;
Bindings::iterator j = attrs->find(state->symbols.create("recurseForDerivations")); Bindings::iterator j = attrs->find(sRecurse);
if (j == attrs->end() || !state->forceBool(*j->value, *j->pos)) return; if (j == attrs->end() || !state->forceBool(*j->value, *j->pos)) {
debug("skip attribute %s", attrPath);
return;
}
} }
Bindings::iterator j = v->attrs->find(sToplevel); bool toplevel2 = false;
bool toplevel2 = j != v->attrs->end() && state->forceBool(*j->value, *j->pos); if (!fromCache) {
Bindings::iterator j = v->attrs->find(sToplevel);
toplevel2 = j != v->attrs->end() && state->forceBool(*j->value, *j->pos);
}
for (auto & i : *v->attrs) { for (auto & i : *v->attrs) {
auto cache2 =
cache ? std::make_unique<JSONObject>(cache->object(i.name)) : nullptr;
doExpr(i.value, doExpr(i.value,
attrPath == "" ? (std::string) i.name : attrPath + "." + (std::string) i.name, attrPath == "" ? (std::string) i.name : attrPath + "." + (std::string) i.name,
toplevel2); toplevel2 || fromCache, cache2 ? cache2.get() : nullptr);
} }
} }
@ -144,7 +181,30 @@ struct CmdSearch : SourceExprCommand, MixJSON
} }
}; };
doExpr(getSourceExpr(*state), "", true); Path jsonCacheFileName = getCacheDir() + "/nix/package-search.json";
if (useCache && pathExists(jsonCacheFileName)) {
Value vRoot;
parseJSON(*state, readFile(jsonCacheFileName), vRoot);
fromCache = true;
doExpr(&vRoot, "", true, nullptr);
}
else {
Path tmpFile = fmt("%s.tmp.%d", jsonCacheFileName, getpid());
std::ofstream jsonCacheFile(tmpFile);
auto cache = writeCache ? std::make_unique<JSONObject>(jsonCacheFile, false) : nullptr;
doExpr(getSourceExpr(*state), "", true, cache.get());
if (rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1)
throw SysError("cannot rename %s to %s", tmpFile, jsonCacheFileName);
}
} }
}; };