lix-project/lix
18
38
Fork 14
Code Issues 286 Code Review (Gerrit) Projects 1 Releases Wiki Activity

Revive 'nix search'

Browse source 
It uses the evaluation cache now rather than the ad hoc JSON cache.
This commit is contained in:
Eelco Dolstra 2020-04-20 15:27:09 +02:00
parent 42a12f9232
commit b69323f8c9
5 changed files with 117 additions and 203 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/nix/flake.cc
View file

@ -695,7 +695,7 @@ struct CmdFlakeShow : FlakeCommand
void run(nix::ref<nix::Store> store) override
{
auto state = getEvalState();
auto flake = lockFlake();
auto flake = std::make_shared<LockedFlake>(lockFlake());
std::function<void(eval_cache::AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit;
@ -815,7 +815,7 @@ struct CmdFlakeShow : FlakeCommand
auto cache = openEvalCache(*state, flake, useEvalCache);
visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake.flake.lockedRef), "");
visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), "");
}
};

41
src/nix/installables.cc
View file

@ -133,6 +133,15 @@ App Installable::toApp(EvalState & state)
return App(state, *toValue(state).first);
}
std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
Installable::getCursor(EvalState & state, bool useEvalCache)
{
auto evalCache =
std::make_shared<nix::eval_cache::EvalCache>(false, Hash(), state,
[&]() { return toValue(state).first; });
return {{evalCache->getRoot(), ""}};
}
struct InstallableStorePath : Installable
{
ref<Store> store;
@ -285,14 +294,14 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
ref<eval_cache::EvalCache> openEvalCache(
EvalState & state,
const flake::LockedFlake & lockedFlake,
std::shared_ptr<flake::LockedFlake> lockedFlake,
bool useEvalCache)
{
return ref(std::make_shared<nix::eval_cache::EvalCache>(
useEvalCache,
lockedFlake.getFingerprint(),
lockedFlake->getFingerprint(),
state,
[&]()
[&state, lockedFlake]()
{
/* For testing whether the evaluation cache is
complete. */
@ -300,7 +309,7 @@ ref<eval_cache::EvalCache> openEvalCache(
throw Error("not everything is cached, but evaluation is not allowed");
auto vFlake = state.allocValue();
flake::callFlake(state, lockedFlake, *vFlake);
flake::callFlake(state, *lockedFlake, *vFlake);
state.forceAttrs(*vFlake);
@ -315,7 +324,8 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
{
auto state = cmd.getEvalState();
auto lockedFlake = lockFlake(*state, flakeRef, cmd.lockFlags);
auto lockedFlake = std::make_shared<flake::LockedFlake>(
lockFlake(*state, flakeRef, cmd.lockFlags));
auto cache = openEvalCache(*state, lockedFlake, true);
auto root = cache->getRoot();
@ -343,7 +353,7 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
attr->getAttr(state->sOutputName)->getString()
};
return {attrPath, lockedFlake.flake.lockedRef, std::move(drvInfo)};
return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
}
throw Error("flake '%s' does not provide attribute %s",
@ -378,6 +388,25 @@ std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state)
flakeRef, concatStringsSep(", ", quoteStrings(attrPaths)));
}
std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
InstallableFlake::getCursor(EvalState & state, bool useEvalCache)
{
auto evalCache = openEvalCache(state,
std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, cmd.lockFlags)),
useEvalCache);
auto root = evalCache->getRoot();
std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> res;
for (auto & attrPath : getActualAttrPaths()) {
auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath));
if (attr) res.push_back({attr, attrPath});
}
return res;
}
// FIXME: extend
std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)";
static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex));

10
src/nix/installables.hh
View file

@ -12,7 +12,7 @@ namespace nix {
struct DrvInfo;
struct SourceExprCommand;
namespace eval_cache { class EvalCache; }
namespace eval_cache { class EvalCache; class AttrCursor; }
struct Buildable
{
@ -57,6 +57,9 @@ struct Installable
{
return {};
}
virtual std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
getCursor(EvalState & state, bool useEvalCache);
};
struct InstallableValue : Installable
@ -100,11 +103,14 @@ struct InstallableFlake : InstallableValue
std::vector<DerivationInfo> toDerivations() override;
std::pair<Value *, Pos> toValue(EvalState & state) override;
std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
getCursor(EvalState & state, bool useEvalCache) override;
};
ref<eval_cache::EvalCache> openEvalCache(
EvalState & state,
const flake::LockedFlake & lockedFlake,
std::shared_ptr<flake::LockedFlake> lockedFlake,
bool useEvalCache);
}

229
src/nix/search.cc
View file

@ -6,8 +6,9 @@
#include "get-drvs.hh"
#include "common-args.hh"
#include "json.hh"
#include "json-to-value.hh"
#include "shared.hh"
#include "eval-cache.hh"
#include "attr-path.hh"
#include <regex>
#include <fstream>
@ -25,7 +26,7 @@ std::string hilite(const std::string & s, const std::smatch & m, std::string pos
m.empty()
? s
: std::string(m.prefix())
+ ANSI_RED + std::string(m.str()) + postfix
+ ANSI_GREEN + std::string(m.str()) + postfix
+ std::string(m.suffix());
}
@ -33,23 +34,9 @@ struct CmdSearch : InstallableCommand, MixJSON
{
std::vector<std::string> res;
bool writeCache = true;
bool useCache = true;
CmdSearch()
{
expectArgs("regex", &res);
mkFlag()
.longName("update-cache")
.shortName('u')
.description("update the package search cache")
.handler([&]() { writeCache = true; useCache = false; });
mkFlag()
.longName("no-cache")
.description("do not use or update the package search cache")
.handler([&]() { writeCache = false; useCache = false; });
}
std::string description() override
@ -61,27 +48,30 @@ struct CmdSearch : InstallableCommand, MixJSON
{
return {
Example{
"To show all available packages:",
"To show all packages in the flake in the current directory:",
"nix search"
},
Example{
"To show any packages containing 'blender' in its name or description:",
"nix search blender"
"To show packages in the 'nixpkgs' flake containing 'blender' in its name or description:",
"nix search nixpkgs blender"
},
Example{
"To search for Firefox or Chromium:",
"nix search 'firefox|chromium'"
"nix search nixpkgs 'firefox|chromium'"
},
Example{
"To search for git and frontend or gui:",
"nix search git 'frontend|gui'"
"To search for packages containing 'git' and either 'frontend' or 'gui':",
"nix search nixpkgs git 'frontend|gui'"
}
};
}
Strings getDefaultFlakeAttrPaths() override
{
return {""};
return {
"packages." + settings.thisSystem.get() + ".",
"legacyPackages." + settings.thisSystem.get() + "."
};
}
void run(ref<Store> store) override
@ -91,9 +81,8 @@ struct CmdSearch : InstallableCommand, MixJSON
// Empty search string should match all packages
// Use "^" here instead of ".*" due to differences in resulting highlighting
// (see #1893 -- libc++ claims empty search string is not in POSIX grammar)
if (res.empty()) {
if (res.empty())
res.push_back("^");
}
std::vector<std::regex> regexes;
regexes.reserve(res.size());
@ -103,181 +92,91 @@ struct CmdSearch : InstallableCommand, MixJSON
auto state = getEvalState();
//auto [value, pos] = installable->toValue(*state);
#if 0
auto jsonOut = json ? std::make_unique<JSONObject>(std::cout) : nullptr;
auto sToplevel = state->symbols.create("_toplevel");
auto sRecurse = state->symbols.create("recurseForDerivations");
uint64_t results = 0;
bool fromCache = false;
std::map<std::string, std::string> results;
std::function<void(Value *, std::string, bool, JSONObject *)> doExpr;
doExpr = [&](Value * v, std::string attrPath, bool toplevel, JSONObject * cache) {
debug("at attribute '%s'", attrPath);
std::function<void(eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath)> visit;
visit = [&](eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath)
{
Activity act(*logger, lvlInfo, actUnknown,
fmt("evaluating '%s'", concatStringsSep(".", attrPath)));
try {
uint found = 0;
auto recurse = [&]()
{
for (const auto & attr : cursor.getAttrs()) {
auto cursor2 = cursor.getAttr(attr);
auto attrPath2(attrPath);
attrPath2.push_back(attr);
visit(*cursor2, attrPath2);
}
};
state->forceValue(*v);
if (cursor.isDerivation()) {
size_t found = 0;
if (v->type == tLambda && toplevel) {
Value * v2 = state->allocValue();
state->autoCallFunction(*state->allocBindings(1), *v, *v2);
v = v2;
state->forceValue(*v);
}
DrvName name(cursor.getAttr("name")->getString());
if (state->isDerivation(*v)) {
auto aMeta = cursor.maybeGetAttr("meta");
auto aDescription = aMeta ? aMeta->maybeGetAttr("description") : nullptr;
auto description = aDescription ? aDescription->getString() : "";
std::replace(description.begin(), description.end(), '\n', ' ');
auto attrPath2 = concatStringsSep(".", attrPath);
DrvInfo drv(*state, attrPath, v->attrs);
std::string description;
std::smatch attrPathMatch;
std::smatch descriptionMatch;
std::smatch nameMatch;
std::string name;
DrvName parsed(drv.queryName());
for (auto &regex : regexes) {
std::regex_search(attrPath, attrPathMatch, regex);
name = parsed.name;
std::regex_search(name, nameMatch, regex);
description = drv.queryMetaString("description");
std::replace(description.begin(), description.end(), '\n', ' ');
for (auto & regex : regexes) {
std::regex_search(attrPath2, attrPathMatch, regex);
std::regex_search(name.name, nameMatch, regex);
std::regex_search(description, descriptionMatch, regex);
if (!attrPathMatch.empty()
|| !nameMatch.empty()
|| !descriptionMatch.empty())
{
found++;
}
}
if (found == res.size()) {
results++;
if (json) {
auto jsonElem = jsonOut->object(attrPath);
jsonElem.attr("pkgName", parsed.name);
jsonElem.attr("version", parsed.version);
auto jsonElem = jsonOut->object(attrPath2);
jsonElem.attr("pkgName", name.name);
jsonElem.attr("version", name.version);
jsonElem.attr("description", description);
} else {
auto name = hilite(parsed.name, nameMatch, "\e[0;2m")
+ std::string(parsed.fullName, parsed.name.length());
results[attrPath] = fmt(
"* %s (%s)\n %s\n",
wrap("\e[0;1m", hilite(attrPath, attrPathMatch, "\e[0;1m")),
wrap("\e[0;2m", hilite(name, nameMatch, "\e[0;2m")),
hilite(description, descriptionMatch, ANSI_NORMAL));
}
}
if (cache) {
cache->attr("type", "derivation");
cache->attr("name", drv.queryName());
cache->attr("system", drv.querySystem());
if (description != "") {
auto meta(cache->object("meta"));
meta.attr("description", description);
auto name2 = hilite(name.name, nameMatch, "\e[0;2m")
+ std::string(name.fullName, name.name.length());
if (results > 1) logger->stdout("");
logger->stdout(
"* %s (%s)",
wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")),
wrap("\e[0;2m", hilite(name2, nameMatch, "\e[0;2m")));
if (description != "")
logger->stdout(
" %s", hilite(description, descriptionMatch, ANSI_NORMAL));
}
}
}
else if (v->type == tAttrs) {
else if (
attrPath.size() == 0
|| (attrPath[0] == "legacyPackages" && attrPath.size() <= 2)
|| (attrPath[0] == "packages" && attrPath.size() <= 2))
recurse();
if (!toplevel) {
auto attrs = v->attrs;
Bindings::iterator j = attrs->find(sRecurse);
if (j == attrs->end() || !state->forceBool(*j->value, *j->pos)) {
debug("skip attribute '%s'", attrPath);
return;
}
}
bool toplevel2 = false;
if (!fromCache) {
Bindings::iterator j = v->attrs->find(sToplevel);
toplevel2 = j != v->attrs->end() && state->forceBool(*j->value, *j->pos);
}
for (auto & i : *v->attrs) {
auto cache2 =
cache ? std::make_unique<JSONObject>(cache->object(i.name)) : nullptr;
doExpr(i.value,
attrPath == "" ? (std::string) i.name : attrPath + "." + (std::string) i.name,
toplevel2 || fromCache, cache2 ? cache2.get() : nullptr);
}
}
} catch (AssertionError & e) {
} catch (Error & e) {
if (!toplevel) {
e.addPrefix(fmt("While evaluating the attribute '%s':\n", attrPath));
} catch (EvalError & e) {
if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages"))
throw;
}
}
};
Path jsonCacheFileName = getCacheDir() + "/nix/package-search.json";
for (auto & [cursor, prefix] : installable->getCursor(*state, true))
visit(*cursor, parseAttrPath(*state, prefix));
if (useCache && pathExists(jsonCacheFileName)) {
warn("using cached results; pass '-u' to update the cache");
Value vRoot;
parseJSON(*state, readFile(jsonCacheFileName), vRoot);
fromCache = true;
doExpr(&vRoot, "", true, nullptr);
}
else {
createDirs(dirOf(jsonCacheFileName));
Path tmpFile = fmt("%s.tmp.%d", jsonCacheFileName, getpid());
std::ofstream jsonCacheFile;
try {
// iostream considered harmful
jsonCacheFile.exceptions(std::ofstream::failbit);
jsonCacheFile.open(tmpFile);
auto cache = writeCache ? std::make_unique<JSONObject>(jsonCacheFile, false) : nullptr;
// FIXME
throw Error("NOT IMPLEMENTED");
//doExpr(getSourceExpr(*state), "", true, cache.get());
} catch (std::exception &) {
/* Fun fact: catching std::ios::failure does not work
due to C++11 ABI shenanigans.
https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66145 */
if (!jsonCacheFile)
throw Error("error writing to %s", tmpFile);
throw;
}
if (writeCache && rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1)
throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName);
}
if (results.size() == 0)
if (!results)
throw Error("no results for the given search term(s)!");
RunPager pager;
for (auto el : results) std::cout << el.second << "\n";
#endif
}
};

36
tests/search.sh
View file

@ -3,43 +3,23 @@ source common.sh
clearStore
clearCache
exit 0 # FIXME
# No packages
(( $(NIX_PATH= nix search -u|wc -l) == 0 ))
# Haven't updated cache, still nothing
(( $(nix search -f search.nix hello|wc -l) == 0 ))
(( $(nix search -f search.nix |wc -l) == 0 ))
# Update cache, search should work
(( $(nix search -f search.nix -u hello|wc -l) > 0 ))
# Use cache
(( $(nix search -f search.nix foo|wc -l) > 0 ))
(( $(nix search foo|wc -l) > 0 ))
# Test --no-cache works
# No results from cache
(( $(nix search --no-cache foo |wc -l) == 0 ))
# Does find results from file pointed at
(( $(nix search -f search.nix --no-cache foo |wc -l) > 0 ))
(( $(nix search -f search.nix '' hello | wc -l) > 0 ))
# Check descriptions are searched
(( $(nix search broken | wc -l) > 0 ))
(( $(nix search -f search.nix '' broken | wc -l) > 0 ))
# Check search that matches nothing
(( $(nix search nosuchpackageexists | wc -l) == 0 ))
(( $(nix search -f search.nix '' nosuchpackageexists | wc -l) == 0 ))
# Search for multiple arguments
(( $(nix search hello empty | wc -l) == 3 ))
(( $(nix search -f search.nix '' hello empty | wc -l) == 2 ))
# Multiple arguments will not exist
(( $(nix search hello broken | wc -l) == 0 ))
(( $(nix search -f search.nix '' hello broken | wc -l) == 0 ))
## Search expressions
# Check that empty search string matches all
nix search|grep -q foo
nix search|grep -q bar
nix search|grep -q hello
nix search -f search.nix '' |grep -q foo
nix search -f search.nix '' |grep -q bar
nix search -f search.nix '' |grep -q hello