Revive 'nix search'

It uses the evaluation cache now rather than the ad hoc JSON cache.

src/nix/flake.cc

@@ -695,7 +695,7 @@ struct CmdFlakeShow : FlakeCommand
     void run(nix::ref<nix::Store> store) override
     {
         auto state = getEvalState();
-        auto flake = lockFlake();
+        auto flake = std::make_shared<LockedFlake>(lockFlake());
 
         std::function<void(eval_cache::AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit;
 
@@ -815,7 +815,7 @@ struct CmdFlakeShow : FlakeCommand
 
         auto cache = openEvalCache(*state, flake, useEvalCache);
 
-        visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake.flake.lockedRef), "");
+        visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), "");
     }
 };
 

src/nix/installables.cc

@@ -133,6 +133,15 @@ App Installable::toApp(EvalState & state)
     return App(state, *toValue(state).first);
 }
 
+std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+Installable::getCursor(EvalState & state, bool useEvalCache)
+{
+    auto evalCache =
+        std::make_shared<nix::eval_cache::EvalCache>(false, Hash(), state,
+            [&]() { return toValue(state).first; });
+    return {{evalCache->getRoot(), ""}};
+}
+
 struct InstallableStorePath : Installable
 {
     ref<Store> store;
@@ -285,14 +294,14 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
 
 ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
-    const flake::LockedFlake & lockedFlake,
+    std::shared_ptr<flake::LockedFlake> lockedFlake,
     bool useEvalCache)
 {
     return ref(std::make_shared<nix::eval_cache::EvalCache>(
         useEvalCache,
-        lockedFlake.getFingerprint(),
+        lockedFlake->getFingerprint(),
         state,
-        [&]()
+        [&state, lockedFlake]()
         {
             /* For testing whether the evaluation cache is
                complete. */
@@ -300,7 +309,7 @@ ref<eval_cache::EvalCache> openEvalCache(
                 throw Error("not everything is cached, but evaluation is not allowed");
 
             auto vFlake = state.allocValue();
-            flake::callFlake(state, lockedFlake, *vFlake);
+            flake::callFlake(state, *lockedFlake, *vFlake);
 
             state.forceAttrs(*vFlake);
 
@@ -315,7 +324,8 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
 {
     auto state = cmd.getEvalState();
 
-    auto lockedFlake = lockFlake(*state, flakeRef, cmd.lockFlags);
+    auto lockedFlake = std::make_shared<flake::LockedFlake>(
+        lockFlake(*state, flakeRef, cmd.lockFlags));
 
     auto cache = openEvalCache(*state, lockedFlake, true);
     auto root = cache->getRoot();
@@ -343,7 +353,7 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
             attr->getAttr(state->sOutputName)->getString()
         };
 
-        return {attrPath, lockedFlake.flake.lockedRef, std::move(drvInfo)};
+        return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
     }
 
     throw Error("flake '%s' does not provide attribute %s",
@@ -378,6 +388,25 @@ std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state)
         flakeRef, concatStringsSep(", ", quoteStrings(attrPaths)));
 }
 
+std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+InstallableFlake::getCursor(EvalState & state, bool useEvalCache)
+{
+    auto evalCache = openEvalCache(state,
+        std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, cmd.lockFlags)),
+        useEvalCache);
+
+    auto root = evalCache->getRoot();
+
+    std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> res;
+
+    for (auto & attrPath : getActualAttrPaths()) {
+        auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath));
+        if (attr) res.push_back({attr, attrPath});
+    }
+
+    return res;
+}
+
 // FIXME: extend
 std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)";
 static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex));

src/nix/installables.hh

@@ -12,7 +12,7 @@ namespace nix {
 struct DrvInfo;
 struct SourceExprCommand;
 
-namespace eval_cache { class EvalCache; }
+namespace eval_cache { class EvalCache; class AttrCursor; }
 
 struct Buildable
 {
@@ -57,6 +57,9 @@ struct Installable
     {
         return {};
     }
+
+    virtual std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+    getCursor(EvalState & state, bool useEvalCache);
 };
 
 struct InstallableValue : Installable
@@ -100,11 +103,14 @@ struct InstallableFlake : InstallableValue
     std::vector<DerivationInfo> toDerivations() override;
 
     std::pair<Value *, Pos> toValue(EvalState & state) override;
+
+    std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+    getCursor(EvalState & state, bool useEvalCache) override;
 };
 
 ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
-    const flake::LockedFlake & lockedFlake,
+    std::shared_ptr<flake::LockedFlake> lockedFlake,
     bool useEvalCache);
 
 }

src/nix/search.cc

@@ -6,8 +6,9 @@
 #include "get-drvs.hh"
 #include "common-args.hh"
 #include "json.hh"
-#include "json-to-value.hh"
 #include "shared.hh"
+#include "eval-cache.hh"
+#include "attr-path.hh"
 
 #include <regex>
 #include <fstream>
@@ -25,7 +26,7 @@ std::string hilite(const std::string & s, const std::smatch & m, std::string pos
         m.empty()
         ? s
         : std::string(m.prefix())
-          + ANSI_RED + std::string(m.str()) + postfix
+          + ANSI_GREEN + std::string(m.str()) + postfix
           + std::string(m.suffix());
 }
 
@@ -33,23 +34,9 @@ struct CmdSearch : InstallableCommand, MixJSON
 {
     std::vector<std::string> res;
 
-    bool writeCache = true;
-    bool useCache = true;
-
     CmdSearch()
     {
         expectArgs("regex", &res);
-
-        mkFlag()
-            .longName("update-cache")
-            .shortName('u')
-            .description("update the package search cache")
-            .handler([&]() { writeCache = true; useCache = false; });
-
-        mkFlag()
-            .longName("no-cache")
-            .description("do not use or update the package search cache")
-            .handler([&]() { writeCache = false; useCache = false; });
     }
 
     std::string description() override
@@ -61,27 +48,30 @@ struct CmdSearch : InstallableCommand, MixJSON
     {
         return {
             Example{
-                "To show all available packages:",
+                "To show all packages in the flake in the current directory:",
                 "nix search"
             },
             Example{
-                "To show any packages containing 'blender' in its name or description:",
+                "To show packages in the 'nixpkgs' flake containing 'blender' in its name or description:",
-                "nix search blender"
+                "nix search nixpkgs blender"
             },
             Example{
                 "To search for Firefox or Chromium:",
-                "nix search 'firefox|chromium'"
+                "nix search nixpkgs 'firefox|chromium'"
             },
             Example{
-                "To search for git and frontend or gui:",
+                "To search for packages containing 'git' and either 'frontend' or 'gui':",
-                "nix search git 'frontend|gui'"
+                "nix search nixpkgs git 'frontend|gui'"
             }
         };
     }
 
     Strings getDefaultFlakeAttrPaths() override
     {
-        return {""};
+        return {
+            "packages." + settings.thisSystem.get() + ".",
+            "legacyPackages." + settings.thisSystem.get() + "."
+        };
     }
 
     void run(ref<Store> store) override
@@ -91,9 +81,8 @@ struct CmdSearch : InstallableCommand, MixJSON
         // Empty search string should match all packages
         // Use "^" here instead of ".*" due to differences in resulting highlighting
         // (see #1893 -- libc++ claims empty search string is not in POSIX grammar)
-        if (res.empty()) {
+        if (res.empty())
             res.push_back("^");
-        }
 
         std::vector<std::regex> regexes;
         regexes.reserve(res.size());
@@ -103,181 +92,91 @@ struct CmdSearch : InstallableCommand, MixJSON
 
         auto state = getEvalState();
 
-        //auto [value, pos] = installable->toValue(*state);
-
-#if 0
         auto jsonOut = json ? std::make_unique<JSONObject>(std::cout) : nullptr;
 
-        auto sToplevel = state->symbols.create("_toplevel");
+        uint64_t results = 0;
-        auto sRecurse = state->symbols.create("recurseForDerivations");
 
-        bool fromCache = false;
+        std::function<void(eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath)> visit;
-
-        std::map<std::string, std::string> results;
-
-        std::function<void(Value *, std::string, bool, JSONObject *)> doExpr;
-
-        doExpr = [&](Value * v, std::string attrPath, bool toplevel, JSONObject * cache) {
-            debug("at attribute '%s'", attrPath);
 
+        visit = [&](eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath)
+        {
+            Activity act(*logger, lvlInfo, actUnknown,
+                fmt("evaluating '%s'", concatStringsSep(".", attrPath)));
             try {
-                uint found = 0;
+                auto recurse = [&]()
+                {
+                    for (const auto & attr : cursor.getAttrs()) {
+                        auto cursor2 = cursor.getAttr(attr);
+                        auto attrPath2(attrPath);
+                        attrPath2.push_back(attr);
+                        visit(*cursor2, attrPath2);
+                    }
+                };
 
-                state->forceValue(*v);
+                if (cursor.isDerivation()) {
+                    size_t found = 0;
 
-                if (v->type == tLambda && toplevel) {
+                    DrvName name(cursor.getAttr("name")->getString());
-                    Value * v2 = state->allocValue();
-                    state->autoCallFunction(*state->allocBindings(1), *v, *v2);
-                    v = v2;
-                    state->forceValue(*v);
-                }
 
-                if (state->isDerivation(*v)) {
+                    auto aMeta = cursor.maybeGetAttr("meta");
+                    auto aDescription = aMeta ? aMeta->maybeGetAttr("description") : nullptr;
+                    auto description = aDescription ? aDescription->getString() : "";
+                    std::replace(description.begin(), description.end(), '\n', ' ');
+                    auto attrPath2 = concatStringsSep(".", attrPath);
 
-                    DrvInfo drv(*state, attrPath, v->attrs);
-                    std::string description;
                     std::smatch attrPathMatch;
                     std::smatch descriptionMatch;
                     std::smatch nameMatch;
-                    std::string name;
 
-                    DrvName parsed(drv.queryName());
+                    for (auto & regex : regexes) {
-
+                        std::regex_search(attrPath2, attrPathMatch, regex);
-                    for (auto &regex : regexes) {
+                        std::regex_search(name.name, nameMatch, regex);
-                        std::regex_search(attrPath, attrPathMatch, regex);
-
-                        name = parsed.name;
-                        std::regex_search(name, nameMatch, regex);
-
-                        description = drv.queryMetaString("description");
-                        std::replace(description.begin(), description.end(), '\n', ' ');
                         std::regex_search(description, descriptionMatch, regex);
-
                         if (!attrPathMatch.empty()
                             || !nameMatch.empty()
                             || !descriptionMatch.empty())
-                        {
                             found++;
-                        }
                     }
 
                     if (found == res.size()) {
+                        results++;
                         if (json) {
-
+                            auto jsonElem = jsonOut->object(attrPath2);
-                            auto jsonElem = jsonOut->object(attrPath);
+                            jsonElem.attr("pkgName", name.name);
-
+                            jsonElem.attr("version", name.version);
-                            jsonElem.attr("pkgName", parsed.name);
-                            jsonElem.attr("version", parsed.version);
                             jsonElem.attr("description", description);
-
                         } else {
-                            auto name = hilite(parsed.name, nameMatch, "\e[0;2m")
+                            auto name2 = hilite(name.name, nameMatch, "\e[0;2m")
-                                + std::string(parsed.fullName, parsed.name.length());
+                                + std::string(name.fullName, name.name.length());
-                            results[attrPath] = fmt(
+                            if (results > 1) logger->stdout("");
-                                "* %s (%s)\n  %s\n",
+                            logger->stdout(
-                                wrap("\e[0;1m", hilite(attrPath, attrPathMatch, "\e[0;1m")),
+                                "* %s (%s)",
-                                wrap("\e[0;2m", hilite(name, nameMatch, "\e[0;2m")),
+                                wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")),
-                                hilite(description, descriptionMatch, ANSI_NORMAL));
+                                wrap("\e[0;2m", hilite(name2, nameMatch, "\e[0;2m")));
-                        }
+                            if (description != "")
-                    }
+                                logger->stdout(
-
+                                    "  %s", hilite(description, descriptionMatch, ANSI_NORMAL));
-                    if (cache) {
-                        cache->attr("type", "derivation");
-                        cache->attr("name", drv.queryName());
-                        cache->attr("system", drv.querySystem());
-                        if (description != "") {
-                            auto meta(cache->object("meta"));
-                            meta.attr("description", description);
                         }
                     }
                 }
 
-                else if (v->type == tAttrs) {
+                else if (
+                    attrPath.size() == 0
+                    || (attrPath[0] == "legacyPackages" && attrPath.size() <= 2)
+                    || (attrPath[0] == "packages" && attrPath.size() <= 2))
+                    recurse();
 
-                    if (!toplevel) {
+            } catch (EvalError & e) {
-                        auto attrs = v->attrs;
+                if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages"))
-                        Bindings::iterator j = attrs->find(sRecurse);
-                        if (j == attrs->end() || !state->forceBool(*j->value, *j->pos)) {
-                            debug("skip attribute '%s'", attrPath);
-                            return;
-                        }
-                    }
-
-                    bool toplevel2 = false;
-                    if (!fromCache) {
-                        Bindings::iterator j = v->attrs->find(sToplevel);
-                        toplevel2 = j != v->attrs->end() && state->forceBool(*j->value, *j->pos);
-                    }
-
-                    for (auto & i : *v->attrs) {
-                        auto cache2 =
-                            cache ? std::make_unique<JSONObject>(cache->object(i.name)) : nullptr;
-                        doExpr(i.value,
-                            attrPath == "" ? (std::string) i.name : attrPath + "." + (std::string) i.name,
-                            toplevel2 || fromCache, cache2 ? cache2.get() : nullptr);
-                    }
-                }
-
-            } catch (AssertionError & e) {
-            } catch (Error & e) {
-                if (!toplevel) {
-                    e.addPrefix(fmt("While evaluating the attribute '%s':\n", attrPath));
                     throw;
-                }
             }
         };
 
-        Path jsonCacheFileName = getCacheDir() + "/nix/package-search.json";
+        for (auto & [cursor, prefix] : installable->getCursor(*state, true))
+            visit(*cursor, parseAttrPath(*state, prefix));
 
-        if (useCache && pathExists(jsonCacheFileName)) {
+        if (!results)
-
-            warn("using cached results; pass '-u' to update the cache");
-
-            Value vRoot;
-            parseJSON(*state, readFile(jsonCacheFileName), vRoot);
-
-            fromCache = true;
-
-            doExpr(&vRoot, "", true, nullptr);
-        }
-
-        else {
-            createDirs(dirOf(jsonCacheFileName));
-
-            Path tmpFile = fmt("%s.tmp.%d", jsonCacheFileName, getpid());
-
-            std::ofstream jsonCacheFile;
-
-            try {
-                // iostream considered harmful
-                jsonCacheFile.exceptions(std::ofstream::failbit);
-                jsonCacheFile.open(tmpFile);
-
-                auto cache = writeCache ? std::make_unique<JSONObject>(jsonCacheFile, false) : nullptr;
-
-                // FIXME
-                throw Error("NOT IMPLEMENTED");
-                //doExpr(getSourceExpr(*state), "", true, cache.get());
-
-            } catch (std::exception &) {
-                /* Fun fact: catching std::ios::failure does not work
-                   due to C++11 ABI shenanigans.
-                   https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66145 */
-                if (!jsonCacheFile)
-                    throw Error("error writing to %s", tmpFile);
-                throw;
-            }
-
-            if (writeCache && rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1)
-                throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName);
-        }
-
-        if (results.size() == 0)
             throw Error("no results for the given search term(s)!");
-
-        RunPager pager;
-        for (auto el : results) std::cout << el.second << "\n";
-#endif
     }
 };
 

tests/search.sh

@@ -3,43 +3,23 @@ source common.sh
 clearStore
 clearCache
 
-exit 0 # FIXME
+(( $(nix search -f search.nix '' hello | wc -l) > 0 ))
-
-# No packages
-(( $(NIX_PATH= nix search -u|wc -l) == 0 ))
-
-# Haven't updated cache, still nothing
-(( $(nix search -f search.nix hello|wc -l) == 0 ))
-(( $(nix search -f search.nix |wc -l) == 0 ))
-
-# Update cache, search should work
-(( $(nix search -f search.nix -u hello|wc -l) > 0 ))
-
-# Use cache
-(( $(nix search -f search.nix foo|wc -l) > 0 ))
-(( $(nix search foo|wc -l) > 0 ))
-
-# Test --no-cache works
-# No results from cache
-(( $(nix search --no-cache foo |wc -l) == 0 ))
-# Does find results from file pointed at
-(( $(nix search -f search.nix --no-cache foo |wc -l) > 0 ))
 
 # Check descriptions are searched
-(( $(nix search broken | wc -l) > 0 ))
+(( $(nix search -f search.nix '' broken | wc -l) > 0 ))
 
 # Check search that matches nothing
-(( $(nix search nosuchpackageexists | wc -l) == 0 ))
+(( $(nix search -f search.nix '' nosuchpackageexists | wc -l) == 0 ))
 
 # Search for multiple arguments
-(( $(nix search hello empty | wc -l) == 3 ))
+(( $(nix search -f search.nix '' hello empty | wc -l) == 2 ))
 
 # Multiple arguments will not exist
-(( $(nix search hello broken | wc -l) == 0 ))
+(( $(nix search -f search.nix '' hello broken | wc -l) == 0 ))
 
 ## Search expressions
 
 # Check that empty search string matches all
-nix search|grep -q foo
+nix search -f search.nix '' |grep -q foo
-nix search|grep -q bar
+nix search -f search.nix '' |grep -q bar
-nix search|grep -q hello
+nix search -f search.nix '' |grep -q hello