From 92e8230215db50bfbd63729c18072d2ed76c267b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hubert=20G=C5=82uchowski?= Date: Wed, 19 Jan 2022 17:46:29 +0100 Subject: [PATCH 1/7] Make `nix search` highlight all regexes --- src/nix/search.cc | 81 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 17 deletions(-) diff --git a/src/nix/search.cc b/src/nix/search.cc index 0d8fdd5c2..3a62251d4 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -20,16 +20,52 @@ std::string wrap(std::string prefix, std::string s) return prefix + s + ANSI_NORMAL; } +#define HILITE_COLOR ANSI_GREEN + std::string hilite(const std::string & s, const std::smatch & m, std::string postfix) { return m.empty() ? s : std::string(m.prefix()) - + ANSI_GREEN + std::string(m.str()) + postfix + + HILITE_COLOR + std::string(m.str()) + postfix + std::string(m.suffix()); } +std::string hilite_all(const std::string &s, std::vector matches, std::string postfix) { + // Don't waste time on trivial highlights + if (matches.size() == 0) + return s; + else if (matches.size() == 1) + return hilite(s, matches[0], postfix); + + std::sort(matches.begin(), matches.end(), [](const auto &a, const auto &b) { + return a.position() < b.position(); + }); + + std::string out; + ssize_t last_end = 0; + for (size_t i = 0; i < matches.size(); i++) { + auto m = matches[i]; + size_t start = m.position(); + out.append(m.prefix().str().substr(last_end)); + // Merge continous matches + ssize_t end = start + m.length(); + while(i + 1 < matches.size() && matches[i+1].position() <= end) { + auto n = matches[++i]; + ssize_t nend = start + (n.position() - start + n.length()); + if(nend > end) + end = nend; + } + out.append(HILITE_COLOR); + out.append(s.substr(start, end - start)); + out.append(postfix); + last_end = end; + } + out.append(s.substr(last_end)); + return out; +} + struct CmdSearch : InstallableCommand, MixJSON { std::vector res; @@ -100,8 +136,6 @@ struct CmdSearch : InstallableCommand, MixJSON }; if (cursor.isDerivation()) { - size_t found = 0; - DrvName name(cursor.getAttr("name")->getString()); auto aMeta = cursor.maybeGetAttr("meta"); @@ -110,21 +144,34 @@ struct CmdSearch : InstallableCommand, MixJSON std::replace(description.begin(), description.end(), '\n', ' '); auto attrPath2 = concatStringsSep(".", attrPath); - std::smatch attrPathMatch; - std::smatch descriptionMatch; - std::smatch nameMatch; + std::vector attrPathMatches; + std::vector descriptionMatches; + std::vector nameMatches; + bool found = false; for (auto & regex : regexes) { - std::regex_search(attrPath2, attrPathMatch, regex); - std::regex_search(name.name, nameMatch, regex); - std::regex_search(description, descriptionMatch, regex); - if (!attrPathMatch.empty() - || !nameMatch.empty() - || !descriptionMatch.empty()) - found++; + std::smatch tmp; + found = false; + + if(std::regex_search(attrPath2, tmp, regex)) { + attrPathMatches.push_back(tmp); + found = true; + } + if(std::regex_search(name.name, tmp, regex)) { + nameMatches.push_back(tmp); + found = true; + } + if(std::regex_search(description, tmp, regex)) { + descriptionMatches.push_back(tmp); + found = true; + } + + if(!found) + break; } - if (found == res.size()) { + if (found) + { results++; if (json) { auto jsonElem = jsonOut->object(attrPath2); @@ -132,15 +179,15 @@ struct CmdSearch : InstallableCommand, MixJSON jsonElem.attr("version", name.version); jsonElem.attr("description", description); } else { - auto name2 = hilite(name.name, nameMatch, "\e[0;2m"); + auto name2 = hilite_all(name.name, nameMatches, "\e[0;2m"); if (results > 1) logger->cout(""); logger->cout( "* %s%s", - wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")), + wrap("\e[0;1m", hilite_all(attrPath2, attrPathMatches, "\e[0;1m")), name.version != "" ? " (" + name.version + ")" : ""); if (description != "") logger->cout( - " %s", hilite(description, descriptionMatch, ANSI_NORMAL)); + " %s", hilite_all(description, descriptionMatches, ANSI_NORMAL)); } } } From 87fdd230251b871f60f2a148ae2ee00b00854017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hubert=20G=C5=82uchowski?= Date: Wed, 19 Jan 2022 19:59:02 +0100 Subject: [PATCH 2/7] Make `nix search` highlight all matches of a regex --- src/nix/search.cc | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/nix/search.cc b/src/nix/search.cc index 3a62251d4..d23ce0f53 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -10,6 +10,7 @@ #include "eval-cache.hh" #include "attr-path.hh" +#include #include #include @@ -48,7 +49,7 @@ std::string hilite_all(const std::string &s, std::vector matches, s for (size_t i = 0; i < matches.size(); i++) { auto m = matches[i]; size_t start = m.position(); - out.append(m.prefix().str().substr(last_end)); + out.append(s.substr(last_end, m.position() - last_end)); // Merge continous matches ssize_t end = start + m.length(); while(i + 1 < matches.size() && matches[i+1].position() <= end) { @@ -150,21 +151,18 @@ struct CmdSearch : InstallableCommand, MixJSON bool found = false; for (auto & regex : regexes) { - std::smatch tmp; found = false; + auto add_all = [&found](std::sregex_iterator it, std::vector& vec){ + const auto end = std::sregex_iterator(); + while(it != end) { + vec.push_back(*it++); + found = true; + } + }; - if(std::regex_search(attrPath2, tmp, regex)) { - attrPathMatches.push_back(tmp); - found = true; - } - if(std::regex_search(name.name, tmp, regex)) { - nameMatches.push_back(tmp); - found = true; - } - if(std::regex_search(description, tmp, regex)) { - descriptionMatches.push_back(tmp); - found = true; - } + add_all(std::sregex_iterator(attrPath2.begin(), attrPath2.end(), regex), attrPathMatches); + add_all(std::sregex_iterator(name.name.begin(), name.name.end(), regex), nameMatches); + add_all(std::sregex_iterator(description.begin(), description.end(), regex), descriptionMatches); if(!found) break; From 9510ad10c54d906ec2597d3baa81d0f0f97b1f77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hubert=20G=C5=82uchowski?= Date: Thu, 20 Jan 2022 17:12:15 +0100 Subject: [PATCH 3/7] Make `hilite_all` take an iterator of matches instead of a vector. --- src/nix/search.cc | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/nix/search.cc b/src/nix/search.cc index d23ce0f53..01211bae3 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -33,27 +33,27 @@ std::string hilite(const std::string & s, const std::smatch & m, std::string pos + std::string(m.suffix()); } -std::string hilite_all(const std::string &s, std::vector matches, std::string postfix) { - // Don't waste time on trivial highlights - if (matches.size() == 0) +template +std::string hilite_all(const std::string &s, Iter matches_first, Iter matches_last, std::string postfix) { + // Avoid copy on zero matches + if (matches_first == matches_last) return s; - else if (matches.size() == 1) - return hilite(s, matches[0], postfix); - std::sort(matches.begin(), matches.end(), [](const auto &a, const auto &b) { + std::sort(matches_first, matches_last, [](const auto &a, const auto &b) { return a.position() < b.position(); }); std::string out; ssize_t last_end = 0; - for (size_t i = 0; i < matches.size(); i++) { - auto m = matches[i]; + + for (Iter it = matches_first; it != matches_last; ++it) { + auto m = *it; size_t start = m.position(); out.append(s.substr(last_end, m.position() - last_end)); // Merge continous matches ssize_t end = start + m.length(); - while(i + 1 < matches.size() && matches[i+1].position() <= end) { - auto n = matches[++i]; + while(it + 1 != matches_last && (*(it + 1)).position() <= end) { + auto n = *++it; ssize_t nend = start + (n.position() - start + n.length()); if(nend > end) end = nend; @@ -177,15 +177,15 @@ struct CmdSearch : InstallableCommand, MixJSON jsonElem.attr("version", name.version); jsonElem.attr("description", description); } else { - auto name2 = hilite_all(name.name, nameMatches, "\e[0;2m"); + auto name2 = hilite_all(name.name, nameMatches.begin(), nameMatches.end(), "\e[0;2m"); if (results > 1) logger->cout(""); logger->cout( "* %s%s", - wrap("\e[0;1m", hilite_all(attrPath2, attrPathMatches, "\e[0;1m")), + wrap("\e[0;1m", hilite_all(attrPath2, attrPathMatches.begin(), attrPathMatches.end(), "\e[0;1m")), name.version != "" ? " (" + name.version + ")" : ""); if (description != "") logger->cout( - " %s", hilite_all(description, descriptionMatches, ANSI_NORMAL)); + " %s", hilite_all(description, descriptionMatches.begin(), descriptionMatches.end(), ANSI_NORMAL)); } } } From b03fe13b5bb5ce540215a7e8df0d63a99f389db5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hubert=20G=C5=82uchowski?= Date: Thu, 20 Jan 2022 17:48:38 +0100 Subject: [PATCH 4/7] Add some tests for multiple regex/match highlighting in `nix search` --- tests/search.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/search.sh b/tests/search.sh index ee3261687..52e12f381 100644 --- a/tests/search.sh +++ b/tests/search.sh @@ -23,3 +23,16 @@ clearCache nix search -f search.nix '' |grep -q foo nix search -f search.nix '' |grep -q bar nix search -f search.nix '' |grep -q hello + +## Tests for multiple regex/match highlighting + +e=$'\x1b' # grep doesn't support \e, \033 or even \x1b +# Multiple overlapping regexes +(( $(nix search -f search.nix '' 'oo' 'foo' 'oo' | grep "$e\[32;1mfoo$e\\[0;1m" | wc -l) == 1 )) +(( $(nix search -f search.nix '' 'broken b' 'en bar' | grep "$e\[32;1mbroken bar$e\\[0m" | wc -l) == 1 )) + +# Multiple matches +# Searching for 'o' should yield the 'o' in 'broken bar', the 'oo' in foo and 'o' in hello +(( $(nix search -f search.nix '' 'o' | grep -Eo "$e\[32;1mo{1,2}$e\[(0|0;1)m" | wc -l) == 3 )) +# Searching for 'b' should yield the 'b' in bar and the two 'b's in 'broken bar' +(( $(nix search -f search.nix '' 'b' | grep -Eo "$e\[32;1mb$e\[(0|0;1)m" | wc -l) == 3 )) From 1e0b7cdc3f3169ae85d71df5b3f80872a8258779 Mon Sep 17 00:00:00 2001 From: Fishhh Date: Fri, 21 Jan 2022 16:13:34 +0100 Subject: [PATCH 5/7] Move hilite_all into libutil and rename it to hiliteMatches The signature was also changed so the function now accepts a vector instead of an iterator --- src/libutil/fmt.cc | 38 ++++++++++++++++++++++++++++++++ src/libutil/fmt.hh | 3 +++ src/nix/search.cc | 54 ++++------------------------------------------ 3 files changed, 45 insertions(+), 50 deletions(-) create mode 100644 src/libutil/fmt.cc diff --git a/src/libutil/fmt.cc b/src/libutil/fmt.cc new file mode 100644 index 000000000..914fb62b2 --- /dev/null +++ b/src/libutil/fmt.cc @@ -0,0 +1,38 @@ +#include + +namespace nix { + +std::string hiliteMatches(const std::string &s, std::vector matches, std::string prefix, std::string postfix) { + // Avoid copy on zero matches + if (matches.size() == 0) + return s; + + std::sort(matches.begin(), matches.end(), [](const auto &a, const auto &b) { + return a.position() < b.position(); + }); + + std::string out; + ssize_t last_end = 0; + + for (auto it = matches.begin(); it != matches.end();) { + auto m = *it; + size_t start = m.position(); + out.append(s.substr(last_end, m.position() - last_end)); + // Merge continous matches + ssize_t end = start + m.length(); + while(++it != matches.end() && (*it).position() <= end) { + auto n = *it; + ssize_t nend = start + (n.position() - start + n.length()); + if(nend > end) + end = nend; + } + out.append(prefix); + out.append(s.substr(start, end - start)); + out.append(postfix); + last_end = end; + } + out.append(s.substr(last_end)); + return out; +} + +} diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index fd335b811..06c4a2049 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -2,6 +2,7 @@ #include #include +#include #include "ansicolor.hh" @@ -154,4 +155,6 @@ inline hintformat hintfmt(std::string plain_string) // we won't be receiving any args in this case, so just print the original string return hintfmt("%s", normaltxt(plain_string)); } + +std::string hiliteMatches(const std::string &s, std::vector matches, std::string prefix, std::string postfix); } diff --git a/src/nix/search.cc b/src/nix/search.cc index 01211bae3..0d10d8c2e 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -9,8 +9,8 @@ #include "shared.hh" #include "eval-cache.hh" #include "attr-path.hh" +#include "fmt.hh" -#include #include #include @@ -21,52 +21,6 @@ std::string wrap(std::string prefix, std::string s) return prefix + s + ANSI_NORMAL; } -#define HILITE_COLOR ANSI_GREEN - -std::string hilite(const std::string & s, const std::smatch & m, std::string postfix) -{ - return - m.empty() - ? s - : std::string(m.prefix()) - + HILITE_COLOR + std::string(m.str()) + postfix - + std::string(m.suffix()); -} - -template -std::string hilite_all(const std::string &s, Iter matches_first, Iter matches_last, std::string postfix) { - // Avoid copy on zero matches - if (matches_first == matches_last) - return s; - - std::sort(matches_first, matches_last, [](const auto &a, const auto &b) { - return a.position() < b.position(); - }); - - std::string out; - ssize_t last_end = 0; - - for (Iter it = matches_first; it != matches_last; ++it) { - auto m = *it; - size_t start = m.position(); - out.append(s.substr(last_end, m.position() - last_end)); - // Merge continous matches - ssize_t end = start + m.length(); - while(it + 1 != matches_last && (*(it + 1)).position() <= end) { - auto n = *++it; - ssize_t nend = start + (n.position() - start + n.length()); - if(nend > end) - end = nend; - } - out.append(HILITE_COLOR); - out.append(s.substr(start, end - start)); - out.append(postfix); - last_end = end; - } - out.append(s.substr(last_end)); - return out; -} - struct CmdSearch : InstallableCommand, MixJSON { std::vector res; @@ -177,15 +131,15 @@ struct CmdSearch : InstallableCommand, MixJSON jsonElem.attr("version", name.version); jsonElem.attr("description", description); } else { - auto name2 = hilite_all(name.name, nameMatches.begin(), nameMatches.end(), "\e[0;2m"); + auto name2 = hiliteMatches(name.name, std::move(nameMatches), ANSI_GREEN, "\e[0;2m"); if (results > 1) logger->cout(""); logger->cout( "* %s%s", - wrap("\e[0;1m", hilite_all(attrPath2, attrPathMatches.begin(), attrPathMatches.end(), "\e[0;1m")), + wrap("\e[0;1m", hiliteMatches(attrPath2, std::move(attrPathMatches), ANSI_GREEN, "\e[0;1m")), name.version != "" ? " (" + name.version + ")" : ""); if (description != "") logger->cout( - " %s", hilite_all(description, descriptionMatches.begin(), descriptionMatches.end(), ANSI_NORMAL)); + " %s", hiliteMatches(description, std::move(descriptionMatches), ANSI_GREEN, ANSI_NORMAL)); } } } From f82a426502dfd8f583eff3e8b0b895989e10d730 Mon Sep 17 00:00:00 2001 From: Fishhh Date: Fri, 21 Jan 2022 19:38:06 +0100 Subject: [PATCH 6/7] Add some tests for hiliteMatches in libutil --- src/libutil/tests/fmt.cc | 68 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 src/libutil/tests/fmt.cc diff --git a/src/libutil/tests/fmt.cc b/src/libutil/tests/fmt.cc new file mode 100644 index 000000000..33772162c --- /dev/null +++ b/src/libutil/tests/fmt.cc @@ -0,0 +1,68 @@ +#include "fmt.hh" + +#include + +#include + +namespace nix { +/* ----------- tests for fmt.hh -------------------------------------------------*/ + + TEST(hiliteMatches, noHighlight) { + ASSERT_STREQ(hiliteMatches("Hello, world!", std::vector(), "(", ")").c_str(), "Hello, world!"); + } + + TEST(hiliteMatches, simpleHighlight) { + std::string str = "Hello, world!"; + std::regex re = std::regex("world"); + auto matches = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator()); + ASSERT_STREQ( + hiliteMatches(str, matches, "(", ")").c_str(), + "Hello, (world)!" + ); + } + + TEST(hiliteMatches, multipleMatches) { + std::string str = "Hello, world, world, world, world, world, world, Hello!"; + std::regex re = std::regex("world"); + auto matches = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator()); + ASSERT_STREQ( + hiliteMatches(str, matches, "(", ")").c_str(), + "Hello, (world), (world), (world), (world), (world), (world), Hello!" + ); + } + + TEST(hiliteMatches, overlappingMatches) { + std::string str = "world, Hello, world, Hello, world, Hello, world, Hello, world!"; + std::regex re = std::regex("Hello, world"); + std::regex re2 = std::regex("world, Hello"); + auto v = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator()); + for(auto it = std::sregex_iterator(str.begin(), str.end(), re2); it != std::sregex_iterator(); ++it) { + v.push_back(*it); + } + ASSERT_STREQ( + hiliteMatches(str, v, "(", ")").c_str(), + "(world, Hello, world, Hello, world, Hello, world, Hello, world)!" + ); + } + + TEST(hiliteMatches, complexOverlappingMatches) { + std::string str = "legacyPackages.x86_64-linux.git-crypt"; + std::vector regexes = { + std::regex("t-cry"), + std::regex("ux\\.git-cry"), + std::regex("git-c"), + std::regex("pt"), + }; + std::vector matches; + for(auto regex : regexes) + { + for(auto it = std::sregex_iterator(str.begin(), str.end(), regex); it != std::sregex_iterator(); ++it) { + matches.push_back(*it); + } + } + ASSERT_STREQ( + hiliteMatches(str, matches, "(", ")").c_str(), + "legacyPackages.x86_64-lin(ux.git-crypt)" + ); + } +} From ffb28eaa1e15f85d3fbf6bfc3a04a4010f9c80c9 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 24 Jan 2022 10:07:02 +0100 Subject: [PATCH 7/7] Add a small documentation for `hiliteMatches` --- src/libutil/fmt.hh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 06c4a2049..1f81bfcfb 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -156,5 +156,12 @@ inline hintformat hintfmt(std::string plain_string) return hintfmt("%s", normaltxt(plain_string)); } +/** + * Highlight all the given matches in the given string `s` by wrapping them + * between `prefix` and `postfix`. + * + * If some matches overlap, then their union will be wrapped rather than the + * individual matches. + */ std::string hiliteMatches(const std::string &s, std::vector matches, std::string prefix, std::string postfix); }