Merge pull request #8398 from polykernel/perf/lazy-eval-replacements-replacestrings

primops: lazy evaluation of replaceStrings replacements
This commit is contained in:
Robert Hensing 2023-05-30 22:49:42 +02:00 committed by GitHub
commit bed2fe2312
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 17 additions and 14 deletions

View file

@ -4,3 +4,4 @@
The number of parallel downloads (also known as substitutions) has been separated from the [`--max-jobs` setting](../command-ref/conf-file.md#conf-max-jobs). The number of parallel downloads (also known as substitutions) has been separated from the [`--max-jobs` setting](../command-ref/conf-file.md#conf-max-jobs).
The new setting is called [`max-substitution-jobs`](../command-ref/conf-file.md#conf-max-substitution-jobs). The new setting is called [`max-substitution-jobs`](../command-ref/conf-file.md#conf-max-substitution-jobs).
The number of parallel downloads is now set to 16 by default (previously, the default was 1 due to the coupling to build jobs). The number of parallel downloads is now set to 16 by default (previously, the default was 1 due to the coupling to build jobs).
- The function `builtins.replaceStrings` is now lazy in the value of its second argument `to`, that is a replacee in `to` is only evaluated when its corresponding pattern in `from` is matched in the string `s`.

View file

@ -3908,13 +3908,8 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a
for (auto elem : args[0]->listItems()) for (auto elem : args[0]->listItems())
from.emplace_back(state.forceString(*elem, pos, "while evaluating one of the strings to replace passed to builtins.replaceStrings")); from.emplace_back(state.forceString(*elem, pos, "while evaluating one of the strings to replace passed to builtins.replaceStrings"));
std::vector<std::pair<std::string, NixStringContext>> to; std::unordered_map<size_t, std::string> cache;
to.reserve(args[1]->listSize()); auto to = args[1]->listItems();
for (auto elem : args[1]->listItems()) {
NixStringContext ctx;
auto s = state.forceString(*elem, ctx, pos, "while evaluating one of the replacement strings passed to builtins.replaceStrings");
to.emplace_back(s, std::move(ctx));
}
NixStringContext context; NixStringContext context;
auto s = state.forceString(*args[2], context, pos, "while evaluating the third argument passed to builtins.replaceStrings"); auto s = state.forceString(*args[2], context, pos, "while evaluating the third argument passed to builtins.replaceStrings");
@ -3925,10 +3920,19 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a
bool found = false; bool found = false;
auto i = from.begin(); auto i = from.begin();
auto j = to.begin(); auto j = to.begin();
for (; i != from.end(); ++i, ++j) size_t j_index = 0;
for (; i != from.end(); ++i, ++j, ++j_index)
if (s.compare(p, i->size(), *i) == 0) { if (s.compare(p, i->size(), *i) == 0) {
found = true; found = true;
res += j->first; auto v = cache.find(j_index);
if (v == cache.end()) {
NixStringContext ctx;
auto ts = state.forceString(**j, ctx, pos, "while evaluating one of the replacement strings passed to builtins.replaceStrings");
v = (cache.emplace(j_index, ts)).first;
for (auto& path : ctx)
context.insert(path);
}
res += v->second;
if (i->empty()) { if (i->empty()) {
if (p < s.size()) if (p < s.size())
res += s[p]; res += s[p];
@ -3936,9 +3940,6 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a
} else { } else {
p += i->size(); p += i->size();
} }
for (auto& path : j->second)
context.insert(path);
j->second.clear();
break; break;
} }
if (!found) { if (!found) {

View file

@ -171,7 +171,7 @@ namespace nix {
hintfmt("value is %s while a string was expected", "an integer"), hintfmt("value is %s while a string was expected", "an integer"),
hintfmt("while evaluating one of the strings to replace passed to builtins.replaceStrings")); hintfmt("while evaluating one of the strings to replace passed to builtins.replaceStrings"));
ASSERT_TRACE2("replaceStrings [ \"old\" ] [ true ] {}", ASSERT_TRACE2("replaceStrings [ \"oo\" ] [ true ] \"foo\"",
TypeError, TypeError,
hintfmt("value is %s while a string was expected", "a Boolean"), hintfmt("value is %s while a string was expected", "a Boolean"),
hintfmt("while evaluating one of the replacement strings passed to builtins.replaceStrings")); hintfmt("while evaluating one of the replacement strings passed to builtins.replaceStrings"));

View file

@ -1 +1 @@
[ "faabar" "fbar" "fubar" "faboor" "fubar" "XaXbXcX" "X" "a_b" ] [ "faabar" "fbar" "fubar" "faboor" "fubar" "XaXbXcX" "X" "a_b" "fubar" ]

View file

@ -8,4 +8,5 @@ with builtins;
(replaceStrings [""] ["X"] "abc") (replaceStrings [""] ["X"] "abc")
(replaceStrings [""] ["X"] "") (replaceStrings [""] ["X"] "")
(replaceStrings ["-"] ["_"] "a-b") (replaceStrings ["-"] ["_"] "a-b")
(replaceStrings ["oo" "XX"] ["u" (throw "unreachable")] "foobar")
] ]