Merge pull request #5887 from pennae/avoid-streams

avoid std::?stream overhead when it's not helpful

src/libexpr/eval.cc

@@ -36,6 +36,19 @@
 namespace nix {
 
 
+static char * allocString(size_t size)
+{
+    char * t;
+#if HAVE_BOEHMGC
+    t = (char *) GC_MALLOC_ATOMIC(size);
+#else
+    t = malloc(size);
+#endif
+    if (!t) throw std::bad_alloc();
+    return t;
+}
+
+
 static char * dupString(const char * s)
 {
     char * t;
@@ -771,17 +784,28 @@ void Value::mkString(std::string_view s)
 }
 
 
+static void copyContextToValue(Value & v, const PathSet & context)
+{
+    if (!context.empty()) {
+        size_t n = 0;
+        v.string.context = (const char * *)
+            allocBytes((context.size() + 1) * sizeof(char *));
+        for (auto & i : context)
+            v.string.context[n++] = dupString(i.c_str());
+        v.string.context[n] = 0;
+    }
+}
+
 void Value::mkString(std::string_view s, const PathSet & context)
 {
     mkString(s);
-    if (!context.empty()) {
+    copyContextToValue(*this, context);
-        size_t n = 0;
+}
-        string.context = (const char * *)
+
-            allocBytes((context.size() + 1) * sizeof(char *));
+void Value::mkStringMove(const char * s, const PathSet & context)
-        for (auto & i : context)
+{
-            string.context[n++] = dupString(i.c_str());
+    mkString(s);
-        string.context[n] = 0;
+    copyContextToValue(*this, context);
-    }
 }
 
 
@@ -1660,13 +1684,34 @@ void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Po
 void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
 {
     PathSet context;
-    std::ostringstream s;
+    std::vector<std::string> s;
+    size_t sSize = 0;
     NixInt n = 0;
     NixFloat nf = 0;
 
     bool first = !forceString;
     ValueType firstType = nString;
 
+    const auto str = [&] {
+        std::string result;
+        result.reserve(sSize);
+        for (const auto & part : s) result += part;
+        return result;
+    };
+    /* c_str() is not str().c_str() because we want to create a string
+       Value. allocating a GC'd string directly and moving it into a
+       Value lets us avoid an allocation and copy. */
+    const auto c_str = [&] {
+        char * result = allocString(sSize + 1);
+        char * tmp = result;
+        for (const auto & part : s) {
+            memcpy(tmp, part.c_str(), part.size());
+            tmp += part.size();
+        }
+        *tmp = 0;
+        return result;
+    };
+
     for (auto & [i_pos, i] : *es) {
         Value vTmp;
         i->eval(state, env, vTmp);
@@ -1696,11 +1741,15 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
                 nf += vTmp.fpoint;
             } else
                 throwEvalError(i_pos, "cannot add %1% to a float", showType(vTmp));
-        } else
+        } else {
+            if (s.empty()) s.reserve(es->size());
             /* skip canonization of first path, which would only be not
             canonized in the first place if it's coming from a ./${foo} type
             path */
-            s << state.coerceToString(i_pos, vTmp, context, false, firstType == nString, !first);
+            s.emplace_back(
+                state.coerceToString(i_pos, vTmp, context, false, firstType == nString, !first));
+            sSize += s.back().size();
+        }
 
         first = false;
     }
@@ -1712,9 +1761,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     else if (firstType == nPath) {
         if (!context.empty())
             throwEvalError(pos, "a string that refers to a store path cannot be appended to a path");
-        v.mkPath(canonPath(s.str()));
+        v.mkPath(canonPath(str()));
     } else
-        v.mkString(s.str(), context);
+        v.mkStringMove(c_str(), context);
 }
 
 

src/libexpr/value.hh

@@ -241,6 +241,8 @@ public:
 
     void mkString(std::string_view s, const PathSet & context);
 
+    void mkStringMove(const char * s, const PathSet & context);
+
     inline void mkString(const Symbol & s)
     {
         mkString(((const std::string &) s).c_str());

src/libutil/json.cc

@@ -7,16 +7,38 @@ namespace nix {
 
 void toJSON(std::ostream & str, const char * start, const char * end)
 {
-    str << '"';
+    constexpr size_t BUF_SIZE = 4096;
-    for (auto i = start; i != end; i++)
+    char buf[BUF_SIZE + 7]; // BUF_SIZE + largest single sequence of puts
-        if (*i == '\"' || *i == '\\') str << '\\' << *i;
+    size_t bufPos = 0;
-        else if (*i == '\n') str << "\\n";
+
-        else if (*i == '\r') str << "\\r";
+    const auto flush = [&] {
-        else if (*i == '\t') str << "\\t";
+        str.write(buf, bufPos);
-        else if (*i >= 0 && *i < 32)
+        bufPos = 0;
-            str << "\\u" << std::setfill('0') << std::setw(4) << std::hex << (uint16_t) *i << std::dec;
+    };
-        else str << *i;
+    const auto put = [&] (char c) {
-    str << '"';
+        buf[bufPos++] = c;
+    };
+
+    put('"');
+    for (auto i = start; i != end; i++) {
+        if (bufPos >= BUF_SIZE) flush();
+        if (*i == '\"' || *i == '\\') { put('\\'); put(*i); }
+        else if (*i == '\n') { put('\\'); put('n'); }
+        else if (*i == '\r') { put('\\'); put('r'); }
+        else if (*i == '\t') { put('\\'); put('t'); }
+        else if (*i >= 0 && *i < 32) {
+            const char hex[17] = "0123456789abcdef";
+            put('\\');
+            put('u');
+            put(hex[(uint16_t(*i) >> 12) & 0xf]);
+            put(hex[(uint16_t(*i) >>  8) & 0xf]);
+            put(hex[(uint16_t(*i) >>  4) & 0xf]);
+            put(hex[(uint16_t(*i) >>  0) & 0xf]);
+        }
+        else put(*i);
+    }
+    put('"');
+    flush();
 }
 
 void toJSON(std::ostream & str, const char * s)

src/libutil/util.hh

@@ -11,6 +11,8 @@
 #include <unistd.h>
 #include <signal.h>
 
+#include <boost/lexical_cast.hpp>
+
 #include <atomic>
 #include <functional>
 #include <map>
@@ -419,21 +421,21 @@ bool statusOk(int status);
 
 /* Parse a string into an integer. */
 template<class N>
-std::optional<N> string2Int(const std::string & s)
+std::optional<N> string2Int(const std::string_view s)
 {
     if (s.substr(0, 1) == "-" && !std::numeric_limits<N>::is_signed)
         return std::nullopt;
-    std::istringstream str(s);
+    try {
-    N n;
+        return boost::lexical_cast<N>(s.data(), s.size());
-    str >> n;
+    } catch (const boost::bad_lexical_cast &) {
-    if (str && str.get() == EOF) return n;
         return std::nullopt;
+    }
 }
 
 /* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or
    'T' denoting a binary unit prefix. */
 template<class N>
-N string2IntWithUnitPrefix(std::string s)
+N string2IntWithUnitPrefix(std::string_view s)
 {
     N multiplier = 1;
     if (!s.empty()) {
@@ -444,7 +446,7 @@ N string2IntWithUnitPrefix(std::string s)
             else if (u == 'G') multiplier = 1ULL << 30;
             else if (u == 'T') multiplier = 1ULL << 40;
             else throw UsageError("invalid unit specifier '%1%'", u);
-            s.resize(s.size() - 1);
+            s.remove_suffix(1);
         }
     }
     if (auto n = string2Int<N>(s))
@@ -454,13 +456,13 @@ N string2IntWithUnitPrefix(std::string s)
 
 /* Parse a string into a float. */
 template<class N>
-std::optional<N> string2Float(const string & s)
+std::optional<N> string2Float(const std::string_view s)
 {
-    std::istringstream str(s);
+    try {
-    N n;
+        return boost::lexical_cast<N>(s.data(), s.size());
-    str >> n;
+    } catch (const boost::bad_lexical_cast &) {
-    if (str && str.get() == EOF) return n;
         return std::nullopt;
+    }
 }