From 7575db522e9008685c4009423398f6900a16bcce Mon Sep 17 00:00:00 2001
From: Jade Lovelace <lix@jade.fyi>
Date: Wed, 29 May 2024 21:12:34 -0700
Subject: [PATCH] Remove 100s of CPU time (10%) from build times (1465s ->
 1302s)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

I saw that boost/lexical_cast was costing about 100s in CPU time on our
compiles. We can fix this trivially by doing explicit template
instantiation in exactly one place and eliminating all other includes of
it, which is a code improvement anyway by hiding the boost.

Before:
```
lix/lix2 » ClangBuildAnalyzer --analyze buildtimeold.bin
Analyzing build trace from 'buildtimeold.bin'...
**** Time summary:
Compilation (551 times):
  Parsing (frontend):         1465.3 s
  Codegen & opts (backend):   1110.9 s

<snip>

**** Expensive headers:
178153 ms: ../src/libcmd/installable-value.hh (included 52 times, avg 3426 ms), included via:
  40x: command.hh
  5x: command-installable-value.hh
  3x: installable-flake.hh
  2x: <direct include>
  2x: installable-attr-path.hh

176217 ms: ../src/libutil/error.hh (included 246 times, avg 716 ms), included via:
  36x: command.hh installable-value.hh installables.hh derived-path.hh config.hh experimental-features.hh
  12x: globals.hh config.hh experimental-features.hh
  11x: file-system.hh file-descriptor.hh
  6x: serialise.hh strings.hh
  6x: <direct include>
  6x: archive.hh serialise.hh strings.hh
  ...

173243 ms: ../src/libstore/store-api.hh (included 152 times, avg 1139 ms), included via:
  55x: <direct include>
  39x: command.hh installable-value.hh installables.hh
  7x: libexpr.hh
  4x: local-store.hh
  4x: command-installable-value.hh installable-value.hh installables.hh
  3x: binary-cache-store.hh
  ...

170482 ms: ../src/libutil/serialise.hh (included 201 times, avg 848 ms), included via:
  37x: command.hh installable-value.hh installables.hh built-path.hh realisation.hh hash.hh
  14x: store-api.hh nar-info.hh hash.hh
  11x: <direct include>
  7x: primops.hh eval.hh attr-set.hh nixexpr.hh value.hh source-path.hh archive.hh
  7x: libexpr.hh value.hh source-path.hh archive.hh
  6x: fetchers.hh hash.hh
  ...

169397 ms: ../src/libcmd/installables.hh (included 53 times, avg 3196 ms), included via:
  40x: command.hh installable-value.hh
  5x: command-installable-value.hh installable-value.hh
  3x: installable-flake.hh installable-value.hh
  2x: <direct include>
  1x: installable-derived-path.hh
  1x: installable-value.hh
  ...

159740 ms: ../src/libutil/strings.hh (included 221 times, avg 722 ms), included via:
  37x: command.hh installable-value.hh installables.hh built-path.hh realisation.hh hash.hh serialise.hh
  19x: <direct include>
  14x: store-api.hh nar-info.hh hash.hh serialise.hh
  11x: serialise.hh
  7x: primops.hh eval.hh attr-set.hh nixexpr.hh value.hh source-path.hh archive.hh serialise.hh
  7x: libexpr.hh value.hh source-path.hh archive.hh serialise.hh
  ...

156796 ms: ../src/libcmd/command.hh (included 51 times, avg 3074 ms), included via:
  42x: <direct include>
  7x: command-installable-value.hh
  2x: installable-attr-path.hh

150392 ms: ../src/libutil/types.hh (included 251 times, avg 599 ms), included via:
  36x: command.hh installable-value.hh installables.hh path.hh
  11x: file-system.hh
  10x: globals.hh
  6x: fetchers.hh
  6x: serialise.hh strings.hh error.hh
  5x: archive.hh
  ...

133101 ms: /nix/store/644b90j1vms44nr18yw3520pzkrg4dd1-boost-1.81.0-dev/include/boost/lexical_cast.hpp (included 226 times, avg 588 ms), included via
:
  37x: command.hh installable-value.hh installables.hh built-path.hh realisation.hh hash.hh serialise.hh strings.hh
  19x: file-system.hh
  11x: store-api.hh nar-info.hh hash.hh serialise.hh strings.hh
  7x: primops.hh eval.hh attr-set.hh nixexpr.hh value.hh source-path.hh archive.hh serialise.hh strings.hh
  7x: libexpr.hh value.hh source-path.hh archive.hh serialise.hh strings.hh
  6x: eval.hh attr-set.hh nixexpr.hh value.hh source-path.hh archive.hh serialise.hh strings.hh
  ...

132887 ms: /nix/store/h2abv2l8irqj942i5rq9wbrj42kbsh5y-gcc-12.3.0/include/c++/12.3.0/memory (included 262 times, avg 507 ms), included via:
  36x: command.hh installable-value.hh installables.hh path.hh types.hh ref.hh
  16x: gtest.h
  11x: file-system.hh types.hh ref.hh
  10x: globals.hh types.hh ref.hh
  10x: json.hpp
  6x: serialise.hh
  ...

  done in 0.6s.
```

After:
```
lix/lix2 » maintainers/buildtime_report.sh build
Processing all files and saving to '/home/jade/lix/lix2/maintainers/../buildtime.bin'...
  done in 0.6s. Run 'ClangBuildAnalyzer --analyze /home/jade/lix/lix2/maintainers/../buildtime.bin' to analyze it.
Analyzing build trace from '/home/jade/lix/lix2/maintainers/../buildtime.bin'...
**** Time summary:
Compilation (551 times):
  Parsing (frontend):         1302.1 s
  Codegen & opts (backend):    956.3 s

<snip>

**** Expensive headers:
178145 ms: ../src/libutil/error.hh (included 246 times, avg 724 ms), included via:
  36x: command.hh installable-value.hh installables.hh derived-path.hh config.hh experimental-features.hh
  12x: globals.hh config.hh experimental-features.hh
  11x: file-system.hh file-descriptor.hh
  6x: <direct include>
  6x: serialise.hh strings.hh
  6x: fetchers.hh hash.hh serialise.hh strings.hh
  ...

154043 ms: ../src/libcmd/installable-value.hh (included 52 times, avg 2962 ms), included via:
  40x: command.hh
  5x: command-installable-value.hh
  3x: installable-flake.hh
  2x: <direct include>
  2x: installable-attr-path.hh

153593 ms: ../src/libstore/store-api.hh (included 152 times, avg 1010 ms), included via:
  55x: <direct include>
  39x: command.hh installable-value.hh installables.hh
  7x: libexpr.hh
  4x: local-store.hh
  4x: command-installable-value.hh installable-value.hh installables.hh
  3x: binary-cache-store.hh
  ...

149948 ms: ../src/libutil/types.hh (included 251 times, avg 597 ms), included via:
  36x: command.hh installable-value.hh installables.hh path.hh
  11x: file-system.hh
  10x: globals.hh
  6x: fetchers.hh
  6x: serialise.hh strings.hh error.hh
  5x: archive.hh
  ...

144560 ms: ../src/libcmd/installables.hh (included 53 times, avg 2727 ms), included via:
  40x: command.hh installable-value.hh
  5x: command-installable-value.hh installable-value.hh
  3x: installable-flake.hh installable-value.hh
  2x: <direct include>
  1x: installable-value.hh
  1x: installable-derived-path.hh
  ...

136585 ms: ../src/libcmd/command.hh (included 51 times, avg 2678 ms), included via:
  42x: <direct include>
  7x: command-installable-value.hh
  2x: installable-attr-path.hh

133394 ms: /nix/store/h2abv2l8irqj942i5rq9wbrj42kbsh5y-gcc-12.3.0/include/c++/12.3.0/memory (included 262 times, avg 509 ms), included via:
  36x: command.hh installable-value.hh installables.hh path.hh types.hh ref.hh
  16x: gtest.h
  11x: file-system.hh types.hh ref.hh
  10x: globals.hh types.hh ref.hh
  10x: json.hpp
  6x: serialise.hh
  ...

89315 ms: ../src/libstore/derived-path.hh (included 178 times, avg 501 ms), included via:
  37x: command.hh installable-value.hh installables.hh
  25x: store-api.hh realisation.hh
  7x: primops.hh eval.hh attr-set.hh nixexpr.hh value.hh context.hh
  6x: eval.hh attr-set.hh nixexpr.hh value.hh context.hh
  6x: libexpr.hh value.hh context.hh
  6x: shared.hh
  ...

87347 ms: /nix/store/h2abv2l8irqj942i5rq9wbrj42kbsh5y-gcc-12.3.0/include/c++/12.3.0/ostream (included 273 times, avg 319 ms), included via:
  35x: command.hh installable-value.hh installables.hh path.hh types.hh ref.hh memory unique_ptr.h
  12x: regex sstream istream
  10x: file-system.hh types.hh ref.hh memory unique_ptr.h
  10x: gtest.h memory unique_ptr.h
  10x: globals.hh types.hh ref.hh memory unique_ptr.h
  6x: fetchers.hh types.hh ref.hh memory unique_ptr.h
  ...

85249 ms: ../src/libutil/config.hh (included 213 times, avg 400 ms), included via:
  37x: command.hh installable-value.hh installables.hh derived-path.hh
  20x: globals.hh
  20x: logging.hh
  16x: store-api.hh logging.hh
  6x: <direct include>
  6x: eval.hh attr-set.hh nixexpr.hh value.hh context.hh derived-path.hh
  ...

  done in 0.5s.
```

Change-Id: I27f0a2d566db17832cd9be935f12efe7f95b92d0
---
 src/libexpr/eval.cc                         |  2 +-
 src/libexpr/lexer.l                         | 10 +++---
 src/libexpr/nixexpr.cc                      |  1 +
 src/libexpr/primops.cc                      |  1 +
 src/libexpr/print.cc                        |  1 +
 src/libmain/progress-bar.cc                 |  1 +
 src/libstore/binary-cache-store.cc          |  1 +
 src/libstore/build/local-derivation-goal.cc |  5 ++-
 src/libstore/daemon.cc                      |  2 ++
 src/libstore/machines.cc                    |  2 +-
 src/libutil/current-process.cc              |  1 +
 src/libutil/file-system.hh                  |  2 --
 src/libutil/logging.cc                      |  1 +
 src/libutil/processes.hh                    |  2 --
 src/libutil/strings.cc                      | 39 +++++++++++++++++++++
 src/libutil/strings.hh                      | 21 ++---------
 src/nix-env/user-env.cc                     |  3 +-
 src/nix/develop.cc                          |  1 +
 18 files changed, 64 insertions(+), 32 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index c9a624eeb..c72b69af2 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -22,6 +22,7 @@
 #include <algorithm>
 #include <chrono>
 #include <iostream>
+#include <sstream>
 #include <cstring>
 #include <optional>
 #include <unistd.h>
@@ -29,7 +30,6 @@
 #include <sys/resource.h>
 #include <fstream>
 #include <functional>
-#include <iostream>
 
 #include <sys/resource.h>
 #include <nlohmann/json.hpp>
diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index 1780f5cb9..1c1cdd673 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -26,10 +26,9 @@
 #pragma clang diagnostic ignored "-Wimplicit-fallthrough"
 #endif
 
-#include <boost/lexical_cast.hpp>
-
 #include "nixexpr.hh"
 #include "parser-tab.hh"
+#include "strings.hh"
 
 using namespace nix;
 
@@ -132,9 +131,10 @@ or          { return OR_KW; }
 
 {ID}        { yylval->id = {yytext, (size_t) yyleng}; return ID; }
 {INT}       { errno = 0;
-              try {
-                  yylval->n = boost::lexical_cast<int64_t>(yytext);
-              } catch (const boost::bad_lexical_cast &) {
+              std::optional<int64_t> numMay = string2Int<int64_t>(yytext);
+              if (numMay.has_value()) {
+                  yylval->n = *numMay;
+              } else {
                   throw ParseError(ErrorInfo{
                       .msg = HintFmt("invalid integer '%1%'", yytext),
                       .pos = state->positions[CUR_POS],
diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc
index 6a1aa8f35..664872882 100644
--- a/src/libexpr/nixexpr.cc
+++ b/src/libexpr/nixexpr.cc
@@ -6,6 +6,7 @@
 #include "escape-string.hh"
 
 #include <cstdlib>
+#include <sstream>
 
 namespace nix {
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index f8ce90ac1..3cc2659fb 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -25,6 +25,7 @@
 
 #include <algorithm>
 #include <cstring>
+#include <sstream>
 #include <regex>
 #include <dlfcn.h>
 
diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc
index c56b0e72e..e387a09fb 100644
--- a/src/libexpr/print.cc
+++ b/src/libexpr/print.cc
@@ -1,6 +1,7 @@
 #include <limits>
 #include <span>
 #include <unordered_set>
+#include <sstream>
 
 #include "escape-string.hh"
 #include "print.hh"
diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc
index 11b2fe800..d83b09cd4 100644
--- a/src/libmain/progress-bar.cc
+++ b/src/libmain/progress-bar.cc
@@ -7,6 +7,7 @@
 #include <atomic>
 #include <map>
 #include <thread>
+#include <sstream>
 #include <iostream>
 #include <chrono>
 
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index e9413cf99..ea643fd31 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -16,6 +16,7 @@
 #include <future>
 #include <regex>
 #include <fstream>
+#include <sstream>
 
 #include <nlohmann/json.hpp>
 
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 479b4ffeb..99468d420 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -64,10 +64,13 @@ extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags,
 
 namespace nix {
 
+namespace {
 /**
  * The system for which Nix is compiled.
  */
-constexpr std::string_view nativeSystem = SYSTEM;
+[[gnu::unused]]
+constexpr const std::string_view nativeSystem = SYSTEM;
+}
 
 void handleDiffHook(
     uid_t uid, uid_t gid,
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 420fc8bfe..641910bd7 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -14,6 +14,8 @@
 #include "derivations.hh"
 #include "args.hh"
 
+#include <sstream>
+
 namespace nix::daemon {
 
 Sink & operator << (Sink & sink, const Logger::Fields & fields)
diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc
index cdd1e1c2c..833482815 100644
--- a/src/libstore/machines.cc
+++ b/src/libstore/machines.cc
@@ -140,7 +140,7 @@ static Machine parseBuilderLine(const std::string & line)
     };
 
     auto parseFloatField = [&](size_t fieldIndex) {
-        const auto result = string2Int<float>(tokens[fieldIndex]);
+        const auto result = string2Float<float>(tokens[fieldIndex]);
         if (!result) {
             throw FormatError("bad machine specification: failed to convert column #%lu in a row: '%s' to 'float'", fieldIndex, line);
         }
diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc
index c64dd1e0d..3c037c33f 100644
--- a/src/libutil/current-process.cc
+++ b/src/libutil/current-process.cc
@@ -5,6 +5,7 @@
 #include "namespaces.hh"
 #include "signals.hh"
 #include "strings.hh"
+#include <math.h>
 
 #ifdef __APPLE__
 # include <mach-o/dyld.h>
diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh
index b9b753980..6c1923d55 100644
--- a/src/libutil/file-system.hh
+++ b/src/libutil/file-system.hh
@@ -13,8 +13,6 @@
 #include <dirent.h>
 #include <unistd.h>
 
-#include <boost/lexical_cast.hpp>
-
 #include <functional>
 #include <optional>
 
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index febbfdb55..b01bb4dd4 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -6,6 +6,7 @@
 #include "terminal.hh"
 
 #include <atomic>
+#include <sstream>
 #include <nlohmann/json.hpp>
 
 namespace nix {
diff --git a/src/libutil/processes.hh b/src/libutil/processes.hh
index 91a4edfd2..b84fc7c4b 100644
--- a/src/libutil/processes.hh
+++ b/src/libutil/processes.hh
@@ -10,8 +10,6 @@
 #include <unistd.h>
 #include <signal.h>
 
-#include <boost/lexical_cast.hpp>
-
 #include <functional>
 #include <map>
 #include <optional>
diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc
index 9cb319cce..947478481 100644
--- a/src/libutil/strings.cc
+++ b/src/libutil/strings.cc
@@ -1,4 +1,6 @@
 #include "strings.hh"
+#include <boost/lexical_cast.hpp>
+#include <stdint.h>
 
 namespace nix {
 
@@ -89,6 +91,43 @@ std::string Rewriter::operator()(std::string s)
     return s;
 }
 
+template<class N>
+std::optional<N> string2Int(const std::string_view s)
+{
+    if (s.substr(0, 1) == "-" && !std::numeric_limits<N>::is_signed)
+        return std::nullopt;
+    try {
+        return boost::lexical_cast<N>(s.data(), s.size());
+    } catch (const boost::bad_lexical_cast &) {
+        return std::nullopt;
+    }
+}
+
+// Explicitly instantiated in one place for faster compilation
+template std::optional<unsigned char>  string2Int<unsigned char>(const std::string_view s);
+template std::optional<unsigned short> string2Int<unsigned short>(const std::string_view s);
+template std::optional<unsigned int> string2Int<unsigned int>(const std::string_view s);
+template std::optional<unsigned long> string2Int<unsigned long>(const std::string_view s);
+template std::optional<unsigned long long> string2Int<unsigned long long>(const std::string_view s);
+template std::optional<signed char> string2Int<signed char>(const std::string_view s);
+template std::optional<signed short> string2Int<signed short>(const std::string_view s);
+template std::optional<signed int> string2Int<signed int>(const std::string_view s);
+template std::optional<signed long> string2Int<signed long>(const std::string_view s);
+template std::optional<signed long long> string2Int<signed long long>(const std::string_view s);
+
+template<class N>
+std::optional<N> string2Float(const std::string_view s)
+{
+    try {
+        return boost::lexical_cast<N>(s.data(), s.size());
+    } catch (const boost::bad_lexical_cast &) {
+        return std::nullopt;
+    }
+}
+
+template std::optional<double> string2Float<double>(const std::string_view s);
+template std::optional<float> string2Float<float>(const std::string_view s);
+
 
 std::string toLower(const std::string & s)
 {
diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh
index daeb5be50..03dff8160 100644
--- a/src/libutil/strings.hh
+++ b/src/libutil/strings.hh
@@ -4,7 +4,6 @@
 #include "error.hh"
 #include "types.hh"
 
-#include <boost/lexical_cast.hpp>
 #include <vector>
 
 namespace nix {
@@ -130,16 +129,7 @@ inline std::string rewriteStrings(std::string s, const StringMap & rewrites)
  * Parse a string into an integer.
  */
 template<class N>
-std::optional<N> string2Int(const std::string_view s)
-{
-    if (s.substr(0, 1) == "-" && !std::numeric_limits<N>::is_signed)
-        return std::nullopt;
-    try {
-        return boost::lexical_cast<N>(s.data(), s.size());
-    } catch (const boost::bad_lexical_cast &) {
-        return std::nullopt;
-    }
-}
+std::optional<N> string2Int(const std::string_view s);
 
 /**
  * Like string2Int(), but support an optional suffix 'K', 'M', 'G' or
@@ -169,14 +159,7 @@ N string2IntWithUnitPrefix(std::string_view s)
  * Parse a string into a float.
  */
 template<class N>
-std::optional<N> string2Float(const std::string_view s)
-{
-    try {
-        return boost::lexical_cast<N>(s.data(), s.size());
-    } catch (const boost::bad_lexical_cast &) {
-        return std::nullopt;
-    }
-}
+std::optional<N> string2Float(const std::string_view s);
 
 
 /**
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 530039ac6..f5dbd06ca 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -9,8 +9,9 @@
 #include "eval-inline.hh"
 #include "profiles.hh"
 #include "print-ambiguous.hh"
-#include <limits>
 
+#include <limits>
+#include <sstream>
 
 namespace nix {
 
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index cd32bb20a..90bdf3bed 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -11,6 +11,7 @@
 
 #include <iterator>
 #include <memory>
+#include <sstream>
 #include <nlohmann/json.hpp>
 #include <algorithm>