Merge pull request #8895 from hercules-ci/gc-before-stats

eval: Run a full GC before printing stats

(cherry picked from commit aeea49609be014b1928c95b7ec28dbedeb4f032a)
Change-Id: I47a23d3a7a47ea61d9a2b5727b638f879f3aaf1e

src/libcmd/command.cc

@@ -98,7 +98,7 @@ EvalCommand::EvalCommand()
 EvalCommand::~EvalCommand()
 {
     if (evalState)
-        evalState->printStats();
+        evalState->maybePrintStats();
 }
 
 ref<Store> EvalCommand::getEvalStore()

src/libexpr/eval.cc

@@ -2477,10 +2477,37 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
     }
 }
 
-void EvalState::printStats()
+bool EvalState::fullGC() {
+#if HAVE_BOEHMGC
+    GC_gcollect();
+    // Check that it ran. We might replace this with a version that uses more
+    // of the boehm API to get this reliably, at a maintenance cost.
+    // We use a 1K margin because technically this has a race condtion, but we
+    // probably won't encounter it in practice, because the CLI isn't concurrent
+    // like that.
+    return GC_get_bytes_since_gc() < 1024;
+#else
+    return false;
+#endif
+}
+
+void EvalState::maybePrintStats()
 {
     bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0";
 
+    if (showStats) {
+        // Make the final heap size more deterministic.
+#if HAVE_BOEHMGC
+        if (!fullGC()) {
+            warn("failed to perform a full GC before reporting stats");
+        }
+#endif
+        printStatistics();
+    }
+}
+
+void EvalState::printStatistics()
+{
     struct rusage buf;
     getrusage(RUSAGE_SELF, &buf);
     float cpuTime = buf.ru_utime.tv_sec + ((float) buf.ru_utime.tv_usec / 1000000);
@@ -2494,105 +2521,104 @@ void EvalState::printStats()
     GC_word heapSize, totalBytes;
     GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes);
 #endif
-    if (showStats) {
+
-        auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-");
+    auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-");
-        std::fstream fs;
+    std::fstream fs;
-        if (outPath != "-")
+    if (outPath != "-")
-            fs.open(outPath, std::fstream::out);
+        fs.open(outPath, std::fstream::out);
-        json topObj = json::object();
+    json topObj = json::object();
-        topObj["cpuTime"] = cpuTime;
+    topObj["cpuTime"] = cpuTime;
-        topObj["envs"] = {
+    topObj["envs"] = {
-            {"number", nrEnvs},
+        {"number", nrEnvs},
-            {"elements", nrValuesInEnvs},
+        {"elements", nrValuesInEnvs},
-            {"bytes", bEnvs},
+        {"bytes", bEnvs},
-        };
+    };
-        topObj["list"] = {
+    topObj["list"] = {
-            {"elements", nrListElems},
+        {"elements", nrListElems},
-            {"bytes", bLists},
+        {"bytes", bLists},
-            {"concats", nrListConcats},
+        {"concats", nrListConcats},
-        };
+    };
-        topObj["values"] = {
+    topObj["values"] = {
-            {"number", nrValues},
+        {"number", nrValues},
-            {"bytes", bValues},
+        {"bytes", bValues},
-        };
+    };
-        topObj["symbols"] = {
+    topObj["symbols"] = {
-            {"number", symbols.size()},
+        {"number", symbols.size()},
-            {"bytes", symbols.totalSize()},
+        {"bytes", symbols.totalSize()},
-        };
+    };
-        topObj["sets"] = {
+    topObj["sets"] = {
-            {"number", nrAttrsets},
+        {"number", nrAttrsets},
-            {"bytes", bAttrsets},
+        {"bytes", bAttrsets},
-            {"elements", nrAttrsInAttrsets},
+        {"elements", nrAttrsInAttrsets},
-        };
+    };
-        topObj["sizes"] = {
+    topObj["sizes"] = {
-            {"Env", sizeof(Env)},
+        {"Env", sizeof(Env)},
-            {"Value", sizeof(Value)},
+        {"Value", sizeof(Value)},
-            {"Bindings", sizeof(Bindings)},
+        {"Bindings", sizeof(Bindings)},
-            {"Attr", sizeof(Attr)},
+        {"Attr", sizeof(Attr)},
-        };
+    };
-        topObj["nrOpUpdates"] = nrOpUpdates;
+    topObj["nrOpUpdates"] = nrOpUpdates;
-        topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied;
+    topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied;
-        topObj["nrThunks"] = nrThunks;
+    topObj["nrThunks"] = nrThunks;
-        topObj["nrAvoided"] = nrAvoided;
+    topObj["nrAvoided"] = nrAvoided;
-        topObj["nrLookups"] = nrLookups;
+    topObj["nrLookups"] = nrLookups;
-        topObj["nrPrimOpCalls"] = nrPrimOpCalls;
+    topObj["nrPrimOpCalls"] = nrPrimOpCalls;
-        topObj["nrFunctionCalls"] = nrFunctionCalls;
+    topObj["nrFunctionCalls"] = nrFunctionCalls;
 #if HAVE_BOEHMGC
-        topObj["gc"] = {
+    topObj["gc"] = {
-            {"heapSize", heapSize},
+        {"heapSize", heapSize},
-            {"totalBytes", totalBytes},
+        {"totalBytes", totalBytes},
-        };
+    };
 #endif
 
-        if (countCalls) {
+    if (countCalls) {
-            topObj["primops"] = primOpCalls;
+        topObj["primops"] = primOpCalls;
-            {
+        {
-                auto& list = topObj["functions"];
+            auto& list = topObj["functions"];
-                list = json::array();
+            list = json::array();
-                for (auto & [fun, count] : functionCalls) {
+            for (auto & [fun, count] : functionCalls) {
-                    json obj = json::object();
+                json obj = json::object();
-                    if (fun->name)
+                if (fun->name)
-                        obj["name"] = (std::string_view) symbols[fun->name];
+                    obj["name"] = (std::string_view) symbols[fun->name];
-                    else
+                else
-                        obj["name"] = nullptr;
+                    obj["name"] = nullptr;
-                    if (auto pos = positions[fun->pos]) {
+                if (auto pos = positions[fun->pos]) {
-                        if (auto path = std::get_if<SourcePath>(&pos.origin))
+                    if (auto path = std::get_if<SourcePath>(&pos.origin))
-                            obj["file"] = path->to_string();
+                        obj["file"] = path->to_string();
-                        obj["line"] = pos.line;
+                    obj["line"] = pos.line;
-                        obj["column"] = pos.column;
+                    obj["column"] = pos.column;
-                    }
-                    obj["count"] = count;
-                    list.push_back(obj);
-                }
-            }
-            {
-                auto list = topObj["attributes"];
-                list = json::array();
-                for (auto & i : attrSelects) {
-                    json obj = json::object();
-                    if (auto pos = positions[i.first]) {
-                        if (auto path = std::get_if<SourcePath>(&pos.origin))
-                            obj["file"] = path->to_string();
-                        obj["line"] = pos.line;
-                        obj["column"] = pos.column;
-                    }
-                    obj["count"] = i.second;
-                    list.push_back(obj);
                 }
+                obj["count"] = count;
+                list.push_back(obj);
             }
         }
+        {
+            auto list = topObj["attributes"];
+            list = json::array();
+            for (auto & i : attrSelects) {
+                json obj = json::object();
+                if (auto pos = positions[i.first]) {
+                    if (auto path = std::get_if<SourcePath>(&pos.origin))
+                        obj["file"] = path->to_string();
+                    obj["line"] = pos.line;
+                    obj["column"] = pos.column;
+                }
+                obj["count"] = i.second;
+                list.push_back(obj);
+            }
+        }
+    }
 
-        if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") {
+    if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") {
-            // XXX: overrides earlier assignment
+        // XXX: overrides earlier assignment
-            topObj["symbols"] = json::array();
+        topObj["symbols"] = json::array();
-            auto &list = topObj["symbols"];
+        auto &list = topObj["symbols"];
-            symbols.dump([&](const std::string & s) { list.emplace_back(s); });
+        symbols.dump([&](const std::string & s) { list.emplace_back(s); });
-        }
+    }
-        if (outPath == "-") {
+    if (outPath == "-") {
-            std::cerr << topObj.dump(2) << std::endl;
+        std::cerr << topObj.dump(2) << std::endl;
-        } else {
+    } else {
-            fs << topObj.dump(2) << std::endl;
+        fs << topObj.dump(2) << std::endl;
-        }
     }
 }
 

src/libexpr/eval.hh

@@ -709,9 +709,25 @@ public:
     void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx);
 
     /**
-     * Print statistics.
+     * Print statistics, if enabled.
+     *
+     * Performs a full memory GC before printing the statistics, so that the
+     * GC statistics are more accurate.
      */
-    void printStats();
+    void maybePrintStats();
+
+    /**
+     * Print statistics, unconditionally, cheaply, without performing a GC first.
+     */
+    void printStatistics();
+
+    /**
+     * Perform a full memory garbage collection - not incremental.
+     *
+     * @return true if Nix was built with GC and a GC was performed, false if not.
+     *              The return value is currently not thread safe - just the return value.
+     */
+    bool fullGC();
 
     /**
      * Realise the given context, and return a mapping from the placeholders

src/nix-build/nix-build.cc

@@ -344,7 +344,7 @@ static void main_nix_build(int argc, char * * argv)
         }
     }
 
-    state->printStats();
+    state->maybePrintStats();
 
     auto buildPaths = [&](const std::vector<DerivedPath> & paths) {
         /* Note: we do this even when !printMissing to efficiently

src/nix-env/nix-env.cc

@@ -1531,7 +1531,7 @@ static int main_nix_env(int argc, char * * argv)
 
         op(globals, std::move(opFlags), std::move(opArgs));
 
-        globals.state->printStats();
+        globals.state->maybePrintStats();
 
         return 0;
     }

src/nix-instantiate/nix-instantiate.cc

@@ -189,7 +189,7 @@ static int main_nix_instantiate(int argc, char * * argv)
                 evalOnly, outputKind, xmlOutputSourceLocation, e);
         }
 
-        state->printStats();
+        state->maybePrintStats();
 
         return 0;
     }