libmain: add progress bar with multiple status lines

Add the log-formats `multiline` and `multiline-with-logs` which offer
multiple current active building status lines.

Change-Id: Idd8afe62f8591b5d8b70e258c5cefa09be4cab03

.gitignore

@@ -1,128 +1,5 @@
-Makefile.config
-perl/Makefile.config
-
-# /
-/aclocal.m4
-/autom4te.cache
-/precompiled-headers.h.gch
-/config.*
-/configure
-/stamp-h1
-/svn-revision
-/libtool
-/config
-
-# /doc/manual/
-/doc/manual/*.1
-/doc/manual/*.5
-/doc/manual/*.8
-/doc/manual/generated/*
-/doc/manual/nix.json
-/doc/manual/conf-file.json
-/doc/manual/language.json
-/doc/manual/xp-features.json
-/doc/manual/src/command-ref/experimental-features-shortlist.md
-/doc/manual/src/contributing/experimental-feature-descriptions.md
-/doc/manual/src/release-notes/rl-next-generated.md
-
-# /scripts/
-/scripts/nix-profile.sh
-/scripts/nix-profile-daemon.sh
-/scripts/nix-profile.fish
-/scripts/nix-profile-daemon.fish
-
-# /src/libexpr/
-/src/libexpr/lexer-tab.cc
-/src/libexpr/lexer-tab.hh
-/src/libexpr/parser-tab.cc
-/src/libexpr/parser-tab.hh
-/src/libexpr/parser-tab.output
-/src/libexpr/nix.tbl
-/src/libexpr/tests
-/tests/unit/libexpr/libnixexpr-tests
-
-# /src/libstore/
-*.gen.*
-/src/libstore/tests
-/tests/unit/libstore/libnixstore-tests
-
-# /src/libutil/
-/src/libutil/tests
-/tests/unit/libutil/libnixutil-tests
-
-/src/nix/nix
-
-/src/nix/doc
-
-# /src/nix-env/
-/src/nix-env/nix-env
-
-# /src/nix-instantiate/
-/src/nix-instantiate/nix-instantiate
-
-# /src/nix-store/
-/src/nix-store/nix-store
-
-/src/nix-prefetch-url/nix-prefetch-url
-
-/src/nix-collect-garbage/nix-collect-garbage
-
-# /src/nix-channel/
-/src/nix-channel/nix-channel
-
-# /src/nix-build/
-/src/nix-build/nix-build
-
-/src/nix-copy-closure/nix-copy-closure
-
-/src/error-demo/error-demo
-
-/src/build-remote/build-remote
-
-# /tests/functional/
-/tests/functional/test-tmp
-/tests/functional/common/vars-and-functions.sh
-/tests/functional/result*
-/tests/functional/restricted-innocent
-/tests/functional/shell
-/tests/functional/shell.drv
-/tests/functional/config.nix
-/tests/functional/ca/config.nix
-/tests/functional/dyn-drv/config.nix
-/tests/functional/repl-result-out
-/tests/functional/debugger-test-out
-/tests/functional/test-libstoreconsumer/test-libstoreconsumer
-
-# /tests/functional/lang/
-/tests/functional/lang/*.out
-/tests/functional/lang/*.out.xml
-/tests/functional/lang/*.err
-/tests/functional/lang/*.ast
-
-/perl/lib/Nix/Config.pm
-/perl/lib/Nix/Store.cc
-
-/misc/systemd/nix-daemon.service
-/misc/systemd/nix-daemon.socket
-/misc/systemd/nix-daemon.conf
-/misc/upstart/nix-daemon.conf
-
-/src/resolve-system-dependencies/resolve-system-dependencies
-
 outputs/
 
-*.a
-*.o
-*.o.tmp
-*.so
-*.dylib
-*.dll
-*.exe
-*.dep
-*~
-*.pc
-*.plist
-
 # GNU Global
 GPATH
 GRTAGS
@@ -132,17 +9,11 @@ GTAGS
 # ccls
 /.ccls-cache
 
-# auto-generated compilation database
-compile_commands.json
-
-nix-rust/target
-
 result
 result-*
 
 .vscode/
 .direnv/
-.envrc.local
 
 # clangd and possibly more
 .cache/

misc/pre-commit.nix

@@ -87,11 +87,12 @@ pre-commit-run {
         "file"
         "header"
       ];
-      # generated files; these will never actually be seen by this
-      # check, and are left here as documentation
       excludes = [
-        "(parser|lexer)-tab\\.hh$"
-        "\\.gen\\.hh$"
+        ''^src/pch/.*$''
+        # generated files; these will never actually be seen by this
+        # check, and are left here as documentation
+        ''(parser|lexer)-tab\.hh$''
+        ''\.gen\.hh$''
       ];
       entry = lib.getExe pkgs.check-headers;
     };

package.nix

@@ -169,7 +169,6 @@ stdenv.mkDerivation (finalAttrs: {
           ./boehmgc-coroutine-sp-fallback.diff
           ./doc
           ./misc
-          ./precompiled-headers.h
           ./src
           ./COPYING
         ]
@@ -449,7 +448,10 @@ stdenv.mkDerivation (finalAttrs: {
           shellHook = ''
             # don't re-run the hook in (other) nested nix-shells
             function lixShellHook() {
-              if [[ $name != lix-shell-env ]]; then
+              # n.b. how the heck does this become -env-env? well, `nix develop` does it:
+              # https://git.lix.systems/lix-project/lix/src/commit/7575db522e9008685c4009423398f6900a16bcce/src/nix/develop.cc#L240-L241
+              # this is, of course, absurd.
+              if [[ $name != lix-shell-env && $name != lix-shell-env-env ]]; then
                 return;
               fi
 

src/build-remote/build-remote.cc

@@ -3,6 +3,7 @@
 #include <algorithm>
 #include <set>
 #include <memory>
+#include <string_view>
 #include <tuple>
 #include <iomanip>
 #if __APPLE__
@@ -20,9 +21,9 @@
 #include "local-store.hh"
 #include "legacy.hh"
 #include "experimental-features.hh"
+#include "hash.hh"
 
 using namespace nix;
-using std::cin;
 
 static void handleAlarm(int sig) {
 }
@@ -35,9 +36,19 @@ std::string escapeUri(std::string uri)
 
 static std::string currentLoad;
 
+static std::string makeLockFilename(const std::string & storeUri) {
+    // We include 48 bytes of escaped URI to give an idea of what the lock
+    // is on, then 16 bytes of hash to disambiguate.
+    // This avoids issues with the escaped URI being very long and causing
+    // path too long errors, while also avoiding any possibility of collision
+    // caused by simple truncation.
+    auto hash = hashString(HashType::htSHA256, storeUri).to_string(Base::Base32, false);
+    return escapeUri(storeUri).substr(0, 48) + "-" + hash.substr(0, 16);
+}
+
 static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
 {
-    return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
+    return openLockFile(fmt("%s/%s-%d", currentLoad, makeLockFilename(m.storeUri), slot), true);
 }
 
 static bool allSupportedLocally(Store & store, const std::set<std::string>& requiredFeatures) {
@@ -263,7 +274,9 @@ connected:
         auto inputs = readStrings<PathSet>(source);
         auto wantedOutputs = readStrings<StringSet>(source);
 
-        AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
+        auto lockFileName = currentLoad + "/" + makeLockFilename(storeUri) + ".upload-lock";
+
+        AutoCloseFD uploadLock = openLockFile(lockFileName, true);
 
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri));

src/libcmd/meson.build

@@ -54,6 +54,7 @@ libcmd = library(
     nlohmann_json,
     lix_doc
   ],
+  cpp_pch : ['../pch/precompiled-headers.hh'],
   install : true,
   # FIXME(Qyriad): is this right?
   install_rpath : libdir,

src/libexpr/meson.build

@@ -145,6 +145,7 @@ libexpr = library(
   include_directories : [
     '../libmain',
   ],
+  cpp_pch : ['../pch/precompiled-headers.hh'],
   install : true,
   # FIXME(Qyriad): is this right?
   install_rpath : libdir,

src/libfetchers/github.cc

@@ -322,7 +322,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
             readFile(
                 store->toRealPath(
                     downloadFile(store, url, "source", false, headers).storePath)));
-        if (json.is_array() && json.size() == 1 && json[0]["id"] != nullptr) {
+        if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
             auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1);
             debug("HEAD revision for '%s' is %s", url, rev.gitRev());
             return rev;

src/libfetchers/meson.build

@@ -30,6 +30,7 @@ libfetchers = library(
     liblixutil,
     nlohmann_json,
   ],
+  cpp_pch : ['../pch/precompiled-headers.hh'],
   install : true,
   # FIXME(Qyriad): is this right?
   install_rpath : libdir,

src/libmain/loggers.cc

@@ -17,6 +17,10 @@ LogFormat parseLogFormat(const std::string & logFormatStr) {
         return LogFormat::bar;
     else if (logFormatStr == "bar-with-logs")
         return LogFormat::barWithLogs;
+    else if (logFormatStr == "multiline")
+        return LogFormat::multiline;
+    else if (logFormatStr == "multiline-with-logs")
+        return LogFormat::multilineWithLogs;
     throw Error("option 'log-format' has an invalid value '%s'", logFormatStr);
 }
 
@@ -35,6 +39,17 @@ Logger * makeDefaultLogger() {
         logger->setPrintBuildLogs(true);
         return logger;
     }
+    case LogFormat::multiline: {
+        auto logger = makeProgressBar();
+        logger->setPrintMultiline(true);
+        return logger;
+    }
+    case LogFormat::multilineWithLogs: {
+        auto logger = makeProgressBar();
+        logger->setPrintMultiline(true);
+        logger->setPrintBuildLogs(true);
+        return logger;
+    }
     default:
         abort();
     }

src/libmain/loggers.hh

@@ -11,6 +11,8 @@ enum class LogFormat {
   internalJSON,
   bar,
   barWithLogs,
+  multiline,
+  multilineWithLogs,
 };
 
 void setLogFormat(const std::string & logFormatStr);

src/libmain/meson.build

@@ -20,6 +20,7 @@ libmain = library(
     liblixutil,
     liblixstore,
   ],
+  cpp_pch : ['../pch/precompiled-headers.hh'],
   install : true,
   # FIXME(Qyriad): is this right?
   install_rpath : libdir,

src/libmain/progress-bar.cc

@@ -73,6 +73,8 @@ private:
 
         std::map<ActivityType, ActivitiesByType> activitiesByType;
 
+        int lastLines = 1;
+
         uint64_t filesLinked = 0, bytesLinked = 0;
 
         uint64_t corruptedPaths = 0, untrustedPaths = 0;
@@ -89,6 +91,7 @@ private:
     std::condition_variable quitCV, updateCV;
 
     bool printBuildLogs = false;
+    bool printMultiline = false;
     bool isTTY;
 
 public:
@@ -103,7 +106,7 @@ public:
             while (state->active) {
                 if (!state->haveUpdate)
                     state.wait_for(updateCV, nextWakeup);
-                nextWakeup = draw(*state);
+                nextWakeup = draw(*state, {});
                 state.wait_for(quitCV, std::chrono::milliseconds(50));
             }
         });
@@ -165,8 +168,7 @@ public:
     void log(State & state, Verbosity lvl, std::string_view s)
     {
         if (state.active) {
-            writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n");
-            draw(state);
+            draw(state, s);
         } else {
             auto s2 = s + ANSI_NORMAL "\n";
             if (!isTTY) s2 = filterANSIEscapes(s2, true);
@@ -354,60 +356,96 @@ public:
         updateCV.notify_one();
     }
 
-    std::chrono::milliseconds draw(State & state)
+    std::chrono::milliseconds draw(State & state, const std::optional<std::string_view> & s)
     {
         auto nextWakeup = A_LONG_TIME;
 
         state.haveUpdate = false;
         if (state.paused || !state.active) return nextWakeup;
 
-        std::string line;
+        auto width = getWindowSize().second;
+        if (width <= 0) {
+            width = std::numeric_limits<decltype(width)>::max();
+        }
 
+        if (printMultiline && state.lastLines) {
+#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
+            system("cls");
+#else
+            writeToStderr("\r\033[K\r");
+            for (auto i = 1; i < state.lastLines; i++) {
+                writeToStderr("\033[1A\r\033[K\r"); // \r
+            }
+#endif
+        }
+
+        state.lastLines = 0;
+
+        if (s != std::nullopt)
+            writeToStderr("\r\e[K" + filterANSIEscapes(s.value(), !isTTY) + ANSI_NORMAL "\n");
+
+        std::string line;
         std::string status = getStatus(state);
         if (!status.empty()) {
             line += '[';
             line += status;
             line += "]";
         }
+        if (printMultiline && !line.empty()) {
+            writeToStderr(filterANSIEscapes(line, false, width));
+            state.lastLines++;
+        }
 
         auto now = std::chrono::steady_clock::now();
 
         if (!state.activities.empty()) {
-            if (!status.empty()) line += " ";
-            auto i = state.activities.rbegin();
-
-            while (i != state.activities.rend()) {
-                if (i->visible && (!i->s.empty() || !i->lastLine.empty())) {
-                    /* Don't show activities until some time has
-                       passed, to avoid displaying very short
-                       activities. */
-                    auto delay = std::chrono::milliseconds(10);
-                    if (i->startTime + delay < now)
-                        break;
-                    else
-                        nextWakeup = std::min(nextWakeup, std::chrono::duration_cast<std::chrono::milliseconds>(delay - (now - i->startTime)));
+            for (auto i = state.activities.begin(); i != state.activities.end(); ++i) {
+                if (!i->visible || (i->s.empty() && i->lastLine.empty())) {
+                    continue;
+                }
+
+                /* Don't show activities until some time has
+                   passed, to avoid displaying very short
+                   activities. */
+                auto delay = std::chrono::milliseconds(10);
+                if (i->startTime + delay >= now) {
+                    nextWakeup = std::min(
+                        nextWakeup,
+                        std::chrono::duration_cast<std::chrono::milliseconds>(
+                            delay - (now - i->startTime)
+                        )
+                    );
+                }
+
+                if (!printMultiline) {
+                    line += i->s;
+                } else {
+                    line = i->s;
                 }
-                ++i;
-            }
 
-            if (i != state.activities.rend()) {
-                line += i->s;
                 if (!i->phase.empty()) {
                     line += " (";
                     line += i->phase;
                     line += ")";
                 }
                 if (!i->lastLine.empty()) {
-                    if (!i->s.empty()) line += ": ";
+                    if (!i->s.empty()) {
+                        line += ": ";
+                    }
                     line += i->lastLine;
                 }
+                if (printMultiline) {
+                    if (state.lastLines)
+                        writeToStderr("\n");
+                    state.lastLines += 1;
+                    writeToStderr(filterANSIEscapes(line, false, width));
+                }
             }
         }
 
-        auto width = getWindowSize().second;
-        if (width <= 0) width = std::numeric_limits<decltype(width)>::max();
-
-        writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K");
+        if (!printMultiline) {
+            writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K");
+        }
 
         return nextWakeup;
     }
@@ -506,9 +544,8 @@ public:
     {
         auto state(state_.lock());
         if (state->active) {
-            std::cerr << "\r\e[K";
             Logger::writeToStdout(s);
-            draw(*state);
+            draw(*state, {});
         } else {
             Logger::writeToStdout(s);
         }
@@ -521,7 +558,7 @@ public:
         std::cerr << fmt("\r\e[K%s ", msg);
         auto s = trim(readLine(STDIN_FILENO));
         if (s.size() != 1) return {};
-        draw(*state);
+        draw(*state, {});
         return s[0];
     }
 
@@ -529,6 +566,11 @@ public:
     {
         this->printBuildLogs = printBuildLogs;
     }
+
+    void setPrintMultiline(bool printMultiline) override
+    {
+        this->printMultiline = printMultiline;
+    }
 };
 
 Logger * makeProgressBar()

src/libstore/local-store.cc

@@ -550,7 +550,12 @@ void LocalStore::openDB(State & state, bool create)
     if (mode == "wal" ) {
          /* persist the WAL files when the DB connection is closed.
          * This allows for read-only connections without any write permissions
-         * on the state directory to succeed on a closed database. */
+         * on the state directory to succeed on a closed database. Setting the
+         * journal_size_limit to 2^40 bytes results in the WAL files getting
+         * truncated to 0 on exit and limits the on disk size of the WAL files
+         * to 2^40 bytes following a checkpoint */
+        if (sqlite3_exec(db, "pragma main.journal_size_limit = 1099511627776;", 0, 0, 0) != SQLITE_OK)
+            SQLiteError::throw_(db, "setting journal_size_limit");
         int enable = 1;
         if (sqlite3_file_control(db, NULL, SQLITE_FCNTL_PERSIST_WAL, &enable) != SQLITE_OK)
             SQLiteError::throw_(db, "setting persistent WAL mode");

src/libstore/meson.build

@@ -220,6 +220,7 @@ libstore = library(
     nlohmann_json,
   ],
   cpp_args : cpp_args,
+  cpp_pch : ['../pch/precompiled-headers.hh'],
   install : true,
   # FIXME(Qyriad): is this right?
   install_rpath : libdir,

src/libutil/logging.hh

@@ -114,6 +114,9 @@ public:
 
     virtual void setPrintBuildLogs(bool printBuildLogs)
     { }
+
+    virtual void setPrintMultiline(bool printMultiline)
+    { }
 };
 
 /**

src/libutil/meson.build

@@ -129,6 +129,7 @@ libutil = library(
     openssl,
     nlohmann_json,
   ],
+  cpp_pch : ['../pch/precompiled-headers.hh'],
   implicit_include_directories : true,
   install : true,
 )

src/nix/meson.build

@@ -89,6 +89,7 @@ nix = executable(
     boehm,
     nlohmann_json,
   ],
+  cpp_pch : ['../pch/precompiled-headers.hh'],
   install : true,
   # FIXME(Qyriad): is this right?
   install_rpath : libdir,

tests/unit/meson.build

@@ -68,6 +68,7 @@ libutil_tester = executable(
     liblixutil_test_support,
     nlohmann_json,
   ],
+  cpp_pch : ['../../src/pch/precompiled-headers.hh'],
 )
 
 test(
@@ -102,6 +103,7 @@ libstore_test_support = library(
   include_directories : include_directories(
     'libstore-support',
   ),
+  cpp_pch : ['../../src/pch/precompiled-headers.hh'],
 )
 liblixstore_test_support = declare_dependency(
   include_directories : include_directories('libstore-support'),
@@ -135,6 +137,7 @@ libstore_tester = executable(
     gtest,
     nlohmann_json,
   ],
+  cpp_pch : ['../../src/pch/precompiled-headers.hh'],
 )
 
 test(
@@ -166,6 +169,7 @@ libexpr_test_support = library(
   include_directories : include_directories(
     'libexpr-support',
   ),
+  cpp_pch : ['../../src/pch/precompiled-headers.hh'],
 )
 liblixexpr_test_support = declare_dependency(
   include_directories : include_directories('libexpr-support'),
@@ -199,6 +203,7 @@ libexpr_tester = executable(
     gtest,
     nlohmann_json,
   ],
+  cpp_pch : ['../../src/pch/precompiled-headers.hh'],
 )
 
 test(
@@ -225,6 +230,7 @@ libcmd_tester = executable(
     gtest,
     boost,
   ],
+  cpp_pch : ['../../src/pch/precompiled-headers.hh'],
 )
 
 test(