From ec5f025ec2325e55834e37ca08f0f40125b041c4 Mon Sep 17 00:00:00 2001 From: Mario Rodas Date: Tue, 28 May 2024 04:20:00 +0000 Subject: [PATCH 01/22] tests: fix functional-timeout grepQuietInvert is a typo introduced by c11836126b5. The test functional-timeout was failing silently because Bash considered the command-not-found error as truthy. Change-Id: Ic13829d02ec55d6ecd63a0f4d34ec0d32379609f --- tests/functional/timeout.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/timeout.sh b/tests/functional/timeout.sh index b179b79a2..e29a85f81 100644 --- a/tests/functional/timeout.sh +++ b/tests/functional/timeout.sh @@ -12,7 +12,7 @@ if [ $status -ne 101 ]; then exit 1 fi -if echo "$messages" | grepQuietInvert "timed out"; then +if echo "$messages" | grepQuietInverse "timed out"; then echo "error: build may have failed for reasons other than timeout; output:" echo "$messages" >&2 exit 1 From 6fd6795bc4faed644c5f3abafdcb21638a119342 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 10:50:54 +0200 Subject: [PATCH 02/22] util.{hh,cc}: Split out environment-variables.{hh,cc} Change-Id: Icff0aa33fda5147bd5dbe256a0b9d6a6c8a2c3f6 --- src/libcmd/editor-for.cc | 1 + src/libmain/loggers.cc | 1 + src/libstore/globals.cc | 1 + src/libstore/globals.hh | 2 +- src/libstore/ssh.cc | 1 + src/libutil/args.cc | 2 + src/libutil/environment-variables.cc | 51 +++++++++++++++++++ src/libutil/environment-variables.hh | 42 +++++++++++++++ src/libutil/error.cc | 1 + src/libutil/filesystem.cc | 1 + src/libutil/logging.cc | 1 + src/libutil/meson.build | 2 + src/libutil/util.cc | 46 +---------------- src/libutil/util.hh | 21 -------- .../libutil-support/tests/characterization.hh | 1 + 15 files changed, 107 insertions(+), 67 deletions(-) create mode 100644 src/libutil/environment-variables.cc create mode 100644 src/libutil/environment-variables.hh diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc index e5f762104..a56a7065e 100644 --- a/src/libcmd/editor-for.cc +++ b/src/libcmd/editor-for.cc @@ -1,5 +1,6 @@ #include "util.hh" #include "editor-for.hh" +#include "environment-variables.hh" #include "source-path.hh" namespace nix { diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index cda5cb939..7e80a31f1 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -1,3 +1,4 @@ +#include "environment-variables.hh" #include "loggers.hh" #include "progress-bar.hh" #include "util.hh" diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 3308cad1f..897d3e8ef 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,3 +1,4 @@ +#include "environment-variables.hh" #include "globals.hh" #include "util.hh" #include "archive.hh" diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 85789f6b5..581de7ff6 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1,9 +1,9 @@ #pragma once ///@file +#include "environment-variables.hh" #include "types.hh" #include "config.hh" -#include "util.hh" #include #include diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 87414fe9c..2f921e2d0 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,3 +1,4 @@ +#include "environment-variables.hh" #include "ssh.hh" #include "finally.hh" diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 520f50c30..bcff653c5 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -2,6 +2,8 @@ #include "args/root.hh" #include "hash.hh" #include "json-utils.hh" +#include "environment-variables.hh" + #include "experimental-features-json.hh" #include diff --git a/src/libutil/environment-variables.cc b/src/libutil/environment-variables.cc new file mode 100644 index 000000000..71c404a0e --- /dev/null +++ b/src/libutil/environment-variables.cc @@ -0,0 +1,51 @@ +#include +#include +#include +#include + +extern char * * environ __attribute__((weak)); + +namespace nix { + +std::optional getEnv(const std::string & key) +{ + char * value = getenv(key.c_str()); + if (!value) return {}; + return std::string(value); +} + +std::optional getEnvNonEmpty(const std::string & key) { + auto value = getEnv(key); + if (value == "") return {}; + return value; +} + +std::map getEnv() +{ + std::map env; + for (size_t i = 0; environ[i]; ++i) { + auto s = environ[i]; + auto eq = strchr(s, '='); + if (!eq) + // invalid env, just keep going + continue; + env.emplace(std::string(s, eq), std::string(eq + 1)); + } + return env; +} + + +void clearEnv() +{ + for (auto & name : getEnv()) + unsetenv(name.first.c_str()); +} + +void replaceEnv(const std::map & newEnv) +{ + clearEnv(); + for (auto & newEnvVar : newEnv) + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); +} + +} diff --git a/src/libutil/environment-variables.hh b/src/libutil/environment-variables.hh new file mode 100644 index 000000000..784b95921 --- /dev/null +++ b/src/libutil/environment-variables.hh @@ -0,0 +1,42 @@ +#pragma once +/** + * @file + * + * Utilities for working with the current process's environment + * variables. + */ + +#include +#include +#include + + +namespace nix { + +/** + * @return an environment variable. + */ +std::optional getEnv(const std::string & key); + +/** + * @return a non empty environment variable. Returns nullopt if the env + * variable is set to "" + */ +std::optional getEnvNonEmpty(const std::string & key); + +/** + * Get the entire environment. + */ +std::map getEnv(); + +/** + * Clear the environment. + */ +void clearEnv(); + +/** + * Replace the entire environment with the given one. + */ +void replaceEnv(const std::map & newEnv); + +} diff --git a/src/libutil/error.cc b/src/libutil/error.cc index b534ff87e..fe73ffc56 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,3 +1,4 @@ +#include "environment-variables.hh" #include "error.hh" #include "position.hh" diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index b637281d0..85871870c 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -2,6 +2,7 @@ #include #include +#include "environment-variables.hh" #include "finally.hh" #include "util.hh" #include "signals.hh" diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 0e63f9bd6..8e4828be4 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -1,3 +1,4 @@ +#include "environment-variables.hh" #include "logging.hh" #include "util.hh" #include "config.hh" diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 58e0bd062..01b3e24b7 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -7,6 +7,7 @@ libutil_sources = files( 'compute-levels.cc', 'config.cc', 'english.cc', + 'environment-variables.cc', 'error.cc', 'escape-char.cc', 'escape-string.cc', @@ -53,6 +54,7 @@ libutil_headers = files( 'config-impl.hh', 'config.hh', 'english.hh', + 'environment-variables.hh', 'error.hh', 'escape-char.hh', 'escape-string.hh', diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 2c0fcc897..97d53ca63 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -4,6 +4,7 @@ #include "serialise.hh" #include "cgroup.hh" #include "signals.hh" +#include "environment-variables.hh" #include #include @@ -43,57 +44,12 @@ #endif -extern char * * environ __attribute__((weak)); - - #ifdef NDEBUG #error "Lix may not be built with assertions disabled (i.e. with -DNDEBUG)." #endif namespace nix { -std::optional getEnv(const std::string & key) -{ - char * value = getenv(key.c_str()); - if (!value) return {}; - return std::string(value); -} - -std::optional getEnvNonEmpty(const std::string & key) { - auto value = getEnv(key); - if (value == "") return {}; - return value; -} - -std::map getEnv() -{ - std::map env; - for (size_t i = 0; environ[i]; ++i) { - auto s = environ[i]; - auto eq = strchr(s, '='); - if (!eq) - // invalid env, just keep going - continue; - env.emplace(std::string(s, eq), std::string(eq + 1)); - } - return env; -} - - -void clearEnv() -{ - for (auto & name : getEnv()) - unsetenv(name.first.c_str()); -} - -void replaceEnv(const std::map & newEnv) -{ - clearEnv(); - for (auto & newEnvVar : newEnv) - setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); -} - - Path absPath(Path path, std::optional dir, bool resolveSymlinks) { if (path.empty() || path[0] != '/') { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 14868776c..151da25b9 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -39,27 +39,6 @@ struct Source; extern const std::string nativeSystem; -/** - * @return an environment variable. - */ -std::optional getEnv(const std::string & key); - -/** - * @return a non empty environment variable. Returns nullopt if the env - * variable is set to "" - */ -std::optional getEnvNonEmpty(const std::string & key); - -/** - * Get the entire environment. - */ -std::map getEnv(); - -/** - * Clear the environment. - */ -void clearEnv(); - /** * @return An absolutized path, resolving paths relative to the * specified directory, or the current directory otherwise. The path diff --git a/tests/unit/libutil-support/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh index 7f570f619..9de20d02c 100644 --- a/tests/unit/libutil-support/tests/characterization.hh +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "environment-variables.hh" #include #include From 81bdf8d2d672e135e68745e6975ad5edafadf13a Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 11:19:08 +0200 Subject: [PATCH 03/22] util.{hh,cc}: Split out terminal.{hh,cc} Change-Id: I9de2296b4012d50f540124001d54d6ca3be4c6da --- src/libcmd/markdown.cc | 1 + src/libexpr/flake/flake.cc | 1 + src/libexpr/print.cc | 1 + src/libmain/progress-bar.cc | 1 + src/libutil/error.cc | 1 + src/libutil/logging.cc | 1 + src/libutil/meson.build | 2 + src/libutil/signals.cc | 1 + src/libutil/suggestions.cc | 4 +- src/libutil/terminal.cc | 104 ++++++++++++++++++++++++++++++++++++ src/libutil/terminal.hh | 40 ++++++++++++++ src/libutil/util.cc | 96 --------------------------------- src/libutil/util.hh | 25 --------- tests/unit/libutil/tests.cc | 1 + 14 files changed, 157 insertions(+), 122 deletions(-) create mode 100644 src/libutil/terminal.cc create mode 100644 src/libutil/terminal.hh diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 668a07763..8b3bbc1b5 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,6 +1,7 @@ #include "markdown.hh" #include "util.hh" #include "finally.hh" +#include "terminal.hh" #include #include diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index e13d1cb93..ac50351ad 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -8,6 +8,7 @@ #include "fetchers.hh" #include "finally.hh" #include "fetch-settings.hh" +#include "terminal.hh" namespace nix { diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index 231bde0a0..c56b0e72e 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -9,6 +9,7 @@ #include "english.hh" #include "signals.hh" #include "eval.hh" +#include "terminal.hh" namespace nix { diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 1a68daca2..f3700f103 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -3,6 +3,7 @@ #include "sync.hh" #include "store-api.hh" #include "names.hh" +#include "terminal.hh" #include #include diff --git a/src/libutil/error.cc b/src/libutil/error.cc index fe73ffc56..f780aabef 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,6 +1,7 @@ #include "environment-variables.hh" #include "error.hh" #include "position.hh" +#include "terminal.hh" #include #include diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 8e4828be4..731d1034d 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -4,6 +4,7 @@ #include "config.hh" #include "source-path.hh" #include "position.hh" +#include "terminal.hh" #include #include diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 01b3e24b7..c890911e9 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -30,6 +30,7 @@ libutil_sources = files( 'source-path.cc', 'suggestions.cc', 'tarfile.cc', + 'terminal.cc', 'thread-pool.cc', 'url.cc', 'url-name.cc', @@ -90,6 +91,7 @@ libutil_headers = files( 'suggestions.hh', 'sync.hh', 'tarfile.hh', + 'terminal.hh', 'thread-pool.hh', 'topo-sort.hh', 'types.hh', diff --git a/src/libutil/signals.cc b/src/libutil/signals.cc index 41fdc9dc8..c0e66f6ed 100644 --- a/src/libutil/signals.cc +++ b/src/libutil/signals.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "error.hh" #include "sync.hh" +#include "terminal.hh" #include diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc index 9510a5f0c..63dcf84b5 100644 --- a/src/libutil/suggestions.cc +++ b/src/libutil/suggestions.cc @@ -1,7 +1,9 @@ #include "suggestions.hh" #include "ansicolor.hh" -#include "util.hh" +#include "terminal.hh" + #include +#include namespace nix { diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc new file mode 100644 index 000000000..b58331d04 --- /dev/null +++ b/src/libutil/terminal.cc @@ -0,0 +1,104 @@ +#include "terminal.hh" +#include "environment-variables.hh" +#include "sync.hh" + +#include +#include + +namespace nix { + +bool shouldANSI() +{ + return isatty(STDERR_FILENO) + && getEnv("TERM").value_or("dumb") != "dumb" + && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value()); +} + +std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) +{ + std::string t, e; + size_t w = 0; + auto i = s.begin(); + + while (w < (size_t) width && i != s.end()) { + + if (*i == '\e') { + std::string e; + e += *i++; + char last = 0; + + if (i != s.end() && *i == '[') { + e += *i++; + // eat parameter bytes + while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; + // eat intermediate bytes + while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; + // eat final byte + if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; + } else { + if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; + } + + if (!filterAll && last == 'm') + t += e; + } + + else if (*i == '\t') { + i++; t += ' '; w++; + while (w < (size_t) width && w % 8) { + t += ' '; w++; + } + } + + else if (*i == '\r' || *i == '\a') + // do nothing for now + i++; + + else { + w++; + // Copy one UTF-8 character. + if ((*i & 0xe0) == 0xc0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } else if ((*i & 0xf0) == 0xe0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } else if ((*i & 0xf8) == 0xf0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } + } else + t += *i++; + } + } + + return t; +} + +static Sync> windowSize{{0, 0}}; + +void updateWindowSize() +{ + struct winsize ws; + if (ioctl(2, TIOCGWINSZ, &ws) == 0) { + auto windowSize_(windowSize.lock()); + windowSize_->first = ws.ws_row; + windowSize_->second = ws.ws_col; + } +} + + +std::pair getWindowSize() +{ + return *windowSize.lock(); +} + +} diff --git a/src/libutil/terminal.hh b/src/libutil/terminal.hh new file mode 100644 index 000000000..43df5bd70 --- /dev/null +++ b/src/libutil/terminal.hh @@ -0,0 +1,40 @@ +#pragma once +///@file + +#include +#include + +namespace nix { + +/** + * Determine whether ANSI escape sequences are appropriate for the + * present output. + */ +bool shouldANSI(); + +/** + * Truncate a string to 'width' printable characters. If 'filterAll' + * is true, all ANSI escape sequences are filtered out. Otherwise, + * some escape sequences (such as colour setting) are copied but not + * included in the character count. Also, tabs are expanded to + * spaces. + */ +std::string filterANSIEscapes(std::string_view s, + bool filterAll = false, + unsigned int width = std::numeric_limits::max()); + +/** + * Recalculate the window size, updating a global variable. Used in the + * `SIGWINCH` signal handler. + */ +void updateWindowSize(); + +/** + * @return the number of rows and columns of the terminal. + * + * The value is cached so this is quick. The cached result is computed + * by `updateWindowSize()`. + */ +std::pair getWindowSize(); + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 97d53ca63..63d9e5248 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1436,83 +1436,6 @@ void ignoreException(Verbosity lvl) } catch (...) { } } -bool shouldANSI() -{ - return isatty(STDERR_FILENO) - && getEnv("TERM").value_or("dumb") != "dumb" - && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value()); -} - -std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) -{ - std::string t, e; - size_t w = 0; - auto i = s.begin(); - - while (w < (size_t) width && i != s.end()) { - - if (*i == '\e') { - std::string e; - e += *i++; - char last = 0; - - if (i != s.end() && *i == '[') { - e += *i++; - // eat parameter bytes - while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; - // eat intermediate bytes - while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; - // eat final byte - if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; - } else { - if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; - } - - if (!filterAll && last == 'm') - t += e; - } - - else if (*i == '\t') { - i++; t += ' '; w++; - while (w < (size_t) width && w % 8) { - t += ' '; w++; - } - } - - else if (*i == '\r' || *i == '\a') - // do nothing for now - i++; - - else { - w++; - // Copy one UTF-8 character. - if ((*i & 0xe0) == 0xc0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } else if ((*i & 0xf0) == 0xe0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } else if ((*i & 0xf8) == 0xf0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } - } else - t += *i++; - } - } - - return t; -} - - constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; std::string base64Encode(std::string_view s) @@ -1630,25 +1553,6 @@ std::pair getLine(std::string_view s) ////////////////////////////////////////////////////////////////////// -static Sync> windowSize{{0, 0}}; - - -void updateWindowSize() -{ - struct winsize ws; - if (ioctl(2, TIOCGWINSZ, &ws) == 0) { - auto windowSize_(windowSize.lock()); - windowSize_->first = ws.ws_row; - windowSize_->second = ws.ws_col; - } -} - - -std::pair getWindowSize() -{ - return *windowSize.lock(); -} - rlim_t savedStackSize = 0; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 151da25b9..1ce7e8312 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -726,23 +726,6 @@ constexpr char treeLast[] = "└───"; constexpr char treeLine[] = "│ "; constexpr char treeNull[] = " "; -/** - * Determine whether ANSI escape sequences are appropriate for the - * present output. - */ -bool shouldANSI(); - -/** - * Truncate a string to 'width' printable characters. If 'filterAll' - * is true, all ANSI escape sequences are filtered out. Otherwise, - * some escape sequences (such as colour setting) are copied but not - * included in the character count. Also, tabs are expanded to - * spaces. - */ -std::string filterANSIEscapes(std::string_view s, - bool filterAll = false, - unsigned int width = std::numeric_limits::max()); - /** * Base64 encoding/decoding. @@ -840,14 +823,6 @@ struct MaintainCount }; -/** - * @return the number of rows and columns of the terminal. - */ -std::pair getWindowSize(); - -void updateWindowSize(); - - /** * Used in various places. */ diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index 720370066..787f79093 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -1,5 +1,6 @@ #include "util.hh" #include "types.hh" +#include "terminal.hh" #include #include From 6b5078c81554ddb36547f8c41805cc94b7738396 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 11:52:13 +0200 Subject: [PATCH 04/22] util.{hh,cc}: Split out file-system.{hh,cc} Change-Id: Ifa89a529e7e34e7291eca87d802d2f569cf2493e --- src/libstore/build/hook-instance.cc | 1 + src/libstore/crypto.cc | 1 + src/libstore/lock.cc | 1 + src/libstore/ssh.hh | 1 + src/libutil/archive.cc | 1 + src/libutil/archive.hh | 1 + src/libutil/canon-path.cc | 2 +- src/libutil/cgroup.cc | 1 + src/libutil/config.cc | 1 + src/libutil/file-system.cc | 678 ++++++++++++++++++ src/libutil/file-system.hh | 270 +++++++ src/libutil/filesystem.cc | 176 ----- src/libutil/hash.hh | 1 + src/libutil/input-accessor.hh | 6 +- src/libutil/meson.build | 3 +- src/libutil/namespaces.cc | 1 + src/libutil/source-path.hh | 2 + src/libutil/tarfile.cc | 1 + src/libutil/util.cc | 501 +------------ src/libutil/util.hh | 239 ------ .../nix-collect-garbage.cc | 1 + tests/unit/libstore/machines.cc | 1 + .../libutil-support/tests/characterization.hh | 2 + tests/unit/libutil/tests.cc | 1 + 24 files changed, 974 insertions(+), 919 deletions(-) create mode 100644 src/libutil/file-system.cc create mode 100644 src/libutil/file-system.hh delete mode 100644 src/libutil/filesystem.cc diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index ea4c2e508..108722cfb 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -1,3 +1,4 @@ +#include "file-system.hh" #include "globals.hh" #include "hook-instance.hh" diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 1027469c9..1e29a812b 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -1,5 +1,6 @@ #include "crypto.hh" #include "util.hh" +#include "file-system.hh" #include "globals.hh" #include diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 80c75eaa5..b52e87ae0 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -1,4 +1,5 @@ #include "lock.hh" +#include "file-system.hh" #include "globals.hh" #include "pathlocks.hh" diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index 94b952af9..0802c6cc0 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "file-system.hh" #include "util.hh" #include "sync.hh" diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index a18c54ebf..c3f39d4b3 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -12,6 +12,7 @@ #include #include "archive.hh" +#include "file-system.hh" #include "util.hh" #include "config.hh" #include "signals.hh" diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 017b6633c..f163a1947 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "serialise.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index 040464532..f678fae94 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -1,5 +1,5 @@ #include "canon-path.hh" -#include "util.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libutil/cgroup.cc b/src/libutil/cgroup.cc index 9320d2371..aa7802f79 100644 --- a/src/libutil/cgroup.cc +++ b/src/libutil/cgroup.cc @@ -2,6 +2,7 @@ #include "cgroup.hh" #include "util.hh" +#include "file-system.hh" #include "finally.hh" #include diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 81efcd507..f6f14878a 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -2,6 +2,7 @@ #include "args.hh" #include "abstract-setting-to-json.hh" #include "experimental-features.hh" +#include "file-system.hh" #include "config-impl.hh" diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc new file mode 100644 index 000000000..e5ba42eb6 --- /dev/null +++ b/src/libutil/file-system.cc @@ -0,0 +1,678 @@ +#include +#include +#include + +#include "environment-variables.hh" +#include "file-system.hh" +#include "finally.hh" +#include "serialise.hh" +#include "util.hh" +#include "signals.hh" +#include "types.hh" + +namespace fs = std::filesystem; + +namespace nix { + +Path absPath(Path path, std::optional dir, bool resolveSymlinks) +{ + if (path.empty() || path[0] != '/') { + if (!dir) { +#ifdef __GNU__ + /* GNU (aka. GNU/Hurd) doesn't have any limitation on path + lengths and doesn't define `PATH_MAX'. */ + char *buf = getcwd(NULL, 0); + if (buf == NULL) +#else + char buf[PATH_MAX]; + if (!getcwd(buf, sizeof(buf))) +#endif + throw SysError("cannot get cwd"); + path = concatStrings(buf, "/", path); +#ifdef __GNU__ + free(buf); +#endif + } else + path = concatStrings(*dir, "/", path); + } + return canonPath(path, resolveSymlinks); +} + + +Path canonPath(PathView path, bool resolveSymlinks) +{ + assert(path != ""); + + std::string s; + s.reserve(256); + + if (path[0] != '/') + throw Error("not an absolute path: '%1%'", path); + + std::string temp; + + /* Count the number of times we follow a symlink and stop at some + arbitrary (but high) limit to prevent infinite loops. */ + unsigned int followCount = 0, maxFollow = 1024; + + while (1) { + + /* Skip slashes. */ + while (!path.empty() && path[0] == '/') path.remove_prefix(1); + if (path.empty()) break; + + /* Ignore `.'. */ + if (path == "." || path.substr(0, 2) == "./") + path.remove_prefix(1); + + /* If `..', delete the last component. */ + else if (path == ".." || path.substr(0, 3) == "../") + { + if (!s.empty()) s.erase(s.rfind('/')); + path.remove_prefix(2); + } + + /* Normal component; copy it. */ + else { + s += '/'; + if (const auto slash = path.find('/'); slash == std::string::npos) { + s += path; + path = {}; + } else { + s += path.substr(0, slash); + path = path.substr(slash); + } + + /* If s points to a symlink, resolve it and continue from there */ + if (resolveSymlinks && isLink(s)) { + if (++followCount >= maxFollow) + throw Error("infinite symlink recursion in path '%1%'", path); + temp = concatStrings(readLink(s), path); + path = temp; + if (!temp.empty() && temp[0] == '/') { + s.clear(); /* restart for symlinks pointing to absolute path */ + } else { + s = dirOf(s); + if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / + s.clear(); + } + } + } + } + } + + return s.empty() ? "/" : std::move(s); +} + +void chmodPath(const Path & path, mode_t mode) +{ + if (chmod(path.c_str(), mode) == -1) + throw SysError("setting permissions on '%s'", path); +} + +Path dirOf(const PathView path) +{ + Path::size_type pos = path.rfind('/'); + if (pos == std::string::npos) + return "."; + return pos == 0 ? "/" : Path(path, 0, pos); +} + + +std::string_view baseNameOf(std::string_view path) +{ + if (path.empty()) + return ""; + + auto last = path.size() - 1; + if (path[last] == '/' && last > 0) + last -= 1; + + auto pos = path.rfind('/', last); + if (pos == std::string::npos) + pos = 0; + else + pos += 1; + + return path.substr(pos, last - pos + 1); +} + + +std::string expandTilde(std::string_view path) +{ + // TODO: expand ~user ? + auto tilde = path.substr(0, 2); + if (tilde == "~/" || tilde == "~") + return getHome() + std::string(path.substr(1)); + else + return std::string(path); +} + + +bool isInDir(std::string_view path, std::string_view dir) +{ + return path.substr(0, 1) == "/" + && path.substr(0, dir.size()) == dir + && path.size() >= dir.size() + 2 + && path[dir.size()] == '/'; +} + + +bool isDirOrInDir(std::string_view path, std::string_view dir) +{ + return path == dir || isInDir(path, dir); +} + + +struct stat stat(const Path & path) +{ + struct stat st; + if (stat(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + + +struct stat lstat(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + +std::optional maybeLstat(const Path & path) +{ + std::optional st{std::in_place}; + if (lstat(path.c_str(), &*st)) + { + if (errno == ENOENT || errno == ENOTDIR) + st.reset(); + else + throw SysError("getting status of '%s'", path); + } + return st; +} + +bool pathExists(const Path & path) +{ + return maybeLstat(path).has_value(); +} + +bool pathAccessible(const Path & path) +{ + try { + return pathExists(path); + } catch (SysError & e) { + // swallow EPERM + if (e.errNo == EPERM) return false; + throw; + } +} + + +Path readLink(const Path & path) +{ + checkInterrupt(); + std::vector buf; + for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { + buf.resize(bufSize); + ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); + if (rlSize == -1) + if (errno == EINVAL) + throw Error("'%1%' is not a symlink", path); + else + throw SysError("reading symbolic link '%1%'", path); + else if (rlSize < bufSize) + return std::string(buf.data(), rlSize); + } +} + + +bool isLink(const Path & path) +{ + struct stat st = lstat(path); + return S_ISLNK(st.st_mode); +} + + +DirEntries readDirectory(DIR *dir, const Path & path) +{ + DirEntries entries; + entries.reserve(64); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { /* sic */ + checkInterrupt(); + std::string name = dirent->d_name; + if (name == "." || name == "..") continue; + entries.emplace_back(name, dirent->d_ino, +#ifdef HAVE_STRUCT_DIRENT_D_TYPE + dirent->d_type +#else + DT_UNKNOWN +#endif + ); + } + if (errno) throw SysError("reading directory '%1%'", path); + + return entries; +} + +DirEntries readDirectory(const Path & path) +{ + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) throw SysError("opening directory '%1%'", path); + + return readDirectory(dir.get(), path); +} + + +unsigned char getFileType(const Path & path) +{ + struct stat st = lstat(path); + if (S_ISDIR(st.st_mode)) return DT_DIR; + if (S_ISLNK(st.st_mode)) return DT_LNK; + if (S_ISREG(st.st_mode)) return DT_REG; + return DT_UNKNOWN; +} + + +std::string readFile(int fd) +{ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + return drainFD(fd, true, st.st_size); +} + + +std::string readFile(const Path & path) +{ + AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; + if (!fd) + throw SysError("opening file '%1%'", path); + return readFile(fd.get()); +} + + +void readFile(const Path & path, Sink & sink) +{ + AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; + if (!fd) + throw SysError("opening file '%s'", path); + drainFD(fd.get(), sink); +} + + +void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) +{ + AutoCloseFD fd{open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode)}; + if (!fd) + throw SysError("opening file '%1%'", path); + try { + writeFull(fd.get(), s); + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; + } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + + +void writeFile(const Path & path, Source & source, mode_t mode, bool sync) +{ + AutoCloseFD fd{open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode)}; + if (!fd) + throw SysError("opening file '%1%'", path); + + std::vector buf(64 * 1024); + + try { + while (true) { + try { + auto n = source.read(buf.data(), buf.size()); + writeFull(fd.get(), {buf.data(), n}); + } catch (EndOfFile &) { break; } + } + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; + } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + +void syncParent(const Path & path) +{ + AutoCloseFD fd{open(dirOf(path).c_str(), O_RDONLY, 0)}; + if (!fd) + throw SysError("opening file '%1%'", path); + fd.fsync(); +} + +static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) +{ + checkInterrupt(); + + std::string name(baseNameOf(path)); + + struct stat st; + if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { + if (errno == ENOENT) return; + throw SysError("getting status of '%1%'", path); + } + + if (!S_ISDIR(st.st_mode)) { + /* We are about to delete a file. Will it likely free space? */ + + switch (st.st_nlink) { + /* Yes: last link. */ + case 1: + bytesFreed += st.st_size; + break; + /* Maybe: yes, if 'auto-optimise-store' or manual optimisation + was performed. Instead of checking for real let's assume + it's an optimised file and space will be freed. + + In worst case we will double count on freed space for files + with exactly two hardlinks for unoptimised packages. + */ + case 2: + bytesFreed += st.st_size; + break; + /* No: 3+ links. */ + default: + break; + } + } + + if (S_ISDIR(st.st_mode)) { + /* Make the directory accessible. */ + const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; + if ((st.st_mode & PERM_MASK) != PERM_MASK) { + if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) + throw SysError("chmod '%1%'", path); + } + + int fd = openat(parentfd, path.c_str(), O_RDONLY); + if (fd == -1) + throw SysError("opening directory '%1%'", path); + AutoCloseDir dir(fdopendir(fd)); + if (!dir) + throw SysError("opening directory '%1%'", path); + for (auto & i : readDirectory(dir.get(), path)) + _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); + } + + int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; + if (unlinkat(parentfd, name.c_str(), flags) == -1) { + if (errno == ENOENT) return; + throw SysError("cannot unlink '%1%'", path); + } +} + +static void _deletePath(const Path & path, uint64_t & bytesFreed) +{ + Path dir = dirOf(path); + if (dir == "") + dir = "/"; + + AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)}; + if (!dirfd) { + if (errno == ENOENT) return; + throw SysError("opening directory '%1%'", path); + } + + _deletePath(dirfd.get(), path, bytesFreed); +} + + +void deletePath(const Path & path) +{ + uint64_t dummy; + deletePath(path, dummy); +} + + +void deletePath(const Path & path, uint64_t & bytesFreed) +{ + //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); + bytesFreed = 0; + _deletePath(path, bytesFreed); +} + +Paths createDirs(const Path & path) +{ + Paths created; + if (path == "/") return created; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + created = createDirs(dirOf(path)); + if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) + throw SysError("creating directory '%1%'", path); + st = lstat(path); + created.push_back(path); + } + + if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) + throw SysError("statting symlink '%1%'", path); + + if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); + + return created; +} + + +////////////////////////////////////////////////////////////////////// + +AutoDelete::AutoDelete() : del{false} {} + +AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p) +{ + del = true; + this->recursive = recursive; +} + +AutoDelete::~AutoDelete() +{ + try { + if (del) { + if (recursive) + deletePath(path); + else { + if (remove(path.c_str()) == -1) + throw SysError("cannot unlink '%1%'", path); + } + } + } catch (...) { + ignoreException(); + } +} + +void AutoDelete::cancel() +{ + del = false; +} + +void AutoDelete::reset(const Path & p, bool recursive) { + path = p; + this->recursive = recursive; + del = true; +} + +////////////////////////////////////////////////////////////////////// + +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + std::atomic & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); + if (includePid) + return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++); + else + return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++); +} + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static std::atomic globalCounter = 0; + std::atomic localCounter = 0; + auto & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { +#if __FreeBSD__ + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError("setting group of directory '%1%'", tmpDir); +#endif + return tmpDir; + } + if (errno != EEXIST) + throw SysError("creating directory '%1%'", tmpDir); + } +} + + +std::pair createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + // FIXME: use O_TMPFILE. + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + closeOnExec(fd.get()); + return {std::move(fd), tmpl}; +} + +void createSymlink(const Path & target, const Path & link) +{ + if (symlink(target.c_str(), link.c_str())) + throw SysError("creating symlink from '%1%' to '%2%'", link, target); +} + +void replaceSymlink(const Path & target, const Path & link) +{ + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + + try { + createSymlink(target, tmp); + } catch (SysError & e) { + if (e.errNo == EEXIST) continue; + throw; + } + + renameFile(tmp, link); + + break; + } +} + +void setWriteTime(const fs::path & p, const struct stat & st) +{ + struct timeval times[2]; + times[0] = { + .tv_sec = st.st_atime, + .tv_usec = 0, + }; + times[1] = { + .tv_sec = st.st_mtime, + .tv_usec = 0, + }; + if (lutimes(p.c_str(), times) != 0) + throw SysError("changing modification time of '%s'", p); +} + +void copy(const fs::directory_entry & from, const fs::path & to, CopyFileFlags flags) +{ + // TODO: Rewrite the `is_*` to use `symlink_status()` + auto statOfFrom = lstat(from.path().c_str()); + auto fromStatus = from.symlink_status(); + + // Mark the directory as writable so that we can delete its children + if (flags.deleteAfter && fs::is_directory(fromStatus)) { + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + } + + + if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { + auto opts = fs::copy_options::overwrite_existing; + + if (!flags.followSymlinks) { + opts |= fs::copy_options::copy_symlinks; + } + + fs::copy(from.path(), to, opts); + } else if (fs::is_directory(fromStatus)) { + fs::create_directory(to); + for (auto & entry : fs::directory_iterator(from.path())) { + copy(entry, to / entry.path().filename(), flags); + } + } else { + throw Error("file '%s' has an unsupported type", from.path()); + } + + setWriteTime(to, statOfFrom); + if (flags.deleteAfter) { + if (!fs::is_symlink(fromStatus)) + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + fs::remove(from.path()); + } +} + + +void copyFile(const Path & oldPath, const Path & newPath, CopyFileFlags flags) +{ + return copy(fs::directory_entry(fs::path(oldPath)), fs::path(newPath), flags); +} + +void renameFile(const Path & oldName, const Path & newName) +{ + fs::rename(oldName, newName); +} + +void moveFile(const Path & oldName, const Path & newName) +{ + try { + renameFile(oldName, newName); + } catch (fs::filesystem_error & e) { + auto oldPath = fs::path(oldName); + auto newPath = fs::path(newName); + // For the move to be as atomic as possible, copy to a temporary + // directory + fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); + Finally removeTemp = [&]() { fs::remove(temp); }; + auto tempCopyTarget = temp / "copy-target"; + if (e.code().value() == EXDEV) { + fs::remove(newPath); + warn("Can’t rename %s as %s, copying instead", oldName, newName); + copy(fs::directory_entry(oldPath), tempCopyTarget, { .deleteAfter = true }); + renameFile(tempCopyTarget, newPath); + } + } +} + +} diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh new file mode 100644 index 000000000..1d91ef334 --- /dev/null +++ b/src/libutil/file-system.hh @@ -0,0 +1,270 @@ +#pragma once +/** + * @file + * + * Utiltities for working with the file sytem and file paths. + */ + +#include "types.hh" +#include "util.hh" + +#include +#include +#include +#include + +#include + +#include +#include + +#ifndef HAVE_STRUCT_DIRENT_D_TYPE +#define DT_UNKNOWN 0 +#define DT_REG 1 +#define DT_LNK 2 +#define DT_DIR 3 +#endif + +namespace nix { + +struct Sink; +struct Source; + +/** + * @return An absolutized path, resolving paths relative to the + * specified directory, or the current directory otherwise. The path + * is also canonicalised. + */ +Path absPath(Path path, + std::optional dir = {}, + bool resolveSymlinks = false); + +/** + * Canonicalise a path by removing all `.` or `..` components and + * double or trailing slashes. Optionally resolves all symlink + * components such that each component of the resulting path is *not* + * a symbolic link. + */ +Path canonPath(PathView path, bool resolveSymlinks = false); + +/** + * Change the permissions of a path + * Not called `chmod` as it shadows and could be confused with + * `int chmod(char *, mode_t)`, which does not handle errors + */ +void chmodPath(const Path & path, mode_t mode); + +/** + * @return The directory part of the given canonical path, i.e., + * everything before the final `/`. If the path is the root or an + * immediate child thereof (e.g., `/foo`), this means `/` + * is returned. + */ +Path dirOf(const PathView path); + +/** + * @return the base name of the given canonical path, i.e., everything + * following the final `/` (trailing slashes are removed). + */ +std::string_view baseNameOf(std::string_view path); + +/** + * Perform tilde expansion on a path. + */ +std::string expandTilde(std::string_view path); + +/** + * Check whether 'path' is a descendant of 'dir'. Both paths must be + * canonicalized. + */ +bool isInDir(std::string_view path, std::string_view dir); + +/** + * Check whether 'path' is equal to 'dir' or a descendant of + * 'dir'. Both paths must be canonicalized. + */ +bool isDirOrInDir(std::string_view path, std::string_view dir); + +/** + * Get status of `path`. + */ +struct stat stat(const Path & path); +struct stat lstat(const Path & path); + +/** + * `lstat` the given path if it exists. + * @return std::nullopt if the path doesn't exist, or an optional containing the result of `lstat` otherwise + */ +std::optional maybeLstat(const Path & path); + +/** + * @return true iff the given path exists. + */ +bool pathExists(const Path & path); + +/** + * A version of pathExists that returns false on a permission error. + * Useful for inferring default paths across directories that might not + * be readable. + * @return true iff the given path can be accessed and exists + */ +bool pathAccessible(const Path & path); + +/** + * Read the contents (target) of a symbolic link. The result is not + * in any way canonicalised. + */ +Path readLink(const Path & path); + +bool isLink(const Path & path); + +/** + * Read the contents of a directory. The entries `.` and `..` are + * removed. + */ +struct DirEntry +{ + std::string name; + ino_t ino; + /** + * one of DT_* + */ + unsigned char type; + DirEntry(std::string name, ino_t ino, unsigned char type) + : name(std::move(name)), ino(ino), type(type) { } +}; + +typedef std::vector DirEntries; + +DirEntries readDirectory(const Path & path); + +unsigned char getFileType(const Path & path); + +/** + * Read the contents of a file into a string. + */ +std::string readFile(int fd); +std::string readFile(const Path & path); +void readFile(const Path & path, Sink & sink); + +/** + * Write a string to a file. + */ +void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); + +void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); + +/** + * Flush a file's parent directory to disk + */ +void syncParent(const Path & path); + +/** + * Delete a path; i.e., in the case of a directory, it is deleted + * recursively. It's not an error if the path does not exist. The + * second variant returns the number of bytes and blocks freed. + */ +void deletePath(const Path & path); + +void deletePath(const Path & path, uint64_t & bytesFreed); + +/** + * Create a directory and all its parents, if necessary. Returns the + * list of created directories, in order of creation. + */ +Paths createDirs(const Path & path); +inline Paths createDirs(PathView path) +{ + return createDirs(Path(path)); +} + +/** + * Create a symlink. + */ +void createSymlink(const Path & target, const Path & link); + +/** + * Atomically create or replace a symlink. + */ +void replaceSymlink(const Path & target, const Path & link); + +void renameFile(const Path & src, const Path & dst); + +/** + * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` + * are on a different filesystem. + * + * Beware that this might not be atomic because of the copy that happens behind + * the scenes + */ +void moveFile(const Path & src, const Path & dst); + +struct CopyFileFlags +{ + /** + * Delete the file after copying. + */ + bool deleteAfter = false; + + /** + * Follow symlinks and copy the eventual target. + */ + bool followSymlinks = false; +}; + +/** + * Recursively copy the content of `oldPath` to `newPath`. If `andDelete` is + * `true`, then also remove `oldPath` (making this equivalent to `moveFile`, but + * with the guaranty that the destination will be “fresh”, with no stale inode + * or file descriptor pointing to it). + */ +void copyFile(const Path & oldPath, const Path & newPath, CopyFileFlags flags); + +/** + * Automatic cleanup of resources. + */ + + +class AutoDelete +{ + Path path; + bool del; + bool recursive; +public: + AutoDelete(); + AutoDelete(const Path & p, bool recursive = true); + ~AutoDelete(); + void cancel(); + void reset(const Path & p, bool recursive = true); + operator Path() const { return path; } + operator PathView() const { return path; } +}; + +struct DIRDeleter +{ + void operator()(DIR * dir) const { + closedir(dir); + } +}; + +typedef std::unique_ptr AutoCloseDir; + +/** + * Create a temporary directory. + */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +/** + * Create a temporary file, returning a file handle and its path. + */ +std::pair createTempFile(const Path & prefix = "nix"); + +/** + * Used in various places. + */ +typedef std::function PathFilter; + +extern PathFilter defaultPathFilter; + +} diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc deleted file mode 100644 index 85871870c..000000000 --- a/src/libutil/filesystem.cc +++ /dev/null @@ -1,176 +0,0 @@ -#include -#include -#include - -#include "environment-variables.hh" -#include "finally.hh" -#include "util.hh" -#include "signals.hh" -#include "types.hh" - -namespace fs = std::filesystem; - -namespace nix { - -static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, - std::atomic & counter) -{ - tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); - if (includePid) - return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++); - else - return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++); -} - -Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter, mode_t mode) -{ - static std::atomic globalCounter = 0; - std::atomic localCounter = 0; - auto & counter(useGlobalCounter ? globalCounter : localCounter); - - while (1) { - checkInterrupt(); - Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), mode) == 0) { -#if __FreeBSD__ - /* Explicitly set the group of the directory. This is to - work around around problems caused by BSD's group - ownership semantics (directories inherit the group of - the parent). For instance, the group of /tmp on - FreeBSD is "wheel", so all directories created in /tmp - will be owned by "wheel"; but if the user is not in - "wheel", then "tar" will fail to unpack archives that - have the setgid bit set on directories. */ - if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError("setting group of directory '%1%'", tmpDir); -#endif - return tmpDir; - } - if (errno != EEXIST) - throw SysError("creating directory '%1%'", tmpDir); - } -} - - -std::pair createTempFile(const Path & prefix) -{ - Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); - // Strictly speaking, this is UB, but who cares... - // FIXME: use O_TMPFILE. - AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); - if (!fd) - throw SysError("creating temporary file '%s'", tmpl); - closeOnExec(fd.get()); - return {std::move(fd), tmpl}; -} - -void createSymlink(const Path & target, const Path & link) -{ - if (symlink(target.c_str(), link.c_str())) - throw SysError("creating symlink from '%1%' to '%2%'", link, target); -} - -void replaceSymlink(const Path & target, const Path & link) -{ - for (unsigned int n = 0; true; n++) { - Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); - - try { - createSymlink(target, tmp); - } catch (SysError & e) { - if (e.errNo == EEXIST) continue; - throw; - } - - renameFile(tmp, link); - - break; - } -} - -void setWriteTime(const fs::path & p, const struct stat & st) -{ - struct timeval times[2]; - times[0] = { - .tv_sec = st.st_atime, - .tv_usec = 0, - }; - times[1] = { - .tv_sec = st.st_mtime, - .tv_usec = 0, - }; - if (lutimes(p.c_str(), times) != 0) - throw SysError("changing modification time of '%s'", p); -} - -void copy(const fs::directory_entry & from, const fs::path & to, CopyFileFlags flags) -{ - // TODO: Rewrite the `is_*` to use `symlink_status()` - auto statOfFrom = lstat(from.path().c_str()); - auto fromStatus = from.symlink_status(); - - // Mark the directory as writable so that we can delete its children - if (flags.deleteAfter && fs::is_directory(fromStatus)) { - fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); - } - - - if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { - auto opts = fs::copy_options::overwrite_existing; - - if (!flags.followSymlinks) { - opts |= fs::copy_options::copy_symlinks; - } - - fs::copy(from.path(), to, opts); - } else if (fs::is_directory(fromStatus)) { - fs::create_directory(to); - for (auto & entry : fs::directory_iterator(from.path())) { - copy(entry, to / entry.path().filename(), flags); - } - } else { - throw Error("file '%s' has an unsupported type", from.path()); - } - - setWriteTime(to, statOfFrom); - if (flags.deleteAfter) { - if (!fs::is_symlink(fromStatus)) - fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); - fs::remove(from.path()); - } -} - - -void copyFile(const Path & oldPath, const Path & newPath, CopyFileFlags flags) -{ - return copy(fs::directory_entry(fs::path(oldPath)), fs::path(newPath), flags); -} - -void renameFile(const Path & oldName, const Path & newName) -{ - fs::rename(oldName, newName); -} - -void moveFile(const Path & oldName, const Path & newName) -{ - try { - renameFile(oldName, newName); - } catch (fs::filesystem_error & e) { - auto oldPath = fs::path(oldName); - auto newPath = fs::path(newName); - // For the move to be as atomic as possible, copy to a temporary - // directory - fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); - Finally removeTemp = [&]() { fs::remove(temp); }; - auto tempCopyTarget = temp / "copy-target"; - if (e.code().value() == EXDEV) { - fs::remove(newPath); - warn("Can’t rename %s as %s, copying instead", oldName, newName); - copy(fs::directory_entry(oldPath), tempCopyTarget, { .deleteAfter = true }); - renameFile(tempCopyTarget, newPath); - } - } -} - -} diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ae3ee40f4..bb2a5e2a7 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "serialise.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libutil/input-accessor.hh b/src/libutil/input-accessor.hh index 740175af0..d8e17c9b1 100644 --- a/src/libutil/input-accessor.hh +++ b/src/libutil/input-accessor.hh @@ -1,8 +1,10 @@ #pragma once ///@file -#include "types.hh" -#include "archive.hh" + +#include +#include +#include namespace nix { diff --git a/src/libutil/meson.build b/src/libutil/meson.build index c890911e9..8af251d1b 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -13,7 +13,7 @@ libutil_sources = files( 'escape-string.cc', 'exit.cc', 'experimental-features.cc', - 'filesystem.cc', + 'file-system.cc', 'git.cc', 'hash.cc', 'hilite.cc', @@ -62,6 +62,7 @@ libutil_headers = files( 'exit.hh', 'experimental-features.hh', 'experimental-features-json.hh', + 'file-system.hh', 'finally.hh', 'fmt.hh', 'git.hh', diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index dec3a7189..4b539f342 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -1,5 +1,6 @@ #if __linux__ +#include "file-system.hh" #include "namespaces.hh" #include "util.hh" #include "finally.hh" diff --git a/src/libutil/source-path.hh b/src/libutil/source-path.hh index 4d4c51062..311e9f7a6 100644 --- a/src/libutil/source-path.hh +++ b/src/libutil/source-path.hh @@ -6,7 +6,9 @@ */ #include "ref.hh" +#include "archive.hh" #include "canon-path.hh" +#include "file-system.hh" #include "repair-flag.hh" #include "input-accessor.hh" diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index f3f04a936..cabb7af9b 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -1,6 +1,7 @@ #include #include +#include "file-system.hh" #include "serialise.hh" #include "tarfile.hh" diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 63d9e5248..ac3071ba8 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -5,6 +5,7 @@ #include "cgroup.hh" #include "signals.hh" #include "environment-variables.hh" +#include "file-system.hh" #include #include @@ -50,352 +51,6 @@ namespace nix { -Path absPath(Path path, std::optional dir, bool resolveSymlinks) -{ - if (path.empty() || path[0] != '/') { - if (!dir) { -#ifdef __GNU__ - /* GNU (aka. GNU/Hurd) doesn't have any limitation on path - lengths and doesn't define `PATH_MAX'. */ - char *buf = getcwd(NULL, 0); - if (buf == NULL) -#else - char buf[PATH_MAX]; - if (!getcwd(buf, sizeof(buf))) -#endif - throw SysError("cannot get cwd"); - path = concatStrings(buf, "/", path); -#ifdef __GNU__ - free(buf); -#endif - } else - path = concatStrings(*dir, "/", path); - } - return canonPath(path, resolveSymlinks); -} - - -Path canonPath(PathView path, bool resolveSymlinks) -{ - assert(path != ""); - - std::string s; - s.reserve(256); - - if (path[0] != '/') - throw Error("not an absolute path: '%1%'", path); - - std::string temp; - - /* Count the number of times we follow a symlink and stop at some - arbitrary (but high) limit to prevent infinite loops. */ - unsigned int followCount = 0, maxFollow = 1024; - - while (1) { - - /* Skip slashes. */ - while (!path.empty() && path[0] == '/') path.remove_prefix(1); - if (path.empty()) break; - - /* Ignore `.'. */ - if (path == "." || path.substr(0, 2) == "./") - path.remove_prefix(1); - - /* If `..', delete the last component. */ - else if (path == ".." || path.substr(0, 3) == "../") - { - if (!s.empty()) s.erase(s.rfind('/')); - path.remove_prefix(2); - } - - /* Normal component; copy it. */ - else { - s += '/'; - if (const auto slash = path.find('/'); slash == std::string::npos) { - s += path; - path = {}; - } else { - s += path.substr(0, slash); - path = path.substr(slash); - } - - /* If s points to a symlink, resolve it and continue from there */ - if (resolveSymlinks && isLink(s)) { - if (++followCount >= maxFollow) - throw Error("infinite symlink recursion in path '%1%'", path); - temp = concatStrings(readLink(s), path); - path = temp; - if (!temp.empty() && temp[0] == '/') { - s.clear(); /* restart for symlinks pointing to absolute path */ - } else { - s = dirOf(s); - if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / - s.clear(); - } - } - } - } - } - - return s.empty() ? "/" : std::move(s); -} - -void chmodPath(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError("setting permissions on '%s'", path); -} - -Path dirOf(const PathView path) -{ - Path::size_type pos = path.rfind('/'); - if (pos == std::string::npos) - return "."; - return pos == 0 ? "/" : Path(path, 0, pos); -} - - -std::string_view baseNameOf(std::string_view path) -{ - if (path.empty()) - return ""; - - auto last = path.size() - 1; - if (path[last] == '/' && last > 0) - last -= 1; - - auto pos = path.rfind('/', last); - if (pos == std::string::npos) - pos = 0; - else - pos += 1; - - return path.substr(pos, last - pos + 1); -} - - -std::string expandTilde(std::string_view path) -{ - // TODO: expand ~user ? - auto tilde = path.substr(0, 2); - if (tilde == "~/" || tilde == "~") - return getHome() + std::string(path.substr(1)); - else - return std::string(path); -} - - -bool isInDir(std::string_view path, std::string_view dir) -{ - return path.substr(0, 1) == "/" - && path.substr(0, dir.size()) == dir - && path.size() >= dir.size() + 2 - && path[dir.size()] == '/'; -} - - -bool isDirOrInDir(std::string_view path, std::string_view dir) -{ - return path == dir || isInDir(path, dir); -} - - -struct stat stat(const Path & path) -{ - struct stat st; - if (stat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); - return st; -} - - -struct stat lstat(const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); - return st; -} - -std::optional maybeLstat(const Path & path) -{ - std::optional st{std::in_place}; - if (lstat(path.c_str(), &*st)) - { - if (errno == ENOENT || errno == ENOTDIR) - st.reset(); - else - throw SysError("getting status of '%s'", path); - } - return st; -} - -bool pathExists(const Path & path) -{ - return maybeLstat(path).has_value(); -} - -bool pathAccessible(const Path & path) -{ - try { - return pathExists(path); - } catch (SysError & e) { - // swallow EPERM - if (e.errNo == EPERM) return false; - throw; - } -} - - -Path readLink(const Path & path) -{ - checkInterrupt(); - std::vector buf; - for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { - buf.resize(bufSize); - ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); - if (rlSize == -1) - if (errno == EINVAL) - throw Error("'%1%' is not a symlink", path); - else - throw SysError("reading symbolic link '%1%'", path); - else if (rlSize < bufSize) - return std::string(buf.data(), rlSize); - } -} - - -bool isLink(const Path & path) -{ - struct stat st = lstat(path); - return S_ISLNK(st.st_mode); -} - - -DirEntries readDirectory(DIR *dir, const Path & path) -{ - DirEntries entries; - entries.reserve(64); - - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { /* sic */ - checkInterrupt(); - std::string name = dirent->d_name; - if (name == "." || name == "..") continue; - entries.emplace_back(name, dirent->d_ino, -#ifdef HAVE_STRUCT_DIRENT_D_TYPE - dirent->d_type -#else - DT_UNKNOWN -#endif - ); - } - if (errno) throw SysError("reading directory '%1%'", path); - - return entries; -} - -DirEntries readDirectory(const Path & path) -{ - AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError("opening directory '%1%'", path); - - return readDirectory(dir.get(), path); -} - - -unsigned char getFileType(const Path & path) -{ - struct stat st = lstat(path); - if (S_ISDIR(st.st_mode)) return DT_DIR; - if (S_ISLNK(st.st_mode)) return DT_LNK; - if (S_ISREG(st.st_mode)) return DT_REG; - return DT_UNKNOWN; -} - - -std::string readFile(int fd) -{ - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError("statting file"); - - return drainFD(fd, true, st.st_size); -} - - -std::string readFile(const Path & path) -{ - AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; - if (!fd) - throw SysError("opening file '%1%'", path); - return readFile(fd.get()); -} - - -void readFile(const Path & path, Sink & sink) -{ - AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; - if (!fd) - throw SysError("opening file '%s'", path); - drainFD(fd.get(), sink); -} - - -void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) -{ - AutoCloseFD fd{open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode)}; - if (!fd) - throw SysError("opening file '%1%'", path); - try { - writeFull(fd.get(), s); - } catch (Error & e) { - e.addTrace({}, "writing file '%1%'", path); - throw; - } - if (sync) - fd.fsync(); - // Explicitly close to make sure exceptions are propagated. - fd.close(); - if (sync) - syncParent(path); -} - - -void writeFile(const Path & path, Source & source, mode_t mode, bool sync) -{ - AutoCloseFD fd{open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode)}; - if (!fd) - throw SysError("opening file '%1%'", path); - - std::vector buf(64 * 1024); - - try { - while (true) { - try { - auto n = source.read(buf.data(), buf.size()); - writeFull(fd.get(), {buf.data(), n}); - } catch (EndOfFile &) { break; } - } - } catch (Error & e) { - e.addTrace({}, "writing file '%1%'", path); - throw; - } - if (sync) - fd.fsync(); - // Explicitly close to make sure exceptions are propagated. - fd.close(); - if (sync) - syncParent(path); -} - -void syncParent(const Path & path) -{ - AutoCloseFD fd{open(dirOf(path).c_str(), O_RDONLY, 0)}; - if (!fd) - throw SysError("opening file '%1%'", path); - fd.fsync(); -} std::string readLine(int fd) { @@ -425,98 +80,6 @@ void writeLine(int fd, std::string s) } -static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) -{ - checkInterrupt(); - - std::string name(baseNameOf(path)); - - struct stat st; - if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { - if (errno == ENOENT) return; - throw SysError("getting status of '%1%'", path); - } - - if (!S_ISDIR(st.st_mode)) { - /* We are about to delete a file. Will it likely free space? */ - - switch (st.st_nlink) { - /* Yes: last link. */ - case 1: - bytesFreed += st.st_size; - break; - /* Maybe: yes, if 'auto-optimise-store' or manual optimisation - was performed. Instead of checking for real let's assume - it's an optimised file and space will be freed. - - In worst case we will double count on freed space for files - with exactly two hardlinks for unoptimised packages. - */ - case 2: - bytesFreed += st.st_size; - break; - /* No: 3+ links. */ - default: - break; - } - } - - if (S_ISDIR(st.st_mode)) { - /* Make the directory accessible. */ - const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; - if ((st.st_mode & PERM_MASK) != PERM_MASK) { - if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) - throw SysError("chmod '%1%'", path); - } - - int fd = openat(parentfd, path.c_str(), O_RDONLY); - if (fd == -1) - throw SysError("opening directory '%1%'", path); - AutoCloseDir dir(fdopendir(fd)); - if (!dir) - throw SysError("opening directory '%1%'", path); - for (auto & i : readDirectory(dir.get(), path)) - _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); - } - - int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; - if (unlinkat(parentfd, name.c_str(), flags) == -1) { - if (errno == ENOENT) return; - throw SysError("cannot unlink '%1%'", path); - } -} - -static void _deletePath(const Path & path, uint64_t & bytesFreed) -{ - Path dir = dirOf(path); - if (dir == "") - dir = "/"; - - AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)}; - if (!dirfd) { - if (errno == ENOENT) return; - throw SysError("opening directory '%1%'", path); - } - - _deletePath(dirfd.get(), path, bytesFreed); -} - - -void deletePath(const Path & path) -{ - uint64_t dummy; - deletePath(path, dummy); -} - - -void deletePath(const Path & path, uint64_t & bytesFreed) -{ - //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); - bytesFreed = 0; - _deletePath(path, bytesFreed); -} - - std::string getUserName() { auto pw = getpwuid(geteuid()); @@ -632,28 +195,6 @@ std::optional getSelfExe() } -Paths createDirs(const Path & path) -{ - Paths created; - if (path == "/") return created; - - struct stat st; - if (lstat(path.c_str(), &st) == -1) { - created = createDirs(dirOf(path)); - if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) - throw SysError("creating directory '%1%'", path); - st = lstat(path); - created.push_back(path); - } - - if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) - throw SysError("statting symlink '%1%'", path); - - if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); - - return created; -} - void readFull(int fd, char * buf, size_t count) { @@ -762,46 +303,6 @@ unsigned int getMaxCPU() ////////////////////////////////////////////////////////////////////// -AutoDelete::AutoDelete() : del{false} {} - -AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p) -{ - del = true; - this->recursive = recursive; -} - -AutoDelete::~AutoDelete() -{ - try { - if (del) { - if (recursive) - deletePath(path); - else { - if (remove(path.c_str()) == -1) - throw SysError("cannot unlink '%1%'", path); - } - } - } catch (...) { - ignoreException(); - } -} - -void AutoDelete::cancel() -{ - del = false; -} - -void AutoDelete::reset(const Path & p, bool recursive) { - path = p; - this->recursive = recursive; - del = true; -} - - - -////////////////////////////////////////////////////////////////////// - - AutoCloseFD::AutoCloseFD() : fd{-1} {} diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 1ce7e8312..63a4fcb6a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -39,135 +39,6 @@ struct Source; extern const std::string nativeSystem; -/** - * @return An absolutized path, resolving paths relative to the - * specified directory, or the current directory otherwise. The path - * is also canonicalised. - */ -Path absPath(Path path, - std::optional dir = {}, - bool resolveSymlinks = false); - -/** - * Canonicalise a path by removing all `.` or `..` components and - * double or trailing slashes. Optionally resolves all symlink - * components such that each component of the resulting path is *not* - * a symbolic link. - */ -Path canonPath(PathView path, bool resolveSymlinks = false); - -/** - * Change the permissions of a path - * Not called `chmod` as it shadows and could be confused with - * `int chmod(char *, mode_t)`, which does not handle errors - */ -void chmodPath(const Path & path, mode_t mode); - -/** - * @return The directory part of the given canonical path, i.e., - * everything before the final `/`. If the path is the root or an - * immediate child thereof (e.g., `/foo`), this means `/` - * is returned. - */ -Path dirOf(const PathView path); - -/** - * @return the base name of the given canonical path, i.e., everything - * following the final `/` (trailing slashes are removed). - */ -std::string_view baseNameOf(std::string_view path); - -/** - * Perform tilde expansion on a path. - */ -std::string expandTilde(std::string_view path); - -/** - * Check whether 'path' is a descendant of 'dir'. Both paths must be - * canonicalized. - */ -bool isInDir(std::string_view path, std::string_view dir); - -/** - * Check whether 'path' is equal to 'dir' or a descendant of - * 'dir'. Both paths must be canonicalized. - */ -bool isDirOrInDir(std::string_view path, std::string_view dir); - -/** - * Get status of `path`. - */ -struct stat stat(const Path & path); -struct stat lstat(const Path & path); - -/** - * `lstat` the given path if it exists. - * @return std::nullopt if the path doesn't exist, or an optional containing the result of `lstat` otherwise - */ -std::optional maybeLstat(const Path & path); - -/** - * @return true iff the given path exists. - */ -bool pathExists(const Path & path); - -/** - * A version of pathExists that returns false on a permission error. - * Useful for inferring default paths across directories that might not - * be readable. - * @return true iff the given path can be accessed and exists - */ -bool pathAccessible(const Path & path); - -/** - * Read the contents (target) of a symbolic link. The result is not - * in any way canonicalised. - */ -Path readLink(const Path & path); - -bool isLink(const Path & path); - -/** - * Read the contents of a directory. The entries `.` and `..` are - * removed. - */ -struct DirEntry -{ - std::string name; - ino_t ino; - /** - * one of DT_* - */ - unsigned char type; - DirEntry(std::string name, ino_t ino, unsigned char type) - : name(std::move(name)), ino(ino), type(type) { } -}; - -typedef std::vector DirEntries; - -DirEntries readDirectory(const Path & path); - -unsigned char getFileType(const Path & path); - -/** - * Read the contents of a file into a string. - */ -std::string readFile(int fd); -std::string readFile(const Path & path); -void readFile(const Path & path, Sink & sink); - -/** - * Write a string to a file. - */ -void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); - -void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); - -/** - * Flush a file's parent directory to disk - */ -void syncParent(const Path & path); - /** * Read a line from a file descriptor. */ @@ -178,14 +49,6 @@ std::string readLine(int fd); */ void writeLine(int fd, std::string s); -/** - * Delete a path; i.e., in the case of a directory, it is deleted - * recursively. It's not an error if the path does not exist. The - * second variant returns the number of bytes and blocks freed. - */ -void deletePath(const Path & path); - -void deletePath(const Path & path, uint64_t & bytesFreed); std::string getUserName(); @@ -238,57 +101,6 @@ Path getStateDir(); */ Path createNixStateDir(); -/** - * Create a directory and all its parents, if necessary. Returns the - * list of created directories, in order of creation. - */ -Paths createDirs(const Path & path); -inline Paths createDirs(PathView path) -{ - return createDirs(Path(path)); -} - -/** - * Create a symlink. - */ -void createSymlink(const Path & target, const Path & link); - -/** - * Atomically create or replace a symlink. - */ -void replaceSymlink(const Path & target, const Path & link); - -void renameFile(const Path & src, const Path & dst); - -/** - * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` - * are on a different filesystem. - * - * Beware that this might not be atomic because of the copy that happens behind - * the scenes - */ -void moveFile(const Path & src, const Path & dst); - -struct CopyFileFlags -{ - /** - * Delete the file after copying. - */ - bool deleteAfter = false; - - /** - * Follow symlinks and copy the eventual target. - */ - bool followSymlinks = false; -}; - -/** - * Recursively copy the content of `oldPath` to `newPath`. If `andDelete` is - * `true`, then also remove `oldPath` (making this equivalent to `moveFile`, but - * with the guaranty that the destination will be “fresh”, with no stale inode - * or file descriptor pointing to it). - */ -void copyFile(const Path & oldPath, const Path & newPath, CopyFileFlags flags); /** * Wrappers arount read()/write() that read/write exactly the @@ -313,27 +125,6 @@ void drainFD(int fd, Sink & sink, bool block = true); */ unsigned int getMaxCPU(); -/** - * Automatic cleanup of resources. - */ - - -class AutoDelete -{ - Path path; - bool del; - bool recursive; -public: - AutoDelete(); - AutoDelete(const Path & p, bool recursive = true); - ~AutoDelete(); - void cancel(); - void reset(const Path & p, bool recursive = true); - operator Path() const { return path; } - operator PathView() const { return path; } -}; - - class AutoCloseFD { int fd; @@ -353,19 +144,6 @@ public: void reset() { *this = {}; } }; - -/** - * Create a temporary directory. - */ -Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); - -/** - * Create a temporary file, returning a file handle and its path. - */ -std::pair createTempFile(const Path & prefix = "nix"); - - class Pipe { public: @@ -375,16 +153,6 @@ public: }; -struct DIRDeleter -{ - void operator()(DIR * dir) const { - closedir(dir); - } -}; - -typedef std::unique_ptr AutoCloseDir; - - class Pid { pid_t pid = -1; @@ -823,13 +591,6 @@ struct MaintainCount }; -/** - * Used in various places. - */ -typedef std::function PathFilter; - -extern PathFilter defaultPathFilter; - /** * Common initialisation performed in child processes. */ diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 1cbba0537..54de68232 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -1,3 +1,4 @@ +#include "file-system.hh" #include "store-api.hh" #include "store-cast.hh" #include "gc-store.hh" diff --git a/tests/unit/libstore/machines.cc b/tests/unit/libstore/machines.cc index 0277237bb..dc192e5be 100644 --- a/tests/unit/libstore/machines.cc +++ b/tests/unit/libstore/machines.cc @@ -1,3 +1,4 @@ +#include "file-system.hh" #include "machines.hh" #include "globals.hh" #include "tests/test-data.hh" diff --git a/tests/unit/libutil-support/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh index 9de20d02c..472fdba3b 100644 --- a/tests/unit/libutil-support/tests/characterization.hh +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -2,6 +2,8 @@ ///@file #include "environment-variables.hh" +#include "file-system.hh" + #include #include diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index 787f79093..c8dc82cb0 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -1,3 +1,4 @@ +#include "file-system.hh" #include "util.hh" #include "types.hh" #include "terminal.hh" From 8cd9aa24a8d48caf22225ce32dff3c5aaa111687 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 12:25:49 +0200 Subject: [PATCH 05/22] util.{hh,cc}: Split out file-descriptor.{hh,cc} Change-Id: I0dd0f9a9c2003fb887e076127e7f825fd3289c76 --- src/libstore/lock.cc | 1 + src/libstore/pathlocks.hh | 2 +- src/libutil/file-descriptor.cc | 251 ++++++++++++++++++ src/libutil/file-descriptor.hh | 89 +++++++ src/libutil/file-system.cc | 12 +- src/libutil/file-system.hh | 3 +- src/libutil/logging.cc | 1 + src/libutil/meson.build | 2 + src/libutil/serialise.hh | 1 + src/libutil/util.cc | 229 ---------------- src/libutil/util.hh | 77 ------ .../repl_characterization/test-session.hh | 2 +- 12 files changed, 349 insertions(+), 321 deletions(-) create mode 100644 src/libutil/file-descriptor.cc create mode 100644 src/libutil/file-descriptor.hh diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index b52e87ae0..05296757d 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -1,4 +1,5 @@ #include "lock.hh" +#include "logging.hh" #include "file-system.hh" #include "globals.hh" #include "pathlocks.hh" diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 4921df352..7fcfa2e40 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "util.hh" +#include "file-descriptor.hh" namespace nix { diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc new file mode 100644 index 000000000..ec22f17ab --- /dev/null +++ b/src/libutil/file-descriptor.cc @@ -0,0 +1,251 @@ +#include "file-system.hh" +#include "finally.hh" +#include "serialise.hh" +#include "signals.hh" + +#include +#include + +namespace nix { + +std::string readFile(int fd) +{ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + return drainFD(fd, true, st.st_size); +} + + +std::string readLine(int fd) +{ + std::string s; + while (1) { + checkInterrupt(); + char ch; + // FIXME: inefficient + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading a line"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; + } + } +} + + +void writeLine(int fd, std::string s) +{ + s += '\n'; + writeFull(fd, s); +} + + +void readFull(int fd, char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + ssize_t res = read(fd, buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("reading from file"); + } + if (res == 0) throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + + +void writeFull(int fd, std::string_view s, bool allowInterrupts) +{ + while (!s.empty()) { + if (allowInterrupts) checkInterrupt(); + ssize_t res = write(fd, s.data(), s.size()); + if (res == -1 && errno != EINTR) + throw SysError("writing to file"); + if (res > 0) + s.remove_prefix(res); + } +} + + +std::string drainFD(int fd, bool block, const size_t reserveSize) +{ + // the parser needs two extra bytes to append terminating characters, other users will + // not care very much about the extra memory. + StringSink sink(reserveSize + 2); + drainFD(fd, sink, block); + return std::move(sink.s); +} + + +void drainFD(int fd, Sink & sink, bool block) +{ + // silence GCC maybe-uninitialized warning in finally + int saved = 0; + + if (!block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) + throw SysError("making file descriptor non-blocking"); + } + + Finally finally([&]() { + if (!block) { + if (fcntl(fd, F_SETFL, saved) == -1) + throw SysError("making file descriptor blocking"); + } + }); + + std::array buf; + while (1) { + checkInterrupt(); + ssize_t rd = read(fd, buf.data(), buf.size()); + if (rd == -1) { + if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) + break; + if (errno != EINTR) + throw SysError("reading from file"); + } + else if (rd == 0) break; + else sink({(char *) buf.data(), (size_t) rd}); + } +} + +AutoCloseFD::AutoCloseFD() : fd{-1} {} + + +AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} + + +AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} +{ + that.fd = -1; +} + + +AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) +{ + close(); + fd = that.fd; + that.fd = -1; + return *this; +} + + +AutoCloseFD::~AutoCloseFD() +{ + try { + close(); + } catch (...) { + ignoreException(); + } +} + + +int AutoCloseFD::get() const +{ + return fd; +} + + +void AutoCloseFD::close() +{ + if (fd != -1) { + if (::close(fd) == -1) + /* This should never happen. */ + throw SysError("closing file descriptor %1%", fd); + fd = -1; + } +} + +void AutoCloseFD::fsync() +{ + if (fd != -1) { + int result; +#if __APPLE__ + result = ::fcntl(fd, F_FULLFSYNC); +#else + result = ::fsync(fd); +#endif + if (result == -1) + throw SysError("fsync file descriptor %1%", fd); + } +} + + +AutoCloseFD::operator bool() const +{ + return fd != -1; +} + + +int AutoCloseFD::release() +{ + int oldFD = fd; + fd = -1; + return oldFD; +} + + +void Pipe::create() +{ + int fds[2]; +#if HAVE_PIPE2 + if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); +#else + if (pipe(fds) != 0) throw SysError("creating pipe"); + closeOnExec(fds[0]); + closeOnExec(fds[1]); +#endif + readSide = AutoCloseFD{fds[0]}; + writeSide = AutoCloseFD{fds[1]}; +} + + +void Pipe::close() +{ + readSide.close(); + writeSide.close(); +} + + +void closeMostFDs(const std::set & exceptions) +{ +#if __linux__ + try { + for (auto & s : readDirectory("/proc/self/fd")) { + auto fd = std::stoi(s.name); + if (!exceptions.count(fd)) { + debug("closing leaked FD %d", fd); + close(fd); + } + } + return; + } catch (SysError &) { + } +#endif + + int maxFD = 0; + maxFD = sysconf(_SC_OPEN_MAX); + for (int fd = 0; fd < maxFD; ++fd) + if (!exceptions.count(fd)) + close(fd); /* ignore result */ +} + + +void closeOnExec(int fd) +{ + int prev; + if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || + fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) + throw SysError("setting close-on-exec flag"); +} + +} diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh new file mode 100644 index 000000000..68324b9d9 --- /dev/null +++ b/src/libutil/file-descriptor.hh @@ -0,0 +1,89 @@ +#pragma once +///@file + +#include "error.hh" + +namespace nix { + +struct Sink; +struct Source; + +/** + * Read a line from a file descriptor. + */ +std::string readLine(int fd); + +/** + * Write a line to a file descriptor. + */ +void writeLine(int fd, std::string s); + +/** + * Read the contents of a file into a string. + */ +std::string readFile(int fd); + +/** + * Wrappers arount read()/write() that read/write exactly the + * requested number of bytes. + */ +void readFull(int fd, char * buf, size_t count); +void writeFull(int fd, std::string_view s, bool allowInterrupts = true); + +/** + * Read a file descriptor until EOF occurs. + */ +std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); + +void drainFD(int fd, Sink & sink, bool block = true); + +class AutoCloseFD +{ + int fd; +public: + AutoCloseFD(); + explicit AutoCloseFD(int fd); + AutoCloseFD(const AutoCloseFD & fd) = delete; + AutoCloseFD(AutoCloseFD&& fd); + ~AutoCloseFD(); + AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; + AutoCloseFD& operator =(AutoCloseFD&& fd) noexcept(false); + int get() const; + explicit operator bool() const; + int release(); + void close(); + void fsync(); + void reset() { *this = {}; } +}; + +class Pipe +{ +public: + AutoCloseFD readSide, writeSide; + void create(); + void close(); +}; + +/** + * Close all file descriptors except those listed in the given set. + * Good practice in child processes. + */ +void closeMostFDs(const std::set & exceptions); + +/** + * Set the close-on-exec flag for the given file descriptor. + */ +void closeOnExec(int fd); + +MakeError(EndOfFile, Error); + +/** + * Create a Unix domain socket. + */ +AutoCloseFD createUnixDomainSocket(); + +/** + * Create a Unix domain socket in listen mode. + */ +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); +} diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index e5ba42eb6..9804c449c 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -3,10 +3,10 @@ #include #include "environment-variables.hh" +#include "file-descriptor.hh" #include "file-system.hh" #include "finally.hh" #include "serialise.hh" -#include "util.hh" #include "signals.hh" #include "types.hh" @@ -278,16 +278,6 @@ unsigned char getFileType(const Path & path) } -std::string readFile(int fd) -{ - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError("statting file"); - - return drainFD(fd, true, st.st_size); -} - - std::string readFile(const Path & path) { AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 1d91ef334..b9b753980 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -6,7 +6,7 @@ */ #include "types.hh" -#include "util.hh" +#include "file-descriptor.hh" #include #include @@ -143,7 +143,6 @@ unsigned char getFileType(const Path & path); /** * Read the contents of a file into a string. */ -std::string readFile(int fd); std::string readFile(const Path & path); void readFile(const Path & path, Sink & sink); diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 731d1034d..8d9e18d09 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -1,4 +1,5 @@ #include "environment-variables.hh" +#include "file-descriptor.hh" #include "logging.hh" #include "util.hh" #include "config.hh" diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 8af251d1b..a4813d9a3 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -13,6 +13,7 @@ libutil_sources = files( 'escape-string.cc', 'exit.cc', 'experimental-features.cc', + 'file-descriptor.cc', 'file-system.cc', 'git.cc', 'hash.cc', @@ -62,6 +63,7 @@ libutil_headers = files( 'exit.hh', 'experimental-features.hh', 'experimental-features-json.hh', + 'file-descriptor.hh', 'file-system.hh', 'finally.hh', 'fmt.hh', diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index d1c791823..d4a14bbf0 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -5,6 +5,7 @@ #include "types.hh" #include "util.hh" +#include "file-descriptor.hh" namespace boost::context { struct stack_context; } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index ac3071ba8..55f3154c9 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -52,34 +52,6 @@ namespace nix { -std::string readLine(int fd) -{ - std::string s; - while (1) { - checkInterrupt(); - char ch; - // FIXME: inefficient - ssize_t rd = read(fd, &ch, 1); - if (rd == -1) { - if (errno != EINTR) - throw SysError("reading a line"); - } else if (rd == 0) - throw EndOfFile("unexpected EOF reading a line"); - else { - if (ch == '\n') return s; - s += ch; - } - } -} - - -void writeLine(int fd, std::string s) -{ - s += '\n'; - writeFull(fd, s); -} - - std::string getUserName() { auto pw = getpwuid(geteuid()); @@ -196,78 +168,6 @@ std::optional getSelfExe() -void readFull(int fd, char * buf, size_t count) -{ - while (count) { - checkInterrupt(); - ssize_t res = read(fd, buf, count); - if (res == -1) { - if (errno == EINTR) continue; - throw SysError("reading from file"); - } - if (res == 0) throw EndOfFile("unexpected end-of-file"); - count -= res; - buf += res; - } -} - - -void writeFull(int fd, std::string_view s, bool allowInterrupts) -{ - while (!s.empty()) { - if (allowInterrupts) checkInterrupt(); - ssize_t res = write(fd, s.data(), s.size()); - if (res == -1 && errno != EINTR) - throw SysError("writing to file"); - if (res > 0) - s.remove_prefix(res); - } -} - - -std::string drainFD(int fd, bool block, const size_t reserveSize) -{ - // the parser needs two extra bytes to append terminating characters, other users will - // not care very much about the extra memory. - StringSink sink(reserveSize + 2); - drainFD(fd, sink, block); - return std::move(sink.s); -} - - -void drainFD(int fd, Sink & sink, bool block) -{ - // silence GCC maybe-uninitialized warning in finally - int saved = 0; - - if (!block) { - saved = fcntl(fd, F_GETFL); - if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) - throw SysError("making file descriptor non-blocking"); - } - - Finally finally([&]() { - if (!block) { - if (fcntl(fd, F_SETFL, saved) == -1) - throw SysError("making file descriptor blocking"); - } - }); - - std::array buf; - while (1) { - checkInterrupt(); - ssize_t rd = read(fd, buf.data(), buf.size()); - if (rd == -1) { - if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) - break; - if (errno != EINTR) - throw SysError("reading from file"); - } - else if (rd == 0) break; - else sink({(char *) buf.data(), (size_t) rd}); - } -} - ////////////////////////////////////////////////////////////////////// unsigned int getMaxCPU() @@ -303,102 +203,6 @@ unsigned int getMaxCPU() ////////////////////////////////////////////////////////////////////// -AutoCloseFD::AutoCloseFD() : fd{-1} {} - - -AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} - - -AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} -{ - that.fd = -1; -} - - -AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) -{ - close(); - fd = that.fd; - that.fd = -1; - return *this; -} - - -AutoCloseFD::~AutoCloseFD() -{ - try { - close(); - } catch (...) { - ignoreException(); - } -} - - -int AutoCloseFD::get() const -{ - return fd; -} - - -void AutoCloseFD::close() -{ - if (fd != -1) { - if (::close(fd) == -1) - /* This should never happen. */ - throw SysError("closing file descriptor %1%", fd); - fd = -1; - } -} - -void AutoCloseFD::fsync() -{ - if (fd != -1) { - int result; -#if __APPLE__ - result = ::fcntl(fd, F_FULLFSYNC); -#else - result = ::fsync(fd); -#endif - if (result == -1) - throw SysError("fsync file descriptor %1%", fd); - } -} - - -AutoCloseFD::operator bool() const -{ - return fd != -1; -} - - -int AutoCloseFD::release() -{ - int oldFD = fd; - fd = -1; - return oldFD; -} - - -void Pipe::create() -{ - int fds[2]; -#if HAVE_PIPE2 - if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); -#else - if (pipe(fds) != 0) throw SysError("creating pipe"); - closeOnExec(fds[0]); - closeOnExec(fds[1]); -#endif - readSide = AutoCloseFD{fds[0]}; - writeSide = AutoCloseFD{fds[1]}; -} - - -void Pipe::close() -{ - readSide.close(); - writeSide.close(); -} ////////////////////////////////////////////////////////////////////// @@ -762,39 +566,6 @@ void runProgram2(const RunOptions & options) } -void closeMostFDs(const std::set & exceptions) -{ -#if __linux__ - try { - for (auto & s : readDirectory("/proc/self/fd")) { - auto fd = std::stoi(s.name); - if (!exceptions.count(fd)) { - debug("closing leaked FD %d", fd); - close(fd); - } - } - return; - } catch (SysError &) { - } -#endif - - int maxFD = 0; - maxFD = sysconf(_SC_OPEN_MAX); - for (int fd = 0; fd < maxFD; ++fd) - if (!exceptions.count(fd)) - close(fd); /* ignore result */ -} - - -void closeOnExec(int fd) -{ - int prev; - if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || - fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) - throw SysError("setting close-on-exec flag"); -} - - ////////////////////////////////////////////////////////////////////// diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 63a4fcb6a..d0e207146 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -39,17 +39,6 @@ struct Source; extern const std::string nativeSystem; -/** - * Read a line from a file descriptor. - */ -std::string readLine(int fd); - -/** - * Write a line to a file descriptor. - */ -void writeLine(int fd, std::string s); - - std::string getUserName(); /** @@ -102,57 +91,12 @@ Path getStateDir(); Path createNixStateDir(); -/** - * Wrappers arount read()/write() that read/write exactly the - * requested number of bytes. - */ -void readFull(int fd, char * buf, size_t count); -void writeFull(int fd, std::string_view s, bool allowInterrupts = true); - -MakeError(EndOfFile, Error); - - -/** - * Read a file descriptor until EOF occurs. - */ -std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); - -void drainFD(int fd, Sink & sink, bool block = true); - /** * If cgroups are active, attempt to calculate the number of CPUs available. * If cgroups are unavailable or if cpu.max is set to "max", return 0. */ unsigned int getMaxCPU(); -class AutoCloseFD -{ - int fd; -public: - AutoCloseFD(); - explicit AutoCloseFD(int fd); - AutoCloseFD(const AutoCloseFD & fd) = delete; - AutoCloseFD(AutoCloseFD&& fd); - ~AutoCloseFD(); - AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; - AutoCloseFD& operator =(AutoCloseFD&& fd) noexcept(false); - int get() const; - explicit operator bool() const; - int release(); - void close(); - void fsync(); - void reset() { *this = {}; } -}; - -class Pipe -{ -public: - AutoCloseFD readSide, writeSide; - void create(); - void close(); -}; - - class Pid { pid_t pid = -1; @@ -172,7 +116,6 @@ public: pid_t release(); }; - /** * Kill all processes running under the specified uid by sending them * a SIGKILL. @@ -279,17 +222,6 @@ public: */ std::vector stringsToCharPtrs(const Strings & ss); -/** - * Close all file descriptors except those listed in the given set. - * Good practice in child processes. - */ -void closeMostFDs(const std::set & exceptions); - -/** - * Set the close-on-exec flag for the given file descriptor. - */ -void closeOnExec(int fd); - MakeError(FormatError, Error); @@ -596,15 +528,6 @@ struct MaintainCount */ void commonChildInit(); -/** - * Create a Unix domain socket. - */ -AutoCloseFD createUnixDomainSocket(); - -/** - * Create a Unix domain socket in listen mode. - */ -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); /** * Bind a Unix domain socket to a path. diff --git a/tests/functional/repl_characterization/test-session.hh b/tests/functional/repl_characterization/test-session.hh index 9961849fc..c77cce6d5 100644 --- a/tests/functional/repl_characterization/test-session.hh +++ b/tests/functional/repl_characterization/test-session.hh @@ -6,7 +6,7 @@ #include #include -#include "util.hh" +#include "file-descriptor.hh" #include "tests/terminal-code-eater.hh" namespace nix { From 9a52e4688ca265155817817f373938428f023966 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 13:14:13 +0200 Subject: [PATCH 06/22] util.{hh,cc}: Split out processes.{hh,cc} Change-Id: I39280dc40ca3f7f9007bc6c898ffcf760e2238b7 --- src/libexpr/primops.cc | 2 +- src/libfetchers/git.cc | 1 + src/libfetchers/mercurial.cc | 3 +- src/libmain/shared.hh | 1 + src/libstore/build/hook-instance.hh | 1 + src/libstore/build/local-derivation-goal.hh | 1 + src/libstore/gc.cc | 1 + src/libstore/globals.cc | 1 + src/libstore/ssh.cc | 1 + src/libstore/ssh.hh | 2 +- src/libutil/meson.build | 2 + src/libutil/namespaces.cc | 5 +- src/libutil/processes.cc | 404 ++++++++++++++++++ src/libutil/processes.hh | 115 +++++ src/libutil/util.cc | 376 +--------------- src/libutil/util.hh | 91 ---- src/nix/upgrade-nix.cc | 1 + .../repl_characterization/test-session.cc | 1 + tests/unit/libutil/tests.cc | 2 +- 19 files changed, 539 insertions(+), 472 deletions(-) create mode 100644 src/libutil/processes.cc create mode 100644 src/libutil/processes.hh diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 64a52dfd6..f8ce90ac1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -9,8 +9,8 @@ #include "json-to-value.hh" #include "names.hh" #include "path-references.hh" +#include "processes.hh" #include "store-api.hh" -#include "util.hh" #include "value-to-json.hh" #include "value-to-xml.hh" #include "primops.hh" diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 4d4bf6cd6..86d9d437f 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1,6 +1,7 @@ #include "fetchers.hh" #include "cache.hh" #include "globals.hh" +#include "processes.hh" #include "tarfile.hh" #include "store-api.hh" #include "url-parts.hh" diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 87fecfca3..086db498f 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -1,7 +1,6 @@ #include "fetchers.hh" #include "cache.hh" -#include "globals.hh" -#include "tarfile.hh" +#include "processes.hh" #include "store-api.hh" #include "url-parts.hh" diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index a7810f77c..907f336f1 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -7,6 +7,7 @@ #include "common-args.hh" #include "path.hh" #include "derived-path.hh" +#include "processes.hh" #include "exit.hh" #include diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/build/hook-instance.hh index d84f62877..481158296 100644 --- a/src/libstore/build/hook-instance.hh +++ b/src/libstore/build/hook-instance.hh @@ -2,6 +2,7 @@ ///@file #include "logging.hh" +#include "processes.hh" #include "serialise.hh" namespace nix { diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index b7f317fb6..f3a83d42f 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -3,6 +3,7 @@ #include "derivation-goal.hh" #include "local-store.hh" +#include "processes.hh" namespace nix { diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 6b37f0af3..60d88f3b2 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,6 +1,7 @@ #include "derivations.hh" #include "globals.hh" #include "local-store.hh" +#include "processes.hh" #include "signals.hh" #include "finally.hh" diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 897d3e8ef..f14e6c91f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -26,6 +26,7 @@ #include "config-impl.hh" #ifdef __APPLE__ +#include "processes.hh" #include #include #endif diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 2f921e2d0..1d8266b9d 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,6 +1,7 @@ #include "environment-variables.hh" #include "ssh.hh" #include "finally.hh" +#include "util.hh" namespace nix { diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index 0802c6cc0..f9c532caa 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -2,7 +2,7 @@ ///@file #include "file-system.hh" -#include "util.hh" +#include "processes.hh" #include "sync.hh" namespace nix { diff --git a/src/libutil/meson.build b/src/libutil/meson.build index a4813d9a3..34715b25e 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -23,6 +23,7 @@ libutil_sources = files( 'namespaces.cc', 'position.cc', 'print-elided.cc', + 'processes.cc', 'references.cc', 'regex.cc', 'serialise.cc', @@ -81,6 +82,7 @@ libutil_headers = files( 'pool.hh', 'position.hh', 'print-elided.hh', + 'processes.hh', 'ref.hh', 'references.hh', 'regex-combinators.hh', diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index 4b539f342..cde442671 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -1,9 +1,10 @@ #if __linux__ #include "file-system.hh" -#include "namespaces.hh" +#include "logging.hh" #include "util.hh" -#include "finally.hh" +#include "namespaces.hh" +#include "processes.hh" #include diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc new file mode 100644 index 000000000..8d1b119b0 --- /dev/null +++ b/src/libutil/processes.cc @@ -0,0 +1,404 @@ +#include "environment-variables.hh" +#include "finally.hh" +#include "logging.hh" +#include "processes.hh" +#include "serialise.hh" +#include "signals.hh" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#ifdef __APPLE__ +# include +#endif + +#ifdef __linux__ +# include +# include +#endif + + +namespace nix { + +Pid::Pid() +{ +} + + +Pid::Pid(pid_t pid) + : pid(pid) +{ +} + + +Pid::~Pid() noexcept(false) +{ + if (pid != -1) kill(); +} + + +void Pid::operator =(pid_t pid) +{ + if (this->pid != -1 && this->pid != pid) kill(); + this->pid = pid; + killSignal = SIGKILL; // reset signal to default +} + + +Pid::operator pid_t() +{ + return pid; +} + + +int Pid::kill() +{ + assert(pid != -1); + + debug("killing process %1%", pid); + + /* Send the requested signal to the child. If it has its own + process group, send the signal to every process in the child + process group (which hopefully includes *all* its children). */ + if (::kill(separatePG ? -pid : pid, killSignal) != 0) { + /* On BSDs, killing a process group will return EPERM if all + processes in the group are zombies (or something like + that). So try to detect and ignore that situation. */ +#if __FreeBSD__ || __APPLE__ + if (errno != EPERM || ::kill(pid, 0) != 0) +#endif + logError(SysError("killing process %d", pid).info()); + } + + return wait(); +} + + +int Pid::wait() +{ + assert(pid != -1); + while (1) { + int status; + int res = waitpid(pid, &status, 0); + if (res == pid) { + pid = -1; + return status; + } + if (errno != EINTR) + throw SysError("cannot get exit status of PID %d", pid); + checkInterrupt(); + } +} + + +void Pid::setSeparatePG(bool separatePG) +{ + this->separatePG = separatePG; +} + + +void Pid::setKillSignal(int signal) +{ + this->killSignal = signal; +} + + +pid_t Pid::release() +{ + pid_t p = pid; + pid = -1; + return p; +} + + +void killUser(uid_t uid) +{ + debug("killing all processes running under uid '%1%'", uid); + + assert(uid != 0); /* just to be safe... */ + + /* The system call kill(-1, sig) sends the signal `sig' to all + users to which the current process can send signals. So we + fork a process, switch to uid, and send a mass kill. */ + + Pid pid = startProcess([&]() { + + if (setuid(uid) == -1) + throw SysError("setting uid"); + + while (true) { +#ifdef __APPLE__ + /* OSX's kill syscall takes a third parameter that, among + other things, determines if kill(-1, signo) affects the + calling process. In the OSX libc, it's set to true, + which means "follow POSIX", which we don't want here + */ + if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; +#else + if (kill(-1, SIGKILL) == 0) break; +#endif + if (errno == ESRCH || errno == EPERM) break; /* no more processes */ + if (errno != EINTR) + throw SysError("cannot kill processes for uid '%1%'", uid); + } + + _exit(0); + }); + + int status = pid.wait(); + if (status != 0) + throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); + + /* !!! We should really do some check to make sure that there are + no processes left running under `uid', but there is no portable + way to do so (I think). The most reliable way may be `ps -eo + uid | grep -q $uid'. */ +} + + +////////////////////////////////////////////////////////////////////// + + +static pid_t doFork(std::function fun) +{ + pid_t pid = fork(); + if (pid != 0) return pid; + fun(); + abort(); +} + +#if __linux__ +static int childEntry(void * arg) +{ + auto main = (std::function *) arg; + (*main)(); + return 1; +} +#endif + + +pid_t startProcess(std::function fun, const ProcessOptions & options) +{ + std::function wrapper = [&]() { + logger = makeSimpleLogger(); + try { +#if __linux__ + if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) + throw SysError("setting death signal"); +#endif + fun(); + } catch (std::exception & e) { + try { + std::cerr << options.errorPrefix << e.what() << "\n"; + } catch (...) { } + } catch (...) { } + if (options.runExitHandlers) + exit(1); + else + _exit(1); + }; + + pid_t pid = -1; + + if (options.cloneFlags) { + #ifdef __linux__ + // Not supported, since then we don't know when to free the stack. + assert(!(options.cloneFlags & CLONE_VM)); + + size_t stackSize = 1 * 1024 * 1024; + auto stack = (char *) mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (stack == MAP_FAILED) throw SysError("allocating stack"); + + Finally freeStack([&]() { munmap(stack, stackSize); }); + + pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); + #else + throw Error("clone flags are only supported on Linux"); + #endif + } else + pid = doFork(wrapper); + + if (pid == -1) throw SysError("unable to fork"); + + return pid; +} + +std::string runProgram(Path program, bool searchPath, const Strings & args, + const std::optional & input, bool isInteractive) +{ + auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive}); + + if (!statusOk(res.first)) + throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); + + return res.second; +} + +// Output = error code + "standard out" output stream +std::pair runProgram(RunOptions && options) +{ + StringSink sink; + options.standardOut = &sink; + + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(sink.s)}; +} + +void runProgram2(const RunOptions & options) +{ + checkInterrupt(); + + assert(!(options.standardIn && options.input)); + + std::unique_ptr source_; + Source * source = options.standardIn; + + if (options.input) { + source_ = std::make_unique(*options.input); + source = source_.get(); + } + + /* Create a pipe. */ + Pipe out, in; + if (options.standardOut) out.create(); + if (source) in.create(); + + ProcessOptions processOptions; + + std::optional>> resumeLoggerDefer; + if (options.isInteractive) { + logger->pause(); + resumeLoggerDefer.emplace( + []() { + logger->resume(); + } + ); + } + + /* Fork. */ + Pid pid = startProcess([&]() { + if (options.environment) + replaceEnv(*options.environment); + if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); + if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + + if (options.chdir && chdir((*options.chdir).c_str()) == -1) + throw SysError("chdir failed"); + if (options.gid && setgid(*options.gid) == -1) + throw SysError("setgid failed"); + /* Drop all other groups if we're setgid. */ + if (options.gid && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + if (options.uid && setuid(*options.uid) == -1) + throw SysError("setuid failed"); + + Strings args_(options.args); + args_.push_front(options.program); + + restoreProcessContext(); + + if (options.searchPath) + execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + // This allows you to refer to a program with a pathname relative + // to the PATH variable. + else + execv(options.program.c_str(), stringsToCharPtrs(args_).data()); + + throw SysError("executing '%1%'", options.program); + }, processOptions); + + out.writeSide.close(); + + std::thread writerThread; + + std::promise promise; + + Finally doJoin([&]() { + if (writerThread.joinable()) + writerThread.join(); + }); + + + if (source) { + in.readSide.close(); + writerThread = std::thread([&]() { + try { + std::vector buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile &) { + break; + } + writeFull(in.writeSide.get(), {buf.data(), n}); + } + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide.close(); + }); + } + + if (options.standardOut) + drainFD(out.readSide.get(), *options.standardOut); + + /* Wait for the child to finish. */ + int status = pid.wait(); + + /* Wait for the writer thread to finish. */ + if (source) promise.get_future().get(); + + if (status) + throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); +} + +std::string statusToString(int status) +{ + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + if (WIFEXITED(status)) + return fmt("failed with exit code %1%", WEXITSTATUS(status)); + else if (WIFSIGNALED(status)) { + int sig = WTERMSIG(status); +#if HAVE_STRSIGNAL + const char * description = strsignal(sig); + return fmt("failed due to signal %1% (%2%)", sig, description); +#else + return fmt("failed due to signal %1%", sig); +#endif + } + else + return "died abnormally"; + } else return "succeeded"; +} + + +bool statusOk(int status) +{ + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +} diff --git a/src/libutil/processes.hh b/src/libutil/processes.hh new file mode 100644 index 000000000..91a4edfd2 --- /dev/null +++ b/src/libutil/processes.hh @@ -0,0 +1,115 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace nix { + +struct Sink; +struct Source; + +class Pid +{ + pid_t pid = -1; + bool separatePG = false; + int killSignal = SIGKILL; +public: + Pid(); + Pid(pid_t pid); + ~Pid() noexcept(false); + void operator =(pid_t pid); + operator pid_t(); + int kill(); + int wait(); + + void setSeparatePG(bool separatePG); + void setKillSignal(int signal); + pid_t release(); +}; + +/** + * Kill all processes running under the specified uid by sending them + * a SIGKILL. + */ +void killUser(uid_t uid); + + +/** + * Fork a process that runs the given function, and return the child + * pid to the caller. + */ +struct ProcessOptions +{ + std::string errorPrefix = ""; + bool dieWithParent = true; + bool runExitHandlers = false; + /** + * use clone() with the specified flags (Linux only) + */ + int cloneFlags = 0; +}; + +pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); + + +/** + * Run a program and return its stdout in a string (i.e., like the + * shell backtick operator). + */ +std::string runProgram(Path program, bool searchPath = false, + const Strings & args = Strings(), + const std::optional & input = {}, bool isInteractive = false); + +struct RunOptions +{ + Path program; + bool searchPath = true; + Strings args; + std::optional uid; + std::optional gid; + std::optional chdir; + std::optional> environment; + std::optional input; + Source * standardIn = nullptr; + Sink * standardOut = nullptr; + bool mergeStderrToStdout = false; + bool isInteractive = false; +}; + +std::pair runProgram(RunOptions && options); + +void runProgram2(const RunOptions & options); + +class ExecError : public Error +{ +public: + int status; + + template + ExecError(int status, const Args & ... args) + : Error(args...), status(status) + { } +}; + +/** + * Convert the exit status of a child as returned by wait() into an + * error string. + */ +std::string statusToString(int status); + +bool statusOk(int status); + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 55f3154c9..702db2afb 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,4 +1,6 @@ #include "util.hh" +#include "processes.hh" + #include "sync.hh" #include "finally.hh" #include "serialise.hh" @@ -208,211 +210,6 @@ unsigned int getMaxCPU() ////////////////////////////////////////////////////////////////////// -Pid::Pid() -{ -} - - -Pid::Pid(pid_t pid) - : pid(pid) -{ -} - - -Pid::~Pid() noexcept(false) -{ - if (pid != -1) kill(); -} - - -void Pid::operator =(pid_t pid) -{ - if (this->pid != -1 && this->pid != pid) kill(); - this->pid = pid; - killSignal = SIGKILL; // reset signal to default -} - - -Pid::operator pid_t() -{ - return pid; -} - - -int Pid::kill() -{ - assert(pid != -1); - - debug("killing process %1%", pid); - - /* Send the requested signal to the child. If it has its own - process group, send the signal to every process in the child - process group (which hopefully includes *all* its children). */ - if (::kill(separatePG ? -pid : pid, killSignal) != 0) { - /* On BSDs, killing a process group will return EPERM if all - processes in the group are zombies (or something like - that). So try to detect and ignore that situation. */ -#if __FreeBSD__ || __APPLE__ - if (errno != EPERM || ::kill(pid, 0) != 0) -#endif - logError(SysError("killing process %d", pid).info()); - } - - return wait(); -} - - -int Pid::wait() -{ - assert(pid != -1); - while (1) { - int status; - int res = waitpid(pid, &status, 0); - if (res == pid) { - pid = -1; - return status; - } - if (errno != EINTR) - throw SysError("cannot get exit status of PID %d", pid); - checkInterrupt(); - } -} - - -void Pid::setSeparatePG(bool separatePG) -{ - this->separatePG = separatePG; -} - - -void Pid::setKillSignal(int signal) -{ - this->killSignal = signal; -} - - -pid_t Pid::release() -{ - pid_t p = pid; - pid = -1; - return p; -} - - -void killUser(uid_t uid) -{ - debug("killing all processes running under uid '%1%'", uid); - - assert(uid != 0); /* just to be safe... */ - - /* The system call kill(-1, sig) sends the signal `sig' to all - users to which the current process can send signals. So we - fork a process, switch to uid, and send a mass kill. */ - - Pid pid = startProcess([&]() { - - if (setuid(uid) == -1) - throw SysError("setting uid"); - - while (true) { -#ifdef __APPLE__ - /* OSX's kill syscall takes a third parameter that, among - other things, determines if kill(-1, signo) affects the - calling process. In the OSX libc, it's set to true, - which means "follow POSIX", which we don't want here - */ - if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; -#else - if (kill(-1, SIGKILL) == 0) break; -#endif - if (errno == ESRCH || errno == EPERM) break; /* no more processes */ - if (errno != EINTR) - throw SysError("cannot kill processes for uid '%1%'", uid); - } - - _exit(0); - }); - - int status = pid.wait(); - if (status != 0) - throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); - - /* !!! We should really do some check to make sure that there are - no processes left running under `uid', but there is no portable - way to do so (I think). The most reliable way may be `ps -eo - uid | grep -q $uid'. */ -} - - -////////////////////////////////////////////////////////////////////// - - -static pid_t doFork(std::function fun) -{ - pid_t pid = fork(); - if (pid != 0) return pid; - fun(); - abort(); -} - - -#if __linux__ -static int childEntry(void * arg) -{ - auto main = (std::function *) arg; - (*main)(); - return 1; -} -#endif - - -pid_t startProcess(std::function fun, const ProcessOptions & options) -{ - std::function wrapper = [&]() { - logger = makeSimpleLogger(); - try { -#if __linux__ - if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) - throw SysError("setting death signal"); -#endif - fun(); - } catch (std::exception & e) { - try { - std::cerr << options.errorPrefix << e.what() << "\n"; - } catch (...) { } - } catch (...) { } - if (options.runExitHandlers) - exit(1); - else - _exit(1); - }; - - pid_t pid = -1; - - if (options.cloneFlags) { - #ifdef __linux__ - // Not supported, since then we don't know when to free the stack. - assert(!(options.cloneFlags & CLONE_VM)); - - size_t stackSize = 1 * 1024 * 1024; - auto stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - Finally freeStack([&]() { munmap(stack, stackSize); }); - - pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); - #else - throw Error("clone flags are only supported on Linux"); - #endif - } else - pid = doFork(wrapper); - - if (pid == -1) throw SysError("unable to fork"); - - return pid; -} - std::vector stringsToCharPtrs(const Strings & ss) { @@ -422,149 +219,6 @@ std::vector stringsToCharPtrs(const Strings & ss) return res; } -std::string runProgram(Path program, bool searchPath, const Strings & args, - const std::optional & input, bool isInteractive) -{ - auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive}); - - if (!statusOk(res.first)) - throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); - - return res.second; -} - -// Output = error code + "standard out" output stream -std::pair runProgram(RunOptions && options) -{ - StringSink sink; - options.standardOut = &sink; - - int status = 0; - - try { - runProgram2(options); - } catch (ExecError & e) { - status = e.status; - } - - return {status, std::move(sink.s)}; -} - -void runProgram2(const RunOptions & options) -{ - checkInterrupt(); - - assert(!(options.standardIn && options.input)); - - std::unique_ptr source_; - Source * source = options.standardIn; - - if (options.input) { - source_ = std::make_unique(*options.input); - source = source_.get(); - } - - /* Create a pipe. */ - Pipe out, in; - if (options.standardOut) out.create(); - if (source) in.create(); - - ProcessOptions processOptions; - - std::optional>> resumeLoggerDefer; - if (options.isInteractive) { - logger->pause(); - resumeLoggerDefer.emplace( - []() { - logger->resume(); - } - ); - } - - /* Fork. */ - Pid pid = startProcess([&]() { - if (options.environment) - replaceEnv(*options.environment); - if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - if (options.mergeStderrToStdout) - if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) - throw SysError("cannot dup stdout into stderr"); - if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - - if (options.chdir && chdir((*options.chdir).c_str()) == -1) - throw SysError("chdir failed"); - if (options.gid && setgid(*options.gid) == -1) - throw SysError("setgid failed"); - /* Drop all other groups if we're setgid. */ - if (options.gid && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - if (options.uid && setuid(*options.uid) == -1) - throw SysError("setuid failed"); - - Strings args_(options.args); - args_.push_front(options.program); - - restoreProcessContext(); - - if (options.searchPath) - execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); - // This allows you to refer to a program with a pathname relative - // to the PATH variable. - else - execv(options.program.c_str(), stringsToCharPtrs(args_).data()); - - throw SysError("executing '%1%'", options.program); - }, processOptions); - - out.writeSide.close(); - - std::thread writerThread; - - std::promise promise; - - Finally doJoin([&]() { - if (writerThread.joinable()) - writerThread.join(); - }); - - - if (source) { - in.readSide.close(); - writerThread = std::thread([&]() { - try { - std::vector buf(8 * 1024); - while (true) { - size_t n; - try { - n = source->read(buf.data(), buf.size()); - } catch (EndOfFile &) { - break; - } - writeFull(in.writeSide.get(), {buf.data(), n}); - } - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - in.writeSide.close(); - }); - } - - if (options.standardOut) - drainFD(out.readSide.get(), *options.standardOut); - - /* Wait for the child to finish. */ - int status = pid.wait(); - - /* Wait for the writer thread to finish. */ - if (source) promise.get_future().get(); - - if (status) - throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); -} - ////////////////////////////////////////////////////////////////////// @@ -648,32 +302,6 @@ std::string Rewriter::operator()(std::string s) } -std::string statusToString(int status) -{ - if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { - if (WIFEXITED(status)) - return fmt("failed with exit code %1%", WEXITSTATUS(status)); - else if (WIFSIGNALED(status)) { - int sig = WTERMSIG(status); -#if HAVE_STRSIGNAL - const char * description = strsignal(sig); - return fmt("failed due to signal %1% (%2%)", sig, description); -#else - return fmt("failed due to signal %1%", sig); -#endif - } - else - return "died abnormally"; - } else return "succeeded"; -} - - -bool statusOk(int status) -{ - return WIFEXITED(status) && WEXITSTATUS(status) == 0; -} - - std::string toLower(const std::string & s) { std::string r(s); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index d0e207146..8d1900131 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -97,78 +97,6 @@ Path createNixStateDir(); */ unsigned int getMaxCPU(); -class Pid -{ - pid_t pid = -1; - bool separatePG = false; - int killSignal = SIGKILL; -public: - Pid(); - Pid(pid_t pid); - ~Pid() noexcept(false); - void operator =(pid_t pid); - operator pid_t(); - int kill(); - int wait(); - - void setSeparatePG(bool separatePG); - void setKillSignal(int signal); - pid_t release(); -}; - -/** - * Kill all processes running under the specified uid by sending them - * a SIGKILL. - */ -void killUser(uid_t uid); - - -/** - * Fork a process that runs the given function, and return the child - * pid to the caller. - */ -struct ProcessOptions -{ - std::string errorPrefix = ""; - bool dieWithParent = true; - bool runExitHandlers = false; - /** - * use clone() with the specified flags (Linux only) - */ - int cloneFlags = 0; -}; - -pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); - - -/** - * Run a program and return its stdout in a string (i.e., like the - * shell backtick operator). - */ -std::string runProgram(Path program, bool searchPath = false, - const Strings & args = Strings(), - const std::optional & input = {}, bool isInteractive = false); - -struct RunOptions -{ - Path program; - bool searchPath = true; - Strings args; - std::optional uid; - std::optional gid; - std::optional chdir; - std::optional> environment; - std::optional input; - Source * standardIn = nullptr; - Sink * standardOut = nullptr; - bool mergeStderrToStdout = false; - bool isInteractive = false; -}; - -std::pair runProgram(RunOptions && options); - -void runProgram2(const RunOptions & options); - /** * Change the stack size. @@ -204,17 +132,6 @@ void restoreMountNamespace(); void unshareFilesystem(); -class ExecError : public Error -{ -public: - int status; - - template - ExecError(int status, const Args & ... args) - : Error(args...), status(status) - { } -}; - /** * Convert a list of strings to a null-terminated vector of `char * *`s. The result must not be accessed beyond the lifetime of the @@ -323,14 +240,6 @@ inline std::string rewriteStrings(std::string s, const StringMap & rewrites) } -/** - * Convert the exit status of a child as returned by wait() into an - * error string. - */ -std::string statusToString(int status); - -bool statusOk(int status); - /** * Parse a string into an integer. diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 3e47fc8f0..cbc28fdd7 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -5,6 +5,7 @@ #include "common-args.hh" #include "local-fs-store.hh" #include "logging.hh" +#include "processes.hh" #include "profiles.hh" #include "store-api.hh" #include "filetransfer.hh" diff --git a/tests/functional/repl_characterization/test-session.cc b/tests/functional/repl_characterization/test-session.cc index c3eb45b85..ab0710b24 100644 --- a/tests/functional/repl_characterization/test-session.cc +++ b/tests/functional/repl_characterization/test-session.cc @@ -5,6 +5,7 @@ #include "test-session.hh" #include "util.hh" #include "escape-char.hh" +#include "processes.hh" namespace nix { diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index c8dc82cb0..a42afd997 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -1,9 +1,9 @@ #include "file-system.hh" #include "util.hh" +#include "processes.hh" #include "types.hh" #include "terminal.hh" -#include #include #include From 2473e1253d37d4bbd3f7aad2ff269ad84747d527 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 13:36:02 +0200 Subject: [PATCH 07/22] util.{hh,cc}: Split out current-process.{hh,cc} Change-Id: I77095b9d37e85310075bada7a076ccd482c28e47 --- src/libmain/shared.cc | 1 + src/libstore/build/local-derivation-goal.cc | 1 - src/libstore/globals.cc | 1 + src/libstore/ssh.cc | 1 + src/libutil/current-process.cc | 109 ++++++++++++++++++++ src/libutil/current-process.hh | 37 +++++++ src/libutil/meson.build | 2 + src/libutil/processes.cc | 1 + src/libutil/util.cc | 87 +--------------- src/libutil/util.hh | 26 ----- src/nix-build/nix-build.cc | 1 + src/nix/edit.cc | 1 + src/nix/main.cc | 1 + src/nix/run.cc | 1 + 14 files changed, 157 insertions(+), 113 deletions(-) create mode 100644 src/libutil/current-process.cc create mode 100644 src/libutil/current-process.hh diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 33323a226..377cd6abd 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -6,6 +6,7 @@ #include "signals.hh" #include "loggers.hh" #include "progress-bar.hh" +#include "current-process.hh" #include #include diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9be780212..c90910d29 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -6,7 +6,6 @@ #include "builtins/buildenv.hh" #include "path-references.hh" #include "finally.hh" -#include "util.hh" #include "archive.hh" #include "compression.hh" #include "daemon.hh" diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index f14e6c91f..30fd4e9e3 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -5,6 +5,7 @@ #include "args.hh" #include "abstract-setting-to-json.hh" #include "compute-levels.hh" +#include "current-process.hh" #include #include diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 1d8266b9d..ccf0aef7f 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,3 +1,4 @@ +#include "current-process.hh" #include "environment-variables.hh" #include "ssh.hh" #include "finally.hh" diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc new file mode 100644 index 000000000..826e25547 --- /dev/null +++ b/src/libutil/current-process.cc @@ -0,0 +1,109 @@ +#include "current-process.hh" +#include "file-system.hh" +#include "logging.hh" +#include "signals.hh" +#include "util.hh" + +#ifdef __APPLE__ +# include +#endif + +#if __linux__ +# include +#endif + +#include +#include + +namespace nix { + +unsigned int getMaxCPU() +{ + #if __linux__ + try { + auto cgroupFS = getCgroupFS(); + if (!cgroupFS) return 0; + + auto cgroups = getCgroups("/proc/self/cgroup"); + auto cgroup = cgroups[""]; + if (cgroup == "") return 0; + + auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; + + auto cpuMax = readFile(cpuFile); + auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); + + if (cpuMaxParts.size() != 2) { + return 0; + } + + auto quota = cpuMaxParts[0]; + auto period = cpuMaxParts[1]; + if (quota != "max") + return std::ceil(std::stoi(quota) / std::stof(period)); + } catch (Error &) { ignoreException(lvlDebug); } + #endif + + return 0; +} + +rlim_t savedStackSize = 0; + +void setStackSize(rlim_t stackSize) +{ + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { + savedStackSize = limit.rlim_cur; + limit.rlim_cur = std::min(stackSize, limit.rlim_max); + if (setrlimit(RLIMIT_STACK, &limit) != 0) { + logger->log( + lvlError, + HintFmt( + "Failed to increase stack size from %1% to %2% (maximum allowed stack size: %3%): %4%", + savedStackSize, + stackSize, + limit.rlim_max, + std::strerror(errno) + ).str() + ); + } + } +} + +void restoreProcessContext(bool restoreMounts) +{ + restoreSignals(); + if (restoreMounts) { + restoreMountNamespace(); + } + + if (savedStackSize) { + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0) { + limit.rlim_cur = savedStackSize; + setrlimit(RLIMIT_STACK, &limit); + } + } +} + +std::optional getSelfExe() +{ + static auto cached = []() -> std::optional + { + #if __linux__ + return readLink("/proc/self/exe"); + #elif __APPLE__ + char buf[1024]; + uint32_t size = sizeof(buf); + if (_NSGetExecutablePath(buf, &size) == 0) + return buf; + else + return std::nullopt; + #else + return std::nullopt; + #endif + }(); + return cached; +} + +} diff --git a/src/libutil/current-process.hh b/src/libutil/current-process.hh new file mode 100644 index 000000000..8d5a2791d --- /dev/null +++ b/src/libutil/current-process.hh @@ -0,0 +1,37 @@ +#pragma once +///@file + +#include +#include + +#include "types.hh" + +namespace nix { + +/** + * If cgroups are active, attempt to calculate the number of CPUs available. + * If cgroups are unavailable or if cpu.max is set to "max", return 0. + */ +unsigned int getMaxCPU(); + + +/** + * Change the stack size. + */ +void setStackSize(rlim_t stackSize); + + +/** + * Restore the original inherited Unix process context (such as signal + * masks, stack size). + + * See startSignalHandlerThread(), saveSignalMask(). + */ +void restoreProcessContext(bool restoreMounts = true); + +/** + * @return the path of the current executable. + */ +std::optional getSelfExe(); + +} diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 34715b25e..ef4270066 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -6,6 +6,7 @@ libutil_sources = files( 'compression.cc', 'compute-levels.cc', 'config.cc', + 'current-process.cc', 'english.cc', 'environment-variables.cc', 'error.cc', @@ -56,6 +57,7 @@ libutil_headers = files( 'compute-levels.hh', 'config-impl.hh', 'config.hh', + 'current-process.hh', 'english.hh', 'environment-variables.hh', 'error.hh', diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc index 8d1b119b0..8639a77f8 100644 --- a/src/libutil/processes.cc +++ b/src/libutil/processes.cc @@ -1,3 +1,4 @@ +#include "current-process.hh" #include "environment-variables.hh" #include "finally.hh" #include "logging.hh" diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 702db2afb..7f9413f8d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,5 +1,6 @@ #include "util.hh" #include "processes.hh" +#include "current-process.hh" #include "sync.hh" #include "finally.hh" @@ -148,59 +149,11 @@ Path createNixStateDir() } -std::optional getSelfExe() -{ - static auto cached = []() -> std::optional - { - #if __linux__ - return readLink("/proc/self/exe"); - #elif __APPLE__ - char buf[1024]; - uint32_t size = sizeof(buf); - if (_NSGetExecutablePath(buf, &size) == 0) - return buf; - else - return std::nullopt; - #else - return std::nullopt; - #endif - }(); - return cached; -} ////////////////////////////////////////////////////////////////////// -unsigned int getMaxCPU() -{ - #if __linux__ - try { - auto cgroupFS = getCgroupFS(); - if (!cgroupFS) return 0; - - auto cgroups = getCgroups("/proc/self/cgroup"); - auto cgroup = cgroups[""]; - if (cgroup == "") return 0; - - auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; - - auto cpuMax = readFile(cpuFile); - auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); - - if (cpuMaxParts.size() != 2) { - return 0; - } - - auto quota = cpuMaxParts[0]; - auto period = cpuMaxParts[1]; - if (quota != "max") - return std::ceil(std::stoi(quota) / std::stof(period)); - } catch (Error &) { ignoreException(lvlDebug); } - #endif - - return 0; -} ////////////////////////////////////////////////////////////////////// @@ -454,28 +407,6 @@ std::pair getLine(std::string_view s) ////////////////////////////////////////////////////////////////////// -rlim_t savedStackSize = 0; - -void setStackSize(rlim_t stackSize) -{ - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { - savedStackSize = limit.rlim_cur; - limit.rlim_cur = std::min(stackSize, limit.rlim_max); - if (setrlimit(RLIMIT_STACK, &limit) != 0) { - logger->log( - lvlError, - HintFmt( - "Failed to increase stack size from %1% to %2% (maximum allowed stack size: %3%): %4%", - savedStackSize, - stackSize, - limit.rlim_max, - std::strerror(errno) - ).str() - ); - } - } -} #if __linux__ static AutoCloseFD fdSavedMountNamespace; @@ -528,22 +459,6 @@ void unshareFilesystem() #endif } -void restoreProcessContext(bool restoreMounts) -{ - restoreSignals(); - if (restoreMounts) { - restoreMountNamespace(); - } - - if (savedStackSize) { - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0) { - limit.rlim_cur = savedStackSize; - setrlimit(RLIMIT_STACK, &limit); - } - } -} - AutoCloseFD createUnixDomainSocket() { AutoCloseFD fdSocket{socket(PF_UNIX, SOCK_STREAM diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 8d1900131..de5f0a2ee 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -71,11 +71,6 @@ std::vector getConfigDirs(); */ Path getDataDir(); -/** - * @return the path of the current executable. - */ -std::optional getSelfExe(); - /** * @return $XDG_STATE_HOME or $HOME/.local/state. * @@ -91,27 +86,6 @@ Path getStateDir(); Path createNixStateDir(); -/** - * If cgroups are active, attempt to calculate the number of CPUs available. - * If cgroups are unavailable or if cpu.max is set to "max", return 0. - */ -unsigned int getMaxCPU(); - - -/** - * Change the stack size. - */ -void setStackSize(rlim_t stackSize); - - -/** - * Restore the original inherited Unix process context (such as signal - * masks, stack size). - - * See startSignalHandlerThread(), saveSignalMask(). - */ -void restoreProcessContext(bool restoreMounts = true); - /** * Save the current mount namespace. Ignored if called more than * once. diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 9a3994842..f1c7bbfb8 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -13,6 +13,7 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" +#include "current-process.hh" #include "derivations.hh" #include "util.hh" #include "shared.hh" diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 66629fab0..d1741a254 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -4,6 +4,7 @@ #include "attr-path.hh" #include "progress-bar.hh" #include "editor-for.hh" +#include "current-process.hh" #include diff --git a/src/nix/main.cc b/src/nix/main.cc index 83d697326..0f98861d5 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -3,6 +3,7 @@ #include "args/root.hh" #include "command.hh" #include "common-args.hh" +#include "current-process.hh" #include "eval.hh" #include "eval-settings.hh" #include "globals.hh" diff --git a/src/nix/run.cc b/src/nix/run.cc index 94d492ca7..e1e896bb4 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -10,6 +10,7 @@ #include "progress-bar.hh" #include "eval.hh" #include "build/personality.hh" +#include "current-process.hh" #if __linux__ #include From e81ed5f12d0702ed402faa8b5ee725f2203db60c Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 13:44:39 +0200 Subject: [PATCH 08/22] util.{hh,cc}: Split out child.{hh,cc} Change-Id: Iec4824e071f537b17dd62dbb8c01b8eec14e9783 --- src/libstore/build/child.cc | 33 +++++++++++++++++++++ src/libstore/build/child.hh | 11 +++++++ src/libstore/build/hook-instance.cc | 1 + src/libstore/build/local-derivation-goal.cc | 1 + src/libstore/meson.build | 2 ++ src/libutil/util.cc | 29 ------------------ src/libutil/util.hh | 6 ---- 7 files changed, 48 insertions(+), 35 deletions(-) create mode 100644 src/libstore/build/child.cc create mode 100644 src/libstore/build/child.hh diff --git a/src/libstore/build/child.cc b/src/libstore/build/child.cc new file mode 100644 index 000000000..a82a5eec9 --- /dev/null +++ b/src/libstore/build/child.cc @@ -0,0 +1,33 @@ +#include "current-process.hh" +#include "logging.hh" + +namespace nix { + +void commonChildInit() +{ + logger = makeSimpleLogger(); + + const static std::string pathNullDevice = "/dev/null"; + restoreProcessContext(false); + + /* Put the child in a separate session (and thus a separate + process group) so that it has no controlling terminal (meaning + that e.g. ssh cannot open /dev/tty) and it doesn't receive + terminal signals. */ + if (setsid() == -1) + throw SysError("creating a new session"); + + /* Dup stderr to stdout. */ + if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) + throw SysError("cannot dup stderr into stdout"); + + /* Reroute stdin to /dev/null. */ + int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); + if (fdDevNull == -1) + throw SysError("cannot open '%1%'", pathNullDevice); + if (dup2(fdDevNull, STDIN_FILENO) == -1) + throw SysError("cannot dup null device into stdin"); + close(fdDevNull); +} + +} diff --git a/src/libstore/build/child.hh b/src/libstore/build/child.hh new file mode 100644 index 000000000..3dfc552b9 --- /dev/null +++ b/src/libstore/build/child.hh @@ -0,0 +1,11 @@ +#pragma once +///@file + +namespace nix { + +/** + * Common initialisation performed in child processes. + */ +void commonChildInit(); + +} diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index 108722cfb..86f72486e 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -1,3 +1,4 @@ +#include "child.hh" #include "file-system.hh" #include "globals.hh" #include "hook-instance.hh" diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index c90910d29..d385ffb34 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -14,6 +14,7 @@ #include "cgroup.hh" #include "personality.hh" #include "namespaces.hh" +#include "child.hh" #include #include diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 98549f6d9..f776e9621 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -79,6 +79,7 @@ libstore_sources = files( 'store-api.cc', 'uds-remote-store.cc', 'worker-protocol.cc', + 'build/child.cc', 'build/derivation-goal.cc', 'build/drv-output-substitution-goal.cc', 'build/entry-points.cc', @@ -96,6 +97,7 @@ libstore_sources = files( libstore_headers = files( 'binary-cache-store.hh', + 'build/child.hh', 'build/derivation-goal.hh', 'build/drv-output-substitution-goal.hh', 'build/goal.hh', diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 7f9413f8d..aff26e32b 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -559,33 +559,4 @@ std::string showBytes(uint64_t bytes) return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); } - -// FIXME: move to libstore/build -void commonChildInit() -{ - logger = makeSimpleLogger(); - - const static std::string pathNullDevice = "/dev/null"; - restoreProcessContext(false); - - /* Put the child in a separate session (and thus a separate - process group) so that it has no controlling terminal (meaning - that e.g. ssh cannot open /dev/tty) and it doesn't receive - terminal signals. */ - if (setsid() == -1) - throw SysError("creating a new session"); - - /* Dup stderr to stdout. */ - if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) - throw SysError("cannot dup stderr into stdout"); - - /* Reroute stdin to /dev/null. */ - int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); - if (fdDevNull == -1) - throw SysError("cannot open '%1%'", pathNullDevice); - if (dup2(fdDevNull, STDIN_FILENO) == -1) - throw SysError("cannot dup null device into stdin"); - close(fdDevNull); -} - } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index de5f0a2ee..e704cd709 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -406,12 +406,6 @@ struct MaintainCount }; -/** - * Common initialisation performed in child processes. - */ -void commonChildInit(); - - /** * Bind a Unix domain socket to a path. */ From 5b5a75979a0b954a5deefe79c8040bac1ad9c76a Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 14:41:48 +0200 Subject: [PATCH 09/22] util.{hh,cc}: Split out unix-domain-socket.{hh,cc} Change-Id: I3f9a628e0f8998b6146f5caa8ae9842361a66b8b --- src/libstore/build/local-derivation-goal.cc | 1 + src/libstore/gc.cc | 1 + src/libstore/uds-remote-store.cc | 1 + src/libutil/file-descriptor.hh | 9 -- src/libutil/meson.build | 2 + src/libutil/unix-domain-socket.cc | 105 ++++++++++++++++++++ src/libutil/unix-domain-socket.hh | 31 ++++++ src/libutil/util.cc | 94 ------------------ src/libutil/util.hh | 11 -- src/nix/daemon.cc | 1 + 10 files changed, 142 insertions(+), 114 deletions(-) create mode 100644 src/libutil/unix-domain-socket.cc create mode 100644 src/libutil/unix-domain-socket.hh diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index d385ffb34..fd871e8a4 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -15,6 +15,7 @@ #include "personality.hh" #include "namespaces.hh" #include "child.hh" +#include "unix-domain-socket.hh" #include #include diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 60d88f3b2..722452b8d 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -4,6 +4,7 @@ #include "processes.hh" #include "signals.hh" #include "finally.hh" +#include "unix-domain-socket.hh" #include #include diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 99589f8b2..226cdf717 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,4 +1,5 @@ #include "uds-remote-store.hh" +#include "unix-domain-socket.hh" #include "worker-protocol.hh" #include diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh index 68324b9d9..f59baa7a0 100644 --- a/src/libutil/file-descriptor.hh +++ b/src/libutil/file-descriptor.hh @@ -77,13 +77,4 @@ void closeOnExec(int fd); MakeError(EndOfFile, Error); -/** - * Create a Unix domain socket. - */ -AutoCloseFD createUnixDomainSocket(); - -/** - * Create a Unix domain socket in listen mode. - */ -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); } diff --git a/src/libutil/meson.build b/src/libutil/meson.build index ef4270066..73c520116 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -35,6 +35,7 @@ libutil_sources = files( 'tarfile.cc', 'terminal.cc', 'thread-pool.cc', + 'unix-domain-socket.cc', 'url.cc', 'url-name.cc', 'util.cc', @@ -102,6 +103,7 @@ libutil_headers = files( 'thread-pool.hh', 'topo-sort.hh', 'types.hh', + 'unix-domain-socket.hh', 'url-parts.hh', 'url-name.hh', 'url.hh', diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc new file mode 100644 index 000000000..9fefcbe1c --- /dev/null +++ b/src/libutil/unix-domain-socket.cc @@ -0,0 +1,105 @@ +#include "file-system.hh" +#include "processes.hh" +#include "unix-domain-socket.hh" +#include "util.hh" + +#include +#include +#include + +namespace nix { + +AutoCloseFD createUnixDomainSocket() +{ + AutoCloseFD fdSocket{socket(PF_UNIX, SOCK_STREAM + #ifdef SOCK_CLOEXEC + | SOCK_CLOEXEC + #endif + , 0)}; + if (!fdSocket) + throw SysError("cannot create Unix domain socket"); + closeOnExec(fdSocket.get()); + return fdSocket; +} + + +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +{ + auto fdSocket = nix::createUnixDomainSocket(); + + bind(fdSocket.get(), path); + + chmodPath(path.c_str(), mode); + + if (listen(fdSocket.get(), 100) == -1) + throw SysError("cannot listen on socket '%1%'", path); + + return fdSocket; +} + +static void bindConnectProcHelper( + std::string_view operationName, auto && operation, + int fd, const std::string & path) +{ + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + // Casting between types like these legacy C library interfaces + // require is forbidden in C++. To maintain backwards + // compatibility, the implementation of the bind/connect functions + // contains some hints to the compiler that allow for this + // special case. + auto * psaddr = reinterpret_cast(&addr); + + if (path.size() + 1 >= sizeof(addr.sun_path)) { + Pipe pipe; + pipe.create(); + Pid pid = startProcess([&] { + try { + pipe.readSide.close(); + Path dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); + if (operation(fd, psaddr, sizeof(addr)) == -1) + throw SysError("cannot %s to socket at '%s'", operationName, path); + writeFull(pipe.writeSide.get(), "0\n"); + } catch (SysError & e) { + writeFull(pipe.writeSide.get(), fmt("%d\n", e.errNo)); + } catch (...) { + writeFull(pipe.writeSide.get(), "-1\n"); + } + }); + pipe.writeSide.close(); + auto errNo = string2Int(chomp(drainFD(pipe.readSide.get()))); + if (!errNo || *errNo == -1) + throw Error("cannot %s to socket at '%s'", operationName, path); + else if (*errNo > 0) { + errno = *errNo; + throw SysError("cannot %s to socket at '%s'", operationName, path); + } + } else { + memcpy(addr.sun_path, path.c_str(), path.size() + 1); + if (operation(fd, psaddr, sizeof(addr)) == -1) + throw SysError("cannot %s to socket at '%s'", operationName, path); + } +} + + +void bind(int fd, const std::string & path) +{ + unlink(path.c_str()); + + bindConnectProcHelper("bind", ::bind, fd, path); +} + + +void connect(int fd, const std::string & path) +{ + bindConnectProcHelper("connect", ::connect, fd, path); +} + +} diff --git a/src/libutil/unix-domain-socket.hh b/src/libutil/unix-domain-socket.hh new file mode 100644 index 000000000..692ad2627 --- /dev/null +++ b/src/libutil/unix-domain-socket.hh @@ -0,0 +1,31 @@ +#pragma once +///@file + +#include "file-descriptor.hh" +#include "types.hh" + +#include + +namespace nix { + +/** + * Create a Unix domain socket. + */ +AutoCloseFD createUnixDomainSocket(); + +/** + * Create a Unix domain socket in listen mode. + */ +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); + +/** + * Bind a Unix domain socket to a path. + */ +void bind(int fd, const std::string & path); + +/** + * Connect to a Unix domain socket. + */ +void connect(int fd, const std::string & path); + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index aff26e32b..f580ef038 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -459,100 +459,6 @@ void unshareFilesystem() #endif } -AutoCloseFD createUnixDomainSocket() -{ - AutoCloseFD fdSocket{socket(PF_UNIX, SOCK_STREAM - #ifdef SOCK_CLOEXEC - | SOCK_CLOEXEC - #endif - , 0)}; - if (!fdSocket) - throw SysError("cannot create Unix domain socket"); - closeOnExec(fdSocket.get()); - return fdSocket; -} - - -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) -{ - auto fdSocket = nix::createUnixDomainSocket(); - - bind(fdSocket.get(), path); - - chmodPath(path.c_str(), mode); - - if (listen(fdSocket.get(), 100) == -1) - throw SysError("cannot listen on socket '%1%'", path); - - return fdSocket; -} - - -static void bindConnectProcHelper( - std::string_view operationName, auto && operation, - int fd, const std::string & path) -{ - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - - // Casting between types like these legacy C library interfaces - // require is forbidden in C++. To maintain backwards - // compatibility, the implementation of the bind/connect functions - // contains some hints to the compiler that allow for this - // special case. - auto * psaddr = reinterpret_cast(&addr); - - if (path.size() + 1 >= sizeof(addr.sun_path)) { - Pipe pipe; - pipe.create(); - Pid pid = startProcess([&] { - try { - pipe.readSide.close(); - Path dir = dirOf(path); - if (chdir(dir.c_str()) == -1) - throw SysError("chdir to '%s' failed", dir); - std::string base(baseNameOf(path)); - if (base.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%s' is too long", base); - memcpy(addr.sun_path, base.c_str(), base.size() + 1); - if (operation(fd, psaddr, sizeof(addr)) == -1) - throw SysError("cannot %s to socket at '%s'", operationName, path); - writeFull(pipe.writeSide.get(), "0\n"); - } catch (SysError & e) { - writeFull(pipe.writeSide.get(), fmt("%d\n", e.errNo)); - } catch (...) { - writeFull(pipe.writeSide.get(), "-1\n"); - } - }); - pipe.writeSide.close(); - auto errNo = string2Int(chomp(drainFD(pipe.readSide.get()))); - if (!errNo || *errNo == -1) - throw Error("cannot %s to socket at '%s'", operationName, path); - else if (*errNo > 0) { - errno = *errNo; - throw SysError("cannot %s to socket at '%s'", operationName, path); - } - } else { - memcpy(addr.sun_path, path.c_str(), path.size() + 1); - if (operation(fd, psaddr, sizeof(addr)) == -1) - throw SysError("cannot %s to socket at '%s'", operationName, path); - } -} - - -void bind(int fd, const std::string & path) -{ - unlink(path.c_str()); - - bindConnectProcHelper("bind", ::bind, fd, path); -} - - -void connect(int fd, const std::string & path) -{ - bindConnectProcHelper("connect", ::connect, fd, path); -} - std::string showBytes(uint64_t bytes) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index e704cd709..216693635 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -406,17 +406,6 @@ struct MaintainCount }; -/** - * Bind a Unix domain socket to a path. - */ -void bind(int fd, const std::string & path); - -/** - * Connect to a Unix domain socket. - */ -void connect(int fd, const std::string & path); - - /** * A Rust/Python-like enumerate() iterator adapter. * diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 9d4afb6d9..113e23bd1 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -14,6 +14,7 @@ #include "legacy.hh" #include "signals.hh" #include "daemon.hh" +#include "unix-domain-socket.hh" #include #include From b91055112035c256fffd44d77f746b977cfdf3ca Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 14:57:20 +0200 Subject: [PATCH 10/22] util.{hh,cc}: Split out strings.{hh,cc} Change-Id: I4f642d1046d56b5db26f1b0296ee16a0e02d444a --- src/libstore/crypto.cc | 2 +- src/libstore/machines.cc | 2 +- src/libstore/names.cc | 2 +- src/libstore/outputs-spec.cc | 3 +- src/libstore/ssh.cc | 3 +- src/libutil/cgroup.cc | 4 +- src/libutil/config.cc | 1 + src/libutil/current-process.cc | 1 + src/libutil/experimental-features.cc | 1 + src/libutil/meson.build | 2 + src/libutil/namespaces.cc | 2 +- src/libutil/regex-combinators.hh | 2 + src/libutil/serialise.hh | 1 + src/libutil/shlex.cc | 2 +- src/libutil/strings.cc | 232 ++++++++++++++++ src/libutil/strings.hh | 256 ++++++++++++++++++ src/libutil/unix-domain-socket.cc | 2 +- src/libutil/url.cc | 2 +- src/libutil/util.cc | 230 +--------------- src/libutil/util.hh | 248 ----------------- .../repl_characterization.cc | 1 + .../repl_characterization/test-session.cc | 2 +- tests/unit/libstore/machines.cc | 1 + .../tests/cli-literate-parser.cc | 1 + tests/unit/libutil/tests.cc | 3 +- 25 files changed, 517 insertions(+), 489 deletions(-) create mode 100644 src/libutil/strings.cc create mode 100644 src/libutil/strings.hh diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 1e29a812b..2e0fd8ca5 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -1,7 +1,7 @@ #include "crypto.hh" -#include "util.hh" #include "file-system.hh" #include "globals.hh" +#include "strings.hh" #include diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 700c9b3dd..cdd1e1c2c 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -1,7 +1,7 @@ #include "machines.hh" -#include "util.hh" #include "globals.hh" #include "store-api.hh" +#include "strings.hh" #include diff --git a/src/libstore/names.cc b/src/libstore/names.cc index 277aabf0f..b903a99f0 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -1,5 +1,5 @@ #include "names.hh" -#include "util.hh" +#include "strings.hh" #include diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc index 21c069223..4422bcd21 100644 --- a/src/libstore/outputs-spec.cc +++ b/src/libstore/outputs-spec.cc @@ -1,10 +1,11 @@ #include #include -#include "util.hh" #include "regex-combinators.hh" #include "outputs-spec.hh" #include "path-regex.hh" +#include "strings.hh" +#include "util.hh" namespace nix { diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index ccf0aef7f..932ebaa52 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -2,7 +2,8 @@ #include "environment-variables.hh" #include "ssh.hh" #include "finally.hh" -#include "util.hh" +#include "logging.hh" +#include "strings.hh" namespace nix { diff --git a/src/libutil/cgroup.cc b/src/libutil/cgroup.cc index aa7802f79..e28e21c3e 100644 --- a/src/libutil/cgroup.cc +++ b/src/libutil/cgroup.cc @@ -1,15 +1,17 @@ +#include "logging.hh" #if __linux__ #include "cgroup.hh" -#include "util.hh" #include "file-system.hh" #include "finally.hh" +#include "strings.hh" #include #include #include #include #include +#include #include #include diff --git a/src/libutil/config.cc b/src/libutil/config.cc index f6f14878a..729b4e596 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -3,6 +3,7 @@ #include "abstract-setting-to-json.hh" #include "experimental-features.hh" #include "file-system.hh" +#include "strings.hh" #include "config-impl.hh" diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 826e25547..41f591b0c 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -3,6 +3,7 @@ #include "logging.hh" #include "signals.hh" #include "util.hh" +#include "strings.hh" #ifdef __APPLE__ # include diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 25beba467..8ebec2956 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -1,6 +1,7 @@ #include "experimental-features.hh" // Required for instances of to_json and from_json for ExperimentalFeature #include "experimental-features-json.hh" +#include "strings.hh" #include "util.hh" #include "nlohmann/json.hpp" diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 73c520116..b662ea455 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -31,6 +31,7 @@ libutil_sources = files( 'shlex.cc', 'signals.cc', 'source-path.cc', + 'strings.cc', 'suggestions.cc', 'tarfile.cc', 'terminal.cc', @@ -96,6 +97,7 @@ libutil_headers = files( 'signals.hh', 'source-path.hh', 'split.hh', + 'strings.hh', 'suggestions.hh', 'sync.hh', 'tarfile.hh', diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index cde442671..d092e6fcc 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -2,9 +2,9 @@ #include "file-system.hh" #include "logging.hh" -#include "util.hh" #include "namespaces.hh" #include "processes.hh" +#include "strings.hh" #include diff --git a/src/libutil/regex-combinators.hh b/src/libutil/regex-combinators.hh index 87d6aa678..37962944e 100644 --- a/src/libutil/regex-combinators.hh +++ b/src/libutil/regex-combinators.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include "strings.hh" + #include namespace nix::regex { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index d4a14bbf0..e46c5624a 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -3,6 +3,7 @@ #include +#include "strings.hh" #include "types.hh" #include "util.hh" #include "file-descriptor.hh" diff --git a/src/libutil/shlex.cc b/src/libutil/shlex.cc index b5f340251..21fa0502a 100644 --- a/src/libutil/shlex.cc +++ b/src/libutil/shlex.cc @@ -1,5 +1,5 @@ #include "shlex.hh" -#include "util.hh" +#include "strings.hh" namespace nix { diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc new file mode 100644 index 000000000..9cb319cce --- /dev/null +++ b/src/libutil/strings.cc @@ -0,0 +1,232 @@ +#include "strings.hh" + +namespace nix { + +std::vector stringsToCharPtrs(const Strings & ss) +{ + std::vector res; + for (auto & s : ss) res.push_back((char *) s.c_str()); + res.push_back(0); + return res; +} + + +template C tokenizeString(std::string_view s, std::string_view separators) +{ + C result; + auto pos = s.find_first_not_of(separators, 0); + while (pos != std::string_view::npos) { + auto end = s.find_first_of(separators, pos + 1); + if (end == std::string_view::npos) end = s.size(); + result.insert(result.end(), std::string(s, pos, end - pos)); + pos = s.find_first_not_of(separators, end); + } + return result; +} + +template Strings tokenizeString(std::string_view s, std::string_view separators); +template StringSet tokenizeString(std::string_view s, std::string_view separators); +template std::vector tokenizeString(std::string_view s, std::string_view separators); + + +std::string chomp(std::string_view s) +{ + size_t i = s.find_last_not_of(" \n\r\t"); + return i == std::string_view::npos ? "" : std::string(s, 0, i + 1); +} + + +std::string trim(std::string_view s, std::string_view whitespace) +{ + auto i = s.find_first_not_of(whitespace); + if (i == s.npos) return ""; + auto j = s.find_last_not_of(whitespace); + return std::string(s, i, j == s.npos ? j : j - i + 1); +} + + +std::string replaceStrings( + std::string res, + std::string_view from, + std::string_view to) +{ + if (from.empty()) return res; + size_t pos = 0; + while ((pos = res.find(from, pos)) != std::string::npos) { + res.replace(pos, from.size(), to); + pos += to.size(); + } + return res; +} + + +Rewriter::Rewriter(std::map rewrites) + : rewrites(std::move(rewrites)) +{ + for (const auto & [k, v] : this->rewrites) { + assert(!k.empty()); + initials.push_back(k[0]); + } + std::ranges::sort(initials); + auto [firstDupe, end] = std::ranges::unique(initials); + initials.erase(firstDupe, end); +} + +std::string Rewriter::operator()(std::string s) +{ + size_t j = 0; + while ((j = s.find_first_of(initials, j)) != std::string::npos) { + size_t skip = 1; + for (auto & [from, to] : rewrites) { + if (s.compare(j, from.size(), from) == 0) { + s.replace(j, from.size(), to); + skip = to.size(); + break; + } + } + j += skip; + } + return s; +} + + +std::string toLower(const std::string & s) +{ + std::string r(s); + for (auto & c : r) + c = std::tolower(c); + return r; +} + + +std::string shellEscape(const std::string_view s) +{ + std::string r; + r.reserve(s.size() + 2); + r += "'"; + for (auto & i : s) + if (i == '\'') r += "'\\''"; else r += i; + r += '\''; + return r; +} + +constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +std::string base64Encode(std::string_view s) +{ + std::string res; + res.reserve((s.size() + 2) / 3 * 4); + int data = 0, nbits = 0; + + for (char c : s) { + data = data << 8 | (unsigned char) c; + nbits += 8; + while (nbits >= 6) { + nbits -= 6; + res.push_back(base64Chars[data >> nbits & 0x3f]); + } + } + + if (nbits) res.push_back(base64Chars[data << (6 - nbits) & 0x3f]); + while (res.size() % 4) res.push_back('='); + + return res; +} + + +std::string base64Decode(std::string_view s) +{ + constexpr char npos = -1; + constexpr std::array base64DecodeChars = [&]() { + std::array result{}; + for (auto& c : result) + c = npos; + for (int i = 0; i < 64; i++) + result[base64Chars[i]] = i; + return result; + }(); + + std::string res; + // Some sequences are missing the padding consisting of up to two '='. + // vvv + res.reserve((s.size() + 2) / 4 * 3); + unsigned int d = 0, bits = 0; + + for (char c : s) { + if (c == '=') break; + if (c == '\n') continue; + + char digit = base64DecodeChars[(unsigned char) c]; + if (digit == npos) + throw Error("invalid character in Base64 string: '%c'", c); + + bits += 6; + d = d << 6 | digit; + if (bits >= 8) { + res.push_back(d >> (bits - 8) & 0xff); + bits -= 8; + } + } + + return res; +} + + +std::string stripIndentation(std::string_view s) +{ + size_t minIndent = 10000; + size_t curIndent = 0; + bool atStartOfLine = true; + + for (auto & c : s) { + if (atStartOfLine && c == ' ') + curIndent++; + else if (c == '\n') { + if (atStartOfLine) + minIndent = std::max(minIndent, curIndent); + curIndent = 0; + atStartOfLine = true; + } else { + if (atStartOfLine) { + minIndent = std::min(minIndent, curIndent); + atStartOfLine = false; + } + } + } + + std::string res; + + size_t pos = 0; + while (pos < s.size()) { + auto eol = s.find('\n', pos); + if (eol == s.npos) eol = s.size(); + if (eol - pos > minIndent) + res.append(s.substr(pos + minIndent, eol - pos - minIndent)); + res.push_back('\n'); + pos = eol + 1; + } + + return res; +} + + +std::pair getLine(std::string_view s) +{ + auto newline = s.find('\n'); + + if (newline == s.npos) { + return {s, ""}; + } else { + auto line = s.substr(0, newline); + if (!line.empty() && line[line.size() - 1] == '\r') + line = line.substr(0, line.size() - 1); + return {line, s.substr(newline + 1)}; + } +} + +std::string showBytes(uint64_t bytes) +{ + return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); +} + +} diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh new file mode 100644 index 000000000..daeb5be50 --- /dev/null +++ b/src/libutil/strings.hh @@ -0,0 +1,256 @@ +#pragma once +///@file + +#include "error.hh" +#include "types.hh" + +#include +#include + +namespace nix { + +/** + * Tree formatting. + */ +constexpr char treeConn[] = "├───"; +constexpr char treeLast[] = "└───"; +constexpr char treeLine[] = "│ "; +constexpr char treeNull[] = " "; + +/** + * Convert a list of strings to a null-terminated vector of `char + * *`s. The result must not be accessed beyond the lifetime of the + * list of strings. + */ +std::vector stringsToCharPtrs(const Strings & ss); + + +MakeError(FormatError, Error); + + +/** + * String tokenizer. + */ +template C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); + + +/** + * Concatenate the given strings with a separator between the + * elements. + */ +template +std::string concatStringsSep(const std::string_view sep, const C & ss) +{ + size_t size = 0; + // need a cast to string_view since this is also called with Symbols + for (const auto & s : ss) size += sep.size() + std::string_view(s).size(); + std::string s; + s.reserve(size); + for (auto & i : ss) { + if (s.size() != 0) s += sep; + s += i; + } + return s; +} + +template +auto concatStrings(Parts && ... parts) + -> std::enable_if_t<(... && std::is_convertible_v), std::string> +{ + std::string_view views[sizeof...(parts)] = { parts... }; + return concatStringsSep({}, views); +} + + +/** + * Add quotes around a collection of strings. + */ +template Strings quoteStrings(const C & c) +{ + Strings res; + for (auto & s : c) + res.push_back("'" + s + "'"); + return res; +} + +/** + * Remove trailing whitespace from a string. + * + * \todo return std::string_view. + */ +std::string chomp(std::string_view s); + + +/** + * Remove whitespace from the start and end of a string. + */ +std::string trim(std::string_view s, std::string_view whitespace = " \n\r\t"); + + +/** + * Replace all occurrences of a string inside another string. + */ +std::string replaceStrings( + std::string s, + std::string_view from, + std::string_view to); + + +/** + * Rewrites a string given a map of replacements, applying the replacements in + * sorted order, only once, considering only the strings appearing in the input + * string in performing replacement. + * + * - Replacements are not performed on intermediate strings. That is, for an input + * `"abb"` with replacements `{"ab" -> "ba"}`, the result is `"bab"`. + * - Transitive replacements are not performed. For example, for the input `"abcde"` + * with replacements `{"a" -> "b", "b" -> "c", "e" -> "b"}`, the result is + * `"bccdb"`. + */ +class Rewriter +{ +private: + std::string initials; + std::map rewrites; + +public: + explicit Rewriter(std::map rewrites); + + std::string operator()(std::string s); +}; + +inline std::string rewriteStrings(std::string s, const StringMap & rewrites) +{ + return Rewriter(rewrites)(s); +} + + + +/** + * Parse a string into an integer. + */ +template +std::optional string2Int(const std::string_view s) +{ + if (s.substr(0, 1) == "-" && !std::numeric_limits::is_signed) + return std::nullopt; + try { + return boost::lexical_cast(s.data(), s.size()); + } catch (const boost::bad_lexical_cast &) { + return std::nullopt; + } +} + +/** + * Like string2Int(), but support an optional suffix 'K', 'M', 'G' or + * 'T' denoting a binary unit prefix. + */ +template +N string2IntWithUnitPrefix(std::string_view s) +{ + N multiplier = 1; + if (!s.empty()) { + char u = std::toupper(*s.rbegin()); + if (std::isalpha(u)) { + if (u == 'K') multiplier = 1ULL << 10; + else if (u == 'M') multiplier = 1ULL << 20; + else if (u == 'G') multiplier = 1ULL << 30; + else if (u == 'T') multiplier = 1ULL << 40; + else throw UsageError("invalid unit specifier '%1%'", u); + s.remove_suffix(1); + } + } + if (auto n = string2Int(s)) + return *n * multiplier; + throw UsageError("'%s' is not an integer", s); +} + +/** + * Parse a string into a float. + */ +template +std::optional string2Float(const std::string_view s) +{ + try { + return boost::lexical_cast(s.data(), s.size()); + } catch (const boost::bad_lexical_cast &) { + return std::nullopt; + } +} + + +/** + * Convert a little-endian integer to host order. + */ +template +T readLittleEndian(unsigned char * p) +{ + T x = 0; + for (size_t i = 0; i < sizeof(x); ++i, ++p) { + x |= ((T) *p) << (i * 8); + } + return x; +} + +/** + * Convert a string to lower case. + */ +std::string toLower(const std::string & s); + + +/** + * Escape a string as a shell word. + */ +std::string shellEscape(const std::string_view s); + +/** + * Base64 encoding/decoding. + */ +std::string base64Encode(std::string_view s); +std::string base64Decode(std::string_view s); + + +/** + * Remove common leading whitespace from the lines in the string + * 's'. For example, if every line is indented by at least 3 spaces, + * then we remove 3 spaces from the start of every line. + */ +std::string stripIndentation(std::string_view s); + +/** + * Get the prefix of 's' up to and excluding the next line break (LF + * optionally preceded by CR), and the remainder following the line + * break. + */ +std::pair getLine(std::string_view s); + +std::string showBytes(uint64_t bytes); + + +/** + * Provide an addition operator between strings and string_views + * inexplicably omitted from the standard library. + */ +inline std::string operator + (const std::string & s1, std::string_view s2) +{ + auto s = s1; + s.append(s2); + return s; +} + +inline std::string operator + (std::string && s, std::string_view s2) +{ + s.append(s2); + return std::move(s); +} + +inline std::string operator + (std::string_view s1, const char * s2) +{ + std::string s; + s.reserve(s1.size() + strlen(s2)); + s.append(s1); + s.append(s2); + return s; +} + +} diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc index 9fefcbe1c..a9a2a415a 100644 --- a/src/libutil/unix-domain-socket.cc +++ b/src/libutil/unix-domain-socket.cc @@ -1,7 +1,7 @@ #include "file-system.hh" #include "processes.hh" #include "unix-domain-socket.hh" -#include "util.hh" +#include "strings.hh" #include #include diff --git a/src/libutil/url.cc b/src/libutil/url.cc index afccc4245..46688cef5 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -1,7 +1,7 @@ #include "url.hh" #include "url-parts.hh" -#include "util.hh" #include "split.hh" +#include "strings.hh" namespace nix { diff --git a/src/libutil/util.cc b/src/libutil/util.cc index f580ef038..099a07622 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,5 +1,6 @@ #include "util.hh" #include "processes.hh" +#include "strings.hh" #include "current-process.hh" #include "sync.hh" @@ -164,116 +165,6 @@ Path createNixStateDir() -std::vector stringsToCharPtrs(const Strings & ss) -{ - std::vector res; - for (auto & s : ss) res.push_back((char *) s.c_str()); - res.push_back(0); - return res; -} - - -////////////////////////////////////////////////////////////////////// - - -template C tokenizeString(std::string_view s, std::string_view separators) -{ - C result; - auto pos = s.find_first_not_of(separators, 0); - while (pos != std::string_view::npos) { - auto end = s.find_first_of(separators, pos + 1); - if (end == std::string_view::npos) end = s.size(); - result.insert(result.end(), std::string(s, pos, end - pos)); - pos = s.find_first_not_of(separators, end); - } - return result; -} - -template Strings tokenizeString(std::string_view s, std::string_view separators); -template StringSet tokenizeString(std::string_view s, std::string_view separators); -template std::vector tokenizeString(std::string_view s, std::string_view separators); - - -std::string chomp(std::string_view s) -{ - size_t i = s.find_last_not_of(" \n\r\t"); - return i == std::string_view::npos ? "" : std::string(s, 0, i + 1); -} - - -std::string trim(std::string_view s, std::string_view whitespace) -{ - auto i = s.find_first_not_of(whitespace); - if (i == s.npos) return ""; - auto j = s.find_last_not_of(whitespace); - return std::string(s, i, j == s.npos ? j : j - i + 1); -} - - -std::string replaceStrings( - std::string res, - std::string_view from, - std::string_view to) -{ - if (from.empty()) return res; - size_t pos = 0; - while ((pos = res.find(from, pos)) != std::string::npos) { - res.replace(pos, from.size(), to); - pos += to.size(); - } - return res; -} - - -Rewriter::Rewriter(std::map rewrites) - : rewrites(std::move(rewrites)) -{ - for (const auto & [k, v] : this->rewrites) { - assert(!k.empty()); - initials.push_back(k[0]); - } - std::ranges::sort(initials); - auto [firstDupe, end] = std::ranges::unique(initials); - initials.erase(firstDupe, end); -} - -std::string Rewriter::operator()(std::string s) -{ - size_t j = 0; - while ((j = s.find_first_of(initials, j)) != std::string::npos) { - size_t skip = 1; - for (auto & [from, to] : rewrites) { - if (s.compare(j, from.size(), from) == 0) { - s.replace(j, from.size(), to); - skip = to.size(); - break; - } - } - j += skip; - } - return s; -} - - -std::string toLower(const std::string & s) -{ - std::string r(s); - for (auto & c : r) - c = std::tolower(c); - return r; -} - - -std::string shellEscape(const std::string_view s) -{ - std::string r; - r.reserve(s.size() + 2); - r += "'"; - for (auto & i : s) - if (i == '\'') r += "'\\''"; else r += i; - r += '\''; - return r; -} void ignoreException(Verbosity lvl) @@ -289,120 +180,6 @@ void ignoreException(Verbosity lvl) } catch (...) { } } -constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - -std::string base64Encode(std::string_view s) -{ - std::string res; - res.reserve((s.size() + 2) / 3 * 4); - int data = 0, nbits = 0; - - for (char c : s) { - data = data << 8 | (unsigned char) c; - nbits += 8; - while (nbits >= 6) { - nbits -= 6; - res.push_back(base64Chars[data >> nbits & 0x3f]); - } - } - - if (nbits) res.push_back(base64Chars[data << (6 - nbits) & 0x3f]); - while (res.size() % 4) res.push_back('='); - - return res; -} - - -std::string base64Decode(std::string_view s) -{ - constexpr char npos = -1; - constexpr std::array base64DecodeChars = [&]() { - std::array result{}; - for (auto& c : result) - c = npos; - for (int i = 0; i < 64; i++) - result[base64Chars[i]] = i; - return result; - }(); - - std::string res; - // Some sequences are missing the padding consisting of up to two '='. - // vvv - res.reserve((s.size() + 2) / 4 * 3); - unsigned int d = 0, bits = 0; - - for (char c : s) { - if (c == '=') break; - if (c == '\n') continue; - - char digit = base64DecodeChars[(unsigned char) c]; - if (digit == npos) - throw Error("invalid character in Base64 string: '%c'", c); - - bits += 6; - d = d << 6 | digit; - if (bits >= 8) { - res.push_back(d >> (bits - 8) & 0xff); - bits -= 8; - } - } - - return res; -} - - -std::string stripIndentation(std::string_view s) -{ - size_t minIndent = 10000; - size_t curIndent = 0; - bool atStartOfLine = true; - - for (auto & c : s) { - if (atStartOfLine && c == ' ') - curIndent++; - else if (c == '\n') { - if (atStartOfLine) - minIndent = std::max(minIndent, curIndent); - curIndent = 0; - atStartOfLine = true; - } else { - if (atStartOfLine) { - minIndent = std::min(minIndent, curIndent); - atStartOfLine = false; - } - } - } - - std::string res; - - size_t pos = 0; - while (pos < s.size()) { - auto eol = s.find('\n', pos); - if (eol == s.npos) eol = s.size(); - if (eol - pos > minIndent) - res.append(s.substr(pos + minIndent, eol - pos - minIndent)); - res.push_back('\n'); - pos = eol + 1; - } - - return res; -} - - -std::pair getLine(std::string_view s) -{ - auto newline = s.find('\n'); - - if (newline == s.npos) { - return {s, ""}; - } else { - auto line = s.substr(0, newline); - if (!line.empty() && line[line.size() - 1] == '\r') - line = line.substr(0, line.size() - 1); - return {line, s.substr(newline + 1)}; - } -} - ////////////////////////////////////////////////////////////////////// @@ -460,9 +237,4 @@ void unshareFilesystem() } -std::string showBytes(uint64_t bytes) -{ - return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); -} - } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 216693635..907bdf4ed 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -106,193 +106,6 @@ void restoreMountNamespace(); void unshareFilesystem(); -/** - * Convert a list of strings to a null-terminated vector of `char - * *`s. The result must not be accessed beyond the lifetime of the - * list of strings. - */ -std::vector stringsToCharPtrs(const Strings & ss); - - -MakeError(FormatError, Error); - - -/** - * String tokenizer. - */ -template C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); - - -/** - * Concatenate the given strings with a separator between the - * elements. - */ -template -std::string concatStringsSep(const std::string_view sep, const C & ss) -{ - size_t size = 0; - // need a cast to string_view since this is also called with Symbols - for (const auto & s : ss) size += sep.size() + std::string_view(s).size(); - std::string s; - s.reserve(size); - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - -template -auto concatStrings(Parts && ... parts) - -> std::enable_if_t<(... && std::is_convertible_v), std::string> -{ - std::string_view views[sizeof...(parts)] = { parts... }; - return concatStringsSep({}, views); -} - - -/** - * Add quotes around a collection of strings. - */ -template Strings quoteStrings(const C & c) -{ - Strings res; - for (auto & s : c) - res.push_back("'" + s + "'"); - return res; -} - -/** - * Remove trailing whitespace from a string. - * - * \todo return std::string_view. - */ -std::string chomp(std::string_view s); - - -/** - * Remove whitespace from the start and end of a string. - */ -std::string trim(std::string_view s, std::string_view whitespace = " \n\r\t"); - - -/** - * Replace all occurrences of a string inside another string. - */ -std::string replaceStrings( - std::string s, - std::string_view from, - std::string_view to); - - -/** - * Rewrites a string given a map of replacements, applying the replacements in - * sorted order, only once, considering only the strings appearing in the input - * string in performing replacement. - * - * - Replacements are not performed on intermediate strings. That is, for an input - * `"abb"` with replacements `{"ab" -> "ba"}`, the result is `"bab"`. - * - Transitive replacements are not performed. For example, for the input `"abcde"` - * with replacements `{"a" -> "b", "b" -> "c", "e" -> "b"}`, the result is - * `"bccdb"`. - */ -class Rewriter -{ -private: - std::string initials; - std::map rewrites; - -public: - explicit Rewriter(std::map rewrites); - - std::string operator()(std::string s); -}; - -inline std::string rewriteStrings(std::string s, const StringMap & rewrites) -{ - return Rewriter(rewrites)(s); -} - - - -/** - * Parse a string into an integer. - */ -template -std::optional string2Int(const std::string_view s) -{ - if (s.substr(0, 1) == "-" && !std::numeric_limits::is_signed) - return std::nullopt; - try { - return boost::lexical_cast(s.data(), s.size()); - } catch (const boost::bad_lexical_cast &) { - return std::nullopt; - } -} - -/** - * Like string2Int(), but support an optional suffix 'K', 'M', 'G' or - * 'T' denoting a binary unit prefix. - */ -template -N string2IntWithUnitPrefix(std::string_view s) -{ - N multiplier = 1; - if (!s.empty()) { - char u = std::toupper(*s.rbegin()); - if (std::isalpha(u)) { - if (u == 'K') multiplier = 1ULL << 10; - else if (u == 'M') multiplier = 1ULL << 20; - else if (u == 'G') multiplier = 1ULL << 30; - else if (u == 'T') multiplier = 1ULL << 40; - else throw UsageError("invalid unit specifier '%1%'", u); - s.remove_suffix(1); - } - } - if (auto n = string2Int(s)) - return *n * multiplier; - throw UsageError("'%s' is not an integer", s); -} - -/** - * Parse a string into a float. - */ -template -std::optional string2Float(const std::string_view s) -{ - try { - return boost::lexical_cast(s.data(), s.size()); - } catch (const boost::bad_lexical_cast &) { - return std::nullopt; - } -} - - -/** - * Convert a little-endian integer to host order. - */ -template -T readLittleEndian(unsigned char * p) -{ - T x = 0; - for (size_t i = 0; i < sizeof(x); ++i, ++p) { - x |= ((T) *p) << (i * 8); - } - return x; -} - -/** - * Convert a string to lower case. - */ -std::string toLower(const std::string & s); - - -/** - * Escape a string as a shell word. - */ -std::string shellEscape(const std::string_view s); - - /** * Exception handling in destructors: print an error message, then * ignore the exception. @@ -301,38 +114,6 @@ void ignoreException(Verbosity lvl = lvlError); -/** - * Tree formatting. - */ -constexpr char treeConn[] = "├───"; -constexpr char treeLast[] = "└───"; -constexpr char treeLine[] = "│ "; -constexpr char treeNull[] = " "; - - -/** - * Base64 encoding/decoding. - */ -std::string base64Encode(std::string_view s); -std::string base64Decode(std::string_view s); - - -/** - * Remove common leading whitespace from the lines in the string - * 's'. For example, if every line is indented by at least 3 spaces, - * then we remove 3 spaces from the start of every line. - */ -std::string stripIndentation(std::string_view s); - - -/** - * Get the prefix of 's' up to and excluding the next line break (LF - * optionally preceded by CR), and the remainder following the line - * break. - */ -std::pair getLine(std::string_view s); - - /** * Get a value for the specified key from an associate container. */ @@ -443,33 +224,4 @@ template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -std::string showBytes(uint64_t bytes); - - -/** - * Provide an addition operator between strings and string_views - * inexplicably omitted from the standard library. - */ -inline std::string operator + (const std::string & s1, std::string_view s2) -{ - auto s = s1; - s.append(s2); - return s; -} - -inline std::string operator + (std::string && s, std::string_view s2) -{ - s.append(s2); - return std::move(s); -} - -inline std::string operator + (std::string_view s1, const char * s2) -{ - std::string s; - s.reserve(s1.size() + strlen(s2)); - s.append(s1); - s.append(s2); - return s; -} - } diff --git a/tests/functional/repl_characterization/repl_characterization.cc b/tests/functional/repl_characterization/repl_characterization.cc index 0d3e5352d..4cc1e7e37 100644 --- a/tests/functional/repl_characterization/repl_characterization.cc +++ b/tests/functional/repl_characterization/repl_characterization.cc @@ -12,6 +12,7 @@ #include "tests/cli-literate-parser.hh" #include "tests/terminal-code-eater.hh" #include "util.hh" +#include "strings.hh" using namespace std::string_literals; diff --git a/tests/functional/repl_characterization/test-session.cc b/tests/functional/repl_characterization/test-session.cc index ab0710b24..be3d407f3 100644 --- a/tests/functional/repl_characterization/test-session.cc +++ b/tests/functional/repl_characterization/test-session.cc @@ -3,9 +3,9 @@ #include #include "test-session.hh" -#include "util.hh" #include "escape-char.hh" #include "processes.hh" +#include "strings.hh" namespace nix { diff --git a/tests/unit/libstore/machines.cc b/tests/unit/libstore/machines.cc index dc192e5be..ba27d85b7 100644 --- a/tests/unit/libstore/machines.cc +++ b/tests/unit/libstore/machines.cc @@ -1,6 +1,7 @@ #include "file-system.hh" #include "machines.hh" #include "globals.hh" +#include "strings.hh" #include "tests/test-data.hh" #include diff --git a/tests/unit/libutil-support/tests/cli-literate-parser.cc b/tests/unit/libutil-support/tests/cli-literate-parser.cc index 822f5e276..69bf9fb46 100644 --- a/tests/unit/libutil-support/tests/cli-literate-parser.cc +++ b/tests/unit/libutil-support/tests/cli-literate-parser.cc @@ -19,6 +19,7 @@ #include "shlex.hh" #include "types.hh" #include "util.hh" +#include "strings.hh" static constexpr const bool DEBUG_PARSER = false; diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index a42afd997..324ebd335 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -1,8 +1,9 @@ #include "file-system.hh" -#include "util.hh" #include "processes.hh" +#include "strings.hh" #include "types.hh" #include "terminal.hh" +#include "util.hh" #include From f79ee66646f66e5117583bcf9a579b7f538ca8bb Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 15:40:03 +0200 Subject: [PATCH 11/22] util.{hh,cc}: Split out users.{hh,cc} Change-Id: I1bd92479a2cb7e5c2c2e1541b80474adb05ea0df --- src/libcmd/repl.cc | 1 + src/libexpr/eval-cache.cc | 2 +- src/libexpr/eval-settings.cc | 3 +- src/libexpr/flake/config.cc | 2 +- src/libexpr/parser.y | 2 +- src/libfetchers/cache.cc | 1 + src/libfetchers/git.cc | 2 +- src/libfetchers/mercurial.cc | 1 + src/libfetchers/registry.cc | 2 +- src/libstore/globals.cc | 2 +- src/libstore/nar-info-disk-cache.cc | 1 + src/libstore/profiles.cc | 2 +- src/libstore/store-api.cc | 1 + src/libutil/file-system.cc | 1 + src/libutil/meson.build | 2 + src/libutil/users.cc | 105 ++++++++++++++++++++++++++++ src/libutil/users.hh | 61 ++++++++++++++++ src/libutil/util.cc | 97 ------------------------- src/libutil/util.hh | 47 ------------- src/nix-channel/nix-channel.cc | 2 +- src/nix-env/nix-env.cc | 2 +- 21 files changed, 185 insertions(+), 154 deletions(-) create mode 100644 src/libutil/users.cc create mode 100644 src/libutil/users.hh diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 02aa5a272..49865aa90 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -31,6 +31,7 @@ #include "print.hh" #include "progress-bar.hh" #include "gc-small-vector.hh" +#include "users.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 5969ee449..90fbfa308 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -1,8 +1,8 @@ #include "eval-cache.hh" #include "sqlite.hh" #include "eval.hh" -#include "eval-inline.hh" #include "store-api.hh" +#include "users.hh" namespace nix::eval_cache { diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 046ca557d..105fd3e9d 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -1,6 +1,7 @@ +#include "file-system.hh" #include "globals.hh" #include "profiles.hh" -#include "eval.hh" +#include "users.hh" #include "eval-settings.hh" namespace nix { diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index b9613462a..35c605de2 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -1,5 +1,5 @@ #include "flake.hh" -#include "globals.hh" +#include "users.hh" #include "fetch-settings.hh" #include diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 5424d8741..91cc7d089 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -19,7 +19,7 @@ #include #include "finally.hh" -#include "util.hh" +#include "users.hh" #include "nixexpr.hh" #include "eval.hh" diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 672e1e0bc..eec77c2dc 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -2,6 +2,7 @@ #include "sqlite.hh" #include "sync.hh" #include "store-api.hh" +#include "users.hh" #include diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 86d9d437f..07cbc781c 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -6,7 +6,7 @@ #include "store-api.hh" #include "url-parts.hh" #include "pathlocks.hh" -#include "util.hh" +#include "users.hh" #include "git.hh" #include "logging.hh" #include "finally.hh" diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 086db498f..4fffa71d3 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -3,6 +3,7 @@ #include "processes.hh" #include "store-api.hh" #include "url-parts.hh" +#include "users.hh" #include "fetch-settings.hh" diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 4b2d61f52..96e9ed1c6 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,6 +1,6 @@ #include "registry.hh" #include "fetchers.hh" -#include "util.hh" +#include "users.hh" #include "globals.hh" #include "store-api.hh" #include "local-fs-store.hh" diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 30fd4e9e3..8ef25c469 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,6 +1,6 @@ #include "environment-variables.hh" #include "globals.hh" -#include "util.hh" +#include "users.hh" #include "archive.hh" #include "args.hh" #include "abstract-setting-to-json.hh" diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index c7176d30f..0413abbf1 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -2,6 +2,7 @@ #include "sync.hh" #include "sqlite.hh" #include "globals.hh" +#include "users.hh" #include #include diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 239047dd6..e8b88693d 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -1,7 +1,7 @@ #include "profiles.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "util.hh" +#include "users.hh" #include #include diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index ed3566f5e..c5631dfd8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -14,6 +14,7 @@ // FIXME this should not be here, see TODO below on // `addMultipleToStore`. #include "worker-protocol.hh" +#include "users.hh" #include #include diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 9804c449c..721bf97e2 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -9,6 +9,7 @@ #include "serialise.hh" #include "signals.hh" #include "types.hh" +#include "users.hh" namespace fs = std::filesystem; diff --git a/src/libutil/meson.build b/src/libutil/meson.build index b662ea455..5e767b447 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -39,6 +39,7 @@ libutil_sources = files( 'unix-domain-socket.cc', 'url.cc', 'url-name.cc', + 'users.cc', 'util.cc', 'xml-writer.cc', ) @@ -109,6 +110,7 @@ libutil_headers = files( 'url-parts.hh', 'url-name.hh', 'url.hh', + 'users.hh', 'util.hh', 'variant-wrapper.hh', 'xml-writer.hh', diff --git a/src/libutil/users.cc b/src/libutil/users.cc new file mode 100644 index 000000000..a9a8a7353 --- /dev/null +++ b/src/libutil/users.cc @@ -0,0 +1,105 @@ +#include "environment-variables.hh" +#include "file-system.hh" +#include "logging.hh" +#include "strings.hh" + +#include +#include +#include + +namespace nix { + +std::string getUserName() +{ + auto pw = getpwuid(geteuid()); + std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); + if (name.empty()) + throw Error("cannot figure out user name"); + return name; +} + +Path getHomeOf(uid_t userId) +{ + std::vector buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + return pw->pw_dir; +} + +Path getHome() +{ + static Path homeDir = []() + { + std::optional unownedUserHomeDir = {}; + auto homeDir = getEnv("HOME"); + if (homeDir) { + // Only use $HOME if doesn't exist or is owned by the current user. + struct stat st; + int result = stat(homeDir->c_str(), &st); + if (result != 0) { + if (errno != ENOENT) { + warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); + homeDir.reset(); + } + } else if (st.st_uid != geteuid()) { + unownedUserHomeDir.swap(homeDir); + } + } + if (!homeDir) { + homeDir = getHomeOf(geteuid()); + if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { + warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); + } + } + return *homeDir; + }(); + return homeDir; +} + + +Path getCacheDir() +{ + auto cacheDir = getEnv("XDG_CACHE_HOME"); + return cacheDir ? *cacheDir : getHome() + "/.cache"; +} + + +Path getConfigDir() +{ + auto configDir = getEnv("XDG_CONFIG_HOME"); + return configDir ? *configDir : getHome() + "/.config"; +} + +std::vector getConfigDirs() +{ + Path configHome = getConfigDir(); + auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); + std::vector result = tokenizeString>(configDirs, ":"); + result.insert(result.begin(), configHome); + return result; +} + + +Path getDataDir() +{ + auto dataDir = getEnv("XDG_DATA_HOME"); + return dataDir ? *dataDir : getHome() + "/.local/share"; +} + +Path getStateDir() +{ + auto stateDir = getEnv("XDG_STATE_HOME"); + return stateDir ? *stateDir : getHome() + "/.local/state"; +} + +Path createNixStateDir() +{ + Path dir = getStateDir() + "/nix"; + createDirs(dir); + return dir; +} + +} diff --git a/src/libutil/users.hh b/src/libutil/users.hh new file mode 100644 index 000000000..3add4c732 --- /dev/null +++ b/src/libutil/users.hh @@ -0,0 +1,61 @@ +#pragma once +///@file + +#include "types.hh" + +#include + +namespace nix { + +std::string getUserName(); + +/** + * @return the given user's home directory from /etc/passwd. + */ +Path getHomeOf(uid_t userId); + +/** + * @return $HOME or the user's home directory from /etc/passwd. + */ +Path getHome(); + +/** + * @return $XDG_CACHE_HOME or $HOME/.cache. + */ +Path getCacheDir(); + +/** + * @return $XDG_CONFIG_HOME or $HOME/.config. + */ +Path getConfigDir(); + +/** + * @return the directories to search for user configuration files + */ +std::vector getConfigDirs(); + +/** + * @return $XDG_DATA_HOME or $HOME/.local/share. + */ +Path getDataDir(); + +/** + * @return $XDG_STATE_HOME or $HOME/.local/state. + * + * @note Not to be confused with settings.nixStateDir. + */ +Path getStateDir(); + +/** + * Create $XDG_STATE_HOME/nix or $HOME/.local/state/nix, and return + * the path to it. + * @note Not to be confused with settings.nixStateDir. + */ +Path createNixStateDir(); + +/** + * Perform tilde expansion on a path. + */ +std::string expandTilde(std::string_view path); + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 099a07622..8e813abc2 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -56,103 +56,6 @@ namespace nix { -std::string getUserName() -{ - auto pw = getpwuid(geteuid()); - std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); - if (name.empty()) - throw Error("cannot figure out user name"); - return name; -} - -Path getHomeOf(uid_t userId) -{ - std::vector buf(16384); - struct passwd pwbuf; - struct passwd * pw; - if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 - || !pw || !pw->pw_dir || !pw->pw_dir[0]) - throw Error("cannot determine user's home directory"); - return pw->pw_dir; -} - -Path getHome() -{ - static Path homeDir = []() - { - std::optional unownedUserHomeDir = {}; - auto homeDir = getEnv("HOME"); - if (homeDir) { - // Only use $HOME if doesn't exist or is owned by the current user. - struct stat st; - int result = stat(homeDir->c_str(), &st); - if (result != 0) { - if (errno != ENOENT) { - warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); - homeDir.reset(); - } - } else if (st.st_uid != geteuid()) { - unownedUserHomeDir.swap(homeDir); - } - } - if (!homeDir) { - homeDir = getHomeOf(geteuid()); - if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { - warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); - } - } - return *homeDir; - }(); - return homeDir; -} - - -Path getCacheDir() -{ - auto cacheDir = getEnv("XDG_CACHE_HOME"); - return cacheDir ? *cacheDir : getHome() + "/.cache"; -} - - -Path getConfigDir() -{ - auto configDir = getEnv("XDG_CONFIG_HOME"); - return configDir ? *configDir : getHome() + "/.config"; -} - -std::vector getConfigDirs() -{ - Path configHome = getConfigDir(); - auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); - std::vector result = tokenizeString>(configDirs, ":"); - result.insert(result.begin(), configHome); - return result; -} - - -Path getDataDir() -{ - auto dataDir = getEnv("XDG_DATA_HOME"); - return dataDir ? *dataDir : getHome() + "/.local/share"; -} - -Path getStateDir() -{ - auto stateDir = getEnv("XDG_STATE_HOME"); - return stateDir ? *stateDir : getHome() + "/.local/state"; -} - -Path createNixStateDir() -{ - Path dir = getStateDir() + "/nix"; - createDirs(dir); - return dir; -} - - - - - ////////////////////////////////////////////////////////////////////// diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 907bdf4ed..e408821b9 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -39,53 +39,6 @@ struct Source; extern const std::string nativeSystem; -std::string getUserName(); - -/** - * @return the given user's home directory from /etc/passwd. - */ -Path getHomeOf(uid_t userId); - -/** - * @return $HOME or the user's home directory from /etc/passwd. - */ -Path getHome(); - -/** - * @return $XDG_CACHE_HOME or $HOME/.cache. - */ -Path getCacheDir(); - -/** - * @return $XDG_CONFIG_HOME or $HOME/.config. - */ -Path getConfigDir(); - -/** - * @return the directories to search for user configuration files - */ -std::vector getConfigDirs(); - -/** - * @return $XDG_DATA_HOME or $HOME/.local/share. - */ -Path getDataDir(); - -/** - * @return $XDG_STATE_HOME or $HOME/.local/state. - * - * @note Not to be confused with settings.nixStateDir. - */ -Path getStateDir(); - -/** - * Create $XDG_STATE_HOME/nix or $HOME/.local/state/nix, and return - * the path to it. - * @note Not to be confused with settings.nixStateDir. - */ -Path createNixStateDir(); - - /** * Save the current mount namespace. Ignored if called more than * once. diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 26003f021..971337b63 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -6,7 +6,7 @@ #include "legacy.hh" #include "fetchers.hh" #include "eval-settings.hh" // for defexpr -#include "util.hh" +#include "users.hh" #include #include diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index ad255a1e1..227a3331c 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -11,7 +11,7 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "user-env.hh" -#include "util.hh" +#include "users.hh" #include "value-to-json.hh" #include "xml-writer.hh" #include "legacy.hh" From 8b6d2d39155e88250c576571a1251769b926ee83 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 16:01:11 +0200 Subject: [PATCH 12/22] util.{hh,cc}: Split out namespaces.{hh,cc} Change-Id: I8fd3f3b50c15ede29d489066b4e8d99c2c4636a6 --- src/libstore/filetransfer.cc | 2 +- src/libutil/current-process.cc | 1 + src/libutil/namespaces.cc | 66 +++++++++++++++++++++++++++++++--- src/libutil/namespaces.hh | 20 +++++++++++ src/libutil/util.cc | 52 --------------------------- src/libutil/util.hh | 20 ----------- src/nix/main.cc | 1 + 7 files changed, 85 insertions(+), 77 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 492463a61..dc656f444 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1,5 +1,5 @@ #include "filetransfer.hh" -#include "util.hh" +#include "namespaces.hh" #include "globals.hh" #include "store-api.hh" #include "s3.hh" diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 41f591b0c..f9a08685f 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -1,6 +1,7 @@ #include "current-process.hh" #include "file-system.hh" #include "logging.hh" +#include "namespaces.hh" #include "signals.hh" #include "util.hh" #include "strings.hh" diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index d092e6fcc..98d3cd306 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -1,5 +1,4 @@ -#if __linux__ - +#include "file-descriptor.hh" #include "file-system.hh" #include "logging.hh" #include "namespaces.hh" @@ -8,8 +7,67 @@ #include +#if __linux__ +# include +# include +#endif + namespace nix { +#if __linux__ +static AutoCloseFD fdSavedMountNamespace; +static AutoCloseFD fdSavedRoot; +#endif + +void saveMountNamespace() +{ +#if __linux__ + static std::once_flag done; + std::call_once(done, []() { + fdSavedMountNamespace = AutoCloseFD{open("/proc/self/ns/mnt", O_RDONLY)}; + if (!fdSavedMountNamespace) + throw SysError("saving parent mount namespace"); + + fdSavedRoot = AutoCloseFD{open("/proc/self/root", O_RDONLY)}; + }); +#endif +} + +void restoreMountNamespace() +{ +#if __linux__ + try { + auto savedCwd = absPath("."); + + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + throw SysError("restoring parent mount namespace"); + + if (fdSavedRoot) { + if (fchdir(fdSavedRoot.get())) + throw SysError("chdir into saved root"); + if (chroot(".")) + throw SysError("chroot into saved root"); + } + + if (chdir(savedCwd.c_str()) == -1) + throw SysError("restoring cwd"); + } catch (Error & e) { + debug(e.msg()); + } +#endif +} + +void unshareFilesystem() +{ +#ifdef __linux__ + if (unshare(CLONE_FS) != 0 && errno != EPERM) + throw SysError("unsharing filesystem state in download thread"); +#endif +} + + +#if __linux__ + static void diagnoseUserNamespaces() { if (!pathExists("/proc/self/ns/user")) { @@ -95,6 +153,6 @@ bool mountAndPidNamespacesSupported() return res; } -} - #endif + +} diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh index 0b7eeb66c..3a920e665 100644 --- a/src/libutil/namespaces.hh +++ b/src/libutil/namespaces.hh @@ -3,6 +3,26 @@ namespace nix { +/** + * Save the current mount namespace. Ignored if called more than + * once. + */ +void saveMountNamespace(); + +/** + * Restore the mount namespace saved by saveMountNamespace(). Ignored + * if saveMountNamespace() was never called. + */ +void restoreMountNamespace(); + +/** + * Cause this thread to not share any FS attributes with the main + * thread, because this causes setns() in restoreMountNamespace() to + * fail. + */ +void unshareFilesystem(); + + #if __linux__ bool userNamespacesSupported(); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 8e813abc2..6d6e55ad1 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -88,56 +88,4 @@ void ignoreException(Verbosity lvl) -#if __linux__ -static AutoCloseFD fdSavedMountNamespace; -static AutoCloseFD fdSavedRoot; -#endif - -void saveMountNamespace() -{ -#if __linux__ - static std::once_flag done; - std::call_once(done, []() { - fdSavedMountNamespace = AutoCloseFD{open("/proc/self/ns/mnt", O_RDONLY)}; - if (!fdSavedMountNamespace) - throw SysError("saving parent mount namespace"); - - fdSavedRoot = AutoCloseFD{open("/proc/self/root", O_RDONLY)}; - }); -#endif -} - -void restoreMountNamespace() -{ -#if __linux__ - try { - auto savedCwd = absPath("."); - - if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) - throw SysError("restoring parent mount namespace"); - - if (fdSavedRoot) { - if (fchdir(fdSavedRoot.get())) - throw SysError("chdir into saved root"); - if (chroot(".")) - throw SysError("chroot into saved root"); - } - - if (chdir(savedCwd.c_str()) == -1) - throw SysError("restoring cwd"); - } catch (Error & e) { - debug(e.msg()); - } -#endif -} - -void unshareFilesystem() -{ -#ifdef __linux__ - if (unshare(CLONE_FS) != 0 && errno != EPERM) - throw SysError("unsharing filesystem state in download thread"); -#endif -} - - } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index e408821b9..867f0a80d 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -39,26 +39,6 @@ struct Source; extern const std::string nativeSystem; -/** - * Save the current mount namespace. Ignored if called more than - * once. - */ -void saveMountNamespace(); - -/** - * Restore the mount namespace saved by saveMountNamespace(). Ignored - * if saveMountNamespace() was never called. - */ -void restoreMountNamespace(); - -/** - * Cause this thread to not share any FS attributes with the main - * thread, because this causes setns() in restoreMountNamespace() to - * fail. - */ -void unshareFilesystem(); - - /** * Exception handling in destructors: print an error message, then * ignore the exception. diff --git a/src/nix/main.cc b/src/nix/main.cc index 0f98861d5..5d92cfd76 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -8,6 +8,7 @@ #include "eval-settings.hh" #include "globals.hh" #include "legacy.hh" +#include "namespaces.hh" #include "shared.hh" #include "store-api.hh" #include "filetransfer.hh" From 93ebb3e7dfb75ce23c1fb46d8e70208a2bad0c02 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 16:09:33 +0200 Subject: [PATCH 13/22] util.{hh,cc}: Move ignoreException to error.{hh,cc} Change-Id: Iae6464217a55c313a983e5c651b26a4a1e446706 --- src/libutil/current-process.cc | 2 +- src/libutil/error.cc | 13 +++++++++++++ src/libutil/error.hh | 6 ++++++ src/libutil/util.cc | 12 ------------ src/libutil/util.hh | 7 ------- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index f9a08685f..c64dd1e0d 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -1,9 +1,9 @@ #include "current-process.hh" +#include "error.hh" #include "file-system.hh" #include "logging.hh" #include "namespaces.hh" #include "signals.hh" -#include "util.hh" #include "strings.hh" #ifdef __APPLE__ diff --git a/src/libutil/error.cc b/src/libutil/error.cc index f780aabef..1a294bf08 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -417,4 +417,17 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s return out; } +void ignoreException(Verbosity lvl) +{ + /* Make sure no exceptions leave this function. + printError() also throws when remote is closed. */ + try { + try { + throw; + } catch (std::exception & e) { + printMsg(lvl, "error (ignored): %1%", e.what()); + } + } catch (...) { } +} + } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 323365d65..0884f9f32 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -202,4 +202,10 @@ public: } }; +/** + * Exception handling in destructors: print an error message, then + * ignore the exception. + */ +void ignoreException(Verbosity lvl = lvlError); + } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 6d6e55ad1..d9bd71b5f 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -70,18 +70,6 @@ namespace nix { -void ignoreException(Verbosity lvl) -{ - /* Make sure no exceptions leave this function. - printError() also throws when remote is closed. */ - try { - try { - throw; - } catch (std::exception & e) { - printMsg(lvl, "error (ignored): %1%", e.what()); - } - } catch (...) { } -} ////////////////////////////////////////////////////////////////////// diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 867f0a80d..c5ca12650 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -39,13 +39,6 @@ struct Source; extern const std::string nativeSystem; -/** - * Exception handling in destructors: print an error message, then - * ignore the exception. - */ -void ignoreException(Verbosity lvl = lvlError); - - /** * Get a value for the specified key from an associate container. From 74513483bc5572d988a059b8e964662d66f1667f Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 16:20:49 +0200 Subject: [PATCH 14/22] util.cc: Delete remaining file Change-Id: I2b47848904f2ce7bd78b83738e99a4c9da627751 --- src/libutil/meson.build | 1 - src/libutil/util.cc | 79 ----------------------------------------- 2 files changed, 80 deletions(-) delete mode 100644 src/libutil/util.cc diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 5e767b447..64d84c714 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -40,7 +40,6 @@ libutil_sources = files( 'url.cc', 'url-name.cc', 'users.cc', - 'util.cc', 'xml-writer.cc', ) diff --git a/src/libutil/util.cc b/src/libutil/util.cc deleted file mode 100644 index d9bd71b5f..000000000 --- a/src/libutil/util.cc +++ /dev/null @@ -1,79 +0,0 @@ -#include "util.hh" -#include "processes.hh" -#include "strings.hh" -#include "current-process.hh" - -#include "sync.hh" -#include "finally.hh" -#include "serialise.hh" -#include "cgroup.hh" -#include "signals.hh" -#include "environment-variables.hh" -#include "file-system.hh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#include -#endif - -#ifdef __linux__ -#include -#include -#include - -#include -#endif - - -#ifdef NDEBUG -#error "Lix may not be built with assertions disabled (i.e. with -DNDEBUG)." -#endif - -namespace nix { - - -////////////////////////////////////////////////////////////////////// - - -////////////////////////////////////////////////////////////////////// - - - - -////////////////////////////////////////////////////////////////////// - - - - - - - -////////////////////////////////////////////////////////////////////// - - - -} From d73c40ff3d74e9b1bac24c104fdf7738df3c3029 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 16:24:19 +0200 Subject: [PATCH 15/22] util.hh: Move stuff to types.hh Change-Id: Ia852306a4b8aac6856dc42bc69e4b58b53a0d67c --- src/libutil/types.hh | 111 +++++++++++++++++++++++++++++++++++++++++++ src/libutil/util.hh | 108 ----------------------------------------- 2 files changed, 111 insertions(+), 108 deletions(-) diff --git a/src/libutil/types.hh b/src/libutil/types.hh index c86f52175..4974634d8 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -4,6 +4,7 @@ #include "ref.hh" #include +#include #include #include #include @@ -51,6 +52,116 @@ struct Explicit { } }; +/** + * Get a value for the specified key from an associate container. + */ +template +const typename T::mapped_type * get(const T & map, const typename T::key_type & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &i->second; +} + +template +typename T::mapped_type * get(T & map, const typename T::key_type & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &i->second; +} + +/** + * Get a value for the specified key from an associate container, or a default value if the key isn't present. + */ +template +const typename T::mapped_type & getOr(T & map, + const typename T::key_type & key, + const typename T::mapped_type & defaultValue) +{ + auto i = map.find(key); + if (i == map.end()) return defaultValue; + return i->second; +} + +/** + * Remove and return the first item from a container. + */ +template +std::optional remove_begin(T & c) +{ + auto i = c.begin(); + if (i == c.end()) return {}; + auto v = std::move(*i); + c.erase(i); + return v; +} + + +/** + * Remove and return the first item from a container. + */ +template +std::optional pop(T & c) +{ + if (c.empty()) return {}; + auto v = std::move(c.front()); + c.pop(); + return v; +} + + +/** + * A RAII helper that increments a counter on construction and + * decrements it on destruction. + */ +template +struct MaintainCount +{ + T & counter; + long delta; + MaintainCount(T & counter, long delta = 1) : counter(counter), delta(delta) { counter += delta; } + ~MaintainCount() { counter -= delta; } +}; + + +/** + * A Rust/Python-like enumerate() iterator adapter. + * + * Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. + */ +template ())), + typename = decltype(std::end(std::declval()))> +constexpr auto enumerate(T && iterable) +{ + struct iterator + { + size_t i; + TIter iter; + constexpr bool operator != (const iterator & other) const { return iter != other.iter; } + constexpr void operator ++ () { ++i; ++iter; } + constexpr auto operator * () const { return std::tie(i, *iter); } + }; + + struct iterable_wrapper + { + T iterable; + constexpr auto begin() { return iterator{ 0, std::begin(iterable) }; } + constexpr auto end() { return iterator{ 0, std::end(iterable) }; } + }; + + return iterable_wrapper{ std::forward(iterable) }; +} + + +/** + * C++17 std::visit boilerplate + */ +template struct overloaded : Ts... { using Ts::operator()...; }; +template overloaded(Ts...) -> overloaded; + + /** * This wants to be a little bit like rust's Cow type. diff --git a/src/libutil/util.hh b/src/libutil/util.hh index c5ca12650..4e20550a7 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -40,114 +40,6 @@ extern const std::string nativeSystem; -/** - * Get a value for the specified key from an associate container. - */ -template -const typename T::mapped_type * get(const T & map, const typename T::key_type & key) -{ - auto i = map.find(key); - if (i == map.end()) return nullptr; - return &i->second; -} - -template -typename T::mapped_type * get(T & map, const typename T::key_type & key) -{ - auto i = map.find(key); - if (i == map.end()) return nullptr; - return &i->second; -} - -/** - * Get a value for the specified key from an associate container, or a default value if the key isn't present. - */ -template -const typename T::mapped_type & getOr(T & map, - const typename T::key_type & key, - const typename T::mapped_type & defaultValue) -{ - auto i = map.find(key); - if (i == map.end()) return defaultValue; - return i->second; -} - -/** - * Remove and return the first item from a container. - */ -template -std::optional remove_begin(T & c) -{ - auto i = c.begin(); - if (i == c.end()) return {}; - auto v = std::move(*i); - c.erase(i); - return v; -} - - -/** - * Remove and return the first item from a container. - */ -template -std::optional pop(T & c) -{ - if (c.empty()) return {}; - auto v = std::move(c.front()); - c.pop(); - return v; -} - - -/** - * A RAII helper that increments a counter on construction and - * decrements it on destruction. - */ -template -struct MaintainCount -{ - T & counter; - long delta; - MaintainCount(T & counter, long delta = 1) : counter(counter), delta(delta) { counter += delta; } - ~MaintainCount() { counter -= delta; } -}; - - -/** - * A Rust/Python-like enumerate() iterator adapter. - * - * Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. - */ -template ())), - typename = decltype(std::end(std::declval()))> -constexpr auto enumerate(T && iterable) -{ - struct iterator - { - size_t i; - TIter iter; - constexpr bool operator != (const iterator & other) const { return iter != other.iter; } - constexpr void operator ++ () { ++i; ++iter; } - constexpr auto operator * () const { return std::tie(i, *iter); } - }; - - struct iterable_wrapper - { - T iterable; - constexpr auto begin() { return iterator{ 0, std::begin(iterable) }; } - constexpr auto end() { return iterator{ 0, std::end(iterable) }; } - }; - - return iterable_wrapper{ std::forward(iterable) }; -} - - -/** - * C++17 std::visit boilerplate - */ -template struct overloaded : Ts... { using Ts::operator()...; }; -template overloaded(Ts...) -> overloaded; } From f0eb650ee8a0eb5244c15fab3b152a1f70e1506e Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 16:27:48 +0200 Subject: [PATCH 16/22] util.hh: Move nativeSystem to local-derivation-goal.cc Change-Id: I74565fbfd3aeedef8f50465808fac712b84e47ad --- src/libstore/build/local-derivation-goal.cc | 5 +++++ src/libutil/error.cc | 2 -- src/libutil/util.hh | 5 ----- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index fd871e8a4..479b4ffeb 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -64,6 +64,11 @@ extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, namespace nix { +/** + * The system for which Nix is compiled. + */ +constexpr std::string_view nativeSystem = SYSTEM; + void handleDiffHook( uid_t uid, uid_t gid, const Path & tryA, const Path & tryB, diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 1a294bf08..2a7fd3d0e 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -10,8 +10,6 @@ namespace nix { -const std::string nativeSystem = SYSTEM; - void BaseError::addTrace(std::shared_ptr && e, HintFmt hint) { err.traces.push_front(Trace { .pos = std::move(e), .hint = hint }); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 4e20550a7..1066f212a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -33,11 +33,6 @@ namespace nix { struct Sink; struct Source; -/** - * The system for which Nix is compiled. - */ -extern const std::string nativeSystem; - From a39ba22ff7112cd3984bbf28d8610d84dd525a0f Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 16:53:21 +0200 Subject: [PATCH 17/22] util.hh: Delete remaining file and clean up headers Change-Id: Ic1f68e6af658e94ef7922841dd3ad4c69551ef56 --- perl/lib/Nix/Store.xs | 1 - src/libcmd/common-eval-args.cc | 1 - src/libcmd/editor-for.cc | 1 - src/libcmd/installable-attr-path.cc | 1 - src/libcmd/installable-attr-path.hh | 1 - src/libcmd/installable-flake.cc | 1 - src/libcmd/installables.cc | 7 +--- src/libcmd/installables.hh | 1 - src/libcmd/markdown.cc | 2 +- src/libcmd/repl-interacter.cc | 7 ++-- src/libexpr/attr-path.cc | 1 - src/libexpr/eval.cc | 1 - src/libexpr/flake/config.cc | 1 + src/libexpr/get-drvs.cc | 1 - src/libexpr/nixexpr.cc | 1 - src/libexpr/print-ambiguous.cc | 3 +- src/libexpr/search-path.cc | 1 - src/libexpr/value-to-json.cc | 1 - src/libexpr/value-to-xml.cc | 4 -- src/libexpr/value/context.hh | 1 - src/libfetchers/fetch-settings.hh | 1 - src/libfetchers/fetch-to-store.hh | 1 - src/libmain/common-args.cc | 1 + src/libmain/loggers.cc | 1 - src/libmain/progress-bar.cc | 1 - src/libmain/shared.cc | 1 - src/libmain/shared.hh | 1 - src/libstore/build/derivation-goal.cc | 1 - src/libstore/common-protocol.cc | 3 -- src/libstore/crypto.hh | 2 +- src/libstore/derivations.cc | 1 - src/libstore/derived-path.hh | 2 +- src/libstore/filetransfer.cc | 1 - src/libstore/filetransfer.hh | 3 +- src/libstore/gc.cc | 6 +-- src/libstore/globals.cc | 5 ++- src/libstore/local-store.hh | 2 - src/libstore/nar-info-disk-cache.cc | 1 + src/libstore/optimise-store.cc | 1 - src/libstore/outputs-spec.cc | 1 - src/libstore/path-references.cc | 4 -- src/libstore/pathlocks.cc | 4 +- src/libstore/remote-store.cc | 1 - src/libstore/serve-protocol.cc | 2 - src/libstore/sqlite.cc | 3 +- src/libstore/store-api.cc | 2 - src/libstore/store-api.hh | 1 + src/libstore/worker-protocol.cc | 1 - src/libutil/archive.cc | 2 +- src/libutil/args.cc | 1 + src/libutil/args.hh | 8 +++- src/libutil/comparator.hh | 2 + src/libutil/compression.cc | 3 -- src/libutil/config-impl.hh | 2 + src/libutil/config.cc | 1 + src/libutil/error.cc | 2 +- src/libutil/experimental-features.cc | 1 - src/libutil/file-descriptor.cc | 1 + src/libutil/file-system.cc | 1 + src/libutil/hash.cc | 3 +- src/libutil/logging.cc | 3 -- src/libutil/meson.build | 1 - src/libutil/references.cc | 4 +- src/libutil/serialise.cc | 1 - src/libutil/serialise.hh | 1 - src/libutil/signals.cc | 1 - src/libutil/signals.hh | 2 - src/libutil/split.hh | 2 - src/libutil/suggestions.cc | 1 + src/libutil/suggestions.hh | 5 ++- src/libutil/tarfile.cc | 1 + src/libutil/thread-pool.cc | 1 + src/libutil/thread-pool.hh | 3 +- src/libutil/url-name.cc | 1 - src/libutil/url-name.hh | 3 -- src/libutil/util.hh | 40 ------------------- src/nix-build/nix-build.cc | 1 - src/nix-env/user-env.cc | 1 - src/nix-instantiate/nix-instantiate.cc | 1 - src/nix-store/dotgraph.cc | 1 - src/nix-store/graphml.cc | 1 - src/nix-store/nix-store.cc | 1 - src/nix/daemon.cc | 1 - src/nix/develop.cc | 1 - src/nix/doctor.cc | 1 - .../repl_characterization.cc | 3 -- tests/unit/libcmd/args.cc | 1 - .../unit/libstore-support/tests/test-data.hh | 2 +- .../libutil-support/tests/characterization.hh | 1 - .../tests/cli-literate-parser.cc | 2 - tests/unit/libutil/logging.cc | 1 - tests/unit/libutil/tests.cc | 3 +- 92 files changed, 50 insertions(+), 161 deletions(-) delete mode 100644 src/libutil/util.hh diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index e96885e4c..4bef020d3 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -11,7 +11,6 @@ #include "derivations.hh" #include "globals.hh" #include "store-api.hh" -#include "util.hh" #include "crypto.hh" #include diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 9beea5aa2..e86213020 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -2,7 +2,6 @@ #include "common-eval-args.hh" #include "shared.hh" #include "filetransfer.hh" -#include "util.hh" #include "eval.hh" #include "fetchers.hh" #include "registry.hh" diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc index a56a7065e..67653d9c9 100644 --- a/src/libcmd/editor-for.cc +++ b/src/libcmd/editor-for.cc @@ -1,4 +1,3 @@ -#include "util.hh" #include "editor-for.hh" #include "environment-variables.hh" #include "source-path.hh" diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index 06e507872..eb15fecc3 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -1,7 +1,6 @@ #include "globals.hh" #include "installable-attr-path.hh" #include "outputs-spec.hh" -#include "util.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" diff --git a/src/libcmd/installable-attr-path.hh b/src/libcmd/installable-attr-path.hh index e9f0c33da..86c2f8219 100644 --- a/src/libcmd/installable-attr-path.hh +++ b/src/libcmd/installable-attr-path.hh @@ -4,7 +4,6 @@ #include "globals.hh" #include "installable-value.hh" #include "outputs-spec.hh" -#include "util.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 46bdd411b..b1ebd339e 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -2,7 +2,6 @@ #include "installable-flake.hh" #include "installable-derived-path.hh" #include "outputs-spec.hh" -#include "util.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 2c18653e4..ab0e4fd1c 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -3,26 +3,21 @@ #include "installable-derived-path.hh" #include "installable-attr-path.hh" #include "installable-flake.hh" +#include "logging.hh" #include "outputs-spec.hh" -#include "util.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" #include "derivations.hh" -#include "eval-inline.hh" #include "eval.hh" #include "eval-settings.hh" -#include "get-drvs.hh" #include "store-api.hh" #include "shared.hh" #include "flake/flake.hh" #include "eval-cache.hh" -#include "url.hh" #include "registry.hh" #include "build-result.hh" -#include -#include #include diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index ad5b4f759..95e8841ca 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "util.hh" #include "path.hh" #include "outputs-spec.hh" #include "derived-path.hh" diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 8b3bbc1b5..dbaab8c19 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,5 +1,5 @@ #include "markdown.hh" -#include "util.hh" +#include "error.hh" #include "finally.hh" #include "terminal.hh" diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 41589cda1..0cf4e34b8 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -1,3 +1,7 @@ +#include "error.hh" +#include "file-system.hh" +#include "logging.hh" +#include #include #include #include @@ -18,11 +22,8 @@ extern "C" { } #endif -#include "signals.hh" #include "finally.hh" #include "repl-interacter.hh" -#include "util.hh" -#include "repl.hh" namespace nix { diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index f4da99a9a..3e60f295d 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -1,6 +1,5 @@ #include "attr-path.hh" #include "eval-inline.hh" -#include "util.hh" namespace nix { diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index a8b37325b..c9a624eeb 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -5,7 +5,6 @@ #include "print-options.hh" #include "shared.hh" #include "types.hh" -#include "util.hh" #include "store-api.hh" #include "derivations.hh" #include "downstream-placeholder.hh" diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index 35c605de2..b330d96f9 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -1,4 +1,5 @@ #include "flake.hh" +#include "logging.hh" #include "users.hh" #include "fetch-settings.hh" diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index e686ffe8c..8c8e142b8 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -1,5 +1,4 @@ #include "get-drvs.hh" -#include "util.hh" #include "eval-inline.hh" #include "derivations.hh" #include "store-api.hh" diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 72a8764e6..6a1aa8f35 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -2,7 +2,6 @@ #include "derivations.hh" #include "eval.hh" #include "symbol-table.hh" -#include "util.hh" #include "print.hh" #include "escape-string.hh" diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc index ec30f5073..bcf86b5c6 100644 --- a/src/libexpr/print-ambiguous.cc +++ b/src/libexpr/print-ambiguous.cc @@ -1,6 +1,7 @@ #include "print-ambiguous.hh" +#include "attr-set.hh" +#include "logging.hh" #include "print.hh" -#include "eval.hh" #include "signals.hh" #include "escape-string.hh" diff --git a/src/libexpr/search-path.cc b/src/libexpr/search-path.cc index 180d5f8b1..a25767496 100644 --- a/src/libexpr/search-path.cc +++ b/src/libexpr/search-path.cc @@ -1,5 +1,4 @@ #include "search-path.hh" -#include "util.hh" namespace nix { diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index ebb3379a8..5743d9057 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -1,6 +1,5 @@ #include "value-to-json.hh" #include "eval-inline.hh" -#include "util.hh" #include "signals.hh" #include "store-api.hh" diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 5d1fbd28d..fda360eff 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -1,11 +1,7 @@ #include "value-to-xml.hh" #include "xml-writer.hh" -#include "eval-inline.hh" -#include "util.hh" #include "signals.hh" -#include - namespace nix { diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 998b70e36..7f23cd3a4 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "util.hh" #include "comparator.hh" #include "derived-path.hh" #include "variant-wrapper.hh" diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index c67a75082..2dc2834fb 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -3,7 +3,6 @@ #include "types.hh" #include "config.hh" -#include "util.hh" #include #include diff --git a/src/libfetchers/fetch-to-store.hh b/src/libfetchers/fetch-to-store.hh index 717450944..b74879637 100644 --- a/src/libfetchers/fetch-to-store.hh +++ b/src/libfetchers/fetch-to-store.hh @@ -3,7 +3,6 @@ #include "source-path.hh" #include "store-api.hh" -#include "util.hh" #include "repair-flag.hh" #include "content-address.hh" diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 12ce289c5..af7f46296 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -2,6 +2,7 @@ #include "args/root.hh" #include "globals.hh" #include "loggers.hh" +#include "logging.hh" namespace nix { diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index 7e80a31f1..80080d616 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -1,7 +1,6 @@ #include "environment-variables.hh" #include "loggers.hh" #include "progress-bar.hh" -#include "util.hh" namespace nix { diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index f3700f103..11b2fe800 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -1,5 +1,4 @@ #include "progress-bar.hh" -#include "util.hh" #include "sync.hh" #include "store-api.hh" #include "names.hh" diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 377cd6abd..f99777a20 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -2,7 +2,6 @@ #include "shared.hh" #include "store-api.hh" #include "gc-store.hh" -#include "util.hh" #include "signals.hh" #include "loggers.hh" #include "progress-bar.hh" diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 907f336f1..b791980fa 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "util.hh" #include "args.hh" #include "args/root.hh" #include "common-args.hh" diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5fa5deb7c..97ba994ad 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -5,7 +5,6 @@ #include "builtins/buildenv.hh" #include "references.hh" #include "finally.hh" -#include "util.hh" #include "archive.hh" #include "compression.hh" #include "common-protocol.hh" diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index f906814bc..456ad2b1f 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -1,11 +1,8 @@ #include "serialise.hh" -#include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" -#include "build-result.hh" #include "common-protocol.hh" #include "common-protocol-impl.hh" -#include "archive.hh" #include "derivations.hh" #include diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh index 35216d470..22588cf97 100644 --- a/src/libstore/crypto.hh +++ b/src/libstore/crypto.hh @@ -1,9 +1,9 @@ #pragma once ///@file -#include "types.hh" #include +#include namespace nix { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index fef680421..4e70804e5 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -3,7 +3,6 @@ #include "store-api.hh" #include "globals.hh" #include "types.hh" -#include "util.hh" #include "split.hh" #include "common-protocol.hh" #include "common-protocol-impl.hh" diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 4d7033df2..c87cf2004 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "util.hh" +#include "config.hh" #include "path.hh" #include "outputs-spec.hh" #include "comparator.hh" diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index dc656f444..cc1656a17 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 3f55995ef..c692f1485 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -1,8 +1,9 @@ #pragma once ///@file +#include "logging.hh" +#include "serialise.hh" #include "types.hh" -#include "hash.hh" #include "config.hh" #include diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 722452b8d..d58e3c8eb 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,18 +1,14 @@ -#include "derivations.hh" #include "globals.hh" #include "local-store.hh" +#include "pathlocks.hh" #include "processes.hh" #include "signals.hh" #include "finally.hh" #include "unix-domain-socket.hh" -#include #include -#include #include -#include -#include #include #include #include diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 8ef25c469..993a8cfa5 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,14 +1,15 @@ #include "environment-variables.hh" #include "globals.hh" +#include "file-system.hh" +#include "logging.hh" +#include "strings.hh" #include "users.hh" -#include "archive.hh" #include "args.hh" #include "abstract-setting-to-json.hh" #include "compute-levels.hh" #include "current-process.hh" #include -#include #include #include #include diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 14f024ca9..808fdc202 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -3,11 +3,9 @@ #include "sqlite.hh" -#include "pathlocks.hh" #include "store-api.hh" #include "indirect-root-store.hh" #include "sync.hh" -#include "util.hh" #include #include diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 0413abbf1..169b63819 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -1,4 +1,5 @@ #include "nar-info-disk-cache.hh" +#include "logging.hh" #include "sync.hh" #include "sqlite.hh" #include "globals.hh" diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index e09072e90..4f02296c3 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -1,4 +1,3 @@ -#include "util.hh" #include "local-store.hh" #include "globals.hh" #include "signals.hh" diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc index 4422bcd21..0d740c324 100644 --- a/src/libstore/outputs-spec.cc +++ b/src/libstore/outputs-spec.cc @@ -5,7 +5,6 @@ #include "outputs-spec.hh" #include "path-regex.hh" #include "strings.hh" -#include "util.hh" namespace nix { diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc index 33cf66ce3..8a02e9006 100644 --- a/src/libstore/path-references.cc +++ b/src/libstore/path-references.cc @@ -1,12 +1,8 @@ #include "path-references.hh" #include "hash.hh" -#include "util.hh" #include "archive.hh" #include -#include -#include -#include namespace nix { diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 3e654c1c9..ced0f30bb 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -1,10 +1,8 @@ #include "pathlocks.hh" -#include "util.hh" +#include "logging.hh" #include "signals.hh" -#include "sync.hh" #include -#include #include #include diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 22d87b027..93b1afabd 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,5 +1,4 @@ #include "serialise.hh" -#include "util.hh" #include "signals.hh" #include "path-with-outputs.hh" #include "gc-store.hh" diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index 2e15d28d5..603137c81 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -1,11 +1,9 @@ #include "serialise.hh" -#include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" #include "build-result.hh" #include "serve-protocol.hh" #include "serve-protocol-impl.hh" -#include "archive.hh" #include "path-info.hh" #include diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 4bd425b46..f40217734 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -1,12 +1,11 @@ #include "sqlite.hh" #include "globals.hh" -#include "util.hh" +#include "logging.hh" #include "signals.hh" #include "url.hh" #include -#include namespace nix { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c5631dfd8..eb9b5c3ef 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,9 +1,7 @@ -#include "crypto.hh" #include "fs-accessor.hh" #include "globals.hh" #include "derivations.hh" #include "store-api.hh" -#include "util.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" #include "url.hh" diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 745fce594..397ebe759 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "logging.hh" #include "nar-info.hh" #include "realisation.hh" #include "path.hh" diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index ad94c79ee..d76a8d2b8 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -1,5 +1,4 @@ #include "serialise.hh" -#include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" #include "build-result.hh" diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index c3f39d4b3..1b42ee4b5 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -13,8 +13,8 @@ #include "archive.hh" #include "file-system.hh" -#include "util.hh" #include "config.hh" +#include "logging.hh" #include "signals.hh" namespace nix { diff --git a/src/libutil/args.cc b/src/libutil/args.cc index bcff653c5..4983e49af 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -5,6 +5,7 @@ #include "environment-variables.hh" #include "experimental-features-json.hh" +#include "logging.hh" #include diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 77f7ff2a8..35a5238c0 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -1,13 +1,17 @@ #pragma once ///@file -#include +#include "experimental-features.hh" +#include "types.hh" +#include #include #include +#include #include +#include +#include -#include "util.hh" namespace nix { diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index cbc2bb4fd..136a3f742 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + #define DECLARE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE) \ PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const; #define DECLARE_EQUAL(prefix, qualification, my_type) \ diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 575a03712..678557a58 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,7 +1,5 @@ #include "compression.hh" #include "tarfile.hh" -#include "util.hh" -#include "finally.hh" #include "signals.hh" #include "logging.hh" @@ -13,7 +11,6 @@ #include #include -#include namespace nix { diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index 9f69e8444..756175f95 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -12,7 +12,9 @@ * instantiation. */ +#include "args.hh" #include "config.hh" +#include "logging.hh" namespace nix { diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 729b4e596..8180886ce 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -3,6 +3,7 @@ #include "abstract-setting-to-json.hh" #include "experimental-features.hh" #include "file-system.hh" +#include "logging.hh" #include "strings.hh" #include "config-impl.hh" diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 2a7fd3d0e..e5d6a9fa8 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,11 +1,11 @@ #include "environment-variables.hh" #include "error.hh" +#include "logging.hh" #include "position.hh" #include "terminal.hh" #include #include -#include "serialise.hh" #include namespace nix { diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 8ebec2956..f1cbfdb16 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -2,7 +2,6 @@ // Required for instances of to_json and from_json for ExperimentalFeature #include "experimental-features-json.hh" #include "strings.hh" -#include "util.hh" #include "nlohmann/json.hpp" diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc index ec22f17ab..a19ceaf2a 100644 --- a/src/libutil/file-descriptor.cc +++ b/src/libutil/file-descriptor.cc @@ -1,5 +1,6 @@ #include "file-system.hh" #include "finally.hh" +#include "logging.hh" #include "serialise.hh" #include "signals.hh" diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 721bf97e2..d573b22b4 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -6,6 +6,7 @@ #include "file-descriptor.hh" #include "file-system.hh" #include "finally.hh" +#include "logging.hh" #include "serialise.hh" #include "signals.hh" #include "types.hh" diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 2c36d9d94..006b5000c 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -1,4 +1,3 @@ -#include #include #include @@ -8,8 +7,8 @@ #include "args.hh" #include "hash.hh" #include "archive.hh" +#include "logging.hh" #include "split.hh" -#include "util.hh" #include #include diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 8d9e18d09..febbfdb55 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -1,15 +1,12 @@ #include "environment-variables.hh" #include "file-descriptor.hh" #include "logging.hh" -#include "util.hh" #include "config.hh" -#include "source-path.hh" #include "position.hh" #include "terminal.hh" #include #include -#include namespace nix { diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 64d84c714..96450fbe2 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -110,7 +110,6 @@ libutil_headers = files( 'url-name.hh', 'url.hh', 'users.hh', - 'util.hh', 'variant-wrapper.hh', 'xml-writer.hh', ) diff --git a/src/libutil/references.cc b/src/libutil/references.cc index 7f59b4c09..6189f69b9 100644 --- a/src/libutil/references.cc +++ b/src/libutil/references.cc @@ -1,9 +1,7 @@ #include "references.hh" #include "hash.hh" -#include "util.hh" -#include "archive.hh" +#include "logging.hh" -#include #include #include #include diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index a6cc919d2..3a8a01f16 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -1,5 +1,4 @@ #include "serialise.hh" -#include "util.hh" #include "signals.hh" #include diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index e46c5624a..c9294ba2d 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -5,7 +5,6 @@ #include "strings.hh" #include "types.hh" -#include "util.hh" #include "file-descriptor.hh" namespace boost::context { struct stack_context; } diff --git a/src/libutil/signals.cc b/src/libutil/signals.cc index c0e66f6ed..a94c2802a 100644 --- a/src/libutil/signals.cc +++ b/src/libutil/signals.cc @@ -1,5 +1,4 @@ #include "signals.hh" -#include "util.hh" #include "error.hh" #include "sync.hh" #include "terminal.hh" diff --git a/src/libutil/signals.hh b/src/libutil/signals.hh index 71593df95..02f8d2ca3 100644 --- a/src/libutil/signals.hh +++ b/src/libutil/signals.hh @@ -1,7 +1,6 @@ #pragma once /// @file -#include "types.hh" #include "error.hh" #include @@ -12,7 +11,6 @@ #include #include -#include namespace nix { diff --git a/src/libutil/split.hh b/src/libutil/split.hh index 4ff940eef..5455b6bff 100644 --- a/src/libutil/split.hh +++ b/src/libutil/split.hh @@ -4,8 +4,6 @@ #include #include -#include "util.hh" - namespace nix { /** diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc index 63dcf84b5..cae52cf29 100644 --- a/src/libutil/suggestions.cc +++ b/src/libutil/suggestions.cc @@ -4,6 +4,7 @@ #include #include +#include namespace nix { diff --git a/src/libutil/suggestions.hh b/src/libutil/suggestions.hh index 9abf5ee5f..3cac1371e 100644 --- a/src/libutil/suggestions.hh +++ b/src/libutil/suggestions.hh @@ -2,8 +2,11 @@ ///@file #include "comparator.hh" -#include "types.hh" + #include +#include +#include +#include namespace nix { diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index cabb7af9b..760a5a65a 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -2,6 +2,7 @@ #include #include "file-system.hh" +#include "logging.hh" #include "serialise.hh" #include "tarfile.hh" diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index a25769d9b..0ff83e997 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -1,4 +1,5 @@ #include "thread-pool.hh" +#include "logging.hh" #include "signals.hh" namespace nix { diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index 0e09fae97..3db7ce88f 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -1,13 +1,12 @@ #pragma once ///@file +#include "error.hh" #include "sync.hh" -#include "util.hh" #include #include #include -#include #include namespace nix { diff --git a/src/libutil/url-name.cc b/src/libutil/url-name.cc index 7c526752c..12c55db9a 100644 --- a/src/libutil/url-name.cc +++ b/src/libutil/url-name.cc @@ -1,4 +1,3 @@ -#include #include #include "url-name.hh" diff --git a/src/libutil/url-name.hh b/src/libutil/url-name.hh index 3a3f88e76..dabd23ca1 100644 --- a/src/libutil/url-name.hh +++ b/src/libutil/url-name.hh @@ -5,9 +5,6 @@ #include #include "url.hh" -#include "url-parts.hh" -#include "util.hh" -#include "split.hh" namespace nix { diff --git a/src/libutil/util.hh b/src/libutil/util.hh deleted file mode 100644 index 1066f212a..000000000 --- a/src/libutil/util.hh +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once -///@file - -#include "types.hh" -#include "error.hh" -#include "logging.hh" -#include "ansicolor.hh" - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include - -#ifndef HAVE_STRUCT_DIRENT_D_TYPE -#define DT_UNKNOWN 0 -#define DT_REG 1 -#define DT_LNK 2 -#define DT_DIR 3 -#endif - -namespace nix { - -struct Sink; -struct Source; - - - - - -} diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index f1c7bbfb8..31d4ac9ff 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -15,7 +15,6 @@ #include "globals.hh" #include "current-process.hh" #include "derivations.hh" -#include "util.hh" #include "shared.hh" #include "path-with-outputs.hh" #include "eval.hh" diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index f0131a458..530039ac6 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -1,5 +1,4 @@ #include "user-env.hh" -#include "util.hh" #include "derivations.hh" #include "store-api.hh" #include "path-with-outputs.hh" diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 3b99b73b4..c0b251ae4 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -7,7 +7,6 @@ #include "attr-path.hh" #include "value-to-xml.hh" #include "value-to-json.hh" -#include "util.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "common-eval-args.hh" diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 577cadceb..2c530999b 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -1,5 +1,4 @@ #include "dotgraph.hh" -#include "util.hh" #include "store-api.hh" #include diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 439557658..3e789a2d8 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -1,5 +1,4 @@ #include "graphml.hh" -#include "util.hh" #include "store-api.hh" #include "derivations.hh" diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 75d8b6c6d..b38e4f4f8 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -11,7 +11,6 @@ #include "serve-protocol.hh" #include "serve-protocol-impl.hh" #include "shared.hh" -#include "util.hh" #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 113e23bd1..f1cc1ee9c 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -5,7 +5,6 @@ #include "local-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" -#include "util.hh" #include "serialise.hh" #include "archive.hh" #include "globals.hh" diff --git a/src/nix/develop.cc b/src/nix/develop.cc index b5543447e..cd32bb20a 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -8,7 +8,6 @@ #include "derivations.hh" #include "progress-bar.hh" #include "run.hh" -#include "util.hh" #include #include diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index da7a1d7a0..4e1cfe8c0 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -6,7 +6,6 @@ #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "util.hh" #include "worker-protocol.hh" using namespace nix; diff --git a/tests/functional/repl_characterization/repl_characterization.cc b/tests/functional/repl_characterization/repl_characterization.cc index 4cc1e7e37..c91d6c1e3 100644 --- a/tests/functional/repl_characterization/repl_characterization.cc +++ b/tests/functional/repl_characterization/repl_characterization.cc @@ -6,12 +6,9 @@ #include #include -#include "escape-string.hh" #include "test-session.hh" #include "tests/characterization.hh" #include "tests/cli-literate-parser.hh" -#include "tests/terminal-code-eater.hh" -#include "util.hh" #include "strings.hh" using namespace std::string_literals; diff --git a/tests/unit/libcmd/args.cc b/tests/unit/libcmd/args.cc index 73550dacf..886467ea0 100644 --- a/tests/unit/libcmd/args.cc +++ b/tests/unit/libcmd/args.cc @@ -10,7 +10,6 @@ #include "filetransfer.hh" #include "shared.hh" #include "store-api.hh" -#include "util.hh" constexpr std::string_view INVALID_CHANNEL = "channel:example"; constexpr std::string_view CHANNEL_URL = "https://nixos.org/channels/example/nixexprs.tar.xz"; diff --git a/tests/unit/libstore-support/tests/test-data.hh b/tests/unit/libstore-support/tests/test-data.hh index 7aa1dd90d..1fec6f912 100644 --- a/tests/unit/libstore-support/tests/test-data.hh +++ b/tests/unit/libstore-support/tests/test-data.hh @@ -1,6 +1,6 @@ #pragma once -#include "util.hh" +#include "environment-variables.hh" #include "types.hh" namespace nix { diff --git a/tests/unit/libutil-support/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh index 472fdba3b..db82476da 100644 --- a/tests/unit/libutil-support/tests/characterization.hh +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -8,7 +8,6 @@ #include -#include "util.hh" #include "types.hh" namespace nix { diff --git a/tests/unit/libutil-support/tests/cli-literate-parser.cc b/tests/unit/libutil-support/tests/cli-literate-parser.cc index 69bf9fb46..f74fe85eb 100644 --- a/tests/unit/libutil-support/tests/cli-literate-parser.cc +++ b/tests/unit/libutil-support/tests/cli-literate-parser.cc @@ -3,7 +3,6 @@ #include "escape-char.hh" #include "libexpr/print.hh" #include "types.hh" -#include "util.hh" #include #include #include @@ -18,7 +17,6 @@ #include "libexpr/print.hh" #include "shlex.hh" #include "types.hh" -#include "util.hh" #include "strings.hh" static constexpr const bool DEBUG_PARSER = false; diff --git a/tests/unit/libutil/logging.cc b/tests/unit/libutil/logging.cc index 94cbe9c51..f99f49527 100644 --- a/tests/unit/libutil/logging.cc +++ b/tests/unit/libutil/logging.cc @@ -2,7 +2,6 @@ #include "logging.hh" #include "nixexpr.hh" -#include "util.hh" #include #include diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index 324ebd335..9a44ad59b 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -3,7 +3,6 @@ #include "strings.hh" #include "types.hh" #include "terminal.hh" -#include "util.hh" #include @@ -11,7 +10,7 @@ namespace nix { -/* ----------- tests for util.hh ------------------------------------------------*/ +/* ----------- tests for libutil ------------------------------------------------*/ /* ---------------------------------------------------------------------------- * absPath From dd4a2c17593283456884db18fa32ea01851fd2c5 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Wed, 29 May 2024 20:51:37 +0200 Subject: [PATCH 18/22] libstore: fix http abuses no longer working while refactoring the curl wrapper we inadvertently broken the immutable flake protocol, because the immutable flake protocol accumulates headers across the entire redirect chain instead of using only the headers given in the final response of the chain. this is a problem because Some Known Providers Of Flake Infrastructure set rel=immutable link headers only in the penultimate entry of the redirect chain, and curl does not regard it as worth returning to us via its response header enumeration mechanisms. fixes https://git.lix.systems/lix-project/lix/issues/358 Change-Id: I645c3932b465cde848bd6a3565925a1e3cbcdda0 --- src/libstore/filetransfer.cc | 70 +++++++++++------------------ tests/unit/libstore/filetransfer.cc | 53 +++++++++++++++++++--- 2 files changed, 73 insertions(+), 50 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 492463a61..076a5ca56 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -51,7 +51,6 @@ struct curlFileTransfer : public FileTransfer std::function dataCallback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object - bool headersProcessed = false; std::string statusMsg; unsigned int attempt = 0; @@ -136,35 +135,11 @@ struct curlFileTransfer : public FileTransfer std::exception_ptr writeException; - std::optional getHeader(const char * name) - { - curl_header * result; - auto e = curl_easy_header(req, name, 0, CURLH_HEADER, -1, &result); - if (e == CURLHE_OK) { - return result->value; - } else if (e == CURLHE_MISSING || e == CURLHE_NOHEADERS) { - return std::nullopt; - } else { - throw nix::Error("unexpected error from curl_easy_header(): %i", e); - } - } - size_t writeCallback(void * contents, size_t size, size_t nmemb) { const size_t realSize = size * nmemb; try { - if (!headersProcessed) { - if (auto h = getHeader("content-encoding")) { - encoding = std::move(*h); - } - if (auto h = getHeader("accept-ranges"); h && *h == "bytes") { - acceptRanges = true; - } - - headersProcessed = true; - } - result.bodySize += realSize; if (successfulStatuses.count(getHTTPStatus()) && this->dataCallback) { @@ -200,7 +175,31 @@ struct curlFileTransfer : public FileTransfer statusMsg = trim(match.str(1)); acceptRanges = false; encoding = ""; - headersProcessed = false; + } else { + auto i = line.find(':'); + if (i != std::string::npos) { + std::string name = toLower(trim(line.substr(0, i))); + + if (name == "etag") { + result.etag = trim(line.substr(i + 1)); + } + + else if (name == "content-encoding") + encoding = trim(line.substr(i + 1)); + + else if (name == "accept-ranges" && toLower(trim(line.substr(i + 1))) == "bytes") + acceptRanges = true; + + else if (name == "link" || name == "x-amz-meta-link") { + auto value = trim(line.substr(i + 1)); + static std::regex linkRegex("<([^>]*)>; rel=\"immutable\"", std::regex::extended | std::regex::icase); + if (std::smatch match; std::regex_match(value, match, linkRegex)) + result.immutableUrl = match.str(1); + else + debug("got invalid link header '%s'", value); + warn("foo %s", value); + } + } } return realSize; } @@ -336,25 +335,6 @@ struct curlFileTransfer : public FileTransfer debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", request.verb(), request.uri, code, httpStatus, result.bodySize); - auto link = getHeader("link"); - if (!link) { - link = getHeader("x-amz-meta-link"); - } - if (link) { - static std::regex linkRegex( - "<([^>]*)>; rel=\"immutable\"", std::regex::extended | std::regex::icase - ); - if (std::smatch match; std::regex_match(*link, match, linkRegex)) { - result.immutableUrl = match.str(1); - } else { - debug("got invalid link header '%s'", *link); - } - } - - if (auto etag = getHeader("etag")) { - result.etag = std::move(*etag); - } - // this has to happen here until we can return an actual future. // wrapping user `callback`s instead is not possible because the // Callback api expects std::functions, and copying Callbacks is diff --git a/tests/unit/libstore/filetransfer.cc b/tests/unit/libstore/filetransfer.cc index 684697c69..0e5c0965e 100644 --- a/tests/unit/libstore/filetransfer.cc +++ b/tests/unit/libstore/filetransfer.cc @@ -23,10 +23,19 @@ using namespace std::chrono_literals; +namespace { + +struct Reply { + std::string status, headers; + std::function content; +}; + +} + namespace nix { static std::tuple -serveHTTP(std::string_view status, std::string_view headers, std::function content) +serveHTTP(std::vector replies) { AutoCloseFD listener(::socket(AF_INET6, SOCK_STREAM, 0)); if (!listener) { @@ -52,7 +61,7 @@ serveHTTP(std::string_view status, std::string_view headers, std::function +serveHTTP(std::string status, std::string headers, std::function content) +{ + return serveHTTP({{{status, headers, content}}}); +} + TEST(FileTransfer, exceptionAbortsDownload) { struct Done @@ -166,4 +183,30 @@ TEST(FileTransfer, NOT_ON_DARWIN(handlesContentEncoding)) ft->download(FileTransferRequest(fmt("http://[::1]:%d/index", port)), sink); EXPECT_EQ(sink.s, original); } + +TEST(FileTransfer, usesIntermediateLinkHeaders) +{ + auto [port, srv] = serveHTTP({ + {"301 ok", + "location: /second\r\n" + "content-length: 0\r\n", + [] { return ""; }}, + {"307 ok", + "location: /third\r\n" + "content-length: 0\r\n", + [] { return ""; }}, + {"307 ok", + "location: /fourth\r\n" + "link: ; rel=\"immutable\"\r\n" + "content-length: 0\r\n", + [] { return ""; }}, + {"200 ok", "content-length: 1\r\n", [] { return "a"; }}, + }); + auto ft = makeFileTransfer(); + FileTransferRequest req(fmt("http://[::1]:%d/first", port)); + req.baseRetryTimeMs = 0; + auto result = ft->download(req); + ASSERT_EQ(result.immutableUrl, "http://foo"); +} + } From afeaa2371c346dd286ea45fcc21456e06b79bb1f Mon Sep 17 00:00:00 2001 From: Qyriad Date: Sat, 25 May 2024 12:19:46 -0600 Subject: [PATCH 19/22] docs: enable non-default TOC folding Folding by default would prevent things like "Ctrl+F for nix-env" from working trivially, but the user should be able to fold if they want to. Change-Id: I5273272289f0f24e1f040c691580acfe33f66bd4 --- doc/manual/book.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/book.toml b/doc/manual/book.toml index 1030f96bb..27c6c4637 100644 --- a/doc/manual/book.toml +++ b/doc/manual/book.toml @@ -11,6 +11,10 @@ additional-js = ["redirects.js"] # to just submit a Gerrit CL by the web for trivial stuff. edit-url-template = "https://github.com/lix-project/lix/tree/main/doc/manual/{path}" git-repository-url = "https://git.lix.systems/lix-project/lix" +# Folding by default would prevent things like "Ctrl+F for nix-env" from working +# trivially, but the user should be able to fold if they want to. +fold.enable = true +fold.level = 30 # Handles replacing @docroot@ with a path to ./src relative to that markdown file, # {{#include handlebars}}, and the @generated@ syntax used within these. it mostly From 68937f2b645ef5d0d34c2b8dba66619951b77c7a Mon Sep 17 00:00:00 2001 From: Qyriad Date: Wed, 29 May 2024 19:40:25 -0600 Subject: [PATCH 20/22] package: dequalify {host,build}Platform for convenience Change-Id: I5ffeac894a5bff101683cf3d566c63b478779962 --- package.nix | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/package.nix b/package.nix index 05503ae98..70db7eb22 100644 --- a/package.nix +++ b/package.nix @@ -85,6 +85,7 @@ let inherit (__forDefaults) canRunInstalled; inherit (lib) fileset; + inherit (stdenv) hostPlatform buildPlatform; version = lib.fileContents ./.version + versionSuffix; @@ -187,13 +188,13 @@ stdenv.mkDerivation (finalAttrs: { dontBuild = false; mesonFlags = - lib.optionals stdenv.hostPlatform.isLinux [ + lib.optionals hostPlatform.isLinux [ # You'd think meson could just find this in PATH, but busybox is in buildInputs, # which don't actually get added to PATH. And buildInputs is correct over # nativeBuildInputs since this should be a busybox executable on the host. "-Dsandbox-shell=${lib.getExe' busybox-sandbox-shell "busybox"}" ] - ++ lib.optional stdenv.hostPlatform.isStatic "-Denable-embedded-sandbox-shell=true" + ++ lib.optional hostPlatform.isStatic "-Denable-embedded-sandbox-shell=true" ++ lib.optional (finalAttrs.dontBuild) "-Denable-build=false" ++ [ # mesonConfigurePhase automatically passes -Dauto_features=enabled, @@ -203,7 +204,7 @@ stdenv.mkDerivation (finalAttrs: { (lib.mesonBool "enable-tests" finalAttrs.doCheck) (lib.mesonBool "enable-docs" canRunInstalled) ] - ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "--cross-file=${mesonCrossFile}"; + ++ lib.optional (hostPlatform != buildPlatform) "--cross-file=${mesonCrossFile}"; # We only include CMake so that Meson can locate toml11, which only ships CMake dependency metadata. dontUseCmakeConfigure = true; @@ -231,7 +232,7 @@ stdenv.mkDerivation (finalAttrs: { jq lsof ] - ++ lib.optional stdenv.hostPlatform.isLinux util-linuxMinimal + ++ lib.optional hostPlatform.isLinux util-linuxMinimal ++ lib.optional (!officialRelease && buildUnreleasedNotes) build-release-notes ++ lib.optional internalApiDocs doxygen; @@ -251,14 +252,14 @@ stdenv.mkDerivation (finalAttrs: { toml11 lix-doc ] - ++ lib.optionals stdenv.hostPlatform.isLinux [ + ++ lib.optionals hostPlatform.isLinux [ libseccomp busybox-sandbox-shell ] ++ lib.optional internalApiDocs rapidcheck - ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid + ++ lib.optional hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies - ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform) aws-sdk-cpp-nix + ++ lib.optional (hostPlatform == buildPlatform) aws-sdk-cpp-nix ++ lib.optionals (finalAttrs.dontBuild) maybePropagatedInputs; checkInputs = [ @@ -278,18 +279,18 @@ stdenv.mkDerivation (finalAttrs: { }; preConfigure = - lib.optionalString (!finalAttrs.dontBuild && !stdenv.hostPlatform.isStatic) '' + lib.optionalString (!finalAttrs.dontBuild && !hostPlatform.isStatic) '' # Copy libboost_context so we don't get all of Boost in our closure. # https://github.com/NixOS/nixpkgs/issues/45462 mkdir -p $out/lib cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib rm -f $out/lib/*.a '' - + lib.optionalString (!finalAttrs.dontBuild && stdenv.hostPlatform.isLinux) '' + + lib.optionalString (!finalAttrs.dontBuild && hostPlatform.isLinux) '' chmod u+w $out/lib/*.so.* patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* '' - + lib.optionalString (!finalAttrs.dontBuild && stdenv.hostPlatform.isDarwin) '' + + lib.optionalString (!finalAttrs.dontBuild && hostPlatform.isDarwin) '' for LIB in $out/lib/*.dylib; do chmod u+w $LIB install_name_tool -id $LIB $LIB @@ -333,7 +334,7 @@ stdenv.mkDerivation (finalAttrs: { mkdir -p $doc/nix-support echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products '' - + lib.optionalString stdenv.hostPlatform.isStatic '' + + lib.optionalString hostPlatform.isStatic '' mkdir -p $out/nix-support echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products '' @@ -364,12 +365,12 @@ stdenv.mkDerivation (finalAttrs: { runHook postInstallCheck ''; - separateDebugInfo = !stdenv.hostPlatform.isStatic && !finalAttrs.dontBuild; + separateDebugInfo = !hostPlatform.isStatic && !finalAttrs.dontBuild; strictDeps = true; # strictoverflow is disabled because we trap on signed overflow instead - hardeningDisable = [ "strictoverflow" ] ++ lib.optional stdenv.hostPlatform.isStatic "pie"; + hardeningDisable = [ "strictoverflow" ] ++ lib.optional hostPlatform.isStatic "pie"; meta = { mainProgram = "nix"; @@ -398,7 +399,7 @@ stdenv.mkDerivation (finalAttrs: { contribNotice, }: let - glibcFix = lib.optionalAttrs (stdenv.buildPlatform.isLinux && glibcLocales != null) { + glibcFix = lib.optionalAttrs (buildPlatform.isLinux && glibcLocales != null) { # Required to make non-NixOS Linux not complain about missing locale files during configure in a dev shell LOCALE_ARCHIVE = "${lib.getLib pkgs.glibcLocales}/lib/locale/locale-archive"; }; @@ -422,7 +423,7 @@ stdenv.mkDerivation (finalAttrs: { env = finalAttrs.env; packages = - lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) clang-tools_llvm + lib.optional (stdenv.cc.isClang && hostPlatform == buildPlatform) clang-tools_llvm ++ [ just nixfmt @@ -435,7 +436,7 @@ stdenv.mkDerivation (finalAttrs: { llvmPackages.clang-unwrapped.dev ] ++ lib.optional (pre-commit-checks ? enabledPackages) pre-commit-checks.enabledPackages - ++ lib.optional (lib.meta.availableOn stdenv.buildPlatform clangbuildanalyzer) clangbuildanalyzer + ++ lib.optional (lib.meta.availableOn buildPlatform clangbuildanalyzer) clangbuildanalyzer ++ finalAttrs.checkInputs; shellHook = '' From eac3546d502522816833d90ac5f8854259d5adff Mon Sep 17 00:00:00 2001 From: Qyriad Date: Wed, 29 May 2024 19:57:58 -0600 Subject: [PATCH 21/22] package: return from shellHook correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If our shellHook is being run from a nested nix-shell (see 7a12bc200¹), then (I think) it is run from a bash function due to the nesting, then `return` is correct. If its `eval`'d though, then there isn't really a correct way to early exit. So we can just unconditionally be executed in a function. Basically, we have IIFE at home. [1]: 7a12bc2007accb5022037b5a04b0e5475a8bb409 Change-Id: Iacad25cbbf66cde2911604e6061e56ad6212af7e --- package.nix | 48 ++++++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/package.nix b/package.nix index 70db7eb22..22cf11cd9 100644 --- a/package.nix +++ b/package.nix @@ -441,32 +441,36 @@ stdenv.mkDerivation (finalAttrs: { shellHook = '' # don't re-run the hook in (other) nested nix-shells - if [[ $name != lix-shell-env ]]; then - return; - fi + function lixShellHook() { + if [[ $name != lix-shell-env ]]; then + return; + fi - PATH=$prefix/bin:$PATH - unset PYTHONPATH - export MANPATH=$out/share/man:$MANPATH + PATH=$prefix/bin:$PATH + unset PYTHONPATH + export MANPATH=$out/share/man:$MANPATH - # Make bash completion work. - XDG_DATA_DIRS+=:$out/share + # Make bash completion work. + XDG_DATA_DIRS+=:$out/share - ${lib.optionalString (pre-commit-checks ? shellHook) pre-commit-checks.shellHook} - # Allow `touch .nocontribmsg` to turn this notice off. - if ! [[ -f .nocontribmsg ]]; then - cat ${contribNotice} - fi + ${lib.optionalString (pre-commit-checks ? shellHook) pre-commit-checks.shellHook} + # Allow `touch .nocontribmsg` to turn this notice off. + if ! [[ -f .nocontribmsg ]]; then + cat ${contribNotice} + fi - # Install the Gerrit commit-msg hook. - # (git common dir is the main .git, including for worktrees) - if gitcommondir=$(git rev-parse --git-common-dir 2>/dev/null) && [[ ! -f "$gitcommondir/hooks/commit-msg" ]]; then - echo 'Installing Gerrit commit-msg hook (adds Change-Id to commit messages)' >&2 - mkdir -p "$gitcommondir/hooks" - curl -s -Lo "$gitcommondir/hooks/commit-msg" https://gerrit.lix.systems/tools/hooks/commit-msg - chmod u+x "$gitcommondir/hooks/commit-msg" - fi - unset gitcommondir + # Install the Gerrit commit-msg hook. + # (git common dir is the main .git, including for worktrees) + if gitcommondir=$(git rev-parse --git-common-dir 2>/dev/null) && [[ ! -f "$gitcommondir/hooks/commit-msg" ]]; then + echo 'Installing Gerrit commit-msg hook (adds Change-Id to commit messages)' >&2 + mkdir -p "$gitcommondir/hooks" + curl -s -Lo "$gitcommondir/hooks/commit-msg" https://gerrit.lix.systems/tools/hooks/commit-msg + chmod u+x "$gitcommondir/hooks/commit-msg" + fi + unset gitcommondir + } + + lixShellHook ''; } ); From 2760818f062b6810da6766639c543aeb1db45522 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Wed, 29 May 2024 20:11:12 -0600 Subject: [PATCH 22/22] package: fix derivation correctness when static Change-Id: I394bb72d9f378cd78acc6cf67a9bb15e342d57c4 --- package.nix | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/package.nix b/package.nix index 22cf11cd9..cc5902a91 100644 --- a/package.nix +++ b/package.nix @@ -201,7 +201,7 @@ stdenv.mkDerivation (finalAttrs: { # so we must explicitly enable or disable features that we are not passing # dependencies for. (lib.mesonEnable "internal-api-docs" internalApiDocs) - (lib.mesonBool "enable-tests" finalAttrs.doCheck) + (lib.mesonBool "enable-tests" finalAttrs.finalPackage.doCheck) (lib.mesonBool "enable-docs" canRunInstalled) ] ++ lib.optional (hostPlatform != buildPlatform) "--cross-file=${mesonCrossFile}"; @@ -286,7 +286,7 @@ stdenv.mkDerivation (finalAttrs: { cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib rm -f $out/lib/*.a '' - + lib.optionalString (!finalAttrs.dontBuild && hostPlatform.isLinux) '' + + lib.optionalString (!finalAttrs.dontBuild && hostPlatform.isLinux && !hostPlatform.isStatic) '' chmod u+w $out/lib/*.so.* patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* '' @@ -417,11 +417,17 @@ stdenv.mkDerivation (finalAttrs: { name = "lix-shell-env"; - inputsFrom = [ finalAttrs ]; + # finalPackage is necessary to propagate stuff that is set by mkDerivation itself, + # like doCheck. + inputsFrom = [ finalAttrs.finalPackage ]; # For Meson to find Boost. env = finalAttrs.env; + # I guess this is necessary because mesonFlags to mkDerivation doesn't propagate in inputsFrom, + # which only propagates stuff set in hooks? idk. + inherit (finalAttrs) mesonFlags; + packages = lib.optional (stdenv.cc.isClang && hostPlatform == buildPlatform) clang-tools_llvm ++ [