From 6fd6795bc4faed644c5f3abafdcb21638a119342 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 10:50:54 +0200 Subject: [PATCH 01/16] util.{hh,cc}: Split out environment-variables.{hh,cc} Change-Id: Icff0aa33fda5147bd5dbe256a0b9d6a6c8a2c3f6 --- src/libcmd/editor-for.cc | 1 + src/libmain/loggers.cc | 1 + src/libstore/globals.cc | 1 + src/libstore/globals.hh | 2 +- src/libstore/ssh.cc | 1 + src/libutil/args.cc | 2 + src/libutil/environment-variables.cc | 51 +++++++++++++++++++ src/libutil/environment-variables.hh | 42 +++++++++++++++ src/libutil/error.cc | 1 + src/libutil/filesystem.cc | 1 + src/libutil/logging.cc | 1 + src/libutil/meson.build | 2 + src/libutil/util.cc | 46 +---------------- src/libutil/util.hh | 21 -------- .../libutil-support/tests/characterization.hh | 1 + 15 files changed, 107 insertions(+), 67 deletions(-) create mode 100644 src/libutil/environment-variables.cc create mode 100644 src/libutil/environment-variables.hh diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc index e5f762104..a56a7065e 100644 --- a/src/libcmd/editor-for.cc +++ b/src/libcmd/editor-for.cc @@ -1,5 +1,6 @@ #include "util.hh" #include "editor-for.hh" +#include "environment-variables.hh" #include "source-path.hh" namespace nix { diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index cda5cb939..7e80a31f1 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -1,3 +1,4 @@ +#include "environment-variables.hh" #include "loggers.hh" #include "progress-bar.hh" #include "util.hh" diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 3308cad1f..897d3e8ef 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,3 +1,4 @@ +#include "environment-variables.hh" #include "globals.hh" #include "util.hh" #include "archive.hh" diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 85789f6b5..581de7ff6 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1,9 +1,9 @@ #pragma once ///@file +#include "environment-variables.hh" #include "types.hh" #include "config.hh" -#include "util.hh" #include #include diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 87414fe9c..2f921e2d0 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,3 +1,4 @@ +#include "environment-variables.hh" #include "ssh.hh" #include "finally.hh" diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 520f50c30..bcff653c5 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -2,6 +2,8 @@ #include "args/root.hh" #include "hash.hh" #include "json-utils.hh" +#include "environment-variables.hh" + #include "experimental-features-json.hh" #include diff --git a/src/libutil/environment-variables.cc b/src/libutil/environment-variables.cc new file mode 100644 index 000000000..71c404a0e --- /dev/null +++ b/src/libutil/environment-variables.cc @@ -0,0 +1,51 @@ +#include +#include +#include +#include + +extern char * * environ __attribute__((weak)); + +namespace nix { + +std::optional getEnv(const std::string & key) +{ + char * value = getenv(key.c_str()); + if (!value) return {}; + return std::string(value); +} + +std::optional getEnvNonEmpty(const std::string & key) { + auto value = getEnv(key); + if (value == "") return {}; + return value; +} + +std::map getEnv() +{ + std::map env; + for (size_t i = 0; environ[i]; ++i) { + auto s = environ[i]; + auto eq = strchr(s, '='); + if (!eq) + // invalid env, just keep going + continue; + env.emplace(std::string(s, eq), std::string(eq + 1)); + } + return env; +} + + +void clearEnv() +{ + for (auto & name : getEnv()) + unsetenv(name.first.c_str()); +} + +void replaceEnv(const std::map & newEnv) +{ + clearEnv(); + for (auto & newEnvVar : newEnv) + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); +} + +} diff --git a/src/libutil/environment-variables.hh b/src/libutil/environment-variables.hh new file mode 100644 index 000000000..784b95921 --- /dev/null +++ b/src/libutil/environment-variables.hh @@ -0,0 +1,42 @@ +#pragma once +/** + * @file + * + * Utilities for working with the current process's environment + * variables. + */ + +#include +#include +#include + + +namespace nix { + +/** + * @return an environment variable. + */ +std::optional getEnv(const std::string & key); + +/** + * @return a non empty environment variable. Returns nullopt if the env + * variable is set to "" + */ +std::optional getEnvNonEmpty(const std::string & key); + +/** + * Get the entire environment. + */ +std::map getEnv(); + +/** + * Clear the environment. + */ +void clearEnv(); + +/** + * Replace the entire environment with the given one. + */ +void replaceEnv(const std::map & newEnv); + +} diff --git a/src/libutil/error.cc b/src/libutil/error.cc index b534ff87e..fe73ffc56 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,3 +1,4 @@ +#include "environment-variables.hh" #include "error.hh" #include "position.hh" diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index b637281d0..85871870c 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -2,6 +2,7 @@ #include #include +#include "environment-variables.hh" #include "finally.hh" #include "util.hh" #include "signals.hh" diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 0e63f9bd6..8e4828be4 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -1,3 +1,4 @@ +#include "environment-variables.hh" #include "logging.hh" #include "util.hh" #include "config.hh" diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 58e0bd062..01b3e24b7 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -7,6 +7,7 @@ libutil_sources = files( 'compute-levels.cc', 'config.cc', 'english.cc', + 'environment-variables.cc', 'error.cc', 'escape-char.cc', 'escape-string.cc', @@ -53,6 +54,7 @@ libutil_headers = files( 'config-impl.hh', 'config.hh', 'english.hh', + 'environment-variables.hh', 'error.hh', 'escape-char.hh', 'escape-string.hh', diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 2c0fcc897..97d53ca63 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -4,6 +4,7 @@ #include "serialise.hh" #include "cgroup.hh" #include "signals.hh" +#include "environment-variables.hh" #include #include @@ -43,57 +44,12 @@ #endif -extern char * * environ __attribute__((weak)); - - #ifdef NDEBUG #error "Lix may not be built with assertions disabled (i.e. with -DNDEBUG)." #endif namespace nix { -std::optional getEnv(const std::string & key) -{ - char * value = getenv(key.c_str()); - if (!value) return {}; - return std::string(value); -} - -std::optional getEnvNonEmpty(const std::string & key) { - auto value = getEnv(key); - if (value == "") return {}; - return value; -} - -std::map getEnv() -{ - std::map env; - for (size_t i = 0; environ[i]; ++i) { - auto s = environ[i]; - auto eq = strchr(s, '='); - if (!eq) - // invalid env, just keep going - continue; - env.emplace(std::string(s, eq), std::string(eq + 1)); - } - return env; -} - - -void clearEnv() -{ - for (auto & name : getEnv()) - unsetenv(name.first.c_str()); -} - -void replaceEnv(const std::map & newEnv) -{ - clearEnv(); - for (auto & newEnvVar : newEnv) - setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); -} - - Path absPath(Path path, std::optional dir, bool resolveSymlinks) { if (path.empty() || path[0] != '/') { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 14868776c..151da25b9 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -39,27 +39,6 @@ struct Source; extern const std::string nativeSystem; -/** - * @return an environment variable. - */ -std::optional getEnv(const std::string & key); - -/** - * @return a non empty environment variable. Returns nullopt if the env - * variable is set to "" - */ -std::optional getEnvNonEmpty(const std::string & key); - -/** - * Get the entire environment. - */ -std::map getEnv(); - -/** - * Clear the environment. - */ -void clearEnv(); - /** * @return An absolutized path, resolving paths relative to the * specified directory, or the current directory otherwise. The path diff --git a/tests/unit/libutil-support/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh index 7f570f619..9de20d02c 100644 --- a/tests/unit/libutil-support/tests/characterization.hh +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "environment-variables.hh" #include #include From 81bdf8d2d672e135e68745e6975ad5edafadf13a Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 11:19:08 +0200 Subject: [PATCH 02/16] util.{hh,cc}: Split out terminal.{hh,cc} Change-Id: I9de2296b4012d50f540124001d54d6ca3be4c6da --- src/libcmd/markdown.cc | 1 + src/libexpr/flake/flake.cc | 1 + src/libexpr/print.cc | 1 + src/libmain/progress-bar.cc | 1 + src/libutil/error.cc | 1 + src/libutil/logging.cc | 1 + src/libutil/meson.build | 2 + src/libutil/signals.cc | 1 + src/libutil/suggestions.cc | 4 +- src/libutil/terminal.cc | 104 ++++++++++++++++++++++++++++++++++++ src/libutil/terminal.hh | 40 ++++++++++++++ src/libutil/util.cc | 96 --------------------------------- src/libutil/util.hh | 25 --------- tests/unit/libutil/tests.cc | 1 + 14 files changed, 157 insertions(+), 122 deletions(-) create mode 100644 src/libutil/terminal.cc create mode 100644 src/libutil/terminal.hh diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 668a07763..8b3bbc1b5 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,6 +1,7 @@ #include "markdown.hh" #include "util.hh" #include "finally.hh" +#include "terminal.hh" #include #include diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index e13d1cb93..ac50351ad 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -8,6 +8,7 @@ #include "fetchers.hh" #include "finally.hh" #include "fetch-settings.hh" +#include "terminal.hh" namespace nix { diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index 231bde0a0..c56b0e72e 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -9,6 +9,7 @@ #include "english.hh" #include "signals.hh" #include "eval.hh" +#include "terminal.hh" namespace nix { diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 1a68daca2..f3700f103 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -3,6 +3,7 @@ #include "sync.hh" #include "store-api.hh" #include "names.hh" +#include "terminal.hh" #include #include diff --git a/src/libutil/error.cc b/src/libutil/error.cc index fe73ffc56..f780aabef 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,6 +1,7 @@ #include "environment-variables.hh" #include "error.hh" #include "position.hh" +#include "terminal.hh" #include #include diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 8e4828be4..731d1034d 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -4,6 +4,7 @@ #include "config.hh" #include "source-path.hh" #include "position.hh" +#include "terminal.hh" #include #include diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 01b3e24b7..c890911e9 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -30,6 +30,7 @@ libutil_sources = files( 'source-path.cc', 'suggestions.cc', 'tarfile.cc', + 'terminal.cc', 'thread-pool.cc', 'url.cc', 'url-name.cc', @@ -90,6 +91,7 @@ libutil_headers = files( 'suggestions.hh', 'sync.hh', 'tarfile.hh', + 'terminal.hh', 'thread-pool.hh', 'topo-sort.hh', 'types.hh', diff --git a/src/libutil/signals.cc b/src/libutil/signals.cc index 41fdc9dc8..c0e66f6ed 100644 --- a/src/libutil/signals.cc +++ b/src/libutil/signals.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "error.hh" #include "sync.hh" +#include "terminal.hh" #include diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc index 9510a5f0c..63dcf84b5 100644 --- a/src/libutil/suggestions.cc +++ b/src/libutil/suggestions.cc @@ -1,7 +1,9 @@ #include "suggestions.hh" #include "ansicolor.hh" -#include "util.hh" +#include "terminal.hh" + #include +#include namespace nix { diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc new file mode 100644 index 000000000..b58331d04 --- /dev/null +++ b/src/libutil/terminal.cc @@ -0,0 +1,104 @@ +#include "terminal.hh" +#include "environment-variables.hh" +#include "sync.hh" + +#include +#include + +namespace nix { + +bool shouldANSI() +{ + return isatty(STDERR_FILENO) + && getEnv("TERM").value_or("dumb") != "dumb" + && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value()); +} + +std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) +{ + std::string t, e; + size_t w = 0; + auto i = s.begin(); + + while (w < (size_t) width && i != s.end()) { + + if (*i == '\e') { + std::string e; + e += *i++; + char last = 0; + + if (i != s.end() && *i == '[') { + e += *i++; + // eat parameter bytes + while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; + // eat intermediate bytes + while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; + // eat final byte + if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; + } else { + if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; + } + + if (!filterAll && last == 'm') + t += e; + } + + else if (*i == '\t') { + i++; t += ' '; w++; + while (w < (size_t) width && w % 8) { + t += ' '; w++; + } + } + + else if (*i == '\r' || *i == '\a') + // do nothing for now + i++; + + else { + w++; + // Copy one UTF-8 character. + if ((*i & 0xe0) == 0xc0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } else if ((*i & 0xf0) == 0xe0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } else if ((*i & 0xf8) == 0xf0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } + } else + t += *i++; + } + } + + return t; +} + +static Sync> windowSize{{0, 0}}; + +void updateWindowSize() +{ + struct winsize ws; + if (ioctl(2, TIOCGWINSZ, &ws) == 0) { + auto windowSize_(windowSize.lock()); + windowSize_->first = ws.ws_row; + windowSize_->second = ws.ws_col; + } +} + + +std::pair getWindowSize() +{ + return *windowSize.lock(); +} + +} diff --git a/src/libutil/terminal.hh b/src/libutil/terminal.hh new file mode 100644 index 000000000..43df5bd70 --- /dev/null +++ b/src/libutil/terminal.hh @@ -0,0 +1,40 @@ +#pragma once +///@file + +#include +#include + +namespace nix { + +/** + * Determine whether ANSI escape sequences are appropriate for the + * present output. + */ +bool shouldANSI(); + +/** + * Truncate a string to 'width' printable characters. If 'filterAll' + * is true, all ANSI escape sequences are filtered out. Otherwise, + * some escape sequences (such as colour setting) are copied but not + * included in the character count. Also, tabs are expanded to + * spaces. + */ +std::string filterANSIEscapes(std::string_view s, + bool filterAll = false, + unsigned int width = std::numeric_limits::max()); + +/** + * Recalculate the window size, updating a global variable. Used in the + * `SIGWINCH` signal handler. + */ +void updateWindowSize(); + +/** + * @return the number of rows and columns of the terminal. + * + * The value is cached so this is quick. The cached result is computed + * by `updateWindowSize()`. + */ +std::pair getWindowSize(); + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 97d53ca63..63d9e5248 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1436,83 +1436,6 @@ void ignoreException(Verbosity lvl) } catch (...) { } } -bool shouldANSI() -{ - return isatty(STDERR_FILENO) - && getEnv("TERM").value_or("dumb") != "dumb" - && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value()); -} - -std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) -{ - std::string t, e; - size_t w = 0; - auto i = s.begin(); - - while (w < (size_t) width && i != s.end()) { - - if (*i == '\e') { - std::string e; - e += *i++; - char last = 0; - - if (i != s.end() && *i == '[') { - e += *i++; - // eat parameter bytes - while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; - // eat intermediate bytes - while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; - // eat final byte - if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; - } else { - if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; - } - - if (!filterAll && last == 'm') - t += e; - } - - else if (*i == '\t') { - i++; t += ' '; w++; - while (w < (size_t) width && w % 8) { - t += ' '; w++; - } - } - - else if (*i == '\r' || *i == '\a') - // do nothing for now - i++; - - else { - w++; - // Copy one UTF-8 character. - if ((*i & 0xe0) == 0xc0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } else if ((*i & 0xf0) == 0xe0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } else if ((*i & 0xf8) == 0xf0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } - } else - t += *i++; - } - } - - return t; -} - - constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; std::string base64Encode(std::string_view s) @@ -1630,25 +1553,6 @@ std::pair getLine(std::string_view s) ////////////////////////////////////////////////////////////////////// -static Sync> windowSize{{0, 0}}; - - -void updateWindowSize() -{ - struct winsize ws; - if (ioctl(2, TIOCGWINSZ, &ws) == 0) { - auto windowSize_(windowSize.lock()); - windowSize_->first = ws.ws_row; - windowSize_->second = ws.ws_col; - } -} - - -std::pair getWindowSize() -{ - return *windowSize.lock(); -} - rlim_t savedStackSize = 0; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 151da25b9..1ce7e8312 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -726,23 +726,6 @@ constexpr char treeLast[] = "└───"; constexpr char treeLine[] = "│ "; constexpr char treeNull[] = " "; -/** - * Determine whether ANSI escape sequences are appropriate for the - * present output. - */ -bool shouldANSI(); - -/** - * Truncate a string to 'width' printable characters. If 'filterAll' - * is true, all ANSI escape sequences are filtered out. Otherwise, - * some escape sequences (such as colour setting) are copied but not - * included in the character count. Also, tabs are expanded to - * spaces. - */ -std::string filterANSIEscapes(std::string_view s, - bool filterAll = false, - unsigned int width = std::numeric_limits::max()); - /** * Base64 encoding/decoding. @@ -840,14 +823,6 @@ struct MaintainCount }; -/** - * @return the number of rows and columns of the terminal. - */ -std::pair getWindowSize(); - -void updateWindowSize(); - - /** * Used in various places. */ diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index 720370066..787f79093 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -1,5 +1,6 @@ #include "util.hh" #include "types.hh" +#include "terminal.hh" #include #include From 6b5078c81554ddb36547f8c41805cc94b7738396 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 11:52:13 +0200 Subject: [PATCH 03/16] util.{hh,cc}: Split out file-system.{hh,cc} Change-Id: Ifa89a529e7e34e7291eca87d802d2f569cf2493e --- src/libstore/build/hook-instance.cc | 1 + src/libstore/crypto.cc | 1 + src/libstore/lock.cc | 1 + src/libstore/ssh.hh | 1 + src/libutil/archive.cc | 1 + src/libutil/archive.hh | 1 + src/libutil/canon-path.cc | 2 +- src/libutil/cgroup.cc | 1 + src/libutil/config.cc | 1 + src/libutil/file-system.cc | 678 ++++++++++++++++++ src/libutil/file-system.hh | 270 +++++++ src/libutil/filesystem.cc | 176 ----- src/libutil/hash.hh | 1 + src/libutil/input-accessor.hh | 6 +- src/libutil/meson.build | 3 +- src/libutil/namespaces.cc | 1 + src/libutil/source-path.hh | 2 + src/libutil/tarfile.cc | 1 + src/libutil/util.cc | 501 +------------ src/libutil/util.hh | 239 ------ .../nix-collect-garbage.cc | 1 + tests/unit/libstore/machines.cc | 1 + .../libutil-support/tests/characterization.hh | 2 + tests/unit/libutil/tests.cc | 1 + 24 files changed, 974 insertions(+), 919 deletions(-) create mode 100644 src/libutil/file-system.cc create mode 100644 src/libutil/file-system.hh delete mode 100644 src/libutil/filesystem.cc diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index ea4c2e508..108722cfb 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -1,3 +1,4 @@ +#include "file-system.hh" #include "globals.hh" #include "hook-instance.hh" diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 1027469c9..1e29a812b 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -1,5 +1,6 @@ #include "crypto.hh" #include "util.hh" +#include "file-system.hh" #include "globals.hh" #include diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 80c75eaa5..b52e87ae0 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -1,4 +1,5 @@ #include "lock.hh" +#include "file-system.hh" #include "globals.hh" #include "pathlocks.hh" diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index 94b952af9..0802c6cc0 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "file-system.hh" #include "util.hh" #include "sync.hh" diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index a18c54ebf..c3f39d4b3 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -12,6 +12,7 @@ #include #include "archive.hh" +#include "file-system.hh" #include "util.hh" #include "config.hh" #include "signals.hh" diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 017b6633c..f163a1947 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "serialise.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index 040464532..f678fae94 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -1,5 +1,5 @@ #include "canon-path.hh" -#include "util.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libutil/cgroup.cc b/src/libutil/cgroup.cc index 9320d2371..aa7802f79 100644 --- a/src/libutil/cgroup.cc +++ b/src/libutil/cgroup.cc @@ -2,6 +2,7 @@ #include "cgroup.hh" #include "util.hh" +#include "file-system.hh" #include "finally.hh" #include diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 81efcd507..f6f14878a 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -2,6 +2,7 @@ #include "args.hh" #include "abstract-setting-to-json.hh" #include "experimental-features.hh" +#include "file-system.hh" #include "config-impl.hh" diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc new file mode 100644 index 000000000..e5ba42eb6 --- /dev/null +++ b/src/libutil/file-system.cc @@ -0,0 +1,678 @@ +#include +#include +#include + +#include "environment-variables.hh" +#include "file-system.hh" +#include "finally.hh" +#include "serialise.hh" +#include "util.hh" +#include "signals.hh" +#include "types.hh" + +namespace fs = std::filesystem; + +namespace nix { + +Path absPath(Path path, std::optional dir, bool resolveSymlinks) +{ + if (path.empty() || path[0] != '/') { + if (!dir) { +#ifdef __GNU__ + /* GNU (aka. GNU/Hurd) doesn't have any limitation on path + lengths and doesn't define `PATH_MAX'. */ + char *buf = getcwd(NULL, 0); + if (buf == NULL) +#else + char buf[PATH_MAX]; + if (!getcwd(buf, sizeof(buf))) +#endif + throw SysError("cannot get cwd"); + path = concatStrings(buf, "/", path); +#ifdef __GNU__ + free(buf); +#endif + } else + path = concatStrings(*dir, "/", path); + } + return canonPath(path, resolveSymlinks); +} + + +Path canonPath(PathView path, bool resolveSymlinks) +{ + assert(path != ""); + + std::string s; + s.reserve(256); + + if (path[0] != '/') + throw Error("not an absolute path: '%1%'", path); + + std::string temp; + + /* Count the number of times we follow a symlink and stop at some + arbitrary (but high) limit to prevent infinite loops. */ + unsigned int followCount = 0, maxFollow = 1024; + + while (1) { + + /* Skip slashes. */ + while (!path.empty() && path[0] == '/') path.remove_prefix(1); + if (path.empty()) break; + + /* Ignore `.'. */ + if (path == "." || path.substr(0, 2) == "./") + path.remove_prefix(1); + + /* If `..', delete the last component. */ + else if (path == ".." || path.substr(0, 3) == "../") + { + if (!s.empty()) s.erase(s.rfind('/')); + path.remove_prefix(2); + } + + /* Normal component; copy it. */ + else { + s += '/'; + if (const auto slash = path.find('/'); slash == std::string::npos) { + s += path; + path = {}; + } else { + s += path.substr(0, slash); + path = path.substr(slash); + } + + /* If s points to a symlink, resolve it and continue from there */ + if (resolveSymlinks && isLink(s)) { + if (++followCount >= maxFollow) + throw Error("infinite symlink recursion in path '%1%'", path); + temp = concatStrings(readLink(s), path); + path = temp; + if (!temp.empty() && temp[0] == '/') { + s.clear(); /* restart for symlinks pointing to absolute path */ + } else { + s = dirOf(s); + if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / + s.clear(); + } + } + } + } + } + + return s.empty() ? "/" : std::move(s); +} + +void chmodPath(const Path & path, mode_t mode) +{ + if (chmod(path.c_str(), mode) == -1) + throw SysError("setting permissions on '%s'", path); +} + +Path dirOf(const PathView path) +{ + Path::size_type pos = path.rfind('/'); + if (pos == std::string::npos) + return "."; + return pos == 0 ? "/" : Path(path, 0, pos); +} + + +std::string_view baseNameOf(std::string_view path) +{ + if (path.empty()) + return ""; + + auto last = path.size() - 1; + if (path[last] == '/' && last > 0) + last -= 1; + + auto pos = path.rfind('/', last); + if (pos == std::string::npos) + pos = 0; + else + pos += 1; + + return path.substr(pos, last - pos + 1); +} + + +std::string expandTilde(std::string_view path) +{ + // TODO: expand ~user ? + auto tilde = path.substr(0, 2); + if (tilde == "~/" || tilde == "~") + return getHome() + std::string(path.substr(1)); + else + return std::string(path); +} + + +bool isInDir(std::string_view path, std::string_view dir) +{ + return path.substr(0, 1) == "/" + && path.substr(0, dir.size()) == dir + && path.size() >= dir.size() + 2 + && path[dir.size()] == '/'; +} + + +bool isDirOrInDir(std::string_view path, std::string_view dir) +{ + return path == dir || isInDir(path, dir); +} + + +struct stat stat(const Path & path) +{ + struct stat st; + if (stat(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + + +struct stat lstat(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + +std::optional maybeLstat(const Path & path) +{ + std::optional st{std::in_place}; + if (lstat(path.c_str(), &*st)) + { + if (errno == ENOENT || errno == ENOTDIR) + st.reset(); + else + throw SysError("getting status of '%s'", path); + } + return st; +} + +bool pathExists(const Path & path) +{ + return maybeLstat(path).has_value(); +} + +bool pathAccessible(const Path & path) +{ + try { + return pathExists(path); + } catch (SysError & e) { + // swallow EPERM + if (e.errNo == EPERM) return false; + throw; + } +} + + +Path readLink(const Path & path) +{ + checkInterrupt(); + std::vector buf; + for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { + buf.resize(bufSize); + ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); + if (rlSize == -1) + if (errno == EINVAL) + throw Error("'%1%' is not a symlink", path); + else + throw SysError("reading symbolic link '%1%'", path); + else if (rlSize < bufSize) + return std::string(buf.data(), rlSize); + } +} + + +bool isLink(const Path & path) +{ + struct stat st = lstat(path); + return S_ISLNK(st.st_mode); +} + + +DirEntries readDirectory(DIR *dir, const Path & path) +{ + DirEntries entries; + entries.reserve(64); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { /* sic */ + checkInterrupt(); + std::string name = dirent->d_name; + if (name == "." || name == "..") continue; + entries.emplace_back(name, dirent->d_ino, +#ifdef HAVE_STRUCT_DIRENT_D_TYPE + dirent->d_type +#else + DT_UNKNOWN +#endif + ); + } + if (errno) throw SysError("reading directory '%1%'", path); + + return entries; +} + +DirEntries readDirectory(const Path & path) +{ + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) throw SysError("opening directory '%1%'", path); + + return readDirectory(dir.get(), path); +} + + +unsigned char getFileType(const Path & path) +{ + struct stat st = lstat(path); + if (S_ISDIR(st.st_mode)) return DT_DIR; + if (S_ISLNK(st.st_mode)) return DT_LNK; + if (S_ISREG(st.st_mode)) return DT_REG; + return DT_UNKNOWN; +} + + +std::string readFile(int fd) +{ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + return drainFD(fd, true, st.st_size); +} + + +std::string readFile(const Path & path) +{ + AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; + if (!fd) + throw SysError("opening file '%1%'", path); + return readFile(fd.get()); +} + + +void readFile(const Path & path, Sink & sink) +{ + AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; + if (!fd) + throw SysError("opening file '%s'", path); + drainFD(fd.get(), sink); +} + + +void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) +{ + AutoCloseFD fd{open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode)}; + if (!fd) + throw SysError("opening file '%1%'", path); + try { + writeFull(fd.get(), s); + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; + } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + + +void writeFile(const Path & path, Source & source, mode_t mode, bool sync) +{ + AutoCloseFD fd{open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode)}; + if (!fd) + throw SysError("opening file '%1%'", path); + + std::vector buf(64 * 1024); + + try { + while (true) { + try { + auto n = source.read(buf.data(), buf.size()); + writeFull(fd.get(), {buf.data(), n}); + } catch (EndOfFile &) { break; } + } + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; + } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + +void syncParent(const Path & path) +{ + AutoCloseFD fd{open(dirOf(path).c_str(), O_RDONLY, 0)}; + if (!fd) + throw SysError("opening file '%1%'", path); + fd.fsync(); +} + +static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) +{ + checkInterrupt(); + + std::string name(baseNameOf(path)); + + struct stat st; + if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { + if (errno == ENOENT) return; + throw SysError("getting status of '%1%'", path); + } + + if (!S_ISDIR(st.st_mode)) { + /* We are about to delete a file. Will it likely free space? */ + + switch (st.st_nlink) { + /* Yes: last link. */ + case 1: + bytesFreed += st.st_size; + break; + /* Maybe: yes, if 'auto-optimise-store' or manual optimisation + was performed. Instead of checking for real let's assume + it's an optimised file and space will be freed. + + In worst case we will double count on freed space for files + with exactly two hardlinks for unoptimised packages. + */ + case 2: + bytesFreed += st.st_size; + break; + /* No: 3+ links. */ + default: + break; + } + } + + if (S_ISDIR(st.st_mode)) { + /* Make the directory accessible. */ + const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; + if ((st.st_mode & PERM_MASK) != PERM_MASK) { + if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) + throw SysError("chmod '%1%'", path); + } + + int fd = openat(parentfd, path.c_str(), O_RDONLY); + if (fd == -1) + throw SysError("opening directory '%1%'", path); + AutoCloseDir dir(fdopendir(fd)); + if (!dir) + throw SysError("opening directory '%1%'", path); + for (auto & i : readDirectory(dir.get(), path)) + _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); + } + + int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; + if (unlinkat(parentfd, name.c_str(), flags) == -1) { + if (errno == ENOENT) return; + throw SysError("cannot unlink '%1%'", path); + } +} + +static void _deletePath(const Path & path, uint64_t & bytesFreed) +{ + Path dir = dirOf(path); + if (dir == "") + dir = "/"; + + AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)}; + if (!dirfd) { + if (errno == ENOENT) return; + throw SysError("opening directory '%1%'", path); + } + + _deletePath(dirfd.get(), path, bytesFreed); +} + + +void deletePath(const Path & path) +{ + uint64_t dummy; + deletePath(path, dummy); +} + + +void deletePath(const Path & path, uint64_t & bytesFreed) +{ + //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); + bytesFreed = 0; + _deletePath(path, bytesFreed); +} + +Paths createDirs(const Path & path) +{ + Paths created; + if (path == "/") return created; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + created = createDirs(dirOf(path)); + if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) + throw SysError("creating directory '%1%'", path); + st = lstat(path); + created.push_back(path); + } + + if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) + throw SysError("statting symlink '%1%'", path); + + if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); + + return created; +} + + +////////////////////////////////////////////////////////////////////// + +AutoDelete::AutoDelete() : del{false} {} + +AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p) +{ + del = true; + this->recursive = recursive; +} + +AutoDelete::~AutoDelete() +{ + try { + if (del) { + if (recursive) + deletePath(path); + else { + if (remove(path.c_str()) == -1) + throw SysError("cannot unlink '%1%'", path); + } + } + } catch (...) { + ignoreException(); + } +} + +void AutoDelete::cancel() +{ + del = false; +} + +void AutoDelete::reset(const Path & p, bool recursive) { + path = p; + this->recursive = recursive; + del = true; +} + +////////////////////////////////////////////////////////////////////// + +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + std::atomic & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); + if (includePid) + return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++); + else + return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++); +} + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static std::atomic globalCounter = 0; + std::atomic localCounter = 0; + auto & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { +#if __FreeBSD__ + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError("setting group of directory '%1%'", tmpDir); +#endif + return tmpDir; + } + if (errno != EEXIST) + throw SysError("creating directory '%1%'", tmpDir); + } +} + + +std::pair createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + // FIXME: use O_TMPFILE. + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + closeOnExec(fd.get()); + return {std::move(fd), tmpl}; +} + +void createSymlink(const Path & target, const Path & link) +{ + if (symlink(target.c_str(), link.c_str())) + throw SysError("creating symlink from '%1%' to '%2%'", link, target); +} + +void replaceSymlink(const Path & target, const Path & link) +{ + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + + try { + createSymlink(target, tmp); + } catch (SysError & e) { + if (e.errNo == EEXIST) continue; + throw; + } + + renameFile(tmp, link); + + break; + } +} + +void setWriteTime(const fs::path & p, const struct stat & st) +{ + struct timeval times[2]; + times[0] = { + .tv_sec = st.st_atime, + .tv_usec = 0, + }; + times[1] = { + .tv_sec = st.st_mtime, + .tv_usec = 0, + }; + if (lutimes(p.c_str(), times) != 0) + throw SysError("changing modification time of '%s'", p); +} + +void copy(const fs::directory_entry & from, const fs::path & to, CopyFileFlags flags) +{ + // TODO: Rewrite the `is_*` to use `symlink_status()` + auto statOfFrom = lstat(from.path().c_str()); + auto fromStatus = from.symlink_status(); + + // Mark the directory as writable so that we can delete its children + if (flags.deleteAfter && fs::is_directory(fromStatus)) { + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + } + + + if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { + auto opts = fs::copy_options::overwrite_existing; + + if (!flags.followSymlinks) { + opts |= fs::copy_options::copy_symlinks; + } + + fs::copy(from.path(), to, opts); + } else if (fs::is_directory(fromStatus)) { + fs::create_directory(to); + for (auto & entry : fs::directory_iterator(from.path())) { + copy(entry, to / entry.path().filename(), flags); + } + } else { + throw Error("file '%s' has an unsupported type", from.path()); + } + + setWriteTime(to, statOfFrom); + if (flags.deleteAfter) { + if (!fs::is_symlink(fromStatus)) + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + fs::remove(from.path()); + } +} + + +void copyFile(const Path & oldPath, const Path & newPath, CopyFileFlags flags) +{ + return copy(fs::directory_entry(fs::path(oldPath)), fs::path(newPath), flags); +} + +void renameFile(const Path & oldName, const Path & newName) +{ + fs::rename(oldName, newName); +} + +void moveFile(const Path & oldName, const Path & newName) +{ + try { + renameFile(oldName, newName); + } catch (fs::filesystem_error & e) { + auto oldPath = fs::path(oldName); + auto newPath = fs::path(newName); + // For the move to be as atomic as possible, copy to a temporary + // directory + fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); + Finally removeTemp = [&]() { fs::remove(temp); }; + auto tempCopyTarget = temp / "copy-target"; + if (e.code().value() == EXDEV) { + fs::remove(newPath); + warn("Can’t rename %s as %s, copying instead", oldName, newName); + copy(fs::directory_entry(oldPath), tempCopyTarget, { .deleteAfter = true }); + renameFile(tempCopyTarget, newPath); + } + } +} + +} diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh new file mode 100644 index 000000000..1d91ef334 --- /dev/null +++ b/src/libutil/file-system.hh @@ -0,0 +1,270 @@ +#pragma once +/** + * @file + * + * Utiltities for working with the file sytem and file paths. + */ + +#include "types.hh" +#include "util.hh" + +#include +#include +#include +#include + +#include + +#include +#include + +#ifndef HAVE_STRUCT_DIRENT_D_TYPE +#define DT_UNKNOWN 0 +#define DT_REG 1 +#define DT_LNK 2 +#define DT_DIR 3 +#endif + +namespace nix { + +struct Sink; +struct Source; + +/** + * @return An absolutized path, resolving paths relative to the + * specified directory, or the current directory otherwise. The path + * is also canonicalised. + */ +Path absPath(Path path, + std::optional dir = {}, + bool resolveSymlinks = false); + +/** + * Canonicalise a path by removing all `.` or `..` components and + * double or trailing slashes. Optionally resolves all symlink + * components such that each component of the resulting path is *not* + * a symbolic link. + */ +Path canonPath(PathView path, bool resolveSymlinks = false); + +/** + * Change the permissions of a path + * Not called `chmod` as it shadows and could be confused with + * `int chmod(char *, mode_t)`, which does not handle errors + */ +void chmodPath(const Path & path, mode_t mode); + +/** + * @return The directory part of the given canonical path, i.e., + * everything before the final `/`. If the path is the root or an + * immediate child thereof (e.g., `/foo`), this means `/` + * is returned. + */ +Path dirOf(const PathView path); + +/** + * @return the base name of the given canonical path, i.e., everything + * following the final `/` (trailing slashes are removed). + */ +std::string_view baseNameOf(std::string_view path); + +/** + * Perform tilde expansion on a path. + */ +std::string expandTilde(std::string_view path); + +/** + * Check whether 'path' is a descendant of 'dir'. Both paths must be + * canonicalized. + */ +bool isInDir(std::string_view path, std::string_view dir); + +/** + * Check whether 'path' is equal to 'dir' or a descendant of + * 'dir'. Both paths must be canonicalized. + */ +bool isDirOrInDir(std::string_view path, std::string_view dir); + +/** + * Get status of `path`. + */ +struct stat stat(const Path & path); +struct stat lstat(const Path & path); + +/** + * `lstat` the given path if it exists. + * @return std::nullopt if the path doesn't exist, or an optional containing the result of `lstat` otherwise + */ +std::optional maybeLstat(const Path & path); + +/** + * @return true iff the given path exists. + */ +bool pathExists(const Path & path); + +/** + * A version of pathExists that returns false on a permission error. + * Useful for inferring default paths across directories that might not + * be readable. + * @return true iff the given path can be accessed and exists + */ +bool pathAccessible(const Path & path); + +/** + * Read the contents (target) of a symbolic link. The result is not + * in any way canonicalised. + */ +Path readLink(const Path & path); + +bool isLink(const Path & path); + +/** + * Read the contents of a directory. The entries `.` and `..` are + * removed. + */ +struct DirEntry +{ + std::string name; + ino_t ino; + /** + * one of DT_* + */ + unsigned char type; + DirEntry(std::string name, ino_t ino, unsigned char type) + : name(std::move(name)), ino(ino), type(type) { } +}; + +typedef std::vector DirEntries; + +DirEntries readDirectory(const Path & path); + +unsigned char getFileType(const Path & path); + +/** + * Read the contents of a file into a string. + */ +std::string readFile(int fd); +std::string readFile(const Path & path); +void readFile(const Path & path, Sink & sink); + +/** + * Write a string to a file. + */ +void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); + +void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); + +/** + * Flush a file's parent directory to disk + */ +void syncParent(const Path & path); + +/** + * Delete a path; i.e., in the case of a directory, it is deleted + * recursively. It's not an error if the path does not exist. The + * second variant returns the number of bytes and blocks freed. + */ +void deletePath(const Path & path); + +void deletePath(const Path & path, uint64_t & bytesFreed); + +/** + * Create a directory and all its parents, if necessary. Returns the + * list of created directories, in order of creation. + */ +Paths createDirs(const Path & path); +inline Paths createDirs(PathView path) +{ + return createDirs(Path(path)); +} + +/** + * Create a symlink. + */ +void createSymlink(const Path & target, const Path & link); + +/** + * Atomically create or replace a symlink. + */ +void replaceSymlink(const Path & target, const Path & link); + +void renameFile(const Path & src, const Path & dst); + +/** + * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` + * are on a different filesystem. + * + * Beware that this might not be atomic because of the copy that happens behind + * the scenes + */ +void moveFile(const Path & src, const Path & dst); + +struct CopyFileFlags +{ + /** + * Delete the file after copying. + */ + bool deleteAfter = false; + + /** + * Follow symlinks and copy the eventual target. + */ + bool followSymlinks = false; +}; + +/** + * Recursively copy the content of `oldPath` to `newPath`. If `andDelete` is + * `true`, then also remove `oldPath` (making this equivalent to `moveFile`, but + * with the guaranty that the destination will be “fresh”, with no stale inode + * or file descriptor pointing to it). + */ +void copyFile(const Path & oldPath, const Path & newPath, CopyFileFlags flags); + +/** + * Automatic cleanup of resources. + */ + + +class AutoDelete +{ + Path path; + bool del; + bool recursive; +public: + AutoDelete(); + AutoDelete(const Path & p, bool recursive = true); + ~AutoDelete(); + void cancel(); + void reset(const Path & p, bool recursive = true); + operator Path() const { return path; } + operator PathView() const { return path; } +}; + +struct DIRDeleter +{ + void operator()(DIR * dir) const { + closedir(dir); + } +}; + +typedef std::unique_ptr AutoCloseDir; + +/** + * Create a temporary directory. + */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +/** + * Create a temporary file, returning a file handle and its path. + */ +std::pair createTempFile(const Path & prefix = "nix"); + +/** + * Used in various places. + */ +typedef std::function PathFilter; + +extern PathFilter defaultPathFilter; + +} diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc deleted file mode 100644 index 85871870c..000000000 --- a/src/libutil/filesystem.cc +++ /dev/null @@ -1,176 +0,0 @@ -#include -#include -#include - -#include "environment-variables.hh" -#include "finally.hh" -#include "util.hh" -#include "signals.hh" -#include "types.hh" - -namespace fs = std::filesystem; - -namespace nix { - -static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, - std::atomic & counter) -{ - tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); - if (includePid) - return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++); - else - return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++); -} - -Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter, mode_t mode) -{ - static std::atomic globalCounter = 0; - std::atomic localCounter = 0; - auto & counter(useGlobalCounter ? globalCounter : localCounter); - - while (1) { - checkInterrupt(); - Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), mode) == 0) { -#if __FreeBSD__ - /* Explicitly set the group of the directory. This is to - work around around problems caused by BSD's group - ownership semantics (directories inherit the group of - the parent). For instance, the group of /tmp on - FreeBSD is "wheel", so all directories created in /tmp - will be owned by "wheel"; but if the user is not in - "wheel", then "tar" will fail to unpack archives that - have the setgid bit set on directories. */ - if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError("setting group of directory '%1%'", tmpDir); -#endif - return tmpDir; - } - if (errno != EEXIST) - throw SysError("creating directory '%1%'", tmpDir); - } -} - - -std::pair createTempFile(const Path & prefix) -{ - Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); - // Strictly speaking, this is UB, but who cares... - // FIXME: use O_TMPFILE. - AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); - if (!fd) - throw SysError("creating temporary file '%s'", tmpl); - closeOnExec(fd.get()); - return {std::move(fd), tmpl}; -} - -void createSymlink(const Path & target, const Path & link) -{ - if (symlink(target.c_str(), link.c_str())) - throw SysError("creating symlink from '%1%' to '%2%'", link, target); -} - -void replaceSymlink(const Path & target, const Path & link) -{ - for (unsigned int n = 0; true; n++) { - Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); - - try { - createSymlink(target, tmp); - } catch (SysError & e) { - if (e.errNo == EEXIST) continue; - throw; - } - - renameFile(tmp, link); - - break; - } -} - -void setWriteTime(const fs::path & p, const struct stat & st) -{ - struct timeval times[2]; - times[0] = { - .tv_sec = st.st_atime, - .tv_usec = 0, - }; - times[1] = { - .tv_sec = st.st_mtime, - .tv_usec = 0, - }; - if (lutimes(p.c_str(), times) != 0) - throw SysError("changing modification time of '%s'", p); -} - -void copy(const fs::directory_entry & from, const fs::path & to, CopyFileFlags flags) -{ - // TODO: Rewrite the `is_*` to use `symlink_status()` - auto statOfFrom = lstat(from.path().c_str()); - auto fromStatus = from.symlink_status(); - - // Mark the directory as writable so that we can delete its children - if (flags.deleteAfter && fs::is_directory(fromStatus)) { - fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); - } - - - if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { - auto opts = fs::copy_options::overwrite_existing; - - if (!flags.followSymlinks) { - opts |= fs::copy_options::copy_symlinks; - } - - fs::copy(from.path(), to, opts); - } else if (fs::is_directory(fromStatus)) { - fs::create_directory(to); - for (auto & entry : fs::directory_iterator(from.path())) { - copy(entry, to / entry.path().filename(), flags); - } - } else { - throw Error("file '%s' has an unsupported type", from.path()); - } - - setWriteTime(to, statOfFrom); - if (flags.deleteAfter) { - if (!fs::is_symlink(fromStatus)) - fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); - fs::remove(from.path()); - } -} - - -void copyFile(const Path & oldPath, const Path & newPath, CopyFileFlags flags) -{ - return copy(fs::directory_entry(fs::path(oldPath)), fs::path(newPath), flags); -} - -void renameFile(const Path & oldName, const Path & newName) -{ - fs::rename(oldName, newName); -} - -void moveFile(const Path & oldName, const Path & newName) -{ - try { - renameFile(oldName, newName); - } catch (fs::filesystem_error & e) { - auto oldPath = fs::path(oldName); - auto newPath = fs::path(newName); - // For the move to be as atomic as possible, copy to a temporary - // directory - fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); - Finally removeTemp = [&]() { fs::remove(temp); }; - auto tempCopyTarget = temp / "copy-target"; - if (e.code().value() == EXDEV) { - fs::remove(newPath); - warn("Can’t rename %s as %s, copying instead", oldName, newName); - copy(fs::directory_entry(oldPath), tempCopyTarget, { .deleteAfter = true }); - renameFile(tempCopyTarget, newPath); - } - } -} - -} diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ae3ee40f4..bb2a5e2a7 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "serialise.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libutil/input-accessor.hh b/src/libutil/input-accessor.hh index 740175af0..d8e17c9b1 100644 --- a/src/libutil/input-accessor.hh +++ b/src/libutil/input-accessor.hh @@ -1,8 +1,10 @@ #pragma once ///@file -#include "types.hh" -#include "archive.hh" + +#include +#include +#include namespace nix { diff --git a/src/libutil/meson.build b/src/libutil/meson.build index c890911e9..8af251d1b 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -13,7 +13,7 @@ libutil_sources = files( 'escape-string.cc', 'exit.cc', 'experimental-features.cc', - 'filesystem.cc', + 'file-system.cc', 'git.cc', 'hash.cc', 'hilite.cc', @@ -62,6 +62,7 @@ libutil_headers = files( 'exit.hh', 'experimental-features.hh', 'experimental-features-json.hh', + 'file-system.hh', 'finally.hh', 'fmt.hh', 'git.hh', diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index dec3a7189..4b539f342 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -1,5 +1,6 @@ #if __linux__ +#include "file-system.hh" #include "namespaces.hh" #include "util.hh" #include "finally.hh" diff --git a/src/libutil/source-path.hh b/src/libutil/source-path.hh index 4d4c51062..311e9f7a6 100644 --- a/src/libutil/source-path.hh +++ b/src/libutil/source-path.hh @@ -6,7 +6,9 @@ */ #include "ref.hh" +#include "archive.hh" #include "canon-path.hh" +#include "file-system.hh" #include "repair-flag.hh" #include "input-accessor.hh" diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index f3f04a936..cabb7af9b 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -1,6 +1,7 @@ #include #include +#include "file-system.hh" #include "serialise.hh" #include "tarfile.hh" diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 63d9e5248..ac3071ba8 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -5,6 +5,7 @@ #include "cgroup.hh" #include "signals.hh" #include "environment-variables.hh" +#include "file-system.hh" #include #include @@ -50,352 +51,6 @@ namespace nix { -Path absPath(Path path, std::optional dir, bool resolveSymlinks) -{ - if (path.empty() || path[0] != '/') { - if (!dir) { -#ifdef __GNU__ - /* GNU (aka. GNU/Hurd) doesn't have any limitation on path - lengths and doesn't define `PATH_MAX'. */ - char *buf = getcwd(NULL, 0); - if (buf == NULL) -#else - char buf[PATH_MAX]; - if (!getcwd(buf, sizeof(buf))) -#endif - throw SysError("cannot get cwd"); - path = concatStrings(buf, "/", path); -#ifdef __GNU__ - free(buf); -#endif - } else - path = concatStrings(*dir, "/", path); - } - return canonPath(path, resolveSymlinks); -} - - -Path canonPath(PathView path, bool resolveSymlinks) -{ - assert(path != ""); - - std::string s; - s.reserve(256); - - if (path[0] != '/') - throw Error("not an absolute path: '%1%'", path); - - std::string temp; - - /* Count the number of times we follow a symlink and stop at some - arbitrary (but high) limit to prevent infinite loops. */ - unsigned int followCount = 0, maxFollow = 1024; - - while (1) { - - /* Skip slashes. */ - while (!path.empty() && path[0] == '/') path.remove_prefix(1); - if (path.empty()) break; - - /* Ignore `.'. */ - if (path == "." || path.substr(0, 2) == "./") - path.remove_prefix(1); - - /* If `..', delete the last component. */ - else if (path == ".." || path.substr(0, 3) == "../") - { - if (!s.empty()) s.erase(s.rfind('/')); - path.remove_prefix(2); - } - - /* Normal component; copy it. */ - else { - s += '/'; - if (const auto slash = path.find('/'); slash == std::string::npos) { - s += path; - path = {}; - } else { - s += path.substr(0, slash); - path = path.substr(slash); - } - - /* If s points to a symlink, resolve it and continue from there */ - if (resolveSymlinks && isLink(s)) { - if (++followCount >= maxFollow) - throw Error("infinite symlink recursion in path '%1%'", path); - temp = concatStrings(readLink(s), path); - path = temp; - if (!temp.empty() && temp[0] == '/') { - s.clear(); /* restart for symlinks pointing to absolute path */ - } else { - s = dirOf(s); - if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / - s.clear(); - } - } - } - } - } - - return s.empty() ? "/" : std::move(s); -} - -void chmodPath(const Path & path, mode_t mode) -{ - if (chmod(path.c_str(), mode) == -1) - throw SysError("setting permissions on '%s'", path); -} - -Path dirOf(const PathView path) -{ - Path::size_type pos = path.rfind('/'); - if (pos == std::string::npos) - return "."; - return pos == 0 ? "/" : Path(path, 0, pos); -} - - -std::string_view baseNameOf(std::string_view path) -{ - if (path.empty()) - return ""; - - auto last = path.size() - 1; - if (path[last] == '/' && last > 0) - last -= 1; - - auto pos = path.rfind('/', last); - if (pos == std::string::npos) - pos = 0; - else - pos += 1; - - return path.substr(pos, last - pos + 1); -} - - -std::string expandTilde(std::string_view path) -{ - // TODO: expand ~user ? - auto tilde = path.substr(0, 2); - if (tilde == "~/" || tilde == "~") - return getHome() + std::string(path.substr(1)); - else - return std::string(path); -} - - -bool isInDir(std::string_view path, std::string_view dir) -{ - return path.substr(0, 1) == "/" - && path.substr(0, dir.size()) == dir - && path.size() >= dir.size() + 2 - && path[dir.size()] == '/'; -} - - -bool isDirOrInDir(std::string_view path, std::string_view dir) -{ - return path == dir || isInDir(path, dir); -} - - -struct stat stat(const Path & path) -{ - struct stat st; - if (stat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); - return st; -} - - -struct stat lstat(const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); - return st; -} - -std::optional maybeLstat(const Path & path) -{ - std::optional st{std::in_place}; - if (lstat(path.c_str(), &*st)) - { - if (errno == ENOENT || errno == ENOTDIR) - st.reset(); - else - throw SysError("getting status of '%s'", path); - } - return st; -} - -bool pathExists(const Path & path) -{ - return maybeLstat(path).has_value(); -} - -bool pathAccessible(const Path & path) -{ - try { - return pathExists(path); - } catch (SysError & e) { - // swallow EPERM - if (e.errNo == EPERM) return false; - throw; - } -} - - -Path readLink(const Path & path) -{ - checkInterrupt(); - std::vector buf; - for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { - buf.resize(bufSize); - ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); - if (rlSize == -1) - if (errno == EINVAL) - throw Error("'%1%' is not a symlink", path); - else - throw SysError("reading symbolic link '%1%'", path); - else if (rlSize < bufSize) - return std::string(buf.data(), rlSize); - } -} - - -bool isLink(const Path & path) -{ - struct stat st = lstat(path); - return S_ISLNK(st.st_mode); -} - - -DirEntries readDirectory(DIR *dir, const Path & path) -{ - DirEntries entries; - entries.reserve(64); - - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { /* sic */ - checkInterrupt(); - std::string name = dirent->d_name; - if (name == "." || name == "..") continue; - entries.emplace_back(name, dirent->d_ino, -#ifdef HAVE_STRUCT_DIRENT_D_TYPE - dirent->d_type -#else - DT_UNKNOWN -#endif - ); - } - if (errno) throw SysError("reading directory '%1%'", path); - - return entries; -} - -DirEntries readDirectory(const Path & path) -{ - AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError("opening directory '%1%'", path); - - return readDirectory(dir.get(), path); -} - - -unsigned char getFileType(const Path & path) -{ - struct stat st = lstat(path); - if (S_ISDIR(st.st_mode)) return DT_DIR; - if (S_ISLNK(st.st_mode)) return DT_LNK; - if (S_ISREG(st.st_mode)) return DT_REG; - return DT_UNKNOWN; -} - - -std::string readFile(int fd) -{ - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError("statting file"); - - return drainFD(fd, true, st.st_size); -} - - -std::string readFile(const Path & path) -{ - AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; - if (!fd) - throw SysError("opening file '%1%'", path); - return readFile(fd.get()); -} - - -void readFile(const Path & path, Sink & sink) -{ - AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; - if (!fd) - throw SysError("opening file '%s'", path); - drainFD(fd.get(), sink); -} - - -void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) -{ - AutoCloseFD fd{open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode)}; - if (!fd) - throw SysError("opening file '%1%'", path); - try { - writeFull(fd.get(), s); - } catch (Error & e) { - e.addTrace({}, "writing file '%1%'", path); - throw; - } - if (sync) - fd.fsync(); - // Explicitly close to make sure exceptions are propagated. - fd.close(); - if (sync) - syncParent(path); -} - - -void writeFile(const Path & path, Source & source, mode_t mode, bool sync) -{ - AutoCloseFD fd{open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode)}; - if (!fd) - throw SysError("opening file '%1%'", path); - - std::vector buf(64 * 1024); - - try { - while (true) { - try { - auto n = source.read(buf.data(), buf.size()); - writeFull(fd.get(), {buf.data(), n}); - } catch (EndOfFile &) { break; } - } - } catch (Error & e) { - e.addTrace({}, "writing file '%1%'", path); - throw; - } - if (sync) - fd.fsync(); - // Explicitly close to make sure exceptions are propagated. - fd.close(); - if (sync) - syncParent(path); -} - -void syncParent(const Path & path) -{ - AutoCloseFD fd{open(dirOf(path).c_str(), O_RDONLY, 0)}; - if (!fd) - throw SysError("opening file '%1%'", path); - fd.fsync(); -} std::string readLine(int fd) { @@ -425,98 +80,6 @@ void writeLine(int fd, std::string s) } -static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) -{ - checkInterrupt(); - - std::string name(baseNameOf(path)); - - struct stat st; - if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { - if (errno == ENOENT) return; - throw SysError("getting status of '%1%'", path); - } - - if (!S_ISDIR(st.st_mode)) { - /* We are about to delete a file. Will it likely free space? */ - - switch (st.st_nlink) { - /* Yes: last link. */ - case 1: - bytesFreed += st.st_size; - break; - /* Maybe: yes, if 'auto-optimise-store' or manual optimisation - was performed. Instead of checking for real let's assume - it's an optimised file and space will be freed. - - In worst case we will double count on freed space for files - with exactly two hardlinks for unoptimised packages. - */ - case 2: - bytesFreed += st.st_size; - break; - /* No: 3+ links. */ - default: - break; - } - } - - if (S_ISDIR(st.st_mode)) { - /* Make the directory accessible. */ - const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; - if ((st.st_mode & PERM_MASK) != PERM_MASK) { - if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) - throw SysError("chmod '%1%'", path); - } - - int fd = openat(parentfd, path.c_str(), O_RDONLY); - if (fd == -1) - throw SysError("opening directory '%1%'", path); - AutoCloseDir dir(fdopendir(fd)); - if (!dir) - throw SysError("opening directory '%1%'", path); - for (auto & i : readDirectory(dir.get(), path)) - _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); - } - - int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; - if (unlinkat(parentfd, name.c_str(), flags) == -1) { - if (errno == ENOENT) return; - throw SysError("cannot unlink '%1%'", path); - } -} - -static void _deletePath(const Path & path, uint64_t & bytesFreed) -{ - Path dir = dirOf(path); - if (dir == "") - dir = "/"; - - AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)}; - if (!dirfd) { - if (errno == ENOENT) return; - throw SysError("opening directory '%1%'", path); - } - - _deletePath(dirfd.get(), path, bytesFreed); -} - - -void deletePath(const Path & path) -{ - uint64_t dummy; - deletePath(path, dummy); -} - - -void deletePath(const Path & path, uint64_t & bytesFreed) -{ - //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); - bytesFreed = 0; - _deletePath(path, bytesFreed); -} - - std::string getUserName() { auto pw = getpwuid(geteuid()); @@ -632,28 +195,6 @@ std::optional getSelfExe() } -Paths createDirs(const Path & path) -{ - Paths created; - if (path == "/") return created; - - struct stat st; - if (lstat(path.c_str(), &st) == -1) { - created = createDirs(dirOf(path)); - if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) - throw SysError("creating directory '%1%'", path); - st = lstat(path); - created.push_back(path); - } - - if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) - throw SysError("statting symlink '%1%'", path); - - if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); - - return created; -} - void readFull(int fd, char * buf, size_t count) { @@ -762,46 +303,6 @@ unsigned int getMaxCPU() ////////////////////////////////////////////////////////////////////// -AutoDelete::AutoDelete() : del{false} {} - -AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p) -{ - del = true; - this->recursive = recursive; -} - -AutoDelete::~AutoDelete() -{ - try { - if (del) { - if (recursive) - deletePath(path); - else { - if (remove(path.c_str()) == -1) - throw SysError("cannot unlink '%1%'", path); - } - } - } catch (...) { - ignoreException(); - } -} - -void AutoDelete::cancel() -{ - del = false; -} - -void AutoDelete::reset(const Path & p, bool recursive) { - path = p; - this->recursive = recursive; - del = true; -} - - - -////////////////////////////////////////////////////////////////////// - - AutoCloseFD::AutoCloseFD() : fd{-1} {} diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 1ce7e8312..63a4fcb6a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -39,135 +39,6 @@ struct Source; extern const std::string nativeSystem; -/** - * @return An absolutized path, resolving paths relative to the - * specified directory, or the current directory otherwise. The path - * is also canonicalised. - */ -Path absPath(Path path, - std::optional dir = {}, - bool resolveSymlinks = false); - -/** - * Canonicalise a path by removing all `.` or `..` components and - * double or trailing slashes. Optionally resolves all symlink - * components such that each component of the resulting path is *not* - * a symbolic link. - */ -Path canonPath(PathView path, bool resolveSymlinks = false); - -/** - * Change the permissions of a path - * Not called `chmod` as it shadows and could be confused with - * `int chmod(char *, mode_t)`, which does not handle errors - */ -void chmodPath(const Path & path, mode_t mode); - -/** - * @return The directory part of the given canonical path, i.e., - * everything before the final `/`. If the path is the root or an - * immediate child thereof (e.g., `/foo`), this means `/` - * is returned. - */ -Path dirOf(const PathView path); - -/** - * @return the base name of the given canonical path, i.e., everything - * following the final `/` (trailing slashes are removed). - */ -std::string_view baseNameOf(std::string_view path); - -/** - * Perform tilde expansion on a path. - */ -std::string expandTilde(std::string_view path); - -/** - * Check whether 'path' is a descendant of 'dir'. Both paths must be - * canonicalized. - */ -bool isInDir(std::string_view path, std::string_view dir); - -/** - * Check whether 'path' is equal to 'dir' or a descendant of - * 'dir'. Both paths must be canonicalized. - */ -bool isDirOrInDir(std::string_view path, std::string_view dir); - -/** - * Get status of `path`. - */ -struct stat stat(const Path & path); -struct stat lstat(const Path & path); - -/** - * `lstat` the given path if it exists. - * @return std::nullopt if the path doesn't exist, or an optional containing the result of `lstat` otherwise - */ -std::optional maybeLstat(const Path & path); - -/** - * @return true iff the given path exists. - */ -bool pathExists(const Path & path); - -/** - * A version of pathExists that returns false on a permission error. - * Useful for inferring default paths across directories that might not - * be readable. - * @return true iff the given path can be accessed and exists - */ -bool pathAccessible(const Path & path); - -/** - * Read the contents (target) of a symbolic link. The result is not - * in any way canonicalised. - */ -Path readLink(const Path & path); - -bool isLink(const Path & path); - -/** - * Read the contents of a directory. The entries `.` and `..` are - * removed. - */ -struct DirEntry -{ - std::string name; - ino_t ino; - /** - * one of DT_* - */ - unsigned char type; - DirEntry(std::string name, ino_t ino, unsigned char type) - : name(std::move(name)), ino(ino), type(type) { } -}; - -typedef std::vector DirEntries; - -DirEntries readDirectory(const Path & path); - -unsigned char getFileType(const Path & path); - -/** - * Read the contents of a file into a string. - */ -std::string readFile(int fd); -std::string readFile(const Path & path); -void readFile(const Path & path, Sink & sink); - -/** - * Write a string to a file. - */ -void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); - -void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); - -/** - * Flush a file's parent directory to disk - */ -void syncParent(const Path & path); - /** * Read a line from a file descriptor. */ @@ -178,14 +49,6 @@ std::string readLine(int fd); */ void writeLine(int fd, std::string s); -/** - * Delete a path; i.e., in the case of a directory, it is deleted - * recursively. It's not an error if the path does not exist. The - * second variant returns the number of bytes and blocks freed. - */ -void deletePath(const Path & path); - -void deletePath(const Path & path, uint64_t & bytesFreed); std::string getUserName(); @@ -238,57 +101,6 @@ Path getStateDir(); */ Path createNixStateDir(); -/** - * Create a directory and all its parents, if necessary. Returns the - * list of created directories, in order of creation. - */ -Paths createDirs(const Path & path); -inline Paths createDirs(PathView path) -{ - return createDirs(Path(path)); -} - -/** - * Create a symlink. - */ -void createSymlink(const Path & target, const Path & link); - -/** - * Atomically create or replace a symlink. - */ -void replaceSymlink(const Path & target, const Path & link); - -void renameFile(const Path & src, const Path & dst); - -/** - * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` - * are on a different filesystem. - * - * Beware that this might not be atomic because of the copy that happens behind - * the scenes - */ -void moveFile(const Path & src, const Path & dst); - -struct CopyFileFlags -{ - /** - * Delete the file after copying. - */ - bool deleteAfter = false; - - /** - * Follow symlinks and copy the eventual target. - */ - bool followSymlinks = false; -}; - -/** - * Recursively copy the content of `oldPath` to `newPath`. If `andDelete` is - * `true`, then also remove `oldPath` (making this equivalent to `moveFile`, but - * with the guaranty that the destination will be “fresh”, with no stale inode - * or file descriptor pointing to it). - */ -void copyFile(const Path & oldPath, const Path & newPath, CopyFileFlags flags); /** * Wrappers arount read()/write() that read/write exactly the @@ -313,27 +125,6 @@ void drainFD(int fd, Sink & sink, bool block = true); */ unsigned int getMaxCPU(); -/** - * Automatic cleanup of resources. - */ - - -class AutoDelete -{ - Path path; - bool del; - bool recursive; -public: - AutoDelete(); - AutoDelete(const Path & p, bool recursive = true); - ~AutoDelete(); - void cancel(); - void reset(const Path & p, bool recursive = true); - operator Path() const { return path; } - operator PathView() const { return path; } -}; - - class AutoCloseFD { int fd; @@ -353,19 +144,6 @@ public: void reset() { *this = {}; } }; - -/** - * Create a temporary directory. - */ -Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); - -/** - * Create a temporary file, returning a file handle and its path. - */ -std::pair createTempFile(const Path & prefix = "nix"); - - class Pipe { public: @@ -375,16 +153,6 @@ public: }; -struct DIRDeleter -{ - void operator()(DIR * dir) const { - closedir(dir); - } -}; - -typedef std::unique_ptr AutoCloseDir; - - class Pid { pid_t pid = -1; @@ -823,13 +591,6 @@ struct MaintainCount }; -/** - * Used in various places. - */ -typedef std::function PathFilter; - -extern PathFilter defaultPathFilter; - /** * Common initialisation performed in child processes. */ diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 1cbba0537..54de68232 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -1,3 +1,4 @@ +#include "file-system.hh" #include "store-api.hh" #include "store-cast.hh" #include "gc-store.hh" diff --git a/tests/unit/libstore/machines.cc b/tests/unit/libstore/machines.cc index 0277237bb..dc192e5be 100644 --- a/tests/unit/libstore/machines.cc +++ b/tests/unit/libstore/machines.cc @@ -1,3 +1,4 @@ +#include "file-system.hh" #include "machines.hh" #include "globals.hh" #include "tests/test-data.hh" diff --git a/tests/unit/libutil-support/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh index 9de20d02c..472fdba3b 100644 --- a/tests/unit/libutil-support/tests/characterization.hh +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -2,6 +2,8 @@ ///@file #include "environment-variables.hh" +#include "file-system.hh" + #include #include diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index 787f79093..c8dc82cb0 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -1,3 +1,4 @@ +#include "file-system.hh" #include "util.hh" #include "types.hh" #include "terminal.hh" From 8cd9aa24a8d48caf22225ce32dff3c5aaa111687 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 12:25:49 +0200 Subject: [PATCH 04/16] util.{hh,cc}: Split out file-descriptor.{hh,cc} Change-Id: I0dd0f9a9c2003fb887e076127e7f825fd3289c76 --- src/libstore/lock.cc | 1 + src/libstore/pathlocks.hh | 2 +- src/libutil/file-descriptor.cc | 251 ++++++++++++++++++ src/libutil/file-descriptor.hh | 89 +++++++ src/libutil/file-system.cc | 12 +- src/libutil/file-system.hh | 3 +- src/libutil/logging.cc | 1 + src/libutil/meson.build | 2 + src/libutil/serialise.hh | 1 + src/libutil/util.cc | 229 ---------------- src/libutil/util.hh | 77 ------ .../repl_characterization/test-session.hh | 2 +- 12 files changed, 349 insertions(+), 321 deletions(-) create mode 100644 src/libutil/file-descriptor.cc create mode 100644 src/libutil/file-descriptor.hh diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index b52e87ae0..05296757d 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -1,4 +1,5 @@ #include "lock.hh" +#include "logging.hh" #include "file-system.hh" #include "globals.hh" #include "pathlocks.hh" diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 4921df352..7fcfa2e40 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "util.hh" +#include "file-descriptor.hh" namespace nix { diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc new file mode 100644 index 000000000..ec22f17ab --- /dev/null +++ b/src/libutil/file-descriptor.cc @@ -0,0 +1,251 @@ +#include "file-system.hh" +#include "finally.hh" +#include "serialise.hh" +#include "signals.hh" + +#include +#include + +namespace nix { + +std::string readFile(int fd) +{ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + return drainFD(fd, true, st.st_size); +} + + +std::string readLine(int fd) +{ + std::string s; + while (1) { + checkInterrupt(); + char ch; + // FIXME: inefficient + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading a line"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; + } + } +} + + +void writeLine(int fd, std::string s) +{ + s += '\n'; + writeFull(fd, s); +} + + +void readFull(int fd, char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + ssize_t res = read(fd, buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("reading from file"); + } + if (res == 0) throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + + +void writeFull(int fd, std::string_view s, bool allowInterrupts) +{ + while (!s.empty()) { + if (allowInterrupts) checkInterrupt(); + ssize_t res = write(fd, s.data(), s.size()); + if (res == -1 && errno != EINTR) + throw SysError("writing to file"); + if (res > 0) + s.remove_prefix(res); + } +} + + +std::string drainFD(int fd, bool block, const size_t reserveSize) +{ + // the parser needs two extra bytes to append terminating characters, other users will + // not care very much about the extra memory. + StringSink sink(reserveSize + 2); + drainFD(fd, sink, block); + return std::move(sink.s); +} + + +void drainFD(int fd, Sink & sink, bool block) +{ + // silence GCC maybe-uninitialized warning in finally + int saved = 0; + + if (!block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) + throw SysError("making file descriptor non-blocking"); + } + + Finally finally([&]() { + if (!block) { + if (fcntl(fd, F_SETFL, saved) == -1) + throw SysError("making file descriptor blocking"); + } + }); + + std::array buf; + while (1) { + checkInterrupt(); + ssize_t rd = read(fd, buf.data(), buf.size()); + if (rd == -1) { + if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) + break; + if (errno != EINTR) + throw SysError("reading from file"); + } + else if (rd == 0) break; + else sink({(char *) buf.data(), (size_t) rd}); + } +} + +AutoCloseFD::AutoCloseFD() : fd{-1} {} + + +AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} + + +AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} +{ + that.fd = -1; +} + + +AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) +{ + close(); + fd = that.fd; + that.fd = -1; + return *this; +} + + +AutoCloseFD::~AutoCloseFD() +{ + try { + close(); + } catch (...) { + ignoreException(); + } +} + + +int AutoCloseFD::get() const +{ + return fd; +} + + +void AutoCloseFD::close() +{ + if (fd != -1) { + if (::close(fd) == -1) + /* This should never happen. */ + throw SysError("closing file descriptor %1%", fd); + fd = -1; + } +} + +void AutoCloseFD::fsync() +{ + if (fd != -1) { + int result; +#if __APPLE__ + result = ::fcntl(fd, F_FULLFSYNC); +#else + result = ::fsync(fd); +#endif + if (result == -1) + throw SysError("fsync file descriptor %1%", fd); + } +} + + +AutoCloseFD::operator bool() const +{ + return fd != -1; +} + + +int AutoCloseFD::release() +{ + int oldFD = fd; + fd = -1; + return oldFD; +} + + +void Pipe::create() +{ + int fds[2]; +#if HAVE_PIPE2 + if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); +#else + if (pipe(fds) != 0) throw SysError("creating pipe"); + closeOnExec(fds[0]); + closeOnExec(fds[1]); +#endif + readSide = AutoCloseFD{fds[0]}; + writeSide = AutoCloseFD{fds[1]}; +} + + +void Pipe::close() +{ + readSide.close(); + writeSide.close(); +} + + +void closeMostFDs(const std::set & exceptions) +{ +#if __linux__ + try { + for (auto & s : readDirectory("/proc/self/fd")) { + auto fd = std::stoi(s.name); + if (!exceptions.count(fd)) { + debug("closing leaked FD %d", fd); + close(fd); + } + } + return; + } catch (SysError &) { + } +#endif + + int maxFD = 0; + maxFD = sysconf(_SC_OPEN_MAX); + for (int fd = 0; fd < maxFD; ++fd) + if (!exceptions.count(fd)) + close(fd); /* ignore result */ +} + + +void closeOnExec(int fd) +{ + int prev; + if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || + fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) + throw SysError("setting close-on-exec flag"); +} + +} diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh new file mode 100644 index 000000000..68324b9d9 --- /dev/null +++ b/src/libutil/file-descriptor.hh @@ -0,0 +1,89 @@ +#pragma once +///@file + +#include "error.hh" + +namespace nix { + +struct Sink; +struct Source; + +/** + * Read a line from a file descriptor. + */ +std::string readLine(int fd); + +/** + * Write a line to a file descriptor. + */ +void writeLine(int fd, std::string s); + +/** + * Read the contents of a file into a string. + */ +std::string readFile(int fd); + +/** + * Wrappers arount read()/write() that read/write exactly the + * requested number of bytes. + */ +void readFull(int fd, char * buf, size_t count); +void writeFull(int fd, std::string_view s, bool allowInterrupts = true); + +/** + * Read a file descriptor until EOF occurs. + */ +std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); + +void drainFD(int fd, Sink & sink, bool block = true); + +class AutoCloseFD +{ + int fd; +public: + AutoCloseFD(); + explicit AutoCloseFD(int fd); + AutoCloseFD(const AutoCloseFD & fd) = delete; + AutoCloseFD(AutoCloseFD&& fd); + ~AutoCloseFD(); + AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; + AutoCloseFD& operator =(AutoCloseFD&& fd) noexcept(false); + int get() const; + explicit operator bool() const; + int release(); + void close(); + void fsync(); + void reset() { *this = {}; } +}; + +class Pipe +{ +public: + AutoCloseFD readSide, writeSide; + void create(); + void close(); +}; + +/** + * Close all file descriptors except those listed in the given set. + * Good practice in child processes. + */ +void closeMostFDs(const std::set & exceptions); + +/** + * Set the close-on-exec flag for the given file descriptor. + */ +void closeOnExec(int fd); + +MakeError(EndOfFile, Error); + +/** + * Create a Unix domain socket. + */ +AutoCloseFD createUnixDomainSocket(); + +/** + * Create a Unix domain socket in listen mode. + */ +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); +} diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index e5ba42eb6..9804c449c 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -3,10 +3,10 @@ #include #include "environment-variables.hh" +#include "file-descriptor.hh" #include "file-system.hh" #include "finally.hh" #include "serialise.hh" -#include "util.hh" #include "signals.hh" #include "types.hh" @@ -278,16 +278,6 @@ unsigned char getFileType(const Path & path) } -std::string readFile(int fd) -{ - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError("statting file"); - - return drainFD(fd, true, st.st_size); -} - - std::string readFile(const Path & path) { AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 1d91ef334..b9b753980 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -6,7 +6,7 @@ */ #include "types.hh" -#include "util.hh" +#include "file-descriptor.hh" #include #include @@ -143,7 +143,6 @@ unsigned char getFileType(const Path & path); /** * Read the contents of a file into a string. */ -std::string readFile(int fd); std::string readFile(const Path & path); void readFile(const Path & path, Sink & sink); diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 731d1034d..8d9e18d09 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -1,4 +1,5 @@ #include "environment-variables.hh" +#include "file-descriptor.hh" #include "logging.hh" #include "util.hh" #include "config.hh" diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 8af251d1b..a4813d9a3 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -13,6 +13,7 @@ libutil_sources = files( 'escape-string.cc', 'exit.cc', 'experimental-features.cc', + 'file-descriptor.cc', 'file-system.cc', 'git.cc', 'hash.cc', @@ -62,6 +63,7 @@ libutil_headers = files( 'exit.hh', 'experimental-features.hh', 'experimental-features-json.hh', + 'file-descriptor.hh', 'file-system.hh', 'finally.hh', 'fmt.hh', diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index d1c791823..d4a14bbf0 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -5,6 +5,7 @@ #include "types.hh" #include "util.hh" +#include "file-descriptor.hh" namespace boost::context { struct stack_context; } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index ac3071ba8..55f3154c9 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -52,34 +52,6 @@ namespace nix { -std::string readLine(int fd) -{ - std::string s; - while (1) { - checkInterrupt(); - char ch; - // FIXME: inefficient - ssize_t rd = read(fd, &ch, 1); - if (rd == -1) { - if (errno != EINTR) - throw SysError("reading a line"); - } else if (rd == 0) - throw EndOfFile("unexpected EOF reading a line"); - else { - if (ch == '\n') return s; - s += ch; - } - } -} - - -void writeLine(int fd, std::string s) -{ - s += '\n'; - writeFull(fd, s); -} - - std::string getUserName() { auto pw = getpwuid(geteuid()); @@ -196,78 +168,6 @@ std::optional getSelfExe() -void readFull(int fd, char * buf, size_t count) -{ - while (count) { - checkInterrupt(); - ssize_t res = read(fd, buf, count); - if (res == -1) { - if (errno == EINTR) continue; - throw SysError("reading from file"); - } - if (res == 0) throw EndOfFile("unexpected end-of-file"); - count -= res; - buf += res; - } -} - - -void writeFull(int fd, std::string_view s, bool allowInterrupts) -{ - while (!s.empty()) { - if (allowInterrupts) checkInterrupt(); - ssize_t res = write(fd, s.data(), s.size()); - if (res == -1 && errno != EINTR) - throw SysError("writing to file"); - if (res > 0) - s.remove_prefix(res); - } -} - - -std::string drainFD(int fd, bool block, const size_t reserveSize) -{ - // the parser needs two extra bytes to append terminating characters, other users will - // not care very much about the extra memory. - StringSink sink(reserveSize + 2); - drainFD(fd, sink, block); - return std::move(sink.s); -} - - -void drainFD(int fd, Sink & sink, bool block) -{ - // silence GCC maybe-uninitialized warning in finally - int saved = 0; - - if (!block) { - saved = fcntl(fd, F_GETFL); - if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) - throw SysError("making file descriptor non-blocking"); - } - - Finally finally([&]() { - if (!block) { - if (fcntl(fd, F_SETFL, saved) == -1) - throw SysError("making file descriptor blocking"); - } - }); - - std::array buf; - while (1) { - checkInterrupt(); - ssize_t rd = read(fd, buf.data(), buf.size()); - if (rd == -1) { - if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) - break; - if (errno != EINTR) - throw SysError("reading from file"); - } - else if (rd == 0) break; - else sink({(char *) buf.data(), (size_t) rd}); - } -} - ////////////////////////////////////////////////////////////////////// unsigned int getMaxCPU() @@ -303,102 +203,6 @@ unsigned int getMaxCPU() ////////////////////////////////////////////////////////////////////// -AutoCloseFD::AutoCloseFD() : fd{-1} {} - - -AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} - - -AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} -{ - that.fd = -1; -} - - -AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) -{ - close(); - fd = that.fd; - that.fd = -1; - return *this; -} - - -AutoCloseFD::~AutoCloseFD() -{ - try { - close(); - } catch (...) { - ignoreException(); - } -} - - -int AutoCloseFD::get() const -{ - return fd; -} - - -void AutoCloseFD::close() -{ - if (fd != -1) { - if (::close(fd) == -1) - /* This should never happen. */ - throw SysError("closing file descriptor %1%", fd); - fd = -1; - } -} - -void AutoCloseFD::fsync() -{ - if (fd != -1) { - int result; -#if __APPLE__ - result = ::fcntl(fd, F_FULLFSYNC); -#else - result = ::fsync(fd); -#endif - if (result == -1) - throw SysError("fsync file descriptor %1%", fd); - } -} - - -AutoCloseFD::operator bool() const -{ - return fd != -1; -} - - -int AutoCloseFD::release() -{ - int oldFD = fd; - fd = -1; - return oldFD; -} - - -void Pipe::create() -{ - int fds[2]; -#if HAVE_PIPE2 - if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); -#else - if (pipe(fds) != 0) throw SysError("creating pipe"); - closeOnExec(fds[0]); - closeOnExec(fds[1]); -#endif - readSide = AutoCloseFD{fds[0]}; - writeSide = AutoCloseFD{fds[1]}; -} - - -void Pipe::close() -{ - readSide.close(); - writeSide.close(); -} ////////////////////////////////////////////////////////////////////// @@ -762,39 +566,6 @@ void runProgram2(const RunOptions & options) } -void closeMostFDs(const std::set & exceptions) -{ -#if __linux__ - try { - for (auto & s : readDirectory("/proc/self/fd")) { - auto fd = std::stoi(s.name); - if (!exceptions.count(fd)) { - debug("closing leaked FD %d", fd); - close(fd); - } - } - return; - } catch (SysError &) { - } -#endif - - int maxFD = 0; - maxFD = sysconf(_SC_OPEN_MAX); - for (int fd = 0; fd < maxFD; ++fd) - if (!exceptions.count(fd)) - close(fd); /* ignore result */ -} - - -void closeOnExec(int fd) -{ - int prev; - if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || - fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) - throw SysError("setting close-on-exec flag"); -} - - ////////////////////////////////////////////////////////////////////// diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 63a4fcb6a..d0e207146 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -39,17 +39,6 @@ struct Source; extern const std::string nativeSystem; -/** - * Read a line from a file descriptor. - */ -std::string readLine(int fd); - -/** - * Write a line to a file descriptor. - */ -void writeLine(int fd, std::string s); - - std::string getUserName(); /** @@ -102,57 +91,12 @@ Path getStateDir(); Path createNixStateDir(); -/** - * Wrappers arount read()/write() that read/write exactly the - * requested number of bytes. - */ -void readFull(int fd, char * buf, size_t count); -void writeFull(int fd, std::string_view s, bool allowInterrupts = true); - -MakeError(EndOfFile, Error); - - -/** - * Read a file descriptor until EOF occurs. - */ -std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); - -void drainFD(int fd, Sink & sink, bool block = true); - /** * If cgroups are active, attempt to calculate the number of CPUs available. * If cgroups are unavailable or if cpu.max is set to "max", return 0. */ unsigned int getMaxCPU(); -class AutoCloseFD -{ - int fd; -public: - AutoCloseFD(); - explicit AutoCloseFD(int fd); - AutoCloseFD(const AutoCloseFD & fd) = delete; - AutoCloseFD(AutoCloseFD&& fd); - ~AutoCloseFD(); - AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; - AutoCloseFD& operator =(AutoCloseFD&& fd) noexcept(false); - int get() const; - explicit operator bool() const; - int release(); - void close(); - void fsync(); - void reset() { *this = {}; } -}; - -class Pipe -{ -public: - AutoCloseFD readSide, writeSide; - void create(); - void close(); -}; - - class Pid { pid_t pid = -1; @@ -172,7 +116,6 @@ public: pid_t release(); }; - /** * Kill all processes running under the specified uid by sending them * a SIGKILL. @@ -279,17 +222,6 @@ public: */ std::vector stringsToCharPtrs(const Strings & ss); -/** - * Close all file descriptors except those listed in the given set. - * Good practice in child processes. - */ -void closeMostFDs(const std::set & exceptions); - -/** - * Set the close-on-exec flag for the given file descriptor. - */ -void closeOnExec(int fd); - MakeError(FormatError, Error); @@ -596,15 +528,6 @@ struct MaintainCount */ void commonChildInit(); -/** - * Create a Unix domain socket. - */ -AutoCloseFD createUnixDomainSocket(); - -/** - * Create a Unix domain socket in listen mode. - */ -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); /** * Bind a Unix domain socket to a path. diff --git a/tests/functional/repl_characterization/test-session.hh b/tests/functional/repl_characterization/test-session.hh index 9961849fc..c77cce6d5 100644 --- a/tests/functional/repl_characterization/test-session.hh +++ b/tests/functional/repl_characterization/test-session.hh @@ -6,7 +6,7 @@ #include #include -#include "util.hh" +#include "file-descriptor.hh" #include "tests/terminal-code-eater.hh" namespace nix { From 9a52e4688ca265155817817f373938428f023966 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 13:14:13 +0200 Subject: [PATCH 05/16] util.{hh,cc}: Split out processes.{hh,cc} Change-Id: I39280dc40ca3f7f9007bc6c898ffcf760e2238b7 --- src/libexpr/primops.cc | 2 +- src/libfetchers/git.cc | 1 + src/libfetchers/mercurial.cc | 3 +- src/libmain/shared.hh | 1 + src/libstore/build/hook-instance.hh | 1 + src/libstore/build/local-derivation-goal.hh | 1 + src/libstore/gc.cc | 1 + src/libstore/globals.cc | 1 + src/libstore/ssh.cc | 1 + src/libstore/ssh.hh | 2 +- src/libutil/meson.build | 2 + src/libutil/namespaces.cc | 5 +- src/libutil/processes.cc | 404 ++++++++++++++++++ src/libutil/processes.hh | 115 +++++ src/libutil/util.cc | 376 +--------------- src/libutil/util.hh | 91 ---- src/nix/upgrade-nix.cc | 1 + .../repl_characterization/test-session.cc | 1 + tests/unit/libutil/tests.cc | 2 +- 19 files changed, 539 insertions(+), 472 deletions(-) create mode 100644 src/libutil/processes.cc create mode 100644 src/libutil/processes.hh diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 64a52dfd6..f8ce90ac1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -9,8 +9,8 @@ #include "json-to-value.hh" #include "names.hh" #include "path-references.hh" +#include "processes.hh" #include "store-api.hh" -#include "util.hh" #include "value-to-json.hh" #include "value-to-xml.hh" #include "primops.hh" diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 4d4bf6cd6..86d9d437f 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1,6 +1,7 @@ #include "fetchers.hh" #include "cache.hh" #include "globals.hh" +#include "processes.hh" #include "tarfile.hh" #include "store-api.hh" #include "url-parts.hh" diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 87fecfca3..086db498f 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -1,7 +1,6 @@ #include "fetchers.hh" #include "cache.hh" -#include "globals.hh" -#include "tarfile.hh" +#include "processes.hh" #include "store-api.hh" #include "url-parts.hh" diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index a7810f77c..907f336f1 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -7,6 +7,7 @@ #include "common-args.hh" #include "path.hh" #include "derived-path.hh" +#include "processes.hh" #include "exit.hh" #include diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/build/hook-instance.hh index d84f62877..481158296 100644 --- a/src/libstore/build/hook-instance.hh +++ b/src/libstore/build/hook-instance.hh @@ -2,6 +2,7 @@ ///@file #include "logging.hh" +#include "processes.hh" #include "serialise.hh" namespace nix { diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index b7f317fb6..f3a83d42f 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -3,6 +3,7 @@ #include "derivation-goal.hh" #include "local-store.hh" +#include "processes.hh" namespace nix { diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 6b37f0af3..60d88f3b2 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,6 +1,7 @@ #include "derivations.hh" #include "globals.hh" #include "local-store.hh" +#include "processes.hh" #include "signals.hh" #include "finally.hh" diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 897d3e8ef..f14e6c91f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -26,6 +26,7 @@ #include "config-impl.hh" #ifdef __APPLE__ +#include "processes.hh" #include #include #endif diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 2f921e2d0..1d8266b9d 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,6 +1,7 @@ #include "environment-variables.hh" #include "ssh.hh" #include "finally.hh" +#include "util.hh" namespace nix { diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index 0802c6cc0..f9c532caa 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -2,7 +2,7 @@ ///@file #include "file-system.hh" -#include "util.hh" +#include "processes.hh" #include "sync.hh" namespace nix { diff --git a/src/libutil/meson.build b/src/libutil/meson.build index a4813d9a3..34715b25e 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -23,6 +23,7 @@ libutil_sources = files( 'namespaces.cc', 'position.cc', 'print-elided.cc', + 'processes.cc', 'references.cc', 'regex.cc', 'serialise.cc', @@ -81,6 +82,7 @@ libutil_headers = files( 'pool.hh', 'position.hh', 'print-elided.hh', + 'processes.hh', 'ref.hh', 'references.hh', 'regex-combinators.hh', diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index 4b539f342..cde442671 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -1,9 +1,10 @@ #if __linux__ #include "file-system.hh" -#include "namespaces.hh" +#include "logging.hh" #include "util.hh" -#include "finally.hh" +#include "namespaces.hh" +#include "processes.hh" #include diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc new file mode 100644 index 000000000..8d1b119b0 --- /dev/null +++ b/src/libutil/processes.cc @@ -0,0 +1,404 @@ +#include "environment-variables.hh" +#include "finally.hh" +#include "logging.hh" +#include "processes.hh" +#include "serialise.hh" +#include "signals.hh" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#ifdef __APPLE__ +# include +#endif + +#ifdef __linux__ +# include +# include +#endif + + +namespace nix { + +Pid::Pid() +{ +} + + +Pid::Pid(pid_t pid) + : pid(pid) +{ +} + + +Pid::~Pid() noexcept(false) +{ + if (pid != -1) kill(); +} + + +void Pid::operator =(pid_t pid) +{ + if (this->pid != -1 && this->pid != pid) kill(); + this->pid = pid; + killSignal = SIGKILL; // reset signal to default +} + + +Pid::operator pid_t() +{ + return pid; +} + + +int Pid::kill() +{ + assert(pid != -1); + + debug("killing process %1%", pid); + + /* Send the requested signal to the child. If it has its own + process group, send the signal to every process in the child + process group (which hopefully includes *all* its children). */ + if (::kill(separatePG ? -pid : pid, killSignal) != 0) { + /* On BSDs, killing a process group will return EPERM if all + processes in the group are zombies (or something like + that). So try to detect and ignore that situation. */ +#if __FreeBSD__ || __APPLE__ + if (errno != EPERM || ::kill(pid, 0) != 0) +#endif + logError(SysError("killing process %d", pid).info()); + } + + return wait(); +} + + +int Pid::wait() +{ + assert(pid != -1); + while (1) { + int status; + int res = waitpid(pid, &status, 0); + if (res == pid) { + pid = -1; + return status; + } + if (errno != EINTR) + throw SysError("cannot get exit status of PID %d", pid); + checkInterrupt(); + } +} + + +void Pid::setSeparatePG(bool separatePG) +{ + this->separatePG = separatePG; +} + + +void Pid::setKillSignal(int signal) +{ + this->killSignal = signal; +} + + +pid_t Pid::release() +{ + pid_t p = pid; + pid = -1; + return p; +} + + +void killUser(uid_t uid) +{ + debug("killing all processes running under uid '%1%'", uid); + + assert(uid != 0); /* just to be safe... */ + + /* The system call kill(-1, sig) sends the signal `sig' to all + users to which the current process can send signals. So we + fork a process, switch to uid, and send a mass kill. */ + + Pid pid = startProcess([&]() { + + if (setuid(uid) == -1) + throw SysError("setting uid"); + + while (true) { +#ifdef __APPLE__ + /* OSX's kill syscall takes a third parameter that, among + other things, determines if kill(-1, signo) affects the + calling process. In the OSX libc, it's set to true, + which means "follow POSIX", which we don't want here + */ + if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; +#else + if (kill(-1, SIGKILL) == 0) break; +#endif + if (errno == ESRCH || errno == EPERM) break; /* no more processes */ + if (errno != EINTR) + throw SysError("cannot kill processes for uid '%1%'", uid); + } + + _exit(0); + }); + + int status = pid.wait(); + if (status != 0) + throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); + + /* !!! We should really do some check to make sure that there are + no processes left running under `uid', but there is no portable + way to do so (I think). The most reliable way may be `ps -eo + uid | grep -q $uid'. */ +} + + +////////////////////////////////////////////////////////////////////// + + +static pid_t doFork(std::function fun) +{ + pid_t pid = fork(); + if (pid != 0) return pid; + fun(); + abort(); +} + +#if __linux__ +static int childEntry(void * arg) +{ + auto main = (std::function *) arg; + (*main)(); + return 1; +} +#endif + + +pid_t startProcess(std::function fun, const ProcessOptions & options) +{ + std::function wrapper = [&]() { + logger = makeSimpleLogger(); + try { +#if __linux__ + if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) + throw SysError("setting death signal"); +#endif + fun(); + } catch (std::exception & e) { + try { + std::cerr << options.errorPrefix << e.what() << "\n"; + } catch (...) { } + } catch (...) { } + if (options.runExitHandlers) + exit(1); + else + _exit(1); + }; + + pid_t pid = -1; + + if (options.cloneFlags) { + #ifdef __linux__ + // Not supported, since then we don't know when to free the stack. + assert(!(options.cloneFlags & CLONE_VM)); + + size_t stackSize = 1 * 1024 * 1024; + auto stack = (char *) mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (stack == MAP_FAILED) throw SysError("allocating stack"); + + Finally freeStack([&]() { munmap(stack, stackSize); }); + + pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); + #else + throw Error("clone flags are only supported on Linux"); + #endif + } else + pid = doFork(wrapper); + + if (pid == -1) throw SysError("unable to fork"); + + return pid; +} + +std::string runProgram(Path program, bool searchPath, const Strings & args, + const std::optional & input, bool isInteractive) +{ + auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive}); + + if (!statusOk(res.first)) + throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); + + return res.second; +} + +// Output = error code + "standard out" output stream +std::pair runProgram(RunOptions && options) +{ + StringSink sink; + options.standardOut = &sink; + + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(sink.s)}; +} + +void runProgram2(const RunOptions & options) +{ + checkInterrupt(); + + assert(!(options.standardIn && options.input)); + + std::unique_ptr source_; + Source * source = options.standardIn; + + if (options.input) { + source_ = std::make_unique(*options.input); + source = source_.get(); + } + + /* Create a pipe. */ + Pipe out, in; + if (options.standardOut) out.create(); + if (source) in.create(); + + ProcessOptions processOptions; + + std::optional>> resumeLoggerDefer; + if (options.isInteractive) { + logger->pause(); + resumeLoggerDefer.emplace( + []() { + logger->resume(); + } + ); + } + + /* Fork. */ + Pid pid = startProcess([&]() { + if (options.environment) + replaceEnv(*options.environment); + if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); + if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + + if (options.chdir && chdir((*options.chdir).c_str()) == -1) + throw SysError("chdir failed"); + if (options.gid && setgid(*options.gid) == -1) + throw SysError("setgid failed"); + /* Drop all other groups if we're setgid. */ + if (options.gid && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + if (options.uid && setuid(*options.uid) == -1) + throw SysError("setuid failed"); + + Strings args_(options.args); + args_.push_front(options.program); + + restoreProcessContext(); + + if (options.searchPath) + execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + // This allows you to refer to a program with a pathname relative + // to the PATH variable. + else + execv(options.program.c_str(), stringsToCharPtrs(args_).data()); + + throw SysError("executing '%1%'", options.program); + }, processOptions); + + out.writeSide.close(); + + std::thread writerThread; + + std::promise promise; + + Finally doJoin([&]() { + if (writerThread.joinable()) + writerThread.join(); + }); + + + if (source) { + in.readSide.close(); + writerThread = std::thread([&]() { + try { + std::vector buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile &) { + break; + } + writeFull(in.writeSide.get(), {buf.data(), n}); + } + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide.close(); + }); + } + + if (options.standardOut) + drainFD(out.readSide.get(), *options.standardOut); + + /* Wait for the child to finish. */ + int status = pid.wait(); + + /* Wait for the writer thread to finish. */ + if (source) promise.get_future().get(); + + if (status) + throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); +} + +std::string statusToString(int status) +{ + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + if (WIFEXITED(status)) + return fmt("failed with exit code %1%", WEXITSTATUS(status)); + else if (WIFSIGNALED(status)) { + int sig = WTERMSIG(status); +#if HAVE_STRSIGNAL + const char * description = strsignal(sig); + return fmt("failed due to signal %1% (%2%)", sig, description); +#else + return fmt("failed due to signal %1%", sig); +#endif + } + else + return "died abnormally"; + } else return "succeeded"; +} + + +bool statusOk(int status) +{ + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +} diff --git a/src/libutil/processes.hh b/src/libutil/processes.hh new file mode 100644 index 000000000..91a4edfd2 --- /dev/null +++ b/src/libutil/processes.hh @@ -0,0 +1,115 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace nix { + +struct Sink; +struct Source; + +class Pid +{ + pid_t pid = -1; + bool separatePG = false; + int killSignal = SIGKILL; +public: + Pid(); + Pid(pid_t pid); + ~Pid() noexcept(false); + void operator =(pid_t pid); + operator pid_t(); + int kill(); + int wait(); + + void setSeparatePG(bool separatePG); + void setKillSignal(int signal); + pid_t release(); +}; + +/** + * Kill all processes running under the specified uid by sending them + * a SIGKILL. + */ +void killUser(uid_t uid); + + +/** + * Fork a process that runs the given function, and return the child + * pid to the caller. + */ +struct ProcessOptions +{ + std::string errorPrefix = ""; + bool dieWithParent = true; + bool runExitHandlers = false; + /** + * use clone() with the specified flags (Linux only) + */ + int cloneFlags = 0; +}; + +pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); + + +/** + * Run a program and return its stdout in a string (i.e., like the + * shell backtick operator). + */ +std::string runProgram(Path program, bool searchPath = false, + const Strings & args = Strings(), + const std::optional & input = {}, bool isInteractive = false); + +struct RunOptions +{ + Path program; + bool searchPath = true; + Strings args; + std::optional uid; + std::optional gid; + std::optional chdir; + std::optional> environment; + std::optional input; + Source * standardIn = nullptr; + Sink * standardOut = nullptr; + bool mergeStderrToStdout = false; + bool isInteractive = false; +}; + +std::pair runProgram(RunOptions && options); + +void runProgram2(const RunOptions & options); + +class ExecError : public Error +{ +public: + int status; + + template + ExecError(int status, const Args & ... args) + : Error(args...), status(status) + { } +}; + +/** + * Convert the exit status of a child as returned by wait() into an + * error string. + */ +std::string statusToString(int status); + +bool statusOk(int status); + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 55f3154c9..702db2afb 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,4 +1,6 @@ #include "util.hh" +#include "processes.hh" + #include "sync.hh" #include "finally.hh" #include "serialise.hh" @@ -208,211 +210,6 @@ unsigned int getMaxCPU() ////////////////////////////////////////////////////////////////////// -Pid::Pid() -{ -} - - -Pid::Pid(pid_t pid) - : pid(pid) -{ -} - - -Pid::~Pid() noexcept(false) -{ - if (pid != -1) kill(); -} - - -void Pid::operator =(pid_t pid) -{ - if (this->pid != -1 && this->pid != pid) kill(); - this->pid = pid; - killSignal = SIGKILL; // reset signal to default -} - - -Pid::operator pid_t() -{ - return pid; -} - - -int Pid::kill() -{ - assert(pid != -1); - - debug("killing process %1%", pid); - - /* Send the requested signal to the child. If it has its own - process group, send the signal to every process in the child - process group (which hopefully includes *all* its children). */ - if (::kill(separatePG ? -pid : pid, killSignal) != 0) { - /* On BSDs, killing a process group will return EPERM if all - processes in the group are zombies (or something like - that). So try to detect and ignore that situation. */ -#if __FreeBSD__ || __APPLE__ - if (errno != EPERM || ::kill(pid, 0) != 0) -#endif - logError(SysError("killing process %d", pid).info()); - } - - return wait(); -} - - -int Pid::wait() -{ - assert(pid != -1); - while (1) { - int status; - int res = waitpid(pid, &status, 0); - if (res == pid) { - pid = -1; - return status; - } - if (errno != EINTR) - throw SysError("cannot get exit status of PID %d", pid); - checkInterrupt(); - } -} - - -void Pid::setSeparatePG(bool separatePG) -{ - this->separatePG = separatePG; -} - - -void Pid::setKillSignal(int signal) -{ - this->killSignal = signal; -} - - -pid_t Pid::release() -{ - pid_t p = pid; - pid = -1; - return p; -} - - -void killUser(uid_t uid) -{ - debug("killing all processes running under uid '%1%'", uid); - - assert(uid != 0); /* just to be safe... */ - - /* The system call kill(-1, sig) sends the signal `sig' to all - users to which the current process can send signals. So we - fork a process, switch to uid, and send a mass kill. */ - - Pid pid = startProcess([&]() { - - if (setuid(uid) == -1) - throw SysError("setting uid"); - - while (true) { -#ifdef __APPLE__ - /* OSX's kill syscall takes a third parameter that, among - other things, determines if kill(-1, signo) affects the - calling process. In the OSX libc, it's set to true, - which means "follow POSIX", which we don't want here - */ - if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; -#else - if (kill(-1, SIGKILL) == 0) break; -#endif - if (errno == ESRCH || errno == EPERM) break; /* no more processes */ - if (errno != EINTR) - throw SysError("cannot kill processes for uid '%1%'", uid); - } - - _exit(0); - }); - - int status = pid.wait(); - if (status != 0) - throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); - - /* !!! We should really do some check to make sure that there are - no processes left running under `uid', but there is no portable - way to do so (I think). The most reliable way may be `ps -eo - uid | grep -q $uid'. */ -} - - -////////////////////////////////////////////////////////////////////// - - -static pid_t doFork(std::function fun) -{ - pid_t pid = fork(); - if (pid != 0) return pid; - fun(); - abort(); -} - - -#if __linux__ -static int childEntry(void * arg) -{ - auto main = (std::function *) arg; - (*main)(); - return 1; -} -#endif - - -pid_t startProcess(std::function fun, const ProcessOptions & options) -{ - std::function wrapper = [&]() { - logger = makeSimpleLogger(); - try { -#if __linux__ - if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) - throw SysError("setting death signal"); -#endif - fun(); - } catch (std::exception & e) { - try { - std::cerr << options.errorPrefix << e.what() << "\n"; - } catch (...) { } - } catch (...) { } - if (options.runExitHandlers) - exit(1); - else - _exit(1); - }; - - pid_t pid = -1; - - if (options.cloneFlags) { - #ifdef __linux__ - // Not supported, since then we don't know when to free the stack. - assert(!(options.cloneFlags & CLONE_VM)); - - size_t stackSize = 1 * 1024 * 1024; - auto stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - Finally freeStack([&]() { munmap(stack, stackSize); }); - - pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); - #else - throw Error("clone flags are only supported on Linux"); - #endif - } else - pid = doFork(wrapper); - - if (pid == -1) throw SysError("unable to fork"); - - return pid; -} - std::vector stringsToCharPtrs(const Strings & ss) { @@ -422,149 +219,6 @@ std::vector stringsToCharPtrs(const Strings & ss) return res; } -std::string runProgram(Path program, bool searchPath, const Strings & args, - const std::optional & input, bool isInteractive) -{ - auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive}); - - if (!statusOk(res.first)) - throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); - - return res.second; -} - -// Output = error code + "standard out" output stream -std::pair runProgram(RunOptions && options) -{ - StringSink sink; - options.standardOut = &sink; - - int status = 0; - - try { - runProgram2(options); - } catch (ExecError & e) { - status = e.status; - } - - return {status, std::move(sink.s)}; -} - -void runProgram2(const RunOptions & options) -{ - checkInterrupt(); - - assert(!(options.standardIn && options.input)); - - std::unique_ptr source_; - Source * source = options.standardIn; - - if (options.input) { - source_ = std::make_unique(*options.input); - source = source_.get(); - } - - /* Create a pipe. */ - Pipe out, in; - if (options.standardOut) out.create(); - if (source) in.create(); - - ProcessOptions processOptions; - - std::optional>> resumeLoggerDefer; - if (options.isInteractive) { - logger->pause(); - resumeLoggerDefer.emplace( - []() { - logger->resume(); - } - ); - } - - /* Fork. */ - Pid pid = startProcess([&]() { - if (options.environment) - replaceEnv(*options.environment); - if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - if (options.mergeStderrToStdout) - if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) - throw SysError("cannot dup stdout into stderr"); - if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - - if (options.chdir && chdir((*options.chdir).c_str()) == -1) - throw SysError("chdir failed"); - if (options.gid && setgid(*options.gid) == -1) - throw SysError("setgid failed"); - /* Drop all other groups if we're setgid. */ - if (options.gid && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - if (options.uid && setuid(*options.uid) == -1) - throw SysError("setuid failed"); - - Strings args_(options.args); - args_.push_front(options.program); - - restoreProcessContext(); - - if (options.searchPath) - execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); - // This allows you to refer to a program with a pathname relative - // to the PATH variable. - else - execv(options.program.c_str(), stringsToCharPtrs(args_).data()); - - throw SysError("executing '%1%'", options.program); - }, processOptions); - - out.writeSide.close(); - - std::thread writerThread; - - std::promise promise; - - Finally doJoin([&]() { - if (writerThread.joinable()) - writerThread.join(); - }); - - - if (source) { - in.readSide.close(); - writerThread = std::thread([&]() { - try { - std::vector buf(8 * 1024); - while (true) { - size_t n; - try { - n = source->read(buf.data(), buf.size()); - } catch (EndOfFile &) { - break; - } - writeFull(in.writeSide.get(), {buf.data(), n}); - } - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - in.writeSide.close(); - }); - } - - if (options.standardOut) - drainFD(out.readSide.get(), *options.standardOut); - - /* Wait for the child to finish. */ - int status = pid.wait(); - - /* Wait for the writer thread to finish. */ - if (source) promise.get_future().get(); - - if (status) - throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); -} - ////////////////////////////////////////////////////////////////////// @@ -648,32 +302,6 @@ std::string Rewriter::operator()(std::string s) } -std::string statusToString(int status) -{ - if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { - if (WIFEXITED(status)) - return fmt("failed with exit code %1%", WEXITSTATUS(status)); - else if (WIFSIGNALED(status)) { - int sig = WTERMSIG(status); -#if HAVE_STRSIGNAL - const char * description = strsignal(sig); - return fmt("failed due to signal %1% (%2%)", sig, description); -#else - return fmt("failed due to signal %1%", sig); -#endif - } - else - return "died abnormally"; - } else return "succeeded"; -} - - -bool statusOk(int status) -{ - return WIFEXITED(status) && WEXITSTATUS(status) == 0; -} - - std::string toLower(const std::string & s) { std::string r(s); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index d0e207146..8d1900131 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -97,78 +97,6 @@ Path createNixStateDir(); */ unsigned int getMaxCPU(); -class Pid -{ - pid_t pid = -1; - bool separatePG = false; - int killSignal = SIGKILL; -public: - Pid(); - Pid(pid_t pid); - ~Pid() noexcept(false); - void operator =(pid_t pid); - operator pid_t(); - int kill(); - int wait(); - - void setSeparatePG(bool separatePG); - void setKillSignal(int signal); - pid_t release(); -}; - -/** - * Kill all processes running under the specified uid by sending them - * a SIGKILL. - */ -void killUser(uid_t uid); - - -/** - * Fork a process that runs the given function, and return the child - * pid to the caller. - */ -struct ProcessOptions -{ - std::string errorPrefix = ""; - bool dieWithParent = true; - bool runExitHandlers = false; - /** - * use clone() with the specified flags (Linux only) - */ - int cloneFlags = 0; -}; - -pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); - - -/** - * Run a program and return its stdout in a string (i.e., like the - * shell backtick operator). - */ -std::string runProgram(Path program, bool searchPath = false, - const Strings & args = Strings(), - const std::optional & input = {}, bool isInteractive = false); - -struct RunOptions -{ - Path program; - bool searchPath = true; - Strings args; - std::optional uid; - std::optional gid; - std::optional chdir; - std::optional> environment; - std::optional input; - Source * standardIn = nullptr; - Sink * standardOut = nullptr; - bool mergeStderrToStdout = false; - bool isInteractive = false; -}; - -std::pair runProgram(RunOptions && options); - -void runProgram2(const RunOptions & options); - /** * Change the stack size. @@ -204,17 +132,6 @@ void restoreMountNamespace(); void unshareFilesystem(); -class ExecError : public Error -{ -public: - int status; - - template - ExecError(int status, const Args & ... args) - : Error(args...), status(status) - { } -}; - /** * Convert a list of strings to a null-terminated vector of `char * *`s. The result must not be accessed beyond the lifetime of the @@ -323,14 +240,6 @@ inline std::string rewriteStrings(std::string s, const StringMap & rewrites) } -/** - * Convert the exit status of a child as returned by wait() into an - * error string. - */ -std::string statusToString(int status); - -bool statusOk(int status); - /** * Parse a string into an integer. diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 3e47fc8f0..cbc28fdd7 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -5,6 +5,7 @@ #include "common-args.hh" #include "local-fs-store.hh" #include "logging.hh" +#include "processes.hh" #include "profiles.hh" #include "store-api.hh" #include "filetransfer.hh" diff --git a/tests/functional/repl_characterization/test-session.cc b/tests/functional/repl_characterization/test-session.cc index c3eb45b85..ab0710b24 100644 --- a/tests/functional/repl_characterization/test-session.cc +++ b/tests/functional/repl_characterization/test-session.cc @@ -5,6 +5,7 @@ #include "test-session.hh" #include "util.hh" #include "escape-char.hh" +#include "processes.hh" namespace nix { diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index c8dc82cb0..a42afd997 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -1,9 +1,9 @@ #include "file-system.hh" #include "util.hh" +#include "processes.hh" #include "types.hh" #include "terminal.hh" -#include #include #include From 2473e1253d37d4bbd3f7aad2ff269ad84747d527 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 13:36:02 +0200 Subject: [PATCH 06/16] util.{hh,cc}: Split out current-process.{hh,cc} Change-Id: I77095b9d37e85310075bada7a076ccd482c28e47 --- src/libmain/shared.cc | 1 + src/libstore/build/local-derivation-goal.cc | 1 - src/libstore/globals.cc | 1 + src/libstore/ssh.cc | 1 + src/libutil/current-process.cc | 109 ++++++++++++++++++++ src/libutil/current-process.hh | 37 +++++++ src/libutil/meson.build | 2 + src/libutil/processes.cc | 1 + src/libutil/util.cc | 87 +--------------- src/libutil/util.hh | 26 ----- src/nix-build/nix-build.cc | 1 + src/nix/edit.cc | 1 + src/nix/main.cc | 1 + src/nix/run.cc | 1 + 14 files changed, 157 insertions(+), 113 deletions(-) create mode 100644 src/libutil/current-process.cc create mode 100644 src/libutil/current-process.hh diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 33323a226..377cd6abd 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -6,6 +6,7 @@ #include "signals.hh" #include "loggers.hh" #include "progress-bar.hh" +#include "current-process.hh" #include #include diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9be780212..c90910d29 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -6,7 +6,6 @@ #include "builtins/buildenv.hh" #include "path-references.hh" #include "finally.hh" -#include "util.hh" #include "archive.hh" #include "compression.hh" #include "daemon.hh" diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index f14e6c91f..30fd4e9e3 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -5,6 +5,7 @@ #include "args.hh" #include "abstract-setting-to-json.hh" #include "compute-levels.hh" +#include "current-process.hh" #include #include diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 1d8266b9d..ccf0aef7f 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,3 +1,4 @@ +#include "current-process.hh" #include "environment-variables.hh" #include "ssh.hh" #include "finally.hh" diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc new file mode 100644 index 000000000..826e25547 --- /dev/null +++ b/src/libutil/current-process.cc @@ -0,0 +1,109 @@ +#include "current-process.hh" +#include "file-system.hh" +#include "logging.hh" +#include "signals.hh" +#include "util.hh" + +#ifdef __APPLE__ +# include +#endif + +#if __linux__ +# include +#endif + +#include +#include + +namespace nix { + +unsigned int getMaxCPU() +{ + #if __linux__ + try { + auto cgroupFS = getCgroupFS(); + if (!cgroupFS) return 0; + + auto cgroups = getCgroups("/proc/self/cgroup"); + auto cgroup = cgroups[""]; + if (cgroup == "") return 0; + + auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; + + auto cpuMax = readFile(cpuFile); + auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); + + if (cpuMaxParts.size() != 2) { + return 0; + } + + auto quota = cpuMaxParts[0]; + auto period = cpuMaxParts[1]; + if (quota != "max") + return std::ceil(std::stoi(quota) / std::stof(period)); + } catch (Error &) { ignoreException(lvlDebug); } + #endif + + return 0; +} + +rlim_t savedStackSize = 0; + +void setStackSize(rlim_t stackSize) +{ + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { + savedStackSize = limit.rlim_cur; + limit.rlim_cur = std::min(stackSize, limit.rlim_max); + if (setrlimit(RLIMIT_STACK, &limit) != 0) { + logger->log( + lvlError, + HintFmt( + "Failed to increase stack size from %1% to %2% (maximum allowed stack size: %3%): %4%", + savedStackSize, + stackSize, + limit.rlim_max, + std::strerror(errno) + ).str() + ); + } + } +} + +void restoreProcessContext(bool restoreMounts) +{ + restoreSignals(); + if (restoreMounts) { + restoreMountNamespace(); + } + + if (savedStackSize) { + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0) { + limit.rlim_cur = savedStackSize; + setrlimit(RLIMIT_STACK, &limit); + } + } +} + +std::optional getSelfExe() +{ + static auto cached = []() -> std::optional + { + #if __linux__ + return readLink("/proc/self/exe"); + #elif __APPLE__ + char buf[1024]; + uint32_t size = sizeof(buf); + if (_NSGetExecutablePath(buf, &size) == 0) + return buf; + else + return std::nullopt; + #else + return std::nullopt; + #endif + }(); + return cached; +} + +} diff --git a/src/libutil/current-process.hh b/src/libutil/current-process.hh new file mode 100644 index 000000000..8d5a2791d --- /dev/null +++ b/src/libutil/current-process.hh @@ -0,0 +1,37 @@ +#pragma once +///@file + +#include +#include + +#include "types.hh" + +namespace nix { + +/** + * If cgroups are active, attempt to calculate the number of CPUs available. + * If cgroups are unavailable or if cpu.max is set to "max", return 0. + */ +unsigned int getMaxCPU(); + + +/** + * Change the stack size. + */ +void setStackSize(rlim_t stackSize); + + +/** + * Restore the original inherited Unix process context (such as signal + * masks, stack size). + + * See startSignalHandlerThread(), saveSignalMask(). + */ +void restoreProcessContext(bool restoreMounts = true); + +/** + * @return the path of the current executable. + */ +std::optional getSelfExe(); + +} diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 34715b25e..ef4270066 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -6,6 +6,7 @@ libutil_sources = files( 'compression.cc', 'compute-levels.cc', 'config.cc', + 'current-process.cc', 'english.cc', 'environment-variables.cc', 'error.cc', @@ -56,6 +57,7 @@ libutil_headers = files( 'compute-levels.hh', 'config-impl.hh', 'config.hh', + 'current-process.hh', 'english.hh', 'environment-variables.hh', 'error.hh', diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc index 8d1b119b0..8639a77f8 100644 --- a/src/libutil/processes.cc +++ b/src/libutil/processes.cc @@ -1,3 +1,4 @@ +#include "current-process.hh" #include "environment-variables.hh" #include "finally.hh" #include "logging.hh" diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 702db2afb..7f9413f8d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,5 +1,6 @@ #include "util.hh" #include "processes.hh" +#include "current-process.hh" #include "sync.hh" #include "finally.hh" @@ -148,59 +149,11 @@ Path createNixStateDir() } -std::optional getSelfExe() -{ - static auto cached = []() -> std::optional - { - #if __linux__ - return readLink("/proc/self/exe"); - #elif __APPLE__ - char buf[1024]; - uint32_t size = sizeof(buf); - if (_NSGetExecutablePath(buf, &size) == 0) - return buf; - else - return std::nullopt; - #else - return std::nullopt; - #endif - }(); - return cached; -} ////////////////////////////////////////////////////////////////////// -unsigned int getMaxCPU() -{ - #if __linux__ - try { - auto cgroupFS = getCgroupFS(); - if (!cgroupFS) return 0; - - auto cgroups = getCgroups("/proc/self/cgroup"); - auto cgroup = cgroups[""]; - if (cgroup == "") return 0; - - auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; - - auto cpuMax = readFile(cpuFile); - auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); - - if (cpuMaxParts.size() != 2) { - return 0; - } - - auto quota = cpuMaxParts[0]; - auto period = cpuMaxParts[1]; - if (quota != "max") - return std::ceil(std::stoi(quota) / std::stof(period)); - } catch (Error &) { ignoreException(lvlDebug); } - #endif - - return 0; -} ////////////////////////////////////////////////////////////////////// @@ -454,28 +407,6 @@ std::pair getLine(std::string_view s) ////////////////////////////////////////////////////////////////////// -rlim_t savedStackSize = 0; - -void setStackSize(rlim_t stackSize) -{ - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { - savedStackSize = limit.rlim_cur; - limit.rlim_cur = std::min(stackSize, limit.rlim_max); - if (setrlimit(RLIMIT_STACK, &limit) != 0) { - logger->log( - lvlError, - HintFmt( - "Failed to increase stack size from %1% to %2% (maximum allowed stack size: %3%): %4%", - savedStackSize, - stackSize, - limit.rlim_max, - std::strerror(errno) - ).str() - ); - } - } -} #if __linux__ static AutoCloseFD fdSavedMountNamespace; @@ -528,22 +459,6 @@ void unshareFilesystem() #endif } -void restoreProcessContext(bool restoreMounts) -{ - restoreSignals(); - if (restoreMounts) { - restoreMountNamespace(); - } - - if (savedStackSize) { - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0) { - limit.rlim_cur = savedStackSize; - setrlimit(RLIMIT_STACK, &limit); - } - } -} - AutoCloseFD createUnixDomainSocket() { AutoCloseFD fdSocket{socket(PF_UNIX, SOCK_STREAM diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 8d1900131..de5f0a2ee 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -71,11 +71,6 @@ std::vector getConfigDirs(); */ Path getDataDir(); -/** - * @return the path of the current executable. - */ -std::optional getSelfExe(); - /** * @return $XDG_STATE_HOME or $HOME/.local/state. * @@ -91,27 +86,6 @@ Path getStateDir(); Path createNixStateDir(); -/** - * If cgroups are active, attempt to calculate the number of CPUs available. - * If cgroups are unavailable or if cpu.max is set to "max", return 0. - */ -unsigned int getMaxCPU(); - - -/** - * Change the stack size. - */ -void setStackSize(rlim_t stackSize); - - -/** - * Restore the original inherited Unix process context (such as signal - * masks, stack size). - - * See startSignalHandlerThread(), saveSignalMask(). - */ -void restoreProcessContext(bool restoreMounts = true); - /** * Save the current mount namespace. Ignored if called more than * once. diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 9a3994842..f1c7bbfb8 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -13,6 +13,7 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" +#include "current-process.hh" #include "derivations.hh" #include "util.hh" #include "shared.hh" diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 66629fab0..d1741a254 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -4,6 +4,7 @@ #include "attr-path.hh" #include "progress-bar.hh" #include "editor-for.hh" +#include "current-process.hh" #include diff --git a/src/nix/main.cc b/src/nix/main.cc index 83d697326..0f98861d5 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -3,6 +3,7 @@ #include "args/root.hh" #include "command.hh" #include "common-args.hh" +#include "current-process.hh" #include "eval.hh" #include "eval-settings.hh" #include "globals.hh" diff --git a/src/nix/run.cc b/src/nix/run.cc index 94d492ca7..e1e896bb4 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -10,6 +10,7 @@ #include "progress-bar.hh" #include "eval.hh" #include "build/personality.hh" +#include "current-process.hh" #if __linux__ #include From e81ed5f12d0702ed402faa8b5ee725f2203db60c Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 13:44:39 +0200 Subject: [PATCH 07/16] util.{hh,cc}: Split out child.{hh,cc} Change-Id: Iec4824e071f537b17dd62dbb8c01b8eec14e9783 --- src/libstore/build/child.cc | 33 +++++++++++++++++++++ src/libstore/build/child.hh | 11 +++++++ src/libstore/build/hook-instance.cc | 1 + src/libstore/build/local-derivation-goal.cc | 1 + src/libstore/meson.build | 2 ++ src/libutil/util.cc | 29 ------------------ src/libutil/util.hh | 6 ---- 7 files changed, 48 insertions(+), 35 deletions(-) create mode 100644 src/libstore/build/child.cc create mode 100644 src/libstore/build/child.hh diff --git a/src/libstore/build/child.cc b/src/libstore/build/child.cc new file mode 100644 index 000000000..a82a5eec9 --- /dev/null +++ b/src/libstore/build/child.cc @@ -0,0 +1,33 @@ +#include "current-process.hh" +#include "logging.hh" + +namespace nix { + +void commonChildInit() +{ + logger = makeSimpleLogger(); + + const static std::string pathNullDevice = "/dev/null"; + restoreProcessContext(false); + + /* Put the child in a separate session (and thus a separate + process group) so that it has no controlling terminal (meaning + that e.g. ssh cannot open /dev/tty) and it doesn't receive + terminal signals. */ + if (setsid() == -1) + throw SysError("creating a new session"); + + /* Dup stderr to stdout. */ + if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) + throw SysError("cannot dup stderr into stdout"); + + /* Reroute stdin to /dev/null. */ + int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); + if (fdDevNull == -1) + throw SysError("cannot open '%1%'", pathNullDevice); + if (dup2(fdDevNull, STDIN_FILENO) == -1) + throw SysError("cannot dup null device into stdin"); + close(fdDevNull); +} + +} diff --git a/src/libstore/build/child.hh b/src/libstore/build/child.hh new file mode 100644 index 000000000..3dfc552b9 --- /dev/null +++ b/src/libstore/build/child.hh @@ -0,0 +1,11 @@ +#pragma once +///@file + +namespace nix { + +/** + * Common initialisation performed in child processes. + */ +void commonChildInit(); + +} diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index 108722cfb..86f72486e 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -1,3 +1,4 @@ +#include "child.hh" #include "file-system.hh" #include "globals.hh" #include "hook-instance.hh" diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index c90910d29..d385ffb34 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -14,6 +14,7 @@ #include "cgroup.hh" #include "personality.hh" #include "namespaces.hh" +#include "child.hh" #include #include diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 98549f6d9..f776e9621 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -79,6 +79,7 @@ libstore_sources = files( 'store-api.cc', 'uds-remote-store.cc', 'worker-protocol.cc', + 'build/child.cc', 'build/derivation-goal.cc', 'build/drv-output-substitution-goal.cc', 'build/entry-points.cc', @@ -96,6 +97,7 @@ libstore_sources = files( libstore_headers = files( 'binary-cache-store.hh', + 'build/child.hh', 'build/derivation-goal.hh', 'build/drv-output-substitution-goal.hh', 'build/goal.hh', diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 7f9413f8d..aff26e32b 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -559,33 +559,4 @@ std::string showBytes(uint64_t bytes) return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); } - -// FIXME: move to libstore/build -void commonChildInit() -{ - logger = makeSimpleLogger(); - - const static std::string pathNullDevice = "/dev/null"; - restoreProcessContext(false); - - /* Put the child in a separate session (and thus a separate - process group) so that it has no controlling terminal (meaning - that e.g. ssh cannot open /dev/tty) and it doesn't receive - terminal signals. */ - if (setsid() == -1) - throw SysError("creating a new session"); - - /* Dup stderr to stdout. */ - if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) - throw SysError("cannot dup stderr into stdout"); - - /* Reroute stdin to /dev/null. */ - int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); - if (fdDevNull == -1) - throw SysError("cannot open '%1%'", pathNullDevice); - if (dup2(fdDevNull, STDIN_FILENO) == -1) - throw SysError("cannot dup null device into stdin"); - close(fdDevNull); -} - } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index de5f0a2ee..e704cd709 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -406,12 +406,6 @@ struct MaintainCount }; -/** - * Common initialisation performed in child processes. - */ -void commonChildInit(); - - /** * Bind a Unix domain socket to a path. */ From 5b5a75979a0b954a5deefe79c8040bac1ad9c76a Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 14:41:48 +0200 Subject: [PATCH 08/16] util.{hh,cc}: Split out unix-domain-socket.{hh,cc} Change-Id: I3f9a628e0f8998b6146f5caa8ae9842361a66b8b --- src/libstore/build/local-derivation-goal.cc | 1 + src/libstore/gc.cc | 1 + src/libstore/uds-remote-store.cc | 1 + src/libutil/file-descriptor.hh | 9 -- src/libutil/meson.build | 2 + src/libutil/unix-domain-socket.cc | 105 ++++++++++++++++++++ src/libutil/unix-domain-socket.hh | 31 ++++++ src/libutil/util.cc | 94 ------------------ src/libutil/util.hh | 11 -- src/nix/daemon.cc | 1 + 10 files changed, 142 insertions(+), 114 deletions(-) create mode 100644 src/libutil/unix-domain-socket.cc create mode 100644 src/libutil/unix-domain-socket.hh diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index d385ffb34..fd871e8a4 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -15,6 +15,7 @@ #include "personality.hh" #include "namespaces.hh" #include "child.hh" +#include "unix-domain-socket.hh" #include #include diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 60d88f3b2..722452b8d 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -4,6 +4,7 @@ #include "processes.hh" #include "signals.hh" #include "finally.hh" +#include "unix-domain-socket.hh" #include #include diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 99589f8b2..226cdf717 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,4 +1,5 @@ #include "uds-remote-store.hh" +#include "unix-domain-socket.hh" #include "worker-protocol.hh" #include diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh index 68324b9d9..f59baa7a0 100644 --- a/src/libutil/file-descriptor.hh +++ b/src/libutil/file-descriptor.hh @@ -77,13 +77,4 @@ void closeOnExec(int fd); MakeError(EndOfFile, Error); -/** - * Create a Unix domain socket. - */ -AutoCloseFD createUnixDomainSocket(); - -/** - * Create a Unix domain socket in listen mode. - */ -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); } diff --git a/src/libutil/meson.build b/src/libutil/meson.build index ef4270066..73c520116 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -35,6 +35,7 @@ libutil_sources = files( 'tarfile.cc', 'terminal.cc', 'thread-pool.cc', + 'unix-domain-socket.cc', 'url.cc', 'url-name.cc', 'util.cc', @@ -102,6 +103,7 @@ libutil_headers = files( 'thread-pool.hh', 'topo-sort.hh', 'types.hh', + 'unix-domain-socket.hh', 'url-parts.hh', 'url-name.hh', 'url.hh', diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc new file mode 100644 index 000000000..9fefcbe1c --- /dev/null +++ b/src/libutil/unix-domain-socket.cc @@ -0,0 +1,105 @@ +#include "file-system.hh" +#include "processes.hh" +#include "unix-domain-socket.hh" +#include "util.hh" + +#include +#include +#include + +namespace nix { + +AutoCloseFD createUnixDomainSocket() +{ + AutoCloseFD fdSocket{socket(PF_UNIX, SOCK_STREAM + #ifdef SOCK_CLOEXEC + | SOCK_CLOEXEC + #endif + , 0)}; + if (!fdSocket) + throw SysError("cannot create Unix domain socket"); + closeOnExec(fdSocket.get()); + return fdSocket; +} + + +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +{ + auto fdSocket = nix::createUnixDomainSocket(); + + bind(fdSocket.get(), path); + + chmodPath(path.c_str(), mode); + + if (listen(fdSocket.get(), 100) == -1) + throw SysError("cannot listen on socket '%1%'", path); + + return fdSocket; +} + +static void bindConnectProcHelper( + std::string_view operationName, auto && operation, + int fd, const std::string & path) +{ + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + // Casting between types like these legacy C library interfaces + // require is forbidden in C++. To maintain backwards + // compatibility, the implementation of the bind/connect functions + // contains some hints to the compiler that allow for this + // special case. + auto * psaddr = reinterpret_cast(&addr); + + if (path.size() + 1 >= sizeof(addr.sun_path)) { + Pipe pipe; + pipe.create(); + Pid pid = startProcess([&] { + try { + pipe.readSide.close(); + Path dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); + if (operation(fd, psaddr, sizeof(addr)) == -1) + throw SysError("cannot %s to socket at '%s'", operationName, path); + writeFull(pipe.writeSide.get(), "0\n"); + } catch (SysError & e) { + writeFull(pipe.writeSide.get(), fmt("%d\n", e.errNo)); + } catch (...) { + writeFull(pipe.writeSide.get(), "-1\n"); + } + }); + pipe.writeSide.close(); + auto errNo = string2Int(chomp(drainFD(pipe.readSide.get()))); + if (!errNo || *errNo == -1) + throw Error("cannot %s to socket at '%s'", operationName, path); + else if (*errNo > 0) { + errno = *errNo; + throw SysError("cannot %s to socket at '%s'", operationName, path); + } + } else { + memcpy(addr.sun_path, path.c_str(), path.size() + 1); + if (operation(fd, psaddr, sizeof(addr)) == -1) + throw SysError("cannot %s to socket at '%s'", operationName, path); + } +} + + +void bind(int fd, const std::string & path) +{ + unlink(path.c_str()); + + bindConnectProcHelper("bind", ::bind, fd, path); +} + + +void connect(int fd, const std::string & path) +{ + bindConnectProcHelper("connect", ::connect, fd, path); +} + +} diff --git a/src/libutil/unix-domain-socket.hh b/src/libutil/unix-domain-socket.hh new file mode 100644 index 000000000..692ad2627 --- /dev/null +++ b/src/libutil/unix-domain-socket.hh @@ -0,0 +1,31 @@ +#pragma once +///@file + +#include "file-descriptor.hh" +#include "types.hh" + +#include + +namespace nix { + +/** + * Create a Unix domain socket. + */ +AutoCloseFD createUnixDomainSocket(); + +/** + * Create a Unix domain socket in listen mode. + */ +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); + +/** + * Bind a Unix domain socket to a path. + */ +void bind(int fd, const std::string & path); + +/** + * Connect to a Unix domain socket. + */ +void connect(int fd, const std::string & path); + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index aff26e32b..f580ef038 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -459,100 +459,6 @@ void unshareFilesystem() #endif } -AutoCloseFD createUnixDomainSocket() -{ - AutoCloseFD fdSocket{socket(PF_UNIX, SOCK_STREAM - #ifdef SOCK_CLOEXEC - | SOCK_CLOEXEC - #endif - , 0)}; - if (!fdSocket) - throw SysError("cannot create Unix domain socket"); - closeOnExec(fdSocket.get()); - return fdSocket; -} - - -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) -{ - auto fdSocket = nix::createUnixDomainSocket(); - - bind(fdSocket.get(), path); - - chmodPath(path.c_str(), mode); - - if (listen(fdSocket.get(), 100) == -1) - throw SysError("cannot listen on socket '%1%'", path); - - return fdSocket; -} - - -static void bindConnectProcHelper( - std::string_view operationName, auto && operation, - int fd, const std::string & path) -{ - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - - // Casting between types like these legacy C library interfaces - // require is forbidden in C++. To maintain backwards - // compatibility, the implementation of the bind/connect functions - // contains some hints to the compiler that allow for this - // special case. - auto * psaddr = reinterpret_cast(&addr); - - if (path.size() + 1 >= sizeof(addr.sun_path)) { - Pipe pipe; - pipe.create(); - Pid pid = startProcess([&] { - try { - pipe.readSide.close(); - Path dir = dirOf(path); - if (chdir(dir.c_str()) == -1) - throw SysError("chdir to '%s' failed", dir); - std::string base(baseNameOf(path)); - if (base.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%s' is too long", base); - memcpy(addr.sun_path, base.c_str(), base.size() + 1); - if (operation(fd, psaddr, sizeof(addr)) == -1) - throw SysError("cannot %s to socket at '%s'", operationName, path); - writeFull(pipe.writeSide.get(), "0\n"); - } catch (SysError & e) { - writeFull(pipe.writeSide.get(), fmt("%d\n", e.errNo)); - } catch (...) { - writeFull(pipe.writeSide.get(), "-1\n"); - } - }); - pipe.writeSide.close(); - auto errNo = string2Int(chomp(drainFD(pipe.readSide.get()))); - if (!errNo || *errNo == -1) - throw Error("cannot %s to socket at '%s'", operationName, path); - else if (*errNo > 0) { - errno = *errNo; - throw SysError("cannot %s to socket at '%s'", operationName, path); - } - } else { - memcpy(addr.sun_path, path.c_str(), path.size() + 1); - if (operation(fd, psaddr, sizeof(addr)) == -1) - throw SysError("cannot %s to socket at '%s'", operationName, path); - } -} - - -void bind(int fd, const std::string & path) -{ - unlink(path.c_str()); - - bindConnectProcHelper("bind", ::bind, fd, path); -} - - -void connect(int fd, const std::string & path) -{ - bindConnectProcHelper("connect", ::connect, fd, path); -} - std::string showBytes(uint64_t bytes) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index e704cd709..216693635 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -406,17 +406,6 @@ struct MaintainCount }; -/** - * Bind a Unix domain socket to a path. - */ -void bind(int fd, const std::string & path); - -/** - * Connect to a Unix domain socket. - */ -void connect(int fd, const std::string & path); - - /** * A Rust/Python-like enumerate() iterator adapter. * diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 9d4afb6d9..113e23bd1 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -14,6 +14,7 @@ #include "legacy.hh" #include "signals.hh" #include "daemon.hh" +#include "unix-domain-socket.hh" #include #include From b91055112035c256fffd44d77f746b977cfdf3ca Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 14:57:20 +0200 Subject: [PATCH 09/16] util.{hh,cc}: Split out strings.{hh,cc} Change-Id: I4f642d1046d56b5db26f1b0296ee16a0e02d444a --- src/libstore/crypto.cc | 2 +- src/libstore/machines.cc | 2 +- src/libstore/names.cc | 2 +- src/libstore/outputs-spec.cc | 3 +- src/libstore/ssh.cc | 3 +- src/libutil/cgroup.cc | 4 +- src/libutil/config.cc | 1 + src/libutil/current-process.cc | 1 + src/libutil/experimental-features.cc | 1 + src/libutil/meson.build | 2 + src/libutil/namespaces.cc | 2 +- src/libutil/regex-combinators.hh | 2 + src/libutil/serialise.hh | 1 + src/libutil/shlex.cc | 2 +- src/libutil/strings.cc | 232 ++++++++++++++++ src/libutil/strings.hh | 256 ++++++++++++++++++ src/libutil/unix-domain-socket.cc | 2 +- src/libutil/url.cc | 2 +- src/libutil/util.cc | 230 +--------------- src/libutil/util.hh | 248 ----------------- .../repl_characterization.cc | 1 + .../repl_characterization/test-session.cc | 2 +- tests/unit/libstore/machines.cc | 1 + .../tests/cli-literate-parser.cc | 1 + tests/unit/libutil/tests.cc | 3 +- 25 files changed, 517 insertions(+), 489 deletions(-) create mode 100644 src/libutil/strings.cc create mode 100644 src/libutil/strings.hh diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 1e29a812b..2e0fd8ca5 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -1,7 +1,7 @@ #include "crypto.hh" -#include "util.hh" #include "file-system.hh" #include "globals.hh" +#include "strings.hh" #include diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 700c9b3dd..cdd1e1c2c 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -1,7 +1,7 @@ #include "machines.hh" -#include "util.hh" #include "globals.hh" #include "store-api.hh" +#include "strings.hh" #include diff --git a/src/libstore/names.cc b/src/libstore/names.cc index 277aabf0f..b903a99f0 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -1,5 +1,5 @@ #include "names.hh" -#include "util.hh" +#include "strings.hh" #include diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc index 21c069223..4422bcd21 100644 --- a/src/libstore/outputs-spec.cc +++ b/src/libstore/outputs-spec.cc @@ -1,10 +1,11 @@ #include #include -#include "util.hh" #include "regex-combinators.hh" #include "outputs-spec.hh" #include "path-regex.hh" +#include "strings.hh" +#include "util.hh" namespace nix { diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index ccf0aef7f..932ebaa52 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -2,7 +2,8 @@ #include "environment-variables.hh" #include "ssh.hh" #include "finally.hh" -#include "util.hh" +#include "logging.hh" +#include "strings.hh" namespace nix { diff --git a/src/libutil/cgroup.cc b/src/libutil/cgroup.cc index aa7802f79..e28e21c3e 100644 --- a/src/libutil/cgroup.cc +++ b/src/libutil/cgroup.cc @@ -1,15 +1,17 @@ +#include "logging.hh" #if __linux__ #include "cgroup.hh" -#include "util.hh" #include "file-system.hh" #include "finally.hh" +#include "strings.hh" #include #include #include #include #include +#include #include #include diff --git a/src/libutil/config.cc b/src/libutil/config.cc index f6f14878a..729b4e596 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -3,6 +3,7 @@ #include "abstract-setting-to-json.hh" #include "experimental-features.hh" #include "file-system.hh" +#include "strings.hh" #include "config-impl.hh" diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 826e25547..41f591b0c 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -3,6 +3,7 @@ #include "logging.hh" #include "signals.hh" #include "util.hh" +#include "strings.hh" #ifdef __APPLE__ # include diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 25beba467..8ebec2956 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -1,6 +1,7 @@ #include "experimental-features.hh" // Required for instances of to_json and from_json for ExperimentalFeature #include "experimental-features-json.hh" +#include "strings.hh" #include "util.hh" #include "nlohmann/json.hpp" diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 73c520116..b662ea455 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -31,6 +31,7 @@ libutil_sources = files( 'shlex.cc', 'signals.cc', 'source-path.cc', + 'strings.cc', 'suggestions.cc', 'tarfile.cc', 'terminal.cc', @@ -96,6 +97,7 @@ libutil_headers = files( 'signals.hh', 'source-path.hh', 'split.hh', + 'strings.hh', 'suggestions.hh', 'sync.hh', 'tarfile.hh', diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index cde442671..d092e6fcc 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -2,9 +2,9 @@ #include "file-system.hh" #include "logging.hh" -#include "util.hh" #include "namespaces.hh" #include "processes.hh" +#include "strings.hh" #include diff --git a/src/libutil/regex-combinators.hh b/src/libutil/regex-combinators.hh index 87d6aa678..37962944e 100644 --- a/src/libutil/regex-combinators.hh +++ b/src/libutil/regex-combinators.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include "strings.hh" + #include namespace nix::regex { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index d4a14bbf0..e46c5624a 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -3,6 +3,7 @@ #include +#include "strings.hh" #include "types.hh" #include "util.hh" #include "file-descriptor.hh" diff --git a/src/libutil/shlex.cc b/src/libutil/shlex.cc index b5f340251..21fa0502a 100644 --- a/src/libutil/shlex.cc +++ b/src/libutil/shlex.cc @@ -1,5 +1,5 @@ #include "shlex.hh" -#include "util.hh" +#include "strings.hh" namespace nix { diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc new file mode 100644 index 000000000..9cb319cce --- /dev/null +++ b/src/libutil/strings.cc @@ -0,0 +1,232 @@ +#include "strings.hh" + +namespace nix { + +std::vector stringsToCharPtrs(const Strings & ss) +{ + std::vector res; + for (auto & s : ss) res.push_back((char *) s.c_str()); + res.push_back(0); + return res; +} + + +template C tokenizeString(std::string_view s, std::string_view separators) +{ + C result; + auto pos = s.find_first_not_of(separators, 0); + while (pos != std::string_view::npos) { + auto end = s.find_first_of(separators, pos + 1); + if (end == std::string_view::npos) end = s.size(); + result.insert(result.end(), std::string(s, pos, end - pos)); + pos = s.find_first_not_of(separators, end); + } + return result; +} + +template Strings tokenizeString(std::string_view s, std::string_view separators); +template StringSet tokenizeString(std::string_view s, std::string_view separators); +template std::vector tokenizeString(std::string_view s, std::string_view separators); + + +std::string chomp(std::string_view s) +{ + size_t i = s.find_last_not_of(" \n\r\t"); + return i == std::string_view::npos ? "" : std::string(s, 0, i + 1); +} + + +std::string trim(std::string_view s, std::string_view whitespace) +{ + auto i = s.find_first_not_of(whitespace); + if (i == s.npos) return ""; + auto j = s.find_last_not_of(whitespace); + return std::string(s, i, j == s.npos ? j : j - i + 1); +} + + +std::string replaceStrings( + std::string res, + std::string_view from, + std::string_view to) +{ + if (from.empty()) return res; + size_t pos = 0; + while ((pos = res.find(from, pos)) != std::string::npos) { + res.replace(pos, from.size(), to); + pos += to.size(); + } + return res; +} + + +Rewriter::Rewriter(std::map rewrites) + : rewrites(std::move(rewrites)) +{ + for (const auto & [k, v] : this->rewrites) { + assert(!k.empty()); + initials.push_back(k[0]); + } + std::ranges::sort(initials); + auto [firstDupe, end] = std::ranges::unique(initials); + initials.erase(firstDupe, end); +} + +std::string Rewriter::operator()(std::string s) +{ + size_t j = 0; + while ((j = s.find_first_of(initials, j)) != std::string::npos) { + size_t skip = 1; + for (auto & [from, to] : rewrites) { + if (s.compare(j, from.size(), from) == 0) { + s.replace(j, from.size(), to); + skip = to.size(); + break; + } + } + j += skip; + } + return s; +} + + +std::string toLower(const std::string & s) +{ + std::string r(s); + for (auto & c : r) + c = std::tolower(c); + return r; +} + + +std::string shellEscape(const std::string_view s) +{ + std::string r; + r.reserve(s.size() + 2); + r += "'"; + for (auto & i : s) + if (i == '\'') r += "'\\''"; else r += i; + r += '\''; + return r; +} + +constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +std::string base64Encode(std::string_view s) +{ + std::string res; + res.reserve((s.size() + 2) / 3 * 4); + int data = 0, nbits = 0; + + for (char c : s) { + data = data << 8 | (unsigned char) c; + nbits += 8; + while (nbits >= 6) { + nbits -= 6; + res.push_back(base64Chars[data >> nbits & 0x3f]); + } + } + + if (nbits) res.push_back(base64Chars[data << (6 - nbits) & 0x3f]); + while (res.size() % 4) res.push_back('='); + + return res; +} + + +std::string base64Decode(std::string_view s) +{ + constexpr char npos = -1; + constexpr std::array base64DecodeChars = [&]() { + std::array result{}; + for (auto& c : result) + c = npos; + for (int i = 0; i < 64; i++) + result[base64Chars[i]] = i; + return result; + }(); + + std::string res; + // Some sequences are missing the padding consisting of up to two '='. + // vvv + res.reserve((s.size() + 2) / 4 * 3); + unsigned int d = 0, bits = 0; + + for (char c : s) { + if (c == '=') break; + if (c == '\n') continue; + + char digit = base64DecodeChars[(unsigned char) c]; + if (digit == npos) + throw Error("invalid character in Base64 string: '%c'", c); + + bits += 6; + d = d << 6 | digit; + if (bits >= 8) { + res.push_back(d >> (bits - 8) & 0xff); + bits -= 8; + } + } + + return res; +} + + +std::string stripIndentation(std::string_view s) +{ + size_t minIndent = 10000; + size_t curIndent = 0; + bool atStartOfLine = true; + + for (auto & c : s) { + if (atStartOfLine && c == ' ') + curIndent++; + else if (c == '\n') { + if (atStartOfLine) + minIndent = std::max(minIndent, curIndent); + curIndent = 0; + atStartOfLine = true; + } else { + if (atStartOfLine) { + minIndent = std::min(minIndent, curIndent); + atStartOfLine = false; + } + } + } + + std::string res; + + size_t pos = 0; + while (pos < s.size()) { + auto eol = s.find('\n', pos); + if (eol == s.npos) eol = s.size(); + if (eol - pos > minIndent) + res.append(s.substr(pos + minIndent, eol - pos - minIndent)); + res.push_back('\n'); + pos = eol + 1; + } + + return res; +} + + +std::pair getLine(std::string_view s) +{ + auto newline = s.find('\n'); + + if (newline == s.npos) { + return {s, ""}; + } else { + auto line = s.substr(0, newline); + if (!line.empty() && line[line.size() - 1] == '\r') + line = line.substr(0, line.size() - 1); + return {line, s.substr(newline + 1)}; + } +} + +std::string showBytes(uint64_t bytes) +{ + return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); +} + +} diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh new file mode 100644 index 000000000..daeb5be50 --- /dev/null +++ b/src/libutil/strings.hh @@ -0,0 +1,256 @@ +#pragma once +///@file + +#include "error.hh" +#include "types.hh" + +#include +#include + +namespace nix { + +/** + * Tree formatting. + */ +constexpr char treeConn[] = "├───"; +constexpr char treeLast[] = "└───"; +constexpr char treeLine[] = "│ "; +constexpr char treeNull[] = " "; + +/** + * Convert a list of strings to a null-terminated vector of `char + * *`s. The result must not be accessed beyond the lifetime of the + * list of strings. + */ +std::vector stringsToCharPtrs(const Strings & ss); + + +MakeError(FormatError, Error); + + +/** + * String tokenizer. + */ +template C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); + + +/** + * Concatenate the given strings with a separator between the + * elements. + */ +template +std::string concatStringsSep(const std::string_view sep, const C & ss) +{ + size_t size = 0; + // need a cast to string_view since this is also called with Symbols + for (const auto & s : ss) size += sep.size() + std::string_view(s).size(); + std::string s; + s.reserve(size); + for (auto & i : ss) { + if (s.size() != 0) s += sep; + s += i; + } + return s; +} + +template +auto concatStrings(Parts && ... parts) + -> std::enable_if_t<(... && std::is_convertible_v), std::string> +{ + std::string_view views[sizeof...(parts)] = { parts... }; + return concatStringsSep({}, views); +} + + +/** + * Add quotes around a collection of strings. + */ +template Strings quoteStrings(const C & c) +{ + Strings res; + for (auto & s : c) + res.push_back("'" + s + "'"); + return res; +} + +/** + * Remove trailing whitespace from a string. + * + * \todo return std::string_view. + */ +std::string chomp(std::string_view s); + + +/** + * Remove whitespace from the start and end of a string. + */ +std::string trim(std::string_view s, std::string_view whitespace = " \n\r\t"); + + +/** + * Replace all occurrences of a string inside another string. + */ +std::string replaceStrings( + std::string s, + std::string_view from, + std::string_view to); + + +/** + * Rewrites a string given a map of replacements, applying the replacements in + * sorted order, only once, considering only the strings appearing in the input + * string in performing replacement. + * + * - Replacements are not performed on intermediate strings. That is, for an input + * `"abb"` with replacements `{"ab" -> "ba"}`, the result is `"bab"`. + * - Transitive replacements are not performed. For example, for the input `"abcde"` + * with replacements `{"a" -> "b", "b" -> "c", "e" -> "b"}`, the result is + * `"bccdb"`. + */ +class Rewriter +{ +private: + std::string initials; + std::map rewrites; + +public: + explicit Rewriter(std::map rewrites); + + std::string operator()(std::string s); +}; + +inline std::string rewriteStrings(std::string s, const StringMap & rewrites) +{ + return Rewriter(rewrites)(s); +} + + + +/** + * Parse a string into an integer. + */ +template +std::optional string2Int(const std::string_view s) +{ + if (s.substr(0, 1) == "-" && !std::numeric_limits::is_signed) + return std::nullopt; + try { + return boost::lexical_cast(s.data(), s.size()); + } catch (const boost::bad_lexical_cast &) { + return std::nullopt; + } +} + +/** + * Like string2Int(), but support an optional suffix 'K', 'M', 'G' or + * 'T' denoting a binary unit prefix. + */ +template +N string2IntWithUnitPrefix(std::string_view s) +{ + N multiplier = 1; + if (!s.empty()) { + char u = std::toupper(*s.rbegin()); + if (std::isalpha(u)) { + if (u == 'K') multiplier = 1ULL << 10; + else if (u == 'M') multiplier = 1ULL << 20; + else if (u == 'G') multiplier = 1ULL << 30; + else if (u == 'T') multiplier = 1ULL << 40; + else throw UsageError("invalid unit specifier '%1%'", u); + s.remove_suffix(1); + } + } + if (auto n = string2Int(s)) + return *n * multiplier; + throw UsageError("'%s' is not an integer", s); +} + +/** + * Parse a string into a float. + */ +template +std::optional string2Float(const std::string_view s) +{ + try { + return boost::lexical_cast(s.data(), s.size()); + } catch (const boost::bad_lexical_cast &) { + return std::nullopt; + } +} + + +/** + * Convert a little-endian integer to host order. + */ +template +T readLittleEndian(unsigned char * p) +{ + T x = 0; + for (size_t i = 0; i < sizeof(x); ++i, ++p) { + x |= ((T) *p) << (i * 8); + } + return x; +} + +/** + * Convert a string to lower case. + */ +std::string toLower(const std::string & s); + + +/** + * Escape a string as a shell word. + */ +std::string shellEscape(const std::string_view s); + +/** + * Base64 encoding/decoding. + */ +std::string base64Encode(std::string_view s); +std::string base64Decode(std::string_view s); + + +/** + * Remove common leading whitespace from the lines in the string + * 's'. For example, if every line is indented by at least 3 spaces, + * then we remove 3 spaces from the start of every line. + */ +std::string stripIndentation(std::string_view s); + +/** + * Get the prefix of 's' up to and excluding the next line break (LF + * optionally preceded by CR), and the remainder following the line + * break. + */ +std::pair getLine(std::string_view s); + +std::string showBytes(uint64_t bytes); + + +/** + * Provide an addition operator between strings and string_views + * inexplicably omitted from the standard library. + */ +inline std::string operator + (const std::string & s1, std::string_view s2) +{ + auto s = s1; + s.append(s2); + return s; +} + +inline std::string operator + (std::string && s, std::string_view s2) +{ + s.append(s2); + return std::move(s); +} + +inline std::string operator + (std::string_view s1, const char * s2) +{ + std::string s; + s.reserve(s1.size() + strlen(s2)); + s.append(s1); + s.append(s2); + return s; +} + +} diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc index 9fefcbe1c..a9a2a415a 100644 --- a/src/libutil/unix-domain-socket.cc +++ b/src/libutil/unix-domain-socket.cc @@ -1,7 +1,7 @@ #include "file-system.hh" #include "processes.hh" #include "unix-domain-socket.hh" -#include "util.hh" +#include "strings.hh" #include #include diff --git a/src/libutil/url.cc b/src/libutil/url.cc index afccc4245..46688cef5 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -1,7 +1,7 @@ #include "url.hh" #include "url-parts.hh" -#include "util.hh" #include "split.hh" +#include "strings.hh" namespace nix { diff --git a/src/libutil/util.cc b/src/libutil/util.cc index f580ef038..099a07622 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,5 +1,6 @@ #include "util.hh" #include "processes.hh" +#include "strings.hh" #include "current-process.hh" #include "sync.hh" @@ -164,116 +165,6 @@ Path createNixStateDir() -std::vector stringsToCharPtrs(const Strings & ss) -{ - std::vector res; - for (auto & s : ss) res.push_back((char *) s.c_str()); - res.push_back(0); - return res; -} - - -////////////////////////////////////////////////////////////////////// - - -template C tokenizeString(std::string_view s, std::string_view separators) -{ - C result; - auto pos = s.find_first_not_of(separators, 0); - while (pos != std::string_view::npos) { - auto end = s.find_first_of(separators, pos + 1); - if (end == std::string_view::npos) end = s.size(); - result.insert(result.end(), std::string(s, pos, end - pos)); - pos = s.find_first_not_of(separators, end); - } - return result; -} - -template Strings tokenizeString(std::string_view s, std::string_view separators); -template StringSet tokenizeString(std::string_view s, std::string_view separators); -template std::vector tokenizeString(std::string_view s, std::string_view separators); - - -std::string chomp(std::string_view s) -{ - size_t i = s.find_last_not_of(" \n\r\t"); - return i == std::string_view::npos ? "" : std::string(s, 0, i + 1); -} - - -std::string trim(std::string_view s, std::string_view whitespace) -{ - auto i = s.find_first_not_of(whitespace); - if (i == s.npos) return ""; - auto j = s.find_last_not_of(whitespace); - return std::string(s, i, j == s.npos ? j : j - i + 1); -} - - -std::string replaceStrings( - std::string res, - std::string_view from, - std::string_view to) -{ - if (from.empty()) return res; - size_t pos = 0; - while ((pos = res.find(from, pos)) != std::string::npos) { - res.replace(pos, from.size(), to); - pos += to.size(); - } - return res; -} - - -Rewriter::Rewriter(std::map rewrites) - : rewrites(std::move(rewrites)) -{ - for (const auto & [k, v] : this->rewrites) { - assert(!k.empty()); - initials.push_back(k[0]); - } - std::ranges::sort(initials); - auto [firstDupe, end] = std::ranges::unique(initials); - initials.erase(firstDupe, end); -} - -std::string Rewriter::operator()(std::string s) -{ - size_t j = 0; - while ((j = s.find_first_of(initials, j)) != std::string::npos) { - size_t skip = 1; - for (auto & [from, to] : rewrites) { - if (s.compare(j, from.size(), from) == 0) { - s.replace(j, from.size(), to); - skip = to.size(); - break; - } - } - j += skip; - } - return s; -} - - -std::string toLower(const std::string & s) -{ - std::string r(s); - for (auto & c : r) - c = std::tolower(c); - return r; -} - - -std::string shellEscape(const std::string_view s) -{ - std::string r; - r.reserve(s.size() + 2); - r += "'"; - for (auto & i : s) - if (i == '\'') r += "'\\''"; else r += i; - r += '\''; - return r; -} void ignoreException(Verbosity lvl) @@ -289,120 +180,6 @@ void ignoreException(Verbosity lvl) } catch (...) { } } -constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - -std::string base64Encode(std::string_view s) -{ - std::string res; - res.reserve((s.size() + 2) / 3 * 4); - int data = 0, nbits = 0; - - for (char c : s) { - data = data << 8 | (unsigned char) c; - nbits += 8; - while (nbits >= 6) { - nbits -= 6; - res.push_back(base64Chars[data >> nbits & 0x3f]); - } - } - - if (nbits) res.push_back(base64Chars[data << (6 - nbits) & 0x3f]); - while (res.size() % 4) res.push_back('='); - - return res; -} - - -std::string base64Decode(std::string_view s) -{ - constexpr char npos = -1; - constexpr std::array base64DecodeChars = [&]() { - std::array result{}; - for (auto& c : result) - c = npos; - for (int i = 0; i < 64; i++) - result[base64Chars[i]] = i; - return result; - }(); - - std::string res; - // Some sequences are missing the padding consisting of up to two '='. - // vvv - res.reserve((s.size() + 2) / 4 * 3); - unsigned int d = 0, bits = 0; - - for (char c : s) { - if (c == '=') break; - if (c == '\n') continue; - - char digit = base64DecodeChars[(unsigned char) c]; - if (digit == npos) - throw Error("invalid character in Base64 string: '%c'", c); - - bits += 6; - d = d << 6 | digit; - if (bits >= 8) { - res.push_back(d >> (bits - 8) & 0xff); - bits -= 8; - } - } - - return res; -} - - -std::string stripIndentation(std::string_view s) -{ - size_t minIndent = 10000; - size_t curIndent = 0; - bool atStartOfLine = true; - - for (auto & c : s) { - if (atStartOfLine && c == ' ') - curIndent++; - else if (c == '\n') { - if (atStartOfLine) - minIndent = std::max(minIndent, curIndent); - curIndent = 0; - atStartOfLine = true; - } else { - if (atStartOfLine) { - minIndent = std::min(minIndent, curIndent); - atStartOfLine = false; - } - } - } - - std::string res; - - size_t pos = 0; - while (pos < s.size()) { - auto eol = s.find('\n', pos); - if (eol == s.npos) eol = s.size(); - if (eol - pos > minIndent) - res.append(s.substr(pos + minIndent, eol - pos - minIndent)); - res.push_back('\n'); - pos = eol + 1; - } - - return res; -} - - -std::pair getLine(std::string_view s) -{ - auto newline = s.find('\n'); - - if (newline == s.npos) { - return {s, ""}; - } else { - auto line = s.substr(0, newline); - if (!line.empty() && line[line.size() - 1] == '\r') - line = line.substr(0, line.size() - 1); - return {line, s.substr(newline + 1)}; - } -} - ////////////////////////////////////////////////////////////////////// @@ -460,9 +237,4 @@ void unshareFilesystem() } -std::string showBytes(uint64_t bytes) -{ - return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); -} - } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 216693635..907bdf4ed 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -106,193 +106,6 @@ void restoreMountNamespace(); void unshareFilesystem(); -/** - * Convert a list of strings to a null-terminated vector of `char - * *`s. The result must not be accessed beyond the lifetime of the - * list of strings. - */ -std::vector stringsToCharPtrs(const Strings & ss); - - -MakeError(FormatError, Error); - - -/** - * String tokenizer. - */ -template C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); - - -/** - * Concatenate the given strings with a separator between the - * elements. - */ -template -std::string concatStringsSep(const std::string_view sep, const C & ss) -{ - size_t size = 0; - // need a cast to string_view since this is also called with Symbols - for (const auto & s : ss) size += sep.size() + std::string_view(s).size(); - std::string s; - s.reserve(size); - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - -template -auto concatStrings(Parts && ... parts) - -> std::enable_if_t<(... && std::is_convertible_v), std::string> -{ - std::string_view views[sizeof...(parts)] = { parts... }; - return concatStringsSep({}, views); -} - - -/** - * Add quotes around a collection of strings. - */ -template Strings quoteStrings(const C & c) -{ - Strings res; - for (auto & s : c) - res.push_back("'" + s + "'"); - return res; -} - -/** - * Remove trailing whitespace from a string. - * - * \todo return std::string_view. - */ -std::string chomp(std::string_view s); - - -/** - * Remove whitespace from the start and end of a string. - */ -std::string trim(std::string_view s, std::string_view whitespace = " \n\r\t"); - - -/** - * Replace all occurrences of a string inside another string. - */ -std::string replaceStrings( - std::string s, - std::string_view from, - std::string_view to); - - -/** - * Rewrites a string given a map of replacements, applying the replacements in - * sorted order, only once, considering only the strings appearing in the input - * string in performing replacement. - * - * - Replacements are not performed on intermediate strings. That is, for an input - * `"abb"` with replacements `{"ab" -> "ba"}`, the result is `"bab"`. - * - Transitive replacements are not performed. For example, for the input `"abcde"` - * with replacements `{"a" -> "b", "b" -> "c", "e" -> "b"}`, the result is - * `"bccdb"`. - */ -class Rewriter -{ -private: - std::string initials; - std::map rewrites; - -public: - explicit Rewriter(std::map rewrites); - - std::string operator()(std::string s); -}; - -inline std::string rewriteStrings(std::string s, const StringMap & rewrites) -{ - return Rewriter(rewrites)(s); -} - - - -/** - * Parse a string into an integer. - */ -template -std::optional string2Int(const std::string_view s) -{ - if (s.substr(0, 1) == "-" && !std::numeric_limits::is_signed) - return std::nullopt; - try { - return boost::lexical_cast(s.data(), s.size()); - } catch (const boost::bad_lexical_cast &) { - return std::nullopt; - } -} - -/** - * Like string2Int(), but support an optional suffix 'K', 'M', 'G' or - * 'T' denoting a binary unit prefix. - */ -template -N string2IntWithUnitPrefix(std::string_view s) -{ - N multiplier = 1; - if (!s.empty()) { - char u = std::toupper(*s.rbegin()); - if (std::isalpha(u)) { - if (u == 'K') multiplier = 1ULL << 10; - else if (u == 'M') multiplier = 1ULL << 20; - else if (u == 'G') multiplier = 1ULL << 30; - else if (u == 'T') multiplier = 1ULL << 40; - else throw UsageError("invalid unit specifier '%1%'", u); - s.remove_suffix(1); - } - } - if (auto n = string2Int(s)) - return *n * multiplier; - throw UsageError("'%s' is not an integer", s); -} - -/** - * Parse a string into a float. - */ -template -std::optional string2Float(const std::string_view s) -{ - try { - return boost::lexical_cast(s.data(), s.size()); - } catch (const boost::bad_lexical_cast &) { - return std::nullopt; - } -} - - -/** - * Convert a little-endian integer to host order. - */ -template -T readLittleEndian(unsigned char * p) -{ - T x = 0; - for (size_t i = 0; i < sizeof(x); ++i, ++p) { - x |= ((T) *p) << (i * 8); - } - return x; -} - -/** - * Convert a string to lower case. - */ -std::string toLower(const std::string & s); - - -/** - * Escape a string as a shell word. - */ -std::string shellEscape(const std::string_view s); - - /** * Exception handling in destructors: print an error message, then * ignore the exception. @@ -301,38 +114,6 @@ void ignoreException(Verbosity lvl = lvlError); -/** - * Tree formatting. - */ -constexpr char treeConn[] = "├───"; -constexpr char treeLast[] = "└───"; -constexpr char treeLine[] = "│ "; -constexpr char treeNull[] = " "; - - -/** - * Base64 encoding/decoding. - */ -std::string base64Encode(std::string_view s); -std::string base64Decode(std::string_view s); - - -/** - * Remove common leading whitespace from the lines in the string - * 's'. For example, if every line is indented by at least 3 spaces, - * then we remove 3 spaces from the start of every line. - */ -std::string stripIndentation(std::string_view s); - - -/** - * Get the prefix of 's' up to and excluding the next line break (LF - * optionally preceded by CR), and the remainder following the line - * break. - */ -std::pair getLine(std::string_view s); - - /** * Get a value for the specified key from an associate container. */ @@ -443,33 +224,4 @@ template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; -std::string showBytes(uint64_t bytes); - - -/** - * Provide an addition operator between strings and string_views - * inexplicably omitted from the standard library. - */ -inline std::string operator + (const std::string & s1, std::string_view s2) -{ - auto s = s1; - s.append(s2); - return s; -} - -inline std::string operator + (std::string && s, std::string_view s2) -{ - s.append(s2); - return std::move(s); -} - -inline std::string operator + (std::string_view s1, const char * s2) -{ - std::string s; - s.reserve(s1.size() + strlen(s2)); - s.append(s1); - s.append(s2); - return s; -} - } diff --git a/tests/functional/repl_characterization/repl_characterization.cc b/tests/functional/repl_characterization/repl_characterization.cc index 0d3e5352d..4cc1e7e37 100644 --- a/tests/functional/repl_characterization/repl_characterization.cc +++ b/tests/functional/repl_characterization/repl_characterization.cc @@ -12,6 +12,7 @@ #include "tests/cli-literate-parser.hh" #include "tests/terminal-code-eater.hh" #include "util.hh" +#include "strings.hh" using namespace std::string_literals; diff --git a/tests/functional/repl_characterization/test-session.cc b/tests/functional/repl_characterization/test-session.cc index ab0710b24..be3d407f3 100644 --- a/tests/functional/repl_characterization/test-session.cc +++ b/tests/functional/repl_characterization/test-session.cc @@ -3,9 +3,9 @@ #include #include "test-session.hh" -#include "util.hh" #include "escape-char.hh" #include "processes.hh" +#include "strings.hh" namespace nix { diff --git a/tests/unit/libstore/machines.cc b/tests/unit/libstore/machines.cc index dc192e5be..ba27d85b7 100644 --- a/tests/unit/libstore/machines.cc +++ b/tests/unit/libstore/machines.cc @@ -1,6 +1,7 @@ #include "file-system.hh" #include "machines.hh" #include "globals.hh" +#include "strings.hh" #include "tests/test-data.hh" #include diff --git a/tests/unit/libutil-support/tests/cli-literate-parser.cc b/tests/unit/libutil-support/tests/cli-literate-parser.cc index 822f5e276..69bf9fb46 100644 --- a/tests/unit/libutil-support/tests/cli-literate-parser.cc +++ b/tests/unit/libutil-support/tests/cli-literate-parser.cc @@ -19,6 +19,7 @@ #include "shlex.hh" #include "types.hh" #include "util.hh" +#include "strings.hh" static constexpr const bool DEBUG_PARSER = false; diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index a42afd997..324ebd335 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -1,8 +1,9 @@ #include "file-system.hh" -#include "util.hh" #include "processes.hh" +#include "strings.hh" #include "types.hh" #include "terminal.hh" +#include "util.hh" #include From f79ee66646f66e5117583bcf9a579b7f538ca8bb Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 15:40:03 +0200 Subject: [PATCH 10/16] util.{hh,cc}: Split out users.{hh,cc} Change-Id: I1bd92479a2cb7e5c2c2e1541b80474adb05ea0df --- src/libcmd/repl.cc | 1 + src/libexpr/eval-cache.cc | 2 +- src/libexpr/eval-settings.cc | 3 +- src/libexpr/flake/config.cc | 2 +- src/libexpr/parser.y | 2 +- src/libfetchers/cache.cc | 1 + src/libfetchers/git.cc | 2 +- src/libfetchers/mercurial.cc | 1 + src/libfetchers/registry.cc | 2 +- src/libstore/globals.cc | 2 +- src/libstore/nar-info-disk-cache.cc | 1 + src/libstore/profiles.cc | 2 +- src/libstore/store-api.cc | 1 + src/libutil/file-system.cc | 1 + src/libutil/meson.build | 2 + src/libutil/users.cc | 105 ++++++++++++++++++++++++++++ src/libutil/users.hh | 61 ++++++++++++++++ src/libutil/util.cc | 97 ------------------------- src/libutil/util.hh | 47 ------------- src/nix-channel/nix-channel.cc | 2 +- src/nix-env/nix-env.cc | 2 +- 21 files changed, 185 insertions(+), 154 deletions(-) create mode 100644 src/libutil/users.cc create mode 100644 src/libutil/users.hh diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 02aa5a272..49865aa90 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -31,6 +31,7 @@ #include "print.hh" #include "progress-bar.hh" #include "gc-small-vector.hh" +#include "users.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 5969ee449..90fbfa308 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -1,8 +1,8 @@ #include "eval-cache.hh" #include "sqlite.hh" #include "eval.hh" -#include "eval-inline.hh" #include "store-api.hh" +#include "users.hh" namespace nix::eval_cache { diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 046ca557d..105fd3e9d 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -1,6 +1,7 @@ +#include "file-system.hh" #include "globals.hh" #include "profiles.hh" -#include "eval.hh" +#include "users.hh" #include "eval-settings.hh" namespace nix { diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index b9613462a..35c605de2 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -1,5 +1,5 @@ #include "flake.hh" -#include "globals.hh" +#include "users.hh" #include "fetch-settings.hh" #include diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 5424d8741..91cc7d089 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -19,7 +19,7 @@ #include #include "finally.hh" -#include "util.hh" +#include "users.hh" #include "nixexpr.hh" #include "eval.hh" diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 672e1e0bc..eec77c2dc 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -2,6 +2,7 @@ #include "sqlite.hh" #include "sync.hh" #include "store-api.hh" +#include "users.hh" #include diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 86d9d437f..07cbc781c 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -6,7 +6,7 @@ #include "store-api.hh" #include "url-parts.hh" #include "pathlocks.hh" -#include "util.hh" +#include "users.hh" #include "git.hh" #include "logging.hh" #include "finally.hh" diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 086db498f..4fffa71d3 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -3,6 +3,7 @@ #include "processes.hh" #include "store-api.hh" #include "url-parts.hh" +#include "users.hh" #include "fetch-settings.hh" diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 4b2d61f52..96e9ed1c6 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,6 +1,6 @@ #include "registry.hh" #include "fetchers.hh" -#include "util.hh" +#include "users.hh" #include "globals.hh" #include "store-api.hh" #include "local-fs-store.hh" diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 30fd4e9e3..8ef25c469 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,6 +1,6 @@ #include "environment-variables.hh" #include "globals.hh" -#include "util.hh" +#include "users.hh" #include "archive.hh" #include "args.hh" #include "abstract-setting-to-json.hh" diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index c7176d30f..0413abbf1 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -2,6 +2,7 @@ #include "sync.hh" #include "sqlite.hh" #include "globals.hh" +#include "users.hh" #include #include diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 239047dd6..e8b88693d 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -1,7 +1,7 @@ #include "profiles.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "util.hh" +#include "users.hh" #include #include diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index ed3566f5e..c5631dfd8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -14,6 +14,7 @@ // FIXME this should not be here, see TODO below on // `addMultipleToStore`. #include "worker-protocol.hh" +#include "users.hh" #include #include diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 9804c449c..721bf97e2 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -9,6 +9,7 @@ #include "serialise.hh" #include "signals.hh" #include "types.hh" +#include "users.hh" namespace fs = std::filesystem; diff --git a/src/libutil/meson.build b/src/libutil/meson.build index b662ea455..5e767b447 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -39,6 +39,7 @@ libutil_sources = files( 'unix-domain-socket.cc', 'url.cc', 'url-name.cc', + 'users.cc', 'util.cc', 'xml-writer.cc', ) @@ -109,6 +110,7 @@ libutil_headers = files( 'url-parts.hh', 'url-name.hh', 'url.hh', + 'users.hh', 'util.hh', 'variant-wrapper.hh', 'xml-writer.hh', diff --git a/src/libutil/users.cc b/src/libutil/users.cc new file mode 100644 index 000000000..a9a8a7353 --- /dev/null +++ b/src/libutil/users.cc @@ -0,0 +1,105 @@ +#include "environment-variables.hh" +#include "file-system.hh" +#include "logging.hh" +#include "strings.hh" + +#include +#include +#include + +namespace nix { + +std::string getUserName() +{ + auto pw = getpwuid(geteuid()); + std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); + if (name.empty()) + throw Error("cannot figure out user name"); + return name; +} + +Path getHomeOf(uid_t userId) +{ + std::vector buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + return pw->pw_dir; +} + +Path getHome() +{ + static Path homeDir = []() + { + std::optional unownedUserHomeDir = {}; + auto homeDir = getEnv("HOME"); + if (homeDir) { + // Only use $HOME if doesn't exist or is owned by the current user. + struct stat st; + int result = stat(homeDir->c_str(), &st); + if (result != 0) { + if (errno != ENOENT) { + warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); + homeDir.reset(); + } + } else if (st.st_uid != geteuid()) { + unownedUserHomeDir.swap(homeDir); + } + } + if (!homeDir) { + homeDir = getHomeOf(geteuid()); + if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { + warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); + } + } + return *homeDir; + }(); + return homeDir; +} + + +Path getCacheDir() +{ + auto cacheDir = getEnv("XDG_CACHE_HOME"); + return cacheDir ? *cacheDir : getHome() + "/.cache"; +} + + +Path getConfigDir() +{ + auto configDir = getEnv("XDG_CONFIG_HOME"); + return configDir ? *configDir : getHome() + "/.config"; +} + +std::vector getConfigDirs() +{ + Path configHome = getConfigDir(); + auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); + std::vector result = tokenizeString>(configDirs, ":"); + result.insert(result.begin(), configHome); + return result; +} + + +Path getDataDir() +{ + auto dataDir = getEnv("XDG_DATA_HOME"); + return dataDir ? *dataDir : getHome() + "/.local/share"; +} + +Path getStateDir() +{ + auto stateDir = getEnv("XDG_STATE_HOME"); + return stateDir ? *stateDir : getHome() + "/.local/state"; +} + +Path createNixStateDir() +{ + Path dir = getStateDir() + "/nix"; + createDirs(dir); + return dir; +} + +} diff --git a/src/libutil/users.hh b/src/libutil/users.hh new file mode 100644 index 000000000..3add4c732 --- /dev/null +++ b/src/libutil/users.hh @@ -0,0 +1,61 @@ +#pragma once +///@file + +#include "types.hh" + +#include + +namespace nix { + +std::string getUserName(); + +/** + * @return the given user's home directory from /etc/passwd. + */ +Path getHomeOf(uid_t userId); + +/** + * @return $HOME or the user's home directory from /etc/passwd. + */ +Path getHome(); + +/** + * @return $XDG_CACHE_HOME or $HOME/.cache. + */ +Path getCacheDir(); + +/** + * @return $XDG_CONFIG_HOME or $HOME/.config. + */ +Path getConfigDir(); + +/** + * @return the directories to search for user configuration files + */ +std::vector getConfigDirs(); + +/** + * @return $XDG_DATA_HOME or $HOME/.local/share. + */ +Path getDataDir(); + +/** + * @return $XDG_STATE_HOME or $HOME/.local/state. + * + * @note Not to be confused with settings.nixStateDir. + */ +Path getStateDir(); + +/** + * Create $XDG_STATE_HOME/nix or $HOME/.local/state/nix, and return + * the path to it. + * @note Not to be confused with settings.nixStateDir. + */ +Path createNixStateDir(); + +/** + * Perform tilde expansion on a path. + */ +std::string expandTilde(std::string_view path); + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 099a07622..8e813abc2 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -56,103 +56,6 @@ namespace nix { -std::string getUserName() -{ - auto pw = getpwuid(geteuid()); - std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); - if (name.empty()) - throw Error("cannot figure out user name"); - return name; -} - -Path getHomeOf(uid_t userId) -{ - std::vector buf(16384); - struct passwd pwbuf; - struct passwd * pw; - if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 - || !pw || !pw->pw_dir || !pw->pw_dir[0]) - throw Error("cannot determine user's home directory"); - return pw->pw_dir; -} - -Path getHome() -{ - static Path homeDir = []() - { - std::optional unownedUserHomeDir = {}; - auto homeDir = getEnv("HOME"); - if (homeDir) { - // Only use $HOME if doesn't exist or is owned by the current user. - struct stat st; - int result = stat(homeDir->c_str(), &st); - if (result != 0) { - if (errno != ENOENT) { - warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); - homeDir.reset(); - } - } else if (st.st_uid != geteuid()) { - unownedUserHomeDir.swap(homeDir); - } - } - if (!homeDir) { - homeDir = getHomeOf(geteuid()); - if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { - warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); - } - } - return *homeDir; - }(); - return homeDir; -} - - -Path getCacheDir() -{ - auto cacheDir = getEnv("XDG_CACHE_HOME"); - return cacheDir ? *cacheDir : getHome() + "/.cache"; -} - - -Path getConfigDir() -{ - auto configDir = getEnv("XDG_CONFIG_HOME"); - return configDir ? *configDir : getHome() + "/.config"; -} - -std::vector getConfigDirs() -{ - Path configHome = getConfigDir(); - auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); - std::vector result = tokenizeString>(configDirs, ":"); - result.insert(result.begin(), configHome); - return result; -} - - -Path getDataDir() -{ - auto dataDir = getEnv("XDG_DATA_HOME"); - return dataDir ? *dataDir : getHome() + "/.local/share"; -} - -Path getStateDir() -{ - auto stateDir = getEnv("XDG_STATE_HOME"); - return stateDir ? *stateDir : getHome() + "/.local/state"; -} - -Path createNixStateDir() -{ - Path dir = getStateDir() + "/nix"; - createDirs(dir); - return dir; -} - - - - - ////////////////////////////////////////////////////////////////////// diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 907bdf4ed..e408821b9 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -39,53 +39,6 @@ struct Source; extern const std::string nativeSystem; -std::string getUserName(); - -/** - * @return the given user's home directory from /etc/passwd. - */ -Path getHomeOf(uid_t userId); - -/** - * @return $HOME or the user's home directory from /etc/passwd. - */ -Path getHome(); - -/** - * @return $XDG_CACHE_HOME or $HOME/.cache. - */ -Path getCacheDir(); - -/** - * @return $XDG_CONFIG_HOME or $HOME/.config. - */ -Path getConfigDir(); - -/** - * @return the directories to search for user configuration files - */ -std::vector getConfigDirs(); - -/** - * @return $XDG_DATA_HOME or $HOME/.local/share. - */ -Path getDataDir(); - -/** - * @return $XDG_STATE_HOME or $HOME/.local/state. - * - * @note Not to be confused with settings.nixStateDir. - */ -Path getStateDir(); - -/** - * Create $XDG_STATE_HOME/nix or $HOME/.local/state/nix, and return - * the path to it. - * @note Not to be confused with settings.nixStateDir. - */ -Path createNixStateDir(); - - /** * Save the current mount namespace. Ignored if called more than * once. diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 26003f021..971337b63 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -6,7 +6,7 @@ #include "legacy.hh" #include "fetchers.hh" #include "eval-settings.hh" // for defexpr -#include "util.hh" +#include "users.hh" #include #include diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index ad255a1e1..227a3331c 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -11,7 +11,7 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "user-env.hh" -#include "util.hh" +#include "users.hh" #include "value-to-json.hh" #include "xml-writer.hh" #include "legacy.hh" From 8b6d2d39155e88250c576571a1251769b926ee83 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 16:01:11 +0200 Subject: [PATCH 11/16] util.{hh,cc}: Split out namespaces.{hh,cc} Change-Id: I8fd3f3b50c15ede29d489066b4e8d99c2c4636a6 --- src/libstore/filetransfer.cc | 2 +- src/libutil/current-process.cc | 1 + src/libutil/namespaces.cc | 66 +++++++++++++++++++++++++++++++--- src/libutil/namespaces.hh | 20 +++++++++++ src/libutil/util.cc | 52 --------------------------- src/libutil/util.hh | 20 ----------- src/nix/main.cc | 1 + 7 files changed, 85 insertions(+), 77 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 492463a61..dc656f444 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1,5 +1,5 @@ #include "filetransfer.hh" -#include "util.hh" +#include "namespaces.hh" #include "globals.hh" #include "store-api.hh" #include "s3.hh" diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 41f591b0c..f9a08685f 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -1,6 +1,7 @@ #include "current-process.hh" #include "file-system.hh" #include "logging.hh" +#include "namespaces.hh" #include "signals.hh" #include "util.hh" #include "strings.hh" diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index d092e6fcc..98d3cd306 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -1,5 +1,4 @@ -#if __linux__ - +#include "file-descriptor.hh" #include "file-system.hh" #include "logging.hh" #include "namespaces.hh" @@ -8,8 +7,67 @@ #include +#if __linux__ +# include +# include +#endif + namespace nix { +#if __linux__ +static AutoCloseFD fdSavedMountNamespace; +static AutoCloseFD fdSavedRoot; +#endif + +void saveMountNamespace() +{ +#if __linux__ + static std::once_flag done; + std::call_once(done, []() { + fdSavedMountNamespace = AutoCloseFD{open("/proc/self/ns/mnt", O_RDONLY)}; + if (!fdSavedMountNamespace) + throw SysError("saving parent mount namespace"); + + fdSavedRoot = AutoCloseFD{open("/proc/self/root", O_RDONLY)}; + }); +#endif +} + +void restoreMountNamespace() +{ +#if __linux__ + try { + auto savedCwd = absPath("."); + + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + throw SysError("restoring parent mount namespace"); + + if (fdSavedRoot) { + if (fchdir(fdSavedRoot.get())) + throw SysError("chdir into saved root"); + if (chroot(".")) + throw SysError("chroot into saved root"); + } + + if (chdir(savedCwd.c_str()) == -1) + throw SysError("restoring cwd"); + } catch (Error & e) { + debug(e.msg()); + } +#endif +} + +void unshareFilesystem() +{ +#ifdef __linux__ + if (unshare(CLONE_FS) != 0 && errno != EPERM) + throw SysError("unsharing filesystem state in download thread"); +#endif +} + + +#if __linux__ + static void diagnoseUserNamespaces() { if (!pathExists("/proc/self/ns/user")) { @@ -95,6 +153,6 @@ bool mountAndPidNamespacesSupported() return res; } -} - #endif + +} diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh index 0b7eeb66c..3a920e665 100644 --- a/src/libutil/namespaces.hh +++ b/src/libutil/namespaces.hh @@ -3,6 +3,26 @@ namespace nix { +/** + * Save the current mount namespace. Ignored if called more than + * once. + */ +void saveMountNamespace(); + +/** + * Restore the mount namespace saved by saveMountNamespace(). Ignored + * if saveMountNamespace() was never called. + */ +void restoreMountNamespace(); + +/** + * Cause this thread to not share any FS attributes with the main + * thread, because this causes setns() in restoreMountNamespace() to + * fail. + */ +void unshareFilesystem(); + + #if __linux__ bool userNamespacesSupported(); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 8e813abc2..6d6e55ad1 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -88,56 +88,4 @@ void ignoreException(Verbosity lvl) -#if __linux__ -static AutoCloseFD fdSavedMountNamespace; -static AutoCloseFD fdSavedRoot; -#endif - -void saveMountNamespace() -{ -#if __linux__ - static std::once_flag done; - std::call_once(done, []() { - fdSavedMountNamespace = AutoCloseFD{open("/proc/self/ns/mnt", O_RDONLY)}; - if (!fdSavedMountNamespace) - throw SysError("saving parent mount namespace"); - - fdSavedRoot = AutoCloseFD{open("/proc/self/root", O_RDONLY)}; - }); -#endif -} - -void restoreMountNamespace() -{ -#if __linux__ - try { - auto savedCwd = absPath("."); - - if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) - throw SysError("restoring parent mount namespace"); - - if (fdSavedRoot) { - if (fchdir(fdSavedRoot.get())) - throw SysError("chdir into saved root"); - if (chroot(".")) - throw SysError("chroot into saved root"); - } - - if (chdir(savedCwd.c_str()) == -1) - throw SysError("restoring cwd"); - } catch (Error & e) { - debug(e.msg()); - } -#endif -} - -void unshareFilesystem() -{ -#ifdef __linux__ - if (unshare(CLONE_FS) != 0 && errno != EPERM) - throw SysError("unsharing filesystem state in download thread"); -#endif -} - - } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index e408821b9..867f0a80d 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -39,26 +39,6 @@ struct Source; extern const std::string nativeSystem; -/** - * Save the current mount namespace. Ignored if called more than - * once. - */ -void saveMountNamespace(); - -/** - * Restore the mount namespace saved by saveMountNamespace(). Ignored - * if saveMountNamespace() was never called. - */ -void restoreMountNamespace(); - -/** - * Cause this thread to not share any FS attributes with the main - * thread, because this causes setns() in restoreMountNamespace() to - * fail. - */ -void unshareFilesystem(); - - /** * Exception handling in destructors: print an error message, then * ignore the exception. diff --git a/src/nix/main.cc b/src/nix/main.cc index 0f98861d5..5d92cfd76 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -8,6 +8,7 @@ #include "eval-settings.hh" #include "globals.hh" #include "legacy.hh" +#include "namespaces.hh" #include "shared.hh" #include "store-api.hh" #include "filetransfer.hh" From 93ebb3e7dfb75ce23c1fb46d8e70208a2bad0c02 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 16:09:33 +0200 Subject: [PATCH 12/16] util.{hh,cc}: Move ignoreException to error.{hh,cc} Change-Id: Iae6464217a55c313a983e5c651b26a4a1e446706 --- src/libutil/current-process.cc | 2 +- src/libutil/error.cc | 13 +++++++++++++ src/libutil/error.hh | 6 ++++++ src/libutil/util.cc | 12 ------------ src/libutil/util.hh | 7 ------- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index f9a08685f..c64dd1e0d 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -1,9 +1,9 @@ #include "current-process.hh" +#include "error.hh" #include "file-system.hh" #include "logging.hh" #include "namespaces.hh" #include "signals.hh" -#include "util.hh" #include "strings.hh" #ifdef __APPLE__ diff --git a/src/libutil/error.cc b/src/libutil/error.cc index f780aabef..1a294bf08 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -417,4 +417,17 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s return out; } +void ignoreException(Verbosity lvl) +{ + /* Make sure no exceptions leave this function. + printError() also throws when remote is closed. */ + try { + try { + throw; + } catch (std::exception & e) { + printMsg(lvl, "error (ignored): %1%", e.what()); + } + } catch (...) { } +} + } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 323365d65..0884f9f32 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -202,4 +202,10 @@ public: } }; +/** + * Exception handling in destructors: print an error message, then + * ignore the exception. + */ +void ignoreException(Verbosity lvl = lvlError); + } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 6d6e55ad1..d9bd71b5f 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -70,18 +70,6 @@ namespace nix { -void ignoreException(Verbosity lvl) -{ - /* Make sure no exceptions leave this function. - printError() also throws when remote is closed. */ - try { - try { - throw; - } catch (std::exception & e) { - printMsg(lvl, "error (ignored): %1%", e.what()); - } - } catch (...) { } -} ////////////////////////////////////////////////////////////////////// diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 867f0a80d..c5ca12650 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -39,13 +39,6 @@ struct Source; extern const std::string nativeSystem; -/** - * Exception handling in destructors: print an error message, then - * ignore the exception. - */ -void ignoreException(Verbosity lvl = lvlError); - - /** * Get a value for the specified key from an associate container. From 74513483bc5572d988a059b8e964662d66f1667f Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 16:20:49 +0200 Subject: [PATCH 13/16] util.cc: Delete remaining file Change-Id: I2b47848904f2ce7bd78b83738e99a4c9da627751 --- src/libutil/meson.build | 1 - src/libutil/util.cc | 79 ----------------------------------------- 2 files changed, 80 deletions(-) delete mode 100644 src/libutil/util.cc diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 5e767b447..64d84c714 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -40,7 +40,6 @@ libutil_sources = files( 'url.cc', 'url-name.cc', 'users.cc', - 'util.cc', 'xml-writer.cc', ) diff --git a/src/libutil/util.cc b/src/libutil/util.cc deleted file mode 100644 index d9bd71b5f..000000000 --- a/src/libutil/util.cc +++ /dev/null @@ -1,79 +0,0 @@ -#include "util.hh" -#include "processes.hh" -#include "strings.hh" -#include "current-process.hh" - -#include "sync.hh" -#include "finally.hh" -#include "serialise.hh" -#include "cgroup.hh" -#include "signals.hh" -#include "environment-variables.hh" -#include "file-system.hh" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#include -#endif - -#ifdef __linux__ -#include -#include -#include - -#include -#endif - - -#ifdef NDEBUG -#error "Lix may not be built with assertions disabled (i.e. with -DNDEBUG)." -#endif - -namespace nix { - - -////////////////////////////////////////////////////////////////////// - - -////////////////////////////////////////////////////////////////////// - - - - -////////////////////////////////////////////////////////////////////// - - - - - - - -////////////////////////////////////////////////////////////////////// - - - -} From d73c40ff3d74e9b1bac24c104fdf7738df3c3029 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 16:24:19 +0200 Subject: [PATCH 14/16] util.hh: Move stuff to types.hh Change-Id: Ia852306a4b8aac6856dc42bc69e4b58b53a0d67c --- src/libutil/types.hh | 111 +++++++++++++++++++++++++++++++++++++++++++ src/libutil/util.hh | 108 ----------------------------------------- 2 files changed, 111 insertions(+), 108 deletions(-) diff --git a/src/libutil/types.hh b/src/libutil/types.hh index c86f52175..4974634d8 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -4,6 +4,7 @@ #include "ref.hh" #include +#include #include #include #include @@ -51,6 +52,116 @@ struct Explicit { } }; +/** + * Get a value for the specified key from an associate container. + */ +template +const typename T::mapped_type * get(const T & map, const typename T::key_type & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &i->second; +} + +template +typename T::mapped_type * get(T & map, const typename T::key_type & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &i->second; +} + +/** + * Get a value for the specified key from an associate container, or a default value if the key isn't present. + */ +template +const typename T::mapped_type & getOr(T & map, + const typename T::key_type & key, + const typename T::mapped_type & defaultValue) +{ + auto i = map.find(key); + if (i == map.end()) return defaultValue; + return i->second; +} + +/** + * Remove and return the first item from a container. + */ +template +std::optional remove_begin(T & c) +{ + auto i = c.begin(); + if (i == c.end()) return {}; + auto v = std::move(*i); + c.erase(i); + return v; +} + + +/** + * Remove and return the first item from a container. + */ +template +std::optional pop(T & c) +{ + if (c.empty()) return {}; + auto v = std::move(c.front()); + c.pop(); + return v; +} + + +/** + * A RAII helper that increments a counter on construction and + * decrements it on destruction. + */ +template +struct MaintainCount +{ + T & counter; + long delta; + MaintainCount(T & counter, long delta = 1) : counter(counter), delta(delta) { counter += delta; } + ~MaintainCount() { counter -= delta; } +}; + + +/** + * A Rust/Python-like enumerate() iterator adapter. + * + * Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. + */ +template ())), + typename = decltype(std::end(std::declval()))> +constexpr auto enumerate(T && iterable) +{ + struct iterator + { + size_t i; + TIter iter; + constexpr bool operator != (const iterator & other) const { return iter != other.iter; } + constexpr void operator ++ () { ++i; ++iter; } + constexpr auto operator * () const { return std::tie(i, *iter); } + }; + + struct iterable_wrapper + { + T iterable; + constexpr auto begin() { return iterator{ 0, std::begin(iterable) }; } + constexpr auto end() { return iterator{ 0, std::end(iterable) }; } + }; + + return iterable_wrapper{ std::forward(iterable) }; +} + + +/** + * C++17 std::visit boilerplate + */ +template struct overloaded : Ts... { using Ts::operator()...; }; +template overloaded(Ts...) -> overloaded; + + /** * This wants to be a little bit like rust's Cow type. diff --git a/src/libutil/util.hh b/src/libutil/util.hh index c5ca12650..4e20550a7 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -40,114 +40,6 @@ extern const std::string nativeSystem; -/** - * Get a value for the specified key from an associate container. - */ -template -const typename T::mapped_type * get(const T & map, const typename T::key_type & key) -{ - auto i = map.find(key); - if (i == map.end()) return nullptr; - return &i->second; -} - -template -typename T::mapped_type * get(T & map, const typename T::key_type & key) -{ - auto i = map.find(key); - if (i == map.end()) return nullptr; - return &i->second; -} - -/** - * Get a value for the specified key from an associate container, or a default value if the key isn't present. - */ -template -const typename T::mapped_type & getOr(T & map, - const typename T::key_type & key, - const typename T::mapped_type & defaultValue) -{ - auto i = map.find(key); - if (i == map.end()) return defaultValue; - return i->second; -} - -/** - * Remove and return the first item from a container. - */ -template -std::optional remove_begin(T & c) -{ - auto i = c.begin(); - if (i == c.end()) return {}; - auto v = std::move(*i); - c.erase(i); - return v; -} - - -/** - * Remove and return the first item from a container. - */ -template -std::optional pop(T & c) -{ - if (c.empty()) return {}; - auto v = std::move(c.front()); - c.pop(); - return v; -} - - -/** - * A RAII helper that increments a counter on construction and - * decrements it on destruction. - */ -template -struct MaintainCount -{ - T & counter; - long delta; - MaintainCount(T & counter, long delta = 1) : counter(counter), delta(delta) { counter += delta; } - ~MaintainCount() { counter -= delta; } -}; - - -/** - * A Rust/Python-like enumerate() iterator adapter. - * - * Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. - */ -template ())), - typename = decltype(std::end(std::declval()))> -constexpr auto enumerate(T && iterable) -{ - struct iterator - { - size_t i; - TIter iter; - constexpr bool operator != (const iterator & other) const { return iter != other.iter; } - constexpr void operator ++ () { ++i; ++iter; } - constexpr auto operator * () const { return std::tie(i, *iter); } - }; - - struct iterable_wrapper - { - T iterable; - constexpr auto begin() { return iterator{ 0, std::begin(iterable) }; } - constexpr auto end() { return iterator{ 0, std::end(iterable) }; } - }; - - return iterable_wrapper{ std::forward(iterable) }; -} - - -/** - * C++17 std::visit boilerplate - */ -template struct overloaded : Ts... { using Ts::operator()...; }; -template overloaded(Ts...) -> overloaded; } From f0eb650ee8a0eb5244c15fab3b152a1f70e1506e Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 16:27:48 +0200 Subject: [PATCH 15/16] util.hh: Move nativeSystem to local-derivation-goal.cc Change-Id: I74565fbfd3aeedef8f50465808fac712b84e47ad --- src/libstore/build/local-derivation-goal.cc | 5 +++++ src/libutil/error.cc | 2 -- src/libutil/util.hh | 5 ----- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index fd871e8a4..479b4ffeb 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -64,6 +64,11 @@ extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, namespace nix { +/** + * The system for which Nix is compiled. + */ +constexpr std::string_view nativeSystem = SYSTEM; + void handleDiffHook( uid_t uid, uid_t gid, const Path & tryA, const Path & tryB, diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 1a294bf08..2a7fd3d0e 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -10,8 +10,6 @@ namespace nix { -const std::string nativeSystem = SYSTEM; - void BaseError::addTrace(std::shared_ptr && e, HintFmt hint) { err.traces.push_front(Trace { .pos = std::move(e), .hint = hint }); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 4e20550a7..1066f212a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -33,11 +33,6 @@ namespace nix { struct Sink; struct Source; -/** - * The system for which Nix is compiled. - */ -extern const std::string nativeSystem; - From a39ba22ff7112cd3984bbf28d8610d84dd525a0f Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 28 May 2024 16:53:21 +0200 Subject: [PATCH 16/16] util.hh: Delete remaining file and clean up headers Change-Id: Ic1f68e6af658e94ef7922841dd3ad4c69551ef56 --- perl/lib/Nix/Store.xs | 1 - src/libcmd/common-eval-args.cc | 1 - src/libcmd/editor-for.cc | 1 - src/libcmd/installable-attr-path.cc | 1 - src/libcmd/installable-attr-path.hh | 1 - src/libcmd/installable-flake.cc | 1 - src/libcmd/installables.cc | 7 +--- src/libcmd/installables.hh | 1 - src/libcmd/markdown.cc | 2 +- src/libcmd/repl-interacter.cc | 7 ++-- src/libexpr/attr-path.cc | 1 - src/libexpr/eval.cc | 1 - src/libexpr/flake/config.cc | 1 + src/libexpr/get-drvs.cc | 1 - src/libexpr/nixexpr.cc | 1 - src/libexpr/print-ambiguous.cc | 3 +- src/libexpr/search-path.cc | 1 - src/libexpr/value-to-json.cc | 1 - src/libexpr/value-to-xml.cc | 4 -- src/libexpr/value/context.hh | 1 - src/libfetchers/fetch-settings.hh | 1 - src/libfetchers/fetch-to-store.hh | 1 - src/libmain/common-args.cc | 1 + src/libmain/loggers.cc | 1 - src/libmain/progress-bar.cc | 1 - src/libmain/shared.cc | 1 - src/libmain/shared.hh | 1 - src/libstore/build/derivation-goal.cc | 1 - src/libstore/common-protocol.cc | 3 -- src/libstore/crypto.hh | 2 +- src/libstore/derivations.cc | 1 - src/libstore/derived-path.hh | 2 +- src/libstore/filetransfer.cc | 1 - src/libstore/filetransfer.hh | 3 +- src/libstore/gc.cc | 6 +-- src/libstore/globals.cc | 5 ++- src/libstore/local-store.hh | 2 - src/libstore/nar-info-disk-cache.cc | 1 + src/libstore/optimise-store.cc | 1 - src/libstore/outputs-spec.cc | 1 - src/libstore/path-references.cc | 4 -- src/libstore/pathlocks.cc | 4 +- src/libstore/remote-store.cc | 1 - src/libstore/serve-protocol.cc | 2 - src/libstore/sqlite.cc | 3 +- src/libstore/store-api.cc | 2 - src/libstore/store-api.hh | 1 + src/libstore/worker-protocol.cc | 1 - src/libutil/archive.cc | 2 +- src/libutil/args.cc | 1 + src/libutil/args.hh | 8 +++- src/libutil/comparator.hh | 2 + src/libutil/compression.cc | 3 -- src/libutil/config-impl.hh | 2 + src/libutil/config.cc | 1 + src/libutil/error.cc | 2 +- src/libutil/experimental-features.cc | 1 - src/libutil/file-descriptor.cc | 1 + src/libutil/file-system.cc | 1 + src/libutil/hash.cc | 3 +- src/libutil/logging.cc | 3 -- src/libutil/meson.build | 1 - src/libutil/references.cc | 4 +- src/libutil/serialise.cc | 1 - src/libutil/serialise.hh | 1 - src/libutil/signals.cc | 1 - src/libutil/signals.hh | 2 - src/libutil/split.hh | 2 - src/libutil/suggestions.cc | 1 + src/libutil/suggestions.hh | 5 ++- src/libutil/tarfile.cc | 1 + src/libutil/thread-pool.cc | 1 + src/libutil/thread-pool.hh | 3 +- src/libutil/url-name.cc | 1 - src/libutil/url-name.hh | 3 -- src/libutil/util.hh | 40 ------------------- src/nix-build/nix-build.cc | 1 - src/nix-env/user-env.cc | 1 - src/nix-instantiate/nix-instantiate.cc | 1 - src/nix-store/dotgraph.cc | 1 - src/nix-store/graphml.cc | 1 - src/nix-store/nix-store.cc | 1 - src/nix/daemon.cc | 1 - src/nix/develop.cc | 1 - src/nix/doctor.cc | 1 - .../repl_characterization.cc | 3 -- tests/unit/libcmd/args.cc | 1 - .../unit/libstore-support/tests/test-data.hh | 2 +- .../libutil-support/tests/characterization.hh | 1 - .../tests/cli-literate-parser.cc | 2 - tests/unit/libutil/logging.cc | 1 - tests/unit/libutil/tests.cc | 3 +- 92 files changed, 50 insertions(+), 161 deletions(-) delete mode 100644 src/libutil/util.hh diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index e96885e4c..4bef020d3 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -11,7 +11,6 @@ #include "derivations.hh" #include "globals.hh" #include "store-api.hh" -#include "util.hh" #include "crypto.hh" #include diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 9beea5aa2..e86213020 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -2,7 +2,6 @@ #include "common-eval-args.hh" #include "shared.hh" #include "filetransfer.hh" -#include "util.hh" #include "eval.hh" #include "fetchers.hh" #include "registry.hh" diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc index a56a7065e..67653d9c9 100644 --- a/src/libcmd/editor-for.cc +++ b/src/libcmd/editor-for.cc @@ -1,4 +1,3 @@ -#include "util.hh" #include "editor-for.hh" #include "environment-variables.hh" #include "source-path.hh" diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index 06e507872..eb15fecc3 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -1,7 +1,6 @@ #include "globals.hh" #include "installable-attr-path.hh" #include "outputs-spec.hh" -#include "util.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" diff --git a/src/libcmd/installable-attr-path.hh b/src/libcmd/installable-attr-path.hh index e9f0c33da..86c2f8219 100644 --- a/src/libcmd/installable-attr-path.hh +++ b/src/libcmd/installable-attr-path.hh @@ -4,7 +4,6 @@ #include "globals.hh" #include "installable-value.hh" #include "outputs-spec.hh" -#include "util.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 46bdd411b..b1ebd339e 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -2,7 +2,6 @@ #include "installable-flake.hh" #include "installable-derived-path.hh" #include "outputs-spec.hh" -#include "util.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 2c18653e4..ab0e4fd1c 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -3,26 +3,21 @@ #include "installable-derived-path.hh" #include "installable-attr-path.hh" #include "installable-flake.hh" +#include "logging.hh" #include "outputs-spec.hh" -#include "util.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" #include "derivations.hh" -#include "eval-inline.hh" #include "eval.hh" #include "eval-settings.hh" -#include "get-drvs.hh" #include "store-api.hh" #include "shared.hh" #include "flake/flake.hh" #include "eval-cache.hh" -#include "url.hh" #include "registry.hh" #include "build-result.hh" -#include -#include #include diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index ad5b4f759..95e8841ca 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "util.hh" #include "path.hh" #include "outputs-spec.hh" #include "derived-path.hh" diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 8b3bbc1b5..dbaab8c19 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,5 +1,5 @@ #include "markdown.hh" -#include "util.hh" +#include "error.hh" #include "finally.hh" #include "terminal.hh" diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 41589cda1..0cf4e34b8 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -1,3 +1,7 @@ +#include "error.hh" +#include "file-system.hh" +#include "logging.hh" +#include #include #include #include @@ -18,11 +22,8 @@ extern "C" { } #endif -#include "signals.hh" #include "finally.hh" #include "repl-interacter.hh" -#include "util.hh" -#include "repl.hh" namespace nix { diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index f4da99a9a..3e60f295d 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -1,6 +1,5 @@ #include "attr-path.hh" #include "eval-inline.hh" -#include "util.hh" namespace nix { diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index a8b37325b..c9a624eeb 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -5,7 +5,6 @@ #include "print-options.hh" #include "shared.hh" #include "types.hh" -#include "util.hh" #include "store-api.hh" #include "derivations.hh" #include "downstream-placeholder.hh" diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index 35c605de2..b330d96f9 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -1,4 +1,5 @@ #include "flake.hh" +#include "logging.hh" #include "users.hh" #include "fetch-settings.hh" diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index e686ffe8c..8c8e142b8 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -1,5 +1,4 @@ #include "get-drvs.hh" -#include "util.hh" #include "eval-inline.hh" #include "derivations.hh" #include "store-api.hh" diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 72a8764e6..6a1aa8f35 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -2,7 +2,6 @@ #include "derivations.hh" #include "eval.hh" #include "symbol-table.hh" -#include "util.hh" #include "print.hh" #include "escape-string.hh" diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc index ec30f5073..bcf86b5c6 100644 --- a/src/libexpr/print-ambiguous.cc +++ b/src/libexpr/print-ambiguous.cc @@ -1,6 +1,7 @@ #include "print-ambiguous.hh" +#include "attr-set.hh" +#include "logging.hh" #include "print.hh" -#include "eval.hh" #include "signals.hh" #include "escape-string.hh" diff --git a/src/libexpr/search-path.cc b/src/libexpr/search-path.cc index 180d5f8b1..a25767496 100644 --- a/src/libexpr/search-path.cc +++ b/src/libexpr/search-path.cc @@ -1,5 +1,4 @@ #include "search-path.hh" -#include "util.hh" namespace nix { diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index ebb3379a8..5743d9057 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -1,6 +1,5 @@ #include "value-to-json.hh" #include "eval-inline.hh" -#include "util.hh" #include "signals.hh" #include "store-api.hh" diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 5d1fbd28d..fda360eff 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -1,11 +1,7 @@ #include "value-to-xml.hh" #include "xml-writer.hh" -#include "eval-inline.hh" -#include "util.hh" #include "signals.hh" -#include - namespace nix { diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 998b70e36..7f23cd3a4 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "util.hh" #include "comparator.hh" #include "derived-path.hh" #include "variant-wrapper.hh" diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index c67a75082..2dc2834fb 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -3,7 +3,6 @@ #include "types.hh" #include "config.hh" -#include "util.hh" #include #include diff --git a/src/libfetchers/fetch-to-store.hh b/src/libfetchers/fetch-to-store.hh index 717450944..b74879637 100644 --- a/src/libfetchers/fetch-to-store.hh +++ b/src/libfetchers/fetch-to-store.hh @@ -3,7 +3,6 @@ #include "source-path.hh" #include "store-api.hh" -#include "util.hh" #include "repair-flag.hh" #include "content-address.hh" diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 12ce289c5..af7f46296 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -2,6 +2,7 @@ #include "args/root.hh" #include "globals.hh" #include "loggers.hh" +#include "logging.hh" namespace nix { diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index 7e80a31f1..80080d616 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -1,7 +1,6 @@ #include "environment-variables.hh" #include "loggers.hh" #include "progress-bar.hh" -#include "util.hh" namespace nix { diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index f3700f103..11b2fe800 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -1,5 +1,4 @@ #include "progress-bar.hh" -#include "util.hh" #include "sync.hh" #include "store-api.hh" #include "names.hh" diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 377cd6abd..f99777a20 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -2,7 +2,6 @@ #include "shared.hh" #include "store-api.hh" #include "gc-store.hh" -#include "util.hh" #include "signals.hh" #include "loggers.hh" #include "progress-bar.hh" diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 907f336f1..b791980fa 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "util.hh" #include "args.hh" #include "args/root.hh" #include "common-args.hh" diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5fa5deb7c..97ba994ad 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -5,7 +5,6 @@ #include "builtins/buildenv.hh" #include "references.hh" #include "finally.hh" -#include "util.hh" #include "archive.hh" #include "compression.hh" #include "common-protocol.hh" diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index f906814bc..456ad2b1f 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -1,11 +1,8 @@ #include "serialise.hh" -#include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" -#include "build-result.hh" #include "common-protocol.hh" #include "common-protocol-impl.hh" -#include "archive.hh" #include "derivations.hh" #include diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh index 35216d470..22588cf97 100644 --- a/src/libstore/crypto.hh +++ b/src/libstore/crypto.hh @@ -1,9 +1,9 @@ #pragma once ///@file -#include "types.hh" #include +#include namespace nix { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index fef680421..4e70804e5 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -3,7 +3,6 @@ #include "store-api.hh" #include "globals.hh" #include "types.hh" -#include "util.hh" #include "split.hh" #include "common-protocol.hh" #include "common-protocol-impl.hh" diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 4d7033df2..c87cf2004 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "util.hh" +#include "config.hh" #include "path.hh" #include "outputs-spec.hh" #include "comparator.hh" diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index dc656f444..cc1656a17 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 3f55995ef..c692f1485 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -1,8 +1,9 @@ #pragma once ///@file +#include "logging.hh" +#include "serialise.hh" #include "types.hh" -#include "hash.hh" #include "config.hh" #include diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 722452b8d..d58e3c8eb 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,18 +1,14 @@ -#include "derivations.hh" #include "globals.hh" #include "local-store.hh" +#include "pathlocks.hh" #include "processes.hh" #include "signals.hh" #include "finally.hh" #include "unix-domain-socket.hh" -#include #include -#include #include -#include -#include #include #include #include diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 8ef25c469..993a8cfa5 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,14 +1,15 @@ #include "environment-variables.hh" #include "globals.hh" +#include "file-system.hh" +#include "logging.hh" +#include "strings.hh" #include "users.hh" -#include "archive.hh" #include "args.hh" #include "abstract-setting-to-json.hh" #include "compute-levels.hh" #include "current-process.hh" #include -#include #include #include #include diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 14f024ca9..808fdc202 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -3,11 +3,9 @@ #include "sqlite.hh" -#include "pathlocks.hh" #include "store-api.hh" #include "indirect-root-store.hh" #include "sync.hh" -#include "util.hh" #include #include diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 0413abbf1..169b63819 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -1,4 +1,5 @@ #include "nar-info-disk-cache.hh" +#include "logging.hh" #include "sync.hh" #include "sqlite.hh" #include "globals.hh" diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index e09072e90..4f02296c3 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -1,4 +1,3 @@ -#include "util.hh" #include "local-store.hh" #include "globals.hh" #include "signals.hh" diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc index 4422bcd21..0d740c324 100644 --- a/src/libstore/outputs-spec.cc +++ b/src/libstore/outputs-spec.cc @@ -5,7 +5,6 @@ #include "outputs-spec.hh" #include "path-regex.hh" #include "strings.hh" -#include "util.hh" namespace nix { diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc index 33cf66ce3..8a02e9006 100644 --- a/src/libstore/path-references.cc +++ b/src/libstore/path-references.cc @@ -1,12 +1,8 @@ #include "path-references.hh" #include "hash.hh" -#include "util.hh" #include "archive.hh" #include -#include -#include -#include namespace nix { diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 3e654c1c9..ced0f30bb 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -1,10 +1,8 @@ #include "pathlocks.hh" -#include "util.hh" +#include "logging.hh" #include "signals.hh" -#include "sync.hh" #include -#include #include #include diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 22d87b027..93b1afabd 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,5 +1,4 @@ #include "serialise.hh" -#include "util.hh" #include "signals.hh" #include "path-with-outputs.hh" #include "gc-store.hh" diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index 2e15d28d5..603137c81 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -1,11 +1,9 @@ #include "serialise.hh" -#include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" #include "build-result.hh" #include "serve-protocol.hh" #include "serve-protocol-impl.hh" -#include "archive.hh" #include "path-info.hh" #include diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 4bd425b46..f40217734 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -1,12 +1,11 @@ #include "sqlite.hh" #include "globals.hh" -#include "util.hh" +#include "logging.hh" #include "signals.hh" #include "url.hh" #include -#include namespace nix { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c5631dfd8..eb9b5c3ef 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,9 +1,7 @@ -#include "crypto.hh" #include "fs-accessor.hh" #include "globals.hh" #include "derivations.hh" #include "store-api.hh" -#include "util.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" #include "url.hh" diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 745fce594..397ebe759 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "logging.hh" #include "nar-info.hh" #include "realisation.hh" #include "path.hh" diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index ad94c79ee..d76a8d2b8 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -1,5 +1,4 @@ #include "serialise.hh" -#include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" #include "build-result.hh" diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index c3f39d4b3..1b42ee4b5 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -13,8 +13,8 @@ #include "archive.hh" #include "file-system.hh" -#include "util.hh" #include "config.hh" +#include "logging.hh" #include "signals.hh" namespace nix { diff --git a/src/libutil/args.cc b/src/libutil/args.cc index bcff653c5..4983e49af 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -5,6 +5,7 @@ #include "environment-variables.hh" #include "experimental-features-json.hh" +#include "logging.hh" #include diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 77f7ff2a8..35a5238c0 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -1,13 +1,17 @@ #pragma once ///@file -#include +#include "experimental-features.hh" +#include "types.hh" +#include #include #include +#include #include +#include +#include -#include "util.hh" namespace nix { diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index cbc2bb4fd..136a3f742 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + #define DECLARE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE) \ PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const; #define DECLARE_EQUAL(prefix, qualification, my_type) \ diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 575a03712..678557a58 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,7 +1,5 @@ #include "compression.hh" #include "tarfile.hh" -#include "util.hh" -#include "finally.hh" #include "signals.hh" #include "logging.hh" @@ -13,7 +11,6 @@ #include #include -#include namespace nix { diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index 9f69e8444..756175f95 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -12,7 +12,9 @@ * instantiation. */ +#include "args.hh" #include "config.hh" +#include "logging.hh" namespace nix { diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 729b4e596..8180886ce 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -3,6 +3,7 @@ #include "abstract-setting-to-json.hh" #include "experimental-features.hh" #include "file-system.hh" +#include "logging.hh" #include "strings.hh" #include "config-impl.hh" diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 2a7fd3d0e..e5d6a9fa8 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,11 +1,11 @@ #include "environment-variables.hh" #include "error.hh" +#include "logging.hh" #include "position.hh" #include "terminal.hh" #include #include -#include "serialise.hh" #include namespace nix { diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 8ebec2956..f1cbfdb16 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -2,7 +2,6 @@ // Required for instances of to_json and from_json for ExperimentalFeature #include "experimental-features-json.hh" #include "strings.hh" -#include "util.hh" #include "nlohmann/json.hpp" diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc index ec22f17ab..a19ceaf2a 100644 --- a/src/libutil/file-descriptor.cc +++ b/src/libutil/file-descriptor.cc @@ -1,5 +1,6 @@ #include "file-system.hh" #include "finally.hh" +#include "logging.hh" #include "serialise.hh" #include "signals.hh" diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 721bf97e2..d573b22b4 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -6,6 +6,7 @@ #include "file-descriptor.hh" #include "file-system.hh" #include "finally.hh" +#include "logging.hh" #include "serialise.hh" #include "signals.hh" #include "types.hh" diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 2c36d9d94..006b5000c 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -1,4 +1,3 @@ -#include #include #include @@ -8,8 +7,8 @@ #include "args.hh" #include "hash.hh" #include "archive.hh" +#include "logging.hh" #include "split.hh" -#include "util.hh" #include #include diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 8d9e18d09..febbfdb55 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -1,15 +1,12 @@ #include "environment-variables.hh" #include "file-descriptor.hh" #include "logging.hh" -#include "util.hh" #include "config.hh" -#include "source-path.hh" #include "position.hh" #include "terminal.hh" #include #include -#include namespace nix { diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 64d84c714..96450fbe2 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -110,7 +110,6 @@ libutil_headers = files( 'url-name.hh', 'url.hh', 'users.hh', - 'util.hh', 'variant-wrapper.hh', 'xml-writer.hh', ) diff --git a/src/libutil/references.cc b/src/libutil/references.cc index 7f59b4c09..6189f69b9 100644 --- a/src/libutil/references.cc +++ b/src/libutil/references.cc @@ -1,9 +1,7 @@ #include "references.hh" #include "hash.hh" -#include "util.hh" -#include "archive.hh" +#include "logging.hh" -#include #include #include #include diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index a6cc919d2..3a8a01f16 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -1,5 +1,4 @@ #include "serialise.hh" -#include "util.hh" #include "signals.hh" #include diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index e46c5624a..c9294ba2d 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -5,7 +5,6 @@ #include "strings.hh" #include "types.hh" -#include "util.hh" #include "file-descriptor.hh" namespace boost::context { struct stack_context; } diff --git a/src/libutil/signals.cc b/src/libutil/signals.cc index c0e66f6ed..a94c2802a 100644 --- a/src/libutil/signals.cc +++ b/src/libutil/signals.cc @@ -1,5 +1,4 @@ #include "signals.hh" -#include "util.hh" #include "error.hh" #include "sync.hh" #include "terminal.hh" diff --git a/src/libutil/signals.hh b/src/libutil/signals.hh index 71593df95..02f8d2ca3 100644 --- a/src/libutil/signals.hh +++ b/src/libutil/signals.hh @@ -1,7 +1,6 @@ #pragma once /// @file -#include "types.hh" #include "error.hh" #include @@ -12,7 +11,6 @@ #include #include -#include namespace nix { diff --git a/src/libutil/split.hh b/src/libutil/split.hh index 4ff940eef..5455b6bff 100644 --- a/src/libutil/split.hh +++ b/src/libutil/split.hh @@ -4,8 +4,6 @@ #include #include -#include "util.hh" - namespace nix { /** diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc index 63dcf84b5..cae52cf29 100644 --- a/src/libutil/suggestions.cc +++ b/src/libutil/suggestions.cc @@ -4,6 +4,7 @@ #include #include +#include namespace nix { diff --git a/src/libutil/suggestions.hh b/src/libutil/suggestions.hh index 9abf5ee5f..3cac1371e 100644 --- a/src/libutil/suggestions.hh +++ b/src/libutil/suggestions.hh @@ -2,8 +2,11 @@ ///@file #include "comparator.hh" -#include "types.hh" + #include +#include +#include +#include namespace nix { diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index cabb7af9b..760a5a65a 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -2,6 +2,7 @@ #include #include "file-system.hh" +#include "logging.hh" #include "serialise.hh" #include "tarfile.hh" diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index a25769d9b..0ff83e997 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -1,4 +1,5 @@ #include "thread-pool.hh" +#include "logging.hh" #include "signals.hh" namespace nix { diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index 0e09fae97..3db7ce88f 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -1,13 +1,12 @@ #pragma once ///@file +#include "error.hh" #include "sync.hh" -#include "util.hh" #include #include #include -#include #include namespace nix { diff --git a/src/libutil/url-name.cc b/src/libutil/url-name.cc index 7c526752c..12c55db9a 100644 --- a/src/libutil/url-name.cc +++ b/src/libutil/url-name.cc @@ -1,4 +1,3 @@ -#include #include #include "url-name.hh" diff --git a/src/libutil/url-name.hh b/src/libutil/url-name.hh index 3a3f88e76..dabd23ca1 100644 --- a/src/libutil/url-name.hh +++ b/src/libutil/url-name.hh @@ -5,9 +5,6 @@ #include #include "url.hh" -#include "url-parts.hh" -#include "util.hh" -#include "split.hh" namespace nix { diff --git a/src/libutil/util.hh b/src/libutil/util.hh deleted file mode 100644 index 1066f212a..000000000 --- a/src/libutil/util.hh +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once -///@file - -#include "types.hh" -#include "error.hh" -#include "logging.hh" -#include "ansicolor.hh" - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include - -#ifndef HAVE_STRUCT_DIRENT_D_TYPE -#define DT_UNKNOWN 0 -#define DT_REG 1 -#define DT_LNK 2 -#define DT_DIR 3 -#endif - -namespace nix { - -struct Sink; -struct Source; - - - - - -} diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index f1c7bbfb8..31d4ac9ff 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -15,7 +15,6 @@ #include "globals.hh" #include "current-process.hh" #include "derivations.hh" -#include "util.hh" #include "shared.hh" #include "path-with-outputs.hh" #include "eval.hh" diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index f0131a458..530039ac6 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -1,5 +1,4 @@ #include "user-env.hh" -#include "util.hh" #include "derivations.hh" #include "store-api.hh" #include "path-with-outputs.hh" diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 3b99b73b4..c0b251ae4 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -7,7 +7,6 @@ #include "attr-path.hh" #include "value-to-xml.hh" #include "value-to-json.hh" -#include "util.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "common-eval-args.hh" diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 577cadceb..2c530999b 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -1,5 +1,4 @@ #include "dotgraph.hh" -#include "util.hh" #include "store-api.hh" #include diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 439557658..3e789a2d8 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -1,5 +1,4 @@ #include "graphml.hh" -#include "util.hh" #include "store-api.hh" #include "derivations.hh" diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 75d8b6c6d..b38e4f4f8 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -11,7 +11,6 @@ #include "serve-protocol.hh" #include "serve-protocol-impl.hh" #include "shared.hh" -#include "util.hh" #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 113e23bd1..f1cc1ee9c 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -5,7 +5,6 @@ #include "local-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" -#include "util.hh" #include "serialise.hh" #include "archive.hh" #include "globals.hh" diff --git a/src/nix/develop.cc b/src/nix/develop.cc index b5543447e..cd32bb20a 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -8,7 +8,6 @@ #include "derivations.hh" #include "progress-bar.hh" #include "run.hh" -#include "util.hh" #include #include diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index da7a1d7a0..4e1cfe8c0 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -6,7 +6,6 @@ #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "util.hh" #include "worker-protocol.hh" using namespace nix; diff --git a/tests/functional/repl_characterization/repl_characterization.cc b/tests/functional/repl_characterization/repl_characterization.cc index 4cc1e7e37..c91d6c1e3 100644 --- a/tests/functional/repl_characterization/repl_characterization.cc +++ b/tests/functional/repl_characterization/repl_characterization.cc @@ -6,12 +6,9 @@ #include #include -#include "escape-string.hh" #include "test-session.hh" #include "tests/characterization.hh" #include "tests/cli-literate-parser.hh" -#include "tests/terminal-code-eater.hh" -#include "util.hh" #include "strings.hh" using namespace std::string_literals; diff --git a/tests/unit/libcmd/args.cc b/tests/unit/libcmd/args.cc index 73550dacf..886467ea0 100644 --- a/tests/unit/libcmd/args.cc +++ b/tests/unit/libcmd/args.cc @@ -10,7 +10,6 @@ #include "filetransfer.hh" #include "shared.hh" #include "store-api.hh" -#include "util.hh" constexpr std::string_view INVALID_CHANNEL = "channel:example"; constexpr std::string_view CHANNEL_URL = "https://nixos.org/channels/example/nixexprs.tar.xz"; diff --git a/tests/unit/libstore-support/tests/test-data.hh b/tests/unit/libstore-support/tests/test-data.hh index 7aa1dd90d..1fec6f912 100644 --- a/tests/unit/libstore-support/tests/test-data.hh +++ b/tests/unit/libstore-support/tests/test-data.hh @@ -1,6 +1,6 @@ #pragma once -#include "util.hh" +#include "environment-variables.hh" #include "types.hh" namespace nix { diff --git a/tests/unit/libutil-support/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh index 472fdba3b..db82476da 100644 --- a/tests/unit/libutil-support/tests/characterization.hh +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -8,7 +8,6 @@ #include -#include "util.hh" #include "types.hh" namespace nix { diff --git a/tests/unit/libutil-support/tests/cli-literate-parser.cc b/tests/unit/libutil-support/tests/cli-literate-parser.cc index 69bf9fb46..f74fe85eb 100644 --- a/tests/unit/libutil-support/tests/cli-literate-parser.cc +++ b/tests/unit/libutil-support/tests/cli-literate-parser.cc @@ -3,7 +3,6 @@ #include "escape-char.hh" #include "libexpr/print.hh" #include "types.hh" -#include "util.hh" #include #include #include @@ -18,7 +17,6 @@ #include "libexpr/print.hh" #include "shlex.hh" #include "types.hh" -#include "util.hh" #include "strings.hh" static constexpr const bool DEBUG_PARSER = false; diff --git a/tests/unit/libutil/logging.cc b/tests/unit/libutil/logging.cc index 94cbe9c51..f99f49527 100644 --- a/tests/unit/libutil/logging.cc +++ b/tests/unit/libutil/logging.cc @@ -2,7 +2,6 @@ #include "logging.hh" #include "nixexpr.hh" -#include "util.hh" #include #include diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index 324ebd335..9a44ad59b 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -3,7 +3,6 @@ #include "strings.hh" #include "types.hh" #include "terminal.hh" -#include "util.hh" #include @@ -11,7 +10,7 @@ namespace nix { -/* ----------- tests for util.hh ------------------------------------------------*/ +/* ----------- tests for libutil ------------------------------------------------*/ /* ---------------------------------------------------------------------------- * absPath