From de552c42cb304ff320bc042a37b2f46ecd20c63b Mon Sep 17 00:00:00 2001 From: Alois Wohlschlager Date: Wed, 21 Aug 2024 17:28:42 +0200 Subject: [PATCH 001/106] Stop the logger in legacy commands again Commit 0dd1d8ca1cdccfc620644a7f690ed35bcd2d1e74 included an accidental revert of 1461e6cdda06f7f461114cce5b415f6d50381311 (actually slightly worse), leading to the progress bar not being stopped properly when a legacy command was invoked with `--log-format bar` (or similar options that show a progress bar). Move the progress bar stopping code to its proper place again to fix this regression. Change-Id: I676333da096d5990b717a387924bb988c9b73fab --- src/nix/main.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index e84e4f310..97f4b4a63 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -353,6 +353,9 @@ void mainWrapped(int argc, char * * argv) argv++; argc--; } + // Clean up the progress bar if shown using --log-format in a legacy command too. + // Otherwise, this is a harmless no-op. + Finally f([] { logger->pause(); }); { auto legacy = (*RegisterLegacyCommand::commands)[programName]; if (legacy) return legacy(argc, argv); @@ -361,7 +364,6 @@ void mainWrapped(int argc, char * * argv) evalSettings.pureEval = true; setLogFormat(LogFormat::bar); - Finally f([] { logger->pause(); }); settings.verboseBuild = false; // FIXME: stop messing about with log verbosity depending on if it is interactive use if (isatty(STDERR_FILENO)) { -- 2.44.1 From fabc9f29b82df97ab3b378dafb8249354f54d1df Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Fri, 23 Aug 2024 15:15:21 -0700 Subject: [PATCH 002/106] Fix comment in `getHome` The logic in the comment is the opposite of the truth. Change-Id: I64add84539209782ffa46431f3db1fb306d90b3f --- src/libutil/users.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/users.cc b/src/libutil/users.cc index a9a8a7353..ce36bad9b 100644 --- a/src/libutil/users.cc +++ b/src/libutil/users.cc @@ -36,7 +36,7 @@ Path getHome() std::optional unownedUserHomeDir = {}; auto homeDir = getEnv("HOME"); if (homeDir) { - // Only use $HOME if doesn't exist or is owned by the current user. + // Only use `$HOME` if it exists and is owned by the current user. struct stat st; int result = stat(homeDir->c_str(), &st); if (result != 0) { -- 2.44.1 From 5fc6fcb31035f79a8e590f07d73dc6cc592e9e29 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Sun, 25 Aug 2024 11:58:10 -0700 Subject: [PATCH 003/106] Thread `ApplyConfigOptions` through config parsing This makes no changes to logic but makes the `ApplyConfigOptions` value available to consumers. Change-Id: I88cf53d38faac8472c556aee55c13d0acbd1e5db --- src/libfetchers/fetch-settings.cc | 2 +- src/libstore/globals.cc | 26 ++++----- src/libstore/globals.hh | 5 +- src/libutil/config-impl.hh | 22 ++++---- src/libutil/config.cc | 85 ++++++++++++++++------------- src/libutil/config.hh | 19 ++++--- tests/unit/libutil/config.cc | 2 +- tests/unit/libutil/paths-setting.cc | 42 +++++--------- 8 files changed, 99 insertions(+), 104 deletions(-) diff --git a/src/libfetchers/fetch-settings.cc b/src/libfetchers/fetch-settings.cc index aeb3c542b..007f2725f 100644 --- a/src/libfetchers/fetch-settings.cc +++ b/src/libfetchers/fetch-settings.cc @@ -7,7 +7,7 @@ namespace nix { -template<> AcceptFlakeConfig BaseSetting::parse(const std::string & str) const +template<> AcceptFlakeConfig BaseSetting::parse(const std::string & str, const ApplyConfigOptions & options) const { if (str == "true") return AcceptFlakeConfig::True; else if (str == "ask") return AcceptFlakeConfig::Ask; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 6cfa3ffac..ab461e739 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -116,14 +116,15 @@ Settings::Settings() void loadConfFile() { - auto applyConfigFile = [&](const Path & path) { + auto applyConfigFile = [&](const ApplyConfigOptions & options) { try { - std::string contents = readFile(path); - globalConfig.applyConfig(contents, path); - } catch (SysError &) { } + std::string contents = readFile(*options.path); + globalConfig.applyConfig(contents, options); + } catch (SysError &) { + } }; - applyConfigFile(settings.nixConfDir + "/nix.conf"); + applyConfigFile(ApplyConfigOptions{.path = settings.nixConfDir + "/nix.conf"}); /* We only want to send overrides to the daemon, i.e. stuff from ~/.nix/nix.conf or the command line. */ @@ -131,14 +132,13 @@ void loadConfFile() auto files = settings.nixUserConfFiles; for (auto file = files.rbegin(); file != files.rend(); file++) { - applyConfigFile(*file); + applyConfigFile(ApplyConfigOptions{.path = *file}); } auto nixConfEnv = getEnv("NIX_CONFIG"); if (nixConfEnv.has_value()) { - globalConfig.applyConfig(nixConfEnv.value(), "NIX_CONFIG"); + globalConfig.applyConfig(nixConfEnv.value(), ApplyConfigOptions{.fromEnvVar = true}); } - } std::vector getUserConfigFiles() @@ -264,7 +264,7 @@ NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { {SandboxMode::smDisabled, false}, }); -template<> SandboxMode BaseSetting::parse(const std::string & str) const +template<> SandboxMode BaseSetting::parse(const std::string & str, const ApplyConfigOptions & options) const { if (str == "true") return smEnabled; else if (str == "relaxed") return smRelaxed; @@ -307,7 +307,7 @@ template<> void BaseSetting::convertToArg(Args & args, const std::s }); } -unsigned int MaxBuildJobsSetting::parse(const std::string & str) const +unsigned int MaxBuildJobsSetting::parse(const std::string & str, const ApplyConfigOptions & options) const { if (str == "auto") return std::max(1U, std::thread::hardware_concurrency()); else { @@ -315,15 +315,15 @@ unsigned int MaxBuildJobsSetting::parse(const std::string & str) const return *n; else throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); + } } -} -Paths PluginFilesSetting::parse(const std::string & str) const +Paths PluginFilesSetting::parse(const std::string & str, const ApplyConfigOptions & options) const { if (pluginsLoaded) throw UsageError("plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); - return BaseSetting::parse(str); + return BaseSetting::parse(str, options); } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index aba99d969..51550b2c3 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -26,7 +26,7 @@ struct MaxBuildJobsSetting : public BaseSetting options->addSetting(this); } - unsigned int parse(const std::string & str) const override; + unsigned int parse(const std::string & str, const ApplyConfigOptions & options) const override; }; struct PluginFilesSetting : public BaseSetting @@ -43,7 +43,7 @@ struct PluginFilesSetting : public BaseSetting options->addSetting(this); } - Paths parse(const std::string & str) const override; + Paths parse(const std::string & str, const ApplyConfigOptions & options) const override; }; const uint32_t maxIdsPerBuild = @@ -1088,6 +1088,7 @@ void loadConfFile(); // Used by the Settings constructor std::vector getUserConfigFiles(); +std::vector getHomeConfigFile(); extern const std::string nixVersion; diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index 024018e00..748107b6e 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -51,14 +51,14 @@ bool BaseSetting::isAppendable() return trait::appendable; } -template<> void BaseSetting::appendOrSet(Strings newValue, bool append); -template<> void BaseSetting::appendOrSet(StringSet newValue, bool append); -template<> void BaseSetting::appendOrSet(StringMap newValue, bool append); -template<> void BaseSetting::appendOrSet(ExperimentalFeatures newValue, bool append); -template<> void BaseSetting::appendOrSet(DeprecatedFeatures newValue, bool append); +template<> void BaseSetting::appendOrSet(Strings newValue, bool append, const ApplyConfigOptions & options); +template<> void BaseSetting::appendOrSet(StringSet newValue, bool append, const ApplyConfigOptions & options); +template<> void BaseSetting::appendOrSet(StringMap newValue, bool append, const ApplyConfigOptions & options); +template<> void BaseSetting::appendOrSet(ExperimentalFeatures newValue, bool append, const ApplyConfigOptions & options); +template<> void BaseSetting::appendOrSet(DeprecatedFeatures newValue, bool append, const ApplyConfigOptions & options); template -void BaseSetting::appendOrSet(T newValue, bool append) +void BaseSetting::appendOrSet(T newValue, bool append, const ApplyConfigOptions & options) { static_assert( !trait::appendable, @@ -69,14 +69,14 @@ void BaseSetting::appendOrSet(T newValue, bool append) } template -void BaseSetting::set(const std::string & str, bool append) +void BaseSetting::set(const std::string & str, bool append, const ApplyConfigOptions & options) { if (experimentalFeatureSettings.isEnabled(experimentalFeature)) { - auto parsed = parse(str); + auto parsed = parse(str, options); if (deprecated && (append || parsed != value)) { warn("deprecated setting '%s' found (set to '%s')", name, str); } - appendOrSet(std::move(parsed), append); + appendOrSet(std::move(parsed), append, options); } else { assert(experimentalFeature); warn("Ignoring setting '%s' because experimental feature '%s' is not enabled", @@ -111,7 +111,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) } #define DECLARE_CONFIG_SERIALISER(TY) \ - template<> TY BaseSetting< TY >::parse(const std::string & str) const; \ + template<> TY BaseSetting< TY >::parse(const std::string & str, const ApplyConfigOptions & options) const; \ template<> std::string BaseSetting< TY >::to_string() const; DECLARE_CONFIG_SERIALISER(std::string) @@ -124,7 +124,7 @@ DECLARE_CONFIG_SERIALISER(ExperimentalFeatures) DECLARE_CONFIG_SERIALISER(DeprecatedFeatures) template -T BaseSetting::parse(const std::string & str) const +T BaseSetting::parse(const std::string & str, const ApplyConfigOptions & options) const { static_assert(std::is_integral::value, "Integer required."); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8e20f1321..333deb388 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -1,4 +1,5 @@ #include "config.hh" +#include "apply-config-options.hh" #include "args.hh" #include "abstract-setting-to-json.hh" #include "experimental-features.hh" @@ -17,7 +18,7 @@ Config::Config(StringMap initials) : AbstractConfig(std::move(initials)) { } -bool Config::set(const std::string & name, const std::string & value) +bool Config::set(const std::string & name, const std::string & value, const ApplyConfigOptions & options) { bool append = false; auto i = _settings.find(name); @@ -30,7 +31,7 @@ bool Config::set(const std::string & name, const std::string & value) } else return false; } - i->second.setting->set(value, append); + i->second.setting->set(value, append, options); i->second.setting->overridden = true; return true; } @@ -91,7 +92,7 @@ void Config::getSettings(std::map & res, bool overridd } -static void applyConfigInner(const std::string & contents, const std::string & path, std::vector> & parsedContents) { +static void applyConfigInner(const std::string & contents, const ApplyConfigOptions & options, std::vector> & parsedContents) { unsigned int pos = 0; while (pos < contents.size()) { @@ -107,7 +108,7 @@ static void applyConfigInner(const std::string & contents, const std::string & p if (tokens.empty()) continue; if (tokens.size() < 2) - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + throw UsageError("illegal configuration line '%1%' in '%2%'", line, options.relativeDisplay()); auto include = false; auto ignoreMissing = false; @@ -119,24 +120,32 @@ static void applyConfigInner(const std::string & contents, const std::string & p } if (include) { - if (tokens.size() != 2) - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - auto p = absPath(tokens[1], dirOf(path)); - if (pathExists(p)) { + if (tokens.size() != 2) { + throw UsageError("illegal configuration line '%1%' in '%2%'", line, options.relativeDisplay()); + } + if (!options.path) { + throw UsageError("can only include configuration '%1%' from files", tokens[1]); + } + auto pathToInclude = absPath(tokens[1], dirOf(*options.path)); + if (pathExists(pathToInclude)) { + auto includeOptions = ApplyConfigOptions { + .path = pathToInclude, + .home = options.home, + }; try { - std::string includedContents = readFile(path); - applyConfigInner(includedContents, p, parsedContents); + std::string includedContents = readFile(pathToInclude); + applyConfigInner(includedContents, includeOptions, parsedContents); } catch (SysError &) { // TODO: Do we actually want to ignore this? Or is it better to fail? } } else if (!ignoreMissing) { - throw Error("file '%1%' included from '%2%' not found", p, path); + throw Error("file '%1%' included from '%2%' not found", pathToInclude, *options.path); } continue; } if (tokens[1] != "=") - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + throw UsageError("illegal configuration line '%1%' in '%2%'", line, options.relativeDisplay()); std::string name = std::move(tokens[0]); @@ -150,20 +159,20 @@ static void applyConfigInner(const std::string & contents, const std::string & p }; } -void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { +void AbstractConfig::applyConfig(const std::string & contents, const ApplyConfigOptions & options) { std::vector> parsedContents; - applyConfigInner(contents, path, parsedContents); + applyConfigInner(contents, options, parsedContents); // First apply experimental-feature related settings for (const auto & [name, value] : parsedContents) if (name == "experimental-features" || name == "extra-experimental-features") - set(name, value); + set(name, value, options); // Then apply other settings for (const auto & [name, value] : parsedContents) if (name != "experimental-features" && name != "extra-experimental-features") - set(name, value); + set(name, value, options); } void Config::resetOverridden() @@ -241,7 +250,7 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category) bool AbstractSetting::isOverridden() const { return overridden; } -template<> std::string BaseSetting::parse(const std::string & str) const +template<> std::string BaseSetting::parse(const std::string & str, const ApplyConfigOptions & options) const { return str; } @@ -251,7 +260,7 @@ template<> std::string BaseSetting::to_string() const return value; } -template<> std::optional BaseSetting>::parse(const std::string & str) const +template<> std::optional BaseSetting>::parse(const std::string & str, const ApplyConfigOptions & options) const { if (str == "") return std::nullopt; @@ -264,7 +273,7 @@ template<> std::string BaseSetting>::to_string() cons return value ? *value : ""; } -template<> bool BaseSetting::parse(const std::string & str) const +template<> bool BaseSetting::parse(const std::string & str, const ApplyConfigOptions & options) const { if (str == "true" || str == "yes" || str == "1") return true; @@ -297,12 +306,12 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & }); } -template<> Strings BaseSetting::parse(const std::string & str) const +template<> Strings BaseSetting::parse(const std::string & str, const ApplyConfigOptions & options) const { return tokenizeString(str); } -template<> void BaseSetting::appendOrSet(Strings newValue, bool append) +template<> void BaseSetting::appendOrSet(Strings newValue, bool append, const ApplyConfigOptions & options) { if (!append) value.clear(); value.insert(value.end(), std::make_move_iterator(newValue.begin()), @@ -314,12 +323,12 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } -template<> StringSet BaseSetting::parse(const std::string & str) const +template<> StringSet BaseSetting::parse(const std::string & str, const ApplyConfigOptions & options) const { return tokenizeString(str); } -template<> void BaseSetting::appendOrSet(StringSet newValue, bool append) +template<> void BaseSetting::appendOrSet(StringSet newValue, bool append, const ApplyConfigOptions & options) { if (!append) value.clear(); value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); @@ -330,7 +339,7 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } -template<> ExperimentalFeatures BaseSetting::parse(const std::string & str) const +template<> ExperimentalFeatures BaseSetting::parse(const std::string & str, const ApplyConfigOptions & options) const { ExperimentalFeatures res{}; for (auto & s : tokenizeString(str)) { @@ -342,7 +351,7 @@ template<> ExperimentalFeatures BaseSetting::parse(const s return res; } -template<> void BaseSetting::appendOrSet(ExperimentalFeatures newValue, bool append) +template<> void BaseSetting::appendOrSet(ExperimentalFeatures newValue, bool append, const ApplyConfigOptions & options) { if (append) value = value | newValue; @@ -359,7 +368,7 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", stringifiedXpFeatures); } -template<> DeprecatedFeatures BaseSetting::parse(const std::string & str) const +template<> DeprecatedFeatures BaseSetting::parse(const std::string & str, const ApplyConfigOptions & options) const { DeprecatedFeatures res{}; for (auto & s : tokenizeString(str)) { @@ -371,7 +380,7 @@ template<> DeprecatedFeatures BaseSetting::parse(const std:: return res; } -template<> void BaseSetting::appendOrSet(DeprecatedFeatures newValue, bool append) +template<> void BaseSetting::appendOrSet(DeprecatedFeatures newValue, bool append, const ApplyConfigOptions & options) { if (append) value = value | newValue; @@ -388,7 +397,7 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", stringifiedDpFeatures); } -template<> StringMap BaseSetting::parse(const std::string & str) const +template<> StringMap BaseSetting::parse(const std::string & str, const ApplyConfigOptions & options) const { StringMap res; for (const auto & s : tokenizeString(str)) { @@ -399,7 +408,7 @@ template<> StringMap BaseSetting::parse(const std::string & str) cons return res; } -template<> void BaseSetting::appendOrSet(StringMap newValue, bool append) +template<> void BaseSetting::appendOrSet(StringMap newValue, bool append, const ApplyConfigOptions & options) { if (!append) value.clear(); value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); @@ -426,7 +435,7 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting; -static Path parsePath(const AbstractSetting & s, const std::string & str) +static Path parsePath(const AbstractSetting & s, const std::string & str, const ApplyConfigOptions & options) { if (str == "") throw UsageError("setting '%s' is a path and paths cannot be empty", s.name); @@ -434,26 +443,26 @@ static Path parsePath(const AbstractSetting & s, const std::string & str) return canonPath(str); } -template<> Path PathsSetting::parse(const std::string & str) const +template<> Path PathsSetting::parse(const std::string & str, const ApplyConfigOptions & options) const { - return parsePath(*this, str); + return parsePath(*this, str, options); } -template<> std::optional PathsSetting>::parse(const std::string & str) const +template<> std::optional PathsSetting>::parse(const std::string & str, const ApplyConfigOptions & options) const { if (str == "") return std::nullopt; else - return parsePath(*this, str); + return parsePath(*this, str, options); } -template<> Paths PathsSetting::parse(const std::string & str) const +template<> Paths PathsSetting::parse(const std::string & str, const ApplyConfigOptions & options) const { auto strings = tokenizeString(str); Paths parsed; for (auto str : strings) { - parsed.push_back(canonPath(str)); + parsed.push_back(parsePath(*this, str, options)); } return parsed; @@ -464,10 +473,10 @@ template class PathsSetting>; template class PathsSetting; -bool GlobalConfig::set(const std::string & name, const std::string & value) +bool GlobalConfig::set(const std::string & name, const std::string & value, const ApplyConfigOptions & options) { for (auto & config : *configRegistrations) - if (config->set(name, value)) return true; + if (config->set(name, value, options)) return true; unknownSettings.emplace(name, value); diff --git a/src/libutil/config.hh b/src/libutil/config.hh index dbca4b406..59cc281c5 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -10,6 +10,7 @@ #include "types.hh" #include "experimental-features.hh" #include "deprecated-features.hh" +#include "apply-config-options.hh" namespace nix { @@ -61,7 +62,7 @@ public: * Sets the value referenced by `name` to `value`. Returns true if the * setting is known, false otherwise. */ - virtual bool set(const std::string & name, const std::string & value) = 0; + virtual bool set(const std::string & name, const std::string & value, const ApplyConfigOptions & options = {}) = 0; struct SettingInfo { @@ -81,7 +82,7 @@ public: * - contents: configuration contents to be parsed and applied * - path: location of the configuration file */ - void applyConfig(const std::string & contents, const std::string & path = ""); + void applyConfig(const std::string & contents, const ApplyConfigOptions & options = {}); /** * Resets the `overridden` flag of all Settings @@ -155,7 +156,7 @@ public: Config(StringMap initials = {}); - bool set(const std::string & name, const std::string & value) override; + bool set(const std::string & name, const std::string & value, const ApplyConfigOptions & options = {}) override; void addSetting(AbstractSetting * setting); @@ -200,7 +201,7 @@ protected: virtual ~AbstractSetting(); - virtual void set(const std::string & value, bool append = false) = 0; + virtual void set(const std::string & value, bool append = false, const ApplyConfigOptions & options = {}) = 0; /** * Whether the type is appendable; i.e. whether the `append` @@ -237,7 +238,7 @@ protected: * * Used by `set()`. */ - virtual T parse(const std::string & str) const; + virtual T parse(const std::string & str, const ApplyConfigOptions & options) const; /** * Append or overwrite `value` with `newValue`. @@ -247,7 +248,7 @@ protected: * * @param append Whether to append or overwrite. */ - virtual void appendOrSet(T newValue, bool append); + virtual void appendOrSet(T newValue, bool append, const ApplyConfigOptions & options); public: @@ -284,7 +285,7 @@ public: * Uses `parse()` to get the value from `str`, and `appendOrSet()` * to set it. */ - void set(const std::string & str, bool append = false) override final; + void set(const std::string & str, bool append = false, const ApplyConfigOptions & options = {}) override final; /** * C++ trick; This is template-specialized to compile-time indicate whether @@ -373,7 +374,7 @@ public: options->addSetting(this); } - T parse(const std::string & str) const override; + T parse(const std::string & str, const ApplyConfigOptions & options) const override; void operator =(const T & v) { this->assign(v); } }; @@ -384,7 +385,7 @@ struct GlobalConfig : public AbstractConfig typedef std::vector ConfigRegistrations; static ConfigRegistrations * configRegistrations; - bool set(const std::string & name, const std::string & value) override; + bool set(const std::string & name, const std::string & value, const ApplyConfigOptions & options = {}) override; void getSettings(std::map & res, bool overriddenOnly = false) override; diff --git a/tests/unit/libutil/config.cc b/tests/unit/libutil/config.cc index 886e70da5..1629969ba 100644 --- a/tests/unit/libutil/config.cc +++ b/tests/unit/libutil/config.cc @@ -80,7 +80,7 @@ namespace nix { class TestSetting : public AbstractSetting { public: TestSetting() : AbstractSetting("test", "test", {}) {} - void set(const std::string & value, bool append) override {} + void set(const std::string & value, bool append, const ApplyConfigOptions & options) override {} std::string to_string() const override { return {}; } bool isAppendable() override { return false; } }; diff --git a/tests/unit/libutil/paths-setting.cc b/tests/unit/libutil/paths-setting.cc index 17cb125c8..c198b25e0 100644 --- a/tests/unit/libutil/paths-setting.cc +++ b/tests/unit/libutil/paths-setting.cc @@ -11,14 +11,13 @@ namespace nix { class PathsSettingTestConfig : public Config { public: - PathsSettingTestConfig() - : Config() - { } + PathsSettingTestConfig() : Config() {} PathsSetting paths{this, Paths(), "paths", "documentation"}; }; -struct PathsSettingTest : public ::testing::Test { +struct PathsSettingTest : public ::testing::Test +{ public: PathsSettingTestConfig mkConfig() { @@ -26,33 +25,27 @@ public: } }; -TEST_F(PathsSettingTest, parse) { +TEST_F(PathsSettingTest, parse) +{ auto config = mkConfig(); // Not an absolute path: - ASSERT_THROW(config.paths.parse("puppy.nix"), Error); + ASSERT_THROW(config.paths.parse("puppy.nix", {}), Error); - ASSERT_THAT( - config.paths.parse("/puppy.nix"), - Eq({"/puppy.nix"}) - ); + ASSERT_THAT(config.paths.parse("/puppy.nix", {}), Eq({"/puppy.nix"})); // Splits on whitespace: ASSERT_THAT( - config.paths.parse("/puppy.nix /doggy.nix"), - Eq({"/puppy.nix", "/doggy.nix"}) + config.paths.parse("/puppy.nix /doggy.nix", {}), Eq({"/puppy.nix", "/doggy.nix"}) ); // Splits on _any_ whitespace: ASSERT_THAT( - config.paths.parse("/puppy.nix \t /doggy.nix\n\n\n/borzoi.nix\r/goldie.nix"), + config.paths.parse("/puppy.nix \t /doggy.nix\n\n\n/borzoi.nix\r/goldie.nix", {}), Eq({"/puppy.nix", "/doggy.nix", "/borzoi.nix", "/goldie.nix"}) ); // Canonicizes paths: - ASSERT_THAT( - config.paths.parse("/puppy/../doggy.nix"), - Eq({"/doggy.nix"}) - ); + ASSERT_THAT(config.paths.parse("/puppy/../doggy.nix", {}), Eq({"/doggy.nix"})); } TEST_F(PathsSettingTest, append) { @@ -61,26 +54,17 @@ TEST_F(PathsSettingTest, append) { ASSERT_TRUE(config.paths.isAppendable()); // Starts with no paths: - ASSERT_THAT( - config.paths.get(), - Eq({}) - ); + ASSERT_THAT(config.paths.get(), Eq({})); // Can append a path: config.paths.set("/puppy.nix", true); - ASSERT_THAT( - config.paths.get(), - Eq({"/puppy.nix"}) - ); + ASSERT_THAT(config.paths.get(), Eq({"/puppy.nix"})); // Can append multiple paths: config.paths.set("/silly.nix /doggy.nix", true); - ASSERT_THAT( - config.paths.get(), - Eq({"/puppy.nix", "/silly.nix", "/doggy.nix"}) - ); + ASSERT_THAT(config.paths.get(), Eq({"/puppy.nix", "/silly.nix", "/doggy.nix"})); } } // namespace nix -- 2.44.1 From 690f07272e58bfe86d12adb0bd6c81c031f930fd Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Sun, 25 Aug 2024 11:58:55 -0700 Subject: [PATCH 004/106] Support relative and `~/` paths in config settings Change-Id: I5566a9858ba255f4ac5051d1368c7dfb24460f0a --- .../relative-and-tilde-paths-in-config.md | 30 ++++++++++ src/libstore/globals.cc | 3 +- src/libutil/config.cc | 14 +++-- src/libutil/file-system.cc | 15 +++++ src/libutil/file-system.hh | 10 ++++ tests/unit/libutil/paths-setting.cc | 55 ++++++++++++++++++- 6 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 doc/manual/rl-next/relative-and-tilde-paths-in-config.md diff --git a/doc/manual/rl-next/relative-and-tilde-paths-in-config.md b/doc/manual/rl-next/relative-and-tilde-paths-in-config.md new file mode 100644 index 000000000..6645496a2 --- /dev/null +++ b/doc/manual/rl-next/relative-and-tilde-paths-in-config.md @@ -0,0 +1,30 @@ +--- +synopsis: Relative and tilde paths in configuration +issues: [fj#482] +cls: [1851, 1863, 1864] +category: Features +credits: [9999years] +--- + +[Configuration settings](@docroot@/command-ref/conf-file.md) can now refer to +files with paths relative to the file they're written in or relative to your +home directory (with `~/`). + +This makes settings like +[`repl-overlays`](@docroot@/command-ref/conf-file.md#conf-repl-overlays) and +[`secret-key-files`](@docroot@/command-ref/conf-file.md#conf-repl-overlays) +much easier to set, especially if you'd like to refer to files in an existing +dotfiles repo cloned into your home directory. + +If you put `repl-overlays = repl.nix` in your `~/.config/nix/nix.conf`, it'll +load `~/.config/nix/repl.nix`. Similarly, you can set `repl-overlays = +~/.dotfiles/repl.nix` to load a file relative to your home directory. + +Configuration files can also +[`include`](@docroot@/command-ref/conf-file.md#file-format) paths relative to +your home directory. + +Only user configuration files (like `$XDG_CONFIG_HOME/nix/nix.conf` or the +files listed in `$NIX_USER_CONF_FILES`) can use tilde paths relative to your +home directory. Configuration listed in the `$NIX_CONFIG` environment variable +may not use relative paths. diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index ab461e739..29ec60105 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -131,8 +131,9 @@ void loadConfFile() globalConfig.resetOverridden(); auto files = settings.nixUserConfFiles; + auto home = getHome(); for (auto file = files.rbegin(); file != files.rend(); file++) { - applyConfigFile(ApplyConfigOptions{.path = *file}); + applyConfigFile(ApplyConfigOptions{.path = *file, .home = home}); } auto nixConfEnv = getEnv("NIX_CONFIG"); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 333deb388..778da1413 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -126,7 +126,7 @@ static void applyConfigInner(const std::string & contents, const ApplyConfigOpti if (!options.path) { throw UsageError("can only include configuration '%1%' from files", tokens[1]); } - auto pathToInclude = absPath(tokens[1], dirOf(*options.path)); + auto pathToInclude = absPath(tildePath(tokens[1], options.home), dirOf(*options.path)); if (pathExists(pathToInclude)) { auto includeOptions = ApplyConfigOptions { .path = pathToInclude, @@ -437,10 +437,16 @@ template class BaseSetting; static Path parsePath(const AbstractSetting & s, const std::string & str, const ApplyConfigOptions & options) { - if (str == "") + if (str == "") { throw UsageError("setting '%s' is a path and paths cannot be empty", s.name); - else - return canonPath(str); + } else { + auto tildeResolvedPath = tildePath(str, options.home); + if (options.path) { + return absPath(tildeResolvedPath, dirOf(*options.path)); + } else { + return canonPath(tildeResolvedPath); + } + } } template<> Path PathsSetting::parse(const std::string & str, const ApplyConfigOptions & options) const diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 631cf076b..234c73163 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -117,6 +117,21 @@ Path realPath(Path const & path) return ret; } +Path tildePath(Path const & path, const std::optional & home) +{ + if (path.starts_with("~/")) { + if (home) { + return *home + "/" + path.substr(2); + } else { + throw UsageError("`~` path not allowed: %1%", path); + } + } else if (path.starts_with('~')) { + throw UsageError("`~` paths must start with `~/`: %1%", path); + } else { + return path; + } +} + void chmodPath(const Path & path, mode_t mode) { if (chmod(path.c_str(), mode) == -1) diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index e49323e84..0a54d1a3b 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -62,6 +62,16 @@ Path canonPath(PathView path, bool resolveSymlinks = false); */ Path realPath(Path const & path); +/** + * Resolve a tilde path like `~/puppy.nix` into an absolute path. + * + * If `home` is given, it's substituted for `~/` at the start of the input + * `path`. Otherwise, an error is thrown. + * + * If the path starts with `~` but not `~/`, an error is thrown. + */ +Path tildePath(Path const & path, const std::optional & home = std::nullopt); + /** * Change the permissions of a path * Not called `chmod` as it shadows and could be confused with diff --git a/tests/unit/libutil/paths-setting.cc b/tests/unit/libutil/paths-setting.cc index c198b25e0..2d37ad525 100644 --- a/tests/unit/libutil/paths-setting.cc +++ b/tests/unit/libutil/paths-setting.cc @@ -48,7 +48,60 @@ TEST_F(PathsSettingTest, parse) ASSERT_THAT(config.paths.parse("/puppy/../doggy.nix", {}), Eq({"/doggy.nix"})); } -TEST_F(PathsSettingTest, append) { +TEST_F(PathsSettingTest, parseRelative) +{ + auto options = ApplyConfigOptions{.path = "/doggy/kinds/config.nix"}; + auto config = mkConfig(); + ASSERT_THAT( + config.paths.parse("puppy.nix", options), + Eq({"/doggy/kinds/puppy.nix"}) + ); + + // Splits on whitespace: + ASSERT_THAT( + config.paths.parse("puppy.nix /doggy.nix", options), Eq({"/doggy/kinds/puppy.nix", "/doggy.nix"}) + ); + + // Canonicizes paths: + ASSERT_THAT(config.paths.parse("../soft.nix", options), Eq({"/doggy/soft.nix"})); + + // Canonicizes paths: + ASSERT_THAT(config.paths.parse("./soft.nix", options), Eq({"/doggy/kinds/soft.nix"})); +} + +TEST_F(PathsSettingTest, parseHome) +{ + auto options = ApplyConfigOptions{ + .path = "/doggy/kinds/config.nix", + .home = "/home/puppy" + }; + auto config = mkConfig(); + + ASSERT_THAT( + config.paths.parse("puppy.nix", options), + Eq({"/doggy/kinds/puppy.nix"}) + ); + + ASSERT_THAT( + config.paths.parse("~/.config/nix/puppy.nix", options), + Eq({"/home/puppy/.config/nix/puppy.nix"}) + ); + + // Splits on whitespace: + ASSERT_THAT( + config.paths.parse("~/puppy.nix ~/doggy.nix", options), + Eq({"/home/puppy/puppy.nix", "/home/puppy/doggy.nix"}) + ); + + // Canonicizes paths: + ASSERT_THAT(config.paths.parse("~/../why.nix", options), Eq({"/home/why.nix"})); + + // Home paths for other users not allowed. Needs to start with `~/`. + ASSERT_THROW(config.paths.parse("~root/config.nix", options), Error); +} + +TEST_F(PathsSettingTest, append) +{ auto config = mkConfig(); ASSERT_TRUE(config.paths.isAppendable()); -- 2.44.1 From 742303dc3aa23462ba3e5e0497ae20487266adb3 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Mon, 26 Aug 2024 11:20:35 -0700 Subject: [PATCH 005/106] Add `getCwd` It's nice for this to be a separate function and not just inline in `absPath`. Prepared as part of cl/1865, though I don't think I actually ended up using it there. Change-Id: I24d9d4a984cee0af587010baf04b3939a1c147ec --- src/libutil/file-system.cc | 14 +++++++++----- src/libutil/file-system.hh | 7 +++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 631cf076b..124a0aef7 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -17,15 +17,19 @@ namespace fs = std::filesystem; namespace nix { +Path getCwd() { + char buf[PATH_MAX]; + if (!getcwd(buf, sizeof(buf))) { + throw SysError("cannot get cwd"); + } + return Path(buf); +} + Path absPath(Path path, std::optional dir, bool resolveSymlinks) { if (path.empty() || path[0] != '/') { if (!dir) { - char buf[PATH_MAX]; - if (!getcwd(buf, sizeof(buf))) { - throw SysError("cannot get cwd"); - } - path = concatStrings(buf, "/", path); + path = concatStrings(getCwd(), "/", path); } else { path = concatStrings(*dir, "/", path); } diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index e49323e84..23685ee08 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -29,6 +29,13 @@ namespace nix { struct Sink; struct Source; +/** + * Get the current working directory. + * + * Throw an error if the current directory cannot get got. + */ +Path getCwd(); + /** * @return An absolutized path, resolving paths relative to the * specified directory, or the current directory otherwise. The path -- 2.44.1 From ca08f1217d8779971d4f2b306a19ad5622360372 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Mon, 26 Aug 2024 11:34:43 -0700 Subject: [PATCH 006/106] rowan: 0.15.15 -> 0.15.16 This fixes an ambiguous pointer comparison warning. See: https://github.com/rust-analyzer/rowan/pull/162 Change-Id: Iaac2c8cab0051eb97211893ad547d8dfa8fda560 --- Cargo.lock | 28 ++++++---------------------- src/lix-doc/Cargo.toml | 2 +- src/lix-doc/meson.build | 4 ---- subprojects/dissimilar-rs.wrap | 8 ++++---- subprojects/expect-test-rs.wrap | 8 ++++---- subprojects/rowan-rs.wrap | 8 ++++---- 6 files changed, 19 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e82e138f5..cf93e2dd4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - [[package]] name = "countme" version = "3.0.1" @@ -16,15 +10,15 @@ checksum = "7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636" [[package]] name = "dissimilar" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86e3bdc80eee6e16b2b6b0f87fbc98c04bee3455e35174c0de1a125d0688c632" +checksum = "59f8e79d1fbf76bdfbde321e902714bf6c49df88a7dda6fc682fc2979226962d" [[package]] name = "expect-test" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d9eafeadd538e68fb28016364c9732d78e420b9ff8853fa5e4058861e9f8d3" +checksum = "9e0be0a561335815e06dab7c62e50353134c796e7a6155402a64bcff66b6a5e0" dependencies = [ "dissimilar", "once_cell", @@ -45,15 +39,6 @@ dependencies = [ "rowan", ] -[[package]] -name = "memoffset" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] - [[package]] name = "once_cell" version = "1.19.0" @@ -71,13 +56,12 @@ dependencies = [ [[package]] name = "rowan" -version = "0.15.15" +version = "0.15.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a58fa8a7ccff2aec4f39cc45bf5f985cec7125ab271cf681c279fd00192b49" +checksum = "0a542b0253fa46e632d27a1dc5cf7b930de4df8659dc6e720b647fc72147ae3d" dependencies = [ "countme", "hashbrown", - "memoffset", "rustc-hash", "text-size", ] diff --git a/src/lix-doc/Cargo.toml b/src/lix-doc/Cargo.toml index 02494862f..b52e6bb3b 100644 --- a/src/lix-doc/Cargo.toml +++ b/src/lix-doc/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/lf-/nix-doc" [dependencies] rnix = "0.11.0" # Necessary because rnix fails to export a critical trait (Rowan's AstNode). -rowan = "0.15.0" +rowan = "0.15.16" [dev-dependencies] expect-test = "1.1.0" diff --git a/src/lix-doc/meson.build b/src/lix-doc/meson.build index 9838984a5..132feebf3 100644 --- a/src/lix-doc/meson.build +++ b/src/lix-doc/meson.build @@ -1,5 +1,3 @@ -# The external crate rowan has an ambiguous pointer comparison warning, which -# we don't want to fail our whole build if werror is on. # FIXME: remove hack once we get rid of meson 1.4 rnix_name = 'rnix-0.11-rs' rowan_name = 'rowan-0.15-rs' @@ -8,8 +6,6 @@ if meson.version().version_compare('< 1.5') rowan_name = 'rowan-rs' endif -subproject(rowan_name, default_options : ['werror=false']) - rnix = dependency(rnix_name) rowan = dependency(rowan_name) diff --git a/subprojects/dissimilar-rs.wrap b/subprojects/dissimilar-rs.wrap index a51407482..5617b775b 100644 --- a/subprojects/dissimilar-rs.wrap +++ b/subprojects/dissimilar-rs.wrap @@ -1,6 +1,6 @@ [wrap-file] method = cargo -directory = dissimilar-1.0.7 -source_url = https://crates.io/api/v1/crates/dissimilar/1.0.7/download -source_filename = dissimilar-1.0.7.tar.gz -source_hash = 86e3bdc80eee6e16b2b6b0f87fbc98c04bee3455e35174c0de1a125d0688c632 +directory = dissimilar-1.0.9 +source_url = https://crates.io/api/v1/crates/dissimilar/1.0.9/download +source_filename = dissimilar-1.0.9.tar.gz +source_hash = 59f8e79d1fbf76bdfbde321e902714bf6c49df88a7dda6fc682fc2979226962d diff --git a/subprojects/expect-test-rs.wrap b/subprojects/expect-test-rs.wrap index 1e2a4a3b9..701c5f0c5 100644 --- a/subprojects/expect-test-rs.wrap +++ b/subprojects/expect-test-rs.wrap @@ -1,6 +1,6 @@ [wrap-file] method = cargo -directory = expect-test-1.4.1 -source_url = https://crates.io/api/v1/crates/expect-test/1.4.1/download -source_filename = expect-test-1.4.1.tar.gz -source_hash = 30d9eafeadd538e68fb28016364c9732d78e420b9ff8853fa5e4058861e9f8d3 +directory = expect-test-1.5.0 +source_url = https://crates.io/api/v1/crates/expect-test/1.5.0/download +source_filename = expect-test-1.5.0.tar.gz +source_hash = 9e0be0a561335815e06dab7c62e50353134c796e7a6155402a64bcff66b6a5e0 diff --git a/subprojects/rowan-rs.wrap b/subprojects/rowan-rs.wrap index f6ab76d69..1baf3f9f2 100644 --- a/subprojects/rowan-rs.wrap +++ b/subprojects/rowan-rs.wrap @@ -1,6 +1,6 @@ [wrap-file] method = cargo -directory = rowan-0.15.15 -source_url = https://crates.io/api/v1/crates/rowan/0.15.15/download -source_filename = rowan-0.15.15.tar.gz -source_hash = 32a58fa8a7ccff2aec4f39cc45bf5f985cec7125ab271cf681c279fd00192b49 +directory = rowan-0.15.16 +source_url = https://crates.io/api/v1/crates/rowan/0.15.16/download +source_filename = rowan-0.15.16.tar.gz +source_hash = 0a542b0253fa46e632d27a1dc5cf7b930de4df8659dc6e720b647fc72147ae3d -- 2.44.1 From 0cc285f87b25365b6050753fba76713332185012 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 22 Aug 2024 22:44:29 -0700 Subject: [PATCH 007/106] treewide: fix a bunch of lints Fixes: - Identifiers starting with _ are prohibited - Some driveby header dependency cleaning which wound up with doing some extra fixups. - Fucking C style casts, man. C++ made these 1000% worse by letting you also do memory corruption with them with references. - Remove casts to Expr * where ExprBlackHole is an incomplete type by introducing an explicitly-cast eBlackHoleAddr as Expr *. - An incredibly illegal cast of the text bytes of the StorePath hash into a size_t directly. You can't DO THAT. Replaced with actually parsing the hash so we get 100% of the bits being entropy, then memcpying the start of the hash. If this shows up in a profile we should just make the hash parser faster with a lookup table or something sensible like that. - This horrendous bit of UB which I thankfully slapped a deprecation warning on, built, and it didn't trigger anywhere so it was dead code and I just deleted it. But holy crap you *cannot* do that. inline void mkString(const Symbol & s) { mkString(((const std::string &) s).c_str()); } - Some wrong lints. Lots of wrong macro lints, one wrong suspicious-sizeof lint triggered by the template being instantiated with only pointers, but the calculation being correct for both pointers and not-pointers. - Exceptions in destructors strike again. I tried to catch the exceptions that might actually happen rather than all the exceptions imaginable. We can let the runtime hard-kill it on other exceptions imo. Change-Id: I71761620846cba64d66ee7ca231b20c061e69710 --- src/libcmd/built-path.hh | 16 +++++---- src/libexpr/eval-error.hh | 3 +- src/libexpr/eval-inline.hh | 6 ++-- src/libexpr/gc-alloc.hh | 1 + src/libexpr/nixexpr.cc | 1 + src/libexpr/value.hh | 35 ++++++++----------- src/libstore/common-protocol-impl.hh | 1 + src/libstore/derived-path.hh | 16 +++++---- .../length-prefixed-protocol-helper.hh | 6 ++-- src/libstore/path.cc | 10 ++++++ src/libstore/path.hh | 8 ++--- src/libstore/pathlocks.hh | 8 +++-- src/libstore/serve-protocol-impl.hh | 1 + src/libstore/sqlite.hh | 2 +- src/libstore/worker-protocol-impl.hh | 1 + src/libutil/error.hh | 5 +-- src/libutil/finally.hh | 1 + src/libutil/hash.cc | 3 +- src/libutil/serialise.cc | 1 + src/libutil/serialise.hh | 25 ++++++++----- src/libutil/signals.cc | 1 + src/libutil/thread-pool.hh | 1 + src/libutil/url.hh | 1 + src/libutil/variant-wrapper.hh | 2 ++ tests/unit/libutil/url.cc | 1 + 25 files changed, 94 insertions(+), 62 deletions(-) diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index da87a33b0..555adb04f 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -20,13 +20,15 @@ struct SingleBuiltPathBuilt { DECLARE_CMP(SingleBuiltPathBuilt); }; -using _SingleBuiltPathRaw = std::variant< +namespace built_path::detail { +using SingleBuiltPathRaw = std::variant< DerivedPathOpaque, SingleBuiltPathBuilt >; +} -struct SingleBuiltPath : _SingleBuiltPathRaw { - using Raw = _SingleBuiltPathRaw; +struct SingleBuiltPath : built_path::detail::SingleBuiltPathRaw { + using Raw = built_path::detail::SingleBuiltPathRaw; using Raw::Raw; using Opaque = DerivedPathOpaque; @@ -65,17 +67,19 @@ struct BuiltPathBuilt { DECLARE_CMP(BuiltPathBuilt); }; -using _BuiltPathRaw = std::variant< +namespace built_path::detail { +using BuiltPathRaw = std::variant< DerivedPath::Opaque, BuiltPathBuilt >; +} /** * A built path. Similar to a DerivedPath, but enriched with the corresponding * output path(s). */ -struct BuiltPath : _BuiltPathRaw { - using Raw = _BuiltPathRaw; +struct BuiltPath : built_path::detail::BuiltPathRaw { + using Raw = built_path::detail::BuiltPathRaw; using Raw::Raw; using Opaque = DerivedPathOpaque; diff --git a/src/libexpr/eval-error.hh b/src/libexpr/eval-error.hh index 19540d612..7114d392f 100644 --- a/src/libexpr/eval-error.hh +++ b/src/libexpr/eval-error.hh @@ -1,9 +1,8 @@ #pragma once ///@file -#include - #include "error.hh" +#include "types.hh" #include "pos-idx.hh" namespace nix { diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 4631b71eb..30badb93f 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -31,7 +31,7 @@ Value * EvalState::allocValue() #endif nrValues++; - return (Value *) p; + return static_cast(p); } @@ -54,10 +54,10 @@ Env & EvalState::allocEnv(size_t size) void * p = *env1AllocCache; *env1AllocCache = GC_NEXT(p); GC_NEXT(p) = nullptr; - env = (Env *) p; + env = static_cast(p); } else #endif - env = (Env *) gcAllocBytes(sizeof(Env) + size * sizeof(Value *)); + env = static_cast(gcAllocBytes(sizeof(Env) + size * sizeof(Value *))); /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */ diff --git a/src/libexpr/gc-alloc.hh b/src/libexpr/gc-alloc.hh index fc034045f..afdd7eeb0 100644 --- a/src/libexpr/gc-alloc.hh +++ b/src/libexpr/gc-alloc.hh @@ -120,6 +120,7 @@ inline T * gcAllocType(size_t howMany = 1) // However, people can and do request zero sized allocations, so we need // to check that neither of our multiplicands were zero before complaining // about it. + // NOLINTNEXTLINE(bugprone-sizeof-expression): yeah we only seem to alloc pointers with this. the calculation *is* correct though! auto checkedSz = checked::Checked(howMany) * sizeof(T); size_t sz = checkedSz.valueWrapping(); if (checkedSz.overflowed()) { diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 08d4b279b..68da254e2 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -11,6 +11,7 @@ namespace nix { ExprBlackHole eBlackHole; +Expr *eBlackHoleAddr = &eBlackHole; // FIXME: remove, because *symbols* are abstract and do not have a single // textual representation; see printIdentifier() diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 57485aa0a..e38b11cbf 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -136,7 +136,9 @@ class ExternalValueBase std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); -extern ExprBlackHole eBlackHole; +/** This is just the address of eBlackHole. It exists because eBlackHole has an + * incomplete type at usage sites so is not possible to cast. */ +extern Expr *eBlackHoleAddr; struct NewValueAs { @@ -196,6 +198,7 @@ private: public: // Discount `using NewValueAs::*;` +// NOLINTNEXTLINE(bugprone-macro-parentheses) #define USING_VALUETYPE(name) using name = NewValueAs::name USING_VALUETYPE(integer_t); USING_VALUETYPE(floating_t); @@ -473,7 +476,7 @@ public: /// Constructs an evil thunk, whose evaluation represents infinite recursion. explicit Value(blackhole_t) : internalType(tThunk) - , thunk({ .env = nullptr, .expr = reinterpret_cast(&eBlackHole) }) + , thunk({ .env = nullptr, .expr = eBlackHoleAddr }) { } Value(Value const & rhs) = default; @@ -513,7 +516,10 @@ public: // type() == nThunk inline bool isThunk() const { return internalType == tThunk; }; inline bool isApp() const { return internalType == tApp; }; - inline bool isBlackhole() const; + inline bool isBlackhole() const + { + return internalType == tThunk && thunk.expr == eBlackHoleAddr; + } // type() == nFunction inline bool isLambda() const { return internalType == tLambda; }; @@ -669,11 +675,6 @@ public: void mkStringMove(const char * s, const NixStringContext & context); - inline void mkString(const Symbol & s) - { - mkString(((const std::string &) s).c_str()); - } - void mkPath(const SourcePath & path); inline void mkPath(const char * path) @@ -732,7 +733,11 @@ public: lambda.fun = f; } - inline void mkBlackhole(); + inline void mkBlackhole() + { + internalType = tThunk; + thunk.expr = eBlackHoleAddr; + } void mkPrimOp(PrimOp * p); @@ -832,18 +837,6 @@ public: } }; - -bool Value::isBlackhole() const -{ - return internalType == tThunk && thunk.expr == (Expr*) &eBlackHole; -} - -void Value::mkBlackhole() -{ - internalType = tThunk; - thunk.expr = (Expr*) &eBlackHole; -} - using ValueVector = GcVector; using ValueMap = GcMap; using ValueVectorMap = std::map; diff --git a/src/libstore/common-protocol-impl.hh b/src/libstore/common-protocol-impl.hh index fd1387e95..387f848ed 100644 --- a/src/libstore/common-protocol-impl.hh +++ b/src/libstore/common-protocol-impl.hh @@ -20,6 +20,7 @@ namespace nix { { \ return LengthPrefixedProtoHelper::read(store, conn); \ } \ + /* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \ TEMPLATE [[nodiscard]] WireFormatGenerator CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \ { \ return LengthPrefixedProtoHelper::write(store, conn, t); \ diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index c87cf2004..c96e0df67 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -78,10 +78,12 @@ struct SingleDerivedPathBuilt { DECLARE_CMP(SingleDerivedPathBuilt); }; -using _SingleDerivedPathRaw = std::variant< +namespace derived_path::detail { +using SingleDerivedPathRaw = std::variant< DerivedPathOpaque, SingleDerivedPathBuilt >; +} /** * A "derived path" is a very simple sort of expression (not a Nix @@ -94,8 +96,8 @@ using _SingleDerivedPathRaw = std::variant< * - built, in which case it is a pair of a derivation path and an * output name. */ -struct SingleDerivedPath : _SingleDerivedPathRaw { - using Raw = _SingleDerivedPathRaw; +struct SingleDerivedPath : derived_path::detail::SingleDerivedPathRaw { + using Raw = derived_path::detail::SingleDerivedPathRaw; using Raw::Raw; using Opaque = DerivedPathOpaque; @@ -201,10 +203,12 @@ struct DerivedPathBuilt { DECLARE_CMP(DerivedPathBuilt); }; -using _DerivedPathRaw = std::variant< +namespace derived_path::detail { +using DerivedPathRaw = std::variant< DerivedPathOpaque, DerivedPathBuilt >; +} /** * A "derived path" is a very simple sort of expression that evaluates @@ -216,8 +220,8 @@ using _DerivedPathRaw = std::variant< * - built, in which case it is a pair of a derivation path and some * output names. */ -struct DerivedPath : _DerivedPathRaw { - using Raw = _DerivedPathRaw; +struct DerivedPath : derived_path::detail::DerivedPathRaw { + using Raw = derived_path::detail::DerivedPathRaw; using Raw::Raw; using Opaque = DerivedPathOpaque; diff --git a/src/libstore/length-prefixed-protocol-helper.hh b/src/libstore/length-prefixed-protocol-helper.hh index 1475d2690..423bc77b8 100644 --- a/src/libstore/length-prefixed-protocol-helper.hh +++ b/src/libstore/length-prefixed-protocol-helper.hh @@ -61,9 +61,9 @@ template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple); template -#define _X std::map -LENGTH_PREFIXED_PROTO_HELPER(Inner, _X); -#undef _X +#define DONT_SUBSTITUTE_KV_TYPE std::map +LENGTH_PREFIXED_PROTO_HELPER(Inner, DONT_SUBSTITUTE_KV_TYPE); +#undef DONT_SUBSTITUTE_KV_TYPE template std::vector diff --git a/src/libstore/path.cc b/src/libstore/path.cc index d4b5fc0dc..82ed20285 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -112,3 +112,13 @@ PathSet Store::printStorePathSet(const StorePathSet & paths) const } } + +std::size_t std::hash::operator()(const nix::StorePath & path) const noexcept +{ + // It's already a cryptographic hash of 160 bits (assuming that nobody gives us bogus ones...), so just parse it. + auto h = nix::Hash::parseNonSRIUnprefixed(path.hashPart(), nix::HashType::SHA1); + // This need not be stable across machines, so bit casting the start of it is fine. + size_t r; + memcpy(&r, h.hash, sizeof(r)); + return r; +} diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 4ca6747b3..09b4710c1 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -2,8 +2,9 @@ ///@file #include +#include -#include "types.hh" +#include "types.hh" // IWYU pragma: keep namespace nix { @@ -89,10 +90,7 @@ const std::string drvExtension = ".drv"; namespace std { template<> struct hash { - std::size_t operator()(const nix::StorePath & path) const noexcept - { - return * (std::size_t *) path.to_string().data(); - } + std::size_t operator()(const nix::StorePath & path) const noexcept; }; } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 7fcfa2e40..d06d031b5 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -49,8 +49,12 @@ struct FdLock ~FdLock() { - if (acquired) - lockFile(fd, ltNone, false); + try { + if (acquired) + lockFile(fd, ltNone, false); + } catch (SysError &) { + ignoreException(); + } } }; diff --git a/src/libstore/serve-protocol-impl.hh b/src/libstore/serve-protocol-impl.hh index cfb1dd574..a1d1c797f 100644 --- a/src/libstore/serve-protocol-impl.hh +++ b/src/libstore/serve-protocol-impl.hh @@ -20,6 +20,7 @@ namespace nix { { \ return LengthPrefixedProtoHelper::read(store, conn); \ } \ + /* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \ TEMPLATE [[nodiscard]] WireFormatGenerator ServeProto::Serialise< T >::write(const Store & store, ServeProto::WriteConn conn, const T & t) \ { \ return LengthPrefixedProtoHelper::write(store, conn, t); \ diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index ca021087f..5740c4e45 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -1,9 +1,9 @@ #pragma once ///@file -#include #include +#include "types.hh" #include "error.hh" struct sqlite3; diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh index b2603b954..d99c59f84 100644 --- a/src/libstore/worker-protocol-impl.hh +++ b/src/libstore/worker-protocol-impl.hh @@ -20,6 +20,7 @@ namespace nix { { \ return LengthPrefixedProtoHelper::read(store, conn); \ } \ + /* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \ TEMPLATE [[nodiscard]] WireFormatGenerator WorkerProto::Serialise< T >::write(const Store & store, WorkerProto::WriteConn conn, const T & t) \ { \ return LengthPrefixedProtoHelper::write(store, conn, t); \ diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 06dfba0df..73c1ccadd 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -16,16 +16,12 @@ */ #include "suggestions.hh" -#include "ref.hh" -#include "types.hh" #include "fmt.hh" #include #include #include -#include #include -#include #include #include @@ -173,6 +169,7 @@ public: }; #define MakeError(newClass, superClass) \ + /* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \ class newClass : public superClass \ { \ public: \ diff --git a/src/libutil/finally.hh b/src/libutil/finally.hh index dc51d7b1e..5e62dfa3b 100644 --- a/src/libutil/finally.hh +++ b/src/libutil/finally.hh @@ -22,6 +22,7 @@ public: Finally(Finally &&other) : fun(std::move(other.fun)) { other.movedFrom = true; } + // NOLINTNEXTLINE(bugprone-exception-escape): the noexcept is declared properly here, the analysis seems broken ~Finally() noexcept(noexcept(fun())) { try { diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 925f71f80..f05d4aa98 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -7,6 +7,7 @@ #include "args.hh" #include "hash.hh" #include "archive.hh" +#include "charptr-cast.hh" #include "logging.hh" #include "split.hh" @@ -129,7 +130,7 @@ std::string Hash::to_string(Base base, bool includeType) const break; case Base::Base64: case Base::SRI: - s += base64Encode(std::string_view(reinterpret_cast(hash), hashSize)); + s += base64Encode(std::string_view(charptr_cast(hash), hashSize)); break; } return s; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index f1db05b0b..a6dd7e200 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -1,4 +1,5 @@ #include "serialise.hh" +#include "charptr-cast.hh" #include "signals.hh" #include diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 8218db440..d6a22b3e9 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -4,6 +4,7 @@ #include #include +#include "charptr-cast.hh" #include "generator.hh" #include "strings.hh" #include "types.hh" @@ -385,7 +386,7 @@ struct SerializingTransform buf[5] = (n >> 40) & 0xff; buf[6] = (n >> 48) & 0xff; buf[7] = (unsigned char) (n >> 56) & 0xff; - return {reinterpret_cast(buf.begin()), 8}; + return {charptr_cast(buf.begin()), 8}; } static Bytes padding(size_t unpadded) @@ -417,6 +418,9 @@ struct SerializingTransform void writePadding(size_t len, Sink & sink); +// NOLINTBEGIN(cppcoreguidelines-avoid-capturing-lambda-coroutines): +// These coroutines do their entire job before the semicolon and are not +// retained, so they live long enough. inline Sink & operator<<(Sink & sink, uint64_t u) { return sink << [&]() -> WireFormatGenerator { co_yield u; }(); @@ -441,6 +445,7 @@ inline Sink & operator<<(Sink & sink, const Error & ex) { return sink << [&]() -> WireFormatGenerator { co_yield ex; }(); } +// NOLINTEND(cppcoreguidelines-avoid-capturing-lambda-coroutines) MakeError(SerialisationError, Error); @@ -448,7 +453,7 @@ template T readNum(Source & source) { unsigned char buf[8]; - source((char *) buf, sizeof(buf)); + source(charptr_cast(buf), sizeof(buf)); auto n = readLittleEndian(buf); @@ -540,13 +545,17 @@ struct FramedSource : Source ~FramedSource() { - if (!eof) { - while (true) { - auto n = readInt(from); - if (!n) break; - std::vector data(n); - from(data.data(), n); + try { + if (!eof) { + while (true) { + auto n = readInt(from); + if (!n) break; + std::vector data(n); + from(data.data(), n); + } } + } catch (...) { + ignoreException(); } } diff --git a/src/libutil/signals.cc b/src/libutil/signals.cc index a94c2802a..04a697d01 100644 --- a/src/libutil/signals.cc +++ b/src/libutil/signals.cc @@ -3,6 +3,7 @@ #include "sync.hh" #include "terminal.hh" +#include #include namespace nix { diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index 3db7ce88f..380e1a2d2 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -4,6 +4,7 @@ #include "error.hh" #include "sync.hh" +#include #include #include #include diff --git a/src/libutil/url.hh b/src/libutil/url.hh index d2413ec0e..a821301ba 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -2,6 +2,7 @@ ///@file #include "error.hh" +#include namespace nix { diff --git a/src/libutil/variant-wrapper.hh b/src/libutil/variant-wrapper.hh index cedcb999c..a809cd2a4 100644 --- a/src/libutil/variant-wrapper.hh +++ b/src/libutil/variant-wrapper.hh @@ -8,6 +8,7 @@ * Force the default versions of all constructors (copy, move, copy * assignment). */ +// NOLINTBEGIN(bugprone-macro-parentheses) #define FORCE_DEFAULT_CONSTRUCTORS(CLASS_NAME) \ CLASS_NAME(const CLASS_NAME &) = default; \ CLASS_NAME(CLASS_NAME &) = default; \ @@ -15,6 +16,7 @@ \ CLASS_NAME & operator =(const CLASS_NAME &) = default; \ CLASS_NAME & operator =(CLASS_NAME &) = default; +// NOLINTEND(bugprone-macro-parentheses) /** * Make a wrapper constructor. All args are forwarded to the diff --git a/tests/unit/libutil/url.cc b/tests/unit/libutil/url.cc index a908631e6..bfd9a228a 100644 --- a/tests/unit/libutil/url.cc +++ b/tests/unit/libutil/url.cc @@ -1,4 +1,5 @@ #include "url.hh" +#include "types.hh" #include namespace nix { -- 2.44.1 From 4f02255c205378427f5831463c0c07e45382b2b2 Mon Sep 17 00:00:00 2001 From: Pierre Bourdon Date: Wed, 5 Jun 2024 06:02:18 +0200 Subject: [PATCH 008/106] libstore: remove static initializers for Store registrations Ref #359. Change-Id: Ia45530ddee25fa9fc399ff10738bb0d8bbc8b221 --- src/libstore/dummy-store.cc | 5 +++- src/libstore/dummy-store.hh | 8 +++++++ src/libstore/globals.cc | 23 +++++++++++++++++++ src/libstore/http-binary-cache-store.cc | 5 +++- src/libstore/http-binary-cache-store.hh | 8 +++++++ src/libstore/legacy-ssh-store.cc | 7 ++++-- src/libstore/legacy-ssh-store.hh | 8 +++++++ src/libstore/local-binary-cache-store.cc | 5 +++- src/libstore/local-binary-cache-store.hh | 8 +++++++ src/libstore/local-store.hh | 3 +++ src/libstore/meson.build | 6 ++++- src/libstore/platform/darwin.cc | 5 ++++ src/libstore/platform/fallback.cc | 4 +++- src/libstore/platform/linux.cc | 4 +++- src/libstore/s3-binary-cache-store.cc | 8 ++++++- src/libstore/s3-binary-cache-store.hh | 2 ++ src/libstore/ssh-store.cc | 6 +++-- .../{ssh-store-config.hh => ssh-store.hh} | 2 ++ src/libstore/store-api.cc | 4 ++-- src/libstore/store-api.hh | 19 ++++----------- src/libstore/uds-remote-store.cc | 4 +++- src/libstore/uds-remote-store.hh | 2 ++ src/nix/main.cc | 2 +- 23 files changed, 118 insertions(+), 30 deletions(-) create mode 100644 src/libstore/dummy-store.hh create mode 100644 src/libstore/http-binary-cache-store.hh create mode 100644 src/libstore/legacy-ssh-store.hh create mode 100644 src/libstore/local-binary-cache-store.hh rename src/libstore/{ssh-store-config.hh => ssh-store.hh} (97%) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index e16f87e4b..f14b7ddd1 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -1,3 +1,4 @@ +#include "dummy-store.hh" #include "store-api.hh" namespace nix { @@ -73,6 +74,8 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store { unsupported("getFSAccessor"); } }; -static RegisterStoreImplementation regDummyStore; +void registerDummyStore() { + StoreImplementations::add(); +} } diff --git a/src/libstore/dummy-store.hh b/src/libstore/dummy-store.hh new file mode 100644 index 000000000..355c011f8 --- /dev/null +++ b/src/libstore/dummy-store.hh @@ -0,0 +1,8 @@ +#pragma once +///@file + +namespace nix { + +void registerDummyStore(); + +} diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 6cfa3ffac..b534882de 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -33,6 +33,16 @@ #include #endif +// All built-in store implementations. +#include "dummy-store.hh" +#include "http-binary-cache-store.hh" +#include "legacy-ssh-store.hh" +#include "local-binary-cache-store.hh" +#include "local-store.hh" +#include "s3-binary-cache-store.hh" +#include "ssh-store.hh" +#include "uds-remote-store.hh" + namespace nix { @@ -396,6 +406,17 @@ static void preloadNSS() }); } +static void registerStoreImplementations() { + registerDummyStore(); + registerHttpBinaryCacheStore(); + registerLegacySSHStore(); + registerLocalBinaryCacheStore(); + registerLocalStore(); + registerS3BinaryCacheStore(); + registerSSHStore(); + registerUDSRemoteStore(); +} + static bool initLibStoreDone = false; void assertLibStoreInitialized() { @@ -433,6 +454,8 @@ void initLibStore() { unsetenv("TMPDIR"); #endif + registerStoreImplementations(); + initLibStoreDone = true; } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 7ceea716a..c68faf552 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -1,3 +1,4 @@ +#include "http-binary-cache-store.hh" #include "binary-cache-store.hh" #include "filetransfer.hh" #include "globals.hh" @@ -194,6 +195,8 @@ protected: } }; -static RegisterStoreImplementation regHttpBinaryCacheStore; +void registerHttpBinaryCacheStore() { + StoreImplementations::add(); +} } diff --git a/src/libstore/http-binary-cache-store.hh b/src/libstore/http-binary-cache-store.hh new file mode 100644 index 000000000..097c5480b --- /dev/null +++ b/src/libstore/http-binary-cache-store.hh @@ -0,0 +1,8 @@ +#pragma once +///@file + +namespace nix { + +void registerHttpBinaryCacheStore(); + +} diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 5650282d6..9e2d65a1c 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -1,4 +1,4 @@ -#include "ssh-store-config.hh" +#include "legacy-ssh-store.hh" #include "archive.hh" #include "pool.hh" #include "remote-store.hh" @@ -8,6 +8,7 @@ #include "store-api.hh" #include "path-with-outputs.hh" #include "ssh.hh" +#include "ssh-store.hh" #include "derivations.hh" namespace nix { @@ -412,6 +413,8 @@ public: { unsupported("queryRealisation"); } }; -static RegisterStoreImplementation regLegacySSHStore; +void registerLegacySSHStore() { + StoreImplementations::add(); +} } diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh new file mode 100644 index 000000000..76298d8d9 --- /dev/null +++ b/src/libstore/legacy-ssh-store.hh @@ -0,0 +1,8 @@ +#pragma once +///@file + +namespace nix { + +void registerLegacySSHStore(); + +} diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 99dd0e0f6..2f6a092e1 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -1,3 +1,4 @@ +#include "local-binary-cache-store.hh" #include "binary-cache-store.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" @@ -124,6 +125,8 @@ std::set LocalBinaryCacheStore::uriSchemes() return {"file"}; } -static RegisterStoreImplementation regLocalBinaryCacheStore; +void registerLocalBinaryCacheStore() { + StoreImplementations::add(); +} } diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh new file mode 100644 index 000000000..727d60cb3 --- /dev/null +++ b/src/libstore/local-binary-cache-store.hh @@ -0,0 +1,8 @@ +#pragma once +///@file + +namespace nix { + +void registerLocalBinaryCacheStore(); + +} diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index f6b553615..1913aa192 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -421,4 +421,7 @@ void canonicaliseTimestampAndPermissions(const Path & path); MakeError(PathInUse, Error); +// Implemented by the relevant platform/ module being used. +void registerLocalStore(); + } diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 5416bd2b5..74f5cd04e 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -118,12 +118,16 @@ libstore_headers = files( 'derived-path-map.hh', 'derived-path.hh', 'downstream-placeholder.hh', + 'dummy-store.hh', 'filetransfer.hh', 'fs-accessor.hh', 'gc-store.hh', 'globals.hh', + 'http-binary-cache-store.hh', 'indirect-root-store.hh', + 'legacy-ssh-store.hh', 'length-prefixed-protocol-helper.hh', + 'local-binary-cache-store.hh', 'local-fs-store.hh', 'local-store.hh', 'lock.hh', @@ -152,8 +156,8 @@ libstore_headers = files( 'serve-protocol-impl.hh', 'serve-protocol.hh', 'sqlite.hh', - 'ssh-store-config.hh', 'ssh.hh', + 'ssh-store.hh', 'store-api.hh', 'store-cast.hh', 'uds-remote-store.hh', diff --git a/src/libstore/platform/darwin.cc b/src/libstore/platform/darwin.cc index 1f7e9be23..078753bff 100644 --- a/src/libstore/platform/darwin.cc +++ b/src/libstore/platform/darwin.cc @@ -261,4 +261,9 @@ void DarwinLocalDerivationGoal::execBuilder(std::string builder, Strings args, S posix_spawn(nullptr, builder.c_str(), nullptr, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); } + +void registerLocalStore() { + StoreImplementations::add(); +} + } diff --git a/src/libstore/platform/fallback.cc b/src/libstore/platform/fallback.cc index 5a01d64c8..0593ec204 100644 --- a/src/libstore/platform/fallback.cc +++ b/src/libstore/platform/fallback.cc @@ -1,5 +1,7 @@ #include "platform/fallback.hh" namespace nix { -static RegisterStoreImplementation regLocalStore; +void registerLocalStore() { + Implementations::add(); +} } diff --git a/src/libstore/platform/linux.cc b/src/libstore/platform/linux.cc index 03b8bc0be..f22fbe58f 100644 --- a/src/libstore/platform/linux.cc +++ b/src/libstore/platform/linux.cc @@ -25,7 +25,9 @@ namespace { constexpr const std::string_view nativeSystem = SYSTEM; } -static RegisterStoreImplementation regLocalStore; +void registerLocalStore() { + StoreImplementations::add(); +} static void readProcLink(const std::string & file, UncheckedRoots & roots) { diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index ffebfda8d..921a2e556 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -526,8 +526,14 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual }; -static RegisterStoreImplementation regS3BinaryCacheStore; +void registerS3BinaryCacheStore() { + StoreImplementations::add(); +} } +#else +namespace nix { +void registerS3BinaryCacheStore() {} +} #endif diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index c62ea5147..baf21e2f4 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -29,4 +29,6 @@ public: virtual const Stats & getS3Stats() = 0; }; +void registerS3BinaryCacheStore(); + } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 80d10eb0f..94c0b7237 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,4 +1,4 @@ -#include "ssh-store-config.hh" +#include "ssh-store.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "remote-store.hh" @@ -110,6 +110,8 @@ ref SSHStore::openConnection() return conn; } -static RegisterStoreImplementation regSSHStore; +void registerSSHStore() { + StoreImplementations::add(); +} } diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store.hh similarity index 97% rename from src/libstore/ssh-store-config.hh rename to src/libstore/ssh-store.hh index bf55d20cf..51951f80b 100644 --- a/src/libstore/ssh-store-config.hh +++ b/src/libstore/ssh-store.hh @@ -26,4 +26,6 @@ struct CommonSSHStoreConfig : virtual StoreConfig )"}; }; +void registerSSHStore(); + } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 6d9fec41b..f921956e8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1472,7 +1472,7 @@ ref openStore(const std::string & uri_, parsedUri.authority.value_or("") + parsedUri.path ); - for (auto implem : *Implementations::registered) { + for (auto implem : *StoreImplementations::registered) { if (implem.uriSchemes.count(parsedUri.scheme)) { auto store = implem.create(parsedUri.scheme, baseURI, params); if (store) { @@ -1526,6 +1526,6 @@ std::list> getDefaultSubstituters() return stores; } -std::vector * Implementations::registered = 0; +std::vector * StoreImplementations::registered = 0; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index efd0e4d9b..2da5cac39 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -50,11 +50,9 @@ namespace nix { * that calls `StoreConfig(params)` (otherwise you're gonna encounter an * `assertion failure` when trying to instantiate it). * - * You can then register the new store using: - * - * ``` - * cpp static RegisterStoreImplementation regStore; - * ``` + * You can then register the new store by defining a registration function + * (using `StoreImplementations::add`) and calling it in + * `registerStoreImplementations` in `globals.cc`. */ MakeError(SubstError, Error); @@ -1004,7 +1002,7 @@ struct StoreFactory std::function ()> getConfig; }; -struct Implementations +struct StoreImplementations { static std::vector * registered; @@ -1027,15 +1025,6 @@ struct Implementations } }; -template -struct RegisterStoreImplementation -{ - RegisterStoreImplementation() - { - Implementations::add(); - } -}; - /** * Display a set of paths in human-readable form (i.e., between quotes diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 226cdf717..44dd45e88 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -88,6 +88,8 @@ void UDSRemoteStore::addIndirectRoot(const Path & path) } -static RegisterStoreImplementation regUDSRemoteStore; +void registerUDSRemoteStore() { + StoreImplementations::add(); +} } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index ff7e9ae3f..8b56e0af0 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -63,4 +63,6 @@ private: std::optional path; }; +void registerUDSRemoteStore(); + } diff --git a/src/nix/main.cc b/src/nix/main.cc index e84e4f310..05c40db03 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -201,7 +201,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs res["args"] = toJSON(); auto stores = nlohmann::json::object(); - for (auto & implem : *Implementations::registered) { + for (auto & implem : *StoreImplementations::registered) { auto storeConfig = implem.getConfig(); auto storeName = storeConfig->name(); auto & j = stores[storeName]; -- 2.44.1 From 4d8984420788d6253d2fd9bfa3ada92be8bfd73b Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 22 Aug 2024 21:02:52 -0700 Subject: [PATCH 009/106] build: remove about 30 cpu-sec of compile time by explicit instantiation Apparently the fmt contraption has some extremely popular overloads, and the boost stuff in there gets built approximately infinite times in every compilation unit. Change-Id: Ideba2db7d6bf8559e4d91974bab636f5ed106198 --- src/libutil/fmt.cc | 14 ++++++++++++++ src/libutil/fmt.hh | 13 +++++++++++-- src/libutil/meson.build | 1 + src/libutil/serialise.cc | 25 +++++++++++++++++++++++++ src/libutil/serialise.hh | 14 +------------- 5 files changed, 52 insertions(+), 15 deletions(-) create mode 100644 src/libutil/fmt.cc diff --git a/src/libutil/fmt.cc b/src/libutil/fmt.cc new file mode 100644 index 000000000..400fb7ea0 --- /dev/null +++ b/src/libutil/fmt.cc @@ -0,0 +1,14 @@ +#include "fmt.hh" // IWYU pragma: keep + +template class boost::basic_format; + +namespace nix { + +// Explicit instantiation saves about 30 cpu-seconds of compile time +template HintFmt::HintFmt(const std::string &, const Uncolored &s); +template HintFmt::HintFmt(const std::string &, const std::string &s); +template HintFmt::HintFmt(const std::string &, const uint64_t &, const char * const &); + +HintFmt::HintFmt(const std::string & literal) : HintFmt("%s", Uncolored(literal)) {} + +} diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index d015f7e5f..7589e51e2 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -3,7 +3,6 @@ #include #include -#include #include // Darwin and FreeBSD stdenv do not define _GNU_SOURCE but do have _Unwind_Backtrace. #if __APPLE__ || __FreeBSD__ @@ -12,6 +11,9 @@ #include #include "ansicolor.hh" +// Explicit instantiation in fmt.cc +extern template class boost::basic_format; + namespace nix { /** @@ -157,7 +159,9 @@ public: * Format the given string literally, without interpolating format * placeholders. */ - HintFmt(const std::string & literal) : HintFmt("%s", Uncolored(literal)) {} + // Moved out of line because it was instantiating the template below in + // every file in the project. + HintFmt(const std::string & literal); /** * Interpolate the given arguments into the format string. @@ -193,6 +197,11 @@ public: } }; +// Explicit instantiations in fmt.cc +extern template HintFmt::HintFmt(const std::string &, const Uncolored &s); +extern template HintFmt::HintFmt(const std::string &, const std::string &s); +extern template HintFmt::HintFmt(const std::string &, const uint64_t &, const char * const &); + std::ostream & operator<<(std::ostream & os, const HintFmt & hf); } diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 7c0e45e4b..e7f986363 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -17,6 +17,7 @@ libutil_sources = files( 'experimental-features.cc', 'file-descriptor.cc', 'file-system.cc', + 'fmt.cc', 'git.cc', 'hash.cc', 'hilite.cc', diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index a6dd7e200..4eda1b7e7 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -9,6 +9,31 @@ namespace nix { +template +T readNum(Source & source) +{ + unsigned char buf[8]; + source(charptr_cast(buf), sizeof(buf)); + + auto n = readLittleEndian(buf); + + if (n > (uint64_t) std::numeric_limits::max()) + throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name()); + + return (T) n; +} + +template bool readNum(Source & source); + +template unsigned char readNum(Source & source); + +template unsigned int readNum(Source & source); + +template unsigned long readNum(Source & source); +template long readNum(Source & source); + +template unsigned long long readNum(Source & source); +template long long readNum(Source & source); void BufferedSink::operator () (std::string_view data) { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index d6a22b3e9..9ad8018d0 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -450,19 +450,7 @@ inline Sink & operator<<(Sink & sink, const Error & ex) MakeError(SerialisationError, Error); template -T readNum(Source & source) -{ - unsigned char buf[8]; - source(charptr_cast(buf), sizeof(buf)); - - auto n = readLittleEndian(buf); - - if (n > (uint64_t) std::numeric_limits::max()) - throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name()); - - return (T) n; -} - +T readNum(Source & source); inline unsigned int readInt(Source & source) { -- 2.44.1 From e6f2af06e6d7cc01e142c21259635094f73de104 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 22 Aug 2024 22:42:58 -0700 Subject: [PATCH 010/106] clang-tidy: fix the fact that we are not linting headers properly This, however, took fixing a pile of lints that we predictably missed because of this bug. Change-Id: I92c36feb4a03f62bc594c2051c7bd7418d13fb08 --- meson/clang-tidy/clang-tidy-runner.py | 89 +++++++++++++++++++++++++++ meson/clang-tidy/clean_compdb.py | 4 +- meson/clang-tidy/meson.build | 29 ++------- 3 files changed, 97 insertions(+), 25 deletions(-) create mode 100755 meson/clang-tidy/clang-tidy-runner.py diff --git a/meson/clang-tidy/clang-tidy-runner.py b/meson/clang-tidy/clang-tidy-runner.py new file mode 100755 index 000000000..c36e66b41 --- /dev/null +++ b/meson/clang-tidy/clang-tidy-runner.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +""" +Runs run-clang-tidy. A bit meta. Maybe it will replace run-clang-tidy one day +because the run-clang-tidy UX is so questionable. +""" + +# I hereby dedicate this script to fuck you meson. +# I cannot simply write my code to invoke a subprocess in a meson file because +# Meson corrupts backslashes in command line args to subprocesses. +# This is allegedly for "Windows support", but last time I checked Windows +# neither needs nor wants you to corrupt its command lines. +# https://github.com/mesonbuild/meson/issues/1564 + +import multiprocessing +import subprocess +import os +import sys +from pathlib import Path + + +def default_concurrency(): + return min(multiprocessing.cpu_count(), + int(os.environ.get("NIX_BUILD_CORES", "16"))) + + +def go(exe: str, plugin_path: Path, compile_commands_json_dir: Path, jobs: int, + paths: list[Path], werror: bool, fix: bool): + args = [ + # XXX: This explicitly invokes it with python because of a nixpkgs bug + # where clang-unwrapped does not patch interpreters in run-clang-tidy. + # However, making clang-unwrapped depend on python is also silly, so idk. + sys.executable, + exe, + '-quiet', + '-load', + plugin_path, + '-p', + compile_commands_json_dir, + '-j', + str(jobs), + '-header-filter', + r'src/[^/]+/.*\.hh' + ] + if werror: + args += ['-warnings-as-errors', '*'] + if fix: + args += ['-fix'] + args += ['--'] + args += paths + os.execvp(sys.executable, args) + + +def main(): + import argparse + + ap = argparse.ArgumentParser(description='Runs run-clang-tidy for you') + ap.add_argument('--jobs', + '-j', + type=int, + default=default_concurrency(), + help='Parallel linting jobs to run') + ap.add_argument('--plugin-path', + type=Path, + help='Path to the Lix clang-tidy plugin') + # FIXME: maybe we should integrate this so it just fixes the compdb for you and throws it in a tempdir? + ap.add_argument( + '--compdb-path', + type=Path, + help= + 'Path to the directory containing the fixed-up compilation database from clean_compdb' + ) + ap.add_argument('--werror', + action='store_true', + help='Warnings get turned into errors') + ap.add_argument('--fix', + action='store_true', + help='Apply fixes for warnings') + ap.add_argument('--run-clang-tidy-path', + default='run-clang-tidy', + help='Path to run-clang-tidy') + ap.add_argument('paths', nargs='*', help='Source paths to check') + args = ap.parse_args() + + go(args.run_clang_tidy_path, args.plugin_path, args.compdb_path, args.jobs, + args.paths, args.werror, args.fix) + + +if __name__ == '__main__': + main() diff --git a/meson/clang-tidy/clean_compdb.py b/meson/clang-tidy/clean_compdb.py index a3fc77204..461488f27 100755 --- a/meson/clang-tidy/clean_compdb.py +++ b/meson/clang-tidy/clean_compdb.py @@ -13,8 +13,8 @@ def process_compdb(compdb: list[dict]) -> list[dict]: out = [] eat_next = False for i, arg in enumerate(args): - if arg == '-fpch-preprocess': - # as used with gcc + if arg in ['-fpch-preprocess', '-fpch-instantiate-templates']: + # -fpch-preprocess as used with gcc, -fpch-instantiate-templates as used by clang continue elif arg == '-include-pch' or (arg == '-include' and args[i + 1] == 'precompiled-headers.hh'): # -include-pch some-pch (clang), or -include some-pch (gcc) diff --git a/meson/clang-tidy/meson.build b/meson/clang-tidy/meson.build index 1cd85a708..1da0661f9 100644 --- a/meson/clang-tidy/meson.build +++ b/meson/clang-tidy/meson.build @@ -57,34 +57,18 @@ build_all_generated_headers = custom_target( ) if lix_clang_tidy_so_found - default_concurrency = run_command(python, '-c', ''' -import multiprocessing -import os -print(min(multiprocessing.cpu_count(), int(os.environ.get("NIX_BUILD_CORES", "16")))) - ''', check : true).stdout() - run_clang_tidy_args = [ - '-load', - lix_clang_tidy_so, - '-p', - # We have to workaround a run-clang-tidy bug too, so we must give the - # directory name rather than the actual compdb file. - # https://github.com/llvm/llvm-project/issues/101440 - meson.current_build_dir(), - '-quiet', - '-j', default_concurrency, + meson.current_source_dir() / 'clang-tidy-runner.py', + '--run-clang-tidy-path', run_clang_tidy, + '--compdb-path', meson.current_build_dir(), + '--plugin-path', lix_clang_tidy_so, ] run_target( 'clang-tidy', command : [ - # XXX: This explicitly invokes it with python because of a nixpkgs bug - # where clang-unwrapped does not patch interpreters in run-clang-tidy. - # However, making clang-unwrapped depend on python is also silly, so idk. python, - run_clang_tidy, run_clang_tidy_args, - '-warnings-as-errors', - '*', + '--werror', ], depends : [ build_all_generated_headers, @@ -94,9 +78,8 @@ print(min(multiprocessing.cpu_count(), int(os.environ.get("NIX_BUILD_CORES", "16 'clang-tidy-fix', command : [ python, - run_clang_tidy, run_clang_tidy_args, - '-fix', + '--fix', ], depends : [ build_all_generated_headers, -- 2.44.1 From 04f8a1483362212f33f97c13472ded4121a18a86 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 23 Aug 2024 16:33:48 -0700 Subject: [PATCH 011/106] tree-wide: shuffle headers around for about 30s compile time This didn't really feel so worth it afterwards, but I did untangle a bunch of stuff that should not have been tangled. The general gist of this change is that variant bullshit was causing a bunch of compile time, and it seems like the only way to deal with variant induced compile time is to keep variant types out of headers. Explicit template instantiation seems to do nothing for them. I also seem to have gotten some back-end time improvement from explicitly instantiating regex, but I don't know why. There is no corresponding front-end time improvement from it: regex is still at the top of the sinners list. **** Templates that took longest to instantiate: 15231 ms: std::basic_regex::_M_compile (28 times, avg 543 ms) 15066 ms: std::__detail::_Compiler>::_Compiler (28 times, avg 538 ms) 12571 ms: std::__detail::_Compiler>::_M_disjunction (28 times, avg 448 ms) 12454 ms: std::__detail::_Compiler>::_M_alternative (28 times, avg 444 ms) 12225 ms: std::__detail::_Compiler>::_M_term (28 times, avg 436 ms) 11363 ms: nlohmann::basic_json<>::parse (21 times, avg 541 ms) 10628 ms: nlohmann::basic_json<>::basic_json (109 times, avg 97 ms) 10134 ms: std::__detail::_Compiler>::_M_atom (28 times, avg 361 ms) Back-end time before messing with the regex: **** Function sets that took longest to compile / optimize: 8076 ms: void boost::io::detail::put<$>(boost::io::detail::put_holder<$> cons... (177 times, avg 45 ms) 4382 ms: std::_Rb_tree<$>::_M_erase(std::_Rb_tree_node<$>*) (1247 times, avg 3 ms) 3137 ms: boost::stacktrace::detail::to_string_impl_base(std::__cxx11::basic_string<$>&, ch... (177 times, avg 16 ms) 2304 ms: std::_Rb_tree<$>::_M_get_insert_hint_unique_pos(std::_Rb_tree_const_... (210 times, avg 10 ms) 2116 ms: bool std::__detail::_Compiler<$>::_M_expression_term<$>(std::__detai... (112 times, avg 18 ms) 2051 ms: std::_Rb_tree_iterator<$> std::_Rb_tree<$>::_M_emplace_hint_unique<$... (244 times, avg 8 ms) 2037 ms: toml::result<$> toml::detail::sequence<$>::invoke<$>(toml::detail::l... (93 times, avg 21 ms) 1928 ms: std::__detail::_Compiler<$>::_M_quantifier() (28 times, avg 68 ms) 1859 ms: nlohmann::json_abi_v3_11_3::detail::serializer<$>::dump(nlohmann::js... (41 times, avg 45 ms) 1824 ms: std::_Function_handler<$>::_M_manager(std::_Any_data&, std::_Any_dat... (973 times, avg 1 ms) 1810 ms: std::__detail::_BracketMatcher<$>::_BracketMatcher(std::__detail::_B... (112 times, avg 16 ms) 1793 ms: nix::fetchers::GitInputScheme::fetch(nix::ref<$>, nix::fetchers::Inp... (1 times, avg 1793 ms) 1759 ms: std::_Rb_tree<$>::_M_get_insert_unique_pos(std::__cxx11::basic_strin... (281 times, avg 6 ms) 1722 ms: bool nlohmann::json_abi_v3_11_3::detail::parser<$>::sax_parse_intern... (19 times, avg 90 ms) 1677 ms: boost::io::basic_altstringbuf<$>::overflow(int) (194 times, avg 8 ms) 1674 ms: std::__cxx11::basic_string<$>::_M_mutate(unsigned long, unsigned lon... (249 times, avg 6 ms) 1660 ms: std::_Rb_tree_node<$>* std::_Rb_tree<$>::_M_copy<$>(std::_Rb_tree_no... (304 times, avg 5 ms) 1599 ms: bool nlohmann::json_abi_v3_11_3::detail::parser<$>::sax_parse_intern... (19 times, avg 84 ms) 1568 ms: void std::__detail::_Compiler<$>::_M_insert_bracket_matcher<$>(bool) (112 times, avg 14 ms) 1541 ms: std::__shared_ptr<$>::~__shared_ptr() (531 times, avg 2 ms) 1539 ms: nlohmann::json_abi_v3_11_3::detail::serializer<$>::dump_escaped(std:... (41 times, avg 37 ms) 1471 ms: void std::__detail::_Compiler<$>::_M_insert_character_class_matcher<... (112 times, avg 13 ms) After messing with the regex (notice std::__detail::_Compiler vanishes here, but I don't know why): **** Function sets that took longest to compile / optimize: 8054 ms: void boost::io::detail::put<$>(boost::io::detail::put_holder<$> cons... (177 times, avg 45 ms) 4313 ms: std::_Rb_tree<$>::_M_erase(std::_Rb_tree_node<$>*) (1217 times, avg 3 ms) 3259 ms: boost::stacktrace::detail::to_string_impl_base(std::__cxx11::basic_string<$>&, ch... (177 times, avg 17 ms) 2314 ms: std::_Rb_tree<$>::_M_get_insert_hint_unique_pos(std::_Rb_tree_const_... (207 times, avg 11 ms) 1923 ms: std::_Rb_tree_iterator<$> std::_Rb_tree<$>::_M_emplace_hint_unique<$... (216 times, avg 8 ms) 1817 ms: bool nlohmann::json_abi_v3_11_3::detail::parser<$>::sax_parse_intern... (18 times, avg 100 ms) 1816 ms: toml::result<$> toml::detail::sequence<$>::invoke<$>(toml::detail::l... (93 times, avg 19 ms) 1788 ms: nlohmann::json_abi_v3_11_3::detail::serializer<$>::dump(nlohmann::js... (40 times, avg 44 ms) 1749 ms: std::_Rb_tree<$>::_M_get_insert_unique_pos(std::__cxx11::basic_strin... (278 times, avg 6 ms) 1724 ms: std::__cxx11::basic_string<$>::_M_mutate(unsigned long, unsigned lon... (248 times, avg 6 ms) 1697 ms: boost::io::basic_altstringbuf<$>::overflow(int) (194 times, avg 8 ms) 1684 ms: nix::fetchers::GitInputScheme::fetch(nix::ref<$>, nix::fetchers::Inp... (1 times, avg 1684 ms) 1680 ms: std::_Rb_tree_node<$>* std::_Rb_tree<$>::_M_copy<$>(std::_Rb_tree_no... (303 times, avg 5 ms) 1589 ms: bool nlohmann::json_abi_v3_11_3::detail::parser<$>::sax_parse_intern... (18 times, avg 88 ms) 1483 ms: non-virtual thunk to boost::wrapexcept<$>::~wrapexcept() (181 times, avg 8 ms) 1447 ms: nlohmann::json_abi_v3_11_3::detail::serializer<$>::dump_escaped(std:... (40 times, avg 36 ms) 1441 ms: std::__shared_ptr<$>::~__shared_ptr() (496 times, avg 2 ms) 1420 ms: boost::stacktrace::basic_stacktrace<$>::init(unsigned long, unsigned... (137 times, avg 10 ms) 1396 ms: boost::basic_format<$>::~basic_format() (194 times, avg 7 ms) 1290 ms: std::__cxx11::basic_string<$>::_M_replace_cold(char*, unsigned long,... (231 times, avg 5 ms) 1258 ms: std::vector<$>::~vector() (354 times, avg 3 ms) 1222 ms: std::__cxx11::basic_string<$>::_M_replace(unsigned long, unsigned lo... (231 times, avg 5 ms) 1194 ms: std::_Rb_tree<$>::_M_get_insert_hint_unique_pos(std::_Rb_tree_const_... (49 times, avg 24 ms) 1186 ms: bool tao::pegtl::internal::sor<$>::match<$>(std::integer_sequence<$>... (1 times, avg 1186 ms) 1149 ms: std::__detail::_Executor<$>::_M_dfs(std::__detail::_Executor<$>::_Ma... (70 times, avg 16 ms) 1123 ms: toml::detail::sequence<$>::invoke(toml::detail::location&) (69 times, avg 16 ms) 1110 ms: nlohmann::json_abi_v3_11_3::basic_json<$>::json_value::destroy(nlohm... (55 times, avg 20 ms) 1079 ms: std::_Function_handler<$>::_M_manager(std::_Any_data&, std::_Any_dat... (541 times, avg 1 ms) 1033 ms: nlohmann::json_abi_v3_11_3::detail::lexer<$>::scan_number() (20 times, avg 51 ms) Change-Id: I10af282bcd4fc39c2d3caae3453e599e4639c70b --- src/build-remote/build-remote.cc | 5 +- src/libcmd/editor-for.cc | 1 + src/libexpr/eval.cc | 1 + src/libexpr/eval.hh | 3 +- src/libexpr/nixexpr.hh | 1 + src/libfetchers/fetchers.hh | 2 + src/libmain/progress-bar.cc | 3 +- src/libmain/shared.cc | 2 + src/libmain/shared.hh | 4 +- src/libstore/binary-cache-store.cc | 1 + src/libstore/build/derivation-goal.cc | 1 + src/libstore/build/entry-points.cc | 1 + src/libstore/build/hook-instance.cc | 1 + src/libstore/build/local-derivation-goal.cc | 4 +- src/libstore/builtins/buildenv.cc | 1 + src/libstore/builtins/fetchurl.cc | 1 + src/libstore/content-address.cc | 1 + src/libstore/daemon.cc | 3 +- src/libstore/derivations.cc | 5 +- src/libstore/derived-path.hh | 1 + src/libstore/filetransfer.cc | 1 + src/libstore/filetransfer.hh | 1 + src/libstore/gc.cc | 1 + src/libstore/legacy-ssh-store.cc | 1 + src/libstore/local-store.cc | 1 + src/libstore/lock.hh | 5 +- src/libstore/machines.hh | 4 +- src/libstore/make-content-addressed.cc | 1 + src/libstore/misc.cc | 2 +- src/libstore/nar-accessor.hh | 4 +- src/libstore/nar-info-disk-cache.cc | 1 + src/libstore/nar-info.cc | 2 +- src/libstore/optimise-store.cc | 1 + src/libstore/parsed-derivations.cc | 1 + src/libstore/path-info.cc | 1 + src/libstore/path-with-outputs.cc | 3 +- src/libstore/platform/darwin.cc | 1 + src/libstore/platform/linux.cc | 1 + src/libstore/profiles.cc | 3 +- src/libstore/remote-store.cc | 4 +- src/libstore/s3-binary-cache-store.cc | 1 + src/libstore/ssh-store.cc | 4 +- src/libstore/store-api.cc | 4 +- src/libutil/args.cc | 5 +- src/libutil/args.hh | 2 + src/libutil/backed-string-view.hh | 69 ++++++++++++++++++++ src/libutil/error.cc | 1 + src/libutil/file-system.cc | 1 + src/libutil/hash.cc | 1 + src/libutil/meson.build | 1 + src/libutil/processes.cc | 1 + src/libutil/regex.cc | 3 + src/libutil/serialise.cc | 22 ++++++- src/libutil/serialise.hh | 7 +- src/libutil/source-path.cc | 1 + src/libutil/strings.hh | 13 ---- src/libutil/types.hh | 72 +-------------------- src/nix-store/nix-store.cc | 2 +- src/nix/doctor.cc | 1 + src/nix/verify.cc | 3 +- src/pch/precompiled-headers.hh | 16 +++++ tests/unit/libstore/derivation.cc | 2 +- tests/unit/libstore/protocol.hh | 1 + tests/unit/libutil/references.cc | 1 + 64 files changed, 190 insertions(+), 125 deletions(-) create mode 100644 src/libutil/backed-string-view.hh diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 2450e80c2..3c7af067b 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -1,11 +1,7 @@ -#include -#include #include #include #include -#include #include -#include #if __APPLE__ #include #endif @@ -18,6 +14,7 @@ #include "build-result.hh" #include "store-api.hh" #include "derivations.hh" +#include "strings.hh" #include "local-store.hh" #include "legacy.hh" #include "experimental-features.hh" diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc index 67653d9c9..868153e90 100644 --- a/src/libcmd/editor-for.cc +++ b/src/libcmd/editor-for.cc @@ -1,6 +1,7 @@ #include "editor-for.hh" #include "environment-variables.hh" #include "source-path.hh" +#include "strings.hh" namespace nix { diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 227cd254a..a87383f7f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -19,6 +19,7 @@ #include "gc-small-vector.hh" #include "fetch-to-store.hh" #include "flake/flakeref.hh" +#include "exit.hh" #include #include diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index eab1f22ef..ba1e4a820 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -12,6 +12,7 @@ #include "experimental-features.hh" #include "search-path.hh" #include "repl-exit-status.hh" +#include "backed-string-view.hh" #include #include @@ -791,4 +792,4 @@ static constexpr std::string_view corepkgsPrefix{"/__corepkgs__/"}; } -#include "eval-inline.hh" +#include "eval-inline.hh" // IWYU pragma: keep diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 52f254813..d16281c39 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -11,6 +11,7 @@ #include "eval-error.hh" #include "pos-idx.hh" #include "pos-table.hh" +#include "strings.hh" namespace nix { diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 40f2b6294..f778908fb 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -7,6 +7,8 @@ #include "path.hh" #include "attrs.hh" #include "url.hh" +#include "ref.hh" +#include "strings.hh" #include diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index f0270df04..cdb15d8c7 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -1,8 +1,9 @@ #include "progress-bar.hh" +#include "file-system.hh" #include "sync.hh" -#include "store-api.hh" #include "names.hh" #include "terminal.hh" +#include "strings.hh" #include #include diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 018e34509..bc9548e09 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -6,6 +6,8 @@ #include "loggers.hh" #include "current-process.hh" #include "terminal.hh" +#include "strings.hh" +#include "exit.hh" #include #include diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index d7d872319..b41efe567 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -7,12 +7,10 @@ #include "path.hh" #include "derived-path.hh" #include "processes.hh" -#include "exit.hh" +#include "strings.hh" #include -#include - namespace nix { diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index fc0569a66..d4197b3df 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -10,6 +10,7 @@ #include "nar-accessor.hh" #include "thread-pool.hh" #include "signals.hh" +#include "strings.hh" #include #include diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index f2c8ccc5f..3ed3cb6bd 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -9,6 +9,7 @@ #include "logging-json.hh" #include "substitution-goal.hh" #include "drv-output-substitution-goal.hh" +#include "strings.hh" #include #include diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index f52f2876f..1f85881fa 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -2,6 +2,7 @@ #include "substitution-goal.hh" #include "derivation-goal.hh" #include "local-store.hh" +#include "strings.hh" namespace nix { diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index d5da80c74..f91a904cc 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -2,6 +2,7 @@ #include "file-system.hh" #include "globals.hh" #include "hook-instance.hh" +#include "strings.hh" namespace nix { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 1571627d6..7553f1e79 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1,15 +1,12 @@ #include "local-derivation-goal.hh" #include "indirect-root-store.hh" -#include "hook-instance.hh" #include "machines.hh" #include "store-api.hh" #include "worker.hh" #include "builtins.hh" #include "builtins/buildenv.hh" #include "path-references.hh" -#include "finally.hh" #include "archive.hh" -#include "compression.hh" #include "daemon.hh" #include "topo-sort.hh" #include "json-utils.hh" @@ -19,6 +16,7 @@ #include "child.hh" #include "unix-domain-socket.hh" #include "mount.hh" +#include "strings.hh" #include #include diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 61c729d27..9fe5f6660 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -1,4 +1,5 @@ #include "buildenv.hh" +#include "strings.hh" #include #include diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 4049d1c6c..062ecdc14 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -3,6 +3,7 @@ #include "store-api.hh" #include "archive.hh" #include "compression.hh" +#include "strings.hh" namespace nix { diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 6aa6d598d..dae311609 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -1,6 +1,7 @@ #include "args.hh" #include "content-address.hh" #include "split.hh" +#include "strings.hh" namespace nix { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 5ac9cd2ef..a9239197b 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -2,7 +2,7 @@ #include "monitor-fd.hh" #include "worker-protocol.hh" #include "worker-protocol-impl.hh" -#include "build-result.hh" +#include "build-result.hh" // IWYU pragma: keep #include "store-api.hh" #include "store-cast.hh" #include "gc-store.hh" @@ -12,6 +12,7 @@ #include "finally.hh" #include "archive.hh" #include "derivations.hh" +#include "strings.hh" #include "args.hh" #include diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 7f41e6865..93baa9f20 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -3,11 +3,12 @@ #include "store-api.hh" #include "globals.hh" #include "types.hh" -#include "split.hh" #include "common-protocol.hh" #include "common-protocol-impl.hh" -#include "fs-accessor.hh" #include "json-utils.hh" +#include "strings.hh" +#include "backed-string-view.hh" + #include #include diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index c96e0df67..b75415aec 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -5,6 +5,7 @@ #include "path.hh" #include "outputs-spec.hh" #include "comparator.hh" +#include "ref.hh" #include diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 11c8a755c..f3e8a5312 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -5,6 +5,7 @@ #include "s3.hh" #include "signals.hh" #include "compression.hh" +#include "strings.hh" #if ENABLE_S3 #include diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index b2ee66312..4352cb81b 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -2,6 +2,7 @@ ///@file #include "box_ptr.hh" +#include "ref.hh" #include "logging.hh" #include "serialise.hh" #include "types.hh" diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 7b4a12dc9..d5903d01e 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -5,6 +5,7 @@ #include "signals.hh" #include "finally.hh" #include "unix-domain-socket.hh" +#include "strings.hh" #include #include diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 9e2d65a1c..4433b411d 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -9,6 +9,7 @@ #include "path-with-outputs.hh" #include "ssh.hh" #include "ssh-store.hh" +#include "strings.hh" #include "derivations.hh" namespace nix { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4c8e2ea2f..49991e38a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -10,6 +10,7 @@ #include "signals.hh" #include "finally.hh" #include "compression.hh" +#include "strings.hh" #include #include diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index 1c268e1fb..ac5ff061b 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -1,11 +1,10 @@ #pragma once ///@file -#include "types.hh" - -#include +#include #include +#include namespace nix { diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index 8516409d4..56d9bed30 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -1,7 +1,9 @@ #pragma once ///@file -#include "types.hh" +#include "ref.hh" +#include +#include namespace nix { diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index abb6e9889..0c592ce22 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -1,5 +1,6 @@ #include "make-content-addressed.hh" #include "references.hh" +#include "strings.hh" namespace nix { diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index a63a28482..7ea7c189a 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,12 +1,12 @@ #include "derivations.hh" #include "parsed-derivations.hh" #include "globals.hh" -#include "local-store.hh" #include "store-api.hh" #include "thread-pool.hh" #include "topo-sort.hh" #include "closure.hh" #include "filetransfer.hh" +#include "strings.hh" namespace nix { diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 5e19bd3c7..4299daf1b 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -1,10 +1,12 @@ #pragma once ///@file +#include "fs-accessor.hh" +#include "ref.hh" + #include #include -#include "fs-accessor.hh" namespace nix { diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 5c0bb17b9..c83a8fbfb 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -4,6 +4,7 @@ #include "sqlite.hh" #include "globals.hh" #include "users.hh" +#include "strings.hh" #include #include diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index e557b4677..0434873df 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -1,4 +1,4 @@ -#include "globals.hh" +#include "strings.hh" #include "nar-info.hh" #include "store-api.hh" diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 9c871b78f..c60e5a85d 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -1,6 +1,7 @@ #include "local-store.hh" #include "globals.hh" #include "signals.hh" +#include "strings.hh" #include #include diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 992a79c6e..1b2ec914d 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -1,4 +1,5 @@ #include "parsed-derivations.hh" +#include "strings.hh" #include #include diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 4dc2823ce..54519a867 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -1,5 +1,6 @@ #include "path-info.hh" #include "store-api.hh" +#include "strings.hh" namespace nix { diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index af6837370..c49531acb 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -1,7 +1,6 @@ #include "path-with-outputs.hh" #include "store-api.hh" - -#include +#include "strings.hh" namespace nix { diff --git a/src/libstore/platform/darwin.cc b/src/libstore/platform/darwin.cc index 078753bff..956fb1e9b 100644 --- a/src/libstore/platform/darwin.cc +++ b/src/libstore/platform/darwin.cc @@ -2,6 +2,7 @@ #include "signals.hh" #include "platform/darwin.hh" #include "regex.hh" +#include "strings.hh" #include #include diff --git a/src/libstore/platform/linux.cc b/src/libstore/platform/linux.cc index f22fbe58f..486d71885 100644 --- a/src/libstore/platform/linux.cc +++ b/src/libstore/platform/linux.cc @@ -5,6 +5,7 @@ #include "signals.hh" #include "platform/linux.hh" #include "regex.hh" +#include "strings.hh" #include #include diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index e8b88693d..d88a3b9fe 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -1,12 +1,11 @@ #include "profiles.hh" -#include "store-api.hh" #include "local-fs-store.hh" #include "users.hh" +#include "strings.hh" #include #include #include -#include #include diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 36223051b..1f94ca03f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -15,6 +15,8 @@ #include "finally.hh" #include "logging.hh" #include "filetransfer.hh" +#include "strings.hh" + #include namespace nix { @@ -64,7 +66,7 @@ void RemoteStore::initConnection(Connection & conn) { /* Send the magic greeting, check for the reply. */ try { - conn.from.endOfFileError = "Nix daemon disconnected unexpectedly (maybe it crashed?)"; + conn.from.specialEndOfFileError = "Nix daemon disconnected unexpectedly (maybe it crashed?)"; conn.to << WORKER_MAGIC_1; conn.to.flush(); diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 921a2e556..b50084860 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -7,6 +7,7 @@ #include "globals.hh" #include "compression.hh" #include "filetransfer.hh" +#include "strings.hh" #include #include diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 94c0b7237..fb60326c1 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,13 +1,11 @@ #include "ssh-store.hh" #include "store-api.hh" -#include "local-fs-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" -#include "remote-fs-accessor.hh" -#include "archive.hh" #include "worker-protocol.hh" #include "pool.hh" #include "ssh.hh" +#include "strings.hh" namespace nix { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index f921956e8..cb0604233 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -5,10 +5,10 @@ #include "nar-info-disk-cache.hh" #include "thread-pool.hh" #include "url.hh" -#include "references.hh" #include "archive.hh" -#include "remote-store.hh" +#include "uds-remote-store.hh" #include "signals.hh" +#include "strings.hh" // FIXME this should not be here, see TODO below on // `addMultipleToStore`. #include "worker-protocol.hh" diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 1342e7c6a..edcab23ac 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -1,10 +1,11 @@ #include "args.hh" #include "args/root.hh" #include "hash.hh" -#include "json-utils.hh" +#include "strings.hh" +#include "json-utils.hh" // IWYU pragma: keep (instances) #include "environment-variables.hh" -#include "experimental-features-json.hh" +#include "experimental-features-json.hh" // IWYU pragma: keep (instances) #include "logging.hh" #include diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 5fdbaba7e..e2bac6415 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -3,6 +3,8 @@ #include "experimental-features.hh" #include "types.hh" +#include "ref.hh" + #include #include #include diff --git a/src/libutil/backed-string-view.hh b/src/libutil/backed-string-view.hh new file mode 100644 index 000000000..96136331c --- /dev/null +++ b/src/libutil/backed-string-view.hh @@ -0,0 +1,69 @@ +#pragma once +/// @file String view that can be either owned or borrowed. +#include +#include +#include + +/** + * This wants to be a little bit like rust's Cow type. + * Some parts of the evaluator benefit greatly from being able to reuse + * existing allocations for strings, but have to be able to also use + * newly allocated storage for values. + * + * We do not define implicit conversions, even with ref qualifiers, + * since those can easily become ambiguous to the reader and can degrade + * into copying behaviour we want to avoid. + */ +class BackedStringView { +private: + std::variant data; + + /** + * Needed to introduce a temporary since operator-> must return + * a pointer. Without this we'd need to store the view object + * even when we already own a string. + */ + class Ptr { + private: + std::string_view view; + public: + Ptr(std::string_view view): view(view) {} + const std::string_view * operator->() const { return &view; } + }; + +public: + BackedStringView(std::string && s): data(std::move(s)) {} + BackedStringView(std::string_view sv): data(sv) {} + template + BackedStringView(const char (& lit)[N]): data(std::string_view(lit)) {} + + BackedStringView(const BackedStringView &) = delete; + BackedStringView & operator=(const BackedStringView &) = delete; + + /** + * We only want move operations defined since the sole purpose of + * this type is to avoid copies. + */ + BackedStringView(BackedStringView && other) = default; + BackedStringView & operator=(BackedStringView && other) = default; + + bool isOwned() const + { + return std::holds_alternative(data); + } + + std::string toOwned() && + { + return isOwned() + ? std::move(std::get(data)) + : std::string(std::get(data)); + } + + std::string_view operator*() const + { + return isOwned() + ? std::get(data) + : std::get(data); + } + Ptr operator->() const { return Ptr(**this); } +}; diff --git a/src/libutil/error.cc b/src/libutil/error.cc index e5d6a9fa8..a7cbfbfd0 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -3,6 +3,7 @@ #include "logging.hh" #include "position.hh" #include "terminal.hh" +#include "strings.hh" #include #include diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 631cf076b..1d266067e 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -10,6 +10,7 @@ #include "logging.hh" #include "serialise.hh" #include "signals.hh" +#include "strings.hh" #include "types.hh" #include "users.hh" diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index f05d4aa98..d383e9802 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -10,6 +10,7 @@ #include "charptr-cast.hh" #include "logging.hh" #include "split.hh" +#include "strings.hh" #include #include diff --git a/src/libutil/meson.build b/src/libutil/meson.build index e7f986363..6566f7f46 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -53,6 +53,7 @@ libutil_headers = files( 'archive.hh', 'args/root.hh', 'args.hh', + 'backed-string-view.hh', 'box_ptr.hh', 'canon-path.hh', 'cgroup.hh', diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc index 61e1ad556..eec592221 100644 --- a/src/libutil/processes.cc +++ b/src/libutil/processes.cc @@ -3,6 +3,7 @@ #include "finally.hh" #include "logging.hh" #include "processes.hh" +#include "strings.hh" #include "serialise.hh" #include "signals.hh" diff --git a/src/libutil/regex.cc b/src/libutil/regex.cc index a9e6c6bee..a12d13550 100644 --- a/src/libutil/regex.cc +++ b/src/libutil/regex.cc @@ -1,6 +1,9 @@ #include #include +// Declared as extern in precompiled-headers.hh +template class std::basic_regex; + namespace nix::regex { std::string quoteRegexChars(const std::string & raw) { diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 4eda1b7e7..f509fedff 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -9,6 +9,21 @@ namespace nix { +namespace { +/** + * Convert a little-endian integer to host order. + */ +template +T readLittleEndian(unsigned char * p) +{ + T x = 0; + for (size_t i = 0; i < sizeof(x); ++i, ++p) { + x |= ((T) *p) << (i * 8); + } + return x; +} +} + template T readNum(Source & source) { @@ -152,7 +167,7 @@ size_t FdSource::readUnbuffered(char * data, size_t len) n = ::read(fd, data, len); } while (n == -1 && errno == EINTR); if (n == -1) { _good = false; throw SysError("reading from file"); } - if (n == 0) { _good = false; throw EndOfFile(std::string(*endOfFileError)); } + if (n == 0) { _good = false; throw EndOfFile(endOfFileError()); } read += n; return n; } @@ -163,6 +178,11 @@ bool FdSource::good() return _good; } +std::string FdSource::endOfFileError() const +{ + return specialEndOfFileError.has_value() ? *specialEndOfFileError : "unexpected end-of-file"; +} + size_t StringSource::read(char * data, size_t len) { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 9ad8018d0..612658b2d 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -1,12 +1,10 @@ #pragma once ///@file -#include #include #include "charptr-cast.hh" #include "generator.hh" -#include "strings.hh" #include "types.hh" #include "file-descriptor.hh" @@ -153,7 +151,10 @@ struct FdSource : BufferedSource { int fd; size_t read = 0; - BackedStringView endOfFileError{"unexpected end-of-file"}; + /** Defaults to "unexpected end-of-file" */ + std::optional specialEndOfFileError; + + std::string endOfFileError() const; FdSource() : fd(-1) { } FdSource(int fd) : fd(fd) { } diff --git a/src/libutil/source-path.cc b/src/libutil/source-path.cc index cfaac20c0..782005ef1 100644 --- a/src/libutil/source-path.cc +++ b/src/libutil/source-path.cc @@ -1,4 +1,5 @@ #include "source-path.hh" +#include "strings.hh" namespace nix { diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh index 7330e2063..ebafab9ad 100644 --- a/src/libutil/strings.hh +++ b/src/libutil/strings.hh @@ -164,19 +164,6 @@ template std::optional string2Float(const std::string_view s); -/** - * Convert a little-endian integer to host order. - */ -template -T readLittleEndian(unsigned char * p) -{ - T x = 0; - for (size_t i = 0; i < sizeof(x); ++i, ++p) { - x |= ((T) *p) << (i * 8); - } - return x; -} - /** * Convert a string to lower case. */ diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 13cb062fb..66c41fe59 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -1,17 +1,15 @@ #pragma once ///@file -#include "ref.hh" - #include #include #include #include -#include +#include #include -#include #include #include +#include // IWYU pragma: keep (this is used literally everywhere) namespace nix { @@ -166,70 +164,4 @@ constexpr auto enumerate(T && iterable) template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; - - -/** - * This wants to be a little bit like rust's Cow type. - * Some parts of the evaluator benefit greatly from being able to reuse - * existing allocations for strings, but have to be able to also use - * newly allocated storage for values. - * - * We do not define implicit conversions, even with ref qualifiers, - * since those can easily become ambiguous to the reader and can degrade - * into copying behaviour we want to avoid. - */ -class BackedStringView { -private: - std::variant data; - - /** - * Needed to introduce a temporary since operator-> must return - * a pointer. Without this we'd need to store the view object - * even when we already own a string. - */ - class Ptr { - private: - std::string_view view; - public: - Ptr(std::string_view view): view(view) {} - const std::string_view * operator->() const { return &view; } - }; - -public: - BackedStringView(std::string && s): data(std::move(s)) {} - BackedStringView(std::string_view sv): data(sv) {} - template - BackedStringView(const char (& lit)[N]): data(std::string_view(lit)) {} - - BackedStringView(const BackedStringView &) = delete; - BackedStringView & operator=(const BackedStringView &) = delete; - - /** - * We only want move operations defined since the sole purpose of - * this type is to avoid copies. - */ - BackedStringView(BackedStringView && other) = default; - BackedStringView & operator=(BackedStringView && other) = default; - - bool isOwned() const - { - return std::holds_alternative(data); - } - - std::string toOwned() && - { - return isOwned() - ? std::move(std::get(data)) - : std::string(std::get(data)); - } - - std::string_view operator*() const - { - return isOwned() - ? std::get(data) - : std::get(data); - } - Ptr operator->() const { return Ptr(**this); } -}; - } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5762b0644..bc43aa7b1 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -1,6 +1,7 @@ #include "archive.hh" #include "derivations.hh" #include "dotgraph.hh" +#include "exit.hh" #include "globals.hh" #include "build-result.hh" #include "store-cast.hh" @@ -17,7 +18,6 @@ #include #include -#include #include #include diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 4e1cfe8c0..17db3f764 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -7,6 +7,7 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "worker-protocol.hh" +#include "exit.hh" using namespace nix; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 8783d4e04..d4ab352f4 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -1,10 +1,9 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" -#include "sync.hh" #include "thread-pool.hh" #include "signals.hh" -#include "references.hh" +#include "exit.hh" #include diff --git a/src/pch/precompiled-headers.hh b/src/pch/precompiled-headers.hh index f52f1cab8..c417d27db 100644 --- a/src/pch/precompiled-headers.hh +++ b/src/pch/precompiled-headers.hh @@ -58,3 +58,19 @@ #include #include + +// This stuff is here to force the compiler to actually apply the extern +// template directives in all compilation units. To borrow a term, under +// complex microarchitectural conditions, clang ignores the extern template +// declaration, as revealed in the profile. +// +// In most cases, extern template works fine in the header itself. We don't +// have any idea why this happens. + +// Here because of all the regexes everywhere (it is infeasible to block instantiation everywhere) +// For some reason this does not actually prevent the instantiation of +// regex::_M_compile, and the regex compiler (my interpretation of what this is +// supposed to do is make the template bits out-of-line), but it *does* prevent +// a bunch of codegen of regex stuff, which seems to save about 30s on-cpu. +// Instantiated in libutil/regex.cc. +extern template class std::basic_regex; diff --git a/tests/unit/libstore/derivation.cc b/tests/unit/libstore/derivation.cc index f6401a095..21e141693 100644 --- a/tests/unit/libstore/derivation.cc +++ b/tests/unit/libstore/derivation.cc @@ -1,8 +1,8 @@ #include #include -#include "experimental-features.hh" #include "derivations.hh" +#include "strings.hh" #include "tests/libstore.hh" #include "tests/characterization.hh" diff --git a/tests/unit/libstore/protocol.hh b/tests/unit/libstore/protocol.hh index 059b4d91c..8212959f0 100644 --- a/tests/unit/libstore/protocol.hh +++ b/tests/unit/libstore/protocol.hh @@ -1,6 +1,7 @@ #include #include +#include "strings.hh" #include "tests/libstore.hh" #include "tests/characterization.hh" diff --git a/tests/unit/libutil/references.cc b/tests/unit/libutil/references.cc index a914e6c70..ef7111e9c 100644 --- a/tests/unit/libutil/references.cc +++ b/tests/unit/libutil/references.cc @@ -1,4 +1,5 @@ #include "references.hh" +#include "strings.hh" #include namespace nix { -- 2.44.1 From a510d1748416ff29b1ed3cab92ac0ad943b6e590 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 23 Aug 2024 16:57:26 -0700 Subject: [PATCH 012/106] build-time: hide boost stacktrace in a .cc file Saves about 16s of CPU time. Not a lot but not nothing. Feels more like the principle of the thing. Change-Id: I0992d4024317c20d6985a7977d5649edfb9f46bb --- src/libutil/fmt.cc | 10 ++++++++++ src/libutil/fmt.hh | 12 +++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/libutil/fmt.cc b/src/libutil/fmt.cc index 400fb7ea0..bff5af020 100644 --- a/src/libutil/fmt.cc +++ b/src/libutil/fmt.cc @@ -1,4 +1,9 @@ #include "fmt.hh" // IWYU pragma: keep +// Darwin and FreeBSD stdenv do not define _GNU_SOURCE but do have _Unwind_Backtrace. +#if __APPLE__ || __FreeBSD__ +#define BOOST_STACKTRACE_GNU_SOURCE_NOT_REQUIRED +#endif +#include template class boost::basic_format; @@ -11,4 +16,9 @@ template HintFmt::HintFmt(const std::string &, const uint64_t &, const char * co HintFmt::HintFmt(const std::string & literal) : HintFmt("%s", Uncolored(literal)) {} +void printStackTrace() +{ + std::cerr << boost::stacktrace::stacktrace() << std::endl; +} + } diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 7589e51e2..ee3e1e2e7 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -4,11 +4,6 @@ #include #include #include -// Darwin and FreeBSD stdenv do not define _GNU_SOURCE but do have _Unwind_Backtrace. -#if __APPLE__ || __FreeBSD__ -#define BOOST_STACKTRACE_GNU_SOURCE_NOT_REQUIRED -#endif -#include #include "ansicolor.hh" // Explicit instantiation in fmt.cc @@ -16,6 +11,9 @@ extern template class boost::basic_format; namespace nix { +/** Prints a C++ stack trace to stderr using boost stacktrace */ +void printStackTrace(); + /** * Values wrapped in this struct are printed in magenta. * @@ -176,14 +174,14 @@ public: std::cerr << "HintFmt received incorrect number of format args. Original format string: '"; std::cerr << format << "'; number of arguments: " << sizeof...(args) << "\n"; // And regardless of the coredump give me a damn stacktrace. - std::cerr << boost::stacktrace::stacktrace() << std::endl; + printStackTrace(); abort(); } } catch (boost::io::format_error & ex) { // Same thing, but for anything that happens in the member initializers. std::cerr << "HintFmt received incorrect format string. Original format string: '"; std::cerr << format << "'; number of arguments: " << sizeof...(args) << "\n"; - std::cerr << boost::stacktrace::stacktrace() << std::endl; + printStackTrace(); abort(); } -- 2.44.1 From bb161a96cf1171a5c4ed3661a0f2f5a93ac10804 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 29 Aug 2024 17:06:39 -0700 Subject: [PATCH 013/106] manual: note that __sandboxProfile allows bypassing the darwin sandbox (but only if it is set to relaxed. no security hole here.) Thanks to lilyball for pointing out this omission in the docs. Change-Id: I2408a943bfe817fe660fe1c8fefef898aaf5f7e9 --- doc/manual/src/installation/multi-user.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/installation/multi-user.md b/doc/manual/src/installation/multi-user.md index 87c956071..a470aa74d 100644 --- a/doc/manual/src/installation/multi-user.md +++ b/doc/manual/src/installation/multi-user.md @@ -36,7 +36,10 @@ All users of the Lix daemon may do the following to bring things into the Nix st - Input-addressed, so they are run in the sandbox with no network access, with the following exceptions: - The (poorly named, since it is not *just* about chroot) property `__noChroot` is set on the derivation and `sandbox` is set to `relaxed`. - - On macOS, the derivation property `__darwinAllowLocalNetworking` allows network access to localhost from input-addressed derivations regardless of the `sandbox` setting value. This property exists with such semantics because macOS has no network namespace equivalent to isolate individual processes' localhost networking. + - On macOS, the derivation property `__darwinAllowLocalNetworking` allows network access to localhost from input-addressed derivations regardless of the `sandbox` setting value. + This property exists with such semantics because macOS has no network namespace equivalent to isolate individual processes' localhost networking. + - On macOS, the derivation property `__sandboxProfile` accepts extra sandbox profile S-expressions, allowing derivations to bypass arbitrary parts of the sandbox without altogether disabling it. + This is only permitted when `sandbox` is set to `relaxed`. - Output-addressed, so they are run with network access but their result must match an expected hash. Trusted users may set any setting, including `sandbox = false`, so the sandbox state can be different at runtime from what is described in `nix.conf` for builds invoked with such settings. -- 2.44.1 From a5c1e73fa8e004a93e37254a3582ba91048c4550 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sun, 25 Aug 2024 13:41:56 +0200 Subject: [PATCH 014/106] libstore: add "is dependency" info to goal whether goal errors are reported via the `ex` member or just printed to the log depends on whether the goal is a toplevel goal or a dependency. if goals are aware of this themselves we can move error printing out of the worker loop, and since a running worker can only be used by running goals it's totally sufficient to keep a `Worker::running` flag for this Change-Id: I6b5cbe6eccee1afa5fde80653c4b968554ddd16f --- src/libstore/build/derivation-goal.cc | 8 ++-- src/libstore/build/derivation-goal.hh | 4 +- .../build/drv-output-substitution-goal.cc | 3 +- .../build/drv-output-substitution-goal.hh | 8 +++- src/libstore/build/goal.hh | 10 ++++- src/libstore/build/local-derivation-goal.hh | 2 + src/libstore/build/substitution-goal.cc | 10 ++++- src/libstore/build/substitution-goal.hh | 8 +++- src/libstore/build/worker.cc | 45 ++++++++++++++----- src/libstore/build/worker.hh | 2 + src/libstore/platform.cc | 18 ++++---- 11 files changed, 86 insertions(+), 32 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3ed3cb6bd..462c381b8 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -58,8 +58,8 @@ namespace nix { DerivationGoal::DerivationGoal(const StorePath & drvPath, - const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) + const OutputsSpec & wantedOutputs, Worker & worker, bool isDependency, BuildMode buildMode) + : Goal(worker, isDependency) , useDerivation(true) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -76,8 +76,8 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) + const OutputsSpec & wantedOutputs, Worker & worker, bool isDependency, BuildMode buildMode) + : Goal(worker, isDependency) , useDerivation(false) , drvPath(drvPath) , wantedOutputs(wantedOutputs) diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index c9638c2b6..77f9fef4b 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -234,10 +234,10 @@ struct DerivationGoal : public Goal std::string machineName; DerivationGoal(const StorePath & drvPath, - const OutputsSpec & wantedOutputs, Worker & worker, + const OutputsSpec & wantedOutputs, Worker & worker, bool isDependency, BuildMode buildMode = bmNormal); DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const OutputsSpec & wantedOutputs, Worker & worker, + const OutputsSpec & wantedOutputs, Worker & worker, bool isDependency, BuildMode buildMode = bmNormal); virtual ~DerivationGoal() noexcept(false); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index e6c7524e9..54d81d7a0 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -9,9 +9,10 @@ namespace nix { DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal( const DrvOutput & id, Worker & worker, + bool isDependency, RepairFlag repair, std::optional ca) - : Goal(worker) + : Goal(worker, isDependency) , id(id) { state = &DrvOutputSubstitutionGoal::init; diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index a660a4f3e..b48c4670b 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -56,7 +56,13 @@ class DrvOutputSubstitutionGoal : public Goal { bool substituterFailed = false; public: - DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + DrvOutputSubstitutionGoal( + const DrvOutput & id, + Worker & worker, + bool isDependency, + RepairFlag repair = NoRepair, + std::optional ca = std::nullopt + ); typedef WorkResult (DrvOutputSubstitutionGoal::*GoalState)(bool inBuildSlot); GoalState state; diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 1bfea8231..1f25fb233 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -60,6 +60,13 @@ struct Goal */ Worker & worker; + /** + * Whether this goal is only a dependency of other goals. Toplevel + * goals that are also dependencies of other toplevel goals do not + * set this, only goals that are exclusively dependencies do this. + */ + const bool isDependency; + /** * Goals that this goal is waiting for. */ @@ -143,8 +150,9 @@ public: */ std::shared_ptr ex; - explicit Goal(Worker & worker) + explicit Goal(Worker & worker, bool isDependency) : worker(worker) + , isDependency(isDependency) { } virtual ~Goal() noexcept(false) diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index f17685af8..37a96b4d1 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -186,6 +186,7 @@ struct LocalDerivationGoal : public DerivationGoal const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, + bool isDependency, BuildMode buildMode ); @@ -198,6 +199,7 @@ struct LocalDerivationGoal : public DerivationGoal const BasicDerivation & drv, const OutputsSpec & wantedOutputs, Worker & worker, + bool isDependency, BuildMode buildMode ); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index d84b65a53..390aabb45 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -6,8 +6,14 @@ namespace nix { -PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) - : Goal(worker) +PathSubstitutionGoal::PathSubstitutionGoal( + const StorePath & storePath, + Worker & worker, + bool isDependency, + RepairFlag repair, + std::optional ca +) + : Goal(worker, isDependency) , storePath(storePath) , repair(repair) , ca(ca) diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index e546fc06f..5d58b34a0 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -80,7 +80,13 @@ struct PathSubstitutionGoal : public Goal std::optional errorMsg = {}); public: - PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + PathSubstitutionGoal( + const StorePath & storePath, + Worker & worker, + bool isDependency, + RepairFlag repair = NoRepair, + std::optional ca = std::nullopt + ); ~PathSubstitutionGoal(); Finished timedOut(Error && ex) override { abort(); }; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 1b4633e64..9e548cc16 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -1,5 +1,6 @@ #include "charptr-cast.hh" #include "worker.hh" +#include "finally.hh" #include "substitution-goal.hh" #include "drv-output-substitution-goal.hh" #include "local-derivation-goal.hh" @@ -59,22 +60,38 @@ std::shared_ptr Worker::makeDerivationGoalCommon( std::shared_ptr Worker::makeDerivationGoal(const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode) { - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr { - return !dynamic_cast(&store) - ? std::make_shared(drvPath, wantedOutputs, *this, buildMode) - : LocalDerivationGoal::makeLocalDerivationGoal(drvPath, wantedOutputs, *this, buildMode); - }); + return makeDerivationGoalCommon( + drvPath, + wantedOutputs, + [&]() -> std::shared_ptr { + return !dynamic_cast(&store) + ? std::make_shared( + drvPath, wantedOutputs, *this, running, buildMode + ) + : LocalDerivationGoal::makeLocalDerivationGoal( + drvPath, wantedOutputs, *this, running, buildMode + ); + } + ); } std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode) { - return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr { - return !dynamic_cast(&store) - ? std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode) - : LocalDerivationGoal::makeLocalDerivationGoal(drvPath, drv, wantedOutputs, *this, buildMode); - }); + return makeDerivationGoalCommon( + drvPath, + wantedOutputs, + [&]() -> std::shared_ptr { + return !dynamic_cast(&store) + ? std::make_shared( + drvPath, drv, wantedOutputs, *this, running, buildMode + ) + : LocalDerivationGoal::makeLocalDerivationGoal( + drvPath, drv, wantedOutputs, *this, running, buildMode + ); + } + ); } @@ -83,7 +100,7 @@ std::shared_ptr Worker::makePathSubstitutionGoal(const Sto std::weak_ptr & goal_weak = substitutionGoals[path]; auto goal = goal_weak.lock(); // FIXME if (!goal) { - goal = std::make_shared(path, *this, repair, ca); + goal = std::make_shared(path, *this, running, repair, ca); goal_weak = goal; wakeUp(goal); } @@ -96,7 +113,7 @@ std::shared_ptr Worker::makeDrvOutputSubstitutionGoal std::weak_ptr & goal_weak = drvOutputSubstitutionGoals[id]; auto goal = goal_weak.lock(); // FIXME if (!goal) { - goal = std::make_shared(id, *this, repair, ca); + goal = std::make_shared(id, *this, running, repair, ca); goal_weak = goal; wakeUp(goal); } @@ -313,6 +330,10 @@ void Worker::run(const Goals & _topGoals) { std::vector topPaths; + assert(!running); + running = true; + Finally const _stop([&] { running = false; }); + for (auto & i : _topGoals) { topGoals.insert(i); if (auto goal = dynamic_cast(i.get())) { diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 7abb966f9..360366f8d 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -47,6 +47,8 @@ class Worker { private: + bool running = false; + /* Note: the worker should only have strong pointers to the top-level goals. */ diff --git a/src/libstore/platform.cc b/src/libstore/platform.cc index 72757e39b..f2c023c82 100644 --- a/src/libstore/platform.cc +++ b/src/libstore/platform.cc @@ -29,17 +29,18 @@ std::shared_ptr LocalDerivationGoal::makeLocalDerivationGoa const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, + bool isDependency, BuildMode buildMode ) { #if __linux__ - return std::make_shared(drvPath, wantedOutputs, worker, buildMode); + return std::make_shared(drvPath, wantedOutputs, worker, isDependency, buildMode); #elif __APPLE__ - return std::make_shared(drvPath, wantedOutputs, worker, buildMode); + return std::make_shared(drvPath, wantedOutputs, worker, isDependency, buildMode); #elif __FreeBSD__ - return std::make_shared(drvPath, wantedOutputs, worker, buildMode); + return std::make_shared(drvPath, wantedOutputs, worker, isDependency, buildMode); #else - return std::make_shared(drvPath, wantedOutputs, worker, buildMode); + return std::make_shared(drvPath, wantedOutputs, worker, isDependency, buildMode); #endif } @@ -48,24 +49,25 @@ std::shared_ptr LocalDerivationGoal::makeLocalDerivationGoa const BasicDerivation & drv, const OutputsSpec & wantedOutputs, Worker & worker, + bool isDependency, BuildMode buildMode ) { #if __linux__ return std::make_shared( - drvPath, drv, wantedOutputs, worker, buildMode + drvPath, drv, wantedOutputs, worker, isDependency, buildMode ); #elif __APPLE__ return std::make_shared( - drvPath, drv, wantedOutputs, worker, buildMode + drvPath, drv, wantedOutputs, worker, isDependency, buildMode ); #elif __FreeBSD__ return std::make_shared( - drvPath, drv, wantedOutputs, worker, buildMode + drvPath, drv, wantedOutputs, worker, isDependency, buildMode ); #else return std::make_shared( - drvPath, drv, wantedOutputs, worker, buildMode + drvPath, drv, wantedOutputs, worker, isDependency, buildMode ); #endif } -- 2.44.1 From 869666cb651f97cdce3a6aabf62073bfe1130cbb Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Thu, 29 Aug 2024 21:06:30 +0200 Subject: [PATCH 015/106] libstore: hide Worker goal factory methods this doesn't serve a great purpose yet except to confine construction of goals to the stack frame of Worker::run() and its child frames. we don't need this yet (and the goal constructors remain fully visible), but in a future change that fully removes the current worker loop we'll need some way of knowing which goals are top-level goals without passing the goals themselves around. once that's possible we can remove visible goals as a concept and rely on build result futures and a scheduler built upon them Change-Id: Ia73cdeffcfb9ba1ce9d69b702dc0bc637a4c4ce6 --- src/libstore/build/derivation-goal.cc | 16 ++--- .../build/drv-output-substitution-goal.cc | 4 +- src/libstore/build/entry-points.cc | 65 ++++++++++--------- src/libstore/build/substitution-goal.cc | 2 +- src/libstore/build/worker.cc | 5 +- src/libstore/build/worker.hh | 63 +++++++++++++++--- 6 files changed, 106 insertions(+), 49 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 462c381b8..3dca7a3c0 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -170,7 +170,7 @@ Goal::WorkResult DerivationGoal::getDerivation(bool inBuildSlot) state = &DerivationGoal::loadDerivation; - return WaitForGoals{{worker.makePathSubstitutionGoal(drvPath)}}; + return WaitForGoals{{worker.goalFactory().makePathSubstitutionGoal(drvPath)}}; } @@ -268,14 +268,14 @@ Goal::WorkResult DerivationGoal::haveDerivation(bool inBuildSlot) if (!status.wanted) continue; if (!status.known) result.goals.insert( - worker.makeDrvOutputSubstitutionGoal( + worker.goalFactory().makeDrvOutputSubstitutionGoal( DrvOutput{status.outputHash, outputName}, buildMode == bmRepair ? Repair : NoRepair ) ); else { auto * cap = getDerivationCA(*drv); - result.goals.insert(worker.makePathSubstitutionGoal( + result.goals.insert(worker.goalFactory().makePathSubstitutionGoal( status.known->path, buildMode == bmRepair ? Repair : NoRepair, cap ? std::optional { *cap } : std::nullopt)); @@ -374,7 +374,7 @@ Goal::WorkResult DerivationGoal::gaveUpOnSubstitution(bool inBuildSlot) addWaiteeDerivedPath = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { if (!inputNode.value.empty()) - result.goals.insert(worker.makeGoal( + result.goals.insert(worker.goalFactory().makeGoal( DerivedPath::Built { .drvPath = inputDrv, .outputs = inputNode.value, @@ -419,7 +419,7 @@ Goal::WorkResult DerivationGoal::gaveUpOnSubstitution(bool inBuildSlot) if (!settings.useSubstitutes) throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - result.goals.insert(worker.makePathSubstitutionGoal(i)); + result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i)); } if (result.goals.empty()) {/* to prevent hang (no wake-up event) */ @@ -475,9 +475,9 @@ Goal::WorkResult DerivationGoal::repairClosure() worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) - result.goals.insert(worker.makePathSubstitutionGoal(i, Repair)); + result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i, Repair)); else - result.goals.insert(worker.makeGoal( + result.goals.insert(worker.goalFactory().makeGoal( DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath2->second), .outputs = OutputsSpec::All { }, @@ -580,7 +580,7 @@ Goal::WorkResult DerivationGoal::inputsRealised(bool inBuildSlot) worker.store.printStorePath(pathResolved), }); - resolvedDrvGoal = worker.makeDerivationGoal( + resolvedDrvGoal = worker.goalFactory().makeDerivationGoal( pathResolved, wantedOutputs, buildMode); state = &DerivationGoal::resolvedFinished; diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 54d81d7a0..9acff14b0 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -112,11 +112,11 @@ Goal::WorkResult DrvOutputSubstitutionGoal::realisationFetched(bool inBuildSlot) ); return tryNext(inBuildSlot); } - result.goals.insert(worker.makeDrvOutputSubstitutionGoal(depId)); + result.goals.insert(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId)); } } - result.goals.insert(worker.makePathSubstitutionGoal(outputInfo->outPath)); + result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath)); if (result.goals.empty()) { return outPathValid(inBuildSlot); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 1f85881fa..a5bb05b24 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -10,11 +10,12 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod { Worker worker(*this, evalStore ? *evalStore : *this); - Goals goals; - for (auto & br : reqs) - goals.insert(worker.makeGoal(br, buildMode)); - - worker.run(goals); + auto goals = worker.run([&](GoalFactory & gf) { + Goals goals; + for (auto & br : reqs) + goals.insert(gf.makeGoal(br, buildMode)); + return goals; + }); StringSet failed; std::shared_ptr ex; @@ -48,17 +49,17 @@ std::vector Store::buildPathsWithResults( std::shared_ptr evalStore) { Worker worker(*this, evalStore ? *evalStore : *this); - - Goals goals; std::vector> state; - for (const auto & req : reqs) { - auto goal = worker.makeGoal(req, buildMode); - goals.insert(goal); - state.push_back({req, goal}); - } - - worker.run(goals); + auto goals = worker.run([&](GoalFactory & gf) { + Goals goals; + for (const auto & req : reqs) { + auto goal = gf.makeGoal(req, buildMode); + goals.insert(goal); + state.push_back({req, goal}); + } + return goals; + }); std::vector results; @@ -72,10 +73,12 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat BuildMode buildMode) { Worker worker(*this, *this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All {}, buildMode); try { - worker.run(Goals{goal}); + auto goals = worker.run([&](GoalFactory & gf) -> Goals { + return Goals{gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)}; + }); + auto goal = *goals.begin(); return goal->buildResult.restrictTo(DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All {}, @@ -95,10 +98,10 @@ void Store::ensurePath(const StorePath & path) if (isValidPath(path)) return; Worker worker(*this, *this); - GoalPtr goal = worker.makePathSubstitutionGoal(path); - Goals goals = {goal}; - worker.run(goals); + auto goals = + worker.run([&](GoalFactory & gf) { return Goals{gf.makePathSubstitutionGoal(path)}; }); + auto goal = *goals.begin(); if (goal->exitCode != Goal::ecSuccess) { if (goal->ex) { @@ -113,23 +116,27 @@ void Store::ensurePath(const StorePath & path) void Store::repairPath(const StorePath & path) { Worker worker(*this, *this); - GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair); - Goals goals = {goal}; - worker.run(goals); + auto goals = worker.run([&](GoalFactory & gf) { + return Goals{gf.makePathSubstitutionGoal(path, Repair)}; + }); + auto goal = *goals.begin(); if (goal->exitCode != Goal::ecSuccess) { /* Since substituting the path didn't work, if we have a valid deriver, then rebuild the deriver. */ auto info = queryPathInfo(path); if (info->deriver && isValidPath(*info->deriver)) { - goals.clear(); - goals.insert(worker.makeGoal(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(*info->deriver), - // FIXME: Should just build the specific output we need. - .outputs = OutputsSpec::All { }, - }, bmRepair)); - worker.run(goals); + worker.run([&](GoalFactory & gf) { + return Goals{gf.makeGoal( + DerivedPath::Built{ + .drvPath = makeConstantStorePathRef(*info->deriver), + // FIXME: Should just build the specific output we need. + .outputs = OutputsSpec::All{}, + }, + bmRepair + )}; + }); } else throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path)); } diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 390aabb45..673b3c503 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -161,7 +161,7 @@ Goal::WorkResult PathSubstitutionGoal::tryNext(bool inBuildSlot) WaitForGoals result; for (auto & i : info->references) if (i != storePath) /* ignore self-references */ - result.goals.insert(worker.makePathSubstitutionGoal(i)); + result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i)); if (result.goals.empty()) {/* to prevent hang (no wake-up event) */ return referencesValid(inBuildSlot); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 9e548cc16..5d0cc920a 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -326,8 +326,9 @@ void Worker::waitForAWhile(GoalPtr goal) } -void Worker::run(const Goals & _topGoals) +Goals Worker::run(std::function req) { + auto _topGoals = req(goalFactory()); std::vector topPaths; assert(!running); @@ -411,6 +412,8 @@ void Worker::run(const Goals & _topGoals) assert(!settings.keepGoing || awake.empty()); assert(!settings.keepGoing || wantingToBuild.empty()); assert(!settings.keepGoing || children.empty()); + + return _topGoals; } void Worker::waitForInput() diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 360366f8d..3fbf457fe 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -40,10 +40,57 @@ struct Child /* Forward definition. */ struct HookInstance; +class GoalFactory +{ +public: + virtual std::shared_ptr makeDerivationGoal( + const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal + ) = 0; + virtual std::shared_ptr makeBasicDerivationGoal( + const StorePath & drvPath, + const BasicDerivation & drv, + const OutputsSpec & wantedOutputs, + BuildMode buildMode = bmNormal + ) = 0; + + /** + * @ref SubstitutionGoal "substitution goal" + */ + virtual std::shared_ptr makePathSubstitutionGoal( + const StorePath & storePath, + RepairFlag repair = NoRepair, + std::optional ca = std::nullopt + ) = 0; + virtual std::shared_ptr makeDrvOutputSubstitutionGoal( + const DrvOutput & id, + RepairFlag repair = NoRepair, + std::optional ca = std::nullopt + ) = 0; + + /** + * Make a goal corresponding to the `DerivedPath`. + * + * It will be a `DerivationGoal` for a `DerivedPath::Built` or + * a `SubstitutionGoal` for a `DerivedPath::Opaque`. + */ + virtual GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0; +}; + +// elaborate hoax to let goals access factory methods while hiding them from the public +class WorkerBase : protected GoalFactory +{ + friend struct DerivationGoal; + friend struct PathSubstitutionGoal; + friend class DrvOutputSubstitutionGoal; + +protected: + GoalFactory & goalFactory() { return *this; } +}; + /** * The worker class. */ -class Worker +class Worker : public WorkerBase { private: @@ -215,19 +262,18 @@ private: std::shared_ptr makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, std::function()> mkDrvGoal); -public: std::shared_ptr makeDerivationGoal( const StorePath & drvPath, - const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); + const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override; std::shared_ptr makeBasicDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, - const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); + const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override; /** * @ref SubstitutionGoal "substitution goal" */ - std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt) override; + std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt) override; /** * Make a goal corresponding to the `DerivedPath`. @@ -235,8 +281,9 @@ public: * It will be a `DerivationGoal` for a `DerivedPath::Built` or * a `SubstitutionGoal` for a `DerivedPath::Opaque`. */ - GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal); + GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override; +public: /** * Unregisters a running child process. */ @@ -245,7 +292,7 @@ public: /** * Loop until the specified top-level goals have finished. */ - void run(const Goals & topGoals); + Goals run(std::function req); /*** * The exit status in case of failure. -- 2.44.1 From e55ec75619b775bf0f5dd60e3da8de9bc8235f68 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sun, 25 Aug 2024 13:41:56 +0200 Subject: [PATCH 016/106] libstore: print dependency errors from DerivationGoal this is not ideal, but it's better than having this stuck in the worker loop itself. setting ex on all failing goals is not problematic because only toplevel goals can ever be observable, all the others are ignored. notably only derivation goals ever set `ex`, substitution goals do not. Change-Id: I02e2164487b2955df053fef3c8e774d557aa638a --- src/libstore/build/derivation-goal.cc | 4 ++++ src/libstore/build/worker.cc | 8 +------- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3dca7a3c0..a75a674d1 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1544,6 +1544,10 @@ Goal::Finished DerivationGoal::done( fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; } + if (ex && isDependency) { + logError(ex->info()); + } + return Finished{ .result = buildResult.success() ? ecSuccess : ecFailed, .ex = ex ? std::make_shared(std::move(*ex)) : nullptr, diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 5d0cc920a..7336ad50f 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -157,19 +157,13 @@ void Worker::goalFinished(GoalPtr goal, Goal::Finished & f) goal->trace("done"); assert(!goal->exitCode.has_value()); goal->exitCode = f.result; + goal->ex = f.ex; permanentFailure |= f.permanentFailure; timedOut |= f.timedOut; hashMismatch |= f.hashMismatch; checkMismatch |= f.checkMismatch; - if (f.ex) { - if (!goal->waiters.empty()) - logError(f.ex->info()); - else - goal->ex = f.ex; - } - for (auto & i : goal->waiters) { if (GoalPtr waiting = i.lock()) { assert(waiting->waitees.count(goal)); -- 2.44.1 From c2b90d235fb5dd721898d8d41d73a51607654890 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Fri, 30 Aug 2024 19:01:30 +0200 Subject: [PATCH 017/106] libstore: don't ContinueImmediately where we can tail call there's no reason to go through the event loop in these cases. returning ContinueImmediately here is just a very convoluted way of jumping to the state we've just set after unwinding one frame of the stack, which never matters in the cases changed here because there are no live RAII guards. Change-Id: I7c00948c22e3caf35e934c1a14ffd2d40efc5547 --- src/libstore/build/derivation-goal.cc | 4 ++-- src/libstore/build/substitution-goal.cc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index a75a674d1..1dda1b1b4 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -649,7 +649,7 @@ Goal::WorkResult DerivationGoal::inputsRealised(bool inBuildSlot) slot to become available, since we don't need one if there is a build hook. */ state = &DerivationGoal::tryToBuild; - return ContinueImmediately{}; + return tryToBuild(inBuildSlot); } void DerivationGoal::started() @@ -772,7 +772,7 @@ Goal::WorkResult DerivationGoal::tryToBuild(bool inBuildSlot) actLock.reset(); state = &DerivationGoal::tryLocalBuild; - return ContinueImmediately{}; + return tryLocalBuild(inBuildSlot); } Goal::WorkResult DerivationGoal::tryLocalBuild(bool inBuildSlot) { diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 673b3c503..1d24938e5 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -188,7 +188,7 @@ Goal::WorkResult PathSubstitutionGoal::referencesValid(bool inBuildSlot) assert(worker.store.isValidPath(i)); state = &PathSubstitutionGoal::tryToRun; - return ContinueImmediately{}; + return tryToRun(inBuildSlot); } @@ -256,7 +256,7 @@ Goal::WorkResult PathSubstitutionGoal::finished(bool inBuildSlot) /* Try the next substitute. */ state = &PathSubstitutionGoal::tryNext; - return ContinueImmediately{}; + return tryNext(inBuildSlot); } worker.markContentsGood(storePath); -- 2.44.1 From e0fd0ba211b38827627218424f92b7d0e059a626 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Fri, 30 Aug 2024 19:01:30 +0200 Subject: [PATCH 018/106] libstore: use notifications for stats counters updating statistics *immediately* when any counter changes declutters things somewhat and makes useful status reports less dependent on the current worker main loop. using callbacks will make it easier to move the worker loop into kj entirely, using only promises for scheduling. Change-Id: I695dfa83111b1ec09b1a54cff268f3c1d7743ed6 --- src/libstore/build/derivation-goal.cc | 6 +- src/libstore/build/derivation-goal.hh | 3 +- .../build/drv-output-substitution-goal.cc | 3 +- .../build/drv-output-substitution-goal.hh | 3 +- src/libstore/build/substitution-goal.cc | 17 ++-- src/libstore/build/substitution-goal.hh | 3 +- src/libstore/build/worker.cc | 36 ++++--- src/libstore/build/worker.hh | 40 +++++--- src/libutil/meson.build | 1 + src/libutil/notifying-counter.hh | 99 +++++++++++++++++++ 10 files changed, 169 insertions(+), 42 deletions(-) create mode 100644 src/libutil/notifying-counter.hh diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 1dda1b1b4..c288003b4 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -71,7 +71,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); trace("created"); - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); + mcExpectedBuilds = worker.expectedBuilds.addTemporarily(1); } @@ -91,7 +91,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store)); trace("created"); - mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); + mcExpectedBuilds = worker.expectedBuilds.addTemporarily(1); /* Prevent the .chroot directory from being garbage-collected. (See isActiveTempFile() in gc.cc.) */ @@ -662,7 +662,7 @@ void DerivationGoal::started() if (hook) msg += fmt(" on '%s'", machineName); act = std::make_unique(*logger, lvlInfo, actBuild, msg, Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", 1, 1}); - mcRunningBuilds = std::make_unique>(worker.runningBuilds); + mcRunningBuilds = worker.runningBuilds.addTemporarily(1); } Goal::WorkResult DerivationGoal::tryToBuild(bool inBuildSlot) diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 77f9fef4b..bf4a3da93 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "notifying-counter.hh" #include "parsed-derivations.hh" #include "lock.hh" #include "outputs-spec.hh" @@ -217,7 +218,7 @@ struct DerivationGoal : public Goal BuildMode buildMode; - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; + NotifyingCounter::Bump mcExpectedBuilds, mcRunningBuilds; std::unique_ptr act; diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 9acff14b0..369b2dd90 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -42,8 +42,7 @@ Goal::WorkResult DrvOutputSubstitutionGoal::tryNext(bool inBuildSlot) return WaitForSlot{}; } - maintainRunningSubstitutions = - std::make_unique>(worker.runningSubstitutions); + maintainRunningSubstitutions = worker.runningSubstitutions.addTemporarily(1); if (subs.size() == 0) { /* None left. Terminate this goal and let someone else deal diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index b48c4670b..8de4d45dd 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "notifying-counter.hh" #include "store-api.hh" #include "goal.hh" #include "realisation.hh" @@ -40,7 +41,7 @@ class DrvOutputSubstitutionGoal : public Goal { */ std::shared_ptr sub; - std::unique_ptr> maintainRunningSubstitutions; + NotifyingCounter::Bump maintainRunningSubstitutions; struct DownloadState { diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 1d24938e5..a798cbde2 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -21,7 +21,7 @@ PathSubstitutionGoal::PathSubstitutionGoal( state = &PathSubstitutionGoal::init; name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); trace("created"); - maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); + maintainExpectedSubstitutions = worker.expectedSubstitutions.addTemporarily(1); } @@ -139,11 +139,11 @@ Goal::WorkResult PathSubstitutionGoal::tryNext(bool inBuildSlot) /* Update the total expected download size. */ auto narInfo = std::dynamic_pointer_cast(info); - maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); + maintainExpectedNar = worker.expectedNarSize.addTemporarily(info->narSize); maintainExpectedDownload = narInfo && narInfo->fileSize - ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) + ? worker.expectedDownloadSize.addTemporarily(narInfo->fileSize) : nullptr; /* Bail out early if this substituter lacks a valid @@ -200,7 +200,7 @@ Goal::WorkResult PathSubstitutionGoal::tryToRun(bool inBuildSlot) return WaitForSlot{}; } - maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); + maintainRunningSubstitutions = worker.runningSubstitutions.addTemporarily(1); outPipe.create(); @@ -268,13 +268,10 @@ Goal::WorkResult PathSubstitutionGoal::finished(bool inBuildSlot) maintainExpectedSubstitutions.reset(); worker.doneSubstitutions++; - if (maintainExpectedDownload) { - auto fileSize = maintainExpectedDownload->delta; - maintainExpectedDownload.reset(); - worker.doneDownloadSize += fileSize; - } + worker.doneDownloadSize += maintainExpectedDownload.delta(); + maintainExpectedDownload.reset(); - worker.doneNarSize += maintainExpectedNar->delta; + worker.doneNarSize += maintainExpectedNar.delta(); maintainExpectedNar.reset(); return done(ecSuccess, BuildResult::Substituted); diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 5d58b34a0..9c7e6f470 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -2,6 +2,7 @@ ///@file #include "lock.hh" +#include "notifying-counter.hh" #include "store-api.hh" #include "goal.hh" @@ -63,7 +64,7 @@ struct PathSubstitutionGoal : public Goal */ Path destPath; - std::unique_ptr> maintainExpectedSubstitutions, + NotifyingCounter::Bump maintainExpectedSubstitutions, maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; typedef WorkResult (PathSubstitutionGoal::*GoalState)(bool inBuildSlot); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 7336ad50f..24c700396 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -320,6 +320,27 @@ void Worker::waitForAWhile(GoalPtr goal) } +void Worker::updateStatistics() +{ + // only update progress info while running. this notably excludes updating + // progress info while destroying, which causes the progress bar to assert + if (running && statisticsOutdated) { + actDerivations.progress( + doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds + ); + actSubstitutions.progress( + doneSubstitutions, + expectedSubstitutions + doneSubstitutions, + runningSubstitutions, + failedSubstitutions + ); + act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); + act.setExpected(actCopyPath, expectedNarSize + doneNarSize); + + statisticsOutdated = false; + } +} + Goals Worker::run(std::function req) { auto _topGoals = req(goalFactory()); @@ -329,6 +350,8 @@ Goals Worker::run(std::function req) running = true; Finally const _stop([&] { running = false; }); + updateStatistics(); + for (auto & i : _topGoals) { topGoals.insert(i); if (auto goal = dynamic_cast(i.get())) { @@ -373,18 +396,7 @@ Goals Worker::run(std::function req) ? nrSubstitutions < std::max(1U, (unsigned int) settings.maxSubstitutionJobs) : nrLocalBuilds < settings.maxBuildJobs; handleWorkResult(goal, goal->work(inSlot)); - - actDerivations.progress( - doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds - ); - actSubstitutions.progress( - doneSubstitutions, - expectedSubstitutions + doneSubstitutions, - runningSubstitutions, - failedSubstitutions - ); - act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); - act.setExpected(actCopyPath, expectedNarSize + doneNarSize); + updateStatistics(); if (topGoals.empty()) break; // stuff may have been cancelled } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 3fbf457fe..9a6ed8449 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "notifying-counter.hh" #include "types.hh" #include "lock.hh" #include "store-api.hh" @@ -213,6 +214,21 @@ private: void childStarted(GoalPtr goal, const std::set & fds, bool inBuildSlot); + /** + * Pass current stats counters to the logger for progress bar updates. + */ + void updateStatistics(); + + bool statisticsOutdated = true; + + /** + * Mark statistics as outdated, such that `updateStatistics` will be called. + */ + void updateStatisticsLater() + { + statisticsOutdated = true; + } + public: const Activity act; @@ -234,19 +250,19 @@ public: HookState hook; - uint64_t expectedBuilds = 0; - uint64_t doneBuilds = 0; - uint64_t failedBuilds = 0; - uint64_t runningBuilds = 0; + NotifyingCounter expectedBuilds{[this] { updateStatisticsLater(); }}; + NotifyingCounter doneBuilds{[this] { updateStatisticsLater(); }}; + NotifyingCounter failedBuilds{[this] { updateStatisticsLater(); }}; + NotifyingCounter runningBuilds{[this] { updateStatisticsLater(); }}; - uint64_t expectedSubstitutions = 0; - uint64_t doneSubstitutions = 0; - uint64_t failedSubstitutions = 0; - uint64_t runningSubstitutions = 0; - uint64_t expectedDownloadSize = 0; - uint64_t doneDownloadSize = 0; - uint64_t expectedNarSize = 0; - uint64_t doneNarSize = 0; + NotifyingCounter expectedSubstitutions{[this] { updateStatisticsLater(); }}; + NotifyingCounter doneSubstitutions{[this] { updateStatisticsLater(); }}; + NotifyingCounter failedSubstitutions{[this] { updateStatisticsLater(); }}; + NotifyingCounter runningSubstitutions{[this] { updateStatisticsLater(); }}; + NotifyingCounter expectedDownloadSize{[this] { updateStatisticsLater(); }}; + NotifyingCounter doneDownloadSize{[this] { updateStatisticsLater(); }}; + NotifyingCounter expectedNarSize{[this] { updateStatisticsLater(); }}; + NotifyingCounter doneNarSize{[this] { updateStatisticsLater(); }}; Worker(Store & store, Store & evalStore); ~Worker(); diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 6566f7f46..1ac31c7eb 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -95,6 +95,7 @@ libutil_headers = files( 'monitor-fd.hh', 'mount.hh', 'namespaces.hh', + 'notifying-counter.hh', 'pool.hh', 'position.hh', 'print-elided.hh', diff --git a/src/libutil/notifying-counter.hh b/src/libutil/notifying-counter.hh new file mode 100644 index 000000000..dc58aac91 --- /dev/null +++ b/src/libutil/notifying-counter.hh @@ -0,0 +1,99 @@ +#pragma once +/// @file + +#include +#include +#include + +namespace nix { + +template +class NotifyingCounter +{ +private: + T counter; + std::function notify; + +public: + class Bump + { + friend class NotifyingCounter; + + struct SubOnFree + { + T delta; + + void operator()(NotifyingCounter * c) const + { + c->add(-delta); + } + }; + + // lightly misuse unique_ptr to get RAII types with destructor callbacks + std::unique_ptr, SubOnFree> at; + + Bump(NotifyingCounter & at, T delta) : at(&at, {delta}) {} + + public: + Bump() = default; + Bump(decltype(nullptr)) {} + + T delta() const + { + return at ? at.get_deleter().delta : 0; + } + + void reset() + { + at.reset(); + } + }; + + explicit NotifyingCounter(std::function notify, T initial = 0) + : counter(initial) + , notify(std::move(notify)) + { + assert(this->notify); + } + + // bumps hold pointers to this, so we should neither copy nor move. + NotifyingCounter(const NotifyingCounter &) = delete; + NotifyingCounter & operator=(const NotifyingCounter &) = delete; + NotifyingCounter(NotifyingCounter &&) = delete; + NotifyingCounter & operator=(NotifyingCounter &&) = delete; + + T get() const + { + return counter; + } + + operator T() const + { + return counter; + } + + void add(T delta) + { + counter += delta; + notify(); + } + + NotifyingCounter & operator+=(T delta) + { + add(delta); + return *this; + } + + NotifyingCounter & operator++(int) + { + return *this += 1; + } + + Bump addTemporarily(T delta) + { + add(delta); + return Bump{*this, delta}; + } +}; + +} -- 2.44.1 From dc0cace604d71d274654525e90d0afed9aa36dc9 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Fri, 30 Aug 2024 19:01:30 +0200 Subject: [PATCH 019/106] libstore: remove queryMissing call from Worker it doesn't have a purpose except cache priming, which is largely irrelevant by default (since another code path already runs this exact query). our store implementations do not benefit that much from this either, and the more bursty load may indeed harm them. Change-Id: I1cc12f8c21cede42524317736d5987f1e43fc9c9 --- src/libstore/build/worker.cc | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 24c700396..0642754e8 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -344,7 +344,6 @@ void Worker::updateStatistics() Goals Worker::run(std::function req) { auto _topGoals = req(goalFactory()); - std::vector topPaths; assert(!running); running = true; @@ -352,22 +351,7 @@ Goals Worker::run(std::function req) updateStatistics(); - for (auto & i : _topGoals) { - topGoals.insert(i); - if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(goal->drvPath), - .outputs = goal->wantedOutputs, - }); - } else if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(DerivedPath::Opaque{goal->storePath}); - } - } - - /* Call queryMissing() to efficiently query substitutes. */ - StorePathSet willBuild, willSubstitute, unknown; - uint64_t downloadSize, narSize; - store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize); + topGoals = _topGoals; debug("entered goal loop"); -- 2.44.1 From a385c5935aa7cfec4de47cfe5620bca2e0e0e55d Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Fri, 30 Aug 2024 19:01:30 +0200 Subject: [PATCH 020/106] libstore: rename Goal::Finished::result to exitCode the more useful type for `result` is BuildResult. Change-Id: If93d9384e8d686eb63b33320f1d565f9b9afbf3a --- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/build/goal.hh | 2 +- src/libstore/build/worker.cc | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index c288003b4..3011d5c6f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1549,7 +1549,7 @@ Goal::Finished DerivationGoal::done( } return Finished{ - .result = buildResult.success() ? ecSuccess : ecFailed, + .exitCode = buildResult.success() ? ecSuccess : ecFailed, .ex = ex ? std::make_shared(std::move(*ex)) : nullptr, .permanentFailure = buildResult.status == BuildResult::PermanentFailure, .timedOut = buildResult.status == BuildResult::TimedOut, diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 1f25fb233..9d9ab53ca 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -124,7 +124,7 @@ public: bool inBuildSlot; }; struct [[nodiscard]] Finished { - ExitCode result; + ExitCode exitCode; std::shared_ptr ex; bool permanentFailure = false; bool timedOut = false; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 0642754e8..f619d574d 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -156,7 +156,7 @@ void Worker::goalFinished(GoalPtr goal, Goal::Finished & f) { goal->trace("done"); assert(!goal->exitCode.has_value()); - goal->exitCode = f.result; + goal->exitCode = f.exitCode; goal->ex = f.ex; permanentFailure |= f.permanentFailure; @@ -171,11 +171,11 @@ void Worker::goalFinished(GoalPtr goal, Goal::Finished & f) waiting->trace(fmt("waitee '%s' done; %d left", goal->name, waiting->waitees.size())); - if (f.result != Goal::ecSuccess) ++waiting->nrFailed; - if (f.result == Goal::ecNoSubstituters) ++waiting->nrNoSubstituters; - if (f.result == Goal::ecIncompleteClosure) ++waiting->nrIncompleteClosure; + if (f.exitCode != Goal::ecSuccess) ++waiting->nrFailed; + if (f.exitCode == Goal::ecNoSubstituters) ++waiting->nrNoSubstituters; + if (f.exitCode == Goal::ecIncompleteClosure) ++waiting->nrIncompleteClosure; - if (waiting->waitees.empty() || (f.result == Goal::ecFailed && !settings.keepGoing)) { + if (waiting->waitees.empty() || (f.exitCode == Goal::ecFailed && !settings.keepGoing)) { /* If we failed and keepGoing is not set, we remove all remaining waitees. */ for (auto & i : waiting->waitees) { -- 2.44.1 From d75df91f74b6819f674f0733143fdf32580af183 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Fri, 30 Aug 2024 19:01:30 +0200 Subject: [PATCH 021/106] libstore: add build result to Goal::Finished it just makes sense to have it too, rather than just the pass/fail information we keep so far. once we turn goals into something more promise-shaped it'll also help detangle the current data flow mess Change-Id: I915cf04d177cad849ea7a5833215d795326f1946 --- src/libstore/build/derivation-goal.cc | 1 + src/libstore/build/drv-output-substitution-goal.cc | 10 ++++++---- src/libstore/build/goal.hh | 1 + src/libstore/build/substitution-goal.cc | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3011d5c6f..b59033bae 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1550,6 +1550,7 @@ Goal::Finished DerivationGoal::done( return Finished{ .exitCode = buildResult.success() ? ecSuccess : ecFailed, + .result = buildResult, .ex = ex ? std::make_shared(std::move(*ex)) : nullptr, .permanentFailure = buildResult.status == BuildResult::PermanentFailure, .timedOut = buildResult.status == BuildResult::TimedOut, diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 369b2dd90..d7a7d257c 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -1,4 +1,5 @@ #include "drv-output-substitution-goal.hh" +#include "build-result.hh" #include "finally.hh" #include "worker.hh" #include "substitution-goal.hh" @@ -27,7 +28,7 @@ Goal::WorkResult DrvOutputSubstitutionGoal::init(bool inBuildSlot) /* If the derivation already exists, we’re done */ if (worker.store.queryRealisation(id)) { - return Finished{ecSuccess}; + return Finished{ecSuccess, std::move(buildResult)}; } subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); @@ -56,7 +57,7 @@ Goal::WorkResult DrvOutputSubstitutionGoal::tryNext(bool inBuildSlot) /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - return Finished{substituterFailed ? ecFailed : ecNoSubstituters}; + return Finished{substituterFailed ? ecFailed : ecNoSubstituters, std::move(buildResult)}; } sub = subs.front(); @@ -133,7 +134,8 @@ Goal::WorkResult DrvOutputSubstitutionGoal::outPathValid(bool inBuildSlot) if (nrFailed > 0) { debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); return Finished{ - nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed + nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, + std::move(buildResult), }; } @@ -144,7 +146,7 @@ Goal::WorkResult DrvOutputSubstitutionGoal::outPathValid(bool inBuildSlot) Goal::WorkResult DrvOutputSubstitutionGoal::finished() { trace("finished"); - return Finished{ecSuccess}; + return Finished{ecSuccess, std::move(buildResult)}; } std::string DrvOutputSubstitutionGoal::key() diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 9d9ab53ca..502ba2a7d 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -125,6 +125,7 @@ public: }; struct [[nodiscard]] Finished { ExitCode exitCode; + BuildResult result; std::shared_ptr ex; bool permanentFailure = false; bool timedOut = false; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index a798cbde2..33715ea6b 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -41,7 +41,7 @@ Goal::Finished PathSubstitutionGoal::done( debug(*errorMsg); buildResult.errorMsg = *errorMsg; } - return Finished{result}; + return Finished{result, std::move(buildResult)}; } -- 2.44.1 From 9d8f433246088baeb878050f3022a6a738db526c Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Sun, 1 Sep 2024 15:10:31 -0700 Subject: [PATCH 022/106] Expand comment on `std::string operator+` Nuts! Change-Id: Ib5bc0606d7c86e57ef76dd7bcc89dce91bd3d50a --- src/libutil/strings.hh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh index ebafab9ad..782807b61 100644 --- a/src/libutil/strings.hh +++ b/src/libutil/strings.hh @@ -200,8 +200,18 @@ std::string showBytes(uint64_t bytes); /** - * Provide an addition operator between strings and string_views + * Provide an addition operator between `std::string` and `std::string_view` * inexplicably omitted from the standard library. + * + * > The reason for this is given in n3512 string_ref: a non-owning reference + * to a string, revision 2 by Jeffrey Yasskin: + * > + * > > I also omitted operator+(basic_string, basic_string_ref) because LLVM + * > > returns a lightweight object from this overload and only performs the + * > > concatenation lazily. If we define this overload, we'll have a hard time + * > > introducing that lightweight concatenation later. + * + * See: https://stackoverflow.com/a/47735624 */ inline std::string operator + (const std::string & s1, std::string_view s2) { -- 2.44.1 From b7b1b9723f7f0a39b47000c18eb4b89252d02c71 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Sun, 1 Sep 2024 15:20:09 -0700 Subject: [PATCH 023/106] Clarify that `diff-hook` no longer needs to be an absolute path See: https://gerrit.lix.systems/c/lix/+/1864 Change-Id: Ic70bfe42b261a83f2cb68b8f102833b739b8e03a --- src/libstore/globals.hh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 51550b2c3..94aa60825 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -637,10 +637,10 @@ public: PathsSetting> diffHook{ this, std::nullopt, "diff-hook", R"( - Absolute path to an executable capable of diffing build - results. The hook is executed if `run-diff-hook` is true, and the - output of a build is known to not be the same. This program is not - executed to determine if two results are the same. + Path to an executable capable of diffing build results. The hook is + executed if `run-diff-hook` is true, and the output of a build is + known to not be the same. This program is not executed to determine + if two results are the same. The diff hook is executed by the same user and group who ran the build. However, the diff hook does not have write access to the -- 2.44.1 From fc4a160878986ac412c3991f55eab5bc139ce698 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Sun, 1 Sep 2024 15:28:59 -0700 Subject: [PATCH 024/106] repl-overlays: Provide an elaborate example This is the repl overlay from my dotfiles, which I think provides a reasonable and ergonomic set of variables. We can iterate on this over time, or (perhaps?) provide a sentinel value like `repl-overlays = ` to include a "suggested default" overlay like this one. Change-Id: I8eba3934c50fbac8367111103e66c7375b8d134e --- src/libexpr/eval-settings.hh | 48 ++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 444e298a0..f7ad2d786 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -185,6 +185,54 @@ struct EvalSettings : Config else { } ``` + + Here's a more elaborate `repl-overlay`, which provides the following + variables: + - The original, unmodified variables are aliased to `original`. + - `legacyPackages.${system}` (if it exists) or `packages.${system}` + (otherwise) is aliased to `pkgs`. + - All attribute set variables with a `${system}` attribute are + abbreviated in the same manner; e.g. `devShells.${system}` is + shortened to `devShells`. + + For example, the following attribute set: + + ```nix + info: final: attrs: let + # Equivalent to nixpkgs `lib.optionalAttrs`. + optionalAttrs = predicate: attrs: + if predicate + then attrs + else {}; + + # If `attrs.${oldName}.${info.currentSystem}` exists, alias `${newName}` to + # it. + collapseRenamed = oldName: newName: + optionalAttrs (builtins.hasAttr oldName attrs + && builtins.hasAttr info.currentSystem attrs.${oldName}) + { + ${newName} = attrs.${oldName}.${info.currentSystem}; + }; + + # Alias `attrs.${oldName}.${info.currentSystem} to `${newName}`. + collapse = name: collapseRenamed name name; + + # Alias all `attrs` keys with an `${info.currentSystem}` attribute. + collapseAll = + builtins.foldl' + (prev: name: prev // collapse name) + {} + (builtins.attrNames attrs); + in + # Preserve the original bindings as `original`. + (optionalAttrs (! attrs ? original) + { + original = attrs; + }) + // (collapseRenamed "packages" "pkgs") + // (collapseRenamed "legacyPackages" "pkgs") + // collapseAll + ``` )"}; }; -- 2.44.1 From 75c0de3e3cbd972328c7d0c6fdea79abbfa204df Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Mon, 26 Aug 2024 09:28:14 -0700 Subject: [PATCH 025/106] Test including relative paths in configuration Change-Id: If6c69a5e16d1ccd223fba392890f08f0032fb754 --- .../libutil-support/tests/characterization.hh | 9 +---- tests/unit/libutil-support/tests/test-data.cc | 16 +++++++++ .../tests/test-data.hh | 13 +++----- tests/unit/libutil/config.cc | 33 +++++++++++++++++++ tests/unit/libutil/data/puppy.conf | 1 + tests/unit/meson.build | 2 +- 6 files changed, 56 insertions(+), 18 deletions(-) create mode 100644 tests/unit/libutil-support/tests/test-data.cc rename tests/unit/{libstore-support => libutil-support}/tests/test-data.hh (53%) create mode 100644 tests/unit/libutil/data/puppy.conf diff --git a/tests/unit/libutil-support/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh index db82476da..2bf606bd8 100644 --- a/tests/unit/libutil-support/tests/characterization.hh +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -9,17 +9,10 @@ #include #include "types.hh" +#include "test-data.hh" namespace nix { -/** - * The path to the unit test data directory. See the contributing guide - * in the manual for further details. - */ -static Path getUnitTestData() { - return getEnv("_NIX_TEST_UNIT_DATA").value(); -} - /** * Whether we should update "golden masters" instead of running tests * against them. See the contributing guide in the manual for further diff --git a/tests/unit/libutil-support/tests/test-data.cc b/tests/unit/libutil-support/tests/test-data.cc new file mode 100644 index 000000000..016bd5e78 --- /dev/null +++ b/tests/unit/libutil-support/tests/test-data.cc @@ -0,0 +1,16 @@ +#include "test-data.hh" +#include "strings.hh" + +namespace nix { + +Path getUnitTestData() +{ + return getEnv("_NIX_TEST_UNIT_DATA").value(); +} + +Path getUnitTestDataPath(std::string_view path) +{ + return absPath(getUnitTestData() + "/" + path); +} + +} diff --git a/tests/unit/libstore-support/tests/test-data.hh b/tests/unit/libutil-support/tests/test-data.hh similarity index 53% rename from tests/unit/libstore-support/tests/test-data.hh rename to tests/unit/libutil-support/tests/test-data.hh index 1fec6f912..794ec6265 100644 --- a/tests/unit/libstore-support/tests/test-data.hh +++ b/tests/unit/libutil-support/tests/test-data.hh @@ -1,25 +1,20 @@ #pragma once -#include "environment-variables.hh" #include "types.hh" +#include "environment-variables.hh" +#include "file-system.hh" namespace nix { -// TODO: These helpers should be available in all unit tests. - /** * The path to the unit test data directory. See the contributing guide * in the manual for further details. */ -static Path getUnitTestData() { - return getEnv("_NIX_TEST_UNIT_DATA").value(); -} +Path getUnitTestData(); /** * Resolve a path under the unit test data directory to an absolute path. */ -static Path getUnitTestDataPath(std::string_view path) { - return absPath(getUnitTestData() + "/" + path); -} +Path getUnitTestDataPath(std::string_view path); } diff --git a/tests/unit/libutil/config.cc b/tests/unit/libutil/config.cc index 1629969ba..1600f4ff0 100644 --- a/tests/unit/libutil/config.cc +++ b/tests/unit/libutil/config.cc @@ -1,5 +1,9 @@ #include "config.hh" #include "args.hh" +#include "file-system.hh" +#include "environment-variables.hh" +#include "logging.hh" +#include "tests/test-data.hh" #include #include @@ -287,6 +291,35 @@ namespace nix { ), Error); } + TEST(Config, includeRelativePath) { + Config config; + Setting setting{&config, "", "puppy", "description"}; + + config.applyConfig("include puppy.conf", { + .path = getUnitTestDataPath("nix.conf") + }); + + std::map settings; + config.getSettings(settings); + ASSERT_FALSE(settings.empty()); + ASSERT_EQ(settings["puppy"].value, "doggy"); + } + + TEST(Config, includeTildePath) { + Config config; + Setting setting{&config, "", "puppy", "description"}; + + config.applyConfig("include ~/puppy.conf", { + .path = "/doesnt-exist", + .home = getUnitTestData() + }); + + std::map settings; + config.getSettings(settings); + ASSERT_FALSE(settings.empty()); + ASSERT_EQ(settings["puppy"].value, "doggy"); + } + TEST(Config, applyConfigInvalidThrows) { Config config; ASSERT_THROW(config.applyConfig("value == key"), UsageError); diff --git a/tests/unit/libutil/data/puppy.conf b/tests/unit/libutil/data/puppy.conf new file mode 100644 index 000000000..805e484ef --- /dev/null +++ b/tests/unit/libutil/data/puppy.conf @@ -0,0 +1 @@ +puppy = doggy diff --git a/tests/unit/meson.build b/tests/unit/meson.build index 55c7566bd..8ff0b5ec5 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -19,6 +19,7 @@ libutil_test_support_sources = files( 'libutil-support/tests/cli-literate-parser.cc', 'libutil-support/tests/hash.cc', 'libutil-support/tests/terminal-code-eater.cc', + 'libutil-support/tests/test-data.cc', ) libutil_test_support = library( 'lixutil-test-support', @@ -95,7 +96,6 @@ libstore_test_support_sources = files( 'libstore-support/tests/derived-path.cc', 'libstore-support/tests/outputs-spec.cc', 'libstore-support/tests/path.cc', - 'libstore-support/tests/test-data.hh', ) libstore_test_support = library( -- 2.44.1 From d7c37324bb77f8348a11f7fbf168fc1a085f2505 Mon Sep 17 00:00:00 2001 From: Alois Wohlschlager Date: Sat, 24 Aug 2024 14:39:45 +0200 Subject: [PATCH 026/106] libstore: declare SandboxMode JSON serialisation in the header The JSON serialisation should be declared in the header so that all translation units can see it when needed, even though it seems that it has not been used anywhere else so far. Unfortunately, this means we cannot use the NLOHMANN_JSON_SERIALIZE_ENUM convenience macro, since it uses a slightly different signature, but the code is not too bad either. Change-Id: I6e2851b250e0b53114d2fecb8011ff1ea9379d0f --- src/libstore/globals.cc | 30 +++++++++++++++++++++++++----- src/libstore/globals.hh | 3 +++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index c114e22dc..ffc2543ef 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -269,11 +269,31 @@ Path Settings::getDefaultSSLCertFile() const std::string nixVersion = PACKAGE_VERSION; -NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { - {SandboxMode::smEnabled, true}, - {SandboxMode::smRelaxed, "relaxed"}, - {SandboxMode::smDisabled, false}, -}); +void to_json(nlohmann::json & j, const SandboxMode & e) +{ + if (e == SandboxMode::smEnabled) { + j = true; + } else if (e == SandboxMode::smRelaxed) { + j = "relaxed"; + } else if (e == SandboxMode::smDisabled) { + j = false; + } else { + abort(); + } +} + +void from_json(const nlohmann::json & j, SandboxMode & e) +{ + if (j == true) { + e = SandboxMode::smEnabled; + } else if (j == "relaxed") { + e = SandboxMode::smRelaxed; + } else if (j == false) { + e = SandboxMode::smDisabled; + } else { + throw Error("Invalid sandbox mode '%s'", std::string(j)); + } +} template<> SandboxMode BaseSetting::parse(const std::string & str, const ApplyConfigOptions & options) const { diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 51550b2c3..ca2a917ed 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -14,6 +14,9 @@ namespace nix { typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode; +void to_json(nlohmann::json & j, const SandboxMode & e); +void from_json(const nlohmann::json & j, SandboxMode & e); + struct MaxBuildJobsSetting : public BaseSetting { MaxBuildJobsSetting(Config * options, -- 2.44.1 From 63ee2cdda36e48f932dd5a1ae7f35f85687ce1f5 Mon Sep 17 00:00:00 2001 From: Alois Wohlschlager Date: Tue, 20 Aug 2024 19:36:49 +0200 Subject: [PATCH 027/106] libfetchers: serialise accept-flake-config properly The AcceptFlakeConfig type used was missing its JSON serialisation definition, so it was incorrectly serialised as an integer, ending up that way for example in the nix.conf manual page. Declare a proper serialisation. Change-Id: If8ec210f9d4dd42fe480c4e97d0a4920eb66a01e --- src/libfetchers/fetch-settings.cc | 26 ++++++++++++++++++++++++++ src/libfetchers/fetch-settings.hh | 3 +++ 2 files changed, 29 insertions(+) diff --git a/src/libfetchers/fetch-settings.cc b/src/libfetchers/fetch-settings.cc index 007f2725f..b278835ad 100644 --- a/src/libfetchers/fetch-settings.cc +++ b/src/libfetchers/fetch-settings.cc @@ -7,6 +7,32 @@ namespace nix { +void to_json(nlohmann::json & j, const AcceptFlakeConfig & e) +{ + if (e == AcceptFlakeConfig::False) { + j = false; + } else if (e == AcceptFlakeConfig::Ask) { + j = "ask"; + } else if (e == AcceptFlakeConfig::True) { + j = true; + } else { + abort(); + } +} + +void from_json(const nlohmann::json & j, AcceptFlakeConfig & e) +{ + if (j == false) { + e = AcceptFlakeConfig::False; + } else if (j == "ask") { + e = AcceptFlakeConfig::Ask; + } else if (j == true) { + e = AcceptFlakeConfig::True; + } else { + throw Error("Invalid accept-flake-config value '%s'", std::string(j)); + } +} + template<> AcceptFlakeConfig BaseSetting::parse(const std::string & str, const ApplyConfigOptions & options) const { if (str == "true") return AcceptFlakeConfig::True; diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 93123463c..0bdc707ec 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -13,6 +13,9 @@ namespace nix { enum class AcceptFlakeConfig { False, Ask, True }; +void to_json(nlohmann::json & j, const AcceptFlakeConfig & e); +void from_json(const nlohmann::json & j, AcceptFlakeConfig & e); + struct FetchSettings : public Config { FetchSettings(); -- 2.44.1 From 4715d557ef1c0dd306cde585820a2a00accd51d6 Mon Sep 17 00:00:00 2001 From: Alois Wohlschlager Date: Sat, 7 Sep 2024 10:37:12 +0200 Subject: [PATCH 028/106] libmain/progress-bar: erase all lines of the multi-line format When the multi-line log format is enabled, the progress bar usually occupies multiple lines on the screen. When stopping the progress bar, only the last line was wiped, leaving all others visible on the screen. Erase all lines belonging to the progress bar to prevent these leftovers. Asking the user for input is theoretically affected by a similar issue, but this is not observed in practice since the only place where the user is asked (whether configuration options coming from flakes should be accepted) does not actually have multiple lines on the progress bar. However, there is no real reason to not fix this either, so let's do it anyway. Change-Id: Iaa5a701874fca32e6f06d85912835d86b8fa7a16 --- src/libmain/progress-bar.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index cdb15d8c7..68349ad26 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -92,7 +92,7 @@ void ProgressBar::resume() nextWakeup = draw(*state, {}); state.wait_for(quitCV, std::chrono::milliseconds(50)); } - writeLogsToStderr("\r\e[K"); + eraseProgressDisplay(*state); }); } @@ -558,7 +558,8 @@ std::optional ProgressBar::ask(std::string_view msg) { auto state(state_.lock()); if (state->paused > 0 || !isatty(STDIN_FILENO)) return {}; - std::cerr << fmt("\r\e[K%s ", msg); + eraseProgressDisplay(*state); + std::cerr << msg; auto s = trim(readLine(STDIN_FILENO)); if (s.size() != 1) return {}; draw(*state, {}); -- 2.44.1 From 92eccfbd68f80e4a5d87556d43927d6dbaabae26 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sun, 1 Sep 2024 01:37:10 +0200 Subject: [PATCH 029/106] libutil: add a result type using boost outcome we're using boost::outcome rather than leaf or stl types because stl types are not available everywhere and leaf does not provide its own storage for error values, relying on thread-locals and the stack. if we want to use promises we won't have a stack and would have to wrap everything into leaf-specific allocating wrappers, so outcome it is. Change-Id: I35111a1f9ed517e7f12a839e2162b1ba6a993f8f --- src/libutil/meson.build | 1 + src/libutil/result.hh | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 src/libutil/result.hh diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 1ac31c7eb..a3f21de59 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -105,6 +105,7 @@ libutil_headers = files( 'regex-combinators.hh', 'regex.hh', 'repair-flag.hh', + 'result.hh', 'serialise.hh', 'shlex.hh', 'signals.hh', diff --git a/src/libutil/result.hh b/src/libutil/result.hh new file mode 100644 index 000000000..b01766fe4 --- /dev/null +++ b/src/libutil/result.hh @@ -0,0 +1,24 @@ +#pragma once +/// @file + +#include +#include +#include +#include + +namespace nix { + +template +using Result = boost::outcome_v2::std_result; + +template +using Outcome = boost::outcome_v2::std_outcome; + +namespace result { + +using boost::outcome_v2::success; +using boost::outcome_v2::failure; + +} + +} -- 2.44.1 From f2a49032a698bd96b37e8df8f02ec403fd0bed0f Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sun, 1 Sep 2024 01:37:10 +0200 Subject: [PATCH 030/106] libstore: turn Worker in a kj event loop user using a proper event loop basis we no longer have to worry about most of the intricacies of poll(), or platform-dependent replacements for it. we may even be able to use the event loop and its promise system for all of our scheduling in the future. we don't do any real async processing yet, this is just preparation to separate the first such change from the huge api design difference with the async framework we chose (kj from capnp): kj::Promise, unlike std::future, doesn't return exceptions unmangled. it instead wraps any non-kj exception into a kj exception, erasing all type information and preserving mostly the what() string in the process. this makes sense in the capnp rpc use case where unrestricted exception types can't be transferred, and since it moves error handling styles closer to a world we'd actually like there's no harm in doing it only here for now Change-Id: I20f888de74d525fb2db36ca30ebba4bcfe9cc838 --- meson.build | 1 + package.nix | 7 + src/libstore/build/derivation-goal.cc | 134 ++++++++++++------ src/libstore/build/derivation-goal.hh | 28 ++-- .../build/drv-output-substitution-goal.cc | 48 ++++--- .../build/drv-output-substitution-goal.hh | 14 +- src/libstore/build/entry-points.cc | 34 +++-- src/libstore/build/goal.hh | 4 +- src/libstore/build/local-derivation-goal.cc | 14 +- src/libstore/build/local-derivation-goal.hh | 2 +- src/libstore/build/substitution-goal.cc | 50 ++++--- src/libstore/build/substitution-goal.hh | 14 +- src/libstore/build/worker.cc | 5 +- src/libstore/build/worker.hh | 4 +- src/libstore/meson.build | 1 + 15 files changed, 225 insertions(+), 135 deletions(-) diff --git a/meson.build b/meson.build index 3a772ac08..b015cc606 100644 --- a/meson.build +++ b/meson.build @@ -229,6 +229,7 @@ configdata += { } boost = dependency('boost', required : true, modules : ['container'], include_type : 'system') +kj = dependency('kj-async', required : true, include_type : 'system') # cpuid only makes sense on x86_64 cpuid_required = is_x64 ? get_option('cpuid') : false diff --git a/package.nix b/package.nix index 73e98bc71..e5ab5eff0 100644 --- a/package.nix +++ b/package.nix @@ -15,6 +15,8 @@ brotli, bzip2, callPackage, + capnproto-lix ? __forDefaults.capnproto-lix, + capnproto, cmake, curl, doxygen, @@ -83,6 +85,9 @@ }); build-release-notes = callPackage ./maintainers/build-release-notes.nix { }; + + # needs explicit c++20 to enable coroutine support + capnproto-lix = capnproto.overrideAttrs { CXXFLAGS = "-std=c++20"; }; }, }: let @@ -220,6 +225,7 @@ stdenv.mkDerivation (finalAttrs: { ninja cmake rustc + capnproto-lix ] ++ [ (lib.getBin lowdown) @@ -260,6 +266,7 @@ stdenv.mkDerivation (finalAttrs: { libsodium toml11 pegtl + capnproto-lix ] ++ lib.optionals hostPlatform.isLinux [ libseccomp diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index b59033bae..827c9f541 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -131,7 +131,7 @@ Goal::Finished DerivationGoal::timedOut(Error && ex) } -Goal::WorkResult DerivationGoal::work(bool inBuildSlot) +kj::Promise> DerivationGoal::work(bool inBuildSlot) noexcept { return (this->*state)(inBuildSlot); } @@ -157,8 +157,8 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) } -Goal::WorkResult DerivationGoal::getDerivation(bool inBuildSlot) -{ +kj::Promise> DerivationGoal::getDerivation(bool inBuildSlot) noexcept +try { trace("init"); /* The first thing to do is to make sure that the derivation @@ -170,16 +170,22 @@ Goal::WorkResult DerivationGoal::getDerivation(bool inBuildSlot) state = &DerivationGoal::loadDerivation; - return WaitForGoals{{worker.goalFactory().makePathSubstitutionGoal(drvPath)}}; + return {WaitForGoals{{worker.goalFactory().makePathSubstitutionGoal(drvPath)}}}; +} catch (...) { + return {std::current_exception()}; } -Goal::WorkResult DerivationGoal::loadDerivation(bool inBuildSlot) -{ +kj::Promise> DerivationGoal::loadDerivation(bool inBuildSlot) noexcept +try { trace("loading derivation"); if (nrFailed != 0) { - return done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); + return {done( + BuildResult::MiscFailure, + {}, + Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)) + )}; } /* `drvPath' should already be a root, but let's be on the safe @@ -202,11 +208,13 @@ Goal::WorkResult DerivationGoal::loadDerivation(bool inBuildSlot) assert(drv); return haveDerivation(inBuildSlot); +} catch (...) { + return {std::current_exception()}; } -Goal::WorkResult DerivationGoal::haveDerivation(bool inBuildSlot) -{ +kj::Promise> DerivationGoal::haveDerivation(bool inBuildSlot) noexcept +try { trace("have derivation"); parsedDrv = std::make_unique(drvPath, *drv); @@ -255,7 +263,7 @@ Goal::WorkResult DerivationGoal::haveDerivation(bool inBuildSlot) /* If they are all valid, then we're done. */ if (allValid && buildMode == bmNormal) { - return done(BuildResult::AlreadyValid, std::move(validOutputs)); + return {done(BuildResult::AlreadyValid, std::move(validOutputs))}; } /* We are first going to try to create the invalid output paths @@ -290,20 +298,29 @@ Goal::WorkResult DerivationGoal::haveDerivation(bool inBuildSlot) return outputsSubstitutionTried(inBuildSlot); } else { state = &DerivationGoal::outputsSubstitutionTried; - return result; + return {std::move(result)}; } +} catch (...) { + return {std::current_exception()}; } -Goal::WorkResult DerivationGoal::outputsSubstitutionTried(bool inBuildSlot) -{ +kj::Promise> DerivationGoal::outputsSubstitutionTried(bool inBuildSlot) noexcept +try { trace("all outputs substituted (maybe)"); assert(drv->type().isPure()); - if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - return done(BuildResult::TransientFailure, {}, - Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", - worker.store.printStorePath(drvPath))); + if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) + { + return {done( + BuildResult::TransientFailure, + {}, + Error( + "some substitutes for the outputs of derivation '%s' failed (usually happens due " + "to networking issues); try '--fallback' to build derivation from source ", + worker.store.printStorePath(drvPath) + ) + )}; } /* If the substitutes form an incomplete closure, then we should @@ -343,7 +360,7 @@ Goal::WorkResult DerivationGoal::outputsSubstitutionTried(bool inBuildSlot) auto [allValid, validOutputs] = checkPathValidity(); if (buildMode == bmNormal && allValid) { - return done(BuildResult::Substituted, std::move(validOutputs)); + return {done(BuildResult::Substituted, std::move(validOutputs))}; } if (buildMode == bmRepair && allValid) { return repairClosure(); @@ -354,13 +371,15 @@ Goal::WorkResult DerivationGoal::outputsSubstitutionTried(bool inBuildSlot) /* Nothing to wait for; tail call */ return gaveUpOnSubstitution(inBuildSlot); +} catch (...) { + return {std::current_exception()}; } /* At least one of the output paths could not be produced using a substitute. So we have to build instead. */ -Goal::WorkResult DerivationGoal::gaveUpOnSubstitution(bool inBuildSlot) -{ +kj::Promise> DerivationGoal::gaveUpOnSubstitution(bool inBuildSlot) noexcept +try { WaitForGoals result; /* At this point we are building all outputs, so if more are wanted there @@ -426,13 +445,15 @@ Goal::WorkResult DerivationGoal::gaveUpOnSubstitution(bool inBuildSlot) return inputsRealised(inBuildSlot); } else { state = &DerivationGoal::inputsRealised; - return result; + return {result}; } +} catch (...) { + return {std::current_exception()}; } -Goal::WorkResult DerivationGoal::repairClosure() -{ +kj::Promise> DerivationGoal::repairClosure() noexcept +try { assert(drv->type().isPure()); /* If we're repairing, we now know that our own outputs are valid. @@ -486,34 +507,44 @@ Goal::WorkResult DerivationGoal::repairClosure() } if (result.goals.empty()) { - return done(BuildResult::AlreadyValid, assertPathValidity()); + return {done(BuildResult::AlreadyValid, assertPathValidity())}; } state = &DerivationGoal::closureRepaired; - return result; + return {result}; +} catch (...) { + return {std::current_exception()}; } -Goal::WorkResult DerivationGoal::closureRepaired(bool inBuildSlot) -{ +kj::Promise> DerivationGoal::closureRepaired(bool inBuildSlot) noexcept +try { trace("closure repaired"); if (nrFailed > 0) throw Error("some paths in the output closure of derivation '%s' could not be repaired", worker.store.printStorePath(drvPath)); - return done(BuildResult::AlreadyValid, assertPathValidity()); + return {done(BuildResult::AlreadyValid, assertPathValidity())}; +} catch (...) { + return {std::current_exception()}; } -Goal::WorkResult DerivationGoal::inputsRealised(bool inBuildSlot) -{ +kj::Promise> DerivationGoal::inputsRealised(bool inBuildSlot) noexcept +try { trace("all inputs realised"); if (nrFailed != 0) { if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - return done(BuildResult::DependencyFailed, {}, Error( + return {done( + BuildResult::DependencyFailed, + {}, + Error( "%s dependencies of derivation '%s' failed to build", - nrFailed, worker.store.printStorePath(drvPath))); + nrFailed, + worker.store.printStorePath(drvPath) + ) + )}; } if (retrySubstitution == RetrySubstitution::YesNeed) { @@ -584,7 +615,7 @@ Goal::WorkResult DerivationGoal::inputsRealised(bool inBuildSlot) pathResolved, wantedOutputs, buildMode); state = &DerivationGoal::resolvedFinished; - return WaitForGoals{{resolvedDrvGoal}}; + return {WaitForGoals{{resolvedDrvGoal}}}; } std::function::ChildNode &)> accumInputPaths; @@ -650,6 +681,8 @@ Goal::WorkResult DerivationGoal::inputsRealised(bool inBuildSlot) build hook. */ state = &DerivationGoal::tryToBuild; return tryToBuild(inBuildSlot); +} catch (...) { + return {std::current_exception()}; } void DerivationGoal::started() @@ -665,8 +698,8 @@ void DerivationGoal::started() mcRunningBuilds = worker.runningBuilds.addTemporarily(1); } -Goal::WorkResult DerivationGoal::tryToBuild(bool inBuildSlot) -{ +kj::Promise> DerivationGoal::tryToBuild(bool inBuildSlot) noexcept +try { trace("trying to build"); /* Obtain locks on all output paths, if the paths are known a priori. @@ -700,7 +733,7 @@ Goal::WorkResult DerivationGoal::tryToBuild(bool inBuildSlot) if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for lock on %s", Magenta(showPaths(lockFiles)))); - return WaitForAWhile{}; + return {WaitForAWhile{}}; } actLock.reset(); @@ -717,7 +750,7 @@ Goal::WorkResult DerivationGoal::tryToBuild(bool inBuildSlot) if (buildMode != bmCheck && allValid) { debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); - return done(BuildResult::AlreadyValid, std::move(validOutputs)); + return {done(BuildResult::AlreadyValid, std::move(validOutputs))}; } /* If any of the outputs already exist but are not valid, delete @@ -765,7 +798,7 @@ Goal::WorkResult DerivationGoal::tryToBuild(bool inBuildSlot) }, hookReply); if (result) { - return std::move(*result); + return {std::move(*result)}; } } @@ -773,13 +806,18 @@ Goal::WorkResult DerivationGoal::tryToBuild(bool inBuildSlot) state = &DerivationGoal::tryLocalBuild; return tryLocalBuild(inBuildSlot); +} catch (...) { + return {std::current_exception()}; } -Goal::WorkResult DerivationGoal::tryLocalBuild(bool inBuildSlot) { +kj::Promise> DerivationGoal::tryLocalBuild(bool inBuildSlot) noexcept +try { throw Error( "unable to build with a primary store that isn't a local store; " "either pass a different '--store' or enable remote builds." "\nhttps://docs.lix.systems/manual/lix/stable/advanced-topics/distributed-builds.html"); +} catch (...) { + return {std::current_exception()}; } @@ -935,8 +973,8 @@ void runPostBuildHook( proc.getStdout()->drainInto(sink); } -Goal::WorkResult DerivationGoal::buildDone(bool inBuildSlot) -{ +kj::Promise> DerivationGoal::buildDone(bool inBuildSlot) noexcept +try { trace("build done"); Finally releaseBuildUser([&](){ this->cleanupHookFinally(); }); @@ -1030,7 +1068,7 @@ Goal::WorkResult DerivationGoal::buildDone(bool inBuildSlot) outputLocks.setDeletion(true); outputLocks.unlock(); - return done(BuildResult::Built, std::move(builtOutputs)); + return {done(BuildResult::Built, std::move(builtOutputs))}; } catch (BuildError & e) { outputLocks.unlock(); @@ -1051,12 +1089,14 @@ Goal::WorkResult DerivationGoal::buildDone(bool inBuildSlot) BuildResult::PermanentFailure; } - return done(st, {}, std::move(e)); + return {done(st, {}, std::move(e))}; } +} catch (...) { + return {std::current_exception()}; } -Goal::WorkResult DerivationGoal::resolvedFinished(bool inBuildSlot) -{ +kj::Promise> DerivationGoal::resolvedFinished(bool inBuildSlot) noexcept +try { trace("resolved derivation finished"); assert(resolvedDrvGoal); @@ -1123,7 +1163,9 @@ Goal::WorkResult DerivationGoal::resolvedFinished(bool inBuildSlot) if (status == BuildResult::AlreadyValid) status = BuildResult::ResolvesToAlreadyValid; - return done(status, std::move(builtOutputs)); + return {done(status, std::move(builtOutputs))}; +} catch (...) { + return {std::current_exception()}; } HookReply DerivationGoal::tryBuildHook(bool inBuildSlot) diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index bf4a3da93..020388d5a 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -213,7 +213,7 @@ struct DerivationGoal : public Goal */ std::optional derivationType; - typedef WorkResult (DerivationGoal::*GoalState)(bool inBuildSlot); + typedef kj::Promise> (DerivationGoal::*GoalState)(bool inBuildSlot) noexcept; GoalState state; BuildMode buildMode; @@ -246,7 +246,7 @@ struct DerivationGoal : public Goal std::string key() override; - WorkResult work(bool inBuildSlot) override; + kj::Promise> work(bool inBuildSlot) noexcept override; /** * Add wanted outputs to an already existing derivation goal. @@ -256,18 +256,18 @@ struct DerivationGoal : public Goal /** * The states. */ - WorkResult getDerivation(bool inBuildSlot); - WorkResult loadDerivation(bool inBuildSlot); - WorkResult haveDerivation(bool inBuildSlot); - WorkResult outputsSubstitutionTried(bool inBuildSlot); - WorkResult gaveUpOnSubstitution(bool inBuildSlot); - WorkResult closureRepaired(bool inBuildSlot); - WorkResult inputsRealised(bool inBuildSlot); - WorkResult tryToBuild(bool inBuildSlot); - virtual WorkResult tryLocalBuild(bool inBuildSlot); - WorkResult buildDone(bool inBuildSlot); + kj::Promise> getDerivation(bool inBuildSlot) noexcept; + kj::Promise> loadDerivation(bool inBuildSlot) noexcept; + kj::Promise> haveDerivation(bool inBuildSlot) noexcept; + kj::Promise> outputsSubstitutionTried(bool inBuildSlot) noexcept; + kj::Promise> gaveUpOnSubstitution(bool inBuildSlot) noexcept; + kj::Promise> closureRepaired(bool inBuildSlot) noexcept; + kj::Promise> inputsRealised(bool inBuildSlot) noexcept; + kj::Promise> tryToBuild(bool inBuildSlot) noexcept; + virtual kj::Promise> tryLocalBuild(bool inBuildSlot) noexcept; + kj::Promise> buildDone(bool inBuildSlot) noexcept; - WorkResult resolvedFinished(bool inBuildSlot); + kj::Promise> resolvedFinished(bool inBuildSlot) noexcept; /** * Is the build hook willing to perform the build? @@ -346,7 +346,7 @@ struct DerivationGoal : public Goal */ virtual void killChild(); - WorkResult repairClosure(); + kj::Promise> repairClosure() noexcept; void started(); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index d7a7d257c..7986123cc 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -22,25 +22,27 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal( } -Goal::WorkResult DrvOutputSubstitutionGoal::init(bool inBuildSlot) -{ +kj::Promise> DrvOutputSubstitutionGoal::init(bool inBuildSlot) noexcept +try { trace("init"); /* If the derivation already exists, we’re done */ if (worker.store.queryRealisation(id)) { - return Finished{ecSuccess, std::move(buildResult)}; + return {Finished{ecSuccess, std::move(buildResult)}}; } subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); return tryNext(inBuildSlot); +} catch (...) { + return {std::current_exception()}; } -Goal::WorkResult DrvOutputSubstitutionGoal::tryNext(bool inBuildSlot) -{ +kj::Promise> DrvOutputSubstitutionGoal::tryNext(bool inBuildSlot) noexcept +try { trace("trying next substituter"); if (!inBuildSlot) { - return WaitForSlot{}; + return {WaitForSlot{}}; } maintainRunningSubstitutions = worker.runningSubstitutions.addTemporarily(1); @@ -57,7 +59,7 @@ Goal::WorkResult DrvOutputSubstitutionGoal::tryNext(bool inBuildSlot) /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - return Finished{substituterFailed ? ecFailed : ecNoSubstituters, std::move(buildResult)}; + return {Finished{substituterFailed ? ecFailed : ecNoSubstituters, std::move(buildResult)}}; } sub = subs.front(); @@ -77,11 +79,13 @@ Goal::WorkResult DrvOutputSubstitutionGoal::tryNext(bool inBuildSlot) }); state = &DrvOutputSubstitutionGoal::realisationFetched; - return WaitForWorld{{downloadState->outPipe.readSide.get()}, true}; + return {WaitForWorld{{downloadState->outPipe.readSide.get()}, true}}; +} catch (...) { + return {std::current_exception()}; } -Goal::WorkResult DrvOutputSubstitutionGoal::realisationFetched(bool inBuildSlot) -{ +kj::Promise> DrvOutputSubstitutionGoal::realisationFetched(bool inBuildSlot) noexcept +try { worker.childTerminated(this); maintainRunningSubstitutions.reset(); @@ -122,31 +126,37 @@ Goal::WorkResult DrvOutputSubstitutionGoal::realisationFetched(bool inBuildSlot) return outPathValid(inBuildSlot); } else { state = &DrvOutputSubstitutionGoal::outPathValid; - return result; + return {std::move(result)}; } +} catch (...) { + return {std::current_exception()}; } -Goal::WorkResult DrvOutputSubstitutionGoal::outPathValid(bool inBuildSlot) -{ +kj::Promise> DrvOutputSubstitutionGoal::outPathValid(bool inBuildSlot) noexcept +try { assert(outputInfo); trace("output path substituted"); if (nrFailed > 0) { debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); - return Finished{ + return {Finished{ nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, std::move(buildResult), - }; + }}; } worker.store.registerDrvOutput(*outputInfo); return finished(); +} catch (...) { + return {std::current_exception()}; } -Goal::WorkResult DrvOutputSubstitutionGoal::finished() -{ +kj::Promise> DrvOutputSubstitutionGoal::finished() noexcept +try { trace("finished"); - return Finished{ecSuccess, std::move(buildResult)}; + return {Finished{ecSuccess, std::move(buildResult)}}; +} catch (...) { + return {std::current_exception()}; } std::string DrvOutputSubstitutionGoal::key() @@ -156,7 +166,7 @@ std::string DrvOutputSubstitutionGoal::key() return "a$" + std::string(id.to_string()); } -Goal::WorkResult DrvOutputSubstitutionGoal::work(bool inBuildSlot) +kj::Promise> DrvOutputSubstitutionGoal::work(bool inBuildSlot) noexcept { return (this->*state)(inBuildSlot); } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 8de4d45dd..f33196665 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -65,20 +65,20 @@ public: std::optional ca = std::nullopt ); - typedef WorkResult (DrvOutputSubstitutionGoal::*GoalState)(bool inBuildSlot); + typedef kj::Promise> (DrvOutputSubstitutionGoal::*GoalState)(bool inBuildSlot) noexcept; GoalState state; - WorkResult init(bool inBuildSlot); - WorkResult tryNext(bool inBuildSlot); - WorkResult realisationFetched(bool inBuildSlot); - WorkResult outPathValid(bool inBuildSlot); - WorkResult finished(); + kj::Promise> init(bool inBuildSlot) noexcept; + kj::Promise> tryNext(bool inBuildSlot) noexcept; + kj::Promise> realisationFetched(bool inBuildSlot) noexcept; + kj::Promise> outPathValid(bool inBuildSlot) noexcept; + kj::Promise> finished() noexcept; Finished timedOut(Error && ex) override { abort(); }; std::string key() override; - WorkResult work(bool inBuildSlot) override; + kj::Promise> work(bool inBuildSlot) noexcept override; JobCategory jobCategory() const override { return JobCategory::Substitution; diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index a5bb05b24..a0f18a02c 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -6,11 +6,17 @@ namespace nix { +static auto runWorker(Worker & worker, auto mkGoals) +{ + return worker.run(mkGoals); +} + void Store::buildPaths(const std::vector & reqs, BuildMode buildMode, std::shared_ptr evalStore) { - Worker worker(*this, evalStore ? *evalStore : *this); + auto aio = kj::setupAsyncIo(); + Worker worker(*this, evalStore ? *evalStore : *this, aio); - auto goals = worker.run([&](GoalFactory & gf) { + auto goals = runWorker(worker, [&](GoalFactory & gf) { Goals goals; for (auto & br : reqs) goals.insert(gf.makeGoal(br, buildMode)); @@ -48,10 +54,12 @@ std::vector Store::buildPathsWithResults( BuildMode buildMode, std::shared_ptr evalStore) { - Worker worker(*this, evalStore ? *evalStore : *this); + auto aio = kj::setupAsyncIo(); + Worker worker(*this, evalStore ? *evalStore : *this, aio); + std::vector> state; - auto goals = worker.run([&](GoalFactory & gf) { + auto goals = runWorker(worker, [&](GoalFactory & gf) { Goals goals; for (const auto & req : reqs) { auto goal = gf.makeGoal(req, buildMode); @@ -72,10 +80,11 @@ std::vector Store::buildPathsWithResults( BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - Worker worker(*this, *this); + auto aio = kj::setupAsyncIo(); + Worker worker(*this, *this, aio); try { - auto goals = worker.run([&](GoalFactory & gf) -> Goals { + auto goals = runWorker(worker, [&](GoalFactory & gf) -> Goals { return Goals{gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)}; }); auto goal = *goals.begin(); @@ -97,10 +106,12 @@ void Store::ensurePath(const StorePath & path) /* If the path is already valid, we're done. */ if (isValidPath(path)) return; - Worker worker(*this, *this); + auto aio = kj::setupAsyncIo(); + Worker worker(*this, *this, aio); - auto goals = - worker.run([&](GoalFactory & gf) { return Goals{gf.makePathSubstitutionGoal(path)}; }); + auto goals = runWorker(worker, [&](GoalFactory & gf) { + return Goals{gf.makePathSubstitutionGoal(path)}; + }); auto goal = *goals.begin(); if (goal->exitCode != Goal::ecSuccess) { @@ -115,9 +126,10 @@ void Store::ensurePath(const StorePath & path) void Store::repairPath(const StorePath & path) { - Worker worker(*this, *this); + auto aio = kj::setupAsyncIo(); + Worker worker(*this, *this, aio); - auto goals = worker.run([&](GoalFactory & gf) { + auto goals = runWorker(worker, [&](GoalFactory & gf) { return Goals{gf.makePathSubstitutionGoal(path, Repair)}; }); auto goal = *goals.begin(); diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 502ba2a7d..189505308 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -1,9 +1,11 @@ #pragma once ///@file +#include "result.hh" #include "types.hh" #include "store-api.hh" #include "build-result.hh" +#include namespace nix { @@ -161,7 +163,7 @@ public: trace("goal destroyed"); } - virtual WorkResult work(bool inBuildSlot) = 0; + virtual kj::Promise> work(bool inBuildSlot) noexcept = 0; virtual void waiteeDone(GoalPtr waitee) { } diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 7553f1e79..4baa525d9 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -149,8 +149,8 @@ void LocalDerivationGoal::killSandbox(bool getStats) } -Goal::WorkResult LocalDerivationGoal::tryLocalBuild(bool inBuildSlot) -{ +kj::Promise> LocalDerivationGoal::tryLocalBuild(bool inBuildSlot) noexcept +try { #if __APPLE__ additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); #endif @@ -159,7 +159,7 @@ Goal::WorkResult LocalDerivationGoal::tryLocalBuild(bool inBuildSlot) state = &DerivationGoal::tryToBuild; outputLocks.unlock(); if (0U != settings.maxBuildJobs) { - return WaitForSlot{}; + return {WaitForSlot{}}; } if (getMachines().empty()) { throw Error( @@ -214,7 +214,7 @@ Goal::WorkResult LocalDerivationGoal::tryLocalBuild(bool inBuildSlot) if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); - return WaitForAWhile{}; + return {WaitForAWhile{}}; } } @@ -250,15 +250,17 @@ Goal::WorkResult LocalDerivationGoal::tryLocalBuild(bool inBuildSlot) state = &DerivationGoal::buildDone; started(); - return WaitForWorld{std::move(fds), true}; + return {WaitForWorld{std::move(fds), true}}; } catch (BuildError & e) { outputLocks.unlock(); buildUser.reset(); auto report = done(BuildResult::InputRejected, {}, std::move(e)); report.permanentFailure = true; - return report; + return {std::move(report)}; } +} catch (...) { + return {std::current_exception()}; } diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 37a96b4d1..cd040bc15 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -213,7 +213,7 @@ struct LocalDerivationGoal : public DerivationGoal /** * The additional states. */ - WorkResult tryLocalBuild(bool inBuildSlot) override; + kj::Promise> tryLocalBuild(bool inBuildSlot) noexcept override; /** * Start building a derivation. diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 33715ea6b..bd0ffcb9b 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -45,21 +45,21 @@ Goal::Finished PathSubstitutionGoal::done( } -Goal::WorkResult PathSubstitutionGoal::work(bool inBuildSlot) +kj::Promise> PathSubstitutionGoal::work(bool inBuildSlot) noexcept { return (this->*state)(inBuildSlot); } -Goal::WorkResult PathSubstitutionGoal::init(bool inBuildSlot) -{ +kj::Promise> PathSubstitutionGoal::init(bool inBuildSlot) noexcept +try { trace("init"); worker.store.addTempRoot(storePath); /* If the path already exists we're done. */ if (!repair && worker.store.isValidPath(storePath)) { - return done(ecSuccess, BuildResult::AlreadyValid); + return {done(ecSuccess, BuildResult::AlreadyValid)}; } if (settings.readOnlyMode) @@ -68,11 +68,13 @@ Goal::WorkResult PathSubstitutionGoal::init(bool inBuildSlot) subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); return tryNext(inBuildSlot); +} catch (...) { + return {std::current_exception()}; } -Goal::WorkResult PathSubstitutionGoal::tryNext(bool inBuildSlot) -{ +kj::Promise> PathSubstitutionGoal::tryNext(bool inBuildSlot) noexcept +try { trace("trying next substituter"); cleanup(); @@ -87,10 +89,10 @@ Goal::WorkResult PathSubstitutionGoal::tryNext(bool inBuildSlot) /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - return done( + return {done( substituterFailed ? ecFailed : ecNoSubstituters, BuildResult::NoSubstituters, - fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath))); + fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)))}; } sub = subs.front(); @@ -167,20 +169,22 @@ Goal::WorkResult PathSubstitutionGoal::tryNext(bool inBuildSlot) return referencesValid(inBuildSlot); } else { state = &PathSubstitutionGoal::referencesValid; - return result; + return {std::move(result)}; } +} catch (...) { + return {std::current_exception()}; } -Goal::WorkResult PathSubstitutionGoal::referencesValid(bool inBuildSlot) -{ +kj::Promise> PathSubstitutionGoal::referencesValid(bool inBuildSlot) noexcept +try { trace("all references realised"); if (nrFailed > 0) { - return done( + return {done( nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, BuildResult::DependencyFailed, - fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath))); + fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)))}; } for (auto & i : info->references) @@ -189,15 +193,17 @@ Goal::WorkResult PathSubstitutionGoal::referencesValid(bool inBuildSlot) state = &PathSubstitutionGoal::tryToRun; return tryToRun(inBuildSlot); +} catch (...) { + return {std::current_exception()}; } -Goal::WorkResult PathSubstitutionGoal::tryToRun(bool inBuildSlot) -{ +kj::Promise> PathSubstitutionGoal::tryToRun(bool inBuildSlot) noexcept +try { trace("trying to run"); if (!inBuildSlot) { - return WaitForSlot{}; + return {WaitForSlot{}}; } maintainRunningSubstitutions = worker.runningSubstitutions.addTemporarily(1); @@ -228,12 +234,14 @@ Goal::WorkResult PathSubstitutionGoal::tryToRun(bool inBuildSlot) }); state = &PathSubstitutionGoal::finished; - return WaitForWorld{{outPipe.readSide.get()}, true}; + return {WaitForWorld{{outPipe.readSide.get()}, true}}; +} catch (...) { + return {std::current_exception()}; } -Goal::WorkResult PathSubstitutionGoal::finished(bool inBuildSlot) -{ +kj::Promise> PathSubstitutionGoal::finished(bool inBuildSlot) noexcept +try { trace("substitute finished"); worker.childTerminated(this); @@ -274,7 +282,9 @@ Goal::WorkResult PathSubstitutionGoal::finished(bool inBuildSlot) worker.doneNarSize += maintainExpectedNar.delta(); maintainExpectedNar.reset(); - return done(ecSuccess, BuildResult::Substituted); + return {done(ecSuccess, BuildResult::Substituted)}; +} catch (...) { + return {std::current_exception()}; } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 9c7e6f470..3c97b19fd 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -67,7 +67,7 @@ struct PathSubstitutionGoal : public Goal NotifyingCounter::Bump maintainExpectedSubstitutions, maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - typedef WorkResult (PathSubstitutionGoal::*GoalState)(bool inBuildSlot); + typedef kj::Promise> (PathSubstitutionGoal::*GoalState)(bool inBuildSlot) noexcept; GoalState state; /** @@ -101,16 +101,16 @@ public: return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); } - WorkResult work(bool inBuildSlot) override; + kj::Promise> work(bool inBuildSlot) noexcept override; /** * The states. */ - WorkResult init(bool inBuildSlot); - WorkResult tryNext(bool inBuildSlot); - WorkResult referencesValid(bool inBuildSlot); - WorkResult tryToRun(bool inBuildSlot); - WorkResult finished(bool inBuildSlot); + kj::Promise> init(bool inBuildSlot) noexcept; + kj::Promise> tryNext(bool inBuildSlot) noexcept; + kj::Promise> referencesValid(bool inBuildSlot) noexcept; + kj::Promise> tryToRun(bool inBuildSlot) noexcept; + kj::Promise> finished(bool inBuildSlot) noexcept; /** * Callback used by the worker to write to the log. diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index f619d574d..ee45c7e3f 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -11,12 +11,13 @@ namespace nix { -Worker::Worker(Store & store, Store & evalStore) +Worker::Worker(Store & store, Store & evalStore, kj::AsyncIoContext & aio) : act(*logger, actRealise) , actDerivations(*logger, actBuilds) , actSubstitutions(*logger, actCopyPaths) , store(store) , evalStore(evalStore) + , aio(aio) { /* Debugging: prevent recursive workers. */ nrLocalBuilds = 0; @@ -379,7 +380,7 @@ Goals Worker::run(std::function req) const bool inSlot = goal->jobCategory() == JobCategory::Substitution ? nrSubstitutions < std::max(1U, (unsigned int) settings.maxSubstitutionJobs) : nrLocalBuilds < settings.maxBuildJobs; - handleWorkResult(goal, goal->work(inSlot)); + handleWorkResult(goal, goal->work(inSlot).wait(aio.waitScope).value()); updateStatistics(); if (topGoals.empty()) break; // stuff may have been cancelled diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 9a6ed8449..6735ea0b9 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -9,6 +9,7 @@ #include "realisation.hh" #include +#include #include namespace nix { @@ -237,6 +238,7 @@ public: Store & store; Store & evalStore; + kj::AsyncIoContext & aio; struct HookState { std::unique_ptr instance; @@ -264,7 +266,7 @@ public: NotifyingCounter expectedNarSize{[this] { updateStatisticsLater(); }}; NotifyingCounter doneNarSize{[this] { updateStatisticsLater(); }}; - Worker(Store & store, Store & evalStore); + Worker(Store & store, Store & evalStore, kj::AsyncIoContext & aio); ~Worker(); /** diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 74f5cd04e..8d8b3422c 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -221,6 +221,7 @@ dependencies = [ aws_s3, aws_sdk_transfer, nlohmann_json, + kj, ] if host_machine.system() == 'freebsd' -- 2.44.1 From c14486ae8d3bbc862c625d948a6b2f4dc0927d5b Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sun, 1 Sep 2024 01:37:10 +0200 Subject: [PATCH 031/106] forbid gcc for compilation, only allow clang while gcc 12 and older miscompile our generators, gcc 13 and older outright crash on kj coroutines. (newer gcc versions may fix this) Change-Id: I19f12c8c147239680eb0fa5a84ef5c7de38c9263 --- flake.nix | 11 ++++++++--- meson.build | 10 +++++++++- package.nix | 4 ++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index 662469479..46f9e43e8 100644 --- a/flake.nix +++ b/flake.nix @@ -99,9 +99,10 @@ ]; stdenvs = [ - "gccStdenv" + # see assertion in package.nix why these two are disabled + # "stdenv" + # "gccStdenv" "clangStdenv" - "stdenv" "libcxxStdenv" "ccacheStdenv" ]; @@ -121,7 +122,11 @@ name = "${stdenvName}Packages"; value = f stdenvName; }) stdenvs - ); + ) + // { + # TODO delete this and reënable gcc stdenvs once gcc compiles kj coros correctly + stdenvPackages = f "clangStdenv"; + }; # Memoize nixpkgs for different platforms for efficiency. nixpkgsFor = forAllSystems ( diff --git a/meson.build b/meson.build index b015cc606..f89f5a016 100644 --- a/meson.build +++ b/meson.build @@ -167,10 +167,18 @@ endif # frees one would expect when the objects are unique_ptrs. these problems # often show up as memory corruption when nesting generators (since we do # treat generators like owned memory) and will cause inexplicable crashs. +# +# gcc 13 does not compile capnp coroutine code correctly. a newer version +# may fix this. (cf. https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102051) +# we allow gcc 13 here anyway because CI uses it for clang-tidy, and when +# the compiler crashes outright if won't produce any bad binaries either. assert( cxx.get_id() != 'gcc' or cxx.version().version_compare('>=13'), - 'GCC 12 and earlier are known to miscompile lix coroutines, use GCC 13 or clang.' + 'GCC is known to miscompile coroutines, use clang.' ) +if cxx.get_id() == 'gcc' + warning('GCC is known to crash while building coroutines, use clang.') +endif # Translate some historical and Mesony CPU names to Lixy CPU names. diff --git a/package.nix b/package.nix index e5ab5eff0..eebf222b7 100644 --- a/package.nix +++ b/package.nix @@ -90,6 +90,10 @@ capnproto-lix = capnproto.overrideAttrs { CXXFLAGS = "-std=c++20"; }; }, }: + +# gcc miscompiles coroutines at least until 13.2, possibly longer +assert stdenv.cc.isClang || lintInsteadOfBuild; + let inherit (__forDefaults) canRunInstalled; inherit (lib) fileset; -- 2.44.1 From 6de6cae3e70caba0ff8fe3d2d6cb7c4fd0022618 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Sun, 8 Sep 2024 15:49:08 -0700 Subject: [PATCH 032/106] repl: Patch editline to recognize Meta-Left & Meta-Right This applies https://github.com/troglobit/editline/pull/70 to our build of editline, which translates `meta-left` and `meta-right` into `fd_word` and `bk_word`. This makes `nix repl` soooo much nicer to use! Note: My terminal renders `meta-left` as `\e\e[C` and `meta-right` as `\e\e[D`. Closes https://git.lix.systems/lix-project/lix/issues/501 Change-Id: I048b10cf17231bbf4e6bf38e1d1d8572cedaa194 --- .../rl-next/alt-left-and-alt-right-in-repl.md | 10 +++++++ package.nix | 29 ++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 doc/manual/rl-next/alt-left-and-alt-right-in-repl.md diff --git a/doc/manual/rl-next/alt-left-and-alt-right-in-repl.md b/doc/manual/rl-next/alt-left-and-alt-right-in-repl.md new file mode 100644 index 000000000..66e89a15e --- /dev/null +++ b/doc/manual/rl-next/alt-left-and-alt-right-in-repl.md @@ -0,0 +1,10 @@ +--- +synopsis: "`Alt+Left` and `Alt+Right` go back/forwards by words in `nix repl`" +issues: [fj#501] +cls: [1883] +category: Fixes +credits: 9999years +--- + +`nix repl` now recognizes `Alt+Left` and `Alt+Right` for navigating by words +when entering input in `nix repl` on more terminals/platforms. diff --git a/package.nix b/package.nix index e5ab5eff0..29824b183 100644 --- a/package.nix +++ b/package.nix @@ -22,6 +22,7 @@ doxygen, editline-lix ? __forDefaults.editline-lix, editline, + fetchpatch, git, gtest, jq, @@ -38,6 +39,7 @@ mercurial, meson, ninja, + ncurses, openssl, pegtl, pkg-config, @@ -81,7 +83,32 @@ boehmgc-nix = boehmgc.override { enableLargeConfig = true; }; editline-lix = editline.overrideAttrs (prev: { - configureFlags = prev.configureFlags or [ ] ++ [ (lib.enableFeature true "sigstop") ]; + patches = (prev.patches or [ ]) ++ [ + # Recognize `Alt-Left` and `Alt-Right` for navigating by words in more + # terminals/shells/platforms. + # + # See: https://github.com/troglobit/editline/pull/70/commits + (fetchpatch { + url = "https://github.com/troglobit/editline/pull/70/commits/d0f2a5bc2300b96b2434c7838184c1dfd6a639f5.diff"; + hash = "sha256-0bbtYDUlk1wA0kpTtlaNI6KaCjLmAesZjcWBJZ+DpyQ="; + }) + + (fetchpatch { + url = "https://github.com/troglobit/editline/pull/70/commits/4c4455353a0a88bee09d5f27c28f81f747682fed.diff"; + hash = "sha256-nVezspwVzeB/8zENeKgwPVum0W1MLv4dOW0967WbX5w="; + }) + ]; + + configureFlags = (prev.configureFlags or [ ]) ++ [ + # Enable SIGSTOP (Ctrl-Z) behavior. + (lib.enableFeature true "sigstop") + # Enable ANSI arrow keys. + (lib.enableFeature true "arrow-keys") + # Use termcap library to query terminal size. + (lib.enableFeature (ncurses != null) "termcap") + ]; + + nativeBuildInputs = (prev.nativeBuildInputs or [ ]) ++ [ ncurses ]; }); build-release-notes = callPackage ./maintainers/build-release-notes.nix { }; -- 2.44.1 From 81c2e0ac8e76ddb3fd3c8e2ce59929853614b1b6 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Wed, 11 Sep 2024 00:27:39 -0700 Subject: [PATCH 033/106] archive: rename ParseSink to NARParseVisitor - Rename the listener to not be called a "sink". If it were a "sink" it would be eating bytes and conform with any of the Nix sink stuff (maybe FileHandle should be a Sink itself! but that's a later CL's problem). This is a parser listener. - Move the RetrieveRegularNARSink thing into store-api.cc, which is its only usage, and fix it to actually do what it is stated to do: crash if its invariants are violated. It's, of course, used to erm, unpack single-file NAR files, generated via a horrible contraption of sources and sinks that looks like a plumbing blueprint. Refactoring that is a future task. - Add a description of the invariants of NARParseVisitor in preparation of refactoring it. Change-Id: Ifca1d74d2947204a1f66349772e54dad0743e944 --- src/libstore/local-store.cc | 2 +- src/libstore/nar-accessor.cc | 3 ++- src/libstore/store-api.cc | 35 ++++++++++++++++++++++++++++-- src/libutil/archive.cc | 42 ++++++++++++++++++++++++++++++------ src/libutil/archive.hh | 37 ++++++------------------------- src/libutil/file-system.hh | 2 +- 6 files changed, 79 insertions(+), 42 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4c8e2ea2f..d3520582e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1215,7 +1215,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, bool narRead = false; Finally cleanup = [&]() { if (!narRead) { - ParseSink sink; + NARParseVisitor sink; try { parseDump(sink, source); } catch (...) { diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index f0dfcb19b..fa7d5e3cb 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -2,6 +2,7 @@ #include "archive.hh" #include +#include #include #include @@ -33,7 +34,7 @@ struct NarAccessor : public FSAccessor NarMember root; - struct NarIndexer : ParseSink, Source + struct NarIndexer : NARParseVisitor, Source { NarAccessor & acc; Source & source; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 6d9fec41b..1619e5062 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -379,6 +379,37 @@ void Store::addMultipleToStore( } } +namespace { +/** + * If the NAR archive contains a single file at top-level, then save + * the contents of the file to `s`. Otherwise assert. + */ +struct RetrieveRegularNARVisitor : NARParseVisitor +{ + Sink & sink; + + RetrieveRegularNARVisitor(Sink & sink) : sink(sink) { } + + void createRegularFile(const Path & path) override + { + } + + void receiveContents(std::string_view data) override + { + sink(data); + } + + void createDirectory(const Path & path) override + { + assert(false && "RetrieveRegularNARVisitor::createDirectory must not be called"); + } + + void createSymlink(const Path & path, const std::string & target) override + { + assert(false && "RetrieveRegularNARVisitor::createSymlink must not be called"); + } +}; +} /* The aim of this function is to compute in one pass the correct ValidPathInfo for @@ -413,7 +444,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, /* Note that fileSink and unusualHashTee must be mutually exclusive, since they both write to caHashSink. Note that that requisite is currently true because the former is only used in the flat case. */ - RetrieveRegularNARSink fileSink { caHashSink }; + RetrieveRegularNARVisitor fileSink { caHashSink }; TeeSink unusualHashTee { narHashSink, caHashSink }; auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != HashType::SHA256 @@ -429,7 +460,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, information to narSink. */ TeeSource tapped { fileSource, narSink }; - ParseSink blank; + NARParseVisitor blank; auto & parseSink = method == FileIngestionMethod::Flat ? fileSink : blank; diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index d4da18f14..78d49026a 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -334,7 +334,7 @@ Generator parse(Source & source) } -static WireFormatGenerator restore(ParseSink & sink, Generator nar) +static WireFormatGenerator restore(NARParseVisitor & sink, Generator nar) { while (auto entry = nar.next()) { co_yield std::visit( @@ -377,12 +377,12 @@ static WireFormatGenerator restore(ParseSink & sink, Generator nar) } } -WireFormatGenerator parseAndCopyDump(ParseSink & sink, Source & source) +WireFormatGenerator parseAndCopyDump(NARParseVisitor & sink, Source & source) { return restore(sink, nar::parse(source)); } -void parseDump(ParseSink & sink, Source & source) +void parseDump(NARParseVisitor & sink, Source & source) { auto parser = parseAndCopyDump(sink, source); while (parser.next()) { @@ -390,7 +390,36 @@ void parseDump(ParseSink & sink, Source & source) } } -struct RestoreSink : ParseSink +/* + * Note [NAR restoration security]: + * It's *critical* that NAR restoration will never overwrite anything even if + * duplicate filenames are passed in. It is inevitable that not all NARs are + * fit to actually successfully restore to the target filesystem; errors may + * occur due to collisions, and this *must* cause the NAR to be rejected. + * + * Although the filenames are blocked from being *the same bytes* by a higher + * layer, filesystems have other ideas on every platform: + * - The store may be on a case-insensitive filesystem like APFS, ext4 with + * casefold directories, zfs with casesensitivity=insensitive + * - The store may be on a Unicode normalizing (or normalization-insensitive) + * filesystem like APFS (where files are looked up by + * hash(normalize(fname))), HFS+ (where file names are always normalized to + * approximately NFD), or zfs with normalization=formC, etc. + * + * It is impossible to know the version of Unicode being used by the underlying + * filesystem, thus it is *impossible* to stop these collisions. + * + * Overwriting files as a result of invalid NARs will cause a security bug like + * CppNix's CVE-2024-45593 (GHSA-h4vv-h3jq-v493) + */ + +/** + * This code restores NARs from disk. + * + * See Note [NAR restoration security] for security invariants in this procedure. + * + */ +struct NARRestoreVisitor : NARParseVisitor { Path dstPath; AutoCloseFD fd; @@ -457,7 +486,7 @@ struct RestoreSink : ParseSink void restorePath(const Path & path, Source & source) { - RestoreSink sink; + NARRestoreVisitor sink; sink.dstPath = path; parseDump(sink, source); } @@ -468,10 +497,9 @@ WireFormatGenerator copyNAR(Source & source) // FIXME: if 'source' is the output of dumpPath() followed by EOF, // we should just forward all data directly without parsing. - static ParseSink parseSink; /* null sink; just parse the NAR */ + static NARParseVisitor parseSink; /* null sink; just parse the NAR */ return parseAndCopyDump(parseSink, source); } - } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index b34d06e3d..5e8db4c53 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -76,8 +76,12 @@ WireFormatGenerator dumpString(std::string_view s); /** * \todo Fix this API, it sucks. + * A visitor for NAR parsing that performs filesystem (or virtual-filesystem) + * actions to restore a NAR. + * + * Methods of this may arbitrarily fail due to filename collisions. */ -struct ParseSink +struct NARParseVisitor { virtual void createDirectory(const Path & path) { }; @@ -90,33 +94,6 @@ struct ParseSink virtual void createSymlink(const Path & path, const std::string & target) { }; }; -/** - * If the NAR archive contains a single file at top-level, then save - * the contents of the file to `s`. Otherwise barf. - */ -struct RetrieveRegularNARSink : ParseSink -{ - bool regular = true; - Sink & sink; - - RetrieveRegularNARSink(Sink & sink) : sink(sink) { } - - void createDirectory(const Path & path) override - { - regular = false; - } - - void receiveContents(std::string_view data) override - { - sink(data); - } - - void createSymlink(const Path & path, const std::string & target) override - { - regular = false; - } -}; - namespace nar { struct MetadataString; @@ -160,8 +137,8 @@ Generator parse(Source & source); } -WireFormatGenerator parseAndCopyDump(ParseSink & sink, Source & source); -void parseDump(ParseSink & sink, Source & source); +WireFormatGenerator parseAndCopyDump(NARParseVisitor & sink, Source & source); +void parseDump(NARParseVisitor & sink, Source & source); void restorePath(const Path & path, Source & source); diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index e49323e84..9fe931556 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -193,7 +193,7 @@ inline Paths createDirs(PathView path) } /** - * Create a symlink. + * Create a symlink. Throws if the symlink exists. */ void createSymlink(const Path & target, const Path & link); -- 2.44.1 From df0137226d7db4bfac455076be9f37dae6f01ffe Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Wed, 11 Sep 2024 09:34:42 -0700 Subject: [PATCH 034/106] editline: Vendor cl/1883 patch to recognize `Alt+Left`/`Alt+Right` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This vendors the patch added in cl/1883 to avoid GitHub garbage-collecting the commits we're referring to. As @emilazy pointed out on GitHub: > GitHub can garbage‐collect unmerged PR commits if they are later > force‐pushed, which means that code review in upstreams can cause > Nixpkgs builds to fail to reproduce in future. See: https://github.com/NixOS/nixpkgs/pull/341131#discussion_r1753046220 See: https://github.com/troglobit/editline/pull/70 See: https://gerrit.lix.systems/c/lix/+/1883 Change-Id: Ifff522f7f23310d6dbe9efc72fd40be5500ae872 --- nix-support/editline.patch | 106 +++++++++++++++++++++++++++++++++++++ package.nix | 12 +---- 2 files changed, 108 insertions(+), 10 deletions(-) create mode 100644 nix-support/editline.patch diff --git a/nix-support/editline.patch b/nix-support/editline.patch new file mode 100644 index 000000000..df31cfdc4 --- /dev/null +++ b/nix-support/editline.patch @@ -0,0 +1,106 @@ +From d0f2a5bc2300b96b2434c7838184c1dfd6a639f5 Mon Sep 17 00:00:00 2001 +From: Rebecca Turner +Date: Sun, 8 Sep 2024 15:42:42 -0700 +Subject: [PATCH 1/2] Recognize Meta+Left and Meta+Right + +Recognize `Alt-Left` and `Alt-Right` for navigating by words in more +terminals/shells/platforms. + +I'm not sure exactly where to find canonical documentation for these +codes, but this seems to match what my terminal produces (macOS + iTerm2 ++ Fish + Tmux). + +It might also be nice to have some more support for editing the bindings +for these characters; sequences of more than one character are not +supported by `el_bind_key` and similar. + +Originally from: https://github.com/troglobit/editline/pull/70 +This patch is applied upstream: https://gerrit.lix.systems/c/lix/+/1883 + +--- + src/editline.c | 29 +++++++++++++++++++++++++++-- + 1 file changed, 27 insertions(+), 2 deletions(-) + +diff --git a/src/editline.c b/src/editline.c +index 5ec9afb..d1cfbbc 100644 +--- a/src/editline.c ++++ b/src/editline.c +@@ -1034,6 +1034,30 @@ static el_status_t meta(void) + return CSeof; + + #ifdef CONFIG_ANSI_ARROWS ++ /* See: https://en.wikipedia.org/wiki/ANSI_escape_code */ ++ /* Recognize ANSI escapes for `Meta+Left` and `Meta+Right`. */ ++ if (c == '\e') { ++ switch (tty_get()) { ++ case '[': ++ { ++ switch (tty_get()) { ++ /* \e\e[C = Meta+Left */ ++ case 'C': return fd_word(); ++ /* \e\e[D = Meta+Right */ ++ case 'D': return bk_word(); ++ default: ++ break; ++ } ++ ++ return el_ring_bell(); ++ } ++ default: ++ break; ++ } ++ ++ return el_ring_bell(); ++ } ++ + /* Also include VT-100 arrows. */ + if (c == '[' || c == 'O') { + switch (tty_get()) { +@@ -1043,6 +1067,7 @@ static el_status_t meta(void) + char seq[4] = { 0 }; + seq[0] = tty_get(); + ++ /* \e[1~ */ + if (seq[0] == '~') + return beg_line(); /* Home */ + +@@ -1050,9 +1075,9 @@ static el_status_t meta(void) + seq[c] = tty_get(); + + if (!strncmp(seq, ";5C", 3)) +- return fd_word(); /* Ctrl+Right */ ++ return fd_word(); /* \e[1;5C = Ctrl+Right */ + if (!strncmp(seq, ";5D", 3)) +- return bk_word(); /* Ctrl+Left */ ++ return bk_word(); /* \e[1;5D = Ctrl+Left */ + + break; + } + +From 4c4455353a0a88bee09d5f27c28f81f747682fed Mon Sep 17 00:00:00 2001 +From: Rebecca Turner +Date: Mon, 9 Sep 2024 09:44:44 -0700 +Subject: [PATCH 2/2] Add support for \e[1;3C and \e[1;3D + +--- + src/editline.c | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/src/editline.c b/src/editline.c +index d1cfbbc..350b5cb 100644 +--- a/src/editline.c ++++ b/src/editline.c +@@ -1074,9 +1074,11 @@ static el_status_t meta(void) + for (c = 1; c < 3; c++) + seq[c] = tty_get(); + +- if (!strncmp(seq, ";5C", 3)) ++ if (!strncmp(seq, ";5C", 3) ++ || !strncmp(seq, ";3C", 3)) + return fd_word(); /* \e[1;5C = Ctrl+Right */ +- if (!strncmp(seq, ";5D", 3)) ++ if (!strncmp(seq, ";5D", 3) ++ || !strncmp(seq, ";3D", 3)) + return bk_word(); /* \e[1;5D = Ctrl+Left */ + + break; diff --git a/package.nix b/package.nix index c9341d4cb..23ebc3a82 100644 --- a/package.nix +++ b/package.nix @@ -87,16 +87,8 @@ # Recognize `Alt-Left` and `Alt-Right` for navigating by words in more # terminals/shells/platforms. # - # See: https://github.com/troglobit/editline/pull/70/commits - (fetchpatch { - url = "https://github.com/troglobit/editline/pull/70/commits/d0f2a5bc2300b96b2434c7838184c1dfd6a639f5.diff"; - hash = "sha256-0bbtYDUlk1wA0kpTtlaNI6KaCjLmAesZjcWBJZ+DpyQ="; - }) - - (fetchpatch { - url = "https://github.com/troglobit/editline/pull/70/commits/4c4455353a0a88bee09d5f27c28f81f747682fed.diff"; - hash = "sha256-nVezspwVzeB/8zENeKgwPVum0W1MLv4dOW0967WbX5w="; - }) + # See: https://github.com/troglobit/editline/pull/70 + ./nix-support/editline.patch ]; configureFlags = (prev.configureFlags or [ ]) ++ [ -- 2.44.1 From 82aa1ccab49c187b6b79c4b27af492060df142a4 Mon Sep 17 00:00:00 2001 From: Alois Wohlschlager Date: Wed, 11 Sep 2024 18:46:54 +0200 Subject: [PATCH 035/106] fish-completion: leave the shell prompt intact When generating shell completions, no logging output should be visible because it would destroy the shell prompt. Originally this was attempted to be done by simply disabling the progress bar (ca946860ce6ce5d4800b0d93d3f83c30d3c953c0), since the situation is particularly bad there (the screen clearing required for the rendering ends up erasing the shell prompt). Due to overlooking the implementation of this hack, it was accidentally undone during a later change (0dd1d8ca1cdccfc620644a7f690ed35bcd2d1e74). Since even with the hack correctly in place, it is still possible to mess up the prompt by logging output (for example warnings for disabled experimental features, or messages generated by `builtins.trace`), simply send it to the bit bucket where it belongs. This was already done for bash and zsh (9d840758a8d195e52e8b7d08cd9c15f6b8259724), and it seems that fish was simply missed at that time. The last trace of the no-longer-working and obsolete hack is deleted too. Fixes: https://git.lix.systems/lix-project/lix/issues/513 Change-Id: I59f1ebf90903034e2059298fa8d76bf970bc3315 --- misc/fish/completion.fish | 2 +- src/libmain/loggers.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/fish/completion.fish b/misc/fish/completion.fish index c6b8ef16a..b7a272c61 100644 --- a/misc/fish/completion.fish +++ b/misc/fish/completion.fish @@ -14,7 +14,7 @@ function _nix_complete # But the variable also misses the current token so it cancels out. set -l nix_arg_to_complete (count $nix_args) - env NIX_GET_COMPLETIONS=$nix_arg_to_complete $nix_args $current_token + env NIX_GET_COMPLETIONS=$nix_arg_to_complete $nix_args $current_token 2>/dev/null end function _nix_accepts_files diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index 8c3c4e355..46438280a 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -7,7 +7,7 @@ namespace nix { LogFormat defaultLogFormat = LogFormat::raw; LogFormat parseLogFormat(const std::string & logFormatStr) { - if (logFormatStr == "raw" || getEnv("NIX_GET_COMPLETIONS")) + if (logFormatStr == "raw") return LogFormat::raw; else if (logFormatStr == "raw-with-logs") return LogFormat::rawWithLogs; -- 2.44.1 From ca1dc3f70bf98e2424b7b2666ee2180675b67451 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Wed, 11 Sep 2024 00:27:39 -0700 Subject: [PATCH 036/106] archive: refactor bad mutable-state API in the NAR parse listener Remove the mutable state stuff that assumes that one file is being written a time. It's true that we don't write multiple files interleaved, but that mutable state is evil. Change-Id: Ia1481da48255d901e4b09a9b783e7af44fae8cff --- src/libstore/nar-accessor.cc | 30 ++++----- src/libstore/store-api.cc | 23 +++++-- src/libutil/archive.cc | 114 ++++++++++++++++++++--------------- src/libutil/archive.hh | 39 +++++++++--- 4 files changed, 127 insertions(+), 79 deletions(-) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index fa7d5e3cb..7600de6e7 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -45,11 +45,12 @@ struct NarAccessor : public FSAccessor uint64_t pos = 0; + public: NarIndexer(NarAccessor & acc, Source & source) : acc(acc), source(source) { } - void createMember(const Path & path, NarMember member) + NarMember & createMember(const Path & path, NarMember member) { size_t level = std::count(path.begin(), path.end(), '/'); while (parents.size() > level) parents.pop(); @@ -63,6 +64,8 @@ struct NarAccessor : public FSAccessor auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); parents.push(&result.first->second); } + + return *parents.top(); } void createDirectory(const Path & path) override @@ -70,28 +73,17 @@ struct NarAccessor : public FSAccessor createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0}); } - void createRegularFile(const Path & path) override + std::unique_ptr createRegularFile(const Path & path, uint64_t size, bool executable) override { - createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); - } + auto & memb = createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); - void closeRegularFile() override - { } - - void isExecutable() override - { - parents.top()->isExecutable = true; - } - - void preallocateContents(uint64_t size) override - { assert(size <= std::numeric_limits::max()); - parents.top()->size = (uint64_t) size; - parents.top()->start = pos; - } + memb.size = (uint64_t) size; + memb.start = pos; + memb.isExecutable = executable; - void receiveContents(std::string_view data) override - { } + return std::make_unique(); + } void createSymlink(const Path & path, const std::string & target) override { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1619e5062..ed8657774 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -386,17 +386,28 @@ namespace { */ struct RetrieveRegularNARVisitor : NARParseVisitor { + struct MyFileHandle : public FileHandle + { + Sink & sink; + + void receiveContents(std::string_view data) override + { + sink(data); + } + + private: + MyFileHandle(Sink & sink) : sink(sink) {} + + friend struct RetrieveRegularNARVisitor; + }; + Sink & sink; RetrieveRegularNARVisitor(Sink & sink) : sink(sink) { } - void createRegularFile(const Path & path) override + std::unique_ptr createRegularFile(const Path & path, uint64_t size, bool executable) override { - } - - void receiveContents(std::string_view data) override - { - sink(data); + return std::unique_ptr(new MyFileHandle{sink}); } void createDirectory(const Path & path) override diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 78d49026a..225483804 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -347,16 +347,13 @@ static WireFormatGenerator restore(NARParseVisitor & sink, Generator }, [&](nar::File f) { return [](auto f, auto & sink) -> WireFormatGenerator { - sink.createRegularFile(f.path); - sink.preallocateContents(f.size); - if (f.executable) { - sink.isExecutable(); - } + auto handle = sink.createRegularFile(f.path, f.size, f.executable); + while (auto block = f.contents.next()) { - sink.receiveContents(std::string_view{block->data(), block->size()}); + handle->receiveContents(std::string_view{block->data(), block->size()}); co_yield *block; } - sink.closeRegularFile(); + handle->close(); }(std::move(f), sink); }, [&](nar::Symlink sl) { @@ -422,8 +419,67 @@ void parseDump(NARParseVisitor & sink, Source & source) struct NARRestoreVisitor : NARParseVisitor { Path dstPath; - AutoCloseFD fd; +private: + class MyFileHandle : public FileHandle + { + AutoCloseFD fd; + + MyFileHandle(AutoCloseFD && fd, uint64_t size, bool executable) : FileHandle(), fd(std::move(fd)) + { + if (executable) { + makeExecutable(); + } + + maybePreallocateContents(size); + } + + void makeExecutable() + { + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); + } + + void maybePreallocateContents(uint64_t len) + { + if (!archiveSettings.preallocateContents) + return; + +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError("preallocating file of %1% bytes", len); + } +#endif + } + + public: + + ~MyFileHandle() = default; + + virtual void close() override + { + /* Call close explicitly to make sure the error is checked */ + fd.close(); + } + + void receiveContents(std::string_view data) override + { + writeFull(fd.get(), data); + } + + friend struct NARRestoreVisitor; + }; + +public: void createDirectory(const Path & path) override { Path p = dstPath + path; @@ -431,49 +487,13 @@ struct NARRestoreVisitor : NARParseVisitor throw SysError("creating directory '%1%'", p); }; - void createRegularFile(const Path & path) override + std::unique_ptr createRegularFile(const Path & path, uint64_t size, bool executable) override { Path p = dstPath + path; - fd = AutoCloseFD{open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666)}; + AutoCloseFD fd = AutoCloseFD{open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666)}; if (!fd) throw SysError("creating file '%1%'", p); - } - void closeRegularFile() override - { - /* Call close explicitly to make sure the error is checked */ - fd.close(); - } - - void isExecutable() override - { - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("fstat"); - if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } - - void preallocateContents(uint64_t len) override - { - if (!archiveSettings.preallocateContents) - return; - -#if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd.get(), 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError("preallocating file of %1% bytes", len); - } -#endif - } - - void receiveContents(std::string_view data) override - { - writeFull(fd.get(), data); + return std::unique_ptr(new MyFileHandle(std::move(fd), size, executable)); } void createSymlink(const Path & path, const std::string & target) override diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 5e8db4c53..c633bee00 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -83,15 +83,40 @@ WireFormatGenerator dumpString(std::string_view s); */ struct NARParseVisitor { - virtual void createDirectory(const Path & path) { }; + /** + * A type-erased file handle specific to this particular NARParseVisitor. + */ + struct FileHandle + { + FileHandle() {} + FileHandle(FileHandle const &) = delete; + FileHandle & operator=(FileHandle &) = delete; - virtual void createRegularFile(const Path & path) { }; - virtual void closeRegularFile() { }; - virtual void isExecutable() { }; - virtual void preallocateContents(uint64_t size) { }; - virtual void receiveContents(std::string_view data) { }; + /** Puts one block of data into the file */ + virtual void receiveContents(std::string_view data) { } - virtual void createSymlink(const Path & path, const std::string & target) { }; + /** + * Explicitly closes the file. Further operations may throw an assert. + * This exists so that closing can fail and throw an exception without doing so in a destructor. + */ + virtual void close() { } + + virtual ~FileHandle() = default; + }; + + virtual void createDirectory(const Path & path) { } + + /** + * Creates a regular file in the extraction output with the given size and executable flag. + * The size is guaranteed to be the true size of the file. + */ + [[nodiscard]] + virtual std::unique_ptr createRegularFile(const Path & path, uint64_t size, bool executable) + { + return std::make_unique(); + } + + virtual void createSymlink(const Path & path, const std::string & target) { } }; namespace nar { -- 2.44.1 From b7fc37b0154b14d677f2d05b06b761373e2d44b6 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 13 Sep 2024 16:26:15 -0700 Subject: [PATCH 037/106] store: add a hint on how to fix Lix installs broken by macOS Sequoia This is not a detailed diagnosis, and it's not worth writing one, tbh. This error basically never happens in normal operation, so diagnosing it by changing the error on macOS is good enough. Relevant: https://git.lix.systems/lix-project/lix-installer/issues/24 Relevant: https://git.lix.systems/lix-project/lix-installer/issues/18 Relevant: https://git.lix.systems/lix-project/lix/issues/521 Change-Id: I03701f917d116575c72a97502b8e1617679447f2 --- src/libstore/lock.cc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 05296757d..377499c0a 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -73,8 +73,16 @@ struct SimpleUserLock : UserLock debug("trying user '%s'", i); struct passwd * pw = getpwnam(i.c_str()); - if (!pw) - throw Error("the user '%s' in the group '%s' does not exist", i, settings.buildUsersGroup); + if (!pw) { +#ifdef __APPLE__ +#define APPLE_HINT "\n\nhint: this may be caused by an update to macOS Sequoia breaking existing Lix installations.\n" \ + "See the macOS Sequoia page on the Lix wiki for detailed repair instructions: https://wiki.lix.systems/link/81" +#else +#define APPLE_HINT +#endif + throw Error("the user '%s' in the group '%s' does not exist" APPLE_HINT, i, settings.buildUsersGroup); +#undef APPLE_HINT + } auto fnUserLock = fmt("%s/userpool/%s", settings.nixStateDir,pw->pw_uid); -- 2.44.1 From 3f07c65510d5f907c75803e7d58884bdafb78132 Mon Sep 17 00:00:00 2001 From: Alois Wohlschlager Date: Tue, 10 Sep 2024 18:05:32 +0200 Subject: [PATCH 038/106] local-store: make extended attribute handling more robust * Move the extended attribute deletion after the hardlink sanity check. We shouldn't be removing extended attributes on random files. * Make the entity owner-writable before attempting to remove extended attributes, since this operation usually requires write access on the file, and we shouldn't fail xattr deletion on a file that has been made unwritable by the builder or a previous canonicalisation pass. Fixes: https://git.lix.systems/lix-project/lix/issues/507 Change-Id: I7e6ccb71649185764cd5210f4a4794ee174afea6 --- src/libstore/local-store.cc | 44 +++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 49991e38a..2da8dd2a6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -664,27 +664,6 @@ static void canonicalisePathMetaData_( if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) throw Error("file '%1%' has an unsupported type", path); -#if __linux__ - /* Remove extended attributes / ACLs. */ - ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); - - if (eaSize < 0) { - if (errno != ENOTSUP && errno != ENODATA) - throw SysError("querying extended attributes of '%s'", path); - } else if (eaSize > 0) { - std::vector eaBuf(eaSize); - - if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) - throw SysError("querying extended attributes of '%s'", path); - - for (auto & eaName: tokenizeString(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { - if (settings.ignoredAcls.get().count(eaName)) continue; - if (lremovexattr(path.c_str(), eaName.c_str()) == -1) - throw SysError("removing extended attribute '%s' from '%s'", eaName, path); - } - } -#endif - /* Fail if the file is not owned by the build user. This prevents us from messing up the ownership/permissions of files hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). @@ -699,6 +678,29 @@ static void canonicalisePathMetaData_( return; } +#if __linux__ + /* Remove extended attributes / ACLs. */ + ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); + + if (eaSize < 0) { + if (errno != ENOTSUP && errno != ENODATA) + throw SysError("querying extended attributes of '%s'", path); + } else if (eaSize > 0) { + std::vector eaBuf(eaSize); + + if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) + throw SysError("querying extended attributes of '%s'", path); + + if (S_ISREG(st.st_mode) || S_ISDIR(st.st_mode)) + chmod(path.c_str(), st.st_mode | S_IWUSR); + for (auto & eaName: tokenizeString(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { + if (settings.ignoredAcls.get().count(eaName)) continue; + if (lremovexattr(path.c_str(), eaName.c_str()) == -1) + throw SysError("removing extended attribute '%s' from '%s'", eaName, path); + } + } +#endif + inodesSeen.insert(Inode(st.st_dev, st.st_ino)); canonicaliseTimestampAndPermissions(path, st); -- 2.44.1 From 727258241fc0b3c02691b72302d2c3092baca275 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 13 Sep 2024 17:00:37 -0700 Subject: [PATCH 039/106] fix: docs issue template was busted Apparently forgejo has a more creative interpretation of \(\) than I was hoping in their markdown parser and thought it was maths. I have no idea then how you put a link in parens next to another square-bracket link, but I am not going to worry about it. There were several more typos, which I also fixed. Fixes: https://git.lix.systems/lix-project/lix/issues/517 Change-Id: I6b144c6881f92ca60ba72a304ce7a0bcb9c6659a --- .github/ISSUE_TEMPLATE/missing_documentation.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md index fa3a5137a..6c632c3e2 100644 --- a/.github/ISSUE_TEMPLATE/missing_documentation.md +++ b/.github/ISSUE_TEMPLATE/missing_documentation.md @@ -2,7 +2,7 @@ name: Missing or incorrect documentation about: Help us improve the reference manual title: '' -labels: documentation +labels: docs assignees: '' --- @@ -19,10 +19,10 @@ assignees: '' -- [ ] checked [latest Lix manual] \([source]\) +- [ ] checked [latest Lix manual] or its [source code] - [ ] checked [documentation issues] and [recent documentation changes] for possible duplicates -[latest Nix manual]: https://docs.lix.systems/manual/lix/nightly -[source]: https://git.lix.systems/lix-project/lix/src/main/doc/manual/src +[latest Lix manual]: https://docs.lix.systems/manual/lix/nightly +[source code]: https://git.lix.systems/lix-project/lix/src/main/doc/manual/src [documentation issues]: https://git.lix.systems/lix-project/lix/issues?labels=151&state=all [recent documentation changes]: https://gerrit.lix.systems/q/p:lix+path:%22%5Edoc/manual/.*%22 -- 2.44.1 From 80202e3ca314c21547c48f3a23d3f629cd9ddb87 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 15 Sep 2024 16:13:22 +0200 Subject: [PATCH 040/106] common-eval-args: raise warning if `--arg` isn't a valid Nix identifier See https://git.lix.systems/lix-project/lix/issues/496. The core idea is to be able to do e.g. nix-instantiate -A some-nonfree-thing --arg config.allowUnfree true which is currently not possible since `config.allowUnfree` is interpreted as attribute name with a dot in it. In order to change that (probably), Jade suggested to find out if there are any folks out there relying on this behavior. For such a use-case, it may still be possible to accept strings, i.e. `--arg '"config.allowUnfree"'. Change-Id: I986c73619fbd87a95b55e2f0ac03feaed3de2d2d --- src/libcmd/common-eval-args.cc | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 9a7c30d57..cbb7edbdd 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -9,8 +9,24 @@ #include "store-api.hh" #include "command.hh" +#include + namespace nix { +static std::regex const identifierRegex("^[A-Za-z_][A-Za-z0-9_'-]*$"); +static void warnInvalidNixIdentifier(const std::string & name) +{ + std::smatch match; + if (!std::regex_match(name, match, identifierRegex)) { + warn("This Nix invocation specifies a value for argument '%s' which isn't a valid \ +Nix identifier. The project is considering to drop support for this \ +or to require quotes around args that aren't valid Nix identifiers. \ +If you depend on this behvior, please reach out in \ +https://git.lix.systems/lix-project/lix/issues/496 so we can discuss \ +your use-case.", name); + } +} + MixEvalArgs::MixEvalArgs() { addFlag({ @@ -18,7 +34,10 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the value *expr* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "expr"}, - .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} + .handler = {[&](std::string name, std::string expr) { + warnInvalidNixIdentifier(name); + autoArgs[name] = 'E' + expr; + }} }); addFlag({ @@ -26,7 +45,10 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the string *string* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "string"}, - .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, + .handler = {[&](std::string name, std::string s) { + warnInvalidNixIdentifier(name); + autoArgs[name] = 'S' + s; + }}, }); addFlag({ -- 2.44.1 From 7ae0409989187d44044c4cd6f274f0a02ec5f826 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Sun, 8 Sep 2024 18:37:28 -0700 Subject: [PATCH 041/106] Remove readline support Lix cannot be built with GNU readline, and we would "rather not" be GPL. Change-Id: I0e86f0f10dab966ab1d1d467fb61fd2de50c00de --- doc/manual/rl-next/readline-support-removed.md | 17 +++++++++++++++++ src/libcmd/repl-interacter.cc | 9 --------- src/libutil/experimental-features.cc | 2 +- 3 files changed, 18 insertions(+), 10 deletions(-) create mode 100644 doc/manual/rl-next/readline-support-removed.md diff --git a/doc/manual/rl-next/readline-support-removed.md b/doc/manual/rl-next/readline-support-removed.md new file mode 100644 index 000000000..6e4faeb67 --- /dev/null +++ b/doc/manual/rl-next/readline-support-removed.md @@ -0,0 +1,17 @@ +--- +synopsis: readline support removed +cls: [1885] +category: Packaging +credits: [9999years] +--- + +Support for building Lix with [`readline`][readline] instead of +[`editline`][editline] has been removed. `readline` support hasn't worked for a +long time (attempting to use it would lead to build errors) and would make Lix +subject to the GPL if it did work. In the future, we're hoping to replace +`editline` with [`rustyline`][rustyline] for improved ergonomics in the `nix +repl`. + +[readline]: https://en.wikipedia.org/wiki/GNU_Readline +[editline]: https://github.com/troglobit/editline +[rustyline]: https://github.com/kkawakam/rustyline diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 6979e3db4..459b048f4 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -8,10 +8,6 @@ #include #include -#ifdef READLINE -#include -#include -#else // editline < 1.15.2 don't wrap their API for C++ usage // (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). // This results in linker errors due to to name-mangling of editline C symbols. @@ -20,7 +16,6 @@ extern "C" { #include } -#endif #include "finally.hh" #include "repl-interacter.hh" @@ -115,17 +110,13 @@ ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleter } catch (SysError & e) { logWarning(e.info()); } -#ifndef READLINE el_hist_size = 1000; -#endif read_history(historyFile.c_str()); auto oldRepl = curRepl; curRepl = repl; Guard restoreRepl([oldRepl] { curRepl = oldRepl; }); -#ifndef READLINE rl_set_complete_func(completionCallback); rl_set_list_possib_func(listPossibleCallback); -#endif return restoreRepl; } diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 35982c28c..bb7a95a24 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -247,7 +247,7 @@ constexpr std::array xpFeatureDetails .tag = Xp::ReplAutomation, .name = "repl-automation", .description = R"( - Makes the repl not use readline/editline, print ENQ (U+0005) when ready for a command, and take commands followed by newline. + Makes the repl not use editline, print ENQ (U+0005) when ready for a command, and take commands followed by newline. )", }, }}; -- 2.44.1 From 4046e019ca6be35b583820df5c3f49e284259319 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Tue, 17 Sep 2024 18:34:01 -0700 Subject: [PATCH 042/106] tests/compression: rewrite This test suite was in desperate need of using the parameterization available with gtest, and was a bunch of useless duplicated code. At least now it's not duplicated code, though it still probably should be more full of property tests. Change-Id: Ia8ccee7ef4f02b2fa40417b79aa8c8f0626ea479 --- tests/unit/libutil/compression.cc | 232 +++++++++++++++++------------- 1 file changed, 132 insertions(+), 100 deletions(-) diff --git a/tests/unit/libutil/compression.cc b/tests/unit/libutil/compression.cc index 3b40db0cd..f629fb0a3 100644 --- a/tests/unit/libutil/compression.cc +++ b/tests/unit/libutil/compression.cc @@ -3,105 +3,137 @@ namespace nix { - /* ---------------------------------------------------------------------------- - * compress / decompress - * --------------------------------------------------------------------------*/ - - TEST(compress, compressWithUnknownMethod) { - ASSERT_THROW(compress("invalid-method", "something-to-compress"), UnknownCompressionMethod); - } - - TEST(compress, noneMethodDoesNothingToTheInput) { - auto o = compress("none", "this-is-a-test"); - - ASSERT_EQ(o, "this-is-a-test"); - } - - TEST(decompress, decompressNoneCompressed) { - auto method = "none"; - auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; - auto o = decompress(method, str); - - ASSERT_EQ(o, str); - } - - TEST(decompress, decompressEmptyCompressed) { - // Empty-method decompression used e.g. by S3 store - // (Content-Encoding == ""). - auto method = ""; - auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; - auto o = decompress(method, str); - - ASSERT_EQ(o, str); - } - - TEST(decompress, decompressXzCompressed) { - auto method = "xz"; - auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; - auto o = decompress(method, compress(method, str)); - - ASSERT_EQ(o, str); - } - - TEST(decompress, decompressBzip2Compressed) { - auto method = "bzip2"; - auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; - auto o = decompress(method, compress(method, str)); - - ASSERT_EQ(o, str); - } - - TEST(decompress, decompressBrCompressed) { - auto method = "br"; - auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; - auto o = decompress(method, compress(method, str)); - - ASSERT_EQ(o, str); - } - - TEST(decompress, decompressInvalidInputThrowsCompressionError) { - auto method = "bzip2"; - auto str = "this is a string that does not qualify as valid bzip2 data"; - - ASSERT_THROW(decompress(method, str), CompressionError); - } - - TEST(decompress, veryLongBrotli) { - auto method = "br"; - auto str = std::string(65536, 'a'); - auto o = decompress(method, compress(method, str)); - - // This is just to not print 64k of "a" for most failures - ASSERT_EQ(o.length(), str.length()); - ASSERT_EQ(o, str); - } - - /* ---------------------------------------------------------------------------- - * compression sinks - * --------------------------------------------------------------------------*/ - - TEST(makeCompressionSink, noneSinkDoesNothingToInput) { - StringSink strSink; - auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; - auto sink = makeCompressionSink("none", strSink); - (*sink)(inputString); - sink->finish(); - - ASSERT_STREQ(strSink.s.c_str(), inputString); - } - - TEST(makeCompressionSink, compressAndDecompress) { - auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; - - StringSink strSink; - auto sink = makeCompressionSink("bzip2", strSink); - (*sink)(inputString); - sink->finish(); - - StringSource strSource{strSink.s}; - auto decompressionSource = makeDecompressionSource("bzip2", strSource); - - ASSERT_STREQ(decompressionSource->drain().c_str(), inputString); - } +/* ---------------------------------------------------------------------------- + * compress / decompress + * --------------------------------------------------------------------------*/ +TEST(compress, compressWithUnknownMethod) +{ + ASSERT_THROW(compress("invalid-method", "something-to-compress"), UnknownCompressionMethod); +} + +TEST(compress, noneMethodDoesNothingToTheInput) +{ + auto o = compress("none", "this-is-a-test"); + + ASSERT_EQ(o, "this-is-a-test"); +} + +TEST(decompress, decompressEmptyString) +{ + // Empty-method decompression used e.g. by S3 store + // (Content-Encoding == ""). + auto o = decompress("", "this-is-a-test"); + + ASSERT_EQ(o, "this-is-a-test"); +} + +/* ---------------------------------------------------------------------------- + * compression sinks + * --------------------------------------------------------------------------*/ + +TEST(makeCompressionSink, noneSinkDoesNothingToInput) +{ + auto method = "none"; + StringSink strSink; + auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + auto sink = makeCompressionSink(method, strSink); + (*sink)(inputString); + sink->finish(); + + ASSERT_STREQ(strSink.s.c_str(), inputString); +} + +/** Tests applied to all compression types */ +class PerTypeCompressionTest : public testing::TestWithParam +{}; + +/** Tests applied to non-passthrough compression types */ +class PerTypeNonNullCompressionTest : public testing::TestWithParam +{}; + +constexpr const char * COMPRESSION_TYPES_NONNULL[] = { + // libarchive + "bzip2", + "compress", + "gzip", + "lzip", + "lzma", + "xz", + "zstd", + // Uses external program via libarchive so cannot be used :( + /* + "grzip", + "lrzip", + "lzop", + "lz4", + */ + // custom + "br", +}; + +INSTANTIATE_TEST_SUITE_P( + compressionNonNull, PerTypeNonNullCompressionTest, testing::ValuesIn(COMPRESSION_TYPES_NONNULL) +); +INSTANTIATE_TEST_SUITE_P( + compressionNonNull, PerTypeCompressionTest, testing::ValuesIn(COMPRESSION_TYPES_NONNULL) +); + +INSTANTIATE_TEST_SUITE_P( + compressionNull, PerTypeCompressionTest, testing::Values("none") +); + +/* --------------------------------------- + * All compression types + * --------------------------------------- */ + +TEST_P(PerTypeCompressionTest, roundTrips) +{ + auto method = GetParam(); + auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + auto o = decompress(method, compress(method, str)); + + ASSERT_EQ(o, str); +} + +TEST_P(PerTypeCompressionTest, longerThanBuffer) +{ + // This is targeted originally at regression testing a brotli bug, but we might as well do it to + // everything + auto method = GetParam(); + auto str = std::string(65536, 'a'); + auto o = decompress(method, compress(method, str)); + + // This is just to not print 64k of "a" for most failures + ASSERT_EQ(o.length(), str.length()); + ASSERT_EQ(o, str); +} + +TEST_P(PerTypeCompressionTest, sinkAndSource) +{ + auto method = GetParam(); + auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + + StringSink strSink; + auto sink = makeCompressionSink(method, strSink); + (*sink)(inputString); + sink->finish(); + + StringSource strSource{strSink.s}; + auto decompressionSource = makeDecompressionSource(method, strSource); + + ASSERT_STREQ(decompressionSource->drain().c_str(), inputString); +} + +/* --------------------------------------- + * Non null compression types + * --------------------------------------- */ + +TEST_P(PerTypeNonNullCompressionTest, bogusInputDecompression) +{ + auto param = GetParam(); + + auto bogus = "this data is bogus and should throw when decompressing"; + ASSERT_THROW(decompress(param, bogus), CompressionError); +} } -- 2.44.1 From ed381cd58a55f65e6b1aa1c17bebe28b1add7a50 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Tue, 17 Sep 2024 20:43:21 -0700 Subject: [PATCH 043/106] package.nix: fix cross for editline editline's dep on ncurses is a runtime one, so it should be in buildInputs, not nativeBuildInputs. CC: https://git.lix.systems/lix-project/lix/issues/527 Change-Id: I631c192a55677b0cc77faa7511986f1fa2205e91 --- package.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package.nix b/package.nix index 23ebc3a82..d71037f54 100644 --- a/package.nix +++ b/package.nix @@ -22,7 +22,6 @@ doxygen, editline-lix ? __forDefaults.editline-lix, editline, - fetchpatch, git, gtest, jq, @@ -100,7 +99,7 @@ (lib.enableFeature (ncurses != null) "termcap") ]; - nativeBuildInputs = (prev.nativeBuildInputs or [ ]) ++ [ ncurses ]; + buildInputs = (prev.buildInputs or [ ]) ++ [ ncurses ]; }); build-release-notes = callPackage ./maintainers/build-release-notes.nix { }; -- 2.44.1 From 2afdf1ed660d0592bf488fa1be68fe0666716421 Mon Sep 17 00:00:00 2001 From: Alois Wohlschlager Date: Wed, 18 Sep 2024 19:25:25 +0200 Subject: [PATCH 044/106] path-info: wipe the progress bar before printing The legitimate output of `nix path-info` may visually interfere with the progress bar, by appending to stale progress output before the latter has been erased. Conveniently, all expensive operations (evaluation or building) have already been performed before, so we can simply wipe the progress bar at this point to fix the issue. Fixes: https://git.lix.systems/lix-project/lix/issues/343 Change-Id: Id9a807a5c882295b3e6fbf841f9c15dc96f67f6e --- src/nix/path-info.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index b14eef467..05578ea53 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -82,6 +82,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON void run(ref store, StorePaths && storePaths) override { + // Wipe the progress bar to prevent interference with the output. + // It's not needed any more because expensive evaluation or builds are already done here. + logger->pause(); + size_t pathLen = 0; for (auto & storePath : storePaths) pathLen = std::max(pathLen, store->printStorePath(storePath).size()); -- 2.44.1 From 789b19a0cfe583586c85657e88d5933d2dbe5715 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Tue, 17 Sep 2024 18:27:22 -0700 Subject: [PATCH 045/106] util: fix brotli decompression of empty input This caused an infinite loop before since it would just keep asking the underlying source for more data. In practice this happened because an HTTP server served a response to a HEAD request (for which curl will not retrieve any body or call our write callback function) with Content-Encoding: br, leading to decompressing nothing at all and going into an infinite loop. This adds a test to make sure none of our compression methods do that again, as well as just patching the HTTP client to never feed empty data into a compression algorithm (since they absolutely have the right to throw CompressionError on unexpectedly-short streams!). Reported on Matrix: https://matrix.to/#/!lymvtcwDJ7ZA9Npq:lix.systems/$8BWQR_zKxCQDJ40C5NnDo4bQPId3pZ_aoDj2ANP7Itc?via=lix.systems&via=matrix.org&via=tchncs.de Change-Id: I027566e280f0f569fdb8df40e5ecbf46c211dad1 --- src/libstore/filetransfer.cc | 2 +- src/libutil/compression.cc | 30 +++++++++++++++++++++++++----- src/libutil/serialise.hh | 5 +++++ tests/unit/libutil/compression.cc | 24 ++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 6 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index f3e8a5312..10c810e49 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -337,7 +337,7 @@ struct curlFileTransfer : public FileTransfer // wrapping user `callback`s instead is not possible because the // Callback api expects std::functions, and copying Callbacks is // not possible due the promises they hold. - if (code == CURLE_OK && !dataCallback) { + if (code == CURLE_OK && !dataCallback && result.data.length() > 0) { result.data = decompress(encoding, result.data); } diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 5152a2146..51c820d55 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -144,6 +144,7 @@ struct BrotliDecompressionSource : Source std::unique_ptr buf; size_t avail_in = 0; const uint8_t * next_in; + std::exception_ptr inputEofException = nullptr; Source * inner; std::unique_ptr state; @@ -167,23 +168,42 @@ struct BrotliDecompressionSource : Source while (len && !BrotliDecoderIsFinished(state.get())) { checkInterrupt(); - while (avail_in == 0) { + while (avail_in == 0 && inputEofException == nullptr) { try { avail_in = inner->read(buf.get(), BUF_SIZE); } catch (EndOfFile &) { + // No more data, but brotli may still have output remaining + // from the last call. + inputEofException = std::current_exception(); break; } next_in = charptr_cast(buf.get()); } - if (!BrotliDecoderDecompressStream( - state.get(), &avail_in, &next_in, &len, &out, nullptr - )) - { + BrotliDecoderResult res = BrotliDecoderDecompressStream( + state.get(), &avail_in, &next_in, &len, &out, nullptr + ); + + switch (res) { + case BROTLI_DECODER_RESULT_SUCCESS: + // We're done here! + goto finish; + case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: + // Grab more input. Don't try if we already have exhausted our input stream. + if (inputEofException != nullptr) { + std::rethrow_exception(inputEofException); + } else { + continue; + } + case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: + // Need more output space: we can only get another buffer by someone calling us again, so get out. + goto finish; + case BROTLI_DECODER_RESULT_ERROR: throw CompressionError("error while decompressing brotli file"); } } +finish: if (begin != out) { return out - begin; } else { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 612658b2d..3a9685e0e 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -77,6 +77,11 @@ struct Source * Store up to ‘len’ in the buffer pointed to by ‘data’, and * return the number of bytes stored. It blocks until at least * one byte is available. + * + * Should not return 0 (generally you want to throw EndOfFile), but nothing + * stops that. + * + * \throws EndOfFile if there is no more data. */ virtual size_t read(char * data, size_t len) = 0; diff --git a/tests/unit/libutil/compression.cc b/tests/unit/libutil/compression.cc index f629fb0a3..8d181f53d 100644 --- a/tests/unit/libutil/compression.cc +++ b/tests/unit/libutil/compression.cc @@ -136,4 +136,28 @@ TEST_P(PerTypeNonNullCompressionTest, bogusInputDecompression) auto bogus = "this data is bogus and should throw when decompressing"; ASSERT_THROW(decompress(param, bogus), CompressionError); } + +TEST_P(PerTypeNonNullCompressionTest, truncatedValidInput) +{ + auto method = GetParam(); + + auto inputString = "the quick brown fox jumps over the lazy doggos"; + auto compressed = compress(method, inputString); + + /* n.b. This also tests zero-length input, which is also invalid. + * As of the writing of this comment, it returns empty output, but is + * allowed to throw a compression error instead. */ + for (int i = 0; i < compressed.length(); ++i) { + auto newCompressed = compressed.substr(compressed.length() - i); + try { + decompress(method, newCompressed); + // Success is acceptable as well, even though it is corrupt data. + // The compression method is not expected to provide integrity, + // just, not break explosively on bad input. + } catch (CompressionError &) { + // Acceptable + } + } +} + } -- 2.44.1 From 2f794733b2af153ff0f6b7ba3a963d84d63c0e03 Mon Sep 17 00:00:00 2001 From: Olivia Crain Date: Tue, 24 Sep 2024 13:10:06 -0500 Subject: [PATCH 046/106] internal-api-docs: allow Doxygen to build regardless of workdir Previously, Doxygen needed to be ran from the project's source root dir due to the relative paths in the config's `INPUT` tag. We now preprocess the relative paths by prefixing them with the absolute path of the project's source root dir. The HTML output remains unchanged. Fixes: https://git.lix.systems/lix-project/lix/issues/240 Change-Id: I85f099c22bfc5fdbf26be27c2db7dcbc8155c8b2 --- doc/internal-api/doxygen.cfg.in | 39 ++++++++++------------------ doc/internal-api/meson.build | 46 ++++++++++++++++++++++++++------- 2 files changed, 49 insertions(+), 36 deletions(-) diff --git a/doc/internal-api/doxygen.cfg.in b/doc/internal-api/doxygen.cfg.in index 73fba6948..662fb4333 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/doc/internal-api/doxygen.cfg.in @@ -33,32 +33,7 @@ GENERATE_LATEX = NO # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -# FIXME Make this list more maintainable somehow. We could maybe generate this -# in the Makefile, but we would need to change how `.in` files are preprocessed -# so they can expand variables despite configure variables. - -INPUT = \ - src/libcmd \ - src/libexpr \ - src/libexpr/flake \ - tests/unit/libexpr \ - tests/unit/libexpr/value \ - tests/unit/libexpr/test \ - tests/unit/libexpr/test/value \ - src/libexpr/value \ - src/libfetchers \ - src/libmain \ - src/libstore \ - src/libstore/build \ - src/libstore/builtins \ - tests/unit/libstore \ - tests/unit/libstore/test \ - src/libutil \ - tests/unit/libutil \ - tests/unit/libutil/test \ - src/nix \ - src/nix-env \ - src/nix-store +INPUT = @INPUT_PATHS@ # If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names # in the source code. If set to NO, only conditional compilation will be @@ -97,3 +72,15 @@ EXPAND_AS_DEFINED = \ DECLARE_WORKER_SERIALISER \ DECLARE_SERVE_SERIALISER \ LENGTH_PREFIXED_PROTO_HELPER + +# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. +# Stripping is only done if one of the specified strings matches the left-hand +# part of the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the path to +# strip. +# +# Note that you can specify absolute paths here, but also relative paths, which +# will be relative from the directory where doxygen is started. +# This tag requires that the tag FULL_PATH_NAMES is set to YES. + +STRIP_FROM_PATH = "@PROJECT_SOURCE_ROOT@" diff --git a/doc/internal-api/meson.build b/doc/internal-api/meson.build index faa30f194..af93b6943 100644 --- a/doc/internal-api/meson.build +++ b/doc/internal-api/meson.build @@ -1,3 +1,35 @@ +internal_api_sources = [ + 'src/libcmd', + 'src/libexpr', + 'src/libexpr/flake', + 'tests/unit/libexpr', + 'tests/unit/libexpr/value', + 'tests/unit/libexpr/test', + 'tests/unit/libexpr/test/value', + 'src/libexpr/value', + 'src/libfetchers', + 'src/libmain', + 'src/libstore', + 'src/libstore/build', + 'src/libstore/builtins', + 'tests/unit/libstore', + 'tests/unit/libstore/test', + 'src/libutil', + 'tests/unit/libutil', + 'tests/unit/libutil/test', + 'src/nix', + 'src/nix-env', + 'src/nix-store', +] + +# We feed Doxygen absolute paths so it can be invoked from any working directory. +internal_api_sources_absolute = [] +foreach src : internal_api_sources + internal_api_sources_absolute += '"' + (meson.project_source_root() / src) + '"' +endforeach + +internal_api_sources_oneline = ' \\\n '.join(internal_api_sources_absolute) + doxygen_cfg = configure_file( input : 'doxygen.cfg.in', output : 'doxygen.cfg', @@ -5,22 +37,16 @@ doxygen_cfg = configure_file( 'PACKAGE_VERSION': meson.project_version(), 'RAPIDCHECK_HEADERS': rapidcheck_meson.get_variable('includedir'), 'docdir' : meson.current_build_dir(), + 'INPUT_PATHS' : internal_api_sources_oneline, + 'PROJECT_SOURCE_ROOT' : meson.project_source_root(), }, ) internal_api_docs = custom_target( 'internal-api-docs', command : [ - bash, - # Meson can you please just give us a `workdir` argument to custom targets... - '-c', - # We have to prefix the doxygen_cfg path with the project build root - # because of the cd in front. - 'cd @0@ && @1@ @2@/@INPUT0@'.format( - meson.project_source_root(), - doxygen.full_path(), - meson.project_build_root(), - ), + doxygen.full_path(), + '@INPUT0@', ], input : [ doxygen_cfg, -- 2.44.1 From eccbe9586a12316252e64ee3db3a453fc19777e4 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Wed, 25 Sep 2024 18:31:34 +0200 Subject: [PATCH 047/106] flake: use clangStdenv for `overlays.default` We don't support GCC anymore for building, so the overlay currently fails to evaluate with error: assertion '((stdenv).cc.isClang || lintInsteadOfBuild)' failed `clangStdenv` seems like a reasonable default now. Noticed while upgrading Lix for our Hydra fork. Change-Id: I948a7c03b3e5648fc7c596f96e1b8053a9e7f92f --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 46f9e43e8..f77f848ca 100644 --- a/flake.nix +++ b/flake.nix @@ -217,7 +217,7 @@ # A Nixpkgs overlay that overrides the 'nix' and # 'nix.perl-bindings' packages. - overlays.default = overlayFor (p: p.stdenv); + overlays.default = overlayFor (p: p.clangStdenv); hydraJobs = { # Binary package for various platforms. -- 2.44.1 From 19e0ce2c03d8e0baa16998b086665664c420c1df Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Sat, 24 Aug 2024 21:11:30 -0700 Subject: [PATCH 048/106] main: log stack traces for std::terminate These stack traces kind of suck for the reasons mentioned on the CppTrace page here (no symbols for inline functions is a major one): https://github.com/jeremy-rifkin/cpptrace I would consider using CppTrace if it were packaged, but to be honest, I think that the more reasonable option is actually to move entirely to out-of-process crash handling and symbolization. The reason for this is that if you want to generate anything of substance on SIGSEGV or really any deadly signal, you are stuck in async-signal-safe land, which is not a place to be trying to run a symbolizer. LLVM does it anyway, probably carefully, and chromium *can* do it on debug builds but in general uses crashpad: https://source.chromium.org/chromium/chromium/src/+/main:base/debug/stack_trace_posix.cc;l=974;drc=82dff63dbf9db05e9274e11d9128af7b9f51ceaa;bpv=1;bpt=1 However, some stack traces are better than *no* stack traces when we get mystery exceptions falling out the bottom of the program. I've also promoted the path for "mystery exceptions falling out the bottom of the program" to hard crash and generate a core dump because although there's been some months since the last one of these, these are nonetheless always *atrociously* diagnosed. We can't improve the crash handling further until either we use Crashpad (which involves more C++ deps, no thanks) or we put in the ostensibly work in progress Rust minidump infrastructure, in which case we need to finish full support for Rust in libutil first. Sample report: Lix crashed. This is a bug. We would appreciate if you report it at https://git.lix.systems/lix-project/lix/issues with the following information included: Exception: std::runtime_error: lol Stack trace: 0# nix::printStackTrace() in /home/jade/lix/lix3/build/src/nix/../libutil/liblixutil.so 1# 0x000073C9862331F2 in /home/jade/lix/lix3/build/src/nix/../libmain/liblixmain.so 2# 0x000073C985F2E21A in /nix/store/p44qan69linp3ii0xrviypsw2j4qdcp2-gcc-13.2.0-lib/lib/libstdc++.so.6 3# 0x000073C985F2E285 in /nix/store/p44qan69linp3ii0xrviypsw2j4qdcp2-gcc-13.2.0-lib/lib/libstdc++.so.6 4# nix::handleExceptions(std::__cxx11::basic_string, std::allocator > const&, std::function) in /home/jade/lix/lix3/build/src/nix/../libmain/liblixmain.so 5# 0x00005CF65B6B048B in /home/jade/lix/lix3/build/src/nix/nix 6# 0x000073C985C8810E in /nix/store/dbcw19dshdwnxdv5q2g6wldj6syyvq7l-glibc-2.39-52/lib/libc.so.6 7# __libc_start_main in /nix/store/dbcw19dshdwnxdv5q2g6wldj6syyvq7l-glibc-2.39-52/lib/libc.so.6 8# 0x00005CF65B610335 in /home/jade/lix/lix3/build/src/nix/nix Change-Id: I1a9f6d349b617fd7145a37159b78ecb9382cb4e9 --- doc/manual/rl-next/stack-traces.md | 26 ++++++++++++++ src/libmain/crash-handler.cc | 41 ++++++++++++++++++++++ src/libmain/crash-handler.hh | 21 +++++++++++ src/libmain/meson.build | 2 ++ src/libmain/shared.cc | 14 +++++--- src/libmain/shared.hh | 2 +- src/libstore/globals.cc | 2 +- tests/unit/libmain/crash.cc | 56 ++++++++++++++++++++++++++++++ tests/unit/meson.build | 7 +++- 9 files changed, 164 insertions(+), 7 deletions(-) create mode 100644 doc/manual/rl-next/stack-traces.md create mode 100644 src/libmain/crash-handler.cc create mode 100644 src/libmain/crash-handler.hh create mode 100644 tests/unit/libmain/crash.cc diff --git a/doc/manual/rl-next/stack-traces.md b/doc/manual/rl-next/stack-traces.md new file mode 100644 index 000000000..e16d6c886 --- /dev/null +++ b/doc/manual/rl-next/stack-traces.md @@ -0,0 +1,26 @@ +--- +synopsis: "Some Lix crashes now produce reporting instructions and a stack trace, then abort" +cls: [1854] +category: Improvements +credits: jade +--- + +Lix, being a C++ program, can crash in a few kinds of ways. +It can obviously do a memory access violation, which will generate a core dump and thus be relatively debuggable. +But, worse, it could throw an unhandled exception, and, in the past, we would just show the message but not where it comes from, in spite of this always being a bug, since we expect all such errors to be translated to a Lix specific error. +Now the latter kind of bug should print reporting instructions, a rudimentary stack trace and (depending on system configuration) generate a core dump. + +Sample output: + +``` +Lix crashed. This is a bug. We would appreciate if you report it along with what caused it at https://git.lix.systems/lix-project/lix/issues with the following information included: + +Exception: std::runtime_error: test exception +Stack trace: + 0# nix::printStackTrace() in /home/jade/lix/lix3/build/src/nix/../libutil/liblixutil.so + 1# 0x000073C9862331F2 in /home/jade/lix/lix3/build/src/nix/../libmain/liblixmain.so + 2# 0x000073C985F2E21A in /nix/store/p44qan69linp3ii0xrviypsw2j4qdcp2-gcc-13.2.0-lib/lib/libstdc++.so.6 + 3# 0x000073C985F2E285 in /nix/store/p44qan69linp3ii0xrviypsw2j4qdcp2-gcc-13.2.0-lib/lib/libstdc++.so.6 + 4# nix::handleExceptions(std::__cxx11::basic_string, std::allocator > const&, std::function) in /home/jade/lix/lix3/build/src/nix/../libmain/liblixmain.so + ... +``` diff --git a/src/libmain/crash-handler.cc b/src/libmain/crash-handler.cc new file mode 100644 index 000000000..3f1b9f7d8 --- /dev/null +++ b/src/libmain/crash-handler.cc @@ -0,0 +1,41 @@ +#include "crash-handler.hh" +#include "fmt.hh" + +#include +#include + +namespace nix { + +namespace { +void onTerminate() +{ + std::cerr << "Lix crashed. This is a bug. We would appreciate if you report it along with what caused it at https://git.lix.systems/lix-project/lix/issues with the following information included:\n\n"; + try { + std::exception_ptr eptr = std::current_exception(); + if (eptr) { + std::rethrow_exception(eptr); + } else { + std::cerr << "std::terminate() called without exception\n"; + } + } catch (const std::exception & ex) { + std::cerr << "Exception: " << boost::core::demangle(typeid(ex).name()) << ": " << ex.what() << "\n"; + } catch (...) { + std::cerr << "Unknown exception! Spooky.\n"; + } + + std::cerr << "Stack trace:\n"; + nix::printStackTrace(); + + std::abort(); +} +} + +void registerCrashHandler() +{ + // DO NOT use this for signals. Boost stacktrace is very much not + // async-signal-safe, and in a world with ASLR, addr2line is pointless. + // + // If you want signals, set up a minidump system and do it out-of-process. + std::set_terminate(onTerminate); +} +} diff --git a/src/libmain/crash-handler.hh b/src/libmain/crash-handler.hh new file mode 100644 index 000000000..4c5641b8c --- /dev/null +++ b/src/libmain/crash-handler.hh @@ -0,0 +1,21 @@ +#pragma once +/// @file Crash handler for Lix that prints back traces (hopefully in instances where it is not just going to crash the process itself). +/* + * Author's note: This will probably be partially/fully supplanted by a + * minidump writer like the following once we get our act together on crashes a + * little bit more: + * https://github.com/rust-minidump/minidump-writer + * https://github.com/EmbarkStudios/crash-handling + * (out of process implementation *should* be able to be done on-demand) + * + * Such an out-of-process implementation could then both make minidumps and + * print stack traces for arbitrarily messed-up process states such that we can + * safely give out backtraces for SIGSEGV and other deadly signals. + */ + +namespace nix { + +/** Registers the Lix crash handler for std::terminate (currently; will support more crashes later). See also detectStackOverflow(). */ +void registerCrashHandler(); + +} diff --git a/src/libmain/meson.build b/src/libmain/meson.build index a7cce287c..a1a888c16 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -1,5 +1,6 @@ libmain_sources = files( 'common-args.cc', + 'crash-handler.cc', 'loggers.cc', 'progress-bar.cc', 'shared.cc', @@ -8,6 +9,7 @@ libmain_sources = files( libmain_headers = files( 'common-args.hh', + 'crash-handler.hh', 'loggers.hh', 'progress-bar.hh', 'shared.hh', diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index bc9548e09..64bd00606 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -1,3 +1,4 @@ +#include "crash-handler.hh" #include "globals.hh" #include "shared.hh" #include "store-api.hh" @@ -118,6 +119,8 @@ static void sigHandler(int signo) { } void initNix() { + registerCrashHandler(); + /* Turn on buffering for cerr. */ static char buf[1024]; std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); @@ -335,12 +338,15 @@ int handleExceptions(const std::string & programName, std::function fun) } catch (BaseError & e) { logError(e.info()); return e.info().status; - } catch (std::bad_alloc & e) { + } catch (const std::bad_alloc & e) { printError(error + "out of memory"); return 1; - } catch (std::exception & e) { - printError(error + e.what()); - return 1; + } catch (const std::exception & e) { + // Random exceptions bubbling into main are cause for bug reports, crash + std::terminate(); + } catch (...) { + // Explicitly do not tolerate non-std exceptions escaping. + std::terminate(); } return 0; diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index b41efe567..49b72a54e 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -111,7 +111,7 @@ struct PrintFreed /** - * Install a SIGSEGV handler to detect stack overflows. + * Install a SIGSEGV handler to detect stack overflows. See also registerCrashHandler(). */ void detectStackOverflow(); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index ffc2543ef..f43b759d2 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -443,7 +443,7 @@ static bool initLibStoreDone = false; void assertLibStoreInitialized() { if (!initLibStoreDone) { printError("The program must call nix::initNix() before calling any libstore library functions."); - abort(); + std::terminate(); }; } diff --git a/tests/unit/libmain/crash.cc b/tests/unit/libmain/crash.cc new file mode 100644 index 000000000..883dc39bd --- /dev/null +++ b/tests/unit/libmain/crash.cc @@ -0,0 +1,56 @@ +#include +#include "crash-handler.hh" + +namespace nix { + +class OopsException : public std::exception +{ + const char * msg; + +public: + OopsException(const char * msg) : msg(msg) {} + const char * what() const noexcept override + { + return msg; + } +}; + +void causeCrashForTesting(std::function fixture) +{ + registerCrashHandler(); + std::cerr << "time to crash\n"; + try { + fixture(); + } catch (...) { + std::terminate(); + } +} + +TEST(CrashHandler, exceptionName) +{ + ASSERT_DEATH( + causeCrashForTesting([]() { throw OopsException{"lol oops"}; }), + "time to crash\nLix crashed.*OopsException: lol oops" + ); +} + +TEST(CrashHandler, unknownTerminate) +{ + ASSERT_DEATH( + causeCrashForTesting([]() { std::terminate(); }), + "time to crash\nLix crashed.*std::terminate\\(\\) called without exception" + ); +} + +TEST(CrashHandler, nonStdException) +{ + ASSERT_DEATH( + causeCrashForTesting([]() { + // NOLINTNEXTLINE(hicpp-exception-baseclass): intentional + throw 4; + }), + "time to crash\nLix crashed.*Unknown exception! Spooky\\." + ); +} + +} diff --git a/tests/unit/meson.build b/tests/unit/meson.build index 8ff0b5ec5..894f48b2a 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -262,9 +262,14 @@ test( protocol : 'gtest', ) +libmain_tests_sources = files( + 'libmain/crash.cc', + 'libmain/progress-bar.cc', +) + libmain_tester = executable( 'liblixmain-tests', - files('libmain/progress-bar.cc'), + libmain_tests_sources, dependencies : [ liblixmain, liblixexpr, -- 2.44.1 From aca19187d0f247de09e79c5c223f88ae82e44b45 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Wed, 25 Sep 2024 15:17:40 -0700 Subject: [PATCH 049/106] fmt: fail hard on bad format strings going into nix::fmt too Previously we would only crash the program for bad HintFmt calls. nix::fmt should also crash. Change-Id: I4ba0abeb8557b208bd9c0be624c022a60446ef7e --- src/libutil/fmt.hh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index ee3e1e2e7..5feefdf90 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -136,11 +136,17 @@ inline std::string fmt(const char * s) template inline std::string fmt(const std::string & fs, const Args &... args) -{ +try { boost::format f(fs); fmt_internal::setExceptions(f); (f % ... % args); return f.str(); +} catch (boost::io::format_error & fe) { + // I don't care who catches this, we do not put up with boost format errors + // Give me a stack trace and a core dump + std::cerr << "nix::fmt threw format error. Original format string: '"; + std::cerr << fs << "'; number of arguments: " << sizeof...(args) << "\n"; + std::terminate(); } /** @@ -174,15 +180,13 @@ public: std::cerr << "HintFmt received incorrect number of format args. Original format string: '"; std::cerr << format << "'; number of arguments: " << sizeof...(args) << "\n"; // And regardless of the coredump give me a damn stacktrace. - printStackTrace(); - abort(); + std::terminate(); } } catch (boost::io::format_error & ex) { // Same thing, but for anything that happens in the member initializers. std::cerr << "HintFmt received incorrect format string. Original format string: '"; std::cerr << format << "'; number of arguments: " << sizeof...(args) << "\n"; - printStackTrace(); - abort(); + std::terminate(); } HintFmt(const HintFmt & hf) : fmt(hf.fmt) {} -- 2.44.1 From c1631b0a39d34267345b41214f1f5e8a77d98cd2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Sep 2024 15:09:44 +0200 Subject: [PATCH 050/106] [security] builtin:fetchurl: Enable TLS verification This is better for privacy and to avoid leaking netrc credentials in a MITM attack, but also the assumption that we check the hash no longer holds in some cases (in particular for impure derivations). Partially reverts https://github.com/NixOS/nix/commit/5db358d4d78aea7204a8f22c5bf2a309267ee038. (cherry picked from commit c04bc17a5a0fdcb725a11ef6541f94730112e7b6) (cherry picked from commit f2f47fa725fc87bfb536de171a2ea81f2789c9fb) (cherry picked from commit 7b39cd631e0d3c3d238015c6f450c59bbc9cbc5b) Upstream-PR: https://github.com/NixOS/nix/pull/11585 Change-Id: Ia973420f6098113da05a594d48394ce1fe41fbb9 --- doc/manual/rl-next/verify-tls.md | 10 ++++ src/libstore/builtins/fetchurl.cc | 3 -- tests/nixos/default.nix | 2 + tests/nixos/fetchurl.nix | 78 +++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 doc/manual/rl-next/verify-tls.md create mode 100644 tests/nixos/fetchurl.nix diff --git a/doc/manual/rl-next/verify-tls.md b/doc/manual/rl-next/verify-tls.md new file mode 100644 index 000000000..608f3347e --- /dev/null +++ b/doc/manual/rl-next/verify-tls.md @@ -0,0 +1,10 @@ +--- +synopsis: "`` now uses TLS verification" +category: Fixes +prs: [11585] +credits: edolstra +--- + +Previously `` did not do TLS verification. This was because the Nix sandbox in the past did not have access to TLS certificates, and Nix checks the hash of the fetched file anyway. However, this can expose authentication data from `netrc` and URLs to man-in-the-middle attackers. In addition, Nix now in some cases (such as when using impure derivations) does *not* check the hash. Therefore we have now enabled TLS verification. This means that downloads by `` will now fail if you're fetching from a HTTPS server that does not have a valid certificate. + +`` is also known as the builtin derivation builder `builtin:fetchurl`. It's not to be confused with the evaluation-time function `builtins.fetchurl`, which was not affected by this issue. diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 062ecdc14..3fb769fe6 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -33,10 +33,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto fetch = [&](const std::string & url) { - /* No need to do TLS verification, because we check the hash of - the result anyway. */ FileTransferRequest request(url); - request.verifyTLS = false; auto raw = fileTransfer->download(std::move(request)); auto decompressor = makeDecompressionSource( diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 20e66f6c1..2d6eaed16 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -157,4 +157,6 @@ in coredumps = runNixOSTestFor "x86_64-linux" ./coredumps; io_uring = runNixOSTestFor "x86_64-linux" ./io_uring; + + fetchurl = runNixOSTestFor "x86_64-linux" ./fetchurl.nix; } diff --git a/tests/nixos/fetchurl.nix b/tests/nixos/fetchurl.nix new file mode 100644 index 000000000..63c639c31 --- /dev/null +++ b/tests/nixos/fetchurl.nix @@ -0,0 +1,78 @@ +# Test whether builtin:fetchurl properly performs TLS certificate +# checks on HTTPS servers. + +{ lib, config, pkgs, ... }: + +let + + makeTlsCert = name: pkgs.runCommand name { + nativeBuildInputs = with pkgs; [ openssl ]; + } '' + mkdir -p $out + openssl req -x509 \ + -subj '/CN=${name}/' -days 49710 \ + -addext 'subjectAltName = DNS:${name}' \ + -keyout "$out/key.pem" -newkey ed25519 \ + -out "$out/cert.pem" -noenc + ''; + + goodCert = makeTlsCert "good"; + badCert = makeTlsCert "bad"; + +in + +{ + name = "fetchurl"; + + nodes = { + machine = { lib, pkgs, ... }: { + services.nginx = { + enable = true; + + virtualHosts."good" = { + addSSL = true; + sslCertificate = "${goodCert}/cert.pem"; + sslCertificateKey = "${goodCert}/key.pem"; + root = pkgs.runCommand "nginx-root" {} '' + mkdir "$out" + echo 'hello world' > "$out/index.html" + ''; + }; + + virtualHosts."bad" = { + addSSL = true; + sslCertificate = "${badCert}/cert.pem"; + sslCertificateKey = "${badCert}/key.pem"; + root = pkgs.runCommand "nginx-root" {} '' + mkdir "$out" + echo 'foobar' > "$out/index.html" + ''; + }; + }; + + security.pki.certificateFiles = [ "${goodCert}/cert.pem" ]; + + networking.hosts."127.0.0.1" = [ "good" "bad" ]; + + virtualisation.writableStore = true; + + nix.settings.experimental-features = "nix-command"; + }; + }; + + testScript = { nodes, ... }: '' + machine.wait_for_unit("nginx") + machine.wait_for_open_port(443) + + out = machine.succeed("curl https://good/index.html") + assert out == "hello world\n" + + # Fetching from a server with a trusted cert should work. + machine.succeed("nix build --no-substitute --expr 'import { url = \"https://good/index.html\"; hash = \"sha256-qUiQTy8PR5uPgZdpSzAYSw0u0cHNKh7A+4XSmaGSpEc=\"; }'") + + # Fetching from a server with an untrusted cert should fail. + err = machine.fail("nix build --no-substitute --expr 'import { url = \"https://bad/index.html\"; hash = \"sha256-rsBwZF/lPuOzdjBZN2E08FjMM3JHyXit0Xi2zN+wAZ8=\"; }' 2>&1") + print(err) + assert "SSL certificate problem: self-signed certificate" in err or "SSL peer certificate or SSH remote key was not OK" in err + ''; +} -- 2.44.1 From 37b22dae04f2da214e6b9bef3427e134280642ca Mon Sep 17 00:00:00 2001 From: Puck Meerburg Date: Thu, 26 Sep 2024 15:12:24 +0000 Subject: [PATCH 051/106] Fix passing custom CA files into the builtin:fetchurl sandbox Without this, verifying TLS certificates would fail on macOS, as well as any system that doesn't have a certificate file at /etc/ssl/certs/ca-certificates.crt, which includes e.g. Fedora. Change-Id: Iaa2e0e9db3747645b5482c82e3e0e4e8f229f5f9 --- src/libstore/build/local-derivation-goal.cc | 19 +++++++++++++------ src/libstore/builtins.hh | 2 +- src/libstore/builtins/fetchurl.cc | 5 ++++- tests/nixos/fetchurl.nix | 6 ++++++ 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 4baa525d9..f14d09652 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1361,13 +1361,20 @@ void LocalDerivationGoal::runChild() bool setUser = true; - /* Make the contents of netrc available to builtin:fetchurl - (which may run under a different uid and/or in a sandbox). */ + /* Make the contents of netrc and the CA certificate bundle + available to builtin:fetchurl (which may run under a + different uid and/or in a sandbox). */ std::string netrcData; - try { - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl" && !derivationType->isSandboxed()) + std::string caFileData; + if (drv->isBuiltin() && drv->builder == "builtin:fetchurl" && !derivationType->isSandboxed()) { + try { netrcData = readFile(settings.netrcFile); - } catch (SysError &) { } + } catch (SysError &) { } + + try { + caFileData = readFile(settings.caFile); + } catch (SysError &) { } + } #if __linux__ if (useChroot) { @@ -1802,7 +1809,7 @@ void LocalDerivationGoal::runChild() e.second = rewriteStrings(e.second, inputRewrites); if (drv->builder == "builtin:fetchurl") - builtinFetchurl(drv2, netrcData); + builtinFetchurl(drv2, netrcData, caFileData); else if (drv->builder == "builtin:buildenv") builtinBuildenv(drv2); else if (drv->builder == "builtin:unpack-channel") diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index d201fb3ac..e20d14b90 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -6,7 +6,7 @@ namespace nix { // TODO: make pluggable. -void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); +void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData, const std::string & caFileData); void builtinUnpackChannel(const BasicDerivation & drv); } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 3fb769fe6..b28eb01d0 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -7,7 +7,7 @@ namespace nix { -void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) +void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData, const std::string & caFileData) { /* Make the host's netrc data available. Too bad curl requires this to be stored in a file. It would be nice if we could just @@ -17,6 +17,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) writeFile(settings.netrcFile, netrcData, 0600); } + settings.caFile = "ca-certificates.crt"; + writeFile(settings.caFile, caFileData, 0600); + auto getAttr = [&](const std::string & name) { auto i = drv.env.find(name); if (i == drv.env.end()) throw Error("attribute '%s' missing", name); diff --git a/tests/nixos/fetchurl.nix b/tests/nixos/fetchurl.nix index 63c639c31..97365d053 100644 --- a/tests/nixos/fetchurl.nix +++ b/tests/nixos/fetchurl.nix @@ -67,6 +67,9 @@ in out = machine.succeed("curl https://good/index.html") assert out == "hello world\n" + out = machine.succeed("cat ${badCert}/cert.pem > /tmp/cafile.pem; curl --cacert /tmp/cafile.pem https://bad/index.html") + assert out == "foobar\n" + # Fetching from a server with a trusted cert should work. machine.succeed("nix build --no-substitute --expr 'import { url = \"https://good/index.html\"; hash = \"sha256-qUiQTy8PR5uPgZdpSzAYSw0u0cHNKh7A+4XSmaGSpEc=\"; }'") @@ -74,5 +77,8 @@ in err = machine.fail("nix build --no-substitute --expr 'import { url = \"https://bad/index.html\"; hash = \"sha256-rsBwZF/lPuOzdjBZN2E08FjMM3JHyXit0Xi2zN+wAZ8=\"; }' 2>&1") print(err) assert "SSL certificate problem: self-signed certificate" in err or "SSL peer certificate or SSH remote key was not OK" in err + + # Fetching from a server with a trusted cert should work via environment variable override. + machine.succeed("NIX_SSL_CERT_FILE=/tmp/cafile.pem nix build --no-substitute --expr 'import { url = \"https://bad/index.html\"; hash = \"sha256-rsBwZF/lPuOzdjBZN2E08FjMM3JHyXit0Xi2zN+wAZ8=\"; }'") ''; } -- 2.44.1 From 4b66e1e24f4952a2e96d680c6b37de2c2c1c76d1 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Tue, 24 Sep 2024 00:21:16 +0200 Subject: [PATCH 052/106] fix internal-api-docs build this one is also run from a gcc stdenv. Change-Id: I91ff6915c6689ece15224f348f54367cff5d2b5a --- package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.nix b/package.nix index d71037f54..ff862bf92 100644 --- a/package.nix +++ b/package.nix @@ -110,7 +110,7 @@ }: # gcc miscompiles coroutines at least until 13.2, possibly longer -assert stdenv.cc.isClang || lintInsteadOfBuild; +assert stdenv.cc.isClang || lintInsteadOfBuild || internalApiDocs; let inherit (__forDefaults) canRunInstalled; -- 2.44.1 From ca9256a789b413b71f424405c8a0d7d37ca36696 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Tue, 24 Sep 2024 00:21:16 +0200 Subject: [PATCH 053/106] libutil: add an async semaphore implementation like a normal semaphore, but with awaitable acquire actions. this is primarily intended as an intermediate concurrency limiting device in the Worker code, but it may find other uses over time. we do not use std::counting_semaphore as a base because the counter of that is not inspectable as will be needed for Worker. we also do not need atomic operations for cross-thread consistency since we don't have multiple threads (thanks to kj event loops being confined to a single thread) Change-Id: Ie2bcb107f3a2c0185138330f7cbba4cec6cbdd95 --- src/libutil/async-semaphore.hh | 122 ++++++++++++++++++++++++++ src/libutil/meson.build | 1 + tests/unit/libutil/async-semaphore.cc | 74 ++++++++++++++++ tests/unit/meson.build | 2 + 4 files changed, 199 insertions(+) create mode 100644 src/libutil/async-semaphore.hh create mode 100644 tests/unit/libutil/async-semaphore.cc diff --git a/src/libutil/async-semaphore.hh b/src/libutil/async-semaphore.hh new file mode 100644 index 000000000..f8db31a68 --- /dev/null +++ b/src/libutil/async-semaphore.hh @@ -0,0 +1,122 @@ +#pragma once +/// @file +/// @brief A semaphore implementation usable from within a KJ event loop. + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace nix { + +class AsyncSemaphore +{ +public: + class [[nodiscard("destroying a semaphore guard releases the semaphore immediately")]] Token + { + struct Release + { + void operator()(AsyncSemaphore * sem) const + { + sem->unsafeRelease(); + } + }; + + std::unique_ptr parent; + + public: + Token() = default; + Token(AsyncSemaphore & parent, kj::Badge) : parent(&parent) {} + + bool valid() const + { + return parent != nullptr; + } + }; + +private: + struct Waiter + { + kj::PromiseFulfiller & fulfiller; + kj::ListLink link; + kj::List & list; + + Waiter(kj::PromiseFulfiller & fulfiller, kj::List & list) + : fulfiller(fulfiller) + , list(list) + { + list.add(*this); + } + + ~Waiter() + { + if (link.isLinked()) { + list.remove(*this); + } + } + }; + + const unsigned capacity_; + unsigned used_ = 0; + kj::List waiters; + + void unsafeRelease() + { + used_ -= 1; + while (used_ < capacity_ && !waiters.empty()) { + used_ += 1; + auto & w = waiters.front(); + w.fulfiller.fulfill(Token{*this, {}}); + waiters.remove(w); + } + } + +public: + explicit AsyncSemaphore(unsigned capacity) : capacity_(capacity) {} + + KJ_DISALLOW_COPY_AND_MOVE(AsyncSemaphore); + + ~AsyncSemaphore() + { + assert(waiters.empty() && "destroyed a semaphore with active waiters"); + } + + std::optional tryAcquire() + { + if (used_ < capacity_) { + used_ += 1; + return Token{*this, {}}; + } else { + return {}; + } + } + + kj::Promise acquire() + { + if (auto t = tryAcquire()) { + return std::move(*t); + } else { + return kj::newAdaptedPromise(waiters); + } + } + + unsigned capacity() const + { + return capacity_; + } + + unsigned used() const + { + return used_; + } + + unsigned available() const + { + return capacity_ - used_; + } +}; +} diff --git a/src/libutil/meson.build b/src/libutil/meson.build index a3f21de59..89eeed133 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -53,6 +53,7 @@ libutil_headers = files( 'archive.hh', 'args/root.hh', 'args.hh', + 'async-semaphore.hh', 'backed-string-view.hh', 'box_ptr.hh', 'canon-path.hh', diff --git a/tests/unit/libutil/async-semaphore.cc b/tests/unit/libutil/async-semaphore.cc new file mode 100644 index 000000000..12b52885d --- /dev/null +++ b/tests/unit/libutil/async-semaphore.cc @@ -0,0 +1,74 @@ +#include "async-semaphore.hh" + +#include +#include + +namespace nix { + +TEST(AsyncSemaphore, counting) +{ + kj::EventLoop loop; + kj::WaitScope waitScope(loop); + + AsyncSemaphore sem(2); + + ASSERT_EQ(sem.available(), 2); + ASSERT_EQ(sem.used(), 0); + + auto a = kj::evalNow([&] { return sem.acquire(); }); + ASSERT_EQ(sem.available(), 1); + ASSERT_EQ(sem.used(), 1); + auto b = kj::evalNow([&] { return sem.acquire(); }); + ASSERT_EQ(sem.available(), 0); + ASSERT_EQ(sem.used(), 2); + + auto c = kj::evalNow([&] { return sem.acquire(); }); + auto d = kj::evalNow([&] { return sem.acquire(); }); + + ASSERT_TRUE(a.poll(waitScope)); + ASSERT_TRUE(b.poll(waitScope)); + ASSERT_FALSE(c.poll(waitScope)); + ASSERT_FALSE(d.poll(waitScope)); + + a = nullptr; + ASSERT_TRUE(c.poll(waitScope)); + ASSERT_FALSE(d.poll(waitScope)); + + { + auto lock = b.wait(waitScope); + ASSERT_FALSE(d.poll(waitScope)); + } + + ASSERT_TRUE(d.poll(waitScope)); + + ASSERT_EQ(sem.available(), 0); + ASSERT_EQ(sem.used(), 2); + c = nullptr; + ASSERT_EQ(sem.available(), 1); + ASSERT_EQ(sem.used(), 1); + d = nullptr; + ASSERT_EQ(sem.available(), 2); + ASSERT_EQ(sem.used(), 0); +} + +TEST(AsyncSemaphore, cancelledWaiter) +{ + kj::EventLoop loop; + kj::WaitScope waitScope(loop); + + AsyncSemaphore sem(1); + + auto a = kj::evalNow([&] { return sem.acquire(); }); + auto b = kj::evalNow([&] { return sem.acquire(); }); + auto c = kj::evalNow([&] { return sem.acquire(); }); + + ASSERT_TRUE(a.poll(waitScope)); + ASSERT_FALSE(b.poll(waitScope)); + + b = nullptr; + a = nullptr; + + ASSERT_TRUE(c.poll(waitScope)); +} + +} diff --git a/tests/unit/meson.build b/tests/unit/meson.build index 8ff0b5ec5..5baf10de2 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -39,6 +39,7 @@ liblixutil_test_support = declare_dependency( ) libutil_tests_sources = files( + 'libutil/async-semaphore.cc', 'libutil/canon-path.cc', 'libutil/checked-arithmetic.cc', 'libutil/chunked-vector.cc', @@ -76,6 +77,7 @@ libutil_tester = executable( liblixexpr_mstatic, liblixutil_test_support, nlohmann_json, + kj, ], cpp_pch : cpp_pch, ) -- 2.44.1 From 531d040e8c2d211408c84ae23421aaa45b3b5a7a Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Tue, 24 Sep 2024 00:21:16 +0200 Subject: [PATCH 054/106] libutil: add async collection mechanism like kj::joinPromisesFailFast this allows waiting for the results of multiple promises at once, but unlike it not all input promises must be complete (or any of them failed) for results to become available. Change-Id: I0e4a37e7bd90651d56b33d0bc5afbadc56cde70c --- src/libutil/async-collect.hh | 101 +++++++++++++++++++++++++++ src/libutil/meson.build | 1 + tests/unit/libutil/async-collect.cc | 104 ++++++++++++++++++++++++++++ tests/unit/meson.build | 1 + 4 files changed, 207 insertions(+) create mode 100644 src/libutil/async-collect.hh create mode 100644 tests/unit/libutil/async-collect.cc diff --git a/src/libutil/async-collect.hh b/src/libutil/async-collect.hh new file mode 100644 index 000000000..9e0b8bad9 --- /dev/null +++ b/src/libutil/async-collect.hh @@ -0,0 +1,101 @@ +#pragma once +/// @file + +#include +#include +#include +#include +#include +#include + +namespace nix { + +template +class AsyncCollect +{ +public: + using Item = std::conditional_t, K, std::pair>; + +private: + kj::ForkedPromise allPromises; + std::list results; + size_t remaining; + + kj::ForkedPromise signal; + kj::Maybe>> notify; + + void oneDone(Item item) + { + results.emplace_back(std::move(item)); + remaining -= 1; + KJ_IF_MAYBE (n, notify) { + (*n)->fulfill(); + notify = nullptr; + } + } + + kj::Promise collectorFor(K key, kj::Promise promise) + { + if constexpr (std::is_void_v) { + return promise.then([this, key{std::move(key)}] { oneDone(std::move(key)); }); + } else { + return promise.then([this, key{std::move(key)}](V v) { + oneDone(Item{std::move(key), std::move(v)}); + }); + } + } + + kj::ForkedPromise waitForAll(kj::Array>> & promises) + { + kj::Vector> wrappers; + for (auto & [key, promise] : promises) { + wrappers.add(collectorFor(std::move(key), std::move(promise))); + } + + return kj::joinPromisesFailFast(wrappers.releaseAsArray()).fork(); + } + +public: + AsyncCollect(kj::Array>> && promises) + : allPromises(waitForAll(promises)) + , remaining(promises.size()) + , signal{nullptr} + { + } + + kj::Promise> next() + { + if (remaining == 0 && results.empty()) { + return {std::nullopt}; + } + + if (!results.empty()) { + auto result = std::move(results.front()); + results.pop_front(); + return {{std::move(result)}}; + } + + if (notify == nullptr) { + auto pair = kj::newPromiseAndFulfiller(); + notify = std::move(pair.fulfiller); + signal = pair.promise.fork(); + } + + return signal.addBranch().exclusiveJoin(allPromises.addBranch()).then([this] { + return next(); + }); + } +}; + +/** + * Collect the results of a list of promises, in order of completion. + * Once any input promise is rejected all promises that have not been + * resolved or rejected will be cancelled and the exception rethrown. + */ +template +AsyncCollect asyncCollect(kj::Array>> promises) +{ + return AsyncCollect(std::move(promises)); +} + +} diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 89eeed133..afca4e021 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -53,6 +53,7 @@ libutil_headers = files( 'archive.hh', 'args/root.hh', 'args.hh', + 'async-collect.hh', 'async-semaphore.hh', 'backed-string-view.hh', 'box_ptr.hh', diff --git a/tests/unit/libutil/async-collect.cc b/tests/unit/libutil/async-collect.cc new file mode 100644 index 000000000..770374d21 --- /dev/null +++ b/tests/unit/libutil/async-collect.cc @@ -0,0 +1,104 @@ +#include "async-collect.hh" + +#include +#include +#include +#include +#include + +namespace nix { + +TEST(AsyncCollect, void) +{ + kj::EventLoop loop; + kj::WaitScope waitScope(loop); + + auto a = kj::newPromiseAndFulfiller(); + auto b = kj::newPromiseAndFulfiller(); + auto c = kj::newPromiseAndFulfiller(); + auto d = kj::newPromiseAndFulfiller(); + + auto collect = asyncCollect(kj::arr( + std::pair(1, std::move(a.promise)), + std::pair(2, std::move(b.promise)), + std::pair(3, std::move(c.promise)), + std::pair(4, std::move(d.promise)) + )); + + auto p = collect.next(); + ASSERT_FALSE(p.poll(waitScope)); + + // collection is ordered + c.fulfiller->fulfill(); + b.fulfiller->fulfill(); + + ASSERT_TRUE(p.poll(waitScope)); + ASSERT_EQ(p.wait(waitScope), 3); + + p = collect.next(); + ASSERT_TRUE(p.poll(waitScope)); + ASSERT_EQ(p.wait(waitScope), 2); + + p = collect.next(); + ASSERT_FALSE(p.poll(waitScope)); + + // exceptions propagate + a.fulfiller->rejectIfThrows([] { throw std::runtime_error("test"); }); + + p = collect.next(); + ASSERT_TRUE(p.poll(waitScope)); + ASSERT_THROW(p.wait(waitScope), kj::Exception); + + // first exception aborts collection + p = collect.next(); + ASSERT_TRUE(p.poll(waitScope)); + ASSERT_THROW(p.wait(waitScope), kj::Exception); +} + +TEST(AsyncCollect, nonVoid) +{ + kj::EventLoop loop; + kj::WaitScope waitScope(loop); + + auto a = kj::newPromiseAndFulfiller(); + auto b = kj::newPromiseAndFulfiller(); + auto c = kj::newPromiseAndFulfiller(); + auto d = kj::newPromiseAndFulfiller(); + + auto collect = asyncCollect(kj::arr( + std::pair(1, std::move(a.promise)), + std::pair(2, std::move(b.promise)), + std::pair(3, std::move(c.promise)), + std::pair(4, std::move(d.promise)) + )); + + auto p = collect.next(); + ASSERT_FALSE(p.poll(waitScope)); + + // collection is ordered + c.fulfiller->fulfill(1); + b.fulfiller->fulfill(2); + + ASSERT_TRUE(p.poll(waitScope)); + ASSERT_EQ(p.wait(waitScope), std::pair(3, 1)); + + p = collect.next(); + ASSERT_TRUE(p.poll(waitScope)); + ASSERT_EQ(p.wait(waitScope), std::pair(2, 2)); + + p = collect.next(); + ASSERT_FALSE(p.poll(waitScope)); + + // exceptions propagate + a.fulfiller->rejectIfThrows([] { throw std::runtime_error("test"); }); + + p = collect.next(); + ASSERT_TRUE(p.poll(waitScope)); + ASSERT_THROW(p.wait(waitScope), kj::Exception); + + // first exception aborts collection + p = collect.next(); + ASSERT_TRUE(p.poll(waitScope)); + ASSERT_THROW(p.wait(waitScope), kj::Exception); +} +} diff --git a/tests/unit/meson.build b/tests/unit/meson.build index 5baf10de2..525fdea0b 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -39,6 +39,7 @@ liblixutil_test_support = declare_dependency( ) libutil_tests_sources = files( + 'libutil/async-collect.cc', 'libutil/async-semaphore.cc', 'libutil/canon-path.cc', 'libutil/checked-arithmetic.cc', -- 2.44.1 From 2265536e853437dc1f36f9c7a20eb2ebeac6ecaa Mon Sep 17 00:00:00 2001 From: zimbatm Date: Thu, 5 Sep 2024 10:29:48 +0200 Subject: [PATCH 055/106] fix(nix fmt): remove the default "." argument When `nix fmt` is called without an argument, Nix appends the "." argument before calling the formatter. The comment in the code is: > Format the current flake out of the box This also happens when formatting sub-folders. This means that the formatter is now unable to distinguish, as an interface, whether the "." argument is coming from the flake or the user's intent to format the current folder. This decision should be up to the formatter. Treefmt, for example, will automatically look up the project's root and format all the files. This is the desired behaviour. But because the "." argument is passed, it cannot function as expected. Upstream-PR: https://github.com/nixos/nix/pull/11438 Change-Id: I60fb6b3ed4ec1b24f81b5f0d76c0be98470817ce --- doc/manual/change-authors.yml | 3 ++ .../rl-next/nix-fmt-default-argument.md | 38 +++++++++++++++++++ src/nix/fmt.cc | 10 +---- tests/functional/fmt.sh | 5 ++- tests/functional/fmt.simple.sh | 2 +- 5 files changed, 48 insertions(+), 10 deletions(-) create mode 100644 doc/manual/rl-next/nix-fmt-default-argument.md diff --git a/doc/manual/change-authors.yml b/doc/manual/change-authors.yml index e18abada1..60c0924c7 100644 --- a/doc/manual/change-authors.yml +++ b/doc/manual/change-authors.yml @@ -147,3 +147,6 @@ winter: yshui: github: yshui + +zimbatm: + github: zimbatm diff --git a/doc/manual/rl-next/nix-fmt-default-argument.md b/doc/manual/rl-next/nix-fmt-default-argument.md new file mode 100644 index 000000000..41b8f85bd --- /dev/null +++ b/doc/manual/rl-next/nix-fmt-default-argument.md @@ -0,0 +1,38 @@ +--- +synopsis: Removing the `.` default argument passed to the `nix fmt` formatter +issues: [] +prs: [11438] +cls: [1902] +category: Breaking Changes +credits: zimbatm +--- + +The underlying formatter no longer receives the ". " default argument when `nix fmt` is called with no arguments. + +This change was necessary as the formatter wasn't able to distinguish between +a user wanting to format the current folder with `nix fmt .` or the generic +`nix fmt`. + +The default behaviour is now the responsibility of the formatter itself, and +allows tools such as treefmt to format the whole tree instead of only the +current directory and below. + +This may cause issues with some formatters: nixfmt, nixpkgs-fmt and alejandra currently format stdin when no arguments are passed. + +Here is a small wrapper example that will restore the previous behaviour for such a formatter: + +```nix +{ + outputs = { self, nixpkgs, systems }: + let + eachSystem = nixpkgs.lib.genAttrs (import systems) (system: nixpkgs.legacyPackages.${system}); + in + { + formatter = eachSystem (pkgs: + pkgs.writeShellScriptBin "formatter" '' + if [[ $# = 0 ]]; set -- .; fi + exec "${pkgs.nixfmt-rfc-style}/bin/nixfmt "$@" + ''); + }; +} +``` diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc index 059904150..f47f2204a 100644 --- a/src/nix/fmt.cc +++ b/src/nix/fmt.cc @@ -39,14 +39,8 @@ struct CmdFmt : SourceExprCommand { Strings programArgs{app.program}; // Propagate arguments from the CLI - if (args.empty()) { - // Format the current flake out of the box - programArgs.push_back("."); - } else { - // User wants more power, let them decide which paths to include/exclude - for (auto &i : args) { - programArgs.push_back(i); - } + for (auto &i : args) { + programArgs.push_back(i); } runProgramInStore(store, UseSearchPath::DontUse, app.program, programArgs); diff --git a/tests/functional/fmt.sh b/tests/functional/fmt.sh index 3c1bd9989..7d6add9b6 100644 --- a/tests/functional/fmt.sh +++ b/tests/functional/fmt.sh @@ -26,7 +26,10 @@ cat << EOF > flake.nix }; } EOF -nix fmt ./file ./folder | grep 'Formatting: ./file ./folder' +# No arguments check +[[ "$(nix fmt)" = "Formatting(0):" ]] +# Argument forwarding check +nix fmt ./file ./folder | grep 'Formatting(2): ./file ./folder' nix flake check nix flake show | grep -P "package 'formatter'" diff --git a/tests/functional/fmt.simple.sh b/tests/functional/fmt.simple.sh index 03109a655..f655846ca 100755 --- a/tests/functional/fmt.simple.sh +++ b/tests/functional/fmt.simple.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -echo Formatting: "${@}" +echo "Formatting(${#}):" "${@}" -- 2.44.1 From 0478949c72310b9749d5b959adad8bdf5c2c0841 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sun, 1 Sep 2024 01:37:10 +0200 Subject: [PATCH 056/106] libstore: turn builder output processing into event loop this removes the rather janky did-you-mean-async poll loop we had so far. sadly kj does not play well with pty file descriptors, so we do have to add our own async input stream that does not eat pty EIO and turns it into an exception. that's still a *lot* better than the old code, and using a real even loop makes everything else easier later. Change-Id: Idd7e0428c59758602cc530bcad224cd2fed4c15e --- src/libstore/build/derivation-goal.cc | 177 +++++++++++++--- src/libstore/build/derivation-goal.hh | 28 ++- .../build/drv-output-substitution-goal.cc | 10 +- .../build/drv-output-substitution-goal.hh | 4 +- src/libstore/build/goal.hh | 25 +-- src/libstore/build/local-derivation-goal.cc | 12 +- src/libstore/build/local-derivation-goal.hh | 2 +- src/libstore/build/substitution-goal.cc | 24 +-- src/libstore/build/substitution-goal.hh | 9 +- src/libstore/build/worker.cc | 191 ++++++------------ src/libstore/build/worker.hh | 42 ++-- 11 files changed, 271 insertions(+), 253 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 827c9f541..f40611b31 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -11,7 +11,10 @@ #include "drv-output-substitution-goal.hh" #include "strings.hh" +#include #include +#include +#include #include #include #include @@ -780,7 +783,7 @@ try { buildResult.startTime = time(0); // inexact state = &DerivationGoal::buildDone; started(); - return WaitForWorld{std::move(a.fds), false}; + return WaitForWorld{std::move(a.promise), false}; }, [&](HookReply::Postpone) -> std::optional { /* Not now; wait until at least one child finishes or @@ -992,9 +995,6 @@ try { buildResult.timesBuilt++; buildResult.stopTime = time(0); - /* So the child is gone now. */ - worker.childTerminated(this); - /* Close the read side of the logger pipe. */ closeReadPipes(); @@ -1266,12 +1266,8 @@ HookReply DerivationGoal::tryBuildHook(bool inBuildSlot) /* Create the log file and pipe. */ Path logFile = openLogFile(); - std::set fds; - fds.insert(hook->fromHook.get()); - fds.insert(hook->builderOut.get()); builderOutFD = &hook->builderOut; - - return HookReply::Accept{std::move(fds)}; + return HookReply::Accept{handleChildOutput()}; } @@ -1331,23 +1327,69 @@ void DerivationGoal::closeLogFile() } -Goal::WorkResult DerivationGoal::handleChildOutput(int fd, std::string_view data) +Goal::Finished DerivationGoal::tooMuchLogs() { - assert(builderOutFD); + killChild(); + return done( + BuildResult::LogLimitExceeded, {}, + Error("%s killed after writing more than %d bytes of log output", + getName(), settings.maxLogSize)); +} - auto tooMuchLogs = [&] { - killChild(); - return done( - BuildResult::LogLimitExceeded, {}, - Error("%s killed after writing more than %d bytes of log output", - getName(), settings.maxLogSize)); - }; +struct DerivationGoal::InputStream final : private kj::AsyncObject +{ + int fd; + kj::UnixEventPort::FdObserver observer; + + InputStream(kj::UnixEventPort & ep, int fd) + : fd(fd) + , observer(ep, fd, kj::UnixEventPort::FdObserver::OBSERVE_READ) + { + int flags = fcntl(fd, F_GETFL); + if (flags < 0) { + throw SysError("fcntl(F_GETFL) failed on fd %i", fd); + } + if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) { + throw SysError("fcntl(F_SETFL) failed on fd %i", fd); + } + } + + kj::Promise read(kj::ArrayPtr buffer) + { + const auto res = ::read(fd, buffer.begin(), buffer.size()); + // closing a pty endpoint causes EIO on the other endpoint. stock kj streams + // do not handle this and throw exceptions we can't ask for errno instead :( + // (we can't use `errno` either because kj may well have mangled it by now.) + if (res == 0 || (res == -1 && errno == EIO)) { + return std::string_view{}; + } + + KJ_NONBLOCKING_SYSCALL(res) {} + + if (res > 0) { + return std::string_view{buffer.begin(), static_cast(res)}; + } + + return observer.whenBecomesReadable().then([this, buffer] { + return read(buffer); + }); + } +}; + +kj::Promise> DerivationGoal::handleBuilderOutput(InputStream & in) noexcept +try { + auto buf = kj::heapArray(4096); + while (true) { + auto data = co_await in.read(buf); + lastChildActivity = worker.aio.provider->getTimer().now(); + + if (data.empty()) { + co_return result::success(); + } - // local & `ssh://`-builds are dealt with here. - if (fd == builderOutFD->get()) { logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { - return tooMuchLogs(); + co_return tooMuchLogs(); } for (auto c : data) @@ -1362,10 +1404,22 @@ Goal::WorkResult DerivationGoal::handleChildOutput(int fd, std::string_view data } if (logSink) (*logSink)(data); - return StillAlive{}; } +} catch (...) { + co_return std::current_exception(); +} + +kj::Promise> DerivationGoal::handleHookOutput(InputStream & in) noexcept +try { + auto buf = kj::heapArray(4096); + while (true) { + auto data = co_await in.read(buf); + lastChildActivity = worker.aio.provider->getTimer().now(); + + if (data.empty()) { + co_return result::success(); + } - if (hook && fd == hook->fromHook.get()) { for (auto c : data) if (c == '\n') { auto json = parseJSONMessage(currentHookLine); @@ -1381,7 +1435,7 @@ Goal::WorkResult DerivationGoal::handleChildOutput(int fd, std::string_view data (fields.size() > 0 ? fields[0].get() : "") + "\n"; logSize += logLine.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { - return tooMuchLogs(); + co_return tooMuchLogs(); } (*logSink)(logLine); } else if (type == resSetPhase && ! fields.is_null()) { @@ -1405,16 +1459,83 @@ Goal::WorkResult DerivationGoal::handleChildOutput(int fd, std::string_view data } else currentHookLine += c; } - - return StillAlive{}; +} catch (...) { + co_return std::current_exception(); } +kj::Promise> DerivationGoal::handleChildOutput() noexcept +try { + assert(builderOutFD); -void DerivationGoal::handleEOF(int fd) + auto builderIn = kj::heap(worker.aio.unixEventPort, builderOutFD->get()); + kj::Own hookIn; + if (hook) { + hookIn = kj::heap(worker.aio.unixEventPort, hook->fromHook.get()); + } + + auto handlers = handleChildStreams(*builderIn, hookIn.get()).attach(std::move(builderIn), std::move(hookIn)); + + if (respectsTimeouts() && settings.buildTimeout != 0) { + handlers = handlers.exclusiveJoin( + worker.aio.provider->getTimer() + .afterDelay(settings.buildTimeout.get() * kj::SECONDS) + .then([this]() -> Outcome { + return timedOut( + Error("%1% timed out after %2% seconds", name, settings.buildTimeout) + ); + }) + ); + } + + return handlers.then([this](auto r) -> Outcome { + if (!currentLogLine.empty()) flushLine(); + return r; + }); +} catch (...) { + return {std::current_exception()}; +} + +kj::Promise> DerivationGoal::monitorForSilence() noexcept { - if (!currentLogLine.empty()) flushLine(); + while (true) { + const auto stash = lastChildActivity; + auto waitUntil = lastChildActivity + settings.maxSilentTime.get() * kj::SECONDS; + co_await worker.aio.provider->getTimer().atTime(waitUntil); + if (lastChildActivity == stash) { + co_return timedOut( + Error("%1% timed out after %2% seconds of silence", name, settings.maxSilentTime) + ); + } + } } +kj::Promise> +DerivationGoal::handleChildStreams(InputStream & builderIn, InputStream * hookIn) noexcept +{ + lastChildActivity = worker.aio.provider->getTimer().now(); + + auto handlers = kj::joinPromisesFailFast([&] { + kj::Vector>> parts{2}; + + parts.add(handleBuilderOutput(builderIn)); + if (hookIn) { + parts.add(handleHookOutput(*hookIn)); + } + + return parts.releaseAsArray(); + }()); + + if (respectsTimeouts() && settings.maxSilentTime != 0) { + handlers = handlers.exclusiveJoin(monitorForSilence().then([](auto r) { + return kj::arr(std::move(r)); + })); + } + + for (auto r : co_await handlers) { + BOOST_OUTCOME_CO_TRYV(r); + } + co_return result::success(); +} void DerivationGoal::flushLine() { diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 020388d5a..46b07fc0b 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -8,6 +8,7 @@ #include "store-api.hh" #include "pathlocks.hh" #include "goal.hh" +#include namespace nix { @@ -17,7 +18,7 @@ struct HookInstance; struct HookReplyBase { struct [[nodiscard]] Accept { - std::set fds; + kj::Promise> promise; }; struct [[nodiscard]] Decline {}; struct [[nodiscard]] Postpone {}; @@ -70,6 +71,8 @@ struct InitialOutput { */ struct DerivationGoal : public Goal { + struct InputStream; + /** * Whether to use an on-disk .drv file. */ @@ -242,7 +245,7 @@ struct DerivationGoal : public Goal BuildMode buildMode = bmNormal); virtual ~DerivationGoal() noexcept(false); - Finished timedOut(Error && ex) override; + Finished timedOut(Error && ex); std::string key() override; @@ -312,13 +315,19 @@ struct DerivationGoal : public Goal virtual void cleanupPostOutputsRegisteredModeCheck(); virtual void cleanupPostOutputsRegisteredModeNonCheck(); - /** - * Callback used by the worker to write to the log. - */ - WorkResult handleChildOutput(int fd, std::string_view data) override; - void handleEOF(int fd) override; +protected: + kj::TimePoint lastChildActivity = kj::minValue; + + kj::Promise> handleChildOutput() noexcept; + kj::Promise> + handleChildStreams(InputStream & builderIn, InputStream * hookIn) noexcept; + kj::Promise> handleBuilderOutput(InputStream & in) noexcept; + kj::Promise> handleHookOutput(InputStream & in) noexcept; + kj::Promise> monitorForSilence() noexcept; + Finished tooMuchLogs(); void flushLine(); +public: /** * Wrappers around the corresponding Store methods that first consult the * derivation. This is currently needed because when there is no drv file @@ -357,6 +366,11 @@ struct DerivationGoal : public Goal void waiteeDone(GoalPtr waitee) override; + virtual bool respectsTimeouts() + { + return false; + } + StorePathSet exportReferences(const StorePathSet & storePaths); JobCategory jobCategory() const override { diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 7986123cc..fdee53699 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -69,24 +69,26 @@ try { some other error occurs), so it must not touch `this`. So put the shared state in a separate refcounted object. */ downloadState = std::make_shared(); - downloadState->outPipe.create(); + auto pipe = kj::newPromiseAndCrossThreadFulfiller(); + downloadState->outPipe = kj::mv(pipe.fulfiller); downloadState->result = std::async(std::launch::async, [downloadState{downloadState}, id{id}, sub{sub}] { + Finally updateStats([&]() { downloadState->outPipe->fulfill(); }); ReceiveInterrupts receiveInterrupts; - Finally updateStats([&]() { downloadState->outPipe.writeSide.close(); }); return sub->queryRealisation(id); }); state = &DrvOutputSubstitutionGoal::realisationFetched; - return {WaitForWorld{{downloadState->outPipe.readSide.get()}, true}}; + return {WaitForWorld{ + pipe.promise.then([]() -> Outcome { return result::success(); }), true + }}; } catch (...) { return {std::current_exception()}; } kj::Promise> DrvOutputSubstitutionGoal::realisationFetched(bool inBuildSlot) noexcept try { - worker.childTerminated(this); maintainRunningSubstitutions.reset(); try { diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index f33196665..a35bf67ee 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -45,7 +45,7 @@ class DrvOutputSubstitutionGoal : public Goal { struct DownloadState { - Pipe outPipe; + kj::Own> outPipe; std::future> result; }; @@ -74,8 +74,6 @@ public: kj::Promise> outPathValid(bool inBuildSlot) noexcept; kj::Promise> finished() noexcept; - Finished timedOut(Error && ex) override { abort(); }; - std::string key() override; kj::Promise> work(bool inBuildSlot) noexcept override; diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 189505308..3f6e8396e 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -114,6 +114,8 @@ struct Goal public: + struct Finished; + struct [[nodiscard]] StillAlive {}; struct [[nodiscard]] WaitForSlot {}; struct [[nodiscard]] WaitForAWhile {}; @@ -122,7 +124,7 @@ public: Goals goals; }; struct [[nodiscard]] WaitForWorld { - std::set fds; + kj::Promise> promise; bool inBuildSlot; }; struct [[nodiscard]] Finished { @@ -167,20 +169,6 @@ public: virtual void waiteeDone(GoalPtr waitee) { } - virtual WorkResult handleChildOutput(int fd, std::string_view data) - { - abort(); - } - - virtual void handleEOF(int fd) - { - } - - virtual bool respectsTimeouts() - { - return false; - } - void trace(std::string_view s); std::string getName() const @@ -188,13 +176,6 @@ public: return name; } - /** - * Callback in case of a timeout. It should wake up its waiters, - * get rid of any running child processes that are being monitored - * by the worker (important!), etc. - */ - virtual Finished timedOut(Error && ex) = 0; - virtual std::string key() = 0; virtual void cleanup() { } diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index f14d09652..040fa7461 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -121,8 +121,6 @@ LocalStore & LocalDerivationGoal::getLocalStore() void LocalDerivationGoal::killChild() { if (pid) { - worker.childTerminated(this); - /* If we're using a build user, then there is a tricky race condition: if we kill the build user before the child has done its setuid() to the build user uid, then it won't be @@ -243,14 +241,14 @@ try { try { /* Okay, we have to build. */ - auto fds = startBuilder(); + auto promise = startBuilder(); /* This state will be reached when we get EOF on the child's log pipe. */ state = &DerivationGoal::buildDone; started(); - return {WaitForWorld{std::move(fds), true}}; + return {WaitForWorld{std::move(promise), true}}; } catch (BuildError & e) { outputLocks.unlock(); @@ -390,7 +388,9 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() cleanupPostOutputsRegisteredModeCheck(); } -std::set LocalDerivationGoal::startBuilder() +// NOTE this one isn't noexcept because it's called from places that expect +// exceptions to signal failure to launch. we should change this some time. +kj::Promise> LocalDerivationGoal::startBuilder() { if ((buildUser && buildUser->getUIDCount() != 1) #if __linux__ @@ -779,7 +779,7 @@ std::set LocalDerivationGoal::startBuilder() msgs.push_back(std::move(msg)); } - return {builderOutPTY.get()}; + return handleChildOutput(); } diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index cd040bc15..6239129ab 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -218,7 +218,7 @@ struct LocalDerivationGoal : public DerivationGoal /** * Start building a derivation. */ - std::set startBuilder(); + kj::Promise> startBuilder(); /** * Fill in the environment for the builder. diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index bd0ffcb9b..058f858d4 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -208,16 +208,17 @@ try { maintainRunningSubstitutions = worker.runningSubstitutions.addTemporarily(1); - outPipe.create(); + auto pipe = kj::newPromiseAndCrossThreadFulfiller(); + outPipe = kj::mv(pipe.fulfiller); thr = std::async(std::launch::async, [this]() { + /* Wake up the worker loop when we're done. */ + Finally updateStats([this]() { outPipe->fulfill(); }); + auto & fetchPath = subPath ? *subPath : storePath; try { ReceiveInterrupts receiveInterrupts; - /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide.close(); }); - Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); @@ -234,7 +235,9 @@ try { }); state = &PathSubstitutionGoal::finished; - return {WaitForWorld{{outPipe.readSide.get()}, true}}; + return {WaitForWorld{ + pipe.promise.then([]() -> Outcome { return result::success(); }), true + }}; } catch (...) { return {std::current_exception()}; } @@ -244,8 +247,6 @@ kj::Promise> PathSubstitutionGoal::finished(bool inBuil try { trace("substitute finished"); - worker.childTerminated(this); - try { thr.get(); } catch (std::exception & e) { @@ -288,22 +289,13 @@ try { } -Goal::WorkResult PathSubstitutionGoal::handleChildOutput(int fd, std::string_view data) -{ - return StillAlive{}; -} - - void PathSubstitutionGoal::cleanup() { try { if (thr.valid()) { // FIXME: signal worker thread to quit. thr.get(); - worker.childTerminated(this); } - - outPipe.close(); } catch (...) { ignoreException(); } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 3c97b19fd..91e256fd7 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -46,7 +46,7 @@ struct PathSubstitutionGoal : public Goal /** * Pipe for the substituter's standard output. */ - Pipe outPipe; + kj::Own> outPipe; /** * The substituter thread. @@ -90,8 +90,6 @@ public: ); ~PathSubstitutionGoal(); - Finished timedOut(Error && ex) override { abort(); }; - /** * We prepend "a$" to the key name to ensure substitution goals * happen before derivation goals. @@ -112,11 +110,6 @@ public: kj::Promise> tryToRun(bool inBuildSlot) noexcept; kj::Promise> finished(bool inBuildSlot) noexcept; - /** - * Callback used by the worker to write to the log. - */ - WorkResult handleChildOutput(int fd, std::string_view data) override; - /* Called by destructor, can't be overridden */ void cleanup() override final; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index ee45c7e3f..284adbc50 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -7,10 +7,19 @@ #include "signals.hh" #include "hook-instance.hh" // IWYU pragma: keep -#include - namespace nix { +namespace { +struct ErrorHandler : kj::TaskSet::ErrorHandler +{ + void taskFailed(kj::Exception && e) override + { + printError("unexpected async failure in Worker: %s", kj::str(e).cStr()); + abort(); + } +} errorHandler; +} + Worker::Worker(Store & store, Store & evalStore, kj::AsyncIoContext & aio) : act(*logger, actRealise) , actDerivations(*logger, actBuilds) @@ -18,6 +27,7 @@ Worker::Worker(Store & store, Store & evalStore, kj::AsyncIoContext & aio) , store(store) , evalStore(evalStore) , aio(aio) + , children(errorHandler) { /* Debugging: prevent recursive workers. */ nrLocalBuilds = 0; @@ -33,6 +43,7 @@ Worker::~Worker() are in trouble, since goals may call childTerminated() etc. in their destructors). */ topGoals.clear(); + children.clear(); assert(expectedSubstitutions == 0); assert(expectedDownloadSize == 0); @@ -209,7 +220,9 @@ void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how) dep->waiters.insert(goal); } }, - [&](Goal::WaitForWorld & w) { childStarted(goal, w.fds, w.inBuildSlot); }, + [&](Goal::WaitForWorld & w) { + childStarted(goal, std::move(w.promise), w.inBuildSlot); + }, [&](Goal::Finished & f) { goalFinished(goal, f); }, }, how @@ -244,16 +257,22 @@ void Worker::wakeUp(GoalPtr goal) } -void Worker::childStarted(GoalPtr goal, const std::set & fds, +void Worker::childStarted(GoalPtr goal, kj::Promise> promise, bool inBuildSlot) { - Child child; - child.goal = goal; - child.goal2 = goal.get(); - child.fds = fds; - child.timeStarted = child.lastOutput = steady_time_point::clock::now(); - child.inBuildSlot = inBuildSlot; - children.emplace_back(child); + children.add(promise + .then([this, goal](auto result) { + if (result.has_value()) { + handleWorkResult(goal, Goal::ContinueImmediately{}); + } else if (result.has_error()) { + handleWorkResult(goal, std::move(result.assume_error())); + } else { + childException = result.assume_exception(); + } + }) + .attach(Finally{[this, goal, inBuildSlot] { + childTerminated(goal, inBuildSlot); + }})); if (inBuildSlot) { switch (goal->jobCategory()) { case JobCategory::Substitution: @@ -269,13 +288,13 @@ void Worker::childStarted(GoalPtr goal, const std::set & fds, } -void Worker::childTerminated(Goal * goal) +void Worker::childTerminated(GoalPtr goal, bool inBuildSlot) { - auto i = std::find_if(children.begin(), children.end(), - [&](const Child & child) { return child.goal2 == goal; }); - if (i == children.end()) return; + if (childFinished) { + childFinished->fulfill(); + } - if (i->inBuildSlot) { + if (inBuildSlot) { switch (goal->jobCategory()) { case JobCategory::Substitution: assert(nrSubstitutions > 0); @@ -290,8 +309,6 @@ void Worker::childTerminated(Goal * goal) } } - children.erase(i); - /* Wake up goals waiting for a build slot. */ for (auto & j : wantingToBuild) { GoalPtr goal = j.lock(); @@ -390,11 +407,15 @@ Goals Worker::run(std::function req) if (topGoals.empty()) break; /* Wait for input. */ - if (!children.empty() || !waitingForAWhile.empty()) + if (!children.isEmpty() || !waitingForAWhile.empty()) waitForInput(); else { assert(!awake.empty()); } + + if (childException) { + std::rethrow_exception(childException); + } } /* If --keep-going is not set, it's possible that the main goal @@ -402,7 +423,7 @@ Goals Worker::run(std::function req) --keep-going *is* set, then they must all be finished now. */ assert(!settings.keepGoing || awake.empty()); assert(!settings.keepGoing || wantingToBuild.empty()); - assert(!settings.keepGoing || children.empty()); + assert(!settings.keepGoing || children.isEmpty()); return _topGoals; } @@ -411,140 +432,52 @@ void Worker::waitForInput() { printMsg(lvlVomit, "waiting for children"); + auto childFinished = [&]{ + auto pair = kj::newPromiseAndFulfiller(); + this->childFinished = kj::mv(pair.fulfiller); + return kj::mv(pair.promise); + }(); + /* Process output from the file descriptors attached to the children, namely log output and output path creation commands. We also use this to detect child termination: if we get EOF on the logger pipe of a build, we assume that the builder has terminated. */ - bool useTimeout = false; - long timeout = 0; + std::optional timeout = 0; auto before = steady_time_point::clock::now(); - /* If we're monitoring for silence on stdout/stderr, or if there - is a build timeout, then wait for input until the first - deadline for any child. */ - auto nearest = steady_time_point::max(); // nearest deadline - if (settings.minFree.get() != 0) - // Periodicallty wake up to see if we need to run the garbage collector. - nearest = before + std::chrono::seconds(10); - for (auto & i : children) { - if (auto goal = i.goal.lock()) { - if (!goal->respectsTimeouts()) continue; - if (0 != settings.maxSilentTime) - nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); - if (0 != settings.buildTimeout) - nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); - } - } - if (nearest != steady_time_point::max()) { - timeout = std::max(1L, (long) std::chrono::duration_cast(nearest - before).count()); - useTimeout = true; + // Periodicallty wake up to see if we need to run the garbage collector. + if (settings.minFree.get() != 0) { + timeout = 10; } /* If we are polling goals that are waiting for a lock, then wake up after a few seconds at most. */ if (!waitingForAWhile.empty()) { - useTimeout = true; if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; timeout = std::max(1L, (long) std::chrono::duration_cast( lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); } else lastWokenUp = steady_time_point::min(); - if (useTimeout) - vomit("sleeping %d seconds", timeout); + if (timeout) + vomit("sleeping %d seconds", *timeout); - /* Use select() to wait for the input side of any logger pipe to - become `available'. Note that `available' (i.e., non-blocking) - includes EOF. */ - std::vector pollStatus; - std::map fdToPollStatus; - for (auto & i : children) { - for (auto & j : i.fds) { - pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); - fdToPollStatus[j] = pollStatus.size() - 1; + auto waitFor = [&] { + if (timeout) { + return aio.provider->getTimer() + .afterDelay(*timeout * kj::SECONDS) + .exclusiveJoin(kj::mv(childFinished)); + } else { + return std::move(childFinished); } - } + }(); - if (poll(pollStatus.data(), pollStatus.size(), - useTimeout ? timeout * 1000 : -1) == -1) { - if (errno == EINTR) return; - throw SysError("waiting for input"); - } + waitFor.wait(aio.waitScope); auto after = steady_time_point::clock::now(); - /* Process all available file descriptors. FIXME: this is - O(children * fds). */ - decltype(children)::iterator i; - for (auto j = children.begin(); j != children.end(); j = i) { - i = std::next(j); - - checkInterrupt(); - - GoalPtr goal = j->goal.lock(); - assert(goal); - - if (!goal->exitCode.has_value() && - 0 != settings.maxSilentTime && - goal->respectsTimeouts() && - after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) - { - handleWorkResult( - goal, - goal->timedOut(Error( - "%1% timed out after %2% seconds of silence", - goal->getName(), - settings.maxSilentTime - )) - ); - continue; - } - - else if (!goal->exitCode.has_value() && - 0 != settings.buildTimeout && - goal->respectsTimeouts() && - after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) - { - handleWorkResult( - goal, - goal->timedOut( - Error("%1% timed out after %2% seconds", goal->getName(), settings.buildTimeout) - ) - ); - continue; - } - - std::set fds2(j->fds); - std::vector buffer(4096); - for (auto & k : fds2) { - const auto fdPollStatusId = get(fdToPollStatus, k); - assert(fdPollStatusId); - assert(*fdPollStatusId < pollStatus.size()); - if (pollStatus.at(*fdPollStatusId).revents) { - ssize_t rd = ::read(k, buffer.data(), buffer.size()); - // FIXME: is there a cleaner way to handle pt close - // than EIO? Is this even standard? - if (rd == 0 || (rd == -1 && errno == EIO)) { - debug("%1%: got EOF", goal->getName()); - goal->handleEOF(k); - handleWorkResult(goal, Goal::ContinueImmediately{}); - j->fds.erase(k); - } else if (rd == -1) { - if (errno != EINTR) - throw SysError("%s: read failed", goal->getName()); - } else { - printMsg(lvlVomit, "%1%: read %2% bytes", - goal->getName(), rd); - std::string_view data(charptr_cast(buffer.data()), rd); - j->lastOutput = after; - handleWorkResult(goal, goal->handleChildOutput(k, data)); - } - } - } - } - if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { lastWokenUp = after; for (auto & i : waitingForAWhile) { diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 6735ea0b9..37d80ba7b 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -21,24 +21,6 @@ class DrvOutputSubstitutionGoal; typedef std::chrono::time_point steady_time_point; -/** - * A mapping used to remember for each child process to what goal it - * belongs, and file descriptors for receiving log data and output - * path creation commands. - */ -struct Child -{ - WeakGoalPtr goal; - Goal * goal2; // ugly hackery - std::set fds; - bool inBuildSlot; - /** - * Time we last got output on stdout/stderr - */ - steady_time_point lastOutput; - steady_time_point timeStarted; -}; - /* Forward definition. */ struct HookInstance; @@ -116,11 +98,6 @@ private: */ WeakGoals wantingToBuild; - /** - * Child processes currently running. - */ - std::list children; - /** * Number of build slots occupied. This includes local builds but does not * include substitutions or remote builds via the build hook. @@ -179,6 +156,8 @@ private: void goalFinished(GoalPtr goal, Goal::Finished & f); void handleWorkResult(GoalPtr goal, Goal::WorkResult how); + kj::Own> childFinished; + /** * Put `goal` to sleep until a build slot becomes available (which * might be right away). @@ -212,9 +191,14 @@ private: * Registers a running child process. `inBuildSlot` means that * the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, const std::set & fds, + void childStarted(GoalPtr goal, kj::Promise> promise, bool inBuildSlot); + /** + * Unregisters a running child process. + */ + void childTerminated(GoalPtr goal, bool inBuildSlot); + /** * Pass current stats counters to the logger for progress bar updates. */ @@ -240,6 +224,11 @@ public: Store & evalStore; kj::AsyncIoContext & aio; +private: + kj::TaskSet children; + std::exception_ptr childException; + +public: struct HookState { std::unique_ptr instance; @@ -302,11 +291,6 @@ private: GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override; public: - /** - * Unregisters a running child process. - */ - void childTerminated(Goal * goal); - /** * Loop until the specified top-level goals have finished. */ -- 2.44.1 From cd1ceffb0ee9544bf14453f94da6b6f0d52f10cd Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sun, 1 Sep 2024 01:37:10 +0200 Subject: [PATCH 057/106] libstore: make waiting for a while a promise this simplifies waitForInput quite a lot, and at the same time makes polling less thundering-herd-y. it even fixes early polling wakeups! Change-Id: I6dfa62ce91729b8880342117d71af5ae33366414 --- src/libstore/build/derivation-goal.cc | 14 ++--- src/libstore/build/goal.cc | 13 +++++ src/libstore/build/goal.hh | 7 ++- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/build/worker.cc | 63 ++++++++------------- src/libstore/build/worker.hh | 20 +------ 6 files changed, 51 insertions(+), 68 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index f40611b31..c95092913 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -736,7 +736,7 @@ try { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for lock on %s", Magenta(showPaths(lockFiles)))); - return {WaitForAWhile{}}; + return waitForAWhile(); } actLock.reset(); @@ -776,32 +776,32 @@ try { auto hookReply = tryBuildHook(inBuildSlot); auto result = std::visit( overloaded{ - [&](HookReply::Accept & a) -> std::optional { + [&](HookReply::Accept & a) -> std::optional>> { /* Yes, it has started doing so. Wait until we get EOF from the hook. */ actLock.reset(); buildResult.startTime = time(0); // inexact state = &DerivationGoal::buildDone; started(); - return WaitForWorld{std::move(a.promise), false}; + return {{WaitForWorld{std::move(a.promise), false}}}; }, - [&](HookReply::Postpone) -> std::optional { + [&](HookReply::Postpone) -> std::optional>> { /* Not now; wait until at least one child finishes or the wake-up timeout expires. */ if (!actLock) actLock = std::make_unique(*logger, lvlTalkative, actBuildWaiting, fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); outputLocks.unlock(); - return WaitForAWhile{}; + return waitForAWhile(); }, - [&](HookReply::Decline) -> std::optional { + [&](HookReply::Decline) -> std::optional>> { /* We should do it ourselves. */ return std::nullopt; }, }, hookReply); if (result) { - return {std::move(*result)}; + return std::move(*result); } } diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 82861ad2b..649093dbd 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -1,4 +1,6 @@ #include "goal.hh" +#include "worker.hh" +#include namespace nix { @@ -15,4 +17,15 @@ void Goal::trace(std::string_view s) debug("%1%: %2%", name, s); } +kj::Promise> Goal::waitForAWhile() +try { + trace("wait for a while"); + /* If we are polling goals that are waiting for a lock, then wake + up after a few seconds at most. */ + co_await worker.aio.provider->getTimer().afterDelay(settings.pollInterval.get() * kj::SECONDS); + co_return ContinueImmediately{}; +} catch (...) { + co_return std::current_exception(); +} + } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 3f6e8396e..fbf767e8d 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -118,7 +118,6 @@ public: struct [[nodiscard]] StillAlive {}; struct [[nodiscard]] WaitForSlot {}; - struct [[nodiscard]] WaitForAWhile {}; struct [[nodiscard]] ContinueImmediately {}; struct [[nodiscard]] WaitForGoals { Goals goals; @@ -140,7 +139,6 @@ public: struct [[nodiscard]] WorkResult : std::variant< StillAlive, WaitForSlot, - WaitForAWhile, ContinueImmediately, WaitForGoals, WaitForWorld, @@ -150,6 +148,11 @@ public: using variant::variant; }; +protected: + kj::Promise> waitForAWhile(); + +public: + /** * Exception containing an error message, if any. */ diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 040fa7461..9ec87f1b6 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -212,7 +212,7 @@ try { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); - return {WaitForAWhile{}}; + return waitForAWhile(); } } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 284adbc50..27d8e6ee1 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -32,7 +32,6 @@ Worker::Worker(Store & store, Store & evalStore, kj::AsyncIoContext & aio) /* Debugging: prevent recursive workers. */ nrLocalBuilds = 0; nrSubstitutions = 0; - lastWokenUp = steady_time_point::min(); } @@ -212,7 +211,6 @@ void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how) overloaded{ [&](Goal::StillAlive) {}, [&](Goal::WaitForSlot) { waitForBuildSlot(goal); }, - [&](Goal::WaitForAWhile) { waitForAWhile(goal); }, [&](Goal::ContinueImmediately) { wakeUp(goal); }, [&](Goal::WaitForGoals & w) { for (auto & dep : w.goals) { @@ -221,12 +219,25 @@ void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how) } }, [&](Goal::WaitForWorld & w) { - childStarted(goal, std::move(w.promise), w.inBuildSlot); + childStarted( + goal, + w.promise.then([](auto r) -> Result { + if (r.has_value()) { + return {Goal::ContinueImmediately{}}; + } else if (r.has_error()) { + return {std::move(r).error()}; + } else { + return r.exception(); + } + }), + w.inBuildSlot + ); }, [&](Goal::Finished & f) { goalFinished(goal, f); }, }, how ); + updateStatistics(); } void Worker::removeGoal(GoalPtr goal) @@ -257,17 +268,15 @@ void Worker::wakeUp(GoalPtr goal) } -void Worker::childStarted(GoalPtr goal, kj::Promise> promise, +void Worker::childStarted(GoalPtr goal, kj::Promise> promise, bool inBuildSlot) { children.add(promise .then([this, goal](auto result) { if (result.has_value()) { - handleWorkResult(goal, Goal::ContinueImmediately{}); - } else if (result.has_error()) { - handleWorkResult(goal, std::move(result.assume_error())); + handleWorkResult(goal, std::move(result.assume_value())); } else { - childException = result.assume_exception(); + childException = result.assume_error(); } }) .attach(Finally{[this, goal, inBuildSlot] { @@ -331,13 +340,6 @@ void Worker::waitForBuildSlot(GoalPtr goal) } -void Worker::waitForAWhile(GoalPtr goal) -{ - debug("wait for a while"); - waitingForAWhile.insert(goal); -} - - void Worker::updateStatistics() { // only update progress info while running. this notably excludes updating @@ -397,8 +399,12 @@ Goals Worker::run(std::function req) const bool inSlot = goal->jobCategory() == JobCategory::Substitution ? nrSubstitutions < std::max(1U, (unsigned int) settings.maxSubstitutionJobs) : nrLocalBuilds < settings.maxBuildJobs; - handleWorkResult(goal, goal->work(inSlot).wait(aio.waitScope).value()); - updateStatistics(); + auto result = goal->work(inSlot); + if (result.poll(aio.waitScope)) { + handleWorkResult(goal, result.wait(aio.waitScope).value()); + } else { + childStarted(goal, std::move(result), false); + } if (topGoals.empty()) break; // stuff may have been cancelled } @@ -407,7 +413,7 @@ Goals Worker::run(std::function req) if (topGoals.empty()) break; /* Wait for input. */ - if (!children.isEmpty() || !waitingForAWhile.empty()) + if (!children.isEmpty()) waitForInput(); else { assert(!awake.empty()); @@ -445,22 +451,12 @@ void Worker::waitForInput() terminated. */ std::optional timeout = 0; - auto before = steady_time_point::clock::now(); // Periodicallty wake up to see if we need to run the garbage collector. if (settings.minFree.get() != 0) { timeout = 10; } - /* If we are polling goals that are waiting for a lock, then wake - up after a few seconds at most. */ - if (!waitingForAWhile.empty()) { - if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; - timeout = std::max(1L, - (long) std::chrono::duration_cast( - lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); - } else lastWokenUp = steady_time_point::min(); - if (timeout) vomit("sleeping %d seconds", *timeout); @@ -475,17 +471,6 @@ void Worker::waitForInput() }(); waitFor.wait(aio.waitScope); - - auto after = steady_time_point::clock::now(); - - if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { - lastWokenUp = after; - for (auto & i : waitingForAWhile) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - waitingForAWhile.clear(); - } } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 37d80ba7b..daa612c06 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -117,16 +117,6 @@ private: std::map> substitutionGoals; std::map> drvOutputSubstitutionGoals; - /** - * Goals sleeping for a few seconds (polling a lock). - */ - WeakGoals waitingForAWhile; - - /** - * Last time the goals in `waitingForAWhile` where woken up. - */ - steady_time_point lastWokenUp; - /** * Cache for pathContentsGood(). */ @@ -164,14 +154,6 @@ private: */ void waitForBuildSlot(GoalPtr goal); - /** - * Wait for a few seconds and then retry this goal. Used when - * waiting for a lock held by another process. This kind of - * polling is inefficient, but POSIX doesn't really provide a way - * to wait for multiple locks in the main select() loop. - */ - void waitForAWhile(GoalPtr goal); - /** * Wake up a goal (i.e., there is something for it to do). */ @@ -191,7 +173,7 @@ private: * Registers a running child process. `inBuildSlot` means that * the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, kj::Promise> promise, + void childStarted(GoalPtr goal, kj::Promise> promise, bool inBuildSlot); /** -- 2.44.1 From bf32085d63ccfa8fb1e0cff2f2ae7156b4679015 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sun, 1 Sep 2024 01:37:10 +0200 Subject: [PATCH 058/106] libstore: simplify Worker::waitForInput with waitForAWhile turned into promised the core functionality of waitForInput is now merely to let gc run every so often if needed Change-Id: I68da342bbc1d67653901cf4502dabfa5bc947628 --- src/libstore/build/worker.cc | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 27d8e6ee1..2cc2828b1 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -438,38 +438,17 @@ void Worker::waitForInput() { printMsg(lvlVomit, "waiting for children"); - auto childFinished = [&]{ + auto waitFor = [&]{ auto pair = kj::newPromiseAndFulfiller(); this->childFinished = kj::mv(pair.fulfiller); return kj::mv(pair.promise); }(); - /* Process output from the file descriptors attached to the - children, namely log output and output path creation commands. - We also use this to detect child termination: if we get EOF on - the logger pipe of a build, we assume that the builder has - terminated. */ - - std::optional timeout = 0; - - // Periodicallty wake up to see if we need to run the garbage collector. if (settings.minFree.get() != 0) { - timeout = 10; + // Periodicallty wake up to see if we need to run the garbage collector. + waitFor = waitFor.exclusiveJoin(aio.provider->getTimer().afterDelay(10 * kj::SECONDS)); } - if (timeout) - vomit("sleeping %d seconds", *timeout); - - auto waitFor = [&] { - if (timeout) { - return aio.provider->getTimer() - .afterDelay(*timeout * kj::SECONDS) - .exclusiveJoin(kj::mv(childFinished)); - } else { - return std::move(childFinished); - } - }(); - waitFor.wait(aio.waitScope); } -- 2.44.1 From 852da07b67564f7a9986f0638aac391d334d4afa Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Tue, 24 Sep 2024 00:21:16 +0200 Subject: [PATCH 059/106] libstore: replace Goal::WaitForSlot with semaphores now that we have an event loop in the worker we can use it and its magical execution suspending properties to replace the slot counts we managed explicitly with semaphores and raii tokens. technically this would not have needed an event loop base to be doable, but it is a whole lot easier to wait for a token to be available if there is a callback mechanism ready for use that doesn't require a whole damn dedicated abstract method in Goal to work, and specific calls to that dedicated method strewn all over the worker implementation Change-Id: I1da7cf386d94e2bbf2dba9b53ff51dbce6a0cff7 --- src/libstore/build/derivation-goal.cc | 7 +- src/libstore/build/derivation-goal.hh | 2 +- .../build/drv-output-substitution-goal.cc | 12 ++- .../build/drv-output-substitution-goal.hh | 2 +- src/libstore/build/goal.hh | 9 +- src/libstore/build/local-derivation-goal.cc | 9 +- src/libstore/build/substitution-goal.cc | 12 ++- src/libstore/build/substitution-goal.hh | 2 +- src/libstore/build/worker.cc | 95 ++++--------------- src/libstore/build/worker.hh | 32 +------ 10 files changed, 58 insertions(+), 124 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index c95092913..3c4257f08 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -134,9 +134,9 @@ Goal::Finished DerivationGoal::timedOut(Error && ex) } -kj::Promise> DerivationGoal::work(bool inBuildSlot) noexcept +kj::Promise> DerivationGoal::work() noexcept { - return (this->*state)(inBuildSlot); + return (this->*state)(slotToken.valid()); } void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) @@ -783,7 +783,7 @@ try { buildResult.startTime = time(0); // inexact state = &DerivationGoal::buildDone; started(); - return {{WaitForWorld{std::move(a.promise), false}}}; + return {{WaitForWorld{std::move(a.promise)}}}; }, [&](HookReply::Postpone) -> std::optional>> { /* Not now; wait until at least one child finishes or @@ -980,6 +980,7 @@ kj::Promise> DerivationGoal::buildDone(bool inBuildSlot try { trace("build done"); + slotToken = {}; Finally releaseBuildUser([&](){ this->cleanupHookFinally(); }); cleanupPreChildKill(); diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 46b07fc0b..d60bb0b4c 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -249,7 +249,7 @@ struct DerivationGoal : public Goal std::string key() override; - kj::Promise> work(bool inBuildSlot) noexcept override; + kj::Promise> work() noexcept override; /** * Add wanted outputs to an already existing derivation goal. diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index fdee53699..6ef00d1ff 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -42,7 +42,10 @@ try { trace("trying next substituter"); if (!inBuildSlot) { - return {WaitForSlot{}}; + return worker.substitutions.acquire().then([this](auto token) { + slotToken = std::move(token); + return work(); + }); } maintainRunningSubstitutions = worker.runningSubstitutions.addTemporarily(1); @@ -81,7 +84,7 @@ try { state = &DrvOutputSubstitutionGoal::realisationFetched; return {WaitForWorld{ - pipe.promise.then([]() -> Outcome { return result::success(); }), true + pipe.promise.then([]() -> Outcome { return result::success(); }) }}; } catch (...) { return {std::current_exception()}; @@ -90,6 +93,7 @@ try { kj::Promise> DrvOutputSubstitutionGoal::realisationFetched(bool inBuildSlot) noexcept try { maintainRunningSubstitutions.reset(); + slotToken = {}; try { outputInfo = downloadState->result.get(); @@ -168,9 +172,9 @@ std::string DrvOutputSubstitutionGoal::key() return "a$" + std::string(id.to_string()); } -kj::Promise> DrvOutputSubstitutionGoal::work(bool inBuildSlot) noexcept +kj::Promise> DrvOutputSubstitutionGoal::work() noexcept { - return (this->*state)(inBuildSlot); + return (this->*state)(slotToken.valid()); } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index a35bf67ee..805b65bfa 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -76,7 +76,7 @@ public: std::string key() override; - kj::Promise> work(bool inBuildSlot) noexcept override; + kj::Promise> work() noexcept override; JobCategory jobCategory() const override { return JobCategory::Substitution; diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index fbf767e8d..1ccf9716b 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "async-semaphore.hh" #include "result.hh" #include "types.hh" #include "store-api.hh" @@ -112,19 +113,20 @@ struct Goal */ BuildResult buildResult; +protected: + AsyncSemaphore::Token slotToken; + public: struct Finished; struct [[nodiscard]] StillAlive {}; - struct [[nodiscard]] WaitForSlot {}; struct [[nodiscard]] ContinueImmediately {}; struct [[nodiscard]] WaitForGoals { Goals goals; }; struct [[nodiscard]] WaitForWorld { kj::Promise> promise; - bool inBuildSlot; }; struct [[nodiscard]] Finished { ExitCode exitCode; @@ -138,7 +140,6 @@ public: struct [[nodiscard]] WorkResult : std::variant< StillAlive, - WaitForSlot, ContinueImmediately, WaitForGoals, WaitForWorld, @@ -168,7 +169,7 @@ public: trace("goal destroyed"); } - virtual kj::Promise> work(bool inBuildSlot) noexcept = 0; + virtual kj::Promise> work() noexcept = 0; virtual void waiteeDone(GoalPtr waitee) { } diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9ec87f1b6..2443cfb5a 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -156,8 +156,11 @@ try { if (!inBuildSlot) { state = &DerivationGoal::tryToBuild; outputLocks.unlock(); - if (0U != settings.maxBuildJobs) { - return {WaitForSlot{}}; + if (worker.localBuilds.capacity() > 0) { + return worker.localBuilds.acquire().then([this](auto token) { + slotToken = std::move(token); + return work(); + }); } if (getMachines().empty()) { throw Error( @@ -248,7 +251,7 @@ try { state = &DerivationGoal::buildDone; started(); - return {WaitForWorld{std::move(promise), true}}; + return {WaitForWorld{std::move(promise)}}; } catch (BuildError & e) { outputLocks.unlock(); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 058f858d4..6d90196fa 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -45,9 +45,9 @@ Goal::Finished PathSubstitutionGoal::done( } -kj::Promise> PathSubstitutionGoal::work(bool inBuildSlot) noexcept +kj::Promise> PathSubstitutionGoal::work() noexcept { - return (this->*state)(inBuildSlot); + return (this->*state)(slotToken.valid()); } @@ -203,7 +203,10 @@ try { trace("trying to run"); if (!inBuildSlot) { - return {WaitForSlot{}}; + return worker.substitutions.acquire().then([this](auto token) { + slotToken = std::move(token); + return work(); + }); } maintainRunningSubstitutions = worker.runningSubstitutions.addTemporarily(1); @@ -236,7 +239,7 @@ try { state = &PathSubstitutionGoal::finished; return {WaitForWorld{ - pipe.promise.then([]() -> Outcome { return result::success(); }), true + pipe.promise.then([]() -> Outcome { return result::success(); }) }}; } catch (...) { return {std::current_exception()}; @@ -248,6 +251,7 @@ try { trace("substitute finished"); try { + slotToken = {}; thr.get(); } catch (std::exception & e) { printError(e.what()); diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 91e256fd7..cef3a4c5c 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -99,7 +99,7 @@ public: return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); } - kj::Promise> work(bool inBuildSlot) noexcept override; + kj::Promise> work() noexcept override; /** * The states. diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 2cc2828b1..e19917d91 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -27,11 +27,13 @@ Worker::Worker(Store & store, Store & evalStore, kj::AsyncIoContext & aio) , store(store) , evalStore(evalStore) , aio(aio) + /* Make sure that we are always allowed to run at least one substitution. + This prevents infinite waiting. */ + , substitutions(std::max(1, settings.maxSubstitutionJobs)) + , localBuilds(settings.maxBuildJobs) , children(errorHandler) { /* Debugging: prevent recursive workers. */ - nrLocalBuilds = 0; - nrSubstitutions = 0; } @@ -210,7 +212,6 @@ void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how) std::visit( overloaded{ [&](Goal::StillAlive) {}, - [&](Goal::WaitForSlot) { waitForBuildSlot(goal); }, [&](Goal::ContinueImmediately) { wakeUp(goal); }, [&](Goal::WaitForGoals & w) { for (auto & dep : w.goals) { @@ -219,19 +220,15 @@ void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how) } }, [&](Goal::WaitForWorld & w) { - childStarted( - goal, - w.promise.then([](auto r) -> Result { - if (r.has_value()) { - return {Goal::ContinueImmediately{}}; - } else if (r.has_error()) { - return {std::move(r).error()}; - } else { - return r.exception(); - } - }), - w.inBuildSlot - ); + childStarted(goal, w.promise.then([](auto r) -> Result { + if (r.has_value()) { + return {Goal::ContinueImmediately{}}; + } else if (r.has_error()) { + return {std::move(r).error()}; + } else { + return r.exception(); + } + })); }, [&](Goal::Finished & f) { goalFinished(goal, f); }, }, @@ -268,8 +265,7 @@ void Worker::wakeUp(GoalPtr goal) } -void Worker::childStarted(GoalPtr goal, kj::Promise> promise, - bool inBuildSlot) +void Worker::childStarted(GoalPtr goal, kj::Promise> promise) { children.add(promise .then([this, goal](auto result) { @@ -279,64 +275,17 @@ void Worker::childStarted(GoalPtr goal, kj::Promise> pr childException = result.assume_error(); } }) - .attach(Finally{[this, goal, inBuildSlot] { - childTerminated(goal, inBuildSlot); + .attach(Finally{[this, goal] { + childTerminated(goal); }})); - if (inBuildSlot) { - switch (goal->jobCategory()) { - case JobCategory::Substitution: - nrSubstitutions++; - break; - case JobCategory::Build: - nrLocalBuilds++; - break; - default: - abort(); - } - } } -void Worker::childTerminated(GoalPtr goal, bool inBuildSlot) +void Worker::childTerminated(GoalPtr goal) { if (childFinished) { childFinished->fulfill(); } - - if (inBuildSlot) { - switch (goal->jobCategory()) { - case JobCategory::Substitution: - assert(nrSubstitutions > 0); - nrSubstitutions--; - break; - case JobCategory::Build: - assert(nrLocalBuilds > 0); - nrLocalBuilds--; - break; - default: - abort(); - } - } - - /* Wake up goals waiting for a build slot. */ - for (auto & j : wantingToBuild) { - GoalPtr goal = j.lock(); - if (goal) wakeUp(goal); - } - - wantingToBuild.clear(); -} - - -void Worker::waitForBuildSlot(GoalPtr goal) -{ - goal->trace("wait for build slot"); - bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution; - if ((!isSubstitutionGoal && nrLocalBuilds < settings.maxBuildJobs) || - (isSubstitutionGoal && nrSubstitutions < settings.maxSubstitutionJobs)) - wakeUp(goal); /* we can do it right away */ - else - wantingToBuild.insert(goal); } @@ -394,16 +343,11 @@ Goals Worker::run(std::function req) awake.clear(); for (auto & goal : awake2) { checkInterrupt(); - /* Make sure that we are always allowed to run at least one substitution. - This prevents infinite waiting. */ - const bool inSlot = goal->jobCategory() == JobCategory::Substitution - ? nrSubstitutions < std::max(1U, (unsigned int) settings.maxSubstitutionJobs) - : nrLocalBuilds < settings.maxBuildJobs; - auto result = goal->work(inSlot); + auto result = goal->work(); if (result.poll(aio.waitScope)) { handleWorkResult(goal, result.wait(aio.waitScope).value()); } else { - childStarted(goal, std::move(result), false); + childStarted(goal, std::move(result)); } if (topGoals.empty()) break; // stuff may have been cancelled @@ -428,7 +372,6 @@ Goals Worker::run(std::function req) exited while some of its subgoals were still active. But if --keep-going *is* set, then they must all be finished now. */ assert(!settings.keepGoing || awake.empty()); - assert(!settings.keepGoing || wantingToBuild.empty()); assert(!settings.keepGoing || children.isEmpty()); return _topGoals; diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index daa612c06..834ecfda3 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "async-semaphore.hh" #include "notifying-counter.hh" #include "types.hh" #include "lock.hh" @@ -93,22 +94,6 @@ private: */ WeakGoals awake; - /** - * Goals waiting for a build slot. - */ - WeakGoals wantingToBuild; - - /** - * Number of build slots occupied. This includes local builds but does not - * include substitutions or remote builds via the build hook. - */ - unsigned int nrLocalBuilds; - - /** - * Number of substitution slots occupied. - */ - unsigned int nrSubstitutions; - /** * Maps used to prevent multiple instantiations of a goal for the * same derivation / path. @@ -148,12 +133,6 @@ private: kj::Own> childFinished; - /** - * Put `goal` to sleep until a build slot becomes available (which - * might be right away). - */ - void waitForBuildSlot(GoalPtr goal); - /** * Wake up a goal (i.e., there is something for it to do). */ @@ -170,16 +149,14 @@ private: void removeGoal(GoalPtr goal); /** - * Registers a running child process. `inBuildSlot` means that - * the process counts towards the jobs limit. + * Registers a running child process. */ - void childStarted(GoalPtr goal, kj::Promise> promise, - bool inBuildSlot); + void childStarted(GoalPtr goal, kj::Promise> promise); /** * Unregisters a running child process. */ - void childTerminated(GoalPtr goal, bool inBuildSlot); + void childTerminated(GoalPtr goal); /** * Pass current stats counters to the logger for progress bar updates. @@ -205,6 +182,7 @@ public: Store & store; Store & evalStore; kj::AsyncIoContext & aio; + AsyncSemaphore substitutions, localBuilds; private: kj::TaskSet children; -- 2.44.1 From ae5d8dae1b796967c40b1f22b844d07f5697033e Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Tue, 24 Sep 2024 00:21:16 +0200 Subject: [PATCH 060/106] libstore: turn Goal::WaitForGoals into a promise also gets rid of explicit strong references to dependencies of any goal, and weak references to dependers as well. those are now only held within promises representing goal completion and thus independent of the goal's relation to each other. the weak references to dependers was only needed for notifications, and that's much better handled entirely by kj itself. Change-Id: I00d06df9090f8d6336ee4bb0c1313a7052fb016b --- src/libstore/build/derivation-goal.cc | 40 ++++--- .../build/drv-output-substitution-goal.cc | 13 ++- src/libstore/build/entry-points.cc | 30 +++-- src/libstore/build/goal.cc | 29 +++++ src/libstore/build/goal.hh | 28 +++-- src/libstore/build/substitution-goal.cc | 10 +- src/libstore/build/worker.cc | 110 ++++++++---------- src/libstore/build/worker.hh | 60 +++++++--- 8 files changed, 192 insertions(+), 128 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3c4257f08..b8c4d278d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -13,8 +13,11 @@ #include #include +#include #include +#include #include +#include #include #include #include @@ -173,7 +176,7 @@ try { state = &DerivationGoal::loadDerivation; - return {WaitForGoals{{worker.goalFactory().makePathSubstitutionGoal(drvPath)}}}; + return waitForGoals(worker.goalFactory().makePathSubstitutionGoal(drvPath)); } catch (...) { return {std::current_exception()}; } @@ -272,13 +275,13 @@ try { /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ - WaitForGoals result; + kj::Vector>> dependencies; if (settings.useSubstitutes) { if (parsedDrv->substitutesAllowed()) { for (auto & [outputName, status] : initialOutputs) { if (!status.wanted) continue; if (!status.known) - result.goals.insert( + dependencies.add( worker.goalFactory().makeDrvOutputSubstitutionGoal( DrvOutput{status.outputHash, outputName}, buildMode == bmRepair ? Repair : NoRepair @@ -286,7 +289,7 @@ try { ); else { auto * cap = getDerivationCA(*drv); - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal( + dependencies.add(worker.goalFactory().makePathSubstitutionGoal( status.known->path, buildMode == bmRepair ? Repair : NoRepair, cap ? std::optional { *cap } : std::nullopt)); @@ -297,11 +300,11 @@ try { } } - if (result.goals.empty()) { /* to prevent hang (no wake-up event) */ + if (dependencies.empty()) { /* to prevent hang (no wake-up event) */ return outputsSubstitutionTried(inBuildSlot); } else { state = &DerivationGoal::outputsSubstitutionTried; - return {std::move(result)}; + return waitForGoals(dependencies.releaseAsArray()); } } catch (...) { return {std::current_exception()}; @@ -383,7 +386,7 @@ try { produced using a substitute. So we have to build instead. */ kj::Promise> DerivationGoal::gaveUpOnSubstitution(bool inBuildSlot) noexcept try { - WaitForGoals result; + kj::Vector>> dependencies; /* At this point we are building all outputs, so if more are wanted there is no need to restart. */ @@ -396,7 +399,7 @@ try { addWaiteeDerivedPath = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { if (!inputNode.value.empty()) - result.goals.insert(worker.goalFactory().makeGoal( + dependencies.add(worker.goalFactory().makeGoal( DerivedPath::Built { .drvPath = inputDrv, .outputs = inputNode.value, @@ -441,14 +444,14 @@ try { if (!settings.useSubstitutes) throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i)); + dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i)); } - if (result.goals.empty()) {/* to prevent hang (no wake-up event) */ + if (dependencies.empty()) {/* to prevent hang (no wake-up event) */ return inputsRealised(inBuildSlot); } else { state = &DerivationGoal::inputsRealised; - return {result}; + return waitForGoals(dependencies.releaseAsArray()); } } catch (...) { return {std::current_exception()}; @@ -491,7 +494,7 @@ try { } /* Check each path (slow!). */ - WaitForGoals result; + kj::Vector>> dependencies; for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; printError( @@ -499,9 +502,9 @@ try { worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i, Repair)); + dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i, Repair)); else - result.goals.insert(worker.goalFactory().makeGoal( + dependencies.add(worker.goalFactory().makeGoal( DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath2->second), .outputs = OutputsSpec::All { }, @@ -509,12 +512,12 @@ try { bmRepair)); } - if (result.goals.empty()) { + if (dependencies.empty()) { return {done(BuildResult::AlreadyValid, assertPathValidity())}; } state = &DerivationGoal::closureRepaired; - return {result}; + return waitForGoals(dependencies.releaseAsArray()); } catch (...) { return {std::current_exception()}; } @@ -614,11 +617,12 @@ try { worker.store.printStorePath(pathResolved), }); - resolvedDrvGoal = worker.goalFactory().makeDerivationGoal( + auto dependency = worker.goalFactory().makeDerivationGoal( pathResolved, wantedOutputs, buildMode); + resolvedDrvGoal = dependency.first; state = &DerivationGoal::resolvedFinished; - return {WaitForGoals{{resolvedDrvGoal}}}; + return waitForGoals(std::move(dependency)); } std::function::ChildNode &)> accumInputPaths; diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 6ef00d1ff..80b2c4cfb 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -4,6 +4,9 @@ #include "worker.hh" #include "substitution-goal.hh" #include "signals.hh" +#include +#include +#include namespace nix { @@ -106,7 +109,7 @@ try { return tryNext(inBuildSlot); } - WaitForGoals result; + kj::Vector>> dependencies; for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { if (depId != id) { if (auto localOutputInfo = worker.store.queryRealisation(depId); @@ -122,17 +125,17 @@ try { ); return tryNext(inBuildSlot); } - result.goals.insert(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId)); + dependencies.add(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId)); } } - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath)); + dependencies.add(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath)); - if (result.goals.empty()) { + if (dependencies.empty()) { return outPathValid(inBuildSlot); } else { state = &DrvOutputSubstitutionGoal::outPathValid; - return {std::move(result)}; + return waitForGoals(dependencies.releaseAsArray()); } } catch (...) { return {std::current_exception()}; diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index a0f18a02c..27c341295 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -17,9 +17,9 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod Worker worker(*this, evalStore ? *evalStore : *this, aio); auto goals = runWorker(worker, [&](GoalFactory & gf) { - Goals goals; + Worker::Targets goals; for (auto & br : reqs) - goals.insert(gf.makeGoal(br, buildMode)); + goals.emplace(gf.makeGoal(br, buildMode)); return goals; }); @@ -60,11 +60,11 @@ std::vector Store::buildPathsWithResults( std::vector> state; auto goals = runWorker(worker, [&](GoalFactory & gf) { - Goals goals; + Worker::Targets goals; for (const auto & req : reqs) { auto goal = gf.makeGoal(req, buildMode); - goals.insert(goal); - state.push_back({req, goal}); + state.push_back({req, goal.first}); + goals.emplace(std::move(goal)); } return goals; }); @@ -84,8 +84,10 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat Worker worker(*this, *this, aio); try { - auto goals = runWorker(worker, [&](GoalFactory & gf) -> Goals { - return Goals{gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)}; + auto goals = runWorker(worker, [&](GoalFactory & gf) { + Worker::Targets goals; + goals.emplace(gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)); + return goals; }); auto goal = *goals.begin(); return goal->buildResult.restrictTo(DerivedPath::Built { @@ -110,7 +112,9 @@ void Store::ensurePath(const StorePath & path) Worker worker(*this, *this, aio); auto goals = runWorker(worker, [&](GoalFactory & gf) { - return Goals{gf.makePathSubstitutionGoal(path)}; + Worker::Targets goals; + goals.emplace(gf.makePathSubstitutionGoal(path)); + return goals; }); auto goal = *goals.begin(); @@ -130,7 +134,9 @@ void Store::repairPath(const StorePath & path) Worker worker(*this, *this, aio); auto goals = runWorker(worker, [&](GoalFactory & gf) { - return Goals{gf.makePathSubstitutionGoal(path, Repair)}; + Worker::Targets goals; + goals.emplace(gf.makePathSubstitutionGoal(path, Repair)); + return goals; }); auto goal = *goals.begin(); @@ -140,14 +146,16 @@ void Store::repairPath(const StorePath & path) auto info = queryPathInfo(path); if (info->deriver && isValidPath(*info->deriver)) { worker.run([&](GoalFactory & gf) { - return Goals{gf.makeGoal( + Worker::Targets goals; + goals.emplace(gf.makeGoal( DerivedPath::Built{ .drvPath = makeConstantStorePathRef(*info->deriver), // FIXME: Should just build the specific output we need. .outputs = OutputsSpec::All{}, }, bmRepair - )}; + )); + return goals; }); } else throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path)); diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 649093dbd..8a2f4ab35 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -1,4 +1,5 @@ #include "goal.hh" +#include "async-collect.hh" #include "worker.hh" #include @@ -28,4 +29,32 @@ try { co_return std::current_exception(); } +kj::Promise> +Goal::waitForGoals(kj::Array>> dependencies) noexcept +try { + auto left = dependencies.size(); + auto collectDeps = asyncCollect(std::move(dependencies)); + + while (auto item = co_await collectDeps.next()) { + left--; + auto & dep = *item; + + trace(fmt("waitee '%s' done; %d left", dep->name, left)); + + if (dep->exitCode != Goal::ecSuccess) ++nrFailed; + if (dep->exitCode == Goal::ecNoSubstituters) ++nrNoSubstituters; + if (dep->exitCode == Goal::ecIncompleteClosure) ++nrIncompleteClosure; + + waiteeDone(dep); + + if (dep->exitCode == ecFailed && !settings.keepGoing) { + co_return result::success(ContinueImmediately{}); + } + } + + co_return result::success(ContinueImmediately{}); +} catch (...) { + co_return result::failure(std::current_exception()); +} + } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 1ccf9716b..e7a500a00 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -6,6 +6,7 @@ #include "types.hh" #include "store-api.hh" #include "build-result.hh" +#include // IWYU pragma: keep #include namespace nix { @@ -70,17 +71,6 @@ struct Goal */ const bool isDependency; - /** - * Goals that this goal is waiting for. - */ - Goals waitees; - - /** - * Goals waiting for this one to finish. Must use weak pointers - * here to prevent cycles. - */ - WeakGoals waiters; - /** * Number of goals we are/were waiting for that have failed. */ @@ -113,6 +103,9 @@ struct Goal */ BuildResult buildResult; + // for use by Worker only. will go away once work() is a promise. + kj::Own> notify; + protected: AsyncSemaphore::Token slotToken; @@ -122,9 +115,6 @@ public: struct [[nodiscard]] StillAlive {}; struct [[nodiscard]] ContinueImmediately {}; - struct [[nodiscard]] WaitForGoals { - Goals goals; - }; struct [[nodiscard]] WaitForWorld { kj::Promise> promise; }; @@ -141,7 +131,6 @@ public: struct [[nodiscard]] WorkResult : std::variant< StillAlive, ContinueImmediately, - WaitForGoals, WaitForWorld, Finished> { @@ -151,6 +140,15 @@ public: protected: kj::Promise> waitForAWhile(); + kj::Promise> + waitForGoals(kj::Array>> dependencies) noexcept; + + template... G> + kj::Promise> + waitForGoals(std::pair, kj::Promise>... goals) noexcept + { + return waitForGoals(kj::arrOf>>(std::move(goals)...)); + } public: diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 6d90196fa..74a63ca21 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -3,6 +3,8 @@ #include "nar-info.hh" #include "signals.hh" #include "finally.hh" +#include +#include namespace nix { @@ -160,16 +162,16 @@ try { /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - WaitForGoals result; + kj::Vector>> dependencies; for (auto & i : info->references) if (i != storePath) /* ignore self-references */ - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i)); + dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i)); - if (result.goals.empty()) {/* to prevent hang (no wake-up event) */ + if (dependencies.empty()) {/* to prevent hang (no wake-up event) */ return referencesValid(inBuildSlot); } else { state = &PathSubstitutionGoal::referencesValid; - return {std::move(result)}; + return waitForGoals(dependencies.releaseAsArray()); } } catch (...) { return {std::current_exception()}; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index e19917d91..68071a94c 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -52,26 +52,28 @@ Worker::~Worker() } -std::shared_ptr Worker::makeDerivationGoalCommon( +std::pair, kj::Promise> Worker::makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, std::function()> mkDrvGoal) { - std::weak_ptr & goal_weak = derivationGoals[drvPath]; - std::shared_ptr goal = goal_weak.lock(); + auto & goal_weak = derivationGoals[drvPath]; + std::shared_ptr goal = goal_weak.goal.lock(); if (!goal) { goal = mkDrvGoal(); - goal_weak = goal; + goal->notify = std::move(goal_weak.fulfiller); + goal_weak.goal = goal; wakeUp(goal); } else { goal->addWantedOutputs(wantedOutputs); } - return goal; + return {goal, goal_weak.promise->addBranch()}; } -std::shared_ptr Worker::makeDerivationGoal(const StorePath & drvPath, - const OutputsSpec & wantedOutputs, BuildMode buildMode) +std::pair, kj::Promise> Worker::makeDerivationGoal( + const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode +) { return makeDerivationGoalCommon( drvPath, @@ -89,8 +91,12 @@ std::shared_ptr Worker::makeDerivationGoal(const StorePath & drv } -std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode) +std::pair, kj::Promise> Worker::makeBasicDerivationGoal( + const StorePath & drvPath, + const BasicDerivation & drv, + const OutputsSpec & wantedOutputs, + BuildMode buildMode +) { return makeDerivationGoalCommon( drvPath, @@ -108,55 +114,63 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -std::shared_ptr Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +std::pair, kj::Promise> +Worker::makePathSubstitutionGoal( + const StorePath & path, RepairFlag repair, std::optional ca +) { - std::weak_ptr & goal_weak = substitutionGoals[path]; - auto goal = goal_weak.lock(); // FIXME + auto & goal_weak = substitutionGoals[path]; + auto goal = goal_weak.goal.lock(); // FIXME if (!goal) { goal = std::make_shared(path, *this, running, repair, ca); - goal_weak = goal; + goal->notify = std::move(goal_weak.fulfiller); + goal_weak.goal = goal; wakeUp(goal); } - return goal; + return {goal, goal_weak.promise->addBranch()}; } -std::shared_ptr Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional ca) +std::pair, kj::Promise> +Worker::makeDrvOutputSubstitutionGoal( + const DrvOutput & id, RepairFlag repair, std::optional ca +) { - std::weak_ptr & goal_weak = drvOutputSubstitutionGoals[id]; - auto goal = goal_weak.lock(); // FIXME + auto & goal_weak = drvOutputSubstitutionGoals[id]; + auto goal = goal_weak.goal.lock(); // FIXME if (!goal) { goal = std::make_shared(id, *this, running, repair, ca); - goal_weak = goal; + goal->notify = std::move(goal_weak.fulfiller); + goal_weak.goal = goal; wakeUp(goal); } - return goal; + return {goal, goal_weak.promise->addBranch()}; } -GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) +std::pair> Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) -> GoalPtr { + [&](const DerivedPath::Built & bfd) -> std::pair> { if (auto bop = std::get_if(&*bfd.drvPath)) return makeDerivationGoal(bop->path, bfd.outputs, buildMode); else throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); }, - [&](const DerivedPath::Opaque & bo) -> GoalPtr { + [&](const DerivedPath::Opaque & bo) -> std::pair> { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); }, }, req.raw()); } -template -static void removeGoal(std::shared_ptr goal, std::map> & goalMap) +template +static void removeGoal(std::shared_ptr goal, auto & goalMap) { /* !!! inefficient */ for (auto i = goalMap.begin(); i != goalMap.end(); ) - if (i->second.lock() == goal) { + if (i->second.goal.lock() == goal) { auto j = i; ++j; goalMap.erase(i); i = j; @@ -177,33 +191,8 @@ void Worker::goalFinished(GoalPtr goal, Goal::Finished & f) hashMismatch |= f.hashMismatch; checkMismatch |= f.checkMismatch; - for (auto & i : goal->waiters) { - if (GoalPtr waiting = i.lock()) { - assert(waiting->waitees.count(goal)); - waiting->waitees.erase(goal); - - waiting->trace(fmt("waitee '%s' done; %d left", goal->name, waiting->waitees.size())); - - if (f.exitCode != Goal::ecSuccess) ++waiting->nrFailed; - if (f.exitCode == Goal::ecNoSubstituters) ++waiting->nrNoSubstituters; - if (f.exitCode == Goal::ecIncompleteClosure) ++waiting->nrIncompleteClosure; - - if (waiting->waitees.empty() || (f.exitCode == Goal::ecFailed && !settings.keepGoing)) { - /* If we failed and keepGoing is not set, we remove all - remaining waitees. */ - for (auto & i : waiting->waitees) { - i->waiters.extract(waiting); - } - waiting->waitees.clear(); - - wakeUp(waiting); - } - - waiting->waiteeDone(goal); - } - } - goal->waiters.clear(); removeGoal(goal); + goal->notify->fulfill(); goal->cleanup(); } @@ -213,12 +202,6 @@ void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how) overloaded{ [&](Goal::StillAlive) {}, [&](Goal::ContinueImmediately) { wakeUp(goal); }, - [&](Goal::WaitForGoals & w) { - for (auto & dep : w.goals) { - goal->waitees.insert(dep); - dep->waiters.insert(goal); - } - }, [&](Goal::WaitForWorld & w) { childStarted(goal, w.promise.then([](auto r) -> Result { if (r.has_value()) { @@ -310,7 +293,7 @@ void Worker::updateStatistics() } } -Goals Worker::run(std::function req) +std::vector Worker::run(std::function req) { auto _topGoals = req(goalFactory()); @@ -320,7 +303,10 @@ Goals Worker::run(std::function req) updateStatistics(); - topGoals = _topGoals; + topGoals.clear(); + for (auto & [goal, _promise] : _topGoals) { + topGoals.insert(goal); + } debug("entered goal loop"); @@ -374,7 +360,11 @@ Goals Worker::run(std::function req) assert(!settings.keepGoing || awake.empty()); assert(!settings.keepGoing || children.isEmpty()); - return _topGoals; + std::vector results; + for (auto & [i, _p] : _topGoals) { + results.push_back(i); + } + return results; } void Worker::waitForInput() diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 834ecfda3..925d289bf 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -28,10 +28,10 @@ struct HookInstance; class GoalFactory { public: - virtual std::shared_ptr makeDerivationGoal( + virtual std::pair, kj::Promise> makeDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal ) = 0; - virtual std::shared_ptr makeBasicDerivationGoal( + virtual std::pair, kj::Promise> makeBasicDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, @@ -41,12 +41,14 @@ public: /** * @ref SubstitutionGoal "substitution goal" */ - virtual std::shared_ptr makePathSubstitutionGoal( + virtual std::pair, kj::Promise> + makePathSubstitutionGoal( const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt ) = 0; - virtual std::shared_ptr makeDrvOutputSubstitutionGoal( + virtual std::pair, kj::Promise> + makeDrvOutputSubstitutionGoal( const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt @@ -58,7 +60,8 @@ public: * It will be a `DerivationGoal` for a `DerivedPath::Built` or * a `SubstitutionGoal` for a `DerivedPath::Opaque`. */ - virtual GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0; + virtual std::pair> + makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0; }; // elaborate hoax to let goals access factory methods while hiding them from the public @@ -94,13 +97,27 @@ private: */ WeakGoals awake; + template + struct CachedGoal + { + std::weak_ptr goal; + kj::Own> promise; + kj::Own> fulfiller; + + CachedGoal() + { + auto pf = kj::newPromiseAndFulfiller(); + promise = kj::heap(pf.promise.fork()); + fulfiller = std::move(pf.fulfiller); + } + }; /** * Maps used to prevent multiple instantiations of a goal for the * same derivation / path. */ - std::map> derivationGoals; - std::map> substitutionGoals; - std::map> drvOutputSubstitutionGoals; + std::map> derivationGoals; + std::map> substitutionGoals; + std::map> drvOutputSubstitutionGoals; /** * Cache for pathContentsGood(). @@ -226,21 +243,31 @@ public: * @ref DerivationGoal "derivation goal" */ private: - std::shared_ptr makeDerivationGoalCommon( + std::pair, kj::Promise> makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, std::function()> mkDrvGoal); - std::shared_ptr makeDerivationGoal( + std::pair, kj::Promise> makeDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override; - std::shared_ptr makeBasicDerivationGoal( + std::pair, kj::Promise> makeBasicDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override; /** * @ref SubstitutionGoal "substitution goal" */ - std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt) override; - std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt) override; + std::pair, kj::Promise> + makePathSubstitutionGoal( + const StorePath & storePath, + RepairFlag repair = NoRepair, + std::optional ca = std::nullopt + ) override; + std::pair, kj::Promise> + makeDrvOutputSubstitutionGoal( + const DrvOutput & id, + RepairFlag repair = NoRepair, + std::optional ca = std::nullopt + ) override; /** * Make a goal corresponding to the `DerivedPath`. @@ -248,13 +275,16 @@ private: * It will be a `DerivationGoal` for a `DerivedPath::Built` or * a `SubstitutionGoal` for a `DerivedPath::Opaque`. */ - GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override; + std::pair> + makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override; public: + using Targets = std::map>; + /** * Loop until the specified top-level goals have finished. */ - Goals run(std::function req); + std::vector run(std::function req); /*** * The exit status in case of failure. -- 2.44.1 From 0e6b3435a14a304b8833c27d2911de7ac4e731d4 Mon Sep 17 00:00:00 2001 From: Olivia Crain Date: Thu, 26 Sep 2024 14:28:25 -0500 Subject: [PATCH 061/106] build: install html manual without using install_subdir In Meson, `install_subdir` is meant to be used with directories in the source directory. When using it to install the HTML manual, we provide it with a path under the build directory. We should instead specify an install directory for the HTML manual as part of the custom target that builds it. What we do currently isn't broken, just semantically incorrect. Changing it does get rid of the following deprecation warning, though: `` Project [...] uses feature deprecated since '0.60.0': install_subdir with empty directory. It worked by accident and is buggy. Use install_emptydir instead. `` Change-Id: I259583b7bdff8ecbb3b342653d70dc5f034c7fad --- doc/manual/meson.build | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/doc/manual/meson.build b/doc/manual/meson.build index f53d41b5d..38aad55b5 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -126,20 +126,18 @@ manual = custom_target( 'manual', 'markdown', ], + install_dir : [ + datadir / 'doc/nix', + false, + ], depfile : 'manual.d', env : { 'RUST_LOG': 'info', 'MDBOOK_SUBSTITUTE_SEARCH': meson.current_build_dir() / 'src', }, ) -manual_html = manual[0] manual_md = manual[1] -install_subdir( - manual_html.full_path(), - install_dir : datadir / 'doc/nix', -) - nix_nested_manpages = [ [ 'nix-env', [ -- 2.44.1 From 624f44bf25902beaf5599bf031ef3610820b1f6a Mon Sep 17 00:00:00 2001 From: Olivia Crain Date: Thu, 26 Sep 2024 14:28:25 -0500 Subject: [PATCH 062/106] build: fix deprecated uses of configure_file Using `configure_file` to copy files has been deprecated since Meson 0.64.0. The intended replacement is the `fs.copyfile` method. This removes the following deprecation warning that arises when a minimum Meson version is specified: `` Project [...] uses feature deprecated since '0.64.0': copy arg in configure_file. Use fs.copyfile instead `` Change-Id: I09ffc92e96311ef9ed594343a0a16d51e74b114a --- meson.build | 8 ++++---- misc/bash/meson.build | 7 +++---- misc/fish/meson.build | 7 +++---- misc/meson.build | 6 +----- misc/zsh/meson.build | 7 +++---- scripts/meson.build | 7 +------ src/nix-channel/meson.build | 6 +----- 7 files changed, 16 insertions(+), 32 deletions(-) diff --git a/meson.build b/meson.build index f89f5a016..53ce6a348 100644 --- a/meson.build +++ b/meson.build @@ -593,10 +593,10 @@ run_command( ) if is_darwin - configure_file( - input : 'misc/launchd/org.nixos.nix-daemon.plist.in', - output : 'org.nixos.nix-daemon.plist', - copy : true, + fs.copyfile( + 'misc/launchd/org.nixos.nix-daemon.plist.in', + 'org.nixos.nix-daemon.plist', + install : true, install_dir : prefix / 'Library/LaunchDaemons', ) endif diff --git a/misc/bash/meson.build b/misc/bash/meson.build index 75acce2ea..178692536 100644 --- a/misc/bash/meson.build +++ b/misc/bash/meson.build @@ -1,8 +1,7 @@ -configure_file( - input : 'completion.sh', - output : 'nix', +fs.copyfile( + 'completion.sh', + 'nix', install : true, install_dir : datadir / 'bash-completion/completions', install_mode : 'rw-r--r--', - copy : true, ) diff --git a/misc/fish/meson.build b/misc/fish/meson.build index d54de9a13..7f9cd0896 100644 --- a/misc/fish/meson.build +++ b/misc/fish/meson.build @@ -1,8 +1,7 @@ -configure_file( - input : 'completion.fish', - output : 'nix.fish', +fs.copyfile( + 'completion.fish', + 'nix.fish', install : true, install_dir : datadir / 'fish/vendor_completions.d', install_mode : 'rw-r--r--', - copy : true, ) diff --git a/misc/meson.build b/misc/meson.build index bf3c157f7..4e2f6aacf 100644 --- a/misc/meson.build +++ b/misc/meson.build @@ -5,8 +5,4 @@ subdir('zsh') subdir('systemd') subdir('flake-registry') -runinpty = configure_file( - copy : true, - input : meson.current_source_dir() / 'runinpty.py', - output : 'runinpty.py', -) +runinpty = fs.copyfile('runinpty.py') diff --git a/misc/zsh/meson.build b/misc/zsh/meson.build index 8063a5cb8..bd388a31f 100644 --- a/misc/zsh/meson.build +++ b/misc/zsh/meson.build @@ -1,10 +1,9 @@ foreach script : [ [ 'completion.zsh', '_nix' ], [ 'run-help-nix' ] ] - configure_file( - input : script[0], - output : script.get(1, script[0]), + fs.copyfile( + script[0], + script.get(1, script[0]), install : true, install_dir : datadir / 'zsh/site-functions', install_mode : 'rw-r--r--', - copy : true, ) endforeach diff --git a/scripts/meson.build b/scripts/meson.build index c916c8efa..e35c6cbb0 100644 --- a/scripts/meson.build +++ b/scripts/meson.build @@ -8,12 +8,7 @@ configure_file( } ) -# https://github.com/mesonbuild/meson/issues/860 -configure_file( - input : 'nix-profile.sh.in', - output : 'nix-profile.sh.in', - copy : true, -) +fs.copyfile('nix-profile.sh.in') foreach rc : [ '.sh', '.fish', '-daemon.sh', '-daemon.fish' ] configure_file( diff --git a/src/nix-channel/meson.build b/src/nix-channel/meson.build index 952dfdb78..97b92d789 100644 --- a/src/nix-channel/meson.build +++ b/src/nix-channel/meson.build @@ -1,5 +1 @@ -configure_file( - input : 'unpack-channel.nix', - output : 'unpack-channel.nix', - copy : true, -) +fs.copyfile('unpack-channel.nix') -- 2.44.1 From b86863d93565a05b7695e3579caa2fe2ab62f3a1 Mon Sep 17 00:00:00 2001 From: Olivia Crain Date: Thu, 26 Sep 2024 22:18:37 -0500 Subject: [PATCH 063/106] build: require meson 1.4.0 or newer This was already the de facto requirement, we use the method `full_path` on a file object (introduced in Meson 1.4.0) in the functional test suite's build. This version of Meson is in NixOS 24.05, so there should be no compatibility issues should this make it into a backported release of Lix. CC: https://git.lix.systems/lix-project/lix/issues/247 Change-Id: I5c640824807353b6eb4287e7ed09c4e89a4bdde2 --- meson.build | 1 + 1 file changed, 1 insertion(+) diff --git a/meson.build b/meson.build index 53ce6a348..ea2050b6b 100644 --- a/meson.build +++ b/meson.build @@ -47,6 +47,7 @@ # in the build directory. project('lix', 'cpp', 'rust', + meson_version : '>=1.4.0', version : run_command('bash', '-c', 'echo -n $(jq -r .version < ./version.json)$VERSION_SUFFIX', check : true).stdout().strip(), default_options : [ 'cpp_std=c++2a', -- 2.44.1 From 4780dd6bc4f33a5340934ac3ff1c47ee682ca395 Mon Sep 17 00:00:00 2001 From: Olivia Crain Date: Tue, 24 Sep 2024 17:23:53 -0500 Subject: [PATCH 064/106] build: let meson add compiler flags for libstdc++ assertions We have manually enabled libstdc++ assertions since cl/797. Meson 1.4.0 (the minimum version we mandate) enables this by default, so we can remove the enabling compiler flag from the list of project arguments. With this patch, `-D_GLIBCXX_ASSERTIONS=1` is still present in the compile command logs when building with both gccStdenv and clangStdenv. See: https://gerrit.lix.systems/c/lix/+/797 See: https://mesonbuild.com/Release-notes-for-1-4-0.html#ndebug-setting-now-controls-c-stdlib-assertions Change-Id: I53483fadfe5cbd11ba35544b437d3a9ee8031631 --- meson.build | 6 ------ 1 file changed, 6 deletions(-) diff --git a/meson.build b/meson.build index ea2050b6b..a8c6b9621 100644 --- a/meson.build +++ b/meson.build @@ -493,12 +493,6 @@ add_project_arguments( '-Wdeprecated-copy', '-Wignored-qualifiers', '-Werror=suggest-override', - # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked - # at ~1% overhead in `nix search`. - # - # FIXME: remove when we get meson 1.4.0 which will default this to on for us: - # https://mesonbuild.com/Release-notes-for-1-4-0.html#ndebug-setting-now-controls-c-stdlib-assertions - '-D_GLIBCXX_ASSERTIONS=1', language : 'cpp', ) -- 2.44.1 From 04daff94e347c03e466577c8c00414f03e27a96f Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Thu, 30 May 2024 13:04:48 +0200 Subject: [PATCH 065/106] libfetchers/git: restore compat with `builtins.fetchGit` from 2.3 Since fb38459d6e58508245553380cccc03c0dbaa1542, each `ref` is appended with `refs/heads` unless it starts with `refs/` already. This regressed two use-cases that worked fine before: * Specifying a commit hash as `ref`: now, if `ref` looks like a commit hash it will be directly passed to `git fetch`. * Specifying a tag without `refs/tags` as prefix: now, the fetcher prepends `refs/*` to a ref that doesn't start with `refs/` and doesn't look like a commit hash. That way, both a branch and a tag specified in `ref` can be fetched. The order of preference in git is * file in `refs/` (e.g. `HEAD`) * file in `refs/tags/` * file in `refs/heads` (i.e. a branch) After fetching `refs/*`, ref is resolved the same way as git does. Change-Id: Idd49b97cbdc8c6fdc8faa5a48bef3dec25e4ccc3 --- doc/manual/rl-next/fetchGit-regression.md | 23 +++++ src/libexpr/primops/fetchTree.cc | 3 +- src/libfetchers/git.cc | 107 ++++++++++++++++++---- tests/functional/fetchGit.sh | 45 +++++++++ 4 files changed, 157 insertions(+), 21 deletions(-) create mode 100644 doc/manual/rl-next/fetchGit-regression.md diff --git a/doc/manual/rl-next/fetchGit-regression.md b/doc/manual/rl-next/fetchGit-regression.md new file mode 100644 index 000000000..f6b4fb9e5 --- /dev/null +++ b/doc/manual/rl-next/fetchGit-regression.md @@ -0,0 +1,23 @@ +--- +synopsis: restore backwards-compatibility of `builtins.fetchGit` with Nix 2.3 +issues: [5291, 5128] +credits: [ma27] +category: Fixes +--- + +Compatibility with `builtins.fetchGit` from Nix 2.3 has been restored as follows: + +* Until now, each `ref` was prefixed with `refs/heads` unless it starts with `refs/` itself. + + Now, this is not done if the `ref` looks like a commit hash. + +* Specifying `builtins.fetchGit { ref = "a-tag"; /* … */ }` was broken because `refs/heads` was appended. + + Now, the fetcher doesn't turn a ref into `refs/heads/ref`, but into `refs/*/ref`. That way, + the value in `ref` can be either a tag or a branch. + +* The ref resolution happens the same way as in git: + + * If `refs/ref` exists, it's used. + * If a tag `refs/tags/ref` exists, it's used. + * If a branch `refs/heads/ref` exists, it's used. diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index b0e14a26e..c98fe2a03 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -394,7 +394,8 @@ static RegisterPrimOp primop_fetchGit({ [Git reference]: https://git-scm.com/book/en/v2/Git-Internals-Git-References By default, the `ref` value is prefixed with `refs/heads/`. - As of 2.3.0, Nix will not prefix `refs/heads/` if `ref` starts with `refs/`. + As of 2.3.0, Nix will not prefix `refs/heads/` if `ref` starts with `refs/` or + if `ref` looks like a commit hash for backwards compatibility with CppNix 2.3. - `submodules` (default: `false`) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7d16d3f57..da60bf331 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1,3 +1,4 @@ +#include "error.hh" #include "fetchers.hh" #include "cache.hh" #include "globals.hh" @@ -257,6 +258,28 @@ std::pair fetchFromWorkdir(ref store, Input & input, co } } // end namespace +static std::optional resolveRefToCachePath( + Input & input, + const Path & cacheDir, + std::vector & gitRefFileCandidates, + std::function condition) +{ + if (input.getRef()->starts_with("refs/")) { + Path fullpath = cacheDir + "/" + *input.getRef(); + if (condition(fullpath)) { + return fullpath; + } + } + + for (auto & candidate : gitRefFileCandidates) { + if (condition(candidate)) { + return candidate; + } + } + + return std::nullopt; +} + struct GitInputScheme : InputScheme { std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override @@ -539,10 +562,13 @@ struct GitInputScheme : InputScheme runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir }); } - Path localRefFile = - input.getRef()->compare(0, 5, "refs/") == 0 - ? cacheDir + "/" + *input.getRef() - : cacheDir + "/refs/heads/" + *input.getRef(); + std::vector gitRefFileCandidates; + for (auto & infix : {"", "tags/", "heads/"}) { + Path p = cacheDir + "/refs/" + infix + *input.getRef(); + gitRefFileCandidates.push_back(p); + } + + Path localRefFile; bool doFetch; time_t now = time(0); @@ -564,29 +590,70 @@ struct GitInputScheme : InputScheme if (allRefs) { doFetch = true; } else { - /* If the local ref is older than ‘tarball-ttl’ seconds, do a - git fetch to update the local ref to the remote ref. */ - struct stat st; - doFetch = stat(localRefFile.c_str(), &st) != 0 || - !isCacheFileWithinTtl(now, st); + std::function condition; + condition = [&now](const Path & path) { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + return stat(path.c_str(), &st) == 0 && + isCacheFileWithinTtl(now, st); + }; + if (auto result = resolveRefToCachePath( + input, + cacheDir, + gitRefFileCandidates, + condition + )) { + localRefFile = *result; + doFetch = false; + } else { + doFetch = true; + } } } + // When having to fetch, we don't know `localRefFile` yet. + // Because git needs to figure out what we're fetching + // (i.e. is it a rev? a branch? a tag?) if (doFetch) { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl)); - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. + auto ref = input.getRef(); + std::string fetchRef; + if (allRefs) { + fetchRef = "refs/*"; + } else if ( + ref->starts_with("refs/") + || *ref == "HEAD" + || std::regex_match(*ref, revRegex)) + { + fetchRef = *ref; + } else { + fetchRef = "refs/*/" + *ref; + } + try { - auto ref = input.getRef(); - auto fetchRef = allRefs - ? "refs/*" - : ref->compare(0, 5, "refs/") == 0 - ? *ref - : ref == "HEAD" - ? *ref - : "refs/heads/" + *ref; - runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, true); + Finally finally([&]() { + if (auto p = resolveRefToCachePath( + input, + cacheDir, + gitRefFileCandidates, + pathExists + )) { + localRefFile = *p; + } + }); + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + runProgram("git", true, { + "-C", repoDir, + "--git-dir", gitDir, + "fetch", + "--quiet", + "--force", + "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) + }, true); } catch (Error & e) { if (!pathExists(localRefFile)) throw; warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index 2c00facc2..492c57602 100644 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -53,8 +53,17 @@ out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = \"file://$repo\" [[ $status == 1 ]] [[ $out =~ 'Cannot find Git revision' ]] +# allow revs as refs (for 2.3 compat) [[ $(nix eval --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; rev = \"$devrev\"; allRefs = true; } + \"/differentbranch\")") = 'different file' ]] +rm -rf "$TEST_ROOT/test-home" +[[ $(nix eval --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; rev = \"$devrev\"; allRefs = true; } + \"/differentbranch\")") = 'different file' ]] + +rm -rf "$TEST_ROOT/test-home" +out=$(nix eval --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; rev = \"$devrev\"; ref = \"lolkek\"; } + \"/differentbranch\")" 2>&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'Cannot find Git revision' ]] + # In pure eval mode, fetchGit without a revision should fail. [[ $(nix eval --impure --raw --expr "builtins.readFile (fetchGit \"file://$repo\" + \"/hello\")") = world ]] (! nix eval --raw --expr "builtins.readFile (fetchGit \"file://$repo\" + \"/hello\")") @@ -228,6 +237,12 @@ export _NIX_FORCE_HTTP=1 rev_tag1_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"refs/tags/tag1\"; }).rev") rev_tag1=$(git -C $repo rev-parse refs/tags/tag1) [[ $rev_tag1_nix = $rev_tag1 ]] + +# Allow fetching tags w/o specifying refs/tags +rm -rf "$TEST_ROOT/test-home" +rev_tag1_nix_alt=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"tag1\"; }).rev") +[[ $rev_tag1_nix_alt = $rev_tag1 ]] + rev_tag2_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"refs/tags/tag2\"; }).rev") rev_tag2=$(git -C $repo rev-parse refs/tags/tag2) [[ $rev_tag2_nix = $rev_tag2 ]] @@ -254,3 +269,33 @@ git -C "$repo" add hello .gitignore git -C "$repo" commit -m 'Bla1' cd "$repo" path11=$(nix eval --impure --raw --expr "(builtins.fetchGit ./.).outPath") + +# test behavior if both branch and tag with same name exist +repo="$TEST_ROOT/git" +rm -rf "$repo"/.git +git init "$repo" +git -C "$repo" config user.email "foobar@example.com" +git -C "$repo" config user.name "Foobar" + +touch "$repo"/test +echo "hello world" > "$repo"/test +git -C "$repo" checkout -b branch +git -C "$repo" add test + +git -C "$repo" commit -m "Init" + +git -C "$repo" tag branch + +echo "goodbye world" > "$repo"/test +git -C "$repo" add test +git -C "$repo" commit -m "Update test" + +path12=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"branch\"; }).outPath") +[[ "$(cat "$path12"/test)" =~ 'hello world' ]] +[[ "$(cat "$repo"/test)" =~ 'goodbye world' ]] + +path13=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"refs/heads/branch\"; }).outPath") +[[ "$(cat "$path13"/test)" =~ 'goodbye world' ]] + +path14=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"refs/tags/branch\"; }).outPath") +[[ "$path14" = "$path12" ]] -- 2.44.1 From 3f7519526f7e2cd3ede01c3910fbfe2ddf0f051f Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Wed, 25 Sep 2024 23:57:46 +0200 Subject: [PATCH 066/106] libstore: have makeLocalDerivationGoal return unique_ptrs these can be unique rather than shared because shared_ptr has a converting constructor. preparatory refactor for something else and not necessary on its own, and the extra allocations we must do for shared_ptr control blocks isn't usually relevant anyway. Change-Id: I5391715545240c6ec8e83a031206edafdfc6462f --- src/libstore/build/local-derivation-goal.hh | 4 ++-- src/libstore/build/worker.cc | 10 +++++----- src/libstore/build/worker.hh | 2 +- src/libstore/platform.cc | 20 ++++++++++---------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 6239129ab..52b7d4c2e 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -182,7 +182,7 @@ struct LocalDerivationGoal : public DerivationGoal * Create a LocalDerivationGoal without an on-disk .drv file, * possibly a platform-specific subclass */ - static std::shared_ptr makeLocalDerivationGoal( + static std::unique_ptr makeLocalDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, @@ -194,7 +194,7 @@ struct LocalDerivationGoal : public DerivationGoal * Create a LocalDerivationGoal for an on-disk .drv file, * possibly a platform-specific subclass */ - static std::shared_ptr makeLocalDerivationGoal( + static std::unique_ptr makeLocalDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 68071a94c..18cdde63a 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -55,7 +55,7 @@ Worker::~Worker() std::pair, kj::Promise> Worker::makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, - std::function()> mkDrvGoal) + std::function()> mkDrvGoal) { auto & goal_weak = derivationGoals[drvPath]; std::shared_ptr goal = goal_weak.goal.lock(); @@ -78,9 +78,9 @@ std::pair, kj::Promise> Worker::makeDeriva return makeDerivationGoalCommon( drvPath, wantedOutputs, - [&]() -> std::shared_ptr { + [&]() -> std::unique_ptr { return !dynamic_cast(&store) - ? std::make_shared( + ? std::make_unique( drvPath, wantedOutputs, *this, running, buildMode ) : LocalDerivationGoal::makeLocalDerivationGoal( @@ -101,9 +101,9 @@ std::pair, kj::Promise> Worker::makeBasicD return makeDerivationGoalCommon( drvPath, wantedOutputs, - [&]() -> std::shared_ptr { + [&]() -> std::unique_ptr { return !dynamic_cast(&store) - ? std::make_shared( + ? std::make_unique( drvPath, drv, wantedOutputs, *this, running, buildMode ) : LocalDerivationGoal::makeLocalDerivationGoal( diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 925d289bf..46adaa145 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -245,7 +245,7 @@ public: private: std::pair, kj::Promise> makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, - std::function()> mkDrvGoal); + std::function()> mkDrvGoal); std::pair, kj::Promise> makeDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override; diff --git a/src/libstore/platform.cc b/src/libstore/platform.cc index f2c023c82..36f8e352a 100644 --- a/src/libstore/platform.cc +++ b/src/libstore/platform.cc @@ -25,7 +25,7 @@ std::shared_ptr LocalStore::makeLocalStore(const Params & params) #endif } -std::shared_ptr LocalDerivationGoal::makeLocalDerivationGoal( +std::unique_ptr LocalDerivationGoal::makeLocalDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, @@ -34,17 +34,17 @@ std::shared_ptr LocalDerivationGoal::makeLocalDerivationGoa ) { #if __linux__ - return std::make_shared(drvPath, wantedOutputs, worker, isDependency, buildMode); + return std::make_unique(drvPath, wantedOutputs, worker, isDependency, buildMode); #elif __APPLE__ - return std::make_shared(drvPath, wantedOutputs, worker, isDependency, buildMode); + return std::make_unique(drvPath, wantedOutputs, worker, isDependency, buildMode); #elif __FreeBSD__ - return std::make_shared(drvPath, wantedOutputs, worker, isDependency, buildMode); + return std::make_unique(drvPath, wantedOutputs, worker, isDependency, buildMode); #else - return std::make_shared(drvPath, wantedOutputs, worker, isDependency, buildMode); + return std::make_unique(drvPath, wantedOutputs, worker, isDependency, buildMode); #endif } -std::shared_ptr LocalDerivationGoal::makeLocalDerivationGoal( +std::unique_ptr LocalDerivationGoal::makeLocalDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, @@ -54,19 +54,19 @@ std::shared_ptr LocalDerivationGoal::makeLocalDerivationGoa ) { #if __linux__ - return std::make_shared( + return std::make_unique( drvPath, drv, wantedOutputs, worker, isDependency, buildMode ); #elif __APPLE__ - return std::make_shared( + return std::make_unique( drvPath, drv, wantedOutputs, worker, isDependency, buildMode ); #elif __FreeBSD__ - return std::make_shared( + return std::make_unique( drvPath, drv, wantedOutputs, worker, isDependency, buildMode ); #else - return std::make_shared( + return std::make_unique( drvPath, drv, wantedOutputs, worker, isDependency, buildMode ); #endif -- 2.44.1 From 1a52e4f755c3fcc50e58f7a0b04269a3839b7eea Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sat, 28 Sep 2024 23:18:35 +0200 Subject: [PATCH 067/106] libstore: fix build tests the new event loop could very occasionally notice that a dependency of some goal has failed, process the failure, cause the depending goal to fail accordingly, and in the doing of the latter two steps let further dependencies that previously have not been reported as failed do their reporting anyway. in such cases a goal could fail with "1 dependencies failed", but more than one dependency failure message was shown. we'll now report the correct number of failed dependency goals in all cases. Change-Id: I5aa95dcb2db4de4fd5fee8acbf5db833531d81a8 --- src/libstore/build/goal.cc | 18 +++++++++++------- tests/functional/build.sh | 6 +++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 8a2f4ab35..5f0ed485c 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -33,18 +33,22 @@ kj::Promise> Goal::waitForGoals(kj::Array>> dependencies) noexcept try { auto left = dependencies.size(); + for (auto & [dep, p] : dependencies) { + p = p.then([this, dep, &left] { + left--; + trace(fmt("waitee '%s' done; %d left", dep->name, left)); + + if (dep->exitCode != Goal::ecSuccess) ++nrFailed; + if (dep->exitCode == Goal::ecNoSubstituters) ++nrNoSubstituters; + if (dep->exitCode == Goal::ecIncompleteClosure) ++nrIncompleteClosure; + }).eagerlyEvaluate(nullptr); + } + auto collectDeps = asyncCollect(std::move(dependencies)); while (auto item = co_await collectDeps.next()) { - left--; auto & dep = *item; - trace(fmt("waitee '%s' done; %d left", dep->name, left)); - - if (dep->exitCode != Goal::ecSuccess) ++nrFailed; - if (dep->exitCode == Goal::ecNoSubstituters) ++nrNoSubstituters; - if (dep->exitCode == Goal::ecIncompleteClosure) ++nrIncompleteClosure; - waiteeDone(dep); if (dep->exitCode == ecFailed && !settings.keepGoing) { diff --git a/tests/functional/build.sh b/tests/functional/build.sh index 356985a64..f80711b31 100644 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -167,9 +167,9 @@ test "$(<<<"$out" grep -E '^error:' | wc -l)" = 4 out="$(nix build -f fod-failing.nix -L x4 2>&1)" && status=0 || status=$? test "$status" = 1 -test "$(<<<"$out" grep -E '^error:' | wc -l)" = 2 -<<<"$out" grepQuiet -E "error: 1 dependencies of derivation '.*-x4\\.drv' failed to build" -<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x2\\.drv'" +test "$(<<<"$out" grep -E '^error:' | wc -l)" -ge 2 +<<<"$out" grepQuiet -E "error: [12] dependencies of derivation '.*-x4\\.drv' failed to build" +<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x[23]\\.drv'" out="$(nix build -f fod-failing.nix -L x4 --keep-going 2>&1)" && status=0 || status=$? test "$status" = 1 -- 2.44.1 From 8fb642b6e09368d10d8357fcb7c508f706fa5f08 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Tue, 24 Sep 2024 00:21:16 +0200 Subject: [PATCH 068/106] libstore: remove Goal::WaitForWorld have DerivationGoal and its subclasses produce a wrapper promise for their intermediate results instead, and return this wrapper promise. Worker already handles promises that do not complete immediately, so we do not have to duplicate this into an entire result type variant. Change-Id: Iae8dbf63cfc742afda4d415922a29ac5a3f39348 --- src/libstore/build/derivation-goal.cc | 15 ++++++++++++++- src/libstore/build/derivation-goal.hh | 2 ++ .../build/drv-output-substitution-goal.cc | 4 +--- src/libstore/build/goal.hh | 4 ---- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/build/substitution-goal.cc | 4 +--- src/libstore/build/worker.cc | 11 ----------- 7 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index b8c4d278d..037b4fb10 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -787,7 +787,7 @@ try { buildResult.startTime = time(0); // inexact state = &DerivationGoal::buildDone; started(); - return {{WaitForWorld{std::move(a.promise)}}}; + return continueOrError(std::move(a.promise)); }, [&](HookReply::Postpone) -> std::optional>> { /* Not now; wait until at least one child finishes or @@ -1756,4 +1756,17 @@ void DerivationGoal::waiteeDone(GoalPtr waitee) } } +kj::Promise> +DerivationGoal::continueOrError(kj::Promise> p) +{ + return p.then([](auto r) -> Result { + if (r.has_value()) { + return ContinueImmediately{}; + } else if (r.has_error()) { + return r.assume_error(); + } else { + return r.assume_exception(); + } + }); +} } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index d60bb0b4c..b1311bd7e 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -327,6 +327,8 @@ protected: Finished tooMuchLogs(); void flushLine(); + static kj::Promise> continueOrError(kj::Promise> p); + public: /** * Wrappers around the corresponding Store methods that first consult the diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 80b2c4cfb..923cbba58 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -86,9 +86,7 @@ try { }); state = &DrvOutputSubstitutionGoal::realisationFetched; - return {WaitForWorld{ - pipe.promise.then([]() -> Outcome { return result::success(); }) - }}; + return pipe.promise.then([]() -> Result { return ContinueImmediately{}; }); } catch (...) { return {std::current_exception()}; } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index e7a500a00..7933fbc31 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -115,9 +115,6 @@ public: struct [[nodiscard]] StillAlive {}; struct [[nodiscard]] ContinueImmediately {}; - struct [[nodiscard]] WaitForWorld { - kj::Promise> promise; - }; struct [[nodiscard]] Finished { ExitCode exitCode; BuildResult result; @@ -131,7 +128,6 @@ public: struct [[nodiscard]] WorkResult : std::variant< StillAlive, ContinueImmediately, - WaitForWorld, Finished> { WorkResult() = delete; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 2443cfb5a..a32f742f4 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -251,7 +251,7 @@ try { state = &DerivationGoal::buildDone; started(); - return {WaitForWorld{std::move(promise)}}; + return continueOrError(std::move(promise)); } catch (BuildError & e) { outputLocks.unlock(); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 74a63ca21..d9d8f1a7d 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -240,9 +240,7 @@ try { }); state = &PathSubstitutionGoal::finished; - return {WaitForWorld{ - pipe.promise.then([]() -> Outcome { return result::success(); }) - }}; + return pipe.promise.then([]() -> Result { return ContinueImmediately{}; }); } catch (...) { return {std::current_exception()}; } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 18cdde63a..1b1bf1d5c 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -202,17 +202,6 @@ void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how) overloaded{ [&](Goal::StillAlive) {}, [&](Goal::ContinueImmediately) { wakeUp(goal); }, - [&](Goal::WaitForWorld & w) { - childStarted(goal, w.promise.then([](auto r) -> Result { - if (r.has_value()) { - return {Goal::ContinueImmediately{}}; - } else if (r.has_error()) { - return {std::move(r).error()}; - } else { - return r.exception(); - } - })); - }, [&](Goal::Finished & f) { goalFinished(goal, f); }, }, how -- 2.44.1 From a5240b23abba2724073f79b79648bf4afb38f70a Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Wed, 25 Sep 2024 23:57:46 +0200 Subject: [PATCH 069/106] libstore: make non-cache goal pointers strong without circular references we do not need weak goal pointers except for caches, which should not prevent goal destructors running. caches though cannot create circular references even when they keep strong references. if we removed goals from caches when their work() is fully finished, not when their destructors are run, we could keep strong pointers in caches. since we do not gain much from this we keep those pointers weak for now. Change-Id: I1d4a6850ff5e264443c90eb4531da89f5e97a3a0 --- src/libstore/build/goal.hh | 7 ------- src/libstore/build/worker.cc | 8 ++------ src/libstore/build/worker.hh | 5 +---- 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 7933fbc31..f808be160 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -21,7 +21,6 @@ class Worker; * A pointer to a goal. */ typedef std::shared_ptr GoalPtr; -typedef std::weak_ptr WeakGoalPtr; struct CompareGoalPtrs { bool operator() (const GoalPtr & a, const GoalPtr & b) const; @@ -31,12 +30,6 @@ struct CompareGoalPtrs { * Set of goals. */ typedef std::set Goals; -typedef std::set> WeakGoals; - -/** - * A map of paths to goals (and the other way around). - */ -typedef std::map WeakGoalMap; /** * Used as a hint to the worker on how to schedule a particular goal. For example, diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 1b1bf1d5c..2894620a1 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -44,6 +44,7 @@ Worker::~Worker() are in trouble, since goals may call childTerminated() etc. in their destructors). */ topGoals.clear(); + awake.clear(); children.clear(); assert(expectedSubstitutions == 0); @@ -310,12 +311,7 @@ std::vector Worker::run(std::function req) /* Call every wake goal (in the ordering established by CompareGoalPtrs). */ while (!awake.empty() && !topGoals.empty()) { - Goals awake2; - for (auto & i : awake) { - GoalPtr goal = i.lock(); - if (goal) awake2.insert(goal); - } - awake.clear(); + Goals awake2 = std::move(awake); for (auto & goal : awake2) { checkInterrupt(); auto result = goal->work(); diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 46adaa145..097e73cf7 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -84,9 +84,6 @@ private: bool running = false; - /* Note: the worker should only have strong pointers to the - top-level goals. */ - /** * The top-level goals of the worker. */ @@ -95,7 +92,7 @@ private: /** * Goals that are ready to do some work. */ - WeakGoals awake; + Goals awake; template struct CachedGoal -- 2.44.1 From 7f4f86795cc4215d0b8906d2203976768c5f7b21 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Wed, 25 Sep 2024 23:57:46 +0200 Subject: [PATCH 070/106] libstore: remove Goal::key this was a debugging aid from day one that should not have any impact on build semantics, and if it *does* have an impact on build semantics then build semantics are seriously broken. keeping the order imposed by these keys will be impossible once we let a real event loop schedule our jobs. Change-Id: I5c313324e1f213ab6453d82f41ae5e59de809a5b --- src/libstore/build/derivation-goal.cc | 10 ---------- src/libstore/build/derivation-goal.hh | 2 -- src/libstore/build/drv-output-substitution-goal.cc | 7 ------- src/libstore/build/drv-output-substitution-goal.hh | 2 -- src/libstore/build/goal.cc | 7 ------- src/libstore/build/goal.hh | 8 +------- src/libstore/build/substitution-goal.hh | 9 --------- tests/functional/build.sh | 7 ++----- 8 files changed, 3 insertions(+), 49 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 037b4fb10..f28285ad8 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -113,16 +113,6 @@ DerivationGoal::~DerivationGoal() noexcept(false) } -std::string DerivationGoal::key() -{ - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); -} - - void DerivationGoal::killChild() { hook.reset(); diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index b1311bd7e..6e8e979d3 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -247,8 +247,6 @@ struct DerivationGoal : public Goal Finished timedOut(Error && ex); - std::string key() override; - kj::Promise> work() noexcept override; /** diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 923cbba58..846268a3a 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -166,13 +166,6 @@ try { return {std::current_exception()}; } -std::string DrvOutputSubstitutionGoal::key() -{ - /* "a$" ensures substitution goals happen before derivation - goals. */ - return "a$" + std::string(id.to_string()); -} - kj::Promise> DrvOutputSubstitutionGoal::work() noexcept { return (this->*state)(slotToken.valid()); diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 805b65bfa..543fa7ed4 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -74,8 +74,6 @@ public: kj::Promise> outPathValid(bool inBuildSlot) noexcept; kj::Promise> finished() noexcept; - std::string key() override; - kj::Promise> work() noexcept override; JobCategory jobCategory() const override { diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 5f0ed485c..957bc2aaf 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -6,13 +6,6 @@ namespace nix { -bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { - std::string s1 = a->key(); - std::string s2 = b->key(); - return s1 < s2; -} - - void Goal::trace(std::string_view s) { debug("%1%: %2%", name, s); diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index f808be160..4436e44b1 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -22,14 +22,10 @@ class Worker; */ typedef std::shared_ptr GoalPtr; -struct CompareGoalPtrs { - bool operator() (const GoalPtr & a, const GoalPtr & b) const; -}; - /** * Set of goals. */ -typedef std::set Goals; +typedef std::set Goals; /** * Used as a hint to the worker on how to schedule a particular goal. For example, @@ -167,8 +163,6 @@ public: return name; } - virtual std::string key() = 0; - virtual void cleanup() { } /** diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index cef3a4c5c..5411afa01 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -90,15 +90,6 @@ public: ); ~PathSubstitutionGoal(); - /** - * We prepend "a$" to the key name to ensure substitution goals - * happen before derivation goals. - */ - std::string key() override - { - return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); - } - kj::Promise> work() noexcept override; /** diff --git a/tests/functional/build.sh b/tests/functional/build.sh index f80711b31..a14f6e3c2 100644 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -146,11 +146,8 @@ out="$(nix build -f fod-failing.nix -L 2>&1)" && status=0 || status=$? test "$status" = 1 # one "hash mismatch" error, one "build of ... failed" test "$(<<<"$out" grep -E '^error:' | wc -l)" = 2 -<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x1\\.drv'" -<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x3\\.drv'" -<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x2\\.drv'" -<<<"$out" grepQuiet -E "likely URL: https://meow.puppy.forge/puppy.tar.gz" -<<<"$out" grepQuiet -vE "likely URL: https://kitty.forge/cat.tar.gz" +<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x.\\.drv'" +<<<"$out" grepQuiet -E "likely URL: " <<<"$out" grepQuiet -E "error: build of '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out' failed" out="$(nix build -f fod-failing.nix -L x1 x2 x3 --keep-going 2>&1)" && status=0 || status=$? -- 2.44.1 From 47ddd119333ab2e7d0c24fb947d99062a79290b9 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Wed, 25 Sep 2024 23:57:46 +0200 Subject: [PATCH 071/106] libstore: extract a real makeGoalCommon makeDerivationGoalCommon had the right idea, but it didn't quite go far enough. let's do the rest and remove the remaining factory duplication. Change-Id: I1fe32446bdfb501e81df56226fd962f85720725b --- src/libstore/build/worker.cc | 64 ++++++++++++++++++------------------ src/libstore/build/worker.hh | 11 +++++-- 2 files changed, 40 insertions(+), 35 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 2894620a1..b1adf6d10 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -53,20 +53,24 @@ Worker::~Worker() } -std::pair, kj::Promise> Worker::makeDerivationGoalCommon( - const StorePath & drvPath, - const OutputsSpec & wantedOutputs, - std::function()> mkDrvGoal) +template G> +std::pair, kj::Promise> Worker::makeGoalCommon( + std::map> & map, + const ID & key, + InvocableR> auto create, + std::invocable auto modify +) { - auto & goal_weak = derivationGoals[drvPath]; - std::shared_ptr goal = goal_weak.goal.lock(); + auto [it, _inserted] = map.try_emplace(key); + auto & goal_weak = it->second; + auto goal = goal_weak.goal.lock(); if (!goal) { - goal = mkDrvGoal(); + goal = create(); goal->notify = std::move(goal_weak.fulfiller); goal_weak.goal = goal; wakeUp(goal); } else { - goal->addWantedOutputs(wantedOutputs); + modify(*goal); } return {goal, goal_weak.promise->addBranch()}; } @@ -76,9 +80,9 @@ std::pair, kj::Promise> Worker::makeDeriva const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode ) { - return makeDerivationGoalCommon( + return makeGoalCommon( + derivationGoals, drvPath, - wantedOutputs, [&]() -> std::unique_ptr { return !dynamic_cast(&store) ? std::make_unique( @@ -87,7 +91,8 @@ std::pair, kj::Promise> Worker::makeDeriva : LocalDerivationGoal::makeLocalDerivationGoal( drvPath, wantedOutputs, *this, running, buildMode ); - } + }, + [&](DerivationGoal & g) { g.addWantedOutputs(wantedOutputs); } ); } @@ -99,9 +104,9 @@ std::pair, kj::Promise> Worker::makeBasicD BuildMode buildMode ) { - return makeDerivationGoalCommon( + return makeGoalCommon( + derivationGoals, drvPath, - wantedOutputs, [&]() -> std::unique_ptr { return !dynamic_cast(&store) ? std::make_unique( @@ -110,7 +115,8 @@ std::pair, kj::Promise> Worker::makeBasicD : LocalDerivationGoal::makeLocalDerivationGoal( drvPath, drv, wantedOutputs, *this, running, buildMode ); - } + }, + [&](DerivationGoal & g) { g.addWantedOutputs(wantedOutputs); } ); } @@ -120,15 +126,12 @@ Worker::makePathSubstitutionGoal( const StorePath & path, RepairFlag repair, std::optional ca ) { - auto & goal_weak = substitutionGoals[path]; - auto goal = goal_weak.goal.lock(); // FIXME - if (!goal) { - goal = std::make_shared(path, *this, running, repair, ca); - goal->notify = std::move(goal_weak.fulfiller); - goal_weak.goal = goal; - wakeUp(goal); - } - return {goal, goal_weak.promise->addBranch()}; + return makeGoalCommon( + substitutionGoals, + path, + [&] { return std::make_unique(path, *this, running, repair, ca); }, + [&](auto &) {} + ); } @@ -137,15 +140,12 @@ Worker::makeDrvOutputSubstitutionGoal( const DrvOutput & id, RepairFlag repair, std::optional ca ) { - auto & goal_weak = drvOutputSubstitutionGoals[id]; - auto goal = goal_weak.goal.lock(); // FIXME - if (!goal) { - goal = std::make_shared(id, *this, running, repair, ca); - goal->notify = std::move(goal_weak.fulfiller); - goal_weak.goal = goal; - wakeUp(goal); - } - return {goal, goal_weak.promise->addBranch()}; + return makeGoalCommon( + drvOutputSubstitutionGoals, + id, + [&] { return std::make_unique(id, *this, running, repair, ca); }, + [&](auto &) {} + ); } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 097e73cf7..1953bbec1 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -2,6 +2,7 @@ ///@file #include "async-semaphore.hh" +#include "concepts.hh" #include "notifying-counter.hh" #include "types.hh" #include "lock.hh" @@ -240,9 +241,13 @@ public: * @ref DerivationGoal "derivation goal" */ private: - std::pair, kj::Promise> makeDerivationGoalCommon( - const StorePath & drvPath, const OutputsSpec & wantedOutputs, - std::function()> mkDrvGoal); + template G> + std::pair, kj::Promise> makeGoalCommon( + std::map> & map, + const ID & key, + InvocableR> auto create, + std::invocable auto modify + ); std::pair, kj::Promise> makeDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override; -- 2.44.1 From ccd28626663d0024f04c31f121586f951b2283ab Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Wed, 25 Sep 2024 23:57:46 +0200 Subject: [PATCH 072/106] libstore: remove worker removeGoal this was immensely inefficient on large caches, as can exist when many derivations are buildable simultaneously. since we have smart pointers to goals we can do cache maintenance in goal deleters instead, and use the exact iterators instead of doing a linear search. this *does* rely on goals being deleted to remove them from the cache, which isn't true for toplevel goals. those would have previously been removed when done in all cases, removing the cache entry when keep-going is set. this is arguably incorrect since it might result in those goals being retried, although that could only happen with dynamic derivations or the likes. (luckily dynamic derivations not complete enough to allow this at all) Change-Id: I8e750b868393588c33e4829333d370f2c509ce99 --- src/libstore/build/worker.cc | 66 +++++++++++++++--------------------- src/libstore/build/worker.hh | 5 --- 2 files changed, 28 insertions(+), 43 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b1adf6d10..94a638725 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -65,7 +65,26 @@ std::pair, kj::Promise> Worker::makeGoalCommon( auto & goal_weak = it->second; auto goal = goal_weak.goal.lock(); if (!goal) { - goal = create(); + struct Deleter + { + std::map> * from; + std::map>::const_iterator iter; + + void operator()(G * g) + { + from->erase(iter); + delete g; + } + }; + + // this dance is necessary to be memory-safe in exceptional cases. + // if *anything* here throws we must still delete the goal object. + goal = [&] { + std::unique_ptr tmp(nullptr, Deleter{&map, it}); + tmp.reset(create().release()); + return tmp; + }(); + goal->notify = std::move(goal_weak.fulfiller); goal_weak.goal = goal; wakeUp(goal); @@ -165,21 +184,6 @@ std::pair> Worker::makeGoal(const DerivedPath & req, } -template -static void removeGoal(std::shared_ptr goal, auto & goalMap) -{ - /* !!! inefficient */ - for (auto i = goalMap.begin(); - i != goalMap.end(); ) - if (i->second.goal.lock() == goal) { - auto j = i; ++j; - goalMap.erase(i); - i = j; - } - else ++i; -} - - void Worker::goalFinished(GoalPtr goal, Goal::Finished & f) { goal->trace("done"); @@ -192,9 +196,16 @@ void Worker::goalFinished(GoalPtr goal, Goal::Finished & f) hashMismatch |= f.hashMismatch; checkMismatch |= f.checkMismatch; - removeGoal(goal); goal->notify->fulfill(); goal->cleanup(); + + if (topGoals.find(goal) != topGoals.end()) { + topGoals.erase(goal); + /* If a top-level goal failed, then kill all other goals + (unless keepGoing was set). */ + if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) + topGoals.clear(); + } } void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how) @@ -210,27 +221,6 @@ void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how) updateStatistics(); } -void Worker::removeGoal(GoalPtr goal) -{ - if (auto drvGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvGoal, derivationGoals); - else if (auto subGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(subGoal, substitutionGoals); - else if (auto subGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(subGoal, drvOutputSubstitutionGoals); - else - assert(false); - - if (topGoals.find(goal) != topGoals.end()) { - topGoals.erase(goal); - /* If a top-level goal failed, then kill all other goals - (unless keepGoing was set). */ - if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) - topGoals.clear(); - } -} - - void Worker::wakeUp(GoalPtr goal) { goal->trace("woken up"); diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 1953bbec1..fb68a0ef3 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -158,11 +158,6 @@ private: */ void waitForInput(); - /** - * Remove a dead goal. - */ - void removeGoal(GoalPtr goal); - /** * Registers a running child process. */ -- 2.44.1 From aa33c34c9be074c9452976aa96d71091325c83ea Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Wed, 25 Sep 2024 23:57:46 +0200 Subject: [PATCH 073/106] libstore: merge ContinueImmediately and StillAlive nothing needs to signal being still active but not actively pollable, only that immediate polling for the next goal work phase is in order. Change-Id: Ia43c1015e94ba4f5f6b9cb92943da608c4a01555 --- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/build/drv-output-substitution-goal.cc | 2 +- src/libstore/build/goal.cc | 6 +++--- src/libstore/build/goal.hh | 2 -- src/libstore/build/substitution-goal.cc | 2 +- src/libstore/build/worker.cc | 3 +-- 6 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index f28285ad8..b4f3aaf6f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1751,7 +1751,7 @@ DerivationGoal::continueOrError(kj::Promise> p) { return p.then([](auto r) -> Result { if (r.has_value()) { - return ContinueImmediately{}; + return StillAlive{}; } else if (r.has_error()) { return r.assume_error(); } else { diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 846268a3a..87a8b0c55 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -86,7 +86,7 @@ try { }); state = &DrvOutputSubstitutionGoal::realisationFetched; - return pipe.promise.then([]() -> Result { return ContinueImmediately{}; }); + return pipe.promise.then([]() -> Result { return StillAlive{}; }); } catch (...) { return {std::current_exception()}; } diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 957bc2aaf..cfdb6717c 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -17,7 +17,7 @@ try { /* If we are polling goals that are waiting for a lock, then wake up after a few seconds at most. */ co_await worker.aio.provider->getTimer().afterDelay(settings.pollInterval.get() * kj::SECONDS); - co_return ContinueImmediately{}; + co_return StillAlive{}; } catch (...) { co_return std::current_exception(); } @@ -45,11 +45,11 @@ try { waiteeDone(dep); if (dep->exitCode == ecFailed && !settings.keepGoing) { - co_return result::success(ContinueImmediately{}); + co_return result::success(StillAlive{}); } } - co_return result::success(ContinueImmediately{}); + co_return result::success(StillAlive{}); } catch (...) { co_return result::failure(std::current_exception()); } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 4436e44b1..17c3d85db 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -103,7 +103,6 @@ public: struct Finished; struct [[nodiscard]] StillAlive {}; - struct [[nodiscard]] ContinueImmediately {}; struct [[nodiscard]] Finished { ExitCode exitCode; BuildResult result; @@ -116,7 +115,6 @@ public: struct [[nodiscard]] WorkResult : std::variant< StillAlive, - ContinueImmediately, Finished> { WorkResult() = delete; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index d9d8f1a7d..cbbd9daf2 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -240,7 +240,7 @@ try { }); state = &PathSubstitutionGoal::finished; - return pipe.promise.then([]() -> Result { return ContinueImmediately{}; }); + return pipe.promise.then([]() -> Result { return StillAlive{}; }); } catch (...) { return {std::current_exception()}; } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 94a638725..b23b2800e 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -212,8 +212,7 @@ void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how) { std::visit( overloaded{ - [&](Goal::StillAlive) {}, - [&](Goal::ContinueImmediately) { wakeUp(goal); }, + [&](Goal::StillAlive) { wakeUp(goal); }, [&](Goal::Finished & f) { goalFinished(goal, f); }, }, how -- 2.44.1 From 8e05cc1e6c5eb1729e06835baf9114c4be8b82ee Mon Sep 17 00:00:00 2001 From: Raito Bezarius Date: Tue, 1 Oct 2024 10:47:10 +0000 Subject: [PATCH 074/106] Revert "libstore: remove worker removeGoal" Revert submission 1946 Reason for revert: regression in building (found via bisection) Reported by users: > error: path '/nix/store/04ca5xwvasz6s3jg0k7njz6rzi0d225w-jq-1.7.1-dev' does not exist in the store Reverted changes: /q/submissionid:1946 Change-Id: I6f1a4b2f7d7ef5ca430e477fc32bca62fd97036b --- src/libstore/build/worker.cc | 66 +++++++++++++++++++++--------------- src/libstore/build/worker.hh | 5 +++ 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b23b2800e..3a49e8ef9 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -65,26 +65,7 @@ std::pair, kj::Promise> Worker::makeGoalCommon( auto & goal_weak = it->second; auto goal = goal_weak.goal.lock(); if (!goal) { - struct Deleter - { - std::map> * from; - std::map>::const_iterator iter; - - void operator()(G * g) - { - from->erase(iter); - delete g; - } - }; - - // this dance is necessary to be memory-safe in exceptional cases. - // if *anything* here throws we must still delete the goal object. - goal = [&] { - std::unique_ptr tmp(nullptr, Deleter{&map, it}); - tmp.reset(create().release()); - return tmp; - }(); - + goal = create(); goal->notify = std::move(goal_weak.fulfiller); goal_weak.goal = goal; wakeUp(goal); @@ -184,6 +165,21 @@ std::pair> Worker::makeGoal(const DerivedPath & req, } +template +static void removeGoal(std::shared_ptr goal, auto & goalMap) +{ + /* !!! inefficient */ + for (auto i = goalMap.begin(); + i != goalMap.end(); ) + if (i->second.goal.lock() == goal) { + auto j = i; ++j; + goalMap.erase(i); + i = j; + } + else ++i; +} + + void Worker::goalFinished(GoalPtr goal, Goal::Finished & f) { goal->trace("done"); @@ -196,16 +192,9 @@ void Worker::goalFinished(GoalPtr goal, Goal::Finished & f) hashMismatch |= f.hashMismatch; checkMismatch |= f.checkMismatch; + removeGoal(goal); goal->notify->fulfill(); goal->cleanup(); - - if (topGoals.find(goal) != topGoals.end()) { - topGoals.erase(goal); - /* If a top-level goal failed, then kill all other goals - (unless keepGoing was set). */ - if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) - topGoals.clear(); - } } void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how) @@ -220,6 +209,27 @@ void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how) updateStatistics(); } +void Worker::removeGoal(GoalPtr goal) +{ + if (auto drvGoal = std::dynamic_pointer_cast(goal)) + nix::removeGoal(drvGoal, derivationGoals); + else if (auto subGoal = std::dynamic_pointer_cast(goal)) + nix::removeGoal(subGoal, substitutionGoals); + else if (auto subGoal = std::dynamic_pointer_cast(goal)) + nix::removeGoal(subGoal, drvOutputSubstitutionGoals); + else + assert(false); + + if (topGoals.find(goal) != topGoals.end()) { + topGoals.erase(goal); + /* If a top-level goal failed, then kill all other goals + (unless keepGoing was set). */ + if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) + topGoals.clear(); + } +} + + void Worker::wakeUp(GoalPtr goal) { goal->trace("woken up"); diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index fb68a0ef3..1953bbec1 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -158,6 +158,11 @@ private: */ void waitForInput(); + /** + * Remove a dead goal. + */ + void removeGoal(GoalPtr goal); + /** * Registers a running child process. */ -- 2.44.1 From d31310bf59a6a51945eb2d671585815b93f4df00 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Mon, 30 Sep 2024 01:31:29 +0200 Subject: [PATCH 075/106] libstore: turn waitForInput into a promise Change-Id: I8355d8d3f6c43a812990c1912b048e5735b07f7b --- src/libstore/build/worker.cc | 11 +++++++---- src/libstore/build/worker.hh | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 3a49e8ef9..7dd75f34a 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -328,7 +328,7 @@ std::vector Worker::run(std::function req) /* Wait for input. */ if (!children.isEmpty()) - waitForInput(); + waitForInput().wait(aio.waitScope).value(); else { assert(!awake.empty()); } @@ -351,8 +351,8 @@ std::vector Worker::run(std::function req) return results; } -void Worker::waitForInput() -{ +kj::Promise> Worker::waitForInput() +try { printMsg(lvlVomit, "waiting for children"); auto waitFor = [&]{ @@ -366,7 +366,10 @@ void Worker::waitForInput() waitFor = waitFor.exclusiveJoin(aio.provider->getTimer().afterDelay(10 * kj::SECONDS)); } - waitFor.wait(aio.waitScope); + co_await waitFor; + co_return result::success(); +} catch (...) { + co_return result::failure(std::current_exception()); } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 1953bbec1..02b98b3aa 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -156,7 +156,7 @@ private: /** * Wait for input to become available. */ - void waitForInput(); + kj::Promise> waitForInput(); /** * Remove a dead goal. -- 2.44.1 From b0c7c1ec664415d451dcd56d2982b369c0c33892 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Mon, 30 Sep 2024 01:31:29 +0200 Subject: [PATCH 076/106] libstore: turn Worker::run() main loop into a promise Change-Id: Ib112ea9a3e67d5cb3d7d0ded30bbd25c96262470 --- src/libstore/build/worker.cc | 29 +++++++++++++++++------------ src/libstore/build/worker.hh | 2 ++ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 7dd75f34a..8ab7fcc86 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -297,6 +297,18 @@ std::vector Worker::run(std::function req) topGoals.insert(goal); } + auto promise = runImpl(); + promise.wait(aio.waitScope).value(); + + std::vector results; + for (auto & [i, _p] : _topGoals) { + results.push_back(i); + } + return results; +} + +kj::Promise> Worker::runImpl() +try { debug("entered goal loop"); while (1) { @@ -313,12 +325,7 @@ std::vector Worker::run(std::function req) Goals awake2 = std::move(awake); for (auto & goal : awake2) { checkInterrupt(); - auto result = goal->work(); - if (result.poll(aio.waitScope)) { - handleWorkResult(goal, result.wait(aio.waitScope).value()); - } else { - childStarted(goal, std::move(result)); - } + childStarted(goal, goal->work()); if (topGoals.empty()) break; // stuff may have been cancelled } @@ -328,7 +335,7 @@ std::vector Worker::run(std::function req) /* Wait for input. */ if (!children.isEmpty()) - waitForInput().wait(aio.waitScope).value(); + (co_await waitForInput()).value(); else { assert(!awake.empty()); } @@ -344,11 +351,9 @@ std::vector Worker::run(std::function req) assert(!settings.keepGoing || awake.empty()); assert(!settings.keepGoing || children.isEmpty()); - std::vector results; - for (auto & [i, _p] : _topGoals) { - results.push_back(i); - } - return results; + co_return result::success(); +} catch (...) { + co_return result::failure(std::current_exception()); } kj::Promise> Worker::waitForInput() diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 02b98b3aa..6da76fe34 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -188,6 +188,8 @@ private: statisticsOutdated = true; } + kj::Promise> runImpl(); + public: const Activity act; -- 2.44.1 From d5db0b1abc1809ad97fa28b82056b19c4a45710a Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Mon, 30 Sep 2024 01:31:30 +0200 Subject: [PATCH 077/106] libstore: turn periodic gc attempt into a promise notably we will check whether we want to do GC at all only once during startup, and we'll only attempt GC every ten seconds rather than every time a goal has finished a partial work call. this shouldn't cause any problems in practice since relying on auto-gc is not deterministic and stores in which builds can fill all remaining free space in merely ten seconds are severely troubled even when gargage collection runs a lot. Change-Id: I1175a56bf7f4e531f8be90157ad88750ff2ddec4 --- src/libstore/build/worker.cc | 36 ++++++++++++++++++++---------------- src/libstore/build/worker.hh | 2 ++ 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 8ab7fcc86..e9904a1f5 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -298,6 +298,13 @@ std::vector Worker::run(std::function req) } auto promise = runImpl(); + + // TODO GC interface? + if (auto localStore = dynamic_cast(&store); localStore && settings.minFree != 0) { + // Periodically wake up to see if we need to run the garbage collector. + promise = promise.exclusiveJoin(boopGC(*localStore)); + } + promise.wait(aio.waitScope).value(); std::vector results; @@ -315,10 +322,6 @@ try { checkInterrupt(); - // TODO GC interface? - if (auto localStore = dynamic_cast(&store)) - localStore->autoGC(false); - /* Call every wake goal (in the ordering established by CompareGoalPtrs). */ while (!awake.empty() && !topGoals.empty()) { @@ -356,22 +359,23 @@ try { co_return result::failure(std::current_exception()); } +kj::Promise> Worker::boopGC(LocalStore & localStore) +try { + while (true) { + co_await aio.provider->getTimer().afterDelay(10 * kj::SECONDS); + localStore.autoGC(false); + } +} catch (...) { + co_return result::failure(std::current_exception()); +} + kj::Promise> Worker::waitForInput() try { printMsg(lvlVomit, "waiting for children"); - auto waitFor = [&]{ - auto pair = kj::newPromiseAndFulfiller(); - this->childFinished = kj::mv(pair.fulfiller); - return kj::mv(pair.promise); - }(); - - if (settings.minFree.get() != 0) { - // Periodicallty wake up to see if we need to run the garbage collector. - waitFor = waitFor.exclusiveJoin(aio.provider->getTimer().afterDelay(10 * kj::SECONDS)); - } - - co_await waitFor; + auto pair = kj::newPromiseAndFulfiller(); + this->childFinished = kj::mv(pair.fulfiller); + co_await pair.promise; co_return result::success(); } catch (...) { co_return result::failure(std::current_exception()); diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 6da76fe34..dc85c43e3 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -20,6 +20,7 @@ namespace nix { struct DerivationGoal; struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; +class LocalStore; typedef std::chrono::time_point steady_time_point; @@ -189,6 +190,7 @@ private: } kj::Promise> runImpl(); + kj::Promise> boopGC(LocalStore & localStore); public: -- 2.44.1 From 732de75f67f030886fd6bb421d49481caa3aa8cf Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Mon, 30 Sep 2024 01:31:30 +0200 Subject: [PATCH 078/106] libstore: remove Worker::wakeUp() Worker::run() is now entirely based on the kj event loop and promises, so we need not handle awakeness of goals manually any more. every goal can instead, once it has finished a partial work call, defer itself to being called again in the next iteration of the loop. same end effect. Change-Id: I320eee2fa60bcebaabd74d1323fa96d1402c1d15 --- src/libstore/build/worker.cc | 33 +++++++-------------------------- src/libstore/build/worker.hh | 10 ---------- 2 files changed, 7 insertions(+), 36 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index e9904a1f5..9317a5ac2 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -44,7 +44,6 @@ Worker::~Worker() are in trouble, since goals may call childTerminated() etc. in their destructors). */ topGoals.clear(); - awake.clear(); children.clear(); assert(expectedSubstitutions == 0); @@ -68,7 +67,10 @@ std::pair, kj::Promise> Worker::makeGoalCommon( goal = create(); goal->notify = std::move(goal_weak.fulfiller); goal_weak.goal = goal; - wakeUp(goal); + // do not start working immediately, this round of the event loop + // may have more calls to this function lined up that'll also run + // modify(). starting early can then cause the goals to misbehave + childStarted(goal, kj::evalLater([goal] { return goal->work(); })); } else { modify(*goal); } @@ -201,7 +203,9 @@ void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how) { std::visit( overloaded{ - [&](Goal::StillAlive) { wakeUp(goal); }, + [&](Goal::StillAlive) { + childStarted(goal, kj::evalLater([goal] { return goal->work(); })); + }, [&](Goal::Finished & f) { goalFinished(goal, f); }, }, how @@ -230,13 +234,6 @@ void Worker::removeGoal(GoalPtr goal) } -void Worker::wakeUp(GoalPtr goal) -{ - goal->trace("woken up"); - awake.insert(goal); -} - - void Worker::childStarted(GoalPtr goal, kj::Promise> promise) { children.add(promise @@ -322,26 +319,11 @@ try { checkInterrupt(); - /* Call every wake goal (in the ordering established by - CompareGoalPtrs). */ - while (!awake.empty() && !topGoals.empty()) { - Goals awake2 = std::move(awake); - for (auto & goal : awake2) { - checkInterrupt(); - childStarted(goal, goal->work()); - - if (topGoals.empty()) break; // stuff may have been cancelled - } - } - if (topGoals.empty()) break; /* Wait for input. */ if (!children.isEmpty()) (co_await waitForInput()).value(); - else { - assert(!awake.empty()); - } if (childException) { std::rethrow_exception(childException); @@ -351,7 +333,6 @@ try { /* If --keep-going is not set, it's possible that the main goal exited while some of its subgoals were still active. But if --keep-going *is* set, then they must all be finished now. */ - assert(!settings.keepGoing || awake.empty()); assert(!settings.keepGoing || children.isEmpty()); co_return result::success(); diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index dc85c43e3..bb51a2114 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -91,11 +91,6 @@ private: */ Goals topGoals; - /** - * Goals that are ready to do some work. - */ - Goals awake; - template struct CachedGoal { @@ -149,11 +144,6 @@ private: kj::Own> childFinished; - /** - * Wake up a goal (i.e., there is something for it to do). - */ - void wakeUp(GoalPtr goal); - /** * Wait for input to become available. */ -- 2.44.1 From 9889c79fe355ce9d1b6ee7814cdeec0fbf70823a Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Mon, 30 Sep 2024 01:31:30 +0200 Subject: [PATCH 079/106] libstore: turn Worker::updateStatistics into a promise we'll now loop to update displayed statistics, and use this loop to limit the update rate to 50 times per second. we could have updated much more frequently before this (once per iteration of `runImpl`), much faster than would ever be useful in practice. aggressive stats updates can even impede progress due to terminal or network delays. Change-Id: Ifba755a2569f73c919b1fbb06a142c0951395d6d --- src/libstore/build/worker.cc | 21 ++++++++++++--------- src/libstore/build/worker.hh | 7 ++++--- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 9317a5ac2..5be706e42 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -258,11 +258,13 @@ void Worker::childTerminated(GoalPtr goal) } -void Worker::updateStatistics() -{ - // only update progress info while running. this notably excludes updating - // progress info while destroying, which causes the progress bar to assert - if (running && statisticsOutdated) { +kj::Promise> Worker::updateStatistics() +try { + while (true) { + statisticsUpdateInhibitor = co_await statisticsUpdateSignal.acquire(); + + // only update progress info while running. this notably excludes updating + // progress info while destroying, which causes the progress bar to assert actDerivations.progress( doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds ); @@ -275,8 +277,11 @@ void Worker::updateStatistics() act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); act.setExpected(actCopyPath, expectedNarSize + doneNarSize); - statisticsOutdated = false; + // limit to 50fps. that should be more than good enough for anything we do + co_await aio.provider->getTimer().afterDelay(20 * kj::MILLISECONDS); } +} catch (...) { + co_return result::failure(std::current_exception()); } std::vector Worker::run(std::function req) @@ -287,14 +292,12 @@ std::vector Worker::run(std::function req) running = true; Finally const _stop([&] { running = false; }); - updateStatistics(); - topGoals.clear(); for (auto & [goal, _promise] : _topGoals) { topGoals.insert(goal); } - auto promise = runImpl(); + auto promise = runImpl().exclusiveJoin(updateStatistics()); // TODO GC interface? if (auto localStore = dynamic_cast(&store); localStore && settings.minFree != 0) { diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index bb51a2114..d6cde8384 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -167,16 +167,17 @@ private: /** * Pass current stats counters to the logger for progress bar updates. */ - void updateStatistics(); + kj::Promise> updateStatistics(); - bool statisticsOutdated = true; + AsyncSemaphore statisticsUpdateSignal{1}; + std::optional statisticsUpdateInhibitor; /** * Mark statistics as outdated, such that `updateStatistics` will be called. */ void updateStatisticsLater() { - statisticsOutdated = true; + statisticsUpdateInhibitor = {}; } kj::Promise> runImpl(); -- 2.44.1 From 9b05636937fe108efbf39a005807e11d36c7b057 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Mon, 30 Sep 2024 01:31:30 +0200 Subject: [PATCH 080/106] libstore: make PathSubstitutionGoal::work *one* promise Change-Id: I38cfe8c7059251b581f1013c4213804f36b985ea --- src/libstore/build/substitution-goal.cc | 128 +++++++++++------------- src/libstore/build/substitution-goal.hh | 12 +-- 2 files changed, 63 insertions(+), 77 deletions(-) diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index cbbd9daf2..8088bf668 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -20,7 +20,6 @@ PathSubstitutionGoal::PathSubstitutionGoal( , repair(repair) , ca(ca) { - state = &PathSubstitutionGoal::init; name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); trace("created"); maintainExpectedSubstitutions = worker.expectedSubstitutions.addTemporarily(1); @@ -48,12 +47,6 @@ Goal::Finished PathSubstitutionGoal::done( kj::Promise> PathSubstitutionGoal::work() noexcept -{ - return (this->*state)(slotToken.valid()); -} - - -kj::Promise> PathSubstitutionGoal::init(bool inBuildSlot) noexcept try { trace("init"); @@ -69,13 +62,13 @@ try { subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - return tryNext(inBuildSlot); + return tryNext(); } catch (...) { return {std::current_exception()}; } -kj::Promise> PathSubstitutionGoal::tryNext(bool inBuildSlot) noexcept +kj::Promise> PathSubstitutionGoal::tryNext() noexcept try { trace("trying next substituter"); @@ -91,10 +84,10 @@ try { /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - return {done( + co_return done( substituterFailed ? ecFailed : ecNoSubstituters, BuildResult::NoSubstituters, - fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)))}; + fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath))); } sub = subs.front(); @@ -107,26 +100,28 @@ try { if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { - return tryNext(inBuildSlot); + co_return co_await tryNext(); } - try { - // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath &) { - return tryNext(inBuildSlot); - } catch (SubstituterDisabled &) { - if (settings.tryFallback) { - return tryNext(inBuildSlot); + do { + try { + // FIXME: make async + info = sub->queryPathInfo(subPath ? *subPath : storePath); + break; + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { + if (!settings.tryFallback) { + throw; + } + } catch (Error & e) { + if (settings.tryFallback) { + logError(e.info()); + } else { + throw; + } } - throw; - } catch (Error & e) { - if (settings.tryFallback) { - logError(e.info()); - return tryNext(inBuildSlot); - } - throw; - } + co_return co_await tryNext(); + } while (false); if (info->path != storePath) { if (info->isContentAddressed(*sub) && info->references.empty()) { @@ -136,7 +131,7 @@ try { } else { printError("asked '%s' for '%s' but got '%s'", sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); - return tryNext(inBuildSlot); + co_return co_await tryNext(); } } @@ -157,7 +152,7 @@ try { { warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", worker.store.printStorePath(storePath), sub->getUri()); - return tryNext(inBuildSlot); + co_return co_await tryNext(); } /* To maintain the closure invariant, we first have to realise the @@ -167,18 +162,16 @@ try { if (i != storePath) /* ignore self-references */ dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i)); - if (dependencies.empty()) {/* to prevent hang (no wake-up event) */ - return referencesValid(inBuildSlot); - } else { - state = &PathSubstitutionGoal::referencesValid; - return waitForGoals(dependencies.releaseAsArray()); + if (!dependencies.empty()) {/* to prevent hang (no wake-up event) */ + (co_await waitForGoals(dependencies.releaseAsArray())).value(); } + co_return co_await referencesValid(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise> PathSubstitutionGoal::referencesValid(bool inBuildSlot) noexcept +kj::Promise> PathSubstitutionGoal::referencesValid() noexcept try { trace("all references realised"); @@ -193,22 +186,18 @@ try { if (i != storePath) /* ignore self-references */ assert(worker.store.isValidPath(i)); - state = &PathSubstitutionGoal::tryToRun; - return tryToRun(inBuildSlot); + return tryToRun(); } catch (...) { return {std::current_exception()}; } -kj::Promise> PathSubstitutionGoal::tryToRun(bool inBuildSlot) noexcept +kj::Promise> PathSubstitutionGoal::tryToRun() noexcept try { trace("trying to run"); - if (!inBuildSlot) { - return worker.substitutions.acquire().then([this](auto token) { - slotToken = std::move(token); - return work(); - }); + if (!slotToken.valid()) { + slotToken = co_await worker.substitutions.acquire(); } maintainRunningSubstitutions = worker.runningSubstitutions.addTemporarily(1); @@ -239,38 +228,39 @@ try { } }); - state = &PathSubstitutionGoal::finished; - return pipe.promise.then([]() -> Result { return StillAlive{}; }); + co_await pipe.promise; + co_return co_await finished(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise> PathSubstitutionGoal::finished(bool inBuildSlot) noexcept +kj::Promise> PathSubstitutionGoal::finished() noexcept try { trace("substitute finished"); - try { - slotToken = {}; - thr.get(); - } catch (std::exception & e) { - printError(e.what()); - - /* Cause the parent build to fail unless --fallback is given, - or the substitute has disappeared. The latter case behaves - the same as the substitute never having existed in the - first place. */ + do { try { - throw; - } catch (SubstituteGone &) { - } catch (...) { - substituterFailed = true; - } + slotToken = {}; + thr.get(); + break; + } catch (std::exception & e) { + printError(e.what()); + /* Cause the parent build to fail unless --fallback is given, + or the substitute has disappeared. The latter case behaves + the same as the substitute never having existed in the + first place. */ + try { + throw; + } catch (SubstituteGone &) { + } catch (...) { + substituterFailed = true; + } + } /* Try the next substitute. */ - state = &PathSubstitutionGoal::tryNext; - return tryNext(inBuildSlot); - } + co_return co_await tryNext(); + } while (false); worker.markContentsGood(storePath); @@ -287,9 +277,9 @@ try { worker.doneNarSize += maintainExpectedNar.delta(); maintainExpectedNar.reset(); - return {done(ecSuccess, BuildResult::Substituted)}; + co_return done(ecSuccess, BuildResult::Substituted); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 5411afa01..dc701bcba 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -67,9 +67,6 @@ struct PathSubstitutionGoal : public Goal NotifyingCounter::Bump maintainExpectedSubstitutions, maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - typedef kj::Promise> (PathSubstitutionGoal::*GoalState)(bool inBuildSlot) noexcept; - GoalState state; - /** * Content address for recomputing store path */ @@ -95,11 +92,10 @@ public: /** * The states. */ - kj::Promise> init(bool inBuildSlot) noexcept; - kj::Promise> tryNext(bool inBuildSlot) noexcept; - kj::Promise> referencesValid(bool inBuildSlot) noexcept; - kj::Promise> tryToRun(bool inBuildSlot) noexcept; - kj::Promise> finished(bool inBuildSlot) noexcept; + kj::Promise> tryNext() noexcept; + kj::Promise> referencesValid() noexcept; + kj::Promise> tryToRun() noexcept; + kj::Promise> finished() noexcept; /* Called by destructor, can't be overridden */ void cleanup() override final; -- 2.44.1 From 3edc272341b22f4d088f30c407b55d91da9bff2c Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Mon, 30 Sep 2024 01:31:30 +0200 Subject: [PATCH 081/106] libstore: turn DrvOutputSubstitutionGoal::work into *one* promise Change-Id: I2d4dcedff0a278d2d8f3d264a9186dfb399275e2 --- .../build/drv-output-substitution-goal.cc | 50 +++++++------------ .../build/drv-output-substitution-goal.hh | 10 ++-- 2 files changed, 22 insertions(+), 38 deletions(-) diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 87a8b0c55..9ffa33d7b 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -19,36 +19,32 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal( : Goal(worker, isDependency) , id(id) { - state = &DrvOutputSubstitutionGoal::init; name = fmt("substitution of '%s'", id.to_string()); trace("created"); } -kj::Promise> DrvOutputSubstitutionGoal::init(bool inBuildSlot) noexcept +kj::Promise> DrvOutputSubstitutionGoal::work() noexcept try { trace("init"); /* If the derivation already exists, we’re done */ if (worker.store.queryRealisation(id)) { - return {Finished{ecSuccess, std::move(buildResult)}}; + co_return Finished{ecSuccess, std::move(buildResult)}; } subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - return tryNext(inBuildSlot); + co_return co_await tryNext(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise> DrvOutputSubstitutionGoal::tryNext(bool inBuildSlot) noexcept +kj::Promise> DrvOutputSubstitutionGoal::tryNext() noexcept try { trace("trying next substituter"); - if (!inBuildSlot) { - return worker.substitutions.acquire().then([this](auto token) { - slotToken = std::move(token); - return work(); - }); + if (!slotToken.valid()) { + slotToken = co_await worker.substitutions.acquire(); } maintainRunningSubstitutions = worker.runningSubstitutions.addTemporarily(1); @@ -65,7 +61,7 @@ try { /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - return {Finished{substituterFailed ? ecFailed : ecNoSubstituters, std::move(buildResult)}}; + co_return Finished{substituterFailed ? ecFailed : ecNoSubstituters, std::move(buildResult)}; } sub = subs.front(); @@ -85,13 +81,13 @@ try { return sub->queryRealisation(id); }); - state = &DrvOutputSubstitutionGoal::realisationFetched; - return pipe.promise.then([]() -> Result { return StillAlive{}; }); + co_await pipe.promise; + co_return co_await realisationFetched(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise> DrvOutputSubstitutionGoal::realisationFetched(bool inBuildSlot) noexcept +kj::Promise> DrvOutputSubstitutionGoal::realisationFetched() noexcept try { maintainRunningSubstitutions.reset(); slotToken = {}; @@ -104,7 +100,7 @@ try { } if (!outputInfo) { - return tryNext(inBuildSlot); + co_return co_await tryNext(); } kj::Vector>> dependencies; @@ -121,7 +117,7 @@ try { worker.store.printStorePath(localOutputInfo->outPath), worker.store.printStorePath(depPath) ); - return tryNext(inBuildSlot); + co_return co_await tryNext(); } dependencies.add(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId)); } @@ -129,17 +125,15 @@ try { dependencies.add(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath)); - if (dependencies.empty()) { - return outPathValid(inBuildSlot); - } else { - state = &DrvOutputSubstitutionGoal::outPathValid; - return waitForGoals(dependencies.releaseAsArray()); + if (!dependencies.empty()) { + (co_await waitForGoals(dependencies.releaseAsArray())).value(); } + co_return co_await outPathValid(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise> DrvOutputSubstitutionGoal::outPathValid(bool inBuildSlot) noexcept +kj::Promise> DrvOutputSubstitutionGoal::outPathValid() noexcept try { assert(outputInfo); trace("output path substituted"); @@ -166,10 +160,4 @@ try { return {std::current_exception()}; } -kj::Promise> DrvOutputSubstitutionGoal::work() noexcept -{ - return (this->*state)(slotToken.valid()); -} - - } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 543fa7ed4..86020705e 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -65,13 +65,9 @@ public: std::optional ca = std::nullopt ); - typedef kj::Promise> (DrvOutputSubstitutionGoal::*GoalState)(bool inBuildSlot) noexcept; - GoalState state; - - kj::Promise> init(bool inBuildSlot) noexcept; - kj::Promise> tryNext(bool inBuildSlot) noexcept; - kj::Promise> realisationFetched(bool inBuildSlot) noexcept; - kj::Promise> outPathValid(bool inBuildSlot) noexcept; + kj::Promise> tryNext() noexcept; + kj::Promise> realisationFetched() noexcept; + kj::Promise> outPathValid() noexcept; kj::Promise> finished() noexcept; kj::Promise> work() noexcept override; -- 2.44.1 From 775292766025380d04004e42fefbdb8ca40b3fa3 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Mon, 30 Sep 2024 01:31:30 +0200 Subject: [PATCH 082/106] libstore: turn DerivationGoal::work into *one* promise Change-Id: Ic2f7bc2bd6a1879ad614e4be81a7214f64eb0e85 --- src/libstore/build/derivation-goal.cc | 186 +++++++++----------- src/libstore/build/derivation-goal.hh | 29 ++- src/libstore/build/local-derivation-goal.cc | 35 ++-- src/libstore/build/local-derivation-goal.hh | 2 +- 4 files changed, 121 insertions(+), 131 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index b4f3aaf6f..5c0452391 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -71,7 +71,6 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, , wantedOutputs(wantedOutputs) , buildMode(buildMode) { - state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); @@ -91,7 +90,6 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation { this->drv = std::make_unique(drv); - state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store)); @@ -129,7 +127,7 @@ Goal::Finished DerivationGoal::timedOut(Error && ex) kj::Promise> DerivationGoal::work() noexcept { - return (this->*state)(slotToken.valid()); + return useDerivation ? getDerivation() : haveDerivation(); } void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) @@ -153,7 +151,7 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) } -kj::Promise> DerivationGoal::getDerivation(bool inBuildSlot) noexcept +kj::Promise> DerivationGoal::getDerivation() noexcept try { trace("init"); @@ -161,18 +159,17 @@ try { exists. If it doesn't, it may be created through a substitute. */ if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) { - return loadDerivation(inBuildSlot); + co_return co_await loadDerivation(); } - - state = &DerivationGoal::loadDerivation; - return waitForGoals(worker.goalFactory().makePathSubstitutionGoal(drvPath)); + (co_await waitForGoals(worker.goalFactory().makePathSubstitutionGoal(drvPath))).value(); + co_return co_await loadDerivation(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise> DerivationGoal::loadDerivation(bool inBuildSlot) noexcept +kj::Promise> DerivationGoal::loadDerivation() noexcept try { trace("loading derivation"); @@ -203,13 +200,13 @@ try { } assert(drv); - return haveDerivation(inBuildSlot); + return haveDerivation(); } catch (...) { return {std::current_exception()}; } -kj::Promise> DerivationGoal::haveDerivation(bool inBuildSlot) noexcept +kj::Promise> DerivationGoal::haveDerivation() noexcept try { trace("have derivation"); @@ -237,7 +234,7 @@ try { }); } - return gaveUpOnSubstitution(inBuildSlot); + co_return co_await gaveUpOnSubstitution(); } for (auto & i : drv->outputsAndOptPaths(worker.store)) @@ -259,7 +256,7 @@ try { /* If they are all valid, then we're done. */ if (allValid && buildMode == bmNormal) { - return {done(BuildResult::AlreadyValid, std::move(validOutputs))}; + co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } /* We are first going to try to create the invalid output paths @@ -290,17 +287,15 @@ try { } } - if (dependencies.empty()) { /* to prevent hang (no wake-up event) */ - return outputsSubstitutionTried(inBuildSlot); - } else { - state = &DerivationGoal::outputsSubstitutionTried; - return waitForGoals(dependencies.releaseAsArray()); + if (!dependencies.empty()) { /* to prevent hang (no wake-up event) */ + (co_await waitForGoals(dependencies.releaseAsArray())).value(); } + co_return co_await outputsSubstitutionTried(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise> DerivationGoal::outputsSubstitutionTried(bool inBuildSlot) noexcept +kj::Promise> DerivationGoal::outputsSubstitutionTried() noexcept try { trace("all outputs substituted (maybe)"); @@ -350,7 +345,7 @@ try { if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) { needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed; - return haveDerivation(inBuildSlot); + return haveDerivation(); } auto [allValid, validOutputs] = checkPathValidity(); @@ -366,7 +361,7 @@ try { worker.store.printStorePath(drvPath)); /* Nothing to wait for; tail call */ - return gaveUpOnSubstitution(inBuildSlot); + return gaveUpOnSubstitution(); } catch (...) { return {std::current_exception()}; } @@ -374,7 +369,7 @@ try { /* At least one of the output paths could not be produced using a substitute. So we have to build instead. */ -kj::Promise> DerivationGoal::gaveUpOnSubstitution(bool inBuildSlot) noexcept +kj::Promise> DerivationGoal::gaveUpOnSubstitution() noexcept try { kj::Vector>> dependencies; @@ -437,14 +432,12 @@ try { dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i)); } - if (dependencies.empty()) {/* to prevent hang (no wake-up event) */ - return inputsRealised(inBuildSlot); - } else { - state = &DerivationGoal::inputsRealised; - return waitForGoals(dependencies.releaseAsArray()); + if (!dependencies.empty()) {/* to prevent hang (no wake-up event) */ + (co_await waitForGoals(dependencies.releaseAsArray())).value(); } + co_return co_await inputsRealised(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } @@ -503,17 +496,17 @@ try { } if (dependencies.empty()) { - return {done(BuildResult::AlreadyValid, assertPathValidity())}; + co_return done(BuildResult::AlreadyValid, assertPathValidity()); } - state = &DerivationGoal::closureRepaired; - return waitForGoals(dependencies.releaseAsArray()); + (co_await waitForGoals(dependencies.releaseAsArray())).value(); + co_return co_await closureRepaired(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise> DerivationGoal::closureRepaired(bool inBuildSlot) noexcept +kj::Promise> DerivationGoal::closureRepaired() noexcept try { trace("closure repaired"); if (nrFailed > 0) @@ -525,14 +518,14 @@ try { } -kj::Promise> DerivationGoal::inputsRealised(bool inBuildSlot) noexcept +kj::Promise> DerivationGoal::inputsRealised() noexcept try { trace("all inputs realised"); if (nrFailed != 0) { if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - return {done( + co_return done( BuildResult::DependencyFailed, {}, Error( @@ -540,12 +533,12 @@ try { nrFailed, worker.store.printStorePath(drvPath) ) - )}; + ); } if (retrySubstitution == RetrySubstitution::YesNeed) { retrySubstitution = RetrySubstitution::AlreadyRetried; - return haveDerivation(inBuildSlot); + co_return co_await haveDerivation(); } /* Gather information necessary for computing the closure and/or @@ -611,8 +604,8 @@ try { pathResolved, wantedOutputs, buildMode); resolvedDrvGoal = dependency.first; - state = &DerivationGoal::resolvedFinished; - return waitForGoals(std::move(dependency)); + (co_await waitForGoals(std::move(dependency))).value(); + co_return co_await resolvedFinished(); } std::function::ChildNode &)> accumInputPaths; @@ -676,10 +669,9 @@ try { /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a build hook. */ - state = &DerivationGoal::tryToBuild; - return tryToBuild(inBuildSlot); + co_return co_await tryToBuild(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } void DerivationGoal::started() @@ -695,8 +687,9 @@ void DerivationGoal::started() mcRunningBuilds = worker.runningBuilds.addTemporarily(1); } -kj::Promise> DerivationGoal::tryToBuild(bool inBuildSlot) noexcept +kj::Promise> DerivationGoal::tryToBuild() noexcept try { +retry: trace("trying to build"); /* Obtain locks on all output paths, if the paths are known a priori. @@ -730,7 +723,9 @@ try { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for lock on %s", Magenta(showPaths(lockFiles)))); - return waitForAWhile(); + (co_await waitForAWhile()).value(); + // we can loop very often, and `co_return co_await` always allocates a new frame + goto retry; } actLock.reset(); @@ -747,7 +742,7 @@ try { if (buildMode != bmCheck && allValid) { debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); - return {done(BuildResult::AlreadyValid, std::move(validOutputs))}; + co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } /* If any of the outputs already exist but are not valid, delete @@ -767,47 +762,56 @@ try { && settings.maxBuildJobs.get() != 0; if (!buildLocally) { - auto hookReply = tryBuildHook(inBuildSlot); - auto result = std::visit( - overloaded{ - [&](HookReply::Accept & a) -> std::optional>> { - /* Yes, it has started doing so. Wait until we get - EOF from the hook. */ - actLock.reset(); - buildResult.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; - started(); - return continueOrError(std::move(a.promise)); - }, - [&](HookReply::Postpone) -> std::optional>> { - /* Not now; wait until at least one child finishes or - the wake-up timeout expires. */ - if (!actLock) - actLock = std::make_unique(*logger, lvlTalkative, actBuildWaiting, - fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); - outputLocks.unlock(); - return waitForAWhile(); - }, - [&](HookReply::Decline) -> std::optional>> { - /* We should do it ourselves. */ - return std::nullopt; - }, - }, - hookReply); - if (result) { - return std::move(*result); + auto hookReply = tryBuildHook(); + switch (hookReply.index()) { + case 0: { + HookReply::Accept & a = std::get<0>(hookReply); + /* Yes, it has started doing so. Wait until we get + EOF from the hook. */ + actLock.reset(); + buildResult.startTime = time(0); // inexact + started(); + auto r = co_await a.promise; + if (r.has_value()) { + co_return co_await buildDone(); + } else if (r.has_error()) { + co_return r.assume_error(); + } else { + co_return r.assume_exception(); + } + } + + case 1: { + HookReply::Decline _ [[gnu::unused]] = std::get<1>(hookReply); + break; + } + + case 2: { + HookReply::Postpone _ [[gnu::unused]] = std::get<2>(hookReply); + /* Not now; wait until at least one child finishes or + the wake-up timeout expires. */ + if (!actLock) + actLock = std::make_unique(*logger, lvlTalkative, actBuildWaiting, + fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); + outputLocks.unlock(); + (co_await waitForAWhile()).value(); + goto retry; + } + + default: + // can't static_assert this because HookReply *subclasses* variant and std::variant_size breaks + assert(false && "unexpected hook reply"); } } actLock.reset(); - state = &DerivationGoal::tryLocalBuild; - return tryLocalBuild(inBuildSlot); + co_return co_await tryLocalBuild(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise> DerivationGoal::tryLocalBuild(bool inBuildSlot) noexcept +kj::Promise> DerivationGoal::tryLocalBuild() noexcept try { throw Error( "unable to build with a primary store that isn't a local store; " @@ -970,7 +974,7 @@ void runPostBuildHook( proc.getStdout()->drainInto(sink); } -kj::Promise> DerivationGoal::buildDone(bool inBuildSlot) noexcept +kj::Promise> DerivationGoal::buildDone() noexcept try { trace("build done"); @@ -1090,7 +1094,7 @@ try { return {std::current_exception()}; } -kj::Promise> DerivationGoal::resolvedFinished(bool inBuildSlot) noexcept +kj::Promise> DerivationGoal::resolvedFinished() noexcept try { trace("resolved derivation finished"); @@ -1163,7 +1167,7 @@ try { return {std::current_exception()}; } -HookReply DerivationGoal::tryBuildHook(bool inBuildSlot) +HookReply DerivationGoal::tryBuildHook() { if (!worker.hook.available || !useDerivation) return HookReply::Decline{}; @@ -1175,7 +1179,7 @@ HookReply DerivationGoal::tryBuildHook(bool inBuildSlot) /* Send the request to the hook. */ worker.hook.instance->sink << "try" - << (inBuildSlot ? 1 : 0) + << (slotToken.valid() ? 1 : 0) << drv->platform << worker.store.printStorePath(drvPath) << parsedDrv->getRequiredSystemFeatures(); @@ -1745,18 +1749,4 @@ void DerivationGoal::waiteeDone(GoalPtr waitee) } } } - -kj::Promise> -DerivationGoal::continueOrError(kj::Promise> p) -{ - return p.then([](auto r) -> Result { - if (r.has_value()) { - return StillAlive{}; - } else if (r.has_error()) { - return r.assume_error(); - } else { - return r.assume_exception(); - } - }); -} } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 6e8e979d3..7505409c0 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -216,9 +216,6 @@ struct DerivationGoal : public Goal */ std::optional derivationType; - typedef kj::Promise> (DerivationGoal::*GoalState)(bool inBuildSlot) noexcept; - GoalState state; - BuildMode buildMode; NotifyingCounter::Bump mcExpectedBuilds, mcRunningBuilds; @@ -257,23 +254,23 @@ struct DerivationGoal : public Goal /** * The states. */ - kj::Promise> getDerivation(bool inBuildSlot) noexcept; - kj::Promise> loadDerivation(bool inBuildSlot) noexcept; - kj::Promise> haveDerivation(bool inBuildSlot) noexcept; - kj::Promise> outputsSubstitutionTried(bool inBuildSlot) noexcept; - kj::Promise> gaveUpOnSubstitution(bool inBuildSlot) noexcept; - kj::Promise> closureRepaired(bool inBuildSlot) noexcept; - kj::Promise> inputsRealised(bool inBuildSlot) noexcept; - kj::Promise> tryToBuild(bool inBuildSlot) noexcept; - virtual kj::Promise> tryLocalBuild(bool inBuildSlot) noexcept; - kj::Promise> buildDone(bool inBuildSlot) noexcept; + kj::Promise> getDerivation() noexcept; + kj::Promise> loadDerivation() noexcept; + kj::Promise> haveDerivation() noexcept; + kj::Promise> outputsSubstitutionTried() noexcept; + kj::Promise> gaveUpOnSubstitution() noexcept; + kj::Promise> closureRepaired() noexcept; + kj::Promise> inputsRealised() noexcept; + kj::Promise> tryToBuild() noexcept; + virtual kj::Promise> tryLocalBuild() noexcept; + kj::Promise> buildDone() noexcept; - kj::Promise> resolvedFinished(bool inBuildSlot) noexcept; + kj::Promise> resolvedFinished() noexcept; /** * Is the build hook willing to perform the build? */ - HookReply tryBuildHook(bool inBuildSlot); + HookReply tryBuildHook(); virtual int getChildStatus(); @@ -325,8 +322,6 @@ protected: Finished tooMuchLogs(); void flushLine(); - static kj::Promise> continueOrError(kj::Promise> p); - public: /** * Wrappers around the corresponding Store methods that first consult the diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index a32f742f4..f3d0bc8b4 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -147,20 +147,18 @@ void LocalDerivationGoal::killSandbox(bool getStats) } -kj::Promise> LocalDerivationGoal::tryLocalBuild(bool inBuildSlot) noexcept +kj::Promise> LocalDerivationGoal::tryLocalBuild() noexcept try { +retry: #if __APPLE__ additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); #endif - if (!inBuildSlot) { - state = &DerivationGoal::tryToBuild; + if (!slotToken.valid()) { outputLocks.unlock(); if (worker.localBuilds.capacity() > 0) { - return worker.localBuilds.acquire().then([this](auto token) { - slotToken = std::move(token); - return work(); - }); + slotToken = co_await worker.localBuilds.acquire(); + co_return co_await tryToBuild(); } if (getMachines().empty()) { throw Error( @@ -215,7 +213,9 @@ try { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); - return waitForAWhile(); + (co_await waitForAWhile()).value(); + // we can loop very often, and `co_return co_await` always allocates a new frame + goto retry; } } @@ -246,22 +246,27 @@ try { /* Okay, we have to build. */ auto promise = startBuilder(); - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; - started(); - return continueOrError(std::move(promise)); + auto r = co_await promise; + if (r.has_value()) { + // all good so far + } else if (r.has_error()) { + co_return r.assume_error(); + } else { + co_return r.assume_exception(); + } } catch (BuildError & e) { outputLocks.unlock(); buildUser.reset(); auto report = done(BuildResult::InputRejected, {}, std::move(e)); report.permanentFailure = true; - return {std::move(report)}; + co_return report; } + + co_return co_await buildDone(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 52b7d4c2e..44bcd2ffe 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -213,7 +213,7 @@ struct LocalDerivationGoal : public DerivationGoal /** * The additional states. */ - kj::Promise> tryLocalBuild(bool inBuildSlot) noexcept override; + kj::Promise> tryLocalBuild() noexcept override; /** * Start building a derivation. -- 2.44.1 From ee0c195eba7d16b796fd9883e3fe88c0d64ff0bf Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 1 Oct 2024 15:30:30 -0700 Subject: [PATCH 083/106] Split ignoreException to avoid suppressing CTRL-C This splits `ignoreException` into `ignoreExceptionExceptInterrupt` (which ignores all exceptions except `Interrupt`, which indicates a SIGINT/CTRL-C) and `ignoreExceptionInDestructor` (which ignores all exceptions, so that destructors do not throw exceptions). This prevents many cases where Nix ignores CTRL-C entirely. See: https://github.com/NixOS/nix/issues/7245 Upstream-PR: https://github.com/NixOS/nix/pull/11618 Change-Id: Ie7d2467eedbe840d1b9fa2e88a4e88e4ab26a87b --- src/libexpr/eval-cache.cc | 6 +++--- src/libmain/shared.cc | 2 +- src/libstore/build/derivation-goal.cc | 4 ++-- src/libstore/build/hook-instance.cc | 3 ++- src/libstore/build/local-derivation-goal.cc | 9 +++++---- src/libstore/build/substitution-goal.cc | 2 +- src/libstore/filetransfer.cc | 2 +- src/libstore/gc.cc | 4 ++-- src/libstore/local-store.cc | 4 ++-- src/libstore/optimise-store.cc | 2 +- src/libstore/pathlocks.cc | 2 +- src/libstore/pathlocks.hh | 3 ++- src/libstore/remote-fs-accessor.cc | 4 ++-- src/libstore/remote-store.cc | 5 +++-- src/libstore/sqlite.cc | 6 +++--- src/libstore/store-api.cc | 2 +- src/libutil/current-process.cc | 2 +- src/libutil/error.cc | 14 +++++++++++++- src/libutil/error.hh | 17 ++++++++++++++++- src/libutil/file-descriptor.cc | 2 +- src/libutil/file-system.cc | 2 +- src/libutil/logging.cc | 2 +- src/libutil/serialise.cc | 2 +- src/libutil/serialise.hh | 4 ++-- src/libutil/signals.cc | 2 +- src/libutil/thread-pool.cc | 5 ++--- src/nix/develop.cc | 2 +- src/nix/flake.cc | 5 ++++- 28 files changed, 76 insertions(+), 43 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 83bfd4fb0..e7336c7e8 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -79,7 +79,7 @@ struct AttrDb state->txn->commit(); state->txn.reset(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -90,7 +90,7 @@ struct AttrDb try { return fun(); } catch (SQLiteError &) { - ignoreException(); + ignoreExceptionExceptInterrupt(); failed = true; return 0; } @@ -329,7 +329,7 @@ static std::shared_ptr makeAttrDb( try { return std::make_shared(cfg, fingerprint, symbols); } catch (SQLiteError &) { - ignoreException(); + ignoreExceptionExceptInterrupt(); return nullptr; } } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 64bd00606..029b457b1 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -395,7 +395,7 @@ RunPager::~RunPager() pid.wait(); } } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5c0452391..fe2bb0145 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -107,7 +107,7 @@ DerivationGoal::~DerivationGoal() noexcept(false) { /* Careful: we should never ever throw an exception from a destructor. */ - try { closeLogFile(); } catch (...) { ignoreException(); } + try { closeLogFile(); } catch (...) { ignoreExceptionInDestructor(); } } @@ -858,7 +858,7 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath) // attempt to recover movePath(oldPath, storePath); } catch (...) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } throw; } diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index f91a904cc..521f34917 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -1,4 +1,5 @@ #include "child.hh" +#include "error.hh" #include "file-system.hh" #include "globals.hh" #include "hook-instance.hh" @@ -86,7 +87,7 @@ HookInstance::~HookInstance() toHook.reset(); if (pid) pid.kill(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index f3d0bc8b4..5c1fcf1f8 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1,4 +1,5 @@ #include "local-derivation-goal.hh" +#include "error.hh" #include "indirect-root-store.hh" #include "machines.hh" #include "store-api.hh" @@ -98,9 +99,9 @@ LocalDerivationGoal::~LocalDerivationGoal() noexcept(false) { /* Careful: we should never ever throw an exception from a destructor. */ - try { deleteTmpDir(false); } catch (...) { ignoreException(); } - try { killChild(); } catch (...) { ignoreException(); } - try { stopDaemon(); } catch (...) { ignoreException(); } + try { deleteTmpDir(false); } catch (...) { ignoreExceptionInDestructor(); } + try { killChild(); } catch (...) { ignoreExceptionInDestructor(); } + try { stopDaemon(); } catch (...) { ignoreExceptionInDestructor(); } } @@ -1249,7 +1250,7 @@ void LocalDerivationGoal::startDaemon() NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SysError &) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } }); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 8088bf668..6dddf6e40 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -291,7 +291,7 @@ void PathSubstitutionGoal::cleanup() thr.get(); } } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 10c810e49..6fe1f9a05 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -115,7 +115,7 @@ struct curlFileTransfer : public FileTransfer if (!done) fail(FileTransferError(Interrupted, {}, "download of '%s' was interrupted", request.uri)); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index d5903d01e..99bf80994 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -923,8 +923,8 @@ void LocalStore::autoGC(bool sync) } catch (...) { // FIXME: we could propagate the exception to the - // future, but we don't really care. - ignoreException(); + // future, but we don't really care. (what??) + ignoreExceptionInDestructor(); } }).detach(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1af0f54de..c3248c2c3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -481,7 +481,7 @@ LocalStore::~LocalStore() unlink(fnTempRoots.c_str()); } } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -1222,7 +1222,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, try { parseDump(sink, source); } catch (...) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } } }; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index c60e5a85d..14381b6e0 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -31,7 +31,7 @@ struct MakeReadOnly /* This will make the path read-only. */ if (path != "") canonicaliseTimestampAndPermissions(path); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } }; diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index ced0f30bb..3225857ec 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -145,7 +145,7 @@ PathLocks::~PathLocks() try { unlock(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index d06d031b5..feb0f5548 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "error.hh" #include "file-descriptor.hh" namespace nix { @@ -53,7 +54,7 @@ struct FdLock if (acquired) lockFile(fd, ltNone, false); } catch (SysError &) { - ignoreException(); + ignoreExceptionInDestructor(); } } }; diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 0689ce74d..59d267873 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -29,7 +29,7 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::str /* FIXME: do this asynchronously. */ writeFile(makeCacheFile(hashPart, "nar"), nar); } catch (...) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } } @@ -41,7 +41,7 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::str nlohmann::json j = listNar(narAccessor, "", true); writeFile(makeCacheFile(hashPart, "ls"), j.dump()); } catch (...) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1f94ca03f..a9f9818be 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,3 +1,4 @@ +#include "error.hh" #include "serialise.hh" #include "signals.hh" #include "path-with-outputs.hh" @@ -855,7 +856,7 @@ RemoteStore::Connection::~Connection() try { to.flush(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -985,7 +986,7 @@ void RemoteStore::ConnectionHandle::withFramedSink(std::function copyPaths( // not be within our control to change that, and we might still want // to at least copy the output paths. if (e.missingFeature == Xp::CaDerivations) - ignoreException(); + ignoreExceptionExceptInterrupt(); else throw; } diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 33cda211b..3b3e46a9a 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -49,7 +49,7 @@ unsigned int getMaxCPU() auto period = cpuMaxParts[1]; if (quota != "max") return std::ceil(std::stoi(quota) / std::stof(period)); - } catch (Error &) { ignoreException(lvlDebug); } + } catch (Error &) { ignoreExceptionInDestructor(lvlDebug); } #endif return 0; diff --git a/src/libutil/error.cc b/src/libutil/error.cc index a7cbfbfd0..027f0b1a5 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -4,6 +4,7 @@ #include "position.hh" #include "terminal.hh" #include "strings.hh" +#include "signals.hh" #include #include @@ -416,7 +417,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s return out; } -void ignoreException(Verbosity lvl) +void ignoreExceptionInDestructor(Verbosity lvl) { /* Make sure no exceptions leave this function. printError() also throws when remote is closed. */ @@ -429,4 +430,15 @@ void ignoreException(Verbosity lvl) } catch (...) { } } +void ignoreExceptionExceptInterrupt(Verbosity lvl) +{ + try { + throw; + } catch (const Interrupted & e) { + throw; + } catch (std::exception & e) { + printMsg(lvl, "error (ignored): %1%", e.what()); + } +} + } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 73c1ccadd..4eff2c2bc 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -204,7 +204,22 @@ public: /** * Exception handling in destructors: print an error message, then * ignore the exception. + * + * If you're not in a destructor, you usually want to use `ignoreExceptionExceptInterrupt()`. + * + * This function might also be used in callbacks whose caller may not handle exceptions, + * but ideally we propagate the exception using an exception_ptr in such cases. + * See e.g. `PackBuilderContext` */ -void ignoreException(Verbosity lvl = lvlError); +void ignoreExceptionInDestructor(Verbosity lvl = lvlError); + +/** + * Not destructor-safe. + * Print an error message, then ignore the exception. + * If the exception is an `Interrupted` exception, rethrow it. + * + * This may be used in a few places where Interrupt can't happen, but that's ok. + */ +void ignoreExceptionExceptInterrupt(Verbosity lvl = lvlError); } diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc index 8385ea402..cbb2bb539 100644 --- a/src/libutil/file-descriptor.cc +++ b/src/libutil/file-descriptor.cc @@ -146,7 +146,7 @@ AutoCloseFD::~AutoCloseFD() try { close(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 1d3eba58f..c4ffb1d0c 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -522,7 +522,7 @@ AutoDelete::~AutoDelete() } } } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 7d9482814..7609e6e39 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -352,7 +352,7 @@ Activity::~Activity() try { logger.stopActivity(id); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index f509fedff..2f5a11a28 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -83,7 +83,7 @@ void BufferedSink::flush() FdSink::~FdSink() { - try { flush(); } catch (...) { ignoreException(); } + try { flush(); } catch (...) { ignoreExceptionInDestructor(); } } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 3a9685e0e..08ea9a135 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -549,7 +549,7 @@ struct FramedSource : Source } } } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -595,7 +595,7 @@ struct FramedSink : nix::BufferedSink to << 0; to.flush(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libutil/signals.cc b/src/libutil/signals.cc index 04a697d01..4e9ed0ba1 100644 --- a/src/libutil/signals.cc +++ b/src/libutil/signals.cc @@ -78,7 +78,7 @@ void triggerInterrupt() try { callback(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } } diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index 0ff83e997..cd380b608 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -109,9 +109,8 @@ void ThreadPool::doWork(bool mainThread) try { std::rethrow_exception(exc); } catch (std::exception & e) { - if (!dynamic_cast(&e) && - !dynamic_cast(&e)) - ignoreException(); + if (!dynamic_cast(&e)) + ignoreExceptionExceptInterrupt(); } catch (...) { } } diff --git a/src/nix/develop.cc b/src/nix/develop.cc index fb144c904..d1615ecdc 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -639,7 +639,7 @@ struct CmdDevelop : Common, MixEnvironment throw Error("package 'nixpkgs#bashInteractive' does not provide a 'bin/bash'"); } catch (Error &) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } // Override SHELL with the one chosen for this environment. diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 15c393c90..0c704a995 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -16,6 +16,7 @@ #include "eval-cache.hh" #include "markdown.hh" #include "terminal.hh" +#include "signals.hh" #include #include @@ -367,9 +368,11 @@ struct CmdFlakeCheck : FlakeCommand auto reportError = [&](const Error & e) { try { throw e; + } catch (Interrupted & e) { + throw; } catch (Error & e) { if (settings.keepGoing) { - ignoreException(); + ignoreExceptionExceptInterrupt(); hasErrors = true; } else -- 2.44.1 From 36073781fb6d625cff891024dbafe98fcb31e60d Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Tue, 1 Oct 2024 16:09:25 -0700 Subject: [PATCH 084/106] editorconfig: Add meson.build Change-Id: Ibb59ddc21f5d3ef7fb4c900e3413e426c201334d --- .editorconfig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.editorconfig b/.editorconfig index bcee9cfce..bbc2a4908 100644 --- a/.editorconfig +++ b/.editorconfig @@ -29,3 +29,7 @@ trim_trailing_whitespace = false indent_style = space indent_size = 2 max_line_length = 0 + +[meson.build] +indent_style = space +indent_size = 2 -- 2.44.1 From 5b1715e63349541c1d021f6426b2ad67a0bf518f Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Wed, 2 Oct 2024 15:39:17 +0200 Subject: [PATCH 085/106] libstore: forbid addWantedGoals when finished due to event loop scheduling behavior it's possible for a derivation goal to fully finish (having seen all paths it was asked to create), but to not notify the worker of this in time to prevent another goal asking the recently-finished goal for more outputs. if this happened the finished goal would ignore the request for more outputs since it considered itself fully done, and the delayed result reporting would cause the requesting goal to assume its request had been honored. if the requested goal had finished *properly* the worker would recreate it instead of asking for more outputs, and this would succeed. it is thus safe to always recreate goals once they are done, so we now do. Change-Id: Ifedd69ca153372c623abe9a9b49cd1523588814f --- src/libstore/build/derivation-goal.cc | 9 ++++- src/libstore/build/derivation-goal.hh | 8 ++++- src/libstore/build/worker.cc | 47 +++++++++++++++++---------- src/libstore/build/worker.hh | 2 +- 4 files changed, 45 insertions(+), 21 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5c0452391..d90a8639d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -130,8 +130,12 @@ kj::Promise> DerivationGoal::work() noexcept return useDerivation ? getDerivation() : haveDerivation(); } -void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) +bool DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) { + if (isDone) { + return false; + } + auto newWanted = wantedOutputs.union_(outputs); switch (needRestart) { case NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed: @@ -148,6 +152,7 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) break; }; wantedOutputs = newWanted; + return true; } @@ -1680,6 +1685,8 @@ Goal::Finished DerivationGoal::done( SingleDrvOutputs builtOutputs, std::optional ex) { + isDone = true; + outputLocks.unlock(); buildResult.status = status; if (ex) diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 7505409c0..1e2f9b55c 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -73,6 +73,12 @@ struct DerivationGoal : public Goal { struct InputStream; + /** + * Whether this goal has completed. Completed goals can not be + * asked for more outputs, a new goal must be created instead. + */ + bool isDone = false; + /** * Whether to use an on-disk .drv file. */ @@ -249,7 +255,7 @@ struct DerivationGoal : public Goal /** * Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const OutputsSpec & outputs); + bool addWantedOutputs(const OutputsSpec & outputs); /** * The states. diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 5be706e42..e063ede71 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -57,24 +57,35 @@ std::pair, kj::Promise> Worker::makeGoalCommon( std::map> & map, const ID & key, InvocableR> auto create, - std::invocable auto modify + InvocableR auto modify ) { auto [it, _inserted] = map.try_emplace(key); - auto & goal_weak = it->second; - auto goal = goal_weak.goal.lock(); - if (!goal) { - goal = create(); - goal->notify = std::move(goal_weak.fulfiller); - goal_weak.goal = goal; - // do not start working immediately, this round of the event loop - // may have more calls to this function lined up that'll also run - // modify(). starting early can then cause the goals to misbehave - childStarted(goal, kj::evalLater([goal] { return goal->work(); })); - } else { - modify(*goal); + // try twice to create the goal. we can only loop if we hit the continue, + // and then we only want to recreate the goal *once*. concurrent accesses + // to the worker are not sound, we want to catch them if at all possible. + for ([[maybe_unused]] auto _attempt : {1, 2}) { + auto & goal_weak = it->second; + auto goal = goal_weak.goal.lock(); + if (!goal) { + goal = create(); + goal->notify = std::move(goal_weak.fulfiller); + goal_weak.goal = goal; + // do not start working immediately. if we are not yet running we + // may create dependencies as though they were toplevel goals, in + // which case the dependencies will not report build errors. when + // we are running we may be called for this same goal more times, + // and then we want to modify rather than recreate when possible. + childStarted(goal, kj::evalLater([goal] { return goal->work(); })); + } else { + if (!modify(*goal)) { + goal_weak = {}; + continue; + } + } + return {goal, goal_weak.promise->addBranch()}; } - return {goal, goal_weak.promise->addBranch()}; + assert(false && "could not make a goal. possible concurrent worker access"); } @@ -94,7 +105,7 @@ std::pair, kj::Promise> Worker::makeDeriva drvPath, wantedOutputs, *this, running, buildMode ); }, - [&](DerivationGoal & g) { g.addWantedOutputs(wantedOutputs); } + [&](DerivationGoal & g) { return g.addWantedOutputs(wantedOutputs); } ); } @@ -118,7 +129,7 @@ std::pair, kj::Promise> Worker::makeBasicD drvPath, drv, wantedOutputs, *this, running, buildMode ); }, - [&](DerivationGoal & g) { g.addWantedOutputs(wantedOutputs); } + [&](DerivationGoal & g) { return g.addWantedOutputs(wantedOutputs); } ); } @@ -132,7 +143,7 @@ Worker::makePathSubstitutionGoal( substitutionGoals, path, [&] { return std::make_unique(path, *this, running, repair, ca); }, - [&](auto &) {} + [&](auto &) { return true; } ); } @@ -146,7 +157,7 @@ Worker::makeDrvOutputSubstitutionGoal( drvOutputSubstitutionGoals, id, [&] { return std::make_unique(id, *this, running, repair, ca); }, - [&](auto &) {} + [&](auto &) { return true; } ); } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index d6cde8384..cd49fb860 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -241,7 +241,7 @@ private: std::map> & map, const ID & key, InvocableR> auto create, - std::invocable auto modify + InvocableR auto modify ); std::pair, kj::Promise> makeDerivationGoal( const StorePath & drvPath, -- 2.44.1 From 5df2cccc4956e53b56ba1613e36d64dc8057c508 Mon Sep 17 00:00:00 2001 From: Alois Wohlschlager Date: Sat, 5 Oct 2024 10:49:34 +0200 Subject: [PATCH 086/106] doc: install the HTML manual again In 0e6b3435a14a304b8833c27d2911de7ac4e731d4, installation of the HTML manual was accidentally dropped: setting install_dir on a custom_target only sets the directory where something is going to be installed if it is installed at all, but does not itself trigger installation. The latter has to be explicitly requested, which is just what we do here to get the manual back. Change-Id: Iff8b791de7e7cb4c8d747c2a9b1154b5fcc32fe0 --- doc/manual/meson.build | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 38aad55b5..35d94740c 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -126,6 +126,7 @@ manual = custom_target( 'manual', 'markdown', ], + install : true, install_dir : [ datadir / 'doc/nix', false, -- 2.44.1 From a3dd07535c183433a3f1f97596e9d2b41f8a33ba Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sat, 5 Oct 2024 14:29:52 +0200 Subject: [PATCH 087/106] fix build test error count checks with async runtime scheduling we can no longer guarantee exact error counts for builds that do not set keepGoing. the old behavior can be recovered with a number of hacks that affect scheduling, but none of those are very easy to follow now advisable. exact error counts will like not be needed for almost all uses except tests, and *those* had better check the actual messages rather than how many they got. more messages can even help to avoid unnecessary rebuilds for most users. Change-Id: I1c9aa7a401227dcaf2e19975b8cb83c5d4f85d64 --- tests/functional/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/build.sh b/tests/functional/build.sh index a14f6e3c2..58fba83aa 100644 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -144,8 +144,8 @@ test "$(<<<"$out" grep -E '^error:' | wc -l)" = 1 # --keep-going and FOD out="$(nix build -f fod-failing.nix -L 2>&1)" && status=0 || status=$? test "$status" = 1 -# one "hash mismatch" error, one "build of ... failed" -test "$(<<<"$out" grep -E '^error:' | wc -l)" = 2 +# at least one "hash mismatch" error, one "build of ... failed" +test "$(<<<"$out" grep -E '^error:' | wc -l)" -ge 2 <<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x.\\.drv'" <<<"$out" grepQuiet -E "likely URL: " <<<"$out" grepQuiet -E "error: build of '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out' failed" -- 2.44.1 From 0d484aa498b3c839991d11afb31bc5fcf368493d Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Sat, 5 Oct 2024 10:40:51 -0700 Subject: [PATCH 088/106] Add release note for CTRL-C improvements I'm very excited for cl/2016, so others will probably be excited also! Let's add a release note. Change-Id: Ic84a4444241aafce4cb6d5a6d1dddb47e7a7dd7b --- doc/manual/rl-next/ctrl-c-improved.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/manual/rl-next/ctrl-c-improved.md diff --git a/doc/manual/rl-next/ctrl-c-improved.md b/doc/manual/rl-next/ctrl-c-improved.md new file mode 100644 index 000000000..c27a0edbb --- /dev/null +++ b/doc/manual/rl-next/ctrl-c-improved.md @@ -0,0 +1,13 @@ +--- +synopsis: Ctrl-C stops Nix commands much more reliably and responsively +issues: [7245, fj#393] +cls: [2016] +prs: [11618] +category: Fixes +credits: [roberth, 9999years] +--- + +CTRL-C will now stop Nix commands much more reliably and responsively. While +there are still some cases where a Nix command can be slow or unresponsive +following a `SIGINT` (please report these as issues!), the vast majority of +signals will now cause the Nix command to quit quickly and consistently. -- 2.44.1 From 896a123605b825cd0a1548fc40da1a757ca30d25 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sat, 5 Oct 2024 00:38:35 +0200 Subject: [PATCH 089/106] libstore: remove Goal::StillAlive this was a triumph. i'm making a note here: huge success. it's hard to overstate my satisfaction! i'm not even angry. i'm being so sincere ri actually, no. we *are* angry. this was one dumbass odyssey. nobody has asked for this. but not doing it would have locked us into old, broken protocols forever or (possibly worse) forced us to write our own async framework building on the old did-you-mean-continuations in Worker. if we had done that we'd be locked into ever more, and ever more complex, manual state management all over the place. this just could not stand. Change-Id: I43a6de1035febff59d2eff83be9ad52af4659871 --- src/libstore/build/derivation-goal.cc | 28 +++++++++---------- src/libstore/build/derivation-goal.hh | 18 ++++++------ .../build/drv-output-substitution-goal.cc | 8 +++--- src/libstore/build/goal.cc | 15 ++++------ src/libstore/build/goal.hh | 20 +++---------- src/libstore/build/local-derivation-goal.cc | 4 +-- src/libstore/build/local-derivation-goal.hh | 2 +- src/libstore/build/substitution-goal.cc | 4 +-- src/libstore/build/substitution-goal.hh | 2 +- src/libstore/build/worker.cc | 18 ++---------- src/libstore/build/worker.hh | 3 +- 11 files changed, 46 insertions(+), 76 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 60389fdd5..c6ac39ddf 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -118,7 +118,7 @@ void DerivationGoal::killChild() } -Goal::Finished DerivationGoal::timedOut(Error && ex) +Goal::WorkResult DerivationGoal::timedOut(Error && ex) { killChild(); return done(BuildResult::TimedOut, {}, std::move(ex)); @@ -728,7 +728,7 @@ retry: if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for lock on %s", Magenta(showPaths(lockFiles)))); - (co_await waitForAWhile()).value(); + co_await waitForAWhile(); // we can loop very often, and `co_return co_await` always allocates a new frame goto retry; } @@ -799,7 +799,7 @@ retry: actLock = std::make_unique(*logger, lvlTalkative, actBuildWaiting, fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); outputLocks.unlock(); - (co_await waitForAWhile()).value(); + co_await waitForAWhile(); goto retry; } @@ -1331,7 +1331,7 @@ void DerivationGoal::closeLogFile() } -Goal::Finished DerivationGoal::tooMuchLogs() +Goal::WorkResult DerivationGoal::tooMuchLogs() { killChild(); return done( @@ -1380,7 +1380,7 @@ struct DerivationGoal::InputStream final : private kj::AsyncObject } }; -kj::Promise> DerivationGoal::handleBuilderOutput(InputStream & in) noexcept +kj::Promise> DerivationGoal::handleBuilderOutput(InputStream & in) noexcept try { auto buf = kj::heapArray(4096); while (true) { @@ -1413,7 +1413,7 @@ try { co_return std::current_exception(); } -kj::Promise> DerivationGoal::handleHookOutput(InputStream & in) noexcept +kj::Promise> DerivationGoal::handleHookOutput(InputStream & in) noexcept try { auto buf = kj::heapArray(4096); while (true) { @@ -1467,7 +1467,7 @@ try { co_return std::current_exception(); } -kj::Promise> DerivationGoal::handleChildOutput() noexcept +kj::Promise> DerivationGoal::handleChildOutput() noexcept try { assert(builderOutFD); @@ -1483,7 +1483,7 @@ try { handlers = handlers.exclusiveJoin( worker.aio.provider->getTimer() .afterDelay(settings.buildTimeout.get() * kj::SECONDS) - .then([this]() -> Outcome { + .then([this]() -> Outcome { return timedOut( Error("%1% timed out after %2% seconds", name, settings.buildTimeout) ); @@ -1491,7 +1491,7 @@ try { ); } - return handlers.then([this](auto r) -> Outcome { + return handlers.then([this](auto r) -> Outcome { if (!currentLogLine.empty()) flushLine(); return r; }); @@ -1499,7 +1499,7 @@ try { return {std::current_exception()}; } -kj::Promise> DerivationGoal::monitorForSilence() noexcept +kj::Promise> DerivationGoal::monitorForSilence() noexcept { while (true) { const auto stash = lastChildActivity; @@ -1513,13 +1513,13 @@ kj::Promise> DerivationGoal::monitorForSilence() n } } -kj::Promise> +kj::Promise> DerivationGoal::handleChildStreams(InputStream & builderIn, InputStream * hookIn) noexcept { lastChildActivity = worker.aio.provider->getTimer().now(); auto handlers = kj::joinPromisesFailFast([&] { - kj::Vector>> parts{2}; + kj::Vector>> parts{2}; parts.add(handleBuilderOutput(builderIn)); if (hookIn) { @@ -1680,7 +1680,7 @@ SingleDrvOutputs DerivationGoal::assertPathValidity() } -Goal::Finished DerivationGoal::done( +Goal::WorkResult DerivationGoal::done( BuildResult::Status status, SingleDrvOutputs builtOutputs, std::optional ex) @@ -1717,7 +1717,7 @@ Goal::Finished DerivationGoal::done( logError(ex->info()); } - return Finished{ + return WorkResult{ .exitCode = buildResult.success() ? ecSuccess : ecFailed, .result = buildResult, .ex = ex ? std::make_shared(std::move(*ex)) : nullptr, diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 1e2f9b55c..0cacf75fd 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -18,7 +18,7 @@ struct HookInstance; struct HookReplyBase { struct [[nodiscard]] Accept { - kj::Promise> promise; + kj::Promise> promise; }; struct [[nodiscard]] Decline {}; struct [[nodiscard]] Postpone {}; @@ -248,7 +248,7 @@ struct DerivationGoal : public Goal BuildMode buildMode = bmNormal); virtual ~DerivationGoal() noexcept(false); - Finished timedOut(Error && ex); + WorkResult timedOut(Error && ex); kj::Promise> work() noexcept override; @@ -319,13 +319,13 @@ struct DerivationGoal : public Goal protected: kj::TimePoint lastChildActivity = kj::minValue; - kj::Promise> handleChildOutput() noexcept; - kj::Promise> + kj::Promise> handleChildOutput() noexcept; + kj::Promise> handleChildStreams(InputStream & builderIn, InputStream * hookIn) noexcept; - kj::Promise> handleBuilderOutput(InputStream & in) noexcept; - kj::Promise> handleHookOutput(InputStream & in) noexcept; - kj::Promise> monitorForSilence() noexcept; - Finished tooMuchLogs(); + kj::Promise> handleBuilderOutput(InputStream & in) noexcept; + kj::Promise> handleHookOutput(InputStream & in) noexcept; + kj::Promise> monitorForSilence() noexcept; + WorkResult tooMuchLogs(); void flushLine(); public: @@ -360,7 +360,7 @@ public: void started(); - Finished done( + WorkResult done( BuildResult::Status status, SingleDrvOutputs builtOutputs = {}, std::optional ex = {}); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 9ffa33d7b..765908aeb 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -30,7 +30,7 @@ try { /* If the derivation already exists, we’re done */ if (worker.store.queryRealisation(id)) { - co_return Finished{ecSuccess, std::move(buildResult)}; + co_return WorkResult{ecSuccess, std::move(buildResult)}; } subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); @@ -61,7 +61,7 @@ try { /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - co_return Finished{substituterFailed ? ecFailed : ecNoSubstituters, std::move(buildResult)}; + co_return WorkResult{substituterFailed ? ecFailed : ecNoSubstituters, std::move(buildResult)}; } sub = subs.front(); @@ -140,7 +140,7 @@ try { if (nrFailed > 0) { debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); - return {Finished{ + return {WorkResult{ nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, std::move(buildResult), }}; @@ -155,7 +155,7 @@ try { kj::Promise> DrvOutputSubstitutionGoal::finished() noexcept try { trace("finished"); - return {Finished{ecSuccess, std::move(buildResult)}}; + return {WorkResult{ecSuccess, std::move(buildResult)}}; } catch (...) { return {std::current_exception()}; } diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index cfdb6717c..5910c37b2 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -11,18 +11,15 @@ void Goal::trace(std::string_view s) debug("%1%: %2%", name, s); } -kj::Promise> Goal::waitForAWhile() -try { +kj::Promise Goal::waitForAWhile() +{ trace("wait for a while"); /* If we are polling goals that are waiting for a lock, then wake up after a few seconds at most. */ - co_await worker.aio.provider->getTimer().afterDelay(settings.pollInterval.get() * kj::SECONDS); - co_return StillAlive{}; -} catch (...) { - co_return std::current_exception(); + return worker.aio.provider->getTimer().afterDelay(settings.pollInterval.get() * kj::SECONDS); } -kj::Promise> +kj::Promise> Goal::waitForGoals(kj::Array>> dependencies) noexcept try { auto left = dependencies.size(); @@ -45,11 +42,11 @@ try { waiteeDone(dep); if (dep->exitCode == ecFailed && !settings.keepGoing) { - co_return result::success(StillAlive{}); + co_return result::success(); } } - co_return result::success(StillAlive{}); + co_return result::success(); } catch (...) { co_return result::failure(std::current_exception()); } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 17c3d85db..03c2cf309 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -99,11 +99,7 @@ protected: AsyncSemaphore::Token slotToken; public: - - struct Finished; - - struct [[nodiscard]] StillAlive {}; - struct [[nodiscard]] Finished { + struct [[nodiscard]] WorkResult { ExitCode exitCode; BuildResult result; std::shared_ptr ex; @@ -113,21 +109,13 @@ public: bool checkMismatch = false; }; - struct [[nodiscard]] WorkResult : std::variant< - StillAlive, - Finished> - { - WorkResult() = delete; - using variant::variant; - }; - protected: - kj::Promise> waitForAWhile(); - kj::Promise> + kj::Promise waitForAWhile(); + kj::Promise> waitForGoals(kj::Array>> dependencies) noexcept; template... G> - kj::Promise> + kj::Promise> waitForGoals(std::pair, kj::Promise>... goals) noexcept { return waitForGoals(kj::arrOf>>(std::move(goals)...)); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 5c1fcf1f8..c8c68f99f 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -214,7 +214,7 @@ retry: if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); - (co_await waitForAWhile()).value(); + co_await waitForAWhile(); // we can loop very often, and `co_return co_await` always allocates a new frame goto retry; } @@ -399,7 +399,7 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() // NOTE this one isn't noexcept because it's called from places that expect // exceptions to signal failure to launch. we should change this some time. -kj::Promise> LocalDerivationGoal::startBuilder() +kj::Promise> LocalDerivationGoal::startBuilder() { if ((buildUser && buildUser->getUIDCount() != 1) #if __linux__ diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 44bcd2ffe..cd6ea2b55 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -218,7 +218,7 @@ struct LocalDerivationGoal : public DerivationGoal /** * Start building a derivation. */ - kj::Promise> startBuilder(); + kj::Promise> startBuilder(); /** * Fill in the environment for the builder. diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 6dddf6e40..c0dd95da5 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -32,7 +32,7 @@ PathSubstitutionGoal::~PathSubstitutionGoal() } -Goal::Finished PathSubstitutionGoal::done( +Goal::WorkResult PathSubstitutionGoal::done( ExitCode result, BuildResult::Status status, std::optional errorMsg) @@ -42,7 +42,7 @@ Goal::Finished PathSubstitutionGoal::done( debug(*errorMsg); buildResult.errorMsg = *errorMsg; } - return Finished{result, std::move(buildResult)}; + return WorkResult{result, std::move(buildResult)}; } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index dc701bcba..41e665779 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -72,7 +72,7 @@ struct PathSubstitutionGoal : public Goal */ std::optional ca; - Finished done( + WorkResult done( ExitCode result, BuildResult::Status status, std::optional errorMsg = {}); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index e063ede71..0ca805b4d 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -193,7 +193,7 @@ static void removeGoal(std::shared_ptr goal, auto & goalMap) } -void Worker::goalFinished(GoalPtr goal, Goal::Finished & f) +void Worker::goalFinished(GoalPtr goal, Goal::WorkResult & f) { goal->trace("done"); assert(!goal->exitCode.has_value()); @@ -210,20 +210,6 @@ void Worker::goalFinished(GoalPtr goal, Goal::Finished & f) goal->cleanup(); } -void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how) -{ - std::visit( - overloaded{ - [&](Goal::StillAlive) { - childStarted(goal, kj::evalLater([goal] { return goal->work(); })); - }, - [&](Goal::Finished & f) { goalFinished(goal, f); }, - }, - how - ); - updateStatistics(); -} - void Worker::removeGoal(GoalPtr goal) { if (auto drvGoal = std::dynamic_pointer_cast(goal)) @@ -250,7 +236,7 @@ void Worker::childStarted(GoalPtr goal, kj::Promise> pr children.add(promise .then([this, goal](auto result) { if (result.has_value()) { - handleWorkResult(goal, std::move(result.assume_value())); + goalFinished(goal, result.assume_value()); } else { childException = result.assume_error(); } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index cd49fb860..d0bf742c5 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -139,8 +139,7 @@ private: */ bool checkMismatch = false; - void goalFinished(GoalPtr goal, Goal::Finished & f); - void handleWorkResult(GoalPtr goal, Goal::WorkResult how); + void goalFinished(GoalPtr goal, Goal::WorkResult & f); kj::Own> childFinished; -- 2.44.1 From 99edc2ae38b533ecd742c172a3531cc8958c4be5 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sat, 5 Oct 2024 00:38:35 +0200 Subject: [PATCH 090/106] libstore: check for interrupts in parallel promise this simplifies the worker loop, and lets us remove it entirely later. note that ideally only one promise waiting for interrupts should exist in the entire system. not one per event loop, one per *process*. extra interrupt waiters make interrupt response nondeterministic and as such aren't great for user experience. if anything wants to react to aborts caused by explicit interruptions, or anything else, those things would be much better served using RAII guards such as Finally (or KJ_DEFER). Change-Id: I41d035ff40172d536e098153c7375b0972110d51 --- src/libstore/build/worker.cc | 11 +++++++---- src/libutil/signals.cc | 7 ++++++- src/libutil/signals.hh | 3 +++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 0ca805b4d..0e8694a6d 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -294,7 +294,13 @@ std::vector Worker::run(std::function req) topGoals.insert(goal); } - auto promise = runImpl().exclusiveJoin(updateStatistics()); + auto onInterrupt = kj::newPromiseAndCrossThreadFulfiller>(); + auto interruptCallback = createInterruptCallback([&] { + return result::failure(std::make_exception_ptr(makeInterrupted())); + }); + + auto promise = + runImpl().exclusiveJoin(updateStatistics()).exclusiveJoin(std::move(onInterrupt.promise)); // TODO GC interface? if (auto localStore = dynamic_cast(&store); localStore && settings.minFree != 0) { @@ -316,9 +322,6 @@ try { debug("entered goal loop"); while (1) { - - checkInterrupt(); - if (topGoals.empty()) break; /* Wait for input. */ diff --git a/src/libutil/signals.cc b/src/libutil/signals.cc index 4e9ed0ba1..dac2964ae 100644 --- a/src/libutil/signals.cc +++ b/src/libutil/signals.cc @@ -12,13 +12,18 @@ std::atomic _isInterrupted = false; thread_local std::function interruptCheck; +Interrupted makeInterrupted() +{ + return Interrupted("interrupted by the user"); +} + void _interrupted() { /* Block user interrupts while an exception is being handled. Throwing an exception while another exception is being handled kills the program! */ if (!std::uncaught_exceptions()) { - throw Interrupted("interrupted by the user"); + throw makeInterrupted(); } } diff --git a/src/libutil/signals.hh b/src/libutil/signals.hh index 02f8d2ca3..538ff94b4 100644 --- a/src/libutil/signals.hh +++ b/src/libutil/signals.hh @@ -16,10 +16,13 @@ namespace nix { /* User interruption. */ +class Interrupted; + extern std::atomic _isInterrupted; extern thread_local std::function interruptCheck; +Interrupted makeInterrupted(); void _interrupted(); void inline checkInterrupt() -- 2.44.1 From a9f2aab22612bea940aa79cfb2eb15cc650ff869 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sat, 5 Oct 2024 00:38:35 +0200 Subject: [PATCH 091/106] libstore: extract Worker::goalFinished specifics there's no reason to have the worker set information on goals that the goals themselves return from their entry point. doing this in the goal `work()` function is much cleaner, and a prerequisite to removing more implicit strong shared references to goals that are currently running. Change-Id: Ibb3e953ab8482a6a21ce2ed659d5023a991e7923 --- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/build/derivation-goal.hh | 2 +- .../build/drv-output-substitution-goal.cc | 2 +- .../build/drv-output-substitution-goal.hh | 2 +- src/libstore/build/goal.cc | 18 ++++++++++++++++++ src/libstore/build/goal.hh | 6 ++++-- src/libstore/build/substitution-goal.cc | 2 +- src/libstore/build/substitution-goal.hh | 2 +- src/libstore/build/worker.cc | 7 ------- 9 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index c6ac39ddf..edee09e13 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -125,7 +125,7 @@ Goal::WorkResult DerivationGoal::timedOut(Error && ex) } -kj::Promise> DerivationGoal::work() noexcept +kj::Promise> DerivationGoal::workImpl() noexcept { return useDerivation ? getDerivation() : haveDerivation(); } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 0cacf75fd..b461b7d0d 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -250,7 +250,7 @@ struct DerivationGoal : public Goal WorkResult timedOut(Error && ex); - kj::Promise> work() noexcept override; + kj::Promise> workImpl() noexcept override; /** * Add wanted outputs to an already existing derivation goal. diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 765908aeb..0d6f3fce0 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -24,7 +24,7 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal( } -kj::Promise> DrvOutputSubstitutionGoal::work() noexcept +kj::Promise> DrvOutputSubstitutionGoal::workImpl() noexcept try { trace("init"); diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 86020705e..f959e2a7b 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -70,7 +70,7 @@ public: kj::Promise> outPathValid() noexcept; kj::Promise> finished() noexcept; - kj::Promise> work() noexcept override; + kj::Promise> workImpl() noexcept override; JobCategory jobCategory() const override { return JobCategory::Substitution; diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 5910c37b2..7fbf43045 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -1,6 +1,7 @@ #include "goal.hh" #include "async-collect.hh" #include "worker.hh" +#include #include namespace nix { @@ -19,6 +20,23 @@ kj::Promise Goal::waitForAWhile() return worker.aio.provider->getTimer().afterDelay(settings.pollInterval.get() * kj::SECONDS); } +kj::Promise> Goal::work() noexcept +try { + BOOST_OUTCOME_CO_TRY(auto result, co_await workImpl()); + + trace("done"); + assert(!exitCode.has_value()); + exitCode = result.exitCode; + ex = result.ex; + + notify->fulfill(); + cleanup(); + + co_return std::move(result); +} catch (...) { + co_return result::failure(std::current_exception()); +} + kj::Promise> Goal::waitForGoals(kj::Array>> dependencies) noexcept try { diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 03c2cf309..10926fffc 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -92,7 +92,7 @@ struct Goal */ BuildResult buildResult; - // for use by Worker only. will go away once work() is a promise. + // for use by Worker and Goal only. will go away once work() is a promise. kj::Own> notify; protected: @@ -121,6 +121,8 @@ protected: return waitForGoals(kj::arrOf>>(std::move(goals)...)); } + virtual kj::Promise> workImpl() noexcept = 0; + public: /** @@ -138,7 +140,7 @@ public: trace("goal destroyed"); } - virtual kj::Promise> work() noexcept = 0; + kj::Promise> work() noexcept; virtual void waiteeDone(GoalPtr waitee) { } diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index c0dd95da5..206bc8649 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -46,7 +46,7 @@ Goal::WorkResult PathSubstitutionGoal::done( } -kj::Promise> PathSubstitutionGoal::work() noexcept +kj::Promise> PathSubstitutionGoal::workImpl() noexcept try { trace("init"); diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 41e665779..18b4262a4 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -87,7 +87,7 @@ public: ); ~PathSubstitutionGoal(); - kj::Promise> work() noexcept override; + kj::Promise> workImpl() noexcept override; /** * The states. diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 0e8694a6d..e90f17678 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -195,19 +195,12 @@ static void removeGoal(std::shared_ptr goal, auto & goalMap) void Worker::goalFinished(GoalPtr goal, Goal::WorkResult & f) { - goal->trace("done"); - assert(!goal->exitCode.has_value()); - goal->exitCode = f.exitCode; - goal->ex = f.ex; - permanentFailure |= f.permanentFailure; timedOut |= f.timedOut; hashMismatch |= f.hashMismatch; checkMismatch |= f.checkMismatch; removeGoal(goal); - goal->notify->fulfill(); - goal->cleanup(); } void Worker::removeGoal(GoalPtr goal) -- 2.44.1 From 7ef44660181b5c9743475ea73bc2e87a5f1d318f Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sat, 5 Oct 2024 00:38:35 +0200 Subject: [PATCH 092/106] libstore: have goals promise WorkResults, not void Change-Id: Idd218ec1572eda84dc47accc0dcd8a954d36f098 --- src/libstore/build/derivation-goal.cc | 6 ++-- .../build/drv-output-substitution-goal.cc | 2 +- src/libstore/build/goal.cc | 9 +++--- src/libstore/build/goal.hh | 12 ++++--- src/libstore/build/substitution-goal.cc | 2 +- src/libstore/build/worker.cc | 16 +++++----- src/libstore/build/worker.hh | 32 ++++++++++--------- 7 files changed, 43 insertions(+), 36 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index edee09e13..7f72efa6a 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -267,7 +267,7 @@ try { /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ - kj::Vector>> dependencies; + kj::Vector>>> dependencies; if (settings.useSubstitutes) { if (parsedDrv->substitutesAllowed()) { for (auto & [outputName, status] : initialOutputs) { @@ -376,7 +376,7 @@ try { produced using a substitute. So we have to build instead. */ kj::Promise> DerivationGoal::gaveUpOnSubstitution() noexcept try { - kj::Vector>> dependencies; + kj::Vector>>> dependencies; /* At this point we are building all outputs, so if more are wanted there is no need to restart. */ @@ -482,7 +482,7 @@ try { } /* Check each path (slow!). */ - kj::Vector>> dependencies; + kj::Vector>>> dependencies; for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; printError( diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 0d6f3fce0..adfed5845 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -103,7 +103,7 @@ try { co_return co_await tryNext(); } - kj::Vector>> dependencies; + kj::Vector>>> dependencies; for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { if (depId != id) { if (auto localOutputInfo = worker.store.queryRealisation(depId); diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 7fbf43045..cf52280ed 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -29,7 +29,7 @@ try { exitCode = result.exitCode; ex = result.ex; - notify->fulfill(); + notify->fulfill(result); cleanup(); co_return std::move(result); @@ -38,24 +38,25 @@ try { } kj::Promise> -Goal::waitForGoals(kj::Array>> dependencies) noexcept +Goal::waitForGoals(kj::Array>>> dependencies) noexcept try { auto left = dependencies.size(); for (auto & [dep, p] : dependencies) { - p = p.then([this, dep, &left] { + p = p.then([this, dep, &left](auto _result) { left--; trace(fmt("waitee '%s' done; %d left", dep->name, left)); if (dep->exitCode != Goal::ecSuccess) ++nrFailed; if (dep->exitCode == Goal::ecNoSubstituters) ++nrNoSubstituters; if (dep->exitCode == Goal::ecIncompleteClosure) ++nrIncompleteClosure; + return _result; }).eagerlyEvaluate(nullptr); } auto collectDeps = asyncCollect(std::move(dependencies)); while (auto item = co_await collectDeps.next()) { - auto & dep = *item; + auto & [dep, _result] = *item; waiteeDone(dep); diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 10926fffc..fbcbbcffc 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -92,8 +92,10 @@ struct Goal */ BuildResult buildResult; + struct WorkResult; + // for use by Worker and Goal only. will go away once work() is a promise. - kj::Own> notify; + kj::Own>> notify; protected: AsyncSemaphore::Token slotToken; @@ -112,13 +114,15 @@ public: protected: kj::Promise waitForAWhile(); kj::Promise> - waitForGoals(kj::Array>> dependencies) noexcept; + waitForGoals(kj::Array>>> dependencies) noexcept; template... G> kj::Promise> - waitForGoals(std::pair, kj::Promise>... goals) noexcept + waitForGoals(std::pair, kj::Promise>>... goals) noexcept { - return waitForGoals(kj::arrOf>>(std::move(goals)...)); + return waitForGoals( + kj::arrOf>>>(std::move(goals)...) + ); } virtual kj::Promise> workImpl() noexcept = 0; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 206bc8649..25a5eb5e4 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -157,7 +157,7 @@ try { /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - kj::Vector>> dependencies; + kj::Vector>>> dependencies; for (auto & i : info->references) if (i != storePath) /* ignore self-references */ dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i)); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index e90f17678..839b56bc8 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -53,7 +53,7 @@ Worker::~Worker() template G> -std::pair, kj::Promise> Worker::makeGoalCommon( +std::pair, kj::Promise>> Worker::makeGoalCommon( std::map> & map, const ID & key, InvocableR> auto create, @@ -89,7 +89,7 @@ std::pair, kj::Promise> Worker::makeGoalCommon( } -std::pair, kj::Promise> Worker::makeDerivationGoal( +std::pair, kj::Promise>> Worker::makeDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode ) { @@ -110,7 +110,7 @@ std::pair, kj::Promise> Worker::makeDeriva } -std::pair, kj::Promise> Worker::makeBasicDerivationGoal( +std::pair, kj::Promise>> Worker::makeBasicDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, @@ -134,7 +134,7 @@ std::pair, kj::Promise> Worker::makeBasicD } -std::pair, kj::Promise> +std::pair, kj::Promise>> Worker::makePathSubstitutionGoal( const StorePath & path, RepairFlag repair, std::optional ca ) @@ -148,7 +148,7 @@ Worker::makePathSubstitutionGoal( } -std::pair, kj::Promise> +std::pair, kj::Promise>> Worker::makeDrvOutputSubstitutionGoal( const DrvOutput & id, RepairFlag repair, std::optional ca ) @@ -162,16 +162,16 @@ Worker::makeDrvOutputSubstitutionGoal( } -std::pair> Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) +std::pair>> Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) -> std::pair> { + [&](const DerivedPath::Built & bfd) -> std::pair>> { if (auto bop = std::get_if(&*bfd.drvPath)) return makeDerivationGoal(bop->path, bfd.outputs, buildMode); else throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); }, - [&](const DerivedPath::Opaque & bo) -> std::pair> { + [&](const DerivedPath::Opaque & bo) -> std::pair>> { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); }, }, req.raw()); diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index d0bf742c5..78e204b5a 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -30,10 +30,12 @@ struct HookInstance; class GoalFactory { public: - virtual std::pair, kj::Promise> makeDerivationGoal( + virtual std::pair, kj::Promise>> + makeDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal ) = 0; - virtual std::pair, kj::Promise> makeBasicDerivationGoal( + virtual std::pair, kj::Promise>> + makeBasicDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, @@ -43,13 +45,13 @@ public: /** * @ref SubstitutionGoal "substitution goal" */ - virtual std::pair, kj::Promise> + virtual std::pair, kj::Promise>> makePathSubstitutionGoal( const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt ) = 0; - virtual std::pair, kj::Promise> + virtual std::pair, kj::Promise>> makeDrvOutputSubstitutionGoal( const DrvOutput & id, RepairFlag repair = NoRepair, @@ -62,7 +64,7 @@ public: * It will be a `DerivationGoal` for a `DerivedPath::Built` or * a `SubstitutionGoal` for a `DerivedPath::Opaque`. */ - virtual std::pair> + virtual std::pair>> makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0; }; @@ -95,12 +97,12 @@ private: struct CachedGoal { std::weak_ptr goal; - kj::Own> promise; - kj::Own> fulfiller; + kj::Own>> promise; + kj::Own>> fulfiller; CachedGoal() { - auto pf = kj::newPromiseAndFulfiller(); + auto pf = kj::newPromiseAndFulfiller>(); promise = kj::heap(pf.promise.fork()); fulfiller = std::move(pf.fulfiller); } @@ -236,29 +238,29 @@ public: */ private: template G> - std::pair, kj::Promise> makeGoalCommon( + std::pair, kj::Promise>> makeGoalCommon( std::map> & map, const ID & key, InvocableR> auto create, InvocableR auto modify ); - std::pair, kj::Promise> makeDerivationGoal( + std::pair, kj::Promise>> makeDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override; - std::pair, kj::Promise> makeBasicDerivationGoal( + std::pair, kj::Promise>> makeBasicDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override; /** * @ref SubstitutionGoal "substitution goal" */ - std::pair, kj::Promise> + std::pair, kj::Promise>> makePathSubstitutionGoal( const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt ) override; - std::pair, kj::Promise> + std::pair, kj::Promise>> makeDrvOutputSubstitutionGoal( const DrvOutput & id, RepairFlag repair = NoRepair, @@ -271,11 +273,11 @@ private: * It will be a `DerivationGoal` for a `DerivedPath::Built` or * a `SubstitutionGoal` for a `DerivedPath::Opaque`. */ - std::pair> + std::pair>> makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override; public: - using Targets = std::map>; + using Targets = std::map>>; /** * Loop until the specified top-level goals have finished. -- 2.44.1 From f389a5407916ba311faa50ec053f7ebac2a608b5 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sat, 5 Oct 2024 00:38:35 +0200 Subject: [PATCH 093/106] libstore: propagate goal exceptions using promises drop childException since it's no longer needed. also makes waitForInput, childFinished, and childTerminated redundant. Change-Id: I05d88ffd323c5b5c909ac21056162f69ffb0eb9f --- src/libstore/build/goal.cc | 1 + src/libstore/build/worker.cc | 63 +++++++++++++----------------------- src/libstore/build/worker.hh | 20 +++--------- 3 files changed, 27 insertions(+), 57 deletions(-) diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index cf52280ed..ef5e8ae96 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -57,6 +57,7 @@ try { while (auto item = co_await collectDeps.next()) { auto & [dep, _result] = *item; + BOOST_OUTCOME_CO_TRYV(_result); waiteeDone(dep); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 839b56bc8..5ca7cde76 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -1,3 +1,4 @@ +#include "async-collect.hh" #include "charptr-cast.hh" #include "worker.hh" #include "finally.hh" @@ -6,6 +7,8 @@ #include "local-derivation-goal.hh" #include "signals.hh" #include "hook-instance.hh" // IWYU pragma: keep +#include +#include namespace nix { @@ -231,20 +234,9 @@ void Worker::childStarted(GoalPtr goal, kj::Promise> pr if (result.has_value()) { goalFinished(goal, result.assume_value()); } else { - childException = result.assume_error(); + goal->notify->fulfill(result.assume_error()); } - }) - .attach(Finally{[this, goal] { - childTerminated(goal); - }})); -} - - -void Worker::childTerminated(GoalPtr goal) -{ - if (childFinished) { - childFinished->fulfill(); - } + })); } @@ -282,9 +274,12 @@ std::vector Worker::run(std::function req) running = true; Finally const _stop([&] { running = false; }); + std::vector results; + topGoals.clear(); for (auto & [goal, _promise] : _topGoals) { topGoals.insert(goal); + results.push_back(goal); } auto onInterrupt = kj::newPromiseAndCrossThreadFulfiller>(); @@ -292,8 +287,9 @@ std::vector Worker::run(std::function req) return result::failure(std::make_exception_ptr(makeInterrupted())); }); - auto promise = - runImpl().exclusiveJoin(updateStatistics()).exclusiveJoin(std::move(onInterrupt.promise)); + auto promise = runImpl(std::move(_topGoals)) + .exclusiveJoin(updateStatistics()) + .exclusiveJoin(std::move(onInterrupt.promise)); // TODO GC interface? if (auto localStore = dynamic_cast(&store); localStore && settings.minFree != 0) { @@ -303,27 +299,24 @@ std::vector Worker::run(std::function req) promise.wait(aio.waitScope).value(); - std::vector results; - for (auto & [i, _p] : _topGoals) { - results.push_back(i); - } return results; } -kj::Promise> Worker::runImpl() +kj::Promise> Worker::runImpl(Targets _topGoals) try { debug("entered goal loop"); - while (1) { + kj::Vector promises(_topGoals.size()); + for (auto & gp : _topGoals) { + promises.add(std::move(gp)); + } + + auto collect = AsyncCollect(promises.releaseAsArray()); + while (auto done = co_await collect.next()) { + // propagate goal exceptions outward + BOOST_OUTCOME_CO_TRYV(done->second); + if (topGoals.empty()) break; - - /* Wait for input. */ - if (!children.isEmpty()) - (co_await waitForInput()).value(); - - if (childException) { - std::rethrow_exception(childException); - } } /* If --keep-going is not set, it's possible that the main goal @@ -346,18 +339,6 @@ try { co_return result::failure(std::current_exception()); } -kj::Promise> Worker::waitForInput() -try { - printMsg(lvlVomit, "waiting for children"); - - auto pair = kj::newPromiseAndFulfiller(); - this->childFinished = kj::mv(pair.fulfiller); - co_await pair.promise; - co_return result::success(); -} catch (...) { - co_return result::failure(std::current_exception()); -} - unsigned int Worker::failingExitStatus() { diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 78e204b5a..26832c3b1 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -84,6 +84,9 @@ protected: */ class Worker : public WorkerBase { +public: + using Targets = std::map>>; + private: bool running = false; @@ -143,13 +146,6 @@ private: void goalFinished(GoalPtr goal, Goal::WorkResult & f); - kj::Own> childFinished; - - /** - * Wait for input to become available. - */ - kj::Promise> waitForInput(); - /** * Remove a dead goal. */ @@ -160,11 +156,6 @@ private: */ void childStarted(GoalPtr goal, kj::Promise> promise); - /** - * Unregisters a running child process. - */ - void childTerminated(GoalPtr goal); - /** * Pass current stats counters to the logger for progress bar updates. */ @@ -181,7 +172,7 @@ private: statisticsUpdateInhibitor = {}; } - kj::Promise> runImpl(); + kj::Promise> runImpl(Targets _topGoals); kj::Promise> boopGC(LocalStore & localStore); public: @@ -197,7 +188,6 @@ public: private: kj::TaskSet children; - std::exception_ptr childException; public: struct HookState { @@ -277,8 +267,6 @@ private: makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override; public: - using Targets = std::map>>; - /** * Loop until the specified top-level goals have finished. */ -- 2.44.1 From 40f154c0edc53e160b70fc60b2b5b8652dfbe84b Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sat, 5 Oct 2024 00:38:35 +0200 Subject: [PATCH 094/106] libstore: remove Worker::topGoals since we now propagate goal exceptions properly we no longer need to check topGoals for a reason to abort early. any early abort reasons, whether by exception or a clean top goal failure, can now be handled by inspecting the goal result in the main loop. this greatly reduces goal-to-goal interactions that do not happen at the main loop level. since the underscore-free name is now available for use as variables we'll migrate to that where we currently use `_topGoals` for locals. Change-Id: I5727c5ea7799647c0a69ab76975b1a03a6558aa6 --- src/libstore/build/worker.cc | 32 +++++++++++++------------------- src/libstore/build/worker.hh | 7 +------ 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 5ca7cde76..ed994f4ea 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -46,7 +46,6 @@ Worker::~Worker() goals that refer to this worker should be gone. (Otherwise we are in trouble, since goals may call childTerminated() etc. in their destructors). */ - topGoals.clear(); children.clear(); assert(expectedSubstitutions == 0); @@ -216,14 +215,6 @@ void Worker::removeGoal(GoalPtr goal) nix::removeGoal(subGoal, drvOutputSubstitutionGoals); else assert(false); - - if (topGoals.find(goal) != topGoals.end()) { - topGoals.erase(goal); - /* If a top-level goal failed, then kill all other goals - (unless keepGoing was set). */ - if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) - topGoals.clear(); - } } @@ -268,7 +259,7 @@ try { std::vector Worker::run(std::function req) { - auto _topGoals = req(goalFactory()); + auto topGoals = req(goalFactory()); assert(!running); running = true; @@ -276,9 +267,7 @@ std::vector Worker::run(std::function req) std::vector results; - topGoals.clear(); - for (auto & [goal, _promise] : _topGoals) { - topGoals.insert(goal); + for (auto & [goal, _promise] : topGoals) { results.push_back(goal); } @@ -287,7 +276,7 @@ std::vector Worker::run(std::function req) return result::failure(std::make_exception_ptr(makeInterrupted())); }); - auto promise = runImpl(std::move(_topGoals)) + auto promise = runImpl(std::move(topGoals)) .exclusiveJoin(updateStatistics()) .exclusiveJoin(std::move(onInterrupt.promise)); @@ -302,21 +291,26 @@ std::vector Worker::run(std::function req) return results; } -kj::Promise> Worker::runImpl(Targets _topGoals) +kj::Promise> Worker::runImpl(Targets topGoals) try { debug("entered goal loop"); - kj::Vector promises(_topGoals.size()); - for (auto & gp : _topGoals) { + kj::Vector promises(topGoals.size()); + for (auto & gp : topGoals) { promises.add(std::move(gp)); } auto collect = AsyncCollect(promises.releaseAsArray()); while (auto done = co_await collect.next()) { // propagate goal exceptions outward - BOOST_OUTCOME_CO_TRYV(done->second); + BOOST_OUTCOME_CO_TRY(auto result, done->second); - if (topGoals.empty()) break; + /* If a top-level goal failed, then kill all other goals + (unless keepGoing was set). */ + if (result.exitCode == Goal::ecFailed && !settings.keepGoing) { + children.clear(); + break; + } } /* If --keep-going is not set, it's possible that the main goal diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 26832c3b1..fa1031907 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -91,11 +91,6 @@ private: bool running = false; - /** - * The top-level goals of the worker. - */ - Goals topGoals; - template struct CachedGoal { @@ -172,7 +167,7 @@ private: statisticsUpdateInhibitor = {}; } - kj::Promise> runImpl(Targets _topGoals); + kj::Promise> runImpl(Targets topGoals); kj::Promise> boopGC(LocalStore & localStore); public: -- 2.44.1 From fc6291e46dda940a69b627a3287b4633bb89f29f Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sat, 5 Oct 2024 00:38:35 +0200 Subject: [PATCH 095/106] libstore: return goal results from Worker::run() this will be needed to move all interesting result fields out of Goal proper and into WorkResult. once that is done we can treat goals as a totally internal construct of the worker mechanism, which also allows us to fully stop exposing unclear intermediate state to Worker users. Change-Id: I98d7778a4b5b2590b7b070bdfc164a22a0ef7190 --- src/libstore/build/entry-points.cc | 8 ++++---- src/libstore/build/worker.cc | 25 ++++++++++--------------- src/libstore/build/worker.hh | 9 +++++---- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 27c341295..73b0f01f3 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -25,7 +25,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod StringSet failed; std::shared_ptr ex; - for (auto & i : goals) { + for (auto & [i, result] : goals) { if (i->ex) { if (ex) logError(i->ex->info()); @@ -89,7 +89,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat goals.emplace(gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)); return goals; }); - auto goal = *goals.begin(); + auto [goal, result] = *goals.begin(); return goal->buildResult.restrictTo(DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All {}, @@ -116,7 +116,7 @@ void Store::ensurePath(const StorePath & path) goals.emplace(gf.makePathSubstitutionGoal(path)); return goals; }); - auto goal = *goals.begin(); + auto [goal, result] = *goals.begin(); if (goal->exitCode != Goal::ecSuccess) { if (goal->ex) { @@ -138,7 +138,7 @@ void Store::repairPath(const StorePath & path) goals.emplace(gf.makePathSubstitutionGoal(path, Repair)); return goals; }); - auto goal = *goals.begin(); + auto [goal, result] = *goals.begin(); if (goal->exitCode != Goal::ecSuccess) { /* Since substituting the path didn't work, if we have a valid diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index ed994f4ea..0f89159e4 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -231,7 +231,7 @@ void Worker::childStarted(GoalPtr goal, kj::Promise> pr } -kj::Promise> Worker::updateStatistics() +kj::Promise> Worker::updateStatistics() try { while (true) { statisticsUpdateInhibitor = co_await statisticsUpdateSignal.acquire(); @@ -257,7 +257,7 @@ try { co_return result::failure(std::current_exception()); } -std::vector Worker::run(std::function req) +Worker::Results Worker::run(std::function req) { auto topGoals = req(goalFactory()); @@ -265,13 +265,7 @@ std::vector Worker::run(std::function req) running = true; Finally const _stop([&] { running = false; }); - std::vector results; - - for (auto & [goal, _promise] : topGoals) { - results.push_back(goal); - } - - auto onInterrupt = kj::newPromiseAndCrossThreadFulfiller>(); + auto onInterrupt = kj::newPromiseAndCrossThreadFulfiller>(); auto interruptCallback = createInterruptCallback([&] { return result::failure(std::make_exception_ptr(makeInterrupted())); }); @@ -286,12 +280,10 @@ std::vector Worker::run(std::function req) promise = promise.exclusiveJoin(boopGC(*localStore)); } - promise.wait(aio.waitScope).value(); - - return results; + return promise.wait(aio.waitScope).value(); } -kj::Promise> Worker::runImpl(Targets topGoals) +kj::Promise> Worker::runImpl(Targets topGoals) try { debug("entered goal loop"); @@ -300,10 +292,13 @@ try { promises.add(std::move(gp)); } + Results results; + auto collect = AsyncCollect(promises.releaseAsArray()); while (auto done = co_await collect.next()) { // propagate goal exceptions outward BOOST_OUTCOME_CO_TRY(auto result, done->second); + results.emplace(done->first, result); /* If a top-level goal failed, then kill all other goals (unless keepGoing was set). */ @@ -318,12 +313,12 @@ try { --keep-going *is* set, then they must all be finished now. */ assert(!settings.keepGoing || children.isEmpty()); - co_return result::success(); + co_return std::move(results); } catch (...) { co_return result::failure(std::current_exception()); } -kj::Promise> Worker::boopGC(LocalStore & localStore) +kj::Promise> Worker::boopGC(LocalStore & localStore) try { while (true) { co_await aio.provider->getTimer().afterDelay(10 * kj::SECONDS); diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index fa1031907..6569de6ee 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -86,6 +86,7 @@ class Worker : public WorkerBase { public: using Targets = std::map>>; + using Results = std::map; private: @@ -154,7 +155,7 @@ private: /** * Pass current stats counters to the logger for progress bar updates. */ - kj::Promise> updateStatistics(); + kj::Promise> updateStatistics(); AsyncSemaphore statisticsUpdateSignal{1}; std::optional statisticsUpdateInhibitor; @@ -167,8 +168,8 @@ private: statisticsUpdateInhibitor = {}; } - kj::Promise> runImpl(Targets topGoals); - kj::Promise> boopGC(LocalStore & localStore); + kj::Promise> runImpl(Targets topGoals); + kj::Promise> boopGC(LocalStore & localStore); public: @@ -265,7 +266,7 @@ public: /** * Loop until the specified top-level goals have finished. */ - std::vector run(std::function req); + Results run(std::function req); /*** * The exit status in case of failure. -- 2.44.1 From 7ff60b7445ccb599edcbf0078d181034f61a0859 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sat, 5 Oct 2024 00:38:35 +0200 Subject: [PATCH 096/106] libstore: move Goal::exitCode to WorkResult the field is simply duplicated between the two, and now that we can return WorkResults from Worker::run we no longer need both of them. Change-Id: I82fc47d050b39b7bb7d1656445630d271f6c9830 --- src/libstore/build/entry-points.cc | 6 +++--- src/libstore/build/goal.cc | 19 ++++++++++--------- src/libstore/build/goal.hh | 5 ----- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 73b0f01f3..3efd31682 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -32,7 +32,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod else ex = i->ex; } - if (i->exitCode != Goal::ecSuccess) { + if (result.exitCode != Goal::ecSuccess) { if (auto i2 = dynamic_cast(i.get())) failed.insert(printStorePath(i2->drvPath)); else if (auto i2 = dynamic_cast(i.get())) @@ -118,7 +118,7 @@ void Store::ensurePath(const StorePath & path) }); auto [goal, result] = *goals.begin(); - if (goal->exitCode != Goal::ecSuccess) { + if (result.exitCode != Goal::ecSuccess) { if (goal->ex) { goal->ex->withExitStatus(worker.failingExitStatus()); throw std::move(*goal->ex); @@ -140,7 +140,7 @@ void Store::repairPath(const StorePath & path) }); auto [goal, result] = *goals.begin(); - if (goal->exitCode != Goal::ecSuccess) { + if (result.exitCode != Goal::ecSuccess) { /* Since substituting the path didn't work, if we have a valid deriver, then rebuild the deriver. */ auto info = queryPathInfo(path); diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index ef5e8ae96..c7dee08b7 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -25,8 +25,6 @@ try { BOOST_OUTCOME_CO_TRY(auto result, co_await workImpl()); trace("done"); - assert(!exitCode.has_value()); - exitCode = result.exitCode; ex = result.ex; notify->fulfill(result); @@ -42,14 +40,17 @@ Goal::waitForGoals(kj::Array>> try { auto left = dependencies.size(); for (auto & [dep, p] : dependencies) { - p = p.then([this, dep, &left](auto _result) { + p = p.then([this, dep, &left](auto _result) -> Result { + BOOST_OUTCOME_TRY(auto result, _result); + left--; trace(fmt("waitee '%s' done; %d left", dep->name, left)); - if (dep->exitCode != Goal::ecSuccess) ++nrFailed; - if (dep->exitCode == Goal::ecNoSubstituters) ++nrNoSubstituters; - if (dep->exitCode == Goal::ecIncompleteClosure) ++nrIncompleteClosure; - return _result; + if (result.exitCode != Goal::ecSuccess) ++nrFailed; + if (result.exitCode == Goal::ecNoSubstituters) ++nrNoSubstituters; + if (result.exitCode == Goal::ecIncompleteClosure) ++nrIncompleteClosure; + + return std::move(result); }).eagerlyEvaluate(nullptr); } @@ -57,11 +58,11 @@ try { while (auto item = co_await collectDeps.next()) { auto & [dep, _result] = *item; - BOOST_OUTCOME_CO_TRYV(_result); + BOOST_OUTCOME_CO_TRY(auto result, _result); waiteeDone(dep); - if (dep->exitCode == ecFailed && !settings.keepGoing) { + if (result.exitCode == ecFailed && !settings.keepGoing) { co_return result::success(); } } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index fbcbbcffc..904b7564b 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -82,11 +82,6 @@ struct Goal */ std::string name; - /** - * Whether the goal is finished. - */ - std::optional exitCode; - /** * Build result. */ -- 2.44.1 From 1caf2afb1d1fffe0ff54244d335c168f4c32cbdf Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sat, 5 Oct 2024 00:38:35 +0200 Subject: [PATCH 097/106] libstore: move Goal::buildResult to WorkResult derivation goals still hold a BuildResult member variable since parts of these results of accumulated in different places, but the Goal class now no longer has such a field. substitution goals don't need it at all, and derivation goals should also be refactored to not drop their buildResult Change-Id: Ic6d3d471cdbe790a6e09a43445e25bedec6ed446 --- src/libstore/build/derivation-goal.hh | 5 +++++ src/libstore/build/drv-output-substitution-goal.cc | 7 +++---- src/libstore/build/entry-points.cc | 4 ++-- src/libstore/build/goal.hh | 5 ----- src/libstore/build/substitution-goal.cc | 2 +- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index b461b7d0d..3885afe4a 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -184,6 +184,11 @@ struct DerivationGoal : public Goal std::map initialOutputs; + /** + * Build result. + */ + BuildResult buildResult; + /** * File descriptor for the log file. */ diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index adfed5845..f04beb884 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -30,7 +30,7 @@ try { /* If the derivation already exists, we’re done */ if (worker.store.queryRealisation(id)) { - co_return WorkResult{ecSuccess, std::move(buildResult)}; + co_return WorkResult{ecSuccess}; } subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); @@ -61,7 +61,7 @@ try { /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - co_return WorkResult{substituterFailed ? ecFailed : ecNoSubstituters, std::move(buildResult)}; + co_return WorkResult{substituterFailed ? ecFailed : ecNoSubstituters}; } sub = subs.front(); @@ -142,7 +142,6 @@ try { debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); return {WorkResult{ nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, - std::move(buildResult), }}; } @@ -155,7 +154,7 @@ try { kj::Promise> DrvOutputSubstitutionGoal::finished() noexcept try { trace("finished"); - return {WorkResult{ecSuccess, std::move(buildResult)}}; + return {WorkResult{ecSuccess}}; } catch (...) { return {std::current_exception()}; } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 3efd31682..edfad3cbb 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -72,7 +72,7 @@ std::vector Store::buildPathsWithResults( std::vector results; for (auto & [req, goalPtr] : state) - results.emplace_back(goalPtr->buildResult.restrictTo(req)); + results.emplace_back(goals[goalPtr].result.restrictTo(req)); return results; } @@ -90,7 +90,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat return goals; }); auto [goal, result] = *goals.begin(); - return goal->buildResult.restrictTo(DerivedPath::Built { + return result.result.restrictTo(DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All {}, }); diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 904b7564b..bceea3c54 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -82,11 +82,6 @@ struct Goal */ std::string name; - /** - * Build result. - */ - BuildResult buildResult; - struct WorkResult; // for use by Worker and Goal only. will go away once work() is a promise. diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 25a5eb5e4..e0ca23a86 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -37,7 +37,7 @@ Goal::WorkResult PathSubstitutionGoal::done( BuildResult::Status status, std::optional errorMsg) { - buildResult.status = status; + BuildResult buildResult{.status = status}; if (errorMsg) { debug(*errorMsg); buildResult.errorMsg = *errorMsg; -- 2.44.1 From 03cbc0ecb9402fe7bbe1a2acd4643995003d7bb2 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sat, 5 Oct 2024 00:38:35 +0200 Subject: [PATCH 098/106] libstore: move Goal::ex to WorkResult yet another duplicated field. it's the last one though. Change-Id: I352df8d306794d262d8c9066f3be78acd40e82cf --- src/libstore/build/entry-points.cc | 12 ++++++------ src/libstore/build/goal.cc | 1 - src/libstore/build/goal.hh | 6 ------ 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index edfad3cbb..808179a4d 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -26,11 +26,11 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod StringSet failed; std::shared_ptr ex; for (auto & [i, result] : goals) { - if (i->ex) { + if (result.ex) { if (ex) - logError(i->ex->info()); + logError(result.ex->info()); else - ex = i->ex; + ex = result.ex; } if (result.exitCode != Goal::ecSuccess) { if (auto i2 = dynamic_cast(i.get())) @@ -119,9 +119,9 @@ void Store::ensurePath(const StorePath & path) auto [goal, result] = *goals.begin(); if (result.exitCode != Goal::ecSuccess) { - if (goal->ex) { - goal->ex->withExitStatus(worker.failingExitStatus()); - throw std::move(*goal->ex); + if (result.ex) { + result.ex->withExitStatus(worker.failingExitStatus()); + throw std::move(*result.ex); } else throw Error(worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); } diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index c7dee08b7..3f30c922b 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -25,7 +25,6 @@ try { BOOST_OUTCOME_CO_TRY(auto result, co_await workImpl()); trace("done"); - ex = result.ex; notify->fulfill(result); cleanup(); diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index bceea3c54..b524d3118 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -118,12 +118,6 @@ protected: virtual kj::Promise> workImpl() noexcept = 0; public: - - /** - * Exception containing an error message, if any. - */ - std::shared_ptr ex; - explicit Goal(Worker & worker, bool isDependency) : worker(worker) , isDependency(isDependency) -- 2.44.1 From 9adf6f4568d5a6b1c61ff93beb12a45b962c2602 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sat, 5 Oct 2024 00:38:35 +0200 Subject: [PATCH 099/106] libstore: remove Goal::notify Goal::work() is a fully usable promise that does not rely on the worker to report completion conditions. as such we no longer need the `notify` field that enabled this interplay. we do have to clear goal caches when destroying the worker though, otherwise goal promises may (incorrectly) keep goals alive due to strong shared pointers created by childStarted. Change-Id: Ie607209aafec064dbdf3464fe207d70ba9ee158a --- src/libstore/build/goal.cc | 1 - src/libstore/build/worker.cc | 12 +++++++----- src/libstore/build/worker.hh | 10 +--------- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 3f30c922b..02b22b8ad 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -26,7 +26,6 @@ try { trace("done"); - notify->fulfill(result); cleanup(); co_return std::move(result); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 0f89159e4..4e8fa38db 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -48,6 +48,10 @@ Worker::~Worker() their destructors). */ children.clear(); + derivationGoals.clear(); + drvOutputSubstitutionGoals.clear(); + substitutionGoals.clear(); + assert(expectedSubstitutions == 0); assert(expectedDownloadSize == 0); assert(expectedNarSize == 0); @@ -71,21 +75,21 @@ std::pair, kj::Promise>> Worker::mak auto goal = goal_weak.goal.lock(); if (!goal) { goal = create(); - goal->notify = std::move(goal_weak.fulfiller); goal_weak.goal = goal; // do not start working immediately. if we are not yet running we // may create dependencies as though they were toplevel goals, in // which case the dependencies will not report build errors. when // we are running we may be called for this same goal more times, // and then we want to modify rather than recreate when possible. - childStarted(goal, kj::evalLater([goal] { return goal->work(); })); + goal_weak.promise = kj::evalLater([goal] { return goal->work(); }).fork(); + childStarted(goal, goal_weak.promise.addBranch()); } else { if (!modify(*goal)) { goal_weak = {}; continue; } } - return {goal, goal_weak.promise->addBranch()}; + return {goal, goal_weak.promise.addBranch()}; } assert(false && "could not make a goal. possible concurrent worker access"); } @@ -224,8 +228,6 @@ void Worker::childStarted(GoalPtr goal, kj::Promise> pr .then([this, goal](auto result) { if (result.has_value()) { goalFinished(goal, result.assume_value()); - } else { - goal->notify->fulfill(result.assume_error()); } })); } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 6569de6ee..923092b51 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -96,15 +96,7 @@ private: struct CachedGoal { std::weak_ptr goal; - kj::Own>> promise; - kj::Own>> fulfiller; - - CachedGoal() - { - auto pf = kj::newPromiseAndFulfiller>(); - promise = kj::heap(pf.promise.fork()); - fulfiller = std::move(pf.fulfiller); - } + kj::ForkedPromise> promise{nullptr}; }; /** * Maps used to prevent multiple instantiations of a goal for the -- 2.44.1 From 649d8cd08fa16c71e3580f16d77c2122540f3195 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sat, 5 Oct 2024 00:38:35 +0200 Subject: [PATCH 100/106] libstore: remove Worker::removeGoal we can use our newfound powers of Goal::work Is A Real Promise to remove completed goals from continuation promises. apart from being much easier to follow it's also a lot more efficient because we have the iterator to the item we are trying to remove, skipping a linear search of the cache. Change-Id: Ie0190d051c5f4b81304d98db478348b20c209df5 --- src/libstore/build/goal.hh | 5 ---- src/libstore/build/worker.cc | 56 ++++++++++++------------------------ src/libstore/build/worker.hh | 7 +---- 3 files changed, 20 insertions(+), 48 deletions(-) diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index b524d3118..de1c92c85 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -82,11 +82,6 @@ struct Goal */ std::string name; - struct WorkResult; - - // for use by Worker and Goal only. will go away once work() is a promise. - kj::Own>> notify; - protected: AsyncSemaphore::Token slotToken; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 4e8fa38db..82acbdb3d 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -71,25 +71,37 @@ std::pair, kj::Promise>> Worker::mak // and then we only want to recreate the goal *once*. concurrent accesses // to the worker are not sound, we want to catch them if at all possible. for ([[maybe_unused]] auto _attempt : {1, 2}) { - auto & goal_weak = it->second; - auto goal = goal_weak.goal.lock(); + auto & cachedGoal = it->second; + auto & goal = cachedGoal.goal; if (!goal) { goal = create(); - goal_weak.goal = goal; // do not start working immediately. if we are not yet running we // may create dependencies as though they were toplevel goals, in // which case the dependencies will not report build errors. when // we are running we may be called for this same goal more times, // and then we want to modify rather than recreate when possible. - goal_weak.promise = kj::evalLater([goal] { return goal->work(); }).fork(); - childStarted(goal, goal_weak.promise.addBranch()); + auto removeWhenDone = [goal, &map, it] { + // c++ lambda coroutine capture semantics are *so* fucked up. + return [](auto goal, auto & map, auto it) -> kj::Promise> { + auto result = co_await goal->work(); + // a concurrent call to makeGoalCommon may have reset our + // cached goal and replaced it with a new instance. don't + // remove the goal in this case, otherwise we will crash. + if (goal == it->second.goal) { + map.erase(it); + } + co_return result; + }(goal, map, it); + }; + cachedGoal.promise = kj::evalLater(std::move(removeWhenDone)).fork(); + childStarted(goal, cachedGoal.promise.addBranch()); } else { if (!modify(*goal)) { - goal_weak = {}; + cachedGoal = {}; continue; } } - return {goal, goal_weak.promise.addBranch()}; + return {goal, cachedGoal.promise.addBranch()}; } assert(false && "could not make a goal. possible concurrent worker access"); } @@ -184,44 +196,14 @@ std::pair>> Worker::makeGoal(const } -template -static void removeGoal(std::shared_ptr goal, auto & goalMap) -{ - /* !!! inefficient */ - for (auto i = goalMap.begin(); - i != goalMap.end(); ) - if (i->second.goal.lock() == goal) { - auto j = i; ++j; - goalMap.erase(i); - i = j; - } - else ++i; -} - - void Worker::goalFinished(GoalPtr goal, Goal::WorkResult & f) { permanentFailure |= f.permanentFailure; timedOut |= f.timedOut; hashMismatch |= f.hashMismatch; checkMismatch |= f.checkMismatch; - - removeGoal(goal); } -void Worker::removeGoal(GoalPtr goal) -{ - if (auto drvGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvGoal, derivationGoals); - else if (auto subGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(subGoal, substitutionGoals); - else if (auto subGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(subGoal, drvOutputSubstitutionGoals); - else - assert(false); -} - - void Worker::childStarted(GoalPtr goal, kj::Promise> promise) { children.add(promise diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 923092b51..fcf0ad8c7 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -95,7 +95,7 @@ private: template struct CachedGoal { - std::weak_ptr goal; + std::shared_ptr goal; kj::ForkedPromise> promise{nullptr}; }; /** @@ -134,11 +134,6 @@ private: void goalFinished(GoalPtr goal, Goal::WorkResult & f); - /** - * Remove a dead goal. - */ - void removeGoal(GoalPtr goal); - /** * Registers a running child process. */ -- 2.44.1 From ed9b7f4f84fd60ad8618645cc1bae2d686ff0db6 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sat, 5 Oct 2024 00:38:35 +0200 Subject: [PATCH 101/106] libstore: remove Worker::{childStarted, goalFinished} these two functions are now nearly trivial and much better inline into makeGoalCommon. keeping them separate also separates information about goal completion flows and how failure information ends up in `Worker`. Change-Id: I6af86996e4a2346583371186595e3013c88fb082 --- src/libstore/build/worker.cc | 30 +++++++++--------------------- src/libstore/build/worker.hh | 7 ------- 2 files changed, 9 insertions(+), 28 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 82acbdb3d..7fc6198cf 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -94,7 +94,15 @@ std::pair, kj::Promise>> Worker::mak }(goal, map, it); }; cachedGoal.promise = kj::evalLater(std::move(removeWhenDone)).fork(); - childStarted(goal, cachedGoal.promise.addBranch()); + children.add(cachedGoal.promise.addBranch().then([this](auto _result) { + if (_result.has_value()) { + auto & result = _result.value(); + permanentFailure |= result.permanentFailure; + timedOut |= result.timedOut; + hashMismatch |= result.hashMismatch; + checkMismatch |= result.checkMismatch; + } + })); } else { if (!modify(*goal)) { cachedGoal = {}; @@ -195,26 +203,6 @@ std::pair>> Worker::makeGoal(const }, req.raw()); } - -void Worker::goalFinished(GoalPtr goal, Goal::WorkResult & f) -{ - permanentFailure |= f.permanentFailure; - timedOut |= f.timedOut; - hashMismatch |= f.hashMismatch; - checkMismatch |= f.checkMismatch; -} - -void Worker::childStarted(GoalPtr goal, kj::Promise> promise) -{ - children.add(promise - .then([this, goal](auto result) { - if (result.has_value()) { - goalFinished(goal, result.assume_value()); - } - })); -} - - kj::Promise> Worker::updateStatistics() try { while (true) { diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index fcf0ad8c7..1a913ca16 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -132,13 +132,6 @@ private: */ bool checkMismatch = false; - void goalFinished(GoalPtr goal, Goal::WorkResult & f); - - /** - * Registers a running child process. - */ - void childStarted(GoalPtr goal, kj::Promise> promise); - /** * Pass current stats counters to the logger for progress bar updates. */ -- 2.44.1 From 51a5025913cd2c901203e1a8d8f8a0df1c1a130b Mon Sep 17 00:00:00 2001 From: Lulu Date: Mon, 7 Oct 2024 09:43:30 +0200 Subject: [PATCH 102/106] Avoid calling memcpy when len == 0 in filetransfer.cc There was a bug report about a potential call to `memcpy` with a null pointer which is not reproducible: https://git.lix.systems/lix-project/lix/issues/492 This occurred in `src/libstore/filetransfer.cc` in `InnerSource::read`. To ensure that this doesn't happen, an early return is added before calling `memcpy` if the length of the data to be copied is 0. This change also adds a test that ensures that when `InnerSource::read` is called with an empty file, it throws an `EndOfFile` exception. Change-Id: Ia18149bee9a3488576c864f28475a3a0c9eadfbb --- src/libstore/filetransfer.cc | 5 ++++- tests/unit/libstore/filetransfer.cc | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 6fe1f9a05..34b92148e 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -6,6 +6,7 @@ #include "signals.hh" #include "compression.hh" #include "strings.hh" +#include #if ENABLE_S3 #include @@ -784,8 +785,10 @@ struct curlFileTransfer : public FileTransfer size_t read(char * data, size_t len) override { - auto readPartial = [this](char * data, size_t len) { + auto readPartial = [this](char * data, size_t len) -> size_t { const auto available = std::min(len, buffered.size()); + if (available == 0u) return 0u; + memcpy(data, buffered.data(), available); buffered.remove_prefix(available); return available; diff --git a/tests/unit/libstore/filetransfer.cc b/tests/unit/libstore/filetransfer.cc index 71e7392fc..fd4d326f0 100644 --- a/tests/unit/libstore/filetransfer.cc +++ b/tests/unit/libstore/filetransfer.cc @@ -150,6 +150,14 @@ TEST(FileTransfer, exceptionAbortsDownload) } } +TEST(FileTransfer, exceptionAbortsRead) +{ + auto [port, srv] = serveHTTP("200 ok", "content-length: 0\r\n", [] { return ""; }); + auto ft = makeFileTransfer(); + char buf[10] = ""; + ASSERT_THROW(ft->download(FileTransferRequest(fmt("http://[::1]:%d/index", port)))->read(buf, 10), EndOfFile); +} + TEST(FileTransfer, NOT_ON_DARWIN(reportsSetupErrors)) { auto [port, srv] = serveHTTP("404 not found", "", [] { return ""; }); -- 2.44.1 From d6e1b11d3e91abc4194744310f36e37a57dd6e1d Mon Sep 17 00:00:00 2001 From: Lulu Date: Sun, 6 Oct 2024 20:47:32 +0200 Subject: [PATCH 103/106] Fix gcc warning -Wsign-compare Add the compile flag '-Wsign-compare' and adapt the code to fix all cases of this warning. Change-Id: I26b08fa5a03e4ac294daf697d32cf9140d84350d --- meson.build | 1 + src/libstore/build/worker.cc | 2 +- tests/unit/libutil/compression.cc | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/meson.build b/meson.build index a8c6b9621..fca9f4ed3 100644 --- a/meson.build +++ b/meson.build @@ -485,6 +485,7 @@ add_project_arguments( # TODO(Qyriad): Yes this is how the autoconf+Make system did it. # It would be nice for our headers to be idempotent instead. '-include', 'config.h', + '-Wsign-compare', '-Wno-deprecated-declarations', '-Wimplicit-fallthrough', '-Werror=switch', diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 7fc6198cf..d1c1abdf8 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -247,7 +247,7 @@ Worker::Results Worker::run(std::function req) .exclusiveJoin(std::move(onInterrupt.promise)); // TODO GC interface? - if (auto localStore = dynamic_cast(&store); localStore && settings.minFree != 0) { + if (auto localStore = dynamic_cast(&store); localStore && settings.minFree != 0u) { // Periodically wake up to see if we need to run the garbage collector. promise = promise.exclusiveJoin(boopGC(*localStore)); } diff --git a/tests/unit/libutil/compression.cc b/tests/unit/libutil/compression.cc index 8d181f53d..6dd8c96df 100644 --- a/tests/unit/libutil/compression.cc +++ b/tests/unit/libutil/compression.cc @@ -1,4 +1,5 @@ #include "compression.hh" +#include #include namespace nix { @@ -147,7 +148,7 @@ TEST_P(PerTypeNonNullCompressionTest, truncatedValidInput) /* n.b. This also tests zero-length input, which is also invalid. * As of the writing of this comment, it returns empty output, but is * allowed to throw a compression error instead. */ - for (int i = 0; i < compressed.length(); ++i) { + for (size_t i = 0u; i < compressed.length(); ++i) { auto newCompressed = compressed.substr(compressed.length() - i); try { decompress(method, newCompressed); -- 2.44.1 From 43e79f443469c55ef4d3a43ce1e455d6eafcd26c Mon Sep 17 00:00:00 2001 From: Lulu Date: Sun, 6 Oct 2024 22:10:40 +0200 Subject: [PATCH 104/106] Fix gcc warning -Wmissing-field-initializers The approach that was taken here was to add default values to the type definitions rather than specify them whenever they are missing. Now the only remaining warning is '-Wunused-parameter' which @jade said is usually counterproductive and that we can just disable it: https://git.lix.systems/lix-project/lix/issues/456#issuecomment-6617 So this change adds the flags '-Wall', '-Wextra' and '-Wno-unused-parameter', so that all warnings are enabled except for '-Wunused-parameter'. Change-Id: Ic223a964d67ab429e8da804c0721ba5e25d53012 --- meson.build | 5 ++--- src/libexpr/parser/state.hh | 2 +- src/libstore/build-result.hh | 4 ++-- src/libstore/build/derivation-goal.hh | 2 +- src/libstore/build/goal.hh | 4 ++-- src/libstore/nar-accessor.cc | 4 ++-- src/libstore/path-with-outputs.hh | 2 +- src/libstore/realisation.hh | 4 ++-- src/libstore/store-api.cc | 2 +- src/libutil/error.hh | 6 +++--- src/libutil/processes.hh | 10 +++++----- 11 files changed, 22 insertions(+), 23 deletions(-) diff --git a/meson.build b/meson.build index fca9f4ed3..c7fe647ce 100644 --- a/meson.build +++ b/meson.build @@ -52,8 +52,7 @@ project('lix', 'cpp', 'rust', default_options : [ 'cpp_std=c++2a', 'rust_std=2021', - # TODO(Qyriad): increase the warning level - 'warning_level=1', + 'warning_level=2', 'debug=true', 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail @@ -485,7 +484,7 @@ add_project_arguments( # TODO(Qyriad): Yes this is how the autoconf+Make system did it. # It would be nice for our headers to be idempotent instead. '-include', 'config.h', - '-Wsign-compare', + '-Wno-unused-parameter', '-Wno-deprecated-declarations', '-Wimplicit-fallthrough', '-Werror=switch', diff --git a/src/libexpr/parser/state.hh b/src/libexpr/parser/state.hh index 1d57e2f5f..3b9b90b94 100644 --- a/src/libexpr/parser/state.hh +++ b/src/libexpr/parser/state.hh @@ -9,7 +9,7 @@ namespace nix::parser { struct StringToken { std::string_view s; - bool hasIndentation; + bool hasIndentation = false; operator std::string_view() const { return s; } }; diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index 9634fb944..846c6c9b9 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -47,7 +47,7 @@ struct BuildResult * @todo This should be an entire ErrorInfo object, not just a * string, for richer information. */ - std::string errorMsg; + std::string errorMsg = {}; std::string toString() const { auto strStatus = [&]() { @@ -90,7 +90,7 @@ struct BuildResult * For derivations, a mapping from the names of the wanted outputs * to actual paths. */ - SingleDrvOutputs builtOutputs; + SingleDrvOutputs builtOutputs = {}; /** * The start/stop times of the build (or one of the rounds, if it diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 3885afe4a..6dd58afd2 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -63,7 +63,7 @@ struct InitialOutputStatus { struct InitialOutput { bool wanted; Hash outputHash; - std::optional known; + std::optional known = {}; }; /** diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index de1c92c85..29540dcd3 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -88,8 +88,8 @@ protected: public: struct [[nodiscard]] WorkResult { ExitCode exitCode; - BuildResult result; - std::shared_ptr ex; + BuildResult result = {}; + std::shared_ptr ex = {}; bool permanentFailure = false; bool timedOut = false; bool hashMismatch = false; diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 7600de6e7..f228004a9 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -20,10 +20,10 @@ struct NarMember file in the NAR. */ uint64_t start = 0, size = 0; - std::string target; + std::string target = {}; /* If this is a directory, all the children of the directory. */ - std::map children; + std::map children = {}; }; struct NarAccessor : public FSAccessor diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 57e03252d..8e2da1908 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -17,7 +17,7 @@ namespace nix { struct StorePathWithOutputs { StorePath path; - std::set outputs; + std::set outputs = {}; std::string to_string(const Store & store) const; diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index f2b228fa0..baeb7a2c9 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -50,7 +50,7 @@ struct Realisation { DrvOutput id; StorePath outPath; - StringSet signatures; + StringSet signatures = {}; /** * The realisations that are required for the current one to be valid. @@ -58,7 +58,7 @@ struct Realisation { * When importing this realisation, the store will first check that all its * dependencies exist, and map to the correct output path */ - std::map dependentRealisations; + std::map dependentRealisations = {}; nlohmann::json toJSON() const; static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index d35a0e6a1..18f80eef8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -829,7 +829,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m { size_t left; StorePathSet valid; - std::exception_ptr exc; + std::exception_ptr exc = {}; }; Sync state_(State{paths.size(), StorePathSet()}); diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 4eff2c2bc..885a2b218 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -70,17 +70,17 @@ inline bool operator<=(const Trace& lhs, const Trace& rhs); inline bool operator>=(const Trace& lhs, const Trace& rhs); struct ErrorInfo { - Verbosity level; + Verbosity level = Verbosity::lvlError; HintFmt msg; std::shared_ptr pos; - std::list traces; + std::list traces = {}; /** * Exit status. */ unsigned int status = 1; - Suggestions suggestions; + Suggestions suggestions = {}; static std::optional programName; }; diff --git a/src/libutil/processes.hh b/src/libutil/processes.hh index dc09a9ba4..dd6e2978e 100644 --- a/src/libutil/processes.hh +++ b/src/libutil/processes.hh @@ -78,11 +78,11 @@ struct RunOptions { Path program; bool searchPath = true; - Strings args; - std::optional uid; - std::optional gid; - std::optional chdir; - std::optional> environment; + Strings args = {}; + std::optional uid = {}; + std::optional gid = {}; + std::optional chdir = {}; + std::optional> environment = {}; bool captureStdout = false; bool mergeStderrToStdout = false; bool isInteractive = false; -- 2.44.1 From 4ea8c9d6436f421dfd63638efd1fd01296bccc3f Mon Sep 17 00:00:00 2001 From: Lulu Date: Tue, 8 Oct 2024 20:05:28 +0200 Subject: [PATCH 105/106] Set c++ version to c++23 I followed @pennae's advice and moved the constructor definition of `AttrName` from the header file `nixexpr.hh` to `nixexpr.cc`. Change-Id: I733f56c25635b366b11ba332ccec38dd7444e793 --- meson.build | 2 +- src/libexpr/nixexpr.cc | 8 ++++++++ src/libexpr/nixexpr.hh | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/meson.build b/meson.build index c7fe647ce..2cf86e985 100644 --- a/meson.build +++ b/meson.build @@ -50,7 +50,7 @@ project('lix', 'cpp', 'rust', meson_version : '>=1.4.0', version : run_command('bash', '-c', 'echo -n $(jq -r .version < ./version.json)$VERSION_SUFFIX', check : true).stdout().strip(), default_options : [ - 'cpp_std=c++2a', + 'cpp_std=c++23', 'rust_std=2021', 'warning_level=2', 'debug=true', diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 68da254e2..4b659b71a 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -21,6 +21,14 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) return printIdentifier(str, s); } +AttrName::AttrName(Symbol s) : symbol(s) +{ +} + +AttrName::AttrName(std::unique_ptr e) : expr(std::move(e)) +{ +} + void Expr::show(const SymbolTable & symbols, std::ostream & str) const { abort(); diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index d16281c39..4e857b321 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -30,8 +30,8 @@ struct AttrName { Symbol symbol; std::unique_ptr expr; - AttrName(Symbol s) : symbol(s) {}; - AttrName(std::unique_ptr e) : expr(std::move(e)) {}; + AttrName(Symbol s); + AttrName(std::unique_ptr e); }; typedef std::vector AttrPath; -- 2.44.1 From 74589afdc92553dd939e37cbc98cfb356550b045 Mon Sep 17 00:00:00 2001 From: Zebreus Date: Wed, 9 Oct 2024 22:19:35 +0200 Subject: [PATCH 106/106] Fix tests on systems with a non-master git defaultBranch Change-Id: Ie73bbed1db9419c9885b9d57e4edb7a4047d5cce --- tests/functional/fetchGit.sh | 6 ++++-- tests/functional/fetchGitRefs.sh | 6 ++++-- tests/functional/flakes/flakes.sh | 29 ++++++++++++++++------------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index 492c57602..d98d889a9 100644 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -8,6 +8,8 @@ clearStore # See https://github.com/NixOS/nix/issues/6195 repo=$TEST_ROOT/./git +default_branch="$(git config init.defaultBranch)" + export _NIX_FORCE_HTTP=1 rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix $TEST_ROOT/worktree $TEST_ROOT/shallow $TEST_ROOT/minimal @@ -47,7 +49,7 @@ git -C $repo checkout -b devtest echo "different file" >> $TEST_ROOT/git/differentbranch git -C $repo add differentbranch git -C $repo commit -m 'Test2' -git -C $repo checkout master +git -C $repo checkout "$default_branch" devrev=$(git -C $repo rev-parse devtest) out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = \"file://$repo\"; rev = \"$devrev\"; }" 2>&1) || status=$? [[ $status == 1 ]] @@ -118,7 +120,7 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyShortRev") = "${rev2:0:7}-dirty" ]] # ... unless we're using an explicit ref or rev. -path3=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") +path3=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"$default_branch\"; }).outPath") [[ $path = $path3 ]] path3=$(nix eval --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; }).outPath") diff --git a/tests/functional/fetchGitRefs.sh b/tests/functional/fetchGitRefs.sh index d643fea04..8a48b6712 100644 --- a/tests/functional/fetchGitRefs.sh +++ b/tests/functional/fetchGitRefs.sh @@ -6,6 +6,8 @@ clearStore repo="$TEST_ROOT/git" +default_branch="$(git config init.defaultBranch)" + rm -rf "$repo" "${repo}-tmp" "$TEST_HOME/.cache/nix" git init "$repo" @@ -16,7 +18,7 @@ echo utrecht > "$repo"/hello git -C "$repo" add hello git -C "$repo" commit -m 'Bla1' -path=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") +path=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = \"$default_branch\"; }).outPath") # Test various combinations of ref names # (taken from the git project) @@ -38,7 +40,7 @@ path=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = \ valid_ref() { { set +x; printf >&2 '\n>>>>>>>>>> valid_ref %s\b <<<<<<<<<<\n' $(printf %s "$1" | sed -n -e l); set -x; } git check-ref-format --branch "$1" >/dev/null - git -C "$repo" branch "$1" master >/dev/null + git -C "$repo" branch "$1" "$default_branch" >/dev/null path1=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath") [[ $path1 = $path ]] git -C "$repo" branch -D "$1" >/dev/null diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 75cc329e5..f41dc365a 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -3,6 +3,9 @@ source ./common.sh requireGit clearStore + +default_branch="$(git config init.defaultBranch)" + rm -rf $TEST_HOME/.cache $TEST_HOME/.config flake1Dir=$TEST_ROOT/flake1 @@ -15,10 +18,10 @@ badFlakeDir=$TEST_ROOT/badFlake flakeGitBare=$TEST_ROOT/flakeGitBare for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $nonFlakeDir; do - # Give one repo a non-main initial branch. + # Give one repo a non-default initial branch. extraArgs= if [[ $repo == $flake2Dir ]]; then - extraArgs="--initial-branch=main" + extraArgs="--initial-branch=notdefault" fi createGitRepo "$repo" "$extraArgs" @@ -152,11 +155,11 @@ nix build -o $TEST_ROOT/result $flake2Dir#bar --no-write-lock-file expect 1 nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' nix build -o $TEST_ROOT/result $flake2Dir#bar --commit-lock-file [[ -e $flake2Dir/flake.lock ]] -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] +[[ -z $(git -C $flake2Dir diff notdefault || echo failed) ]] # Rerunning the build should not change the lockfile. nix build -o $TEST_ROOT/result $flake2Dir#bar -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] +[[ -z $(git -C $flake2Dir diff notdefault || echo failed) ]] # Building with a lockfile should not require a fetch of the registry. nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2Dir#bar --refresh @@ -165,7 +168,7 @@ nix build -o $TEST_ROOT/result --no-use-registries $flake2Dir#bar --refresh # Updating the flake should not change the lockfile. nix flake lock $flake2Dir -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] +[[ -z $(git -C $flake2Dir diff notdefault || echo failed) ]] # Now we should be able to build the flake in pure mode. nix build -o $TEST_ROOT/result flake2#bar @@ -200,7 +203,7 @@ nix build -o $TEST_ROOT/result $flake3Dir#"sth sth" nix build -o $TEST_ROOT/result $flake3Dir#"sth%20sth" # Check whether it saved the lockfile -[[ -n $(git -C $flake3Dir diff master) ]] +[[ -n $(git -C $flake3Dir diff "$default_branch") ]] git -C $flake3Dir add flake.lock @@ -286,7 +289,7 @@ nix build -o $TEST_ROOT/result $flake3Dir#sth --commit-lock-file Flake lock file updates: "?" Added input 'nonFlake': - 'git+file://"*"/flakes/flakes/nonFlake?ref=refs/heads/master&rev="*"' "*" + 'git+file://"*"/flakes/flakes/nonFlake?ref=refs/heads/$default_branch&rev="*"' "*" "?" Added input 'nonFlakeFile': 'path:"*"/flakes/flakes/nonFlake/README.md?lastModified="*"&narHash=sha256-cPh6hp48IOdRxVV3xGd0PDgSxgzj5N/2cK0rMPNaR4o%3D' "*" "?" Added input 'nonFlakeFile2': @@ -313,10 +316,10 @@ nix build -o $TEST_ROOT/result flake4#xyzzy # Test 'nix flake update' and --override-flake. nix flake lock $flake3Dir -[[ -z $(git -C $flake3Dir diff master || echo failed) ]] +[[ -z $(git -C $flake3Dir diff "$default_branch" || echo failed) ]] nix flake update --flake "$flake3Dir" --override-flake flake2 nixpkgs -[[ ! -z $(git -C "$flake3Dir" diff master || echo failed) ]] +[[ ! -z $(git -C "$flake3Dir" diff "$default_branch" || echo failed) ]] # Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore git -C $flake3Dir checkout -b removeXyzzy @@ -350,7 +353,7 @@ EOF nix flake lock $flake3Dir git -C $flake3Dir add flake.nix flake.lock git -C $flake3Dir commit -m 'Remove packages.xyzzy' -git -C $flake3Dir checkout master +git -C $flake3Dir checkout "$default_branch" # Test whether fuzzy-matching works for registry entries. (! nix build -o $TEST_ROOT/result flake4/removeXyzzy#xyzzy) @@ -499,7 +502,7 @@ nix flake lock $flake3Dir --override-input flake2/flake1 file://$TEST_ROOT/flake nix flake lock $flake3Dir --override-input flake2/flake1 flake1 [[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]] -nix flake lock $flake3Dir --override-input flake2/flake1 flake1/master/$hash1 +nix flake lock $flake3Dir --override-input flake2/flake1 "flake1/$default_branch/$hash1" [[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash1 ]] nix flake lock $flake3Dir @@ -510,8 +513,8 @@ nix flake update flake2/flake1 --flake "$flake3Dir" [[ $(jq -r .nodes.flake1_2.locked.rev "$flake3Dir/flake.lock") =~ $hash2 ]] # Test updating multiple inputs. -nix flake lock "$flake3Dir" --override-input flake1 flake1/master/$hash1 -nix flake lock "$flake3Dir" --override-input flake2/flake1 flake1/master/$hash1 +nix flake lock "$flake3Dir" --override-input flake1 "flake1/$default_branch/$hash1" +nix flake lock "$flake3Dir" --override-input flake2/flake1 "flake1/$default_branch/$hash1" [[ $(jq -r .nodes.flake1.locked.rev "$flake3Dir/flake.lock") =~ $hash1 ]] [[ $(jq -r .nodes.flake1_2.locked.rev "$flake3Dir/flake.lock") =~ $hash1 ]] -- 2.44.1