From 8c09888de4ada45ffdcb733879847aa24d3b00d8 Mon Sep 17 00:00:00 2001 From: Clemens Tolboom Date: Sat, 13 Feb 2021 15:18:08 +0100 Subject: [PATCH 01/75] Use long options instead of short ones It is a little hard to learn what the options mean. --- doc/manual/src/quick-start.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 651134c25..829e50d8a 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -19,7 +19,7 @@ to subsequent chapters. channel: ```console - $ nix-env -qaP + $ nix-env --query --available --attr-path nixpkgs.docbook_xml_dtd_43 docbook-xml-4.3 nixpkgs.docbook_xml_dtd_45 docbook-xml-4.5 nixpkgs.firefox firefox-33.0.2 @@ -31,7 +31,7 @@ to subsequent chapters. 1. Install some packages from the channel: ```console - $ nix-env -iA nixpkgs.hello + $ nix-env --install --attr nixpkgs.hello ``` This should download pre-built packages; it should not build them @@ -49,13 +49,13 @@ to subsequent chapters. 1. Uninstall a package: ```console - $ nix-env -e hello + $ nix-env --uninstall hello ``` 1. You can also test a package without installing it: ```console - $ nix-shell -p hello + $ nix-shell --profile hello ``` This builds or downloads GNU Hello and its dependencies, then drops @@ -76,7 +76,7 @@ to subsequent chapters. ```console $ nix-channel --update nixpkgs - $ nix-env -u '*' + $ nix-env --upgrade '*' ``` The latter command will upgrade each installed package for which @@ -95,5 +95,5 @@ to subsequent chapters. them: ```console - $ nix-collect-garbage -d + $ nix-collect-garbage --delete-old ``` From ddb40ddd476164a316d5cee299d807856c251040 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 19 Feb 2023 20:00:02 -0500 Subject: [PATCH 02/75] Update doc/manual/src/quick-start.md Co-authored-by: Valentin Gagarin --- doc/manual/src/quick-start.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 829e50d8a..1d2688ede 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -55,7 +55,7 @@ to subsequent chapters. 1. You can also test a package without installing it: ```console - $ nix-shell --profile hello + $ nix-shell --packages hello ``` This builds or downloads GNU Hello and its dependencies, then drops From 5bb6e3bfafbcd101962b39ce579ee56d670cd83a Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Thu, 2 Mar 2023 17:24:58 +0100 Subject: [PATCH 03/75] NixRepl::mainLoop: restore old curRepl on function exit This fixes completion callbacks after entering and leaving a nested debugger. --- src/libcmd/repl.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index e3afb1531..29c838ec0 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -252,7 +252,9 @@ void NixRepl::mainLoop() el_hist_size = 1000; #endif read_history(historyFile.c_str()); + auto oldRepl = curRepl; curRepl = this; + Finally restoreRepl([&] { curRepl = oldRepl; }); #ifndef READLINE rl_set_complete_func(completionCallback); rl_set_list_possib_func(listPossibleCallback); From e91596eb6922157aaba17a858dc52244dd0e5688 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Mon, 13 Mar 2023 21:08:52 +0100 Subject: [PATCH 04/75] Allow specifying alternative paths for reading/writing flake locks This allows having multiple separate lockfiles for a single project, which can be useful for testing against different versions of nixpkgs; it also allows tracking custom input overrides for remote flakes without requiring local clones of these flakes. For example, if I want to build Nix against my locally pinned nixpkgs, and have a lock file tracking this override independently of future updates to said nixpkgs: nix flake lock --output-lock-file /tmp/nix-flake.lock --override-input nixpkgs flake:nixpkgs nix build --reference-lock-file /tmp/nix-flake.lock Co-Authored-By: Will Fancher --- src/libcmd/installables.cc | 22 ++++++++++++++++++++++ src/libexpr/flake/flake.cc | 31 +++++++++++++++++++------------ src/libexpr/flake/flake.hh | 6 ++++++ 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 5ecf6293f..ce42e7770 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -102,6 +102,28 @@ MixFlakeOptions::MixFlakeOptions() }} }); + addFlag({ + .longName = "reference-lock-file", + .description = "Read the given lock file instead of `flake.lock` within the top-level flake", + .category = category, + .labels = {"flake-lock-path"}, + .handler = {[&](std::string lockFilePath) { + lockFlags.referenceLockFilePath = lockFilePath; + }}, + .completer = completePath + }); + + addFlag({ + .longName = "output-lock-file", + .description = "Write the given lock file instead of `flake.lock` within the top-level flake", + .category = category, + .labels = {"flake-lock-path"}, + .handler = {[&](std::string lockFilePath) { + lockFlags.outputLockFilePath = lockFilePath; + }}, + .completer = completePath + }); + addFlag({ .longName = "inputs-from", .description = "Use the inputs of the specified flake as registry entries.", diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 336eb274d..64ec4bdd2 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -337,7 +337,8 @@ LockedFlake lockFlake( // FIXME: symlink attack auto oldLockFile = LockFile::read( - flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"); + lockFlags.referenceLockFilePath.value_or( + flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock")); debug("old lock file: %s", oldLockFile); @@ -619,13 +620,20 @@ LockedFlake lockFlake( debug("new lock file: %s", newLockFile); + auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + auto sourcePath = topRef.input.getSourcePath(); + auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt; + if (lockFlags.outputLockFilePath) { + outputLockFilePath = lockFlags.outputLockFilePath; + } + /* Check whether we need to / can write the new lock file. */ - if (!(newLockFile == oldLockFile)) { + if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) { auto diff = LockFile::diff(oldLockFile, newLockFile); if (lockFlags.writeLockFile) { - if (auto sourcePath = topRef.input.getSourcePath()) { + if (outputLockFilePath) { if (auto unlockedInput = newLockFile.isUnlocked()) { if (fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); @@ -633,25 +641,24 @@ LockedFlake lockFlake( if (!lockFlags.updateLockFile) throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); - auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; - - auto path = *sourcePath + "/" + relPath; - - bool lockFileExists = pathExists(path); + bool lockFileExists = pathExists(*outputLockFilePath); if (lockFileExists) { auto s = chomp(diff); if (s.empty()) - warn("updating lock file '%s'", path); + warn("updating lock file '%s'", *outputLockFilePath); else - warn("updating lock file '%s':\n%s", path, s); + warn("updating lock file '%s':\n%s", *outputLockFilePath, s); } else - warn("creating lock file '%s'", path); + warn("creating lock file '%s'", *outputLockFilePath); - newLockFile.write(path); + newLockFile.write(*outputLockFilePath); std::optional commitMessage = std::nullopt; if (lockFlags.commitLockFile) { + if (lockFlags.outputLockFilePath) { + throw Error("--commit-lock-file and --output-lock-file are currently incompatible"); + } std::string cm; cm = fetchSettings.commitLockFileSummary.get(); diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 10301d8aa..b5db56312 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -117,6 +117,12 @@ struct LockFlags /* Whether to commit changes to flake.lock. */ bool commitLockFile = false; + /* The path to a lock file to read instead of the `flake.lock` file in the top-level flake */ + std::optional referenceLockFilePath = std::nullopt; + + /* The path to a lock file to write to instead of the `flake.lock` file in the top-level flake */ + std::optional outputLockFilePath = std::nullopt; + /* Flake inputs to be overridden. */ std::map inputOverrides; From 3a1de4c3fe176256514455e1ca951bf28f53bd71 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Tue, 14 Mar 2023 12:02:03 +0100 Subject: [PATCH 05/75] Apply review suggestions Co-authored-by: Eelco Dolstra --- src/libcmd/installables.cc | 4 ++-- src/libexpr/flake/flake.hh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index ce42e7770..c2fda7c19 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -104,7 +104,7 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "reference-lock-file", - .description = "Read the given lock file instead of `flake.lock` within the top-level flake", + .description = "Read the given lock file instead of `flake.lock` within the top-level flake.", .category = category, .labels = {"flake-lock-path"}, .handler = {[&](std::string lockFilePath) { @@ -115,7 +115,7 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "output-lock-file", - .description = "Write the given lock file instead of `flake.lock` within the top-level flake", + .description = "Write the given lock file instead of `flake.lock` within the top-level flake.", .category = category, .labels = {"flake-lock-path"}, .handler = {[&](std::string lockFilePath) { diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index b5db56312..e7e201395 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -118,10 +118,10 @@ struct LockFlags bool commitLockFile = false; /* The path to a lock file to read instead of the `flake.lock` file in the top-level flake */ - std::optional referenceLockFilePath = std::nullopt; + std::optional referenceLockFilePath; /* The path to a lock file to write to instead of the `flake.lock` file in the top-level flake */ - std::optional outputLockFilePath = std::nullopt; + std::optional outputLockFilePath; /* Flake inputs to be overridden. */ std::map inputOverrides; From 95dabbadd8a4e37da6739c93d65c70941097a09c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 13 Mar 2023 08:15:23 -0400 Subject: [PATCH 06/75] Prioritize testing in the maintainers process docs PRs that don't increase our ongoing obligations (i.e. by adding new features) but do increase test coverage of existing features are good things to merge for the health of the project, and thus good to prioritize. --- maintainers/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/maintainers/README.md b/maintainers/README.md index 476a5f51e..36d40a0ab 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -69,6 +69,7 @@ Issues on the board progress through the following states: 2. [security](https://github.com/NixOS/nix/labels/security) 3. [regression](https://github.com/NixOS/nix/labels/regression) 4. [bug](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Abug+sort%3Areactions-%2B1-desc) + 5. [tests of existing functionality](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Atests+-label%3Afeature+sort%3Areactions-%2B1-desc) - [oldest pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Acreated-asc) - [most popular pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Areactions-%2B1-desc) @@ -91,7 +92,7 @@ Issues on the board progress through the following states: Contributors who took the time to implement concrete change proposals should not wait indefinitely. - - Prioritise fixing bugs over documentation, improvements or new features + - Prioritise fixing bugs and testing over documentation, improvements or new features The team values stability and accessibility higher than raw functionality. From ea207a2eedbcb6c17055f7d92a26d9f7209c84b6 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Sun, 19 Mar 2023 14:11:19 +0100 Subject: [PATCH 07/75] Add tests for alternate lockfile path functionality --- tests/flakes/flakes.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/flakes/flakes.sh b/tests/flakes/flakes.sh index 5c922d7c5..fcbb01b05 100644 --- a/tests/flakes/flakes.sh +++ b/tests/flakes/flakes.sh @@ -96,7 +96,9 @@ json=$(nix flake metadata flake1 --json | jq .) hash1=$(echo "$json" | jq -r .revision) echo -n '# foo' >> $flake1Dir/flake.nix +flake1OriginalCommit=$(git -C $flake1Dir rev-parse HEAD) git -C $flake1Dir commit -a -m 'Foo' +flake1NewCommit=$(git -C $flake1Dir rev-parse HEAD) hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) [[ $hash1 != $hash2 ]] @@ -491,3 +493,14 @@ nix store delete $(nix store add-path $badFlakeDir) [[ $(nix-instantiate --eval flake:git+file://$flake3Dir -A x) = 123 ]] [[ $(nix-instantiate -I flake3=flake:flake3 --eval '' -A x) = 123 ]] [[ $(NIX_PATH=flake3=flake:flake3 nix-instantiate --eval '' -A x) = 123 ]] + +# Test alternate lockfile paths. +nix flake lock $flake2Dir --output-lock-file flake2.lock +cmp $flake2Dir/flake.lock flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one + +nix flake lock $flake2Dir --output-lock-file flake2-overridden.lock --override-input flake1 git+file://$flake1Dir?rev=$flake1OriginalCommit +expectStderr 1 cmp $flake2Dir/flake.lock flake2-overridden.lock +nix flake metadata $flake2Dir --reference-lock-file flake2-overridden.lock | grepQuiet $flake1OriginalCommit + +# reference-lock-file can only be used if allow-dirty is set. +expectStderr 1 nix flake metadata $flake2Dir --no-allow-dirty --reference-lock-file flake2-overridden.lock From f1c9d83697e074c32f4efdcb2845bc25edc48f13 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Sun, 19 Mar 2023 14:12:49 +0100 Subject: [PATCH 08/75] Only allow reference lock files when allow-dirty is set --- src/libexpr/flake/flake.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 64ec4bdd2..0d55ce23e 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -334,6 +334,9 @@ LockedFlake lockFlake( } try { + if (!fetchSettings.allowDirty && lockFlags.referenceLockFilePath) { + throw Error("reference lock file was provided, but the `allow-dirty` setting is set to false"); + } // FIXME: symlink attack auto oldLockFile = LockFile::read( From 3c3bd0767f70309417999ebfa34cb89cc78d4a3e Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Sun, 19 Mar 2023 14:14:30 +0100 Subject: [PATCH 09/75] Create test lockfiles in TEST_ROOT --- tests/flakes/flakes.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/flakes/flakes.sh b/tests/flakes/flakes.sh index fcbb01b05..f2e216435 100644 --- a/tests/flakes/flakes.sh +++ b/tests/flakes/flakes.sh @@ -495,12 +495,12 @@ nix store delete $(nix store add-path $badFlakeDir) [[ $(NIX_PATH=flake3=flake:flake3 nix-instantiate --eval '' -A x) = 123 ]] # Test alternate lockfile paths. -nix flake lock $flake2Dir --output-lock-file flake2.lock -cmp $flake2Dir/flake.lock flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one +nix flake lock $flake2Dir --output-lock-file $TEST_ROOT/flake2.lock +cmp $flake2Dir/flake.lock $TEST_ROOT/flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one -nix flake lock $flake2Dir --output-lock-file flake2-overridden.lock --override-input flake1 git+file://$flake1Dir?rev=$flake1OriginalCommit -expectStderr 1 cmp $flake2Dir/flake.lock flake2-overridden.lock -nix flake metadata $flake2Dir --reference-lock-file flake2-overridden.lock | grepQuiet $flake1OriginalCommit +nix flake lock $flake2Dir --output-lock-file $TEST_ROOT/flake2-overridden.lock --override-input flake1 git+file://$flake1Dir?rev=$flake1OriginalCommit +expectStderr 1 cmp $flake2Dir/flake.lock $TEST_ROOT/flake2-overridden.lock +nix flake metadata $flake2Dir --reference-lock-file $TEST_ROOT/flake2-overridden.lock | grepQuiet $flake1OriginalCommit # reference-lock-file can only be used if allow-dirty is set. -expectStderr 1 nix flake metadata $flake2Dir --no-allow-dirty --reference-lock-file flake2-overridden.lock +expectStderr 1 nix flake metadata $flake2Dir --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock From 7c4dea3cf3ef8cc0185163e69d9f3cb3a0fc95ac Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 16 Jan 2023 19:48:39 -0500 Subject: [PATCH 10/75] Punt on improper global flags for now See the note in the test. We don't want these flags showing up for commands where they are irrelevant. Eventually, this needs a proper fix, but it need not be a blocker for stabilize: for a quick-n-dirty punt, just put these flags behind the `nix-command` unstable feature. This is fine because they are only relevant for commands which we don't need to stabilize for a while. --- src/nix/main.cc | 3 +++ tests/experimental-features.sh | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/nix/main.cc b/src/nix/main.cc index c79d39459..378e53a9e 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -82,6 +82,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .description = "Print full build logs on standard error.", .category = loggingCategory, .handler = {[&]() { logger->setPrintBuildLogs(true); }}, + .experimentalFeature = Xp::NixCommand, }); addFlag({ @@ -97,6 +98,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .description = "Disable substituters and consider all previously downloaded files up-to-date.", .category = miscCategory, .handler = {[&]() { useNet = false; }}, + .experimentalFeature = Xp::NixCommand, }); addFlag({ @@ -104,6 +106,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .description = "Consider all previously downloaded files out-of-date.", .category = miscCategory, .handler = {[&]() { refresh = true; }}, + .experimentalFeature = Xp::NixCommand, }); } diff --git a/tests/experimental-features.sh b/tests/experimental-features.sh index 3be77d5cc..f718585f3 100644 --- a/tests/experimental-features.sh +++ b/tests/experimental-features.sh @@ -21,3 +21,15 @@ both_ways store gc --help expect 1 nix --experimental-features 'nix-command' show-config --flake-registry 'https://no' nix --experimental-features 'nix-command flakes' show-config --flake-registry 'https://no' + +# Double check this is stable +nix --experimental-features '' --help + +# These 3 arguments are currently given to all commands, which is wrong (as not +# all care). To deal with fixing later, we simply make them require the +# nix-command experimental features --- it so happens that the commands we wish +# stabilizing to do not need them anyways. +for arg in '--print-build-logs' '--offline' '--refresh'; do + nix --experimental-features 'nix-command' "$arg" --help + ! nix --experimental-features '' "$arg" --help +done From d1d1ae7a3b97059af09dd5a5dde2e37ada0fddac Mon Sep 17 00:00:00 2001 From: Noah Snelson Date: Mon, 27 Mar 2023 17:12:49 -0700 Subject: [PATCH 11/75] Documentation: list experimental features in manual Lists all current experimental features in the `nix.conf` manual. --- src/libutil/config.hh | 2 +- src/libutil/experimental-features.cc | 91 +++++++++++++++++++++++----- src/libutil/experimental-features.hh | 5 +- 3 files changed, 81 insertions(+), 17 deletions(-) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 748d6043b..59a766034 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -367,7 +367,7 @@ extern GlobalConfig globalConfig; struct ExperimentalFeatureSettings : Config { Setting> experimentalFeatures{this, {}, "experimental-features", - "Experimental Nix features to enable."}; + getExperimentalFeaturesList()}; /** * Check whether the given experimental feature is enabled. diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 58d762ebb..fc8590674 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -5,18 +5,61 @@ namespace nix { -std::map stringifiedXpFeatures = { - { Xp::CaDerivations, "ca-derivations" }, - { Xp::ImpureDerivations, "impure-derivations" }, - { Xp::Flakes, "flakes" }, - { Xp::NixCommand, "nix-command" }, - { Xp::RecursiveNix, "recursive-nix" }, - { Xp::NoUrlLiterals, "no-url-literals" }, - { Xp::FetchClosure, "fetch-closure" }, - { Xp::ReplFlake, "repl-flake" }, - { Xp::AutoAllocateUids, "auto-allocate-uids" }, - { Xp::Cgroups, "cgroups" }, - { Xp::DiscardReferences, "discard-references" }, +std::map> stringifiedXpFeatures = { + { Xp::CaDerivations, {"ca-derivations", R"( + Allows derivations to be content-addressed in order to prevent rebuilds + when changes to the derivation do not result in changes to the + derivation's output. See + [__contentAddressed](../language/advanced-attributes.md#adv-attr-__contentAddressed) + for more info. + )"} }, + { Xp::ImpureDerivations, {"impure-derivations", R"( + Allows derivations to produce non-fixed outputs by setting the `__impure` + derivation attribute to `true`. See [these release + notes](../release-notes/rl-2.8.md) for an example. + )"} }, + { Xp::Flakes, {"flakes", R"( + Allows for derivations to be packaged in flakes. See the manual entry for + [`nix flake`](../command-ref/new-cli/nix3-flake.md) or this [detailed + introduction](https://www.tweag.io/blog/2020-05-25-flakes/) for more info. + )"} }, + { Xp::NixCommand, {"nix-command", R"( + Allows the usage of the new `nix` CLI subcommands, such as `nix build`, `nix + develop`, `nix run`, etc. See the manual for + [`nix`](../command-ref/new-cli/nix.md) for more info. + )"} }, + { Xp::RecursiveNix, {"recursive-nix", R"( + Allow Nix derivations to call Nix in order to recursively build derivations. + See [this + commit](https://github.com/edolstra/nix/commit/1a27aa7d64ffe6fc36cfca4d82bdf51c4d8cf717) + for more info. + )"} }, + { Xp::NoUrlLiterals, {"no-url-literals", R"( + Disallows unquoted URLs as part of the Nix language syntax. See [RFC + 45](https://github.com/NixOS/rfcs/pull/45) for more info. + )"} }, + { Xp::FetchClosure, {"fetch-closure", R"( + Enables the use of the `fetchClosure` function in the standard library. See + the docs for [`fetchClosure`](../language/builtins.md#builtins-fetchClosure) + for more info. + )"} }, + { Xp::ReplFlake, {"repl-flake", R"( + Allows the user to enter a Nix REPL within a flake, e.g. `nix repl nixpkgs` + or `nix repl .#foo`. + )"} }, + { Xp::AutoAllocateUids, {"auto-allocate-uids", R"( + Allows Nix to automatically pick UIDs for builds, rather than creating + `nixbld*` user accounts. See [here](#conf-auto-allocate-uids) for more info. + )"} }, + { Xp::Cgroups, {"cgroups", R"( + Allows Nix to execute builds inside cgroups. See + [`use-cgroups`](#conf-use-cgroups) for more info. + )"} }, + { Xp::DiscardReferences, {"discard-references", R"( + Enables the use of the `unsafeDiscardReferences` attribute in derivations + that use structured attributes. This disables scanning of outputs for + runtime dependencies. + )"} }, }; const std::optional parseExperimentalFeature(const std::string_view & name) @@ -26,8 +69,11 @@ const std::optional parseExperimentalFeature(const std::str static auto reverseXpMap = []() { auto reverseXpMap = std::make_unique(); - for (auto & [feature, name] : stringifiedXpFeatures) + std::string_view name; + for (auto & [feature, featureStringPair] : stringifiedXpFeatures) { + name = featureStringPair.first; (*reverseXpMap)[name] = feature; + } return reverseXpMap; }(); @@ -41,7 +87,24 @@ std::string_view showExperimentalFeature(const ExperimentalFeature feature) { const auto ret = get(stringifiedXpFeatures, feature); assert(ret); - return *ret; + return ret->first; +} + +std::string getExperimentalFeaturesList() { + std::string experimentalFeaturesList = R"( + Experimental Nix features to enable. + Current experimental features are the following: + +)"; + + std::string experimentalFeatureString; + for (auto& [feature, featureStringPair] : stringifiedXpFeatures) { + experimentalFeatureString = " - `" + featureStringPair.first + "`\n"; + experimentalFeatureString += featureStringPair.second + "\n\n"; + experimentalFeaturesList += experimentalFeatureString; + } + + return experimentalFeaturesList; } std::set parseFeatures(const std::set & rawFeatures) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index ac372e03e..1f1852705 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -10,8 +10,8 @@ namespace nix { /** * The list of available experimental features. * - * If you update this, don’t forget to also change the map defining their - * string representation in the corresponding `.cc` file. + * If you update this, don’t forget to also change the map defining their string + * representation and documentation in the corresponding `.cc` file as well. **/ enum struct ExperimentalFeature { @@ -36,6 +36,7 @@ using Xp = ExperimentalFeature; const std::optional parseExperimentalFeature( const std::string_view & name); std::string_view showExperimentalFeature(const ExperimentalFeature); +std::string getExperimentalFeaturesList(); std::ostream & operator<<( std::ostream & str, From 6c651599b0fa6c049e5044608fcde1daefb17142 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 11 Jan 2023 09:39:24 +0100 Subject: [PATCH 12/75] review process: all PRs must be triaged in order to make the development process more transparent for everyone, all pull requests should go through the triage process before getting merged. this ensures that all team members are aware of what is going on, and that rationale for decisions is kept track of in the meeting notes for posterity. (ideally all that should go into the commit history, but this is a more invasive process change that needs further deliberation.) having all team members take at least a brief but conscious look at each change may also help with keeping our quality standards, as more reviewers are more likely to remind each other of our shared values. --- maintainers/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/maintainers/README.md b/maintainers/README.md index 476a5f51e..95f58acaa 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -56,7 +56,7 @@ Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.u The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19/views/1) for tracking its work. -Issues on the board progress through the following states: +Items on the board progress through the following states: - No Status @@ -79,6 +79,8 @@ Issues on the board progress through the following states: If there is disagreement on the general idea behind an issue or pull request, it is moved to _To discuss_, otherwise to _In review_. + To ensure process quality and reliability, all pull requests must be triaged before merging. + - To discuss Pull requests and issues that are deemed important and controversial are discussed by the team during discussion meetings. From 2585bcaa500f874871d8ac34a83fca638770b78f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Apr 2023 18:11:16 -0400 Subject: [PATCH 13/75] Rework a few things with the experimental features list - Use struct not `std::pair`, designated initializers - Use `constexpr` array that we can index by enum tag - It no longer segfaults; not sure why. --- src/libutil/experimental-features.cc | 209 +++++++++++++++++---------- src/libutil/experimental-features.hh | 5 +- src/libutil/util.hh | 10 +- 3 files changed, 142 insertions(+), 82 deletions(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index fc8590674..305b0cb11 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -5,75 +5,131 @@ namespace nix { -std::map> stringifiedXpFeatures = { - { Xp::CaDerivations, {"ca-derivations", R"( - Allows derivations to be content-addressed in order to prevent rebuilds - when changes to the derivation do not result in changes to the - derivation's output. See - [__contentAddressed](../language/advanced-attributes.md#adv-attr-__contentAddressed) - for more info. - )"} }, - { Xp::ImpureDerivations, {"impure-derivations", R"( - Allows derivations to produce non-fixed outputs by setting the `__impure` - derivation attribute to `true`. See [these release - notes](../release-notes/rl-2.8.md) for an example. - )"} }, - { Xp::Flakes, {"flakes", R"( - Allows for derivations to be packaged in flakes. See the manual entry for - [`nix flake`](../command-ref/new-cli/nix3-flake.md) or this [detailed - introduction](https://www.tweag.io/blog/2020-05-25-flakes/) for more info. - )"} }, - { Xp::NixCommand, {"nix-command", R"( - Allows the usage of the new `nix` CLI subcommands, such as `nix build`, `nix - develop`, `nix run`, etc. See the manual for - [`nix`](../command-ref/new-cli/nix.md) for more info. - )"} }, - { Xp::RecursiveNix, {"recursive-nix", R"( - Allow Nix derivations to call Nix in order to recursively build derivations. - See [this - commit](https://github.com/edolstra/nix/commit/1a27aa7d64ffe6fc36cfca4d82bdf51c4d8cf717) - for more info. - )"} }, - { Xp::NoUrlLiterals, {"no-url-literals", R"( - Disallows unquoted URLs as part of the Nix language syntax. See [RFC - 45](https://github.com/NixOS/rfcs/pull/45) for more info. - )"} }, - { Xp::FetchClosure, {"fetch-closure", R"( - Enables the use of the `fetchClosure` function in the standard library. See - the docs for [`fetchClosure`](../language/builtins.md#builtins-fetchClosure) - for more info. - )"} }, - { Xp::ReplFlake, {"repl-flake", R"( - Allows the user to enter a Nix REPL within a flake, e.g. `nix repl nixpkgs` - or `nix repl .#foo`. - )"} }, - { Xp::AutoAllocateUids, {"auto-allocate-uids", R"( - Allows Nix to automatically pick UIDs for builds, rather than creating - `nixbld*` user accounts. See [here](#conf-auto-allocate-uids) for more info. - )"} }, - { Xp::Cgroups, {"cgroups", R"( - Allows Nix to execute builds inside cgroups. See - [`use-cgroups`](#conf-use-cgroups) for more info. - )"} }, - { Xp::DiscardReferences, {"discard-references", R"( - Enables the use of the `unsafeDiscardReferences` attribute in derivations - that use structured attributes. This disables scanning of outputs for - runtime dependencies. - )"} }, +struct ExperimentalFeatureDetails +{ + ExperimentalFeature tag; + std::string_view name; + std::string_view description; }; +constexpr std::array xpFeatureDetails = {{ + { + .tag = Xp::CaDerivations, + .name = "ca-derivations", + .description = R"( + Allows derivations to be content-addressed in order to prevent rebuilds + when changes to the derivation do not result in changes to the + derivation's output. See + [__contentAddressed](../language/advanced-attributes.md#adv-attr-__contentAddressed) + for more info. + )", + }, + { + .tag = Xp::ImpureDerivations, + .name = "impure-derivations", + .description = R"( + Allows derivations to produce non-fixed outputs by setting the `__impure` + derivation attribute to `true`. See [these release + notes](../release-notes/rl-2.8.md) for an example. + )", + }, + { + .tag = Xp::Flakes, + .name = "flakes", + .description = R"( + Allows for derivations to be packaged in flakes. See the manual entry for + [`nix flake`](../command-ref/new-cli/nix3-flake.md) or this [detailed + introduction](https://www.tweag.io/blog/2020-05-25-flakes/) for more info. + )", + }, + { + .tag = Xp::NixCommand, + .name = "nix-command", + .description = R"( + Allows the usage of the new `nix` CLI subcommands, such as `nix build`, `nix + develop`, `nix run`, etc. See the manual for + [`nix`](../command-ref/new-cli/nix.md) for more info. + )", + }, + { + .tag = Xp::RecursiveNix, + .name = "recursive-nix", + .description = R"( + Allow Nix derivations to call Nix in order to recursively build derivations. + See [this + commit](https://github.com/edolstra/nix/commit/1a27aa7d64ffe6fc36cfca4d82bdf51c4d8cf717) + for more info. + )", + }, + { + .tag = Xp::NoUrlLiterals, + .name = "no-url-literals", + .description = R"( + Disallows unquoted URLs as part of the Nix language syntax. See [RFC + 45](https://github.com/NixOS/rfcs/pull/45) for more info. + )", + }, + { + .tag = Xp::FetchClosure, + .name = "fetch-closure", + .description = R"( + Enables the use of the `fetchClosure` function in the standard library. See + the docs for [`fetchClosure`](../language/builtins.md#builtins-fetchClosure) + for more info. + )", + }, + { + .tag = Xp::ReplFlake, + .name = "repl-flake", + .description = R"( + Allows the user to enter a Nix REPL within a flake, e.g. `nix repl nixpkgs` + or `nix repl .#foo`. + )", + }, + { + .tag = Xp::AutoAllocateUids, + .name = "auto-allocate-uids", + .description = R"( + Allows Nix to automatically pick UIDs for builds, rather than creating + `nixbld*` user accounts. See [here](#conf-auto-allocate-uids) for more info. + )", + }, + { + .tag = Xp::Cgroups, + .name = "cgroups", + .description = R"( + Allows Nix to execute builds inside cgroups. See + [`use-cgroups`](#conf-use-cgroups) for more info. + )", + }, + { + .tag = Xp::DiscardReferences, + .name = "discard-references", + .description = R"( + Enables the use of the `unsafeDiscardReferences` attribute in derivations + that use structured attributes. This disables scanning of outputs for + runtime dependencies. + )", + }, +}}; + +static_assert( + []() constexpr { + for (auto [index, feature] : enumerate(xpFeatureDetails)) + if (index != (size_t)feature.tag) + return false; + return true; + }(), + "array order does not match enum tag order"); + const std::optional parseExperimentalFeature(const std::string_view & name) { using ReverseXpMap = std::map; - static auto reverseXpMap = []() - { + static std::unique_ptr reverseXpMap = [](){ auto reverseXpMap = std::make_unique(); - std::string_view name; - for (auto & [feature, featureStringPair] : stringifiedXpFeatures) { - name = featureStringPair.first; - (*reverseXpMap)[name] = feature; - } + for (auto & xpFeature : xpFeatureDetails) + (*reverseXpMap)[xpFeature.name] = xpFeature.tag; return reverseXpMap; }(); @@ -83,25 +139,29 @@ const std::optional parseExperimentalFeature(const std::str return std::nullopt; } -std::string_view showExperimentalFeature(const ExperimentalFeature feature) +std::string_view showExperimentalFeature(const ExperimentalFeature tag) { - const auto ret = get(stringifiedXpFeatures, feature); - assert(ret); - return ret->first; + assert((size_t)tag < xpFeatureDetails.size()); + return xpFeatureDetails[(size_t)tag].name; } std::string getExperimentalFeaturesList() { std::string experimentalFeaturesList = R"( - Experimental Nix features to enable. - Current experimental features are the following: + Experimental Nix features to enable. + Current experimental features are the following: )"; - std::string experimentalFeatureString; - for (auto& [feature, featureStringPair] : stringifiedXpFeatures) { - experimentalFeatureString = " - `" + featureStringPair.first + "`\n"; - experimentalFeatureString += featureStringPair.second + "\n\n"; - experimentalFeaturesList += experimentalFeatureString; + for (auto & xpFeature : xpFeatureDetails) { + experimentalFeaturesList += std::string {} + /* length of this first string must be 12, matching the indent of + the descriptions in the xpFeatureDetails literal. FIXME compute + markdown in a less hacky way. */ + + " - " + + "`" + xpFeature.name + "`" + + "\n" + + xpFeature.description + + "\n\n"; } return experimentalFeaturesList; @@ -110,10 +170,9 @@ std::string getExperimentalFeaturesList() { std::set parseFeatures(const std::set & rawFeatures) { std::set res; - for (auto & rawFeature : rawFeatures) { + for (auto & rawFeature : rawFeatures) if (auto feature = parseExperimentalFeature(rawFeature)) res.insert(*feature); - } return res; } diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 7c2f872c5..650fa6533 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -11,8 +11,9 @@ namespace nix { /** * The list of available experimental features. * - * If you update this, don’t forget to also change the map defining their string - * representation and documentation in the corresponding `.cc` file as well. + * If you update this, don’t forget to also change the map defining + * their string representation and documentation in the corresponding + * `.cc` file as well. */ enum struct ExperimentalFeature { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 6c2706cc1..94c9efb74 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -730,16 +730,16 @@ constexpr auto enumerate(T && iterable) { size_t i; TIter iter; - bool operator != (const iterator & other) const { return iter != other.iter; } - void operator ++ () { ++i; ++iter; } - auto operator * () const { return std::tie(i, *iter); } + constexpr bool operator != (const iterator & other) const { return iter != other.iter; } + constexpr void operator ++ () { ++i; ++iter; } + constexpr auto operator * () const { return std::tie(i, *iter); } }; struct iterable_wrapper { T iterable; - auto begin() { return iterator{ 0, std::begin(iterable) }; } - auto end() { return iterator{ 0, std::end(iterable) }; } + constexpr auto begin() { return iterator{ 0, std::begin(iterable) }; } + constexpr auto end() { return iterator{ 0, std::end(iterable) }; } }; return iterable_wrapper{ std::forward(iterable) }; From 32d72b1696c085d17c82c9f9b02f31ac32aa148f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Apr 2023 18:57:46 -0400 Subject: [PATCH 14/75] Add more API docs to `experimental-features.hh` --- src/libutil/experimental-features.hh | 33 +++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 650fa6533..3c479dbd9 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -35,27 +35,54 @@ enum struct ExperimentalFeature */ using Xp = ExperimentalFeature; +/** + * Parse an experimental feature (enum value) from its name. Experimental + * feature flag names are hyphenated and do not contain spaces. + */ const std::optional parseExperimentalFeature( const std::string_view & name); + +/** + * Show the name of an experimental feature. This is the opposite of + * parseExperimentalFeature(). + */ std::string_view showExperimentalFeature(const ExperimentalFeature); + +/** + * Compute the documentation of all experimental features. + * + * This a markdown bulleted list where each item is first (a) the + * experimental feature flag name in backticks, and then (b) the + * description of the experimental feature. + */ std::string getExperimentalFeaturesList(); +/** + * Shorthand for `str << showExperimentalFeature(feature)`. + */ std::ostream & operator<<( std::ostream & str, const ExperimentalFeature & feature); /** - * Parse a set of strings to the corresponding set of experimental features, - * ignoring (but warning for) any unkwown feature. + * Parse a set of strings to the corresponding set of experimental + * features, ignoring (but warning for) any unknown feature. */ std::set parseFeatures(const std::set &); +/** + * An experimental feature was required for some (experimental) + * operation, but was not enabled. + */ class MissingExperimentalFeature : public Error { public: + /** + * The experimental feature that was required but not enabled. + */ ExperimentalFeature missingFeature; - MissingExperimentalFeature(ExperimentalFeature); + MissingExperimentalFeature(ExperimentalFeature missingFeature); }; /** From 9383520b755af3f496f7ecb562d10575367e274d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Apr 2023 20:28:10 -0400 Subject: [PATCH 15/75] Move `querySubstitutablePathInfos` from `LocalStore` to `Store` The code is not local-store-specific, so we should share it with all stores. More uniform behavior is better, and a less store-specific functionality is more maintainable. This fixes a FIXME added in f73d911628 by @edolstra himself. --- src/libstore/local-store.cc | 48 ------------------------------------- src/libstore/local-store.hh | 3 --- src/libstore/store-api.cc | 48 +++++++++++++++++++++++++++++++++++++ src/libstore/store-api.hh | 2 +- 4 files changed, 49 insertions(+), 52 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index dbba0c91f..f58d90895 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1134,54 +1134,6 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -// FIXME: move this, it's not specific to LocalStore. -void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) -{ - if (!settings.useSubstitutes) return; - for (auto & sub : getDefaultSubstituters()) { - for (auto & path : paths) { - if (infos.count(path.first)) - // Choose first succeeding substituter. - continue; - - auto subPath(path.first); - - // Recompute store path so that we can use a different store root. - if (path.second) { - subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); - if (sub->storeDir == storeDir) - assert(subPath == path.first); - if (subPath != path.first) - debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); - } else if (sub->storeDir != storeDir) continue; - - debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); - try { - auto info = sub->queryPathInfo(subPath); - - if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) - continue; - - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - infos.insert_or_assign(path.first, SubstitutablePathInfo{ - info->deriver, - info->references, - narInfo ? narInfo->fileSize : 0, - info->narSize}); - } catch (InvalidPath &) { - } catch (SubstituterDisabled &) { - } catch (Error & e) { - if (settings.tryFallback) - logError(e.info()); - else - throw; - } - } - } -} - - void LocalStore::registerValidPath(const ValidPathInfo & info) { registerValidPaths({{info.path, info}}); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 2eaf451bf..1b668b6fd 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -134,9 +134,6 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathCAMap & paths, - SubstitutablePathInfos & infos) override; - bool pathInfoIsUntrusted(const ValidPathInfo &) override; bool realisationIsUntrusted(const Realisation & ) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 60e87918a..8fc3ed46a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -507,6 +507,54 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path) return outputPaths; } + +void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) +{ + if (!settings.useSubstitutes) return; + for (auto & sub : getDefaultSubstituters()) { + for (auto & path : paths) { + if (infos.count(path.first)) + // Choose first succeeding substituter. + continue; + + auto subPath(path.first); + + // Recompute store path so that we can use a different store root. + if (path.second) { + subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); + if (sub->storeDir == storeDir) + assert(subPath == path.first); + if (subPath != path.first) + debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); + } else if (sub->storeDir != storeDir) continue; + + debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); + try { + auto info = sub->queryPathInfo(subPath); + + if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) + continue; + + auto narInfo = std::dynamic_pointer_cast( + std::shared_ptr(info)); + infos.insert_or_assign(path.first, SubstitutablePathInfo{ + info->deriver, + info->references, + narInfo ? narInfo->fileSize : 0, + info->narSize}); + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { + } catch (Error & e) { + if (settings.tryFallback) + logError(e.info()); + else + throw; + } + } + } +} + + bool Store::isValidPath(const StorePath & storePath) { { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 58d7b848e..4d1047380 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -456,7 +456,7 @@ public: * resulting ‘infos’ map. */ virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, - SubstitutablePathInfos & infos) { return; }; + SubstitutablePathInfos & infos); /** * Import a path into the store. From 7963d0c68dfd31a1aa179a8bd8f37dd790701d7c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Apr 2023 23:11:21 -0400 Subject: [PATCH 16/75] Try auto-labeling a few more things --- .github/labeler.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/labeler.yml b/.github/labeler.yml index dc502b6d5..fce0d3aeb 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -2,5 +2,22 @@ - doc/manual/* - src/nix/**/*.md +"store": + - src/libstore/store-api.* + - src/libstore/*-store.* + +"fetching": + - src/libfetchers/**/* + +"repl": + - src/libcmd/repl.* + - src/nix/repl.* + +"new-cli": + - src/nix/**/* + "tests": + # Unit tests + - src/*/tests/**/* + # Functional and integration tests - tests/**/* From 1c55544a428c61448dcaa4b431f20bde52c3991e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 1 Apr 2023 22:05:06 +0200 Subject: [PATCH 17/75] eval: Fix crash on missing printValue tBlackhole case Fixes #8119 --- src/libexpr/eval.cc | 10 ++++++++++ tests/lang.sh | 7 +++++++ 2 files changed, 17 insertions(+) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 584bbc879..22337a3ff 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -173,7 +173,17 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, case tFloat: str << fpoint; break; + case tBlackhole: + // Although we know for sure that it's going to be an infinite recursion + // when this value is accessed _in the current context_, it's likely + // that the user will misinterpret a simpler «infinite recursion» output + // as a definitive statement about the value, while in fact it may be + // a valid value after `builtins.trace` and perhaps some other steps + // have completed. + str << "«potential infinite recursion»"; + break; default: + printError("Nix evaluator internal error: Value::print(): invalid value type %1%", internalType); abort(); } } diff --git a/tests/lang.sh b/tests/lang.sh index cdb4174eb..8170cb39d 100644 --- a/tests/lang.sh +++ b/tests/lang.sh @@ -5,12 +5,19 @@ export NIX_REMOTE=dummy:// export NIX_STORE_DIR=/nix/store nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grepQuiet Hello +nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>/dev/null | grepQuiet 123 nix-instantiate --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 nix-instantiate --trace-verbose --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuiet Hello nix-instantiate --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuietInverse Hello nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grepQuietInverse Hello expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' | grepQuiet Hello +nix-instantiate --eval -E 'let x = builtins.trace { x = x; } true; in x' \ + 2>&1 | grepQuiet -E 'trace: { x = «potential infinite recursion»; }' + +nix-instantiate --eval -E 'let x = { repeating = x; tracing = builtins.trace x true; }; in x.tracing'\ + 2>&1 | grepQuiet -F 'trace: { repeating = «repeated»; tracing = «potential infinite recursion»; }' + set +x fail=0 From 4a0b893d5e5d5cbf2f39e52d1651bcff8b4943a9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 16 Jan 2023 23:01:18 -0500 Subject: [PATCH 18/75] Stuctured command stability Prior to this, there was an ad-hoc whitelist in `main.cc`. Now, every command states its stability. In a future PR, we will adjust the manual to take advantage of this new information in the JSON. (It will be easier to do that once we have some experimental feature docs to link too; see #5930 and #7798.) --- src/libutil/args.cc | 17 +++++++++++++++-- src/libutil/args.hh | 2 ++ src/nix/doctor.cc | 8 ++++++++ src/nix/main.cc | 6 ++---- src/nix/repl.cc | 8 ++++++++ src/nix/upgrade-nix.cc | 8 ++++++++ tests/experimental-features.sh | 11 ++++++++--- 7 files changed, 51 insertions(+), 9 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index fc009592c..081dbeb28 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -236,8 +236,6 @@ nlohmann::json Args::toJSON() auto flags = nlohmann::json::object(); for (auto & [name, flag] : longFlags) { - /* Skip experimental flags when listing flags. */ - if (!experimentalFeatureSettings.isEnabled(flag->experimentalFeature)) continue; auto j = nlohmann::json::object(); if (flag->aliases.count(name)) continue; if (flag->shortName) @@ -249,6 +247,11 @@ nlohmann::json Args::toJSON() j["arity"] = flag->handler.arity; if (!flag->labels.empty()) j["labels"] = flag->labels; + // TODO With C++23 use `std::optional::tranform` + if (auto & xp = flag->experimentalFeature) + j["experimental-feature"] = showExperimentalFeature(*xp); + else + j["experimental-feature"] = nullptr; flags[name] = std::move(j); } @@ -345,6 +348,11 @@ Strings argvToStrings(int argc, char * * argv) return args; } +std::optional Command::experimentalFeature () +{ + return { Xp::NixCommand }; +} + MultiCommand::MultiCommand(const Commands & commands_) : commands(commands_) { @@ -408,6 +416,11 @@ nlohmann::json MultiCommand::toJSON() cat["id"] = command->category(); cat["description"] = trim(categories[command->category()]); j["category"] = std::move(cat); + // TODO With C++23 use `std::optional::tranform` + if (auto xp = command->experimentalFeature()) + cat["experimental-feature"] = showExperimentalFeature(*xp); + else + cat["experimental-feature"] = nullptr; cmds[name] = std::move(j); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 07d0d8eae..d90129796 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -236,6 +236,8 @@ struct Command : virtual public Args static constexpr Category catDefault = 0; + virtual std::optional experimentalFeature (); + virtual Category category() { return catDefault; } }; diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 7da4549a1..354b03cf6 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -39,6 +39,14 @@ struct CmdDoctor : StoreCommand { bool success = true; + /** + * This command is stable before the others + */ + std::optional experimentalFeature() override + { + return std::nullopt; + } + std::string description() override { return "check your system for potential problems and print a PASS or FAIL for each check"; diff --git a/src/nix/main.cc b/src/nix/main.cc index c2e1dda74..4d4164333 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -423,10 +423,8 @@ void mainWrapped(int argc, char * * argv) if (!args.command) throw UsageError("no subcommand specified"); - if (args.command->first != "repl" - && args.command->first != "doctor" - && args.command->first != "upgrade-nix") - experimentalFeatureSettings.require(Xp::NixCommand); + experimentalFeatureSettings.require( + args.command->second->experimentalFeature()); if (args.useNet && !haveInternet()) { warn("you don't have Internet access; disabling some network-dependent features"); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 7aa8774e9..bb14f3f99 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -12,6 +12,14 @@ struct CmdRepl : RawInstallablesCommand evalSettings.pureEval = false; } + /** + * This command is stable before the others + */ + std::optional experimentalFeature() override + { + return std::nullopt; + } + std::vector files; Strings getDefaultFlakeAttrPaths() override diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 17796d6b8..2295d86d0 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -32,6 +32,14 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand }); } + /** + * This command is stable before the others + */ + std::optional experimentalFeature() override + { + return std::nullopt; + } + std::string description() override { return "upgrade Nix to the stable version declared in Nixpkgs"; diff --git a/tests/experimental-features.sh b/tests/experimental-features.sh index f718585f3..a4d55f5f4 100644 --- a/tests/experimental-features.sh +++ b/tests/experimental-features.sh @@ -15,15 +15,20 @@ function both_ways { # Simple case, the configuration effects the running command both_ways show-config -# Complicated case, earlier args effect later args +# Skipping for now, because we actually *do* want these to show up in +# the manual, just be marked experimental. Will reenable once the manual +# generation takes advantage of the JSON metadata on this. -both_ways store gc --help +# both_ways store gc --help expect 1 nix --experimental-features 'nix-command' show-config --flake-registry 'https://no' nix --experimental-features 'nix-command flakes' show-config --flake-registry 'https://no' -# Double check this is stable +# Double check these are stable nix --experimental-features '' --help +nix --experimental-features '' doctor --help +nix --experimental-features '' repl --help +nix --experimental-features '' upgrade-nix --help # These 3 arguments are currently given to all commands, which is wrong (as not # all care). To deal with fixing later, we simply make them require the From 62cacc371f7e0d5745c28a8e1a77ddbc834df31c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 3 Apr 2023 17:59:41 +0200 Subject: [PATCH 19/75] Fix BuildResult.toString() for NoSubstituters --- src/libstore/build-result.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index e50ddbb8c..a12c599d9 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -53,6 +53,7 @@ struct BuildResult case LogLimitExceeded: return "LogLimitExceeded"; case NotDeterministic: return "NotDeterministic"; case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid"; + case NoSubstituters: return "NoSubstituters"; default: return "Unknown"; }; }(); From ed7885017c25940f571dc58e94ca47fa84780f9d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 3 Apr 2023 18:01:30 +0200 Subject: [PATCH 20/75] Fix systemd logging for lvlNotice: eqv to lvlInfo, not lvlVomit --- src/libutil/logging.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 7cac75ce1..28b385461 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -65,7 +65,7 @@ public: switch (lvl) { case lvlError: c = '3'; break; case lvlWarn: c = '4'; break; - case lvlInfo: c = '5'; break; + case lvlNotice: case lvlInfo: c = '5'; break; case lvlTalkative: case lvlChatty: c = '6'; break; default: c = '7'; } From 3dac4c7874b876dc28d522aa4eddd8b4deb64378 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 3 Apr 2023 18:03:20 +0200 Subject: [PATCH 21/75] Add explicit case statements where -Wswitch-enum would report them --- src/libcmd/repl.cc | 2 ++ src/libexpr/eval.cc | 1 + src/libstore/nar-accessor.cc | 1 + src/libutil/logging.cc | 3 ++- src/nix-store/nix-store.cc | 2 +- 5 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index e3afb1531..e75ac2cf4 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -1024,6 +1024,8 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m str << v.fpoint; break; + case nThunk: + case nExternal: default: str << ANSI_RED "«unknown»" ANSI_NORMAL; break; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 22337a3ff..a5e9636ae 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2337,6 +2337,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v case nFloat: return v1.fpoint == v2.fpoint; + case nThunk: // Must not be left by forceValue default: error("cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).debugThrow(); } diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 9a0003588..f0dfcb19b 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -275,6 +275,7 @@ json listNar(ref accessor, const Path & path, bool recurse) obj["type"] = "symlink"; obj["target"] = accessor->readLink(path); break; + case FSAccessor::Type::tMissing: default: throw Error("path '%s' does not exist in NAR", path); } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 28b385461..741e1d1f5 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -67,7 +67,8 @@ public: case lvlWarn: c = '4'; break; case lvlNotice: case lvlInfo: c = '5'; break; case lvlTalkative: case lvlChatty: c = '6'; break; - default: c = '7'; + case lvlDebug: case lvlVomit: c = '7'; + default: c = '7'; break; } prefix = std::string("<") + c + ">"; } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index a62cb874f..2d784376d 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -443,7 +443,7 @@ static void opQuery(Strings opFlags, Strings opArgs) break; } - default: + default: case qDefault: abort(); } } From 9470ee877dabcf133469467a23649066e2bcce5c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 3 Apr 2023 18:15:12 +0200 Subject: [PATCH 22/75] Allow open switch-enum in 5 places --- src/libexpr/eval.cc | 8 ++++++++ src/libexpr/flake/flake.cc | 4 ++++ src/libexpr/primops.cc | 4 ++++ src/libstore/filetransfer.cc | 5 +++++ 4 files changed, 21 insertions(+) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index a5e9636ae..7f2065656 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -239,6 +239,9 @@ std::string_view showType(ValueType type) std::string showType(const Value & v) { + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (v.internalType) { case tString: return v.string.context ? "a string with context" : "a string"; case tPrimOp: @@ -252,16 +255,21 @@ std::string showType(const Value & v) default: return std::string(showType(v.type())); } + #pragma GCC diagnostic pop } PosIdx Value::determinePos(const PosIdx pos) const { + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (internalType) { case tAttrs: return attrs->pos; case tLambda: return lambda.fun->pos; case tApp: return app.left->determinePos(pos); default: return pos; } + #pragma GCC diagnostic pop } bool Value::isTrivial() const diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 81e94848a..7c6cd091d 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -125,6 +125,9 @@ static FlakeInput parseFlakeInput(EvalState & state, follows.insert(follows.begin(), lockRootPath.begin(), lockRootPath.end()); input.follows = follows; } else { + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (attr.value->type()) { case nString: attrs.emplace(state.symbols[attr.name], attr.value->string.s); @@ -139,6 +142,7 @@ static FlakeInput parseFlakeInput(EvalState & state, throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", state.symbols[attr.name], showType(*attr.value)); } + #pragma GCC diagnostic pop } } catch (Error & e) { e.addTrace( diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 72faeada8..f1bce2fb6 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -577,6 +577,9 @@ struct CompareValues return v1->integer < v2->fpoint; if (v1->type() != v2->type()) state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow(); + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (v1->type()) { case nInt: return v1->integer < v2->integer; @@ -599,6 +602,7 @@ struct CompareValues } default: state.error("cannot compare %s with %s; values of that type are incomparable", showType(*v1), showType(*v2)).debugThrow(); + #pragma GCC diagnostic pop } } catch (Error & e) { if (!errorCtx.empty()) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 1ba399a29..2346accbe 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -407,6 +407,10 @@ struct curlFileTransfer : public FileTransfer err = Misc; } else { // Don't bother retrying on certain cURL errors either + + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (code) { case CURLE_FAILED_INIT: case CURLE_URL_MALFORMAT: @@ -427,6 +431,7 @@ struct curlFileTransfer : public FileTransfer default: // Shut up warnings break; } + #pragma GCC diagnostic pop } attempt++; From fba7be80eb0fab54b570623d3991739cf3da0759 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 3 Apr 2023 18:11:42 +0200 Subject: [PATCH 23/75] Enable -Werror=switch-enum switch statements must now match all enum values or disable the warning. Explicit is good. This has helped us find two bugs, after solving another one by debugging. From now on, adding to an enum will raise errors where they are not explicitly handled, which is good for productivity, and helps us decide the correct behavior in all usages. Notably still excluded from this though are the cases where the warning is disabled by local pragmas. fromTOML.cc did not build despite a top-level pragma, so I've had to resort to a makefile solution for that. --- local.mk | 4 +++- mk/patterns.mk | 4 ++-- src/libexpr/local.mk | 2 ++ src/libutil/logging.cc | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/local.mk b/local.mk index 6a7074e8e..6951c179e 100644 --- a/local.mk +++ b/local.mk @@ -1,6 +1,8 @@ clean-files += Makefile.config -GLOBAL_CXXFLAGS += -Wno-deprecated-declarations +GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch +# Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. +ERROR_SWITCH_ENUM = -Werror=switch-enum $(foreach i, config.h $(wildcard src/lib*/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) diff --git a/mk/patterns.mk b/mk/patterns.mk index 86a724806..c81150260 100644 --- a/mk/patterns.mk +++ b/mk/patterns.mk @@ -1,10 +1,10 @@ $(buildprefix)%.o: %.cc @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP $(buildprefix)%.o: %.cpp @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep, $@) -MP $(buildprefix)%.o: %.c @mkdir -p "$(dir $@)" diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 2171e769b..d243b9cec 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -46,3 +46,5 @@ $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh + +src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 741e1d1f5..5a2dd99af 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -68,7 +68,7 @@ public: case lvlNotice: case lvlInfo: c = '5'; break; case lvlTalkative: case lvlChatty: c = '6'; break; case lvlDebug: case lvlVomit: c = '7'; - default: c = '7'; break; + default: c = '7'; break; // should not happen, and missing enum case is reported by -Werror=switch-enum } prefix = std::string("<") + c + ">"; } From bf2c5c3958c258584ac66519124d05ccf446aaeb Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 3 Apr 2023 18:38:11 +0200 Subject: [PATCH 24/75] nix-store.cc: Refactor, remove qDefault --- src/nix-store/nix-store.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 2d784376d..7035e6a7b 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -277,17 +277,17 @@ static void printTree(const StorePath & path, static void opQuery(Strings opFlags, Strings opArgs) { enum QueryType - { qDefault, qOutputs, qRequisites, qReferences, qReferrers + { qOutputs, qRequisites, qReferences, qReferrers , qReferrersClosure, qDeriver, qBinding, qHash, qSize , qTree, qGraph, qGraphML, qResolve, qRoots }; - QueryType query = qDefault; + std::optional query; bool useOutput = false; bool includeOutputs = false; bool forceRealise = false; std::string bindingName; for (auto & i : opFlags) { - QueryType prev = query; + std::optional prev = query; if (i == "--outputs") query = qOutputs; else if (i == "--requisites" || i == "-R") query = qRequisites; else if (i == "--references") query = qReferences; @@ -312,15 +312,15 @@ static void opQuery(Strings opFlags, Strings opArgs) else if (i == "--force-realise" || i == "--force-realize" || i == "-f") forceRealise = true; else if (i == "--include-outputs") includeOutputs = true; else throw UsageError("unknown flag '%1%'", i); - if (prev != qDefault && prev != query) + if (prev && prev != query) throw UsageError("query type '%1%' conflicts with earlier flag", i); } - if (query == qDefault) query = qOutputs; + if (!query) query = qOutputs; RunPager pager; - switch (query) { + switch (*query) { case qOutputs: { for (auto & i : opArgs) { @@ -443,7 +443,7 @@ static void opQuery(Strings opFlags, Strings opArgs) break; } - default: case qDefault: + default: abort(); } } From 7076d37047d73e82cb51c315fede4906a76f57b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 3 Apr 2023 20:47:21 +0200 Subject: [PATCH 25/75] Fix the flaky `nix-profile` test Exclude the `error (ignored)` from the message that is checked by the install conflict test. Fix https://github.com/NixOS/nix/issues/8140 --- tests/nix-profile.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index 652e8a8f2..4ef5b484a 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -144,6 +144,7 @@ expect 1 nix profile install $flake2Dir diff -u <( nix --offline profile install $flake2Dir 2>&1 1> /dev/null \ | grep -vE "^warning: " \ + | grep -vE "^error \(ignored\): " \ || true ) <(cat << EOF error: An existing package already provides the following file: From bdeeffff967a513b6a35165bb476d24491a03e23 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Apr 2023 19:16:10 -0400 Subject: [PATCH 26/75] Apply suggestions from code review Co-authored-by: Valentin Gagarin --- src/libutil/experimental-features.cc | 41 ++++++++++++++-------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 305b0cb11..a64e9715a 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -17,11 +17,11 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::CaDerivations, .name = "ca-derivations", .description = R"( - Allows derivations to be content-addressed in order to prevent rebuilds + Allow derivations to be content-addressed in order to prevent rebuilds when changes to the derivation do not result in changes to the derivation's output. See - [__contentAddressed](../language/advanced-attributes.md#adv-attr-__contentAddressed) - for more info. + [__contentAddressed](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed) + for details. )", }, { @@ -37,18 +37,16 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::Flakes, .name = "flakes", .description = R"( - Allows for derivations to be packaged in flakes. See the manual entry for - [`nix flake`](../command-ref/new-cli/nix3-flake.md) or this [detailed - introduction](https://www.tweag.io/blog/2020-05-25-flakes/) for more info. + Enable flakes. See the manual entry for + [`nix flake`](../command-ref/new-cli/nix3-flake.md) for details. )", }, { .tag = Xp::NixCommand, .name = "nix-command", .description = R"( - Allows the usage of the new `nix` CLI subcommands, such as `nix build`, `nix - develop`, `nix run`, etc. See the manual for - [`nix`](../command-ref/new-cli/nix.md) for more info. + Enable the new `nix` subcommands. See the manual on + [`nix`](@docroot@/command-ref/new-cli/nix.md) for details. )", }, { @@ -73,17 +71,14 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::FetchClosure, .name = "fetch-closure", .description = R"( - Enables the use of the `fetchClosure` function in the standard library. See - the docs for [`fetchClosure`](../language/builtins.md#builtins-fetchClosure) - for more info. + Enable the use of the [`fetchClosure`](@docroot@/language/builtins.md#builtins-fetchClosure) built-in function in the Nix language. )", }, { .tag = Xp::ReplFlake, .name = "repl-flake", .description = R"( - Allows the user to enter a Nix REPL within a flake, e.g. `nix repl nixpkgs` - or `nix repl .#foo`. + Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands. )", }, { @@ -91,7 +86,7 @@ constexpr std::array xpFeatureDetails = {{ .name = "auto-allocate-uids", .description = R"( Allows Nix to automatically pick UIDs for builds, rather than creating - `nixbld*` user accounts. See [here](#conf-auto-allocate-uids) for more info. + `nixbld*` user accounts. See the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting for details. )", }, { @@ -99,15 +94,15 @@ constexpr std::array xpFeatureDetails = {{ .name = "cgroups", .description = R"( Allows Nix to execute builds inside cgroups. See - [`use-cgroups`](#conf-use-cgroups) for more info. + the [`use-cgroups`](#conf-use-cgroups) setting for details. )", }, { .tag = Xp::DiscardReferences, .name = "discard-references", .description = R"( - Enables the use of the `unsafeDiscardReferences` attribute in derivations - that use structured attributes. This disables scanning of outputs for + Allow the use of the [`unsafeDiscardReferences`](@docroot@/language/advanced-attributes.html#adv-attr-unsafeDiscardReferences) attribute in derivations + that use [structured attributes](@docroot@/language/advanced-attributes.html#adv-attr-structuredAttrs). This disables scanning of outputs for runtime dependencies. )", }, @@ -147,8 +142,14 @@ std::string_view showExperimentalFeature(const ExperimentalFeature tag) std::string getExperimentalFeaturesList() { std::string experimentalFeaturesList = R"( - Experimental Nix features to enable. - Current experimental features are the following: + Experimental features that can be enabled. + + Example: + + ``` + experimental-features = nix-command flakes + + Experimental features available: )"; From f912ab89cc78a57513f7090ecfc799adb1b464a6 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 28 Mar 2023 11:51:47 +0200 Subject: [PATCH 27/75] only triage non-trivial pull requests the team perceives triaging literally everything as too much overhead, and there is a desire to allow for more parallel action. --- maintainers/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/maintainers/README.md b/maintainers/README.md index 95f58acaa..298c1d5da 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -79,7 +79,8 @@ Items on the board progress through the following states: If there is disagreement on the general idea behind an issue or pull request, it is moved to _To discuss_, otherwise to _In review_. - To ensure process quality and reliability, all pull requests must be triaged before merging. + To ensure process quality and reliability, all non-trivial pull requests must be triaged before merging. + What constitutes a trivial pull request is up to maintainers' judgement. - To discuss From 53d0836347ea262884658036f97bf19ecbdc5c26 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Apr 2023 22:57:11 -0400 Subject: [PATCH 28/75] Assemble experimental feature docs outside of Nix itself Instead of constructing a markdown list in C++ (which involved all sorts of nasty string literals), export some JSON and assemble it with the manual build system. Besides following the precedent set with other dumped data, this is a better separate of content and presentation; if we decide for example we want to display this information in a different way, or in a different section of the manual, it will become much easier to do so. --- .gitignore | 2 ++ doc/manual/generate-xp-features.nix | 11 ++++++++++ doc/manual/local.mk | 13 +++++++++-- doc/manual/utils.nix | 6 +++--- src/libutil/config.hh | 17 +++++++++++++-- src/libutil/experimental-features.cc | 32 ++++++---------------------- src/libutil/experimental-features.hh | 6 ++---- src/nix/main.cc | 5 +++++ 8 files changed, 55 insertions(+), 37 deletions(-) create mode 100644 doc/manual/generate-xp-features.nix diff --git a/.gitignore b/.gitignore index e326966d6..53442751f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,9 +19,11 @@ perl/Makefile.config /doc/manual/nix.json /doc/manual/conf-file.json /doc/manual/builtins.json +/doc/manual/xp-features.json /doc/manual/src/SUMMARY.md /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md +/doc/manual/src/command-ref/experimental-features.md /doc/manual/src/language/builtins.md # /scripts/ diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix new file mode 100644 index 000000000..db1ba6092 --- /dev/null +++ b/doc/manual/generate-xp-features.nix @@ -0,0 +1,11 @@ +with builtins; +with import ./utils.nix; + +let + showExperimentalFeature = name: doc: + squash '' + - [`${name}`](#xp-feature-${name}) + + ${indent " " doc} + ''; +in xps: indent " " (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index df941d460..0d4b9d640 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -88,12 +88,12 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli @cp $< $@ @$(call process-includes,$@,$@) -$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix +$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)' @mv $@.tmp $@ -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix +$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ @@ -106,6 +106,15 @@ $(d)/conf-file.json: $(bindir)/nix $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ +$(d)/src/command-ref/experimental-features.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix + @rm -rf $@ $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' + @mv $@.tmp $@ + +$(d)/xp-features.json: $(bindir)/nix + $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp + @mv $@.tmp $@ + $(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix @cat doc/manual/src/language/builtins-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index 5eacce0dd..f78e6bb02 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -74,10 +74,10 @@ rec { if aliases == [] then "" else "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; - indent = prefix: s: - concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s)); - in result; + indent = prefix: s: + concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s)); + showSettings = args: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting args) settingsInfo)); } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index a001056f7..8b0fe6555 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -371,8 +371,21 @@ extern GlobalConfig globalConfig; struct ExperimentalFeatureSettings : Config { - Setting> experimentalFeatures{this, {}, "experimental-features", - getExperimentalFeaturesList()}; + Setting> experimentalFeatures{ + this, {}, "experimental-features", + R"( + Experimental features that are enabled. + + Example: + + ``` + experimental-features = nix-command flakes + ``` + + Experimental features available: + + {{#include experimental-features.md}} + )"}; /** * Check whether the given experimental feature is enabled. diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index a64e9715a..010ab1d68 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -140,32 +140,12 @@ std::string_view showExperimentalFeature(const ExperimentalFeature tag) return xpFeatureDetails[(size_t)tag].name; } -std::string getExperimentalFeaturesList() { - std::string experimentalFeaturesList = R"( - Experimental features that can be enabled. - - Example: - - ``` - experimental-features = nix-command flakes - - Experimental features available: - -)"; - - for (auto & xpFeature : xpFeatureDetails) { - experimentalFeaturesList += std::string {} - /* length of this first string must be 12, matching the indent of - the descriptions in the xpFeatureDetails literal. FIXME compute - markdown in a less hacky way. */ - + " - " - + "`" + xpFeature.name + "`" - + "\n" - + xpFeature.description - + "\n\n"; - } - - return experimentalFeaturesList; +nlohmann::json documentExperimentalFeatures() { + StringMap res; + for (auto & xpFeature : xpFeatureDetails) + res[std::string { xpFeature.name }] = + trim(stripIndentation(xpFeature.description)); + return (nlohmann::json) res; } std::set parseFeatures(const std::set & rawFeatures) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 3c479dbd9..8ef66263a 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -51,11 +51,9 @@ std::string_view showExperimentalFeature(const ExperimentalFeature); /** * Compute the documentation of all experimental features. * - * This a markdown bulleted list where each item is first (a) the - * experimental feature flag name in backticks, and then (b) the - * description of the experimental feature. + * See `doc/manual` for how this information is used. */ -std::string getExperimentalFeaturesList(); +nlohmann::json documentExperimentalFeatures(); /** * Shorthand for `str << showExperimentalFeature(feature)`. diff --git a/src/nix/main.cc b/src/nix/main.cc index 4d4164333..62b1f98d1 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -374,6 +374,11 @@ void mainWrapped(int argc, char * * argv) return; } + if (argc == 2 && std::string(argv[1]) == "__dump-xp-features") { + logger->cout(documentExperimentalFeatures().dump()); + return; + } + Finally printCompletions([&]() { if (completions) { From ed5d0c13620a54c8db9715e98f1a85734c1b0ad3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 5 Apr 2023 10:48:59 +0200 Subject: [PATCH 29/75] add anchor to 'local store' --- doc/manual/src/glossary.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index b56d857d1..0888954f9 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -54,7 +54,7 @@ invoked, the Nix store can be referred to as a "_local_" or a "_remote_" one: - + A *local store* exists on the filesystem of + + A [local store]{#gloss-local-store} exists on the filesystem of the machine where Nix is invoked. You can use other local stores by passing the `--store` flag to the `nix` command. Local stores can be used for building derivations. @@ -65,9 +65,10 @@ served by the `nix-serve` Perl script. [store]: #gloss-store + [local store]: #gloss-local-store - [chroot store]{#gloss-chroot-store}\ - A local store whose canonical path is anything other than `/nix/store`. + A [local store] whose canonical path is anything other than `/nix/store`. - [binary cache]{#gloss-binary-cache}\ A *binary cache* is a Nix store which uses a different format: its @@ -175,7 +176,7 @@ - [validity]{#gloss-validity}\ A store path is valid if all [store object]s in its [closure] can be read from the [store]. - For a local store, this means: + For a [local store], this means: - The store path leads to an existing [store object] in that [store]. - The store path is listed in the Nix database as being valid. - All paths in the store path's [closure] are valid. From cab52660922e96a54780a87b4a7511eee2598d2a Mon Sep 17 00:00:00 2001 From: "Mike (stew) O'Connor" Date: Wed, 5 Apr 2023 10:50:34 +0200 Subject: [PATCH 30/75] add a definition of the nix database to the glossary Co-authored-by: Valentin Gagarin --- doc/manual/src/glossary.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 0888954f9..fe5920746 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -73,10 +73,9 @@ - [binary cache]{#gloss-binary-cache}\ A *binary cache* is a Nix store which uses a different format: its metadata and signatures are kept in `.narinfo` files rather than in a - Nix database. This different format simplifies serving store objects - over the network, but cannot host builds. Examples of binary caches - include S3 buckets and the [NixOS binary - cache](https://cache.nixos.org). + [Nix database]. This different format simplifies serving store objects + over the network, but cannot host builds. Examples of binary caches + include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). - [store path]{#gloss-store-path}\ The location of a [store object] in the file system, i.e., an @@ -109,7 +108,7 @@ [fixed-output derivations](#gloss-fixed-output-derivation). - [substitute]{#gloss-substitute}\ - A substitute is a command invocation stored in the Nix database that + A substitute is a command invocation stored in the [Nix database] that describes how to build a store object, bypassing the normal build mechanism (i.e., derivations). Typically, the substitute builds the store object by downloading a pre-built version of the store object @@ -128,6 +127,14 @@ builder can rely on external inputs such as the network or the system time) but the Nix model assumes it. + - Nix database{#gloss-nix-database}\ + An SQlite database to track [reference]s between [store object]s. + This is an implementation detail of the [local store]. + + Default location: `/nix/var/nix/db`. + + [Nix database]: #gloss-nix-database + - [Nix expression]{#gloss-nix-expression}\ A high-level description of software packages and compositions thereof. Deploying software using Nix entails writing Nix @@ -178,7 +185,7 @@ For a [local store], this means: - The store path leads to an existing [store object] in that [store]. - - The store path is listed in the Nix database as being valid. + - The store path is listed in the [Nix database] as being valid. - All paths in the store path's [closure] are valid. [validity]: #gloss-validity From 1ac10808baf878267eba8e758d985f24349f18c5 Mon Sep 17 00:00:00 2001 From: milahu Date: Thu, 16 Feb 2023 10:51:54 +0100 Subject: [PATCH 31/75] docs: fix nix-shell commands --- doc/manual/src/command-ref/nix-shell.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 840bccd25..6e5a198a8 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -118,7 +118,8 @@ shell in which to build it: ```console $ nix-shell '' -A pan [nix-shell]$ eval ${unpackPhase:-unpackPhase} -[nix-shell]$ cd pan-* +[nix-shell]$ cd $sourceRoot +[nix-shell]$ eval ${patchPhase:-patchPhase} [nix-shell]$ eval ${configurePhase:-configurePhase} [nix-shell]$ eval ${buildPhase:-buildPhase} [nix-shell]$ ./pan/gui/pan From faefaac87568fe3ff04f4c2624f293763de2ae1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 5 Apr 2023 17:20:04 +0200 Subject: [PATCH 32/75] Explicitely define `LockFile::operator!=` It should be syntethised in terms of `operator==`, but the GCC version used on aarch64-linux doesn't implement that (see https://hydra.nixos.org/build/214848896=. So explicitely define it. Fix https://github.com/NixOS/nix/issues/8159 --- src/libexpr/flake/lockfile.cc | 5 +++++ src/libexpr/flake/lockfile.hh | 3 +++ 2 files changed, 8 insertions(+) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index a74e68c9c..ba2fd46f0 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -234,6 +234,11 @@ bool LockFile::operator ==(const LockFile & other) const return toJSON() == other.toJSON(); } +bool LockFile::operator !=(const LockFile & other) const +{ + return !(*this == other); +} + InputPath parseInputPath(std::string_view s) { InputPath path; diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 6512509c5..4616a77f6 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -67,6 +67,9 @@ struct LockFile std::optional isUnlocked() const; bool operator ==(const LockFile & other) const; + // Needed for old gcc versions that don't syntethise it (like gcc 8.2.2 + // that is still the default on aarch64-linux) + bool operator !=(const LockFile & other) const; std::shared_ptr findInput(const InputPath & path); From 91856396317995aa38dc7244357596b8de27f937 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Thu, 6 Apr 2023 13:25:22 +0200 Subject: [PATCH 33/75] =?UTF-8?q?Document=20the=20concept=20of=20=E2=80=9C?= =?UTF-8?q?experimental=20feature=E2=80=9D=20(#5930)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a page explaining what “experimental features” are, when and how they should be used Co-authored-by: Valentin Gagarin Co-authored-by: Eelco Dolstra Co-authored-by: John Ericson --- doc/manual/src/SUMMARY.md.in | 1 + .../src/contributing/experimental-features.md | 91 +++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 doc/manual/src/contributing/experimental-features.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 4b654567f..5bf274550 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -95,6 +95,7 @@ - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) + - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/contributing/experimental-features.md new file mode 100644 index 000000000..f1db22751 --- /dev/null +++ b/doc/manual/src/contributing/experimental-features.md @@ -0,0 +1,91 @@ +This section describes the notion of *experimental features*, and how it fits into the big picture of the development of Nix. + +# What are experimental features? + +Experimental features are considered unstable, which means that they can be changed or removed at any time. +Users must explicitly enable them by toggling the associated [experimental feature flags](@docroot@/command-ref/conf-file.md#conf-experimental-features). +This allows accessing unstable functionality without unwittingly relying on it. + +Experimental feature flags were first introduced in [Nix 2.4](@docroot@/release-notes/rl-2.4.md). +Before that, Nix did have experimental features, but they were not guarded by flags and were merely documented as unstable. +This was a source of confusion and controversy. + +# When should a new feature be marked experimental? + +A change in the Nix codebase should be guarded by an experimental feature flag if it is considered likely to be reverted or adapted in a backwards-incompatible manner after gathering more experience with it in practice. + +Examples: + +- Changes to the Nix language, such as new built-ins, syntactic or semantic changes, etc. +- Changes to the command-line interface + +# Lifecycle of an experimental feature + +Experimental features have to be treated on a case-by-case basis. +However, the standard workflow for an experimental feature is as follows: + +- A new feature is implemented in a *pull request* + - It is guarded by an experimental feature flag that is disabled by default +- The pull request is merged, the *experimental* feature ends up in a release + - Using the feature requires explicitly enabling it, signifying awareness of the potential risks + - Being experimental, the feature can still be changed arbitrarily +- The feature can be *removed* + - The associated experimental feature flag is also removed +- The feature can be declared *stable* + - The associated experimental feature flag is removed + - There should be enough evidence of users having tried the feature, such as feedback, fixed bugs, demonstrations of how it is put to use + - Maintainers must feel confident that: + - The feature is designed and implemented sensibly, that it is fit for purpose + - Potential interactions are well-understood + - Stabilising the feature will not incur an outsized maintenance burden in the future + +The following diagram illustrates the process: + +``` + .------. + | idea | + '------' + | + discussion, design, implementation + | + | .-------. + | | | + v v | + .--------------. review + | pull request | | + '--------------' | + | ^ | | + | | '-------' + .---' '----. + | | + merge user feedback, + | (breaking) changes + | | + '---. .----' + | | + v | + +--------------+ + .---| experimental |----. + | +--------------+ | + | | +decision to stabilise decision against + | keeping the feature + | | + v v + +--------+ +---------+ + | stable | | removed | + +--------+ +---------+ +``` + +# Relation to the RFC process + +Experimental features and [RFCs](https://github.com/NixOS/rfcs/) both allow approaching substantial changes while minimizing the risk. +However they serve different purposes: + +- An experimental feature enables developers to iterate on and deliver a new idea without committing to it or requiring a costly long-running fork. + It is primarily an issue of *implementation*, targeting Nix developers and early testers. +- The goal of an RFC is to make explicit all the implications of a change: + Explain why it is wanted, which new use-cases it enables, which interface changes it requires, etc. + It is primarily an issue of *design* and *communication*, targeting the broader community. + +This means that experimental features and RFCs are orthogonal mechanisms, and can be used independently or together as needed. From 8a7790f46aaf608bd7ed4ec6042b99bd36d7118e Mon Sep 17 00:00:00 2001 From: Noah Snelson Date: Wed, 5 Apr 2023 20:10:11 -0700 Subject: [PATCH 34/75] Expand documentation for `experimental-features` Adds examples and additional information to the `impure-derivations`, `recursive-nix`, and `no-url-literals` experimental feature documentation. --- src/libutil/experimental-features.cc | 111 +++++++++++++++++++++++---- 1 file changed, 97 insertions(+), 14 deletions(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 010ab1d68..b7879ff69 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -17,9 +17,9 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::CaDerivations, .name = "ca-derivations", .description = R"( - Allow derivations to be content-addressed in order to prevent rebuilds - when changes to the derivation do not result in changes to the - derivation's output. See + Allow derivations to be content-addressed in order to prevent + rebuilds when changes to the derivation do not result in changes to + the derivation's output. See [__contentAddressed](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed) for details. )", @@ -28,17 +28,36 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::ImpureDerivations, .name = "impure-derivations", .description = R"( - Allows derivations to produce non-fixed outputs by setting the `__impure` - derivation attribute to `true`. See [these release - notes](../release-notes/rl-2.8.md) for an example. + Allow derivations to produce non-fixed outputs by setting the + `__impure` derivation attribute to `true`. An impure derivation can + have differing outputs each time it is built. + + Example: + + ``` + derivation { + name = "impure"; + builder = /bin/sh; + __impure = true; # mark this derivation as impure + args = [ "-c" "read -n 10 random < /dev/random; echo $random > $out" ]; + system = builtins.currentSystem; + } + ``` + + Each time this derivation is built, it can produce a different + output (as the builder outputs random bytes to `$out`). Impure + derivations also have access to the network, and only fixed-output + or other impure derivations can rely on impure derivations. Finally, + an impure derivation cannot also be + [content-addressed](#xp-feature-ca-derivations). )", }, { .tag = Xp::Flakes, .name = "flakes", .description = R"( - Enable flakes. See the manual entry for - [`nix flake`](../command-ref/new-cli/nix3-flake.md) for details. + Enable flakes. See the manual entry for [`nix + flake`](../command-ref/new-cli/nix3-flake.md) for details. )", }, { @@ -53,18 +72,82 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::RecursiveNix, .name = "recursive-nix", .description = R"( - Allow Nix derivations to call Nix in order to recursively build derivations. - See [this - commit](https://github.com/edolstra/nix/commit/1a27aa7d64ffe6fc36cfca4d82bdf51c4d8cf717) - for more info. + Allow derivation builders to call Nix, and thus build derivations + recursively. + + Example: + + ``` + with import {}; + + runCommand "foo" + { + buildInputs = [ nix jq ]; + NIX_PATH = "nixpkgs=${}"; + } + '' + hello=$(nix-build -E '(import {}).hello.overrideDerivation (args: { name = "recursive-hello"; })') + + mkdir -p $out/bin + ln -s $hello/bin/hello $out/bin/hello + '' + ``` + + An important restriction on recursive builders is disallowing + arbitrary substitutions. For example, running + + ``` + nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10 + ``` + + in the above `runCommand` script would be disallowed, as this could + lead to derivations with hidden dependencies or breaking + reproducibility by relying on the current state of the Nix store. An + exception would be if + `/nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10` were + already in the build inputs or built by a previous recursive Nix + call. )", }, { .tag = Xp::NoUrlLiterals, .name = "no-url-literals", .description = R"( - Disallows unquoted URLs as part of the Nix language syntax. See [RFC - 45](https://github.com/NixOS/rfcs/pull/45) for more info. + Disallow unquoted URLs as part of the Nix language syntax. The Nix + language allows for URL literals, like so: + + ``` + $ nix repl + Welcome to Nix 2.15.0. Type :? for help. + + nix-repl> http://foo + "http://foo" + ``` + + But enabling this experimental feature will cause the Nix parser to + throw an error when encountering a URL literal: + + ``` + $ nix repl --extra-experimental-features 'no-url-literals' + Welcome to Nix 2.15.0. Type :? for help. + + nix-repl> http://foo + error: URL literals are disabled + + at «string»:1:1: + + 1| http://bar + | ^ + + ``` + + While this is currently an experimental feature, unquoted URLs are + being deprecated and their usage is discouraged. + + The reason is that, as opposed to path literals, URLs have no + special properties that distinguish them from regular strings, URLs + containing parameters have to be quoted anyway, and unquoted URLs + may confuse external tooling. )", }, { From e399cb49c245de909417cfbe39a4c64fe0632653 Mon Sep 17 00:00:00 2001 From: Noah Snelson Date: Thu, 6 Apr 2023 15:02:19 -0700 Subject: [PATCH 35/75] Fix typo in `no-url-literals` experimental feature docs Co-authored-by: Valentin Gagarin --- src/libutil/experimental-features.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index b7879ff69..aa3367639 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -136,7 +136,7 @@ constexpr std::array xpFeatureDetails = {{ at «string»:1:1: - 1| http://bar + 1| http://foo | ^ ``` From 6c4049b38a2e694d904b4bc4f3525efe68f2f611 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 6 Apr 2023 11:37:02 -0400 Subject: [PATCH 36/75] Link the new general documentation on xp features on the setting --- src/libutil/config.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 8b0fe6555..b21f8a422 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -385,6 +385,8 @@ struct ExperimentalFeatureSettings : Config { Experimental features available: {{#include experimental-features.md}} + + Experimental features are [further documented](@docroot@/contributing/experimental-features.md) in the contribution guide. )"}; /** From bc192a95ef4c777d9ca0ad025e0fb2b7f18879f7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 6 Apr 2023 12:24:11 -0400 Subject: [PATCH 37/75] Describe active experimental features in the contributing guide They are put in the manual separate pages under the new overarching description of experimental features. The settings page just lists the valid experimental feature names (so people know what a valid setting entry looks like), with links to those pages. It doesn't attempt to describe each experimental feature as that is too much information for the configuration settings section. --- .gitignore | 3 ++- doc/manual/generate-xp-features-shortlist.nix | 9 ++++++++ doc/manual/generate-xp-features.nix | 22 ++++++++++++++----- doc/manual/local.mk | 14 ++++++++---- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/utils.nix | 3 +++ src/libutil/config.hh | 2 +- src/libutil/experimental-features.cc | 2 +- 8 files changed, 43 insertions(+), 13 deletions(-) create mode 100644 doc/manual/generate-xp-features-shortlist.nix diff --git a/.gitignore b/.gitignore index 53442751f..ffaf52be8 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,8 @@ perl/Makefile.config /doc/manual/src/SUMMARY.md /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md -/doc/manual/src/command-ref/experimental-features.md +/doc/manual/src/command-ref/experimental-features-shortlist.md +/doc/manual/src/contributing/experimental-features /doc/manual/src/language/builtins.md # /scripts/ diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix new file mode 100644 index 000000000..b2095bc27 --- /dev/null +++ b/doc/manual/generate-xp-features-shortlist.nix @@ -0,0 +1,9 @@ +with builtins; +with import ./utils.nix; + +let + showExperimentalFeature = name: doc: + '' + - [`${name}`](@docroot@/contributing/experimental-features/${name}.md) + ''; +in xps: indent " " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix index db1ba6092..ff64edcf7 100644 --- a/doc/manual/generate-xp-features.nix +++ b/doc/manual/generate-xp-features.nix @@ -1,11 +1,21 @@ +xps: + with builtins; with import ./utils.nix; let - showExperimentalFeature = name: doc: - squash '' - - [`${name}`](#xp-feature-${name}) + makePage = { name, value }: + { + name = "${name}.md"; + inherit value; + feature = name; + }; - ${indent " " doc} - ''; -in xps: indent " " (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) + featurePages = map makePage (attrsToList xps); + + tableOfContents = let + showEntry = page: + " - [${page.feature}](contributing/experimental-features/${page.name})"; + in concatStringsSep "\n" (map showEntry featurePages) + "\n"; + +in (listToAttrs featurePages) // { "SUMMARY.md" = tableOfContents; } diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 0d4b9d640..ae55ae4a9 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -81,10 +81,11 @@ $(d)/%.8: $(d)/src/command-ref/%.md $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md @printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp @cat $^ >> $^.tmp + @$(call process-includes,$^,$^.tmp) $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-features @cp $< $@ @$(call process-includes,$@,$@) @@ -93,7 +94,7 @@ $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)' @mv $@.tmp $@ -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features.md $(bindir)/nix +$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ @@ -106,11 +107,16 @@ $(d)/conf-file.json: $(bindir)/nix $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/command-ref/experimental-features.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix +$(d)/src/contributing/experimental-features: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ +$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix + @rm -rf $@ $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))' + @mv $@.tmp $@ + $(d)/xp-features.json: $(bindir)/nix $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp @mv $@.tmp $@ @@ -154,7 +160,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-features $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 5bf274550..298644044 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -96,6 +96,7 @@ - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) - [Experimental Features](contributing/experimental-features.md) +{{#include ./contributing/experimental-features/SUMMARY.md}} - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index f78e6bb02..82544935a 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -5,6 +5,9 @@ rec { concatStrings = concatStringsSep ""; + attrsToList = a: + map (name: { inherit name; value = a.${name}; }) (builtins.attrNames a); + replaceStringsRec = from: to: string: # recursively replace occurrences of `from` with `to` within `string` # example: diff --git a/src/libutil/config.hh b/src/libutil/config.hh index b21f8a422..6baba0167 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -384,7 +384,7 @@ struct ExperimentalFeatureSettings : Config { Experimental features available: - {{#include experimental-features.md}} + {{#include experimental-features-shortlist.md}} Experimental features are [further documented](@docroot@/contributing/experimental-features.md) in the contribution guide. )"}; diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index aa3367639..1cd0dbb79 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -57,7 +57,7 @@ constexpr std::array xpFeatureDetails = {{ .name = "flakes", .description = R"( Enable flakes. See the manual entry for [`nix - flake`](../command-ref/new-cli/nix3-flake.md) for details. + flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details. )", }, { From 9207f945822764a041a485009759f0a895468e94 Mon Sep 17 00:00:00 2001 From: matthewcroughan Date: Mon, 26 Dec 2022 20:21:08 +0000 Subject: [PATCH 38/75] Add `Store::isTrustedClient()` This function returns true or false depending on whether the Nix client is trusted or not. Mostly relevant when speaking to a remote store with a daemon. We include this information in `nix ping store` and `nix doctor` Co-Authored-By: John Ericson --- doc/manual/src/release-notes/rl-next.md | 6 +++ src/libstore/build/local-derivation-goal.cc | 5 ++- src/libstore/daemon.cc | 9 ++++ src/libstore/daemon.hh | 1 - src/libstore/dummy-store.cc | 8 ++++ src/libstore/http-binary-cache-store.cc | 12 ++++++ src/libstore/legacy-ssh-store.cc | 9 ++++ src/libstore/local-binary-cache-store.cc | 4 ++ src/libstore/local-store.cc | 5 +++ src/libstore/local-store.hh | 2 + src/libstore/remote-store.cc | 46 +++++++++++++++++++++ src/libstore/remote-store.hh | 3 ++ src/libstore/s3-binary-cache-store.cc | 10 +++++ src/libstore/store-api.hh | 12 ++++++ src/libstore/worker-protocol.hh | 3 +- src/nix/doctor.cc | 13 ++++++ src/nix/ping-store.cc | 5 +++ tests/legacy-ssh-store.sh | 4 ++ tests/local-store.sh | 3 ++ tests/local.mk | 1 + tests/remote-store.sh | 11 +++++ 21 files changed, 169 insertions(+), 3 deletions(-) create mode 100644 tests/legacy-ssh-store.sh diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index ae159de8f..af1850bdc 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -39,3 +39,9 @@ * `man nix-store-` and `man nix-env-` * `nix-store --help --` and `nix-env --help --`. + +* Nix when used as a client now checks whether the store (the server) trusts the client. + (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.) + This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores. + + `nix store ping` and `nix doctor` now display this information. diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index e22180670..58d6901d3 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1415,6 +1415,9 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo virtual void addBuildLog(const StorePath & path, std::string_view log) override { unsupported("addBuildLog"); } + + std::optional isTrustedClient() override + { return NotTrusted; } }; @@ -1467,7 +1470,7 @@ void LocalDerivationGoal::startDaemon() FdSink to(remote.get()); try { daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive); + NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SysError &) { ignoreException(); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 656ad4587..63898f8dc 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1032,6 +1032,15 @@ void processConnection( if (GET_PROTOCOL_MINOR(clientVersion) >= 33) to << nixVersion; + if (GET_PROTOCOL_MINOR(clientVersion) >= 35) { + // We and the underlying store both need to trust the client for + // it to be trusted. + auto temp = trusted + ? store->isTrustedClient() + : std::optional { NotTrusted }; + worker_proto::write(*store, to, temp); + } + /* Send startup error messages to the client. */ tunnelLogger->startWork(); diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 67340a05b..1964c0d99 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -6,7 +6,6 @@ namespace nix::daemon { -enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; enum RecursiveFlag : bool { NotRecursive = false, Recursive = true }; void processConnection( diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 16e5fafd7..ae2777d0c 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -39,6 +39,14 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store callback(nullptr); } + /** + * The dummy store is incapable of *not* trusting! :) + */ + virtual std::optional isTrustedClient() override + { + return Trusted; + } + static std::set uriSchemes() { return {"dummy"}; } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 238fd1d98..85c5eed4c 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -194,6 +194,18 @@ protected: }}); } + /** + * This isn't actually necessary read only. We support "upsert" now, so we + * have a notion of authentication via HTTP POST/PUT. + * + * For now, we conservatively say we don't know. + * + * \todo try to expose our HTTP authentication status. + */ + std::optional isTrustedClient() override + { + return std::nullopt; + } }; static RegisterStoreImplementation regHttpBinaryCacheStore; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 98322b045..eb471d8fc 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -389,6 +389,15 @@ public: return conn->remoteVersion; } + /** + * The legacy ssh protocol doesn't support checking for trusted-user. + * Try using ssh-ng:// instead if you want to know. + */ + std::optional isTrustedClient() override + { + return std::nullopt; + } + void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept override // TODO: Implement diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index e5ee6fc15..5481dd762 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -95,6 +95,10 @@ protected: return paths; } + std::optional isTrustedClient() override + { + return Trusted; + } }; void LocalBinaryCacheStore::init() diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f58d90895..e0ad50f6d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1685,6 +1685,11 @@ unsigned int LocalStore::getProtocol() return PROTOCOL_VERSION; } +std::optional LocalStore::isTrustedClient() +{ + return Trusted; +} + #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1b668b6fd..6deaa051f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -204,6 +204,8 @@ public: unsigned int getProtocol() override; + std::optional isTrustedClient() override; + void vacuumDB(); void repairPath(const StorePath & path) override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d24d83117..e128c3a29 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -42,6 +42,40 @@ void write(const Store & store, Sink & out, const StorePath & storePath) } +std::optional read(const Store & store, Source & from, Phantom> _) +{ + auto temp = readNum(from); + switch (temp) { + case 0: + return std::nullopt; + case 1: + return { Trusted }; + case 2: + return { NotTrusted }; + default: + throw Error("Invalid trusted status from remote"); + } +} + +void write(const Store & store, Sink & out, const std::optional & optTrusted) +{ + if (!optTrusted) + out << (uint8_t)0; + else { + switch (*optTrusted) { + case Trusted: + out << (uint8_t)1; + break; + case NotTrusted: + out << (uint8_t)2; + break; + default: + assert(false); + }; + } +} + + ContentAddress read(const Store & store, Source & from, Phantom _) { return parseContentAddress(readString(from)); @@ -226,6 +260,13 @@ void RemoteStore::initConnection(Connection & conn) conn.daemonNixVersion = readString(conn.from); } + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) { + conn.remoteTrustsUs = worker_proto::read(*this, conn.from, Phantom> {}); + } else { + // We don't know the answer; protocol to old. + conn.remoteTrustsUs = std::nullopt; + } + auto ex = conn.processStderr(); if (ex) std::rethrow_exception(ex); } @@ -1082,6 +1123,11 @@ unsigned int RemoteStore::getProtocol() return conn->daemonVersion; } +std::optional RemoteStore::isTrustedClient() +{ + auto conn(getConnection()); + return conn->remoteTrustsUs; +} void RemoteStore::flushBadConnections() { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index f5f45f853..f220dbc20 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -144,6 +144,8 @@ public: unsigned int getProtocol() override; + std::optional isTrustedClient() override; + void flushBadConnections(); struct Connection @@ -151,6 +153,7 @@ public: FdSink to; FdSource from; unsigned int daemonVersion; + std::optional remoteTrustsUs; std::optional daemonNixVersion; std::chrono::time_point startTime; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index ac82147ee..d2fc6abaf 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -509,6 +509,16 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual return paths; } + /** + * For now, we conservatively say we don't know. + * + * \todo try to expose our S3 authentication status. + */ + std::optional isTrustedClient() override + { + return std::nullopt; + } + static std::set uriSchemes() { return {"s3"}; } }; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 4d1047380..9c6e80486 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -89,6 +89,7 @@ const uint32_t exportMagic = 0x4558494e; enum BuildMode { bmNormal, bmRepair, bmCheck }; +enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; struct BuildResult; @@ -815,6 +816,17 @@ public: return 0; }; + /** + * @return/ whether store trusts *us*. + * + * `std::nullopt` means we do not know. + * + * @note This is the opposite of the StoreConfig::isTrusted + * store setting. That is about whether *we* trust the store. + */ + virtual std::optional isTrustedClient() = 0; + + virtual Path toRealPath(const Path & storePath) { return storePath; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 697dd2b8c..c7a6f8688 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -10,7 +10,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 34) +#define PROTOCOL_VERSION (1 << 8 | 35) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -103,6 +103,7 @@ MAKE_WORKER_PROTO(, DerivedPath); MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, DrvOutput); MAKE_WORKER_PROTO(, BuildResult); +MAKE_WORKER_PROTO(, std::optional); MAKE_WORKER_PROTO(template, std::vector); MAKE_WORKER_PROTO(template, std::set); diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 354b03cf6..1aa6831d3 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -33,6 +33,10 @@ bool checkFail(const std::string & msg) { return false; } +void checkInfo(const std::string & msg) { + notice(ANSI_BLUE "[INFO] " ANSI_NORMAL + msg); +} + } struct CmdDoctor : StoreCommand @@ -63,6 +67,7 @@ struct CmdDoctor : StoreCommand success &= checkProfileRoots(store); } success &= checkStoreProtocol(store->getProtocol()); + checkTrustedUser(store); if (!success) throw Exit(2); @@ -138,6 +143,14 @@ struct CmdDoctor : StoreCommand return checkPass("Client protocol matches store protocol."); } + + void checkTrustedUser(ref store) + { + std::string_view trusted = store->isTrustedClient() + ? "trusted" + : "not trusted"; + checkInfo(fmt("You are %s by store uri: %s", trusted, store->getUri())); + } }; static auto rCmdDoctor = registerCommand("doctor"); diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 5c44510ab..ec450e8e0 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -28,15 +28,20 @@ struct CmdPingStore : StoreCommand, MixJSON store->connect(); if (auto version = store->getVersion()) notice("Version: %s", *version); + if (auto trusted = store->isTrustedClient()) + notice("Trusted: %s", *trusted); } else { nlohmann::json res; Finally printRes([&]() { logger->cout("%s", res); }); + res["url"] = store->getUri(); store->connect(); if (auto version = store->getVersion()) res["version"] = *version; + if (auto trusted = store->isTrustedClient()) + res["trusted"] = *trusted; } } }; diff --git a/tests/legacy-ssh-store.sh b/tests/legacy-ssh-store.sh new file mode 100644 index 000000000..71b716b84 --- /dev/null +++ b/tests/legacy-ssh-store.sh @@ -0,0 +1,4 @@ +source common.sh + +# Check that store ping trusted doesn't yet work with ssh:// +nix --store ssh://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e 'has("trusted") | not' diff --git a/tests/local-store.sh b/tests/local-store.sh index 0247346f1..89502f864 100644 --- a/tests/local-store.sh +++ b/tests/local-store.sh @@ -17,3 +17,6 @@ PATH2=$(nix path-info --store "$PWD/x" $CORRECT_PATH) PATH3=$(nix path-info --store "local?root=$PWD/x" $CORRECT_PATH) [ $CORRECT_PATH == $PATH3 ] + +# Ensure store ping trusted works with local store +nix --store ./x store ping --json | jq -e '.trusted' diff --git a/tests/local.mk b/tests/local.mk index ccd76eeac..7d54d94e5 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -17,6 +17,7 @@ nix_tests = \ ca/gc.sh \ gc.sh \ remote-store.sh \ + legacy-ssh-store.sh \ lang.sh \ experimental-features.sh \ fetchMercurial.sh \ diff --git a/tests/remote-store.sh b/tests/remote-store.sh index 1ae126794..ea32a20d3 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -5,8 +5,19 @@ clearStore # Ensure "fake ssh" remote store works just as legacy fake ssh would. nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store doctor +# Ensure that store ping trusted works with ssh-ng:// +nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e '.trusted' + startDaemon +if isDaemonNewer "2.15pre0"; then + # Ensure that ping works trusted with new daemon + nix store ping --json | jq -e '.trusted' +else + # And the the field is absent with the old daemon + nix store ping --json | jq -e 'has("trusted") | not' +fi + # Test import-from-derivation through the daemon. [[ $(nix eval --impure --raw --expr ' with import ./config.nix; From 214f1d67910e491e0bd7e1f09f0623fdffd82d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 7 Apr 2023 09:16:40 +0200 Subject: [PATCH 39/75] Rename and protect `BufferedSink::write` The `write` name is ambiguous and could lead to some funny bugs like https://github.com/NixOS/nix/pull/8173#issuecomment-1500009480. So rename it to the more explicit `writeUnbuffered`. Besides, this method shouldn't be (and isn't) used outside of the class implementation, so mark it `protected`. This makes it more symetrical to `BufferedSource` which uses a `protected readUnbuffered` method. --- src/libutil/compression.cc | 6 +++--- src/libutil/compression.hh | 2 +- src/libutil/hash.cc | 2 +- src/libutil/hash.hh | 2 +- src/libutil/serialise.cc | 6 +++--- src/libutil/serialise.hh | 8 +++++--- 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 89180e7a7..ba0847cde 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -23,7 +23,7 @@ struct ChunkedCompressionSink : CompressionSink { uint8_t outbuf[32 * 1024]; - void write(std::string_view data) override + void writeUnbuffered(std::string_view data) override { const size_t CHUNK_SIZE = sizeof(outbuf) << 2; while (!data.empty()) { @@ -103,7 +103,7 @@ struct ArchiveCompressionSink : CompressionSink throw Error(reason, archive_error_string(this->archive)); } - void write(std::string_view data) override + void writeUnbuffered(std::string_view data) override { ssize_t result = archive_write_data(archive, data.data(), data.length()); if (result <= 0) check(result); @@ -136,7 +136,7 @@ struct NoneSink : CompressionSink warn("requested compression level '%d' not supported by compression method 'none'", level); } void finish() override { flush(); } - void write(std::string_view data) override { nextSink(data); } + void writeUnbuffered(std::string_view data) override { nextSink(data); } }; struct BrotliDecompressionSink : ChunkedCompressionSink diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index 3892831c2..4e53a7b3c 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -12,7 +12,7 @@ namespace nix { struct CompressionSink : BufferedSink, FinishSink { using BufferedSink::operator (); - using BufferedSink::write; + using BufferedSink::writeUnbuffered; using FinishSink::finish; }; diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 5735e4715..0c8f5727d 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -343,7 +343,7 @@ HashSink::~HashSink() delete ctx; } -void HashSink::write(std::string_view data) +void HashSink::writeUnbuffered(std::string_view data) { bytes += data.size(); update(ht, *ctx, data); diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index be1fdba2a..ae3ee40f4 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -197,7 +197,7 @@ public: HashSink(HashType ht); HashSink(const HashSink & h); ~HashSink(); - void write(std::string_view data) override; + void writeUnbuffered(std::string_view data) override; HashResult finish() override; HashResult currentHash(); }; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 7476e6f6c..62d426e5c 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -20,7 +20,7 @@ void BufferedSink::operator () (std::string_view data) buffer size. */ if (bufPos + data.size() >= bufSize) { flush(); - write(data); + writeUnbuffered(data); break; } /* Otherwise, copy the bytes to the buffer. Flush the buffer @@ -38,7 +38,7 @@ void BufferedSink::flush() if (bufPos == 0) return; size_t n = bufPos; bufPos = 0; // don't trigger the assert() in ~BufferedSink() - write({buffer.get(), n}); + writeUnbuffered({buffer.get(), n}); } @@ -48,7 +48,7 @@ FdSink::~FdSink() } -void FdSink::write(std::string_view data) +void FdSink::writeUnbuffered(std::string_view data) { written += data.size(); try { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 2cf527023..cdb991036 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -53,7 +53,9 @@ struct BufferedSink : virtual Sink void flush(); - virtual void write(std::string_view data) = 0; +protected: + + virtual void writeUnbuffered(std::string_view data) = 0; }; @@ -133,7 +135,7 @@ struct FdSink : BufferedSink ~FdSink(); - void write(std::string_view data) override; + void writeUnbuffered(std::string_view data) override; bool good() override; @@ -520,7 +522,7 @@ struct FramedSink : nix::BufferedSink } } - void write(std::string_view data) override + void writeUnbuffered(std::string_view data) override { /* Don't send more data if the remote has encountered an error. */ From 6960de73affeb7a6d061837b1088937a0295dd93 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 7 Apr 2023 13:08:32 +0200 Subject: [PATCH 40/75] Typo --- src/libexpr/flake/lockfile.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 4616a77f6..868ab89ad 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -67,7 +67,7 @@ struct LockFile std::optional isUnlocked() const; bool operator ==(const LockFile & other) const; - // Needed for old gcc versions that don't syntethise it (like gcc 8.2.2 + // Needed for old gcc versions that don't synthesize it (like gcc 8.2.2 // that is still the default on aarch64-linux) bool operator !=(const LockFile & other) const; From fe9cbe838c3d59e2768b92d8cab0e5a2674f5bfb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 17 Feb 2023 18:37:35 -0500 Subject: [PATCH 41/75] Create `Derivation::fromJSON` And test, of course --- src/libstore/content-address.hh | 5 ++ src/libstore/derivations.cc | 105 +++++++++++++++++++++++++++++++ src/libstore/derivations.hh | 35 ++++++++++- src/libstore/tests/derivation.cc | 61 ++++++++++++++---- src/libutil/comparator.hh | 10 +-- 5 files changed, 197 insertions(+), 19 deletions(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 19fdfc1eb..368a7ec48 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -3,6 +3,7 @@ #include #include "hash.hh" +#include "comparator.hh" namespace nix { @@ -30,6 +31,8 @@ struct TextHash { * Hash of the contents of the text/file. */ Hash hash; + + GENERATE_CMP(TextHash, me->hash); }; /** @@ -46,6 +49,8 @@ struct FixedOutputHash { Hash hash; std::string printMethodAlgo() const; + + GENERATE_CMP(FixedOutputHash, me->method, me->hash); }; /** diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 06cc69056..ea4abb352 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -889,6 +889,7 @@ std::optional Derivation::tryResolve( return resolved; } + const Hash impureOutputHash = hashString(htSHA256, "impure"); nlohmann::json DerivationOutput::toJSON( @@ -916,6 +917,73 @@ nlohmann::json DerivationOutput::toJSON( return res; } + +DerivationOutput DerivationOutput::fromJSON( + const Store & store, std::string_view drvName, std::string_view outputName, + const nlohmann::json & _json) +{ + std::set keys; + auto json = (std::map) _json; + + for (const auto & [key, _] : json) + keys.insert(key); + + auto methodAlgo = [&]() -> std::pair { + std::string hashAlgo = json["hashAlgo"]; + auto method = FileIngestionMethod::Flat; + if (hashAlgo.substr(0, 2) == "r:") { + method = FileIngestionMethod::Recursive; + hashAlgo = hashAlgo.substr(2); + } + auto hashType = parseHashType(hashAlgo); + return { method, hashType }; + }; + + if (keys == (std::set { "path" })) { + return DerivationOutput::InputAddressed { + .path = store.parseStorePath((std::string) json["path"]), + }; + } + + else if (keys == (std::set { "path", "hashAlgo", "hash" })) { + auto [method, hashType] = methodAlgo(); + auto dof = DerivationOutput::CAFixed { + .hash = { + .method = method, + .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType), + }, + }; + if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) + throw Error("Path doesn't match derivation output"); + return dof; + } + + else if (keys == (std::set { "hashAlgo" })) { + auto [method, hashType] = methodAlgo(); + return DerivationOutput::CAFloating { + .method = method, + .hashType = hashType, + }; + } + + else if (keys == (std::set { })) { + return DerivationOutput::Deferred {}; + } + + else if (keys == (std::set { "hashAlgo", "impure" })) { + auto [method, hashType] = methodAlgo(); + return DerivationOutput::Impure { + .method = method, + .hashType = hashType, + }; + } + + else { + throw Error("invalid JSON for derivation output"); + } +} + + nlohmann::json Derivation::toJSON(const Store & store) const { nlohmann::json res = nlohmann::json::object(); @@ -950,4 +1018,41 @@ nlohmann::json Derivation::toJSON(const Store & store) const return res; } + +Derivation Derivation::fromJSON( + const Store & store, std::string_view drvName, + const nlohmann::json & json) +{ + Derivation res; + + { + auto & outputsObj = json["outputs"]; + for (auto & [outputName, output] : outputsObj.items()) { + res.outputs.insert_or_assign( + outputName, + DerivationOutput::fromJSON(store, drvName, outputName, output)); + } + } + + { + auto & inputsList = json["inputSrcs"]; + for (auto & input : inputsList) + res.inputSrcs.insert(store.parseStorePath(static_cast(input))); + } + + { + auto & inputDrvsObj = json["inputDrvs"]; + for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) + res.inputDrvs[store.parseStorePath(inputDrvPath)] = + static_cast(inputOutputs); + } + + res.platform = json["system"]; + res.builder = json["builder"]; + res.args = json["args"]; + res.env = json["env"]; + + return res; +} + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index e12bd2119..0722dc61d 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -7,6 +7,7 @@ #include "content-address.hh" #include "repair-flag.hh" #include "sync.hh" +#include "comparator.hh" #include #include @@ -24,6 +25,8 @@ class Store; struct DerivationOutputInputAddressed { StorePath path; + + GENERATE_CMP(DerivationOutputInputAddressed, me->path); }; /** @@ -44,6 +47,8 @@ struct DerivationOutputCAFixed * @param outputName The name of this output. */ StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; + + GENERATE_CMP(DerivationOutputCAFixed, me->hash); }; /** @@ -62,13 +67,17 @@ struct DerivationOutputCAFloating * How the serialization will be hashed */ HashType hashType; + + GENERATE_CMP(DerivationOutputCAFloating, me->method, me->hashType); }; /** * Input-addressed output which depends on a (CA) derivation whose hash * isn't known yet. */ -struct DerivationOutputDeferred {}; +struct DerivationOutputDeferred { + GENERATE_CMP(DerivationOutputDeferred); +}; /** * Impure output which is moved to a content-addressed location (like @@ -85,6 +94,8 @@ struct DerivationOutputImpure * How the serialization will be hashed */ HashType hashType; + + GENERATE_CMP(DerivationOutputImpure, me->method, me->hashType); }; typedef std::variant< @@ -125,6 +136,11 @@ struct DerivationOutput : _DerivationOutputRaw const Store & store, std::string_view drvName, std::string_view outputName) const; + static DerivationOutput fromJSON( + const Store & store, + std::string_view drvName, + std::string_view outputName, + const nlohmann::json & json); }; typedef std::map DerivationOutputs; @@ -273,6 +289,15 @@ struct BasicDerivation DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const; static std::string_view nameFromPath(const StorePath & storePath); + + GENERATE_CMP(BasicDerivation, + me->outputs, + me->inputSrcs, + me->platform, + me->builder, + me->args, + me->env, + me->name); }; struct Derivation : BasicDerivation @@ -313,6 +338,14 @@ struct Derivation : BasicDerivation Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } nlohmann::json toJSON(const Store & store) const; + static Derivation fromJSON( + const Store & store, + std::string_view drvName, + const nlohmann::json & json); + + GENERATE_CMP(Derivation, + static_cast(*me), + me->inputDrvs); }; diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 12be8504d..8993cd80b 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -11,15 +11,29 @@ class DerivationTest : public LibStoreTest { }; -#define TEST_JSON(TYPE, NAME, STR, VAL, ...) \ - TEST_F(DerivationTest, TYPE ## _ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (TYPE { VAL }).toJSON(*store __VA_OPT__(,) __VA_ARGS__)); \ +#define TEST_JSON(NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (DerivationOutput { VAL }).toJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME)); \ + } \ + \ + TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + DerivationOutput { VAL }, \ + DerivationOutput::fromJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME, \ + STR ## _json)); \ } -TEST_JSON(DerivationOutput, inputAddressed, +TEST_JSON(inputAddressed, R"({ "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" })", @@ -28,7 +42,7 @@ TEST_JSON(DerivationOutput, inputAddressed, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, caFixed, +TEST_JSON(caFixed, R"({ "hashAlgo": "r:sha256", "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", @@ -42,7 +56,7 @@ TEST_JSON(DerivationOutput, caFixed, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, caFloating, +TEST_JSON(caFloating, R"({ "hashAlgo": "r:sha256" })", @@ -52,12 +66,12 @@ TEST_JSON(DerivationOutput, caFloating, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, deferred, +TEST_JSON(deferred, R"({ })", DerivationOutput::Deferred { }, "drv-name", "output-name") -TEST_JSON(DerivationOutput, impure, +TEST_JSON(impure, R"({ "hashAlgo": "r:sha256", "impure": true @@ -68,7 +82,27 @@ TEST_JSON(DerivationOutput, impure, }), "drv-name", "output-name") -TEST_JSON(Derivation, impure, +#undef TEST_JSON + +#define TEST_JSON(NAME, STR, VAL, DRV_NAME) \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (Derivation { VAL }).toJSON(*store)); \ + } \ + \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + Derivation { VAL }, \ + Derivation::fromJSON( \ + *store, \ + DRV_NAME, \ + STR ## _json)); \ + } + +TEST_JSON(simple, R"({ "inputSrcs": [ "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" @@ -117,7 +151,8 @@ TEST_JSON(Derivation, impure, }, }; drv; - })) + }), + "drv-name") #undef TEST_JSON diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index 2b5424b3d..9f661c5c3 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -17,12 +17,12 @@ * } * ``` */ -#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \ +#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, ...) \ bool operator COMPARATOR(const MY_TYPE& other) const { \ - const MY_TYPE* me = this; \ - auto fields1 = std::make_tuple( FIELDS ); \ - me = &other; \ - auto fields2 = std::make_tuple( FIELDS ); \ + __VA_OPT__(const MY_TYPE* me = this;) \ + auto fields1 = std::make_tuple( __VA_ARGS__ ); \ + __VA_OPT__(me = &other;) \ + auto fields2 = std::make_tuple( __VA_ARGS__ ); \ return fields1 COMPARATOR fields2; \ } #define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) From b200784cec056da53378eb043cae4fad188e4e6f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Mar 2023 11:06:52 -0400 Subject: [PATCH 42/75] Include the name in the JSON for derivations This is non-breaking change in the to-JSON direction. This *is* a breaking change in the from-JSON direction, but we don't care, as that is brand new in this PR. `nix show-derivation --help` currently has the sole public documentation of this format, it is updated accordingly. --- src/libstore/derivations.cc | 8 ++++++-- src/libstore/derivations.hh | 1 - src/libstore/tests/derivation.cc | 3 ++- src/nix/show-derivation.md | 3 +++ 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index ea4abb352..9f74f1c30 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -988,6 +988,8 @@ nlohmann::json Derivation::toJSON(const Store & store) const { nlohmann::json res = nlohmann::json::object(); + res["name"] = name; + { nlohmann::json & outputsObj = res["outputs"]; outputsObj = nlohmann::json::object(); @@ -1020,17 +1022,19 @@ nlohmann::json Derivation::toJSON(const Store & store) const Derivation Derivation::fromJSON( - const Store & store, std::string_view drvName, + const Store & store, const nlohmann::json & json) { Derivation res; + res.name = json["name"]; + { auto & outputsObj = json["outputs"]; for (auto & [outputName, output] : outputsObj.items()) { res.outputs.insert_or_assign( outputName, - DerivationOutput::fromJSON(store, drvName, outputName, output)); + DerivationOutput::fromJSON(store, res.name, outputName, output)); } } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 0722dc61d..38727d77a 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -340,7 +340,6 @@ struct Derivation : BasicDerivation nlohmann::json toJSON(const Store & store) const; static Derivation fromJSON( const Store & store, - std::string_view drvName, const nlohmann::json & json); GENERATE_CMP(Derivation, diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 8993cd80b..80ee52fd0 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -98,12 +98,12 @@ TEST_JSON(impure, Derivation { VAL }, \ Derivation::fromJSON( \ *store, \ - DRV_NAME, \ STR ## _json)); \ } TEST_JSON(simple, R"({ + "name": "my-derivation", "inputSrcs": [ "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" ], @@ -126,6 +126,7 @@ TEST_JSON(simple, })", ({ Derivation drv; + drv.name = "my-derivation"; drv.inputSrcs = { store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), }; diff --git a/src/nix/show-derivation.md b/src/nix/show-derivation.md index 1d37c6f5a..17d88a5f9 100644 --- a/src/nix/show-derivation.md +++ b/src/nix/show-derivation.md @@ -51,6 +51,9 @@ The JSON output is a JSON object whose keys are the store paths of the derivations, and whose values are a JSON object with the following fields: +* `name`: The name of the derivation. This is used when calculating the + store paths of the derivation's outputs. + * `outputs`: Information about the output paths of the derivation. This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these From 4e9f32f993aaa9f7995919e480e0e920d946184d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Mar 2023 16:57:36 -0500 Subject: [PATCH 43/75] Liberate `checkDerivationOutputs` from `LocalStore` Make it instead a method on `Derivation` that can work with any store. We will need this for a CLI command to create a derivation. --- src/libstore/derivations.cc | 60 ++++++++++++++++++++++++++++++++++++ src/libstore/derivations.hh | 8 +++++ src/libstore/local-store.cc | 61 ++----------------------------------- src/libstore/local-store.hh | 2 -- 4 files changed, 70 insertions(+), 61 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 9f74f1c30..abdfb1978 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -890,6 +890,66 @@ std::optional Derivation::tryResolve( } +void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const +{ + assert(drvPath.isDerivation()); + std::string drvName(drvPath.name()); + drvName = drvName.substr(0, drvName.size() - drvExtension.size()); + + if (drvName != name) { + throw Error("Derivation '%s' has name '%s' which does not match its path", store.printStorePath(drvPath), name); + } + + auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) + { + auto j = env.find(varName); + if (j == env.end() || store.parseStorePath(j->second) != actual) + throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", + store.printStorePath(drvPath), varName, store.printStorePath(actual)); + }; + + + // Don't need the answer, but do this anyways to assert is proper + // combination. The code below is more general and naturally allows + // combinations that are currently prohibited. + type(); + + std::optional hashesModulo; + for (auto & i : outputs) { + std::visit(overloaded { + [&](const DerivationOutput::InputAddressed & doia) { + if (!hashesModulo) { + // somewhat expensive so we do lazily + hashesModulo = hashDerivationModulo(store, *this, true); + } + auto currentOutputHash = get(hashesModulo->hashes, i.first); + if (!currentOutputHash) + throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", + store.printStorePath(drvPath), store.printStorePath(doia.path), i.first); + StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName); + if (doia.path != recomputed) + throw Error("derivation '%s' has incorrect output '%s', should be '%s'", + store.printStorePath(drvPath), store.printStorePath(doia.path), store.printStorePath(recomputed)); + envHasRightPath(doia.path, i.first); + }, + [&](const DerivationOutput::CAFixed & dof) { + StorePath path = store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); + envHasRightPath(path, i.first); + }, + [&](const DerivationOutput::CAFloating &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Deferred &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Impure &) { + /* Nothing to check */ + }, + }, i.second.raw()); + } +} + + const Hash impureOutputHash = hashString(htSHA256, "impure"); nlohmann::json DerivationOutput::toJSON( diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 38727d77a..9e7ceeb5d 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -333,6 +333,14 @@ struct Derivation : BasicDerivation Store & store, const std::map, StorePath> & inputDrvOutputs) const; + /* Check that the derivation is valid and does not present any + illegal states. + + This is mainly a matter of checking the outputs, where our C++ + representation supports all sorts of combinations we do not yet + allow. */ + void checkInvariants(Store & store, const StorePath & drvPath) const; + Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e0ad50f6d..c3be1b461 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -710,62 +710,6 @@ void canonicalisePathMetaData(const Path & path, canonicalisePathMetaData(path, uidRange, inodesSeen); } - -void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv) -{ - assert(drvPath.isDerivation()); - std::string drvName(drvPath.name()); - drvName = drvName.substr(0, drvName.size() - drvExtension.size()); - - auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) - { - auto j = drv.env.find(varName); - if (j == drv.env.end() || parseStorePath(j->second) != actual) - throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", - printStorePath(drvPath), varName, printStorePath(actual)); - }; - - - // Don't need the answer, but do this anyways to assert is proper - // combination. The code below is more general and naturally allows - // combinations that are currently prohibited. - drv.type(); - - std::optional hashesModulo; - for (auto & i : drv.outputs) { - std::visit(overloaded { - [&](const DerivationOutput::InputAddressed & doia) { - if (!hashesModulo) { - // somewhat expensive so we do lazily - hashesModulo = hashDerivationModulo(*this, drv, true); - } - auto currentOutputHash = get(hashesModulo->hashes, i.first); - if (!currentOutputHash) - throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", - printStorePath(drvPath), printStorePath(doia.path), i.first); - StorePath recomputed = makeOutputPath(i.first, *currentOutputHash, drvName); - if (doia.path != recomputed) - throw Error("derivation '%s' has incorrect output '%s', should be '%s'", - printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); - envHasRightPath(doia.path, i.first); - }, - [&](const DerivationOutput::CAFixed & dof) { - StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); - envHasRightPath(path, i.first); - }, - [&](const DerivationOutput::CAFloating &) { - /* Nothing to check */ - }, - [&](const DerivationOutput::Deferred &) { - /* Nothing to check */ - }, - [&](const DerivationOutput::Impure &) { - /* Nothing to check */ - }, - }, i.second.raw()); - } -} - void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) { experimentalFeatureSettings.require(Xp::CaDerivations); @@ -876,7 +820,7 @@ uint64_t LocalStore::addValidPath(State & state, derivations). Note that if this throws an error, then the DB transaction is rolled back, so the path validity registration above is undone. */ - if (checkOutputs) checkDerivationOutputs(info.path, drv); + if (checkOutputs) drv.checkInvariants(*this, info.path); for (auto & i : drv.outputsAndOptPaths(*this)) { /* Floating CA derivations have indeterminate output paths until @@ -1175,8 +1119,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - checkDerivationOutputs(i.path, - readInvalidDerivation(i.path)); + readInvalidDerivation(i.path).checkInvariants(*this, i.path); } /* Do a topological sort of the paths. This will throw an diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 6deaa051f..7e0849961 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -270,8 +270,6 @@ private: std::pair createTempDirInStore(); - void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); - typedef std::unordered_set InodeHash; InodeHash loadInodeHash(); From 2b98af2e62f8a70cfa132f9bac7049a198309df8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Apr 2023 13:59:19 -0400 Subject: [PATCH 44/75] `nix show-derivation` -> `nix derivation show` --- doc/manual/src/glossary.md | 2 +- src/nix/derivation.cc | 25 +++++++++++++++++++++++++ src/nix/main.cc | 1 + src/nix/show-derivation.cc | 4 ++-- src/nix/show-derivation.md | 6 +++--- tests/ca/build.sh | 2 +- tests/impure-derivations.sh | 4 ++-- 7 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 src/nix/derivation.cc diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index fe5920746..4eedb2e93 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -15,7 +15,7 @@ Example: `/nix/store/g946hcz4c8mdvq2g8vxx42z51qb71rvp-git-2.38.1.drv` - See [`nix show-derivation`](./command-ref/new-cli/nix3-show-derivation.md) (experimental) for displaying the contents of store derivations. + See [`nix derivation show`](./command-ref/new-cli/nix3-derivation-show.md) (experimental) for displaying the contents of store derivations. [store derivation]: #gloss-store-derivation diff --git a/src/nix/derivation.cc b/src/nix/derivation.cc new file mode 100644 index 000000000..cd3975a4f --- /dev/null +++ b/src/nix/derivation.cc @@ -0,0 +1,25 @@ +#include "command.hh" + +using namespace nix; + +struct CmdDerivation : virtual NixMultiCommand +{ + CmdDerivation() : MultiCommand(RegisterCommand::getCommandsFor({"derivation"})) + { } + + std::string description() override + { + return "Work with derivations, Nix's notion of a build plan."; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix derivation' requires a sub-command."); + command->second->run(); + } +}; + +static auto rCmdDerivation = registerCommand("derivation"); diff --git a/src/nix/main.cc b/src/nix/main.cc index 4d4164333..f943f77bb 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -127,6 +127,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs {"optimise-store", {"store", "optimise"}}, {"ping-store", {"store", "ping"}}, {"sign-paths", {"store", "sign"}}, + {"show-derivation", {"derivation", "show"}}, {"to-base16", {"hash", "to-base16"}}, {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 4a406ae08..0a55b36f2 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -1,5 +1,5 @@ // FIXME: integrate this with nix path-info? -// FIXME: rename to 'nix store show-derivation' or 'nix debug show-derivation'? +// FIXME: rename to 'nix store derivation show' or 'nix debug derivation show'? #include "command.hh" #include "common-args.hh" @@ -61,4 +61,4 @@ struct CmdShowDerivation : InstallablesCommand } }; -static auto rCmdShowDerivation = registerCommand("show-derivation"); +static auto rCmdShowDerivation = registerCommand2({"derivation", "show"}); diff --git a/src/nix/show-derivation.md b/src/nix/show-derivation.md index 17d88a5f9..1ebe22f65 100644 --- a/src/nix/show-derivation.md +++ b/src/nix/show-derivation.md @@ -8,7 +8,7 @@ R""( [store derivation]: ../../glossary.md#gloss-store-derivation ```console - # nix show-derivation nixpkgs#hello + # nix derivation show nixpkgs#hello { "/nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv": { … @@ -20,14 +20,14 @@ R""( NixOS system: ```console - # nix show-derivation -r /run/current-system + # nix derivation show -r /run/current-system ``` * Print all files fetched using `fetchurl` by Firefox's dependency graph: ```console - # nix show-derivation -r nixpkgs#firefox \ + # nix derivation show -r nixpkgs#firefox \ | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' \ | uniq | sort ``` diff --git a/tests/ca/build.sh b/tests/ca/build.sh index 98e1c5125..7754ad276 100644 --- a/tests/ca/build.sh +++ b/tests/ca/build.sh @@ -3,7 +3,7 @@ source common.sh drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1) -nix show-derivation "$drv" --arg seed 1 +nix derivation show "$drv" --arg seed 1 buildAttr () { local derivationPath=$1 diff --git a/tests/impure-derivations.sh b/tests/impure-derivations.sh index 7595fdd35..c7dadf397 100644 --- a/tests/impure-derivations.sh +++ b/tests/impure-derivations.sh @@ -37,8 +37,8 @@ path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnIm (! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation' drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .) -[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] -[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] +[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] +[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] # Fixed-output derivations *can* depend on impure derivations. path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out) From 27597f813126bf0e866c6f6ba35dc7d3761843f8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Apr 2023 14:05:53 -0400 Subject: [PATCH 45/75] Rename files to reflect new `nix derivation show` name This will match the files we added for `nix add derivation` in the rest of this PR. --- src/nix/{show-derivation.cc => derivation-show.cc} | 2 +- src/nix/{show-derivation.md => derivation-show.md} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/nix/{show-derivation.cc => derivation-show.cc} (97%) rename src/nix/{show-derivation.md => derivation-show.md} (100%) diff --git a/src/nix/show-derivation.cc b/src/nix/derivation-show.cc similarity index 97% rename from src/nix/show-derivation.cc rename to src/nix/derivation-show.cc index 0a55b36f2..bf637246d 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/derivation-show.cc @@ -33,7 +33,7 @@ struct CmdShowDerivation : InstallablesCommand std::string doc() override { return - #include "show-derivation.md" + #include "derivation-show.md" ; } diff --git a/src/nix/show-derivation.md b/src/nix/derivation-show.md similarity index 100% rename from src/nix/show-derivation.md rename to src/nix/derivation-show.md From 59e072871410ba88694cf2597079a9f0f115e458 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 17 Feb 2023 18:37:35 -0500 Subject: [PATCH 46/75] Create `nix derivation add` command Also refine `nix derivation show`'s docs very slightly. --- src/nix/derivation-add.cc | 45 +++++++++++++++++++++++++++++++++++++ src/nix/derivation-add.md | 18 +++++++++++++++ src/nix/derivation-show.md | 9 ++++---- tests/ca/derivation-json.sh | 26 +++++++++++++++++++++ tests/derivation-json.sh | 12 ++++++++++ tests/local.mk | 2 ++ 6 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 src/nix/derivation-add.cc create mode 100644 src/nix/derivation-add.md create mode 100644 tests/ca/derivation-json.sh create mode 100644 tests/derivation-json.sh diff --git a/src/nix/derivation-add.cc b/src/nix/derivation-add.cc new file mode 100644 index 000000000..4d91d4538 --- /dev/null +++ b/src/nix/derivation-add.cc @@ -0,0 +1,45 @@ +// FIXME: rename to 'nix plan add' or 'nix derivation add'? + +#include "command.hh" +#include "common-args.hh" +#include "store-api.hh" +#include "archive.hh" +#include "derivations.hh" +#include + +using namespace nix; +using json = nlohmann::json; + +struct CmdAddDerivation : MixDryRun, StoreCommand +{ + std::string description() override + { + return "Add a store derivation"; + } + + std::string doc() override + { + return + #include "derivation-add.md" + ; + } + + Category category() override { return catUtility; } + + void run(ref store) override + { + auto json = nlohmann::json::parse(drainFD(STDIN_FILENO)); + + auto drv = Derivation::fromJSON(*store, json); + + auto drvPath = writeDerivation(*store, drv, NoRepair, /* read only */ dryRun); + + drv.checkInvariants(*store, drvPath); + + writeDerivation(*store, drv, NoRepair, dryRun); + + logger->cout("%s", store->printStorePath(drvPath)); + } +}; + +static auto rCmdAddDerivation = registerCommand2({"derivation", "add"}); diff --git a/src/nix/derivation-add.md b/src/nix/derivation-add.md new file mode 100644 index 000000000..f116681ab --- /dev/null +++ b/src/nix/derivation-add.md @@ -0,0 +1,18 @@ +R""( + +# Description + +This command reads from standard input a JSON representation of a +[store derivation] to which an [*installable*](./nix.md#installables) evaluates. + +Store derivations are used internally by Nix. They are store paths with +extension `.drv` that represent the build-time dependency graph to which +a Nix expression evaluates. + +[store derivation]: ../../glossary.md#gloss-store-derivation + +The JSON format is documented under the [`derivation show`] command. + +[`derivation show`]: ./nix3-derivation-show.md + +)"" diff --git a/src/nix/derivation-show.md b/src/nix/derivation-show.md index 1ebe22f65..1296e2885 100644 --- a/src/nix/derivation-show.md +++ b/src/nix/derivation-show.md @@ -39,10 +39,11 @@ R""( # Description This command prints on standard output a JSON representation of the -[store derivation]s to which [*installables*](./nix.md#installables) evaluate. Store derivations -are used internally by Nix. They are store paths with extension `.drv` -that represent the build-time dependency graph to which a Nix -expression evaluates. +[store derivation]s to which [*installables*](./nix.md#installables) evaluate. + +Store derivations are used internally by Nix. They are store paths with +extension `.drv` that represent the build-time dependency graph to which +a Nix expression evaluates. By default, this command only shows top-level derivations, but with `--recursive`, it also shows their dependencies. diff --git a/tests/ca/derivation-json.sh b/tests/ca/derivation-json.sh new file mode 100644 index 000000000..3615177e9 --- /dev/null +++ b/tests/ca/derivation-json.sh @@ -0,0 +1,26 @@ +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 + +drvPath=$(nix-instantiate ../simple.nix) + +nix derivation show $drvPath | jq .[] > $TEST_HOME/simple.json + +drvPath2=$(nix derivation add < $TEST_HOME/simple.json) + +[[ "$drvPath" = "$drvPath2" ]] + +# Content-addressed derivations can be renamed. +jq '.name = "foo"' < $TEST_HOME/simple.json > $TEST_HOME/foo.json +drvPath3=$(nix derivation add --dry-run < $TEST_HOME/foo.json) +# With --dry-run nothing is actually written +[[ ! -e "$drvPath3" ]] + +# Without --dry-run it is actually written +drvPath4=$(nix derivation add < $TEST_HOME/foo.json) +[[ "$drvPath4" = "$drvPath3" ]] +[[ -e "$drvPath3" ]] + +# The modified derivation read back as JSON matches +nix derivation show $drvPath3 | jq .[] > $TEST_HOME/foo-read.json +diff $TEST_HOME/foo.json $TEST_HOME/foo-read.json diff --git a/tests/derivation-json.sh b/tests/derivation-json.sh new file mode 100644 index 000000000..b6be5d977 --- /dev/null +++ b/tests/derivation-json.sh @@ -0,0 +1,12 @@ +source common.sh + +drvPath=$(nix-instantiate simple.nix) + +nix derivation show $drvPath | jq .[] > $TEST_HOME/simple.json + +drvPath2=$(nix derivation add < $TEST_HOME/simple.json) + +[[ "$drvPath" = "$drvPath2" ]] + +# Input addressed derivations cannot be renamed. +jq '.name = "foo"' < $TEST_HOME/simple.json | expectStderr 1 nix derivation add | grepQuiet "has incorrect output" diff --git a/tests/local.mk b/tests/local.mk index 7d54d94e5..6cb466e8e 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -102,6 +102,8 @@ nix_tests = \ eval-store.sh \ why-depends.sh \ ca/why-depends.sh \ + derivation-json.sh \ + ca/derivation-json.sh \ import-derivation.sh \ ca/import-derivation.sh \ nix_path.sh \ From 9d1105824f99ada218387eb8791643f66b25c3f8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 3 Apr 2023 09:18:33 -0400 Subject: [PATCH 47/75] Add release notes for `nix derivation {add,show}` --- doc/manual/src/release-notes/rl-next.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index af1850bdc..5b62836bf 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -45,3 +45,14 @@ This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores. `nix store ping` and `nix doctor` now display this information. + +* A new command `nix derivation add` is created, to allow adding derivations to the store without involving the Nix language. + It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store. + It uses the same JSON layout as `nix show-derivation`, and is its inverse. + +* `nix show-derivation` has been renamed to `nix derivation show`. + This matches `nix derivation add`, and avoids bloating the top-level namespace. + The old name is still kept as an alias for compatibility, however. + +* The `nix derivation {add,show}` JSON format now includes the derivation name as a top-level field. + This is useful in general, but especially necessary for the `add` direction, as otherwise we would need to pass in the name out of band for certain cases. From 2c53ef1bfee7c7afea889f42b9ef13e1007ad228 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Fri, 3 Feb 2023 17:50:01 +0100 Subject: [PATCH 48/75] Disable GC inside coroutines on mac OS --- src/libexpr/eval.cc | 20 ++++++++++++++++++++ src/libutil/serialise.cc | 38 +++++++++++++++++++++++++++++++++----- src/libutil/serialise.hh | 10 ++++++++++ 3 files changed, 63 insertions(+), 5 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7f2065656..f7c056998 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -345,6 +345,20 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) } +class BoehmDisableGC : public DisableGC { +public: + BoehmDisableGC() { +#if HAVE_BOEHMGC + GC_disable(); +#endif + }; + virtual ~BoehmDisableGC() override { +#if HAVE_BOEHMGC + GC_enable(); +#endif + }; +}; + static bool gcInitialised = false; void initGC() @@ -368,6 +382,12 @@ void initGC() StackAllocator::defaultAllocator = &boehmGCStackAllocator; + + /* Used to disable GC when entering coroutines on macOS */ + DisableGC::create = []() { + return std::dynamic_pointer_cast(std::make_shared()); + }; + /* Set the initial heap size to something fairly big (25% of physical RAM, up to a maximum of 384 MiB) so that in most cases we don't need to garbage collect at all. (Collection has a diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 7476e6f6c..fac2a0df8 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -186,6 +186,10 @@ static DefaultStackAllocator defaultAllocatorSingleton; StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton; +std::shared_ptr (*DisableGC::create)() = []() { + return std::dynamic_pointer_cast(std::make_shared()); +}; + std::unique_ptr sourceToSink(std::function fun) { struct SourceToSink : FinishSink @@ -206,7 +210,11 @@ std::unique_ptr sourceToSink(std::function fun) if (in.empty()) return; cur = in; - if (!coro) + if (!coro) { +#if __APPLE__ + /* Disable GC when entering the coroutine on macOS, since it doesn't find the main thread stack in this case */ + auto disablegc = DisableGC::create(); +#endif coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) { LambdaSource source([&](char *out, size_t out_len) { if (cur.empty()) { @@ -223,17 +231,28 @@ std::unique_ptr sourceToSink(std::function fun) }); fun(source); }); + } if (!*coro) { abort(); } - if (!cur.empty()) (*coro)(false); + if (!cur.empty()) { +#if __APPLE__ + auto disablegc = DisableGC::create(); +#endif + (*coro)(false); + } } void finish() override { if (!coro) return; if (!*coro) abort(); - (*coro)(true); + { +#if __APPLE__ + auto disablegc = DisableGC::create(); +#endif + (*coro)(true); + } if (*coro) abort(); } }; @@ -264,18 +283,27 @@ std::unique_ptr sinkToSource( size_t read(char * data, size_t len) override { - if (!coro) + if (!coro) { +#if __APPLE__ + auto disablegc = DisableGC::create(); +#endif coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { LambdaSink sink([&](std::string_view data) { if (!data.empty()) yield(std::string(data)); }); fun(sink); }); + } if (!*coro) { eof(); abort(); } if (pos == cur.size()) { - if (!cur.empty()) (*coro)(); + if (!cur.empty()) { +#if __APPLE__ + auto disablegc = DisableGC::create(); +#endif + (*coro)(); + } cur = coro->get(); pos = 0; } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 2cf527023..612196fd2 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -551,4 +551,14 @@ struct StackAllocator { static StackAllocator *defaultAllocator; }; +/* Disabling GC when entering a coroutine (on macos). + ::create is to avoid boehm gc dependency in libutil. + */ +class DisableGC { +public: + DisableGC() {}; + virtual ~DisableGC() {}; + static std::shared_ptr (*create)(); +}; + } From 00bc34430b9e8e687cd3f1681e9f7b5baa7c717e Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Fri, 10 Feb 2023 15:04:17 +0100 Subject: [PATCH 49/75] DisableGC: replace by CoroutineContext, std::shared_ptr --- src/libexpr/eval.cc | 20 +++++++++++--------- src/libutil/serialise.cc | 38 ++++++++++++++++++++------------------ src/libutil/serialise.hh | 10 +++------- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f7c056998..e5e8d2b22 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -344,20 +344,22 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) } } - -class BoehmDisableGC : public DisableGC { +#if HAVE_BOEHMGC +/* Disable GC while this object lives. Used by CoroutineContext. + * + * Boehm keeps a count of GC_disable() and GC_enable() calls, + * and only enables GC when the count matches. + */ +class BoehmDisableGC { public: BoehmDisableGC() { -#if HAVE_BOEHMGC GC_disable(); -#endif }; - virtual ~BoehmDisableGC() override { -#if HAVE_BOEHMGC + ~BoehmDisableGC() { GC_enable(); -#endif }; }; +#endif static bool gcInitialised = false; @@ -384,8 +386,8 @@ void initGC() /* Used to disable GC when entering coroutines on macOS */ - DisableGC::create = []() { - return std::dynamic_pointer_cast(std::make_shared()); + create_disable_gc = []() -> std::shared_ptr { + return std::make_shared(); }; /* Set the initial heap size to something fairly big (25% of diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index fac2a0df8..0bdeec0cd 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -186,8 +186,21 @@ static DefaultStackAllocator defaultAllocatorSingleton; StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton; -std::shared_ptr (*DisableGC::create)() = []() { - return std::dynamic_pointer_cast(std::make_shared()); +std::shared_ptr (*create_disable_gc)() = []() -> std::shared_ptr { + return {}; +}; + +/* This class is used for entry and exit hooks on coroutines */ +class CoroutineContext { +#if __APPLE__ + /* Disable GC when entering the coroutine on macOS, since it doesn't find the main thread stack in this case. + * std::shared_ptr performs type-erasure, so it will call the right + * deleter. */ + const std::shared_ptr disable_gc = create_disable_gc(); +#endif +public: + CoroutineContext() {}; + ~CoroutineContext() {}; }; std::unique_ptr sourceToSink(std::function fun) @@ -211,10 +224,7 @@ std::unique_ptr sourceToSink(std::function fun) cur = in; if (!coro) { -#if __APPLE__ - /* Disable GC when entering the coroutine on macOS, since it doesn't find the main thread stack in this case */ - auto disablegc = DisableGC::create(); -#endif + CoroutineContext ctx; coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) { LambdaSource source([&](char *out, size_t out_len) { if (cur.empty()) { @@ -236,9 +246,7 @@ std::unique_ptr sourceToSink(std::function fun) if (!*coro) { abort(); } if (!cur.empty()) { -#if __APPLE__ - auto disablegc = DisableGC::create(); -#endif + CoroutineContext ctx; (*coro)(false); } } @@ -248,9 +256,7 @@ std::unique_ptr sourceToSink(std::function fun) if (!coro) return; if (!*coro) abort(); { -#if __APPLE__ - auto disablegc = DisableGC::create(); -#endif + CoroutineContext ctx; (*coro)(true); } if (*coro) abort(); @@ -284,9 +290,7 @@ std::unique_ptr sinkToSource( size_t read(char * data, size_t len) override { if (!coro) { -#if __APPLE__ - auto disablegc = DisableGC::create(); -#endif + CoroutineContext ctx; coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { LambdaSink sink([&](std::string_view data) { if (!data.empty()) yield(std::string(data)); @@ -299,9 +303,7 @@ std::unique_ptr sinkToSource( if (pos == cur.size()) { if (!cur.empty()) { -#if __APPLE__ - auto disablegc = DisableGC::create(); -#endif + CoroutineContext ctx; (*coro)(); } cur = coro->get(); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 612196fd2..b895db779 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -552,13 +552,9 @@ struct StackAllocator { }; /* Disabling GC when entering a coroutine (on macos). - ::create is to avoid boehm gc dependency in libutil. + mutable to avoid boehm gc dependency in libutil. */ -class DisableGC { -public: - DisableGC() {}; - virtual ~DisableGC() {}; - static std::shared_ptr (*create)(); -}; +extern std::shared_ptr (*create_disable_gc)(); + } From 58d24a4cb630a6162d6f7d2a85e41949fdf2ad36 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Wed, 1 Mar 2023 15:07:00 +0100 Subject: [PATCH 50/75] Always disable GC in a coroutine unless the patch is applied --- boehmgc-coroutine-sp-fallback.diff | 22 +++++++++++++++++++--- src/libexpr/eval.cc | 4 +++- src/libutil/serialise.cc | 9 ++++----- src/libutil/serialise.hh | 4 ++-- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff index 2826486fb..5066d8278 100644 --- a/boehmgc-coroutine-sp-fallback.diff +++ b/boehmgc-coroutine-sp-fallback.diff @@ -1,8 +1,8 @@ diff --git a/darwin_stop_world.c b/darwin_stop_world.c -index 3dbaa3fb..36a1d1f7 100644 +index 0468aaec..b348d869 100644 --- a/darwin_stop_world.c +++ b/darwin_stop_world.c -@@ -352,6 +352,7 @@ GC_INNER void GC_push_all_stacks(void) +@@ -356,6 +356,7 @@ GC_INNER void GC_push_all_stacks(void) int nthreads = 0; word total_size = 0; mach_msg_type_number_t listcount = (mach_msg_type_number_t)THREAD_TABLE_SZ; @@ -10,7 +10,7 @@ index 3dbaa3fb..36a1d1f7 100644 if (!EXPECT(GC_thr_initialized, TRUE)) GC_thr_init(); -@@ -407,6 +408,19 @@ GC_INNER void GC_push_all_stacks(void) +@@ -411,6 +412,19 @@ GC_INNER void GC_push_all_stacks(void) GC_push_all_stack_sections(lo, hi, p->traced_stack_sect); } if (altstack_lo) { @@ -30,6 +30,22 @@ index 3dbaa3fb..36a1d1f7 100644 total_size += altstack_hi - altstack_lo; GC_push_all_stack(altstack_lo, altstack_hi); } +diff --git a/include/gc.h b/include/gc.h +index edab6c22..f2c61282 100644 +--- a/include/gc.h ++++ b/include/gc.h +@@ -2172,6 +2172,11 @@ GC_API void GC_CALL GC_win32_free_heap(void); + (*GC_amiga_allocwrapper_do)(a,GC_malloc_atomic_ignore_off_page) + #endif /* _AMIGA && !GC_AMIGA_MAKINGLIB */ + ++#if !__APPLE__ ++/* Patch doesn't work on apple */ ++#define NIX_BOEHM_PATCH_VERSION 1 ++#endif ++ + #ifdef __cplusplus + } /* extern "C" */ + #endif diff --git a/pthread_stop_world.c b/pthread_stop_world.c index b5d71e62..aed7b0bf 100644 --- a/pthread_stop_world.c diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e5e8d2b22..7dcd10d50 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -385,10 +385,12 @@ void initGC() StackAllocator::defaultAllocator = &boehmGCStackAllocator; +#if NIX_BOEHM_PATCH_VERSION != 1 /* Used to disable GC when entering coroutines on macOS */ - create_disable_gc = []() -> std::shared_ptr { + create_coro_gc_hook = []() -> std::shared_ptr { return std::make_shared(); }; +#endif /* Set the initial heap size to something fairly big (25% of physical RAM, up to a maximum of 384 MiB) so that in most cases diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 0bdeec0cd..6e53239f5 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -186,18 +186,17 @@ static DefaultStackAllocator defaultAllocatorSingleton; StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton; -std::shared_ptr (*create_disable_gc)() = []() -> std::shared_ptr { +std::shared_ptr (*create_coro_gc_hook)() = []() -> std::shared_ptr { return {}; }; /* This class is used for entry and exit hooks on coroutines */ class CoroutineContext { -#if __APPLE__ - /* Disable GC when entering the coroutine on macOS, since it doesn't find the main thread stack in this case. + /* Disable GC when entering the coroutine without the boehm patch, + * since it doesn't find the main thread stack in this case. * std::shared_ptr performs type-erasure, so it will call the right * deleter. */ - const std::shared_ptr disable_gc = create_disable_gc(); -#endif + const std::shared_ptr coro_gc_hook = create_coro_gc_hook(); public: CoroutineContext() {}; ~CoroutineContext() {}; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index b895db779..ba6dbf619 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -551,10 +551,10 @@ struct StackAllocator { static StackAllocator *defaultAllocator; }; -/* Disabling GC when entering a coroutine (on macos). +/* Disabling GC when entering a coroutine (without the boehm patch). mutable to avoid boehm gc dependency in libutil. */ -extern std::shared_ptr (*create_disable_gc)(); +extern std::shared_ptr (*create_coro_gc_hook)(); } From 62ddd8633c06c32baad71b72ce4c82c54c2df512 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Fri, 3 Mar 2023 11:43:47 +0100 Subject: [PATCH 51/75] Add talkative msg for coro gc debug --- src/libexpr/eval.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7dcd10d50..18cfd9531 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -386,6 +386,7 @@ void initGC() #if NIX_BOEHM_PATCH_VERSION != 1 + printTalkative("Unpatched BoehmGC, disabling GC inside coroutines"); /* Used to disable GC when entering coroutines on macOS */ create_coro_gc_hook = []() -> std::shared_ptr { return std::make_shared(); From 0746951be1563b1dd590690f9ee48a2fe964bd93 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 7 Apr 2023 09:55:28 -0400 Subject: [PATCH 52/75] Finish converting existing comments for internal API docs (#8146) * Finish converting existing comments for internal API docs 99% of this was just reformatting existing comments. Only two exceptions: - Expanded upon `BuildResult::status` compat note - Split up file-level `symbol-table.hh` doc comments to get per-definition docs Also fixed a few whitespace goofs, turning leading tabs to spaces and removing trailing spaces. Picking up from #8133 * Fix two things from comments * Use triple-backtick not indent for `dumpPath` * Convert GNU-style `\`..'` quotes to markdown style in API docs This will render correctly. --- src/libcmd/command-installable-value.cc | 4 +- src/libcmd/editor-for.hh | 6 +- src/libcmd/installable-value.cc | 2 +- src/libexpr/attr-path.hh | 4 +- src/libexpr/attr-set.hh | 26 +- src/libexpr/eval-cache.hh | 10 +- src/libexpr/eval-inline.hh | 4 +- src/libexpr/eval.hh | 238 ++++++--- src/libexpr/flake/flake.hh | 111 ++-- src/libexpr/flake/flakeref.hh | 14 +- src/libexpr/flake/lockfile.hh | 20 +- src/libexpr/get-drvs.hh | 22 +- src/libexpr/nixexpr.hh | 16 +- src/libexpr/primops.hh | 16 +- src/libexpr/symbol-table.hh | 26 +- src/libexpr/value.hh | 95 ++-- src/libexpr/value/context.hh | 46 +- src/libfetchers/fetchers.hh | 41 +- src/libmain/shared.hh | 60 ++- src/libstore/binary-cache-store.hh | 18 +- src/libstore/build-result.hh | 56 +- src/libstore/build/derivation-goal.hh | 174 +++++-- .../build/drv-output-substitution-goal.hh | 34 +- src/libstore/build/goal.hh | 70 ++- src/libstore/build/hook-instance.hh | 16 +- src/libstore/build/local-derivation-goal.hh | 221 +++++--- src/libstore/build/substitution-goal.hh | 62 ++- src/libstore/build/worker.hh | 186 ++++--- src/libstore/crypto.hh | 16 +- src/libstore/derivations.hh | 16 +- src/libstore/fs-accessor.hh | 17 +- src/libstore/gc-store.hh | 85 ++-- src/libstore/globals.hh | 64 ++- src/libstore/local-fs-store.hh | 4 +- src/libstore/local-store.hh | 145 ++++-- src/libstore/lock.hh | 14 +- src/libstore/log-store.hh | 6 +- src/libstore/nar-accessor.hh | 22 +- src/libstore/nar-info-disk-cache.hh | 6 +- src/libstore/path-info.hh | 83 +-- src/libstore/path-with-outputs.hh | 23 +- src/libstore/pathlocks.hh | 12 +- src/libstore/profiles.hh | 42 +- src/libstore/remote-store.hh | 14 +- src/libstore/sqlite.hh | 42 +- src/libstore/store-api.hh | 22 +- src/libutil/ansicolor.hh | 7 +- src/libutil/archive.hh | 52 +- src/libutil/canon-path.hh | 6 +- src/libutil/error.hh | 32 +- src/libutil/logging.hh | 23 +- src/libutil/thread-pool.hh | 16 +- src/libutil/util.hh | 478 ++++++++++++------ 53 files changed, 1907 insertions(+), 938 deletions(-) diff --git a/src/libcmd/command-installable-value.cc b/src/libcmd/command-installable-value.cc index d7581534b..7e0c15eb8 100644 --- a/src/libcmd/command-installable-value.cc +++ b/src/libcmd/command-installable-value.cc @@ -4,8 +4,8 @@ namespace nix { void InstallableValueCommand::run(ref store, ref installable) { - auto installableValue = InstallableValue::require(installable); - run(store, installableValue); + auto installableValue = InstallableValue::require(installable); + run(store, installableValue); } } diff --git a/src/libcmd/editor-for.hh b/src/libcmd/editor-for.hh index f752bd849..c8c4e9d9b 100644 --- a/src/libcmd/editor-for.hh +++ b/src/libcmd/editor-for.hh @@ -5,8 +5,10 @@ namespace nix { -/* Helper function to generate args that invoke $EDITOR on - filename:lineno. */ +/** + * Helper function to generate args that invoke $EDITOR on + * filename:lineno. + */ Strings editorFor(const Path & file, uint32_t line); } diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 30f80edb2..3a7ede4e2 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -22,7 +22,7 @@ InstallableValue::getCursor(EvalState & state) static UsageError nonValueInstallable(Installable & installable) { - return UsageError("installable '%s' does not correspond to a Nix language value", installable.what()); + return UsageError("installable '%s' does not correspond to a Nix language value", installable.what()); } InstallableValue & InstallableValue::require(Installable & installable) diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index d0d05b1a1..b2bfb5d04 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -17,7 +17,9 @@ std::pair findAlongAttrPath( Bindings & autoArgs, Value & vIn); -/* Heuristic to find the filename and lineno or a nix value. */ +/** + * Heuristic to find the filename and lineno or a nix value. + */ std::pair findPackageFilename(EvalState & state, Value & v, std::string what); std::vector parseAttrPath(EvalState & state, std::string_view s); diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 3fe54408b..31215f880 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -13,7 +13,9 @@ namespace nix { class EvalState; struct Value; -/* Map one attribute name to its value. */ +/** + * Map one attribute name to its value. + */ struct Attr { /* the placement of `name` and `pos` in this struct is important. @@ -37,10 +39,12 @@ static_assert(sizeof(Attr) == 2 * sizeof(uint32_t) + sizeof(Value *), "avoid introducing any padding into Attr if at all possible, and do not " "introduce new fields that need not be present for almost every instance."); -/* Bindings contains all the attributes of an attribute set. It is defined - by its size and its capacity, the capacity being the number of Attr - elements allocated after this structure, while the size corresponds to - the number of elements already inserted in this structure. */ +/** + * Bindings contains all the attributes of an attribute set. It is defined + * by its size and its capacity, the capacity being the number of Attr + * elements allocated after this structure, while the size corresponds to + * the number of elements already inserted in this structure. + */ class Bindings { public: @@ -95,7 +99,9 @@ public: size_t capacity() { return capacity_; } - /* Returns the attributes in lexicographically sorted order. */ + /** + * Returns the attributes in lexicographically sorted order. + */ std::vector lexicographicOrder(const SymbolTable & symbols) const { std::vector res; @@ -112,9 +118,11 @@ public: friend class EvalState; }; -/* A wrapper around Bindings that ensures that its always in sorted - order at the end. The only way to consume a BindingsBuilder is to - call finish(), which sorts the bindings. */ +/** + * A wrapper around Bindings that ensures that its always in sorted + * order at the end. The only way to consume a BindingsBuilder is to + * call finish(), which sorts the bindings. + */ class BindingsBuilder { Bindings * bindings; diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index c90882edc..46c4999c8 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -110,8 +110,10 @@ public: ref getAttr(std::string_view name); - /* Get an attribute along a chain of attrsets. Note that this does - not auto-call functors or functions. */ + /** + * Get an attribute along a chain of attrsets. Note that this does + * not auto-call functors or functions. + */ OrSuggestions> findAlongAttrPath(const std::vector & attrPath, bool force = false); std::string getString(); @@ -130,7 +132,9 @@ public: Value & forceValue(); - /* Force creation of the .drv file in the Nix store. */ + /** + * Force creation of the .drv file in the Nix store. + */ StorePath forceDerivation(); }; diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index f8ddd2acc..a988fa40c 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -5,7 +5,9 @@ namespace nix { -/* Note: Various places expect the allocated memory to be zeroed. */ +/** + * Note: Various places expect the allocated memory to be zeroed. + */ [[gnu::always_inline]] inline void * allocBytes(size_t n) { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index a1b54951e..b3b985683 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -43,7 +43,10 @@ struct PrimOp struct Env { Env * up; - unsigned short prevWith:14; // nr of levels up to next `with' environment + /** + * Number of of levels up to next `with` environment + */ + unsigned short prevWith:14; enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2; Value * values[0]; }; @@ -56,8 +59,10 @@ std::unique_ptr mapStaticEnvBindings(const SymbolTable & st, const Stati void copyContext(const Value & v, PathSet & context); -/* Cache for calls to addToStore(); maps source paths to the store - paths. */ +/** + * Cache for calls to addToStore(); maps source paths to the store + * paths. + */ typedef std::map SrcToStore; @@ -69,7 +74,9 @@ typedef std::pair SearchPathElem; typedef std::list SearchPath; -/* Initialise the Boehm GC, if applicable. */ +/** + * Initialise the Boehm GC, if applicable. + */ void initGC(); @@ -144,26 +151,36 @@ public: sOutputSpecified; Symbol sDerivationNix; - /* If set, force copying files to the Nix store even if they - already exist there. */ + /** + * If set, force copying files to the Nix store even if they + * already exist there. + */ RepairFlag repair; - /* The allowed filesystem paths in restricted or pure evaluation - mode. */ + /** + * The allowed filesystem paths in restricted or pure evaluation + * mode. + */ std::optional allowedPaths; Bindings emptyBindings; - /* Store used to materialise .drv files. */ + /** + * Store used to materialise .drv files. + */ const ref store; - /* Store used to build stuff. */ + /** + * Store used to build stuff. + */ const ref buildStore; RootValue vCallFlake = nullptr; RootValue vImportedDrvToDerivation = nullptr; - /* Debugger */ + /** + * Debugger + */ void (* debugRepl)(ref es, const ValMap & extraEnv); bool debugStop; bool debugQuit; @@ -219,7 +236,9 @@ public: private: SrcToStore srcToStore; - /* A cache from path names to parse trees. */ + /** + * A cache from path names to parse trees. + */ #if HAVE_BOEHMGC typedef std::map, traceable_allocator>> FileParseCache; #else @@ -227,7 +246,9 @@ private: #endif FileParseCache fileParseCache; - /* A cache from path names to values. */ + /** + * A cache from path names to values. + */ #if HAVE_BOEHMGC typedef std::map, traceable_allocator>> FileEvalCache; #else @@ -239,17 +260,25 @@ private: std::map> searchPathResolved; - /* Cache used by checkSourcePath(). */ + /** + * Cache used by checkSourcePath(). + */ std::unordered_map resolvedPaths; - /* Cache used by prim_match(). */ + /** + * Cache used by prim_match(). + */ std::shared_ptr regexCache; #if HAVE_BOEHMGC - /* Allocation cache for GC'd Value objects. */ + /** + * Allocation cache for GC'd Value objects. + */ std::shared_ptr valueAllocCache; - /* Allocation cache for size-1 Env objects. */ + /** + * Allocation cache for size-1 Env objects. + */ std::shared_ptr env1AllocCache; #endif @@ -265,47 +294,65 @@ public: SearchPath getSearchPath() { return searchPath; } - /* Allow access to a path. */ + /** + * Allow access to a path. + */ void allowPath(const Path & path); - /* Allow access to a store path. Note that this gets remapped to - the real store path if `store` is a chroot store. */ + /** + * Allow access to a store path. Note that this gets remapped to + * the real store path if `store` is a chroot store. + */ void allowPath(const StorePath & storePath); - /* Allow access to a store path and return it as a string. */ + /** + * Allow access to a store path and return it as a string. + */ void allowAndSetStorePathString(const StorePath & storePath, Value & v); - /* Check whether access to a path is allowed and throw an error if - not. Otherwise return the canonicalised path. */ + /** + * Check whether access to a path is allowed and throw an error if + * not. Otherwise return the canonicalised path. + */ Path checkSourcePath(const Path & path); void checkURI(const std::string & uri); - /* When using a diverted store and 'path' is in the Nix store, map - 'path' to the diverted location (e.g. /nix/store/foo is mapped - to /home/alice/my-nix/nix/store/foo). However, this is only - done if the context is not empty, since otherwise we're - probably trying to read from the actual /nix/store. This is - intended to distinguish between import-from-derivation and - sources stored in the actual /nix/store. */ + /** + * When using a diverted store and 'path' is in the Nix store, map + * 'path' to the diverted location (e.g. /nix/store/foo is mapped + * to /home/alice/my-nix/nix/store/foo). However, this is only + * done if the context is not empty, since otherwise we're + * probably trying to read from the actual /nix/store. This is + * intended to distinguish between import-from-derivation and + * sources stored in the actual /nix/store. + */ Path toRealPath(const Path & path, const PathSet & context); - /* Parse a Nix expression from the specified file. */ + /** + * Parse a Nix expression from the specified file. + */ Expr * parseExprFromFile(const Path & path); Expr * parseExprFromFile(const Path & path, std::shared_ptr & staticEnv); - /* Parse a Nix expression from the specified string. */ + /** + * Parse a Nix expression from the specified string. + */ Expr * parseExprFromString(std::string s, const Path & basePath, std::shared_ptr & staticEnv); Expr * parseExprFromString(std::string s, const Path & basePath); Expr * parseStdin(); - /* Evaluate an expression read from the given file to normal - form. Optionally enforce that the top-level expression is - trivial (i.e. doesn't require arbitrary computation). */ + /** + * Evaluate an expression read from the given file to normal + * form. Optionally enforce that the top-level expression is + * trivial (i.e. doesn't require arbitrary computation). + */ void evalFile(const Path & path, Value & v, bool mustBeTrivial = false); - /* Like `evalFile`, but with an already parsed expression. */ + /** + * Like `evalFile`, but with an already parsed expression. + */ void cacheFile( const Path & path, const Path & resolvedPath, @@ -315,37 +362,52 @@ public: void resetFileCache(); - /* Look up a file in the search path. */ + /** + * Look up a file in the search path. + */ Path findFile(const std::string_view path); Path findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); - /* If the specified search path element is a URI, download it. */ + /** + * If the specified search path element is a URI, download it. + */ std::pair resolveSearchPathElem(const SearchPathElem & elem); - /* Evaluate an expression to normal form, storing the result in - value `v'. */ + /** + * Evaluate an expression to normal form + * + * @param [out] v The resulting is stored here. + */ void eval(Expr * e, Value & v); - /* Evaluation the expression, then verify that it has the expected - type. */ + /** + * Evaluation the expression, then verify that it has the expected + * type. + */ inline bool evalBool(Env & env, Expr * e); inline bool evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx); inline void evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx); - /* If `v' is a thunk, enter it and overwrite `v' with the result - of the evaluation of the thunk. If `v' is a delayed function - application, call the function and overwrite `v' with the - result. Otherwise, this is a no-op. */ + /** + * If `v` is a thunk, enter it and overwrite `v` with the result + * of the evaluation of the thunk. If `v` is a delayed function + * application, call the function and overwrite `v` with the + * result. Otherwise, this is a no-op. + */ inline void forceValue(Value & v, const PosIdx pos); template inline void forceValue(Value & v, Callable getPos); - /* Force a value, then recursively force list elements and - attributes. */ + /** + * Force a value, then recursively force list elements and + * attributes. + */ void forceValueDeep(Value & v); - /* Force `v', and then verify that it has the expected type. */ + /** + * Force `v`, and then verify that it has the expected type. + */ NixInt forceInt(Value & v, const PosIdx pos, std::string_view errorCtx); NixFloat forceFloat(Value & v, const PosIdx pos, std::string_view errorCtx); bool forceBool(Value & v, const PosIdx pos, std::string_view errorCtx); @@ -356,7 +418,10 @@ public: inline void forceAttrs(Value & v, Callable getPos, std::string_view errorCtx); inline void forceList(Value & v, const PosIdx pos, std::string_view errorCtx); - void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx); // either lambda or primop + /** + * @param v either lambda or primop + */ + void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx); std::string_view forceString(Value & v, const PosIdx pos, std::string_view errorCtx); std::string_view forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx); std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx); @@ -367,17 +432,23 @@ public: void addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame = false) const; public: - /* Return true iff the value `v' denotes a derivation (i.e. a - set with attribute `type = "derivation"'). */ + /** + * @return true iff the value `v` denotes a derivation (i.e. a + * set with attribute `type = "derivation"`). + */ bool isDerivation(Value & v); std::optional tryAttrsToString(const PosIdx pos, Value & v, PathSet & context, bool coerceMore = false, bool copyToStore = true); - /* String coercion. Converts strings, paths and derivations to a - string. If `coerceMore' is set, also converts nulls, integers, - booleans and lists to a string. If `copyToStore' is set, - referenced paths are copied to the Nix store as a side effect. */ + /** + * String coercion. + * + * Converts strings, paths and derivations to a + * string. If `coerceMore` is set, also converts nulls, integers, + * booleans and lists to a string. If `copyToStore` is set, + * referenced paths are copied to the Nix store as a side effect. + */ BackedStringView coerceToString(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx, bool coerceMore = false, bool copyToStore = true, @@ -385,21 +456,31 @@ public: StorePath copyPathToStore(PathSet & context, const Path & path); - /* Path coercion. Converts strings, paths and derivations to a - path. The result is guaranteed to be a canonicalised, absolute - path. Nothing is copied to the store. */ + /** + * Path coercion. + * + * Converts strings, paths and derivations to a + * path. The result is guaranteed to be a canonicalised, absolute + * path. Nothing is copied to the store. + */ Path coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); - /* Like coerceToPath, but the result must be a store path. */ + /** + * Like coerceToPath, but the result must be a store path. + */ StorePath coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); public: - /* The base environment, containing the builtin functions and - values. */ + /** + * The base environment, containing the builtin functions and + * values. + */ Env & baseEnv; - /* The same, but used during parsing to resolve variables. */ + /** + * The same, but used during parsing to resolve variables. + */ std::shared_ptr staticBaseEnv; // !!! should be private private: @@ -449,8 +530,10 @@ private: public: - /* Do a deep equality test between two values. That is, list - elements and attributes are compared recursively. */ + /** + * Do a deep equality test between two values. That is, list + * elements and attributes are compared recursively. + */ bool eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx); bool isFunctor(Value & fun); @@ -464,11 +547,15 @@ public: callFunction(fun, 1, args, vRes, pos); } - /* Automatically call a function for which each argument has a - default value or has a binding in the `args' map. */ + /** + * Automatically call a function for which each argument has a + * default value or has a binding in the `args` map. + */ void autoCallFunction(Bindings & args, Value & fun, Value & res); - /* Allocation primitives. */ + /** + * Allocation primitives. + */ inline Value * allocValue(); inline Env & allocEnv(size_t size); @@ -488,10 +575,13 @@ public: void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); - /* Print statistics. */ + /** + * Print statistics. + */ void printStats(); - /* Realise the given context, and return a mapping from the placeholders + /** + * Realise the given context, and return a mapping from the placeholders * used to construct the associated value to their final store path */ [[nodiscard]] StringMap realiseContext(const PathSet & context); @@ -551,11 +641,15 @@ struct DebugTraceStacker { DebugTrace trace; }; -/* Return a string representing the type of the value `v'. */ +/** + * @return A string representing the type of the value `v`. + */ std::string_view showType(ValueType type); std::string showType(const Value & v); -/* If `path' refers to a directory, then append "/default.nix". */ +/** + * If `path` refers to a directory, then append "/default.nix". + */ Path resolveExprPath(Path path); struct InvalidPathError : EvalError diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index b6f710288..c1d1b71e5 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -18,7 +18,8 @@ struct FlakeInput; typedef std::map FlakeInputs; -/* FlakeInput is the 'Flake'-level parsed form of the "input" entries +/** + * FlakeInput is the 'Flake'-level parsed form of the "input" entries * in the flake file. * * A FlakeInput is normally constructed by the 'parseFlakeInput' @@ -42,7 +43,12 @@ typedef std::map FlakeInputs; struct FlakeInput { std::optional ref; - bool isFlake = true; // true = process flake to get outputs, false = (fetched) static source path + /** + * true = process flake to get outputs + * + * false = (fetched) static source path + */ + bool isFlake = true; std::optional follows; FlakeInputs overrides; }; @@ -56,23 +62,42 @@ struct ConfigFile void apply(); }; -/* The contents of a flake.nix file. */ +/** + * The contents of a flake.nix file. + */ struct Flake { - FlakeRef originalRef; // the original flake specification (by the user) - FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake - FlakeRef lockedRef; // the specific local store result of invoking the fetcher - bool forceDirty = false; // pretend that 'lockedRef' is dirty + /** + * The original flake specification (by the user) + */ + FlakeRef originalRef; + /** + * registry references and caching resolved to the specific underlying flake + */ + FlakeRef resolvedRef; + /** + * the specific local store result of invoking the fetcher + */ + FlakeRef lockedRef; + /** + * pretend that 'lockedRef' is dirty + */ + bool forceDirty = false; std::optional description; std::shared_ptr sourceInfo; FlakeInputs inputs; - ConfigFile config; // 'nixConfig' attribute + /** + * 'nixConfig' attribute + */ + ConfigFile config; ~Flake(); }; Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup); -/* Fingerprint of a locked flake; used as a cache key. */ +/** + * Fingerprint of a locked flake; used as a cache key. + */ typedef Hash Fingerprint; struct LockedFlake @@ -85,50 +110,72 @@ struct LockedFlake struct LockFlags { - /* Whether to ignore the existing lock file, creating a new one - from scratch. */ + /** + * Whether to ignore the existing lock file, creating a new one + * from scratch. + */ bool recreateLockFile = false; - /* Whether to update the lock file at all. If set to false, if any - change to the lock file is needed (e.g. when an input has been - added to flake.nix), you get a fatal error. */ + /** + * Whether to update the lock file at all. If set to false, if any + * change to the lock file is needed (e.g. when an input has been + * added to flake.nix), you get a fatal error. + */ bool updateLockFile = true; - /* Whether to write the lock file to disk. If set to true, if the - any changes to the lock file are needed and the flake is not - writable (i.e. is not a local Git working tree or similar), you - get a fatal error. If set to false, Nix will use the modified - lock file in memory only, without writing it to disk. */ + /** + * Whether to write the lock file to disk. If set to true, if the + * any changes to the lock file are needed and the flake is not + * writable (i.e. is not a local Git working tree or similar), you + * get a fatal error. If set to false, Nix will use the modified + * lock file in memory only, without writing it to disk. + */ bool writeLockFile = true; - /* Whether to use the registries to lookup indirect flake - references like 'nixpkgs'. */ + /** + * Whether to use the registries to lookup indirect flake + * references like 'nixpkgs'. + */ std::optional useRegistries = std::nullopt; - /* Whether to apply flake's nixConfig attribute to the configuration */ + /** + * Whether to apply flake's nixConfig attribute to the configuration + */ bool applyNixConfig = false; - /* Whether unlocked flake references (i.e. those without a Git - revision or similar) without a corresponding lock are - allowed. Unlocked flake references with a lock are always - allowed. */ + /** + * Whether unlocked flake references (i.e. those without a Git + * revision or similar) without a corresponding lock are + * allowed. Unlocked flake references with a lock are always + * allowed. + */ bool allowUnlocked = true; - /* Whether to commit changes to flake.lock. */ + /** + * Whether to commit changes to flake.lock. + */ bool commitLockFile = false; - /* The path to a lock file to read instead of the `flake.lock` file in the top-level flake */ + /** + * The path to a lock file to read instead of the `flake.lock` file in the top-level flake + */ std::optional referenceLockFilePath; - /* The path to a lock file to write to instead of the `flake.lock` file in the top-level flake */ + /** + * The path to a lock file to write to instead of the `flake.lock` file in the top-level flake + */ std::optional outputLockFilePath; - /* Flake inputs to be overridden. */ + /** + * Flake inputs to be overridden. + */ std::map inputOverrides; - /* Flake inputs to be updated. This means that any existing lock - for those inputs will be ignored. */ + /** + * Flake inputs to be updated. This means that any existing lock + * for those inputs will be ignored. + */ std::set inputUpdates; }; diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 23d19adb1..a7c9208c0 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -14,7 +14,8 @@ class Store; typedef std::string FlakeId; -/* A flake reference specifies how to fetch a flake or raw source +/** + * A flake reference specifies how to fetch a flake or raw source * (e.g. from a Git repository). It is created from a URL-like syntax * (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{ * type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local @@ -33,14 +34,17 @@ typedef std::string FlakeId; * be lazy), but the fetcher can be invoked at any time via the * FlakeRef to ensure the store is populated with this input. */ - struct FlakeRef { - /* Fetcher-specific representation of the input, sufficient to - perform the fetch operation. */ + /** + * Fetcher-specific representation of the input, sufficient to + * perform the fetch operation. + */ fetchers::Input input; - /* sub-path within the fetched input that represents this input */ + /** + * sub-path within the fetched input that represents this input + */ Path subdir; bool operator==(const FlakeRef & other) const; diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 6512509c5..0ac731b5d 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -16,9 +16,11 @@ typedef std::vector InputPath; struct LockedNode; -/* A node in the lock file. It has outgoing edges to other nodes (its - inputs). Only the root node has this type; all other nodes have - type LockedNode. */ +/** + * A node in the lock file. It has outgoing edges to other nodes (its + * inputs). Only the root node has this type; all other nodes have + * type LockedNode. + */ struct Node : std::enable_shared_from_this { typedef std::variant, InputPath> Edge; @@ -28,7 +30,9 @@ struct Node : std::enable_shared_from_this virtual ~Node() { } }; -/* A non-root node in the lock file. */ +/** + * A non-root node in the lock file. + */ struct LockedNode : Node { FlakeRef lockedRef, originalRef; @@ -63,7 +67,9 @@ struct LockFile void write(const Path & path) const; - /* Check whether this lock file has any unlocked inputs. */ + /** + * Check whether this lock file has any unlocked inputs. + */ std::optional isUnlocked() const; bool operator ==(const LockFile & other) const; @@ -74,7 +80,9 @@ struct LockFile static std::string diff(const LockFile & oldLocks, const LockFile & newLocks); - /* Check that every 'follows' input target exists. */ + /** + * Check that every 'follows' input target exists. + */ void check(); }; diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 51ef7782a..584d64ac1 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -26,7 +26,10 @@ private: mutable std::string outputName; Outputs outputs; - bool failed = false; // set if we get an AssertionError + /** + * Set if we get an AssertionError + */ + bool failed = false; Bindings * attrs = nullptr, * meta = nullptr; @@ -35,7 +38,10 @@ private: bool checkMeta(Value & v); public: - std::string attrPath; /* path towards the derivation */ + /** + * path towards the derivation + */ + std::string attrPath; DrvInfo(EvalState & state) : state(&state) { }; DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs); @@ -47,8 +53,10 @@ public: StorePath requireDrvPath() const; StorePath queryOutPath() const; std::string queryOutputName() const; - /** Return the unordered map of output names to (optional) output paths. - * The "outputs to install" are determined by `meta.outputsToInstall`. */ + /** + * Return the unordered map of output names to (optional) output paths. + * The "outputs to install" are determined by `meta.outputsToInstall`. + */ Outputs queryOutputs(bool withPaths = true, bool onlyOutputsToInstall = false); StringSet queryMetaNames(); @@ -80,8 +88,10 @@ typedef std::list DrvInfos; #endif -/* If value `v' denotes a derivation, return a DrvInfo object - describing it. Otherwise return nothing. */ +/** + * If value `v` denotes a derivation, return a DrvInfo object + * describing it. Otherwise return nothing. + */ std::optional getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures); diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 4079a7b24..c2f817c9a 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -22,7 +22,9 @@ MakeError(UndefinedVarError, Error); MakeError(MissingArgumentError, EvalError); MakeError(RestrictedPathError, Error); -/* Position objects. */ +/** + * Position objects. + */ struct Pos { uint32_t line; @@ -133,7 +135,9 @@ class EvalState; struct StaticEnv; -/* An attribute path is a sequence of attribute names. */ +/** + * An attribute path is a sequence of attribute names. + */ struct AttrName { Symbol symbol; @@ -213,11 +217,11 @@ struct ExprVar : Expr or function argument) or from a "with". */ bool fromWith; - /* In the former case, the value is obtained by going `level' + /* In the former case, the value is obtained by going `level` levels up from the current environment and getting the - `displ'th value in that environment. In the latter case, the - value is obtained by getting the attribute named `name' from - the set stored in the environment that is `level' levels up + `displ`th value in that environment. In the latter case, the + value is obtained by getting the attribute named `name` from + the set stored in the environment that is `level` levels up from the current one.*/ Level level; Displacement displ; diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 1c5ce219f..4ae73fe1f 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -23,9 +23,11 @@ struct RegisterPrimOp typedef std::vector PrimOps; static PrimOps * primOps; - /* You can register a constant by passing an arity of 0. fun - will get called during EvalState initialization, so there - may be primops not yet added and builtins is not yet sorted. */ + /** + * You can register a constant by passing an arity of 0. fun + * will get called during EvalState initialization, so there + * may be primops not yet added and builtins is not yet sorted. + */ RegisterPrimOp( std::string name, size_t arity, @@ -38,10 +40,14 @@ struct RegisterPrimOp may wish to use them in limited contexts without globally enabling them. */ -/* Load a ValueInitializer from a DSO and return whatever it initializes */ +/** + * Load a ValueInitializer from a DSO and return whatever it initializes + */ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Value & v); -/* Execute a program and parse its output */ +/** + * Execute a program and parse its output + */ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v); } diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index c97a0a2db..967a186dd 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -10,15 +10,11 @@ namespace nix { -/* Symbol table used by the parser and evaluator to represent and look - up identifiers and attributes efficiently. SymbolTable::create() - converts a string into a symbol. Symbols have the property that - they can be compared efficiently (using an equality test), - because the symbol table stores only one copy of each string. */ - -/* This class mainly exists to give us an operator<< for ostreams. We could also - return plain strings from SymbolTable, but then we'd have to wrap every - instance of a symbol that is fmt()ed, which is inconvenient and error-prone. */ +/** + * This class mainly exists to give us an operator<< for ostreams. We could also + * return plain strings from SymbolTable, but then we'd have to wrap every + * instance of a symbol that is fmt()ed, which is inconvenient and error-prone. + */ class SymbolStr { friend class SymbolTable; @@ -47,6 +43,11 @@ public: friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol); }; +/** + * Symbols have the property that they can be compared efficiently + * (using an equality test), because the symbol table stores only one + * copy of each string. + */ class Symbol { friend class SymbolTable; @@ -66,6 +67,10 @@ public: bool operator!=(const Symbol other) const { return id != other.id; } }; +/** + * Symbol table used by the parser and evaluator to represent and look + * up identifiers and attributes efficiently. + */ class SymbolTable { private: @@ -74,6 +79,9 @@ private: public: + /** + * converts a string into a symbol. + */ Symbol create(std::string_view s) { // Most symbols are looked up more than once, so we trade off insertion performance diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index bfae4ee94..7739f99df 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -36,9 +36,11 @@ typedef enum { tFloat } InternalType; -// This type abstracts over all actual value types in the language, -// grouping together implementation details like tList*, different function -// types, and types in non-normal form (so thunks and co.) +/** + * This type abstracts over all actual value types in the language, + * grouping together implementation details like tList*, different function + * types, and types in non-normal form (so thunks and co.) + */ typedef enum { nThunk, nInt, @@ -70,38 +72,51 @@ class XMLWriter; typedef int64_t NixInt; typedef double NixFloat; -/* External values must descend from ExternalValueBase, so that +/** + * External values must descend from ExternalValueBase, so that * type-agnostic nix functions (e.g. showType) can be implemented */ class ExternalValueBase { friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); protected: - /* Print out the value */ + /** + * Print out the value + */ virtual std::ostream & print(std::ostream & str) const = 0; public: - /* Return a simple string describing the type */ + /** + * Return a simple string describing the type + */ virtual std::string showType() const = 0; - /* Return a string to be used in builtins.typeOf */ + /** + * Return a string to be used in builtins.typeOf + */ virtual std::string typeOf() const = 0; - /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an + /** + * Coerce the value to a string. Defaults to uncoercable, i.e. throws an * error. */ virtual std::string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const; - /* Compare to another value of the same type. Defaults to uncomparable, + /** + * Compare to another value of the same type. Defaults to uncomparable, * i.e. always false. */ virtual bool operator ==(const ExternalValueBase & b) const; - /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ + /** + * Print the value as JSON. Defaults to unconvertable, i.e. throws an error + */ virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict, PathSet & context, bool copyToStore = true) const; - /* Print the value as XML. Defaults to unevaluated */ + /** + * Print the value as XML. Defaults to unevaluated + */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, const PosIdx pos) const; @@ -146,26 +161,28 @@ public: NixInt integer; bool boolean; - /* Strings in the evaluator carry a so-called `context' which - is a list of strings representing store paths. This is to - allow users to write things like + /** + * Strings in the evaluator carry a so-called `context` which + * is a list of strings representing store paths. This is to + * allow users to write things like - "--with-freetype2-library=" + freetype + "/lib" + * "--with-freetype2-library=" + freetype + "/lib" - where `freetype' is a derivation (or a source to be copied - to the store). If we just concatenated the strings without - keeping track of the referenced store paths, then if the - string is used as a derivation attribute, the derivation - will not have the correct dependencies in its inputDrvs and - inputSrcs. + * where `freetype` is a derivation (or a source to be copied + * to the store). If we just concatenated the strings without + * keeping track of the referenced store paths, then if the + * string is used as a derivation attribute, the derivation + * will not have the correct dependencies in its inputDrvs and + * inputSrcs. - The semantics of the context is as follows: when a string - with context C is used as a derivation attribute, then the - derivations in C will be added to the inputDrvs of the - derivation, and the other store paths in C will be added to - the inputSrcs of the derivations. + * The semantics of the context is as follows: when a string + * with context C is used as a derivation attribute, then the + * derivations in C will be added to the inputDrvs of the + * derivation, and the other store paths in C will be added to + * the inputSrcs of the derivations. - For canonicity, the store paths should be in sorted order. */ + * For canonicity, the store paths should be in sorted order. + */ struct { const char * s; const char * * context; // must be in sorted order @@ -197,8 +214,10 @@ public: NixFloat fpoint; }; - // Returns the normal type of a Value. This only returns nThunk if the - // Value hasn't been forceValue'd + /** + * Returns the normal type of a Value. This only returns nThunk if + * the Value hasn't been forceValue'd + */ inline ValueType type() const { switch (internalType) { @@ -217,8 +236,10 @@ public: abort(); } - /* After overwriting an app node, be sure to clear pointers in the - Value to ensure that the target isn't kept alive unnecessarily. */ + /** + * After overwriting an app node, be sure to clear pointers in the + * Value to ensure that the target isn't kept alive unnecessarily. + */ inline void clearValue() { app.left = app.right = 0; @@ -366,9 +387,11 @@ public: PosIdx determinePos(const PosIdx pos) const; - /* Check whether forcing this value requires a trivial amount of - computation. In particular, function applications are - non-trivial. */ + /** + * Check whether forcing this value requires a trivial amount of + * computation. In particular, function applications are + * non-trivial. + */ bool isTrivial() const; NixStringContext getContext(const Store &); @@ -414,7 +437,9 @@ typedef std::map ValueVectorMap; #endif -/* A value allocated in traceable memory. */ +/** + * A value allocated in traceable memory. + */ typedef std::shared_ptr RootValue; RootValue allocRootValue(Value * v); diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index d467b4f1d..8719602d8 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -28,34 +28,37 @@ public: class Store; -/* Plain opaque path to some store object. - - Encoded as just the path: ‘’. -*/ +/** + * Plain opaque path to some store object. + * + * Encoded as just the path: ‘’. + */ struct NixStringContextElem_Opaque { StorePath path; GENERATE_CMP(NixStringContextElem_Opaque, me->path); }; -/* Path to a derivation and its entire build closure. - - The path doesn't just refer to derivation itself and its closure, but - also all outputs of all derivations in that closure (including the - root derivation). - - Encoded in the form ‘=’. -*/ +/** + * Path to a derivation and its entire build closure. + * + * The path doesn't just refer to derivation itself and its closure, but + * also all outputs of all derivations in that closure (including the + * root derivation). + * + * Encoded in the form ‘=’. + */ struct NixStringContextElem_DrvDeep { StorePath drvPath; GENERATE_CMP(NixStringContextElem_DrvDeep, me->drvPath); }; -/* Derivation output. - - Encoded in the form ‘!!’. -*/ +/** + * Derivation output. + * + * Encoded in the form ‘!!’. + */ struct NixStringContextElem_Built { StorePath drvPath; std::string output; @@ -84,11 +87,12 @@ struct NixStringContextElem : _NixStringContextElem_Raw { return static_cast(*this); } - /* Decode a context string, one of: - - ‘’ - - ‘=’ - - ‘!!’ - */ + /** + * Decode a context string, one of: + * - ‘’ + * - ‘=’ + * - ‘!!’ + */ static NixStringContextElem parse(const Store & store, std::string_view s); std::string to_string(const Store & store) const; }; diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index acdecea57..498ad7e4d 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -21,14 +21,14 @@ struct Tree struct InputScheme; -/* The Input object is generated by a specific fetcher, based on the +/** + * The Input object is generated by a specific fetcher, based on the * user-supplied input attribute in the flake.nix file, and contains * the information that the specific fetcher needs to perform the * actual fetch. The Input object is most commonly created via the * "fromURL()" or "fromAttrs()" static functions which are provided * the url or attrset specified in the flake file. */ - struct Input { friend struct InputScheme; @@ -38,7 +38,9 @@ struct Input bool locked = false; bool direct = true; - /* path of the parent of this input, used for relative path resolution */ + /** + * path of the parent of this input, used for relative path resolution + */ std::optional parent; public: @@ -56,27 +58,35 @@ public: Attrs toAttrs() const; - /* Check whether this is a "direct" input, that is, not - one that goes through a registry. */ + /** + * Check whether this is a "direct" input, that is, not + * one that goes through a registry. + */ bool isDirect() const { return direct; } - /* Check whether this is a "locked" input, that is, - one that contains a commit hash or content hash. */ + /** + * Check whether this is a "locked" input, that is, + * one that contains a commit hash or content hash. + */ bool isLocked() const { return locked; } - /* Check whether the input carries all necessary info required - for cache insertion and substitution. - These fields are used to uniquely identify cached trees - within the "tarball TTL" window without necessarily - indicating that the input's origin is unchanged. */ + /** + * Check whether the input carries all necessary info required + * for cache insertion and substitution. + * These fields are used to uniquely identify cached trees + * within the "tarball TTL" window without necessarily + * indicating that the input's origin is unchanged. + */ bool hasAllInfo() const; bool operator ==(const Input & other) const; bool contains(const Input & other) const; - /* Fetch the input into the Nix store, returning the location in - the Nix store and the locked input. */ + /** + * Fetch the input into the Nix store, returning the location in + * the Nix store and the locked input. + */ std::pair fetch(ref store) const; Input applyOverrides( @@ -105,7 +115,8 @@ public: }; -/* The InputScheme represents a type of fetcher. Each fetcher +/** + * The InputScheme represents a type of fetcher. Each fetcher * registers with nix at startup time. When processing an input for a * flake, each scheme is given an opportunity to "recognize" that * input from the url or attributes in the flake file's specification diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index d915a4a65..7a9e83c6c 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -25,7 +25,9 @@ public: int handleExceptions(const std::string & programName, std::function fun); -/* Don't forget to call initPlugins() after settings are initialized! */ +/** + * Don't forget to call initPlugins() after settings are initialized! + */ void initNix(); void parseCmdLine(int argc, char * * argv, @@ -36,7 +38,9 @@ void parseCmdLine(const std::string & programName, const Strings & args, void printVersion(const std::string & programName); -/* Ugh. No better place to put this. */ +/** + * Ugh. No better place to put this. + */ void printGCWarning(); class Store; @@ -75,11 +79,15 @@ struct LegacyArgs : public MixCommonArgs }; -/* Show the manual page for the specified program. */ +/** + * Show the manual page for the specified program. + */ void showManPage(const std::string & name); -/* The constructor of this class starts a pager if stdout is a - terminal and $PAGER is set. Stdout is redirected to the pager. */ +/** + * The constructor of this class starts a pager if stdout is a + * terminal and $PAGER is set. Stdout is redirected to the pager. + */ class RunPager { public: @@ -110,28 +118,34 @@ struct PrintFreed }; -/* Install a SIGSEGV handler to detect stack overflows. */ +/** + * Install a SIGSEGV handler to detect stack overflows. + */ void detectStackOverflow(); -/* Pluggable behavior to run in case of a stack overflow. - - Default value: defaultStackOverflowHandler. - - This is called by the handler installed by detectStackOverflow(). - - This gives Nix library consumers a limit opportunity to report the error - condition. The handler should exit the process. - See defaultStackOverflowHandler() for a reference implementation. - - NOTE: Use with diligence, because this runs in the signal handler, with very - limited stack space and a potentially a corrupted heap, all while the failed - thread is blocked indefinitely. All functions called must be reentrant. */ +/** + * Pluggable behavior to run in case of a stack overflow. + * + * Default value: defaultStackOverflowHandler. + * + * This is called by the handler installed by detectStackOverflow(). + * + * This gives Nix library consumers a limit opportunity to report the error + * condition. The handler should exit the process. + * See defaultStackOverflowHandler() for a reference implementation. + * + * NOTE: Use with diligence, because this runs in the signal handler, with very + * limited stack space and a potentially a corrupted heap, all while the failed + * thread is blocked indefinitely. All functions called must be reentrant. + */ extern std::function stackOverflowHandler; -/* The default, robust implementation of stackOverflowHandler. - - Prints an error message directly to stderr using a syscall instead of the - logger. Exits the process immediately after. */ +/** + * The default, robust implementation of stackOverflowHandler. + * + * Prints an error message directly to stderr using a syscall instead of the + * logger. Exits the process immediately after. + */ void defaultStackOverflowHandler(siginfo_t * info, void * ctx); } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 5e52d7844..49f271d24 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -46,6 +46,11 @@ struct BinaryCacheStoreConfig : virtual StoreConfig )"}; }; + +/** + * @note subclasses must implement at least one of the two + * virtual getFile() methods. + */ class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store, public virtual LogStore @@ -75,14 +80,15 @@ public: std::string && data, const std::string & mimeType); - /* Note: subclasses must implement at least one of the two - following getFile() methods. */ - - /* Dump the contents of the specified file to a sink. */ + /** + * Dump the contents of the specified file to a sink. + */ virtual void getFile(const std::string & path, Sink & sink); - /* Fetch the specified file and call the specified callback with - the result. A subclass may implement this asynchronously. */ + /** + * Fetch the specified file and call the specified callback with + * the result. A subclass may implement this asynchronously. + */ virtual void getFile( const std::string & path, Callback> callback) noexcept; diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index a12c599d9..27d1a1b6c 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -12,9 +12,12 @@ namespace nix { struct BuildResult { - /* Note: don't remove status codes, and only add new status codes - at the end of the list, to prevent client/server - incompatibilities in the nix-store --serve protocol. */ + /** + * @note This is directly used in the nix-store --serve protocol. + * That means we need to worry about compatability across versions. + * Therefore, don't remove status codes, and only add new status + * codes at the end of the list. + */ enum Status { Built = 0, Substituted, @@ -22,8 +25,10 @@ struct BuildResult PermanentFailure, InputRejected, OutputRejected, - TransientFailure, // possibly transient - CachedFailure, // no longer used + /// possibly transient + TransientFailure, + /// no longer used + CachedFailure, TimedOut, MiscFailure, DependencyFailed, @@ -33,7 +38,12 @@ struct BuildResult NoSubstituters, } status = MiscFailure; - // FIXME: include entire ErrorInfo object. + /** + * Information about the error if the build failed. + * + * @todo This should be an entire ErrorInfo object, not just a + * string, for richer information. + */ std::string errorMsg; std::string toString() const { @@ -60,27 +70,39 @@ struct BuildResult return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg); } - /* How many times this build was performed. */ + /** + * How many times this build was performed. + */ unsigned int timesBuilt = 0; - /* If timesBuilt > 1, whether some builds did not produce the same - result. (Note that 'isNonDeterministic = false' does not mean - the build is deterministic, just that we don't have evidence of - non-determinism.) */ + /** + * If timesBuilt > 1, whether some builds did not produce the same + * result. (Note that 'isNonDeterministic = false' does not mean + * the build is deterministic, just that we don't have evidence of + * non-determinism.) + */ bool isNonDeterministic = false; - /* The derivation we built or the store path we substituted. */ + /** + * The derivation we built or the store path we substituted. + */ DerivedPath path; - /* For derivations, a mapping from the names of the wanted outputs - to actual paths. */ + /** + * For derivations, a mapping from the names of the wanted outputs + * to actual paths. + */ DrvOutputs builtOutputs; - /* The start/stop times of the build (or one of the rounds, if it - was repeated). */ + /** + * The start/stop times of the build (or one of the rounds, if it + * was repeated). + */ time_t startTime = 0, stopTime = 0; - /* User and system CPU time the build took. */ + /** + * User and system CPU time the build took. + */ std::optional cpuUser, cpuSystem; bool success() diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index f43ce22af..3a6f0c2d9 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -16,8 +16,10 @@ struct HookInstance; typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ +/** + * Unless we are repairing, we don't both to test validity and just assume it, + * so the choices are `Absent` or `Valid`. + */ enum struct PathStatus { Corrupt, Absent, @@ -27,11 +29,15 @@ enum struct PathStatus { struct InitialOutputStatus { StorePath path; PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ + /** + * Valid in the store, and additionally non-corrupt if we are repairing + */ bool isValid() const { return status == PathStatus::Valid; } - /* Merely present, allowed to be corrupt */ + /** + * Merely present, allowed to be corrupt + */ bool isPresent() const { return status == PathStatus::Corrupt || status == PathStatus::Valid; @@ -46,59 +52,87 @@ struct InitialOutput { struct DerivationGoal : public Goal { - /* Whether to use an on-disk .drv file. */ + /** + * Whether to use an on-disk .drv file. + */ bool useDerivation; - /* The path of the derivation. */ + /** The path of the derivation. */ StorePath drvPath; - /* The goal for the corresponding resolved derivation */ + /** + * The goal for the corresponding resolved derivation + */ std::shared_ptr resolvedDrvGoal; - /* The specific outputs that we need to build. Empty means all of - them. */ + /** + * The specific outputs that we need to build. Empty means all of + * them. + */ OutputsSpec wantedOutputs; - /* Mapping from input derivations + output names to actual store - paths. This is filled in by waiteeDone() as each dependency - finishes, before inputsRealised() is reached, */ + /** + * Mapping from input derivations + output names to actual store + * paths. This is filled in by waiteeDone() as each dependency + * finishes, before inputsRealised() is reached. + */ std::map, StorePath> inputDrvOutputs; - /* Whether additional wanted outputs have been added. */ + /** + * Whether additional wanted outputs have been added. + */ bool needRestart = false; - /* Whether to retry substituting the outputs after building the - inputs. This is done in case of an incomplete closure. */ + /** + * Whether to retry substituting the outputs after building the + * inputs. This is done in case of an incomplete closure. + */ bool retrySubstitution = false; - /* Whether we've retried substitution, in which case we won't try - again. */ + /** + * Whether we've retried substitution, in which case we won't try + * again. + */ bool retriedSubstitution = false; - /* The derivation stored at drvPath. */ + /** + * The derivation stored at drvPath. + */ std::unique_ptr drv; std::unique_ptr parsedDrv; - /* The remainder is state held during the build. */ + /** + * The remainder is state held during the build. + */ - /* Locks on (fixed) output paths. */ + /** + * Locks on (fixed) output paths. + */ PathLocks outputLocks; - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ + /** + * All input paths (that is, the union of FS closures of the + * immediate input paths). + */ StorePathSet inputPaths; std::map initialOutputs; - /* File descriptor for the log file. */ + /** + * File descriptor for the log file. + */ AutoCloseFD fdLogFile; std::shared_ptr logFileSink, logSink; - /* Number of bytes received from the builder's stdout/stderr. */ + /** + * Number of bytes received from the builder's stdout/stderr. + */ unsigned long logSize; - /* The most recent log lines. */ + /** + * The most recent log lines. + */ std::list logTail; std::string currentLogLine; @@ -106,10 +140,14 @@ struct DerivationGoal : public Goal std::string currentHookLine; - /* The build hook. */ + /** + * The build hook. + */ std::unique_ptr hook; - /* The sort of derivation we are building. */ + /** + * The sort of derivation we are building. + */ DerivationType derivationType; typedef void (DerivationGoal::*GoalState)(); @@ -121,12 +159,16 @@ struct DerivationGoal : public Goal std::unique_ptr act; - /* Activity that denotes waiting for a lock. */ + /** + * Activity that denotes waiting for a lock. + */ std::unique_ptr actLock; std::map builderActivities; - /* The remote machine on which we're building. */ + /** + * The remote machine on which we're building. + */ std::string machineName; DerivationGoal(const StorePath & drvPath, @@ -143,10 +185,14 @@ struct DerivationGoal : public Goal void work() override; - /* Add wanted outputs to an already existing derivation goal. */ + /** + * Add wanted outputs to an already existing derivation goal. + */ void addWantedOutputs(const OutputsSpec & outputs); - /* The states. */ + /** + * The states. + */ void getDerivation(); void loadDerivation(); void haveDerivation(); @@ -160,28 +206,42 @@ struct DerivationGoal : public Goal void resolvedFinished(); - /* Is the build hook willing to perform the build? */ + /** + * Is the build hook willing to perform the build? + */ HookReply tryBuildHook(); virtual int getChildStatus(); - /* Check that the derivation outputs all exist and register them - as valid. */ + /** + * Check that the derivation outputs all exist and register them + * as valid. + */ virtual DrvOutputs registerOutputs(); - /* Open a log file and a pipe to it. */ + /** + * Open a log file and a pipe to it. + */ Path openLogFile(); - /* Sign the newly built realisation if the store allows it */ + /** + * Sign the newly built realisation if the store allows it + */ virtual void signRealisation(Realisation&) {} - /* Close the log file. */ + /** + * Close the log file. + */ void closeLogFile(); - /* Close the read side of the logger pipe. */ + /** + * Close the read side of the logger pipe. + */ virtual void closeReadPipes(); - /* Cleanup hooks for buildDone() */ + /** + * Cleanup hooks for buildDone() + */ virtual void cleanupHookFinally(); virtual void cleanupPreChildKill(); virtual void cleanupPostChildKill(); @@ -191,30 +251,40 @@ struct DerivationGoal : public Goal virtual bool isReadDesc(int fd); - /* Callback used by the worker to write to the log. */ + /** + * Callback used by the worker to write to the log. + */ void handleChildOutput(int fd, std::string_view data) override; void handleEOF(int fd) override; void flushLine(); - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ + /** + * Wrappers around the corresponding Store methods that first consult the + * derivation. This is currently needed because when there is no drv file + * there also is no DB entry. + */ std::map> queryPartialDerivationOutputMap(); OutputPathMap queryDerivationOutputMap(); - /* Update 'initialOutputs' to determine the current status of the - outputs of the derivation. Also returns a Boolean denoting - whether all outputs are valid and non-corrupt, and a - 'DrvOutputs' structure containing the valid and wanted - outputs. */ + /** + * Update 'initialOutputs' to determine the current status of the + * outputs of the derivation. Also returns a Boolean denoting + * whether all outputs are valid and non-corrupt, and a + * 'DrvOutputs' structure containing the valid and wanted + * outputs. + */ std::pair checkPathValidity(); - /* Aborts if any output is not valid or corrupt, and otherwise - returns a 'DrvOutputs' structure containing the wanted - outputs. */ + /** + * Aborts if any output is not valid or corrupt, and otherwise + * returns a 'DrvOutputs' structure containing the wanted + * outputs. + */ DrvOutputs assertPathValidity(); - /* Forcibly kill the child process, if any. */ + /** + * Forcibly kill the child process, if any. + */ virtual void killChild(); void repairClosure(); diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 3b6620b76..697ddb283 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -11,24 +11,34 @@ namespace nix { class Worker; -// Substitution of a derivation output. -// This is done in three steps: -// 1. Fetch the output info from a substituter -// 2. Substitute the corresponding output path -// 3. Register the output info +/** + * Substitution of a derivation output. + * This is done in three steps: + * 1. Fetch the output info from a substituter + * 2. Substitute the corresponding output path + * 3. Register the output info + */ class DrvOutputSubstitutionGoal : public Goal { - // The drv output we're trying to substitue + /** + * The drv output we're trying to substitue + */ DrvOutput id; - // The realisation corresponding to the given output id. - // Will be filled once we can get it. + /** + * The realisation corresponding to the given output id. + * Will be filled once we can get it. + */ std::shared_ptr outputInfo; - /* The remaining substituters. */ + /** + * The remaining substituters. + */ std::list> subs; - /* The current substituter. */ + /** + * The current substituter. + */ std::shared_ptr sub; struct DownloadState @@ -39,7 +49,9 @@ class DrvOutputSubstitutionGoal : public Goal { std::shared_ptr downloadState; - /* Whether a substituter failed. */ + /** + * Whether a substituter failed. + */ bool substituterFailed = false; public: diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 924a8bbd5..f4bf6f38b 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -7,11 +7,15 @@ namespace nix { -/* Forward definition. */ +/** + * Forward definition. + */ struct Goal; class Worker; -/* A pointer to a goal. */ +/** + * A pointer to a goal. + */ typedef std::shared_ptr GoalPtr; typedef std::weak_ptr WeakGoalPtr; @@ -19,48 +23,72 @@ struct CompareGoalPtrs { bool operator() (const GoalPtr & a, const GoalPtr & b) const; }; -/* Set of goals. */ +/** + * Set of goals. + */ typedef std::set Goals; typedef std::set> WeakGoals; -/* A map of paths to goals (and the other way around). */ +/** + * A map of paths to goals (and the other way around). + */ typedef std::map WeakGoalMap; struct Goal : public std::enable_shared_from_this { typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - /* Backlink to the worker. */ + /** + * Backlink to the worker. + */ Worker & worker; - /* Goals that this goal is waiting for. */ + /** + * Goals that this goal is waiting for. + */ Goals waitees; - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ + /** + * Goals waiting for this one to finish. Must use weak pointers + * here to prevent cycles. + */ WeakGoals waiters; - /* Number of goals we are/were waiting for that have failed. */ + /** + * Number of goals we are/were waiting for that have failed. + */ size_t nrFailed = 0; - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ + /** + * Number of substitution goals we are/were waiting for that + * failed because there are no substituters. + */ size_t nrNoSubstituters = 0; - /* Number of substitution goals we are/were waiting for that - failed because they had unsubstitutable references. */ + /** + * Number of substitution goals we are/were waiting for that + * failed because they had unsubstitutable references. + */ size_t nrIncompleteClosure = 0; - /* Name of this goal for debugging purposes. */ + /** + * Name of this goal for debugging purposes. + */ std::string name; - /* Whether the goal is finished. */ + /** + * Whether the goal is finished. + */ ExitCode exitCode = ecBusy; - /* Build result. */ + /** + * Build result. + */ BuildResult buildResult; - /* Exception containing an error message, if any. */ + /** + * Exception containing an error message, if any. + */ std::optional ex; Goal(Worker & worker, DerivedPath path) @@ -96,9 +124,11 @@ struct Goal : public std::enable_shared_from_this return name; } - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ + /** + * Callback in case of a timeout. It should wake up its waiters, + * get rid of any running child processes that are being monitored + * by the worker (important!), etc. + */ virtual void timedOut(Error && ex) = 0; virtual std::string key() = 0; diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/build/hook-instance.hh index 6bf60b297..d84f62877 100644 --- a/src/libstore/build/hook-instance.hh +++ b/src/libstore/build/hook-instance.hh @@ -8,16 +8,24 @@ namespace nix { struct HookInstance { - /* Pipes for talking to the build hook. */ + /** + * Pipes for talking to the build hook. + */ Pipe toHook; - /* Pipe for the hook's standard output/error. */ + /** + * Pipe for the hook's standard output/error. + */ Pipe fromHook; - /* Pipe for the builder's standard output/error. */ + /** + * Pipe for the builder's standard output/error. + */ Pipe builderOut; - /* The process ID of the hook. */ + /** + * The process ID of the hook. + */ Pid pid; FdSink sink; diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 1c4b4e3fe..42d32a31a 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -10,49 +10,75 @@ struct LocalDerivationGoal : public DerivationGoal { LocalStore & getLocalStore(); - /* User selected for running the builder. */ + /** + * User selected for running the builder. + */ std::unique_ptr buildUser; - /* The process ID of the builder. */ + /** + * The process ID of the builder. + */ Pid pid; - /* The cgroup of the builder, if any. */ + /** + * The cgroup of the builder, if any. + */ std::optional cgroup; - /* The temporary directory. */ + /** + * The temporary directory. + */ Path tmpDir; - /* The path of the temporary directory in the sandbox. */ + /** + * The path of the temporary directory in the sandbox. + */ Path tmpDirInSandbox; - /* Master side of the pseudoterminal used for the builder's - standard output/error. */ + /** + * Master side of the pseudoterminal used for the builder's + * standard output/error. + */ AutoCloseFD builderOut; - /* Pipe for synchronising updates to the builder namespaces. */ + /** + * Pipe for synchronising updates to the builder namespaces. + */ Pipe userNamespaceSync; - /* The mount namespace and user namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ + /** + * The mount namespace and user namespace of the builder, used to add additional + * paths to the sandbox as a result of recursive Nix calls. + */ AutoCloseFD sandboxMountNamespace; AutoCloseFD sandboxUserNamespace; - /* On Linux, whether we're doing the build in its own user - namespace. */ + /** + * On Linux, whether we're doing the build in its own user + * namespace. + */ bool usingUserNamespace = true; - /* Whether we're currently doing a chroot build. */ + /** + * Whether we're currently doing a chroot build. + */ bool useChroot = false; Path chrootRootDir; - /* RAII object to delete the chroot directory. */ + /** + * RAII object to delete the chroot directory. + */ std::shared_ptr autoDelChroot; - /* Whether to run the build in a private network namespace. */ + /** + * Whether to run the build in a private network namespace. + */ bool privateNetwork = false; - /* Stuff we need to pass to initChild(). */ + /** + * Stuff we need to pass to initChild(). + */ struct ChrootPath { Path source; bool optional; @@ -71,30 +97,35 @@ struct LocalDerivationGoal : public DerivationGoal SandboxProfile additionalSandboxProfile; #endif - /* Hash rewriting. */ + /** + * Hash rewriting. + */ StringMap inputRewrites, outputRewrites; typedef map RedirectedOutputs; RedirectedOutputs redirectedOutputs; - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. + /** + * The outputs paths used during the build. + * + * - Input-addressed derivations or fixed content-addressed outputs are + * sometimes built when some of their outputs already exist, and can not + * be hidden via sandboxing. We use temporary locations instead and + * rewrite after the build. Otherwise the regular predetermined paths are + * put here. + * + * - Floating content-addressed derivations do not know their final build + * output paths until the outputs are hashed, so random locations are + * used, and then renamed. The randomness helps guard against hidden + * self-references. */ OutputPathMap scratchOutputs; - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ + /** + * Path registration info from the previous round, if we're + * building multiple times. Since this contains the hash, it + * allows us to compare whether two rounds produced the same + * result. + */ std::map prevInfos; uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } @@ -102,25 +133,37 @@ struct LocalDerivationGoal : public DerivationGoal const static Path homeDir; - /* The recursive Nix daemon socket. */ + /** + * The recursive Nix daemon socket. + */ AutoCloseFD daemonSocket; - /* The daemon main thread. */ + /** + * The daemon main thread. + */ std::thread daemonThread; - /* The daemon worker threads. */ + /** + * The daemon worker threads. + */ std::vector daemonWorkerThreads; - /* Paths that were added via recursive Nix calls. */ + /** + * Paths that were added via recursive Nix calls. + */ StorePathSet addedPaths; - /* Realisations that were added via recursive Nix calls. */ + /** + * Realisations that were added via recursive Nix calls. + */ std::set addedDrvOutputs; - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ + /** + * Recursive Nix calls are only allowed to build or realize paths + * in the original input closure or added via a recursive Nix call + * (so e.g. you can't do 'nix-store -r /nix/store/' where + * /nix/store/ is some arbitrary path in a binary cache). + */ bool isAllowed(const StorePath & path) { return inputPaths.count(path) || addedPaths.count(path); @@ -138,55 +181,81 @@ struct LocalDerivationGoal : public DerivationGoal virtual ~LocalDerivationGoal() override; - /* Whether we need to perform hash rewriting if there are valid output paths. */ + /** + * Whether we need to perform hash rewriting if there are valid output paths. + */ bool needsHashRewrite(); - /* The additional states. */ + /** + * The additional states. + */ void tryLocalBuild() override; - /* Start building a derivation. */ + /** + * Start building a derivation. + */ void startBuilder(); - /* Fill in the environment for the builder. */ + /** + * Fill in the environment for the builder. + */ void initEnv(); - /* Setup tmp dir location. */ + /** + * Setup tmp dir location. + */ void initTmpDir(); - /* Write a JSON file containing the derivation attributes. */ + /** + * Write a JSON file containing the derivation attributes. + */ void writeStructuredAttrs(); void startDaemon(); void stopDaemon(); - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ + /** + * Add 'path' to the set of paths that may be referenced by the + * outputs, and make it appear in the sandbox. + */ void addDependency(const StorePath & path); - /* Make a file owned by the builder. */ + /** + * Make a file owned by the builder. + */ void chownToBuilder(const Path & path); int getChildStatus() override; - /* Run the builder's process. */ + /** + * Run the builder's process. + */ void runChild(); - /* Check that the derivation outputs all exist and register them - as valid. */ + /** + * Check that the derivation outputs all exist and register them + * as valid. + */ DrvOutputs registerOutputs() override; void signRealisation(Realisation &) override; - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ + /** + * Check that an output meets the requirements specified by the + * 'outputChecks' attribute (or the legacy + * '{allowed,disallowed}{References,Requisites}' attributes). + */ void checkOutputs(const std::map & outputs); - /* Close the read side of the logger pipe. */ + /** + * Close the read side of the logger pipe. + */ void closeReadPipes() override; - /* Cleanup hooks for buildDone() */ + /** + * Cleanup hooks for buildDone() + */ void cleanupHookFinally() override; void cleanupPreChildKill() override; void cleanupPostChildKill() override; @@ -196,24 +265,36 @@ struct LocalDerivationGoal : public DerivationGoal bool isReadDesc(int fd) override; - /* Delete the temporary directory, if we have one. */ + /** + * Delete the temporary directory, if we have one. + */ void deleteTmpDir(bool force); - /* Forcibly kill the child process, if any. */ + /** + * Forcibly kill the child process, if any. + */ void killChild() override; - /* Kill any processes running under the build user UID or in the - cgroup of the build. */ + /** + * Kill any processes running under the build user UID or in the + * cgroup of the build. + */ void killSandbox(bool getStats); - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ + /** + * Create alternative path calculated from but distinct from the + * input, so we can avoid overwriting outputs (or other store paths) + * that already exist. + */ StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ + + /** + * Make a path to another based on the output name along with the + * derivation hash. + * + * @todo Add option to randomize, so we can audit whether our + * rewrites caught everything + */ StorePath makeFallbackPath(std::string_view outputName); }; diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 1add9eb14..c2b7fc95a 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -11,38 +11,58 @@ class Worker; struct PathSubstitutionGoal : public Goal { - /* The store path that should be realised through a substitute. */ + /** + * The store path that should be realised through a substitute. + */ StorePath storePath; - /* The path the substituter refers to the path as. This will be - different when the stores have different names. */ + /** + * The path the substituter refers to the path as. This will be + * different when the stores have different names. + */ std::optional subPath; - /* The remaining substituters. */ + /** + * The remaining substituters. + */ std::list> subs; - /* The current substituter. */ + /** + * The current substituter. + */ std::shared_ptr sub; - /* Whether a substituter failed. */ + /** + * Whether a substituter failed. + */ bool substituterFailed = false; - /* Path info returned by the substituter's query info operation. */ + /** + * Path info returned by the substituter's query info operation. + */ std::shared_ptr info; - /* Pipe for the substituter's standard output. */ + /** + * Pipe for the substituter's standard output. + */ Pipe outPipe; - /* The substituter thread. */ + /** + * The substituter thread. + */ std::thread thr; std::promise promise; - /* Whether to try to repair a valid path. */ + /** + * Whether to try to repair a valid path. + */ RepairFlag repair; - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ + /** + * Location where we're downloading the substitute. Differs from + * storePath when doing a repair. + */ Path destPath; std::unique_ptr> maintainExpectedSubstitutions, @@ -51,7 +71,9 @@ struct PathSubstitutionGoal : public Goal typedef void (PathSubstitutionGoal::*GoalState)(); GoalState state; - /* Content address for recomputing store path */ + /** + * Content address for recomputing store path + */ std::optional ca; void done( @@ -65,16 +87,20 @@ public: void timedOut(Error && ex) override { abort(); }; + /** + * We prepend "a$" to the key name to ensure substitution goals + * happen before derivation goals. + */ std::string key() override { - /* "a$" ensures substitution goals happen before derivation - goals. */ return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); } void work() override; - /* The states. */ + /** + * The states. + */ void init(); void tryNext(); void gotInfo(); @@ -82,7 +108,9 @@ public: void tryToRun(); void finished(); - /* Callback used by the worker to write to the log. */ + /** + * Callback used by the worker to write to the log. + */ void handleChildOutput(int fd, std::string_view data) override; void handleEOF(int fd) override; diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index d840b3b3f..48a1a27fa 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -17,24 +17,29 @@ struct DerivationGoal; struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; -/* Workaround for not being able to declare a something like - - class PathSubstitutionGoal : public Goal; - - even when Goal is a complete type. - - This is still a static cast. The purpose of exporting it is to define it in - a place where `PathSubstitutionGoal` is concrete, and use it in a place where it - is opaque. */ +/** + * Workaround for not being able to declare a something like + * + * ```c++ + * class PathSubstitutionGoal : public Goal; + * ``` + * even when Goal is a complete type. + * + * This is still a static cast. The purpose of exporting it is to define it in + * a place where `PathSubstitutionGoal` is concrete, and use it in a place where it + * is opaque. + */ GoalPtr upcast_goal(std::shared_ptr subGoal); GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ +/** + * A mapping used to remember for each child process to what goal it + * belongs, and file descriptors for receiving log data and output + * path creation commands. + */ struct Child { WeakGoalPtr goal; @@ -42,14 +47,19 @@ struct Child std::set fds; bool respectTimeouts; bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ + /** + * Time we last got output on stdout/stderr + */ + steady_time_point lastOutput; steady_time_point timeStarted; }; /* Forward definition. */ struct HookInstance; -/* The worker class. */ +/** + * The worker class. + */ class Worker { private: @@ -57,38 +67,58 @@ private: /* Note: the worker should only have strong pointers to the top-level goals. */ - /* The top-level goals of the worker. */ + /** + * The top-level goals of the worker. + */ Goals topGoals; - /* Goals that are ready to do some work. */ + /** + * Goals that are ready to do some work. + */ WeakGoals awake; - /* Goals waiting for a build slot. */ + /** + * Goals waiting for a build slot. + */ WeakGoals wantingToBuild; - /* Child processes currently running. */ + /** + * Child processes currently running. + */ std::list children; - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ + /** + * Number of build slots occupied. This includes local builds and + * substitutions but not remote builds via the build hook. + */ unsigned int nrLocalBuilds; - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ + /** + * Maps used to prevent multiple instantiations of a goal for the + * same derivation / path. + */ std::map> derivationGoals; std::map> substitutionGoals; std::map> drvOutputSubstitutionGoals; - /* Goals waiting for busy paths to be unlocked. */ + /** + * Goals waiting for busy paths to be unlocked. + */ WeakGoals waitingForAnyGoal; - /* Goals sleeping for a few seconds (polling a lock). */ + /** + * Goals sleeping for a few seconds (polling a lock). + */ WeakGoals waitingForAWhile; - /* Last time the goals in `waitingForAWhile' where woken up. */ + /** + * Last time the goals in `waitingForAWhile` where woken up. + */ steady_time_point lastWokenUp; - /* Cache for pathContentsGood(). */ + /** + * Cache for pathContentsGood(). + */ std::map pathContentsGoodCache; public: @@ -97,17 +127,25 @@ public: const Activity actDerivations; const Activity actSubstitutions; - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ + /** + * Set if at least one derivation had a BuildError (i.e. permanent + * failure). + */ bool permanentFailure; - /* Set if at least one derivation had a timeout. */ + /** + * Set if at least one derivation had a timeout. + */ bool timedOut; - /* Set if at least one derivation fails with a hash mismatch. */ + /** + * Set if at least one derivation fails with a hash mismatch. + */ bool hashMismatch; - /* Set if at least one derivation is not deterministic in check mode. */ + /** + * Set if at least one derivation is not deterministic in check mode. + */ bool checkMismatch; Store & store; @@ -129,16 +167,22 @@ public: uint64_t expectedNarSize = 0; uint64_t doneNarSize = 0; - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ + /** + * Whether to ask the build hook if it can build a derivation. If + * it answers with "decline-permanently", we don't try again. + */ bool tryBuildHook = true; Worker(Store & store, Store & evalStore); ~Worker(); - /* Make a goal (with caching). */ + /** + * Make a goal (with caching). + */ - /* derivation goal */ + /** + * derivation goal + */ private: std::shared_ptr makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, @@ -151,56 +195,80 @@ public: const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); - /* substitution goal */ + /** + * substitution goal + */ std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - /* Remove a dead goal. */ + /** + * Remove a dead goal. + */ void removeGoal(GoalPtr goal); - /* Wake up a goal (i.e., there is something for it to do). */ + /** + * Wake up a goal (i.e., there is something for it to do). + */ void wakeUp(GoalPtr goal); - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ + /** + * Return the number of local build and substitution processes + * currently running (but not remote builds via the build + * hook). + */ unsigned int getNrLocalBuilds(); - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ + /** + * Registers a running child process. `inBuildSlot` means that + * the process counts towards the jobs limit. + */ void childStarted(GoalPtr goal, const std::set & fds, bool inBuildSlot, bool respectTimeouts); - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ + /** + * Unregisters a running child process. `wakeSleepers` should be + * false if there is no sense in waking up goals that are sleeping + * because they can't run yet (e.g., there is no free build slot, + * or the hook would still say `postpone`). + */ void childTerminated(Goal * goal, bool wakeSleepers = true); - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ + /** + * Put `goal` to sleep until a build slot becomes available (which + * might be right away). + */ void waitForBuildSlot(GoalPtr goal); - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ + /** + * Wait for any goal to finish. Pretty indiscriminate way to + * wait for some resource that some other goal is holding. + */ void waitForAnyGoal(GoalPtr goal); - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ + /** + * Wait for a few seconds and then retry this goal. Used when + * waiting for a lock held by another process. This kind of + * polling is inefficient, but POSIX doesn't really provide a way + * to wait for multiple locks in the main select() loop. + */ void waitForAWhile(GoalPtr goal); - /* Loop until the specified top-level goals have finished. */ + /** + * Loop until the specified top-level goals have finished. + */ void run(const Goals & topGoals); - /* Wait for input to become available. */ + /** + * Wait for input to become available. + */ void waitForInput(); unsigned int exitStatus(); - /* Check whether the given valid path exists and has the right - contents. */ + /** + * Check whether the given valid path exists and has the right + * contents. + */ bool pathContentsGood(const StorePath & path); void markContentsGood(const StorePath & path); diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh index a98f2a3b8..35216d470 100644 --- a/src/libstore/crypto.hh +++ b/src/libstore/crypto.hh @@ -12,8 +12,10 @@ struct Key std::string name; std::string key; - /* Construct Key from a string in the format - ‘:’. */ + /** + * Construct Key from a string in the format + * ‘:’. + */ Key(std::string_view s); std::string to_string() const; @@ -29,7 +31,9 @@ struct SecretKey : Key { SecretKey(std::string_view s); - /* Return a detached signature of the given string. */ + /** + * Return a detached signature of the given string. + */ std::string signDetached(std::string_view s) const; PublicKey toPublicKey() const; @@ -53,8 +57,10 @@ private: typedef std::map PublicKeys; -/* Return true iff ‘sig’ is a correct signature over ‘data’ using one - of the given public keys. */ +/** + * @return true iff ‘sig’ is a correct signature over ‘data’ using one + * of the given public keys. + */ bool verifyDetached(const std::string & data, const std::string & sig, const PublicKeys & publicKeys); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 9e7ceeb5d..ccdde36ca 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -258,8 +258,14 @@ struct DerivationType : _DerivationTypeRaw { struct BasicDerivation { - DerivationOutputs outputs; /* keyed on symbolic IDs */ - StorePathSet inputSrcs; /* inputs that are sources */ + /** + * keyed on symbolic IDs + */ + DerivationOutputs outputs; + /** + * inputs that are sources + */ + StorePathSet inputSrcs; std::string platform; Path builder; Strings args; @@ -429,12 +435,12 @@ void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; * * A fixed-output derivation is a derivation whose outputs have a * specified content hash and hash algorithm. (Currently they must have - * exactly one output (`out'), which is specified using the `outputHash' - * and `outputHashAlgo' attributes, but the algorithm doesn't assume + * exactly one output (`out`), which is specified using the `outputHash` + * and `outputHashAlgo` attributes, but the algorithm doesn't assume * this.) We don't want changes to such derivations to propagate upwards * through the dependency graph, changing output paths everywhere. * - * For instance, if we change the url in a call to the `fetchurl' + * For instance, if we change the url in a call to the `fetchurl` * function, we do not want to rebuild everything depending on it---after * all, (the hash of) the file being downloaded is unchanged. So the * *output paths* should not change. On the other hand, the *derivation diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index 1c98a42d7..1df19e647 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -5,8 +5,10 @@ namespace nix { -/* An abstract class for accessing a filesystem-like structure, such - as a (possibly remote) Nix store or the contents of a NAR file. */ +/** + * An abstract class for accessing a filesystem-like structure, such + * as a (possibly remote) Nix store or the contents of a NAR file. + */ class FSAccessor { public: @@ -15,8 +17,17 @@ public: struct Stat { Type type = tMissing; - uint64_t fileSize = 0; // regular files only + /** + * regular files only + */ + uint64_t fileSize = 0; + /** + * regular files only + */ bool isExecutable = false; // regular files only + /** + * regular files only + */ uint64_t narOffset = 0; // regular files only }; diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index 17f043a63..2c26c65c4 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -12,19 +12,20 @@ typedef std::unordered_map> Roots; struct GCOptions { - /* Garbage collector operation: - - - `gcReturnLive': return the set of paths reachable from - (i.e. in the closure of) the roots. - - - `gcReturnDead': return the set of paths not reachable from - the roots. - - - `gcDeleteDead': actually delete the latter set. - - - `gcDeleteSpecific': delete the paths listed in - `pathsToDelete', insofar as they are not reachable. - */ + /** + * Garbage collector operation: + * + * - `gcReturnLive`: return the set of paths reachable from + * (i.e. in the closure of) the roots. + * + * - `gcReturnDead`: return the set of paths not reachable from + * the roots. + * + * - `gcDeleteDead`: actually delete the latter set. + * + * - `gcDeleteSpecific`: delete the paths listed in + * `pathsToDelete`, insofar as they are not reachable. + */ typedef enum { gcReturnLive, gcReturnDead, @@ -34,28 +35,38 @@ struct GCOptions GCAction action{gcDeleteDead}; - /* If `ignoreLiveness' is set, then reachability from the roots is - ignored (dangerous!). However, the paths must still be - unreferenced *within* the store (i.e., there can be no other - store paths that depend on them). */ + /** + * If `ignoreLiveness` is set, then reachability from the roots is + * ignored (dangerous!). However, the paths must still be + * unreferenced *within* the store (i.e., there can be no other + * store paths that depend on them). + */ bool ignoreLiveness{false}; - /* For `gcDeleteSpecific', the paths to delete. */ + /** + * For `gcDeleteSpecific`, the paths to delete. + */ StorePathSet pathsToDelete; - /* Stop after at least `maxFreed' bytes have been freed. */ + /** + * Stop after at least `maxFreed` bytes have been freed. + */ uint64_t maxFreed{std::numeric_limits::max()}; }; struct GCResults { - /* Depending on the action, the GC roots, or the paths that would - be or have been deleted. */ + /** + * Depending on the action, the GC roots, or the paths that would + * be or have been deleted. + */ PathSet paths; - /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the - number of bytes that would be or was freed. */ + /** + * For `gcReturnDead`, `gcDeleteDead` and `gcDeleteSpecific`, the + * number of bytes that would be or was freed. + */ uint64_t bytesFreed = 0; }; @@ -64,21 +75,27 @@ struct GcStore : public virtual Store { inline static std::string operationName = "Garbage collection"; - /* Add an indirect root, which is merely a symlink to `path' from - /nix/var/nix/gcroots/auto/. `path' is supposed - to be a symlink to a store path. The garbage collector will - automatically remove the indirect root when it finds that - `path' has disappeared. */ + /** + * Add an indirect root, which is merely a symlink to `path` from + * `/nix/var/nix/gcroots/auto/`. `path` is supposed + * to be a symlink to a store path. The garbage collector will + * automatically remove the indirect root when it finds that + * `path` has disappeared. + */ virtual void addIndirectRoot(const Path & path) = 0; - /* Find the roots of the garbage collector. Each root is a pair - (link, storepath) where `link' is the path of the symlink - outside of the Nix store that point to `storePath'. If - 'censor' is true, privacy-sensitive information about roots - found in /proc is censored. */ + /** + * Find the roots of the garbage collector. Each root is a pair + * `(link, storepath)` where `link` is the path of the symlink + * outside of the Nix store that point to `storePath`. If + * `censor` is true, privacy-sensitive information about roots + * found in `/proc` is censored. + */ virtual Roots findRoots(bool censor) = 0; - /* Perform a garbage collection. */ + /** + * Perform a garbage collection. + */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; }; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 9b04c9e78..63c7389da 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -72,30 +72,46 @@ public: Path nixPrefix; - /* The directory where we store sources and derived files. */ + /** + * The directory where we store sources and derived files. + */ Path nixStore; Path nixDataDir; /* !!! fix */ - /* The directory where we log various operations. */ + /** + * The directory where we log various operations. + */ Path nixLogDir; - /* The directory where state is stored. */ + /** + * The directory where state is stored. + */ Path nixStateDir; - /* The directory where system configuration files are stored. */ + /** + * The directory where system configuration files are stored. + */ Path nixConfDir; - /* A list of user configuration files to load. */ + /** + * A list of user configuration files to load. + */ std::vector nixUserConfFiles; - /* The directory where the main programs are stored. */ + /** + * The directory where the main programs are stored. + */ Path nixBinDir; - /* The directory where the man pages are stored. */ + /** + * The directory where the man pages are stored. + */ Path nixManDir; - /* File name of the socket the daemon listens to. */ + /** + * File name of the socket the daemon listens to. + */ Path nixDaemonSocketFile; Setting storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", @@ -121,7 +137,9 @@ public: )", {"build-fallback"}}; - /* Whether to show build log output in real time. */ + /** + * Whether to show build log output in real time. + */ bool verboseBuild = true; Setting logLines{this, 10, "log-lines", @@ -157,8 +175,10 @@ public: )", {"build-cores"}, false}; - /* Read-only mode. Don't copy stuff to the store, don't change - the database. */ + /** + * Read-only mode. Don't copy stuff to the store, don't change + * the database. + */ bool readOnlyMode = false; Setting thisSystem{ @@ -458,7 +478,9 @@ public: )", {"env-keep-derivations"}}; - /* Whether to lock the Nix client and worker to the same CPU. */ + /** + * Whether to lock the Nix client and worker to the same CPU. + */ bool lockCPU; Setting sandboxMode{ @@ -997,8 +1019,10 @@ public: // FIXME: don't use a global variable. extern Settings settings; -/* This should be called after settings are initialized, but before - anything else */ +/** + * This should be called after settings are initialized, but before + * anything else + */ void initPlugins(); void loadConfFile(); @@ -1008,12 +1032,16 @@ std::vector getUserConfigFiles(); extern const std::string nixVersion; -/* NB: This is not sufficient. You need to call initNix() */ +/** + * NB: This is not sufficient. You need to call initNix() + */ void initLibStore(); -/* It's important to initialize before doing _anything_, which is why we - call upon the programmer to handle this correctly. However, we only add - this in a key locations, so as not to litter the code. */ +/** + * It's important to initialize before doing _anything_, which is why we + * call upon the programmer to handle this correctly. However, we only add + * this in a key locations, so as not to litter the code. + */ void assertLibStoreInitialized(); } diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 1e7f31a09..a03bb88f5 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -48,7 +48,9 @@ public: void narFromPath(const StorePath & path, Sink & sink) override; ref getFSAccessor() override; - /* Register a permanent GC root. */ + /** + * Register a permanent GC root. + */ Path addPermRoot(const StorePath & storePath, const Path & gcRoot); virtual Path getRealStoreDir() { return realStoreDir; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 7e0849961..55add18dd 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -19,10 +19,14 @@ namespace nix { -/* Nix store and database schema version. Version 1 (or 0) was Nix <= - 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. - Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is - Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */ +/** + * Nix store and database schema version. + * + * Version 1 (or 0) was Nix <= + * 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. + * Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is + * Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. + */ const int nixSchemaVersion = 10; @@ -51,30 +55,40 @@ class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, { private: - /* Lock file used for upgrading. */ + /** + * Lock file used for upgrading. + */ AutoCloseFD globalLock; struct State { - /* The SQLite database object. */ + /** + * The SQLite database object. + */ SQLite db; struct Stmts; std::unique_ptr stmts; - /* The last time we checked whether to do an auto-GC, or an - auto-GC finished. */ + /** + * The last time we checked whether to do an auto-GC, or an + * auto-GC finished. + */ std::chrono::time_point lastGCCheck; - /* Whether auto-GC is running. If so, get gcFuture to wait for - the GC to finish. */ + /** + * Whether auto-GC is running. If so, get gcFuture to wait for + * the GC to finish. + */ bool gcRunning = false; std::shared_future gcFuture; - /* How much disk space was available after the previous - auto-GC. If the current available disk space is below - minFree but not much below availAfterGC, then there is no - point in starting a new GC. */ + /** + * How much disk space was available after the previous + * auto-GC. If the current available disk space is below + * minFree but not much below availAfterGC, then there is no + * point in starting a new GC. + */ uint64_t availAfterGC = std::numeric_limits::max(); std::unique_ptr publicKeys; @@ -97,11 +111,15 @@ private: public: - // Hack for build-remote.cc. + /** + * Hack for build-remote.cc. + */ PathSet locksHeld; - /* Initialise the local store, upgrading the schema if - necessary. */ + /** + * Initialise the local store, upgrading the schema if + * necessary. + */ LocalStore(const Params & params); LocalStore(std::string scheme, std::string path, const Params & params); @@ -110,7 +128,9 @@ public: static std::set uriSchemes() { return {}; } - /* Implementations of abstract store API methods. */ + /** + * Implementations of abstract store API methods. + */ std::string getUri() override; @@ -155,13 +175,19 @@ private: void createTempRootsFile(); - /* The file to which we write our temporary roots. */ + /** + * The file to which we write our temporary roots. + */ Sync _fdTempRoots; - /* The global GC lock. */ + /** + * The global GC lock. + */ Sync _fdGCLock; - /* Connection to the garbage collector. */ + /** + * Connection to the garbage collector. + */ Sync _fdRootsSocket; public: @@ -180,24 +206,30 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override; - /* Optimise the disk space usage of the Nix store by hard-linking - files with the same contents. */ + /** + * Optimise the disk space usage of the Nix store by hard-linking + * files with the same contents. + */ void optimiseStore(OptimiseStats & stats); void optimiseStore() override; - /* Optimise a single store path. Optionally, test the encountered - symlinks for corruption. */ + /** + * Optimise a single store path. Optionally, test the encountered + * symlinks for corruption. + */ void optimisePath(const Path & path, RepairFlag repair); bool verifyStore(bool checkContents, RepairFlag repair) override; - /* Register the validity of a path, i.e., that `path' exists, that - the paths referenced by it exists, and in the case of an output - path of a derivation, that it has been produced by a successful - execution of the derivation (or something equivalent). Also - register the hash of the file system contents of the path. The - hash must be a SHA-256 hash. */ + /** + * Register the validity of a path, i.e., that `path` exists, that + * the paths referenced by it exists, and in the case of an output + * path of a derivation, that it has been produced by a successful + * execution of the derivation (or something equivalent). Also + * register the hash of the file system contents of the path. The + * hash must be a SHA-256 hash. + */ void registerValidPath(const ValidPathInfo & info); void registerValidPaths(const ValidPathInfos & infos); @@ -212,12 +244,16 @@ public: void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - /* If free disk space in /nix/store if below minFree, delete - garbage until it exceeds maxFree. */ + /** + * If free disk space in /nix/store if below minFree, delete + * garbage until it exceeds maxFree. + */ void autoGC(bool sync = true); - /* Register the store path 'output' as the output named 'outputName' of - derivation 'deriver'. */ + /** + * Register the store path 'output' as the output named 'outputName' of + * derivation 'deriver'. + */ void registerDrvOutput(const Realisation & info) override; void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override; void cacheDrvOutputMapping( @@ -247,7 +283,9 @@ private: void invalidatePath(State & state, const StorePath & path); - /* Delete a path from the Nix store. */ + /** + * Delete a path from the Nix store. + */ void invalidatePathChecked(const StorePath & path); void verifyPath(const Path & path, const StringSet & store, @@ -280,8 +318,10 @@ private: bool isValidPath_(State & state, const StorePath & path); void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers); - /* Add signatures to a ValidPathInfo or Realisation using the secret keys - specified by the ‘secret-key-files’ option. */ + /** + * Add signatures to a ValidPathInfo or Realisation using the secret keys + * specified by the ‘secret-key-files’ option. + */ void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); @@ -311,18 +351,23 @@ typedef std::pair Inode; typedef std::set InodesSeen; -/* "Fix", or canonicalise, the meta-data of the files in a store path - after it has been built. In particular: - - the last modification date on each file is set to 1 (i.e., - 00:00:01 1/1/1970 UTC) - - the permissions are set of 444 or 555 (i.e., read-only with or - without execute permission; setuid bits etc. are cleared) - - the owner and group are set to the Nix user and group, if we're - running as root. - If uidRange is not empty, this function will throw an error if it - encounters files owned by a user outside of the closed interval - [uidRange->first, uidRange->second]. -*/ +/** + * "Fix", or canonicalise, the meta-data of the files in a store path + * after it has been built. In particular: + * + * - the last modification date on each file is set to 1 (i.e., + * 00:00:01 1/1/1970 UTC) + * + * - the permissions are set of 444 or 555 (i.e., read-only with or + * without execute permission; setuid bits etc. are cleared) + * + * - the owner and group are set to the Nix user and group, if we're + * running as root. + * + * If uidRange is not empty, this function will throw an error if it + * encounters files owned by a user outside of the closed interval + * [uidRange->first, uidRange->second]. + */ void canonicalisePathMetaData( const Path & path, std::optional> uidRange, diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index 7be3ba314..1c268e1fb 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -13,14 +13,18 @@ struct UserLock { virtual ~UserLock() { } - /* Get the first and last UID. */ + /** + * Get the first and last UID. + */ std::pair getUIDRange() { auto first = getUID(); return {first, first + getUIDCount() - 1}; } - /* Get the first UID. */ + /** + * Get the first UID. + */ virtual uid_t getUID() = 0; virtual uid_t getUIDCount() = 0; @@ -30,8 +34,10 @@ struct UserLock virtual std::vector getSupplementaryGIDs() = 0; }; -/* Acquire a user lock for a UID range of size `nrIds`. Note that this - may return nullptr if no user is available. */ +/** + * Acquire a user lock for a UID range of size `nrIds`. Note that this + * may return nullptr if no user is available. + */ std::unique_ptr acquireUserLock(uid_t nrIds, bool useUserNamespace); bool useBuildUsers(); diff --git a/src/libstore/log-store.hh b/src/libstore/log-store.hh index 7aeec73b2..a84f7dbeb 100644 --- a/src/libstore/log-store.hh +++ b/src/libstore/log-store.hh @@ -10,8 +10,10 @@ struct LogStore : public virtual Store { inline static std::string operationName = "Build log storage and retrieval"; - /* Return the build log of the specified store path, if available, - or null otherwise. */ + /** + * Return the build log of the specified store path, if available, + * or null otherwise. + */ std::optional getBuildLog(const StorePath & path); virtual std::optional getBuildLogExact(const StorePath & path) = 0; diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 940e537b6..5e19bd3c7 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -10,24 +10,30 @@ namespace nix { struct Source; -/* Return an object that provides access to the contents of a NAR - file. */ +/** + * Return an object that provides access to the contents of a NAR + * file. + */ ref makeNarAccessor(std::string && nar); ref makeNarAccessor(Source & source); -/* Create a NAR accessor from a NAR listing (in the format produced by - listNar()). The callback getNarBytes(offset, length) is used by the - readFile() method of the accessor to get the contents of files - inside the NAR. */ +/** + * Create a NAR accessor from a NAR listing (in the format produced by + * listNar()). The callback getNarBytes(offset, length) is used by the + * readFile() method of the accessor to get the contents of files + * inside the NAR. + */ typedef std::function GetNarBytes; ref makeLazyNarAccessor( const std::string & listing, GetNarBytes getNarBytes); -/* Write a JSON representation of the contents of a NAR (except file - contents). */ +/** + * Write a JSON representation of the contents of a NAR (except file + * contents). + */ nlohmann::json listNar(ref accessor, const Path & path, bool recurse); } diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index c596f2d71..bbd1d05d5 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -43,8 +43,10 @@ public: const std::string & uri, const DrvOutput & id) = 0; }; -/* Return a singleton cache object that can be used concurrently by - multiple threads. */ +/** + * Return a singleton cache object that can be used concurrently by + * multiple threads. + */ ref getNarInfoDiskCache(); ref getTestNarInfoDiskCache(Path dbPath); diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index b28bf751c..9af1309a0 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -19,8 +19,14 @@ struct SubstitutablePathInfo { std::optional deriver; StorePathSet references; - uint64_t downloadSize; /* 0 = unknown or inapplicable */ - uint64_t narSize; /* 0 = unknown */ + /** + * 0 = unknown or inapplicable + */ + uint64_t downloadSize; + /** + * 0 = unknown + */ + uint64_t narSize; }; typedef std::map SubstitutablePathInfos; @@ -30,35 +36,40 @@ struct ValidPathInfo { StorePath path; std::optional deriver; - // TODO document this + /** + * \todo document this + */ Hash narHash; StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only - /* Whether the path is ultimately trusted, that is, it's a - derivation output that was built locally. */ + /** + * Whether the path is ultimately trusted, that is, it's a + * derivation output that was built locally. + */ bool ultimate = false; StringSet sigs; // note: not necessarily verified - /* If non-empty, an assertion that the path is content-addressed, - i.e., that the store path is computed from a cryptographic hash - of the contents of the path, plus some other bits of data like - the "name" part of the path. Such a path doesn't need - signatures, since we don't have to trust anybody's claim that - the path is the output of a particular derivation. (In the - extensional store model, we have to trust that the *contents* - of an output path of a derivation were actually produced by - that derivation. In the intensional model, we have to trust - that a particular output path was produced by a derivation; the - path then implies the contents.) - - Ideally, the content-addressability assertion would just be a Boolean, - and the store path would be computed from the name component, ‘narHash’ - and ‘references’. However, we support many types of content addresses. - */ + /** + * If non-empty, an assertion that the path is content-addressed, + * i.e., that the store path is computed from a cryptographic hash + * of the contents of the path, plus some other bits of data like + * the "name" part of the path. Such a path doesn't need + * signatures, since we don't have to trust anybody's claim that + * the path is the output of a particular derivation. (In the + * extensional store model, we have to trust that the *contents* + * of an output path of a derivation were actually produced by + * that derivation. In the intensional model, we have to trust + * that a particular output path was produced by a derivation; the + * path then implies the contents.) + * + * Ideally, the content-addressability assertion would just be a Boolean, + * and the store path would be computed from the name component, ‘narHash’ + * and ‘references’. However, we support many types of content addresses. + */ std::optional ca; bool operator == (const ValidPathInfo & i) const @@ -69,27 +80,35 @@ struct ValidPathInfo && references == i.references; } - /* Return a fingerprint of the store path to be used in binary - cache signatures. It contains the store path, the base-32 - SHA-256 hash of the NAR serialisation of the path, the size of - the NAR, and the sorted references. The size field is strictly - speaking superfluous, but might prevent endless/excessive data - attacks. */ + /** + * Return a fingerprint of the store path to be used in binary + * cache signatures. It contains the store path, the base-32 + * SHA-256 hash of the NAR serialisation of the path, the size of + * the NAR, and the sorted references. The size field is strictly + * speaking superfluous, but might prevent endless/excessive data + * attacks. + */ std::string fingerprint(const Store & store) const; void sign(const Store & store, const SecretKey & secretKey); - /* Return true iff the path is verifiably content-addressed. */ + /** + * @return true iff the path is verifiably content-addressed. + */ bool isContentAddressed(const Store & store) const; static const size_t maxSigs = std::numeric_limits::max(); - /* Return the number of signatures on this .narinfo that were - produced by one of the specified keys, or maxSigs if the path - is content-addressed. */ + /** + * Return the number of signatures on this .narinfo that were + * produced by one of the specified keys, or maxSigs if the path + * is content-addressed. + */ size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; - /* Verify a single signature. */ + /** + * Verify a single signature. + */ bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; Strings shortRefs() const; diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index a845b0e5f..d75850868 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -6,13 +6,14 @@ namespace nix { -/* This is a deprecated old type just for use by the old CLI, and older - versions of the RPC protocols. In new code don't use it; you want - `DerivedPath` instead. - - `DerivedPath` is better because it handles more cases, and does so more - explicitly without devious punning tricks. -*/ +/** + * This is a deprecated old type just for use by the old CLI, and older + * versions of the RPC protocols. In new code don't use it; you want + * `DerivedPath` instead. + * + * `DerivedPath` is better because it handles more cases, and does so more + * explicitly without devious punning tricks. + */ struct StorePathWithOutputs { StorePath path; @@ -31,9 +32,11 @@ std::pair parsePathWithOutputs(std::string_view s); class Store; -/* Split a string specifying a derivation and a set of outputs - (/nix/store/hash-foo!out1,out2,...) into the derivation path - and the outputs. */ +/** + * Split a string specifying a derivation and a set of outputs + * (/nix/store/hash-foo!out1,out2,...) into the derivation path + * and the outputs. + */ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs); StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index ba8f2749b..4921df352 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -5,12 +5,16 @@ namespace nix { -/* Open (possibly create) a lock file and return the file descriptor. - -1 is returned if create is false and the lock could not be opened - because it doesn't exist. Any other error throws an exception. */ +/** + * Open (possibly create) a lock file and return the file descriptor. + * -1 is returned if create is false and the lock could not be opened + * because it doesn't exist. Any other error throws an exception. + */ AutoCloseFD openLockFile(const Path & path, bool create); -/* Delete an open lock file. */ +/** + * Delete an open lock file. + */ void deleteLockFile(const Path & path, int fd); enum LockType { ltRead, ltWrite, ltNone }; diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 6a5965390..4e1f42e83 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "types.hh" + #include "types.hh" #include "pathlocks.hh" #include @@ -24,9 +24,11 @@ struct Generation typedef std::list Generations; -/* Returns the list of currently present generations for the specified - profile, sorted by generation number. Also returns the number of - the current generation. */ +/** + * Returns the list of currently present generations for the specified + * profile, sorted by generation number. Also returns the number of + * the current generation. + */ std::pair> findGenerations(Path profile); class LocalFSStore; @@ -47,26 +49,32 @@ void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, void switchLink(Path link, Path target); -/* Roll back a profile to the specified generation, or to the most - recent one older than the current. */ +/** + * Roll back a profile to the specified generation, or to the most + * recent one older than the current. + */ void switchGeneration( const Path & profile, std::optional dstGen, bool dryRun); -/* Ensure exclusive access to a profile. Any command that modifies - the profile first acquires this lock. */ +/** + * Ensure exclusive access to a profile. Any command that modifies + * the profile first acquires this lock. + */ void lockProfile(PathLocks & lock, const Path & profile); -/* Optimistic locking is used by long-running operations like `nix-env - -i'. Instead of acquiring the exclusive lock for the entire - duration of the operation, we just perform the operation - optimistically (without an exclusive lock), and check at the end - whether the profile changed while we were busy (i.e., the symlink - target changed). If so, the operation is restarted. Restarting is - generally cheap, since the build results are still in the Nix - store. Most of the time, only the user environment has to be - rebuilt. */ +/** + * Optimistic locking is used by long-running operations like `nix-env + * -i'. Instead of acquiring the exclusive lock for the entire + * duration of the operation, we just perform the operation + * optimistically (without an exclusive lock), and check at the end + * whether the profile changed while we were busy (i.e., the symlink + * target changed). If so, the operation is restarted. Restarting is + * generally cheap, since the build results are still in the Nix + * store. Most of the time, only the user environment has to be + * rebuilt. + */ std::string optimisticLockProfile(const Path & profile); /** diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index f220dbc20..1c45f543e 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -32,8 +32,10 @@ struct RemoteStoreConfig : virtual StoreConfig "Maximum age of a connection before it is closed."}; }; -/* FIXME: RemoteStore is a misnomer - should be something like - DaemonStore. */ +/** + * \todo RemoteStore is a misnomer - should be something like + * DaemonStore. + */ class RemoteStore : public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore, @@ -69,7 +71,9 @@ public: void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; - /* Add a content-addressable store path. `dump` will be drained. */ + /** + * Add a content-addressable store path. `dump` will be drained. + */ ref addCAToStore( Source & dump, std::string_view name, @@ -77,7 +81,9 @@ public: const StorePathSet & references, RepairFlag repair); - /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ + /** + * Add a content-addressable store path. Does not support references. `dump` will be drained. + */ StorePath addToStoreFromDump(Source & dump, std::string_view name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 86410f998..b735838ec 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -11,7 +11,9 @@ struct sqlite3_stmt; namespace nix { -/* RAII wrapper to close a SQLite database automatically. */ +/** + * RAII wrapper to close a SQLite database automatically. + */ struct SQLite { sqlite3 * db = 0; @@ -23,7 +25,9 @@ struct SQLite ~SQLite(); operator sqlite3 * () { return db; } - /* Disable synchronous mode, set truncate journal mode. */ + /** + * Disable synchronous mode, set truncate journal mode. + */ void isCache(); void exec(const std::string & stmt); @@ -31,7 +35,9 @@ struct SQLite uint64_t getLastInsertedRowId(); }; -/* RAII wrapper to create and destroy SQLite prepared statements. */ +/** + * RAII wrapper to create and destroy SQLite prepared statements. + */ struct SQLiteStmt { sqlite3 * db = 0; @@ -43,7 +49,9 @@ struct SQLiteStmt ~SQLiteStmt(); operator sqlite3_stmt * () { return stmt; } - /* Helper for binding / executing statements. */ + /** + * Helper for binding / executing statements. + */ class Use { friend struct SQLiteStmt; @@ -56,7 +64,9 @@ struct SQLiteStmt ~Use(); - /* Bind the next parameter. */ + /** + * Bind the next parameter. + */ Use & operator () (std::string_view value, bool notNull = true); Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); @@ -64,11 +74,15 @@ struct SQLiteStmt int step(); - /* Execute a statement that does not return rows. */ + /** + * Execute a statement that does not return rows. + */ void exec(); - /* For statements that return 0 or more rows. Returns true iff - a row is available. */ + /** + * For statements that return 0 or more rows. Returns true iff + * a row is available. + */ bool next(); std::string getStr(int col); @@ -82,8 +96,10 @@ struct SQLiteStmt } }; -/* RAII helper that ensures transactions are aborted unless explicitly - committed. */ +/** + * RAII helper that ensures transactions are aborted unless explicitly + * committed. + */ struct SQLiteTxn { bool active = false; @@ -125,8 +141,10 @@ MakeError(SQLiteBusy, SQLiteError); void handleSQLiteBusy(const SQLiteBusy & e); -/* Convenience function for retrying a SQLite transaction when the - database is busy. */ +/** + * Convenience function for retrying a SQLite transaction when the + * database is busy. + */ template T retrySQLite(F && fun) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9c6e80486..3cb48bff5 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -411,17 +411,17 @@ public: { unsupported("queryReferrers"); } /** - * @return all currently valid derivations that have `path' as an + * @return all currently valid derivations that have `path` as an * output. * - * (Note that the result of `queryDeriver()' is the derivation that - * was actually used to produce `path', which may not exist + * (Note that the result of `queryDeriver()` is the derivation that + * was actually used to produce `path`, which may not exist * anymore.) */ virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; }; /** - * Query the outputs of the derivation denoted by `path'. + * Query the outputs of the derivation denoted by `path`. */ virtual StorePathSet queryDerivationOutputs(const StorePath & path); @@ -513,7 +513,7 @@ public: /** * Like addToStore(), but the contents of the path are contained - * in `dump', which is either a NAR serialisation (if recursive == + * in `dump`, which is either a NAR serialisation (if recursive == * true) or simply the contents of a regular file (if recursive == * false). * `dump` may be drained @@ -634,8 +634,8 @@ public: /** * @return a string representing information about the path that - * can be loaded into the database using `nix-store --load-db' or - * `nix-store --register-validity'. + * can be loaded into the database using `nix-store --load-db` or + * `nix-store --register-validity`. */ std::string makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash); @@ -715,12 +715,12 @@ public: /** * @param [out] out Place in here the set of all store paths in the - * file system closure of `storePath'; that is, all paths than can - * be directly or indirectly reached from it. `out' is not cleared. + * file system closure of `storePath`; that is, all paths than can + * be directly or indirectly reached from it. `out` is not cleared. * * @param flipDirection If true, the set of paths that can reach - * `storePath' is returned; that is, the closures under the - * `referrers' relation instead of the `references' relation is + * `storePath` is returned; that is, the closures under the + * `referrers` relation instead of the `references` relation is * returned. */ virtual void computeFSClosure(const StorePathSet & paths, diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh index 54721649c..86becafa6 100644 --- a/src/libutil/ansicolor.hh +++ b/src/libutil/ansicolor.hh @@ -1,9 +1,12 @@ #pragma once -///@file +/** + * @file + * + * @brief Some ANSI escape sequences. + */ namespace nix { -/* Some ANSI escape sequences. */ #define ANSI_NORMAL "\e[0m" #define ANSI_BOLD "\e[1m" #define ANSI_FAINT "\e[2m" diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 60e33dd40..2cf164a41 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -20,39 +20,41 @@ namespace nix { * * The format is as follows: * - * IF path points to a REGULAR FILE: - * dump(path) = attrs( - * [ ("type", "regular") - * , ("contents", contents(path)) - * ]) + * ``` + * IF path points to a REGULAR FILE: + * dump(path) = attrs( + * [ ("type", "regular") + * , ("contents", contents(path)) + * ]) * - * IF path points to a DIRECTORY: - * dump(path) = attrs( - * [ ("type", "directory") - * , ("entries", concat(map(f, sort(entries(path))))) - * ]) - * where f(fn) = attrs( - * [ ("name", fn) - * , ("file", dump(path + "/" + fn)) - * ]) + * IF path points to a DIRECTORY: + * dump(path) = attrs( + * [ ("type", "directory") + * , ("entries", concat(map(f, sort(entries(path))))) + * ]) + * where f(fn) = attrs( + * [ ("name", fn) + * , ("file", dump(path + "/" + fn)) + * ]) * - * where: + * where: * - * attrs(as) = concat(map(attr, as)) + encN(0) - * attrs((a, b)) = encS(a) + encS(b) + * attrs(as) = concat(map(attr, as)) + encN(0) + * attrs((a, b)) = encS(a) + encS(b) * - * encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) + * encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) * - * encN(n) = 64-bit little-endian encoding of n. + * encN(n) = 64-bit little-endian encoding of n. * - * contents(path) = the contents of a regular file. + * contents(path) = the contents of a regular file. * - * sort(strings) = lexicographic sort by 8-bit value (strcmp). + * sort(strings) = lexicographic sort by 8-bit value (strcmp). * - * entries(path) = the entries of a directory, without `.' and - * `..'. + * entries(path) = the entries of a directory, without `.` and + * `..`. * - * `+' denotes string concatenation. + * `+` denotes string concatenation. + * ``` */ void dumpPath(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); @@ -88,7 +90,7 @@ struct ParseSink /** * If the NAR archive contains a single file at top-level, then save - * the contents of the file to `s'. Otherwise barf. + * the contents of the file to `s`. Otherwise barf. */ struct RetrieveRegularNARSink : ParseSink { diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index 76e48c4f2..614883c06 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -192,8 +192,10 @@ public: */ bool isAllowed(const std::set & allowed) const; - /* Return a representation `x` of `path` relative to `this`, i.e. - `CanonPath(this.makeRelative(x), this) == path`. */ + /** + * Return a representation `x` of `path` relative to `this`, i.e. + * `CanonPath(this.makeRelative(x), this) == path`. + */ std::string makeRelative(const CanonPath & path) const; }; diff --git a/src/libutil/error.hh b/src/libutil/error.hh index eafc6a540..6a0923081 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -1,5 +1,19 @@ #pragma once -///@file +/** + * @file + * + * @brief This file defines two main structs/classes used in nix error handling. + * + * ErrorInfo provides a standard payload of error information, with conversion to string + * happening in the logger rather than at the call site. + * + * BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains + * an ErrorInfo. + * + * ErrorInfo structs are sent to the logger as part of an exception, or directly with the + * logError or logWarning macros. + * See libutil/tests/logging.cc for usage examples. + */ #include "suggestions.hh" #include "ref.hh" @@ -27,22 +41,6 @@ namespace nix { -/* - - This file defines two main structs/classes used in nix error handling. - - ErrorInfo provides a standard payload of error information, with conversion to string - happening in the logger rather than at the call site. - - BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains - an ErrorInfo. - - ErrorInfo structs are sent to the logger as part of an exception, or directly with the - logError or logWarning macros. - - See libutil/tests/logging.cc for usage examples. - - */ typedef enum { lvlError = 0, diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 576068c22..5aa6bee95 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -183,12 +183,17 @@ bool handleJSONLogMessage(const std::string & msg, const Activity & act, std::map & activities, bool trusted); -extern Verbosity verbosity; /* suppress msgs > this */ +/** + * suppress msgs > this + */ +extern Verbosity verbosity; -/* Print a message with the standard ErrorInfo format. - In general, use these 'log' macros for reporting problems that may require user - intervention or that need more explanation. Use the 'print' macros for more - lightweight status messages. */ +/** + * Print a message with the standard ErrorInfo format. + * In general, use these 'log' macros for reporting problems that may require user + * intervention or that need more explanation. Use the 'print' macros for more + * lightweight status messages. + */ #define logErrorInfo(level, errorInfo...) \ do { \ if ((level) <= nix::verbosity) { \ @@ -199,9 +204,11 @@ extern Verbosity verbosity; /* suppress msgs > this */ #define logError(errorInfo...) logErrorInfo(lvlError, errorInfo) #define logWarning(errorInfo...) logErrorInfo(lvlWarn, errorInfo) -/* Print a string message if the current log level is at least the specified - level. Note that this has to be implemented as a macro to ensure that the - arguments are evaluated lazily. */ +/** + * Print a string message if the current log level is at least the specified + * level. Note that this has to be implemented as a macro to ensure that the + * arguments are evaluated lazily. + */ #define printMsgUsing(loggerParam, level, args...) \ do { \ auto __lvl = level; \ diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index 14b32279c..0e09fae97 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -40,15 +40,15 @@ public: /** * Execute work items until the queue is empty. - * - * \note Note that work items are allowed to add new items to the - * queue; this is handled correctly. * - * Queue processing stops prematurely if any work item throws an - * exception. This exception is propagated to the calling thread. If - * multiple work items throw an exception concurrently, only one - * item is propagated; the others are printed on stderr and - * otherwise ignored. + * \note Note that work items are allowed to add new items to the + * queue; this is handled correctly. + * + * Queue processing stops prematurely if any work item throws an + * exception. This exception is propagated to the calling thread. If + * multiple work items throw an exception concurrently, only one + * item is propagated; the others are printed on stderr and + * otherwise ignored. */ void process(); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 6c2706cc1..56160baaf 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -33,77 +33,112 @@ struct Sink; struct Source; -/* The system for which Nix is compiled. */ +/** + * The system for which Nix is compiled. + */ extern const std::string nativeSystem; -/* Return an environment variable. */ +/** + * @return an environment variable. + */ std::optional getEnv(const std::string & key); -/* Return a non empty environment variable. Returns nullopt if the env -variable is set to "" */ +/** + * @return a non empty environment variable. Returns nullopt if the env + * variable is set to "" + */ std::optional getEnvNonEmpty(const std::string & key); -/* Get the entire environment. */ +/** + * Get the entire environment. + */ std::map getEnv(); -/* Clear the environment. */ +/** + * Clear the environment. + */ void clearEnv(); -/* Return an absolutized path, resolving paths relative to the - specified directory, or the current directory otherwise. The path - is also canonicalised. */ +/** + * @return An absolutized path, resolving paths relative to the + * specified directory, or the current directory otherwise. The path + * is also canonicalised. + */ Path absPath(Path path, std::optional dir = {}, bool resolveSymlinks = false); -/* Canonicalise a path by removing all `.' or `..' components and - double or trailing slashes. Optionally resolves all symlink - components such that each component of the resulting path is *not* - a symbolic link. */ +/** + * Canonicalise a path by removing all `.` or `..` components and + * double or trailing slashes. Optionally resolves all symlink + * components such that each component of the resulting path is *not* + * a symbolic link. + */ Path canonPath(PathView path, bool resolveSymlinks = false); -/* Return the directory part of the given canonical path, i.e., - everything before the final `/'. If the path is the root or an - immediate child thereof (e.g., `/foo'), this means `/' - is returned.*/ +/** + * @return The directory part of the given canonical path, i.e., + * everything before the final `/`. If the path is the root or an + * immediate child thereof (e.g., `/foo`), this means `/` + * is returned. + */ Path dirOf(const PathView path); -/* Return the base name of the given canonical path, i.e., everything - following the final `/' (trailing slashes are removed). */ +/** + * @return the base name of the given canonical path, i.e., everything + * following the final `/` (trailing slashes are removed). + */ std::string_view baseNameOf(std::string_view path); -/* Perform tilde expansion on a path. */ +/** + * Perform tilde expansion on a path. + */ std::string expandTilde(std::string_view path); -/* Check whether 'path' is a descendant of 'dir'. Both paths must be - canonicalized. */ +/** + * Check whether 'path' is a descendant of 'dir'. Both paths must be + * canonicalized. + */ bool isInDir(std::string_view path, std::string_view dir); -/* Check whether 'path' is equal to 'dir' or a descendant of - 'dir'. Both paths must be canonicalized. */ +/** + * Check whether 'path' is equal to 'dir' or a descendant of + * 'dir'. Both paths must be canonicalized. + */ bool isDirOrInDir(std::string_view path, std::string_view dir); -/* Get status of `path'. */ +/** + * Get status of `path`. + */ struct stat stat(const Path & path); struct stat lstat(const Path & path); -/* Return true iff the given path exists. */ +/** + * @return true iff the given path exists. + */ bool pathExists(const Path & path); -/* Read the contents (target) of a symbolic link. The result is not - in any way canonicalised. */ +/** + * Read the contents (target) of a symbolic link. The result is not + * in any way canonicalised. + */ Path readLink(const Path & path); bool isLink(const Path & path); -/* Read the contents of a directory. The entries `.' and `..' are - removed. */ +/** + * Read the contents of a directory. The entries `.` and `..` are + * removed. + */ struct DirEntry { std::string name; ino_t ino; - unsigned char type; // one of DT_* + /** + * one of DT_* + */ + unsigned char type; DirEntry(std::string name, ino_t ino, unsigned char type) : name(std::move(name)), ino(ino), type(type) { } }; @@ -114,74 +149,110 @@ DirEntries readDirectory(const Path & path); unsigned char getFileType(const Path & path); -/* Read the contents of a file into a string. */ +/** + * Read the contents of a file into a string. + */ std::string readFile(int fd); std::string readFile(const Path & path); void readFile(const Path & path, Sink & sink); -/* Write a string to a file. */ +/** + * Write a string to a file. + */ void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); -/* Flush a file's parent directory to disk */ +/** + * Flush a file's parent directory to disk + */ void syncParent(const Path & path); -/* Read a line from a file descriptor. */ +/** + * Read a line from a file descriptor. + */ std::string readLine(int fd); -/* Write a line to a file descriptor. */ +/** + * Write a line to a file descriptor. + */ void writeLine(int fd, std::string s); -/* Delete a path; i.e., in the case of a directory, it is deleted - recursively. It's not an error if the path does not exist. The - second variant returns the number of bytes and blocks freed. */ +/** + * Delete a path; i.e., in the case of a directory, it is deleted + * recursively. It's not an error if the path does not exist. The + * second variant returns the number of bytes and blocks freed. + */ void deletePath(const Path & path); void deletePath(const Path & path, uint64_t & bytesFreed); std::string getUserName(); -/* Return the given user's home directory from /etc/passwd. */ +/** + * @return the given user's home directory from /etc/passwd. + */ Path getHomeOf(uid_t userId); -/* Return $HOME or the user's home directory from /etc/passwd. */ +/** + * @return $HOME or the user's home directory from /etc/passwd. + */ Path getHome(); -/* Return $XDG_CACHE_HOME or $HOME/.cache. */ +/** + * @return $XDG_CACHE_HOME or $HOME/.cache. + */ Path getCacheDir(); -/* Return $XDG_CONFIG_HOME or $HOME/.config. */ +/** + * @return $XDG_CONFIG_HOME or $HOME/.config. + */ Path getConfigDir(); -/* Return the directories to search for user configuration files */ +/** + * @return the directories to search for user configuration files + */ std::vector getConfigDirs(); -/* Return $XDG_DATA_HOME or $HOME/.local/share. */ +/** + * @return $XDG_DATA_HOME or $HOME/.local/share. + */ Path getDataDir(); -/* Return the path of the current executable. */ +/** + * @return the path of the current executable. + */ std::optional getSelfExe(); -/* Return $XDG_STATE_HOME or $HOME/.local/state. */ +/** + * @return $XDG_STATE_HOME or $HOME/.local/state. + */ Path getStateDir(); -/* Create the Nix state directory and return the path to it. */ +/** + * Create the Nix state directory and return the path to it. + */ Path createNixStateDir(); -/* Create a directory and all its parents, if necessary. Returns the - list of created directories, in order of creation. */ +/** + * Create a directory and all its parents, if necessary. Returns the + * list of created directories, in order of creation. + */ Paths createDirs(const Path & path); inline Paths createDirs(PathView path) { return createDirs(Path(path)); } -/* Create a symlink. */ +/** + * Create a symlink. + */ void createSymlink(const Path & target, const Path & link, std::optional mtime = {}); -/* Atomically create or replace a symlink. */ +/** + * Atomically create or replace a symlink. + */ void replaceSymlink(const Path & target, const Path & link, std::optional mtime = {}); @@ -197,24 +268,32 @@ void renameFile(const Path & src, const Path & dst); void moveFile(const Path & src, const Path & dst); -/* Wrappers arount read()/write() that read/write exactly the - requested number of bytes. */ +/** + * Wrappers arount read()/write() that read/write exactly the + * requested number of bytes. + */ void readFull(int fd, char * buf, size_t count); void writeFull(int fd, std::string_view s, bool allowInterrupts = true); MakeError(EndOfFile, Error); -/* Read a file descriptor until EOF occurs. */ +/** + * Read a file descriptor until EOF occurs. + */ std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); void drainFD(int fd, Sink & sink, bool block = true); -/* If cgroups are active, attempt to calculate the number of CPUs available. - If cgroups are unavailable or if cpu.max is set to "max", return 0. */ +/** + * If cgroups are active, attempt to calculate the number of CPUs available. + * If cgroups are unavailable or if cpu.max is set to "max", return 0. + */ unsigned int getMaxCPU(); -/* Automatic cleanup of resources. */ +/** + * Automatic cleanup of resources. + */ class AutoDelete @@ -252,11 +331,15 @@ public: }; -/* Create a temporary directory. */ +/** + * Create a temporary directory. + */ Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); -/* Create a temporary file, returning a file handle and its path. */ +/** + * Create a temporary file, returning a file handle and its path. + */ std::pair createTempFile(const Path & prefix = "nix"); @@ -299,27 +382,36 @@ public: }; -/* Kill all processes running under the specified uid by sending them - a SIGKILL. */ +/** + * Kill all processes running under the specified uid by sending them + * a SIGKILL. + */ void killUser(uid_t uid); -/* Fork a process that runs the given function, and return the child - pid to the caller. */ +/** + * Fork a process that runs the given function, and return the child + * pid to the caller. + */ struct ProcessOptions { std::string errorPrefix = ""; bool dieWithParent = true; bool runExitHandlers = false; bool allowVfork = false; - int cloneFlags = 0; // use clone() with the specified flags (Linux only) + /** + * use clone() with the specified flags (Linux only) + */ + int cloneFlags = 0; }; pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); -/* Run a program and return its stdout in a string (i.e., like the - shell backtick operator). */ +/** + * Run a program and return its stdout in a string (i.e., like the + * shell backtick operator). + */ std::string runProgram(Path program, bool searchPath = false, const Strings & args = Strings(), const std::optional & input = {}); @@ -344,25 +436,35 @@ std::pair runProgram(RunOptions && options); void runProgram2(const RunOptions & options); -/* Change the stack size. */ +/** + * Change the stack size. + */ void setStackSize(size_t stackSize); -/* Restore the original inherited Unix process context (such as signal - masks, stack size). */ +/** + * Restore the original inherited Unix process context (such as signal + * masks, stack size). + */ void restoreProcessContext(bool restoreMounts = true); -/* Save the current mount namespace. Ignored if called more than - once. */ +/** + * Save the current mount namespace. Ignored if called more than + * once. + */ void saveMountNamespace(); -/* Restore the mount namespace saved by saveMountNamespace(). Ignored - if saveMountNamespace() was never called. */ +/** + * Restore the mount namespace saved by saveMountNamespace(). Ignored + * if saveMountNamespace() was never called. + */ void restoreMountNamespace(); -/* Cause this thread to not share any FS attributes with the main - thread, because this causes setns() in restoreMountNamespace() to - fail. */ +/** + * Cause this thread to not share any FS attributes with the main + * thread, because this causes setns() in restoreMountNamespace() to + * fail. + */ void unshareFilesystem(); @@ -377,16 +479,22 @@ public: { } }; -/* Convert a list of strings to a null-terminated vector of char - *'s. The result must not be accessed beyond the lifetime of the - list of strings. */ +/** + * Convert a list of strings to a null-terminated vector of `char + * *`s. The result must not be accessed beyond the lifetime of the + * list of strings. + */ std::vector stringsToCharPtrs(const Strings & ss); -/* Close all file descriptors except those listed in the given set. - Good practice in child processes. */ +/** + * Close all file descriptors except those listed in the given set. + * Good practice in child processes. + */ void closeMostFDs(const std::set & exceptions); -/* Set the close-on-exec flag for the given file descriptor. */ +/** + * Set the close-on-exec flag for the given file descriptor. + */ void closeOnExec(int fd); @@ -412,12 +520,16 @@ MakeError(Interrupted, BaseError); MakeError(FormatError, Error); -/* String tokenizer. */ +/** + * String tokenizer. + */ template C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); -/* Concatenate the given strings with a separator between the - elements. */ +/** + * Concatenate the given strings with a separator between the + * elements. + */ template std::string concatStringsSep(const std::string_view sep, const C & ss) { @@ -442,7 +554,9 @@ auto concatStrings(Parts && ... parts) } -/* Add quotes around a collection of strings. */ +/** + * Add quotes around a collection of strings. + */ template Strings quoteStrings(const C & c) { Strings res; @@ -451,16 +565,23 @@ template Strings quoteStrings(const C & c) return res; } -/* Remove trailing whitespace from a string. FIXME: return - std::string_view. */ +/** + * Remove trailing whitespace from a string. + * + * \todo return std::string_view. + */ std::string chomp(std::string_view s); -/* Remove whitespace from the start and end of a string. */ +/** + * Remove whitespace from the start and end of a string. + */ std::string trim(std::string_view s, std::string_view whitespace = " \n\r\t"); -/* Replace all occurrences of a string inside another string. */ +/** + * Replace all occurrences of a string inside another string. + */ std::string replaceStrings( std::string s, std::string_view from, @@ -470,14 +591,18 @@ std::string replaceStrings( std::string rewriteStrings(std::string s, const StringMap & rewrites); -/* Convert the exit status of a child as returned by wait() into an - error string. */ +/** + * Convert the exit status of a child as returned by wait() into an + * error string. + */ std::string statusToString(int status); bool statusOk(int status); -/* Parse a string into an integer. */ +/** + * Parse a string into an integer. + */ template std::optional string2Int(const std::string_view s) { @@ -490,8 +615,10 @@ std::optional string2Int(const std::string_view s) } } -/* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or - 'T' denoting a binary unit prefix. */ +/** + * Like string2Int(), but support an optional suffix 'K', 'M', 'G' or + * 'T' denoting a binary unit prefix. + */ template N string2IntWithUnitPrefix(std::string_view s) { @@ -512,7 +639,9 @@ N string2IntWithUnitPrefix(std::string_view s) throw UsageError("'%s' is not an integer", s); } -/* Parse a string into a float. */ +/** + * Parse a string into a float. + */ template std::optional string2Float(const std::string_view s) { @@ -524,7 +653,9 @@ std::optional string2Float(const std::string_view s) } -/* Convert a little-endian integer to host order. */ +/** + * Convert a little-endian integer to host order. + */ template T readLittleEndian(unsigned char * p) { @@ -536,66 +667,90 @@ T readLittleEndian(unsigned char * p) } -/* Return true iff `s' starts with `prefix'. */ +/** + * @return true iff `s` starts with `prefix`. + */ bool hasPrefix(std::string_view s, std::string_view prefix); -/* Return true iff `s' ends in `suffix'. */ +/** + * @return true iff `s` ends in `suffix`. + */ bool hasSuffix(std::string_view s, std::string_view suffix); -/* Convert a string to lower case. */ +/** + * Convert a string to lower case. + */ std::string toLower(const std::string & s); -/* Escape a string as a shell word. */ +/** + * Escape a string as a shell word. + */ std::string shellEscape(const std::string_view s); -/* Exception handling in destructors: print an error message, then - ignore the exception. */ +/** + * Exception handling in destructors: print an error message, then + * ignore the exception. + */ void ignoreException(Verbosity lvl = lvlError); -/* Tree formatting. */ +/** + * Tree formatting. + */ constexpr char treeConn[] = "├───"; constexpr char treeLast[] = "└───"; constexpr char treeLine[] = "│ "; constexpr char treeNull[] = " "; -/* Determine whether ANSI escape sequences are appropriate for the - present output. */ +/** + * Determine whether ANSI escape sequences are appropriate for the + * present output. + */ bool shouldANSI(); -/* Truncate a string to 'width' printable characters. If 'filterAll' - is true, all ANSI escape sequences are filtered out. Otherwise, - some escape sequences (such as colour setting) are copied but not - included in the character count. Also, tabs are expanded to - spaces. */ +/** + * Truncate a string to 'width' printable characters. If 'filterAll' + * is true, all ANSI escape sequences are filtered out. Otherwise, + * some escape sequences (such as colour setting) are copied but not + * included in the character count. Also, tabs are expanded to + * spaces. + */ std::string filterANSIEscapes(std::string_view s, bool filterAll = false, unsigned int width = std::numeric_limits::max()); -/* Base64 encoding/decoding. */ +/** + * Base64 encoding/decoding. + */ std::string base64Encode(std::string_view s); std::string base64Decode(std::string_view s); -/* Remove common leading whitespace from the lines in the string - 's'. For example, if every line is indented by at least 3 spaces, - then we remove 3 spaces from the start of every line. */ +/** + * Remove common leading whitespace from the lines in the string + * 's'. For example, if every line is indented by at least 3 spaces, + * then we remove 3 spaces from the start of every line. + */ std::string stripIndentation(std::string_view s); -/* Get the prefix of 's' up to and excluding the next line break (LF - optionally preceded by CR), and the remainder following the line - break. */ +/** + * Get the prefix of 's' up to and excluding the next line break (LF + * optionally preceded by CR), and the remainder following the line + * break. + */ std::pair getLine(std::string_view s); -/* Get a value for the specified key from an associate container. */ +/** + * Get a value for the specified key from an associate container. + */ template const typename T::mapped_type * get(const T & map, const typename T::key_type & key) { @@ -612,7 +767,9 @@ typename T::mapped_type * get(T & map, const typename T::key_type & key) return &i->second; } -/* Get a value for the specified key from an associate container, or a default value if the key isn't present. */ +/** + * Get a value for the specified key from an associate container, or a default value if the key isn't present. + */ template const typename T::mapped_type & getOr(T & map, const typename T::key_type & key, @@ -623,7 +780,9 @@ const typename T::mapped_type & getOr(T & map, return i->second; } -/* Remove and return the first item from a container. */ +/** + * Remove and return the first item from a container. + */ template std::optional remove_begin(T & c) { @@ -635,7 +794,9 @@ std::optional remove_begin(T & c) } -/* Remove and return the first item from a container. */ +/** + * Remove and return the first item from a container. + */ template std::optional pop(T & c) { @@ -650,8 +811,10 @@ template class Callback; -/* Start a thread that handles various signals. Also block those signals - on the current thread (and thus any threads created by it). */ +/** + * Start a thread that handles various signals. Also block those signals + * on the current thread (and thus any threads created by it). + */ void startSignalHandlerThread(); struct InterruptCallback @@ -659,16 +822,20 @@ struct InterruptCallback virtual ~InterruptCallback() { }; }; -/* Register a function that gets called on SIGINT (in a non-signal - context). */ +/** + * Register a function that gets called on SIGINT (in a non-signal + * context). + */ std::unique_ptr createInterruptCallback( std::function callback); void triggerInterrupt(); -/* A RAII class that causes the current thread to receive SIGUSR1 when - the signal handler thread receives SIGINT. That is, this allows - SIGINT to be multiplexed to multiple threads. */ +/** + * A RAII class that causes the current thread to receive SIGUSR1 when + * the signal handler thread receives SIGINT. That is, this allows + * SIGINT to be multiplexed to multiple threads. + */ struct ReceiveInterrupts { pthread_t target; @@ -682,8 +849,10 @@ struct ReceiveInterrupts -/* A RAII helper that increments a counter on construction and - decrements it on destruction. */ +/** + * A RAII helper that increments a counter on construction and + * decrements it on destruction. + */ template struct MaintainCount { @@ -694,33 +863,50 @@ struct MaintainCount }; -/* Return the number of rows and columns of the terminal. */ +/** + * @return the number of rows and columns of the terminal. + */ std::pair getWindowSize(); -/* Used in various places. */ +/** + * Used in various places. + */ typedef std::function PathFilter; extern PathFilter defaultPathFilter; -/* Common initialisation performed in child processes. */ +/** + * Common initialisation performed in child processes. + */ void commonChildInit(); -/* Create a Unix domain socket. */ +/** + * Create a Unix domain socket. + */ AutoCloseFD createUnixDomainSocket(); -/* Create a Unix domain socket in listen mode. */ +/** + * Create a Unix domain socket in listen mode. + */ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); -/* Bind a Unix domain socket to a path. */ +/** + * Bind a Unix domain socket to a path. + */ void bind(int fd, const std::string & path); -/* Connect to a Unix domain socket. */ +/** + * Connect to a Unix domain socket. + */ void connect(int fd, const std::string & path); -// A Rust/Python-like enumerate() iterator adapter. -// Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. +/** + * A Rust/Python-like enumerate() iterator adapter. + * + * Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. + */ template ())), typename = decltype(std::end(std::declval()))> @@ -746,7 +932,9 @@ constexpr auto enumerate(T && iterable) } -// C++17 std::visit boilerplate +/** + * C++17 std::visit boilerplate + */ template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; @@ -754,8 +942,10 @@ template overloaded(Ts...) -> overloaded; std::string showBytes(uint64_t bytes); -/* Provide an addition operator between strings and string_views - inexplicably omitted from the standard library. */ +/** + * Provide an addition operator between strings and string_views + * inexplicably omitted from the standard library. + */ inline std::string operator + (const std::string & s1, std::string_view s2) { auto s = s1; From 8f44edc83ea072bd9559f40fb6e308ee501afbf0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 7 Apr 2023 11:44:34 -0400 Subject: [PATCH 53/75] Tidy up and comment daemon CLI Some of the factoring out was taken from #7912 by @mupdt. Thanks! No behavior should be changed in this commit. Co-Authored-By: mupdt <25388474+mupdt@users.noreply.github.com> Co-authored-by: Robert Hensing --- src/nix/daemon.cc | 213 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 157 insertions(+), 56 deletions(-) diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 7e4a7ba86..7ae7b4ea6 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -1,3 +1,5 @@ +///@file + #include "command.hh" #include "shared.hh" #include "local-store.hh" @@ -34,6 +36,19 @@ using namespace nix; using namespace nix::daemon; +/** + * Settings related to authenticating clients for the Nix daemon. + * + * For pipes we have little good information about the client side, but + * for Unix domain sockets we do. So currently these options implemented + * mandatory access control based on user names and group names (looked + * up and translated to UID/GIDs in the CLI process that runs the code + * in this file). + * + * No code outside of this file knows about these settings (this is not + * exposed in a header); all authentication and authorization happens in + * `daemon.cc`. + */ struct AuthorizationSettings : Config { Setting trustedUsers{ @@ -54,7 +69,9 @@ struct AuthorizationSettings : Config { > directories that are otherwise inacessible to them. )"}; - /* ?Who we trust to use the daemon in safe ways */ + /** + * Who we trust to use the daemon in safe ways + */ Setting allowedUsers{ this, {"*"}, "allowed-users", R"( @@ -112,8 +129,36 @@ static void setSigChldAction(bool autoReap) throw SysError("setting SIGCHLD handler"); } +/** + * @return Is the given user a member of this group? + * + * @param user User specified by username. + * + * @param group Group the user might be a member of. + */ +static bool matchUser(std::string_view user, const struct group & gr) +{ + for (char * * mem = gr.gr_mem; *mem; mem++) + if (user == std::string_view(*mem)) return true; + return false; +} -bool matchUser(const std::string & user, const std::string & group, const Strings & users) + +/** + * Does the given user (specified by user name and primary group name) + * match the given user/group whitelist? + * + * If the list allows all users: Yes. + * + * If the username is in the set: Yes. + * + * If the groupname is in the set: Yes. + * + * If the user is in another group which is in the set: yes. + * + * Otherwise: No. + */ +static bool matchUser(const std::string & user, const std::string & group, const Strings & users) { if (find(users.begin(), users.end(), "*") != users.end()) return true; @@ -126,8 +171,7 @@ bool matchUser(const std::string & user, const std::string & group, const String if (group == i.substr(1)) return true; struct group * gr = getgrnam(i.c_str() + 1); if (!gr) continue; - for (char * * mem = gr->gr_mem; *mem; mem++) - if (user == std::string(*mem)) return true; + if (matchUser(user, *gr)) return true; } return false; @@ -145,7 +189,9 @@ struct PeerInfo }; -// Get the identity of the caller, if possible. +/** + * Get the identity of the caller, if possible. + */ static PeerInfo getPeerInfo(int remote) { PeerInfo peer = { false, 0, false, 0, false, 0 }; @@ -179,6 +225,9 @@ static PeerInfo getPeerInfo(int remote) #define SD_LISTEN_FDS_START 3 +/** + * Open a store without a path info cache. + */ static ref openUncachedStore() { Store::Params params; // FIXME: get params from somewhere @@ -187,7 +236,44 @@ static ref openUncachedStore() return openStore(settings.storeUri, params); } +/** + * Authenticate a potential client + * + * @param peer Information about other end of the connection, the client which + * wants to communicate with us. + * + * @return A pair of a `TrustedFlag`, whether the potential client is trusted, + * and the name of the user (useful for printing messages). + * + * If the potential client is not allowed to talk to us, we throw an `Error`. + */ +static std::pair authPeer(const PeerInfo & peer) +{ + TrustedFlag trusted = NotTrusted; + struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; + std::string user = pw ? pw->pw_name : std::to_string(peer.uid); + + struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; + std::string group = gr ? gr->gr_name : std::to_string(peer.gid); + + const Strings & trustedUsers = authorizationSettings.trustedUsers; + const Strings & allowedUsers = authorizationSettings.allowedUsers; + + if (matchUser(user, group, trustedUsers)) + trusted = Trusted; + + if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) + throw Error("user '%1%' is not allowed to connect to the Nix daemon", user); + + return { trusted, std::move(user) }; +} + + +/** + * Run a server. The loop opens a socket and accepts new connections from that + * socket. + */ static void daemonLoop() { if (chdir("/") == -1) @@ -231,23 +317,9 @@ static void daemonLoop() closeOnExec(remote.get()); - TrustedFlag trusted = NotTrusted; PeerInfo peer = getPeerInfo(remote.get()); - - struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; - std::string user = pw ? pw->pw_name : std::to_string(peer.uid); - - struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; - std::string group = gr ? gr->gr_name : std::to_string(peer.gid); - - Strings trustedUsers = authorizationSettings.trustedUsers; - Strings allowedUsers = authorizationSettings.allowedUsers; - - if (matchUser(user, group, trustedUsers)) - trusted = Trusted; - - if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) - throw Error("user '%1%' is not allowed to connect to the Nix daemon", user); + auto [_trusted, user] = authPeer(peer); + auto trusted = _trusted; printInfo((std::string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : ""), peer.pidKnown ? std::to_string(peer.pid) : "", @@ -294,45 +366,74 @@ static void daemonLoop() } } +/** + * Forward a standard IO connection to the given remote store. + * + * We just act as a middleman blindly ferry output between the standard + * input/output and the remote store connection, not processing anything. + * + * Loops until standard input disconnects, or an error is encountered. + */ +static void forwardStdioConnection(RemoteStore & store) { + auto conn = store.openConnectionWrapper(); + int from = conn->from.fd; + int to = conn->to.fd; + + auto nfds = std::max(from, STDIN_FILENO) + 1; + while (true) { + fd_set fds; + FD_ZERO(&fds); + FD_SET(from, &fds); + FD_SET(STDIN_FILENO, &fds); + if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) + throw SysError("waiting for data from client or server"); + if (FD_ISSET(from, &fds)) { + auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from daemon socket to stdout"); + else if (res == 0) + throw EndOfFile("unexpected EOF from daemon socket"); + } + if (FD_ISSET(STDIN_FILENO, &fds)) { + auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from stdin to daemon socket"); + else if (res == 0) + return; + } + } +} + +/** + * Process a client connecting to us via standard input/output + * + * Unlike `forwardStdioConnection()` we do process commands ourselves in + * this case, not delegating to another daemon. + * + * @note `Trusted` is unconditionally passed because in this mode we + * blindly trust the standard streams. Limiting access to those is + * explicitly not `nix-daemon`'s responsibility. + */ +static void processStdioConnection(ref store) +{ + FdSource from(STDIN_FILENO); + FdSink to(STDOUT_FILENO); + processConnection(store, from, to, Trusted, NotRecursive); +} + +/** + * Entry point shared between the new CLI `nix daemon` and old CLI + * `nix-daemon`. + */ static void runDaemon(bool stdio) { if (stdio) { - if (auto store = openUncachedStore().dynamic_pointer_cast()) { - auto conn = store->openConnectionWrapper(); - int from = conn->from.fd; - int to = conn->to.fd; + auto store = openUncachedStore(); - auto nfds = std::max(from, STDIN_FILENO) + 1; - while (true) { - fd_set fds; - FD_ZERO(&fds); - FD_SET(from, &fds); - FD_SET(STDIN_FILENO, &fds); - if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) - throw SysError("waiting for data from client or server"); - if (FD_ISSET(from, &fds)) { - auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from daemon socket to stdout"); - else if (res == 0) - throw EndOfFile("unexpected EOF from daemon socket"); - } - if (FD_ISSET(STDIN_FILENO, &fds)) { - auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from stdin to daemon socket"); - else if (res == 0) - return; - } - } - } else { - FdSource from(STDIN_FILENO); - FdSink to(STDOUT_FILENO); - /* Auth hook is empty because in this mode we blindly trust the - standard streams. Limiting access to those is explicitly - not `nix-daemon`'s responsibility. */ - processConnection(openUncachedStore(), from, to, Trusted, NotRecursive); - } + if (auto remoteStore = store.dynamic_pointer_cast()) + forwardStdioConnection(*remoteStore); + else + processStdioConnection(store); } else daemonLoop(); } From 8ffc05e7ec7da1bff88fafeab9617bcf04005d62 Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Sat, 8 Apr 2023 22:29:43 +0200 Subject: [PATCH 54/75] add check for librapidcheck declare RAPIDCHECK_HEADERS as variable --- configure.ac | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index f1f45f868..e587bd563 100644 --- a/configure.ac +++ b/configure.ac @@ -289,13 +289,24 @@ PKG_CHECK_MODULES([GTEST], [gtest_main]) # Look for rapidcheck. +AC_ARG_VAR([RAPIDCHECK_HEADERS], [include path of gtest headers shipped by RAPIDCHECK]) # No pkg-config yet, https://github.com/emil-e/rapidcheck/issues/302 AC_LANG_PUSH(C++) AC_SUBST(RAPIDCHECK_HEADERS) [CXXFLAGS="-I $RAPIDCHECK_HEADERS $CXXFLAGS"] +[LIBS="-lrapidcheck -lgtest $LIBS"] AC_CHECK_HEADERS([rapidcheck/gtest.h], [], [], [#include ]) -dnl No good for C++ libs with mangled symbols -dnl AC_CHECK_LIB([rapidcheck], []) +dnl AC_CHECK_LIB doesn't work for C++ libs with mangled symbols +AC_LINK_IFELSE([ + AC_LANG_PROGRAM([[ + #include + #include + ]], [[ + return RUN_ALL_TESTS(); + ]]) + ], + [], + [AC_MSG_ERROR([librapidcheck is not found.])]) AC_LANG_POP(C++) fi From ca6b759f4ed95faba814ce0fda1668f09eb8faf3 Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Sun, 9 Apr 2023 02:33:53 +0200 Subject: [PATCH 55/75] fix failing configure in nix-tests --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index dc64bdfcf..a4ee80b32 100644 --- a/flake.nix +++ b/flake.nix @@ -219,6 +219,7 @@ enableParallelBuilding = true; + configureFlags = testConfigureFlags; # otherwise configure fails dontBuild = true; doInstallCheck = true; From 73eb6a2a578c85fe4a9399bdc20b4ff943fb49b4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 9 Apr 2023 11:01:23 -0400 Subject: [PATCH 56/75] Single page for experimental feature descriptions As requested by @fricklerhandwerk. --- .gitignore | 2 +- doc/manual/generate-xp-features-shortlist.nix | 2 +- doc/manual/generate-xp-features.nix | 22 +++++-------------- doc/manual/local.mk | 6 ++--- doc/manual/src/SUMMARY.md.in | 1 - .../src/contributing/experimental-features.md | 4 ++++ 6 files changed, 15 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index ffaf52be8..8ceff4ef2 100644 --- a/.gitignore +++ b/.gitignore @@ -24,7 +24,7 @@ perl/Makefile.config /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md /doc/manual/src/command-ref/experimental-features-shortlist.md -/doc/manual/src/contributing/experimental-features +/doc/manual/src/contributing/experimental-feature-descriptions.md /doc/manual/src/language/builtins.md # /scripts/ diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix index b2095bc27..30e211c96 100644 --- a/doc/manual/generate-xp-features-shortlist.nix +++ b/doc/manual/generate-xp-features-shortlist.nix @@ -4,6 +4,6 @@ with import ./utils.nix; let showExperimentalFeature = name: doc: '' - - [`${name}`](@docroot@/contributing/experimental-features/${name}.md) + - [`${name}`](@docroot@/contributing/experimental-features.md#xp-feature-${name}) ''; in xps: indent " " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix index ff64edcf7..adb94355c 100644 --- a/doc/manual/generate-xp-features.nix +++ b/doc/manual/generate-xp-features.nix @@ -1,21 +1,11 @@ -xps: - with builtins; with import ./utils.nix; let - makePage = { name, value }: - { - name = "${name}.md"; - inherit value; - feature = name; - }; + showExperimentalFeature = name: doc: + squash '' + ## [`${name}`]{#xp-feature-${name}} - featurePages = map makePage (attrsToList xps); - - tableOfContents = let - showEntry = page: - " - [${page.feature}](contributing/experimental-features/${page.name})"; - in concatStringsSep "\n" (map showEntry featurePages) + "\n"; - -in (listToAttrs featurePages) // { "SUMMARY.md" = tableOfContents; } + ${doc} + ''; +in xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index ae55ae4a9..63e7e61e4 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -85,7 +85,7 @@ $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-features +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md @cp $< $@ @$(call process-includes,$@,$@) @@ -107,7 +107,7 @@ $(d)/conf-file.json: $(bindir)/nix $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/contributing/experimental-features: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix +$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ @@ -160,7 +160,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-features $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 298644044..5bf274550 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -96,7 +96,6 @@ - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) - [Experimental Features](contributing/experimental-features.md) -{{#include ./contributing/experimental-features/SUMMARY.md}} - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/contributing/experimental-features.md index f1db22751..ad5cffa91 100644 --- a/doc/manual/src/contributing/experimental-features.md +++ b/doc/manual/src/contributing/experimental-features.md @@ -89,3 +89,7 @@ However they serve different purposes: It is primarily an issue of *design* and *communication*, targeting the broader community. This means that experimental features and RFCs are orthogonal mechanisms, and can be used independently or together as needed. + +# Currently available experimental features + +{{#include ./experimental-feature-descriptions.md}} From 4e0804c920558575a4b3486df1e595445bf67555 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 9 Apr 2023 22:42:20 +0200 Subject: [PATCH 57/75] Deduplicate string literal rendering, fix 4909 --- src/libcmd/repl.cc | 20 ++++++-------------- src/libexpr/eval.cc | 13 +++---------- src/libexpr/nixexpr.cc | 19 ++++--------------- src/libexpr/value/print.cc | 26 ++++++++++++++++++++++++++ src/libexpr/value/print.hh | 30 ++++++++++++++++++++++++++++++ src/libstore/derivations.cc | 9 +++++++++ tests/repl.sh | 8 ++++++++ 7 files changed, 86 insertions(+), 39 deletions(-) create mode 100644 src/libexpr/value/print.cc create mode 100644 src/libexpr/value/print.hh diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 57848a5d3..1366622c7 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -40,6 +40,7 @@ extern "C" { #include "markdown.hh" #include "local-fs-store.hh" #include "progress-bar.hh" +#include "value/print.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW @@ -894,17 +895,6 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m } -std::ostream & printStringValue(std::ostream & str, const char * string) { - str << "\""; - for (const char * i = string; *i; i++) - if (*i == '\"' || *i == '\\') str << "\\" << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else str << *i; - str << "\""; - return str; -} // FIXME: lot of cut&paste from Nix's eval.cc. @@ -922,12 +912,14 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m break; case nBool: - str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL; + str << ANSI_CYAN; + printLiteral(str, v.boolean); + str << ANSI_NORMAL; break; case nString: str << ANSI_WARNING; - printStringValue(str, v.string.s); + printLiteral(str, v.string.s); str << ANSI_NORMAL; break; @@ -967,7 +959,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m if (isVarName(i.first)) str << i.first; else - printStringValue(str, i.first.c_str()); + printLiteral(str, i.first); str << " = "; if (seen.count(i.second)) str << "«repeated»"; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 18cfd9531..06208897f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -9,6 +9,7 @@ #include "filetransfer.hh" #include "function-trace.hh" #include "profiles.hh" +#include "value/print.hh" #include #include @@ -104,18 +105,10 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, str << integer; break; case tBool: - str << (boolean ? "true" : "false"); + printLiteral(str, boolean); break; case tString: - str << "\""; - for (const char * i = string.s; *i; i++) - if (*i == '\"' || *i == '\\') str << "\\" << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else if (*i == '$' && *(i+1) == '{') str << "\\" << *i; - else str << *i; - str << "\""; + printLiteral(str, string.s); break; case tPath: str << path; // !!! escaping? diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index eb6f062b4..ca6df0af3 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -3,6 +3,7 @@ #include "eval.hh" #include "symbol-table.hh" #include "util.hh" +#include "value/print.hh" #include @@ -62,18 +63,6 @@ Pos::operator std::shared_ptr() const /* Displaying abstract syntax trees. */ -static void showString(std::ostream & str, std::string_view s) -{ - str << '"'; - for (auto c : s) - if (c == '"' || c == '\\' || c == '$') str << "\\" << c; - else if (c == '\n') str << "\\n"; - else if (c == '\r') str << "\\r"; - else if (c == '\t') str << "\\t"; - else str << c; - str << '"'; -} - std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) { std::string_view s = symbol; @@ -85,7 +74,7 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) else { char c = s[0]; if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { - showString(str, s); + printLiteral(str, s); return str; } for (auto c : s) @@ -93,7 +82,7 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '\'' || c == '-')) { - showString(str, s); + printLiteral(str, s); return str; } str << s; @@ -118,7 +107,7 @@ void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const void ExprString::show(const SymbolTable & symbols, std::ostream & str) const { - showString(str, s); + printLiteral(str, s); } void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const diff --git a/src/libexpr/value/print.cc b/src/libexpr/value/print.cc new file mode 100644 index 000000000..cf97def46 --- /dev/null +++ b/src/libexpr/value/print.cc @@ -0,0 +1,26 @@ +#include "value/print.hh" + +namespace nix { + +std::ostream & +printLiteral(std::ostream & str, const std::string_view string) { + str << "\""; + for (auto i = string.begin(); i != string.end(); ++i) { + if (*i == '\"' || *i == '\\') str << "\\" << *i; + else if (*i == '\n') str << "\\n"; + else if (*i == '\r') str << "\\r"; + else if (*i == '\t') str << "\\t"; + else if (*i == '$' && *(i+1) == '{') str << "\\" << *i; + else str << *i; + } + str << "\""; + return str; +} + +std::ostream & +printLiteral(std::ostream & str, bool boolean) { + str << (boolean ? "true" : "false"); + return str; +} + +} diff --git a/src/libexpr/value/print.hh b/src/libexpr/value/print.hh new file mode 100644 index 000000000..31c94eb85 --- /dev/null +++ b/src/libexpr/value/print.hh @@ -0,0 +1,30 @@ +#pragma once +/** + * @file + * @brief Common printing functions for the Nix language + * + * While most types come with their own methods for printing, they share some + * functions that are placed here. + */ + +#include + +namespace nix { + /** + * Print a string as a Nix string literal. + * + * Quotes and fairly minimal escaping are added. + * + * @param s The logical string + */ + std::ostream & printLiteral(std::ostream & o, std::string_view s); + inline std::ostream & printLiteral(std::ostream & o, const char * s) { + return printLiteral(o, std::string_view(s)); + } + inline std::ostream & printLiteral(std::ostream & o, const std::string & s) { + return printLiteral(o, std::string_view(s)); + } + + /** Print `true` or `false`. */ + std::ostream & printLiteral(std::ostream & o, bool b); +} diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index abdfb1978..9948862e5 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -313,6 +313,15 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi } +/** + * Print a derivation string literal to an std::string. + * + * This syntax does not generalize to the expression language, which needs to + * escape `$`. + * + * @param res Where to print to + * @param s Which logical string to print + */ static void printString(std::string & res, std::string_view s) { boost::container::small_vector buffer; diff --git a/tests/repl.sh b/tests/repl.sh index be8adb742..2b3789521 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -79,6 +79,14 @@ testReplResponse ' "result: ${a}" ' "result: 2" +# check dollar escaping https://github.com/NixOS/nix/issues/4909 +# note the escaped \, +# \\ +# because the second argument is a regex +testReplResponse ' +"$" + "{hi}" +' '"\\${hi}"' + testReplResponse ' drvPath ' '".*-simple.drv"' \ From deb7f4b466574c12a8384f1c76df0141c6f7ca69 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 11 Apr 2023 11:29:35 +0200 Subject: [PATCH 58/75] Nitpicks --- src/libutil/config.hh | 4 ++-- src/libutil/experimental-features.cc | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 6baba0167..162626791 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -382,11 +382,11 @@ struct ExperimentalFeatureSettings : Config { experimental-features = nix-command flakes ``` - Experimental features available: + The following experimental features are available: {{#include experimental-features-shortlist.md}} - Experimental features are [further documented](@docroot@/contributing/experimental-features.md) in the contribution guide. + Experimental features are [further documented in the manual](@docroot@/contributing/experimental-features.md). )"}; /** diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 1cd0dbb79..5b4418714 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -204,7 +204,7 @@ const std::optional parseExperimentalFeature(const std::str { using ReverseXpMap = std::map; - static std::unique_ptr reverseXpMap = [](){ + static std::unique_ptr reverseXpMap = []() { auto reverseXpMap = std::make_unique(); for (auto & xpFeature : xpFeatureDetails) (*reverseXpMap)[xpFeature.name] = xpFeature.tag; @@ -223,7 +223,8 @@ std::string_view showExperimentalFeature(const ExperimentalFeature tag) return xpFeatureDetails[(size_t)tag].name; } -nlohmann::json documentExperimentalFeatures() { +nlohmann::json documentExperimentalFeatures() +{ StringMap res; for (auto & xpFeature : xpFeatureDetails) res[std::string { xpFeature.name }] = From 7f5ca6192d091090bc71ab7bf96dd4acf0f1d376 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 10 Apr 2023 14:12:10 +0100 Subject: [PATCH 59/75] Add script to reproduce issue by inducing heavy load. --- repro-7998.sh | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 repro-7998.sh diff --git a/repro-7998.sh b/repro-7998.sh new file mode 100755 index 000000000..b022d31ab --- /dev/null +++ b/repro-7998.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -eux +LOG_FILE=/tmp/repro-7998.log +rm -f "$LOG_FILE" +for i in {1..8}; do + ( + while true; do + nix-build \ + --argstr uuid $(uuidgen) \ + --arg drvCount $((RANDOM % 256)) \ + -E ' + { uuid ? "00000000-0000-0000-0000-000000000000", drvCount ? 0 }: + with import { }; + let + mkDrv = name: buildInputs: + stdenv.mkDerivation { + inherit name; + inherit buildInputs; + unpackPhase = "date +\"${uuid} %F %T\" >date.txt"; + installPhase = "mkdir -p $out; cp date.txt $out/"; + }; + mkDrvs = n: + let + name = "repro-7998-${toString n}"; + buildInputs = if n == 0 then [ ] else [ (mkDrvs (n - 1)) ]; + in mkDrv name buildInputs; + in mkDrvs drvCount + ' + done 2>&1 | tee -a "$LOG_FILE" + ) & +done +read # Press enter to stop +pkill -KILL -f repro-7998.sh From 7c56e842133afe14812270c34cda3dc0a3da8aa6 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 11 Apr 2023 10:22:07 +0100 Subject: [PATCH 60/75] Warn after a second of being busy instead of immediately. Getting the occasional SQLITE_BUSY is expected when the database is being accessed concurrently. The retry will likely succeed so it is pointless to warn immediately. Instead we track how long each retrySQLite block has been running, and only begin warning after a second has elapsed (and then every 10 seconds subsequently). --- src/libstore/sqlite.cc | 9 ++------- src/libstore/sqlite.hh | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 871f2f3be..c57e58fe0 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -239,14 +239,9 @@ SQLiteTxn::~SQLiteTxn() } } -void handleSQLiteBusy(const SQLiteBusy & e) +void handleSQLiteBusy(const SQLiteBusy & e, bool shouldWarn) { - static std::atomic lastWarned{0}; - - time_t now = time(0); - - if (now > lastWarned + 10) { - lastWarned = now; + if (shouldWarn) { logWarning({ .msg = hintfmt(e.what()) }); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index b735838ec..e2c9e28f8 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -139,7 +139,7 @@ protected: MakeError(SQLiteBusy, SQLiteError); -void handleSQLiteBusy(const SQLiteBusy & e); +void handleSQLiteBusy(const SQLiteBusy & e, bool shouldWarn); /** * Convenience function for retrying a SQLite transaction when the @@ -148,11 +148,22 @@ void handleSQLiteBusy(const SQLiteBusy & e); template T retrySQLite(F && fun) { + time_t nextWarning = time(0) + 1; + while (true) { try { return fun(); + } catch (SQLiteBusy & e) { - handleSQLiteBusy(e); + time_t now = time(0); + bool shouldWarn = false; + + if (now > nextWarning) { + nextWarning = now + 10; + shouldWarn = true; + } + + handleSQLiteBusy(e, shouldWarn); } } } From da322ebda62470acbf5373374c4cee8236705c2f Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 11 Apr 2023 10:47:53 +0100 Subject: [PATCH 61/75] Revert "Add script to reproduce issue by inducing heavy load." This reverts commit 213b838f9cfb820d2bc76d7c6edc468b27029945. --- repro-7998.sh | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100755 repro-7998.sh diff --git a/repro-7998.sh b/repro-7998.sh deleted file mode 100755 index b022d31ab..000000000 --- a/repro-7998.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -set -eux -LOG_FILE=/tmp/repro-7998.log -rm -f "$LOG_FILE" -for i in {1..8}; do - ( - while true; do - nix-build \ - --argstr uuid $(uuidgen) \ - --arg drvCount $((RANDOM % 256)) \ - -E ' - { uuid ? "00000000-0000-0000-0000-000000000000", drvCount ? 0 }: - with import { }; - let - mkDrv = name: buildInputs: - stdenv.mkDerivation { - inherit name; - inherit buildInputs; - unpackPhase = "date +\"${uuid} %F %T\" >date.txt"; - installPhase = "mkdir -p $out; cp date.txt $out/"; - }; - mkDrvs = n: - let - name = "repro-7998-${toString n}"; - buildInputs = if n == 0 then [ ] else [ (mkDrvs (n - 1)) ]; - in mkDrv name buildInputs; - in mkDrvs drvCount - ' - done 2>&1 | tee -a "$LOG_FILE" - ) & -done -read # Press enter to stop -pkill -KILL -f repro-7998.sh From e570a91661f0742f07dd148e4e75e943d7806f2f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 11 Apr 2023 12:40:14 +0200 Subject: [PATCH 62/75] Release notes --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/release-notes/rl-2.15.md | 58 +++++++++++++++++++++++++ doc/manual/src/release-notes/rl-next.md | 56 ------------------------ 3 files changed, 59 insertions(+), 56 deletions(-) create mode 100644 doc/manual/src/release-notes/rl-2.15.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 5bf274550..f783d5908 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -99,6 +99,7 @@ - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md) - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md) - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md new file mode 100644 index 000000000..133121999 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -0,0 +1,58 @@ +# Release 2.15 (2023-04-11) + +* Commands which take installables on the command line can now read them from the standard input if + passed the `--stdin` flag. This is primarily useful when you have a large amount of paths which + exceed the OS argument limit. + +* The `nix-hash` command now supports Base64 and SRI. Use the flags `--base64` + or `--sri` to specify the format of output hash as Base64 or SRI, and `--to-base64` + or `--to-sri` to convert a hash to Base64 or SRI format, respectively. + + As the choice of hash formats is no longer binary, the `--base16` flag is also added + to explicitly specify the Base16 format, which is still the default. + +* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. + + The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. + Using this is better and more clear than relying on the now-removed `.drv` special handling. + + For example, + ```shell-session + $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv + ``` + + now gives info about the derivation itself, while + + ```shell-session + $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^* + ``` + provides information about each of its outputs. + +* The experimental command `nix describe-stores` has been removed. + +* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md). + +* Documentation for operations of `nix-store` and `nix-env` are now available on separate pages of the manual. + They include all common options that can be specified and common environment variables that affect these commands. + + These pages can be viewed offline with `man` using + + * `man nix-store-` and `man nix-env-` + * `nix-store --help --` and `nix-env --help --`. + +* Nix when used as a client now checks whether the store (the server) trusts the client. + (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.) + This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores. + + `nix store ping` and `nix doctor` now display this information. + +* The new command `nix derivation add` allows adding derivations to the store without involving the Nix language. + It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store. + It uses the same JSON layout as `nix derivation show`, and is its inverse. + +* `nix show-derivation` has been renamed to `nix derivation show`. + This matches `nix derivation add`, and avoids bloating the top-level namespace. + The old name is still kept as an alias for compatibility, however. + +* The `nix derivation {add,show}` JSON format now includes the derivation name as a top-level field. + This is useful in general, but especially necessary for the `add` direction, as otherwise we would need to pass in the name out of band for certain cases. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 5b62836bf..78ae99f4b 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,58 +1,2 @@ # Release X.Y (202?-??-??) -* Commands which take installables on the command line can now read them from the standard input if - passed the `--stdin` flag. This is primarily useful when you have a large amount of paths which - exceed the OS arg limit. - -* The `nix-hash` command now supports Base64 and SRI. Use the flags `--base64` - or `--sri` to specify the format of output hash as Base64 or SRI, and `--to-base64` - or `--to-sri` to convert a hash to Base64 or SRI format, respectively. - - As the choice of hash formats is no longer binary, the `--base16` flag is also added - to explicitly specify the Base16 format, which is still the default. - -* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. - - The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. - Using this is better and more clear than relying on the now-removed `.drv` special handling. - - For example, - ```shell-session - $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv - ``` - - now gives info about the derivation itself, while - - ```shell-session - $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^* - ``` - provides information about each of its outputs. - -* The experimental command `nix describe-stores` has been removed. - -* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md). - -* Documentation for operations of `nix-store` and `nix-env` are now available on separate pages of the manual. - They include all common options that can be specified and common environment variables that affect these commands. - - These pages can be viewed offline with `man` using - - * `man nix-store-` and `man nix-env-` - * `nix-store --help --` and `nix-env --help --`. - -* Nix when used as a client now checks whether the store (the server) trusts the client. - (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.) - This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores. - - `nix store ping` and `nix doctor` now display this information. - -* A new command `nix derivation add` is created, to allow adding derivations to the store without involving the Nix language. - It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store. - It uses the same JSON layout as `nix show-derivation`, and is its inverse. - -* `nix show-derivation` has been renamed to `nix derivation show`. - This matches `nix derivation add`, and avoids bloating the top-level namespace. - The old name is still kept as an alias for compatibility, however. - -* The `nix derivation {add,show}` JSON format now includes the derivation name as a top-level field. - This is useful in general, but especially necessary for the `add` direction, as otherwise we would need to pass in the name out of band for certain cases. From 450e5ec6185e2e1102e67ec7a348a0dc8955692d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 11 Apr 2023 10:46:38 -0400 Subject: [PATCH 63/75] Do not gate or hide experimental settings This is somewhat hacky fix just for 2.15. I unintentionally hid them from the manual, when no one wanted to hide them that (including myself). I also required the experimental feature to be enabled in an order-dependent way, which is not good. The simplest fix for this immanent release is just to always show them, and always allow them to be set. Effectively undoes some changes from aa663b7e89d3d02248d37ee9f68b52770b247018 --- src/libutil/config.cc | 16 +++---------- tests/experimental-features.sh | 42 ++++++++++++++++++---------------- 2 files changed, 25 insertions(+), 33 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8d63536d6..5ff8d91ba 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -70,17 +70,10 @@ void AbstractConfig::reapplyUnknownSettings() set(s.first, s.second); } -// Whether we should process the option. Excludes aliases, which are handled elsewhere, and disabled features. -static bool applicable(const Config::SettingData & sd) -{ - return !sd.isAlias - && experimentalFeatureSettings.isEnabled(sd.setting->experimentalFeature); -} - void Config::getSettings(std::map & res, bool overriddenOnly) { for (auto & opt : _settings) - if (applicable(opt.second) && (!overriddenOnly || opt.second.setting->overridden)) + if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden)) res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } @@ -154,7 +147,7 @@ nlohmann::json Config::toJSON() { auto res = nlohmann::json::object(); for (auto & s : _settings) - if (applicable(s.second)) + if (!s.second.isAlias) res.emplace(s.first, s.second.setting->toJSON()); return res; } @@ -163,7 +156,7 @@ std::string Config::toKeyValue() { auto res = std::string(); for (auto & s : _settings) - if (applicable(s.second)) + if (s.second.isAlias) res += fmt("%s = %s\n", s.first, s.second.setting->to_string()); return res; } @@ -171,9 +164,6 @@ std::string Config::toKeyValue() void Config::convertToArgs(Args & args, const std::string & category) { for (auto & s : _settings) { - /* We do include args for settings gated on disabled - experimental-features. The args themselves however will also be - gated on any experimental feature the underlying setting is. */ if (!s.second.isAlias) s.second.setting->convertToArg(args, category); } diff --git a/tests/experimental-features.sh b/tests/experimental-features.sh index a4d55f5f4..73554da8c 100644 --- a/tests/experimental-features.sh +++ b/tests/experimental-features.sh @@ -1,25 +1,27 @@ source common.sh -# Without flakes, flake options should not show up -# With flakes, flake options should show up - -function both_ways { - nix --experimental-features 'nix-command' "$@" | grepQuietInverse flake - nix --experimental-features 'nix-command flakes' "$@" | grepQuiet flake - - # Also, the order should not matter - nix "$@" --experimental-features 'nix-command' | grepQuietInverse flake - nix "$@" --experimental-features 'nix-command flakes' | grepQuiet flake -} - -# Simple case, the configuration effects the running command -both_ways show-config - -# Skipping for now, because we actually *do* want these to show up in -# the manual, just be marked experimental. Will reenable once the manual -# generation takes advantage of the JSON metadata on this. - -# both_ways store gc --help +# Skipping these two for now, because we actually *do* want flags and +# config settings to always show up in the manual, just be marked +# experimental. Will reenable once the manual generation takes advantage +# of the JSON metadata on this. +# +# # Without flakes, flake options should not show up +# # With flakes, flake options should show up +# +# function grep_both_ways { +# nix --experimental-features 'nix-command' "$@" | grepQuietInverse flake +# nix --experimental-features 'nix-command flakes' "$@" | grepQuiet flake +# +# # Also, the order should not matter +# nix "$@" --experimental-features 'nix-command' | grepQuietInverse flake +# nix "$@" --experimental-features 'nix-command flakes' | grepQuiet flake +# } +# +# # Simple case, the configuration effects the running command +# grep_both_ways show-config +# +# # Medium case, the configuration effects --help +# grep_both_ways store gc --help expect 1 nix --experimental-features 'nix-command' show-config --flake-registry 'https://no' nix --experimental-features 'nix-command flakes' show-config --flake-registry 'https://no' From de3df3009bf003f327d35e246d5904d93273e2e9 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 11 Apr 2023 16:03:37 +0100 Subject: [PATCH 64/75] Move warning timing logic into handleSQLiteBusy. --- src/libstore/sqlite.cc | 6 ++++-- src/libstore/sqlite.hh | 13 ++----------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index c57e58fe0..df334c23c 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -239,9 +239,11 @@ SQLiteTxn::~SQLiteTxn() } } -void handleSQLiteBusy(const SQLiteBusy & e, bool shouldWarn) +void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning) { - if (shouldWarn) { + time_t now = time(0); + if (now > nextWarning) { + nextWarning = now + 10; logWarning({ .msg = hintfmt(e.what()) }); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index e2c9e28f8..6e14852cb 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -139,7 +139,7 @@ protected: MakeError(SQLiteBusy, SQLiteError); -void handleSQLiteBusy(const SQLiteBusy & e, bool shouldWarn); +void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning); /** * Convenience function for retrying a SQLite transaction when the @@ -153,17 +153,8 @@ T retrySQLite(F && fun) while (true) { try { return fun(); - } catch (SQLiteBusy & e) { - time_t now = time(0); - bool shouldWarn = false; - - if (now > nextWarning) { - nextWarning = now + 10; - shouldWarn = true; - } - - handleSQLiteBusy(e, shouldWarn); + handleSQLiteBusy(e, nextWarning); } } } From a4b6d1d9a3a3234258790dd58c7041ee8fe01929 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 11 Apr 2023 20:16:37 +0200 Subject: [PATCH 65/75] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index c910885a0..752490696 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.15.0 \ No newline at end of file +2.16.0 From ee97f107e8749d61ae5e310a2d457637ccc109ce Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 13 Apr 2023 13:38:12 -0400 Subject: [PATCH 66/75] Push `getFSAccessor` `unsupported(...)` down `Store` class hierarchy More progress on issue #5729. Instead of having it by the default method in `Store` itself, have it be the implementation in `DummyStore` and `LegacySSHStore`. Then just the implementations which fail to provide the method pay the "penalty" of dealing with the icky `unimplemented` function for non-compliance. Combined with my other recent PRs, this finally makes `Store` have no `unsupported` calls! --- src/libstore/dummy-store.cc | 3 +++ src/libstore/legacy-ssh-store.cc | 3 +++ src/libstore/store-api.hh | 3 +-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index ae2777d0c..74d6ed3b5 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -71,6 +71,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept override { callback(nullptr); } + + virtual ref getFSAccessor() override + { unsupported("getFSAccessor"); } }; static RegisterStoreImplementation regDummyStore; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index eb471d8fc..d2ddbbe5f 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -342,6 +342,9 @@ public: void ensurePath(const StorePath & path) override { unsupported("ensurePath"); } + virtual ref getFSAccessor() override + { unsupported("getFSAccessor"); } + void computeFSClosure(const StorePathSet & paths, StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false) override diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 3cb48bff5..5bee272bf 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -678,8 +678,7 @@ public: /** * @return An object to access files in the Nix store. */ - virtual ref getFSAccessor() - { unsupported("getFSAccessor"); } + virtual ref getFSAccessor() = 0; /** * Repair the contents of the given path by redownloading it using From 9e8f2090365a509656dead69bc91fb6615cf9d05 Mon Sep 17 00:00:00 2001 From: Raphael Robatsch Date: Thu, 13 Apr 2023 16:22:45 +0200 Subject: [PATCH 67/75] Display valid installable in InstallableDerivedPath::parse warning The warning message should produce an installable name that can be passed to `nix build`, `nix path-info`, etc. again. Since the CLI expects that the .drv path and the output names are separated by a caret, the warning message must also separate the .drv path and output names with a caret. However, `DerivedPath::Built.to_string()` uses an exclamation point as the separator instead. This commit adds a `separator` argument to the to_string method. This changes the warning message from: If this command is now failing try again with '/nix/store/foo.drv!*' to: If this command is now failing try again with '/nix/store/foo.drv^*' --- src/libcmd/installable-derived-path.cc | 2 +- src/libstore/derived-path.cc | 13 +++++++------ src/libstore/derived-path.hh | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc index 6ecf54b7c..35f6c5bfe 100644 --- a/src/libcmd/installable-derived-path.cc +++ b/src/libcmd/installable-derived-path.cc @@ -47,7 +47,7 @@ InstallableDerivedPath InstallableDerivedPath::parse( }; warn( "The interpretation of store paths arguments ending in `.drv` recently changed. If this command is now failing try again with '%s'", - oldDerivedPath.to_string(*store)); + oldDerivedPath.to_string(*store, '^')); }; return DerivedPath::Opaque { .path = std::move(storePath), diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index e5f0f1b33..6baca70a3 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -59,18 +59,19 @@ std::string DerivedPath::Opaque::to_string(const Store & store) const return store.printStorePath(path); } -std::string DerivedPath::Built::to_string(const Store & store) const +std::string DerivedPath::Built::to_string(const Store & store, char separator) const { return store.printStorePath(drvPath) - + "!" + + separator + outputs.to_string(); } -std::string DerivedPath::to_string(const Store & store) const +std::string DerivedPath::to_string(const Store & store, char separator) const { - return std::visit( - [&](const auto & req) { return req.to_string(store); }, - this->raw()); + return std::visit(overloaded { + [&](const DerivedPath::Built & req) { return req.to_string(store, separator); }, + [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + }, this->raw()); } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 2155776b1..9fc64e2f2 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -48,7 +48,7 @@ struct DerivedPathBuilt { StorePath drvPath; OutputsSpec outputs; - std::string to_string(const Store & store) const; + std::string to_string(const Store & store, char separator = '!') const; static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view); nlohmann::json toJSON(ref store) const; @@ -81,7 +81,7 @@ struct DerivedPath : _DerivedPathRaw { return static_cast(*this); } - std::string to_string(const Store & store) const; + std::string to_string(const Store & store, char separator = '!') const; static DerivedPath parse(const Store & store, std::string_view); }; From d93e76fbb80c9f407ddac128357350f54add68fb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 6 Apr 2023 11:09:01 -0400 Subject: [PATCH 68/75] Start cross-referencing experimental features - Create a glossary entry for experimental features. - Have the man page experimental feature notice link `nix-commmand`. (Eventually this should be programmed, based on whether the command is experimental, and if so what experimental feature does it depend on.) - Document which installables depend on which experimental features. I tried to use the same style (bold warning and block quote) that the top of the man page uses. Co-authored-by: Valentin Gagarin --- doc/manual/generate-manpage.nix | 4 ++- .../src/command-ref/experimental-commands.md | 2 +- doc/manual/src/glossary.md | 6 +++++ .../src/language/advanced-attributes.md | 26 +++++++++++++++---- src/nix/nix.md | 14 +++++++++- 5 files changed, 44 insertions(+), 8 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 86f2ca567..0b3a23801 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -10,7 +10,9 @@ let result = '' > **Warning** \ - > This program is **experimental** and its interface is subject to change. + > This program is + > [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) + > and its interface is subject to change. # Name diff --git a/doc/manual/src/command-ref/experimental-commands.md b/doc/manual/src/command-ref/experimental-commands.md index cfa6f8b73..286ddc6d6 100644 --- a/doc/manual/src/command-ref/experimental-commands.md +++ b/doc/manual/src/command-ref/experimental-commands.md @@ -1,6 +1,6 @@ # Experimental Commands -This section lists experimental commands. +This section lists [experimental commands](@docroot@/contributing/experimental-features.md#xp-feature-nix-command). > **Warning** > diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 4eedb2e93..a9782be5c 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -225,3 +225,9 @@ [string]: ./language/values.md#type-string [path]: ./language/values.md#type-path [attribute name]: ./language/values.md#attribute-set + + - [experimental feature]{#gloss-experimental-feature}\ + Not yet stabilized functionality guarded by named experimental feature flags. + These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. + + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 3e8c48890..307971434 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -208,12 +208,26 @@ Derivations can declare some infrequently used optional attributes. about converting to and from base-32 notation.) - [`__contentAddressed`]{#adv-attr-__contentAddressed} - If this **experimental** attribute is set to true, then the derivation + > **Warning** + > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > + > To use this attribute, you must enable the + > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature. + > For example, in [nix.conf](../command-ref/conf-file.md) you could add: + > + > ``` + > extra-experimental-features = ca-derivations + > ``` + + If this attribute is set to `true`, then the derivation outputs will be stored in a content-addressed location rather than the traditional input-addressed one. - This only has an effect if the `ca-derivations` experimental feature is enabled. - Setting this attribute also requires setting `outputHashMode` and `outputHashAlgo` like for *fixed-output derivations* (see above). + Setting this attribute also requires setting + [`outputHashMode`](#adv-attr-outputHashMode) + and + [`outputHashAlgo`](#adv-attr-outputHashAlgo) + like for *fixed-output derivations* (see above). - [`passAsFile`]{#adv-attr-passAsFile}\ A list of names of attributes that should be passed via files rather @@ -307,9 +321,11 @@ Derivations can declare some infrequently used optional attributes. - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\ > **Warning** - > This is an experimental feature. + > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). > - > To enable it, add the following to [nix.conf](../command-ref/conf-file.md): + > To use this attribute, you must enable the + > [`discard-references`](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) experimental feature. + > For example, in [nix.conf](../command-ref/conf-file.md) you could add: > > ``` > extra-experimental-features = discard-references diff --git a/src/nix/nix.md b/src/nix/nix.md index e1865b31c..1ef6c7fcd 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -48,12 +48,17 @@ manual](https://nixos.org/manual/nix/stable/). # Installables +> **Warning** \ +> Installables are part of the unstable +> [`nix-command` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-nix-command), +> and subject to change without notice. + Many `nix` subcommands operate on one or more *installables*. These are command line arguments that represent something that can be realised in the Nix store. The following types of installable are supported by most commands: -- [Flake output attribute](#flake-output-attribute) +- [Flake output attribute](#flake-output-attribute) (experimental) - [Store path](#store-path) - [Nix file](#nix-file), optionally qualified by an attribute path - [Nix expression](#nix-expression), optionally qualified by an attribute path @@ -63,6 +68,13 @@ That is, Nix will operate on the default flake output attribute of the flake in ### Flake output attribute +> **Warning** \ +> Flake output attribute installables depend on both the +> [`flakes`](@docroot@/contributing/experimental-features.md#xp-feature-flakes) +> and +> [`nix-command`](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> experimental features, and subject to change without notice. + Example: `nixpkgs#hello` These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a From ee420ac64e7d1f51f5abcb069dbe84cd6ff707ce Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 14 Apr 2023 20:45:11 -0400 Subject: [PATCH 69/75] Legacy vs non-legacy `to_string`/`parse` for `DerivedPath` As requested by @roberth, it is good to call out the specific instances we care about, which is `!` for the RPC protocols, and `^` for humans. This doesn't take advantage of parametricity as much, but since the human and computer interfaces are good to decouple anyways (we don't care if they drift further apart over time in the slightest) some separation and slight duplication is fine. Also, unit test both round trips. --- src/libcmd/installable-derived-path.cc | 2 +- src/libstore/derived-path.cc | 37 +++++++++++++++++++++----- src/libstore/derived-path.hh | 30 ++++++++++++++++++--- src/libstore/remote-store.cc | 4 +-- src/libstore/tests/derived-path.cc | 8 ++++++ 5 files changed, 69 insertions(+), 12 deletions(-) diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc index 35f6c5bfe..6ecf54b7c 100644 --- a/src/libcmd/installable-derived-path.cc +++ b/src/libcmd/installable-derived-path.cc @@ -47,7 +47,7 @@ InstallableDerivedPath InstallableDerivedPath::parse( }; warn( "The interpretation of store paths arguments ending in `.drv` recently changed. If this command is now failing try again with '%s'", - oldDerivedPath.to_string(*store, '^')); + oldDerivedPath.to_string(*store)); }; return DerivedPath::Opaque { .path = std::move(storePath), diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 6baca70a3..9a2ffda39 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -59,17 +59,32 @@ std::string DerivedPath::Opaque::to_string(const Store & store) const return store.printStorePath(path); } -std::string DerivedPath::Built::to_string(const Store & store, char separator) const +std::string DerivedPath::Built::to_string(const Store & store) const { return store.printStorePath(drvPath) - + separator + + '^' + outputs.to_string(); } -std::string DerivedPath::to_string(const Store & store, char separator) const +std::string DerivedPath::Built::to_string_legacy(const Store & store) const +{ + return store.printStorePath(drvPath) + + '!' + + outputs.to_string(); +} + +std::string DerivedPath::to_string(const Store & store) const { return std::visit(overloaded { - [&](const DerivedPath::Built & req) { return req.to_string(store, separator); }, + [&](const DerivedPath::Built & req) { return req.to_string(store); }, + [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + }, this->raw()); +} + +std::string DerivedPath::to_string_legacy(const Store & store) const +{ + return std::visit(overloaded { + [&](const DerivedPath::Built & req) { return req.to_string_legacy(store); }, [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, }, this->raw()); } @@ -88,14 +103,24 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi }; } -DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator) { - size_t n = s.find("!"); + size_t n = s.find(separator); return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1)); } +DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +{ + return parseWith(store, s, "^"); +} + +DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s) +{ + return parseWith(store, s, "!"); +} + RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const { RealisedPath::Set res; diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 9fc64e2f2..5f7acbebc 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -48,8 +48,18 @@ struct DerivedPathBuilt { StorePath drvPath; OutputsSpec outputs; - std::string to_string(const Store & store, char separator = '!') const; - static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view); + /** + * Uses `^` as the separator + */ + std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * The caller splits on the separator, so it works for both variants. + */ + static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs); nlohmann::json toJSON(ref store) const; GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs); @@ -81,8 +91,22 @@ struct DerivedPath : _DerivedPathRaw { return static_cast(*this); } - std::string to_string(const Store & store, char separator = '!') const; + /** + * Uses `^` as the separator + */ + std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * Uses `^` as the separator + */ static DerivedPath parse(const Store & store, std::string_view); + /** + * Uses `!` as the separator + */ + static DerivedPath parseLegacy(const Store & store, std::string_view); }; /** diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index e128c3a29..b862902d1 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -90,12 +90,12 @@ void write(const Store & store, Sink & out, const ContentAddress & ca) DerivedPath read(const Store & store, Source & from, Phantom _) { auto s = readString(from); - return DerivedPath::parse(store, s); + return DerivedPath::parseLegacy(store, s); } void write(const Store & store, Sink & out, const DerivedPath & req) { - out << req.to_string(store); + out << req.to_string_legacy(store); } diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc index d1ac2c5e7..e6d32dbd0 100644 --- a/src/libstore/tests/derived-path.cc +++ b/src/libstore/tests/derived-path.cc @@ -51,6 +51,14 @@ TEST_F(DerivedPathTest, force_init) { } +RC_GTEST_FIXTURE_PROP( + DerivedPathTest, + prop_legacy_round_rip, + (const DerivedPath & o)) +{ + RC_ASSERT(o == DerivedPath::parseLegacy(*store, o.to_string_legacy(*store))); +} + RC_GTEST_FIXTURE_PROP( DerivedPathTest, prop_round_rip, From 9c74df5bb4f41c938a4f6942492f5ce92b0ef371 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 15 Apr 2023 20:56:51 +0200 Subject: [PATCH 70/75] Format Co-authored-by: Eelco Dolstra Co-authored-by: John Ericson --- src/libexpr/value/print.cc | 6 ++++-- src/libstore/derivations.cc | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libexpr/value/print.cc b/src/libexpr/value/print.cc index cf97def46..c67ff9f87 100644 --- a/src/libexpr/value/print.cc +++ b/src/libexpr/value/print.cc @@ -3,7 +3,8 @@ namespace nix { std::ostream & -printLiteral(std::ostream & str, const std::string_view string) { +printLiteral(std::ostream & str, const std::string_view string) +{ str << "\""; for (auto i = string.begin(); i != string.end(); ++i) { if (*i == '\"' || *i == '\\') str << "\\" << *i; @@ -18,7 +19,8 @@ printLiteral(std::ostream & str, const std::string_view string) { } std::ostream & -printLiteral(std::ostream & str, bool boolean) { +printLiteral(std::ostream & str, bool boolean) +{ str << (boolean ? "true" : "false"); return str; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 9948862e5..7eb5cd275 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -314,7 +314,7 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi /** - * Print a derivation string literal to an std::string. + * Print a derivation string literal to an `std::string`. * * This syntax does not generalize to the expression language, which needs to * escape `$`. From 1e2dd669bcdd8df6cdaac061e035828626906447 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 16 Apr 2023 12:56:31 +0200 Subject: [PATCH 71/75] printLiteral: Do not overload --- src/libcmd/repl.cc | 6 +++--- src/libexpr/eval.cc | 4 ++-- src/libexpr/nixexpr.cc | 6 +++--- src/libexpr/value/print.cc | 4 ++-- src/libexpr/value/print.hh | 12 ++++++------ 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 1366622c7..41cf77424 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -913,13 +913,13 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m case nBool: str << ANSI_CYAN; - printLiteral(str, v.boolean); + printLiteralBool(str, v.boolean); str << ANSI_NORMAL; break; case nString: str << ANSI_WARNING; - printLiteral(str, v.string.s); + printLiteralString(str, v.string.s); str << ANSI_NORMAL; break; @@ -959,7 +959,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m if (isVarName(i.first)) str << i.first; else - printLiteral(str, i.first); + printLiteralString(str, i.first); str << " = "; if (seen.count(i.second)) str << "«repeated»"; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 06208897f..bd05fc156 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -105,10 +105,10 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, str << integer; break; case tBool: - printLiteral(str, boolean); + printLiteralBool(str, boolean); break; case tString: - printLiteral(str, string.s); + printLiteralString(str, string.s); break; case tPath: str << path; // !!! escaping? diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index ca6df0af3..1b5d522d3 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -74,7 +74,7 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) else { char c = s[0]; if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { - printLiteral(str, s); + printLiteralString(str, s); return str; } for (auto c : s) @@ -82,7 +82,7 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '\'' || c == '-')) { - printLiteral(str, s); + printLiteralString(str, s); return str; } str << s; @@ -107,7 +107,7 @@ void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const void ExprString::show(const SymbolTable & symbols, std::ostream & str) const { - printLiteral(str, s); + printLiteralString(str, s); } void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const diff --git a/src/libexpr/value/print.cc b/src/libexpr/value/print.cc index c67ff9f87..bd241d9d8 100644 --- a/src/libexpr/value/print.cc +++ b/src/libexpr/value/print.cc @@ -3,7 +3,7 @@ namespace nix { std::ostream & -printLiteral(std::ostream & str, const std::string_view string) +printLiteralString(std::ostream & str, const std::string_view string) { str << "\""; for (auto i = string.begin(); i != string.end(); ++i) { @@ -19,7 +19,7 @@ printLiteral(std::ostream & str, const std::string_view string) } std::ostream & -printLiteral(std::ostream & str, bool boolean) +printLiteralBool(std::ostream & str, bool boolean) { str << (boolean ? "true" : "false"); return str; diff --git a/src/libexpr/value/print.hh b/src/libexpr/value/print.hh index 31c94eb85..98dd2008d 100644 --- a/src/libexpr/value/print.hh +++ b/src/libexpr/value/print.hh @@ -17,14 +17,14 @@ namespace nix { * * @param s The logical string */ - std::ostream & printLiteral(std::ostream & o, std::string_view s); - inline std::ostream & printLiteral(std::ostream & o, const char * s) { - return printLiteral(o, std::string_view(s)); + std::ostream & printLiteralString(std::ostream & o, std::string_view s); + inline std::ostream & printLiteralString(std::ostream & o, const char * s) { + return printLiteralString(o, std::string_view(s)); } - inline std::ostream & printLiteral(std::ostream & o, const std::string & s) { - return printLiteral(o, std::string_view(s)); + inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) { + return printLiteralString(o, std::string_view(s)); } /** Print `true` or `false`. */ - std::ostream & printLiteral(std::ostream & o, bool b); + std::ostream & printLiteralBool(std::ostream & o, bool b); } From 28a5cdde02964306e7eb443f696c8d5d59ebf9e9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 16 Apr 2023 13:10:45 +0200 Subject: [PATCH 72/75] libexpr/value/print.* -> libexpr/print.* Generalizes the file to sensibly allow printing any part of the language syntax. --- src/libcmd/repl.cc | 2 +- src/libexpr/eval.cc | 2 +- src/libexpr/nixexpr.cc | 2 +- src/libexpr/{value => }/print.cc | 2 +- src/libexpr/{value => }/print.hh | 0 5 files changed, 4 insertions(+), 4 deletions(-) rename src/libexpr/{value => }/print.cc (96%) rename src/libexpr/{value => }/print.hh (100%) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 41cf77424..806dce024 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -40,7 +40,7 @@ extern "C" { #include "markdown.hh" #include "local-fs-store.hh" #include "progress-bar.hh" -#include "value/print.hh" +#include "print.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index bd05fc156..6668add8c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -9,7 +9,7 @@ #include "filetransfer.hh" #include "function-trace.hh" #include "profiles.hh" -#include "value/print.hh" +#include "print.hh" #include #include diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 1b5d522d3..d8f3cd701 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -3,7 +3,7 @@ #include "eval.hh" #include "symbol-table.hh" #include "util.hh" -#include "value/print.hh" +#include "print.hh" #include diff --git a/src/libexpr/value/print.cc b/src/libexpr/print.cc similarity index 96% rename from src/libexpr/value/print.cc rename to src/libexpr/print.cc index bd241d9d8..282903b72 100644 --- a/src/libexpr/value/print.cc +++ b/src/libexpr/print.cc @@ -1,4 +1,4 @@ -#include "value/print.hh" +#include "print.hh" namespace nix { diff --git a/src/libexpr/value/print.hh b/src/libexpr/print.hh similarity index 100% rename from src/libexpr/value/print.hh rename to src/libexpr/print.hh From b6125772d7d5f82d48873fc93a7f261832154b14 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 16 Apr 2023 14:07:35 +0200 Subject: [PATCH 73/75] libexpr: Move identifier-like printing to print.cc --- src/libcmd/repl.cc | 6 ++--- src/libexpr/nixexpr.cc | 27 +++-------------------- src/libexpr/print.cc | 50 ++++++++++++++++++++++++++++++++++++++++++ src/libexpr/print.hh | 18 +++++++++++++++ 4 files changed, 73 insertions(+), 28 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 806dce024..80c08bf1c 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -426,6 +426,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix) } +// FIXME: DRY and match or use the parser static bool isVarName(std::string_view s) { if (s.size() == 0) return false; @@ -956,10 +957,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m sorted.emplace(state->symbols[i.name], i.value); for (auto & i : sorted) { - if (isVarName(i.first)) - str << i.first; - else - printLiteralString(str, i.first); + printAttributeName(str, i.first); str << " = "; if (seen.count(i.second)) str << "«repeated»"; diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index d8f3cd701..1557cbbeb 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -61,33 +61,12 @@ Pos::operator std::shared_ptr() const return pos; } -/* Displaying abstract syntax trees. */ - +// FIXME: remove, because *symbols* are abstract and do not have a single +// textual representation; see printIdentifier() std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) { std::string_view s = symbol; - - if (s.empty()) - str << "\"\""; - else if (s == "if") // FIXME: handle other keywords - str << '"' << s << '"'; - else { - char c = s[0]; - if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { - printLiteralString(str, s); - return str; - } - for (auto c : s) - if (!((c >= 'a' && c <= 'z') || - (c >= 'A' && c <= 'Z') || - (c >= '0' && c <= '9') || - c == '_' || c == '\'' || c == '-')) { - printLiteralString(str, s); - return str; - } - str << s; - } - return str; + return printIdentifier(str, s); } void Expr::show(const SymbolTable & symbols, std::ostream & str) const diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index 282903b72..d08672cfc 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -25,4 +25,54 @@ printLiteralBool(std::ostream & str, bool boolean) return str; } +std::ostream & +printIdentifier(std::ostream & str, std::string_view s) { + if (s.empty()) + str << "\"\""; + else if (s == "if") // FIXME: handle other keywords + str << '"' << s << '"'; + else { + char c = s[0]; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { + printLiteralString(str, s); + return str; + } + for (auto c : s) + if (!((c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + c == '_' || c == '\'' || c == '-')) { + printLiteralString(str, s); + return str; + } + str << s; + } + return str; +} + +// FIXME: keywords +static bool isVarName(std::string_view s) +{ + if (s.size() == 0) return false; + char c = s[0]; + if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false; + for (auto & i : s) + if (!((i >= 'a' && i <= 'z') || + (i >= 'A' && i <= 'Z') || + (i >= '0' && i <= '9') || + i == '_' || i == '-' || i == '\'')) + return false; + return true; +} + +std::ostream & +printAttributeName(std::ostream & str, std::string_view name) { + if (isVarName(name)) + str << name; + else + printLiteralString(str, name); + return str; +} + + } diff --git a/src/libexpr/print.hh b/src/libexpr/print.hh index 98dd2008d..f9cfc3964 100644 --- a/src/libexpr/print.hh +++ b/src/libexpr/print.hh @@ -27,4 +27,22 @@ namespace nix { /** Print `true` or `false`. */ std::ostream & printLiteralBool(std::ostream & o, bool b); + + /** + * Print a string as an attribute name in the Nix expression language syntax. + * + * Prints a quoted string if necessary. + */ + std::ostream & printAttributeName(std::ostream & o, std::string_view s); + + /** + * Print a string as an identifier in the Nix expression language syntax. + * + * FIXME: "identifier" is ambiguous. Identifiers do not have a single + * textual representation. They can be used in variable references, + * let bindings, left-hand sides or attribute names in a select + * expression, or something else entirely, like JSON. Use one of the + * `print*` functions instead. + */ + std::ostream & printIdentifier(std::ostream & o, std::string_view s); } From ba9ae691b6b0d8f6e23a2a9468aef51a75f16cfe Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 16 Apr 2023 10:44:03 -0400 Subject: [PATCH 74/75] Add `optionalString` to manual Nix lang utilities Use it everywhere it could be also. --- doc/manual/generate-manpage.nix | 28 +++++++++++++++------------- doc/manual/utils.nix | 4 +++- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 0b3a23801..d04eecf55 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -31,19 +31,18 @@ let showSynopsis = command: args: let - showArgument = arg: "*${arg.label}*" + (if arg ? arity then "" else "..."); + showArgument = arg: "*${arg.label}*" + optionalString (! arg ? arity) "..."; arguments = concatStringsSep " " (map showArgument args); in '' `${command}` [*option*...] ${arguments} ''; - maybeSubcommands = if details ? commands && details.commands != {} - then '' + maybeSubcommands = optionalString (details ? commands && details.commands != {}) + '' where *subcommand* is one of the following: ${subcommands} - '' - else ""; + ''; subcommands = if length categories > 1 then listCategories @@ -65,12 +64,11 @@ let * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description} ''; - maybeDocumentation = - if details ? doc - then replaceStrings ["@stores@"] [storeDocs] details.doc - else ""; + maybeDocumentation = optionalString + (details ? doc) + (replaceStrings ["@stores@"] [storeDocs] details.doc); - maybeOptions = if details.flags == {} then "" else '' + maybeOptions = optionalString (details.flags != {}) '' # Options ${showOptions details.flags toplevel.flags} @@ -80,15 +78,19 @@ let let allOptions = options // commonOptions; showCategory = cat: '' - ${if cat != "" then "**${cat}:**" else ""} + ${optionalString (cat != "") "**${cat}:**"} ${listOptions (filterAttrs (n: v: v.category == cat) allOptions)} ''; listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts)); showOption = name: option: let - shortName = if option ? shortName then "/ `-${option.shortName}`" else ""; - labels = if option ? labels then (concatStringsSep " " (map (s: "*${s}*") option.labels)) else ""; + shortName = optionalString + (option ? shortName) + ("/ `-${option.shortName}`"); + labels = optionalString + (option ? labels) + (concatStringsSep " " (map (s: "*${s}*") option.labels)); in trim '' - `--${name}` ${shortName} ${labels} diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index 82544935a..e69cbe658 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -42,6 +42,8 @@ rec { filterAttrs = pred: set: listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set)); + optionalString = cond: string: if cond then string else ""; + showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value }: let result = squash '' @@ -74,7 +76,7 @@ rec { else "*machine-specific*"; showAliases = aliases: - if aliases == [] then "" else + optionalString (aliases != []) "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; in result; From 9800c1e8074d248f75ea9bed1b5a0f76e799863d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 10 Apr 2023 12:02:35 -0400 Subject: [PATCH 75/75] Mark experimental configuration settings programmatically Fix #8162 The test is changed to compare `nlohmann::json` values, not strings of dumped JSON, which allows us to format things more nicely. --- doc/manual/utils.nix | 20 +++++++++++++++- src/libstore/globals.hh | 20 ---------------- src/libutil/config.cc | 4 ++++ src/libutil/tests/config.cc | 48 ++++++++++++++++++++++++++++++++++--- 4 files changed, 68 insertions(+), 24 deletions(-) diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index e69cbe658..9043dd8cd 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -44,7 +44,7 @@ rec { optionalString = cond: string: if cond then string else ""; - showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value }: + showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }: let result = squash '' - ${if useAnchors @@ -54,10 +54,28 @@ rec { ${indent " " body} ''; + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > This setting is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + + To change this setting, you need to make sure the corresponding experimental feature, + [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), + is enabled. + For example, include the following in [`nix.conf`](#): + + ``` + extra-experimental-features = ${experimentalFeature} + ${name} = ... + ``` + ''; + # separate body to cleanly handle indentation body = '' ${description} + ${experimentalFeatureNote} + **Default:** ${showDefault documentDefault defaultValue} ${showAliases aliases} diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 63c7389da..f598ed4a8 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -328,16 +328,6 @@ public: users in `build-users-group`. UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. - - > **Warning** - > This is an experimental feature. - - To enable it, add the following to [`nix.conf`](#): - - ``` - extra-experimental-features = auto-allocate-uids - auto-allocate-uids = true - ``` )"}; Setting startId{this, @@ -367,16 +357,6 @@ public: Cgroups are required and enabled automatically for derivations that require the `uid-range` system feature. - - > **Warning** - > This is an experimental feature. - - To enable it, add the following to [`nix.conf`](#): - - ``` - extra-experimental-features = cgroups - use-cgroups = true - ``` )"}; #endif diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 5ff8d91ba..a42f3a849 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -191,6 +191,10 @@ std::map AbstractSetting::toJSONObject() std::map obj; obj.emplace("description", description); obj.emplace("aliases", aliases); + if (experimentalFeature) + obj.emplace("experimentalFeature", *experimentalFeature); + else + obj.emplace("experimentalFeature", nullptr); return obj; } diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc index 8be6730dd..f250e934e 100644 --- a/src/libutil/tests/config.cc +++ b/src/libutil/tests/config.cc @@ -156,12 +156,54 @@ namespace nix { } TEST(Config, toJSONOnNonEmptyConfig) { + using nlohmann::literals::operator "" _json; Config config; - std::map settings; - Setting setting{&config, "", "name-of-the-setting", "description"}; + Setting setting{ + &config, + "", + "name-of-the-setting", + "description", + }; setting.assign("value"); - ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","documentDefault":true,"value":"value"}})#"); + ASSERT_EQ(config.toJSON(), + R"#({ + "name-of-the-setting": { + "aliases": [], + "defaultValue": "", + "description": "description\n", + "documentDefault": true, + "value": "value", + "experimentalFeature": null + } + })#"_json); + } + + TEST(Config, toJSONOnNonEmptyConfigWithExperimentalSetting) { + using nlohmann::literals::operator "" _json; + Config config; + Setting setting{ + &config, + "", + "name-of-the-setting", + "description", + {}, + true, + Xp::Flakes, + }; + setting.assign("value"); + + ASSERT_EQ(config.toJSON(), + R"#({ + "name-of-the-setting": { + "aliases": [], + "defaultValue": "", + "description": "description\n", + "documentDefault": true, + "value": "value", + "experimentalFeature": "flakes" + } + })#"_json); } TEST(Config, setSettingAlias) {