From 02bff90e7bbc04244b3d566d82c945156cb0885b Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 1 Sep 2022 22:02:38 +0200 Subject: [PATCH 001/179] nix flake show: don't evaluate derivations for foreign systems by default --- src/nix/flake.cc | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3967f1102..5232534c6 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -957,6 +957,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun struct CmdFlakeShow : FlakeCommand, MixJSON { bool showLegacy = false; + bool showForeign = false; CmdFlakeShow() { @@ -965,6 +966,11 @@ struct CmdFlakeShow : FlakeCommand, MixJSON .description = "Show the contents of the `legacyPackages` output.", .handler = {&showLegacy, true} }); + addFlag({ + .longName = "foreign", + .description = "Show the contents of outputs for foreign systems.", + .handler = {&showForeign, true} + }); } std::string description() override @@ -985,6 +991,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto state = getEvalState(); auto flake = std::make_shared(lockFlake()); + auto localSystem = std::string(settings.thisSystem.get()); std::functioncout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--foreign' to show)", headerPrefix)); + else { + logger->warn(fmt("%s omitted (use '--foreign' to show)", concatStringsSep(".", attrPathS))); + } + } else { + if (visitor.isDerivation()) + showDerivation(); + else + throw Error("expected a derivation"); + } } else if (attrPath.size() > 0 && attrPathS[0] == "hydraJobs") { From 6da4cc92d8c546939818b65ba4f1b4ce65d88d6e Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 7 Sep 2022 20:28:30 +0200 Subject: [PATCH 002/179] showForeign -> showAllSystems --- src/nix/flake.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 5232534c6..bb8be872b 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -957,7 +957,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun struct CmdFlakeShow : FlakeCommand, MixJSON { bool showLegacy = false; - bool showForeign = false; + bool showAllSystems = false; CmdFlakeShow() { @@ -967,9 +967,9 @@ struct CmdFlakeShow : FlakeCommand, MixJSON .handler = {&showLegacy, true} }); addFlag({ - .longName = "foreign", - .description = "Show the contents of outputs for foreign systems.", - .handler = {&showForeign, true} + .longName = "all-systems", + .description = "Show the contents of outputs for all systems.", + .handler = {&showAllSystems, true} }); } @@ -1082,11 +1082,11 @@ struct CmdFlakeShow : FlakeCommand, MixJSON || (attrPath.size() == 3 && (attrPathS[0] == "checks" || attrPathS[0] == "packages" || attrPathS[0] == "devShells")) ) { - if (!showForeign && std::string(attrPathS[1]) != localSystem) { + if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { if (!json) - logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--foreign' to show)", headerPrefix)); + logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); else { - logger->warn(fmt("%s omitted (use '--foreign' to show)", concatStringsSep(".", attrPathS))); + logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); } } else { if (visitor.isDerivation()) From d0f2da214badd4cded6fb00e97af4588da49c0d0 Mon Sep 17 00:00:00 2001 From: sternenseemann Date: Sun, 16 Oct 2022 01:32:55 +0200 Subject: [PATCH 003/179] primops: make nature of foldl' strictness clearer * Clarify the documentation of foldl': That the arguments are forced before application (?) of `op` is necessarily true. What is important to stress is that we force every application of `op`, even when the value turns out to be unused. * Move the example before the comment about strictness to make it less confusing: It is a general example and doesn't really showcase anything about foldl' strictness. * Add test cases which nail down aspects of foldl' strictness: * The initial accumulator value is not forced unconditionally. * Applications of op are forced. * The list elements are not forced unconditionally. --- src/libexpr/primops.cc | 6 +++--- .../lang/eval-fail-foldlStrict-strict-op-application.nix | 5 +++++ tests/lang/eval-okay-foldlStrict-lazy-elements.exp | 1 + tests/lang/eval-okay-foldlStrict-lazy-elements.nix | 9 +++++++++ .../eval-okay-foldlStrict-lazy-initial-accumulator.exp | 1 + .../eval-okay-foldlStrict-lazy-initial-accumulator.nix | 6 ++++++ 6 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 tests/lang/eval-fail-foldlStrict-strict-op-application.nix create mode 100644 tests/lang/eval-okay-foldlStrict-lazy-elements.exp create mode 100644 tests/lang/eval-okay-foldlStrict-lazy-elements.nix create mode 100644 tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp create mode 100644 tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 28b998474..ba4b7c67a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2908,9 +2908,9 @@ static RegisterPrimOp primop_foldlStrict({ .doc = R"( Reduce a list by applying a binary operator, from left to right, e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) - ...`. The operator is applied strictly, i.e., its arguments are - evaluated first. For example, `foldl' (x: y: x + y) 0 [1 2 3]` - evaluates to 6. + ...`. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6. + The return value of each application of `op` is evaluated immediately, + even for intermediate values. )", .fun = prim_foldlStrict, }); diff --git a/tests/lang/eval-fail-foldlStrict-strict-op-application.nix b/tests/lang/eval-fail-foldlStrict-strict-op-application.nix new file mode 100644 index 000000000..1620cc76e --- /dev/null +++ b/tests/lang/eval-fail-foldlStrict-strict-op-application.nix @@ -0,0 +1,5 @@ +# Tests that the result of applying op is forced even if the value is never used +builtins.foldl' + (_: f: f null) + null + [ (_: throw "Not the final value, but is still forced!") (_: 23) ] diff --git a/tests/lang/eval-okay-foldlStrict-lazy-elements.exp b/tests/lang/eval-okay-foldlStrict-lazy-elements.exp new file mode 100644 index 000000000..d81cc0710 --- /dev/null +++ b/tests/lang/eval-okay-foldlStrict-lazy-elements.exp @@ -0,0 +1 @@ +42 diff --git a/tests/lang/eval-okay-foldlStrict-lazy-elements.nix b/tests/lang/eval-okay-foldlStrict-lazy-elements.nix new file mode 100644 index 000000000..c666e07f3 --- /dev/null +++ b/tests/lang/eval-okay-foldlStrict-lazy-elements.nix @@ -0,0 +1,9 @@ +# Tests that the rhs argument of op is not forced unconditionally +let + lst = builtins.foldl' + (acc: x: acc ++ [ x ]) + [ ] + [ 42 (throw "this shouldn't be evaluated") ]; +in + +builtins.head lst diff --git a/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp b/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp new file mode 100644 index 000000000..d81cc0710 --- /dev/null +++ b/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp @@ -0,0 +1 @@ +42 diff --git a/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix b/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix new file mode 100644 index 000000000..abcd5366a --- /dev/null +++ b/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix @@ -0,0 +1,6 @@ +# Checks that the nul value for the accumulator is not forced unconditionally. +# Some languages provide a foldl' that is strict in this argument, but Nix does not. +builtins.foldl' + (_: x: x) + (throw "This is never forced") + [ "but the results of applying op are" 42 ] From c5fd34a14e2a271b66839b35f500caeae3368be2 Mon Sep 17 00:00:00 2001 From: Graham Bennett Date: Sat, 22 Oct 2022 14:24:25 +0100 Subject: [PATCH 004/179] Build with C++20 --- Makefile | 2 +- perl/Makefile | 2 +- src/libcmd/nix-cmd.pc.in | 2 +- src/libexpr/nix-expr.pc.in | 2 +- src/libmain/nix-main.pc.in | 2 +- src/libstore/nix-store.pc.in | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index c1a1ce2c7..42d11638b 100644 --- a/Makefile +++ b/Makefile @@ -36,4 +36,4 @@ endif include mk/lib.mk -GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17 -I src +GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++20 -I src diff --git a/perl/Makefile b/perl/Makefile index 708f86882..2d759e6fc 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -1,6 +1,6 @@ makefiles = local.mk -GLOBAL_CXXFLAGS += -g -Wall -std=c++17 -I ../src +GLOBAL_CXXFLAGS += -g -Wall -std=c++20 -I ../src -include Makefile.config diff --git a/src/libcmd/nix-cmd.pc.in b/src/libcmd/nix-cmd.pc.in index 1761a9f41..a21d93f1d 100644 --- a/src/libcmd/nix-cmd.pc.in +++ b/src/libcmd/nix-cmd.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixcmd -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++20 diff --git a/src/libexpr/nix-expr.pc.in b/src/libexpr/nix-expr.pc.in index 80f7a492b..95d452ca8 100644 --- a/src/libexpr/nix-expr.pc.in +++ b/src/libexpr/nix-expr.pc.in @@ -7,4 +7,4 @@ Description: Nix Package Manager Version: @PACKAGE_VERSION@ Requires: nix-store bdw-gc Libs: -L${libdir} -lnixexpr -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++20 diff --git a/src/libmain/nix-main.pc.in b/src/libmain/nix-main.pc.in index 37b03dcd4..b46ce1990 100644 --- a/src/libmain/nix-main.pc.in +++ b/src/libmain/nix-main.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixmain -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++20 diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in index 6d67b1e03..385169a13 100644 --- a/src/libstore/nix-store.pc.in +++ b/src/libstore/nix-store.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixstore -lnixutil -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++20 From 4563e803633d44ec57dfc4515f0c42c8855c018b Mon Sep 17 00:00:00 2001 From: Graham Bennett Date: Sat, 22 Oct 2022 14:25:35 +0100 Subject: [PATCH 005/179] Fix C++20 warnings --- src/libstore/binary-cache-store.cc | 2 +- src/libstore/build/worker.cc | 2 +- src/libstore/globals.cc | 6 +++--- src/libutil/args.cc | 2 +- src/libutil/config.cc | 8 ++++---- src/libutil/config.hh | 12 ++++++++---- 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index a26770c79..9ab941075 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -378,7 +378,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, auto callbackPtr = std::make_shared(std::move(callback)); getFile(narInfoFile, - {[=](std::future> fut) { + {[=,this](std::future> fut) { try { auto data = fut.get(); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b192fbc77..75406bec9 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -276,7 +276,7 @@ void Worker::run(const Goals & _topGoals) if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); else { - if (awake.empty() && 0 == settings.maxBuildJobs) + if (awake.empty() && 0U == settings.maxBuildJobs) { if (getMachines().empty()) throw Error("unable to start any build; either increase '--max-jobs' " diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index ff658c428..c264014be 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -218,19 +218,19 @@ template<> void BaseSetting::convertToArg(Args & args, const std::s .longName = name, .description = "Enable sandboxing.", .category = category, - .handler = {[=]() { override(smEnabled); }} + .handler = {[this]() { override(smEnabled); }} }); args.addFlag({ .longName = "no-" + name, .description = "Disable sandboxing.", .category = category, - .handler = {[=]() { override(smDisabled); }} + .handler = {[this]() { override(smDisabled); }} }); args.addFlag({ .longName = "relaxed-" + name, .description = "Enable sandboxing, but allow builds to disable it.", .category = category, - .handler = {[=]() { override(smRelaxed); }} + .handler = {[this]() { override(smRelaxed); }} }); } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 753980fd4..2930913d6 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -324,7 +324,7 @@ MultiCommand::MultiCommand(const Commands & commands_) expectArgs({ .label = "subcommand", .optional = true, - .handler = {[=](std::string s) { + .handler = {[=,this](std::string s) { assert(!command); auto i = commands.find(s); if (i == commands.end()) { diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 9bb412b4f..b349f2d80 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -209,7 +209,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) .description = fmt("Set the `%s` setting.", name), .category = category, .labels = {"value"}, - .handler = {[=](std::string s) { overridden = true; set(s); }}, + .handler = {[this](std::string s) { overridden = true; set(s); }}, }); if (isAppendable()) @@ -218,7 +218,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) .description = fmt("Append to the `%s` setting.", name), .category = category, .labels = {"value"}, - .handler = {[=](std::string s) { overridden = true; set(s, true); }}, + .handler = {[this](std::string s) { overridden = true; set(s, true); }}, }); } @@ -270,13 +270,13 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & .longName = name, .description = fmt("Enable the `%s` setting.", name), .category = category, - .handler = {[=]() { override(true); }} + .handler = {[this]() { override(true); }} }); args.addFlag({ .longName = "no-" + name, .description = fmt("Disable the `%s` setting.", name), .category = category, - .handler = {[=]() { override(false); }} + .handler = {[this]() { override(false); }} }); } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 79ec0f9cf..7ac43c854 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -250,11 +250,15 @@ public: operator const T &() const { return value; } operator T &() { return value; } const T & get() const { return value; } - bool operator ==(const T & v2) const { return value == v2; } - bool operator !=(const T & v2) const { return value != v2; } - void operator =(const T & v) { assign(v); } + template + bool operator ==(const U & v2) const { return value == v2; } + template + bool operator !=(const U & v2) const { return value != v2; } + template + void operator =(const U & v) { assign(v); } virtual void assign(const T & v) { value = v; } - void setDefault(const T & v) { if (!overridden) value = v; } + template + void setDefault(const U & v) { if (!overridden) value = v; } void set(const std::string & str, bool append = false) override; From 180538672a017488f9c23a4f72da5e5738a7570c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 5 Dec 2022 11:01:46 +0100 Subject: [PATCH 006/179] disallow selecting shell prompt in code samples this is a quick half-fix for command line examples, as discussed discussed in [1]. [1]: https://github.com/NixOS/nix/pull/7389 examples which look like this $ foo bar baz are confusing for Unix shell beginners, because it's hard to discern what is supposed to be entered into the actual command line when the convention of prefixing `$` is not known, as barely any real-world shell looks that way any more. this change prevents selecting the prompt part with the mouse in the HTML representation of the Nix manual. it does not prevent selecting the output part of the shell example. it also does not address that the copy button provided by mdBook takes the entire sample, including the prompts, into the clipboard. --- doc/manual/custom.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/custom.css b/doc/manual/custom.css index 69d48d4a7..b90f5423f 100644 --- a/doc/manual/custom.css +++ b/doc/manual/custom.css @@ -5,3 +5,7 @@ h1:not(:first-of-type) { h2 { margin-top: 1em; } + +.hljs-meta { + user-select: none; +} From aaef82e192fbee108f9c2f64f4dd1fa2cd21d72b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Dec 2022 14:12:05 +0100 Subject: [PATCH 007/179] Document the release process --- maintainers/release-process.md | 166 +++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 maintainers/release-process.md diff --git a/maintainers/release-process.md b/maintainers/release-process.md new file mode 100644 index 000000000..68063f50c --- /dev/null +++ b/maintainers/release-process.md @@ -0,0 +1,166 @@ +# Nix release process + +## Release artifacts + +The release process is intended to create the following for each +release: + +* A Git tag + +* Binary tarballs in https://releases.nixos.org/?prefix=nix/ + +* Docker images + +* Closures in https://cache.nixos.org + +* (Optionally) Updated `fallback-paths.nix` in Nixpkgs + +* Update the manual on https://nixos.org/manual/nix/stable/ + +## Creating a new release from master + +* Make sure that https://hydra.nixos.org/jobset/nix/master is green. + +* In a checkout of the Nix repo, make sure you're on `master` and do a + `git pull`. + +* Move the contents of `doc/manual/src/release-notes/rl-next.md` + (except the first line) to + `doc/manual/src/release-notes/rl-$VERSION.md` (where `$VERSION` is + the contents of `.version` *without* the patch level, e.g. `2.12` + rather than `2.12.0`). + +* Add a header to `doc/manual/src/release-notes/rl-$VERSION.md` like + + ``` + # Release 2.12 (2022-12-06) + ``` + +* Proof-read / edit / rearrange the release notes. Breaking changes + and highlights should go to the top. + +* Add a link to the release notes to `doc/manual/src/SUMMARY.md.in` + (*not* `SUMMARY.md`), e.g. + + ``` + - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) + ``` + +* Run + + ```console + $ git checkout -b release-notes + $ git add doc/manual/src/release-notes/rl-$VERSION.md + $ git commit -a -m 'Release notes' + $ git push --set-upstream edolstra release-notes + ``` + +* Create a PR for `release-notes` and auto-merge it. + +* Wait for the PR to be merged. + +* Branch the release: + + ```console + $ git checkout master + $ git pull + $ git checkout -b $VERSION-maintenance + ``` + +* Mark the release as stable: + + ```console + $ git cherry-pick f673551e71942a52b6d7ae66af8b67140904a76a + ``` + + This removes the link to `rl-next.md` from the manual and sets + `officialRelease = true` in `flake.nix`. + +* Push the release branch: + + ```console + $ git push + ``` + +* Create a jobset for the release branch on Hydra as follows: + + * Go to the jobset of the previous release + (e.g. https://hydra.nixos.org/jobset/nix/maintenance-2.11). + + * Select `Actions -> Clone this jobset`. + + * Set identifier to `maintenance-$VERSION`. + + * Set description to `$VERSION release branch`. + + * Set flake URL to `github:NixOS/nix/$VERSION-maintenance`. + + * Hit `Create jobset`. + +* Wait for the new jobset to evaluate and build. If impatient, go to + the evaluation and select `Actions -> Bump builds to front of + queue`. + +* When the jobset evaluation has succeeded building, take note of the + evaluation ID (e.g. `1780832` in + https://hydra.nixos.org/eval/1780832). + +* Tag the release and upload the release artifacts to + `releases.nixos.org` and dockerhub: + + ```console + $ IS_LATEST=1 ./maintainers/upload-release.pl + ``` + + Note: `IS_LATEST=1` causes the `latest-release` branch to be + force-updated. This is used by the `nixos.org` website to get the + latest Nix manual. + + TODO: This script requires the right AWS credentials. Document. + + TODO: This script currently requires a + `/home/eelco/Dev/nix-pristine` and + `/home/eelco/Dev/nixpkgs-pristine`. + +* Prepare for the next point release by editing `.version` to + e.g. + + ```console + $ echo -n 2.12.1 > .version + $ git commit -a -m 'Bump version' + $ git push + ``` + + Note the `-n`: `.version` must not end in a newline. + + Commit and push this to the maintenance branch. + +* Bump the version of `master`: + + ```console + $ git checkout master + $ echo -n 2.13.0 > .version + $ git checkout -b bump-2.13 + $ git commit -a -m 'Bump version' + $ git push + ``` + + Make a PR and auto-merge it. + +## Creating a point release + +* Wait for the desired evaluation of the maintenance jobset to finish + building. + +* Run + + ```console + $ IS_LATEST=1 ./maintainers/upload-release.pl + ``` + + Omit `IS_LATEST=1` when creating a point release that is not on the + most recent stable branch. This prevents `nixos.org` to going back + to an older release. + +* Bump the version number of the release branch as above (e.g. to + `2.12.2`). From 98364a0df234f0febfddbcdde4a627405d6b19f1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Dec 2022 14:33:12 +0100 Subject: [PATCH 008/179] Mention milestone --- maintainers/release-process.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index 68063f50c..5eeb0c206 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -147,6 +147,10 @@ release: Make a PR and auto-merge it. +* Create a milestone for the next release, move all unresolved issues + from the previous milestone, and close the previous milestone. Set + the date for the next milestone 6 weeks from now. + ## Creating a point release * Wait for the desired evaluation of the maintenance jobset to finish From f2b9e1f8cb682d37533feed7d5b1d1a4ee13ad81 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Dec 2022 15:26:26 +0100 Subject: [PATCH 009/179] Update maintainers/release-process.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- maintainers/release-process.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index 5eeb0c206..b266b35e0 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -15,7 +15,7 @@ release: * (Optionally) Updated `fallback-paths.nix` in Nixpkgs -* Update the manual on https://nixos.org/manual/nix/stable/ +* An updated manual on https://nixos.org/manual/nix/stable/ ## Creating a new release from master From 758c2fdbfe93ab5d3582cc83dfef80f20387630b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Dec 2022 15:27:45 +0100 Subject: [PATCH 010/179] Update maintainers/release-process.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- maintainers/release-process.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index b266b35e0..e1f8e7d75 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -59,7 +59,7 @@ release: * Wait for the PR to be merged. -* Branch the release: +* Create a branch for the release: ```console $ git checkout master From 31e3c7d6fd5ce6c009ef2b7025aa1de43050ca2e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Dec 2022 17:07:51 +0100 Subject: [PATCH 011/179] Mention Discourse --- maintainers/release-process.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index e1f8e7d75..759e80d3c 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -79,7 +79,7 @@ release: * Push the release branch: ```console - $ git push + $ git push --set-upstream origin $VERSION-maintenance ``` * Create a jobset for the release branch on Hydra as follows: @@ -151,6 +151,9 @@ release: from the previous milestone, and close the previous milestone. Set the date for the next milestone 6 weeks from now. +* Post an announcement on Discourse, including the contents of + `rl-$VERSION.md`. + ## Creating a point release * Wait for the desired evaluation of the maintenance jobset to finish From 27392a3b48bad24385ddf1c359f7c04164b6aabc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Dec 2022 17:46:42 +0100 Subject: [PATCH 012/179] More tweaks --- maintainers/release-process.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index 759e80d3c..27452b37e 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -139,10 +139,12 @@ release: ```console $ git checkout master - $ echo -n 2.13.0 > .version - $ git checkout -b bump-2.13 + $ git pull + $ NEW_VERSION=2.13.0 + $ echo -n $NEW_VERSION > .version + $ git checkout -b bump-$NEW_VERSION $ git commit -a -m 'Bump version' - $ git push + $ git push --set-upstream origin bump-$NEW_VERSION ``` Make a PR and auto-merge it. From acb69a7beb434e5a5a25462f6d4e38d771eca7a1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Dec 2022 13:29:50 +0100 Subject: [PATCH 013/179] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> Co-authored-by: Robert Hensing --- maintainers/release-process.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index 27452b37e..565539bf6 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -52,10 +52,10 @@ release: $ git checkout -b release-notes $ git add doc/manual/src/release-notes/rl-$VERSION.md $ git commit -a -m 'Release notes' - $ git push --set-upstream edolstra release-notes + $ git push --set-upstream $REMOTE release-notes ``` -* Create a PR for `release-notes` and auto-merge it. +* Create a PR for `release-notes`. * Wait for the PR to be merged. @@ -122,17 +122,16 @@ release: `/home/eelco/Dev/nix-pristine` and `/home/eelco/Dev/nixpkgs-pristine`. + TODO: trigger nixos.org netlify: https://docs.netlify.com/configure-builds/build-hooks/ * Prepare for the next point release by editing `.version` to e.g. ```console - $ echo -n 2.12.1 > .version + $ echo 2.12.1 > .version $ git commit -a -m 'Bump version' $ git push ``` - Note the `-n`: `.version` must not end in a newline. - Commit and push this to the maintenance branch. * Bump the version of `master`: @@ -173,3 +172,8 @@ release: * Bump the version number of the release branch as above (e.g. to `2.12.2`). + +## Recovering from mistakes + +`upload-release.pl` should be idempotent. For instance a wrong `IS_LATEST` value can be fixed that way, by running the script on the actual latest release. + From 6eed1325ffbe50214620e974685e36731e8730b6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 12 Dec 2022 17:04:06 +0100 Subject: [PATCH 014/179] Apply suggestions from code review Co-authored-by: Valentin Gagarin --- maintainers/release-process.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index 565539bf6..75c1470b8 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -17,11 +17,11 @@ release: * An updated manual on https://nixos.org/manual/nix/stable/ -## Creating a new release from master +## Creating a new release from the `master` branch -* Make sure that https://hydra.nixos.org/jobset/nix/master is green. +* Make sure that the [Hydra `master` jobset](https://hydra.nixos.org/jobset/nix/master) succeeds. -* In a checkout of the Nix repo, make sure you're on `master` and do a +* In a checkout of the Nix repo, make sure you're on `master` and run `git pull`. * Move the contents of `doc/manual/src/release-notes/rl-next.md` @@ -103,10 +103,10 @@ release: * When the jobset evaluation has succeeded building, take note of the evaluation ID (e.g. `1780832` in - https://hydra.nixos.org/eval/1780832). + `https://hydra.nixos.org/eval/1780832`). * Tag the release and upload the release artifacts to - `releases.nixos.org` and dockerhub: + [`releases.nixos.org`](https://releases.nixos.org/) and [Docker Hub](https://hub.docker.com/): ```console $ IS_LATEST=1 ./maintainers/upload-release.pl @@ -114,7 +114,7 @@ release: Note: `IS_LATEST=1` causes the `latest-release` branch to be force-updated. This is used by the `nixos.org` website to get the - latest Nix manual. + [latest Nix manual](https://nixos.org/manual/nixpkgs/unstable/). TODO: This script requires the right AWS credentials. Document. @@ -146,13 +146,13 @@ release: $ git push --set-upstream origin bump-$NEW_VERSION ``` - Make a PR and auto-merge it. + Make a pull request and auto-merge it. * Create a milestone for the next release, move all unresolved issues from the previous milestone, and close the previous milestone. Set the date for the next milestone 6 weeks from now. -* Post an announcement on Discourse, including the contents of +* Post an [announcement on Discourse](https://discourse.nixos.org/c/announcements/8), including the contents of `rl-$VERSION.md`. ## Creating a point release From 15f7fa59bec1dbb4fa94bc1c2f9018b1f14f0d65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 23 Sep 2022 23:30:29 +0200 Subject: [PATCH 015/179] `unsafeDiscardReferences` Adds a new boolean structured attribute `outputChecks..unsafeDiscardReferences` which disables scanning an output for runtime references. __structuredAttrs = true; outputChecks.out.unsafeDiscardReferences = true; This is useful when creating filesystem images containing their own embedded Nix store: they are self-contained blobs of data with no runtime dependencies. Setting this attribute requires the experimental feature `discard-references` to be enabled. --- .../src/language/advanced-attributes.md | 6 ++--- doc/manual/src/release-notes/rl-next.md | 12 +++++++++ src/libstore/build/local-derivation-goal.cc | 27 ++++++++++++++++--- src/libutil/experimental-features.cc | 1 + src/libutil/experimental-features.hh | 1 + tests/check-refs.nix | 9 +++++++ tests/check-refs.sh | 9 +++++++ 7 files changed, 58 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 2e7e80ed0..f02425b13 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -207,13 +207,13 @@ Derivations can declare some infrequently used optional attributes. the hash in either hexadecimal or base-32 notation. (See the [`nix-hash` command](../command-ref/nix-hash.md) for information about converting to and from base-32 notation.) - + - [`__contentAddressed`]{#adv-attr-__contentAddressed} If this **experimental** attribute is set to true, then the derivation outputs will be stored in a content-addressed location rather than the traditional input-addressed one. - This only has an effect if the `ca-derivation` experimental feature is enabled. - + This only has an effect if the `ca-derivations` experimental feature is enabled. + Setting this attribute also requires setting `outputHashMode` and `outputHashAlgo` like for *fixed-output derivations* (see above). - [`passAsFile`]{#adv-attr-passAsFile}\ diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 9f491efc8..51320d50a 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -21,3 +21,15 @@ * Error traces have been reworked to provide detailed explanations and more accurate error locations. A short excerpt of the trace is now shown by default when an error occurs. + +* In derivations that use structured attributes, you can now use `unsafeDiscardReferences` + to disable scanning a given output for runtime dependencies: + ```nix + __structuredAttrs = true; + outputChecks.out.unsafeDiscardReferences = true; + ``` + This is useful e.g. when generating self-contained filesystem images with + their own embedded Nix store: hashes found inside such an image refer + to the embedded store and not to the host's Nix store. + + This requires the `discard-references` experimental feature. diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9d869d513..65340d6ce 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2319,11 +2319,30 @@ DrvOutputs LocalDerivationGoal::registerOutputs() buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, inodesSeen); - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + bool discardReferences = false; + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { + if (auto output = get(*outputChecks, outputName)) { + if (auto unsafeDiscardReferences = get(*output, "unsafeDiscardReferences")) { + settings.requireExperimentalFeature(Xp::DiscardReferences); + if (!unsafeDiscardReferences->is_boolean()) + throw Error("attribute 'outputChecks.\"%s\".unsafeDiscardReferences' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); + discardReferences = unsafeDiscardReferences->get(); + } + } + } + } - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = scanForReferences(blank, actualPath, referenceablePaths); + StorePathSet references; + if (discardReferences) + debug("discarding references of output '%s'", outputName); + else { + debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + + /* Pass blank Sink as we are not ready to hash data at this stage. */ + NullSink blank; + references = scanForReferences(blank, actualPath, referenceablePaths); + } outputReferencesIfUnregistered.insert_or_assign( outputName, diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index e0902971e..58d762ebb 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -16,6 +16,7 @@ std::map stringifiedXpFeatures = { { Xp::ReplFlake, "repl-flake" }, { Xp::AutoAllocateUids, "auto-allocate-uids" }, { Xp::Cgroups, "cgroups" }, + { Xp::DiscardReferences, "discard-references" }, }; const std::optional parseExperimentalFeature(const std::string_view & name) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index af775feb0..ac372e03e 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -25,6 +25,7 @@ enum struct ExperimentalFeature ReplFlake, AutoAllocateUids, Cgroups, + DiscardReferences, }; /** diff --git a/tests/check-refs.nix b/tests/check-refs.nix index 9d90b0920..a5e40c9e5 100644 --- a/tests/check-refs.nix +++ b/tests/check-refs.nix @@ -67,4 +67,13 @@ rec { disallowedReferences = [test5]; }; + test11 = makeTest 11 { + __structuredAttrs = true; + outputChecks.out = { + unsafeDiscardReferences = true; + allowedReferences = []; + }; + buildCommand = ''echo ${dep} > "''${outputs[out]}"''; + }; + } diff --git a/tests/check-refs.sh b/tests/check-refs.sh index 16bbabc40..65a72552a 100644 --- a/tests/check-refs.sh +++ b/tests/check-refs.sh @@ -40,3 +40,12 @@ nix-build -o $RESULT check-refs.nix -A test7 # test10 should succeed (no disallowed references). nix-build -o $RESULT check-refs.nix -A test10 + +if isDaemonNewer 2.12pre20230103; then + enableFeatures discard-references + restartDaemon + + # test11 should succeed. + test11=$(nix-build -o $RESULT check-refs.nix -A test11) + [[ -z $(nix-store -q --references "$test11") ]] +fi From 3c968191f1d54a172d2b245d080c67012f766f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Tue, 3 Jan 2023 18:53:01 +0100 Subject: [PATCH 016/179] move `unsafeDiscardReferences` out of `outputChecks` It's not a check. --- doc/manual/src/release-notes/rl-next.md | 2 +- src/libstore/build/local-derivation-goal.cc | 14 ++++++-------- tests/check-refs.nix | 6 ++---- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 51320d50a..1e8a035b0 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -26,7 +26,7 @@ to disable scanning a given output for runtime dependencies: ```nix __structuredAttrs = true; - outputChecks.out.unsafeDiscardReferences = true; + unsafeDiscardReferences.out = true; ``` This is useful e.g. when generating self-contained filesystem images with their own embedded Nix store: hashes found inside such an image refer diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 65340d6ce..d4ea68f93 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2321,14 +2321,12 @@ DrvOutputs LocalDerivationGoal::registerOutputs() bool discardReferences = false; if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { - if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { - if (auto output = get(*outputChecks, outputName)) { - if (auto unsafeDiscardReferences = get(*output, "unsafeDiscardReferences")) { - settings.requireExperimentalFeature(Xp::DiscardReferences); - if (!unsafeDiscardReferences->is_boolean()) - throw Error("attribute 'outputChecks.\"%s\".unsafeDiscardReferences' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); - discardReferences = unsafeDiscardReferences->get(); - } + if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { + settings.requireExperimentalFeature(Xp::DiscardReferences); + if (auto output = get(*udr, outputName)) { + if (!output->is_boolean()) + throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); + discardReferences = output->get(); } } } diff --git a/tests/check-refs.nix b/tests/check-refs.nix index a5e40c9e5..99d69a226 100644 --- a/tests/check-refs.nix +++ b/tests/check-refs.nix @@ -69,10 +69,8 @@ rec { test11 = makeTest 11 { __structuredAttrs = true; - outputChecks.out = { - unsafeDiscardReferences = true; - allowedReferences = []; - }; + unsafeDiscardReferences.out = true; + outputChecks.out.allowedReferences = []; buildCommand = ''echo ${dep} > "''${outputs[out]}"''; }; From 5c0d3b476f617723b90d6e9daf0d8b53ac25bbac Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 12 Jan 2023 00:19:18 +0100 Subject: [PATCH 017/179] release-process: Add "create a backport label" --- maintainers/release-process.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index 75c1470b8..b52d218f5 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -152,6 +152,8 @@ release: from the previous milestone, and close the previous milestone. Set the date for the next milestone 6 weeks from now. +* Create a backport label + * Post an [announcement on Discourse](https://discourse.nixos.org/c/announcements/8), including the contents of `rl-$VERSION.md`. From 1fc74afbbae1ae3e78e3c0d2096e81c1e24f9d52 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 12 Jan 2023 13:42:17 -0800 Subject: [PATCH 018/179] nix/show-config: allow getting the value of a specific setting Instead of needing to run `nix show-config --json | jq -r '."warn-dirty".value'` to view the value of `warn-dirty`, you can now run `nix show-config warn-dirty`. --- src/nix/show-config.cc | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 29944e748..3530584f9 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -9,15 +9,44 @@ using namespace nix; struct CmdShowConfig : Command, MixJSON { + std::optional name; + + CmdShowConfig() { + expectArgs({ + .label = {"name"}, + .optional = true, + .handler = {&name}, + }); + } + std::string description() override { - return "show the Nix configuration"; + return "show the Nix configuration or the value of a specific setting"; } Category category() override { return catUtility; } void run() override { + if (name) { + if (json) { + throw UsageError("'--json' is not supported when specifying a setting name"); + } + + std::map settings; + globalConfig.getSettings(settings); + auto setting = settings.find(*name); + + if (setting == settings.end()) { + throw Error("could not find setting '%1%'", *name); + } else { + const auto & value = setting->second.value; + logger->cout("%s", value); + } + + return; + } + if (json) { // FIXME: use appropriate JSON types (bool, ints, etc). logger->cout("%s", globalConfig.toJSON().dump()); From 7f195d058c0077a98c48116c7198d97ab194c74a Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Fri, 13 Jan 2023 07:57:55 -0800 Subject: [PATCH 019/179] tests/config: test retrieving a single setting's value with `nix show-config ` --- tests/config.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/config.sh b/tests/config.sh index 3d0da3cef..723f575ed 100644 --- a/tests/config.sh +++ b/tests/config.sh @@ -51,3 +51,8 @@ exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 [[ $prev != $exp_cores ]] [[ $exp_cores == "4242" ]] [[ $exp_features == "flakes nix-command" ]] + +# Test that it's possible to retrieve a single setting's value +val=$(nix show-config | grep '^warn-dirty' | cut -d '=' -f 2 | xargs) +val2=$(nix show-config warn-dirty) +[[ $val == $val2 ]] From 763c1dfc2b0875251385d9b228c0fc1bfd654bf4 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Mon, 16 Jan 2023 11:35:12 -0800 Subject: [PATCH 020/179] Expand installation.md Changes the `quick-start.md` to recommend a multi-user install, since single-user is not supported on MacOS and https://nixos.org/download.html recommends multi-user. Expands `installation.md` to reflect wording on https://nixos.org/download.html --- doc/manual/src/installation/installation.md | 40 +++++++++++++++++++-- doc/manual/src/quick-start.md | 12 +++---- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/installation/installation.md b/doc/manual/src/installation/installation.md index b40c5b95f..dafdeb667 100644 --- a/doc/manual/src/installation/installation.md +++ b/doc/manual/src/installation/installation.md @@ -1,2 +1,38 @@ -This section describes how to install and configure Nix for first-time -use. +# Installation + +This section describes how to install and configure Nix for first-time use. + +The current recommended option on Linux and MacOS is [multi-user](#multi-user). + +## Multi-user + +This installation offers better sharing, improved isolation, and more security +over a single user installation. + +This option requires either: + +* Linux running systemd, with SELinux disabled +* MacOS + +```console +$ bash <(curl -L https://nixos.org/nix/install) --daemon +``` + +## Single-user + +> Single-user is not supported on Mac. + +This installation has less requirements than the multi-user install, however it +cannot offer equivalent sharing, isolation, or security. + +This option is suitable for systems without systemd. + +```console +$ bash <(curl -L https://nixos.org/nix/install) --no-daemon +``` + +## Distributions + +The Nix community maintains installers for several distributions. + +They can be found in the [`nix-community/nix-installers`](https://github.com/nix-community/nix-installers) repository. diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index b54e73500..e11549984 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -4,16 +4,16 @@ This chapter is for impatient people who don't like reading documentation. For more in-depth information you are kindly referred to subsequent chapters. -1. Install single-user Nix by running the following: +1. Install Nix by running the following: ```console - $ bash <(curl -L https://nixos.org/nix/install) + $ bash <(curl -L https://nixos.org/nix/install) --daemon ``` - This will install Nix in `/nix`. The install script will create - `/nix` using `sudo`, so make sure you have sufficient rights. (For - other installation methods, see - [here](installation/installation.md).) + The install script will use `sudo`, so make sure you have sufficient rights. + On Linux, `--daemon` can be omitted for a single-user install. + + For other installation methods, see [here](installation/installation.md). 1. See what installable packages are currently available in the channel: From a5919f4754be6f4a9fe091e0ee5538ad85af0050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 10:26:50 +0200 Subject: [PATCH 021/179] =?UTF-8?q?Move=20the=20default=20profiles=20to=20?= =?UTF-8?q?the=20user=E2=80=99s=20home?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rather than using `/nix/var/nix/{profiles,gcroots}/per-user/`, put the user profiles and gcroots under `$XDG_DATA_DIR/nix/{profiles,gcroots}`. This means that the daemon no longer needs to manage these paths itself (they are fully handled client-side). In particular, it doesn’t have to `chown` them anymore (removing one need for root). This does change the layout of the gc-roots created by nix-env, and is likely to break some stuff, so I’m not sure how to properly handle that. --- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/local-store.cc | 16 ------------- src/libstore/local-store.hh | 2 -- src/libstore/profiles.cc | 26 ++++++++++++++++----- src/libstore/profiles.hh | 4 ++++ src/libstore/store-api.hh | 3 --- src/libutil/util.cc | 18 ++++++++------ src/libutil/util.hh | 3 +++ src/nix-channel/nix-channel.cc | 4 +++- src/nix/daemon.cc | 1 - tests/remote-store.sh | 4 ---- 11 files changed, 42 insertions(+), 41 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9ab9b17fe..4fec8fb88 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1517,7 +1517,7 @@ void LocalDerivationGoal::startDaemon() try { daemon::processConnection(store, from, to, daemon::NotTrusted, daemon::Recursive, - [&](Store & store) { store.createUser("nobody", 65535); }); + [&](Store & store) {}); debug("terminated daemon connection"); } catch (SysError &) { ignoreException(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index be21e3ca0..82edaa9bf 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -201,8 +201,6 @@ LocalStore::LocalStore(const Params & params) throw SysError("could not set permissions on '%s' to 755", perUserDir); } - createUser(getUserName(), getuid()); - /* Optionally, create directories and set permissions for a multi-user install. */ if (getuid() == 0 && settings.buildUsersGroup != "") { @@ -1824,20 +1822,6 @@ void LocalStore::signPathInfo(ValidPathInfo & info) } -void LocalStore::createUser(const std::string & userName, uid_t userId) -{ - for (auto & dir : { - fmt("%s/profiles/per-user/%s", stateDir, userName), - fmt("%s/gcroots/per-user/%s", stateDir, userName) - }) { - createDirs(dir); - if (chmod(dir.c_str(), 0755) == -1) - throw SysError("changing permissions of directory '%s'", dir); - if (chown(dir.c_str(), userId, getgid()) == -1) - throw SysError("changing owner of directory '%s'", dir); - } -} - std::optional> LocalStore::queryRealisationCore_( LocalStore::State & state, const DrvOutput & id) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 06d36a7d5..a84eb7c26 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -281,8 +281,6 @@ private: void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); - void createUser(const std::string & userName, uid_t userId) override; - // XXX: Make a generic `Store` method FixedOutputHash hashCAPath( const FileIngestionMethod & method, diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 3e4188188..29ce13b8d 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -280,16 +280,30 @@ std::string optimisticLockProfile(const Path & profile) } +Path profilesDir() +{ + auto profileRoot = getDataDir() + "/nix/profiles"; + createDirs(profileRoot); + return profileRoot; +} + + Path getDefaultProfile() { Path profileLink = getHome() + "/.nix-profile"; try { - if (!pathExists(profileLink)) { - replaceSymlink( - getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), - profileLink); + // Migrate from the “old-style” profiles stored under `/nix/var`: + // If the link exists and points to the old location, rewrite it to the + // new one (otherwise keep-it as-it-is as it might have been + // intentionnally changed, in which case we shouldn’t touch it) + auto legacyProfile = getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()); + if (!pathExists(profileLink) || + (isLink(profileLink) && + readLink(profileLink) == legacyProfile) + ) { + replaceSymlink(profilesDir() + "/profile", profileLink); } return absPath(readLink(profileLink), dirOf(profileLink)); } catch (Error &) { diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 408ca039c..73667a798 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -68,6 +68,10 @@ void lockProfile(PathLocks & lock, const Path & profile); rebuilt. */ std::string optimisticLockProfile(const Path & profile); +/* Creates and returns the path to a directory suitable for storing the user’s + profiles. */ +Path profilesDir(); + /* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create it. */ Path getDefaultProfile(); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9eab4b4e5..5807392a7 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -657,9 +657,6 @@ public: return toRealPath(printStorePath(storePath)); } - virtual void createUser(const std::string & userName, uid_t userId) - { } - /* * Synchronises the options of the client with those of the daemon * (a no-op when there’s no daemon) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 993dc1cb6..40faa4bf2 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -537,6 +537,16 @@ std::string getUserName() return name; } +Path getHomeOf(uid_t userId) +{ + std::vector buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + return pw->pw_dir; +} Path getHome() { @@ -558,13 +568,7 @@ Path getHome() } } if (!homeDir) { - std::vector buf(16384); - struct passwd pwbuf; - struct passwd * pw; - if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 - || !pw || !pw->pw_dir || !pw->pw_dir[0]) - throw Error("cannot determine user's home directory"); - homeDir = pw->pw_dir; + homeDir = getHomeOf(geteuid()); if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 9b149de80..266da0ae3 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -137,6 +137,9 @@ void deletePath(const Path & path, uint64_t & bytesFreed); std::string getUserName(); +/* Return the given user's home directory from /etc/passwd. */ +Path getHomeOf(uid_t userId); + /* Return $HOME or the user's home directory from /etc/passwd. */ Path getHome(); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index cf52b03b4..263d85eea 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -1,9 +1,11 @@ +#include "profiles.hh" #include "shared.hh" #include "globals.hh" #include "filetransfer.hh" #include "store-api.hh" #include "legacy.hh" #include "fetchers.hh" +#include "util.hh" #include #include @@ -166,7 +168,7 @@ static int main_nix_channel(int argc, char ** argv) nixDefExpr = home + "/.nix-defexpr"; // Figure out the name of the channels profile. - profile = fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()); + profile = profilesDir() + "/channels"; enum { cNone, diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index c527fdb0a..19fbbf155 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -248,7 +248,6 @@ static void daemonLoop() querySetting("build-users-group", "") == "") throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); #endif - store.createUser(user, peer.uid); }); exit(0); diff --git a/tests/remote-store.sh b/tests/remote-store.sh index 31210ab47..1ae126794 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -30,7 +30,3 @@ NIX_REMOTE= nix-store --dump-db > $TEST_ROOT/d2 cmp $TEST_ROOT/d1 $TEST_ROOT/d2 killDaemon - -user=$(whoami) -[ -e $NIX_STATE_DIR/gcroots/per-user/$user ] -[ -e $NIX_STATE_DIR/profiles/per-user/$user ] From 0601050755ba39f52fdd2a0fc0478baaf9b1c7b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 10:26:50 +0200 Subject: [PATCH 022/179] Migrate the old profiles to the new location MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make sure that we don’t just create the new profiles directory, but that we also migrate every existing profile to it. --- src/libstore/profiles.cc | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 29ce13b8d..d185a898c 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -293,17 +293,29 @@ Path getDefaultProfile() Path profileLink = getHome() + "/.nix-profile"; try { // Migrate from the “old-style” profiles stored under `/nix/var`: - // If the link exists and points to the old location, rewrite it to the - // new one (otherwise keep-it as-it-is as it might have been - // intentionnally changed, in which case we shouldn’t touch it) + // If the link exists and points to the old location, then: + // - Rewrite it to point to the new location + // - For every generation of the old default profile, create a symlink + // from the new directory to it (so that all the previous profiles + // and generations are still available). auto legacyProfile = getuid() == 0 ? settings.nixStateDir + "/profiles/default" : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()); - if (!pathExists(profileLink) || - (isLink(profileLink) && - readLink(profileLink) == legacyProfile) - ) { - replaceSymlink(profilesDir() + "/profile", profileLink); + auto newProfile = profilesDir() + "/profile"; + if (!pathExists(profileLink) + || (isLink(profileLink) + && readLink(profileLink) == legacyProfile)) { + warn("Migrating the default profile"); + replaceSymlink(newProfile, profileLink); + replaceSymlink(legacyProfile, newProfile); + if (pathExists(legacyProfile)) { + for (auto & oldGen : findGenerations(legacyProfile).first) { + replaceSymlink( + oldGen.path, + dirOf(newProfile) + "/" + + std::string(baseNameOf(oldGen.path))); + } + } } return absPath(readLink(profileLink), dirOf(profileLink)); } catch (Error &) { From 1f02aa4098a61e60062e3ecdb713810e24efa25a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 10:26:50 +0200 Subject: [PATCH 023/179] Test the migration of the user profiles --- tests/common.sh.in | 2 +- tests/local.mk | 1 + tests/user-envs-migration.sh | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 tests/user-envs-migration.sh diff --git a/tests/common.sh.in b/tests/common.sh.in index 73c2d2309..74bbbc8ca 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -62,7 +62,7 @@ readLink() { } clearProfiles() { - profiles="$NIX_STATE_DIR"/profiles + profiles="$HOME"/.local/share/nix/profiles rm -rf $profiles } diff --git a/tests/local.mk b/tests/local.mk index 5ac1ede32..b900e1534 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -17,6 +17,7 @@ nix_tests = \ fetchMercurial.sh \ gc-auto.sh \ user-envs.sh \ + user-envs-migration.sh \ binary-cache.sh \ multiple-outputs.sh \ ca/build.sh \ diff --git a/tests/user-envs-migration.sh b/tests/user-envs-migration.sh new file mode 100644 index 000000000..467c28fbb --- /dev/null +++ b/tests/user-envs-migration.sh @@ -0,0 +1,35 @@ +# Test that the migration of user environments +# (https://github.com/NixOS/nix/pull/5226) does preserve everything + +source common.sh + +if isDaemonNewer "2.4pre20211005"; then + exit 99 +fi + + +killDaemon +unset NIX_REMOTE + +clearStore +clearProfiles +rm -rf ~/.nix-profile + +# Fill the environment using the older Nix +PATH_WITH_NEW_NIX="$PATH" +export PATH="$NIX_DAEMON_PACKAGE/bin:$PATH" + +nix-env -f user-envs.nix -i foo-1.0 +nix-env -f user-envs.nix -i bar-0.1 + +# Migrate to the new profile dir, and ensure that everything’s there +export PATH="$PATH_WITH_NEW_NIX" +nix-env -q # Trigger the migration +( [[ -L ~/.nix-profile ]] && \ + [[ $(readlink ~/.nix-profile) == ~/.local/share/nix/profiles/profile ]] ) || \ + fail "The nix profile should point to the new location" + +(nix-env -q | grep foo && nix-env -q | grep bar && \ + [[ -e ~/.nix-profile/bin/foo ]] && \ + [[ $(nix-env --list-generations | wc -l) == 2 ]]) || + fail "The nix profile should have the same content as before the migration" From c80621dbacc501f8bf43863438a3df087436d13e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 17 Jan 2023 14:15:00 +0100 Subject: [PATCH 024/179] Don't try to migrate existing profiles Doing so would be more dangerous than useful, better leave them as-is if they already exist --- src/libstore/profiles.cc | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index d185a898c..ff62d0972 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -292,30 +292,9 @@ Path getDefaultProfile() { Path profileLink = getHome() + "/.nix-profile"; try { - // Migrate from the “old-style” profiles stored under `/nix/var`: - // If the link exists and points to the old location, then: - // - Rewrite it to point to the new location - // - For every generation of the old default profile, create a symlink - // from the new directory to it (so that all the previous profiles - // and generations are still available). - auto legacyProfile = getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()); - auto newProfile = profilesDir() + "/profile"; - if (!pathExists(profileLink) - || (isLink(profileLink) - && readLink(profileLink) == legacyProfile)) { - warn("Migrating the default profile"); - replaceSymlink(newProfile, profileLink); - replaceSymlink(legacyProfile, newProfile); - if (pathExists(legacyProfile)) { - for (auto & oldGen : findGenerations(legacyProfile).first) { - replaceSymlink( - oldGen.path, - dirOf(newProfile) + "/" - + std::string(baseNameOf(oldGen.path))); - } - } + auto profile = profilesDir() + "/profile"; + if (!pathExists(profileLink)) { + replaceSymlink(profile, profileLink); } return absPath(readLink(profileLink), dirOf(profileLink)); } catch (Error &) { From 6bdf4edb77bea3fdc006e9b8131a43f4dcd0ee42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 17 Jan 2023 14:15:53 +0100 Subject: [PATCH 025/179] Keep the default profile the same MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's used as the “system” profile in a bunch of places, so better not touch it. Besides, it doesn't hurt to keep it since it's owned by root any way, so it doesn't have the `chown` problem that the user profiles had and that led to wanting to move them on the client-side. --- src/libstore/profiles.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index ff62d0972..b202351ce 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -292,7 +292,10 @@ Path getDefaultProfile() { Path profileLink = getHome() + "/.nix-profile"; try { - auto profile = profilesDir() + "/profile"; + auto profile = + getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : profilesDir() + "/profile"; if (!pathExists(profileLink)) { replaceSymlink(profile, profileLink); } From 3272a2d26fd24647509cddfc2267310416a41cd1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 18 Jan 2023 12:16:27 +0100 Subject: [PATCH 026/179] maintainers: Document backporting quirk Closes #7623 --- maintainers/backporting.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 maintainers/backporting.md diff --git a/maintainers/backporting.md b/maintainers/backporting.md new file mode 100644 index 000000000..fe860bc61 --- /dev/null +++ b/maintainers/backporting.md @@ -0,0 +1,8 @@ + +# Backporting + +Backports are handled by the backport action. + +Since GitHub Actions can not trigger actions, the backport PR needs to be re-triggered by another actor. This is achieved by closing and reopening the backport PR. + +This specifically affects the `installer_test` check, but note that it only runs after the other tests, so it may take a while to appear. From 4d50995effdaf1d04453293d1afa56e9c8ce6f24 Mon Sep 17 00:00:00 2001 From: Eric Wolf Date: Fri, 20 Jan 2023 10:31:26 +0100 Subject: [PATCH 027/179] Fix url parsing for urls using `file+` `file+https://example.org/test.mp4` should not be rejected with `unexpected authority`. --- src/libutil/tests/url.cc | 21 +++++++++++++++++++++ src/libutil/url.cc | 6 +++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc index c3b233797..e0c438b4d 100644 --- a/src/libutil/tests/url.cc +++ b/src/libutil/tests/url.cc @@ -99,6 +99,27 @@ namespace nix { ASSERT_EQ(parsed, expected); } + TEST(parseURL, parsesFilePlusHttpsUrl) { + auto s = "file+https://www.example.org/video.mp4"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "file+https://www.example.org/video.mp4", + .base = "https://www.example.org/video.mp4", + .scheme = "file+https", + .authority = "www.example.org", + .path = "/video.mp4", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, rejectsAuthorityInUrlsWithFileTransportation) { + auto s = "file://www.example.org/video.mp4"; + ASSERT_THROW(parseURL(s), Error); + } TEST(parseURL, parseIPv4Address) { auto s = "http://127.0.0.1:8080/file.tar.gz?download=fast&when=now#hello"; diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 5b7abeb49..4e43455e1 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -30,13 +30,13 @@ ParsedURL parseURL(const std::string & url) auto & query = match[6]; auto & fragment = match[7]; - auto isFile = scheme.find("file") != std::string::npos; + auto transportIsFile = parseUrlScheme(scheme).transport == "file"; - if (authority && *authority != "" && isFile) + if (authority && *authority != "" && transportIsFile) throw BadURL("file:// URL '%s' has unexpected authority '%s'", url, *authority); - if (isFile && path.empty()) + if (transportIsFile && path.empty()) path = "/"; return ParsedURL{ From 74026bb1014460d511534da8a955bee6948716ee Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 20 Jan 2023 14:43:26 +0100 Subject: [PATCH 028/179] tests: Move NixOS tests to tests/nixos This will allow contributors to find them more easily. --- flake.nix | 14 +++++++------- tests/{ => nixos/containers}/containers.nix | 0 tests/{ => nixos/containers}/id-test.nix | 0 tests/{ => nixos/containers}/systemd-nspawn.nix | 0 tests/{ => nixos}/github-flakes.nix | 0 tests/{ => nixos}/nix-copy-closure.nix | 0 tests/{ => nixos}/nss-preload.nix | 0 tests/{ => nixos}/remote-builds.nix | 0 tests/{ => nixos}/setuid.nix | 0 tests/{ => nixos}/sourcehut-flakes.nix | 0 10 files changed, 7 insertions(+), 7 deletions(-) rename tests/{ => nixos/containers}/containers.nix (100%) rename tests/{ => nixos/containers}/id-test.nix (100%) rename tests/{ => nixos/containers}/systemd-nspawn.nix (100%) rename tests/{ => nixos}/github-flakes.nix (100%) rename tests/{ => nixos}/nix-copy-closure.nix (100%) rename tests/{ => nixos}/nss-preload.nix (100%) rename tests/{ => nixos}/remote-builds.nix (100%) rename tests/{ => nixos}/setuid.nix (100%) rename tests/{ => nixos}/sourcehut-flakes.nix (100%) diff --git a/flake.nix b/flake.nix index 573373c42..0dc8c4c2e 100644 --- a/flake.nix +++ b/flake.nix @@ -475,37 +475,37 @@ }; # System tests. - tests.remoteBuilds = import ./tests/remote-builds.nix { + tests.remoteBuilds = import ./tests/nixos/remote-builds.nix { system = "x86_64-linux"; inherit nixpkgs; overlay = self.overlays.default; }; - tests.nix-copy-closure = import ./tests/nix-copy-closure.nix { + tests.nix-copy-closure = import ./tests/nixos/nix-copy-closure.nix { system = "x86_64-linux"; inherit nixpkgs; overlay = self.overlays.default; }; - tests.nssPreload = (import ./tests/nss-preload.nix rec { + tests.nssPreload = (import ./tests/nixos/nss-preload.nix rec { system = "x86_64-linux"; inherit nixpkgs; overlay = self.overlays.default; }); - tests.githubFlakes = (import ./tests/github-flakes.nix rec { + tests.githubFlakes = (import ./tests/nixos/github-flakes.nix rec { system = "x86_64-linux"; inherit nixpkgs; overlay = self.overlays.default; }); - tests.sourcehutFlakes = (import ./tests/sourcehut-flakes.nix rec { + tests.sourcehutFlakes = (import ./tests/nixos/sourcehut-flakes.nix rec { system = "x86_64-linux"; inherit nixpkgs; overlay = self.overlays.default; }); - tests.containers = (import ./tests/containers.nix rec { + tests.containers = (import ./tests/nixos/containers/containers.nix rec { system = "x86_64-linux"; inherit nixpkgs; overlay = self.overlays.default; @@ -514,7 +514,7 @@ tests.setuid = nixpkgs.lib.genAttrs ["i686-linux" "x86_64-linux"] (system: - import ./tests/setuid.nix rec { + import ./tests/nixos/setuid.nix rec { inherit nixpkgs system; overlay = self.overlays.default; }); diff --git a/tests/containers.nix b/tests/nixos/containers/containers.nix similarity index 100% rename from tests/containers.nix rename to tests/nixos/containers/containers.nix diff --git a/tests/id-test.nix b/tests/nixos/containers/id-test.nix similarity index 100% rename from tests/id-test.nix rename to tests/nixos/containers/id-test.nix diff --git a/tests/systemd-nspawn.nix b/tests/nixos/containers/systemd-nspawn.nix similarity index 100% rename from tests/systemd-nspawn.nix rename to tests/nixos/containers/systemd-nspawn.nix diff --git a/tests/github-flakes.nix b/tests/nixos/github-flakes.nix similarity index 100% rename from tests/github-flakes.nix rename to tests/nixos/github-flakes.nix diff --git a/tests/nix-copy-closure.nix b/tests/nixos/nix-copy-closure.nix similarity index 100% rename from tests/nix-copy-closure.nix rename to tests/nixos/nix-copy-closure.nix diff --git a/tests/nss-preload.nix b/tests/nixos/nss-preload.nix similarity index 100% rename from tests/nss-preload.nix rename to tests/nixos/nss-preload.nix diff --git a/tests/remote-builds.nix b/tests/nixos/remote-builds.nix similarity index 100% rename from tests/remote-builds.nix rename to tests/nixos/remote-builds.nix diff --git a/tests/setuid.nix b/tests/nixos/setuid.nix similarity index 100% rename from tests/setuid.nix rename to tests/nixos/setuid.nix diff --git a/tests/sourcehut-flakes.nix b/tests/nixos/sourcehut-flakes.nix similarity index 100% rename from tests/sourcehut-flakes.nix rename to tests/nixos/sourcehut-flakes.nix From 261c25601d9a4efd5245e3ef161fb52bf0543083 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 20 Jan 2023 15:32:31 +0100 Subject: [PATCH 029/179] Use the official, documented NixOS runTest interface --- flake.nix | 55 ++++++++++----------------- tests/nixos/containers/containers.nix | 11 ++---- tests/nixos/github-flakes.nix | 15 ++------ tests/nixos/nix-copy-closure.nix | 17 +++++---- tests/nixos/nss-preload.nix | 16 +++----- tests/nixos/remote-builds.nix | 14 ++----- tests/nixos/setuid.nix | 11 +++--- tests/nixos/sourcehut-flakes.nix | 14 ++----- 8 files changed, 56 insertions(+), 97 deletions(-) diff --git a/flake.nix b/flake.nix index 0dc8c4c2e..31d7f1fcf 100644 --- a/flake.nix +++ b/flake.nix @@ -401,6 +401,18 @@ }; }; + nixos-lib = import (nixpkgs + "/nixos/lib") { }; + + # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests + runNixOSTestFor = system: test: nixos-lib.runTest { + imports = [ test ]; + hostPkgs = nixpkgsFor.${system}; + defaults = { + nixpkgs.pkgs = nixpkgsFor.${system}; + }; + _module.args.nixpkgs = nixpkgs; + }; + in { # A Nixpkgs overlay that overrides the 'nix' and @@ -475,49 +487,22 @@ }; # System tests. - tests.remoteBuilds = import ./tests/nixos/remote-builds.nix { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }; + tests.remoteBuilds = runNixOSTestFor "x86_64-linux" ./tests/nixos/remote-builds.nix; - tests.nix-copy-closure = import ./tests/nixos/nix-copy-closure.nix { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }; + tests.nix-copy-closure = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy-closure.nix; - tests.nssPreload = (import ./tests/nixos/nss-preload.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }); + tests.nssPreload = runNixOSTestFor "x86_64-linux" ./tests/nixos/nss-preload.nix; - tests.githubFlakes = (import ./tests/nixos/github-flakes.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }); + tests.githubFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/github-flakes.nix; - tests.sourcehutFlakes = (import ./tests/nixos/sourcehut-flakes.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }); + tests.sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/sourcehut-flakes.nix; - tests.containers = (import ./tests/nixos/containers/containers.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }); + tests.containers = runNixOSTestFor "x86_64-linux" ./tests/nixos/containers/containers.nix; tests.setuid = nixpkgs.lib.genAttrs ["i686-linux" "x86_64-linux"] - (system: - import ./tests/nixos/setuid.nix rec { - inherit nixpkgs system; - overlay = self.overlays.default; - }); + (system: runNixOSTestFor system ./tests/nixos/setuid.nix); + # Make sure that nix-env still produces the exact same result # on a particular version of Nixpkgs. diff --git a/tests/nixos/containers/containers.nix b/tests/nixos/containers/containers.nix index a4856b2df..c8ee78a4a 100644 --- a/tests/nixos/containers/containers.nix +++ b/tests/nixos/containers/containers.nix @@ -1,12 +1,7 @@ # Test whether we can run a NixOS container inside a Nix build using systemd-nspawn. -{ nixpkgs, system, overlay }: +{ lib, nixpkgs, ... }: -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; - -makeTest ({ +{ name = "containers"; nodes = @@ -65,4 +60,4 @@ makeTest ({ host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") ''; -}) +} diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index a8b036b17..e4d347691 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -1,14 +1,9 @@ -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; - +{ lib, config, nixpkgs, ... }: let + pkgs = config.nodes.client.nixpkgs.pkgs; # Generate a fake root CA and a fake api.github.com / github.com / channels.nixos.org certificate. - cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; } + cert = pkgs.runCommand "cert" { nativeBuildInputs = [ pkgs.openssl ]; } '' mkdir -p $out @@ -92,8 +87,6 @@ let ''; in -makeTest ( - { name = "github-flakes"; @@ -207,4 +200,4 @@ makeTest ( client.succeed("nix build nixpkgs#fuse --tarball-ttl 0") ''; -}) +} diff --git a/tests/nixos/nix-copy-closure.nix b/tests/nixos/nix-copy-closure.nix index 2dc164ae4..66cbfb033 100644 --- a/tests/nixos/nix-copy-closure.nix +++ b/tests/nixos/nix-copy-closure.nix @@ -1,13 +1,16 @@ # Test ‘nix-copy-closure’. -{ nixpkgs, system, overlay }: +{ lib, config, nixpkgs, hostPkgs, ... }: -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; +let + pkgs = config.nodes.client.nixpkgs.pkgs; -makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; pkgD = pkgs.tmux; in { + pkgA = pkgs.cowsay; + pkgB = pkgs.wget; + pkgC = pkgs.hello; + pkgD = pkgs.tmux; + +in { name = "nix-copy-closure"; nodes = @@ -74,4 +77,4 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; pkgD = pk # ) # client.succeed("nix-store --check-validity ${pkgC}") ''; -}) +} diff --git a/tests/nixos/nss-preload.nix b/tests/nixos/nss-preload.nix index 5a6ff3f68..cef62e95b 100644 --- a/tests/nixos/nss-preload.nix +++ b/tests/nixos/nss-preload.nix @@ -1,11 +1,9 @@ -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; +{ lib, config, nixpkgs, ... }: let + + pkgs = config.nodes.client.nixpkgs.pkgs; + nix-fetch = pkgs.writeText "fetch.nix" '' derivation { # This derivation is an copy from what is available over at @@ -41,9 +39,7 @@ let ''; in -makeTest ( - -rec { +{ name = "nss-preload"; nodes = { @@ -122,4 +118,4 @@ rec { nix-build ${nix-fetch} >&2 """) ''; -}) +} diff --git a/tests/nixos/remote-builds.nix b/tests/nixos/remote-builds.nix index 9f88217fe..696cd2652 100644 --- a/tests/nixos/remote-builds.nix +++ b/tests/nixos/remote-builds.nix @@ -1,15 +1,9 @@ # Test Nix's remote build feature. -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; - -makeTest ( +{ config, lib, hostPkgs, ... }: let + pkgs = config.nodes.client.nixpkgs.pkgs; # The configuration of the remote builders. builder = @@ -75,7 +69,7 @@ in # Create an SSH key on the client. subprocess.run([ - "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" ], capture_output=True, check=True) client.succeed("mkdir -p -m 700 /root/.ssh") client.copy_from_host("key", "/root/.ssh/id_ed25519") @@ -109,4 +103,4 @@ in builder1.block() client.succeed("nix-build ${expr nodes.client.config 4}") ''; -}) +} diff --git a/tests/nixos/setuid.nix b/tests/nixos/setuid.nix index 6784615e4..2b66320dd 100644 --- a/tests/nixos/setuid.nix +++ b/tests/nixos/setuid.nix @@ -1,13 +1,12 @@ # Verify that Linux builds cannot create setuid or setgid binaries. -{ nixpkgs, system, overlay }: +{ lib, config, nixpkgs, ... }: -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; +let + pkgs = config.nodes.machine.nixpkgs.pkgs; -makeTest { +in +{ name = "setuid"; nodes.machine = diff --git a/tests/nixos/sourcehut-flakes.nix b/tests/nixos/sourcehut-flakes.nix index b77496ab6..a76fed020 100644 --- a/tests/nixos/sourcehut-flakes.nix +++ b/tests/nixos/sourcehut-flakes.nix @@ -1,12 +1,8 @@ -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") -{ - inherit system; - extraConfigurations = [{ nixpkgs.overlays = [ overlay ]; }]; -}; +{ lib, config, hostPkgs, nixpkgs, ... }: let + pkgs = config.nodes.sourcehut.nixpkgs.pkgs; + # Generate a fake root CA and a fake git.sr.ht certificate. cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; } '' @@ -64,8 +60,6 @@ let in -makeTest ( - { name = "sourcehut-flakes"; @@ -164,4 +158,4 @@ makeTest ( client.succeed("nix build nixpkgs#fuse --tarball-ttl 0") ''; - }) +} From 3c08a3e6b68135af126a05e9e3be417b604edded Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 20 Jan 2023 15:35:46 +0100 Subject: [PATCH 030/179] PR template: Specify path to integration tests --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a268d7caf..344f9405f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -21,7 +21,7 @@ Maintainers: tick if completed or explain if not relevant - [ ] tests, as appropriate - functional tests - `tests/**.sh` - unit tests - `src/*/tests` - - integration tests + - integration tests - `tests/nixos/*` - [ ] documentation in the manual - [ ] code and comments are self-explanatory - [ ] commit message explains why the change was made From a91709a604df24026771550638694075ce34fe45 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 23 Jan 2023 15:30:58 -0500 Subject: [PATCH 031/179] Try to fix #7669 The issue *seems* to be the cross jobs, which are missing the `CXXFLAGS` needed to get rapidcheck. PR #6538 would be really nice to resurrect which will prevent the `configureFlags` from going out of sync between the regular build and the cross build again. --- flake.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 5796c54ea..e3f8c728c 100644 --- a/flake.nix +++ b/flake.nix @@ -653,6 +653,7 @@ inherit system crossSystem; overlays = [ self.overlays.default ]; }; + inherit (nixpkgsCross) lib; in with commonDeps { pkgs = nixpkgsCross; }; nixpkgsCross.stdenv.mkDerivation { name = "nix-${version}"; @@ -665,7 +666,11 @@ nativeBuildInputs = nativeBuildDeps; buildInputs = buildDeps ++ propagatedDeps; - configureFlags = [ "--sysconfdir=/etc" "--disable-doc-gen" ]; + configureFlags = [ + "CXXFLAGS=-I${lib.getDev nixpkgsCross.rapidcheck}/extras/gtest/include" + "--sysconfdir=/etc" + "--disable-doc-gen" + ]; enableParallelBuilding = true; From 57f9dcaeb262f521738c25a05f295b74b910ab9c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Jan 2023 22:00:50 +0000 Subject: [PATCH 032/179] Bump zeebe-io/backport-action from 1.0.1 to 1.1.0 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 1.0.1 to 1.1.0. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v1.0.1...v1.1.0) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 9f8d14509..ca5af260f 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v1.0.1 + uses: zeebe-io/backport-action@v1.1.0 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From dc4aa383e991fbe9fa20b047c6812e4f34a332ea Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Tue, 24 Jan 2023 00:15:12 +0100 Subject: [PATCH 033/179] doc: fix anchor links in and to glossary --- doc/manual/src/command-ref/nix-store.md | 4 ++-- doc/manual/src/glossary.md | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index f6017481c..403cf285d 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -82,8 +82,8 @@ paths. Realisation is a somewhat overloaded term: produced through substitutes. If there are no (successful) substitutes, realisation fails. -[valid]: ../glossary.md#validity -[substitutes]: ../glossary.md#substitute +[valid]: ../glossary.md#gloss-validity +[substitutes]: ../glossary.md#gloss-substitute The output path of each derivation is printed on standard output. (For non-derivations argument, the argument itself is printed.) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 5a49af8dc..6004df833 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -156,6 +156,8 @@ to path `Q`, then `Q` is in the closure of `P`. Further, if `Q` references `R` then `R` is also in the closure of `P`. + [closure]: #gloss-closure + - [output path]{#gloss-output-path}\ A [store path] produced by a [derivation]. @@ -172,6 +174,8 @@ - The store path is listed in the Nix database as being valid. - All paths in the store path's [closure] are valid. + [validity]: #gloss-validity + - [user environment]{#gloss-user-env}\ An automatically generated store object that consists of a set of symlinks to “active” applications, i.e., other store paths. These From f58759816de03ac2d233576b8740410a68e9192f Mon Sep 17 00:00:00 2001 From: Andrea Ciceri Date: Wed, 23 Nov 2022 17:25:04 +0100 Subject: [PATCH 034/179] Tighten up the `exportReferencesGraph` tests Add an `$` at the end of the `grep` regex. Without it, `checkRef foo` would always imply `checkRef foo.drv`. We want to tell these situations apart to more precisely test what is going on. --- tests/export-graph.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/export-graph.sh b/tests/export-graph.sh index a1449b34e..4954a6cbc 100644 --- a/tests/export-graph.sh +++ b/tests/export-graph.sh @@ -4,7 +4,7 @@ clearStore clearProfiles checkRef() { - nix-store -q --references $TEST_ROOT/result | grep -q "$1" || fail "missing reference $1" + nix-store -q --references $TEST_ROOT/result | grep -q "$1"'$' || fail "missing reference $1" } # Test the export of the runtime dependency graph. From 0664ba0a67a2e1917ca66fed2174a587d27f3369 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 24 Jan 2023 14:39:45 +0100 Subject: [PATCH 035/179] Revert "fixup: remove boehmgc patch" It is still necessary. Please do your research, or f ask the author, which happens to be me. An evaluator like this is not an environment where "it compiles, so it works" will ever hold. This reverts commit 1c40182b12d5fd462c891b597e1a3f9b912502d5. --- boehmgc-coroutine-sp-fallback.diff | 77 ++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 boehmgc-coroutine-sp-fallback.diff diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff new file mode 100644 index 000000000..8fdafbecb --- /dev/null +++ b/boehmgc-coroutine-sp-fallback.diff @@ -0,0 +1,77 @@ +diff --git a/darwin_stop_world.c b/darwin_stop_world.c +index 3dbaa3fb..36a1d1f7 100644 +--- a/darwin_stop_world.c ++++ b/darwin_stop_world.c +@@ -352,6 +352,7 @@ GC_INNER void GC_push_all_stacks(void) + int nthreads = 0; + word total_size = 0; + mach_msg_type_number_t listcount = (mach_msg_type_number_t)THREAD_TABLE_SZ; ++ size_t stack_limit; + if (!EXPECT(GC_thr_initialized, TRUE)) + GC_thr_init(); + +@@ -407,6 +408,19 @@ GC_INNER void GC_push_all_stacks(void) + GC_push_all_stack_sections(lo, hi, p->traced_stack_sect); + } + if (altstack_lo) { ++ // When a thread goes into a coroutine, we lose its original sp until ++ // control flow returns to the thread. ++ // While in the coroutine, the sp points outside the thread stack, ++ // so we can detect this and push the entire thread stack instead, ++ // as an approximation. ++ // We assume that the coroutine has similarly added its entire stack. ++ // This could be made accurate by cooperating with the application ++ // via new functions and/or callbacks. ++ stack_limit = pthread_get_stacksize_np(p->id); ++ if (altstack_lo >= altstack_hi || altstack_lo < altstack_hi - stack_limit) { // sp outside stack ++ altstack_lo = altstack_hi - stack_limit; ++ } ++ + total_size += altstack_hi - altstack_lo; + GC_push_all_stack(altstack_lo, altstack_hi); + } +diff --git a/pthread_stop_world.c b/pthread_stop_world.c +index 4b2c429..1fb4c52 100644 +--- a/pthread_stop_world.c ++++ b/pthread_stop_world.c +@@ -673,6 +673,8 @@ GC_INNER void GC_push_all_stacks(void) + struct GC_traced_stack_sect_s *traced_stack_sect; + pthread_t self = pthread_self(); + word total_size = 0; ++ size_t stack_limit; ++ pthread_attr_t pattr; + + if (!EXPECT(GC_thr_initialized, TRUE)) + GC_thr_init(); +@@ -722,6 +724,31 @@ GC_INNER void GC_push_all_stacks(void) + hi = p->altstack + p->altstack_size; + /* FIXME: Need to scan the normal stack too, but how ? */ + /* FIXME: Assume stack grows down */ ++ } else { ++ if (pthread_getattr_np(p->id, &pattr)) { ++ ABORT("GC_push_all_stacks: pthread_getattr_np failed!"); ++ } ++ if (pthread_attr_getstacksize(&pattr, &stack_limit)) { ++ ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!"); ++ } ++ if (pthread_attr_destroy(&pattr)) { ++ ABORT("GC_push_all_stacks: pthread_attr_destroy failed!"); ++ } ++ // When a thread goes into a coroutine, we lose its original sp until ++ // control flow returns to the thread. ++ // While in the coroutine, the sp points outside the thread stack, ++ // so we can detect this and push the entire thread stack instead, ++ // as an approximation. ++ // We assume that the coroutine has similarly added its entire stack. ++ // This could be made accurate by cooperating with the application ++ // via new functions and/or callbacks. ++ #ifndef STACK_GROWS_UP ++ if (lo >= hi || lo < hi - stack_limit) { // sp outside stack ++ lo = hi - stack_limit; ++ } ++ #else ++ #error "STACK_GROWS_UP not supported in boost_coroutine2 (as of june 2021), so we don't support it in Nix." ++ #endif + } + GC_push_all_stack_sections(lo, hi, traced_stack_sect); + # ifdef STACK_GROWS_UP From 8270dccf60948fe8a6b6817c81d67eee98c07df3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 24 Jan 2023 14:57:18 +0100 Subject: [PATCH 036/179] Actually complete the revert --- flake.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 573373c42..773e693ba 100644 --- a/flake.nix +++ b/flake.nix @@ -128,9 +128,14 @@ }); propagatedDeps = - [ (boehmgc.override { + [ ((boehmgc.override { enableLargeConfig = true; + }).overrideAttrs(o: { + patches = (o.patches or []) ++ [ + ./boehmgc-coroutine-sp-fallback.diff + ]; }) + ) nlohmann_json ]; }; From 46054f932b6a7dceef127389f83bf7438d1731d0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 24 Jan 2023 15:11:55 +0100 Subject: [PATCH 037/179] Update boehmgc-coroutine-sp-fallback.diff --- boehmgc-coroutine-sp-fallback.diff | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff index 8fdafbecb..2826486fb 100644 --- a/boehmgc-coroutine-sp-fallback.diff +++ b/boehmgc-coroutine-sp-fallback.diff @@ -31,19 +31,19 @@ index 3dbaa3fb..36a1d1f7 100644 GC_push_all_stack(altstack_lo, altstack_hi); } diff --git a/pthread_stop_world.c b/pthread_stop_world.c -index 4b2c429..1fb4c52 100644 +index b5d71e62..aed7b0bf 100644 --- a/pthread_stop_world.c +++ b/pthread_stop_world.c -@@ -673,6 +673,8 @@ GC_INNER void GC_push_all_stacks(void) - struct GC_traced_stack_sect_s *traced_stack_sect; - pthread_t self = pthread_self(); - word total_size = 0; +@@ -768,6 +768,8 @@ STATIC void GC_restart_handler(int sig) + /* world is stopped. Should not fail if it isn't. */ + GC_INNER void GC_push_all_stacks(void) + { + size_t stack_limit; + pthread_attr_t pattr; - - if (!EXPECT(GC_thr_initialized, TRUE)) - GC_thr_init(); -@@ -722,6 +724,31 @@ GC_INNER void GC_push_all_stacks(void) + GC_bool found_me = FALSE; + size_t nthreads = 0; + int i; +@@ -851,6 +853,31 @@ GC_INNER void GC_push_all_stacks(void) hi = p->altstack + p->altstack_size; /* FIXME: Need to scan the normal stack too, but how ? */ /* FIXME: Assume stack grows down */ From 734c5fdcd62ead1b28e7f7fa3962ce0b542d5846 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Tue, 24 Jan 2023 16:37:50 +0100 Subject: [PATCH 038/179] Fix 'destructor called on non-final ...' warning clangStdenv compiles with a single warning: ``` warning: destructor called on non-final 'nix::PosAdapter' that has virtual functions but non-virtual destructor [-Wdelete-non-abstract-non-virtual-dtor] ``` This fixes the warning by making the destructor of PosAdapter virtual, deffering to the correct destructor from the concrete child classes. This has no impact in the end, as none of these classes have specific destructors. Technicaly, it may be faster not to have this indirection, but as per the warning, there is only one place where we have to delete abstract PosAdapter values. Not worth bikesheding I guess. --- src/libutil/error.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 7d236028c..0ebeaba61 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -74,6 +74,8 @@ struct AbstractPos virtual void print(std::ostream & out) const = 0; std::optional getCodeLines() const; + + virtual ~AbstractPos() = default; }; std::ostream & operator << (std::ostream & str, const AbstractPos & pos); From 816031173c8624f2c0b4fec6dc97f3b05a7132b7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 24 Jan 2023 18:53:46 -0500 Subject: [PATCH 039/179] Fix the 2.13 changelog It is just the new CLI that gets the `^` syntax. The old CLI already has a (slightly different) `!` syntax. Fixes #7682 --- doc/manual/src/release-notes/rl-2.13.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-2.13.md b/doc/manual/src/release-notes/rl-2.13.md index 2ebf19f60..d8db57a48 100644 --- a/doc/manual/src/release-notes/rl-2.13.md +++ b/doc/manual/src/release-notes/rl-2.13.md @@ -25,11 +25,11 @@ * Allow explicitly selecting outputs in a store derivation installable, just like we can do with other sorts of installables. For example, ```shell-session - # nix-build /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^dev + # nix build /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^dev ``` now works just as ```shell-session - # nix-build glibc^dev + # nix build glibc^dev ``` does already. From 75892710f833e4f5fa304eacfd0bfb45634702ec Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 24 Jan 2023 19:19:19 -0500 Subject: [PATCH 040/179] Fix the coverage job See https://hydra.nixos.org/build/206790960 --- flake.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flake.nix b/flake.nix index 62ea3cf00..2e9e73934 100644 --- a/flake.nix +++ b/flake.nix @@ -465,6 +465,10 @@ src = self; + configureFlags = [ + "CXXFLAGS=-I${lib.getDev pkgs.rapidcheck}/extras/gtest/include" + ]; + enableParallelBuilding = true; nativeBuildInputs = nativeBuildDeps; From f465e378c431e9efa84e03a0446a1380ceb18485 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 25 Jan 2023 08:58:41 -0500 Subject: [PATCH 041/179] Update doc/manual/src/release-notes/rl-2.13.md Co-authored-by: Eelco Dolstra --- doc/manual/src/release-notes/rl-2.13.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-2.13.md b/doc/manual/src/release-notes/rl-2.13.md index d8db57a48..168708113 100644 --- a/doc/manual/src/release-notes/rl-2.13.md +++ b/doc/manual/src/release-notes/rl-2.13.md @@ -29,7 +29,7 @@ ``` now works just as ```shell-session - # nix build glibc^dev + # nix build nixpkgs#glibc^dev ``` does already. From a96156c58f94165e8dc61981ab9fa6e97a963e96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sol=C3=A8ne=20Rapenne?= Date: Wed, 14 Dec 2022 14:01:29 +0100 Subject: [PATCH 042/179] warnings: enhance the case of untrusted substituter for untrusted user --- src/libstore/daemon.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 12596ba49..3731b21b8 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -222,7 +222,8 @@ struct ClientSettings else if (!hasSuffix(s, "/") && trusted.count(s + "/")) subs.push_back(s + "/"); else - warn("ignoring untrusted substituter '%s'", s); + warn("ignoring untrusted substituter '%s', you are not a trusted user.\n" + "More information about 'trusted-substituters' option in nix.conf man page", s); res = subs; return true; }; From 64951d9125fc223bbeb939b1c774533a8c6ded98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sol=C3=A8ne=20Rapenne?= Date: Tue, 3 Jan 2023 15:35:28 +0100 Subject: [PATCH 043/179] Update src/libstore/daemon.cc Co-authored-by: Valentin Gagarin --- src/libstore/daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 3731b21b8..e2a7dab35 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -223,7 +223,7 @@ struct ClientSettings subs.push_back(s + "/"); else warn("ignoring untrusted substituter '%s', you are not a trusted user.\n" - "More information about 'trusted-substituters' option in nix.conf man page", s); + "Run `man nix.conf` for more information on the `substituters` configuration option.", s); res = subs; return true; }; From 6b2729c81e1e0d37ba3680e36df4769d35d13c12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sol=C3=A8ne=20Rapenne?= Date: Fri, 20 Jan 2023 09:46:28 +0100 Subject: [PATCH 044/179] improve documentation about substituters and trusted users MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt --- src/libstore/globals.hh | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 7111def92..c3ccb5e11 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -570,11 +570,15 @@ public: {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, "trusted-public-keys", R"( - A whitespace-separated list of public keys. When paths are copied - from another Nix store (such as a binary cache), they must be - signed with one of these keys. For example: - `cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=`. + A whitespace-separated list of public keys. + + At least one of the following condition must be met + for Nix to accept copying a store object from another + Nix store (such as a substituter): + + - the store object has been signed using a key in the trusted keys list + - the [`require-sigs`](#conf-require-sigs) option has been set to `false` + - the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object) )", {"binary-cache-public-keys"}}; @@ -670,13 +674,14 @@ public: independently. Lower value means higher priority. The default is `https://cache.nixos.org`, with a Priority of 40. - Nix will copy a store path from a remote store only if one - of the following is true: + At least one of the following conditions must be met for Nix to use + a substituter: - - the store object is signed by one of the [`trusted-public-keys`](#conf-trusted-public-keys) - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list - - the [`require-sigs`](#conf-require-sigs) option has been set to `false` - - the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object) + - the user calling Nix is in the [`trusted-users`](#conf-trusted-users) list + + In addition, each store path should be trusted as described + in [`trusted-public-keys`](#conf-trusted-public-keys) )", {"binary-caches"}}; From 79c084cb598221f5910c429127e8da33cff6a206 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 27 Jan 2023 10:13:05 +0100 Subject: [PATCH 045/179] Add a test for `nix flake show` --- tests/flakes/common.sh | 4 ++++ tests/flakes/show.sh | 39 +++++++++++++++++++++++++++++++++++++++ tests/local.mk | 1 + 3 files changed, 44 insertions(+) create mode 100644 tests/flakes/show.sh diff --git a/tests/flakes/common.sh b/tests/flakes/common.sh index c333733c2..fdcdd4aef 100644 --- a/tests/flakes/common.sh +++ b/tests/flakes/common.sh @@ -20,6 +20,10 @@ writeSimpleFlake() { foo = import ./simple.nix; default = foo; }; + packages.someOtherSystem = rec { + foo = import ./simple.nix; + default = foo; + }; # To test "nix flake init". legacyPackages.x86_64-linux.hello = import ./simple.nix; diff --git a/tests/flakes/show.sh b/tests/flakes/show.sh new file mode 100644 index 000000000..995de8dc3 --- /dev/null +++ b/tests/flakes/show.sh @@ -0,0 +1,39 @@ +source ./common.sh + +flakeDir=$TEST_ROOT/flake +mkdir -p "$flakeDir" + +writeSimpleFlake "$flakeDir" +cd "$flakeDir" + + +# By default: Only show the packages content for the current system and no +# legacyPackages at all +nix flake show --json > show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.packages.someOtherSystem.default == {}; +assert show_output.packages.${builtins.currentSystem}.default.name == "simple"; +assert show_output.legacyPackages.${builtins.currentSystem} == {}; +true +' + +# With `--all-systems`, show the packages for all systems +nix flake show --json --all-systems > show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.packages.someOtherSystem.default.name == "simple"; +assert show_output.legacyPackages.${builtins.currentSystem} == {}; +true +' + +# With `--legacy`, show the legacy packages +nix flake show --json --legacy > show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.legacyPackages.${builtins.currentSystem}.hello.name == "simple"; +true +' diff --git a/tests/local.mk b/tests/local.mk index 5ac1ede32..c8ee82e1a 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -113,6 +113,7 @@ nix_tests = \ store-ping.sh \ fetchClosure.sh \ completions.sh \ + flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ toString-path.sh From 1cba5984a68a489c4a56691032e4c87991c678f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Wed, 25 Jan 2023 15:10:35 +0100 Subject: [PATCH 046/179] getDefaultNixPath: actually respect `{restrict,pure}-eval` Previously, getDefaultNixPath was called too early: at initialisation time, before CLI and config have been processed, when `restrictEval` and `pureEval` both have their default value `false`. Call it when initialising the EvalState instead, and use `setDefault`. --- src/libexpr/eval.cc | 36 +++++++++++++++++++++--------------- src/libexpr/eval.hh | 4 ++-- tests/nix_path.sh | 5 +++++ tests/restricted.sh | 3 +++ 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1828b8c2e..a48968656 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -519,6 +519,7 @@ EvalState::EvalState( static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); /* Initialise the Nix expression search path. */ + evalSettings.nixPath.setDefault(evalSettings.getDefaultNixPath()); if (!evalSettings.pureEval) { for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); @@ -2472,30 +2473,35 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { EvalSettings::EvalSettings() { - auto var = getEnv("NIX_PATH"); - if (var) nixPath = parseNixPath(*var); } +/* impure => NIX_PATH or a default path + * restrict-eval => NIX_PATH + * pure-eval => empty + */ Strings EvalSettings::getDefaultNixPath() { - Strings res; - auto add = [&](const Path & p, const std::string & s = std::string()) { - if (pathExists(p)) { - if (s.empty()) { - res.push_back(p); - } else { - res.push_back(s + "=" + p); - } - } - }; + if (pureEval) + return {}; + + auto var = getEnv("NIX_PATH"); + if (var) { + return parseNixPath(*var); + } else if (restrictEval) { + return {}; + } else { + Strings res; + auto add = [&](const Path & p, const std::optional & s = std::nullopt) { + if (pathExists(p)) + res.push_back(s ? *s + "=" + p : p); + }; - if (!evalSettings.restrictEval && !evalSettings.pureEval) { add(getHome() + "/.nix-defexpr/channels"); add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs"); add(settings.nixStateDir + "/profiles/per-user/root/channels"); - } - return res; + return res; + } } bool EvalSettings::isPseudoUrl(std::string_view s) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e4d5906bd..876a6ae0e 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -570,7 +570,7 @@ struct EvalSettings : Config { EvalSettings(); - static Strings getDefaultNixPath(); + Strings getDefaultNixPath(); static bool isPseudoUrl(std::string_view s); @@ -580,7 +580,7 @@ struct EvalSettings : Config "Whether builtin functions that allow executing native code should be enabled."}; Setting nixPath{ - this, getDefaultNixPath(), "nix-path", + this, {}, "nix-path", "List of directories to be searched for `<...>` file references."}; Setting restrictEval{ diff --git a/tests/nix_path.sh b/tests/nix_path.sh index 2b222b4a1..d16fb4bb2 100644 --- a/tests/nix_path.sh +++ b/tests/nix_path.sh @@ -12,3 +12,8 @@ nix-instantiate --eval -E '' --restrict-eval [[ $(nix-instantiate --find-file by-absolute-path/simple.nix) = $PWD/simple.nix ]] [[ $(nix-instantiate --find-file by-relative-path/simple.nix) = $PWD/simple.nix ]] + +unset NIX_PATH + +[[ $(nix-instantiate --option nix-path by-relative-path=. --find-file by-relative-path/simple.nix) = "$PWD/simple.nix" ]] +[[ $(NIX_PATH= nix-instantiate --option nix-path by-relative-path=. --find-file by-relative-path/simple.nix) = "$PWD/simple.nix" ]] diff --git a/tests/restricted.sh b/tests/restricted.sh index 9bd16cf51..3b6ee2af1 100644 --- a/tests/restricted.sh +++ b/tests/restricted.sh @@ -17,6 +17,9 @@ nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel' (! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ') nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ' -I src=. +# no default NIX_PATH +(unset NIX_PATH; ! nix-instantiate --restrict-eval --find-file .) + p=$(nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)") cmp $p restricted.sh From dba9173a1d8cb1dd40e5922d009cb3a434e081c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 27 Jan 2023 15:25:07 +0100 Subject: [PATCH 047/179] Document default `nix-path` value --- src/libexpr/eval.hh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 876a6ae0e..2340ef67b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -581,7 +581,14 @@ struct EvalSettings : Config Setting nixPath{ this, {}, "nix-path", - "List of directories to be searched for `<...>` file references."}; + R"( + List of directories to be searched for `<...>` file references. + + If [pure evaluation](#conf-pure-eval) is disabled, + this is initialised using the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) + environment variable, or, if it is unset and [restricted evaluation](#conf-restrict-eval) + is disabled, a default search path including the user's and `root`'s channels. + )"}; Setting restrictEval{ this, false, "restrict-eval", From 02e81cdf6276dbaaa5ecb720fb3244d5cf2c5e8e Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 27 Jan 2023 23:59:48 +0100 Subject: [PATCH 048/179] apply showAllSystems to legacyPackages as well --- src/nix/flake.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 2e4d2cf3a..d2f68c37c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1121,6 +1121,12 @@ struct CmdFlakeShow : FlakeCommand, MixJSON else { logger->warn(fmt("%s omitted (use '--legacy' to show)", concatStringsSep(".", attrPathS))); } + } else if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { + if (!json) + logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); + else { + logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); + } } else { if (visitor.isDerivation()) showDerivation(); From 6772e9ed0ac7853cefdeb6b354c81cba531829f2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 29 Jan 2023 12:31:34 -0500 Subject: [PATCH 049/179] Allow programs to have custom names Logic modeled after that for libraries. --- mk/programs.mk | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mk/programs.mk b/mk/programs.mk index 0fc1990f7..204409332 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -3,6 +3,9 @@ programs-list := # Build a program with symbolic name $(1). The program is defined by # various variables prefixed by ‘$(1)_’: # +# - $(1)_NAME: the name of the program (e.g. ‘foo’); defaults to +# $(1). +# # - $(1)_DIR: the directory where the (non-installed) program will be # placed. # @@ -23,11 +26,12 @@ programs-list := # - $(1)_INSTALL_DIR: the directory where the program will be # installed; defaults to $(bindir). define build-program + $(1)_NAME ?= $(1) _d := $(buildprefix)$$($(1)_DIR) _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs)))) _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH)) - $(1)_PATH := $$(_d)/$(1) + $(1)_PATH := $$(_d)/$$($(1)_NAME) $$(eval $$(call create-dir, $$(_d))) @@ -38,7 +42,7 @@ define build-program ifdef $(1)_INSTALL_DIR - $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1) + $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME) $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) From ec0c0efec6d7eed868b424434748b97651e02bcc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 29 Jan 2023 12:37:32 -0500 Subject: [PATCH 050/179] Allow unit test infra to be reused across libs' tests This allows using Arbitrary "instances" defined in libstore-tests in libexpr-tests, something we will leverage in a moment. --- .gitignore | 6 +++--- src/libexpr/tests/error_traces.cc | 2 +- src/libexpr/tests/json.cc | 2 +- .../tests/{libexprtests.hh => libexpr.hh} | 8 ++++---- src/libexpr/tests/local.mk | 4 +++- src/libexpr/tests/primops.cc | 2 +- src/libexpr/tests/trivial.cc | 2 +- src/libexpr/tests/value/context.cc | 2 +- .../tests/{libstoretests.hh => libstore.hh} | 0 src/libstore/tests/local.mk | 18 ++++++++++++++++-- src/libstore/tests/path.cc | 8 ++------ src/libstore/tests/path.hh | 15 +++++++++++++++ src/libutil/tests/local.mk | 2 ++ 13 files changed, 50 insertions(+), 21 deletions(-) rename src/libexpr/tests/{libexprtests.hh => libexpr.hh} (95%) rename src/libstore/tests/{libstoretests.hh => libstore.hh} (100%) create mode 100644 src/libstore/tests/path.hh diff --git a/.gitignore b/.gitignore index 8e0db013f..47815fde6 100644 --- a/.gitignore +++ b/.gitignore @@ -37,14 +37,14 @@ perl/Makefile.config /src/libexpr/parser-tab.hh /src/libexpr/parser-tab.output /src/libexpr/nix.tbl -/src/libexpr/tests/libexpr-tests +/src/libexpr/tests/libnixexpr-tests # /src/libstore/ *.gen.* -/src/libstore/tests/libstore-tests +/src/libstore/tests/libnixstore-tests # /src/libutil/ -/src/libutil/tests/libutil-tests +/src/libutil/tests/libnixutil-tests /src/nix/nix diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc index 5e2213f69..24e95ac39 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/src/libexpr/tests/error_traces.cc @@ -1,7 +1,7 @@ #include #include -#include "libexprtests.hh" +#include "tests/libexpr.hh" namespace nix { diff --git a/src/libexpr/tests/json.cc b/src/libexpr/tests/json.cc index f1ea1b197..411bc0ac3 100644 --- a/src/libexpr/tests/json.cc +++ b/src/libexpr/tests/json.cc @@ -1,4 +1,4 @@ -#include "libexprtests.hh" +#include "tests/libexpr.hh" #include "value-to-json.hh" namespace nix { diff --git a/src/libexpr/tests/libexprtests.hh b/src/libexpr/tests/libexpr.hh similarity index 95% rename from src/libexpr/tests/libexprtests.hh rename to src/libexpr/tests/libexpr.hh index 03e468fbb..8534d9567 100644 --- a/src/libexpr/tests/libexprtests.hh +++ b/src/libexpr/tests/libexpr.hh @@ -7,18 +7,19 @@ #include "eval-inline.hh" #include "store-api.hh" +#include "tests/libstore.hh" namespace nix { - class LibExprTest : public ::testing::Test { + class LibExprTest : public LibStoreTest { public: static void SetUpTestSuite() { - initLibStore(); + LibStoreTest::SetUpTestSuite(); initGC(); } protected: LibExprTest() - : store(openStore("dummy://")) + : LibStoreTest() , state({}, store) { } @@ -36,7 +37,6 @@ namespace nix { return state.symbols.create(value); } - ref store; EvalState state; }; diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk index e483575a4..6b766d82a 100644 --- a/src/libexpr/tests/local.mk +++ b/src/libexpr/tests/local.mk @@ -2,6 +2,8 @@ check: libexpr-tests_RUN programs += libexpr-tests +libexpr-tests_NAME := libnixexpr-tests + libexpr-tests_DIR := $(d) libexpr-tests_INSTALL_DIR := @@ -12,6 +14,6 @@ libexpr-tests_SOURCES := \ libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -libexpr-tests_LIBS = libexpr libutil libstore libfetchers +libexpr-tests_LIBS = libstore-tests libexpr libutil libstore libfetchers libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock diff --git a/src/libexpr/tests/primops.cc b/src/libexpr/tests/primops.cc index 9cdcf64a1..e1d3ac503 100644 --- a/src/libexpr/tests/primops.cc +++ b/src/libexpr/tests/primops.cc @@ -1,7 +1,7 @@ #include #include -#include "libexprtests.hh" +#include "tests/libexpr.hh" namespace nix { class CaptureLogger : public Logger diff --git a/src/libexpr/tests/trivial.cc b/src/libexpr/tests/trivial.cc index 8ce276e52..171727ac7 100644 --- a/src/libexpr/tests/trivial.cc +++ b/src/libexpr/tests/trivial.cc @@ -1,4 +1,4 @@ -#include "libexprtests.hh" +#include "tests/libexpr.hh" namespace nix { // Testing of trivial expressions diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index d5c9d3bce..0bcd79625 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -1,6 +1,6 @@ #include "value/context.hh" -#include "libexprtests.hh" +#include "tests/libexpr.hh" namespace nix { diff --git a/src/libstore/tests/libstoretests.hh b/src/libstore/tests/libstore.hh similarity index 100% rename from src/libstore/tests/libstoretests.hh rename to src/libstore/tests/libstore.hh diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk index a2cf8a0cf..c24712ccf 100644 --- a/src/libstore/tests/local.mk +++ b/src/libstore/tests/local.mk @@ -1,6 +1,20 @@ -check: libstore-tests_RUN +check: libstore-tests-exe_RUN -programs += libstore-tests +programs += libstore-tests-exe + +libstore-tests-exe_NAME = libnixstore-tests + +libstore-tests-exe_DIR := $(d) + +libstore-tests-exe_INSTALL_DIR := + +libstore-tests-exe_LIBS = libstore-tests + +libstore-tests-exe_LDFLAGS := $(GTEST_LIBS) + +libraries += libstore-tests + +libstore-tests_NAME = libnixstore-tests libstore-tests_DIR := $(d) diff --git a/src/libstore/tests/path.cc b/src/libstore/tests/path.cc index 8ea252c92..2fc6d7e48 100644 --- a/src/libstore/tests/path.cc +++ b/src/libstore/tests/path.cc @@ -7,7 +7,8 @@ #include "path-regex.hh" #include "store-api.hh" -#include "libstoretests.hh" +#include "tests/libstore.hh" +#include "tests/path.hh" namespace nix { @@ -73,11 +74,6 @@ void showValue(const StorePath & p, std::ostream & os) { namespace rc { using namespace nix; -template<> -struct Arbitrary { - static Gen arbitrary(); -}; - Gen Arbitrary::arbitrary() { auto len = *gen::inRange(1, StorePath::MaxPathLen); diff --git a/src/libstore/tests/path.hh b/src/libstore/tests/path.hh new file mode 100644 index 000000000..0ac9bb9f2 --- /dev/null +++ b/src/libstore/tests/path.hh @@ -0,0 +1,15 @@ +#pragma once + +#include + +#include "path.hh" + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index 815e18560..4f2a3f294 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -2,6 +2,8 @@ check: libutil-tests_RUN programs += libutil-tests +libutil-tests_NAME := libnixutil-tests + libutil-tests_DIR := $(d) libutil-tests_INSTALL_DIR := From ecd3e4ebd777dca694d0c60431641f054ec2d1ce Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 29 Jan 2023 13:52:38 -0500 Subject: [PATCH 051/179] More property tests Also put proper comparison methods on `DerivedPath` and `NixStringContextElem`, which is needed for the tests but good in general. --- src/libexpr/tests/local.mk | 2 +- src/libexpr/tests/value/context.cc | 42 +++++++++++++++++++++++- src/libexpr/tests/value/context.hh | 15 +++++++++ src/libexpr/value/context.hh | 17 +++++++--- src/libstore/derived-path.hh | 11 ++++--- src/libstore/tests/derived-path.cc | 52 ++++++++++++++++++++++++++++++ src/libstore/tests/derived-path.hh | 18 +++++++++++ src/libstore/tests/local.mk | 2 +- src/libstore/tests/outputs-spec.cc | 32 ++++++++++++++++++ src/libstore/tests/outputs-spec.hh | 17 ++++++++++ src/libstore/tests/path.cc | 23 ++++++++++--- src/libstore/tests/path.hh | 17 ++++++++-- src/libutil/tests/hash.cc | 22 ++++++++++++- src/libutil/tests/hash.hh | 15 +++++++++ src/libutil/tests/local.mk | 18 +++++++++-- 15 files changed, 279 insertions(+), 24 deletions(-) create mode 100644 src/libexpr/tests/value/context.hh create mode 100644 src/libstore/tests/derived-path.cc create mode 100644 src/libstore/tests/derived-path.hh create mode 100644 src/libstore/tests/outputs-spec.hh create mode 100644 src/libutil/tests/hash.hh diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk index 6b766d82a..3e5504f71 100644 --- a/src/libexpr/tests/local.mk +++ b/src/libexpr/tests/local.mk @@ -14,6 +14,6 @@ libexpr-tests_SOURCES := \ libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -libexpr-tests_LIBS = libstore-tests libexpr libutil libstore libfetchers +libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index 0bcd79625..75934ab31 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -1,6 +1,10 @@ -#include "value/context.hh" +#include +#include +#include +#include "tests/path.hh" #include "tests/libexpr.hh" +#include "tests/value/context.hh" namespace nix { @@ -70,3 +74,39 @@ TEST_F(NixStringContextElemTest, built) { } } + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, 2)) { + case 0: + return gen::just((NixStringContextElem) NixStringContextElem::Opaque { + .path = *gen::arbitrary(), + }); + case 1: + return gen::just((NixStringContextElem) NixStringContextElem::DrvDeep { + .drvPath = *gen::arbitrary(), + }); + default: + return gen::just((NixStringContextElem) NixStringContextElem::Built { + .drvPath = *gen::arbitrary(), + .output = (*gen::arbitrary()).name, + }); + } +} + +} + +namespace nix { + +RC_GTEST_FIXTURE_PROP( + NixStringContextElemTest, + prop_round_rip, + (const NixStringContextElem & o)) +{ + RC_ASSERT(o == NixStringContextElem::parse(store(), o.to_string(store()))); +} + +} diff --git a/src/libexpr/tests/value/context.hh b/src/libexpr/tests/value/context.hh new file mode 100644 index 000000000..83505962f --- /dev/null +++ b/src/libexpr/tests/value/context.hh @@ -0,0 +1,15 @@ +#pragma once + +#include + +#include + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index d8008c436..721563cba 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -1,9 +1,10 @@ #pragma once #include "util.hh" +#include "comparator.hh" #include "path.hh" -#include +#include #include @@ -31,7 +32,9 @@ class Store; Encoded as just the path: ‘’. */ struct NixStringContextElem_Opaque { - StorePath path; + StorePath path; + + GENERATE_CMP(NixStringContextElem_Opaque, me->path); }; /* Path to a derivation and its entire build closure. @@ -43,7 +46,9 @@ struct NixStringContextElem_Opaque { Encoded in the form ‘=’. */ struct NixStringContextElem_DrvDeep { - StorePath drvPath; + StorePath drvPath; + + GENERATE_CMP(NixStringContextElem_DrvDeep, me->drvPath); }; /* Derivation output. @@ -51,8 +56,10 @@ struct NixStringContextElem_DrvDeep { Encoded in the form ‘!!’. */ struct NixStringContextElem_Built { - StorePath drvPath; - std::string output; + StorePath drvPath; + std::string output; + + GENERATE_CMP(NixStringContextElem_Built, me->drvPath, me->output); }; using _NixStringContextElem_Raw = std::variant< diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 4edff7467..9e0cce377 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -4,8 +4,9 @@ #include "path.hh" #include "realisation.hh" #include "outputs-spec.hh" +#include "comparator.hh" -#include +#include #include @@ -27,8 +28,7 @@ struct DerivedPathOpaque { std::string to_string(const Store & store) const; static DerivedPathOpaque parse(const Store & store, std::string_view); - bool operator < (const DerivedPathOpaque & b) const - { return path < b.path; } + GENERATE_CMP(DerivedPathOpaque, me->path); }; /** @@ -51,8 +51,7 @@ struct DerivedPathBuilt { static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view); nlohmann::json toJSON(ref store) const; - bool operator < (const DerivedPathBuilt & b) const - { return std::make_pair(drvPath, outputs) < std::make_pair(b.drvPath, b.outputs); } + GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs); }; using _DerivedPathRaw = std::variant< @@ -96,6 +95,8 @@ struct BuiltPathBuilt { nlohmann::json toJSON(ref store) const; static BuiltPathBuilt parse(const Store & store, std::string_view); + + GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs); }; using _BuiltPathRaw = std::variant< diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc new file mode 100644 index 000000000..09887d1f0 --- /dev/null +++ b/src/libstore/tests/derived-path.cc @@ -0,0 +1,52 @@ +#include + +#include +#include +#include + +#include "tests/derived-path.hh" +#include "tests/libstore.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, 1)) { + case 0: + return gen::just((DerivedPath) DerivedPath::Opaque { + .path = *gen::arbitrary(), + }); + default: + return gen::just((DerivedPath) DerivedPath::Built { + .drvPath = *gen::arbitrary(), + .outputs = *gen::arbitrary(), + }); + } +} + +} + +namespace nix { + +class DerivedPathTest : public LibStoreTest +{ +}; + +// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is +// no a real fixture. +// +// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args +TEST_F(DerivedPathTest, force_init) +{ +} + +RC_GTEST_FIXTURE_PROP( + DerivedPathTest, + prop_round_rip, + (const DerivedPath & o)) +{ + RC_ASSERT(o == DerivedPath::parse(*store, o.to_string(*store))); +} + +} diff --git a/src/libstore/tests/derived-path.hh b/src/libstore/tests/derived-path.hh new file mode 100644 index 000000000..0161449ef --- /dev/null +++ b/src/libstore/tests/derived-path.hh @@ -0,0 +1,18 @@ +#pragma once + +#include + +#include + +#include "tests/path.hh" +#include "tests/outputs-spec.hh" + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk index c24712ccf..03becc7d1 100644 --- a/src/libstore/tests/local.mk +++ b/src/libstore/tests/local.mk @@ -24,6 +24,6 @@ libstore-tests_SOURCES := $(wildcard $(d)/*.cc) libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil -libstore-tests_LIBS = libstore libutil +libstore-tests_LIBS = libutil-tests libstore libutil libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libstore/tests/outputs-spec.cc b/src/libstore/tests/outputs-spec.cc index 06e4cabbd..984d1d963 100644 --- a/src/libstore/tests/outputs-spec.cc +++ b/src/libstore/tests/outputs-spec.cc @@ -2,6 +2,7 @@ #include #include +#include namespace nix { @@ -199,3 +200,34 @@ TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Expl #undef TEST_JSON } + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, 1)) { + case 0: + return gen::just((OutputsSpec) OutputsSpec::All { }); + default: + return gen::just((OutputsSpec) OutputsSpec::Names { + *gen::nonEmpty(gen::container(gen::map( + gen::arbitrary(), + [](StorePathName n) { return n.name; }))), + }); + } +} + +} + +namespace nix { + +RC_GTEST_PROP( + OutputsSpec, + prop_round_rip, + (const OutputsSpec & o)) +{ + RC_ASSERT(o == OutputsSpec::parse(o.to_string())); +} + +} diff --git a/src/libstore/tests/outputs-spec.hh b/src/libstore/tests/outputs-spec.hh new file mode 100644 index 000000000..2d455c817 --- /dev/null +++ b/src/libstore/tests/outputs-spec.hh @@ -0,0 +1,17 @@ +#pragma once + +#include + +#include + +#include + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff --git a/src/libstore/tests/path.cc b/src/libstore/tests/path.cc index 2fc6d7e48..430aa0099 100644 --- a/src/libstore/tests/path.cc +++ b/src/libstore/tests/path.cc @@ -7,6 +7,7 @@ #include "path-regex.hh" #include "store-api.hh" +#include "tests/hash.hh" #include "tests/libstore.hh" #include "tests/path.hh" @@ -74,12 +75,14 @@ void showValue(const StorePath & p, std::ostream & os) { namespace rc { using namespace nix; -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { - auto len = *gen::inRange(1, StorePath::MaxPathLen); + auto len = *gen::inRange( + 1, + StorePath::MaxPathLen - std::string_view { HASH_PART }.size()); - std::string pre { HASH_PART "-" }; - pre.reserve(pre.size() + len); + std::string pre; + pre.reserve(len); for (size_t c = 0; c < len; ++c) { switch (auto i = *gen::inRange(0, 10 + 2 * 26 + 6)) { @@ -114,7 +117,17 @@ Gen Arbitrary::arbitrary() } } - return gen::just(StorePath { pre }); + return gen::just(StorePathName { + .name = std::move(pre), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(StorePath { + *gen::arbitrary(), + (*gen::arbitrary()).name, + }); } } // namespace rc diff --git a/src/libstore/tests/path.hh b/src/libstore/tests/path.hh index 0ac9bb9f2..d7f1a8988 100644 --- a/src/libstore/tests/path.hh +++ b/src/libstore/tests/path.hh @@ -1,12 +1,25 @@ #pragma once -#include +#include -#include "path.hh" +#include + +namespace nix { + +struct StorePathName { + std::string name; +}; + +} namespace rc { using namespace nix; +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + template<> struct Arbitrary { static Gen arbitrary(); diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc index 412c03030..e4e928b3b 100644 --- a/src/libutil/tests/hash.cc +++ b/src/libutil/tests/hash.cc @@ -1,5 +1,12 @@ -#include "hash.hh" +#include + +#include #include +#include + +#include + +#include "tests/hash.hh" namespace nix { @@ -73,3 +80,16 @@ namespace nix { "c7d329eeb6dd26545e96e55b874be909"); } } + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + Hash hash(htSHA1); + for (size_t i = 0; i < hash.hashSize; ++i) + hash.hash[i] = *gen::arbitrary(); + return gen::just(hash); +} + +} diff --git a/src/libutil/tests/hash.hh b/src/libutil/tests/hash.hh new file mode 100644 index 000000000..9e9650e6e --- /dev/null +++ b/src/libutil/tests/hash.hh @@ -0,0 +1,15 @@ +#pragma once + +#include + +#include + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index 4f2a3f294..167915439 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -2,7 +2,19 @@ check: libutil-tests_RUN programs += libutil-tests -libutil-tests_NAME := libnixutil-tests +libutil-tests-exe_NAME = libnixutil-tests + +libutil-tests-exe_DIR := $(d) + +libutil-tests-exe_INSTALL_DIR := + +libutil-tests-exe_LIBS = libutil-tests + +libutil-tests-exe_LDFLAGS := $(GTEST_LIBS) + +libraries += libutil-tests + +libutil-tests_NAME = libnixutil-tests libutil-tests_DIR := $(d) @@ -10,8 +22,8 @@ libutil-tests_INSTALL_DIR := libutil-tests_SOURCES := $(wildcard $(d)/*.cc) -libutil-tests_CXXFLAGS += -I src/libutil -I src/libexpr +libutil-tests_CXXFLAGS += -I src/libutil libutil-tests_LIBS = libutil -libutil-tests_LDFLAGS := $(GTEST_LIBS) +libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) From 7408776b003d8a7670c5d8eb393f22899a662b25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 30 Jan 2023 10:32:23 +0100 Subject: [PATCH 052/179] Fix the release-notes Slightly butchered during the merge --- doc/manual/src/release-notes/rl-next.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 2f0f89cc6..7e8344e63 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,3 +1,5 @@ +# Release X.Y (202?-??-??) + * A new function `builtins.readFileType` is available. It is similar to `builtins.readDir` but acts on a single file or directory. @@ -6,7 +8,7 @@ discovered by making multiple syscalls. This change makes these operations lazy such that these lookups will only be performed if the attribute is used. This optimization affects a minority of filesystems and operating systems. - + * In derivations that use structured attributes, you can now use `unsafeDiscardReferences` to disable scanning a given output for runtime dependencies: ```nix @@ -17,4 +19,4 @@ their own embedded Nix store: hashes found inside such an image refer to the embedded store and not to the host's Nix store. - This requires the `discard-references` experimental feature. \ No newline at end of file + This requires the `discard-references` experimental feature. From bc6e65e26fd54f9c27a0344b86fc813820a7e9e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 30 Jan 2023 10:44:10 +0100 Subject: [PATCH 053/179] Fix the flakes/show test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don't hardcode “x86_64-linux” as this won't work too nicely on other platforms --- tests/flakes/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/flakes/common.sh b/tests/flakes/common.sh index fdcdd4aef..9d79080cd 100644 --- a/tests/flakes/common.sh +++ b/tests/flakes/common.sh @@ -26,7 +26,7 @@ writeSimpleFlake() { }; # To test "nix flake init". - legacyPackages.x86_64-linux.hello = import ./simple.nix; + legacyPackages.$system.hello = import ./simple.nix; }; } EOF From 3ac9f1658a482b3bf94b4a78263ea8e1ebea5b13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 30 Jan 2023 11:21:52 +0100 Subject: [PATCH 054/179] Fix the flakes init test Things leading to another... --- tests/flakes/init.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/flakes/init.sh b/tests/flakes/init.sh index 36cb9956a..2d4c77ba1 100644 --- a/tests/flakes/init.sh +++ b/tests/flakes/init.sh @@ -41,8 +41,8 @@ cat > $templatesDir/trivial/flake.nix < Date: Mon, 30 Jan 2023 14:49:45 +0100 Subject: [PATCH 055/179] doc: add `__structuredAttrs`, `outputChecks`, `unsafeDiscardReferences` --- .../src/language/advanced-attributes.md | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index f02425b13..a164b3997 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -255,3 +255,67 @@ Derivations can declare some infrequently used optional attributes. > substituted. Thus it is usually a good idea to align `system` with > `builtins.currentSystem` when setting `allowSubstitutes` to > `false`. For most trivial derivations this should be the case. + + - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ + If the special attribute `__structuredAttrs` is set to `true`, the other derivation + attributes are serialised in JSON format and made available to the + builder via the file `.attrs.json` in the builder’s temporary + directory. This obviates the need for `passAsFile` since JSON files + have no size restrictions, unlike process environments. + + It also makes it possible to tweak derivation settings in a structured way; see + [`outputChecks`](#adv-attr-outputChecks) for example. + + As a convenience to Bash builders, + Nix writes a script named `.attrs.sh` to the builder’s directory + that initialises shell variables corresponding to all attributes + that are representable in Bash. This includes non-nested + (associative) arrays. For example, the attribute `hardening.format = true` + ends up as the Bash associative array element `${hardening[format]}`. + + - [`outputChecks`]{#adv-attr-outputChecks}\ + When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks` + attribute allows defining checks per-output. + + In addition to + [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites), + [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), + the following attributes are available: + + - `maxSize` defines the maximum size of the output path. + - `maxClosureSize` defines the maximum size of the output's closure. + - `ignoreSelfRefs` controls whether self-references should be considered when + checking for allowed references/requisites. + + ```nix + __structuredAttrs = true; + + outputChecks.out = { + # The closure of 'out' must not be larger than 256 MiB. + maxClosureSize = 256 * 1024 * 1024; + + # It must not refer to the C compiler or to the 'dev' output. + disallowedRequisites = [ stdenv.cc "dev" ]; + }; + + outputChecks.dev = { + # The 'dev' output must not be larger than 128 KiB. + maxSize = 128 * 1024; + }; + ``` + + - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\ + When using [structured attributes](#adv-attr-structuredAttrs), the **experimental** + attribute `unsafeDiscardReferences` is a per-output boolean which, if set to `true`, + disables scanning the build output for runtime dependencies altogether. + + ```nix + __structuredAttrs = true; + unsafeDiscardReferences.out = true; + ``` + + This is useful, for example, when generating self-contained filesystem images with + their own embedded Nix store: hashes found inside such an image refer + to the embedded store and not to the host's Nix store. + + This is only allowed if the `discard-references` experimental feature is enabled. From f3e272ba02c3167b65a635389394f97a733440ca Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Jan 2023 09:35:25 -0500 Subject: [PATCH 056/179] Avoid some `StorePath` <-> `Path` round trips Avoid needless work and throwing away invariants. These conversions date back to when `StorePath` was in Rust and there were issues with it missing utility methods. --- src/libstore/store-api.cc | 8 ++++---- src/nix/verify.cc | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5130409d4..31bd9318f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -742,13 +742,13 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const Path & path) { + auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future> fut) { + queryPathInfo(path, {[path, this, &state_, &wakeup](std::future> fut) { auto state(state_.lock()); try { auto info = fut.get(); - state->valid.insert(parseStorePath(path)); + state->valid.insert(path); } catch (InvalidPath &) { } catch (...) { state->exc = std::current_exception(); @@ -760,7 +760,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m }; for (auto & path : paths) - pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME + pool.enqueue(std::bind(doQuery, path)); pool.process(); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index efa2434dc..0b306cc11 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -81,14 +81,14 @@ struct CmdVerify : StorePathsCommand ThreadPool pool; - auto doPath = [&](const Path & storePath) { + auto doPath = [&](const StorePath & storePath) { try { checkInterrupt(); MaintainCount> mcActive(active); update(); - auto info = store->queryPathInfo(store->parseStorePath(storePath)); + auto info = store->queryPathInfo(storePath); // Note: info->path can be different from storePath // for binary cache stores when using --all (since we @@ -173,7 +173,7 @@ struct CmdVerify : StorePathsCommand }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); + pool.enqueue(std::bind(doPath, storePath)); pool.process(); From 974a983351283a644228b10731e4f9d2ff01e533 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Jan 2023 09:59:55 -0500 Subject: [PATCH 057/179] Shrink diff in two places Stuff crept in there. --- src/libstore/content-address.cc | 3 ++- src/libstore/make-content-addressed.cc | 14 ++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a98e34cb8..a51646d0f 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -17,8 +17,9 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m) return ""; case FileIngestionMethod::Recursive: return "r:"; + default: + throw Error("impossible, caught both cases"); } - assert(false); } std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 3ee64c77a..ff9f5cdaa 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -31,12 +31,14 @@ std::map makeContentAddressed( for (auto & ref : oldInfo->references) { if (ref == path) refs.self = true; - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) { - rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); - refs.others.insert(std::move(replacement)); + else { + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) { + rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); + refs.others.insert(std::move(replacement)); + } } } From 02e745ba5b3dca3b53d88a49d86f5b22d047a08d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Jan 2023 10:12:44 -0500 Subject: [PATCH 058/179] Separate `path.hh` from `content-address.hh` It is good to separate concerns; `StorePath` (in general) has nothing to do with `ContentAddress` anyways. This reduces the diff from #3746. --- src/libstore/path.hh | 5 ++--- src/libstore/realisation.hh | 3 +++ src/libstore/store-api.hh | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 6a8f027f9..1e5579b90 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,6 +1,7 @@ #pragma once -#include "content-address.hh" +#include + #include "types.hh" namespace nix { @@ -66,8 +67,6 @@ public: typedef std::set StorePathSet; typedef std::vector StorePaths; -typedef std::map> StorePathCAMap; - /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 62561fce3..48d0283de 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -1,5 +1,8 @@ #pragma once +#include + +#include "hash.hh" #include "path.hh" #include #include "comparator.hh" diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5807392a7..4d8db3596 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -87,6 +87,8 @@ enum BuildMode { bmNormal, bmRepair, bmCheck }; struct BuildResult; +typedef std::map> StorePathCAMap; + struct StoreConfig : public Config { using Config::Config; From 560142fec0debefe7f983e60663dc3db4bdd79a9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Jan 2023 10:55:08 -0500 Subject: [PATCH 059/179] Make per-variant Arbitrary impls too This is a nice idea that @roberth requested. If we could factor our a generic `std::variant` impl as a follow-up it would be even better! --- src/libexpr/tests/value/context.cc | 35 +++++++++++++++++++++--------- src/libexpr/tests/value/context.hh | 15 +++++++++++++ src/libstore/tests/derived-path.cc | 24 ++++++++++++++------ src/libstore/tests/derived-path.hh | 10 +++++++++ 4 files changed, 67 insertions(+), 17 deletions(-) diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index 75934ab31..083359b7a 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -78,22 +78,37 @@ TEST_F(NixStringContextElemTest, built) { namespace rc { using namespace nix; +Gen Arbitrary::arbitrary() +{ + return gen::just(NixStringContextElem::Opaque { + .path = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(NixStringContextElem::DrvDeep { + .drvPath = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(NixStringContextElem::Built { + .drvPath = *gen::arbitrary(), + .output = (*gen::arbitrary()).name, + }); +} + Gen Arbitrary::arbitrary() { switch (*gen::inRange(0, 2)) { case 0: - return gen::just((NixStringContextElem) NixStringContextElem::Opaque { - .path = *gen::arbitrary(), - }); + return gen::just(*gen::arbitrary()); case 1: - return gen::just((NixStringContextElem) NixStringContextElem::DrvDeep { - .drvPath = *gen::arbitrary(), - }); + return gen::just(*gen::arbitrary()); default: - return gen::just((NixStringContextElem) NixStringContextElem::Built { - .drvPath = *gen::arbitrary(), - .output = (*gen::arbitrary()).name, - }); + return gen::just(*gen::arbitrary()); } } diff --git a/src/libexpr/tests/value/context.hh b/src/libexpr/tests/value/context.hh index 83505962f..54d21760e 100644 --- a/src/libexpr/tests/value/context.hh +++ b/src/libexpr/tests/value/context.hh @@ -7,6 +7,21 @@ namespace rc { using namespace nix; +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + template<> struct Arbitrary { static Gen arbitrary(); diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc index 09887d1f0..d1ac2c5e7 100644 --- a/src/libstore/tests/derived-path.cc +++ b/src/libstore/tests/derived-path.cc @@ -10,18 +10,28 @@ namespace rc { using namespace nix; +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Opaque { + .path = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Built { + .drvPath = *gen::arbitrary(), + .outputs = *gen::arbitrary(), + }); +} + Gen Arbitrary::arbitrary() { switch (*gen::inRange(0, 1)) { case 0: - return gen::just((DerivedPath) DerivedPath::Opaque { - .path = *gen::arbitrary(), - }); + return gen::just(*gen::arbitrary()); default: - return gen::just((DerivedPath) DerivedPath::Built { - .drvPath = *gen::arbitrary(), - .outputs = *gen::arbitrary(), - }); + return gen::just(*gen::arbitrary()); } } diff --git a/src/libstore/tests/derived-path.hh b/src/libstore/tests/derived-path.hh index 0161449ef..3bc812440 100644 --- a/src/libstore/tests/derived-path.hh +++ b/src/libstore/tests/derived-path.hh @@ -10,6 +10,16 @@ namespace rc { using namespace nix; +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + template<> struct Arbitrary { static Gen arbitrary(); From b26562c629604b76c58b789ee520581cc5549433 Mon Sep 17 00:00:00 2001 From: dramforever Date: Mon, 23 Jan 2023 23:06:57 +0800 Subject: [PATCH 060/179] InstallableFlake: Handle missing attr in getCursors Handle the case where none of getActualAttrPaths() actually exists, in which case instead of returning an empty vector. This fixes the case where the user misspells the attribute name in nix search. Instead of getting no search results, now it shows an error with suggestions. Also remove InstallableFlake::getCursor() override since it's now equivalent to the base class version. --- src/libcmd/installables.cc | 44 +++++++++++--------------------------- src/libcmd/installables.hh | 4 ---- 2 files changed, 13 insertions(+), 35 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 5090ea6d2..f8632a535 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -696,46 +696,28 @@ InstallableFlake::getCursors(EvalState & state) std::vector> res; - for (auto & attrPath : getActualAttrPaths()) { - auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); - if (attr) res.push_back(ref(*attr)); - } - - return res; -} - -ref InstallableFlake::getCursor(EvalState & state) -{ - auto lockedFlake = getLockedFlake(); - - auto cache = openEvalCache(state, lockedFlake); - auto root = cache->getRoot(); - Suggestions suggestions; - auto attrPaths = getActualAttrPaths(); for (auto & attrPath : attrPaths) { debug("trying flake output attribute '%s'", attrPath); - auto attrOrSuggestions = root->findAlongAttrPath( - parseAttrPath(state, attrPath), - true - ); - - if (!attrOrSuggestions) { - suggestions += attrOrSuggestions.getSuggestions(); - continue; + auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); + if (attr) { + res.push_back(ref(*attr)); + } else { + suggestions += attr.getSuggestions(); } - - return *attrOrSuggestions; } - throw Error( - suggestions, - "flake '%s' does not provide attribute %s", - flakeRef, - showAttrPaths(attrPaths)); + if (res.size() == 0) + throw Error( + suggestions, + "flake '%s' does not provide attribute %s", + flakeRef, + showAttrPaths(attrPaths)); + + return res; } std::shared_ptr InstallableFlake::getLockedFlake() const diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 3d12639b0..d489315df 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -198,10 +198,6 @@ struct InstallableFlake : InstallableValue std::vector> getCursors(EvalState & state) override; - /* Get a cursor to the first attrpath in getActualAttrPaths() that - exists, or throw an exception with suggestions if none exists. */ - ref getCursor(EvalState & state) override; - std::shared_ptr getLockedFlake() const; FlakeRef nixpkgsFlakeRef() const override; From 377d5eb38815d8290ab444c81c7826f46ba79844 Mon Sep 17 00:00:00 2001 From: dramforever Date: Mon, 30 Jan 2023 23:56:27 +0800 Subject: [PATCH 061/179] Installable::getCursors: Cleanup - Clarify doc comments, Installables::getCursors returns non-empty vector - Use vector::at in Installable::getCursor instead of checking for empty vector and throwing an exception with error message. --- src/libcmd/installables.cc | 7 +++---- src/libcmd/installables.hh | 8 ++++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index f8632a535..24f458f1a 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -379,10 +379,9 @@ Installable::getCursors(EvalState & state) ref Installable::getCursor(EvalState & state) { - auto cursors = getCursors(state); - if (cursors.empty()) - throw Error("cannot find flake attribute '%s'", what()); - return cursors[0]; + /* Although getCursors should return at least one element, in case it doesn't, + bound check to avoid an undefined behavior for vector[0] */ + return getCursors(state).at(0); } static StorePath getDeriver( diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index d489315df..da6a3addd 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -103,9 +103,13 @@ struct Installable return {}; } + /* Get a cursor to each value this Installable could refer to. However + if none exists, throw exception instead of returning empty vector. */ virtual std::vector> getCursors(EvalState & state); + /* Get the first and most preferred cursor this Installable could refer + to, or throw an exception if none exists. */ virtual ref getCursor(EvalState & state); @@ -193,8 +197,8 @@ struct InstallableFlake : InstallableValue std::pair toValue(EvalState & state) override; - /* Get a cursor to every attrpath in getActualAttrPaths() that - exists. */ + /* Get a cursor to every attrpath in getActualAttrPaths() + that exists. However if none exists, throw an exception. */ std::vector> getCursors(EvalState & state) override; From 6b779e4b07c42e5780feda5daf6dedc6c49e50d7 Mon Sep 17 00:00:00 2001 From: dramforever Date: Mon, 23 Jan 2023 23:07:34 +0800 Subject: [PATCH 062/179] Fix extra "." in CmdSearch::getDefaultFlakeAttrPaths No other getDefaultFlakeAttrPaths implementation has this trailing dot, and the dot can show up in error messages like: error: flake '...' does not provide attribute 'packages.x86_64-linux.', ... --- src/nix/search.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/search.cc b/src/nix/search.cc index d2a31607d..4fa1e7837 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -56,8 +56,8 @@ struct CmdSearch : InstallableCommand, MixJSON Strings getDefaultFlakeAttrPaths() override { return { - "packages." + settings.thisSystem.get() + ".", - "legacyPackages." + settings.thisSystem.get() + "." + "packages." + settings.thisSystem.get(), + "legacyPackages." + settings.thisSystem.get() }; } From e21aa43212a4326631ecb14a673b68f0b3bbc58e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Jan 2023 11:14:25 -0500 Subject: [PATCH 063/179] Delete dead code The references set seems to have been unused since `LegacySSHStore` references were first created in caa5793b4a74049ee37dd88eb1c5b785456ce40d. The method decls never were upstream, and accidentally added by me in 062533f7cdb74026096ca8c7d5b6e393893d59ef (probably due to `git rerere`). Sorry! This reduces the diff from #3746. --- src/libstore/legacy-ssh-store.cc | 1 - src/libstore/path-info.hh | 6 ------ 2 files changed, 7 deletions(-) diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index e1a4e13a3..2c9dd2680 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -134,7 +134,6 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor /* Hash will be set below. FIXME construct ValidPathInfo at end. */ auto info = std::make_shared(path, Hash::dummy); - PathSet references; auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index b4b54e593..a7fcbd232 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -81,12 +81,6 @@ struct ValidPathInfo /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf() const; - void insertReferencePossiblyToSelf(StorePath && ref); - void setReferencesPossiblyToSelf(StorePathSet && refs); - static const size_t maxSigs = std::numeric_limits::max(); /* Return the number of signatures on this .narinfo that were From 16fa8c2e0e1c03c5cdf759358574a7030bb0dbf7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Jan 2023 12:08:57 -0500 Subject: [PATCH 064/179] Fix configure.ac rapidcheck tests - `AC_LANG_PUSH(C++)` is needed for the header check - The library check is hopeless (without lots of third-party macros I don't feel like getting) because name mangling Pkg-config would make all this easier. I previously opened https://github.com/emil-e/rapidcheck/issues/302, I should write a PR too. --- configure.ac | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 0066bc389..09b3651b9 100644 --- a/configure.ac +++ b/configure.ac @@ -276,8 +276,11 @@ PKG_CHECK_MODULES([GTEST], [gtest_main]) # Look for rapidcheck. # No pkg-config yet, https://github.com/emil-e/rapidcheck/issues/302 +AC_LANG_PUSH(C++) AC_CHECK_HEADERS([rapidcheck/gtest.h], [], [], [#include ]) -AC_CHECK_LIB([rapidcheck], []) +dnl No good for C++ libs with mangled symbols +dnl AC_CHECK_LIB([rapidcheck], []) +AC_LANG_POP(C++) # Look for nlohmann/json. From c5d4c50abaa5977db7dd7da2112f9ac289d4171e Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Mon, 30 Jan 2023 23:34:35 -0800 Subject: [PATCH 065/179] manual: Document that the store timestamp is now 1, not 0 Commit 14bc3ce3d6d5745717fa19b8b43b5fdd117ff757 (0.13~43) changed the timestamps in the Nix store from 0 to 1. Update the nix-store man page to match. Signed-off-by: Anders Kaseorg --- doc/manual/src/command-ref/nix-store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index 403cf285d..31fdd7806 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -633,7 +633,7 @@ written to standard output. A NAR archive is like a TAR or Zip archive, but it contains only the information that Nix considers important. For instance, timestamps are -elided because all files in the Nix store have their timestamp set to 0 +elided because all files in the Nix store have their timestamp set to 1 anyway. Likewise, all permissions are left out except for the execute bit, because all files in the Nix store have 444 or 555 permission. From 28648ed78413c9a681501e37473cfebe39bedb1b Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Tue, 31 Jan 2023 13:24:23 +0100 Subject: [PATCH 066/179] nix store ping: add --json flag --- src/nix/ping-store.cc | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 3c3b7bb45..0d7a75075 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -2,9 +2,11 @@ #include "shared.hh" #include "store-api.hh" +#include + using namespace nix; -struct CmdPingStore : StoreCommand +struct CmdPingStore : StoreCommand, MixJSON { std::string description() override { @@ -20,10 +22,19 @@ struct CmdPingStore : StoreCommand void run(ref store) override { - notice("Store URL: %s", store->getUri()); - store->connect(); - if (auto version = store->getVersion()) - notice("Version: %s", *version); + if (!json) { + notice("Store URL: %s", store->getUri()); + store->connect(); + if (auto version = store->getVersion()) + notice("Version: %s", *version); + } else { + nlohmann::json res; + res["url"] = store->getUri(); + store->connect(); + if (auto version = store->getVersion()) + res["version"] = *version; + logger->cout("%s", res); + } } }; From 4f4a6074e41bb635bc35ac14000b11d341536650 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Tue, 31 Jan 2023 15:10:39 +0100 Subject: [PATCH 067/179] nix store ping: try to print json if connect() fails aswell --- src/nix/ping-store.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 0d7a75075..5c44510ab 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -1,6 +1,7 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" +#include "finally.hh" #include @@ -29,11 +30,13 @@ struct CmdPingStore : StoreCommand, MixJSON notice("Version: %s", *version); } else { nlohmann::json res; + Finally printRes([&]() { + logger->cout("%s", res); + }); res["url"] = store->getUri(); store->connect(); if (auto version = store->getVersion()) res["version"] = *version; - logger->cout("%s", res); } } }; From 4757b3f04e0a4568ab40ab86b9251945874a5662 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Tue, 31 Jan 2023 15:10:54 +0100 Subject: [PATCH 068/179] tests/store-ping: test nix store ping --json --- tests/store-ping.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/store-ping.sh b/tests/store-ping.sh index f9427cf0a..9846c7d3d 100644 --- a/tests/store-ping.sh +++ b/tests/store-ping.sh @@ -1,13 +1,17 @@ source common.sh STORE_INFO=$(nix store ping 2>&1) +STORE_INFO_JSON=$(nix store ping --json) echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then DAEMON_VERSION=$($NIX_DAEMON_PACKAGE/bin/nix-daemon --version | cut -d' ' -f3) echo "$STORE_INFO" | grep "Version: $DAEMON_VERSION" + [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]] fi expect 127 NIX_REMOTE=unix:$PWD/store nix store ping || \ fail "nix store ping on a non-existent store should fail" + +[[ "$(echo "$STORE_INFO_JSON" | jq -r ".url")" == "${NIX_REMOTE:-local}" ]] From 60d48eda2354dfa6d23ed3feb7b23ac25b34edcf Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 31 Jan 2023 18:16:31 +0100 Subject: [PATCH 069/179] nix flake show: Ignore empty attrsets For frameworks it's important that structures are as lazy as possible to prevent infinite recursions, performance issues and errors that aren't related to the thing to evaluate. As a consequence, they have to emit more attributes than strictly (sic) necessary. However, these attributes with empty values are not useful to the user so we omit them. --- src/nix/flake.cc | 62 +++++++++++++++++++++++++++++++++++++++++++- tests/flakes/show.sh | 27 +++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index d2f68c37c..c025bc7a6 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1002,6 +1002,61 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto flake = std::make_shared(lockFlake()); auto localSystem = std::string(settings.thisSystem.get()); + std::function &attrPath, + const Symbol &attr)> hasContent; + + // For frameworks it's important that structures are as lazy as possible + // to prevent infinite recursions, performance issues and errors that + // aren't related to the thing to evaluate. As a consequence, they have + // to emit more attributes than strictly (sic) necessary. + // However, these attributes with empty values are not useful to the user + // so we omit them. + hasContent = [&]( + eval_cache::AttrCursor & visitor, + const std::vector &attrPath, + const Symbol &attr) -> bool + { + auto attrPath2(attrPath); + attrPath2.push_back(attr); + auto attrPathS = state->symbols.resolve(attrPath2); + const auto & attrName = state->symbols[attr]; + + auto visitor2 = visitor.getAttr(attrName); + + if ((attrPathS[0] == "apps" + || attrPathS[0] == "checks" + || attrPathS[0] == "devShells" + || attrPathS[0] == "legacyPackages" + || attrPathS[0] == "packages") + && (attrPathS.size() == 1 || attrPathS.size() == 2)) { + for (const auto &subAttr : visitor2->getAttrs()) { + if (hasContent(*visitor2, attrPath2, subAttr)) { + return true; + } + } + return false; + } + + if ((attrPathS.size() == 1) + && (attrPathS[0] == "formatter" + || attrPathS[0] == "nixosConfigurations" + || attrPathS[0] == "nixosModules" + || attrPathS[0] == "overlays" + )) { + for (const auto &subAttr : visitor2->getAttrs()) { + if (hasContent(*visitor2, attrPath2, subAttr)) { + return true; + } + } + return false; + } + + // If we don't recognize it, it's probably content + return true; + }; + std::function & attrPath, @@ -1027,7 +1082,12 @@ struct CmdFlakeShow : FlakeCommand, MixJSON { if (!json) logger->cout("%s", headerPrefix); - auto attrs = visitor.getAttrs(); + std::vector attrs; + for (const auto &attr : visitor.getAttrs()) { + if (hasContent(visitor, attrPath, attr)) + attrs.push_back(attr); + } + for (const auto & [i, attr] : enumerate(attrs)) { const auto & attrName = state->symbols[attr]; bool last = i + 1 == attrs.size(); diff --git a/tests/flakes/show.sh b/tests/flakes/show.sh index 995de8dc3..dd13264b9 100644 --- a/tests/flakes/show.sh +++ b/tests/flakes/show.sh @@ -37,3 +37,30 @@ in assert show_output.legacyPackages.${builtins.currentSystem}.hello.name == "simple"; true ' + +# Test that attributes are only reported when they have actual content +cat >flake.nix < show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output == { }; +true +' From 0983a0bd3050d02659f7c58555b8cbcfffed2c3b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Feb 2023 10:04:28 -0500 Subject: [PATCH 070/179] Shrink diff in one place --- src/libstore/make-content-addressed.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index ff9f5cdaa..42de79226 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -35,10 +35,9 @@ std::map makeContentAddressed( auto i = remappings.find(ref); auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? - if (replacement != ref) { + if (replacement != ref) rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); - refs.others.insert(std::move(replacement)); - } + refs.others.insert(std::move(replacement)); } } From db759b1bc23c64b2aa6bdd0c5444a6d864488671 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Feb 2023 10:07:54 -0500 Subject: [PATCH 071/179] Undo style change `&` without space before is far more common on this codebase than I thought, so it is not worth changing just this one file. Maybe we will adopt a formatter someday but until then this is fine. --- src/libstore/misc.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 1c187535d..b28768459 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -21,16 +21,16 @@ void Store::computeFSClosure(const StorePathSet & startPaths, StorePathSet res; StorePathSet referrers; queryReferrers(path, referrers); - for (auto & ref : referrers) + for (auto& ref : referrers) if (ref != path) res.insert(ref); if (includeOutputs) - for (auto & i : queryValidDerivers(path)) + for (auto& i : queryValidDerivers(path)) res.insert(i); if (includeDerivers && path.isDerivation()) - for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); return res; @@ -40,12 +40,12 @@ void Store::computeFSClosure(const StorePathSet & startPaths, std::future> & fut) { StorePathSet res; auto info = fut.get(); - for (auto & ref : info->references) + for (auto& ref : info->references) if (ref != path) res.insert(ref); if (includeOutputs && path.isDerivation()) - for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); From 59d3175649a6bbdde76d1dfcf476c11392add827 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Feb 2023 10:09:25 -0500 Subject: [PATCH 072/179] Put back TODO I don't think the `narHash` is in need of documentation more than the other undocumented fields, but regardless this change has nothing to do with that field and so we should leave the comment as is. --- src/libstore/path-info.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 663d94540..35aced472 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -29,6 +29,7 @@ struct ValidPathInfo { StorePath path; std::optional deriver; + // TODO document this Hash narHash; StorePathSet references; time_t registrationTime = 0; From ee9eb83a842eb97d0180fd9d349d30ff27fdb485 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Feb 2023 11:25:56 -0500 Subject: [PATCH 073/179] Remove some designated initializers With the switch to C++20, the rules became more strict, and we can no longer initialize base classes. Make them comments instead. (BTW https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p2287r1.html this offers some new syntax for this use-case. Hopefully this will be adopted and we can eventually use it.) --- perl/lib/Nix/Store.xs | 2 +- src/libexpr/primops.cc | 4 ++-- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/tarball.cc | 2 +- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/content-address.cc | 4 ++-- src/libstore/local-store.cc | 2 +- src/libstore/make-content-addressed.cc | 2 +- src/libstore/path-info.cc | 4 ++-- src/libstore/store-api.cc | 4 ++-- src/nix-store/nix-store.cc | 2 +- src/nix/add-to-store.cc | 2 +- src/nix/prefetch.cc | 2 +- src/nix/profile.cc | 2 +- 16 files changed, 21 insertions(+), 21 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index f19fb20bf..314183383 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -299,7 +299,7 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) .method = method, .hash = h, }, - .references = {}, + /* .references = */ {}, }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8b54c4477..4e2f92276 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1287,7 +1287,7 @@ drvName, Bindings * attrs, Value & v) .method = method, .hash = h, }, - .references = {}, + /* .references = */ {}, }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", @@ -2103,7 +2103,7 @@ static void addPath( .method = method, .hash = *expectedHash, }, - .references = {}, + /* .references = */ {}, }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e194462e4..69395ad3d 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -240,7 +240,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, .hash = *expectedHash, }, - .references = {} + /* .references = */ {} }); if (state.store->isValidPath(expectedPath)) { diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 3936eadfe..dae4998f9 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -215,7 +215,7 @@ StorePath Input::computeStorePath(Store & store) const .method = FileIngestionMethod::Recursive, .hash = *narHash, }, - .references = {}, + /* .references = */ {}, }); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 302046610..b6f72bb1f 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -78,7 +78,7 @@ DownloadFileResult downloadFile( .method = FileIngestionMethod::Flat, .hash = hash, }, - .references = {}, + /* .references = */ {}, }, hashString(htSHA256, sink.s), }; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9cb0f74f6..5617e2c42 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -313,7 +313,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n .method = method, .hash = nar.first, }, - .references = { + /* .references = */ { .others = references, .self = false, }, @@ -431,7 +431,7 @@ StorePath BinaryCacheStore::addToStore( .method = method, .hash = h, }, - .references = { + /* .references = */ { .others = references, .self = false, }, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 8fdc9dce1..cfd5db3b4 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2498,7 +2498,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() .method = outputHash.method, .hash = got, }, - .references = rewriteRefs(), + /* .references = */ rewriteRefs(), }, Hash::dummy, }; diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a51646d0f..39a31f0a0 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -167,13 +167,13 @@ ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { [&](const TextHash & h) -> ContentAddressWithReferences { return TextInfo { h, - .references = {}, + /* .references = */ {}, }; }, [&](const FixedOutputHash & h) -> ContentAddressWithReferences { return FixedOutputInfo { h, - .references = {}, + /* .references = */ {}, }; }, }, ca); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c6f870dde..9d2ea7156 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1419,7 +1419,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name .method = method, .hash = hash, }, - .references = { + /* .references = */ { .others = references, .self = false, }, diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 42de79226..59a452918 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -56,7 +56,7 @@ std::map makeContentAddressed( .method = FileIngestionMethod::Recursive, .hash = narModuloHash, }, - .references = std::move(refs), + /* .references = */ std::move(refs), }, Hash::dummy, }; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 5944afd06..ff85b3932 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -31,7 +31,7 @@ std::optional ValidPathInfo::contentAddressWithRef assert(references.count(path) == 0); return TextInfo { th, - .references = references, + /* .references = */ references, }; }, [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { @@ -43,7 +43,7 @@ std::optional ValidPathInfo::contentAddressWithRef } return FixedOutputInfo { foh, - .references = { + /* .references = */ { .others = std::move(refs), .self = hasSelfReference, }, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3c0c26706..295ce4953 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -234,7 +234,7 @@ std::pair Store::computeStorePathForPath(std::string_view name, .method = method, .hash = h, }, - .references = {}, + /* .references = */ {}, }; return std::make_pair(makeFixedOutputPath(name, caInfo), h); } @@ -443,7 +443,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, .method = method, .hash = hash, }, - .references = {}, + /* .references = */ {}, }, narHash, }; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5b261ecc6..28ddf2f4a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -220,7 +220,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) .method = method, .hash = Hash::parseAny(hash, hashAlgo), }, - .references = {}, + /* .references = */ {}, }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 81dbc09a6..5de1aebfc 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -49,7 +49,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand .method = std::move(ingestionMethod), .hash = std::move(hash), }, - .references = {}, + /* .references = */ {}, }, narHash, }; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index df9933d29..bc270f66d 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -72,7 +72,7 @@ std::tuple prefetchFile( .method = ingestionMethod, .hash = *expectedHash, }, - .references = {}, + /* .references = */ {}, }); if (store->isValidPath(*storePath)) hash = expectedHash; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 345505532..e552e8975 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -206,7 +206,7 @@ struct ProfileManifest .method = FileIngestionMethod::Recursive, .hash = narHash, }, - .references = { + /* .references = */ { .others = std::move(references), .self = false, }, From 57a4258426260da9faf8c1ffd003edac330d50b8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Feb 2023 20:27:35 +0100 Subject: [PATCH 074/179] Remove an unused capture --- src/libstore/store-api.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 31bd9318f..601efa1cc 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -744,7 +744,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(path, {[path, this, &state_, &wakeup](std::future> fut) { + queryPathInfo(path, {[path, &state_, &wakeup](std::future> fut) { auto state(state_.lock()); try { auto info = fut.get(); From e8ca49f6efeb54a9e1ed624f658e6790a3121ac9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Feb 2023 20:34:44 +0100 Subject: [PATCH 075/179] Fix clang compilation --- src/libexpr/flake/lockfile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index a3ed90e1f..a74e68c9c 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -219,7 +219,7 @@ std::optional LockFile::isUnlocked() const visit(root); for (auto & i : nodes) { - if (i == root) continue; + if (i == ref(root)) continue; auto node = i.dynamic_pointer_cast(); if (node && !node->lockedRef.input.isLocked()) return node->lockedRef; From 32ca59649b0af8d8d77905274fe63de3a3fc0d1e Mon Sep 17 00:00:00 2001 From: Jamie Quigley Date: Wed, 1 Feb 2023 20:43:26 +0000 Subject: [PATCH 076/179] nix-shell: Colour the prompt red if the user is root This matches the nixos prompt colours - green for standard user, red for root --- src/nix-build/nix-build.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 049838bb1..da76c2ace 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -536,7 +536,9 @@ static void main_nix_build(int argc, char * * argv) "SHELL=%5%; " "BASH=%5%; " "set +e; " - R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s" + R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && )s" + + (getuid() == 0 ? R"s(PS1='\n\[\033[1;31m\][nix-shell:\w]\$\[\033[0m\] '; )s" + : R"s(PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s") + "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; " "unset NIX_ENFORCE_PURITY; " "shopt -u nullglob; " From 92edc3836904c2448387be11330a6e7c62593873 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Wed, 1 Feb 2023 20:05:56 -0500 Subject: [PATCH 077/179] Don't send plugin-files to the daemon. This is radically unsafe and the daemon has already loaded its plugins anyway. Fixes cachix/devenv#276 --- src/libstore/remote-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ff57a77ca..d1296627a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -266,6 +266,7 @@ void RemoteStore::setOptions(Connection & conn) overrides.erase(settings.useSubstitutes.name); overrides.erase(loggerSettings.showTrace.name); overrides.erase(settings.experimentalFeatures.name); + overrides.erase(settings.pluginFiles.name); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; From 895c525d04901022b7e0c4586ee12774fb4f8c6f Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 2 Feb 2023 06:03:45 -0500 Subject: [PATCH 078/179] daemon: Warn on old clients passing unexpected plugin-files. The setting itself was already ignored due to exception trying to set pluginFiles. --- src/libstore/daemon.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e2a7dab35..d6621ec0b 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -236,6 +236,10 @@ struct ClientSettings // the daemon, as that could cause some pretty weird stuff if (parseFeatures(tokenizeString(value)) != settings.experimentalFeatures.get()) debug("Ignoring the client-specified experimental features"); + } else if (name == settings.pluginFiles.name) { + if (tokenizeString(value) != settings.pluginFiles.get()) + warn("Ignoring the client-specified plugin-files.\n" + "The client specifying plugins to the daemon never made sense, and was removed in Nix >=2.14."); } else if (trusted || name == settings.buildTimeout.name From 51013da9211b4f18ca97b7194788121c92094bb1 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 28 Jan 2023 11:48:23 +0100 Subject: [PATCH 079/179] perl: run `initLibStore()` on `openStore()` Since #7478 it's mandatory that `initLibStore()` is called for store operations. However that's not the case when running `openStore()` in Perl using the perl-bindings. That breaks e.g. `hydra-eval-jobset` when built against Nix 2.13 which uses small portions of the store API. --- perl/lib/Nix/Store.xs | 1 + 1 file changed, 1 insertion(+) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 54ad1680c..de91dc28d 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -26,6 +26,7 @@ static ref store() static std::shared_ptr _store; if (!_store) { try { + initLibStore(); loadConfFile(); settings.lockCPU = false; _store = openStore(); From 479c0117840a5dc710019db006c5940b29d98dcc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 2 Feb 2023 12:02:03 -0500 Subject: [PATCH 080/179] Get rid of the `authHook` parameter on `processConnection` This is (morally) dead code. As @edolstra pointed out in https://github.com/NixOS/nix/pull/5226#discussion_r1073470813, this is no longer needed. I created this in 8d4162ff9e940ea9e2f97b07f3030a722695901a, so it is fitting that I now destroy it :). --- src/libstore/build/local-derivation-goal.cc | 3 +-- src/libstore/daemon.cc | 7 +------ src/libstore/daemon.hh | 7 +------ src/nix/daemon.cc | 11 ++--------- 4 files changed, 5 insertions(+), 23 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 572f71045..8ff83f748 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1516,8 +1516,7 @@ void LocalDerivationGoal::startDaemon() FdSink to(remote.get()); try { daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, - [&](Store & store) {}); + daemon::NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SysError &) { ignoreException(); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index d6621ec0b..d1f69fe2d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -985,8 +985,7 @@ void processConnection( FdSource & from, FdSink & to, TrustedFlag trusted, - RecursiveFlag recursive, - std::function authHook) + RecursiveFlag recursive) { auto monitor = !recursive ? std::make_unique(from.fd) : nullptr; @@ -1029,10 +1028,6 @@ void processConnection( try { - /* If we can't accept clientVersion, then throw an error - *here* (not above). */ - authHook(*store); - tunnelLogger->stopWork(); to.flush(); diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 67755d54e..8c765615c 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -13,11 +13,6 @@ void processConnection( FdSource & from, FdSink & to, TrustedFlag trusted, - RecursiveFlag recursive, - /* Arbitrary hook to check authorization / initialize user data / whatever - after the protocol has been negotiated. The idea is that this function - and everything it calls doesn't know about this stuff, and the - `nix-daemon` handles that instead. */ - std::function authHook); + RecursiveFlag recursive); } diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 19fbbf155..96568fb8f 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -241,14 +241,7 @@ static void daemonLoop() // Handle the connection. FdSource from(remote.get()); FdSink to(remote.get()); - processConnection(openUncachedStore(), from, to, trusted, NotRecursive, [&](Store & store) { -#if 0 - /* Prevent users from doing something very dangerous. */ - if (geteuid() == 0 && - querySetting("build-users-group", "") == "") - throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); -#endif - }); + processConnection(openUncachedStore(), from, to, trusted, NotRecursive); exit(0); }, options); @@ -301,7 +294,7 @@ static void runDaemon(bool stdio) /* Auth hook is empty because in this mode we blindly trust the standard streams. Limiting access to those is explicitly not `nix-daemon`'s responsibility. */ - processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); + processConnection(openUncachedStore(), from, to, Trusted, NotRecursive); } } else daemonLoop(); From a47e055e090a85884413dfc7889897f4e3d831fb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 2 Feb 2023 13:20:44 -0500 Subject: [PATCH 081/179] Move `trustedUsers` and `allowedUsers` to separate config struct These settings are not needed for libstore at all, they are just used by the nix daemon *command* for authorization on unix domain sockets. My moving them to a new configuration struct just in that file, we avoid them leaking anywhere else. Also, it is good to break up the mammoth `Settings` struct in general. Issue #5638 tracks this. The message is not changed because I do not want to regress in convenience to the user. Just saying "this connection is not trusted" doesn't tell them out to fix the issue. The ideal thing to do would be to somehow parameterize `processCommand` on how the error should be displayed, so different sorts of connections can display different information to the user based on how authentication is performed for the connection in question. This, however, is a good bit more work, so it is left for the future. This came up with me thinking about the tcp:// store (#5265). The larger project is not TCP *per se*, but the idea that it should be possible for something else to manage access control to services like the Nix Daemon, and those services simply trust or trust the incoming connection as they are told. This is a more capability-oriented way of thinking about trust than "every server implements its own auth separately" as we are used to today. Its very great that libstore itself already implements just this model, and so via this refactor I basically want to "enshrine" that so it continues to be the case. --- src/libstore/daemon.cc | 13 +++++++++++-- src/libstore/globals.hh | 34 ++-------------------------------- src/nix/daemon.cc | 41 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 52 insertions(+), 36 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index d6621ec0b..591693a64 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -529,7 +529,14 @@ static void performOp(TunnelLogger * logger, ref store, mode = (BuildMode) readInt(from); /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + clients. + + FIXME: layer violation in this message: the daemon code (i.e. + this file) knows whether a client/connection is trusted, but it + does not how how the client was authenticated. The mechanism + need not be getting the UID of the other end of a Unix Domain + Socket. + */ if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); } @@ -546,7 +553,9 @@ static void performOp(TunnelLogger * logger, ref store, mode = (BuildMode) readInt(from); /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + clients. + + FIXME: layer violation; see above. */ if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index c3ccb5e11..42981219d 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -279,8 +279,8 @@ public: If the build users group is empty, builds will be performed under the uid of the Nix process (that is, the uid of the caller if `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if - `NIX_REMOTE` is `daemon`). Obviously, this should not be used in - multi-user settings with untrusted users. + `NIX_REMOTE` is `daemon`). Obviously, this should not be used + with a nix daemon accessible to untrusted clients. Defaults to `nixbld` when running as root, *empty* otherwise. )", @@ -696,24 +696,6 @@ public: )", {"trusted-binary-caches"}}; - Setting trustedUsers{ - this, {"root"}, "trusted-users", - R"( - A list of names of users (separated by whitespace) that have - additional rights when connecting to the Nix daemon, such as the - ability to specify additional binary caches, or to import unsigned - NARs. You can also specify groups by prefixing them with `@`; for - instance, `@wheel` means all users in the `wheel` group. The default - is `root`. - - > **Warning** - > - > Adding a user to `trusted-users` is essentially equivalent to - > giving that user root access to the system. For example, the user - > can set `sandbox-paths` and thereby obtain read access to - > directories that are otherwise inacessible to them. - )"}; - Setting ttlNegativeNarInfoCache{ this, 3600, "narinfo-cache-negative-ttl", R"( @@ -736,18 +718,6 @@ public: mismatch if the build isn't reproducible. )"}; - /* ?Who we trust to use the daemon in safe ways */ - Setting allowedUsers{ - this, {"*"}, "allowed-users", - R"( - A list of names of users (separated by whitespace) that are allowed - to connect to the Nix daemon. As with the `trusted-users` option, - you can specify groups by prefixing them with `@`. Also, you can - allow all users by specifying `*`. The default is `*`. - - Note that trusted users are always allowed to connect. - )"}; - Setting printMissing{this, true, "print-missing", "Whether to print what paths need to be built or downloaded."}; diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 19fbbf155..b391ebed5 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -34,6 +34,43 @@ using namespace nix; using namespace nix::daemon; +struct UserSettings : Config { + + Setting trustedUsers{ + this, {"root"}, "trusted-users", + R"( + A list of names of users (separated by whitespace) that have + additional rights when connecting to the Nix daemon, such as the + ability to specify additional binary caches, or to import unsigned + NARs. You can also specify groups by prefixing them with `@`; for + instance, `@wheel` means all users in the `wheel` group. The default + is `root`. + + > **Warning** + > + > Adding a user to `trusted-users` is essentially equivalent to + > giving that user root access to the system. For example, the user + > can set `sandbox-paths` and thereby obtain read access to + > directories that are otherwise inacessible to them. + )"}; + + /* ?Who we trust to use the daemon in safe ways */ + Setting allowedUsers{ + this, {"*"}, "allowed-users", + R"( + A list of names of users (separated by whitespace) that are allowed + to connect to the Nix daemon. As with the `trusted-users` option, + you can specify groups by prefixing them with `@`. Also, you can + allow all users by specifying `*`. The default is `*`. + + Note that trusted users are always allowed to connect. + )"}; +}; + +UserSettings userSettings; + +static GlobalConfig::Register rSettings(&userSettings); + #ifndef __linux__ #define SPLICE_F_MOVE 0 static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags) @@ -203,8 +240,8 @@ static void daemonLoop() struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; std::string group = gr ? gr->gr_name : std::to_string(peer.gid); - Strings trustedUsers = settings.trustedUsers; - Strings allowedUsers = settings.allowedUsers; + Strings trustedUsers = userSettings.trustedUsers; + Strings allowedUsers = userSettings.allowedUsers; if (matchUser(user, group, trustedUsers)) trusted = Trusted; From f20d3726dd9031a4ec311c77f30caa7cc493138b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 29 Jan 2023 15:57:15 +0100 Subject: [PATCH 082/179] advertise transport encoding in http transfers to MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tl;dr: With this 1 line change I was able to get a speedup of 1.5x on 1Gbit/s wan connections by enabling zstd compression in nginx. Also nix already supported all common compression format for http transfer, webservers usually only enable them if they are advertised through the Accept-Encoding header. This pull requests makes nix advertises content compression support for zstd, br, gzip and deflate. It's particular useful to add transparent compression for binary caches that serve packages from the host nix store in particular nix-serve, nix-serve-ng and harmonia. I tried so far gzip, brotli and zstd, whereas only zstd was able to bring me performance improvements for 1Gbit/s WAN connections. The following nginx configuration was used in combination with the [zstd module](https://github.com/tokers/zstd-nginx-module) and [harmonia](https://github.com/nix-community/harmonia/) ```nix { services.nginx.virtualHosts."cache.yourhost.com" = { locations."/".extraConfig = '' proxy_pass http://127.0.0.1:5000; proxy_set_header Host $host; proxy_redirect http:// https://; proxy_http_version 1.1; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; zstd on; zstd_types application/x-nix-archive; ''; }; } ``` For testing I unpacked a linux kernel tarball to the nix store using this command `nix-prefetch-url --unpack https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-6.1.8.tar.gz`. Before: ```console $ nix build && rm -rf /tmp/hello && time ./result/bin/nix copy --no-check-sigs --from https://cache.thalheim.io --to 'file:///tmp/hello?compression=none' '/nix/store/j42mahch5f0jvfmayhzwbb88sw36fvah-linux-6.1.8.tar.gz' warning: Git tree '/scratch/joerg/nix' is dirty real 0m18,375s user 0m2,889s sys 0m1,558s ``` After: ```console $ nix build && rm -rf /tmp/hello && time ./result/bin/nix copy --no-check-sigs --from https://cache.thalheim.io --to 'file:///tmp/hello?compression=none' '/nix/store/j42mahch5f0jvfmayhzwb b88sw36fvah-linux-6.1.8.tar.gz' real 0m11,884s user 0m4,130s sys 0m1,439s ``` Signed-off-by: Jörg Thalheim Update src/libstore/filetransfer.cc Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- src/libstore/filetransfer.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 756bd4423..b25089ec3 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -101,6 +101,7 @@ struct curlFileTransfer : public FileTransfer this->result.data.append(data); }) { + requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz"); if (!request.expectedETag.empty()) requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); if (!request.mimeType.empty()) From 45fa297e4052acef962d9d124241e7abd02f58af Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 3 Feb 2023 11:21:47 -0500 Subject: [PATCH 083/179] Factor out `InstallableStorePath` to its own file, dedup `nix app` had something called `InstallableDerivedPath` which is actually the same thing. We go with the later's name because it has become more correct. I originally did this change (more hurriedly) as part of #6225 --- a mini store-only Nix and a full Nix need to share this code. In the first RFC meeting for https://github.com/NixOS/rfcs/pull/134 we discussed how some splitting of the massive `installables.cc` could begin prior, as that is a good thing anyways. (@edolstra's words, not mine!) This would be one such PR. --- src/libcmd/installable-derived-path.cc | 70 ++++++++++++++++++++++++++ src/libcmd/installable-derived-path.hh | 28 +++++++++++ src/libcmd/installables.cc | 70 ++------------------------ src/nix/app.cc | 27 +--------- 4 files changed, 104 insertions(+), 91 deletions(-) create mode 100644 src/libcmd/installable-derived-path.cc create mode 100644 src/libcmd/installable-derived-path.hh diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc new file mode 100644 index 000000000..a9921b901 --- /dev/null +++ b/src/libcmd/installable-derived-path.cc @@ -0,0 +1,70 @@ +#include "installable-derived-path.hh" +#include "derivations.hh" + +namespace nix { + +std::string InstallableDerivedPath::what() const +{ + return derivedPath.to_string(*store); +} + +DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths() +{ + return {{.path = derivedPath, .info = {} }}; +} + +std::optional InstallableDerivedPath::getStorePath() +{ + return std::visit(overloaded { + [&](const DerivedPath::Built & bfd) { + return bfd.drvPath; + }, + [&](const DerivedPath::Opaque & bo) { + return bo.path; + }, + }, derivedPath.raw()); +} + +InstallableDerivedPath InstallableDerivedPath::parse( + ref store, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec) +{ + auto derivedPath = std::visit(overloaded { + // If the user did not use ^, we treat the output more liberally. + [&](const ExtendedOutputsSpec::Default &) -> DerivedPath { + // First, we accept a symlink chain or an actual store path. + auto storePath = store->followLinksToStorePath(prefix); + // Second, we see if the store path ends in `.drv` to decide what sort + // of derived path they want. + // + // This handling predates the `^` syntax. The `^*` in + // `/nix/store/hash-foo.drv^*` unambiguously means "do the + // `DerivedPath::Built` case", so plain `/nix/store/hash-foo.drv` could + // also unambiguously mean "do the DerivedPath::Opaque` case". + // + // Issue #7261 tracks reconsidering this `.drv` dispatching. + return storePath.isDerivation() + ? (DerivedPath) DerivedPath::Built { + .drvPath = std::move(storePath), + .outputs = OutputsSpec::All {}, + } + : (DerivedPath) DerivedPath::Opaque { + .path = std::move(storePath), + }; + }, + // If the user did use ^, we just do exactly what is written. + [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath { + return DerivedPath::Built { + .drvPath = store->parseStorePath(prefix), + .outputs = outputSpec, + }; + }, + }, extendedOutputsSpec.raw()); + return InstallableDerivedPath { + store, + std::move(derivedPath), + }; +} + +} diff --git a/src/libcmd/installable-derived-path.hh b/src/libcmd/installable-derived-path.hh new file mode 100644 index 000000000..042878b91 --- /dev/null +++ b/src/libcmd/installable-derived-path.hh @@ -0,0 +1,28 @@ +#pragma once + +#include "installables.hh" + +namespace nix { + +struct InstallableDerivedPath : Installable +{ + ref store; + DerivedPath derivedPath; + + InstallableDerivedPath(ref store, DerivedPath && derivedPath) + : store(store), derivedPath(std::move(derivedPath)) + { } + + std::string what() const override; + + DerivedPathsWithInfo toDerivedPaths() override; + + std::optional getStorePath() override; + + static InstallableDerivedPath parse( + ref store, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec); +}; + +} diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 24f458f1a..5a75ed7af 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -1,5 +1,6 @@ #include "globals.hh" #include "installables.hh" +#include "installable-derived-path.hh" #include "outputs-spec.hh" #include "util.hh" #include "command.hh" @@ -396,38 +397,6 @@ static StorePath getDeriver( return *derivers.begin(); } -struct InstallableStorePath : Installable -{ - ref store; - DerivedPath req; - - InstallableStorePath(ref store, DerivedPath && req) - : store(store), req(std::move(req)) - { } - - std::string what() const override - { - return req.to_string(*store); - } - - DerivedPathsWithInfo toDerivedPaths() override - { - return {{.path = req, .info = {} }}; - } - - std::optional getStorePath() override - { - return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; - }, - [&](const DerivedPath::Opaque & bo) { - return bo.path; - }, - }, req.raw()); - } -}; - struct InstallableAttrPath : InstallableValue { SourceExprCommand & cmd; @@ -792,41 +761,10 @@ std::vector> SourceExprCommand::parseInstallables( auto prefix = std::move(prefix_); auto extendedOutputsSpec = std::move(extendedOutputsSpec_); - auto found = prefix.find('/'); - if (found != std::string::npos) { + if (prefix.find('/') != std::string::npos) { try { - auto derivedPath = std::visit(overloaded { - // If the user did not use ^, we treat the output more liberally. - [&](const ExtendedOutputsSpec::Default &) -> DerivedPath { - // First, we accept a symlink chain or an actual store path. - auto storePath = store->followLinksToStorePath(prefix); - // Second, we see if the store path ends in `.drv` to decide what sort - // of derived path they want. - // - // This handling predates the `^` syntax. The `^*` in - // `/nix/store/hash-foo.drv^*` unambiguously means "do the - // `DerivedPath::Built` case", so plain `/nix/store/hash-foo.drv` could - // also unambiguously mean "do the DerivedPath::Opaque` case". - // - // Issue #7261 tracks reconsidering this `.drv` dispatching. - return storePath.isDerivation() - ? (DerivedPath) DerivedPath::Built { - .drvPath = std::move(storePath), - .outputs = OutputsSpec::All {}, - } - : (DerivedPath) DerivedPath::Opaque { - .path = std::move(storePath), - }; - }, - // If the user did use ^, we just do exactly what is written. - [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath { - return DerivedPath::Built { - .drvPath = store->parseStorePath(prefix), - .outputs = outputSpec, - }; - }, - }, extendedOutputsSpec.raw()); - result.push_back(std::make_shared(store, std::move(derivedPath))); + result.push_back(std::make_shared( + InstallableDerivedPath::parse(store, prefix, extendedOutputsSpec))); continue; } catch (BadStorePath &) { } catch (...) { diff --git a/src/nix/app.cc b/src/nix/app.cc index 08cd0ccd4..5cd65136f 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -1,4 +1,5 @@ #include "installables.hh" +#include "installable-derived-path.hh" #include "store-api.hh" #include "eval-inline.hh" #include "eval-cache.hh" @@ -8,30 +9,6 @@ namespace nix { -struct InstallableDerivedPath : Installable -{ - ref store; - const DerivedPath derivedPath; - - InstallableDerivedPath(ref store, const DerivedPath & derivedPath) - : store(store) - , derivedPath(derivedPath) - { - } - - std::string what() const override { return derivedPath.to_string(*store); } - - DerivedPathsWithInfo toDerivedPaths() override - { - return {{derivedPath}}; - } - - std::optional getStorePath() override - { - return std::nullopt; - } -}; - /** * Return the rewrites that are needed to resolve a string whose context is * included in `dependencies`. @@ -146,7 +123,7 @@ App UnresolvedApp::resolve(ref evalStore, ref store) for (auto & ctxElt : unresolved.context) installableContext.push_back( - std::make_shared(store, ctxElt)); + std::make_shared(store, DerivedPath { ctxElt })); auto builtContext = Installable::build(evalStore, store, Realise::Outputs, installableContext); res.program = resolveString(*store, unresolved.program, builtContext); From 6352e20bc8dba9477e25eedbe7bea91cbe1614f9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 4 Feb 2023 18:26:43 -0500 Subject: [PATCH 084/179] Remove `--derivation` from test It doesn't do anything here, and in the next commit `show-derivation will no longer accept this flag. --- tests/ca/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ca/build.sh b/tests/ca/build.sh index 92f8b429a..cc225c6c8 100644 --- a/tests/ca/build.sh +++ b/tests/ca/build.sh @@ -3,7 +3,7 @@ source common.sh drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 1) -nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drv" --arg seed 1 +nix --experimental-features 'nix-command ca-derivations' show-derivation "$drv" --arg seed 1 buildAttr () { local derivationPath=$1 From 44bea52ae3ca9569250eb1f50100f2c3260cc688 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 4 Feb 2023 18:27:17 -0500 Subject: [PATCH 085/179] Scope down `--derivation` to just the commands that use it Per the old FIXME, this flag was on too many commands, and mostly ignored. Now it is just on the commands where it actually has an effect. Per https://github.com/NixOS/nix/issues/7261, I would still like to get rid of it entirely, but that is a separate project. This change should be good with or without doing that. --- src/libcmd/command.cc | 10 ++++++++++ src/libcmd/command.hh | 12 ++++++++---- src/libcmd/installables.cc | 7 ------- src/nix/diff-closures.cc | 2 +- src/nix/why-depends.cc | 2 +- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 0740ea960..517cdf617 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -127,6 +127,16 @@ ref EvalCommand::getEvalState() return ref(evalState); } +MixOperateOnOptions::MixOperateOnOptions() +{ + addFlag({ + .longName = "derivation", + .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.", + .category = installablesCategory, + .handler = {&operateOn, OperateOn::Derivation}, + }); +} + BuiltPathsCommand::BuiltPathsCommand(bool recursive) : recursive(recursive) { diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 3b4b40981..d16bdbc4b 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -96,9 +96,6 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions std::optional expr; bool readOnlyMode = false; - // FIXME: move this; not all commands (e.g. 'nix run') use it. - OperateOn operateOn = OperateOn::Output; - SourceExprCommand(bool supportReadOnlyMode = false); std::vector> parseInstallables( @@ -153,8 +150,15 @@ private: std::string _installable{"."}; }; +struct MixOperateOnOptions : virtual Args +{ + OperateOn operateOn = OperateOn::Output; + + MixOperateOnOptions(); +}; + /* A command that operates on zero or more store paths. */ -struct BuiltPathsCommand : public InstallablesCommand +struct BuiltPathsCommand : InstallablesCommand, virtual MixOperateOnOptions { private: diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 24f458f1a..ff8261b09 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -167,13 +167,6 @@ SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode) .handler = {&expr} }); - addFlag({ - .longName = "derivation", - .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.", - .category = installablesCategory, - .handler = {&operateOn, OperateOn::Derivation}, - }); - if (supportReadOnlyMode) { addFlag({ .longName = "read-only", diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 0621d662c..3489cc132 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -106,7 +106,7 @@ void printClosureDiff( using namespace nix; -struct CmdDiffClosures : SourceExprCommand +struct CmdDiffClosures : SourceExprCommand, MixOperateOnOptions { std::string _before, _after; diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 76125e5e4..a3a9dc698 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -27,7 +27,7 @@ static std::string filterPrintable(const std::string & s) return res; } -struct CmdWhyDepends : SourceExprCommand +struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions { std::string _package, _dependency; bool all = false; From b8e96351f761ebc688f2f2f6a13345576c008396 Mon Sep 17 00:00:00 2001 From: Peter Becich Date: Sun, 5 Feb 2023 12:00:48 -0800 Subject: [PATCH 086/179] fix `clang11Stdenv` dev shell documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `clang11StdenvPackages` does not exist ``` │ └───x86_64-linux │ ├───ccacheStdenv: development environment 'nix' │ ├───clang11Stdenv: development environment 'nix' │ ├───clangStdenv: development environment 'nix' │ ├───default: development environment 'nix' │ ├───gccStdenv: development environment 'nix' │ ├───libcxxStdenv: development environment 'nix' │ └───stdenv: development environment 'nix' ``` --- doc/manual/src/contributing/hacking.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 9dbafcc0a..e2046acc8 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -45,13 +45,13 @@ To get a shell with a different compilation environment (e.g. stdenv, gccStdenv, clangStdenv, clang11Stdenv, ccacheStdenv): ```console -$ nix-shell -A devShells.x86_64-linux.clang11StdenvPackages +$ nix-shell -A devShells.x86_64-linux.clang11Stdenv ``` or if you have a flake-enabled nix: ```console -$ nix develop .#clang11StdenvPackages +$ nix develop .#clang11Stdenv ``` Note: you can use `ccacheStdenv` to drastically improve rebuild From 3050005211951a1053e79634b1eb2fee1ad30a90 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Mon, 6 Feb 2023 17:53:03 +0100 Subject: [PATCH 087/179] build-remote: don't warn when all local build slots are taken Previously, build-remote would show a warning if all build slots were taken, even if they would open up later. This caused a lot of spam in the logs. Disable this warning when maxJobs > 0. See #6263 --- src/build-remote/build-remote.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 6b81ecc49..e197af847 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -112,10 +112,14 @@ static int main_build_remote(int argc, char * * argv) drvPath = store->parseStorePath(readString(source)); auto requiredFeatures = readStrings>(source); - auto canBuildLocally = amWilling + /* It would be possible to build locally after some builds clear out, + so don't show the warning now: */ + bool couldBuildLocally = settings.maxBuildJobs > 0 && ( neededSystem == settings.thisSystem || settings.extraPlatforms.get().count(neededSystem) > 0) && allSupportedLocally(*store, requiredFeatures); + /* It's possible to build this locally right now: */ + bool canBuildLocally = amWilling && couldBuildLocally; /* Error ignored here, will be caught later */ mkdir(currentLoad.c_str(), 0777); @@ -214,7 +218,7 @@ static int main_build_remote(int argc, char * * argv) % concatStringsSep(", ", m.supportedFeatures) % concatStringsSep(", ", m.mandatoryFeatures); - printMsg(canBuildLocally ? lvlChatty : lvlWarn, error); + printMsg(couldBuildLocally ? lvlChatty : lvlWarn, error); std::cerr << "# decline\n"; } From 6fdce7a9df0af7ea6266ded6c287b1dfc9d3eada Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Feb 2023 22:00:49 +0000 Subject: [PATCH 088/179] Bump cachix/install-nix-action from 18 to 19 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 18 to 19. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v18...v19) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dafba6d85..325579a5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v19 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/cachix-action@v12 if: needs.check_secrets.outputs.cachix == 'true' @@ -58,7 +58,7 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v19 - uses: cachix/cachix-action@v12 with: name: '${{ env.CACHIX_NAME }}' @@ -77,7 +77,7 @@ jobs: steps: - uses: actions/checkout@v3 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v19 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -102,7 +102,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v19 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV - uses: cachix/cachix-action@v12 From 9670cac1ab0119628916dfd5a7df5751ee27530a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 7 Feb 2023 11:11:50 +0100 Subject: [PATCH 089/179] Backport PR title: Bot-based -> Automatic In hope to reduce friction in contributor-facing communication, use a more common word. "bot" needless jargon for "something the machine does itself". --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index ca5af260f..558cfa804 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -27,6 +27,6 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} github_workspace: ${{ github.workspace }} pull_description: |- - Bot-based backport to `${target_branch}`, triggered by a label in #${pull_number}. + Automatic backport to `${target_branch}`, triggered by a label in #${pull_number}. # should be kept in sync with `uses` version: v0.0.5 From 2915db7b28ffab0aee9cfc3759f1ad0ee11b4cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Tue, 7 Feb 2023 11:38:09 +0100 Subject: [PATCH 090/179] doc: fixup 7714 --- .../src/language/advanced-attributes.md | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index a164b3997..5a63236e5 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -260,7 +260,7 @@ Derivations can declare some infrequently used optional attributes. If the special attribute `__structuredAttrs` is set to `true`, the other derivation attributes are serialised in JSON format and made available to the builder via the file `.attrs.json` in the builder’s temporary - directory. This obviates the need for `passAsFile` since JSON files + directory. This obviates the need for [`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions, unlike process environments. It also makes it possible to tweak derivation settings in a structured way; see @@ -282,11 +282,13 @@ Derivations can declare some infrequently used optional attributes. [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), the following attributes are available: - - `maxSize` defines the maximum size of the output path. + - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object). - `maxClosureSize` defines the maximum size of the output's closure. - `ignoreSelfRefs` controls whether self-references should be considered when checking for allowed references/requisites. + Example: + ```nix __structuredAttrs = true; @@ -305,9 +307,20 @@ Derivations can declare some infrequently used optional attributes. ``` - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\ - When using [structured attributes](#adv-attr-structuredAttrs), the **experimental** - attribute `unsafeDiscardReferences` is a per-output boolean which, if set to `true`, - disables scanning the build output for runtime dependencies altogether. + > **Warning** + > This is an experimental feature. + > + > To enable it, add the following to [nix.conf](../command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = discard-references + > ``` + + When using [structured attributes](#adv-attr-structuredAttrs), the + attribute `unsafeDiscardReferences` is an attribute set with a boolean value for each output name. + If set to `true`, it disables scanning the output for runtime dependencies. + + Example: ```nix __structuredAttrs = true; @@ -317,5 +330,3 @@ Derivations can declare some infrequently used optional attributes. This is useful, for example, when generating self-contained filesystem images with their own embedded Nix store: hashes found inside such an image refer to the embedded store and not to the host's Nix store. - - This is only allowed if the `discard-references` experimental feature is enabled. From 631ba6442a2fd2475b213cd463ca811d21761f36 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Tue, 7 Feb 2023 12:08:00 +0100 Subject: [PATCH 091/179] build-remote: store maxBuildJobs before forcing it to 1 --- src/build-remote/build-remote.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index e197af847..174435e7c 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -72,6 +72,7 @@ static int main_build_remote(int argc, char * * argv) settings.set(name, value); } + auto maxBuildJobs = settings.maxBuildJobs; settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work initPlugins(); @@ -114,7 +115,7 @@ static int main_build_remote(int argc, char * * argv) /* It would be possible to build locally after some builds clear out, so don't show the warning now: */ - bool couldBuildLocally = settings.maxBuildJobs > 0 + bool couldBuildLocally = maxBuildJobs > 0 && ( neededSystem == settings.thisSystem || settings.extraPlatforms.get().count(neededSystem) > 0) && allSupportedLocally(*store, requiredFeatures); From c20394245d602d43c7a601253bebdf0fab498b52 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 19 Jan 2023 15:48:38 +0100 Subject: [PATCH 092/179] add "instantiate" to glossary --- doc/manual/src/glossary.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 6004df833..d0aff34e2 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -19,6 +19,13 @@ [store derivation]: #gloss-store-derivation + - [instantiate]{#gloss-instantiate}, instantiation\ + Translate a [derivation] into a [store derivation]. + + See [`nix-instantiate`](./command-ref/nix-instantiate.md). + + [instantiate]: #gloss-instantiate + - [realise]{#gloss-realise}, realisation\ Ensure a [store path] is [valid][validity]. From 81e75e4bf6083046155a0c1fd6e312b43a60f7da Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 3 Feb 2023 21:42:42 +0100 Subject: [PATCH 093/179] Add some progress indication when fetching submodules --- src/libfetchers/git.cc | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 1f7d7c07d..fb80d248c 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -615,15 +615,24 @@ struct GitInputScheme : InputScheme AutoDelete delTmpGitDir(tmpGitDir, true); runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir }); - // TODO: repoDir might lack the ref (it only checks if rev - // exists, see FIXME above) so use a big hammer and fetch - // everything to ensure we get the rev. - runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", - "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); + + { + // TODO: repoDir might lack the ref (it only checks if rev + // exists, see FIXME above) so use a big hammer and fetch + // everything to ensure we get the rev. + Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", tmpDir)); + runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", + "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); + } runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); + runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); - runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); + + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", tmpDir)); + runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); + } filter = isNotDotGitDirectory; } else { From 2edd5cf6184a7955ba82a2aed8bbfa870bfeb15f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Feb 2023 15:35:32 +0100 Subject: [PATCH 094/179] Fix the origin URL used for fetching submodules We cannot use 'actualUrl', because for file:// repos that's not the original URL that the repo was fetched from. This is a problem since submodules may be relative to the original URL. Fixes e.g. nix eval --impure --json --expr 'builtins.fetchTree { type = "git"; url = "/path/to/blender"; submodules = true; }' where /path/to/blender is a clone of https://github.com/blender/blender.git (which has several relative submodules like '../blender-addons.git'). --- src/libfetchers/git.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index fb80d248c..8981476e1 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -627,7 +627,17 @@ struct GitInputScheme : InputScheme runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); - runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); + /* Ensure that we use the correct origin for fetching + submodules. This matters for submodules with relative + URLs. */ + if (isLocal) { + writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + gitDir + "/config")); + + /* Restore the config.bare setting we may have just + nuked. */ + runProgram("git", true, { "-C", tmpDir, "config", "core.bare", "false" }); + } else + runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", actualUrl }); { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", tmpDir)); From a8fe0dc16c18c260a1bb9e88f467e4c929cf22dc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Feb 2023 15:50:35 +0100 Subject: [PATCH 095/179] Speed up fetching submodules Previously we would completely refetch the submodules from the network, even though the repo might already have them. Now we copy the .git/modules directory from the repo as an optimisation. This speeds up evaluating builtins.fetchTree { type = "git"; url = "/path/to/blender"; submodules = true; } (where /path/to/blender already has the needed submodules) from 121s to 57s. This is still pretty inefficient and a hack, but a better solution is best done on the lazy-trees branch. This change also help in the case where the repo already has the submodules but the origin is unfetchable for whatever reason (e.g. there have been cases where Nix in a GitHub action doesn't have the right authentication set up). --- src/libfetchers/git.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8981476e1..9908db214 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -639,6 +639,14 @@ struct GitInputScheme : InputScheme } else runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", actualUrl }); + /* As an optimisation, copy the modules directory of the + source repo if it exists. */ + auto modulesPath = repoDir + "/" + gitDir + "/modules"; + if (pathExists(modulesPath)) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", actualUrl)); + runProgram("cp", true, { "-R", "--", modulesPath, tmpGitDir + "/modules" }); + } + { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", tmpDir)); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); From 72b18f05a22f46f383e0476262dc363ac0318aa6 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 7 Feb 2023 16:36:10 +0100 Subject: [PATCH 096/179] Add a basic daemon authorization test --- flake.nix | 2 + tests/nixos/authorization.nix | 79 +++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 tests/nixos/authorization.nix diff --git a/flake.nix b/flake.nix index 0325f33b1..7f0b738f8 100644 --- a/flake.nix +++ b/flake.nix @@ -499,6 +499,8 @@ }; # System tests. + tests.authorization = runNixOSTestFor "x86_64-linux" ./tests/nixos/authorization.nix; + tests.remoteBuilds = runNixOSTestFor "x86_64-linux" ./tests/nixos/remote-builds.nix; tests.nix-copy-closure = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy-closure.nix; diff --git a/tests/nixos/authorization.nix b/tests/nixos/authorization.nix new file mode 100644 index 000000000..4b13ec65e --- /dev/null +++ b/tests/nixos/authorization.nix @@ -0,0 +1,79 @@ +{ + name = "authorization"; + + nodes.machine = { + virtualisation.writableStore = true; + # TODO add a test without allowed-users setting. allowed-users is uncommon among NixOS users. + nix.settings.allowed-users = ["alice" "bob"]; + nix.settings.trusted-users = ["alice"]; + + users.users.alice.isNormalUser = true; + users.users.bob.isNormalUser = true; + users.users.mallory.isNormalUser = true; + + nix.settings.experimental-features = "nix-command"; + }; + + testScript = + let + pathFour = "/nix/store/20xfy868aiic0r0flgzq4n5dq1yvmxkn-four"; + in + '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + exec 1>&2 + echo kSELDhobKaF8/VdxIxdP7EQe+Q > one + diff $(nix store add-file one) one + """) + machine.succeed(""" + su --login alice -c ' + set -x + cd ~ + echo ehHtmfuULXYyBV6NBk6QUi8iE0 > two + ls + diff $(echo $(nix store add-file two)) two' 1>&2 + """) + machine.succeed(""" + su --login bob -c ' + set -x + cd ~ + echo 0Jw8RNp7cK0W2AdNbcquofcOVk > three + diff $(nix store add-file three) three + ' 1>&2 + """) + + # We're going to check that a path is not created + machine.succeed(""" + ! [[ -e ${pathFour} ]] + """) + machine.succeed(""" + su --login mallory -c ' + set -x + cd ~ + echo 5mgtDj0ohrWkT50TLR0f4tIIxY > four; + (! diff $(nix store add-file four) four 2>&1) | grep -F "cannot open connection to remote store" + (! diff $(nix store add-file four) four 2>&1) | grep -F "Connection reset by peer" + ! [[ -e ${pathFour} ]] + ' 1>&2 + """) + + # Check that the file _can_ be added, and matches the expected path we were checking + machine.succeed(""" + exec 1>&2 + echo 5mgtDj0ohrWkT50TLR0f4tIIxY > four + four="$(nix store add-file four)" + diff $four four + diff <(echo $four) <(echo ${pathFour}) + """) + + machine.succeed(""" + su --login alice -c 'nix-store --verify --repair' + """) + + machine.succeed(""" + set -x + su --login bob -c '(! nix-store --verify --repair 2>&1)' | tee diag 1>&2 + grep -F "you are not privileged to repair paths" diag + """) + ''; +} From 7a6daf61e8a3b6fd91e6d89334fe6b59fe07a2a6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Feb 2023 22:22:50 +0100 Subject: [PATCH 097/179] Fix activity message --- src/libfetchers/git.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 9908db214..3871cf0dd 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -620,7 +620,7 @@ struct GitInputScheme : InputScheme // TODO: repoDir might lack the ref (it only checks if rev // exists, see FIXME above) so use a big hammer and fetch // everything to ensure we get the rev. - Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", tmpDir)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir)); runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); } @@ -648,7 +648,7 @@ struct GitInputScheme : InputScheme } { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", tmpDir)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", repoDir)); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); } From 9a7dc5d71879a32ff06305aa843abf299b39c90f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Feb 2023 22:46:25 +0100 Subject: [PATCH 098/179] Add some tests --- tests/fetchGitSubmodules.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 50da4cb97..08ccaa3cd 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -104,3 +104,28 @@ noSubmoduleRepoBaseline=$(nix eval --raw --expr "(builtins.fetchGit { url = file noSubmoduleRepo=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath") [[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] + +# Test relative submodule URLs. +rm $TEST_HOME/.cache/nix/fetcher-cache* +rm -rf $rootRepo/.git $rootRepo/.gitmodules $rootRepo/sub +initGitRepo $rootRepo +git -C $rootRepo submodule add ../gitSubmodulesSub sub +git -C $rootRepo commit -m "Add submodule" +rev2=$(git -C $rootRepo rev-parse HEAD) +pathWithRelative=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev2\"; submodules = true; }).outPath") +diff -r -x .gitmodules $pathWithSubmodules $pathWithRelative + +# Test clones that have an upstream with relative submodule URLs. +rm $TEST_HOME/.cache/nix/fetcher-cache* +cloneRepo=$TEST_ROOT/a/b/gitSubmodulesClone # NB /a/b to make the relative path not work relative to $cloneRepo +git clone $rootRepo $cloneRepo +pathIndirect=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath") +[[ $pathIndirect = $pathWithRelative ]] + +# Test that if the clone has the submodule already, we're not fetching +# it again. +git -C $cloneRepo submodule update --init +rm $TEST_HOME/.cache/nix/fetcher-cache* +rm -rf $subRepo +pathSubmoduleGone=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath") +[[ $pathSubmoduleGone = $pathWithRelative ]] From 0a82d2ca491507f727d44a96f07d48934862da1b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Feb 2023 22:48:31 +0100 Subject: [PATCH 099/179] Ask release notes for incompatible changes, not bug fixes --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 344f9405f..db69e51db 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -25,4 +25,4 @@ Maintainers: tick if completed or explain if not relevant - [ ] documentation in the manual - [ ] code and comments are self-explanatory - [ ] commit message explains why the change was made - - [ ] new feature or bug fix: updated release notes + - [ ] new feature or incompatible change: updated release notes From fb2f7f5dcc6b37a4f39f59d9f477d3fa57d79095 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Jan 2023 17:31:27 +0100 Subject: [PATCH 100/179] Fix auto-uid-allocation in Docker containers This didn't work because sandboxing doesn't work in Docker. However, the sandboxing check is done lazily - after clone(CLONE_NEWNS) fails, we retry with sandboxing disabled. But at that point, we've already done UID allocation under the assumption that user namespaces are enabled. So let's get rid of the "goto fallback" logic and just detect early whether user / mount namespaces are enabled. This commit also gets rid of a compatibility hack for some ancient Linux kernels (<2.13). --- src/libstore/build/local-derivation-goal.cc | 82 +++++---------------- src/libutil/namespaces.cc | 63 ++++++++++++++++ src/libutil/namespaces.hh | 9 +++ 3 files changed, 91 insertions(+), 63 deletions(-) create mode 100644 src/libutil/namespaces.cc create mode 100644 src/libutil/namespaces.hh diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 8ff83f748..16955e326 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -16,6 +16,7 @@ #include "json-utils.hh" #include "cgroup.hh" #include "personality.hh" +#include "namespaces.hh" #include #include @@ -167,7 +168,8 @@ void LocalDerivationGoal::killSandbox(bool getStats) } -void LocalDerivationGoal::tryLocalBuild() { +void LocalDerivationGoal::tryLocalBuild() +{ unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { state = &DerivationGoal::tryToBuild; @@ -205,6 +207,17 @@ void LocalDerivationGoal::tryLocalBuild() { #endif } + #if __linux__ + if (useChroot) { + if (!mountNamespacesSupported()) { + if (!settings.sandboxFallback) + throw Error("this system does not support mount namespaces, which are required for sandboxing"); + debug("auto-disabling sandboxing because mount namespaces are not available"); + useChroot = false; + } + } + #endif + if (useBuildUsers()) { if (!buildUser) buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1, useChroot); @@ -888,12 +901,7 @@ void LocalDerivationGoal::startBuilder() userNamespaceSync.create(); - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; - static bool userNamespacesEnabled = - pathExists(maxUserNamespaces) - && trim(readFile(maxUserNamespaces)) != "0"; - - usingUserNamespace = userNamespacesEnabled; + usingUserNamespace = userNamespacesSupported(); Pid helper = startProcess([&]() { @@ -920,64 +928,15 @@ void LocalDerivationGoal::startBuilder() flags |= CLONE_NEWUSER; pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unprivileged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - * Details: https://salsa.debian.org/kernel-team/linux/-/commit/d98e00eda6bea437e39b9e80444eee84a32438a6 - */ - usingUserNamespace = false; - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (child == -1) { - switch(errno) { - case EPERM: - case EINVAL: { - int errno_ = errno; - if (!userNamespacesEnabled && errno==EPERM) - notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); - if (userNamespacesEnabled) { - Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; - if (pathExists(procSysKernelUnprivilegedUsernsClone) - && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { - notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); - } - } - Path procSelfNsUser = "/proc/self/ns/user"; - if (!pathExists(procSelfNsUser)) - notice("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (settings.sandboxFallback) - _exit(1); - /* Mention sandbox-fallback in the error message so the user - knows that having it disabled contributed to the - unrecoverability of this failure */ - throw SysError(errno_, "creating sandboxed builder process using clone(), without sandbox-fallback"); - } - default: - throw SysError("creating sandboxed builder process using clone()"); - } - } + + if (child == -1) + throw SysError("creating sandboxed builder process using clone()"); writeFull(builderOut.writeSide.get(), fmt("%d %d\n", usingUserNamespace, child)); _exit(0); }); - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) + if (helper.wait() != 0) throw Error("unable to start build process"); userNamespaceSync.readSide = -1; @@ -1045,9 +1004,6 @@ void LocalDerivationGoal::startBuilder() } else #endif { -#if __linux__ - fallback: -#endif pid = startProcess([&]() { runChild(); }); diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc new file mode 100644 index 000000000..0c3c3cbdd --- /dev/null +++ b/src/libutil/namespaces.cc @@ -0,0 +1,63 @@ +#include "namespaces.hh" +#include "util.hh" + +#if __linux__ + +namespace nix { + +bool userNamespacesSupported() +{ + static bool res = [&]() -> bool + { + if (!pathExists("/proc/self/ns/user")) { + notice("'/proc/self/ns/user' does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); + return false; + } + + Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; + if (!pathExists(maxUserNamespaces) || + trim(readFile(maxUserNamespaces)) == "0") + { + notice("user namespaces appear to be disabled; they are required for sandboxing; check '/proc/sys/user/max_user_namespaces'"); + return false; + } + + Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; + if (pathExists(procSysKernelUnprivilegedUsernsClone) + && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") + { + notice("user namespaces appear to be disabled; they are required for sandboxing; check '/proc/sys/kernel/unprivileged_userns_clone'"); + return false; + } + + Pid pid = startProcess([&]() + { + auto res = unshare(CLONE_NEWUSER); + _exit(res ? 1 : 0); + }); + + return pid.wait() == 0; + }(); + return res; +} + +bool mountNamespacesSupported() +{ + static bool res = [&]() -> bool + { + bool useUserNamespace = userNamespacesSupported(); + + Pid pid = startProcess([&]() + { + auto res = unshare(CLONE_NEWNS | (useUserNamespace ? CLONE_NEWUSER : 0)); + _exit(res ? 1 : 0); + }); + + return pid.wait() == 0; + }(); + return res; +} + +} + +#endif diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh new file mode 100644 index 000000000..4ed6cb683 --- /dev/null +++ b/src/libutil/namespaces.hh @@ -0,0 +1,9 @@ +#pragma once + +namespace nix { + +bool userNamespacesSupported(); + +bool mountNamespacesSupported(); + +} From bc1d9fd8b5a14334af1d0455e6b4d595cae959d5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Jan 2023 15:25:56 +0100 Subject: [PATCH 101/179] Check whether we can use PID namespaces In unprivileged podman containers, /proc is not fully visible (there are other filesystems mounted on subdirectories of /proc). Therefore we can't mount a new /proc in the sandbox that matches the PID namespace of the sandbox. So this commit automatically disables sandboxing if /proc is not fully visible. --- src/libstore/build/local-derivation-goal.cc | 6 ++-- src/libutil/namespaces.cc | 37 ++++++++++++++++++--- src/libutil/namespaces.hh | 2 ++ 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 16955e326..a99da0b5f 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -209,10 +209,10 @@ void LocalDerivationGoal::tryLocalBuild() #if __linux__ if (useChroot) { - if (!mountNamespacesSupported()) { + if (!mountNamespacesSupported() || !pidNamespacesSupported()) { if (!settings.sandboxFallback) - throw Error("this system does not support mount namespaces, which are required for sandboxing"); - debug("auto-disabling sandboxing because mount namespaces are not available"); + throw Error("this system does not support the kernel namespaces that are required for sandboxing"); + debug("auto-disabling sandboxing because the prerequisite namespaces are not available"); useChroot = false; } } diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index 0c3c3cbdd..222f0d11b 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -1,5 +1,8 @@ #include "namespaces.hh" #include "util.hh" +#include "finally.hh" + +#include #if __linux__ @@ -7,10 +10,10 @@ namespace nix { bool userNamespacesSupported() { - static bool res = [&]() -> bool + static auto res = [&]() -> bool { if (!pathExists("/proc/self/ns/user")) { - notice("'/proc/self/ns/user' does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); + debug("'/proc/self/ns/user' does not exist; your kernel was likely built without CONFIG_USER_NS=y"); return false; } @@ -18,7 +21,7 @@ bool userNamespacesSupported() if (!pathExists(maxUserNamespaces) || trim(readFile(maxUserNamespaces)) == "0") { - notice("user namespaces appear to be disabled; they are required for sandboxing; check '/proc/sys/user/max_user_namespaces'"); + debug("user namespaces appear to be disabled; check '/proc/sys/user/max_user_namespaces'"); return false; } @@ -26,7 +29,7 @@ bool userNamespacesSupported() if (pathExists(procSysKernelUnprivilegedUsernsClone) && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { - notice("user namespaces appear to be disabled; they are required for sandboxing; check '/proc/sys/kernel/unprivileged_userns_clone'"); + debug("user namespaces appear to be disabled; check '/proc/sys/kernel/unprivileged_userns_clone'"); return false; } @@ -43,7 +46,7 @@ bool userNamespacesSupported() bool mountNamespacesSupported() { - static bool res = [&]() -> bool + static auto res = [&]() -> bool { bool useUserNamespace = userNamespacesSupported(); @@ -58,6 +61,30 @@ bool mountNamespacesSupported() return res; } +bool pidNamespacesSupported() +{ + static auto res = [&]() -> bool + { + /* Check whether /proc is fully visible, i.e. there are no + filesystems mounted on top of files inside /proc. If this + is not the case, then we cannot mount a new /proc inside + the sandbox that matches the sandbox's PID namespace. + See https://lore.kernel.org/lkml/87tvsrjai0.fsf@xmission.com/T/. */ + auto fp = fopen("/proc/mounts", "r"); + if (!fp) return false; + Finally delFP = [&]() { fclose(fp); }; + + while (auto ent = getmntent(fp)) + if (hasPrefix(std::string_view(ent->mnt_dir), "/proc/")) { + debug("PID namespaces do not work because /proc is not fully visible; disabling sandboxing"); + return false; + } + + return true; + }(); + return res; +} + } #endif diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh index 4ed6cb683..ad7bb559e 100644 --- a/src/libutil/namespaces.hh +++ b/src/libutil/namespaces.hh @@ -6,4 +6,6 @@ bool userNamespacesSupported(); bool mountNamespacesSupported(); +bool pidNamespacesSupported(); + } From d834de2894b5addc5a4a8c5088debd56a8517db1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Jan 2023 16:52:01 +0100 Subject: [PATCH 102/179] Fix macOS build --- src/libutil/namespaces.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index 222f0d11b..b1cdbfe03 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -1,11 +1,11 @@ +#if __linux__ + #include "namespaces.hh" #include "util.hh" #include "finally.hh" #include -#if __linux__ - namespace nix { bool userNamespacesSupported() From 4e61877b5c64096fa3ea63bf5ead7e17e1ddef66 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Jan 2023 16:52:31 +0100 Subject: [PATCH 103/179] More #ifdef --- src/libutil/namespaces.hh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh index ad7bb559e..34e54d5ad 100644 --- a/src/libutil/namespaces.hh +++ b/src/libutil/namespaces.hh @@ -2,10 +2,14 @@ namespace nix { +#if __linux__ + bool userNamespacesSupported(); bool mountNamespacesSupported(); bool pidNamespacesSupported(); +#endif + } From c5c0617d6fe6810c35ec56d3116ef523f3f38904 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Feb 2023 22:59:46 +0100 Subject: [PATCH 104/179] Mention --no-sandbox if sandboxing is unsupported --- src/libstore/build/local-derivation-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index a99da0b5f..e1cc504f8 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -211,7 +211,7 @@ void LocalDerivationGoal::tryLocalBuild() if (useChroot) { if (!mountNamespacesSupported() || !pidNamespacesSupported()) { if (!settings.sandboxFallback) - throw Error("this system does not support the kernel namespaces that are required for sandboxing"); + throw Error("this system does not support the kernel namespaces that are required for sandboxing; use '--no-sandbox' to disable sandboxing"); debug("auto-disabling sandboxing because the prerequisite namespaces are not available"); useChroot = false; } From 0a70b411e1afaa22d8b01560de908246042daf10 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Feb 2023 23:01:39 +0100 Subject: [PATCH 105/179] Print debug message if a namespace test fails --- src/libutil/namespaces.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index b1cdbfe03..fdd52d92b 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -39,7 +39,12 @@ bool userNamespacesSupported() _exit(res ? 1 : 0); }); - return pid.wait() == 0; + bool supported = pid.wait() == 0; + + if (!supported) + debug("user namespaces do not work on this system"); + + return supported; }(); return res; } @@ -56,7 +61,12 @@ bool mountNamespacesSupported() _exit(res ? 1 : 0); }); - return pid.wait() == 0; + bool supported = pid.wait() == 0; + + if (!supported) + debug("mount namespaces do not work on this system"); + + return supported; }(); return res; } From 32db5e63494436b67d71d9393220dcafc34e5227 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Thu, 26 Jan 2023 22:50:36 +0100 Subject: [PATCH 106/179] docs: Fix broken anchor link --- doc/manual/src/language/operators.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 1f918bd4d..208d65839 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -116,7 +116,7 @@ The result is a string. [store path]: ../glossary.md#gloss-store-path [store]: ../glossary.md#gloss-store -[Path and string concatenation]: #path-and-string-concatenation +[String and path concatenation]: #string-and-path-concatenation ## Update @@ -133,7 +133,7 @@ If an attribute name is present in both, the attribute value from the latter is Comparison is -- [arithmetic] for [number]s +- [arithmetic] for [number]s - lexicographic for [string]s and [path]s - item-wise lexicographic for [list]s: elements at the same index in both lists are compared according to their type and skipped if they are equal. From d910dfe97892dcf26e83b6d591d46fbdfe2c0944 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Tue, 31 Jan 2023 14:20:51 +0100 Subject: [PATCH 107/179] docs: Fix formatting of || operator This is a workaround for [mdBook#2000](https://github.com/rust-lang/mdBook/issues/2000) --- doc/manual/src/language/operators.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 208d65839..90b325597 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -24,7 +24,7 @@ | [Equality] | *expr* `==` *expr* | none | 11 | | Inequality | *expr* `!=` *expr* | none | 11 | | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | -| Logical disjunction (`OR`) | *bool* `\|\|` *bool* | left | 13 | +| Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | | [Logical implication] | *bool* `->` *bool* | none | 14 | [string]: ./values.md#type-string From 9aeaf98c4b0e99a3af45517a2a54f7715a396542 Mon Sep 17 00:00:00 2001 From: David Dunn <26876072+doubledup@users.noreply.github.com> Date: Wed, 8 Feb 2023 00:32:27 +0200 Subject: [PATCH 108/179] Make install command in documentation compatible with fish shell (#7474) Use a pipe for all install commands Co-authored-by: Valentin Gagarin --- doc/manual/src/contributing/hacking.md | 2 +- doc/manual/src/installation/env-variables.md | 4 ++-- doc/manual/src/installation/installing-binary.md | 8 ++++---- doc/manual/src/quick-start.md | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index e2046acc8..53f49602b 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -219,7 +219,7 @@ After the CI run completes, you can check the output to extract the installer UR 5. To generate an install command, plug this `install_url` and your GitHub username into this template: ```console - sh <(curl -L ) --tarball-url-prefix https://-nix-install-tests.cachix.org/serve + curl -L | sh -s -- --tarball-url-prefix https://-nix-install-tests.cachix.org/serve ```