From 50f9f335c92a88e8c9bd3421e6befbcd06cac4ec Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Wed, 30 Mar 2022 16:35:26 -0400 Subject: [PATCH 01/27] profile!: consistent use of url/uri. create new version --- src/nix/profile.cc | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index f35947ddb..b151e48d6 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -97,19 +97,30 @@ struct ProfileManifest auto json = nlohmann::json::parse(readFile(manifestPath)); auto version = json.value("version", 0); - if (version != 1) - throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version); + std::string sUrl; + std::string sOriginalUrl; + switch(version){ + case 1: + sUrl = "uri"; + sOriginalUrl = "originalUri"; + break; + case 2: + sUrl = "url"; + sOriginalUrl = "originalUrl"; + break; + default: + throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version); + } for (auto & e : json["elements"]) { ProfileElement element; for (auto & p : e["storePaths"]) element.storePaths.insert(state.store->parseStorePath((std::string) p)); element.active = e["active"]; - if (e.value("uri", "") != "") { - auto originalUrl = e.value("originalUrl", e["originalUri"]); + if (e.value(sUrl,"") != "") { element.source = ProfileElementSource{ - parseFlakeRef(originalUrl), - parseFlakeRef(e["uri"]), + parseFlakeRef(e[sOriginalUrl]), + parseFlakeRef(e[sUrl]), e["attrPath"] }; } @@ -144,13 +155,13 @@ struct ProfileManifest obj["active"] = element.active; if (element.source) { obj["originalUrl"] = element.source->originalRef.to_string(); - obj["uri"] = element.source->resolvedRef.to_string(); + obj["url"] = element.source->resolvedRef.to_string(); obj["attrPath"] = element.source->attrPath; } array.push_back(obj); } nlohmann::json json; - json["version"] = 1; + json["version"] = 2; json["elements"] = array; return json.dump(); } From 7492030ed7d1d570b71e832b610150054a9f095b Mon Sep 17 00:00:00 2001 From: Artturin Date: Thu, 31 Mar 2022 22:14:53 +0300 Subject: [PATCH 02/27] scripts/install-systemd-multi-user.sh: fix another typo --- scripts/install-systemd-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 1d92c5388..62397127a 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -90,7 +90,7 @@ poly_configure_nix_daemon_service() { ln -sfn /nix/var/nix/profiles/default/$TMPFILES_SRC $TMPFILES_DEST _sudo "to run systemd-tmpfiles once to pick that path up" \ - systemd-tmpfiles create --prefix=/nix/var/nix + systemd-tmpfiles --create --prefix=/nix/var/nix _sudo "to set up the nix-daemon service" \ systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC" From fdfe737867920ed0ec29ba8ffb7d19854d8658bb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 1 Apr 2022 12:40:49 +0200 Subject: [PATCH 03/27] Fix handling of outputHash when outputHashAlgo is not specified https://hydra.nixos.org/build/171351131 --- src/libexpr/primops.cc | 7 +++---- tests/ca/content-addressed.nix | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 969391725..73817dbdd 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -991,7 +991,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * bool contentAddressed = false; bool isImpure = false; std::optional outputHash; - std::optional outputHashAlgo; + std::string outputHashAlgo; std::optional ingestionMethod; StringSet outputs; @@ -1190,8 +1190,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * .errPos = posDrvName }); - std::optional ht = parseHashTypeOpt(outputHashAlgo.value_or("sha256")); - Hash h = newHashAllowEmpty(*outputHash, ht); + auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo)); auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); auto outPath = state.store->makeFixedOutputPath(method, h, drvName); @@ -1212,7 +1211,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * .errPos = posDrvName }); - auto ht = parseHashType(outputHashAlgo.value_or("sha256")); + auto ht = parseHashTypeOpt(outputHashAlgo).value_or(htSHA256); auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive); for (auto & i : outputs) { diff --git a/tests/ca/content-addressed.nix b/tests/ca/content-addressed.nix index d328fc92c..1be3eeb6e 100644 --- a/tests/ca/content-addressed.nix +++ b/tests/ca/content-addressed.nix @@ -64,8 +64,7 @@ rec { dependentFixedOutput = mkDerivation { name = "dependent-fixed-output"; outputHashMode = "recursive"; - outputHashAlgo = "sha256"; - outputHash = "sha256-QvtAMbUl/uvi+LCObmqOhvNOapHdA2raiI4xG5zI5pA="; + outputHash = "sha512-7aJcmSuEuYP5tGKcmGY8bRr/lrCjJlOxP2mIUjO/vMQeg6gx/65IbzRWES8EKiPDOs9z+wF30lEfcwxM/cT4pw=="; buildCommand = '' cat ${dependentCA}/dep echo foo > $out From 435848cef12d065b209c82c96ce3a8bfa5e6a051 Mon Sep 17 00:00:00 2001 From: aszlig Date: Fri, 1 Apr 2022 09:23:43 -0700 Subject: [PATCH 04/27] libutil: Fix restoring mount namespace I regularly pass around simple scripts by using nix-shell as the script interpreter, eg. like this: #!/usr/bin/env nix-shell #!nix-shell -p dd_rescue coreutils bash -i bash While this works most of the time, I recently had one occasion where it would not and the above would result in the following: $ sudo ./myscript.sh bash: ./myscript.sh: No such file or directory Note the "sudo" here, because this error only occurs if we're root. The reason for the latter is because running Nix as root means that we can directly access the store, which makes sure we use a filesystem namespace to make the store writable. XXX - REWORD! So when stracing the process, I stumbled on the following sequence: openat(AT_FDCWD, "/proc/self/ns/mnt", O_RDONLY) = 3 unshare(CLONE_NEWNS) = 0 ... later ... getcwd("/the/real/cwd", 4096) = 14 setns(3, CLONE_NEWNS) = 0 getcwd("/", 4096) = 2 In the whole strace output there are no calls to chdir() whatsoever, so I decided to look into the kernel source to see what else could change directories and found this[1]: /* Update the pwd and root */ set_fs_pwd(fs, &root); set_fs_root(fs, &root); The set_fs_pwd() call is roughly equivalent to a chdir() syscall and this is called when the setns() syscall is invoked[2]. [1]: https://github.com/torvalds/linux/blob/b14ffae378aa1db993e62b01392e70d1e585fb23/fs/namespace.c#L4659 [2]: https://github.com/torvalds/linux/blob/b14ffae378aa1db993e62b01392e70d1e585fb23/kernel/nsproxy.c#L346 --- src/libutil/util.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 59e3aad6d..e62672717 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -1688,13 +1689,17 @@ void setStackSize(size_t stackSize) #endif } +#if __linux__ static AutoCloseFD fdSavedMountNamespace; +std::optional savedCwd; +#endif void saveMountNamespace() { #if __linux__ static std::once_flag done; std::call_once(done, []() { + savedCwd.emplace(std::filesystem::current_path()); AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY); if (!fd) throw SysError("saving parent mount namespace"); @@ -1712,6 +1717,12 @@ void restoreMountNamespace() } catch (Error & e) { debug(e.msg()); } + try { + if (savedCwd) + std::filesystem::current_path(*savedCwd); + } catch (std::filesystem::filesystem_error const &e) { + debug(e.what()); + } #endif } From 7f5caaa7c0f151520d05d4662415ac09d4cf34b0 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Fri, 1 Apr 2022 10:24:31 -0700 Subject: [PATCH 05/27] libutil: Don't use std::filesystem Just in case making libutil depend on std::filesystem is unacceptable, here is the non-filesystem approach. --- src/libutil/util.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index e62672717..0b18f1027 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -1691,7 +1690,7 @@ void setStackSize(size_t stackSize) #if __linux__ static AutoCloseFD fdSavedMountNamespace; -std::optional savedCwd; +std::optional savedCwd; #endif void saveMountNamespace() @@ -1699,7 +1698,11 @@ void saveMountNamespace() #if __linux__ static std::once_flag done; std::call_once(done, []() { - savedCwd.emplace(std::filesystem::current_path()); + char* cwd = getcwd(NULL, 0); + if (cwd == NULL) throw SysError("getting cwd"); + savedCwd.emplace(cwd); + free(cwd); + AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY); if (!fd) throw SysError("saving parent mount namespace"); @@ -1717,11 +1720,8 @@ void restoreMountNamespace() } catch (Error & e) { debug(e.msg()); } - try { - if (savedCwd) - std::filesystem::current_path(*savedCwd); - } catch (std::filesystem::filesystem_error const &e) { - debug(e.what()); + if (savedCwd && chdir(savedCwd->c_str()) == -1) { + throw SysError("restoring cwd"); } #endif } From 2a45cf54e4201a894254604676dcb51f4c1a471c Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Fri, 1 Apr 2022 12:20:34 -0700 Subject: [PATCH 06/27] libutil: Properly guard self-allocating getcwd on GNU It's a GNU extension, as pointed out by pennae. --- src/libutil/util.cc | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 0b18f1027..28ab77adc 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1698,10 +1698,19 @@ void saveMountNamespace() #if __linux__ static std::once_flag done; std::call_once(done, []() { - char* cwd = getcwd(NULL, 0); - if (cwd == NULL) throw SysError("getting cwd"); +#ifdef __GNU__ + // getcwd allocating its return value is a GNU extension. + char *cwd = getcwd(NULL, 0); + if (cwd == NULL) +#else + char cwd[PATH_MAX]; + if (!getcwd(cwd, sizeof(cwd))) +#endif + throw SysError("getting cwd"); savedCwd.emplace(cwd); +#ifdef __GNU__ free(cwd); +#endif AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY); if (!fd) From c1e2ce4515c12ff0b4b1c3ab6d1e057507104979 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Thu, 31 Mar 2022 20:30:53 -0400 Subject: [PATCH 07/27] fix(run): set applyNixConfig lockFlag --- src/nix/run.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/run.cc b/src/nix/run.cc index 23e893fbf..25a8fa8d3 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -182,6 +182,7 @@ struct CmdRun : InstallableCommand { auto state = getEvalState(); + lockFlags.applyNixConfig = true; auto app = installable->toApp(*state).resolve(getEvalStore(), store); Strings allArgs{app.program}; From a4a1de69dcc3c6e0c40a093d67b5f20568a5f31e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Apr 2022 16:49:39 +0200 Subject: [PATCH 08/27] Add missing #include --- src/libstore/build-result.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index cb6d19b8e..7f9bcce6d 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -1,6 +1,7 @@ #pragma once #include "realisation.hh" +#include "derived-path.hh" #include #include From e5b70d47aa656833e4609106f9f1ef48b67664cc Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Mon, 4 Apr 2022 08:19:49 -0700 Subject: [PATCH 09/27] libutil: cleanup savedCwd logic Co-authored-by: Eelco Dolstra --- src/libutil/util.cc | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 28ab77adc..9b34d5d72 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1698,20 +1698,7 @@ void saveMountNamespace() #if __linux__ static std::once_flag done; std::call_once(done, []() { -#ifdef __GNU__ - // getcwd allocating its return value is a GNU extension. - char *cwd = getcwd(NULL, 0); - if (cwd == NULL) -#else - char cwd[PATH_MAX]; - if (!getcwd(cwd, sizeof(cwd))) -#endif - throw SysError("getting cwd"); - savedCwd.emplace(cwd); -#ifdef __GNU__ - free(cwd); -#endif - + savedCwd = absPath("."); AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY); if (!fd) throw SysError("saving parent mount namespace"); From e135d223f66f77f2b03a11b979d714e582364013 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Mon, 4 Apr 2022 08:32:45 -0700 Subject: [PATCH 10/27] libutil: save fd to cwd instead of cwd itself --- src/libutil/util.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 9b34d5d72..a00a03978 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1690,7 +1690,7 @@ void setStackSize(size_t stackSize) #if __linux__ static AutoCloseFD fdSavedMountNamespace; -std::optional savedCwd; +static AutoCloseFD fdSavedCwd; #endif void saveMountNamespace() @@ -1698,11 +1698,15 @@ void saveMountNamespace() #if __linux__ static std::once_flag done; std::call_once(done, []() { - savedCwd = absPath("."); AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY); if (!fd) throw SysError("saving parent mount namespace"); fdSavedMountNamespace = std::move(fd); + + fd = open("/proc/self/cwd", O_RDONLY); + if (!fd) + throw SysError("saving cwd"); + fdSavedCwd = std::move(fd); }); #endif } @@ -1716,7 +1720,7 @@ void restoreMountNamespace() } catch (Error & e) { debug(e.msg()); } - if (savedCwd && chdir(savedCwd->c_str()) == -1) { + if (fdSavedCwd && fchdir(fdSavedCwd.get()) == -1) { throw SysError("restoring cwd"); } #endif From f89b0f7846f2177b65eebdbff7f24a255d66819e Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Mon, 4 Apr 2022 08:33:59 -0700 Subject: [PATCH 11/27] libutil: `try` restoring the cwd from fdSavedCwd --- src/libutil/util.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index a00a03978..1f800f3f4 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1717,12 +1717,12 @@ void restoreMountNamespace() try { if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) throw SysError("restoring parent mount namespace"); + if (fdSavedCwd && fchdir(fdSavedCwd.get()) == -1) { + throw SysError("restoring cwd"); + } } catch (Error & e) { debug(e.msg()); } - if (fdSavedCwd && fchdir(fdSavedCwd.get()) == -1) { - throw SysError("restoring cwd"); - } #endif } From 10b9c1b2b269b96dcb8b3d298491fa143d1663e8 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Mon, 4 Apr 2022 10:16:30 -0700 Subject: [PATCH 12/27] libutil: save cwd fd in restoreMountNamespace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This doesn't work very well (maybe I'm misunderstanding the desired implementation): : ~/w/vc/nix ; doas outputs/out/bin/nix --experimental-features 'nix-command flakes' develop -c pwd pwd: couldn't find directory entry in ‘../../../..’ with matching i-node --- src/libutil/util.cc | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 1f800f3f4..701545589 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1690,7 +1690,6 @@ void setStackSize(size_t stackSize) #if __linux__ static AutoCloseFD fdSavedMountNamespace; -static AutoCloseFD fdSavedCwd; #endif void saveMountNamespace() @@ -1702,11 +1701,6 @@ void saveMountNamespace() if (!fd) throw SysError("saving parent mount namespace"); fdSavedMountNamespace = std::move(fd); - - fd = open("/proc/self/cwd", O_RDONLY); - if (!fd) - throw SysError("saving cwd"); - fdSavedCwd = std::move(fd); }); #endif } @@ -1715,6 +1709,10 @@ void restoreMountNamespace() { #if __linux__ try { + AutoCloseFD fdSavedCwd = open("/proc/self/cwd", O_RDONLY); + if (!fdSavedCwd) { + throw SysError("saving cwd"); + } if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) throw SysError("restoring parent mount namespace"); if (fdSavedCwd && fchdir(fdSavedCwd.get()) == -1) { From 56009b2639dc878be11f94d096fae56ac14dcd1d Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Mon, 4 Apr 2022 10:21:56 -0700 Subject: [PATCH 13/27] libutil: don't save cwd fd, use path instead MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Saving the cwd fd didn't actually work well -- prior to this commit, the following would happen: : ~/w/vc/nix ; doas outputs/out/bin/nix --experimental-features 'nix-command flakes' run nixpkgs#coreutils -- --coreutils-prog=pwd pwd: couldn't find directory entry in ‘../../../..’ with matching i-node : ~/w/vc/nix ; doas outputs/out/bin/nix --experimental-features 'nix-command flakes' develop -c pwd pwd: couldn't find directory entry in ‘../../../..’ with matching i-node --- src/libutil/util.cc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 701545589..3132e7161 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1709,13 +1709,11 @@ void restoreMountNamespace() { #if __linux__ try { - AutoCloseFD fdSavedCwd = open("/proc/self/cwd", O_RDONLY); - if (!fdSavedCwd) { - throw SysError("saving cwd"); - } + auto savedCwd = absPath("."); + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) throw SysError("restoring parent mount namespace"); - if (fdSavedCwd && fchdir(fdSavedCwd.get()) == -1) { + if (chdir(savedCwd.c_str()) == -1) { throw SysError("restoring cwd"); } } catch (Error & e) { From 5abe3f4aa61f33ac2124a624763e067257541871 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 5 Apr 2022 11:03:43 +0200 Subject: [PATCH 14/27] Allow `welcomeText` when checking a flake template Fix https://github.com/NixOS/nix/issues/6321 --- src/nix/flake.cc | 2 +- tests/flakes.sh | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 47a380238..ce7dc101a 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -463,7 +463,7 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *v.attrs) { std::string name(attr.name); - if (name != "path" && name != "description") + if (name != "path" && name != "description" && name != "welcomeText") throw Error("template '%s' has unsupported attribute '%s'", attrPath, name); } } catch (Error & e) { diff --git a/tests/flakes.sh b/tests/flakes.sh index ea629ae70..46e6a7982 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -376,6 +376,9 @@ cat > $templatesDir/flake.nix < Date: Tue, 5 Apr 2022 13:34:25 +0200 Subject: [PATCH 15/27] manual: Add some anchor targets for the nix.conf options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For each `nix.conf` option, add an empty html node with a unique `id` that can be used as an anchor target. Also make the name of the option be a link to that target so that it’s easily discoverable. We can’t rewrite the whole list as an html definition list like it’s done for the builtins because these options also appear in a man page, and the manpage renderer (lowdown) can’t render arbitrary html. But the hack here allows to keep the manpage and have the links in the html version. Fix https://github.com/NixOS/nix/issues/5745 --- doc/manual/generate-options.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/generate-options.nix b/doc/manual/generate-options.nix index 84d90beb6..2d586fa1b 100644 --- a/doc/manual/generate-options.nix +++ b/doc/manual/generate-options.nix @@ -6,7 +6,8 @@ options: concatStrings (map (name: let option = options.${name}; in - " - `${name}` \n\n" + " - [`${name}`](#conf-${name})" + + "

\n\n" + concatStrings (map (s: " ${s}\n") (splitLines option.description)) + "\n\n" + (if option.documentDefault then " **Default:** " + ( From 9a640afc1e63bed28926cb44fe85137fb7d2d2d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 5 Apr 2022 11:33:46 +0200 Subject: [PATCH 16/27] doctor: Always show the output Fix https://github.com/NixOS/nix/issues/6342 --- src/nix/doctor.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 4f3003448..ea87e3d87 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -24,12 +24,12 @@ std::string formatProtocol(unsigned int proto) } bool checkPass(const std::string & msg) { - logger->log(ANSI_GREEN "[PASS] " ANSI_NORMAL + msg); + notice(ANSI_GREEN "[PASS] " ANSI_NORMAL + msg); return true; } bool checkFail(const std::string & msg) { - logger->log(ANSI_RED "[FAIL] " ANSI_NORMAL + msg); + notice(ANSI_RED "[FAIL] " ANSI_NORMAL + msg); return false; } From f98d76ff1aeaaf18ea68a86f9eb91dd73b2d6ce4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Apr 2022 14:13:26 +0200 Subject: [PATCH 17/27] rl-2.7.md: Fix title --- doc/manual/src/release-notes/rl-2.7.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-2.7.md b/doc/manual/src/release-notes/rl-2.7.md index dc92a9cde..2f3879422 100644 --- a/doc/manual/src/release-notes/rl-2.7.md +++ b/doc/manual/src/release-notes/rl-2.7.md @@ -1,4 +1,4 @@ -# Release X.Y (2022-03-07) +# Release 2.7 (2022-03-07) * Nix will now make some helpful suggestions when you mistype something on the command line. For instance, if you type `nix build From 8d6c937d6a233106dcf9c3e0f0e1c60148b7a71e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Apr 2022 16:41:40 +0200 Subject: [PATCH 18/27] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/82891b5e2c2359d7e58d08849e4c89511ab94234' (2021-09-28) → 'github:NixOS/nixpkgs/530a53dcbc9437363471167a5e4762c5fcfa34a1' (2022-02-19) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 61eccb73c..cd79fa85e 100644 --- a/flake.lock +++ b/flake.lock @@ -18,11 +18,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1632864508, - "narHash": "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM=", + "lastModified": 1645296114, + "narHash": "sha256-y53N7TyIkXsjMpOG7RhvqJFGDacLs9HlyHeSTBioqYU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "82891b5e2c2359d7e58d08849e4c89511ab94234", + "rev": "530a53dcbc9437363471167a5e4762c5fcfa34a1", "type": "github" }, "original": { From 1fa03934791189f02208e1807d822128b216f087 Mon Sep 17 00:00:00 2001 From: Daniel Pauls Date: Tue, 5 Apr 2022 21:16:11 +0200 Subject: [PATCH 19/27] libutil: Reserve memory when en/decoding base64 The size of the output when encoding to and decoding from base64 is (roughly) known so we can allocate it in advance to prevent reallocation. --- src/libutil/util.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 59e3aad6d..cca66d5bf 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1471,6 +1471,7 @@ constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuv std::string base64Encode(std::string_view s) { std::string res; + res.reserve((s.size() + 2) / 3 * 4); int data = 0, nbits = 0; for (char c : s) { @@ -1502,6 +1503,9 @@ std::string base64Decode(std::string_view s) }(); std::string res; + // Some sequences are missing the padding consisting of up to two '='. + // vvv + res.reserve((s.size() + 2) / 4 * 3); unsigned int d = 0, bits = 0; for (char c : s) { From 513652d5947c1923a004318ea453cab786cabda1 Mon Sep 17 00:00:00 2001 From: Daniel Pauls Date: Tue, 5 Apr 2022 22:13:45 +0200 Subject: [PATCH 20/27] tokenizeString: Fix semantic mistake `string_view::find_first_not_of(...)` and `string_view::find_first_of(...)` return `string_view::npos` on error not `string::npos`. --- src/libutil/util.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 59e3aad6d..8ab385837 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1259,9 +1259,9 @@ template C tokenizeString(std::string_view s, std::string_view separato { C result; auto pos = s.find_first_not_of(separators, 0); - while (pos != std::string::npos) { + while (pos != std::string_view::npos) { auto end = s.find_first_of(separators, pos + 1); - if (end == std::string::npos) end = s.size(); + if (end == std::string_view::npos) end = s.size(); result.insert(result.end(), std::string(s, pos, end - pos)); pos = s.find_first_not_of(separators, end); } From 589f6f267b009bc2856597995db360f910e69a6f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Apr 2022 11:52:51 +0200 Subject: [PATCH 21/27] fetchClosure: Don't allow URL query parameters Allowing this is a potential security hole, since it allows the user to specify parameters like 'local-nar-cache'. --- src/libexpr/primops/fetchClosure.cc | 9 ++++++++- tests/fetchClosure.sh | 12 ++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index efeb93daf..821eba698 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -61,6 +61,12 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args .errPos = pos }); + if (!parsedURL.query.empty()) + throw Error({ + .msg = hintfmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl), + .errPos = pos + }); + auto fromStore = openStore(parsedURL.to_string()); if (toCA) { @@ -87,7 +93,8 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args }); } } else { - copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); + if (!state.store->isValidPath(*fromPath)) + copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); toPath = fromPath; } diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh index 0c905ac43..96e4bb741 100644 --- a/tests/fetchClosure.sh +++ b/tests/fetchClosure.sh @@ -56,3 +56,15 @@ nix copy --to file://$cacheDir $caPath fromPath = $caPath; } ") = $caPath ]] + +# Check that URL query parameters aren't allowed. +clearStore +narCache=$TEST_ROOT/nar-cache +rm -rf $narCache +(! nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir?local-nar-cache=$narCache\"; + fromPath = $caPath; + } +") +(! [ -e $narCache ]) From 318936366d5198123ae9ba0292538529f706dce8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Apr 2022 12:41:18 +0200 Subject: [PATCH 22/27] Fix empty 'nix copy' error message This was caused by SubstitutionGoal not setting the errorMsg field in its BuildResult. We now get a more descriptive message than in 2.7.0, e.g. error: path '/nix/store/13mh...' is required, but there is no substituter that can build it instead of the misleading (since there was no build) error: build of '/nix/store/13mh...' failed Fixes #6295. --- .../build/drv-output-substitution-goal.cc | 2 +- src/libstore/build/substitution-goal.cc | 19 ++++++++++++++----- src/libstore/build/substitution-goal.hh | 5 ++++- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index e50292c1e..b7f7b5ab1 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -41,7 +41,7 @@ void DrvOutputSubstitutionGoal::tryNext() if (subs.size() == 0) { /* None left. Terminate this goal and let someone else deal with it. */ - debug("drv output '%s' is required, but there is no substituter that can provide it", id.to_string()); + debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 31e6dbc9f..ca5218627 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -24,9 +24,16 @@ PathSubstitutionGoal::~PathSubstitutionGoal() } -void PathSubstitutionGoal::done(ExitCode result, BuildResult::Status status) +void PathSubstitutionGoal::done( + ExitCode result, + BuildResult::Status status, + std::optional errorMsg) { buildResult.status = status; + if (errorMsg) { + debug(*errorMsg); + buildResult.errorMsg = *errorMsg; + } amDone(result); } @@ -67,12 +74,14 @@ void PathSubstitutionGoal::tryNext() if (subs.size() == 0) { /* None left. Terminate this goal and let someone else deal with it. */ - debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)); /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - done(substituterFailed ? ecFailed : ecNoSubstituters, BuildResult::NoSubstituters); + done( + substituterFailed ? ecFailed : ecNoSubstituters, + BuildResult::NoSubstituters, + fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath))); if (substituterFailed) { worker.failedSubstitutions++; @@ -169,10 +178,10 @@ void PathSubstitutionGoal::referencesValid() trace("all references realised"); if (nrFailed > 0) { - debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); done( nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, - BuildResult::DependencyFailed); + BuildResult::DependencyFailed, + fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath))); return; } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 946f13841..a73f8e666 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -53,7 +53,10 @@ struct PathSubstitutionGoal : public Goal /* Content address for recomputing store path */ std::optional ca; - void done(ExitCode result, BuildResult::Status status); + void done( + ExitCode result, + BuildResult::Status status, + std::optional errorMsg = {}); public: PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); From a7b12c6bd90cd44432bdaf45cb32b0af916d499c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Apr 2022 13:34:25 +0200 Subject: [PATCH 23/27] curl: Use --fail to catch errors --- scripts/check-hydra-status.sh | 2 +- scripts/install.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/check-hydra-status.sh b/scripts/check-hydra-status.sh index 5e2f03429..e62705e94 100644 --- a/scripts/check-hydra-status.sh +++ b/scripts/check-hydra-status.sh @@ -14,7 +14,7 @@ curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master someBuildFailed=0 for buildId in $BUILDS_FOR_LATEST_EVAL; do - buildInfo=$(curl -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") + buildInfo=$(curl --fail -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") finished=$(echo "$buildInfo" | jq -r '.finished') diff --git a/scripts/install.in b/scripts/install.in index 38d1fb36f..af5f71080 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -82,7 +82,7 @@ if [ "$(uname -s)" != "Darwin" ]; then fi if command -v curl > /dev/null 2>&1; then - fetch() { curl -L "$1" -o "$2"; } + fetch() { curl --fail -L "$1" -o "$2"; } elif command -v wget > /dev/null 2>&1; then fetch() { wget "$1" -O "$2"; } else From 1e1cd6e7a926097683da366983cc362c2430867d Mon Sep 17 00:00:00 2001 From: Daniel Pauls Date: Wed, 6 Apr 2022 17:33:23 +0200 Subject: [PATCH 24/27] libfetchers: Fix assertion The filter expects all paths to have a prefix of the raw `actualUrl`, but `Store::addToStore(...)` provides absolute canonicalized paths. To fix this create an absolute and canonicalized path from the `actualUrl` and use it instead. Fixes #6195. --- src/libfetchers/git.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index d75c5d3ae..f8433bc28 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -285,9 +285,11 @@ struct GitInputScheme : InputScheme auto files = tokenizeString>( runProgram("git", true, gitOpts), "\0"s); + Path actualPath(absPath(actualUrl)); + PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, actualUrl)); - std::string file(p, actualUrl.size() + 1); + assert(hasPrefix(p, actualPath)); + std::string file(p, actualPath.size() + 1); auto st = lstat(p); @@ -300,13 +302,13 @@ struct GitInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); // FIXME: maybe we should use the timestamp of the last // modified dirty file? input.attrs.insert_or_assign( "lastModified", - hasHead ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); + hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); return {std::move(storePath), input}; } From b9c969a86667a7b16a7efa1df0e3d090ef2ead72 Mon Sep 17 00:00:00 2001 From: Rehno Lindeque Date: Wed, 6 Apr 2022 12:20:39 -0400 Subject: [PATCH 25/27] nix flake check: Warn about deprecated nixosModule output --- src/nix/flake.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index ce7dc101a..a876bb3af 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -508,6 +508,7 @@ struct CmdFlakeCheck : FlakeCommand name == "defaultBundler" ? "bundlers..default" : name == "overlay" ? "overlays.default" : name == "devShell" ? "devShells..default" : + name == "nixosModule" ? "nixosModules.default" : ""; if (replacement != "") warn("flake output attribute '%s' is deprecated; use '%s' instead", name, replacement); From 5ff4c42608bdf54d6f889f196f59a6c20c1631d6 Mon Sep 17 00:00:00 2001 From: Rehno Lindeque Date: Wed, 6 Apr 2022 12:24:35 -0400 Subject: [PATCH 26/27] Update release notes --- doc/manual/src/release-notes/rl-next.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 4f3c9ce41..8c8c0fd41 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -32,3 +32,11 @@ paths. Like fixed-output derivations, impure derivations have access to the network. Only fixed-output derivations and impure derivations can depend on an impure derivation. + +* The `nixosModule` flake output attribute has been renamed consistent + with the `.default` renames in nix 2.7. + + * `nixosModule` → `nixosModules.default` + + As before, the old output will continue to work, but `nix flake check` will + issue a warning about it. From 305d3a0ec3c7d53a5ceffee239c6cd4949f99423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 7 Apr 2022 17:31:12 +0200 Subject: [PATCH 27/27] Test fetchgit with path containing a `.` segment --- tests/fetchGit.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index ac23be5c0..9179e2071 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -7,7 +7,9 @@ fi clearStore -repo=$TEST_ROOT/git +# Intentionally not in a canonical form +# See https://github.com/NixOS/nix/issues/6195 +repo=$TEST_ROOT/./git export _NIX_FORCE_HTTP=1