From c5a488afc05372dd46e05559ef04ac9969808507 Mon Sep 17 00:00:00 2001 From: jakobrs Date: Thu, 19 Mar 2020 19:15:55 +0100 Subject: [PATCH 01/32] Remove the --delete option for --gc Running `nix-store --gc --delete` will, as of Nix 2.3.3, simply fail because the --delete option conflicts with the --delete operation. $ nix-store --gc --delete error: only one operation may be specified Try 'nix-store --help' for more information. Furthermore, it has been broken since at least Nix 0.16 (which was released sometime in 2010), which means that any scripts which depend on it should have been broken at least nine years ago. This commit simply formally removes the option. There should be no actual difference in behaviour as far as the user is concerned: it errors with the exact same error message. The manual has been edited to remove any references to the (now gone) --delete option. Other information: * Path for Nix 0.16 used: /nix/store/rp3sgmskn0p0pj1ia2qwd5al6f6pinz4-nix-0.16 --- doc/manual/command-ref/nix-store.xml | 17 ++++------------- src/nix-store/nix-store.cc | 1 - 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index 113a3c2e4..e8bbd16e8 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -360,7 +360,6 @@ EOF - bytes @@ -407,14 +406,6 @@ the Nix store not reachable via file system references from a set of - - - This operation performs an actual garbage - collection. All dead paths are removed from the - store. This is the default. - - - By default, all unreachable paths are deleted. The following @@ -444,10 +435,10 @@ and keep-derivations variables in the Nix configuration file. -With , the collector prints the total -number of freed bytes when it finishes (or when it is interrupted). -With , it prints the number of bytes that -would be freed. +By default, the collector prints the total number of freed bytes +when it finishes (or when it is interrupted). With +, it prints the number of bytes that would +be freed. diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 45e152c47..c14d900ae 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -577,7 +577,6 @@ static void opGC(Strings opFlags, Strings opArgs) if (*i == "--print-roots") printRoots = true; else if (*i == "--print-live") options.action = GCOptions::gcReturnLive; else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead; - else if (*i == "--delete") options.action = GCOptions::gcDeleteDead; else if (*i == "--max-freed") { long long maxFreed = getIntArg(*i, i, opFlags.end(), true); options.maxFreed = maxFreed >= 0 ? maxFreed : 0; From afb78ebd34bff9a701d70041abc2ff211390584e Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Fri, 20 Mar 2020 21:21:56 +0100 Subject: [PATCH 02/32] libstore: disable resolve-system-dependencies hook This is used to determine the dependency tree of impure libraries so nix knows what paths to open in the sandbox. With the less restrictive defaults it isn't needed anymore. --- src/libstore/globals.hh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 782870547..3aa3653f3 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -311,12 +311,7 @@ public: Setting printMissing{this, true, "print-missing", "Whether to print what paths need to be built or downloaded."}; - Setting preBuildHook{this, -#if __APPLE__ - nixLibexecDir + "/nix/resolve-system-dependencies", -#else - "", -#endif + Setting preBuildHook{this, "", "pre-build-hook", "A program to run just before a build to set derivation-specific build settings."}; From 7f2df903d91cd21ab05223344ee4dec0a7d52c41 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Fri, 20 Mar 2020 21:31:20 +0100 Subject: [PATCH 03/32] libstore: relax default sandbox-paths on darwin --- src/libstore/globals.cc | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index a8945996e..1a2fcbe22 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -20,13 +20,6 @@ namespace nix { must be deleted and recreated on startup.) */ #define DEFAULT_SOCKET_PATH "/daemon-socket/socket" -/* chroot-like behavior from Apple's sandbox */ -#if __APPLE__ - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh" -#else - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "" -#endif - Settings settings; static GlobalConfig::Register r1(&settings); @@ -68,7 +61,12 @@ Settings::Settings() sandboxPaths = tokenizeString("/bin/sh=" SANDBOX_SHELL); #endif - allowedImpureHostPrefixes = tokenizeString(DEFAULT_ALLOWED_IMPURE_PREFIXES); + +/* chroot-like behavior from Apple's sandbox */ +#if __APPLE__ + sandboxPaths = tokenizeString("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /private/tmp /private/var/tmp /usr/lib"); + allowedImpureHostPrefixes = tokenizeString("/System/Library /usr/lib /dev /bin/sh"); +#endif } void loadConfFile() From f6c122aaeb08cc3d9e89465b440b25c7e0c87d9e Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Fri, 20 Mar 2020 21:58:45 +0100 Subject: [PATCH 04/32] sandbox: allow pty devices Nix now runs builds with a pseudo-terminal to enable colored build output. --- src/libstore/sandbox-defaults.sb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb index 0299d1ee4..c09ce1729 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/sandbox-defaults.sb @@ -71,6 +71,12 @@ (literal "/dev/zero") (subpath "/dev/fd")) +; Allow pseudo-terminals. +(allow file* + (literal "/dev/ptmx") + (regex #"^/dev/pty[a-z]+") + (regex #"^/dev/ttys[0-9]+")) + ; Does nothing, but reduces build noise. (allow file* (literal "/dev/dtracehelper")) From 2e9bc1245c125f96ce53210751940067d4cf3f1c Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Fri, 20 Mar 2020 22:12:30 +0100 Subject: [PATCH 05/32] sandbox: fix /bin/sh on catalina Sadly 10.15 changed /bin/sh to a shim which executes bash, this means it can't be used anymore without also opening up the sandbox to allow bash. Failed to exec /bin/bash as variant for /bin/sh (1: Operation not permitted). --- src/libstore/globals.cc | 2 +- src/libstore/sandbox-defaults.sb | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 1a2fcbe22..7e97f3c22 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -64,7 +64,7 @@ Settings::Settings() /* chroot-like behavior from Apple's sandbox */ #if __APPLE__ - sandboxPaths = tokenizeString("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /private/tmp /private/var/tmp /usr/lib"); + sandboxPaths = tokenizeString("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /bin/bash /private/tmp /private/var/tmp /usr/lib"); allowedImpureHostPrefixes = tokenizeString("/System/Library /usr/lib /dev /bin/sh"); #endif } diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb index c09ce1729..351037822 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/sandbox-defaults.sb @@ -91,3 +91,7 @@ (literal "/etc") (literal "/var") (literal "/private/var/tmp")) + +; This is used by /bin/sh on macOS 10.15 and later. +(allow file* + (literal "/private/var/select/sh")) From 9450dece24edd34613d887c7c0a1f2e58d86ecb4 Mon Sep 17 00:00:00 2001 From: Philipp Middendorf Date: Sat, 21 Mar 2020 09:31:39 +0100 Subject: [PATCH 06/32] installer: also test for xz to unpack --- scripts/install.in | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/install.in b/scripts/install.in index 4500fd54a..6709f00d4 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -36,6 +36,7 @@ tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")" require_util curl "download the binary tarball" require_util tar "unpack the binary tarball" +require_util xz "unpack the binary tarball" echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..." curl -L "$url" -o "$tarball" || oops "failed to download '$url'" From 26851dd2c22690838d391ef85b90a99fc00bf9ea Mon Sep 17 00:00:00 2001 From: Greg Price Date: Sat, 21 Mar 2020 22:03:58 -0700 Subject: [PATCH 07/32] installer: Set files read-only when copying into store After installing Nix, I found that all the files and directories initially copied into the store were writable, with mode 644 or 755: drwxr-xr-x 9 root root 4096 Dec 31 1969 /nix/store/ddmmzn4ggz1f66lwxjy64n89864yj9w9-nix-2.3.3 The reason is that that's how they were in the unpacked tarball, and the install-multi-user script used `rsync -p` without doing anything else to affect the permissions. The plain `install` script for a single-user install takes care to do a `chmod -R a-w` on each store path copied. We could do the same here with one more command; or we can pass `--chmod` to rsync, to have it write the files with the desired modes in the first place. Tested the new `rsync` command on both a Linux machine with a reasonably-modern rsync (3.1.3) and a Mac with its default, ancient, rsync 2.6.9, and it works as expected on both. Thankfully the latter is just new enough to have `--chmod`, which dates to rsync 2.6.7. --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 13762cba3..35341543e 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -567,7 +567,7 @@ install_from_extracted_nix() { cd "$EXTRACTED_NIX_PATH" _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \ - rsync -rlpt ./store/* "$NIX_ROOT/store/" + rsync -rlpt --chmod=-w ./store/* "$NIX_ROOT/store/" if [ -d "$NIX_INSTALLED_NIX" ]; then echo " Alright! We have our first nix at $NIX_INSTALLED_NIX" From e40e01c1dd58c12a763d9614ff38dd7a984c9787 Mon Sep 17 00:00:00 2001 From: Greg Price Date: Sat, 21 Mar 2020 22:20:02 -0700 Subject: [PATCH 08/32] doc: Files in the store have modes 444/555, not 644/755 This line has been this way since it was written, in 9e08f5efe in 2006. I think it was just a small mistake then; Eelco's thesis earlier that year says the permission on each file is set to 0444 or 0555 in a derivation's output as part of the build process. In any case I'm pretty sure that's the behavior now. --- doc/manual/command-ref/nix-store.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index e8bbd16e8..1ddb5408d 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -1139,7 +1139,7 @@ the information that Nix considers important. For instance, timestamps are elided because all files in the Nix store have their timestamp set to 0 anyway. Likewise, all permissions are left out except for the execute bit, because all files in the Nix store have -644 or 755 permission. +444 or 555 permission. Also, a NAR archive is canonical, meaning that “equal” paths always produce the same NAR archive. For instance, From c34a20e1f60486140f0ddc5657c077d2fdc9c7f5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Nov 2018 19:55:48 +0100 Subject: [PATCH 09/32] EvalState::allocAttr(): Add convenience method (cherry picked from commit c02da997570ac0d9b595d787bea8cb5a4e3cc1f5) --- src/libexpr/attr-set.cc | 6 ++++++ src/libexpr/eval.hh | 1 + 2 files changed, 7 insertions(+) diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index 0785897d2..b1d61a285 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -43,6 +43,12 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name) } +Value * EvalState::allocAttr(Value & vAttrs, const std::string & name) +{ + return allocAttr(vAttrs, symbols.create(name)); +} + + void Bindings::sort() { std::sort(begin(), end()); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index eac53b894..5efad8707 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -272,6 +272,7 @@ public: Env & allocEnv(size_t size); Value * allocAttr(Value & vAttrs, const Symbol & name); + Value * allocAttr(Value & vAttrs, const std::string & name); Bindings * allocBindings(size_t capacity); From 5a7e7fc35f934872a9e3e9e9043ff6894b36637a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Feb 2020 15:53:59 +0100 Subject: [PATCH 10/32] Use std::string_view (cherry picked from commit 6529490cc10018d5191e50c482ac1180b96b1a3c) --- src/libstore/derivations.cc | 6 +++--- src/libstore/derivations.hh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 205b90e55..973ddc86a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -65,7 +65,7 @@ bool BasicDerivation::isBuiltin() const StorePath writeDerivation(ref store, - const Derivation & drv, const string & name, RepairFlag repair) + const Derivation & drv, std::string_view name, RepairFlag repair) { auto references = cloneStorePathSet(drv.inputSrcs); for (auto & i : drv.inputDrvs) @@ -73,8 +73,8 @@ StorePath writeDerivation(ref store, /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be held during a garbage collection). */ - string suffix = name + drvExtension; - string contents = drv.unparse(*store, false); + auto suffix = std::string(name) + drvExtension; + auto contents = drv.unparse(*store, false); return settings.readOnlyMode ? store->computeStorePathForText(suffix, contents, references) : store->addTextToStore(suffix, contents, references, repair); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index c2df66229..7222d25e5 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -79,7 +79,7 @@ class Store; /* Write a derivation to the Nix store, and return its path. */ StorePath writeDerivation(ref store, - const Derivation & drv, const string & name, RepairFlag repair = NoRepair); + const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair); /* Read a derivation from a file. */ Derivation readDerivation(const Store & store, const Path & drvPath); From edc34cc1a2865777cee6b325f705fb218955b593 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 May 2019 21:09:52 +0200 Subject: [PATCH 11/32] Add function for quoting strings (cherry picked from commit 7dcf5b011a0942ecf953f2b607c4c8d0e9e652c7) --- src/libstore/build.cc | 2 +- src/libstore/store-api.cc | 7 +------ src/libutil/util.cc | 22 ---------------------- src/libutil/util.hh | 22 ++++++++++++++++++++-- 4 files changed, 22 insertions(+), 31 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9c6aedfa5..7efa81a62 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1904,7 +1904,7 @@ void DerivationGoal::startBuilder() concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), worker.store.printStorePath(drvPath), settings.thisSystem, - concatStringsSep(", ", settings.systemFeatures)); + concatStringsSep(", ", settings.systemFeatures)); if (drv->isBuiltin()) preloadNSS(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b043feb0a..aacbe5e0b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -741,12 +741,7 @@ std::string Store::showPaths(const StorePathSet & paths) string showPaths(const PathSet & paths) { - string s; - for (auto & i : paths) { - if (s.size() != 0) s += ", "; - s += "'" + i + "'"; - } - return s; + return concatStringsSep(", ", quoteStrings(paths)); } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 012f1d071..52a1394f2 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1205,28 +1205,6 @@ template StringSet tokenizeString(std::string_view s, const string & separators) template vector tokenizeString(std::string_view s, const string & separators); -string concatStringsSep(const string & sep, const Strings & ss) -{ - string s; - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - - -string concatStringsSep(const string & sep, const StringSet & ss) -{ - string s; - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - - string chomp(const string & s) { size_t i = s.find_last_not_of(" \n\r\t"); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 3bfebcd15..559ed4332 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -345,8 +345,26 @@ template C tokenizeString(std::string_view s, const string & separators /* Concatenate the given strings with a separator between the elements. */ -string concatStringsSep(const string & sep, const Strings & ss); -string concatStringsSep(const string & sep, const StringSet & ss); +template +string concatStringsSep(const string & sep, const C & ss) +{ + string s; + for (auto & i : ss) { + if (s.size() != 0) s += sep; + s += i; + } + return s; +} + + +/* Add quotes around a collection of strings. */ +template Strings quoteStrings(const C & c) +{ + Strings res; + for (auto & s : c) + res.push_back("'" + s + "'"); + return res; +} /* Remove trailing whitespace from a string. */ From 1eb952d27ac0714a99a5f7b0e8c7034e7ac0bc0d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Apr 2019 14:20:58 +0200 Subject: [PATCH 12/32] findAlongAttrPath(): Throw AttrPathNotFound (cherry picked from commit 6b0ca8e803710342af70e257935724c5ad84ca04) --- src/libexpr/attr-path.cc | 4 ++-- src/libexpr/attr-path.hh | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 06b472d8b..843585631 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -70,7 +70,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); if (a == v->attrs->end()) - throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath); + throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath); v = &*a->value; } @@ -82,7 +82,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, % attrPath % showType(*v)); if (attrIndex >= v->listSize()) - throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath); + throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath); v = v->listElems()[attrIndex]; } diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index 716e5ba27..fcccc39c8 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -7,6 +7,8 @@ namespace nix { +MakeError(AttrPathNotFound, Error); + Value * findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Value & vIn); From c1ca4f0accdb96295b27dadacd20b3db745e1d2d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 7 Feb 2020 14:08:24 +0100 Subject: [PATCH 13/32] findAlongAttrPath(): Return position (cherry picked from commit 0b013a54dc570395bed887369f8dd622b8ce337b) --- src/libexpr/attr-path.cc | 9 ++++++--- src/libexpr/attr-path.hh | 2 +- src/nix-build/nix-build.cc | 2 +- src/nix-env/nix-env.cc | 4 ++-- src/nix-instantiate/nix-instantiate.cc | 2 +- src/nix-prefetch-url/nix-prefetch-url.cc | 2 +- src/nix/installables.cc | 2 +- src/nix/upgrade-nix.cc | 2 +- 8 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 843585631..1e22f5708 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -32,7 +32,7 @@ static Strings parseAttrPath(const string & s) } -Value * findAlongAttrPath(EvalState & state, const string & attrPath, +std::pair findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Value & vIn) { Strings tokens = parseAttrPath(attrPath); @@ -41,6 +41,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, Error(format("attribute selection path '%1%' does not match expression") % attrPath); Value * v = &vIn; + Pos pos = noPos; for (auto & attr : tokens) { @@ -72,6 +73,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, if (a == v->attrs->end()) throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath); v = &*a->value; + pos = *a->pos; } else if (apType == apIndex) { @@ -85,11 +87,12 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath); v = v->listElems()[attrIndex]; + pos = noPos; } } - return v; + return {v, pos}; } @@ -98,7 +101,7 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what) Value * v2; try { auto dummyArgs = state.allocBindings(0); - v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v); + v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v).first; } catch (Error &) { throw Error("package '%s' has no source location information", what); } diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index fcccc39c8..3e78e2899 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -9,7 +9,7 @@ namespace nix { MakeError(AttrPathNotFound, Error); -Value * findAlongAttrPath(EvalState & state, const string & attrPath, +std::pair findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Value & vIn); /* Heuristic to find the filename and lineno or a nix value. */ diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 27ec7d0fe..1bda159d0 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -314,7 +314,7 @@ static void _main(int argc, char * * argv) state->eval(e, vRoot); for (auto & i : attrPaths) { - Value & v(*findAlongAttrPath(*state, i, *autoArgs, vRoot)); + Value & v(*findAlongAttrPath(*state, i, *autoArgs, vRoot).first); state->forceValue(v); getDerivations(*state, v, "", *autoArgs, drvs, false); } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 106dfe0b6..c7e553d0c 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -178,7 +178,7 @@ static void loadDerivations(EvalState & state, Path nixExprPath, Value vRoot; loadSourceExpr(state, nixExprPath, vRoot); - Value & v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot)); + Value & v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot).first); getDerivations(state, v, pathPrefix, autoArgs, elems, true); @@ -408,7 +408,7 @@ static void queryInstSources(EvalState & state, Value vRoot; loadSourceExpr(state, instSource.nixExprPath, vRoot); for (auto & i : args) { - Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot)); + Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot).first); getDerivations(state, v, "", *instSource.autoArgs, elems, true); } break; diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 5a886d69d..617d927a4 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -39,7 +39,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, state.eval(e, vRoot); for (auto & i : attrPaths) { - Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot)); + Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot).first); state.forceValue(v); PathSet context; diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index cc0891811..18ced94b1 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -120,7 +120,7 @@ static int _main(int argc, char * * argv) Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); Value vRoot; state->evalFile(path, vRoot); - Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot)); + Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); state->forceAttrs(v); /* Extract the URI. */ diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 8ce6bd06e..eb6fca62b 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -193,7 +193,7 @@ struct InstallableAttrPath : InstallableValue Bindings & autoArgs = *cmd.getAutoArgs(state); - Value * v = findAlongAttrPath(state, attrPath, autoArgs, *source); + auto v = findAlongAttrPath(state, attrPath, autoArgs, *source).first; state.forceValue(*v); return v; diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 87f1f9d1b..c05c29517 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -145,7 +145,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto v = state->allocValue(); state->eval(state->parseExprFromString(*res.data, "/no-such-path"), *v); Bindings & bindings(*state->allocBindings(0)); - auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v); + auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v).first; return store->parseStorePath(state->forceString(*v2)); } From 231a8aa2c2196a3ba6d6bd3dc9d6c50e23e20a8d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 7 Feb 2020 14:22:01 +0100 Subject: [PATCH 14/32] nix edit: Support non-derivation attributes E.g. $ nix edit .#nixosConfigurations.bla now works. (cherry picked from commit d2032edb2f86e955a8a7724a27c0c3225f386500) --- src/libexpr/attr-path.cc | 2 +- src/libexpr/attr-path.hh | 1 + src/nix/command.hh | 9 ++------- src/nix/edit.cc | 10 ++++++++-- src/nix/eval.cc | 2 +- src/nix/installables.cc | 10 +++++----- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 1e22f5708..4545bfd72 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -103,7 +103,7 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what) auto dummyArgs = state.allocBindings(0); v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v).first; } catch (Error &) { - throw Error("package '%s' has no source location information", what); + throw NoPositionInfo("package '%s' has no source location information", what); } // FIXME: is it possible to extract the Pos object instead of doing this diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index 3e78e2899..fce160da7 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -8,6 +8,7 @@ namespace nix { MakeError(AttrPathNotFound, Error); +MakeError(NoPositionInfo, Error); std::pair findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Value & vIn); diff --git a/src/nix/command.hh b/src/nix/command.hh index 00eb46903..a954a7d04 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -3,17 +3,12 @@ #include "args.hh" #include "common-eval-args.hh" #include "path.hh" +#include "eval.hh" namespace nix { extern std::string programPath; -struct Value; -class Bindings; -class EvalState; -struct Pos; -class Store; - /* A command that requires a Nix store. */ struct StoreCommand : virtual Command { @@ -48,7 +43,7 @@ struct Installable Buildable toBuildable(); - virtual Value * toValue(EvalState & state) + virtual std::pair toValue(EvalState & state) { throw Error("argument '%s' cannot be evaluated", what()); } diff --git a/src/nix/edit.cc b/src/nix/edit.cc index ca410cd1f..1683eada0 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -29,9 +29,15 @@ struct CmdEdit : InstallableCommand { auto state = getEvalState(); - auto v = installable->toValue(*state); + auto [v, pos] = installable->toValue(*state); - Pos pos = findDerivationFilename(*state, *v, installable->what()); + try { + pos = findDerivationFilename(*state, *v, installable->what()); + } catch (NoPositionInfo &) { + } + + if (pos == noPos) + throw Error("cannot find position information for '%s", installable->what()); stopProgressBar(); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 276fdf003..6398fc58e 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -52,7 +52,7 @@ struct CmdEval : MixJSON, InstallableCommand auto state = getEvalState(); - auto v = installable->toValue(*state); + auto v = installable->toValue(*state).first; PathSet context; stopProgressBar(); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index eb6fca62b..013218cd9 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -121,7 +121,7 @@ struct InstallableValue : Installable { auto state = cmd.getEvalState(); - auto v = toValue(*state); + auto v = toValue(*state).first; Bindings & autoArgs = *cmd.getAutoArgs(*state); @@ -169,11 +169,11 @@ struct InstallableExpr : InstallableValue std::string what() override { return text; } - Value * toValue(EvalState & state) override + std::pair toValue(EvalState & state) override { auto v = state.allocValue(); state.eval(state.parseExprFromString(text, absPath(".")), *v); - return v; + return {v, noPos}; } }; @@ -187,7 +187,7 @@ struct InstallableAttrPath : InstallableValue std::string what() override { return attrPath; } - Value * toValue(EvalState & state) override + std::pair toValue(EvalState & state) override { auto source = cmd.getSourceExpr(state); @@ -196,7 +196,7 @@ struct InstallableAttrPath : InstallableValue auto v = findAlongAttrPath(state, attrPath, autoArgs, *source).first; state.forceValue(*v); - return v; + return {v, noPos}; } }; From 76e7d958ed8a1502cb060300093c120311d10cac Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jan 2020 21:58:07 +0100 Subject: [PATCH 15/32] Fix coverage build https://hydra.nixos.org/build/110757285 (cherry picked from commit b430a81a1fbf6c792ba49e3aefe46256263430e5) --- src/libexpr/function-trace.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/function-trace.cc b/src/libexpr/function-trace.cc index af1486f78..c6057b384 100644 --- a/src/libexpr/function-trace.cc +++ b/src/libexpr/function-trace.cc @@ -1,4 +1,5 @@ #include "function-trace.hh" +#include "logging.hh" namespace nix { From f9611c7ae48abd4506a93691c25f067a0e8e73b4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 17:17:15 +0200 Subject: [PATCH 16/32] buildenv: Eliminate global variables, other cleanup (cherry picked from commit b82f75464d1e5ae9a00d8004e5dd7b1ca05059e4) --- src/libstore/build.cc | 1 + src/libstore/builtins.hh | 1 - src/libstore/builtins/buildenv.cc | 128 ++++++++++++++---------------- src/libstore/builtins/buildenv.hh | 21 +++++ 4 files changed, 81 insertions(+), 70 deletions(-) create mode 100644 src/libstore/builtins/buildenv.hh diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 7efa81a62..dc6a2c606 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -6,6 +6,7 @@ #include "archive.hh" #include "affinity.hh" #include "builtins.hh" +#include "builtins/buildenv.hh" #include "download.hh" #include "finally.hh" #include "compression.hh" diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index 87d6ce665..66597e456 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -6,7 +6,6 @@ namespace nix { // TODO: make pluggable. void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); -void builtinBuildenv(const BasicDerivation & drv); void builtinUnpackChannel(const BasicDerivation & drv); } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 096593886..c1c85d0bf 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -1,4 +1,4 @@ -#include "builtins.hh" +#include "buildenv.hh" #include #include @@ -7,16 +7,14 @@ namespace nix { -typedef std::map Priorities; - -// FIXME: change into local variables. - -static Priorities priorities; - -static unsigned long symlinks; +struct State +{ + std::map priorities; + unsigned long symlinks = 0; +}; /* For each activated package, create symlinks */ -static void createLinks(const Path & srcDir, const Path & dstDir, int priority) +static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority) { DirEntries srcFiles; @@ -67,7 +65,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) auto res = lstat(dstFile.c_str(), &dstSt); if (res == 0) { if (S_ISDIR(dstSt.st_mode)) { - createLinks(srcFile, dstFile, priority); + createLinks(state, srcFile, dstFile, priority); continue; } else if (S_ISLNK(dstSt.st_mode)) { auto target = canonPath(dstFile, true); @@ -77,8 +75,8 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) throw SysError(format("unlinking '%1%'") % dstFile); if (mkdir(dstFile.c_str(), 0755) == -1) throw SysError(format("creating directory '%1%'")); - createLinks(target, dstFile, priorities[dstFile]); - createLinks(srcFile, dstFile, priority); + createLinks(state, target, dstFile, state.priorities[dstFile]); + createLinks(state, srcFile, dstFile, priority); continue; } } else if (errno != ENOENT) @@ -90,7 +88,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) auto res = lstat(dstFile.c_str(), &dstSt); if (res == 0) { if (S_ISLNK(dstSt.st_mode)) { - auto prevPriority = priorities[dstFile]; + auto prevPriority = state.priorities[dstFile]; if (prevPriority == priority) throw Error( "packages '%1%' and '%2%' have the same priority %3%; " @@ -109,67 +107,30 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) } createSymlink(srcFile, dstFile); - priorities[dstFile] = priority; - symlinks++; + state.priorities[dstFile] = priority; + state.symlinks++; } } -typedef std::set FileProp; - -static FileProp done; -static FileProp postponed = FileProp{}; - -static Path out; - -static void addPkg(const Path & pkgDir, int priority) +void buildProfile(const Path & out, Packages && pkgs) { - if (!done.insert(pkgDir).second) return; - createLinks(pkgDir, out, priority); + State state; - try { - for (const auto & p : tokenizeString>( - readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n")) - if (!done.count(p)) - postponed.insert(p); - } catch (SysError & e) { - if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw; - } -} + std::set done, postponed; -struct Package { - Path path; - bool active; - int priority; - Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} -}; + auto addPkg = [&](const Path & pkgDir, int priority) { + if (!done.insert(pkgDir).second) return; + createLinks(state, pkgDir, out, priority); -typedef std::vector Packages; - -void builtinBuildenv(const BasicDerivation & drv) -{ - auto getAttr = [&](const string & name) { - auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error("attribute '%s' missing", name); - return i->second; - }; - - out = getAttr("out"); - createDirs(out); - - /* Convert the stuff we get from the environment back into a - * coherent data type. */ - Packages pkgs; - auto derivations = tokenizeString(getAttr("derivations")); - while (!derivations.empty()) { - /* !!! We're trusting the caller to structure derivations env var correctly */ - auto active = derivations.front(); derivations.pop_front(); - auto priority = stoi(derivations.front()); derivations.pop_front(); - auto outputs = stoi(derivations.front()); derivations.pop_front(); - for (auto n = 0; n < outputs; n++) { - auto path = derivations.front(); derivations.pop_front(); - pkgs.emplace_back(path, active != "false", priority); + try { + for (const auto & p : tokenizeString>( + readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n")) + if (!done.count(p)) + postponed.insert(p); + } catch (SysError & e) { + if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw; } - } + }; /* Symlink to the packages that have been installed explicitly by the * user. Process in priority order to reduce unnecessary @@ -189,13 +150,42 @@ void builtinBuildenv(const BasicDerivation & drv) */ auto priorityCounter = 1000; while (!postponed.empty()) { - auto pkgDirs = postponed; - postponed = FileProp{}; + std::set pkgDirs; + postponed.swap(pkgDirs); for (const auto & pkgDir : pkgDirs) addPkg(pkgDir, priorityCounter++); } - printError("created %d symlinks in user environment", symlinks); + printError("created %d symlinks in user environment", state.symlinks); +} + +void builtinBuildenv(const BasicDerivation & drv) +{ + auto getAttr = [&](const string & name) { + auto i = drv.env.find(name); + if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + return i->second; + }; + + Path out = getAttr("out"); + createDirs(out); + + /* Convert the stuff we get from the environment back into a + * coherent data type. */ + Packages pkgs; + auto derivations = tokenizeString(getAttr("derivations")); + while (!derivations.empty()) { + /* !!! We're trusting the caller to structure derivations env var correctly */ + auto active = derivations.front(); derivations.pop_front(); + auto priority = stoi(derivations.front()); derivations.pop_front(); + auto outputs = stoi(derivations.front()); derivations.pop_front(); + for (auto n = 0; n < outputs; n++) { + auto path = derivations.front(); derivations.pop_front(); + pkgs.emplace_back(path, active != "false", priority); + } + } + + buildProfile(out, std::move(pkgs)); createSymlink(getAttr("manifest"), out + "/manifest.nix"); } diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh new file mode 100644 index 000000000..0a37459b0 --- /dev/null +++ b/src/libstore/builtins/buildenv.hh @@ -0,0 +1,21 @@ +#pragma once + +#include "derivations.hh" +#include "store-api.hh" + +namespace nix { + +struct Package { + Path path; + bool active; + int priority; + Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} +}; + +typedef std::vector Packages; + +void buildProfile(const Path & out, Packages && pkgs); + +void builtinBuildenv(const BasicDerivation & drv); + +} From 4260a22a55d7afc931b22e8c41282fbd0ed18a77 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 Jan 2020 15:34:06 +0100 Subject: [PATCH 17/32] absPath(): Use std::optional (cherry picked from commit 1bf9eb21b75f0d93d9c1633ea2e6fdf840047e79) --- src/libutil/util.cc | 6 +++--- src/libutil/util.hh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 52a1394f2..097ff210a 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -97,10 +97,10 @@ void replaceEnv(std::map newEnv) } -Path absPath(Path path, Path dir) +Path absPath(Path path, std::optional dir) { if (path[0] != '/') { - if (dir == "") { + if (!dir) { #ifdef __GNU__ /* GNU (aka. GNU/Hurd) doesn't have any limitation on path lengths and doesn't define `PATH_MAX'. */ @@ -116,7 +116,7 @@ Path absPath(Path path, Path dir) free(buf); #endif } - path = dir + "/" + path; + path = *dir + "/" + path; } return canonPath(path); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 559ed4332..8c2cef9e1 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -46,7 +46,7 @@ void clearEnv(); /* Return an absolutized path, resolving paths relative to the specified directory, or the current directory otherwise. The path is also canonicalised. */ -Path absPath(Path path, Path dir = ""); +Path absPath(Path path, std::optional dir = {}); /* Canonicalise a path by removing all `.' or `..' components and double or trailing slashes. Optionally resolves all symlink From 7a8de57d3e7e53a4f6d8060c7201f3fbbe6cd325 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 24 Mar 2020 14:17:10 +0100 Subject: [PATCH 18/32] Pretty-print 'nix why-depends' / 'nix-store -q --tree' output Extracted from 678301072f05b650dc15c5edb4c25f08f0d6cace. --- src/libutil/util.hh | 34 ++++++++++++++++++++++++++++++++++ src/nix-store/nix-store.cc | 15 +++++---------- src/nix/why-depends.cc | 7 +------ tests/dependencies.sh | 8 ++++---- 4 files changed, 44 insertions(+), 20 deletions(-) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 8c2cef9e1..acef682cd 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -451,6 +451,13 @@ void ignoreException(); #define ANSI_BLUE "\e[34;1m" +/* Tree formatting. */ +constexpr char treeConn[] = "├───"; +constexpr char treeLast[] = "└───"; +constexpr char treeLine[] = "│ "; +constexpr char treeNull[] = " "; + + /* Truncate a string to 'width' printable characters. If 'filterAll' is true, all ANSI escape sequences are filtered out. Otherwise, some escape sequences (such as colour setting) are copied but not @@ -576,4 +583,31 @@ extern PathFilter defaultPathFilter; AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); +// A Rust/Python-like enumerate() iterator adapter. +// Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. +template ())), + typename = decltype(std::end(std::declval()))> +constexpr auto enumerate(T && iterable) +{ + struct iterator + { + size_t i; + TIter iter; + bool operator != (const iterator & other) const { return iter != other.iter; } + void operator ++ () { ++i; ++iter; } + auto operator * () const { return std::tie(i, *iter); } + }; + + struct iterable_wrapper + { + T iterable; + auto begin() { return iterator{ 0, std::begin(iterable) }; } + auto end() { return iterator{ 0, std::end(iterable) }; } + }; + + return iterable_wrapper{ std::forward(iterable) }; +} + + } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index c14d900ae..806ab7563 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -229,12 +229,6 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput, /* Some code to print a tree representation of a derivation dependency graph. Topological sorting is used to keep the tree relatively flat. */ - -const string treeConn = "+---"; -const string treeLine = "| "; -const string treeNull = " "; - - static void printTree(const StorePath & path, const string & firstPad, const string & tailPad, StorePathSet & done) { @@ -254,10 +248,11 @@ static void printTree(const StorePath & path, auto sorted = store->topoSortPaths(info->references); reverse(sorted.begin(), sorted.end()); - for (auto i = sorted.begin(); i != sorted.end(); ++i) { - auto j = i; ++j; - printTree(*i, tailPad + treeConn, - j == sorted.end() ? tailPad + treeNull : tailPad + treeLine, + for (const auto &[n, i] : enumerate(sorted)) { + bool last = n + 1 == sorted.size(); + printTree(i, + tailPad + (last ? treeLast : treeConn), + tailPad + (last ? treeNull : treeLine), done); } } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index c24ae7c8e..d3b7a674a 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -143,11 +143,6 @@ struct CmdWhyDepends : SourceExprCommand and `dependency`. */ std::function printNode; - const string treeConn = "╠═══"; - const string treeLast = "╚═══"; - const string treeLine = "║ "; - const string treeNull = " "; - struct BailOut { }; printNode = [&](Node & node, const string & firstPad, const string & tailPad) { @@ -157,7 +152,7 @@ struct CmdWhyDepends : SourceExprCommand std::cout << fmt("%s%s%s%s" ANSI_NORMAL "\n", firstPad, node.visited ? "\e[38;5;244m" : "", - firstPad != "" ? "=> " : "", + firstPad != "" ? "→ " : "", pathS); if (node.path == dependencyPath && !all diff --git a/tests/dependencies.sh b/tests/dependencies.sh index df204d185..8d0fdc10f 100644 --- a/tests/dependencies.sh +++ b/tests/dependencies.sh @@ -6,7 +6,7 @@ drvPath=$(nix-instantiate dependencies.nix) echo "derivation is $drvPath" -nix-store -q --tree "$drvPath" | grep ' +---.*builder1.sh' +nix-store -q --tree "$drvPath" | grep '───.*builder1.sh' # Test Graphviz graph generation. nix-store -q --graph "$drvPath" > $TEST_ROOT/graph @@ -22,9 +22,9 @@ nix-store -q --graph "$outPath" > $TEST_ROOT/graph if test -n "$dot"; then # Does it parse? $dot < $TEST_ROOT/graph -fi +fi -nix-store -q --tree "$outPath" | grep '+---.*dependencies-input-2' +nix-store -q --tree "$outPath" | grep '───.*dependencies-input-2' echo "output path is $outPath" @@ -49,4 +49,4 @@ nix-store -q --referrers-closure "$input2OutPath" | grep "$outPath" # Check that the derivers are set properly. test $(nix-store -q --deriver "$outPath") = "$drvPath" -nix-store -q --deriver "$input2OutPath" | grep -q -- "-input-2.drv" +nix-store -q --deriver "$input2OutPath" | grep -q -- "-input-2.drv" From 777e21e596ec85f18eddf8b286d9ba4995010835 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 11 Feb 2020 23:50:16 +0100 Subject: [PATCH 19/32] nix path-info --json: Print hash in SRI format (cherry picked from commit 442e665d6d3fcbdee7dece2f62a597142f8784b1) --- src/libstore/store-api.cc | 6 ++++-- src/libstore/store-api.hh | 1 + src/nix/path-info.cc | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index aacbe5e0b..b9e894a9a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -441,7 +441,9 @@ string Store::makeValidityRegistration(const StorePathSet & paths, void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid) + bool includeImpureInfo, bool showClosureSize, + Base hashBase, + AllowInvalidFlag allowInvalid) { auto jsonList = jsonOut.list(); @@ -453,7 +455,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store auto info = queryPathInfo(storePath); jsonPath - .attr("narHash", info->narHash.to_string()) + .attr("narHash", info->narHash.to_string(hashBase)) .attr("narSize", info->narSize); { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e0484de13..c57a6935e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -561,6 +561,7 @@ public: each path is included. */ void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, + Base hashBase = Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); /* Return the size of the closure of the specified path, that is, diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index bffa7b356..45ec297d2 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -89,7 +89,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON store->pathInfoToJSON(jsonRoot, // FIXME: preserve order? storePathsToSet(storePaths), - true, showClosureSize, AllowInvalid); + true, showClosureSize, SRI, AllowInvalid); } else { From 6b824c78f1f6c2de95b594a440118e259d28def0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Dec 2019 15:56:37 +0100 Subject: [PATCH 20/32] nix: Add --refresh as an alias for --tarball-ttl 0 (cherry picked from commit e721f99817bb7154d8098c902e25f84521a90b7f) --- src/nix/main.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/nix/main.cc b/src/nix/main.cc index 41d44f431..5cb1ad25e 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -92,6 +92,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .longName("no-net") .description("disable substituters and consider all previously downloaded files up-to-date") .handler([&]() { useNet = false; }); + + mkFlag() + .longName("refresh") + .description("consider all previously downloaded files out-of-date") + .handler([&]() { settings.tarballTtl = 0; }); } void printFlags(std::ostream & out) override From c85097da7c1ef06e2f1912c922110ff358de3aa2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 28 Jan 2020 17:34:48 +0100 Subject: [PATCH 21/32] Fix --refresh with --no-net https://hydra.nixos.org/build/110879699 (cherry picked from commit 5bbe793abf18414878a069399d1759673d693fb6) --- src/nix/main.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index 5cb1ad25e..3b5f5516f 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -55,6 +55,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { bool printBuildLogs = false; bool useNet = true; + bool refresh = false; NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { @@ -96,7 +97,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs mkFlag() .longName("refresh") .description("consider all previously downloaded files out-of-date") - .handler([&]() { settings.tarballTtl = 0; }); + .handler([&]() { refresh = true; }); } void printFlags(std::ostream & out) override @@ -181,6 +182,9 @@ void mainWrapped(int argc, char * * argv) downloadSettings.connectTimeout = 1; } + if (args.refresh) + settings.tarballTtl = 0; + args.command->prepare(); args.command->run(); } From 0a10854f857e519d1455ff0f0be1f8401c40a11c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 24 Mar 2020 14:26:13 +0100 Subject: [PATCH 22/32] Misc changes from the flakes branch --- .gitignore | 2 +- Makefile.config.in | 1 + src/libexpr/eval.hh | 2 +- src/libstore/build.cc | 16 ++++++++-------- src/libstore/builtins/buildenv.cc | 2 +- src/libstore/http-binary-cache-store.cc | 5 +++-- src/libstore/local-store.cc | 13 +++---------- src/libstore/nar-info-disk-cache.cc | 7 +------ src/libstore/nar-info-disk-cache.hh | 2 +- src/libstore/parsed-derivations.cc | 6 +++++- src/libstore/parsed-derivations.hh | 10 ++++++---- src/libstore/profiles.cc | 18 ++++++++++++++++++ src/libstore/profiles.hh | 4 ++++ src/libstore/sqlite.cc | 25 +++++++++++++++++++++++-- src/libstore/sqlite.hh | 10 +++++++--- src/libstore/store-api.hh | 5 +++++ src/libutil/hash.hh | 12 ++++++++++++ src/libutil/util.hh | 5 +++-- src/nix-env/nix-env.cc | 17 ++--------------- tests/binary-cache.sh | 4 ++-- tests/config.nix.in | 2 +- tests/fetchGit.sh | 2 ++ 22 files changed, 110 insertions(+), 60 deletions(-) diff --git a/.gitignore b/.gitignore index e10c75418..e3186fa76 100644 --- a/.gitignore +++ b/.gitignore @@ -47,7 +47,7 @@ perl/Makefile.config /src/libexpr/nix.tbl # /src/libstore/ -/src/libstore/*.gen.hh +*.gen.* /src/nix/nix diff --git a/Makefile.config.in b/Makefile.config.in index fe609ce06..e7a12089a 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -37,6 +37,7 @@ prefix = @prefix@ sandbox_shell = @sandbox_shell@ storedir = @storedir@ sysconfdir = @sysconfdir@ +system = @system@ doc_generate = @doc_generate@ xmllint = @xmllint@ xsltproc = @xsltproc@ diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 5efad8707..1485dc7fe 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -368,7 +368,7 @@ struct EvalSettings : Config "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; Setting traceFunctionCalls{this, false, "trace-function-calls", - "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"}; + "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."}; }; extern EvalSettings evalSettings; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index dc6a2c606..0e3a23a4d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1398,7 +1398,7 @@ void DerivationGoal::tryToBuild() few seconds and then retry this goal. */ PathSet lockFiles; for (auto & outPath : drv->outputPaths()) - lockFiles.insert(worker.store.toRealPath(worker.store.printStorePath(outPath))); + lockFiles.insert(worker.store.Store::toRealPath(outPath)); if (!outputLocks.lockPaths(lockFiles, "", false)) { worker.waitForAWhile(shared_from_this()); @@ -1429,7 +1429,7 @@ void DerivationGoal::tryToBuild() for (auto & i : drv->outputs) { if (worker.store.isValidPath(i.second.path)) continue; debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path)); - deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second.path))); + deletePath(worker.store.Store::toRealPath(i.second.path)); } /* Don't do a remote build if the derivation has the attribute @@ -1686,7 +1686,7 @@ void DerivationGoal::buildDone() /* Delete unused redirected outputs (when doing hash rewriting). */ for (auto & i : redirectedOutputs) - deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second))); + deletePath(worker.store.Store::toRealPath(i.second)); /* Delete the chroot (if we were using one). */ autoDelChroot.reset(); /* this runs the destructor */ @@ -2072,7 +2072,7 @@ void DerivationGoal::startBuilder() environment using bind-mounts. We put it in the Nix store to ensure that we can create hard-links to non-directory inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.toRealPath(worker.store.printStorePath(drvPath)) + ".chroot"; + chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; deletePath(chrootRootDir); /* Clean up the chroot directory automatically. */ @@ -2551,7 +2551,7 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); void DerivationGoal::writeStructuredAttrs() { - auto & structuredAttrs = parsedDrv->getStructuredAttrs(); + auto structuredAttrs = parsedDrv->getStructuredAttrs(); if (!structuredAttrs) return; auto json = *structuredAttrs; @@ -2917,7 +2917,7 @@ void DerivationGoal::addDependency(const StorePath & path) #if __linux__ - Path source = worker.store.toRealPath(worker.store.printStorePath(path)); + Path source = worker.store.Store::toRealPath(path); Path target = chrootRootDir + worker.store.printStorePath(path); debug("bind-mounting %s -> %s", target, source); @@ -3579,7 +3579,7 @@ void DerivationGoal::registerOutputs() if (needsHashRewrite()) { auto r = redirectedOutputs.find(i.second.path); if (r != redirectedOutputs.end()) { - auto redirected = worker.store.toRealPath(worker.store.printStorePath(r->second)); + auto redirected = worker.store.Store::toRealPath(r->second); if (buildMode == bmRepair && redirectedBadOutputs.count(i.second.path) && pathExists(redirected)) @@ -3672,7 +3672,7 @@ void DerivationGoal::registerOutputs() BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI))); - Path actualDest = worker.store.toRealPath(worker.store.printStorePath(dest)); + Path actualDest = worker.store.Store::toRealPath(dest); if (worker.store.isValidPath(dest)) std::rethrow_exception(delayedException); diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index c1c85d0bf..1b802d908 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -156,7 +156,7 @@ void buildProfile(const Path & out, Packages && pkgs) addPkg(pkgDir, priorityCounter++); } - printError("created %d symlinks in user environment", state.symlinks); + debug("created %d symlinks in user environment", state.symlinks); } void builtinBuildenv(const BasicDerivation & drv) diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index d4ae36662..011794c62 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -163,10 +163,11 @@ static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr { + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; if (std::string(uri, 0, 7) != "http://" && std::string(uri, 0, 8) != "https://" && - (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://") - ) return 0; + (!forceHttp || std::string(uri, 0, 7) != "file://")) + return 0; auto store = std::make_shared(params, uri); store->init(); return store; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index cd2e86f29..ae7513ad8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -298,9 +298,7 @@ void LocalStore::openDB(State & state, bool create) /* Open the Nix database. */ string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - if (sqlite3_open_v2(dbPath.c_str(), &db.db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) - throw Error(format("cannot open Nix database '%1%'") % dbPath); + state.db = SQLite(dbPath, create); #ifdef __CYGWIN__ /* The cygwin version of sqlite3 has a patch which calls @@ -312,11 +310,6 @@ void LocalStore::openDB(State & state, bool create) SetDllDirectoryW(L""); #endif - if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(db, "setting timeout"); - - db.exec("pragma foreign_keys = 1"); - /* !!! check whether sqlite has been built with foreign key support */ @@ -350,7 +343,7 @@ void LocalStore::openDB(State & state, bool create) /* Initialise the database schema, if necessary. */ if (create) { - const char * schema = + static const char schema[] = #include "schema.sql.gen.hh" ; db.exec(schema); @@ -1275,7 +1268,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) else hashSink = std::make_unique(info->narHash.type, storePathToHash(printStorePath(info->path))); - dumpPath(toRealPath(printStorePath(i)), *hashSink); + dumpPath(Store::toRealPath(i), *hashSink); auto current = hashSink->finish(); if (info->narHash != nullHash && info->narHash != current.first) { diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 907645d86..442541330 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -78,12 +78,7 @@ public: state->db = SQLite(dbPath); - if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(state->db, "setting timeout"); - - // We can always reproduce the cache. - state->db.exec("pragma synchronous = off"); - state->db.exec("pragma main.journal_mode = truncate"); + state->db.isCache(); state->db.exec(schema); diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 878acbb87..04de2c5eb 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -10,7 +10,7 @@ class NarInfoDiskCache public: typedef enum { oValid, oInvalid, oUnknown } Outcome; - virtual ~NarInfoDiskCache() { }; + virtual ~NarInfoDiskCache() { } virtual void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) = 0; diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index d0f289a0f..45c033c66 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -1,5 +1,7 @@ #include "parsed-derivations.hh" +#include + namespace nix { ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv) @@ -9,13 +11,15 @@ ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv) auto jsonAttr = drv.env.find("__json"); if (jsonAttr != drv.env.end()) { try { - structuredAttrs = nlohmann::json::parse(jsonAttr->second); + structuredAttrs = std::make_unique(nlohmann::json::parse(jsonAttr->second)); } catch (std::exception & e) { throw Error("cannot process __json attribute of '%s': %s", drvPath.to_string(), e.what()); } } } +ParsedDerivation::~ParsedDerivation() { } + std::optional ParsedDerivation::getStringAttr(const std::string & name) const { if (structuredAttrs) { diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index cec868754..f4df5dd54 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,6 +1,6 @@ #include "derivations.hh" -#include +#include namespace nix { @@ -8,15 +8,17 @@ class ParsedDerivation { StorePath drvPath; BasicDerivation & drv; - std::optional structuredAttrs; + std::unique_ptr structuredAttrs; public: ParsedDerivation(StorePath && drvPath, BasicDerivation & drv); - const std::optional & getStructuredAttrs() const + ~ParsedDerivation(); + + const nlohmann::json * getStructuredAttrs() const { - return structuredAttrs; + return structuredAttrs.get(); } std::optional getStringAttr(const std::string & name) const; diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index dae3f2d32..2bef51878 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -256,4 +256,22 @@ string optimisticLockProfile(const Path & profile) } +Path getDefaultProfile() +{ + Path profileLink = getHome() + "/.nix-profile"; + try { + if (!pathExists(profileLink)) { + replaceSymlink( + getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), + profileLink); + } + return absPath(readLink(profileLink), dirOf(profileLink)); + } catch (Error &) { + return profileLink; + } +} + + } diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 5fa1533de..78645d8b6 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -64,4 +64,8 @@ void lockProfile(PathLocks & lock, const Path & profile); rebuilt. */ string optimisticLockProfile(const Path & profile); +/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create + it. */ +Path getDefaultProfile(); + } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index a061d64f3..eb1daafc5 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -25,11 +25,16 @@ namespace nix { throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path); } -SQLite::SQLite(const Path & path) +SQLite::SQLite(const Path & path, bool create) { if (sqlite3_open_v2(path.c_str(), &db, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) throw Error(format("cannot open SQLite database '%s'") % path); + + if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) + throwSQLiteError(db, "setting timeout"); + + exec("pragma foreign_keys = 1"); } SQLite::~SQLite() @@ -42,6 +47,12 @@ SQLite::~SQLite() } } +void SQLite::isCache() +{ + exec("pragma synchronous = off"); + exec("pragma main.journal_mode = truncate"); +} + void SQLite::exec(const std::string & stmt) { retrySQLite([&]() { @@ -94,6 +105,16 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool return *this; } +SQLiteStmt::Use & SQLiteStmt::Use::operator () (const unsigned char * data, size_t len, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_blob(stmt, curArg++, data, len, SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) { if (notNull) { diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index bd012d9b9..fd04c9b07 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -5,8 +5,8 @@ #include "types.hh" -class sqlite3; -class sqlite3_stmt; +struct sqlite3; +struct sqlite3_stmt; namespace nix { @@ -15,13 +15,16 @@ struct SQLite { sqlite3 * db = 0; SQLite() { } - SQLite(const Path & path); + SQLite(const Path & path, bool create = true); SQLite(const SQLite & from) = delete; SQLite& operator = (const SQLite & from) = delete; SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; } ~SQLite(); operator sqlite3 * () { return db; } + /* Disable synchronous mode, set truncate journal mode. */ + void isCache(); + void exec(const std::string & stmt); }; @@ -52,6 +55,7 @@ struct SQLiteStmt /* Bind the next parameter. */ Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); Use & bind(); // null diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index c57a6935e..0fa59be6a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -677,6 +677,11 @@ public: return storePath; } + Path toRealPath(const StorePath & storePath) + { + return toRealPath(printStorePath(storePath)); + } + virtual void createUser(const std::string & userName, uid_t userId) { } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ffa43ecf5..ea9fca3e7 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -80,6 +80,18 @@ struct Hash or base-64. By default, this is prefixed by the hash type (e.g. "sha256:"). */ std::string to_string(Base base = Base32, bool includeType = true) const; + + std::string gitRev() const + { + assert(type == htSHA1); + return to_string(Base16, false); + } + + std::string gitShortRev() const + { + assert(type == htSHA1); + return std::string(to_string(Base16, false), 0, 7); + } }; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index acef682cd..7c3a30242 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -476,10 +476,11 @@ string base64Decode(const string & s); /* Get a value for the specified key from an associate container, or a default value if the key doesn't exist. */ template -std::optional get(const T & map, const std::string & key) +std::optional get(const T & map, const typename T::key_type & key) { auto i = map.find(key); - return i == map.end() ? std::optional() : i->second; + if (i == map.end()) return {}; + return std::optional(i->second); } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index c7e553d0c..e47b00acf 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1428,21 +1428,8 @@ static int _main(int argc, char * * argv) if (globals.profile == "") globals.profile = getEnv("NIX_PROFILE").value_or(""); - if (globals.profile == "") { - Path profileLink = getHome() + "/.nix-profile"; - try { - if (!pathExists(profileLink)) { - replaceSymlink( - getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), - profileLink); - } - globals.profile = absPath(readLink(profileLink), dirOf(profileLink)); - } catch (Error &) { - globals.profile = profileLink; - } - } + if (globals.profile == "") + globals.profile = getDefaultProfile(); op(globals, opFlags, opArgs); diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index eb58ae7c1..a3c3c7847 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -48,7 +48,7 @@ basicTests # Test HttpBinaryCacheStore. -export _NIX_FORCE_HTTP_BINARY_CACHE_STORE=1 +export _NIX_FORCE_HTTP=1 basicTests @@ -126,7 +126,7 @@ badKey="$(cat $TEST_ROOT/pk2)" res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3)) otherKey="$(cat $TEST_ROOT/pk3)" -_NIX_FORCE_HTTP_BINARY_CACHE_STORE= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath +_NIX_FORCE_HTTP= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath # Downloading should fail if we don't provide a key. diff --git a/tests/config.nix.in b/tests/config.nix.in index 51aed539c..0ec2eba6b 100644 --- a/tests/config.nix.in +++ b/tests/config.nix.in @@ -3,7 +3,7 @@ rec { path = "@coreutils@"; - system = builtins.currentSystem; + system = "@system@"; shared = builtins.getEnv "_NIX_TEST_SHARED"; diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 4c46bdf04..ed8fa14d6 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -9,6 +9,8 @@ clearStore repo=$TEST_ROOT/git +export _NIX_FORCE_HTTP=1 + rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2 git init $repo From 7313aa267b5be1e5264e4577e7bc3daec2fef282 Mon Sep 17 00:00:00 2001 From: Greg Price Date: Tue, 24 Mar 2020 21:15:01 -0700 Subject: [PATCH 23/32] installer: Fix terminal colors. The install-multi-user script uses blue, green, and red colors, as well as bold and underline, to add helpful formatting that helps structure its rather voluminous output. Unfortunately, the terminal escape sequences it uses are not quite well-formed. The relevant information is all there, just obscured by some extra noise, a leading parameter `38`. Empirically, the result is: * On macOS, in both Terminal.app and iTerm2, the spurious `38` is ignored, the rest of the escape sequence is applied, and the colors show up as intended. * On Linux, in at least gnome-terminal and xterm, the spurious `38` and the next parameter after it are ignored, and what's left is applied. So in the sequence `38;4;32`, the 4 (underline) is ignored but the 32 (green) takes effect; in a more typical sequence like `38;34`, the 34 (blue) is ignored and nothing happens. These codes are all unchanged since this script's origins as a Darwin-only script -- so the fact that they work fine in common macOS terminals goes some way to explain how the bug arose. Happily, we can make the colors work as intended by just deleting the extra `38;`. Tested in all four terminals mentioned above; the new codes work correctly on all of them, and on the two macOS terminals they work exactly the same as before. --- In a bit more technical detail -- perhaps more than anyone, me included, ever wanted to know, but now that I've gone and learned it I'll write it down anyway :) -- here's what's happening in these codes: An ECMA-48 "control sequence" begins with `\033[` aka "CSI", contains any number of parameters as semicolon-separated decimal numbers (plus sometimes other wrinkles), and ends with a byte from 0x40..0x7e. In our case, with `m` aka "SGR", "Select Graphic Rendition". An SGR control sequence `\033[...m` sets colors, fonts, text styles, etc. In particular a parameter `31` means red, `32` green, `34` blue, `4` underline, and `0` means reset to normal. Those are all we use. There is also a `38`. This is used for setting colors too... but it needs arguments. `38;5;nn` is color nn from a 256-color palette, and `38;2;rr;gg;bb` has the given RGB values. There is no meaning defined for `38;1` or `38;34` etc. On seeing a parameter `38` followed by an unrecognized argument for it, apparently some implementations (as seen on macOS) discard only the `38` and others (as seen on Linux) discard the argument too before resuming. --- scripts/install-multi-user.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 35341543e..a0f1deb98 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -13,12 +13,12 @@ set -o pipefail # however tracking which bits came from which would be impossible. readonly ESC='\033[0m' -readonly BOLD='\033[38;1m' -readonly BLUE='\033[38;34m' -readonly BLUE_UL='\033[38;4;34m' -readonly GREEN='\033[38;32m' -readonly GREEN_UL='\033[38;4;32m' -readonly RED='\033[38;31m' +readonly BOLD='\033[1m' +readonly BLUE='\033[34m' +readonly BLUE_UL='\033[4;34m' +readonly GREEN='\033[32m' +readonly GREEN_UL='\033[4;32m' +readonly RED='\033[31m' readonly NIX_USER_COUNT="32" readonly NIX_BUILD_GROUP_ID="30000" From e322a16523c3c0644a900453abbe32bd0fd982e3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2020 14:29:29 +0200 Subject: [PATCH 24/32] Remove global -I flags (cherry picked from commit 2c692a3b144523bca68dd6de618124ba6c9bb332) --- local.mk | 4 +++- mk/precompiled-headers.mk | 4 ++-- src/build-remote/build-remote.cc | 2 +- src/libexpr/local.mk | 2 ++ src/libexpr/primops/fromTOML.cc | 2 +- src/libmain/local.mk | 2 ++ src/libstore/local.mk | 3 ++- src/nix-build/nix-build.cc | 2 +- src/nix-channel/nix-channel.cc | 2 +- src/nix-collect-garbage/nix-collect-garbage.cc | 2 +- src/nix-copy-closure/nix-copy-closure.cc | 2 +- src/nix-daemon/nix-daemon.cc | 2 +- src/nix-env/nix-env.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- src/nix-prefetch-url/nix-prefetch-url.cc | 4 ++-- src/nix-store/nix-store.cc | 2 +- src/nix/local.mk | 2 ++ tests/plugins/local.mk | 2 ++ 18 files changed, 27 insertions(+), 16 deletions(-) diff --git a/local.mk b/local.mk index 55f85a044..d254c10fe 100644 --- a/local.mk +++ b/local.mk @@ -6,9 +6,11 @@ dist-files += configure config.h.in perl/configure clean-files += Makefile.config -GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix -Wno-deprecated-declarations +GLOBAL_CXXFLAGS += -Wno-deprecated-declarations $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) $(GCH) $(PCH): src/libutil/util.hh config.h + +GCH_CXXFLAGS = -I src/libutil diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index 1a727ba1b..1c0452dc2 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -8,14 +8,14 @@ GCH = $(buildprefix)precompiled-headers.h.gch $(GCH): precompiled-headers.h @rm -f $@ @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) + $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) PCH = $(buildprefix)precompiled-headers.h.pch $(PCH): precompiled-headers.h @rm -f $@ @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) + $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) clean-files += $(GCH) $(PCH) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 69d1c6f7e..00340b787 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -17,7 +17,7 @@ #include "store-api.hh" #include "derivations.hh" #include "local-store.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" using namespace nix; using std::cin; diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 8a9b3c2ea..a4ccab376 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -6,6 +6,8 @@ libexpr_DIR := $(d) libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc +libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain -I src/libexpr + libexpr_LIBS = libutil libstore libnixrust libexpr_LDFLAGS = diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index a84e569e9..c43324dbb 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -1,7 +1,7 @@ #include "primops.hh" #include "eval-inline.hh" -#include "cpptoml/cpptoml.h" +#include "../../cpptoml/cpptoml.h" namespace nix { diff --git a/src/libmain/local.mk b/src/libmain/local.mk index 0c80f5a0a..a8eed6c65 100644 --- a/src/libmain/local.mk +++ b/src/libmain/local.mk @@ -6,6 +6,8 @@ libmain_DIR := $(d) libmain_SOURCES := $(wildcard $(d)/*.cc) +libmain_CXXFLAGS += -I src/libutil -I src/libstore + libmain_LDFLAGS = $(OPENSSL_LIBS) libmain_LIBS = libstore libutil diff --git a/src/libstore/local.mk b/src/libstore/local.mk index ac68c2342..91acef368 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -31,7 +31,8 @@ ifeq ($(HAVE_SECCOMP), 1) libstore_LDFLAGS += -lseccomp endif -libstore_CXXFLAGS = \ +libstore_CXXFLAGS += \ + -I src/libutil -I src/libstore \ -DNIX_PREFIX=\"$(prefix)\" \ -DNIX_STORE_DIR=\"$(storedir)\" \ -DNIX_DATA_DIR=\"$(datadir)\" \ diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 1bda159d0..0a058a31b 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -16,7 +16,7 @@ #include "get-drvs.hh" #include "common-eval-args.hh" #include "attr-path.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" using namespace nix; using namespace std::string_literals; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 1b337a712..a2639579d 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -2,7 +2,7 @@ #include "globals.hh" #include "download.hh" #include "store-api.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" #include #include diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index d4060ac93..aa5ada3a6 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -2,7 +2,7 @@ #include "profiles.hh" #include "shared.hh" #include "globals.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" #include #include diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index f87035760..b10184718 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -1,6 +1,6 @@ #include "shared.hh" #include "store-api.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" using namespace nix; diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 134898561..e68d1b1be 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -6,7 +6,7 @@ #include "globals.hh" #include "derivations.hh" #include "finally.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" #include "daemon.hh" #include diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index e47b00acf..1a2bb42a3 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -13,7 +13,7 @@ #include "json.hh" #include "value-to-json.hh" #include "xml-writer.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" #include #include diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 617d927a4..6c99d1181 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -9,7 +9,7 @@ #include "util.hh" #include "store-api.hh" #include "common-eval-args.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" #include #include diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 18ced94b1..2b9254659 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -6,9 +6,9 @@ #include "eval-inline.hh" #include "common-eval-args.hh" #include "attr-path.hh" -#include "legacy.hh" #include "finally.hh" -#include "progress-bar.hh" +#include "../nix/legacy.hh" +#include "../nix/progress-bar.hh" #include "tarfile.hh" #include diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 806ab7563..fcc00175a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -9,7 +9,7 @@ #include "util.hh" #include "worker-protocol.hh" #include "graphml.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" #include #include diff --git a/src/nix/local.mk b/src/nix/local.mk index 51dad101f..50a18efd7 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -15,6 +15,8 @@ nix_SOURCES := \ $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ +nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain + nix_LIBS = libexpr libmain libstore libutil libnixrust nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system diff --git a/tests/plugins/local.mk b/tests/plugins/local.mk index 1d2bac052..82ad99402 100644 --- a/tests/plugins/local.mk +++ b/tests/plugins/local.mk @@ -7,3 +7,5 @@ libplugintest_SOURCES := $(d)/plugintest.cc libplugintest_ALLOW_UNDEFINED := 1 libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1 + +libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr From 367577d9a6ec465d773f7b3ba53a5656253f514d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2020 17:00:40 +0200 Subject: [PATCH 25/32] Fix macOS build --- src/resolve-system-dependencies/local.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk index bf65f7905..f0e82e023 100644 --- a/src/resolve-system-dependencies/local.mk +++ b/src/resolve-system-dependencies/local.mk @@ -6,6 +6,8 @@ resolve-system-dependencies_DIR := $(d) resolve-system-dependencies_INSTALL_DIR := $(libexecdir)/nix +resolve-system-dependencies_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain + resolve-system-dependencies_LIBS := libstore libmain libutil libnixrust resolve-system-dependencies_SOURCES := $(d)/resolve-system-dependencies.cc From e1a94ad852e91fcdcf4efb60fe7e9b9e328df7ac Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2020 19:14:17 +0200 Subject: [PATCH 26/32] Backport 'nix dev-shell' from the flakes branch This also adds a '--profile' option to 'nix build' (replacing 'nix-env --set'). --- src/libutil/util.cc | 11 ++ src/libutil/util.hh | 12 +- src/nix/build.cc | 17 ++- src/nix/command.cc | 94 ++++++++++++ src/nix/command.hh | 62 ++++---- src/nix/installables.cc | 5 + src/nix/installables.hh | 45 ++++++ src/nix/run.cc | 120 ++++++--------- src/nix/shell.cc | 319 ++++++++++++++++++++++++++++++++++++++++ 9 files changed, 569 insertions(+), 116 deletions(-) create mode 100644 src/nix/installables.hh create mode 100644 src/nix/shell.cc diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 097ff210a..332c1c43a 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -478,6 +478,17 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, } +std::pair createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + return {std::move(fd), tmpl}; +} + + std::string getUserName() { auto pw = getpwuid(geteuid()); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 7c3a30242..1f85c7c46 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -122,10 +122,6 @@ void deletePath(const Path & path); void deletePath(const Path & path, unsigned long long & bytesFreed); -/* Create a temporary directory. */ -Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); - std::string getUserName(); /* Return $HOME or the user's home directory from /etc/passwd. */ @@ -205,6 +201,14 @@ public: }; +/* Create a temporary directory. */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +/* Create a temporary file, returning a file handle and its path. */ +std::pair createTempFile(const Path & prefix = "nix"); + + class Pipe { public: diff --git a/src/nix/build.cc b/src/nix/build.cc index 3c9d2df39..0b0762836 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -5,7 +5,7 @@ using namespace nix; -struct CmdBuild : MixDryRun, InstallablesCommand +struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile { Path outLink = "result"; @@ -40,6 +40,10 @@ struct CmdBuild : MixDryRun, InstallablesCommand "To build the build.x86_64-linux attribute from release.nix:", "nix build -f release.nix build.x86_64-linux" }, + Example{ + "To make a profile point at GNU Hello:", + "nix build --profile /tmp/profile nixpkgs.hello" + }, }; } @@ -49,18 +53,19 @@ struct CmdBuild : MixDryRun, InstallablesCommand if (dryRun) return; - for (size_t i = 0; i < buildables.size(); ++i) { - auto & b(buildables[i]); - - if (outLink != "") - for (auto & output : b.outputs) + if (outLink != "") { + for (size_t i = 0; i < buildables.size(); ++i) { + for (auto & output : buildables[i].outputs) if (auto store2 = store.dynamic_pointer_cast()) { std::string symlink = outLink; if (i) symlink += fmt("-%d", i); if (output.first != "out") symlink += fmt("-%s", output.first); store2->addPermRoot(output.second, absPath(symlink), true); } + } } + + updateProfile(buildables); } }; diff --git a/src/nix/command.cc b/src/nix/command.cc index 442bc6c53..99b24d2a2 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -2,6 +2,9 @@ #include "store-api.hh" #include "derivations.hh" #include "nixexpr.hh" +#include "profiles.hh" + +extern char * * environ; namespace nix { @@ -96,4 +99,95 @@ Strings editorFor(const Pos & pos) return args; } +MixProfile::MixProfile() +{ + mkFlag() + .longName("profile") + .description("profile to update") + .labels({"path"}) + .dest(&profile); +} + +void MixProfile::updateProfile(const StorePath & storePath) +{ + if (!profile) return; + auto store = getStore().dynamic_pointer_cast(); + if (!store) throw Error("'--profile' is not supported for this Nix store"); + auto profile2 = absPath(*profile); + switchLink(profile2, + createGeneration( + ref(store), + profile2, store->printStorePath(storePath))); +} + +void MixProfile::updateProfile(const Buildables & buildables) +{ + if (!profile) return; + + std::optional result; + + for (auto & buildable : buildables) { + for (auto & output : buildable.outputs) { + if (result) + throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple"); + result = output.second.clone(); + } + } + + if (!result) + throw Error("'--profile' requires that the arguments produce a single store path, but there are none"); + + updateProfile(*result); +} + +MixDefaultProfile::MixDefaultProfile() +{ + profile = getDefaultProfile(); +} + +MixEnvironment::MixEnvironment() : ignoreEnvironment(false) { + mkFlag() + .longName("ignore-environment") + .shortName('i') + .description("clear the entire environment (except those specified with --keep)") + .set(&ignoreEnvironment, true); + + mkFlag() + .longName("keep") + .shortName('k') + .description("keep specified environment variable") + .arity(1) + .labels({"name"}) + .handler([&](std::vector ss) { keep.insert(ss.front()); }); + + mkFlag() + .longName("unset") + .shortName('u') + .description("unset specified environment variable") + .arity(1) + .labels({"name"}) + .handler([&](std::vector ss) { unset.insert(ss.front()); }); +} + +void MixEnvironment::setEnviron() { + if (ignoreEnvironment) { + if (!unset.empty()) + throw UsageError("--unset does not make sense with --ignore-environment"); + + for (const auto & var : keep) { + auto val = getenv(var.c_str()); + if (val) stringsEnv.emplace_back(fmt("%s=%s", var.c_str(), val)); + } + + vectorEnv = stringsToCharPtrs(stringsEnv); + environ = vectorEnv.data(); + } else { + if (!keep.empty()) + throw UsageError("--keep does not make sense without --ignore-environment"); + + for (const auto & var : unset) + unsetenv(var.c_str()); + } +} + } diff --git a/src/nix/command.hh b/src/nix/command.hh index a954a7d04..23f5c9898 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -1,5 +1,6 @@ #pragma once +#include "installables.hh" #include "args.hh" #include "common-eval-args.hh" #include "path.hh" @@ -22,34 +23,7 @@ private: std::shared_ptr _store; }; -struct Buildable -{ - std::optional drvPath; - std::map outputs; -}; - -typedef std::vector Buildables; - -struct Installable -{ - virtual ~Installable() { } - - virtual std::string what() = 0; - - virtual Buildables toBuildables() - { - throw Error("argument '%s' cannot be built", what()); - } - - Buildable toBuildable(); - - virtual std::pair toValue(EvalState & state) - { - throw Error("argument '%s' cannot be evaluated", what()); - } -}; - -struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs +struct SourceExprCommand : virtual StoreCommand, MixEvalArgs { Path file; @@ -184,4 +158,36 @@ std::set toDerivations(ref store, filename:lineno. */ Strings editorFor(const Pos & pos); +struct MixProfile : virtual StoreCommand +{ + std::optional profile; + + MixProfile(); + + /* If 'profile' is set, make it point at 'storePath'. */ + void updateProfile(const StorePath & storePath); + + /* If 'profile' is set, make it point at the store path produced + by 'buildables'. */ + void updateProfile(const Buildables & buildables); +}; + +struct MixDefaultProfile : MixProfile +{ + MixDefaultProfile(); +}; + +struct MixEnvironment : virtual Args { + + StringSet keep, unset; + Strings stringsEnv; + std::vector vectorEnv; + bool ignoreEnvironment; + + MixEnvironment(); + + /* Modify global environ based on ignoreEnvironment, keep, and unset. It's expected that exec will be called before this class goes out of scope, otherwise environ will become invalid. */ + void setEnviron(); +}; + } diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 013218cd9..f464d0aa1 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -109,6 +109,11 @@ struct InstallableStorePath : Installable bs.push_back(std::move(b)); return bs; } + + std::optional getStorePath() override + { + return storePath.clone(); + } }; struct InstallableValue : Installable diff --git a/src/nix/installables.hh b/src/nix/installables.hh new file mode 100644 index 000000000..503984220 --- /dev/null +++ b/src/nix/installables.hh @@ -0,0 +1,45 @@ +#pragma once + +#include "util.hh" +#include "path.hh" +#include "eval.hh" + +#include + +namespace nix { + +struct Buildable +{ + std::optional drvPath; + std::map outputs; +}; + +typedef std::vector Buildables; + +struct Installable +{ + virtual ~Installable() { } + + virtual std::string what() = 0; + + virtual Buildables toBuildables() + { + throw Error("argument '%s' cannot be built", what()); + } + + Buildable toBuildable(); + + virtual std::pair toValue(EvalState & state) + { + throw Error("argument '%s' cannot be evaluated", what()); + } + + /* Return a value only if this installable is a store path or a + symlink to it. */ + virtual std::optional getStorePath() + { + return {}; + } +}; + +} diff --git a/src/nix/run.cc b/src/nix/run.cc index f885c5e49..8e30264c0 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -8,6 +8,7 @@ #include "fs-accessor.hh" #include "progress-bar.hh" #include "affinity.hh" +#include "eval.hh" #if __linux__ #include @@ -19,11 +20,46 @@ using namespace nix; std::string chrootHelperName = "__run_in_chroot"; -struct CmdRun : InstallablesCommand +struct RunCommon : virtual Command +{ + void runProgram(ref store, + const std::string & program, + const Strings & args) + { + stopProgressBar(); + + restoreSignals(); + + restoreAffinity(); + + /* If this is a diverted store (i.e. its "logical" location + (typically /nix/store) differs from its "physical" location + (e.g. /home/eelco/nix/store), then run the command in a + chroot. For non-root users, this requires running it in new + mount and user namespaces. Unfortunately, + unshare(CLONE_NEWUSER) doesn't work in a multithreaded + program (which "nix" is), so we exec() a single-threaded + helper program (chrootHelper() below) to do the work. */ + auto store2 = store.dynamic_pointer_cast(); + + if (store2 && store->storeDir != store2->realStoreDir) { + Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, program }; + for (auto & arg : args) helperArgs.push_back(arg); + + execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); + + throw SysError("could not execute chroot helper"); + } + + execvp(program.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("unable to execute '%s'", program); + } +}; + +struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment { std::vector command = { "bash" }; - StringSet keep, unset; - bool ignoreEnvironment = false; CmdRun() { @@ -37,28 +73,6 @@ struct CmdRun : InstallablesCommand if (ss.empty()) throw UsageError("--command requires at least one argument"); command = ss; }); - - mkFlag() - .longName("ignore-environment") - .shortName('i') - .description("clear the entire environment (except those specified with --keep)") - .set(&ignoreEnvironment, true); - - mkFlag() - .longName("keep") - .shortName('k') - .description("keep specified environment variable") - .arity(1) - .labels({"name"}) - .handler([&](std::vector ss) { keep.insert(ss.front()); }); - - mkFlag() - .longName("unset") - .shortName('u') - .description("unset specified environment variable") - .arity(1) - .labels({"name"}) - .handler([&](std::vector ss) { unset.insert(ss.front()); }); } std::string description() override @@ -94,35 +108,13 @@ struct CmdRun : InstallablesCommand auto accessor = store->getFSAccessor(); - if (ignoreEnvironment) { - - if (!unset.empty()) - throw UsageError("--unset does not make sense with --ignore-environment"); - - std::map kept; - for (auto & var : keep) { - auto s = getenv(var.c_str()); - if (s) kept[var] = s; - } - - clearEnv(); - - for (auto & var : kept) - setenv(var.first.c_str(), var.second.c_str(), 1); - - } else { - - if (!keep.empty()) - throw UsageError("--keep does not make sense without --ignore-environment"); - - for (auto & var : unset) - unsetenv(var.c_str()); - } std::unordered_set done; std::queue todo; for (auto & path : outPaths) todo.push(path.clone()); + setEnviron(); + auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); while (!todo.empty()) { @@ -142,38 +134,10 @@ struct CmdRun : InstallablesCommand setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1); - std::string cmd = *command.begin(); Strings args; for (auto & arg : command) args.push_back(arg); - stopProgressBar(); - - restoreSignals(); - - restoreAffinity(); - - /* If this is a diverted store (i.e. its "logical" location - (typically /nix/store) differs from its "physical" location - (e.g. /home/eelco/nix/store), then run the command in a - chroot. For non-root users, this requires running it in new - mount and user namespaces. Unfortunately, - unshare(CLONE_NEWUSER) doesn't work in a multithreaded - program (which "nix" is), so we exec() a single-threaded - helper program (chrootHelper() below) to do the work. */ - auto store2 = store.dynamic_pointer_cast(); - - if (store2 && store->storeDir != store2->realStoreDir) { - Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, cmd }; - for (auto & arg : args) helperArgs.push_back(arg); - - execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); - - throw SysError("could not execute chroot helper"); - } - - execvp(cmd.c_str(), stringsToCharPtrs(args).data()); - - throw SysError("unable to exec '%s'", cmd); + runProgram(store, *command.begin(), args); } }; diff --git a/src/nix/shell.cc b/src/nix/shell.cc new file mode 100644 index 000000000..82b1da552 --- /dev/null +++ b/src/nix/shell.cc @@ -0,0 +1,319 @@ +#include "eval.hh" +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "affinity.hh" +#include "progress-bar.hh" + +#include + +using namespace nix; + +struct Var +{ + bool exported; + std::string value; // quoted string or array +}; + +struct BuildEnvironment +{ + std::map env; + std::string bashFunctions; +}; + +BuildEnvironment readEnvironment(const Path & path) +{ + BuildEnvironment res; + + std::set exported; + + debug("reading environment file '%s'", path); + + auto file = readFile(path); + + auto pos = file.cbegin(); + + static std::string varNameRegex = + R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re"; + + static std::regex declareRegex( + "^declare -x (" + varNameRegex + ")" + + R"re((?:="((?:[^"\\]|\\.)*)")?\n)re"); + + static std::string simpleStringRegex = + R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re"; + + static std::string quotedStringRegex = + R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re"; + + static std::string arrayRegex = + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")*\)))re"; + + static std::regex varRegex( + "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + arrayRegex + ")\n"); + + static std::regex functionRegex( + "^" + varNameRegex + " \\(\\) *\n"); + + while (pos != file.end()) { + + std::smatch match; + + if (std::regex_search(pos, file.cend(), match, declareRegex)) { + pos = match[0].second; + exported.insert(match[1]); + } + + else if (std::regex_search(pos, file.cend(), match, varRegex)) { + pos = match[0].second; + res.env.insert({match[1], Var { (bool) exported.count(match[1]), match[2] }}); + } + + else if (std::regex_search(pos, file.cend(), match, functionRegex)) { + res.bashFunctions = std::string(pos, file.cend()); + break; + } + + else throw Error("shell environment '%s' has unexpected line '%s'", + path, file.substr(pos - file.cbegin(), 60)); + } + + return res; +} + +/* Given an existing derivation, return the shell environment as + initialised by stdenv's setup script. We do this by building a + modified derivation with the same dependencies and nearly the same + initial environment variables, that just writes the resulting + environment to a file and exits. */ +StorePath getDerivationEnvironment(ref store, Derivation drv) +{ + auto builder = baseNameOf(drv.builder); + if (builder != "bash") + throw Error("'nix shell' only works on derivations that use 'bash' as their builder"); + + drv.args = { + "-c", + "set -e; " + "export IN_NIX_SHELL=impure; " + "export dontAddDisableDepTrack=1; " + "if [[ -n $stdenv ]]; then " + " source $stdenv/setup; " + "fi; " + "export > $out; " + "set >> $out "}; + + /* Remove derivation checks. */ + drv.env.erase("allowedReferences"); + drv.env.erase("allowedRequisites"); + drv.env.erase("disallowedReferences"); + drv.env.erase("disallowedRequisites"); + + // FIXME: handle structured attrs + + /* Rehash and write the derivation. FIXME: would be nice to use + 'buildDerivation', but that's privileged. */ + auto drvName = drv.env["name"] + "-env"; + for (auto & output : drv.outputs) + drv.env.erase(output.first); + drv.env["out"] = ""; + drv.env["outputs"] = "out"; + Hash h = hashDerivationModulo(*store, drv, true); + auto shellOutPath = store->makeOutputPath("out", h, drvName); + drv.outputs.insert_or_assign("out", DerivationOutput(shellOutPath.clone(), "", "")); + drv.env["out"] = store->printStorePath(shellOutPath); + auto shellDrvPath2 = writeDerivation(store, drv, drvName); + + /* Build the derivation. */ + store->buildPaths({shellDrvPath2}); + + assert(store->isValidPath(shellOutPath)); + + return shellOutPath; +} + +struct Common : InstallableCommand, MixProfile +{ + std::set ignoreVars{ + "BASHOPTS", + "EUID", + "HOME", // FIXME: don't ignore in pure mode? + "NIX_BUILD_TOP", + "NIX_ENFORCE_PURITY", + "NIX_LOG_FD", + "PPID", + "PWD", + "SHELLOPTS", + "SHLVL", + "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt + "TEMP", + "TEMPDIR", + "TERM", + "TMP", + "TMPDIR", + "TZ", + "UID", + }; + + void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out) + { + out << "nix_saved_PATH=\"$PATH\"\n"; + + for (auto & i : buildEnvironment.env) { + if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) { + out << fmt("%s=%s\n", i.first, i.second.value); + if (i.second.exported) + out << fmt("export %s\n", i.first); + } + } + + out << "PATH=\"$PATH:$nix_saved_PATH\"\n"; + + out << buildEnvironment.bashFunctions << "\n"; + + // FIXME: set outputs + + out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n"; + for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) + out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); + + out << "eval \"$shellHook\"\n"; + } + + StorePath getShellOutPath(ref store) + { + auto path = installable->getStorePath(); + if (path && hasSuffix(path->to_string(), "-env")) + return path->clone(); + else { + auto drvs = toDerivations(store, {installable}); + + if (drvs.size() != 1) + throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations", + installable->what(), drvs.size()); + + auto & drvPath = *drvs.begin(); + + return getDerivationEnvironment(store, store->derivationFromPath(drvPath)); + } + } + + BuildEnvironment getBuildEnvironment(ref store) + { + auto shellOutPath = getShellOutPath(store); + + updateProfile(shellOutPath); + + return readEnvironment(store->printStorePath(shellOutPath)); + } +}; + +struct CmdDevShell : Common, MixEnvironment +{ + std::vector command; + + CmdDevShell() + { + mkFlag() + .longName("command") + .shortName('c') + .description("command and arguments to be executed insted of an interactive shell") + .labels({"command", "args"}) + .arity(ArityAny) + .handler([&](std::vector ss) { + if (ss.empty()) throw UsageError("--command requires at least one argument"); + command = ss; + }); + } + + std::string description() override + { + return "run a bash shell that provides the build environment of a derivation"; + } + + Examples examples() override + { + return { + Example{ + "To get the build environment of GNU hello:", + "nix dev-shell nixpkgs.hello" + }, + Example{ + "To store the build environment in a profile:", + "nix dev-shell --profile /tmp/my-shell nixpkgs.hello" + }, + Example{ + "To use a build environment previously recorded in a profile:", + "nix dev-shell /tmp/my-shell" + }, + }; + } + + void run(ref store) override + { + auto buildEnvironment = getBuildEnvironment(store); + + auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); + + std::ostringstream ss; + makeRcScript(buildEnvironment, ss); + + ss << fmt("rm -f '%s'\n", rcFilePath); + + if (!command.empty()) { + std::vector args; + for (auto s : command) + args.push_back(shellEscape(s)); + ss << fmt("exec %s\n", concatStringsSep(" ", args)); + } + + writeFull(rcFileFd.get(), ss.str()); + + stopProgressBar(); + + auto shell = getEnv("SHELL").value_or("bash"); + + setEnviron(); + + auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; + + restoreAffinity(); + restoreSignals(); + + execvp(shell.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("executing shell '%s'", shell); + } +}; + +struct CmdPrintDevEnv : Common +{ + std::string description() override + { + return "print shell code that can be sourced by bash to reproduce the build environment of a derivation"; + } + + Examples examples() override + { + return { + Example{ + "To apply the build environment of GNU hello to the current shell:", + ". <(nix print-dev-env nixpkgs.hello)" + }, + }; + } + + void run(ref store) override + { + auto buildEnvironment = getBuildEnvironment(store); + + stopProgressBar(); + + makeRcScript(buildEnvironment, std::cout); + } +}; + +static auto r1 = registerCommand("print-dev-env"); +static auto r2 = registerCommand("dev-shell"); From 12556e570924315eb25ad6057b6c2c5162955e4f Mon Sep 17 00:00:00 2001 From: mlatus Date: Tue, 31 Mar 2020 19:40:16 +0800 Subject: [PATCH 27/32] fix placeholder not substituted in passAsFile --- src/libstore/build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0e3a23a4d..527d7ac42 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2465,7 +2465,7 @@ void DerivationGoal::initTmpDir() { auto hash = hashString(htSHA256, i.first); string fn = ".attr-" + hash.to_string(Base32, false); Path p = tmpDir + "/" + fn; - writeFile(p, i.second); + writeFile(p, rewriteStrings(i.second, inputRewrites)); chownToBuilder(p); env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; } From 3166b97174ab879dac3b19eb3f257e73a943fef0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 31 Mar 2020 13:45:28 +0200 Subject: [PATCH 28/32] nix shell -> nix dev-shell --- src/nix/shell.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 82b1da552..71e640667 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -92,7 +92,7 @@ StorePath getDerivationEnvironment(ref store, Derivation drv) { auto builder = baseNameOf(drv.builder); if (builder != "bash") - throw Error("'nix shell' only works on derivations that use 'bash' as their builder"); + throw Error("'nix dev-shell' only works on derivations that use 'bash' as their builder"); drv.args = { "-c", From 4fc4eb6c93511c61a278ba0e5555ffb24435ab59 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Thu, 2 Apr 2020 17:04:00 +0200 Subject: [PATCH 29/32] libexpr: remove unused attrError The attrError variable is no longer used but still allocated on every call to the findAlongAttrPath function. --- src/libexpr/attr-path.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 4545bfd72..76d101b98 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -37,9 +37,6 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr { Strings tokens = parseAttrPath(attrPath); - Error attrError = - Error(format("attribute selection path '%1%' does not match expression") % attrPath); - Value * v = &vIn; Pos pos = noPos; From 63fa92605b2b0a8a6d948a27e1d331cc46b199bd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Apr 2020 11:54:48 +0200 Subject: [PATCH 30/32] nix-env: Refuse to operate on a new-style profile This prevents users from accidentally nuking their profile via nix-env. (cherry picked from commit 021634e3e3edb327089d33ab41b743f0a40126da) --- src/nix-env/user-env.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 717431b7a..f852916d8 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -15,6 +15,8 @@ namespace nix { DrvInfos queryInstalled(EvalState & state, const Path & userEnv) { DrvInfos elems; + if (pathExists(userEnv + "/manifest.json")) + throw Error("profile '%s' is incompatible with 'nix-env'; please use 'nix profile' instead", userEnv); Path manifestFile = userEnv + "/manifest.nix"; if (pathExists(manifestFile)) { Value v; From 5e7ccdc9e3ddd61dc85e20c898001345bfb497a5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 3 Apr 2020 20:06:26 +0200 Subject: [PATCH 31/32] Publish a tarball containing the crates we depend on This is needed since we no longer produce a source tarball. (cherry picked from commit bf70a047a0b2da606f65a88f13ce2994065cd9c4) --- release.nix | 94 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 53 insertions(+), 41 deletions(-) diff --git a/release.nix b/release.nix index 9b591229a..65b035957 100644 --- a/release.nix +++ b/release.nix @@ -12,52 +12,64 @@ let builtins.readFile ./.version + (if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}"); + # Create a "vendor" directory that contains the crates listed in + # Cargo.lock. This allows Nix to be built without network access. + vendoredCrates' = + let + lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock); + + files = map (pkg: import { + url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download"; + sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)"; + }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package); + + in pkgs.runCommand "cargo-vendor-dir" {} + '' + mkdir -p $out/vendor + + cat > $out/vendor/config < "$dir/.cargo-checksum.json" + + # Clean up some cruft from the winapi crates. FIXME: find + # a way to remove winapi* from our dependencies. + if [[ $dir =~ /winapi ]]; then + find $dir -name "*.a" -print0 | xargs -0 rm -f -- + fi + + mv "$dir" $out/vendor/ + + rm -rf $out/vendor/tmp + '') files)} + ''; + jobs = rec { - # Create a "vendor" directory that contains the crates listed in - # Cargo.lock. This allows Nix to be built without network access. vendoredCrates = - let - lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock); - - files = map (pkg: import { - url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download"; - sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)"; - }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package); - - in pkgs.runCommand "cargo-vendor-dir" {} + with pkgs; + runCommand "vendored-crates" {} '' - mkdir -p $out/vendor - - cat > $out/vendor/config < "$dir/.cargo-checksum.json" - - # Clean up some cruft from the winapi crates. FIXME: find - # a way to remove winapi* from our dependencies. - if [[ $dir =~ /winapi ]]; then - find $dir -name "*.a" -print0 | xargs -0 rm -f -- - fi - - mv "$dir" $out/vendor/ - - rm -rf $out/vendor/tmp - '') files)} + mkdir -p $out/nix-support + name=nix-vendored-crates-${version} + fn=$out/$name.tar.xz + tar cvfJ $fn -C ${vendoredCrates'} vendor \ + --owner=0 --group=0 --mode=u+rw,uga+r \ + --transform "s,vendor,$name," + echo "file crates-tarball $fn" >> $out/nix-support/hydra-build-products ''; - build = pkgs.lib.genAttrs systems (system: let pkgs = import nixpkgs { inherit system; }; in @@ -89,7 +101,7 @@ let patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* ''} - ln -sfn ${vendoredCrates}/vendor/ nix-rust/vendor + ln -sfn ${vendoredCrates'}/vendor/ nix-rust/vendor (cd perl; autoreconf --install --force --verbose) ''; From c976cb0b8ab5d0f2c4ab8c9826fc7db56e2f1b3e Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Sat, 4 Apr 2020 12:27:39 -0700 Subject: [PATCH 32/32] Don't retry on "unsupported protocol" error When encountering an unsupported protocol, there's no need to retry. Chances are, it won't suddenly be supported between retry attempts; error instead. Otherwise, you see something like the following: $ nix-env -i -f git://git@github.com/foo/bar warning: unable to download 'git://git@github.com/foo/bar': Unsupported protocol (1); retrying in 335 ms warning: unable to download 'git://git@github.com/foo/bar': Unsupported protocol (1); retrying in 604 ms warning: unable to download 'git://git@github.com/foo/bar': Unsupported protocol (1); retrying in 1340 ms warning: unable to download 'git://git@github.com/foo/bar': Unsupported protocol (1); retrying in 2685 ms With this change, you now see: $ nix-env -i -f git://git@github.com/foo/bar error: unable to download 'git://git@github.com/foo/bar': Unsupported protocol (1) --- src/libstore/download.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 149c84765..5967d0425 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -390,6 +390,7 @@ struct CurlDownloader : public Downloader case CURLE_SSL_CACERT_BADFILE: case CURLE_TOO_MANY_REDIRECTS: case CURLE_WRITE_ERROR: + case CURLE_UNSUPPORTED_PROTOCOL: err = Misc; break; default: // Shut up warnings