From ca93b26db6d2d73e702ea8ecdd0a98f17ace2c7b Mon Sep 17 00:00:00 2001 From: Antoine Eiche Date: Wed, 29 Apr 2020 14:39:37 +0200 Subject: [PATCH 01/12] Only call grantpt on MacOS systems The commit 3cc1125595d97b4ab7369e37e4ad22f4cfecb8b2 adds a `grantpt` call on the builder pseudo terminal fd. This call is actually only required for MacOS, but it however requires a RW access to /dev/pts which is only RO bindmounted in the Bazel Linux sandbox. So, Nix can not be actually run in the Bazel Linux sandbox for unneeded reasons. --- src/libstore/build.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 572634765..147093fae 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2250,10 +2250,13 @@ void DerivationGoal::startBuilder() if (chown(slaveName.c_str(), buildUser->getUID(), 0)) throw SysError("changing owner of pseudoterminal slave"); - } else { + } +#if __APPLE__ + else { if (grantpt(builderOut.readSide.get())) throw SysError("granting access to pseudoterminal slave"); } +#endif #if 0 // Mount the pt in the sandbox so that the "tty" command works. From d1229859c2612587bfd04111bae6584c0c9df051 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Wed, 29 Apr 2020 22:48:29 +0200 Subject: [PATCH 02/12] Fix displaying error-position in `builtins.fetch{Tree,Tarball}` Without dereferencing this pointer, you'd get an error like this: ``` error: unsupported argument 'abc' to 'fetchTarball', at 0x13627e8 ``` --- src/libexpr/primops/fetchTree.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 43c58485a..c5a0d9886 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -108,7 +108,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError("unsupported argument '%s' to '%s', at %s", - attr.name, who, attr.pos); + attr.name, who, *attr.pos); } if (!url) From 2fcfc6c2c6ff144131d669ebfb2caa63c8b41417 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Apr 2020 13:05:29 +0200 Subject: [PATCH 03/12] nix dev-shell: Refactor script for getting the environment --- src/nix/get-env.sh | 8 ++++++++ src/nix/local.mk | 2 ++ src/nix/shell.cc | 18 ++++++++---------- 3 files changed, 18 insertions(+), 10 deletions(-) create mode 100644 src/nix/get-env.sh diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh new file mode 100644 index 000000000..782f97b75 --- /dev/null +++ b/src/nix/get-env.sh @@ -0,0 +1,8 @@ +set -e +export IN_NIX_SHELL=impure +export dontAddDisableDepTrack=1 +if [[ -n $stdenv ]]; then + source $stdenv/setup +fi +export > $out +set >> $out diff --git a/src/nix/local.mk b/src/nix/local.mk index 033675e89..15cef55bb 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -27,3 +27,5 @@ $(foreach name, \ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh + +src/nix/shell.cc: src/nix/get-env.sh.gen.hh diff --git a/src/nix/shell.cc b/src/nix/shell.cc index fbfe24bbe..c8bf35a13 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -83,6 +83,10 @@ BuildEnvironment readEnvironment(const Path & path) return res; } +const static std::string getEnvSh = + #include "get-env.sh.gen.hh" + ; + /* Given an existing derivation, return the shell environment as initialised by stdenv's setup script. We do this by building a modified derivation with the same dependencies and nearly the same @@ -94,16 +98,9 @@ StorePath getDerivationEnvironment(ref store, Derivation drv) if (builder != "bash") throw Error("'nix dev-shell' only works on derivations that use 'bash' as their builder"); - drv.args = { - "-c", - "set -e; " - "export IN_NIX_SHELL=impure; " - "export dontAddDisableDepTrack=1; " - "if [[ -n $stdenv ]]; then " - " source $stdenv/setup; " - "fi; " - "export > $out; " - "set >> $out "}; + auto getEnvShPath = store->addTextToStore("get-env.sh", getEnvSh, {}); + + drv.args = {store->printStorePath(getEnvShPath)}; /* Remove derivation checks. */ drv.env.erase("allowedReferences"); @@ -120,6 +117,7 @@ StorePath getDerivationEnvironment(ref store, Derivation drv) drv.env.erase(output.first); drv.env["out"] = ""; drv.env["outputs"] = "out"; + drv.inputSrcs.insert(std::move(getEnvShPath)); Hash h = hashDerivationModulo(*store, drv, true); auto shellOutPath = store->makeOutputPath("out", h, drvName); drv.outputs.insert_or_assign("out", DerivationOutput(shellOutPath.clone(), "", "")); From efe6c186eabeb85830be7bef9aea4dd7eb2357e7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Apr 2020 14:39:26 +0200 Subject: [PATCH 04/12] nix dev-shell: Support structured attrs Tested against https://github.com/NixOS/nixpkgs/pull/72074. Fixes #3540. --- src/nix/get-env.sh | 1 + src/nix/shell.cc | 45 ++++++++++++++++++++++++++++++++------------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index 782f97b75..a25ec43a9 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -1,4 +1,5 @@ set -e +if [ -e .attrs.sh ]; then source .attrs.sh; fi export IN_NIX_SHELL=impure export dontAddDisableDepTrack=1 if [[ -n $stdenv ]]; then diff --git a/src/nix/shell.cc b/src/nix/shell.cc index c8bf35a13..f0ffceaa7 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -13,7 +13,8 @@ using namespace nix; struct Var { - bool exported; + bool exported = true; + bool associative = false; std::string value; // quoted string or array }; @@ -48,11 +49,17 @@ BuildEnvironment readEnvironment(const Path & path) static std::string quotedStringRegex = R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re"; - static std::string arrayRegex = - R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")*\)))re"; + static std::string indexedArrayRegex = + R"re((?:\(( *\[[0-9]+]="(?:[^"\\]|\\.)*")**\)))re"; static std::regex varRegex( - "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + arrayRegex + ")\n"); + "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n"); + + /* Note: we distinguish between an indexed and associative array + using the space before the closing parenthesis. Will + undoubtedly regret this some day. */ + static std::regex assocArrayRegex( + "^(" + varNameRegex + ")=" + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")* *\)))re" + "\n"); static std::regex functionRegex( "^" + varNameRegex + " \\(\\) *\n"); @@ -68,7 +75,12 @@ BuildEnvironment readEnvironment(const Path & path) else if (std::regex_search(pos, file.cend(), match, varRegex)) { pos = match[0].second; - res.env.insert({match[1], Var { (bool) exported.count(match[1]), match[2] }}); + res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }}); + } + + else if (std::regex_search(pos, file.cend(), match, assocArrayRegex)) { + pos = match[0].second; + res.env.insert({match[1], Var { .associative = true, .value = match[2] }}); } else if (std::regex_search(pos, file.cend(), match, functionRegex)) { @@ -92,8 +104,10 @@ const static std::string getEnvSh = modified derivation with the same dependencies and nearly the same initial environment variables, that just writes the resulting environment to a file and exits. */ -StorePath getDerivationEnvironment(ref store, Derivation drv) +StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) { + auto drv = store->derivationFromPath(drvPath); + auto builder = baseNameOf(drv.builder); if (builder != "bash") throw Error("'nix dev-shell' only works on derivations that use 'bash' as their builder"); @@ -108,11 +122,12 @@ StorePath getDerivationEnvironment(ref store, Derivation drv) drv.env.erase("disallowedReferences"); drv.env.erase("disallowedRequisites"); - // FIXME: handle structured attrs - /* Rehash and write the derivation. FIXME: would be nice to use 'buildDerivation', but that's privileged. */ - auto drvName = drv.env["name"] + "-env"; + auto drvName = std::string(drvPath.name()); + assert(hasSuffix(drvName, ".drv")); + drvName.resize(drvName.size() - 4); + drvName += "-env"; for (auto & output : drv.outputs) drv.env.erase(output.first); drv.env["out"] = ""; @@ -161,9 +176,13 @@ struct Common : InstallableCommand, MixProfile for (auto & i : buildEnvironment.env) { if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) { - out << fmt("%s=%s\n", i.first, i.second.value); - if (i.second.exported) - out << fmt("export %s\n", i.first); + if (i.second.associative) + out << fmt("declare -A %s=(%s)\n", i.first, i.second.value); + else { + out << fmt("%s=%s\n", i.first, i.second.value); + if (i.second.exported) + out << fmt("export %s\n", i.first); + } } } @@ -194,7 +213,7 @@ struct Common : InstallableCommand, MixProfile auto & drvPath = *drvs.begin(); - return getDerivationEnvironment(store, store->derivationFromPath(drvPath)); + return getDerivationEnvironment(store, drvPath); } } From 0135fd6ec4dcd15771008b867f28047e5c2bb605 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Apr 2020 14:46:51 +0200 Subject: [PATCH 05/12] nix dev-shell: Unset shellHook This avoids inheriting the caller's shellHook, which can happen when running a dev-shell inside a dev-shell. --- src/nix/shell.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index f0ffceaa7..9c45a935d 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -172,6 +172,8 @@ struct Common : InstallableCommand, MixProfile void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out) { + out << "unset shellHook\n"; + out << "nix_saved_PATH=\"$PATH\"\n"; for (auto & i : buildEnvironment.env) { From a15f918cbac51dee2e82b4cb0535e7ae5f4ff587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 30 Apr 2020 14:57:40 +0200 Subject: [PATCH 06/12] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 26 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 20 +++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..5c9e1d9cc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,26 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**Steps To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**`nix-env --version` output: ** + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..392ed30c6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: improvement +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. From 625868b33dab37c0750081843be8091e22808376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 30 Apr 2020 15:01:23 +0200 Subject: [PATCH 07/12] Update issue templates --- .github/ISSUE_TEMPLATE.md | 27 --------------------------- .github/ISSUE_TEMPLATE/bug_report.md | 10 ++++++++-- 2 files changed, 8 insertions(+), 29 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 3372b1f03..000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,27 +0,0 @@ - diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 5c9e1d9cc..e6d346bc1 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -8,19 +8,25 @@ assignees: '' --- **Describe the bug** + A clear and concise description of what the bug is. +If you have a problem with a specific package or NixOS, +you probably want to file an issue at https://github.com/NixOS/nixpkgs/issues. + **Steps To Reproduce** -Steps to reproduce the behavior: + 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error **Expected behavior** + A clear and concise description of what you expected to happen. -**`nix-env --version` output: ** +**`nix-env --version` output** **Additional context** + Add any other context about the problem here. From 533343628dcd4cacc8ff74d429372bfbd860a683 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Thu, 30 Apr 2020 22:01:28 +0200 Subject: [PATCH 08/12] Revamp README.md --- README.md | 57 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 9c0c87887..9aaca6ed3 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,55 @@ +# Nix + [![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos) [![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions) -Nix, the purely functional package manager ------------------------------------------- +Nix is a powerful package manager for Linux and other Unix systems that makes package +management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual) +for more details. -Nix is a new take on package management that is fairly unique. Because of its -purity aspects, a lot of issues found in traditional package managers don't -appear with Nix. +## Installation -To find out more about the tool, usage and installation instructions, please -read the manual, which is available on the Nix website at -. +On Linux and macOS the easiest way to Install Nix is to run the following shell command +(as a user other than root): -## Contributing +``` +$ curl -L https://nixos.org/nix/install | sh +``` -Take a look at the [Hacking Section](https://nixos.org/nix/manual/#chap-hacking) -of the manual. It helps you to get started with building Nix from source. +Information on additional installation methods is available on the [Nix download page](https://nixos.org/download.html). + +## Building And Developing + +### Building Nix + +You can build Nix via Nix via one of the targets provided by [release.nix](./release.nix): + +``` +$ nix-build ./release.nix -A build.aarch64-linux +$ nix-build ./release.nix -A build.x86_64-darwin +$ nix-build ./release.nix -A build.i686-linux +$ nix-build ./release.nix -A build.x86_64-linux +``` + +### Development Environment + +You can use the provided `shell.nix` to easily bootstrap working development environment: + +``` +$ nix-shell +$ ./bootstrap.sh +$ ./configure +$ make +``` + +## Additional Resources + +- [Nix manual](https://nixos.org/nix/manual) +- [Nix jobs on nixos.hydra.org](https://hydra.nixos.org/project/nix) +- [Nix - A One Pager](https://github.com/tazjin/nix-1p) +- [NixOS Discourse](https://discourse.nixos.org/) +- #nix / #nixos on irc.freenode.net ## License -Nix is released under the LGPL v2.1 +Nix is released under the [LGPL v2.1](./COPYING) From 536512d273f1d5821f65e9df77fe8f47b823ce6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 1 May 2020 11:17:38 +0200 Subject: [PATCH 09/12] Update README.md Co-authored-by: worldofpeace --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9aaca6ed3..454b0ffa0 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ $ nix-build ./release.nix -A build.x86_64-linux ### Development Environment -You can use the provided `shell.nix` to easily bootstrap working development environment: +You can use the provided `shell.nix` to easily bootstrap a working development environment: ``` $ nix-shell From 30616d8e86f7ef635de561905eab0022000547b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 1 May 2020 11:19:18 +0200 Subject: [PATCH 10/12] Update README.md Co-authored-by: worldofpeace --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 454b0ffa0..0f2f9bd1a 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ $ make ## Additional Resources - [Nix manual](https://nixos.org/nix/manual) -- [Nix jobs on nixos.hydra.org](https://hydra.nixos.org/project/nix) +- [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) - [Nix - A One Pager](https://github.com/tazjin/nix-1p) - [NixOS Discourse](https://discourse.nixos.org/) - #nix / #nixos on irc.freenode.net From 2a434fc62bdb4391620d8030aec6b8d4a9f8ba95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 1 May 2020 11:20:39 +0200 Subject: [PATCH 11/12] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0f2f9bd1a..44687a150 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ $ make - [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) - [Nix - A One Pager](https://github.com/tazjin/nix-1p) - [NixOS Discourse](https://discourse.nixos.org/) -- #nix / #nixos on irc.freenode.net +- [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos) ## License From 404a94ab6914405948640bedb92d1f9ddd502a8a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 1 May 2020 11:47:26 +0200 Subject: [PATCH 12/12] Tweak README.md --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 44687a150..a1588284d 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Information on additional installation methods is available on the [Nix download ### Building Nix -You can build Nix via Nix via one of the targets provided by [release.nix](./release.nix): +You can build Nix using one of the targets provided by [release.nix](./release.nix): ``` $ nix-build ./release.nix -A build.aarch64-linux @@ -33,7 +33,7 @@ $ nix-build ./release.nix -A build.x86_64-linux ### Development Environment -You can use the provided `shell.nix` to easily bootstrap a working development environment: +You can use the provided `shell.nix` to get a working development environment: ``` $ nix-shell @@ -46,10 +46,9 @@ $ make - [Nix manual](https://nixos.org/nix/manual) - [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) -- [Nix - A One Pager](https://github.com/tazjin/nix-1p) - [NixOS Discourse](https://discourse.nixos.org/) - [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos) ## License -Nix is released under the [LGPL v2.1](./COPYING) +Nix is released under the [LGPL v2.1](./COPYING).