From 2b8c63f303de7d1da133cb3e9a00d509dca40f57 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 May 2019 20:22:14 +0200 Subject: [PATCH 1/6] Add 'nix dev-shell' and 'nix print-dev-env' command 'nix dev-shell' is intended to replace nix-shell. It supports flakes, e.g. $ nix dev-shell nixpkgs:hello starts a bash shell providing an environment for building 'hello'. Like Lorri (and unlike nix-shell), it computes the build environment by building a modified top-level derivation that writes the environment after running $stdenv/setup to $out and exits. This provides some caching, so it's faster than nix-shell in some cases (especially for packages with lots of dependencies, where the setup script takes a long time). There also is a command 'nix print-dev-env' that prints out shell code for setting up the build environment in an existing shell, e.g. $ . <(nix print-dev-env nixpkgs:hello) https://github.com/tweag/nix/issues/21 --- src/nix/shell.cc | 276 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 src/nix/shell.cc diff --git a/src/nix/shell.cc b/src/nix/shell.cc new file mode 100644 index 000000000..14d88faeb --- /dev/null +++ b/src/nix/shell.cc @@ -0,0 +1,276 @@ +#include "eval.hh" +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "affinity.hh" +#include "progress-bar.hh" + +using namespace nix; + +struct BuildEnvironment +{ + // FIXME: figure out which vars should be exported. + std::map env; + std::map functions; +}; + +BuildEnvironment readEnvironment(const Path & path) +{ + BuildEnvironment res; + + auto lines = tokenizeString(readFile(path), "\n"); + + auto getLine = + [&]() { + if (lines.empty()) + throw Error("shell environment '%s' ends unexpectedly", path); + auto line = lines.front(); + lines.pop_front(); + return line; + }; + + while (!lines.empty()) { + auto line = getLine(); + + auto eq = line.find('='); + if (eq != std::string::npos) { + std::string name(line, 0, eq); + std::string value(line, eq + 1); + // FIXME: parse arrays + res.env.insert({name, value}); + } + + else if (hasSuffix(line, " () ")) { + std::string name(line, 0, line.size() - 4); + // FIXME: validate name + auto l = getLine(); + if (l != "{ ") throw Error("shell environment '%s' has unexpected line '%s'", path, l); + std::string body; + while ((l = getLine()) != "}") { + body += l; + body += '\n'; + } + res.functions.insert({name, body}); + } + + else throw Error("shell environment '%s' has unexpected line '%s'", path, line); + } + + return res; +} + +/* Given an existing derivation, return the shell environment as + initialised by stdenv's setup script. We do this by building a + modified derivation with the same dependencies and nearly the same + initial environment variables, that just writes the resulting + environment to a file and exits. */ +BuildEnvironment getDerivationEnvironment(ref store, Derivation drv) +{ + auto builder = baseNameOf(drv.builder); + if (builder != "bash") + throw Error("'nix shell' only works on derivations that use 'bash' as their builder"); + + drv.args = {"-c", "set -e; if [[ -n $stdenv ]]; then source $stdenv/setup; fi; set > $out"}; + + /* Remove derivation checks. */ + drv.env.erase("allowedReferences"); + drv.env.erase("allowedRequisites"); + drv.env.erase("disallowedReferences"); + drv.env.erase("disallowedRequisites"); + + // FIXME: handle structured attrs + + /* Rehash and write the derivation. FIXME: would be nice to use + 'buildDerivation', but that's privileged. */ + auto drvName = drv.env["name"] + "-env"; + for (auto & output : drv.outputs) + drv.env.erase(output.first); + drv.env["out"] = ""; + drv.env["outputs"] = "out"; + drv.outputs["out"] = DerivationOutput("", "", ""); + Hash h = hashDerivationModulo(*store, drv); + Path shellOutPath = store->makeOutputPath("out", h, drvName); + drv.outputs["out"].path = shellOutPath; + drv.env["out"] = shellOutPath; + Path shellDrvPath2 = writeDerivation(store, drv, drvName); + + /* Build the derivation. */ + store->buildPaths({shellDrvPath2}); + + assert(store->isValidPath(shellOutPath)); + + return readEnvironment(shellOutPath); +} + +struct Common : InstallableCommand +{ + /* + std::set keepVars{ + "DISPLAY", + "HOME", + "IN_NIX_SHELL", + "LOGNAME", + "NIX_BUILD_SHELL", + "PAGER", + "PATH", + "TERM", + "TZ", + "USER", + }; + */ + + std::set ignoreVars{ + "BASHOPTS", + "EUID", + "NIX_BUILD_TOP", + "PPID", + "PWD", + "SHELLOPTS", + "SHLVL", + "TEMP", + "TEMPDIR", + "TMP", + "TMPDIR", + "UID", + }; + + void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out) + { + for (auto & i : buildEnvironment.env) { + // FIXME: shellEscape + // FIXME: figure out what to export + // FIXME: handle arrays + if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) + out << fmt("export %s=%s\n", i.first, i.second); + } + + for (auto & i : buildEnvironment.functions) { + out << fmt("%s () {\n%s\n}\n", i.first, i.second); + } + + // FIXME: set outputs + + out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n"; + for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) + out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); + } +}; + +std::pair createTempFile(const Path & prefix = "nix") +{ + Path tmpl(getEnv("TMPDIR", "/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + return {std::move(fd), tmpl}; +} + +struct CmdDevShell : Common +{ + + std::string name() override + { + return "dev-shell"; + } + + std::string description() override + { + return "run a bash shell that provides the build environment of a derivation"; + } + + Examples examples() override + { + return { + Example{ + "To get the build environment of GNU hello:", + "nix dev-shell nixpkgs:hello" + }, + Example{ + "To get the build environment of the default package of flake in the current directory:", + "nix dev-shell" + }, + }; + } + + void run(ref store) override + { + auto drvs = toDerivations(store, {installable}); + + if (drvs.size() != 1) + throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations", + installable->what(), drvs.size()); + + auto & drvPath = *drvs.begin(); + + auto buildEnvironment = getDerivationEnvironment(store, store->derivationFromPath(drvPath)); + + auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); + + std::ostringstream ss; + makeRcScript(buildEnvironment, ss); + + ss << fmt("rm -f '%s'\n", rcFilePath); + + writeFull(rcFileFd.get(), ss.str()); + + stopProgressBar(); + + auto shell = getEnv("SHELL", "bash"); + + auto args = Strings{baseNameOf(shell), "--rcfile", rcFilePath}; + + restoreAffinity(); + restoreSignals(); + + execvp(shell.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("executing shell '%s'", shell); + } +}; + +struct CmdPrintDevEnv : Common +{ + + std::string name() override + { + return "print-dev-env"; + } + + std::string description() override + { + return "print shell code that can be sourced by bash to reproduce the build environment of a derivation"; + } + + Examples examples() override + { + return { + Example{ + "To apply the build environment of GNU hello to the current shell:", + ". <(nix print-dev-env nixpkgs:hello)" + }, + }; + } + + void run(ref store) override + { + auto drvs = toDerivations(store, {installable}); + + if (drvs.size() != 1) + throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations", + installable->what(), drvs.size()); + + auto & drvPath = *drvs.begin(); + + auto buildEnvironment = getDerivationEnvironment(store, store->derivationFromPath(drvPath)); + + stopProgressBar(); + + makeRcScript(buildEnvironment, std::cout); + } +}; + +static RegisterCommand r1(make_ref()); +static RegisterCommand r2(make_ref()); From 7dcf5b011a0942ecf953f2b607c4c8d0e9e652c7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 May 2019 21:09:52 +0200 Subject: [PATCH 2/6] Add function for quoting strings --- src/libstore/build.cc | 2 +- src/libstore/store-api.cc | 7 +------ src/libutil/util.cc | 22 ---------------------- src/libutil/util.hh | 22 ++++++++++++++++++++-- 4 files changed, 22 insertions(+), 31 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 53a0c743b..9730c75e2 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1805,7 +1805,7 @@ void DerivationGoal::startBuilder() concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), drvPath, settings.thisSystem, - concatStringsSep(", ", settings.systemFeatures)); + concatStringsSep(", ", settings.systemFeatures)); if (drv->isBuiltin()) preloadNSS(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c13ff1156..8fabeeea4 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -726,12 +726,7 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) string showPaths(const PathSet & paths) { - string s; - for (auto & i : paths) { - if (s.size() != 0) s += ", "; - s += "'" + i + "'"; - } - return s; + return concatStringsSep(", ", quoteStrings(paths)); } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index f4f86c5c8..5598415f5 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1167,28 +1167,6 @@ template StringSet tokenizeString(const string & s, const string & separators); template vector tokenizeString(const string & s, const string & separators); -string concatStringsSep(const string & sep, const Strings & ss) -{ - string s; - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - - -string concatStringsSep(const string & sep, const StringSet & ss) -{ - string s; - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - - string chomp(const string & s) { size_t i = s.find_last_not_of(" \n\r\t"); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 9f239bff3..8bd57d2e4 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -334,8 +334,26 @@ template C tokenizeString(const string & s, const string & separators = /* Concatenate the given strings with a separator between the elements. */ -string concatStringsSep(const string & sep, const Strings & ss); -string concatStringsSep(const string & sep, const StringSet & ss); +template +string concatStringsSep(const string & sep, const C & ss) +{ + string s; + for (auto & i : ss) { + if (s.size() != 0) s += sep; + s += i; + } + return s; +} + + +/* Add quotes around a collection of strings. */ +template Strings quoteStrings(const C & c) +{ + Strings res; + for (auto & s : c) + res.push_back("'" + s + "'"); + return res; +} /* Remove trailing whitespace from a string. */ From 2919c496ea10a99d08baffbef485cfb719233b9f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 May 2019 21:10:13 +0200 Subject: [PATCH 3/6] nix dev-shell: Use 'provides.devShell' by default Thus $ nix dev-shell will now build the 'provides.devShell' attribute from the flake in the current directory. If it doesn't exist, it falls back to 'provides.defaultPackage'. --- flake.nix | 5 +++++ shell.nix | 6 ++++-- src/nix/command.hh | 5 +++++ src/nix/installables.cc | 42 ++++++++++++++++++++++++++++++----------- src/nix/shell.cc | 6 ++++++ 5 files changed, 51 insertions(+), 13 deletions(-) diff --git a/flake.nix b/flake.nix index 95ec5d952..ab316c7c6 100644 --- a/flake.nix +++ b/flake.nix @@ -17,5 +17,10 @@ packages.nix = hydraJobs.build.x86_64-linux; defaultPackage = packages.nix; + + devShell = import ./shell.nix { + nixpkgs = deps.nixpkgs; + }; + }; } diff --git a/shell.nix b/shell.nix index 8167f87a2..d7e63bad3 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,8 @@ -{ useClang ? false }: +{ useClang ? false +, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz +}: -with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz) {}; +with import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; }; with import ./release-common.nix { inherit pkgs; }; diff --git a/src/nix/command.hh b/src/nix/command.hh index 6d43261ac..640c6cd16 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -88,6 +88,11 @@ struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs std::shared_ptr parseInstallable( ref store, const std::string & installable); + virtual Strings getDefaultFlakeAttrPaths() + { + return {"defaultPackage"}; + } + private: std::shared_ptr evalState; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index c3ca87aa7..db67952e1 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -142,13 +142,18 @@ struct InstallableAttrPath : InstallableValue struct InstallableFlake : InstallableValue { FlakeRef flakeRef; - std::string attrPath; + Strings attrPaths; + bool searchPackages = false; - InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef, const std::string & attrPath) - : InstallableValue(cmd), flakeRef(flakeRef), attrPath(attrPath) + InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef, Strings attrPaths) + : InstallableValue(cmd), flakeRef(flakeRef), attrPaths(std::move(attrPaths)) { } - std::string what() override { return flakeRef.to_string() + ":" + attrPath; } + InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef, std::string attrPath) + : InstallableValue(cmd), flakeRef(flakeRef), attrPaths{attrPath}, searchPackages(true) + { } + + std::string what() override { return flakeRef.to_string() + ":" + *attrPaths.begin(); } Value * toValue(EvalState & state) override { @@ -166,18 +171,31 @@ struct InstallableFlake : InstallableValue auto emptyArgs = state.allocBindings(0); - if (auto aPackages = *vProvides->attrs->get(state.symbols.create("packages"))) { + // As a convenience, look for the attribute in + // 'provides.packages'. + if (searchPackages) { + if (auto aPackages = *vProvides->attrs->get(state.symbols.create("packages"))) { + try { + auto * v = findAlongAttrPath(state, *attrPaths.begin(), *emptyArgs, *aPackages->value); + state.forceValue(*v); + return v; + } catch (AttrPathNotFound & e) { + } + } + } + + // Otherwise, look for it in 'provides'. + for (auto & attrPath : attrPaths) { try { - auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *aPackages->value); + auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *vProvides); state.forceValue(*v); return v; } catch (AttrPathNotFound & e) { } } - auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *vProvides); - state.forceValue(*v); - return v; + throw Error("flake '%s' does not provide attribute %s", + flakeRef, concatStringsSep(", ", quoteStrings(attrPaths))); } }; @@ -216,7 +234,8 @@ std::vector> SourceExprCommand::parseInstallables( else if (hasPrefix(s, "nixpkgs.")) { bool static warned; warnOnce(warned, "the syntax 'nixpkgs.' is deprecated; use 'nixpkgs:' instead"); - result.push_back(std::make_shared(*this, FlakeRef("nixpkgs"), std::string(s, 8))); + result.push_back(std::make_shared(*this, FlakeRef("nixpkgs"), + Strings{"packages." + std::string(s, 8)})); } else if ((colon = s.rfind(':')) != std::string::npos) { @@ -233,7 +252,8 @@ std::vector> SourceExprCommand::parseInstallables( if (storePath != "") result.push_back(std::make_shared(storePath)); else - result.push_back(std::make_shared(*this, FlakeRef(s, true), "defaultPackage")); + result.push_back(std::make_shared(*this, FlakeRef(s, true), + getDefaultFlakeAttrPaths())); } else diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 14d88faeb..2e17111d6 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -125,6 +125,7 @@ struct Common : InstallableCommand "BASHOPTS", "EUID", "NIX_BUILD_TOP", + "NIX_ENFORCE_PURITY", "PPID", "PWD", "SHELLOPTS", @@ -156,6 +157,11 @@ struct Common : InstallableCommand for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); } + + Strings getDefaultFlakeAttrPaths() override + { + return {"devShell", "defaultPackage"}; + } }; std::pair createTempFile(const Path & prefix = "nix") From dea18ff99913349ef0f54b80d45c8dfdc8b31f65 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 May 2019 21:13:19 +0200 Subject: [PATCH 4/6] nix dev-shell: Execute shellHook --- src/nix/shell.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 2e17111d6..d3ecf8de4 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -156,6 +156,8 @@ struct Common : InstallableCommand out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n"; for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); + + out << "eval \"$shellHook\"\n"; } Strings getDefaultFlakeAttrPaths() override From 8ec77614f63e14d1869734b0d21a646667bbf88b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 May 2019 21:28:41 +0200 Subject: [PATCH 5/6] Move createTempFile to libutil --- src/libutil/util.cc | 11 +++++++++++ src/libutil/util.hh | 12 ++++++++---- src/nix/shell.cc | 10 ---------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 5598415f5..75b73fcfa 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -461,6 +461,17 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, } +std::pair createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR", "/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + return {std::move(fd), tmpl}; +} + + static Lazy getHome2([]() { Path homeDir = getEnv("HOME"); if (homeDir.empty()) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 8bd57d2e4..6c9d7c2eb 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -118,10 +118,6 @@ void deletePath(const Path & path); void deletePath(const Path & path, unsigned long long & bytesFreed); -/* Create a temporary directory. */ -Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); - /* Return $HOME or the user's home directory from /etc/passwd. */ Path getHome(); @@ -199,6 +195,14 @@ public: }; +/* Create a temporary directory. */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +/* Create a temporary file, returning a file handle and its path. */ +std::pair createTempFile(const Path & prefix = "nix"); + + class Pipe { public: diff --git a/src/nix/shell.cc b/src/nix/shell.cc index d3ecf8de4..95028f10e 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -166,16 +166,6 @@ struct Common : InstallableCommand } }; -std::pair createTempFile(const Path & prefix = "nix") -{ - Path tmpl(getEnv("TMPDIR", "/tmp") + "/" + prefix + ".XXXXXX"); - // Strictly speaking, this is UB, but who cares... - AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); - if (!fd) - throw SysError("creating temporary file '%s'", tmpl); - return {std::move(fd), tmpl}; -} - struct CmdDevShell : Common { From 7ba0f98e644c31ce0c16db10aa87f896937e0ddf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 May 2019 21:28:52 +0200 Subject: [PATCH 6/6] nix dev-shell: Less purity --- src/nix/shell.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 95028f10e..0813d122c 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -124,6 +124,7 @@ struct Common : InstallableCommand std::set ignoreVars{ "BASHOPTS", "EUID", + "HOME", // FIXME: don't ignore in pure mode? "NIX_BUILD_TOP", "NIX_ENFORCE_PURITY", "PPID", @@ -134,11 +135,15 @@ struct Common : InstallableCommand "TEMPDIR", "TMP", "TMPDIR", + "TZ", "UID", }; void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out) { + out << "export IN_NIX_SHELL=1\n"; + out << "nix_saved_PATH=\"$PATH\"\n"; + for (auto & i : buildEnvironment.env) { // FIXME: shellEscape // FIXME: figure out what to export @@ -147,6 +152,8 @@ struct Common : InstallableCommand out << fmt("export %s=%s\n", i.first, i.second); } + out << "PATH=\"$PATH:$nix_saved_PATH\"\n"; + for (auto & i : buildEnvironment.functions) { out << fmt("%s () {\n%s\n}\n", i.first, i.second); }