Backport 'nix dev-shell' from the flakes branch

This also adds a '--profile' option to 'nix build' (replacing 'nix-env
--set').
This commit is contained in:
Eelco Dolstra 2020-03-30 19:14:17 +02:00
parent 367577d9a6
commit e1a94ad852
9 changed files with 569 additions and 116 deletions

View file

@ -478,6 +478,17 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix,
} }
std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
{
Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
// Strictly speaking, this is UB, but who cares...
AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
if (!fd)
throw SysError("creating temporary file '%s'", tmpl);
return {std::move(fd), tmpl};
}
std::string getUserName() std::string getUserName()
{ {
auto pw = getpwuid(geteuid()); auto pw = getpwuid(geteuid());

View file

@ -122,10 +122,6 @@ void deletePath(const Path & path);
void deletePath(const Path & path, unsigned long long & bytesFreed); void deletePath(const Path & path, unsigned long long & bytesFreed);
/* Create a temporary directory. */
Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
std::string getUserName(); std::string getUserName();
/* Return $HOME or the user's home directory from /etc/passwd. */ /* Return $HOME or the user's home directory from /etc/passwd. */
@ -205,6 +201,14 @@ public:
}; };
/* Create a temporary directory. */
Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
/* Create a temporary file, returning a file handle and its path. */
std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix");
class Pipe class Pipe
{ {
public: public:

View file

@ -5,7 +5,7 @@
using namespace nix; using namespace nix;
struct CmdBuild : MixDryRun, InstallablesCommand struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
{ {
Path outLink = "result"; Path outLink = "result";
@ -40,6 +40,10 @@ struct CmdBuild : MixDryRun, InstallablesCommand
"To build the build.x86_64-linux attribute from release.nix:", "To build the build.x86_64-linux attribute from release.nix:",
"nix build -f release.nix build.x86_64-linux" "nix build -f release.nix build.x86_64-linux"
}, },
Example{
"To make a profile point at GNU Hello:",
"nix build --profile /tmp/profile nixpkgs.hello"
},
}; };
} }
@ -49,18 +53,19 @@ struct CmdBuild : MixDryRun, InstallablesCommand
if (dryRun) return; if (dryRun) return;
for (size_t i = 0; i < buildables.size(); ++i) { if (outLink != "") {
auto & b(buildables[i]); for (size_t i = 0; i < buildables.size(); ++i) {
for (auto & output : buildables[i].outputs)
if (outLink != "")
for (auto & output : b.outputs)
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) { if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
std::string symlink = outLink; std::string symlink = outLink;
if (i) symlink += fmt("-%d", i); if (i) symlink += fmt("-%d", i);
if (output.first != "out") symlink += fmt("-%s", output.first); if (output.first != "out") symlink += fmt("-%s", output.first);
store2->addPermRoot(output.second, absPath(symlink), true); store2->addPermRoot(output.second, absPath(symlink), true);
} }
}
} }
updateProfile(buildables);
} }
}; };

View file

@ -2,6 +2,9 @@
#include "store-api.hh" #include "store-api.hh"
#include "derivations.hh" #include "derivations.hh"
#include "nixexpr.hh" #include "nixexpr.hh"
#include "profiles.hh"
extern char * * environ;
namespace nix { namespace nix {
@ -96,4 +99,95 @@ Strings editorFor(const Pos & pos)
return args; return args;
} }
MixProfile::MixProfile()
{
mkFlag()
.longName("profile")
.description("profile to update")
.labels({"path"})
.dest(&profile);
}
void MixProfile::updateProfile(const StorePath & storePath)
{
if (!profile) return;
auto store = getStore().dynamic_pointer_cast<LocalFSStore>();
if (!store) throw Error("'--profile' is not supported for this Nix store");
auto profile2 = absPath(*profile);
switchLink(profile2,
createGeneration(
ref<LocalFSStore>(store),
profile2, store->printStorePath(storePath)));
}
void MixProfile::updateProfile(const Buildables & buildables)
{
if (!profile) return;
std::optional<StorePath> result;
for (auto & buildable : buildables) {
for (auto & output : buildable.outputs) {
if (result)
throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple");
result = output.second.clone();
}
}
if (!result)
throw Error("'--profile' requires that the arguments produce a single store path, but there are none");
updateProfile(*result);
}
MixDefaultProfile::MixDefaultProfile()
{
profile = getDefaultProfile();
}
MixEnvironment::MixEnvironment() : ignoreEnvironment(false) {
mkFlag()
.longName("ignore-environment")
.shortName('i')
.description("clear the entire environment (except those specified with --keep)")
.set(&ignoreEnvironment, true);
mkFlag()
.longName("keep")
.shortName('k')
.description("keep specified environment variable")
.arity(1)
.labels({"name"})
.handler([&](std::vector<std::string> ss) { keep.insert(ss.front()); });
mkFlag()
.longName("unset")
.shortName('u')
.description("unset specified environment variable")
.arity(1)
.labels({"name"})
.handler([&](std::vector<std::string> ss) { unset.insert(ss.front()); });
}
void MixEnvironment::setEnviron() {
if (ignoreEnvironment) {
if (!unset.empty())
throw UsageError("--unset does not make sense with --ignore-environment");
for (const auto & var : keep) {
auto val = getenv(var.c_str());
if (val) stringsEnv.emplace_back(fmt("%s=%s", var.c_str(), val));
}
vectorEnv = stringsToCharPtrs(stringsEnv);
environ = vectorEnv.data();
} else {
if (!keep.empty())
throw UsageError("--keep does not make sense without --ignore-environment");
for (const auto & var : unset)
unsetenv(var.c_str());
}
}
} }

View file

@ -1,5 +1,6 @@
#pragma once #pragma once
#include "installables.hh"
#include "args.hh" #include "args.hh"
#include "common-eval-args.hh" #include "common-eval-args.hh"
#include "path.hh" #include "path.hh"
@ -22,34 +23,7 @@ private:
std::shared_ptr<Store> _store; std::shared_ptr<Store> _store;
}; };
struct Buildable struct SourceExprCommand : virtual StoreCommand, MixEvalArgs
{
std::optional<StorePath> drvPath;
std::map<std::string, StorePath> outputs;
};
typedef std::vector<Buildable> Buildables;
struct Installable
{
virtual ~Installable() { }
virtual std::string what() = 0;
virtual Buildables toBuildables()
{
throw Error("argument '%s' cannot be built", what());
}
Buildable toBuildable();
virtual std::pair<Value *, Pos> toValue(EvalState & state)
{
throw Error("argument '%s' cannot be evaluated", what());
}
};
struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs
{ {
Path file; Path file;
@ -184,4 +158,36 @@ std::set<StorePath> toDerivations(ref<Store> store,
filename:lineno. */ filename:lineno. */
Strings editorFor(const Pos & pos); Strings editorFor(const Pos & pos);
struct MixProfile : virtual StoreCommand
{
std::optional<Path> profile;
MixProfile();
/* If 'profile' is set, make it point at 'storePath'. */
void updateProfile(const StorePath & storePath);
/* If 'profile' is set, make it point at the store path produced
by 'buildables'. */
void updateProfile(const Buildables & buildables);
};
struct MixDefaultProfile : MixProfile
{
MixDefaultProfile();
};
struct MixEnvironment : virtual Args {
StringSet keep, unset;
Strings stringsEnv;
std::vector<char*> vectorEnv;
bool ignoreEnvironment;
MixEnvironment();
/* Modify global environ based on ignoreEnvironment, keep, and unset. It's expected that exec will be called before this class goes out of scope, otherwise environ will become invalid. */
void setEnviron();
};
} }

View file

@ -109,6 +109,11 @@ struct InstallableStorePath : Installable
bs.push_back(std::move(b)); bs.push_back(std::move(b));
return bs; return bs;
} }
std::optional<StorePath> getStorePath() override
{
return storePath.clone();
}
}; };
struct InstallableValue : Installable struct InstallableValue : Installable

45
src/nix/installables.hh Normal file
View file

@ -0,0 +1,45 @@
#pragma once
#include "util.hh"
#include "path.hh"
#include "eval.hh"
#include <optional>
namespace nix {
struct Buildable
{
std::optional<StorePath> drvPath;
std::map<std::string, StorePath> outputs;
};
typedef std::vector<Buildable> Buildables;
struct Installable
{
virtual ~Installable() { }
virtual std::string what() = 0;
virtual Buildables toBuildables()
{
throw Error("argument '%s' cannot be built", what());
}
Buildable toBuildable();
virtual std::pair<Value *, Pos> toValue(EvalState & state)
{
throw Error("argument '%s' cannot be evaluated", what());
}
/* Return a value only if this installable is a store path or a
symlink to it. */
virtual std::optional<StorePath> getStorePath()
{
return {};
}
};
}

View file

@ -8,6 +8,7 @@
#include "fs-accessor.hh" #include "fs-accessor.hh"
#include "progress-bar.hh" #include "progress-bar.hh"
#include "affinity.hh" #include "affinity.hh"
#include "eval.hh"
#if __linux__ #if __linux__
#include <sys/mount.h> #include <sys/mount.h>
@ -19,11 +20,46 @@ using namespace nix;
std::string chrootHelperName = "__run_in_chroot"; std::string chrootHelperName = "__run_in_chroot";
struct CmdRun : InstallablesCommand struct RunCommon : virtual Command
{
void runProgram(ref<Store> store,
const std::string & program,
const Strings & args)
{
stopProgressBar();
restoreSignals();
restoreAffinity();
/* If this is a diverted store (i.e. its "logical" location
(typically /nix/store) differs from its "physical" location
(e.g. /home/eelco/nix/store), then run the command in a
chroot. For non-root users, this requires running it in new
mount and user namespaces. Unfortunately,
unshare(CLONE_NEWUSER) doesn't work in a multithreaded
program (which "nix" is), so we exec() a single-threaded
helper program (chrootHelper() below) to do the work. */
auto store2 = store.dynamic_pointer_cast<LocalStore>();
if (store2 && store->storeDir != store2->realStoreDir) {
Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, program };
for (auto & arg : args) helperArgs.push_back(arg);
execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data());
throw SysError("could not execute chroot helper");
}
execvp(program.c_str(), stringsToCharPtrs(args).data());
throw SysError("unable to execute '%s'", program);
}
};
struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment
{ {
std::vector<std::string> command = { "bash" }; std::vector<std::string> command = { "bash" };
StringSet keep, unset;
bool ignoreEnvironment = false;
CmdRun() CmdRun()
{ {
@ -37,28 +73,6 @@ struct CmdRun : InstallablesCommand
if (ss.empty()) throw UsageError("--command requires at least one argument"); if (ss.empty()) throw UsageError("--command requires at least one argument");
command = ss; command = ss;
}); });
mkFlag()
.longName("ignore-environment")
.shortName('i')
.description("clear the entire environment (except those specified with --keep)")
.set(&ignoreEnvironment, true);
mkFlag()
.longName("keep")
.shortName('k')
.description("keep specified environment variable")
.arity(1)
.labels({"name"})
.handler([&](std::vector<std::string> ss) { keep.insert(ss.front()); });
mkFlag()
.longName("unset")
.shortName('u')
.description("unset specified environment variable")
.arity(1)
.labels({"name"})
.handler([&](std::vector<std::string> ss) { unset.insert(ss.front()); });
} }
std::string description() override std::string description() override
@ -94,35 +108,13 @@ struct CmdRun : InstallablesCommand
auto accessor = store->getFSAccessor(); auto accessor = store->getFSAccessor();
if (ignoreEnvironment) {
if (!unset.empty())
throw UsageError("--unset does not make sense with --ignore-environment");
std::map<std::string, std::string> kept;
for (auto & var : keep) {
auto s = getenv(var.c_str());
if (s) kept[var] = s;
}
clearEnv();
for (auto & var : kept)
setenv(var.first.c_str(), var.second.c_str(), 1);
} else {
if (!keep.empty())
throw UsageError("--keep does not make sense without --ignore-environment");
for (auto & var : unset)
unsetenv(var.c_str());
}
std::unordered_set<StorePath> done; std::unordered_set<StorePath> done;
std::queue<StorePath> todo; std::queue<StorePath> todo;
for (auto & path : outPaths) todo.push(path.clone()); for (auto & path : outPaths) todo.push(path.clone());
setEnviron();
auto unixPath = tokenizeString<Strings>(getEnv("PATH").value_or(""), ":"); auto unixPath = tokenizeString<Strings>(getEnv("PATH").value_or(""), ":");
while (!todo.empty()) { while (!todo.empty()) {
@ -142,38 +134,10 @@ struct CmdRun : InstallablesCommand
setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1); setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1);
std::string cmd = *command.begin();
Strings args; Strings args;
for (auto & arg : command) args.push_back(arg); for (auto & arg : command) args.push_back(arg);
stopProgressBar(); runProgram(store, *command.begin(), args);
restoreSignals();
restoreAffinity();
/* If this is a diverted store (i.e. its "logical" location
(typically /nix/store) differs from its "physical" location
(e.g. /home/eelco/nix/store), then run the command in a
chroot. For non-root users, this requires running it in new
mount and user namespaces. Unfortunately,
unshare(CLONE_NEWUSER) doesn't work in a multithreaded
program (which "nix" is), so we exec() a single-threaded
helper program (chrootHelper() below) to do the work. */
auto store2 = store.dynamic_pointer_cast<LocalStore>();
if (store2 && store->storeDir != store2->realStoreDir) {
Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, cmd };
for (auto & arg : args) helperArgs.push_back(arg);
execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data());
throw SysError("could not execute chroot helper");
}
execvp(cmd.c_str(), stringsToCharPtrs(args).data());
throw SysError("unable to exec '%s'", cmd);
} }
}; };

319
src/nix/shell.cc Normal file
View file

@ -0,0 +1,319 @@
#include "eval.hh"
#include "command.hh"
#include "common-args.hh"
#include "shared.hh"
#include "store-api.hh"
#include "derivations.hh"
#include "affinity.hh"
#include "progress-bar.hh"
#include <regex>
using namespace nix;
struct Var
{
bool exported;
std::string value; // quoted string or array
};
struct BuildEnvironment
{
std::map<std::string, Var> env;
std::string bashFunctions;
};
BuildEnvironment readEnvironment(const Path & path)
{
BuildEnvironment res;
std::set<std::string> exported;
debug("reading environment file '%s'", path);
auto file = readFile(path);
auto pos = file.cbegin();
static std::string varNameRegex =
R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re";
static std::regex declareRegex(
"^declare -x (" + varNameRegex + ")" +
R"re((?:="((?:[^"\\]|\\.)*)")?\n)re");
static std::string simpleStringRegex =
R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re";
static std::string quotedStringRegex =
R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re";
static std::string arrayRegex =
R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")*\)))re";
static std::regex varRegex(
"^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + arrayRegex + ")\n");
static std::regex functionRegex(
"^" + varNameRegex + " \\(\\) *\n");
while (pos != file.end()) {
std::smatch match;
if (std::regex_search(pos, file.cend(), match, declareRegex)) {
pos = match[0].second;
exported.insert(match[1]);
}
else if (std::regex_search(pos, file.cend(), match, varRegex)) {
pos = match[0].second;
res.env.insert({match[1], Var { (bool) exported.count(match[1]), match[2] }});
}
else if (std::regex_search(pos, file.cend(), match, functionRegex)) {
res.bashFunctions = std::string(pos, file.cend());
break;
}
else throw Error("shell environment '%s' has unexpected line '%s'",
path, file.substr(pos - file.cbegin(), 60));
}
return res;
}
/* Given an existing derivation, return the shell environment as
initialised by stdenv's setup script. We do this by building a
modified derivation with the same dependencies and nearly the same
initial environment variables, that just writes the resulting
environment to a file and exits. */
StorePath getDerivationEnvironment(ref<Store> store, Derivation drv)
{
auto builder = baseNameOf(drv.builder);
if (builder != "bash")
throw Error("'nix shell' only works on derivations that use 'bash' as their builder");
drv.args = {
"-c",
"set -e; "
"export IN_NIX_SHELL=impure; "
"export dontAddDisableDepTrack=1; "
"if [[ -n $stdenv ]]; then "
" source $stdenv/setup; "
"fi; "
"export > $out; "
"set >> $out "};
/* Remove derivation checks. */
drv.env.erase("allowedReferences");
drv.env.erase("allowedRequisites");
drv.env.erase("disallowedReferences");
drv.env.erase("disallowedRequisites");
// FIXME: handle structured attrs
/* Rehash and write the derivation. FIXME: would be nice to use
'buildDerivation', but that's privileged. */
auto drvName = drv.env["name"] + "-env";
for (auto & output : drv.outputs)
drv.env.erase(output.first);
drv.env["out"] = "";
drv.env["outputs"] = "out";
Hash h = hashDerivationModulo(*store, drv, true);
auto shellOutPath = store->makeOutputPath("out", h, drvName);
drv.outputs.insert_or_assign("out", DerivationOutput(shellOutPath.clone(), "", ""));
drv.env["out"] = store->printStorePath(shellOutPath);
auto shellDrvPath2 = writeDerivation(store, drv, drvName);
/* Build the derivation. */
store->buildPaths({shellDrvPath2});
assert(store->isValidPath(shellOutPath));
return shellOutPath;
}
struct Common : InstallableCommand, MixProfile
{
std::set<string> ignoreVars{
"BASHOPTS",
"EUID",
"HOME", // FIXME: don't ignore in pure mode?
"NIX_BUILD_TOP",
"NIX_ENFORCE_PURITY",
"NIX_LOG_FD",
"PPID",
"PWD",
"SHELLOPTS",
"SHLVL",
"SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt
"TEMP",
"TEMPDIR",
"TERM",
"TMP",
"TMPDIR",
"TZ",
"UID",
};
void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out)
{
out << "nix_saved_PATH=\"$PATH\"\n";
for (auto & i : buildEnvironment.env) {
if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) {
out << fmt("%s=%s\n", i.first, i.second.value);
if (i.second.exported)
out << fmt("export %s\n", i.first);
}
}
out << "PATH=\"$PATH:$nix_saved_PATH\"\n";
out << buildEnvironment.bashFunctions << "\n";
// FIXME: set outputs
out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n";
for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"})
out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i);
out << "eval \"$shellHook\"\n";
}
StorePath getShellOutPath(ref<Store> store)
{
auto path = installable->getStorePath();
if (path && hasSuffix(path->to_string(), "-env"))
return path->clone();
else {
auto drvs = toDerivations(store, {installable});
if (drvs.size() != 1)
throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations",
installable->what(), drvs.size());
auto & drvPath = *drvs.begin();
return getDerivationEnvironment(store, store->derivationFromPath(drvPath));
}
}
BuildEnvironment getBuildEnvironment(ref<Store> store)
{
auto shellOutPath = getShellOutPath(store);
updateProfile(shellOutPath);
return readEnvironment(store->printStorePath(shellOutPath));
}
};
struct CmdDevShell : Common, MixEnvironment
{
std::vector<std::string> command;
CmdDevShell()
{
mkFlag()
.longName("command")
.shortName('c')
.description("command and arguments to be executed insted of an interactive shell")
.labels({"command", "args"})
.arity(ArityAny)
.handler([&](std::vector<std::string> ss) {
if (ss.empty()) throw UsageError("--command requires at least one argument");
command = ss;
});
}
std::string description() override
{
return "run a bash shell that provides the build environment of a derivation";
}
Examples examples() override
{
return {
Example{
"To get the build environment of GNU hello:",
"nix dev-shell nixpkgs.hello"
},
Example{
"To store the build environment in a profile:",
"nix dev-shell --profile /tmp/my-shell nixpkgs.hello"
},
Example{
"To use a build environment previously recorded in a profile:",
"nix dev-shell /tmp/my-shell"
},
};
}
void run(ref<Store> store) override
{
auto buildEnvironment = getBuildEnvironment(store);
auto [rcFileFd, rcFilePath] = createTempFile("nix-shell");
std::ostringstream ss;
makeRcScript(buildEnvironment, ss);
ss << fmt("rm -f '%s'\n", rcFilePath);
if (!command.empty()) {
std::vector<std::string> args;
for (auto s : command)
args.push_back(shellEscape(s));
ss << fmt("exec %s\n", concatStringsSep(" ", args));
}
writeFull(rcFileFd.get(), ss.str());
stopProgressBar();
auto shell = getEnv("SHELL").value_or("bash");
setEnviron();
auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath};
restoreAffinity();
restoreSignals();
execvp(shell.c_str(), stringsToCharPtrs(args).data());
throw SysError("executing shell '%s'", shell);
}
};
struct CmdPrintDevEnv : Common
{
std::string description() override
{
return "print shell code that can be sourced by bash to reproduce the build environment of a derivation";
}
Examples examples() override
{
return {
Example{
"To apply the build environment of GNU hello to the current shell:",
". <(nix print-dev-env nixpkgs.hello)"
},
};
}
void run(ref<Store> store) override
{
auto buildEnvironment = getBuildEnvironment(store);
stopProgressBar();
makeRcScript(buildEnvironment, std::cout);
}
};
static auto r1 = registerCommand<CmdPrintDevEnv>("print-dev-env");
static auto r2 = registerCommand<CmdDevShell>("dev-shell");