From 499b0388759db0f9f385da402a4bba551268aa99 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 12 Oct 2019 19:02:57 -0400 Subject: [PATCH 01/60] Fix sandbox fallback settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tmpDirInSandbox is different when in sandboxed vs. non-sandboxed. Since we don’t know ahead of time here whether sandboxing is enabled, we need to reset all of the env vars we’ve set previously. This fixes the issue encountered in https://github.com/NixOS/nixpkgs/issues/70856. --- src/libstore/build.cc | 60 ++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0e0f8a545..273f25363 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -956,6 +956,9 @@ private: /* Fill in the environment for the builder. */ void initEnv(); + /* Setup tmp dir location. */ + void initTmpDir(); + /* Write a JSON file containing the derivation attributes. */ void writeStructuredAttrs(); @@ -2383,6 +2386,7 @@ void DerivationGoal::startBuilder() if (res != 0 && settings.sandboxFallback) { useChroot = false; tmpDirInSandbox = tmpDir; + initTmpDir(); goto fallback; } else if (res != 0) throw Error("unable to start build process"); @@ -2438,32 +2442,7 @@ void DerivationGoal::startBuilder() } -void DerivationGoal::initEnv() -{ - env.clear(); - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = worker.store.storeDir; - - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); - +void DerivationGoal::initTmpDir() { /* In non-structured mode, add all bindings specified in the derivation via the environment, except those listed in the passAsFile attribute. Those are passed as file names pointing @@ -2501,6 +2480,35 @@ void DerivationGoal::initEnv() inode of the current directory doesn't appear in .. (because getdents returns the inode of the mount point). */ env["PWD"] = tmpDirInSandbox; +} + +void DerivationGoal::initEnv() +{ + env.clear(); + + /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when + PATH is not set. We don't want this, so we fill it in with some dummy + value. */ + env["PATH"] = "/path-not-set"; + + /* Set HOME to a non-existing path to prevent certain programs from using + /etc/passwd (or NIS, or whatever) to locate the home directory (for + example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd + if HOME is not set, but they will just assume that the settings file + they are looking for does not exist if HOME is set but points to some + non-existing path. */ + env["HOME"] = homeDir; + + /* Tell the builder where the Nix store is. Usually they + shouldn't care, but this is useful for purity checking (e.g., + the compiler or linker might only want to accept paths to files + in the store or in the build directory). */ + env["NIX_STORE"] = worker.store.storeDir; + + /* The maximum number of cores to utilize for parallel building. */ + env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); + + initTmpDir(); /* Compatibility hack with Nix <= 0.7: if this is a fixed-output derivation, tell the builder, so that for instance `fetchurl' From 96c84937c49435525d0733437aa88902b2c8caf6 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sun, 13 Oct 2019 16:41:49 -0400 Subject: [PATCH 02/60] Move tmpDirInSandbox to initTmpDir --- src/libstore/build.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 273f25363..03a069ae2 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1961,13 +1961,6 @@ void DerivationGoal::startBuilder() auto drvName = storePathToName(drvPath); tmpDir = createTempDir("", "nix-build-" + drvName, false, false, 0700); - /* In a sandbox, for determinism, always use the same temporary - directory. */ -#if __linux__ - tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; -#else - tmpDirInSandbox = tmpDir; -#endif chownToBuilder(tmpDir); /* Substitute output placeholders with the actual output paths. */ @@ -2385,7 +2378,6 @@ void DerivationGoal::startBuilder() int res = helper.wait(); if (res != 0 && settings.sandboxFallback) { useChroot = false; - tmpDirInSandbox = tmpDir; initTmpDir(); goto fallback; } else if (res != 0) @@ -2443,6 +2435,14 @@ void DerivationGoal::startBuilder() void DerivationGoal::initTmpDir() { + /* In a sandbox, for determinism, always use the same temporary + directory. */ +#if __linux__ + tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; +#else + tmpDirInSandbox = tmpDir; +#endif + /* In non-structured mode, add all bindings specified in the derivation via the environment, except those listed in the passAsFile attribute. Those are passed as file names pointing From 7c568d4c6e3291f01833aa9cf3779dfdcdaa4505 Mon Sep 17 00:00:00 2001 From: xbreak Date: Sat, 12 Oct 2019 13:00:19 +0000 Subject: [PATCH 03/60] Downloader: Warn if no trusted CA file has been configured --- src/libstore/download.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index cdf56e09d..e5f8556b7 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -285,6 +285,7 @@ struct CurlDownloader : public Downloader } if (request.verifyTLS) { + debug("verify TLS: Nix CA file = '%s'", settings.caFile); if (settings.caFile != "") curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str()); } else { From 3e85c57a6cbf46d5f0fe8a89b368a43abd26daba Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 1 Nov 2019 13:27:40 -0400 Subject: [PATCH 04/60] Pass --static flag to pkg-config when necessary --- configure.ac | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/configure.ac b/configure.ac index ebe6d4267..20969c14f 100644 --- a/configure.ac +++ b/configure.ac @@ -170,6 +170,18 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then LIBS="-latomic $LIBS" fi +PKG_PROG_PKG_CONFIG + +AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared], + [Build shared libraries for Nix [default=yes]]), + shared=$enableval, shared=yes) +if test "$shared" = yes; then + AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.]) +else + AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.]) + PKG_CONFIG="$PKG_CONFIG --static" +fi + # Look for OpenSSL, a required dependency. PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) @@ -302,16 +314,6 @@ AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH], sandbox_shell=$withval) AC_SUBST(sandbox_shell) -AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared], - [Build shared libraries for Nix [default=yes]]), - shared=$enableval, shared=yes) -if test "$shared" = yes; then - AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.]) -else - AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.]) -fi - - # Expand all variables in config.status. test "$prefix" = NONE && prefix=$ac_default_prefix test "$exec_prefix" = NONE && exec_prefix='${prefix}' From c4d7c76b641d82b2696fef73ce0ac160043c18da Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Oct 2018 16:01:26 +0200 Subject: [PATCH 05/60] Recursive Nix support This allows Nix builders to call Nix to build derivations, with some limitations. Example: let nixpkgs = fetchTarball channel:nixos-18.03; in with import {}; runCommand "foo" { buildInputs = [ nix jq ]; NIX_PATH = "nixpkgs=${nixpkgs}"; } '' hello=$(nix-build -E '(import {}).hello.overrideDerivation (args: { name = "hello-3.5"; })') $hello/bin/hello mkdir -p $out/bin ln -s $hello/bin/hello $out/bin/hello nix path-info -r --json $hello | jq . '' This derivation makes a recursive Nix call to build GNU Hello and symlinks it from its $out, i.e. # ll ./result/bin/ lrwxrwxrwx 1 root root 63 Jan 1 1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello # nix-store -qR ./result /nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131 /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5 /nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo This is implemented as follows: * Before running the outer builder, Nix creates a Unix domain socket '.nix-socket' in the builder's temporary directory and sets $NIX_REMOTE to point to it. It starts a thread to process connections to this socket. (Thus you don't need to have nix-daemon running.) * The daemon thread uses a wrapper store (RestrictedStore) to keep track of paths added through recursive Nix calls, to implement some restrictions (see below), and to do some censorship (e.g. for purity, queryPathInfo() won't return impure information such as signatures and timestamps). * After the build finishes, the output paths are scanned for references to the paths added through recursive Nix calls (in addition to the inputs closure). Thus, in the example above, $out has a reference to $hello. The main restriction on recursive Nix calls is that they cannot do arbitrary substitutions. For example, doing nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10 is forbidden unless /nix/store/kmwd... is in the inputs closure or previously built by a recursive Nix call. This is to prevent irreproducible derivations that have hidden dependencies on substituters or the current store contents. Building a derivation is fine, however, and Nix will use substitutes if available. In other words, the builder has to present proof that it knows how to build a desired store path from scratch by constructing a derivation graph for that path. Probably we should also disallow instantiating/building fixed-output derivations (specifically, those that access the network, but currently we have no way to mark fixed-output derivations that don't access the network). Otherwise sandboxed derivations can bypass sandbox restrictions and access the network. When sandboxing is enabled, we make paths appear in the sandbox of the builder by entering the mount namespace of the builder and bind-mounting each path. This is tricky because we do a pivot_root() in the builder to change the root directory of its mount namespace, and thus the host /nix/store is not visible in the mount namespace of the builder. To get around this, just before doing pivot_root(), we branch a second mount namespace that shares its /nix/store mountpoint with the parent. Recursive Nix currently doesn't work on macOS in sandboxed mode (because we can't change the sandbox policy of a running build) and in non-root mode (because setns() barfs). --- src/libstore/build.cc | 403 +++++++++++++++++++++++++++++++++-- src/libstore/daemon.cc | 153 ++++++++----- src/libstore/daemon.hh | 6 +- src/nix-daemon/nix-daemon.cc | 8 +- 4 files changed, 492 insertions(+), 78 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index ada1cf8ec..2fd27cfa0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -13,6 +13,7 @@ #include "nar-info.hh" #include "parsed-derivations.hh" #include "machines.hh" +#include "daemon.hh" #include #include @@ -33,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -765,9 +767,6 @@ private: immediate input paths). */ PathSet inputPaths; - /* Referenceable paths (i.e., input and output paths). */ - PathSet allPaths; - /* Outputs that are already valid. If we're repairing, these are the outputs that are valid *and* not corrupt. */ PathSet validPaths; @@ -805,9 +804,13 @@ private: /* Pipe for the builder's standard output/error. */ Pipe builderOut; - /* Pipe for synchronising updates to the builder user namespace. */ + /* Pipe for synchronising updates to the builder namespaces. */ Pipe userNamespaceSync; + /* The mount namespace of the builder, used to add additional + paths to the sandbox as a result of recursive Nix calls. */ + AutoCloseFD sandboxMountNamespace; + /* The build hook. */ std::unique_ptr hook; @@ -886,6 +889,26 @@ private: /* The remote machine on which we're building. */ std::string machineName; + /* The recursive Nix daemon socket. */ + AutoCloseFD daemonSocket; + + /* The daemon main thread. */ + std::thread daemonThread; + + /* Paths that were added via recursive Nix calls. */ + PathSet addedPaths; + + /* Recursive Nix calls are only allowed to build or realize paths + in the original input closure or added via a recursive Nix call + (so e.g. you can't do 'nix-store -r /nix/store/' where + /nix/store/ is some arbitrary path in a binary cache). */ + bool isAllowed(const Path & path) + { + return inputPaths.count(path) || addedPaths.count(path); + } + + friend class RestrictedStore; + public: DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); @@ -942,6 +965,14 @@ private: /* Write a JSON file containing the derivation attributes. */ void writeStructuredAttrs(); + void startDaemon(); + + void stopDaemon(); + + /* Add 'path' to the set of paths that may be referenced by the + outputs, and make it appear in the sandbox. */ + void addDependency(const Path & path); + /* Make a file owned by the builder. */ void chownToBuilder(const Path & path); @@ -1043,6 +1074,7 @@ DerivationGoal::~DerivationGoal() /* Careful: we should never ever throw an exception from a destructor. */ try { killChild(); } catch (...) { ignoreException(); } + try { stopDaemon(); } catch (...) { ignoreException(); } try { deleteTmpDir(false); } catch (...) { ignoreException(); } try { closeLogFile(); } catch (...) { ignoreException(); } } @@ -1332,12 +1364,6 @@ void DerivationGoal::inputsRealised() /* Gather information necessary for computing the closure and/or running the build hook. */ - /* The outputs are referenceable paths. */ - for (auto & i : drv->outputs) { - debug(format("building path '%1%'") % i.second.path); - allPaths.insert(i.second.path); - } - /* Determine the full set of input paths. */ /* First, the input derivations. */ @@ -1362,8 +1388,6 @@ void DerivationGoal::inputsRealised() debug(format("added input paths %1%") % showPaths(inputPaths)); - allPaths.insert(inputPaths.begin(), inputPaths.end()); - /* Is this a fixed-output derivation? */ fixedOutput = drv->isFixedOutput(); @@ -1527,6 +1551,8 @@ void DerivationGoal::buildDone() uid and then messing around with our output. */ Finally releaseBuildUser([&]() { buildUser.reset(); }); + sandboxMountNamespace = -1; + /* Since we got an EOF on the logger pipe, the builder is presumed to have terminated. In fact, the builder could also have simply have closed its end of the pipe, so just to be sure, @@ -1558,6 +1584,9 @@ void DerivationGoal::buildDone() root. */ if (buildUser) buildUser->kill(); + /* Terminate the recursive Nix daemon. */ + stopDaemon(); + bool diskFull = false; try { @@ -2217,6 +2246,10 @@ void DerivationGoal::startBuilder() } } + /* Fire up a Nix daemon to process recursive Nix calls from the + builder. */ + startDaemon(); + /* Run the builder. */ printMsg(lvlChatty, format("executing builder '%1%'") % drv->builder); @@ -2391,6 +2424,12 @@ void DerivationGoal::startBuilder() writeFile("/proc/" + std::to_string(pid) + "/gid_map", (format("%d %d 1") % sandboxGid % hostGid).str()); + /* Save the mount namespace of the child. We have to do this + *before* the child does a chroot. */ + sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); + if (sandboxMountNamespace.get() == -1) + throw SysError("getting sandbox mount namespace"); + /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); userNamespaceSync.writeSide = -1; @@ -2621,6 +2660,302 @@ void DerivationGoal::writeStructuredAttrs() } +/* A wrapper around LocalStore that only allows building/querying of + paths that are in the input closures of the build or were added via + recursive Nix calls. */ +struct RestrictedStore : public LocalFSStore +{ + ref next; + + DerivationGoal & goal; + + RestrictedStore(const Params & params, ref next, DerivationGoal & goal) + : Store(params), LocalFSStore(params), next(next), goal(goal) + { } + + Path getRealStoreDir() override + { return next->realStoreDir; } + + std::string getUri() override + { return next->getUri(); } + + PathSet queryAllValidPaths() override + { + PathSet paths; + for (auto & p : goal.inputPaths) paths.insert(p); + for (auto & p : goal.addedPaths) paths.insert(p); + return paths; + } + + void queryPathInfoUncached(const Path & path, + Callback> callback) noexcept override + { + if (goal.isAllowed(path)) { + try { + /* Censor impure information. */ + auto info = std::make_shared(*next->queryPathInfo(path)); + info->deriver.clear(); + info->registrationTime = 0; + info->ultimate = false; + info->sigs.clear(); + callback(info); + } catch (InvalidPath &) { + callback(nullptr); + } + } else + callback(nullptr); + }; + + void queryReferrers(const Path & path, PathSet & referrers) override + { } + + PathSet queryDerivationOutputs(const Path & path) override + { throw Error("queryDerivationOutputs"); } + + StringSet queryDerivationOutputNames(const Path & path) override + { throw Error("queryDerivationOutputNames"); } + + Path queryPathFromHashPart(const string & hashPart) override + { throw Error("queryPathFromHashPart"); } + + Path addToStore(const string & name, const Path & srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override + { throw Error("addToStore"); + } + + Path addToStoreFromDump(const string & dump, const string & name, + bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + { + auto path = next->addToStoreFromDump(dump, name, recursive, hashAlgo, repair); + goal.addDependency(path); + return path; + } + + Path addTextToStore(const string & name, const string & s, + const PathSet & references, RepairFlag repair = NoRepair) override + { + auto path = next->addTextToStore(name, s, references, repair); + goal.addDependency(path); + return path; + } + + void narFromPath(const Path & path, Sink & sink) override + { + if (!goal.isAllowed(path)) + throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", path); + LocalFSStore::narFromPath(path, sink); + } + + void ensurePath(const Path & path) override + { + if (!goal.isAllowed(path)) + throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", path); + /* Nothing to be done; 'path' must already be valid. */ + } + + void buildPaths(const PathSet & paths, BuildMode buildMode) override + { + if (buildMode != bmNormal) throw Error("unsupported build mode"); + + PathSet newPaths; + + for (auto & path : paths) { + DrvPathWithOutputs i = parseDrvPathWithOutputs(path); + if (isDerivation(i.first)) { + if (!goal.isAllowed(i.first)) + throw InvalidPath("cannot build unknown path '%s' in recursive Nix", i.first); + auto drv = derivationFromPath(i.first); + for (auto & output : drv.outputs) + if (wantOutput(output.first, i.second)) + newPaths.insert(output.second.path); + } else if (!goal.isAllowed(path)) + throw InvalidPath("cannot build unknown path '%s' in recursive Nix", path); + } + + next->buildPaths(paths, buildMode); + + PathSet closure; + next->computeFSClosure(newPaths, closure); + for (auto & path : closure) + goal.addDependency(path); + } + + BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode = bmNormal) override + { unsupported("buildDerivation"); } + + void addTempRoot(const Path & path) + { } + + void addIndirectRoot(const Path & path) + { } + + Roots findRoots() + { return Roots(); } + + void collectGarbage(const GCOptions & options, GCResults & results) + { } + + void addSignatures(const Path & storePath, const StringSet & sigs) + { unsupported("addSignatures"); } + + void queryMissing(const PathSet & targets, + PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize) + { + /* This is slightly impure since it leaks information to the + client about what paths will be built/substituted or are + already present. Probably not a big deal. */ + + PathSet allowed; + for (auto & path : targets) { + DrvPathWithOutputs i = parseDrvPathWithOutputs(path); + if (goal.isAllowed(i.first)) + allowed.insert(i.first); + else + unknown.insert(i.first); + } + + next->queryMissing(allowed, willBuild, willSubstitute, + unknown, downloadSize, narSize); + } +}; + + +void DerivationGoal::startDaemon() +{ + Store::Params params; + params["path-info-cache-size"] = "0"; + params["store"] = worker.store.storeDir; + params["root"] = worker.store.rootDir; + params["state"] = "/no-such-path"; + params["log"] = "/no-such-path"; + auto store = make_ref(params, + ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), + *this); + + addedPaths.clear(); + + auto socketName = ".nix-socket"; + Path socketPath = tmpDir + "/" + socketName; + env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName; + + daemonSocket = createUnixDomainSocket(socketPath, 0600); + + chownToBuilder(socketPath); + + daemonThread = std::thread([this, store]() { + + while (true) { + + /* Accept a connection. */ + struct sockaddr_un remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + AutoCloseFD remote = accept(daemonSocket.get(), + (struct sockaddr *) &remoteAddr, &remoteAddrLen); + if (!remote) { + if (errno == EINTR) continue; + if (errno == EINVAL) break; + throw SysError("accepting connection"); + } + + closeOnExec(remote.get()); + + debug("received daemon connection"); + + // FIXME: process on a separate thread. + FdSource from(remote.get()); + FdSink to(remote.get()); + try { + daemon::processConnection(store, from, to, + daemon::NotTrusted, daemon::Recursive, "nobody", 65535); + } catch (SysError &) { + ignoreException(); + } + + debug("terminated daemon connection"); + } + + debug("daemon shutting down"); + }); +} + + +void DerivationGoal::stopDaemon() +{ + if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) + throw SysError("shutting down daemon socket"); + + if (daemonThread.joinable()) + daemonThread.join(); + + daemonSocket = -1; +} + + +void DerivationGoal::addDependency(const Path & path) +{ + worker.store.assertStorePath(path); + + if (isAllowed(path)) return; + + addedPaths.insert(path); + + /* If we're doing a sandbox build, then we have to make the path + appear in the sandbox. */ + if (useChroot) { + + debug("materialising '%s' in the sandbox", path); + + #if __linux__ + + Path source = worker.store.toRealPath(path); + Path target = chrootRootDir + path; + debug("bind-mounting %s -> %s", target, source); + + if (pathExists(target)) + throw Error("store path '%s' already exists in the sandbox", path); + + struct stat st; + if (lstat(source.c_str(), &st)) + throw SysError("getting attributes of path '%s'", source); + + if (S_ISDIR(st.st_mode)) { + + /* Bind-mount the path into the sandbox. This requires + entering its mount namespace, which is not possible + in multithreaded programs. So we do this in a + child process.*/ + Pid child(startProcess([&]() { + + if (setns(sandboxMountNamespace.get(), 0) == -1) + throw SysError("entering sandbox mount namespace"); + + createDirs(target); + + if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) + throw SysError("bind mount from '%s' to '%s' failed", source, target); + + _exit(0); + })); + + int status = child.wait(); + if (status != 0) + throw Error("could not add path '%s' to sandbox", path); + + } else + linkOrCopy(source, target); + + #else + throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", path); + #endif + + } +} + + void DerivationGoal::chownToBuilder(const Path & path) { if (!buildUser) return; @@ -2756,15 +3091,30 @@ void DerivationGoal::runChild() outside of the namespace. Making a subtree private is local to the namespace, though, so setting MS_PRIVATE does not affect the outside world. */ - if (mount(0, "/", 0, MS_REC|MS_PRIVATE, 0) == -1) { - throw SysError("unable to make '/' private mount"); - } + if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) + throw SysError("unable to make '/' private"); /* Bind-mount chroot directory to itself, to treat it as a different filesystem from /, as needed for pivot_root. */ if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) throw SysError(format("unable to bind mount '%1%'") % chrootRootDir); + /* Bind-mount the sandbox's Nix store onto itself so that + we can mark it as a "shared" subtree, allowing bind + mounts made in *this* mount namespace to be propagated + into the child namespace created by the + unshare(CLONE_NEWNS) call below. + + Marking chrootRootDir as MS_SHARED causes pivot_root() + to fail with EINVAL. Don't know why. */ + Path chrootStoreDir = chrootRootDir + worker.store.storeDir; + + if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1) + throw SysError("unable to bind mount the Nix store", chrootStoreDir); + + if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1) + throw SysError("unable to make '%s' shared", chrootStoreDir); + /* Set up a nearly empty /dev, unless the user asked to bind-mount the host /dev. */ Strings ss; @@ -2866,6 +3216,19 @@ void DerivationGoal::runChild() } } + /* Unshare this mount namespace. This is necessary because + pivot_root() below changes the root of the mount + namespace. This means that the call to setns() in + addDependency() would hide the host's filesystem, + making it impossible to bind-mount paths from the host + Nix store into the sandbox. Therefore, we save the + pre-pivot_root namespace in + sandboxMountNamespace. Since we made /nix/store a + shared subtree above, this allows addDependency() to + make paths appear in the sandbox. */ + if (unshare(CLONE_NEWNS) == -1) + throw SysError("unsharing mount namespace"); + /* Do the chroot(). */ if (chdir(chrootRootDir.c_str()) == -1) throw SysError(format("cannot change directory to '%1%'") % chrootRootDir); @@ -3182,6 +3545,14 @@ void DerivationGoal::registerOutputs() std::exception_ptr delayedException; + /* The paths that can be referenced are the input closures, the + output paths, and any paths that have been built via recursive + Nix calls. */ + PathSet referenceablePaths; + for (auto & p : inputPaths) referenceablePaths.insert(p); + for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path); + for (auto & p : addedPaths) referenceablePaths.insert(p); + /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ @@ -3317,7 +3688,7 @@ void DerivationGoal::registerOutputs() verify later on whether nobody has messed with the store. */ debug("scanning for references inside '%1%'", path); HashResult hash; - PathSet references = scanForReferences(actualPath, allPaths, hash); + PathSet references = scanForReferences(actualPath, referenceablePaths, hash); if (buildMode == bmCheck) { if (!worker.store.isValidPath(path)) continue; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 64147027b..d3c17e772 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -186,8 +186,75 @@ struct RetrieveRegularNARSink : ParseSink } }; +struct ClientSettings +{ + bool keepFailed; + bool keepGoing; + bool tryFallback; + Verbosity verbosity; + unsigned int maxBuildJobs; + time_t maxSilentTime; + bool verboseBuild; + unsigned int buildCores; + bool useSubstitutes; + StringMap overrides; + + void apply(TrustedFlag trusted) + { + settings.keepFailed = keepFailed; + settings.keepGoing = keepGoing; + settings.tryFallback = tryFallback; + nix::verbosity = verbosity; + settings.maxBuildJobs.assign(maxBuildJobs); + settings.maxSilentTime = maxSilentTime; + settings.verboseBuild = verboseBuild; + settings.buildCores = buildCores; + settings.useSubstitutes = useSubstitutes; + + for (auto & i : overrides) { + auto & name(i.first); + auto & value(i.second); + + auto setSubstituters = [&](Setting & res) { + if (name != res.name && res.aliases.count(name) == 0) + return false; + StringSet trusted = settings.trustedSubstituters; + for (auto & s : settings.substituters.get()) + trusted.insert(s); + Strings subs; + auto ss = tokenizeString(value); + for (auto & s : ss) + if (trusted.count(s)) + subs.push_back(s); + else + warn("ignoring untrusted substituter '%s'", s); + res = subs; + return true; + }; + + try { + if (name == "ssh-auth-sock") // obsolete + ; + else if (trusted + || name == settings.buildTimeout.name + || name == "connect-timeout" + || (name == "builders" && value == "")) + settings.set(name, value); + else if (setSubstituters(settings.substituters)) + ; + else if (setSubstituters(settings.extraSubstituters)) + ; + else + warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); + } catch (UsageError & e) { + warn(e.what()); + } + } + } +}; + static void performOp(TunnelLogger * logger, ref store, - bool trusted, unsigned int clientVersion, + TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, Source & from, BufferedSink & to, unsigned int op) { switch (op) { @@ -464,70 +531,37 @@ static void performOp(TunnelLogger * logger, ref store, } case wopSetOptions: { - settings.keepFailed = readInt(from); - settings.keepGoing = readInt(from); - settings.tryFallback = readInt(from); - verbosity = (Verbosity) readInt(from); - settings.maxBuildJobs.assign(readInt(from)); - settings.maxSilentTime = readInt(from); + + ClientSettings clientSettings; + + clientSettings.keepFailed = readInt(from); + clientSettings.keepGoing = readInt(from); + clientSettings.tryFallback = readInt(from); + clientSettings.verbosity = (Verbosity) readInt(from); + clientSettings.maxBuildJobs = readInt(from); + clientSettings.maxSilentTime = readInt(from); readInt(from); // obsolete useBuildHook - settings.verboseBuild = lvlError == (Verbosity) readInt(from); + clientSettings.verboseBuild = lvlError == (Verbosity) readInt(from); readInt(from); // obsolete logType readInt(from); // obsolete printBuildTrace - settings.buildCores = readInt(from); - settings.useSubstitutes = readInt(from); + clientSettings.buildCores = readInt(from); + clientSettings.useSubstitutes = readInt(from); - StringMap overrides; if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { unsigned int n = readInt(from); for (unsigned int i = 0; i < n; i++) { string name = readString(from); string value = readString(from); - overrides.emplace(name, value); + clientSettings.overrides.emplace(name, value); } } logger->startWork(); - for (auto & i : overrides) { - auto & name(i.first); - auto & value(i.second); - - auto setSubstituters = [&](Setting & res) { - if (name != res.name && res.aliases.count(name) == 0) - return false; - StringSet trusted = settings.trustedSubstituters; - for (auto & s : settings.substituters.get()) - trusted.insert(s); - Strings subs; - auto ss = tokenizeString(value); - for (auto & s : ss) - if (trusted.count(s)) - subs.push_back(s); - else - warn("ignoring untrusted substituter '%s'", s); - res = subs; - return true; - }; - - try { - if (name == "ssh-auth-sock") // obsolete - ; - else if (trusted - || name == settings.buildTimeout.name - || name == "connect-timeout" - || (name == "builders" && value == "")) - settings.set(name, value); - else if (setSubstituters(settings.substituters)) - ; - else if (setSubstituters(settings.extraSubstituters)) - ; - else - warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); - } catch (UsageError & e) { - warn(e.what()); - } - } + // FIXME: use some setting in recursive mode. Will need to use + // non-global variables. + if (!recursive) + clientSettings.apply(trusted); logger->stopWork(); break; @@ -694,11 +728,12 @@ void processConnection( ref store, FdSource & from, FdSink & to, - bool trusted, + TrustedFlag trusted, + RecursiveFlag recursive, const std::string & userName, uid_t userId) { - MonitorFdHup monitor(from.fd); + auto monitor = !recursive ? std::make_unique(from.fd) : nullptr; /* Exchange the greeting. */ unsigned int magic = readInt(from); @@ -712,7 +747,9 @@ void processConnection( auto tunnelLogger = new TunnelLogger(to, clientVersion); auto prevLogger = nix::logger; - logger = tunnelLogger; + // FIXME + if (!recursive) + logger = tunnelLogger; unsigned int opCount = 0; @@ -721,8 +758,10 @@ void processConnection( prevLogger->log(lvlDebug, fmt("%d operations", opCount)); }); - if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) - setAffinityTo(readInt(from)); + if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) { + auto affinity = readInt(from); + setAffinityTo(affinity); + } readInt(from); // obsolete reserveSpace @@ -760,7 +799,7 @@ void processConnection( opCount++; try { - performOp(tunnelLogger, store, trusted, clientVersion, from, to, op); + performOp(tunnelLogger, store, trusted, recursive, clientVersion, from, to, op); } catch (Error & e) { /* If we're not in a state where we can send replies, then something went wrong processing the input of the diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index fa26ba9c2..266932013 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -3,11 +3,15 @@ namespace nix::daemon { +enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; +enum RecursiveFlag : bool { NotRecursive = false, Recursive = true }; + void processConnection( ref store, FdSource & from, FdSink & to, - bool trusted, + TrustedFlag trusted, + RecursiveFlag recursive, const std::string & userName, uid_t userId); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index da17ddcba..9314b6161 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -192,7 +192,7 @@ static void daemonLoop(char * * argv) closeOnExec(remote.get()); - bool trusted = false; + TrustedFlag trusted = NotTrusted; PeerInfo peer = getPeerInfo(remote.get()); struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; @@ -205,7 +205,7 @@ static void daemonLoop(char * * argv) Strings allowedUsers = settings.allowedUsers; if (matchUser(user, group, trustedUsers)) - trusted = true; + trusted = Trusted; if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) throw Error(format("user '%1%' is not allowed to connect to the Nix daemon") % user); @@ -239,7 +239,7 @@ static void daemonLoop(char * * argv) /* Handle the connection. */ FdSource from(remote.get()); FdSink to(remote.get()); - processConnection(openUncachedStore(), from, to, trusted, user, peer.uid); + processConnection(openUncachedStore(), from, to, trusted, NotRecursive, user, peer.uid); exit(0); }, options); @@ -321,7 +321,7 @@ static int _main(int argc, char * * argv) } else { FdSource from(STDIN_FILENO); FdSink to(STDOUT_FILENO); - processConnection(openUncachedStore(), from, to, true, "root", 0); + processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, "root", 0); } } else { daemonLoop(argv); From c921074c1921d4bf392991a44c5644877f9c567a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Nov 2019 12:55:05 +0100 Subject: [PATCH 06/60] RestrictedStore: Implement addToStore() --- src/libstore/build.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 2fd27cfa0..e8b23d396 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2721,7 +2721,14 @@ struct RestrictedStore : public LocalFSStore Path addToStore(const string & name, const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override - { throw Error("addToStore"); + { throw Error("addToStore"); } + + void addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, + std::shared_ptr accessor = 0) override + { + next->addToStore(info, narSource, repair, checkSigs, accessor); + goal.addDependency(info.path); } Path addToStoreFromDump(const string & dump, const string & name, From 2af9561316606892ce0337939c01f3c0d43d75bd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Nov 2019 13:23:58 +0100 Subject: [PATCH 07/60] Add a test for recursive Nix --- tests/local.mk | 3 ++- tests/recursive.sh | 67 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 tests/recursive.sh diff --git a/tests/local.mk b/tests/local.mk index 187f96ea2..affdc3cc3 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -30,7 +30,8 @@ nix_tests = \ search.sh \ nix-copy-ssh.sh \ post-hook.sh \ - function-trace.sh + function-trace.sh \ + recursive.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/recursive.sh b/tests/recursive.sh new file mode 100644 index 000000000..25204caf1 --- /dev/null +++ b/tests/recursive.sh @@ -0,0 +1,67 @@ +source common.sh + +clearStore + +export unreachable=$(nix add-to-store ./recursive.sh) + +nix build -o $TEST_ROOT/result -L '( + with import ./config.nix; + with import ; + mkDerivation { + name = "recursive"; + dummy = builtins.toFile "dummy" "bla bla"; + SHELL = shell; + + # Note: this is a string without context. + unreachable = builtins.getEnv "unreachable"; + + buildCommand = '\'\'' + mkdir $out + PATH=${nixBinDir}:$PATH + opts="--experimental-features nix-command" + + # Check that we can query/build paths in our input closure. + nix $opts path-info $dummy + nix $opts build $dummy + + # Make sure we cannot query/build paths not in out input closure. + [[ -e $unreachable ]] + (! nix $opts path-info $unreachable) + (! nix $opts build $unreachable) + + # Add something to the store. + echo foobar > foobar + foobar=$(nix $opts add-to-store ./foobar) + + nix $opts path-info $foobar + nix $opts build $foobar + + # Add it to our closure. + ln -s $foobar $out/foobar + + [[ $(nix $opts path-info --all | wc -l) -eq 3 ]] + + # Build a derivation. + nix $opts build -L '\''( + derivation { + name = "inner1"; + builder = builtins.getEnv "SHELL"; + system = builtins.getEnv "system"; + fnord = builtins.toFile "fnord" "fnord"; + args = [ "-c" "echo $fnord blaat > $out" ]; + } + )'\'' + + [[ $(nix $opts path-info --json ./result) =~ fnord ]] + + ln -s $(nix $opts path-info ./result) $out/inner1 + '\'\''; + }) +' + +[[ $(cat $TEST_ROOT/result/inner1) =~ blaat ]] + +# Make sure the recursively created paths are in the closure. +nix path-info -r $TEST_ROOT/result | grep foobar +nix path-info -r $TEST_ROOT/result | grep fnord +nix path-info -r $TEST_ROOT/result | grep inner1 From c119ab9db0edf65379593883c5ed5253549ebbd0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Nov 2019 13:48:58 +0100 Subject: [PATCH 08/60] Enable recursive Nix using a feature Derivations that want to use recursion should now set requiredSystemFeatures = [ "recursive-nix" ]; to make the daemon socket appear. Also, Nix should be configured with "experimental-features = recursive-nix". --- src/libstore/build.cc | 5 ++++- src/libstore/globals.cc | 2 +- tests/recursive.sh | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index e8b23d396..e77512ca4 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2248,7 +2248,8 @@ void DerivationGoal::startBuilder() /* Fire up a Nix daemon to process recursive Nix calls from the builder. */ - startDaemon(); + if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix")) + startDaemon(); /* Run the builder. */ printMsg(lvlChatty, format("executing builder '%1%'") % drv->builder); @@ -2832,6 +2833,8 @@ struct RestrictedStore : public LocalFSStore void DerivationGoal::startDaemon() { + settings.requireExperimentalFeature("recursive-nix"); + Store::Params params; params["path-info-cache-size"] = "0"; params["store"] = worker.store.storeDir; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 249c36673..042aaea95 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -95,7 +95,7 @@ StringSet Settings::getDefaultSystemFeatures() /* For backwards compatibility, accept some "features" that are used in Nixpkgs to route builds to certain machines but don't actually require anything special on the machines. */ - StringSet features{"nixos-test", "benchmark", "big-parallel"}; + StringSet features{"nixos-test", "benchmark", "big-parallel", "recursive-nix"}; #if __linux__ if (access("/dev/kvm", R_OK | W_OK) == 0) diff --git a/tests/recursive.sh b/tests/recursive.sh index 25204caf1..62896c015 100644 --- a/tests/recursive.sh +++ b/tests/recursive.sh @@ -4,7 +4,7 @@ clearStore export unreachable=$(nix add-to-store ./recursive.sh) -nix build -o $TEST_ROOT/result -L '( +nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L '( with import ./config.nix; with import ; mkDerivation { @@ -15,6 +15,8 @@ nix build -o $TEST_ROOT/result -L '( # Note: this is a string without context. unreachable = builtins.getEnv "unreachable"; + requiredSystemFeatures = [ "recursive-nix" ]; + buildCommand = '\'\'' mkdir $out PATH=${nixBinDir}:$PATH From 69326f3637f1560407711838e7298d736274ffd4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Nov 2019 14:27:28 +0100 Subject: [PATCH 09/60] Recursive Nix: Handle concurrent client connections --- src/libstore/build.cc | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index e77512ca4..63557b560 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -895,6 +895,9 @@ private: /* The daemon main thread. */ std::thread daemonThread; + /* The daemon worker threads. */ + std::vector daemonWorkerThreads; + /* Paths that were added via recursive Nix calls. */ PathSet addedPaths; @@ -2875,17 +2878,19 @@ void DerivationGoal::startDaemon() debug("received daemon connection"); - // FIXME: process on a separate thread. - FdSource from(remote.get()); - FdSink to(remote.get()); - try { - daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, "nobody", 65535); - } catch (SysError &) { - ignoreException(); - } + auto workerThread = std::thread([this, store, remote{std::move(remote)}]() { + FdSource from(remote.get()); + FdSink to(remote.get()); + try { + daemon::processConnection(store, from, to, + daemon::NotTrusted, daemon::Recursive, "nobody", 65535); + debug("terminated daemon connection"); + } catch (SysError &) { + ignoreException(); + } + }); - debug("terminated daemon connection"); + daemonWorkerThreads.push_back(std::move(workerThread)); } debug("daemon shutting down"); @@ -2901,6 +2906,12 @@ void DerivationGoal::stopDaemon() if (daemonThread.joinable()) daemonThread.join(); + // FIXME: should prune worker threads more quickly. + // FIXME: shutdown the client socket to speed up worker termination. + for (auto & thread : daemonWorkerThreads) + thread.join(); + daemonWorkerThreads.clear(); + daemonSocket = -1; } From 52ffe2797a6a943bac968e92ad443a567dfdf80e Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Thu, 7 Nov 2019 00:11:57 +0100 Subject: [PATCH 10/60] doc: Document `--dry-run` option for `nix-build` --- doc/manual/command-ref/nix-build.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/manual/command-ref/nix-build.xml b/doc/manual/command-ref/nix-build.xml index 40fe7a43f..c1b783c87 100644 --- a/doc/manual/command-ref/nix-build.xml +++ b/doc/manual/command-ref/nix-build.xml @@ -30,6 +30,7 @@ attrPath + @@ -98,6 +99,10 @@ also . + + Show what store paths would be built or downloaded + + / outlink From a08f35392256b1ef23947857e41a9b12b1591245 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Fri, 8 Nov 2019 09:38:54 +0000 Subject: [PATCH 11/60] gitignore /precompiled-headers.h.?ch --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b75c5d489..fd62dfb38 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ perl/Makefile.config # / /aclocal.m4 /autom4te.cache +/precompiled-headers.h.gch +/precompiled-headers.h.pch /config.* /configure /nix.spec From 6c041e84139fc06110f38d940d6117badcc1fd0a Mon Sep 17 00:00:00 2001 From: Eric Culp Date: Thu, 31 Oct 2019 17:55:43 -0700 Subject: [PATCH 12/60] Replace $TMPDIR with $TEST_ROOT in tests/fetchurl.sh $TMPDIR isn't necessarily set and would cause this test to fail. --- tests/fetchurl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 7319ced2b..2535651b0 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -35,7 +35,7 @@ clearStore hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh) hash32=$(nix hash-file --type sha512 --base16 ./fetchurl.sh) -mirror=$TMPDIR/hashed-mirror +mirror=$TEST_ROOT/hashed-mirror rm -rf $mirror mkdir -p $mirror/sha512 ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32 From 2ba9f227154e04aefc4ed47f9f23c97a7aefa269 Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Sun, 10 Nov 2019 10:02:22 +0100 Subject: [PATCH 13/60] De-duplicate struct PrimOp forward declaration --- src/libexpr/value.hh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index e1ec87d3b..6743d7fd8 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -35,7 +35,6 @@ struct Env; struct Expr; struct ExprLambda; struct PrimOp; -struct PrimOp; class Symbol; struct Pos; class EvalState; From 96e6e680c1a298613756571e4984c340ce887c2a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 10 Nov 2019 11:14:26 -0500 Subject: [PATCH 14/60] Fix extra ; warnings involving MakeError --- src/libexpr/json-to-value.hh | 2 +- src/libexpr/nixexpr.hh | 16 ++++++++-------- src/libstore/build.cc | 2 +- src/libstore/store-api.hh | 12 ++++++------ src/libutil/serialise.hh | 2 +- src/libutil/thread-pool.hh | 2 +- src/libutil/types.hh | 4 ++-- src/libutil/util.hh | 6 +++--- 8 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/libexpr/json-to-value.hh b/src/libexpr/json-to-value.hh index 33f35b16c..3b0fdae11 100644 --- a/src/libexpr/json-to-value.hh +++ b/src/libexpr/json-to-value.hh @@ -6,7 +6,7 @@ namespace nix { -MakeError(JSONParseError, EvalError) +MakeError(JSONParseError, EvalError); void parseJSON(EvalState & state, const string & s, Value & v); diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 665a42987..f7e9105a4 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -9,14 +9,14 @@ namespace nix { -MakeError(EvalError, Error) -MakeError(ParseError, Error) -MakeError(AssertionError, EvalError) -MakeError(ThrownError, AssertionError) -MakeError(Abort, EvalError) -MakeError(TypeError, EvalError) -MakeError(UndefinedVarError, Error) -MakeError(RestrictedPathError, Error) +MakeError(EvalError, Error); +MakeError(ParseError, Error); +MakeError(AssertionError, EvalError); +MakeError(ThrownError, AssertionError); +MakeError(Abort, EvalError); +MakeError(TypeError, EvalError); +MakeError(UndefinedVarError, Error); +MakeError(RestrictedPathError, Error); /* Position objects. */ diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 68c2f2ce3..a1b9c92f8 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1515,7 +1515,7 @@ void replaceValidPath(const Path & storePath, const Path tmpPath) } -MakeError(NotDeterministic, BuildError) +MakeError(NotDeterministic, BuildError); void DerivationGoal::buildDone() diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 250e1a4bc..b860b48be 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -20,12 +20,12 @@ namespace nix { -MakeError(SubstError, Error) -MakeError(BuildError, Error) /* denotes a permanent build failure */ -MakeError(InvalidPath, Error) -MakeError(Unsupported, Error) -MakeError(SubstituteGone, Error) -MakeError(SubstituterDisabled, Error) +MakeError(SubstError, Error); +MakeError(BuildError, Error) /* denotes a permanent build failure */; +MakeError(InvalidPath, Error); +MakeError(Unsupported, Error); +MakeError(SubstituteGone, Error); +MakeError(SubstituterDisabled, Error); struct BasicDerivation; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 0120aeecb..128e287f3 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -274,7 +274,7 @@ Sink & operator << (Sink & sink, const Strings & s); Sink & operator << (Sink & sink, const StringSet & s); -MakeError(SerialisationError, Error) +MakeError(SerialisationError, Error); template diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index bb16b639a..b22e0d162 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -11,7 +11,7 @@ namespace nix { -MakeError(ThreadPoolShutDown, Error) +MakeError(ThreadPoolShutDown, Error); /* A simple thread pool that executes a queue of work items (lambdas). */ diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 92bf469b5..4bc91828b 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -116,9 +116,9 @@ public: { \ public: \ using superClass::superClass; \ - }; + } -MakeError(Error, BaseError) +MakeError(Error, BaseError); class SysError : public Error { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 3493e80b5..5f1d1d181 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -160,7 +160,7 @@ void readFull(int fd, unsigned char * buf, size_t count); void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts = true); void writeFull(int fd, const string & s, bool allowInterrupts = true); -MakeError(EndOfFile, Error) +MakeError(EndOfFile, Error); /* Read a file descriptor until EOF occurs. */ @@ -333,10 +333,10 @@ void inline checkInterrupt() _interrupted(); } -MakeError(Interrupted, BaseError) +MakeError(Interrupted, BaseError); -MakeError(FormatError, Error) +MakeError(FormatError, Error); /* String tokenizer. */ From 4c34054673edd67cd1cfd81ab2972f27e518c78a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 10 Nov 2019 11:23:35 -0500 Subject: [PATCH 15/60] Remove unneeded semicolons --- src/libexpr/eval.cc | 2 +- src/libutil/args.hh | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f60e8d3ab..34e527c52 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -432,7 +432,7 @@ Path EvalState::toRealPath(const Path & path, const PathSet & context) !context.empty() && store->isInStore(path) ? store->toRealPath(path) : path; -}; +} Value * EvalState::addConstant(const string & name, Value & v) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index ad5fcca39..f8df39b5c 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -70,23 +70,23 @@ public: Args & args; Flag::ptr flag; friend class Args; - FlagMaker(Args & args) : args(args), flag(std::make_shared()) { }; + FlagMaker(Args & args) : args(args), flag(std::make_shared()) { } public: ~FlagMaker(); - FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; }; - FlagMaker & shortName(char s) { flag->shortName = s; return *this; }; - FlagMaker & description(const std::string & s) { flag->description = s; return *this; }; - FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; }; - FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; }; - FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; }; - FlagMaker & handler(std::function)> handler) { flag->handler = handler; return *this; }; - FlagMaker & handler(std::function handler) { flag->handler = [handler](std::vector) { handler(); }; return *this; }; + FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; } + FlagMaker & shortName(char s) { flag->shortName = s; return *this; } + FlagMaker & description(const std::string & s) { flag->description = s; return *this; } + FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; } + FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; } + FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; } + FlagMaker & handler(std::function)> handler) { flag->handler = handler; return *this; } + FlagMaker & handler(std::function handler) { flag->handler = [handler](std::vector) { handler(); }; return *this; } FlagMaker & handler(std::function handler) { flag->arity = 1; flag->handler = [handler](std::vector ss) { handler(std::move(ss[0])); }; return *this; - }; - FlagMaker & category(const std::string & s) { flag->category = s; return *this; }; + } + FlagMaker & category(const std::string & s) { flag->category = s; return *this; } template FlagMaker & dest(T * dest) @@ -94,7 +94,7 @@ public: flag->arity = 1; flag->handler = [=](std::vector ss) { *dest = ss[0]; }; return *this; - }; + } template FlagMaker & set(T * dest, const T & val) @@ -102,7 +102,7 @@ public: flag->arity = 0; flag->handler = [=](std::vector ss) { *dest = val; }; return *this; - }; + } FlagMaker & mkHashTypeFlag(HashType * ht); }; From 07294e988cc54f1c2570c567662cf39eabbd610c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Sun, 10 Nov 2019 17:29:13 +0100 Subject: [PATCH 16/60] manual: nix-shell: Elaborate on using `-p` with expressions. This documents the outcome of the change in https://github.com/NixOS/nix/issues/454: > We can also automatically add parentheses in the generated > `buildInputs`, so you can type `nix-shell -p "expr"` > instead of `"(expr"). --- doc/manual/command-ref/nix-shell.xml | 24 +++++++++++++++++++----- doc/manual/command-ref/opt-common.xml | 9 ++++++++- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml index bb4a4e420..766482460 100644 --- a/doc/manual/command-ref/nix-shell.xml +++ b/doc/manual/command-ref/nix-shell.xml @@ -39,7 +39,12 @@ - packages + + + packages + expressions + + path @@ -189,8 +194,8 @@ also . NIX_BUILD_SHELL - - Shell used to start the interactive environment. + + Shell used to start the interactive environment. Defaults to the bash found in PATH. @@ -222,8 +227,9 @@ $ nix-shell '<nixpkgs>' -A pan --pure \ --command 'export NIX_DEBUG=1; export NIX_CORES=8; return' -Nix expressions can also be given on the command line. For instance, -the following starts a shell containing the packages +Nix expressions can also be given on the command line using the +-E and -p flags. +For instance, the following starts a shell containing the packages sqlite and libX11: @@ -238,6 +244,14 @@ $ nix-shell -p sqlite xorg.libX11 … -L/nix/store/j1zg5v…-sqlite-3.8.0.2/lib -L/nix/store/0gmcz9…-libX11-1.6.1/lib … +Note that -p accepts multiple full nix expressions that +are valid in the buildInputs = [ ... ] shown above, +not only package names. So the following is also legal: + + +$ nix-shell -p sqlite 'git.override { withManual = false; }' + + The -p flag looks up Nixpkgs in the Nix search path. You can override it by passing or setting NIX_PATH. For example, the following gives you a shell diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml index 6a8107be9..0383bfaed 100644 --- a/doc/manual/command-ref/opt-common.xml +++ b/doc/manual/command-ref/opt-common.xml @@ -323,7 +323,14 @@ Nix expressions to be parsed and evaluated, rather than as a list of file names of Nix expressions. (nix-instantiate, nix-build - and nix-shell only.) + and nix-shell only.) + + For nix-shell, this option is commonly used + to give you a shell in which you can build the packages returned + by the expression. If you want to get a shell which contain the + built packages ready for use, give your + expression to the nix-shell -p convenience flag + instead. From 8669db1dcc9378bd61438e6c5978aea5d53d546f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 10 Nov 2019 16:21:59 -0500 Subject: [PATCH 17/60] Clean up semicolon and comma Thanks @bhipple for catching! --- src/libstore/store-api.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b860b48be..1862fa267 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -21,7 +21,7 @@ namespace nix { MakeError(SubstError, Error); -MakeError(BuildError, Error) /* denotes a permanent build failure */; +MakeError(BuildError, Error); // denotes a permanent build failure MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); From 0352c1a4f8fdf911a3def97be1ab3d97a1817000 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Nov 2019 17:18:17 +0100 Subject: [PATCH 18/60] Typo --- src/build-remote/build-remote.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 279ae62f6..27dd35784 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -188,7 +188,7 @@ static int _main(int argc, char * * argv) Store::Params storeParams; if (hasPrefix(bestMachine->storeUri, "ssh://")) { - storeParams["max-connections"] ="1"; + storeParams["max-connections"] = "1"; storeParams["log-fd"] = "4"; if (bestMachine->sshKey != "") storeParams["ssh-key"] = bestMachine->sshKey; From 3e2c77d001a31952e62bfbb825bc7e034678ae12 Mon Sep 17 00:00:00 2001 From: Chuck Date: Fri, 15 Nov 2019 11:55:36 -0800 Subject: [PATCH 19/60] Check for and repair bad .links entries A corrupt entry in .links prevents adding a fixed version of that file to the store in any path. The user experience is that corruption present in the store 'spreads' to new paths added to the store: (With store optimisation enabled) 1. A file in the store gets corrupted somehow (eg: filesystem bug). 2. The user tries to add a thing to the store which contains a good copy of the corrupted file. 3. The file being added to the store is hashed, found to match the bad .links entry, and is replaced by a link to the bad .links entry. (The .links entry's hash is not verified during add -- this would impose a substantial performance burden.) 4. The user observes that the thing in the store that is supposed to be a copy of what they were trying to add is not a correct copy -- some files have different contents! Running "nix-store --verify --check-contents --repair" does not fix the problem. This change makes "nix-store --verify --check-contents --repair" fix this problem. Bad .links entries are simply removed, allowing future attempts to insert a good copy of the file to succeed. --- src/libstore/local-store.cc | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b5b0f1ba7..2a21f816c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1238,7 +1238,35 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Optionally, check the content hashes (slow). */ if (checkContents) { - printInfo("checking hashes..."); + + printInfo("checking link hashes..."); + + AutoCloseDir dir(opendir(linksDir.c_str())); + if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ + checkInterrupt(); + if (strcmp(dirent->d_name, ".") == 0 || strcmp(dirent->d_name, "..") == 0) continue; + Path linkPath = linksDir + "/" + dirent->d_name; + string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); + if (hash != dirent->d_name) { + printError(format("link '%1%' was modified! " + "expected hash '%2%', got '%3%'") + % linkPath % dirent->d_name % hash); + if (repair) { + if (unlink(linkPath.c_str()) == 0) + printError(format("Removed link '%1%'") % linkPath); + else + throw SysError(format("removing corrupt link '%1%'") % linkPath); + } else { + errors = true; + } + } + } + if (errno) throw SysError(format("reading directory '%1%'") % linksDir); + + printInfo("checking store hashes..."); Hash nullHash(htSHA256); From ba87b08f8529e4d9f8c58d8c625152058ceadb75 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 22 Nov 2019 16:06:44 +0100 Subject: [PATCH 20/60] getEnv(): Return std::optional This allows distinguishing between an empty value and no value. --- src/libexpr/eval.cc | 12 ++++++------ src/libexpr/primops.cc | 2 +- src/libstore/build.cc | 4 ++-- src/libstore/gc.cc | 6 +++--- src/libstore/globals.cc | 22 +++++++++++----------- src/libstore/globals.hh | 2 +- src/libstore/local-store.hh | 2 +- src/libstore/ssh.cc | 2 +- src/libutil/util.cc | 35 +++++++++++++++-------------------- src/libutil/util.hh | 2 +- src/nix-build/nix-build.cc | 17 +++++++++-------- src/nix-daemon/nix-daemon.cc | 5 +++-- src/nix-env/nix-env.cc | 2 +- src/nix/command.cc | 2 +- src/nix/doctor.cc | 4 ++-- src/nix/run.cc | 2 +- src/nix/upgrade-nix.cc | 2 +- 17 files changed, 60 insertions(+), 63 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 3c3d1a324..bb6b23ff1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -217,7 +217,7 @@ void initGC() that GC_expand_hp() causes a lot of virtual, but not physical (resident) memory to be allocated. This might be a problem on systems that don't overcommit. */ - if (!getenv("GC_INITIAL_HEAP_SIZE")) { + if (!getEnv("GC_INITIAL_HEAP_SIZE")) { size_t size = 32 * 1024 * 1024; #if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES) size_t maxSize = 384 * 1024 * 1024; @@ -306,7 +306,7 @@ EvalState::EvalState(const Strings & _searchPath, ref store) , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) { - countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0"; + countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; assert(gcInitialised); @@ -314,7 +314,7 @@ EvalState::EvalState(const Strings & _searchPath, ref store) /* Initialise the Nix expression search path. */ if (!evalSettings.pureEval) { - Strings paths = parseNixPath(getEnv("NIX_PATH", "")); + Strings paths = parseNixPath(getEnv("NIX_PATH").value_or("")); for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : paths) addToSearchPath(i); } @@ -1753,7 +1753,7 @@ bool EvalState::eqValues(Value & v1, Value & v2) void EvalState::printStats() { - bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0"; + bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0"; struct rusage buf; getrusage(RUSAGE_SELF, &buf); @@ -1769,7 +1769,7 @@ void EvalState::printStats() GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes); #endif if (showStats) { - auto outPath = getEnv("NIX_SHOW_STATS_PATH","-"); + auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-"); std::fstream fs; if (outPath != "-") fs.open(outPath, std::fstream::out); @@ -1861,7 +1861,7 @@ void EvalState::printStats() } } - if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") { + if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") { auto list = topObj.list("symbols"); symbols.dump([&](const std::string & s) { list.elem(s); }); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index deaffa605..2cc03fe61 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -469,7 +469,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v) { string name = state.forceStringNoCtx(*args[0], pos); - mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name)); + mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or("")); } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 35a5388c6..67ccfcee0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2503,7 +2503,7 @@ void DerivationGoal::initEnv() already know the cryptographic hash of the output). */ if (fixedOutput) { for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) - env[i] = getEnv(i); + env[i] = getEnv(i).value_or(""); } /* Currently structured log messages piggyback on stderr, but we @@ -3075,7 +3075,7 @@ void DerivationGoal::runChild() /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ - Path globalTmpDir = canonPath(getEnv("TMPDIR", "/tmp"), true); + Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true); /* They don't like trailing slashes on subpath directives */ if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 7d3da1cc8..842980fb7 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -870,11 +870,11 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) void LocalStore::autoGC(bool sync) { - static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE", ""); + static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE"); auto getAvail = [this]() -> uint64_t { - if (!fakeFreeSpaceFile.empty()) - return std::stoll(readFile(fakeFreeSpaceFile)); + if (fakeFreeSpaceFile) + return std::stoll(readFile(*fakeFreeSpaceFile)); struct statvfs st; if (statvfs(realStoreDir.c_str(), &st)) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 249c36673..aa0823f41 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -32,20 +32,20 @@ static GlobalConfig::Register r1(&settings); Settings::Settings() : nixPrefix(NIX_PREFIX) - , nixStore(canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)))) - , nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR))) - , nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR))) - , nixStateDir(canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR))) - , nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR))) - , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR))) - , nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR))) + , nixStore(canonPath(getEnv("NIX_STORE_DIR").value_or(getEnv("NIX_STORE").value_or(NIX_STORE_DIR)))) + , nixDataDir(canonPath(getEnv("NIX_DATA_DIR").value_or(NIX_DATA_DIR))) + , nixLogDir(canonPath(getEnv("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) + , nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) + , nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) + , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR))) + , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) , nixManDir(canonPath(NIX_MAN_DIR)) , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; - lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; + lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; - caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", "")); + caFile = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); if (caFile == "") { for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) if (pathExists(fn)) { @@ -56,9 +56,9 @@ Settings::Settings() /* Backwards compatibility. */ auto s = getEnv("NIX_REMOTE_SYSTEMS"); - if (s != "") { + if (s) { Strings ss; - for (auto & p : tokenizeString(s, ":")) + for (auto & p : tokenizeString(*s, ":")) ss.push_back("@" + p); builders = concatStringsSep(" ", ss); } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 1221e4db7..3eb2f698c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -66,7 +66,7 @@ public: /* File name of the socket the daemon listens to. */ Path nixDaemonSocketFile; - Setting storeUri{this, getEnv("NIX_REMOTE", "auto"), "store", + Setting storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", "The default Nix store to use."}; Setting keepFailed{this, false, "keep-failed", diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 5aa6b0519..1d4b88898 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -107,7 +107,7 @@ private: public: // Hack for build-remote.cc. - PathSet locksHeld = tokenizeString(getEnv("NIX_HELD_LOCKS")); + PathSet locksHeld; /* Initialise the local store, upgrading the schema if necessary. */ diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 5e0e44935..ac3ccd63d 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -16,7 +16,7 @@ SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool void SSHMaster::addCommonSSHOpts(Strings & args) { - for (auto & i : tokenizeString(getEnv("NIX_SSHOPTS"))) + for (auto & i : tokenizeString(getEnv("NIX_SSHOPTS").value_or(""))) args.push_back(i); if (!keyFile.empty()) args.insert(args.end(), {"-i", keyFile}); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 79716c5a7..b4d444a67 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -58,10 +58,11 @@ std::string SysError::addErrno(const std::string & s) } -string getEnv(const string & key, const string & def) +std::optional getEnv(const std::string & key) { char * value = getenv(key.c_str()); - return value ? string(value) : def; + if (!value) return {}; + return std::string(value); } @@ -438,7 +439,7 @@ void deletePath(const Path & path, unsigned long long & bytesFreed) static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, int & counter) { - tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR", "/tmp") : tmpRoot, true); + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); if (includePid) return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); else @@ -480,7 +481,7 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, std::string getUserName() { auto pw = getpwuid(geteuid()); - std::string name = pw ? pw->pw_name : getEnv("USER", ""); + std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); if (name.empty()) throw Error("cannot figure out user name"); return name; @@ -488,8 +489,8 @@ std::string getUserName() static Lazy getHome2([]() { - Path homeDir = getEnv("HOME"); - if (homeDir.empty()) { + auto homeDir = getEnv("HOME"); + if (!homeDir) { std::vector buf(16384); struct passwd pwbuf; struct passwd * pw; @@ -498,7 +499,7 @@ static Lazy getHome2([]() { throw Error("cannot determine user's home directory"); homeDir = pw->pw_dir; } - return homeDir; + return *homeDir; }); Path getHome() { return getHome2(); } @@ -506,25 +507,21 @@ Path getHome() { return getHome2(); } Path getCacheDir() { - Path cacheDir = getEnv("XDG_CACHE_HOME"); - if (cacheDir.empty()) - cacheDir = getHome() + "/.cache"; - return cacheDir; + auto cacheDir = getEnv("XDG_CACHE_HOME"); + return cacheDir ? *cacheDir : getHome() + "/.cache"; } Path getConfigDir() { - Path configDir = getEnv("XDG_CONFIG_HOME"); - if (configDir.empty()) - configDir = getHome() + "/.config"; - return configDir; + auto configDir = getEnv("XDG_CONFIG_HOME"); + return configDir ? *configDir : getHome() + "/.config"; } std::vector getConfigDirs() { Path configHome = getConfigDir(); - string configDirs = getEnv("XDG_CONFIG_DIRS"); + string configDirs = getEnv("XDG_CONFIG_DIRS").value_or(""); std::vector result = tokenizeString>(configDirs, ":"); result.insert(result.begin(), configHome); return result; @@ -533,10 +530,8 @@ std::vector getConfigDirs() Path getDataDir() { - Path dataDir = getEnv("XDG_DATA_HOME"); - if (dataDir.empty()) - dataDir = getHome() + "/.local/share"; - return dataDir; + auto dataDir = getEnv("XDG_DATA_HOME"); + return dataDir ? *dataDir : getHome() + "/.local/share"; } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index e8fed353c..b9f9ea882 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -35,7 +35,7 @@ extern const std::string nativeSystem; /* Return an environment variable. */ -string getEnv(const string & key, const string & def = ""); +std::optional getEnv(const std::string & key); /* Get the entire environment. */ std::map getEnv(); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 1c5d71681..c506f9b0a 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -344,9 +344,9 @@ static void _main(int argc, char * * argv) /* Figure out what bash shell to use. If $NIX_BUILD_SHELL is not set, then build bashInteractive from . */ - auto shell = getEnv("NIX_BUILD_SHELL", ""); + auto shell = getEnv("NIX_BUILD_SHELL"); - if (shell == "") { + if (!shell) { try { auto expr = state->parseExprFromString("(import {}).bashInteractive", absPath(".")); @@ -382,7 +382,8 @@ static void _main(int argc, char * * argv) // Set the environment. auto env = getEnv(); - auto tmp = getEnv("TMPDIR", getEnv("XDG_RUNTIME_DIR", "/tmp")); + auto tmp = getEnv("TMPDIR"); + if (!tmp) tmp = getEnv("XDG_RUNTIME_DIR").value_or("/tmp"); if (pure) { decltype(env) newEnv; @@ -394,7 +395,7 @@ static void _main(int argc, char * * argv) env["__ETC_PROFILE_SOURCED"] = "1"; } - env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmp; + env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = *tmp; env["NIX_STORE"] = store->storeDir; env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores); @@ -439,8 +440,8 @@ static void _main(int argc, char * * argv) (Path) tmpDir, (pure ? "" : "p=$PATH; "), (pure ? "" : "PATH=$PATH:$p; unset p; "), - dirOf(shell), - shell, + dirOf(*shell), + *shell, (getenv("TZ") ? (string("export TZ='") + getenv("TZ") + "'; ") : ""), envCommand)); @@ -460,9 +461,9 @@ static void _main(int argc, char * * argv) restoreSignals(); - execvp(shell.c_str(), argPtrs.data()); + execvp(shell->c_str(), argPtrs.data()); - throw SysError("executing shell '%s'", shell); + throw SysError("executing shell '%s'", *shell); } else { diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 65ae88faf..9a4d60726 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -161,8 +161,9 @@ static void daemonLoop(char * * argv) AutoCloseFD fdSocket; /* Handle socket-based activation by systemd. */ - if (getEnv("LISTEN_FDS") != "") { - if (getEnv("LISTEN_PID") != std::to_string(getpid()) || getEnv("LISTEN_FDS") != "1") + auto listenFds = getEnv("LISTEN_FDS"); + if (listenFds) { + if (getEnv("LISTEN_PID") != std::to_string(getpid()) || listenFds != "1") throw Error("unexpected systemd environment variables"); fdSocket = SD_LISTEN_FDS_START; closeOnExec(fdSocket.get()); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 199dc92aa..a9c743955 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1425,7 +1425,7 @@ static int _main(int argc, char * * argv) globals.instSource.autoArgs = myArgs.getAutoArgs(*globals.state); if (globals.profile == "") - globals.profile = getEnv("NIX_PROFILE", ""); + globals.profile = getEnv("NIX_PROFILE").value_or(""); if (globals.profile == "") { Path profileLink = getHome() + "/.nix-profile"; diff --git a/src/nix/command.cc b/src/nix/command.cc index 97c2fcf1c..724f03e5d 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -156,7 +156,7 @@ void StorePathCommand::run(ref store) Strings editorFor(const Pos & pos) { - auto editor = getEnv("EDITOR", "cat"); + auto editor = getEnv("EDITOR").value_or("cat"); auto args = tokenizeString(editor); if (pos.line > 0 && ( editor.find("emacs") != std::string::npos || diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 795236c5f..481c93b45 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -68,7 +68,7 @@ struct CmdDoctor : StoreCommand { PathSet dirs; - for (auto & dir : tokenizeString(getEnv("PATH"), ":")) + for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) if (pathExists(dir + "/nix-env")) dirs.insert(dirOf(canonPath(dir + "/nix-env", true))); @@ -87,7 +87,7 @@ struct CmdDoctor : StoreCommand { PathSet dirs; - for (auto & dir : tokenizeString(getEnv("PATH"), ":")) { + for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) { Path profileDir = dirOf(dir); try { Path userEnv = canonPath(profileDir, true); diff --git a/src/nix/run.cc b/src/nix/run.cc index 90b76d666..fd4f92282 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -128,7 +128,7 @@ struct CmdRun : InstallablesCommand std::queue todo; for (auto & path : outPaths) todo.push(path); - auto unixPath = tokenizeString(getEnv("PATH"), ":"); + auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); while (!todo.empty()) { Path path = todo.front(); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 35c44a70c..6c9e37d04 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -106,7 +106,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand { Path where; - for (auto & dir : tokenizeString(getEnv("PATH"), ":")) + for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) if (pathExists(dir + "/nix-env")) { where = dir; break; From 1c3ccba0f5e658fa5d593ddef8aaaf6728f945a5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 22 Nov 2019 16:27:48 +0100 Subject: [PATCH 21/60] Remove $NIX_USER_PROFILE_DIR This is not used anywhere. --- scripts/nix-profile-daemon.sh.in | 1 - scripts/nix-profile.sh.in | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 47655080a..e0c24954a 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -2,7 +2,6 @@ if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi __ETC_PROFILE_NIX_SOURCED=1 -export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER" export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index e15f7cd46..0768c805a 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -5,7 +5,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then NIX_LINK=$HOME/.nix-profile - NIX_USER_PROFILE_DIR=@localstatedir@/nix/profiles/per-user/$USER # Append ~/.nix-defexpr/channels to $NIX_PATH so that # paths work when the user has fetched the Nixpkgs channel. @@ -35,5 +34,5 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then fi export PATH="$NIX_LINK/bin:$PATH" - unset NIX_LINK NIX_USER_PROFILE_DIR + unset NIX_LINK fi From ec9dd9a5aeb9b7fca170d86b8906ffac7e5ef057 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 22 Nov 2019 22:08:51 +0100 Subject: [PATCH 22/60] Provide a default value for NIX_PATH --- scripts/nix-profile-daemon.sh.in | 1 - scripts/nix-profile.sh.in | 5 ----- src/libexpr/eval.cc | 14 +++++++++++++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index e0c24954a..4bc7c6fc5 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -24,5 +24,4 @@ else done fi -export NIX_PATH="nixpkgs=@localstatedir@/nix/profiles/per-user/root/channels/nixpkgs:@localstatedir@/nix/profiles/per-user/root/channels" export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH" diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 0768c805a..8cba1c522 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -5,11 +5,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then NIX_LINK=$HOME/.nix-profile - - # Append ~/.nix-defexpr/channels to $NIX_PATH so that - # paths work when the user has fetched the Nixpkgs channel. - export NIX_PATH=${NIX_PATH:+$NIX_PATH:}$HOME/.nix-defexpr/channels - # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index bb6b23ff1..65466a0b4 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -275,6 +275,17 @@ static Strings parseNixPath(const string & s) } +static Strings getDefaultNixPath() +{ + Strings res; + auto add = [&](const Path & p) { if (pathExists(p)) { res.push_back(p); } }; + add(getHome() + "/.nix-defexpr/channels"); + add("nixpkgs=" + settings.nixStateDir + "/nix/profiles/per-user/root/channels/nixpkgs"); + add(settings.nixStateDir + "/nix/profiles/per-user/root/channels"); + return res; +} + + EvalState::EvalState(const Strings & _searchPath, ref store) : sWith(symbols.create("")) , sOutPath(symbols.create("outPath")) @@ -314,7 +325,8 @@ EvalState::EvalState(const Strings & _searchPath, ref store) /* Initialise the Nix expression search path. */ if (!evalSettings.pureEval) { - Strings paths = parseNixPath(getEnv("NIX_PATH").value_or("")); + auto nixPath = getEnv("NIX_PATH"); + auto paths = nixPath ? parseNixPath(*nixPath) : getDefaultNixPath(); for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : paths) addToSearchPath(i); } From d12d69ea1a871d631d77c8ef5e8468b4a2bff80f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 22 Nov 2019 23:07:35 +0100 Subject: [PATCH 23/60] Turn NIX_PATH into a config setting This allows it to be set in nix.conf. --- src/libexpr/eval.cc | 31 +++++++++++++++++-------------- src/libexpr/eval.hh | 7 +++++++ 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 65466a0b4..62812f90a 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -275,17 +275,6 @@ static Strings parseNixPath(const string & s) } -static Strings getDefaultNixPath() -{ - Strings res; - auto add = [&](const Path & p) { if (pathExists(p)) { res.push_back(p); } }; - add(getHome() + "/.nix-defexpr/channels"); - add("nixpkgs=" + settings.nixStateDir + "/nix/profiles/per-user/root/channels/nixpkgs"); - add(settings.nixStateDir + "/nix/profiles/per-user/root/channels"); - return res; -} - - EvalState::EvalState(const Strings & _searchPath, ref store) : sWith(symbols.create("")) , sOutPath(symbols.create("outPath")) @@ -325,10 +314,8 @@ EvalState::EvalState(const Strings & _searchPath, ref store) /* Initialise the Nix expression search path. */ if (!evalSettings.pureEval) { - auto nixPath = getEnv("NIX_PATH"); - auto paths = nixPath ? parseNixPath(*nixPath) : getDefaultNixPath(); for (auto & i : _searchPath) addToSearchPath(i); - for (auto & i : paths) addToSearchPath(i); + for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); } addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); @@ -1986,6 +1973,22 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { } +EvalSettings::EvalSettings() +{ + auto var = getEnv("NIX_PATH"); + if (var) nixPath = parseNixPath(*var); +} + +Strings EvalSettings::getDefaultNixPath() +{ + Strings res; + auto add = [&](const Path & p) { if (pathExists(p)) { res.push_back(p); } }; + add(getHome() + "/.nix-defexpr/channels"); + add("nixpkgs=" + settings.nixStateDir + "/nix/profiles/per-user/root/channels/nixpkgs"); + add(settings.nixStateDir + "/nix/profiles/per-user/root/channels"); + return res; +} + EvalSettings evalSettings; static GlobalConfig::Register r1(&evalSettings); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 61ee4a73b..9d075a48a 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -339,9 +339,16 @@ struct InvalidPathError : EvalError struct EvalSettings : Config { + EvalSettings(); + + static Strings getDefaultNixPath(); + Setting enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", "Whether builtin functions that allow executing native code should be enabled."}; + Setting nixPath{this, getDefaultNixPath(), "nix-path", + "List of directories to be searched for <...> file references."}; + Setting restrictEval{this, false, "restrict-eval", "Whether to restrict file system access to paths in $NIX_PATH, " "and network access to the URI prefixes listed in 'allowed-uris'."}; From cd55f91ad2075d5ebb506950dfbc4161cab22db5 Mon Sep 17 00:00:00 2001 From: Puck Meerburg Date: Mon, 25 Nov 2019 12:37:14 +0000 Subject: [PATCH 24/60] Ensure enough space in attrset bindings when using both __overrides and dynamic attributes --- src/libexpr/eval.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 62812f90a..3a9b7a3c0 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -875,7 +875,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) if (hasOverrides) { Value * vOverrides = (*v.attrs)[overrides->second.displ].value; state.forceAttrs(*vOverrides); - Bindings * newBnds = state.allocBindings(v.attrs->size() + vOverrides->attrs->size()); + Bindings * newBnds = state.allocBindings(v.attrs->capacity() + vOverrides->attrs->size()); for (auto & i : *v.attrs) newBnds->push_back(i); for (auto & i : *vOverrides->attrs) { From cdadbf770855e828a0f64d70a47db43e920ad137 Mon Sep 17 00:00:00 2001 From: Puck Meerburg Date: Mon, 25 Nov 2019 13:03:54 +0000 Subject: [PATCH 25/60] Add testcase for attrset using __overrides and dynamic attrs --- tests/lang/eval-okay-attrs6.exp | 1 + tests/lang/eval-okay-attrs6.nix | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 tests/lang/eval-okay-attrs6.exp create mode 100644 tests/lang/eval-okay-attrs6.nix diff --git a/tests/lang/eval-okay-attrs6.exp b/tests/lang/eval-okay-attrs6.exp new file mode 100644 index 000000000..b46938032 --- /dev/null +++ b/tests/lang/eval-okay-attrs6.exp @@ -0,0 +1 @@ +{ __overrides = { bar = "qux"; }; bar = "qux"; foo = "bar"; } diff --git a/tests/lang/eval-okay-attrs6.nix b/tests/lang/eval-okay-attrs6.nix new file mode 100644 index 000000000..2e5c85483 --- /dev/null +++ b/tests/lang/eval-okay-attrs6.nix @@ -0,0 +1,4 @@ +rec { + "${"foo"}" = "bar"; + __overrides = { bar = "qux"; }; +} From 4e70652ee33641a6cfddbfb9b7d741515f1bf98a Mon Sep 17 00:00:00 2001 From: Arnout Engelen Date: Mon, 25 Nov 2019 18:00:05 +0100 Subject: [PATCH 26/60] Document builtins.placeholder --- doc/manual/expressions/builtins.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index f39b393bf..394e1fc32 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -1120,6 +1120,16 @@ Evaluates to [ "foo" ]. + + builtins.placeholder + output + + Return a placeholder string for the specified + output that will be substituted by the + corresponding output path at build time. Typical outputs would be + "out", "bin" or + "dev". + builtins.readDir From 1ec6e6e11e2dbca8803305757d472dabe5878129 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 26 Nov 2019 19:48:34 +0100 Subject: [PATCH 27/60] Add feature to disable URL literals E.g. $ nix-build '' -A hello --experimental-features no-url-literals error: URL literals are disabled, at /nix/store/vsjamkzh15r3c779q2711az826hqgvzr-nixpkgs-20.03pre194957.bef773ed53f/nixpkgs/pkgs/top-level/all-packages.nix:1236:11 Helps with implementing https://github.com/NixOS/rfcs/pull/45. --- src/libexpr/parser.y | 8 +++++++- src/libstore/globals.cc | 9 +++++++-- src/libstore/globals.hh | 2 ++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 93834bec6..10a057062 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -20,6 +20,7 @@ #include "nixexpr.hh" #include "eval.hh" +#include "globals.hh" namespace nix { @@ -401,7 +402,12 @@ expr_simple new ExprVar(data->symbols.create("__nixPath"))), new ExprString(data->symbols.create(path))); } - | URI { $$ = new ExprString(data->symbols.create($1)); } + | URI { + static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals"); + if (noURLLiterals) + throw ParseError("URL literals are disabled, at %s", CUR_POS); + $$ = new ExprString(data->symbols.create($1)); + } | '(' expr ')' { $$ = $2; } /* Let expressions `let {..., body = ...}' are just desugared into `(rec {..., body = ...}).body'. */ diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index aa0823f41..6d3b95740 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -105,10 +105,15 @@ StringSet Settings::getDefaultSystemFeatures() return features; } -void Settings::requireExperimentalFeature(const std::string & name) +bool Settings::isExperimentalFeatureEnabled(const std::string & name) { auto & f = experimentalFeatures.get(); - if (std::find(f.begin(), f.end(), name) == f.end()) + return std::find(f.begin(), f.end(), name) != f.end(); +} + +void Settings::requireExperimentalFeature(const std::string & name) +{ + if (!isExperimentalFeatureEnabled(name)) throw Error("experimental Nix feature '%s' is disabled", name); } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 3eb2f698c..9f395425a 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -357,6 +357,8 @@ public: Setting experimentalFeatures{this, {}, "experimental-features", "Experimental Nix features to enable."}; + bool isExperimentalFeatureEnabled(const std::string & name); + void requireExperimentalFeature(const std::string & name); }; From 89db9353d7cc5aaf360c48d17aeb0e9ddd940451 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 26 Nov 2019 20:08:25 +0100 Subject: [PATCH 28/60] Doh --- src/libstore/local-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b5b0f1ba7..e9dd3f96e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -54,6 +54,7 @@ LocalStore::LocalStore(const Params & params) , trashDir(realStoreDir + "/trash") , tempRootsDir(stateDir + "/temproots") , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) + , locksHeld(tokenizeString(getEnv("NIX_HELD_LOCKS").value_or(""))) { auto state(_state.lock()); From c13193017fdcb910627698e343c2df4183d91f8f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 26 Nov 2019 20:12:15 +0100 Subject: [PATCH 29/60] Disallow empty store path names Fixes #3239. --- src/libstore/local-store.cc | 2 ++ src/libstore/store-api.cc | 9 +++++++-- tests/referrers.sh | 4 ++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e9dd3f96e..7d127840a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -585,6 +585,8 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs) { + checkStoreName(storePathToName(info.path)); + if (info.ca != "" && !info.isContentAddressed(*this)) throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", info.path); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 54430d3ba..0758907e7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -90,17 +90,22 @@ void checkStoreName(const string & name) "Path names are alphanumeric and can include the symbols %1% " "and must not begin with a period. " "Note: If '%2%' is a source file and you cannot rename it on " - "disk, builtins.path { name = ... } can be used to give it an " + "disk, 'builtins.path { name = ... }' can be used to give it an " "alternative name.") % validChars % name; + if (name.empty()) + throw Error(baseError % "it is an empty string"); + /* Disallow names starting with a dot for possible security reasons (e.g., "." and ".."). */ - if (string(name, 0, 1) == ".") + if (name[0] == '.') throw Error(baseError % "it is illegal to start the name with a period"); + /* Disallow names longer than 211 characters. ext4’s max is 256, but we need extra space for the hash and .chroot extensions. */ if (name.length() > 211) throw Error(baseError % "name must be less than 212 characters"); + for (auto & i : name) if (!((i >= 'A' && i <= 'Z') || (i >= 'a' && i <= 'z') || diff --git a/tests/referrers.sh b/tests/referrers.sh index 8ab8e5ddf..614dd8f5b 100644 --- a/tests/referrers.sh +++ b/tests/referrers.sh @@ -4,9 +4,9 @@ clearStore max=500 -reference=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +reference=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bla touch $reference -(echo $reference && echo && echo 0) | nix-store --register-validity +(echo $reference && echo && echo 0) | nix-store --register-validity echo "making registration..." From ec5e7b44ff1ce14eb930948df1b89f07abbf16f9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 26 Nov 2019 20:26:22 +0100 Subject: [PATCH 30/60] Simplify --- src/libstore/local-store.cc | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0d44665b0..60e7bb7af 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1244,30 +1244,24 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printInfo("checking link hashes..."); - AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); - - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { /* sic */ - checkInterrupt(); - if (strcmp(dirent->d_name, ".") == 0 || strcmp(dirent->d_name, "..") == 0) continue; - Path linkPath = linksDir + "/" + dirent->d_name; + for (auto & link : readDirectory(linksDir)) { + printMsg(lvlTalkative, "checking contents of '%s'", link.name); + Path linkPath = linksDir + "/" + link.name; string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); - if (hash != dirent->d_name) { - printError(format("link '%1%' was modified! " - "expected hash '%2%', got '%3%'") - % linkPath % dirent->d_name % hash); + if (hash != link.name) { + printError( + "link '%s' was modified! expected hash '%s', got '%s'", + linkPath, link.name, hash); if (repair) { if (unlink(linkPath.c_str()) == 0) - printError(format("Removed link '%1%'") % linkPath); + printError("removed link '%s'", linkPath); else - throw SysError(format("removing corrupt link '%1%'") % linkPath); + throw SysError("removing corrupt link '%s'", linkPath); } else { errors = true; } } } - if (errno) throw SysError(format("reading directory '%1%'") % linksDir); printInfo("checking store hashes..."); From 0be8d7784ffad0131c2e69fe25350fe5d2132e34 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 26 Nov 2019 20:33:46 +0100 Subject: [PATCH 31/60] Typo --- doc/manual/command-ref/nix-build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/command-ref/nix-build.xml b/doc/manual/command-ref/nix-build.xml index c1b783c87..886d25910 100644 --- a/doc/manual/command-ref/nix-build.xml +++ b/doc/manual/command-ref/nix-build.xml @@ -100,7 +100,7 @@ also . - Show what store paths would be built or downloaded + Show what store paths would be built or downloaded. / From 313106d5491c7c6279f3c962348462ca04cd2e0e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 26 Nov 2019 21:07:44 +0100 Subject: [PATCH 32/60] Fix clang warnings --- src/libstore/nar-info-disk-cache.hh | 2 ++ src/libstore/remote-store.hh | 2 +- src/libstore/ssh-store.cc | 2 +- src/libstore/store-api.cc | 2 +- src/nix/command.hh | 3 +++ 5 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 11e6c55ca..285873f7e 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -10,6 +10,8 @@ class NarInfoDiskCache public: typedef enum { oValid, oInvalid, oUnknown } Outcome; + virtual ~NarInfoDiskCache() { }; + virtual void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) = 0; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 84de4c460..728df8b00 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -148,7 +148,7 @@ public: std::string getUri() override; - bool sameMachine() + bool sameMachine() override { return true; } private: diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 93e117389..b9d0c41ea 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -35,7 +35,7 @@ public: return uriScheme + host; } - bool sameMachine() + bool sameMachine() override { return false; } void narFromPath(const Path & path, Sink & sink) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 0758907e7..7e548ecd2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -216,7 +216,7 @@ static std::string makeType(string && type, const PathSet & references) type += ":"; type += i; } - return type; + return std::move(type); } diff --git a/src/nix/command.hh b/src/nix/command.hh index 26c15e646..2db22dba2 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -16,6 +16,7 @@ struct Pos; run() method. */ struct Command : virtual Args { + virtual ~Command() { } virtual std::string name() = 0; virtual void prepare() { }; virtual void run() = 0; @@ -58,6 +59,8 @@ typedef std::vector Buildables; struct Installable { + virtual ~Installable() { } + virtual std::string what() = 0; virtual Buildables toBuildables() From abb8ef619ba2fab3ae16fb5b5430215905bac723 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 26 Nov 2019 21:08:56 +0100 Subject: [PATCH 33/60] Fix macOS build https://hydra.nixos.org/build/107457009 --- src/libmain/shared.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index d3dbfbc44..81fb6b05a 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -155,7 +155,7 @@ void initNix() sshd). This breaks build users because they don't have access to the TMPDIR, in particular in ‘nix-store --serve’. */ #if __APPLE__ - if (getuid() == 0 && hasPrefix(getEnv("TMPDIR"), "/var/folders/")) + if (getuid() == 0 && hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) unsetenv("TMPDIR"); #endif } From 11da5b2816e11b081d8ff38db4330addd2014f7e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Mar 2019 14:12:20 +0100 Subject: [PATCH 34/60] Add some Rust code --- .gitignore | 2 ++ Makefile | 1 + nix-rust/Cargo.lock | 59 +++++++++++++++++++++++++++++++++++++++++++ nix-rust/Cargo.toml | 12 +++++++++ nix-rust/local.mk | 7 ++++++ nix-rust/src/lib.rs | 48 +++++++++++++++++++++++++++++++++++ release-common.nix | 1 + shell.nix | 2 +- src/nix/local.mk | 4 +-- src/nix/test.cc | 61 +++++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 nix-rust/Cargo.lock create mode 100644 nix-rust/Cargo.toml create mode 100644 nix-rust/local.mk create mode 100644 nix-rust/src/lib.rs create mode 100644 src/nix/test.cc diff --git a/.gitignore b/.gitignore index fd62dfb38..f58e1f90f 100644 --- a/.gitignore +++ b/.gitignore @@ -119,3 +119,5 @@ GPATH GRTAGS GSYMS GTAGS + +nix-rust/target diff --git a/Makefile b/Makefile index 866c0961e..469070533 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ makefiles = \ mk/precompiled-headers.mk \ local.mk \ + nix-rust/local.mk \ src/libutil/local.mk \ src/libstore/local.mk \ src/libmain/local.mk \ diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock new file mode 100644 index 000000000..cfaedc3ed --- /dev/null +++ b/nix-rust/Cargo.lock @@ -0,0 +1,59 @@ +[[package]] +name = "cfg-if" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "filetime" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "libc" +version = "0.2.50" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "nix-rust" +version = "0.1.0" +dependencies = [ + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "tar 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "redox_syscall" +version = "0.1.51" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "tar" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", + "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "xattr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "11d43355396e872eefb45ce6342e4374ed7bc2b3a502d1b28e36d6e23c05d1f4" +"checksum filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a2df5c1a8c4be27e7707789dc42ae65976e60b394afd293d1419ab915833e646" +"checksum libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)" = "aab692d7759f5cd8c859e169db98ae5b52c924add2af5fbbca11d12fefb567c1" +"checksum redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)" = "423e376fffca3dfa06c9e9790a9ccd282fafb3cc6e6397d01dbf64f9bacc6b85" +"checksum tar 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)" = "c2167ff53da2a661702b3299f71a91b61b1dffef36b4b2884b1f9c67254c0133" +"checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml new file mode 100644 index 000000000..2d2bf6752 --- /dev/null +++ b/nix-rust/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "nix-rust" +version = "0.1.0" +authors = ["Eelco Dolstra "] + +[lib] +name = "nixrust" +crate-type = ["staticlib"] + +[dependencies] +tar = "0.4" +libc = "0.2" diff --git a/nix-rust/local.mk b/nix-rust/local.mk new file mode 100644 index 000000000..c69e3b9ea --- /dev/null +++ b/nix-rust/local.mk @@ -0,0 +1,7 @@ +libnixrust_PATH := $(d)/target/release/libnixrust.a +libnixrust_INSTALL_PATH := $(libnixrust_PATH) +libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl +libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE) + +$(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml + $(trace-gen) cd nix-rust && RUSTC_BOOTSTRAP=1 cargo build --release && touch target/release/libnixrust.a diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs new file mode 100644 index 000000000..abcb89055 --- /dev/null +++ b/nix-rust/src/lib.rs @@ -0,0 +1,48 @@ +extern crate libc; +extern crate tar; + +use std::fs; +use std::io; +use std::os::unix::fs::OpenOptionsExt; +use std::path::Path; +use tar::Archive; + +#[no_mangle] +pub extern "C" fn unpack_tarfile(data: &[u8], dest_dir: &str) -> bool { + // FIXME: handle errors. + + let dest_dir = Path::new(dest_dir); + + let mut tar = Archive::new(data); + + for file in tar.entries().unwrap() { + let mut file = file.unwrap(); + + let dest_file = dest_dir.join(file.header().path().unwrap()); + + fs::create_dir_all(dest_file.parent().unwrap()).unwrap(); + + match file.header().entry_type() { + tar::EntryType::Directory => { + fs::create_dir(dest_file).unwrap(); + } + tar::EntryType::Regular => { + let mode = if file.header().mode().unwrap() & libc::S_IXUSR == 0 { + 0o666 + } else { + 0o777 + }; + let mut f = fs::OpenOptions::new() + .create(true) + .write(true) + .mode(mode) + .open(dest_file) + .unwrap(); + io::copy(&mut file, &mut f).unwrap(); + } + t => panic!("Unsupported tar entry type '{:?}'.", t), + } + } + + true +} diff --git a/release-common.nix b/release-common.nix index 63f39f005..dd5f939d9 100644 --- a/release-common.nix +++ b/release-common.nix @@ -51,6 +51,7 @@ rec { openssl pkgconfig sqlite boehmgc boost nlohmann_json + rustc cargo # Tests git diff --git a/shell.nix b/shell.nix index 9c504f024..21ed47121 100644 --- a/shell.nix +++ b/shell.nix @@ -7,7 +7,7 @@ with import ./release-common.nix { inherit pkgs; }; (if useClang then clangStdenv else stdenv).mkDerivation { name = "nix"; - buildInputs = buildDeps ++ tarballDeps ++ perlDeps; + buildInputs = buildDeps ++ tarballDeps ++ perlDeps ++ [ pkgs.rustfmt ]; inherit configureFlags; diff --git a/src/nix/local.mk b/src/nix/local.mk index c09efd1fc..a145cf9b1 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -15,9 +15,9 @@ nix_SOURCES := \ $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ -nix_LIBS = libexpr libmain libstore libutil +nix_LIBS = libexpr libmain libstore libutil libnixrust -nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system -Lnix-rust/target/release $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ diff --git a/src/nix/test.cc b/src/nix/test.cc new file mode 100644 index 000000000..70e0a903d --- /dev/null +++ b/src/nix/test.cc @@ -0,0 +1,61 @@ +#include "command.hh" +#include "store-api.hh" +#include "common-args.hh" + +using namespace nix; + +namespace rust { + +// Depending on the internal representation of Rust slices is slightly +// evil... +template struct Slice +{ + const T * ptr; + size_t size; + + Slice(const T * ptr, size_t size) : ptr(ptr), size(size) + { + assert(ptr); + } +}; + +struct StringSlice : Slice +{ + StringSlice(const std::string & s): Slice(s.data(), s.size()) { } +}; + +} + +extern "C" { + bool unpack_tarfile(rust::Slice data, rust::StringSlice dest_dir); +} + +struct CmdTest : StoreCommand +{ + CmdTest() + { + } + + std::string name() override + { + return "test"; + } + + std::string description() override + { + return "bla bla"; + } + + void run(ref store) override + { + auto data = readFile("./nix-2.2.tar"); + + std::string destDir = "./dest"; + + deletePath(destDir); + + unpack_tarfile({(uint8_t*) data.data(), data.size()}, destDir); + } +}; + +static RegisterCommand r(make_ref()); From e60f6bd4ce831ced94fafeb527c429b6f88159ac Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Mar 2019 20:45:56 +0100 Subject: [PATCH 35/60] Enable Rust code to call C++ Source objects --- nix-rust/src/lib.rs | 19 +++++++++++++++++-- src/nix/test.cc | 34 ++++++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index abcb89055..799d52e31 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -7,13 +7,28 @@ use std::os::unix::fs::OpenOptionsExt; use std::path::Path; use tar::Archive; +/// A wrapper around Nix's Source class that provides the Read trait. +#[repr(C)] +pub struct Source { + fun: extern "C" fn(this: *mut libc::c_void, data: &mut [u8]) -> usize, + this: *mut libc::c_void, +} + +impl std::io::Read for Source { + fn read(&mut self, buf: &mut [u8]) -> std::result::Result { + let n = (self.fun)(self.this, buf); + assert!(n <= buf.len()); + Ok(n) + } +} + #[no_mangle] -pub extern "C" fn unpack_tarfile(data: &[u8], dest_dir: &str) -> bool { +pub extern "C" fn unpack_tarfile(source: Source, dest_dir: &str) -> bool { // FIXME: handle errors. let dest_dir = Path::new(dest_dir); - let mut tar = Archive::new(data); + let mut tar = Archive::new(source); for file in tar.entries().unwrap() { let mut file = file.unwrap(); diff --git a/src/nix/test.cc b/src/nix/test.cc index 70e0a903d..7f1f0d47a 100644 --- a/src/nix/test.cc +++ b/src/nix/test.cc @@ -1,6 +1,7 @@ #include "command.hh" #include "store-api.hh" #include "common-args.hh" +#include "compression.hh" using namespace nix; @@ -10,10 +11,10 @@ namespace rust { // evil... template struct Slice { - const T * ptr; + T * ptr; size_t size; - Slice(const T * ptr, size_t size) : ptr(ptr), size(size) + Slice(T * ptr, size_t size) : ptr(ptr), size(size) { assert(ptr); } @@ -21,13 +22,30 @@ template struct Slice struct StringSlice : Slice { - StringSlice(const std::string & s): Slice(s.data(), s.size()) { } + StringSlice(const std::string & s): Slice((char *) s.data(), s.size()) {} +}; + +struct Source +{ + size_t (*fun)(void * source_this, rust::Slice data); + nix::Source * _this; + + Source(nix::Source & _this) + : fun(sourceWrapper), _this(&_this) + {} + + // FIXME: how to propagate exceptions? + static size_t sourceWrapper(void * _this, rust::Slice data) + { + auto n = ((nix::Source *) _this)->read(data.ptr, data.size); + return n; + } }; } extern "C" { - bool unpack_tarfile(rust::Slice data, rust::StringSlice dest_dir); + bool unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); } struct CmdTest : StoreCommand @@ -48,13 +66,17 @@ struct CmdTest : StoreCommand void run(ref store) override { - auto data = readFile("./nix-2.2.tar"); + auto source = sinkToSource([&](Sink & sink) { + auto decompressor = makeDecompressionSink("bzip2", sink); + readFile("./nix-2.2.tar.bz2", *decompressor); + decompressor->finish(); + }); std::string destDir = "./dest"; deletePath(destDir); - unpack_tarfile({(uint8_t*) data.data(), data.size()}, destDir); + unpack_tarfile(*source, destDir); } }; From 045708db4343174f30f3647776971c852f72a9e8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Mar 2019 23:40:35 +0100 Subject: [PATCH 36/60] Make a builtin builder This was the last function using a shell script, so this allows us to get rid of tar, coreutils, bash etc. --- corepkgs/config.nix.in | 10 +----- corepkgs/unpack-channel.nix | 35 +++------------------ nix-rust/src/lib.rs | 5 ++- src/libstore/build.cc | 2 ++ src/libstore/builtins.hh | 1 + src/libstore/builtins/unpack-channel.cc | 39 +++++++++++++++++++++++ src/libstore/local.mk | 2 +- src/{nix/test.cc => libstore/rust.hh} | 41 +------------------------ src/nix/local.mk | 4 +-- 9 files changed, 55 insertions(+), 84 deletions(-) create mode 100644 src/libstore/builtins/unpack-channel.cc rename src/{nix/test.cc => libstore/rust.hh} (53%) diff --git a/corepkgs/config.nix.in b/corepkgs/config.nix.in index 32ce6b399..4ea182d8f 100644 --- a/corepkgs/config.nix.in +++ b/corepkgs/config.nix.in @@ -1,3 +1,4 @@ +# FIXME: remove this file? let fromEnv = var: def: let val = builtins.getEnv var; in @@ -17,13 +18,4 @@ in rec { nixLocalstateDir = "@localstatedir@"; nixSysconfDir = "@sysconfdir@"; nixStoreDir = fromEnv "NIX_STORE_DIR" "@storedir@"; - - # If Nix is installed in the Nix store, then automatically add it as - # a dependency to the core packages. This ensures that they work - # properly in a chroot. - chrootDeps = - if dirOf nixPrefix == builtins.storeDir then - [ (builtins.storePath nixPrefix) ] - else - [ ]; } diff --git a/corepkgs/unpack-channel.nix b/corepkgs/unpack-channel.nix index d39a20637..10515bc8b 100644 --- a/corepkgs/unpack-channel.nix +++ b/corepkgs/unpack-channel.nix @@ -1,39 +1,12 @@ -with import ; - -let - - builder = builtins.toFile "unpack-channel.sh" - '' - mkdir $out - cd $out - xzpat="\.xz\$" - gzpat="\.gz\$" - if [[ "$src" =~ $xzpat ]]; then - ${xz} -d < $src | ${tar} xf - ${tarFlags} - elif [[ "$src" =~ $gzpat ]]; then - ${gzip} -d < $src | ${tar} xf - ${tarFlags} - else - ${bzip2} -d < $src | ${tar} xf - ${tarFlags} - fi - if [ * != $channelName ]; then - mv * $out/$channelName - fi - ''; - -in - { name, channelName, src }: derivation { - system = builtins.currentSystem; - builder = shell; - args = [ "-e" builder ]; - inherit name channelName src; + builder = "builtin:unpack-channel"; - PATH = "${nixBinDir}:${coreutils}"; + system = "builtin"; + + inherit name channelName src; # No point in doing this remotely. preferLocalBuild = true; - - inherit chrootDeps; } diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index 799d52e31..ac6dee543 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -33,7 +33,7 @@ pub extern "C" fn unpack_tarfile(source: Source, dest_dir: &str) -> bool { for file in tar.entries().unwrap() { let mut file = file.unwrap(); - let dest_file = dest_dir.join(file.header().path().unwrap()); + let dest_file = dest_dir.join(file.path().unwrap()); fs::create_dir_all(dest_file.parent().unwrap()).unwrap(); @@ -55,6 +55,9 @@ pub extern "C" fn unpack_tarfile(source: Source, dest_dir: &str) -> bool { .unwrap(); io::copy(&mut file, &mut f).unwrap(); } + tar::EntryType::Symlink => { + std::os::unix::fs::symlink(file.header().link_name().unwrap().unwrap(), dest_file).unwrap(); + } t => panic!("Unsupported tar entry type '{:?}'.", t), } } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 51a9fa35b..efbb7fc9f 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3128,6 +3128,8 @@ void DerivationGoal::runChild() builtinFetchurl(drv2, netrcData); else if (drv->builder == "builtin:buildenv") builtinBuildenv(drv2); + else if (drv->builder == "builtin:unpack-channel") + builtinUnpackChannel(drv2); else throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8)); _exit(0); diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index 0d2da873e..87d6ce665 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -7,5 +7,6 @@ namespace nix { // TODO: make pluggable. void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); void builtinBuildenv(const BasicDerivation & drv); +void builtinUnpackChannel(const BasicDerivation & drv); } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc new file mode 100644 index 000000000..88202ec6b --- /dev/null +++ b/src/libstore/builtins/unpack-channel.cc @@ -0,0 +1,39 @@ +#include "rust.hh" +#include "builtins.hh" +#include "compression.hh" + +namespace nix { + +void builtinUnpackChannel(const BasicDerivation & drv) +{ + auto getAttr = [&](const string & name) { + auto i = drv.env.find(name); + if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + return i->second; + }; + + Path out = getAttr("out"); + auto channelName = getAttr("channelName"); + auto src = getAttr("src"); + + createDirs(out); + + auto source = sinkToSource([&](Sink & sink) { + auto decompressor = + hasSuffix(src, ".bz2") ? makeDecompressionSink("bzip2", sink) : + hasSuffix(src, ".xz") ? makeDecompressionSink("xz", sink) : + makeDecompressionSink("none", sink); + readFile(src, *decompressor); + decompressor->finish(); + }); + + unpack_tarfile(*source, out); + + auto entries = readDirectory(out); + if (entries.size() != 1) + throw Error("channel tarball '%s' contains more than one file", src); + if (rename((out + "/" + entries[0].name).c_str(), (out + "/" + channelName).c_str()) == -1) + throw SysError("renaming channel directory"); +} + +} diff --git a/src/libstore/local.mk b/src/libstore/local.mk index d690fea28..d3254554d 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -6,7 +6,7 @@ libstore_DIR := $(d) libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) -libstore_LIBS = libutil +libstore_LIBS = libutil libnixrust libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread ifneq ($(OS), FreeBSD) diff --git a/src/nix/test.cc b/src/libstore/rust.hh similarity index 53% rename from src/nix/test.cc rename to src/libstore/rust.hh index 7f1f0d47a..7e6c2f54d 100644 --- a/src/nix/test.cc +++ b/src/libstore/rust.hh @@ -1,9 +1,4 @@ -#include "command.hh" -#include "store-api.hh" -#include "common-args.hh" -#include "compression.hh" - -using namespace nix; +#include "serialise.hh" namespace rust { @@ -47,37 +42,3 @@ struct Source extern "C" { bool unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); } - -struct CmdTest : StoreCommand -{ - CmdTest() - { - } - - std::string name() override - { - return "test"; - } - - std::string description() override - { - return "bla bla"; - } - - void run(ref store) override - { - auto source = sinkToSource([&](Sink & sink) { - auto decompressor = makeDecompressionSink("bzip2", sink); - readFile("./nix-2.2.tar.bz2", *decompressor); - decompressor->finish(); - }); - - std::string destDir = "./dest"; - - deletePath(destDir); - - unpack_tarfile(*source, destDir); - } -}; - -static RegisterCommand r(make_ref()); diff --git a/src/nix/local.mk b/src/nix/local.mk index a145cf9b1..c09efd1fc 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -15,9 +15,9 @@ nix_SOURCES := \ $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ -nix_LIBS = libexpr libmain libstore libutil libnixrust +nix_LIBS = libexpr libmain libstore libutil -nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system -Lnix-rust/target/release +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ From 6a9c815734de3549a5116796830bd4ebfd006025 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Mar 2019 23:49:14 +0100 Subject: [PATCH 37/60] Remove most of This is no longer needed. --- .gitignore | 1 + corepkgs/config.nix.in | 8 -------- tests/{config.nix => config.nix.in} | 6 ++---- tests/local.mk | 2 +- 4 files changed, 4 insertions(+), 13 deletions(-) rename tests/{config.nix => config.nix.in} (86%) diff --git a/.gitignore b/.gitignore index f58e1f90f..f64031e12 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,7 @@ perl/Makefile.config /tests/restricted-innocent /tests/shell /tests/shell.drv +/tests/config.nix # /tests/lang/ /tests/lang/*.out diff --git a/corepkgs/config.nix.in b/corepkgs/config.nix.in index 4ea182d8f..cb9945944 100644 --- a/corepkgs/config.nix.in +++ b/corepkgs/config.nix.in @@ -4,14 +4,6 @@ let let val = builtins.getEnv var; in if val != "" then val else def; in rec { - shell = "@bash@"; - coreutils = "@coreutils@"; - bzip2 = "@bzip2@"; - gzip = "@gzip@"; - xz = "@xz@"; - tar = "@tar@"; - tarFlags = "@tarFlags@"; - tr = "@tr@"; nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@"; nixPrefix = "@prefix@"; nixLibexecDir = fromEnv "NIX_LIBEXEC_DIR" "@libexecdir@"; diff --git a/tests/config.nix b/tests/config.nix.in similarity index 86% rename from tests/config.nix rename to tests/config.nix.in index 6ba91065b..51aed539c 100644 --- a/tests/config.nix +++ b/tests/config.nix.in @@ -1,9 +1,7 @@ -with import ; - rec { - inherit shell; + shell = "@bash@"; - path = coreutils; + path = "@coreutils@"; system = builtins.currentSystem; diff --git a/tests/local.mk b/tests/local.mk index 187f96ea2..f1d215eec 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -39,4 +39,4 @@ tests-environment = NIX_REMOTE= $(bash) -e clean-files += $(d)/common.sh -installcheck: $(d)/common.sh $(d)/plugins/libplugintest.$(SO_EXT) +installcheck: $(d)/common.sh $(d)/config.nix $(d)/plugins/libplugintest.$(SO_EXT) From 87b7b25e13206bcfe200f45df822787612f159f7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 28 Mar 2019 00:02:10 +0100 Subject: [PATCH 38/60] Clean up the configure script --- configure.ac | 20 +++----------------- perl/lib/Nix/Config.pm.in | 4 ---- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/configure.ac b/configure.ac index 26f16b846..8e38f2b8e 100644 --- a/configure.ac +++ b/configure.ac @@ -117,26 +117,15 @@ fi ]) NEED_PROG(bash, bash) -NEED_PROG(patch, patch) AC_PATH_PROG(xmllint, xmllint, false) AC_PATH_PROG(xsltproc, xsltproc, false) AC_PATH_PROG(flex, flex, false) AC_PATH_PROG(bison, bison, false) -NEED_PROG(sed, sed) -NEED_PROG(tar, tar) -NEED_PROG(bzip2, bzip2) -NEED_PROG(gzip, gzip) -NEED_PROG(xz, xz) AC_PATH_PROG(dot, dot) AC_PATH_PROG(lsof, lsof, lsof) -NEED_PROG(cat, cat) -NEED_PROG(tr, tr) -AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH], - [path of cat, mkdir, etc.]), - coreutils=$withval, coreutils=$(dirname $cat)) -AC_SUBST(coreutils) +AC_SUBST(coreutils, [$(dirname $(type -p cat))]) AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH], @@ -167,7 +156,8 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then LIBS="-latomic $LIBS" fi -# Look for OpenSSL, a required dependency. +# Look for OpenSSL, a required dependency. FIXME: this is only (maybe) +# used by S3BinaryCacheStore. PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) @@ -177,11 +167,9 @@ AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true], AC_CHECK_HEADERS([bzlib.h], [true], [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) - # Look for SQLite, a required dependency. PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"]) - # Look for libcurl, a required dependency. PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"]) @@ -204,13 +192,11 @@ PKG_CHECK_MODULES([SODIUM], [libsodium], have_sodium=1], [have_sodium=]) AC_SUBST(HAVE_SODIUM, [$have_sodium]) - # Look for liblzma, a required dependency. PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"]) AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt], [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])]) - # Look for libbrotli{enc,dec}. PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]) diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in index 67a20c3f4..bc1749e60 100644 --- a/perl/lib/Nix/Config.pm.in +++ b/perl/lib/Nix/Config.pm.in @@ -11,10 +11,6 @@ $logDir = $ENV{"NIX_LOG_DIR"} || "@nixlocalstatedir@/log/nix"; $confDir = $ENV{"NIX_CONF_DIR"} || "@nixsysconfdir@/nix"; $storeDir = $ENV{"NIX_STORE_DIR"} || "@nixstoredir@"; -$bzip2 = "@bzip2@"; -$xz = "@xz@"; -$curl = "@curl@"; - $useBindings = 1; %config = (); From 0dbb249b36f1d058e63982d26746fde48911cce4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 4 Jul 2019 17:06:38 +0200 Subject: [PATCH 39/60] Update Rust dependencies --- nix-rust/Cargo.lock | 61 +++++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock index cfaedc3ed..ea7ac72db 100644 --- a/nix-rust/Cargo.lock +++ b/nix-rust/Cargo.lock @@ -1,59 +1,82 @@ [[package]] name = "cfg-if" -version = "0.1.7" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "filetime" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "libc" -version = "0.2.50" +version = "0.2.58" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "nix-rust" version = "0.1.0" dependencies = [ - "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", - "tar 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "redox_syscall" -version = "0.1.51" +version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "tar" -version = "0.4.22" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", + "filetime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "winapi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "xattr" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", ] [metadata] -"checksum cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "11d43355396e872eefb45ce6342e4374ed7bc2b3a502d1b28e36d6e23c05d1f4" -"checksum filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a2df5c1a8c4be27e7707789dc42ae65976e60b394afd293d1419ab915833e646" -"checksum libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)" = "aab692d7759f5cd8c859e169db98ae5b52c924add2af5fbbca11d12fefb567c1" -"checksum redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)" = "423e376fffca3dfa06c9e9790a9ccd282fafb3cc6e6397d01dbf64f9bacc6b85" -"checksum tar 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)" = "c2167ff53da2a661702b3299f71a91b61b1dffef36b4b2884b1f9c67254c0133" +"checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33" +"checksum filetime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "450537dc346f0c4d738dda31e790da1da5d4bd12145aad4da0d03d713cb3794f" +"checksum libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)" = "6281b86796ba5e4366000be6e9e18bf35580adf9e63fbe2294aadb587613a319" +"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +"checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3" +"checksum winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" "checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" From d722e2175ef826cd60f05608fb7d58aa82261549 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Jul 2019 01:01:18 +0200 Subject: [PATCH 40/60] Include cargo dependencies in the Nix tarball --- nix-rust/local.mk | 4 +++- release.nix | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/nix-rust/local.mk b/nix-rust/local.mk index c69e3b9ea..ce88fa51b 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -4,4 +4,6 @@ libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE) $(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml - $(trace-gen) cd nix-rust && RUSTC_BOOTSTRAP=1 cargo build --release && touch target/release/libnixrust.a + $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release -Z offline && touch target/release/libnixrust.a + +dist-files += $(d)/vendor diff --git a/release.nix b/release.nix index 1d7c2e2d6..64aa35e11 100644 --- a/release.nix +++ b/release.nix @@ -10,6 +10,43 @@ let jobs = rec { + # Create a "vendor" directory that contains the crates listed in + # Cargo.lock, and include it in the Nix tarball. This allows Nix + # to be built without network access. + vendoredCrates = + let + lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock); + + files = map (pkg: import { + url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download"; + sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)"; + }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package); + + in pkgs.runCommand "cargo-vendor-dir" {} + '' + mkdir -p $out/vendor + + cat > $out/vendor/config < "$dir/.cargo-checksum.json" + + mv "$dir" $out/vendor/ + rmdir $out/vendor/tmp + '') files)} + ''; + tarball = with pkgs; @@ -38,6 +75,8 @@ let distPhase = '' + cp -prd ${vendoredCrates}/vendor/ nix-rust/vendor/ + runHook preDist make dist mkdir -p $out/tarballs From afb021893beb598213d4225de5edd746baa81d7a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Jul 2019 01:19:30 +0200 Subject: [PATCH 41/60] Reduce the size of the vendor directory by removing some winapi cruft --- release.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 64aa35e11..9cf4c74f2 100644 --- a/release.nix +++ b/release.nix @@ -42,8 +42,15 @@ let # Add just enough metadata to keep Cargo happy. printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json" + # Clean up some cruft from the winapi crates. FIXME: find + # a way to remove winapi* from our dependencies. + if [[ $dir =~ /winapi ]]; then + find $dir -name "*.a" -print0 | xargs -0 rm -f -- + fi + mv "$dir" $out/vendor/ - rmdir $out/vendor/tmp + + rm -rf $out/vendor/tmp '') files)} ''; From 343ebcc048c69e103db3f87a9339388bd5dd93cb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Jul 2019 01:50:35 +0200 Subject: [PATCH 42/60] Only pass '-Z offline' to cargo if we have a vendor directory --- nix-rust/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix-rust/local.mk b/nix-rust/local.mk index ce88fa51b..a5e017cef 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -4,6 +4,6 @@ libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE) $(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml - $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release -Z offline && touch target/release/libnixrust.a + $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release $$(if [[ -d vendor ]]; then echo -Z offline; fi) && touch target/release/libnixrust.a dist-files += $(d)/vendor From 8110b4ebb29174ecd4b22510da0285abf604b8a7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Sep 2019 21:55:32 +0200 Subject: [PATCH 43/60] Rust cleanup --- nix-rust/Cargo.toml | 1 + nix-rust/src/error.rs | 5 +++ nix-rust/src/foreign.rs | 14 +++++++++ nix-rust/src/lib.rs | 68 ++++------------------------------------- nix-rust/src/tarfile.rs | 47 ++++++++++++++++++++++++++++ src/libstore/rust.hh | 2 +- 6 files changed, 74 insertions(+), 63 deletions(-) create mode 100644 nix-rust/src/error.rs create mode 100644 nix-rust/src/foreign.rs create mode 100644 nix-rust/src/tarfile.rs diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml index 2d2bf6752..8b56dc89c 100644 --- a/nix-rust/Cargo.toml +++ b/nix-rust/Cargo.toml @@ -2,6 +2,7 @@ name = "nix-rust" version = "0.1.0" authors = ["Eelco Dolstra "] +edition = "2018" [lib] name = "nixrust" diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs new file mode 100644 index 000000000..28d0abdef --- /dev/null +++ b/nix-rust/src/error.rs @@ -0,0 +1,5 @@ +#[derive(Debug)] +pub enum Error { + Misc(String), + Foreign(libc::c_void), // == std::exception_ptr +} diff --git a/nix-rust/src/foreign.rs b/nix-rust/src/foreign.rs new file mode 100644 index 000000000..7bce7753c --- /dev/null +++ b/nix-rust/src/foreign.rs @@ -0,0 +1,14 @@ +/// A wrapper around Nix's Source class that provides the Read trait. +#[repr(C)] +pub struct Source { + fun: extern "C" fn(this: *mut libc::c_void, data: &mut [u8]) -> usize, + this: *mut libc::c_void, +} + +impl std::io::Read for Source { + fn read(&mut self, buf: &mut [u8]) -> std::result::Result { + let n = (self.fun)(self.this, buf); + assert!(n <= buf.len()); + Ok(n) + } +} diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index ac6dee543..192ca29e4 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -1,66 +1,10 @@ -extern crate libc; -extern crate tar; +mod error; +mod foreign; +mod tarfile; -use std::fs; -use std::io; -use std::os::unix::fs::OpenOptionsExt; -use std::path::Path; -use tar::Archive; - -/// A wrapper around Nix's Source class that provides the Read trait. -#[repr(C)] -pub struct Source { - fun: extern "C" fn(this: *mut libc::c_void, data: &mut [u8]) -> usize, - this: *mut libc::c_void, -} - -impl std::io::Read for Source { - fn read(&mut self, buf: &mut [u8]) -> std::result::Result { - let n = (self.fun)(self.this, buf); - assert!(n <= buf.len()); - Ok(n) - } -} +pub use error::Error; #[no_mangle] -pub extern "C" fn unpack_tarfile(source: Source, dest_dir: &str) -> bool { - // FIXME: handle errors. - - let dest_dir = Path::new(dest_dir); - - let mut tar = Archive::new(source); - - for file in tar.entries().unwrap() { - let mut file = file.unwrap(); - - let dest_file = dest_dir.join(file.path().unwrap()); - - fs::create_dir_all(dest_file.parent().unwrap()).unwrap(); - - match file.header().entry_type() { - tar::EntryType::Directory => { - fs::create_dir(dest_file).unwrap(); - } - tar::EntryType::Regular => { - let mode = if file.header().mode().unwrap() & libc::S_IXUSR == 0 { - 0o666 - } else { - 0o777 - }; - let mut f = fs::OpenOptions::new() - .create(true) - .write(true) - .mode(mode) - .open(dest_file) - .unwrap(); - io::copy(&mut file, &mut f).unwrap(); - } - tar::EntryType::Symlink => { - std::os::unix::fs::symlink(file.header().link_name().unwrap().unwrap(), dest_file).unwrap(); - } - t => panic!("Unsupported tar entry type '{:?}'.", t), - } - } - - true +pub extern "C" fn unpack_tarfile(source: foreign::Source, dest_dir: &str) { + tarfile::unpack_tarfile(source, dest_dir).unwrap(); } diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs new file mode 100644 index 000000000..696118e4d --- /dev/null +++ b/nix-rust/src/tarfile.rs @@ -0,0 +1,47 @@ +use crate::{foreign::Source, Error}; +use std::fs; +use std::io; +use std::os::unix::fs::OpenOptionsExt; +use std::path::Path; +use tar::Archive; + +pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> { + let dest_dir = Path::new(dest_dir); + + let mut tar = Archive::new(source); + + for file in tar.entries().unwrap() { + let mut file = file.unwrap(); + + let dest_file = dest_dir.join(file.path().unwrap()); + + fs::create_dir_all(dest_file.parent().unwrap()).unwrap(); + + match file.header().entry_type() { + tar::EntryType::Directory => { + fs::create_dir(dest_file).unwrap(); + } + tar::EntryType::Regular => { + let mode = if file.header().mode().unwrap() & libc::S_IXUSR == 0 { + 0o666 + } else { + 0o777 + }; + let mut f = fs::OpenOptions::new() + .create(true) + .write(true) + .mode(mode) + .open(dest_file) + .unwrap(); + io::copy(&mut file, &mut f).unwrap(); + } + tar::EntryType::Symlink => { + std::os::unix::fs::symlink(file.header().link_name().unwrap().unwrap(), dest_file) + .unwrap(); + } + t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))), + } + } + + Ok(()) +} diff --git a/src/libstore/rust.hh b/src/libstore/rust.hh index 7e6c2f54d..4366e4723 100644 --- a/src/libstore/rust.hh +++ b/src/libstore/rust.hh @@ -40,5 +40,5 @@ struct Source } extern "C" { - bool unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); + void unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); } From f738cd4d976f4f72159bbcbfa7b451c33f0ea74a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 01:15:20 +0200 Subject: [PATCH 44/60] More Rust FFI adventures We can now convert Rust Errors to C++ exceptions. At the Rust->C++ FFI boundary, Result will cause Error to be converted to and thrown as a C++ exception. --- nix-rust/src/error.rs | 27 ++++++++++- nix-rust/src/lib.rs | 29 ++++++++++-- nix-rust/src/tarfile.rs | 20 ++++---- src/libstore/builtins/unpack-channel.cc | 2 +- src/libstore/rust.cc | 12 +++++ src/libstore/rust.hh | 61 ++++++++++++++++++++++++- 6 files changed, 133 insertions(+), 18 deletions(-) create mode 100644 src/libstore/rust.cc diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs index 28d0abdef..a2003be6f 100644 --- a/nix-rust/src/error.rs +++ b/nix-rust/src/error.rs @@ -1,5 +1,30 @@ #[derive(Debug)] pub enum Error { + IOError(std::io::Error), Misc(String), - Foreign(libc::c_void), // == std::exception_ptr + Foreign(CppException), +} + +impl From for Error { + fn from(err: std::io::Error) -> Self { + Error::IOError(err) + } +} + +impl From for CppException { + fn from(err: Error) -> Self { + match err { + Error::Foreign(ex) => ex, + Error::Misc(s) => unsafe { make_error(&s) }, + Error::IOError(err) => unsafe { make_error(&err.to_string()) }, + } + } +} + +#[repr(C)] +#[derive(Debug)] +pub struct CppException(*const libc::c_void); // == std::exception_ptr* + +extern "C" { + fn make_error(s: &str) -> CppException; } diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index 192ca29e4..b6b0d746d 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -4,7 +4,30 @@ mod tarfile; pub use error::Error; -#[no_mangle] -pub extern "C" fn unpack_tarfile(source: foreign::Source, dest_dir: &str) { - tarfile::unpack_tarfile(source, dest_dir).unwrap(); +pub struct CBox { + ptr: *mut libc::c_void, + phantom: std::marker::PhantomData, +} + +impl CBox { + fn new(t: T) -> Self { + unsafe { + let size = std::mem::size_of::(); + let ptr = libc::malloc(size); + eprintln!("PTR = {:?}, SIZE = {}", ptr, size); + *(ptr as *mut T) = t; // FIXME: probably UB + Self { + ptr, + phantom: std::marker::PhantomData, + } + } + } +} + +#[no_mangle] +pub extern "C" fn unpack_tarfile( + source: foreign::Source, + dest_dir: &str, +) -> CBox> { + CBox::new(tarfile::unpack_tarfile(source, dest_dir).map_err(|err| err.into())) } diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs index 696118e4d..797aa5064 100644 --- a/nix-rust/src/tarfile.rs +++ b/nix-rust/src/tarfile.rs @@ -10,19 +10,19 @@ pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> { let mut tar = Archive::new(source); - for file in tar.entries().unwrap() { - let mut file = file.unwrap(); + for file in tar.entries()? { + let mut file = file?; - let dest_file = dest_dir.join(file.path().unwrap()); + let dest_file = dest_dir.join(file.path()?); - fs::create_dir_all(dest_file.parent().unwrap()).unwrap(); + fs::create_dir_all(dest_file.parent().unwrap())?; match file.header().entry_type() { tar::EntryType::Directory => { - fs::create_dir(dest_file).unwrap(); + fs::create_dir(dest_file)?; } tar::EntryType::Regular => { - let mode = if file.header().mode().unwrap() & libc::S_IXUSR == 0 { + let mode = if file.header().mode()? & libc::S_IXUSR == 0 { 0o666 } else { 0o777 @@ -31,13 +31,11 @@ pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> { .create(true) .write(true) .mode(mode) - .open(dest_file) - .unwrap(); - io::copy(&mut file, &mut f).unwrap(); + .open(dest_file)?; + io::copy(&mut file, &mut f)?; } tar::EntryType::Symlink => { - std::os::unix::fs::symlink(file.header().link_name().unwrap().unwrap(), dest_file) - .unwrap(); + std::os::unix::fs::symlink(file.header().link_name()?.unwrap(), dest_file)?; } t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))), } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index 88202ec6b..2da26d98e 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -27,7 +27,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) decompressor->finish(); }); - unpack_tarfile(*source, out); + unpack_tarfile(*source, out).use()->unwrap(); auto entries = readDirectory(out); if (entries.size() != 1) diff --git a/src/libstore/rust.cc b/src/libstore/rust.cc new file mode 100644 index 000000000..a616d83a6 --- /dev/null +++ b/src/libstore/rust.cc @@ -0,0 +1,12 @@ +#include "logging.hh" +#include "rust.hh" + +namespace nix { + +extern "C" std::exception_ptr * make_error(rust::StringSlice s) +{ + // FIXME: leak + return new std::exception_ptr(std::make_exception_ptr(Error(std::string(s.ptr, s.size)))); +} + +} diff --git a/src/libstore/rust.hh b/src/libstore/rust.hh index 4366e4723..4c7720a44 100644 --- a/src/libstore/rust.hh +++ b/src/libstore/rust.hh @@ -4,7 +4,8 @@ namespace rust { // Depending on the internal representation of Rust slices is slightly // evil... -template struct Slice +template +struct Slice { T * ptr; size_t size; @@ -37,8 +38,64 @@ struct Source } }; +/* C++ representation of Rust's Result. */ +template +struct Result +{ + unsigned int tag; + + union { + T data; + std::exception_ptr * exc; + }; + + /* Rethrow the wrapped exception or return the wrapped value. */ + T unwrap() + { + if (tag == 0) + return data; + else if (tag == 1) + std::rethrow_exception(*exc); + else + abort(); + } +}; + +template +struct CBox +{ + T * ptr; + + T * operator ->() + { + return ptr; + } + + CBox(T * ptr) : ptr(ptr) { } + CBox(const CBox &) = delete; + CBox(CBox &&) = delete; + + ~CBox() + { + free(ptr); + } +}; + +// Grrr, this is only needed because 'extern "C"' functions don't +// support non-POD return types (and CBox has a destructor so it's not +// POD). +template +struct CBox2 +{ + T * ptr; + CBox use() + { + return CBox(ptr); + } +}; + } extern "C" { - void unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); + rust::CBox2>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); } From b7fba16613aed5c2b2a0b4f98e3e0d32b0cddd40 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 12:43:07 +0200 Subject: [PATCH 45/60] Move code around --- src/libstore/rust.hh | 97 +------------------ src/{libstore/rust.cc => libutil/rust-ffi.cc} | 0 src/libutil/rust-ffi.hh | 97 +++++++++++++++++++ 3 files changed, 98 insertions(+), 96 deletions(-) rename src/{libstore/rust.cc => libutil/rust-ffi.cc} (100%) create mode 100644 src/libutil/rust-ffi.hh diff --git a/src/libstore/rust.hh b/src/libstore/rust.hh index 4c7720a44..dccc98b4b 100644 --- a/src/libstore/rust.hh +++ b/src/libstore/rust.hh @@ -1,100 +1,5 @@ #include "serialise.hh" - -namespace rust { - -// Depending on the internal representation of Rust slices is slightly -// evil... -template -struct Slice -{ - T * ptr; - size_t size; - - Slice(T * ptr, size_t size) : ptr(ptr), size(size) - { - assert(ptr); - } -}; - -struct StringSlice : Slice -{ - StringSlice(const std::string & s): Slice((char *) s.data(), s.size()) {} -}; - -struct Source -{ - size_t (*fun)(void * source_this, rust::Slice data); - nix::Source * _this; - - Source(nix::Source & _this) - : fun(sourceWrapper), _this(&_this) - {} - - // FIXME: how to propagate exceptions? - static size_t sourceWrapper(void * _this, rust::Slice data) - { - auto n = ((nix::Source *) _this)->read(data.ptr, data.size); - return n; - } -}; - -/* C++ representation of Rust's Result. */ -template -struct Result -{ - unsigned int tag; - - union { - T data; - std::exception_ptr * exc; - }; - - /* Rethrow the wrapped exception or return the wrapped value. */ - T unwrap() - { - if (tag == 0) - return data; - else if (tag == 1) - std::rethrow_exception(*exc); - else - abort(); - } -}; - -template -struct CBox -{ - T * ptr; - - T * operator ->() - { - return ptr; - } - - CBox(T * ptr) : ptr(ptr) { } - CBox(const CBox &) = delete; - CBox(CBox &&) = delete; - - ~CBox() - { - free(ptr); - } -}; - -// Grrr, this is only needed because 'extern "C"' functions don't -// support non-POD return types (and CBox has a destructor so it's not -// POD). -template -struct CBox2 -{ - T * ptr; - CBox use() - { - return CBox(ptr); - } -}; - -} +#include "rust-ffi.hh" extern "C" { rust::CBox2>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); diff --git a/src/libstore/rust.cc b/src/libutil/rust-ffi.cc similarity index 100% rename from src/libstore/rust.cc rename to src/libutil/rust-ffi.cc diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh new file mode 100644 index 000000000..a488b96d6 --- /dev/null +++ b/src/libutil/rust-ffi.hh @@ -0,0 +1,97 @@ +#include "serialise.hh" + +namespace rust { + +// Depending on the internal representation of Rust slices is slightly +// evil... +template +struct Slice +{ + T * ptr; + size_t size; + + Slice(T * ptr, size_t size) : ptr(ptr), size(size) + { + assert(ptr); + } +}; + +struct StringSlice : Slice +{ + StringSlice(const std::string & s): Slice((char *) s.data(), s.size()) {} +}; + +struct Source +{ + size_t (*fun)(void * source_this, rust::Slice data); + nix::Source * _this; + + Source(nix::Source & _this) + : fun(sourceWrapper), _this(&_this) + {} + + // FIXME: how to propagate exceptions? + static size_t sourceWrapper(void * _this, rust::Slice data) + { + auto n = ((nix::Source *) _this)->read(data.ptr, data.size); + return n; + } +}; + +/* C++ representation of Rust's Result. */ +template +struct Result +{ + unsigned int tag; + + union { + T data; + std::exception_ptr * exc; + }; + + /* Rethrow the wrapped exception or return the wrapped value. */ + T unwrap() + { + if (tag == 0) + return data; + else if (tag == 1) + std::rethrow_exception(*exc); + else + abort(); + } +}; + +template +struct CBox +{ + T * ptr; + + T * operator ->() + { + return ptr; + } + + CBox(T * ptr) : ptr(ptr) { } + CBox(const CBox &) = delete; + CBox(CBox &&) = delete; + + ~CBox() + { + free(ptr); + } +}; + +// Grrr, this is only needed because 'extern "C"' functions don't +// support non-POD return types (and CBox has a destructor so it's not +// POD). +template +struct CBox2 +{ + T * ptr; + CBox use() + { + return CBox(ptr); + } +}; + +} From d14b1c261cd3dfb4be2da943d901a394c3f23205 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 12:44:31 +0200 Subject: [PATCH 46/60] Shut up some rust warnings --- nix-rust/src/error.rs | 1 + nix-rust/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs index a2003be6f..519007ea0 100644 --- a/nix-rust/src/error.rs +++ b/nix-rust/src/error.rs @@ -26,5 +26,6 @@ impl From for CppException { pub struct CppException(*const libc::c_void); // == std::exception_ptr* extern "C" { + #[allow(improper_ctypes)] // YOLO fn make_error(s: &str) -> CppException; } diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index b6b0d746d..48952d8b9 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -5,7 +5,7 @@ mod tarfile; pub use error::Error; pub struct CBox { - ptr: *mut libc::c_void, + pub ptr: *mut libc::c_void, phantom: std::marker::PhantomData, } From d33dd6e6c09d87a59989057ad622a6265ddec2e0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 13:10:46 +0200 Subject: [PATCH 47/60] Move code around --- nix-rust/src/lib.rs | 1 - src/libstore/builtins/unpack-channel.cc | 4 ++-- src/libstore/local.mk | 2 +- src/libutil/local.mk | 2 ++ src/libutil/rust-ffi.cc | 2 +- src/{libstore/rust.hh => libutil/tarfile.cc} | 10 +++++++++- src/libutil/tarfile.hh | 7 +++++++ 7 files changed, 22 insertions(+), 6 deletions(-) rename src/{libstore/rust.hh => libutil/tarfile.cc} (53%) create mode 100644 src/libutil/tarfile.hh diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index 48952d8b9..1b88ac8af 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -14,7 +14,6 @@ impl CBox { unsafe { let size = std::mem::size_of::(); let ptr = libc::malloc(size); - eprintln!("PTR = {:?}, SIZE = {}", ptr, size); *(ptr as *mut T) = t; // FIXME: probably UB Self { ptr, diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index 2da26d98e..5fc68cd66 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -1,6 +1,6 @@ -#include "rust.hh" #include "builtins.hh" #include "compression.hh" +#include "tarfile.hh" namespace nix { @@ -27,7 +27,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) decompressor->finish(); }); - unpack_tarfile(*source, out).use()->unwrap(); + unpackTarfile(*source, out); auto entries = readDirectory(out); if (entries.size() != 1) diff --git a/src/libstore/local.mk b/src/libstore/local.mk index d3254554d..d690fea28 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -6,7 +6,7 @@ libstore_DIR := $(d) libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) -libstore_LIBS = libutil libnixrust +libstore_LIBS = libutil libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread ifneq ($(OS), FreeBSD) diff --git a/src/libutil/local.mk b/src/libutil/local.mk index e41a67d1f..35c1f6c13 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -7,3 +7,5 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context + +libutil_LIBS = libnixrust diff --git a/src/libutil/rust-ffi.cc b/src/libutil/rust-ffi.cc index a616d83a6..931d29542 100644 --- a/src/libutil/rust-ffi.cc +++ b/src/libutil/rust-ffi.cc @@ -1,5 +1,5 @@ #include "logging.hh" -#include "rust.hh" +#include "rust-ffi.hh" namespace nix { diff --git a/src/libstore/rust.hh b/src/libutil/tarfile.cc similarity index 53% rename from src/libstore/rust.hh rename to src/libutil/tarfile.cc index dccc98b4b..ae6d512bd 100644 --- a/src/libstore/rust.hh +++ b/src/libutil/tarfile.cc @@ -1,6 +1,14 @@ -#include "serialise.hh" #include "rust-ffi.hh" extern "C" { rust::CBox2>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); } + +namespace nix { + +void unpackTarfile(Source & source, Path destDir) +{ + unpack_tarfile(source, destDir).use()->unwrap(); +} + +} diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh new file mode 100644 index 000000000..c3e95fb0c --- /dev/null +++ b/src/libutil/tarfile.hh @@ -0,0 +1,7 @@ +#include "serialise.hh" + +namespace nix { + +void unpackTarfile(Source & source, Path destDir); + +} From f2bd8470926686361602e545d63a69d4bfc22f90 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 15:03:17 +0200 Subject: [PATCH 48/60] Ignore tar header entries In particular, these are emitted by 'git archive' (in fetchGit). --- nix-rust/src/tarfile.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs index 797aa5064..9e32a3f00 100644 --- a/nix-rust/src/tarfile.rs +++ b/nix-rust/src/tarfile.rs @@ -37,6 +37,7 @@ pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> { tar::EntryType::Symlink => { std::os::unix::fs::symlink(file.header().link_name()?.unwrap(), dest_file)?; } + tar::EntryType::XGlobalHeader | tar::EntryType::XHeader => {} t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))), } } From 8918bae09828133259acb36d6aef60ffbfad252c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 15:25:43 +0200 Subject: [PATCH 49/60] Drop remaining uses of external "tar" Also, fetchGit now runs in O(1) memory since we pipe the output of 'git archive' directly into unpackTarball() (rather than first reading it all into memory). --- src/libexpr/primops/fetchGit.cc | 11 +++++++---- src/libstore/builtins/unpack-channel.cc | 12 +----------- src/libstore/download.cc | 12 ++++++++---- src/libutil/serialise.hh | 1 - src/libutil/tarfile.cc | 22 +++++++++++++++++++++- src/libutil/tarfile.hh | 5 ++++- src/nix-prefetch-url/nix-prefetch-url.cc | 4 ++-- 7 files changed, 43 insertions(+), 24 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 7ef3b3823..9d0c64291 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -4,6 +4,7 @@ #include "store-api.hh" #include "pathlocks.hh" #include "hash.hh" +#include "tarfile.hh" #include @@ -164,14 +165,16 @@ GitInfo exportGit(ref store, const std::string & uri, if (e.errNo != ENOENT) throw; } - // FIXME: should pipe this, or find some better way to extract a - // revision. - auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev }); + auto source = sinkToSource([&](Sink & sink) { + RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); + gitOptions.standardOut = &sink; + runProgram2(gitOptions); + }); Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - runProgram("tar", true, { "x", "-C", tmpDir }, tar); + unpackTarfile(*source, tmpDir); gitInfo.storePath = store->addToStore(name, tmpDir); diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index 5fc68cd66..d18e3ddaf 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -1,5 +1,4 @@ #include "builtins.hh" -#include "compression.hh" #include "tarfile.hh" namespace nix { @@ -18,16 +17,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) createDirs(out); - auto source = sinkToSource([&](Sink & sink) { - auto decompressor = - hasSuffix(src, ".bz2") ? makeDecompressionSink("bzip2", sink) : - hasSuffix(src, ".xz") ? makeDecompressionSink("xz", sink) : - makeDecompressionSink("none", sink); - readFile(src, *decompressor); - decompressor->finish(); - }); - - unpackTarfile(*source, out); + unpackTarfile(src, out); auto entries = readDirectory(out); if (entries.size() != 1) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index e80663dff..61e88c5c1 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -8,6 +8,7 @@ #include "compression.hh" #include "pathlocks.hh" #include "finally.hh" +#include "tarfile.hh" #ifdef ENABLE_S3 #include @@ -903,12 +904,15 @@ CachedDownloadResult Downloader::downloadCached( unpackedStorePath = ""; } if (unpackedStorePath.empty()) { - printInfo(format("unpacking '%1%'...") % url); + printInfo("unpacking '%s'...", url); Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); - // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"}); - unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); + unpackTarfile(store->toRealPath(storePath), tmpDir, baseNameOf(url)); + auto members = readDirectory(tmpDir); + if (members.size() != 1) + throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); + auto topDir = tmpDir + "/" + members.begin()->name; + unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); } replaceSymlink(unpackedStorePath, unpackedLink); storePath = unpackedStorePath; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 128e287f3..5780c93a6 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -77,7 +77,6 @@ struct BufferedSource : Source size_t read(unsigned char * data, size_t len) override; - bool hasData(); protected: diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index ae6d512bd..f7d3ad417 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -1,4 +1,5 @@ #include "rust-ffi.hh" +#include "compression.hh" extern "C" { rust::CBox2>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); @@ -6,9 +7,28 @@ extern "C" { namespace nix { -void unpackTarfile(Source & source, Path destDir) +void unpackTarfile(Source & source, const Path & destDir) { unpack_tarfile(source, destDir).use()->unwrap(); } +void unpackTarfile(const Path & tarFile, const Path & destDir, + std::optional baseName) +{ + if (!baseName) baseName = baseNameOf(tarFile); + + auto source = sinkToSource([&](Sink & sink) { + // FIXME: look at first few bytes to determine compression type. + auto decompressor = + // FIXME: add .gz support + hasSuffix(*baseName, ".bz2") ? makeDecompressionSink("bzip2", sink) : + hasSuffix(*baseName, ".xz") ? makeDecompressionSink("xz", sink) : + makeDecompressionSink("none", sink); + readFile(tarFile, *decompressor); + decompressor->finish(); + }); + + unpackTarfile(*source, destDir); +} + } diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index c3e95fb0c..ce0911e2a 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -2,6 +2,9 @@ namespace nix { -void unpackTarfile(Source & source, Path destDir); +void unpackTarfile(Source & source, const Path & destDir); + +void unpackTarfile(const Path & tarFile, const Path & destDir, + std::optional baseName = {}); } diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index f54706a8a..78c883833 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -9,6 +9,7 @@ #include "legacy.hh" #include "finally.hh" #include "progress-bar.hh" +#include "tarfile.hh" #include @@ -192,8 +193,7 @@ static int _main(int argc, char * * argv) if (hasSuffix(baseNameOf(uri), ".zip")) runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked}); else - // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", tmpFile, "-C", unpacked}); + unpackTarfile(tmpFile, unpacked, baseNameOf(uri)); /* If the archive unpacks to a single file/directory, then use that as the top-level. */ From 88f8063917a4261abfabfe94bc33c63f7405d1d5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 26 Nov 2019 22:45:15 +0100 Subject: [PATCH 50/60] -Z offline -> --offline --- nix-rust/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix-rust/local.mk b/nix-rust/local.mk index a5e017cef..fda1c3dee 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -4,6 +4,6 @@ libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE) $(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml - $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release $$(if [[ -d vendor ]]; then echo -Z offline; fi) && touch target/release/libnixrust.a + $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release $$(if [[ -d vendor ]]; then echo --offline; fi) && touch target/release/libnixrust.a dist-files += $(d)/vendor From e6c1d1b474b1f14753b27f35b2aad4051ffd9e8d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 26 Nov 2019 22:46:36 +0100 Subject: [PATCH 51/60] Update Cargo.lock --- nix-rust/Cargo.lock | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock index ea7ac72db..0112ed471 100644 --- a/nix-rust/Cargo.lock +++ b/nix-rust/Cargo.lock @@ -1,29 +1,31 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. [[package]] name = "cfg-if" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "filetime" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "libc" -version = "0.2.58" +version = "0.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "nix-rust" version = "0.1.0" dependencies = [ - "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -37,15 +39,15 @@ name = "tar" version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "filetime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "winapi" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -67,16 +69,16 @@ name = "xattr" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", ] [metadata] -"checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33" -"checksum filetime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "450537dc346f0c4d738dda31e790da1da5d4bd12145aad4da0d03d713cb3794f" -"checksum libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)" = "6281b86796ba5e4366000be6e9e18bf35580adf9e63fbe2294aadb587613a319" +"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +"checksum filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1ff6d4dab0aa0c8e6346d46052e93b13a16cf847b54ed357087c35011048cc7d" +"checksum libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)" = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8" "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" "checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3" -"checksum winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770" +"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" "checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" From dbc4f9d478814f3ce4ee23531502247d02c85911 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Nov 2019 00:17:24 +0100 Subject: [PATCH 52/60] Fix macOS build https://hydra.nixos.org/build/107466992 --- nix-rust/src/tarfile.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs index 9e32a3f00..379d9098f 100644 --- a/nix-rust/src/tarfile.rs +++ b/nix-rust/src/tarfile.rs @@ -22,7 +22,7 @@ pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> { fs::create_dir(dest_file)?; } tar::EntryType::Regular => { - let mode = if file.header().mode()? & libc::S_IXUSR == 0 { + let mode = if file.header().mode()? & (libc::S_IXUSR as u32) == 0 { 0o666 } else { 0o777 From 949dc848940b68ffd1327a278df4472e98a455dc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Nov 2019 14:17:15 +0100 Subject: [PATCH 53/60] Fix segfault on i686-linux https://hydra.nixos.org/build/107467517 Seems that on i686-linux, gcc and rustc disagree on how to return 1-word structs: gcc has the caller pass a pointer to the result, while rustc has the callee return the result in a register. Work around this by using a bare pointer. --- src/libutil/rust-ffi.hh | 13 ------------- src/libutil/tarfile.cc | 6 ++++-- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh index a488b96d6..663758bfc 100644 --- a/src/libutil/rust-ffi.hh +++ b/src/libutil/rust-ffi.hh @@ -81,17 +81,4 @@ struct CBox } }; -// Grrr, this is only needed because 'extern "C"' functions don't -// support non-POD return types (and CBox has a destructor so it's not -// POD). -template -struct CBox2 -{ - T * ptr; - CBox use() - { - return CBox(ptr); - } -}; - } diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index f7d3ad417..2cc7793fd 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -2,14 +2,16 @@ #include "compression.hh" extern "C" { - rust::CBox2>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); + rust::Result> * + unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); } namespace nix { void unpackTarfile(Source & source, const Path & destDir) { - unpack_tarfile(source, destDir).use()->unwrap(); + rust::Source source2(source); + rust::CBox(unpack_tarfile(source2, destDir))->unwrap(); } void unpackTarfile(const Path & tarFile, const Path & destDir, From f553a8bdea96afe18b97dee6eba91c0bbc545b79 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Nov 2019 14:18:57 +0100 Subject: [PATCH 54/60] When OPTIMIZE=0, build rust code in debug mode --- nix-rust/local.mk | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/nix-rust/local.mk b/nix-rust/local.mk index fda1c3dee..c147f9874 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -1,9 +1,19 @@ -libnixrust_PATH := $(d)/target/release/libnixrust.a +ifeq ($(OPTIMIZE), 1) + RUST_MODE = --release + RUST_DIR = release +else + RUST_MODE = + RUST_DIR = debug +endif + +libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.a libnixrust_INSTALL_PATH := $(libnixrust_PATH) -libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl +libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE) -$(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml - $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release $$(if [[ -d vendor ]]; then echo --offline; fi) && touch target/release/libnixrust.a +$(d)/target/$(RUST_DIR)/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml + $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \ + cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \ + && touch target/$(RUST_DIR)/libnixrust.a dist-files += $(d)/vendor From 895ce1bb6cc77324c377baa08eb98666768e5c6d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Nov 2019 17:33:59 +0100 Subject: [PATCH 55/60] make clean: Delete nix-rust/target --- nix-rust/local.mk | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nix-rust/local.mk b/nix-rust/local.mk index c147f9874..ed1e640c9 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -17,3 +17,8 @@ $(d)/target/$(RUST_DIR)/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml && touch target/$(RUST_DIR)/libnixrust.a dist-files += $(d)/vendor + +clean: clean-rust + +clean-rust: + $(suppress) rm -rfv nix-rust/target From 2d6f1ddbb5f2ae2f3153785d7a4e5321c85495fd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 28 Nov 2019 13:52:42 +0100 Subject: [PATCH 56/60] Remove builtins.valueSize Fixes #3246. --- src/libexpr/eval.cc | 91 ------------------------------------------ src/libexpr/eval.hh | 1 - src/libexpr/primops.cc | 8 ---- src/libexpr/value.hh | 9 ----- 4 files changed, 109 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 3a9b7a3c0..b89a67b19 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -614,13 +614,9 @@ Value * EvalState::allocValue() Env & EvalState::allocEnv(size_t size) { - if (size > std::numeric_limits::max()) - throw Error("environment size %d is too big", size); - nrEnvs++; nrValuesInEnvs += size; Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *)); - env->size = (decltype(Env::size)) size; env->type = Env::Plain; /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */ @@ -1868,93 +1864,6 @@ void EvalState::printStats() } -size_t valueSize(Value & v) -{ - std::set seen; - - auto doString = [&](const char * s) -> size_t { - if (!seen.insert(s).second) return 0; - return strlen(s) + 1; - }; - - std::function doValue; - std::function doEnv; - - doValue = [&](Value & v) -> size_t { - if (!seen.insert(&v).second) return 0; - - size_t sz = sizeof(Value); - - switch (v.type) { - case tString: - sz += doString(v.string.s); - if (v.string.context) - for (const char * * p = v.string.context; *p; ++p) - sz += doString(*p); - break; - case tPath: - sz += doString(v.path); - break; - case tAttrs: - if (seen.insert(v.attrs).second) { - sz += sizeof(Bindings) + sizeof(Attr) * v.attrs->capacity(); - for (auto & i : *v.attrs) - sz += doValue(*i.value); - } - break; - case tList1: - case tList2: - case tListN: - if (seen.insert(v.listElems()).second) { - sz += v.listSize() * sizeof(Value *); - for (size_t n = 0; n < v.listSize(); ++n) - sz += doValue(*v.listElems()[n]); - } - break; - case tThunk: - sz += doEnv(*v.thunk.env); - break; - case tApp: - sz += doValue(*v.app.left); - sz += doValue(*v.app.right); - break; - case tLambda: - sz += doEnv(*v.lambda.env); - break; - case tPrimOpApp: - sz += doValue(*v.primOpApp.left); - sz += doValue(*v.primOpApp.right); - break; - case tExternal: - if (!seen.insert(v.external).second) break; - sz += v.external->valueSize(seen); - break; - default: - ; - } - - return sz; - }; - - doEnv = [&](Env & env) -> size_t { - if (!seen.insert(&env).second) return 0; - - size_t sz = sizeof(Env) + sizeof(Value *) * env.size; - - if (env.type != Env::HasWithExpr) - for (size_t i = 0; i < env.size; ++i) - if (env.values[i]) - sz += doValue(*env.values[i]); - - if (env.up) sz += doEnv(*env.up); - - return sz; - }; - - return doValue(v); -} - - string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const { throw TypeError(format("cannot coerce %1% to a string, at %2%") % diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 9d075a48a..419b703fc 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -37,7 +37,6 @@ struct PrimOp struct Env { Env * up; - unsigned short size; // used by ‘valueSize’ unsigned short prevWith:14; // nr of levels up to next `with' environment enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2; Value * values[0]; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 2cc03fe61..d693a3b20 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -506,13 +506,6 @@ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value } -void prim_valueSize(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - /* We're not forcing the argument on purpose. */ - mkInt(v, valueSize(*args[0])); -} - - /************************************************************* * Derivations *************************************************************/ @@ -2206,7 +2199,6 @@ void EvalState::createBaseEnv() // Debugging addPrimOp("__trace", 2, prim_trace); - addPrimOp("__valueSize", 1, prim_valueSize); // Paths addPrimOp("__toPath", 1, prim_toPath); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 6743d7fd8..689373873 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -62,9 +62,6 @@ class ExternalValueBase /* Return a string to be used in builtins.typeOf */ virtual string typeOf() const = 0; - /* How much space does this value take up */ - virtual size_t valueSize(std::set & seen) const = 0; - /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an * error */ @@ -255,12 +252,6 @@ static inline void mkPathNoCopy(Value & v, const char * s) void mkPath(Value & v, const char * s); -/* Compute the size in bytes of the given value, including all values - and environments reachable from it. Static expressions (Exprs) are - not included. */ -size_t valueSize(Value & v); - - #if HAVE_BOEHMGC typedef std::vector > ValueVector; typedef std::map, gc_allocator > > ValueMap; From 895ed4cef09729609d9f1c0b0abe1cc86356c086 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 28 Nov 2019 15:10:18 +0100 Subject: [PATCH 57/60] Remove RPM spec file Closes #3225. Closes #3226. --- .gitignore | 1 - local.mk | 2 +- nix.spec.in | 173 ---------------------------------------------------- 3 files changed, 1 insertion(+), 175 deletions(-) delete mode 100644 nix.spec.in diff --git a/.gitignore b/.gitignore index fd62dfb38..3f9758b9d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ perl/Makefile.config /precompiled-headers.h.pch /config.* /configure -/nix.spec /stamp-h1 /svn-revision /libtool diff --git a/local.mk b/local.mk index 4b380176f..921ea91cd 100644 --- a/local.mk +++ b/local.mk @@ -2,7 +2,7 @@ ifeq ($(MAKECMDGOALS), dist) dist-files += $(shell cat .dist-files) endif -dist-files += configure config.h.in nix.spec perl/configure +dist-files += configure config.h.in perl/configure clean-files += Makefile.config diff --git a/nix.spec.in b/nix.spec.in deleted file mode 100644 index 6b9e37637..000000000 --- a/nix.spec.in +++ /dev/null @@ -1,173 +0,0 @@ -%undefine _hardened_build - -%global nixbld_user "nix-builder-" -%global nixbld_group "nixbld" - -# NOTE: BUILD on EL7 requires -# - Centos / RHEL7 software collection repository -# yum install centos-release-scl -# -# - Recent boost backport -# curl https://copr.fedorainfracloud.org/coprs/whosthere/boost/repo/epel-7/whosthere-boost-epel-7.repo -o /etc/yum.repos.d/whosthere-boost-epel-7.repo -# - -# Disable documentation generation -# necessary on some platforms -%bcond_without docgen - -Summary: The Nix software deployment system -Name: nix -Version: @PACKAGE_VERSION@ -Release: 2%{?dist} -License: LGPLv2+ -Group: Applications/System -URL: http://nixos.org/ -Source0: %{name}-%{version}.tar.bz2 - -Requires: curl -Requires: bzip2 -Requires: gzip -Requires: xz -BuildRequires: bison -BuildRequires: boost-devel >= 1.60 -BuildRequires: bzip2-devel - -# for RHEL <= 7, we need software collections for a C++14 compatible compatible compiler -%if 0%{?rhel} -BuildRequires: devtoolset-7-gcc -BuildRequires: devtoolset-7-gcc-c++ -%endif - -BuildRequires: flex -BuildRequires: libcurl-devel -BuildRequires: libseccomp-devel -BuildRequires: openssl-devel -BuildRequires: sqlite-devel -BuildRequires: xz-devel - -%description -Nix is a purely functional package manager. It allows multiple -versions of a package to be installed side-by-side, ensures that -dependency specifications are complete, supports atomic upgrades and -rollbacks, allows non-root users to install software, and has many -other features. It is the basis of the NixOS Linux distribution, but -it can be used equally well under other Unix systems. - -%package devel -Summary: Development files for %{name} -Requires: %{name}%{?_isa} = %{version}-%{release} - -%description devel -The %{name}-devel package contains libraries and header files for -developing applications that use %{name}. - - -%package doc -Summary: Documentation files for %{name} -BuildArch: noarch -Requires: %{name} = %{version}-%{release} - -%description doc -The %{name}-doc package contains documentation files for %{name}. - -%prep -%setup -q - - -%build -%if 0%{?rhel} -source /opt/rh/devtoolset-7/enable -%endif -extraFlags= -# - override docdir so large documentation files are owned by the -# -doc subpackage -# - set localstatedir by hand to the preferred nix value -%configure --localstatedir=/nix/var \ - %{!?without_docgen:--disable-doc-gen} \ - --docdir=%{_defaultdocdir}/%{name}-doc-%{version} \ - $extraFlags -make V=1 %{?_smp_mflags} - - -%install -%if 0%{?rhel} -source /opt/rh/devtoolset-7/enable -%endif - -make DESTDIR=$RPM_BUILD_ROOT install - -find $RPM_BUILD_ROOT -name '*.la' -exec rm -f {} ';' - -# make the store -mkdir -p $RPM_BUILD_ROOT/nix/store -chmod 1775 $RPM_BUILD_ROOT/nix/store - -# make per-user directories -for d in profiles gcroots; -do - mkdir -p $RPM_BUILD_ROOT/nix/var/nix/$d/per-user - chmod 755 $RPM_BUILD_ROOT/nix/var/nix/$d/per-user -done - -# fix permission of nix profile -# (until this is fixed in the relevant Makefile) -chmod -x $RPM_BUILD_ROOT%{_sysconfdir}/profile.d/nix.sh - -# we ship this file in the base package -rm -f $RPM_BUILD_ROOT%{_defaultdocdir}/%{name}-doc-%{version}/README - -# Get rid of Upstart job. -rm -rf $RPM_BUILD_ROOT%{_sysconfdir}/init - - -%clean -rm -rf $RPM_BUILD_ROOT - - -%pre -getent group %{nixbld_group} >/dev/null || groupadd -r %{nixbld_group} -for i in $(seq 10); -do - getent passwd %{nixbld_user}$i >/dev/null || \ - useradd -r -g %{nixbld_group} -G %{nixbld_group} -d /var/empty \ - -s %{_sbindir}/nologin \ - -c "Nix build user $i" %{nixbld_user}$i -done - -%post -chgrp %{nixbld_group} /nix/store -%if ! 0%{?rhel} || 0%{?rhel} >= 7 -# Enable and start Nix worker -systemctl enable nix-daemon.socket nix-daemon.service -systemctl start nix-daemon.socket -%endif - -%files -%license COPYING -%{_bindir}/nix* -%{_libdir}/*.so -%{_prefix}/libexec/* -%if ! 0%{?rhel} || 0%{?rhel} >= 7 -%{_prefix}/lib/systemd/system/nix-daemon.socket -%{_prefix}/lib/systemd/system/nix-daemon.service -%endif -%{_datadir}/nix -#%if ! %{without docgen} -#%{_mandir}/man1/*.1* -#%{_mandir}/man5/*.5* -#%{_mandir}/man8/*.8* -#%endif -%config(noreplace) %{_sysconfdir}/profile.d/nix.sh -%config(noreplace) %{_sysconfdir}/profile.d/nix-daemon.sh -/nix - -%files devel -%{_includedir}/nix -%{_prefix}/lib/pkgconfig/*.pc - - -#%if ! %{without docgen} -#%files doc -#%docdir %{_defaultdocdir}/%{name}-doc-%{version} -#%{_defaultdocdir}/%{name}-doc-%{version} -#%endif From 39954a958623431acb8642372f881cbdb7bb789d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 29 Nov 2019 18:27:40 +0100 Subject: [PATCH 58/60] Make libnixrust a dynamic library This is a hack to fix the build on macOS, which was failing because libnixrust.a contains compiler builtins that clash with libclang_rt.osx.a. There's probably a better solution... https://hydra.nixos.org/build/107473280 --- nix-rust/Cargo.toml | 2 +- nix-rust/local.mk | 26 ++++++++++++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml index 8b56dc89c..c4f4ceb09 100644 --- a/nix-rust/Cargo.toml +++ b/nix-rust/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [lib] name = "nixrust" -crate-type = ["staticlib"] +crate-type = ["cdylib"] [dependencies] tar = "0.4" diff --git a/nix-rust/local.mk b/nix-rust/local.mk index ed1e640c9..7645d5394 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -6,15 +6,29 @@ else RUST_DIR = debug endif -libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.a -libnixrust_INSTALL_PATH := $(libnixrust_PATH) +libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.$(SO_EXT) +libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT) libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl -libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE) +libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl -$(d)/target/$(RUST_DIR)/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml +ifeq ($(OS), Darwin) +libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup" +else +libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR)) +libnixrust_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$(libdir) +endif + +$(libnixrust_PATH): $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \ - cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \ - && touch target/$(RUST_DIR)/libnixrust.a + $(libnixrust_BUILD_FLAGS) \ + cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \ + && touch target/$(RUST_DIR)/libnixrust.$(SO_EXT) + +$(libnixrust_INSTALL_PATH): $(libnixrust_PATH) + $(target-gen) cp $^ $@ +ifeq ($(OS), Darwin) + install_name_tool -id $@ $@ +endif dist-files += $(d)/vendor From 8737980e75bf14cae278f596ac26577bec94b3f9 Mon Sep 17 00:00:00 2001 From: Brian Wignall Date: Sat, 30 Nov 2019 19:04:14 -0500 Subject: [PATCH 59/60] Fix typos --- src/libstore/globals.hh | 2 +- src/libstore/store-api.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 9f395425a..41e511c6c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -319,7 +319,7 @@ public: "A program to run just before a build to set derivation-specific build settings."}; Setting postBuildHook{this, "", "post-build-hook", - "A program to run just after each succesful build."}; + "A program to run just after each successful build."}; Setting netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", "Path to the netrc file used to obtain usernames/passwords for downloads."}; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ec32e995d..8e4b133d5 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -569,7 +569,7 @@ public: unsigned long long & downloadSize, unsigned long long & narSize); /* Sort a set of paths topologically under the references - relation. If p refers to q, then p preceeds q in this list. */ + relation. If p refers to q, then p precedes q in this list. */ Paths topoSortPaths(const PathSet & paths); /* Export multiple paths in the format expected by ‘nix-store From c1d18050b4cfed9eba68d4d21b397c6cce035e37 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Dec 2019 19:18:47 +0100 Subject: [PATCH 60/60] Disable recursive Nix test on macOS https://hydra.nixos.org/build/107724274 --- tests/recursive.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/recursive.sh b/tests/recursive.sh index 62896c015..394ae5ddb 100644 --- a/tests/recursive.sh +++ b/tests/recursive.sh @@ -1,5 +1,8 @@ source common.sh +# FIXME +if [[ $(uname) != Linux ]]; then exit; fi + clearStore export unreachable=$(nix add-to-store ./recursive.sh)