diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index bbaabf93c..000000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1,3 +0,0 @@ -# These are supported funding model platforms - -custom: https://nixos.org/nixos/foundation.html diff --git a/README.md b/README.md index 3173c6c44..48cb1685c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos) + Nix, the purely functional package manager ------------------------------------------ diff --git a/doc/manual/advanced-topics/advanced-topics.xml b/doc/manual/advanced-topics/advanced-topics.xml index c304367aa..871b7eb1d 100644 --- a/doc/manual/advanced-topics/advanced-topics.xml +++ b/doc/manual/advanced-topics/advanced-topics.xml @@ -7,6 +7,8 @@ Advanced Topics + + diff --git a/doc/manual/advanced-topics/cores-vs-jobs.xml b/doc/manual/advanced-topics/cores-vs-jobs.xml new file mode 100644 index 000000000..eba645faf --- /dev/null +++ b/doc/manual/advanced-topics/cores-vs-jobs.xml @@ -0,0 +1,121 @@ + + +Tuning Cores and Jobs + +Nix has two relevant settings with regards to how your CPU cores +will be utilized: and +. This chapter will talk about what +they are, how they interact, and their configuration trade-offs. + + + + + + Dictates how many separate derivations will be built at the same + time. If you set this to zero, the local machine will do no + builds. Nix will still substitute from binary caches, and build + remotely if remote builders are configured. + + + + + + Suggests how many cores each derivation should use. Similar to + make -j. + + + + +The setting determines the value of +NIX_BUILD_CORES. NIX_BUILD_CORES is equal +to , unless +equals 0, in which case NIX_BUILD_CORES +will be the total number of cores in the system. + +The total number of consumed cores is a simple multiplication, + * NIX_BUILD_CORES. + +The balance on how to set these two independent variables depends +upon each builder's workload and hardware. Here are a few example +scenarios on a machine with 24 cores: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Balancing 24 Build Cores
NIX_BUILD_CORESMaximum ProcessesResult
1242424 + One derivation will be built at a time, each one can use 24 + cores. Undersold if a job can’t use 24 cores. +
46624 + Four derivations will be built at once, each given access to + six cores. +
126672 + 12 derivations will be built at once, each given access to six + cores. This configuration is over-sold. If all 12 derivations + being built simultaneously try to use all six cores, the + machine's performance will be degraded due to extensive context + switching between the 12 builds. +
241124 + 24 derivations can build at the same time, each using a single + core. Never oversold, but derivations which require many cores + will be very slow to compile. +
24024576 + 24 derivations can build at the same time, each using all the + available cores of the machine. Very likely to be oversold, + and very likely to suffer context switches. +
+ +It is up to the derivations' build script to respect +host's requested cores-per-build by following the value of the +NIX_BUILD_CORES environment variable. + + diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml new file mode 100644 index 000000000..3dc43ee79 --- /dev/null +++ b/doc/manual/advanced-topics/post-build-hook.xml @@ -0,0 +1,160 @@ + + +Using the <xref linkend="conf-post-build-hook" /> +Uploading to an S3-compatible binary cache after each build + + +
+ Implementation Caveats + Here we use the post-build hook to upload to a binary cache. + This is a simple and working example, but it is not suitable for all + use cases. + + The post build hook program runs after each executed build, + and blocks the build loop. The build loop exits if the hook program + fails. + + Concretely, this implementation will make Nix slow or unusable + when the internet is slow or unreliable. + + A more advanced implementation might pass the store paths to a + user-supplied daemon or queue for processing the store paths outside + of the build loop. +
+ +
+ Prerequisites + + + This tutorial assumes you have configured an S3-compatible binary cache + according to the instructions at + , and + that the root user's default AWS profile can + upload to the bucket. + +
+ +
+ Set up a Signing Key + Use nix-store --generate-binary-cache-key to + create our public and private signing keys. We will sign paths + with the private key, and distribute the public key for verifying + the authenticity of the paths. + + +# nix-store --generate-binary-cache-key example-nix-cache-1 /etc/nix/key.private /etc/nix/key.public +# cat /etc/nix/key.public +example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= + + +Then, add the public key and the cache URL to your +nix.conf's +and like: + + +substituters = https://cache.nixos.org/ s3://example-nix-cache +trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= + + +we will restart the Nix daemon a later step. +
+ +
+ Implementing the build hook + Write the following script to + /etc/nix/upload-to-cache.sh: + + + +#!/bin/sh + +set -eu +set -f # disable globbing +export IFS=' ' + +echo "Signing paths" $OUT_PATHS +nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS +echo "Uploading paths" $OUT_PATHS +exec nix copy --to 's3://example-nix-cache' $OUT_PATHS + + + + Should <literal>$OUT_PATHS</literal> be quoted? + + The $OUT_PATHS variable is a space-separated + list of Nix store paths. In this case, we expect and want the + shell to perform word splitting to make each output path its + own argument to nix sign-paths. Nix guarantees + the paths will not contain any spaces, however a store path + might contain glob characters. The set -f + disables globbing in the shell. + + + + Then make sure the hook program is executable by the root user: + +# chmod +x /etc/nix/upload-to-cache.sh + +
+ +
+ Updating Nix Configuration + + Edit /etc/nix/nix.conf to run our hook, + by adding the following configuration snippet at the end: + + +post-build-hook = /etc/nix/upload-to-cache.sh + + +Then, restart the nix-daemon. +
+ +
+ Testing + + Build any derivation, for example: + + +$ nix-build -E '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)' +these derivations will be built: + /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv +building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'... +running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... +post-build-hook: Signing paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +post-build-hook: Uploading paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +/nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example + + + Then delete the path from the store, and try substituting it from the binary cache: + +$ rm ./result +$ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example + + +Now, copy the path back from the cache: + +$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'... +warning: you did not specify '--add-root'; the result might be removed by the garbage collector +/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example + +
+
+ Conclusion + + We now have a Nix installation configured to automatically sign and + upload every local build to a remote binary cache. + + + + Before deploying this to production, be sure to consider the + implementation caveats in . + +
+ diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 09aad2e05..c7b540892 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -238,8 +238,9 @@ false. linkend='opt-cores'>--cores command line switch and defaults to 1. The value 0 means that the builder should use all available CPU cores in the - system. + system. + See also . diff-hook @@ -482,8 +483,10 @@ builtins.fetchurl { max-free - This option defines after how many free bytes to stop collecting - garbage once the min-free condition gets triggered. + When a garbage collection is triggered by the + min-free option, it stops as soon as + max-free bytes are available. The default is + infinity (i.e. delete all garbage). @@ -498,7 +501,10 @@ builtins.fetchurl { regardless). It can be overridden using the () - command line switch. + command line switch. + + See also . + max-silent-time @@ -524,9 +530,11 @@ builtins.fetchurl { min-free - When the disk reaches min-free bytes of free disk space during a build, nix - will start to garbage-collection until max-free bytes are available on the disk. - A value of 0 (the default) means that this feature is disabled. + When free disk space in /nix/store + drops below min-free during a build, Nix + performs a garbage-collection until max-free + bytes are available or there is no more garbage. A value of + 0 (the default) disables this feature. @@ -656,6 +664,62 @@ password my-password + + post-build-hook + + Optional. The path to a program to execute after each build. + + This option is only settable in the global + nix.conf, or on the command line by trusted + users. + + When using the nix-daemon, the daemon executes the hook as + root. If the nix-daemon is not involved, the + hook runs as the user executing the nix-build. + + + The hook executes after an evaluation-time build. + The hook does not execute on substituted paths. + The hook's output always goes to the user's terminal. + If the hook fails, the build succeeds but no further builds execute. + The hook executes synchronously, and blocks other builds from progressing while it runs. + + + The program executes with no arguments. The program's environment + contains the following environment variables: + + + + DRV_PATH + + The derivation for the built paths. + Example: + /nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv + + + + + + OUT_PATHS + + Output paths of the built derivation, separated by a space character. + Example: + /nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev + /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc + /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info + /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man + /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23. + + + + + + See for an example + implementation. + + + + repeat How many times to repeat builds to check whether diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index fc999d336..35926f3da 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -12,7 +12,7 @@ if ! [ -e "$self/.reginfo" ]; then echo "$0: incomplete installer (.reginfo is missing)" >&2 fi -if [ -z "$USER" ]; then +if [ -z "$USER" ] && ! USER=$(id -u -n); then echo "$0: \$USER is not set" >&2 exit 1 fi diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 10ce1abf5..32ce00bbc 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -832,8 +832,14 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, { PathSet context; Path path = state.coerceToPath(pos, *args[0], context); - if (!context.empty()) - throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format( + "cannot check the existence of '%1%', since path '%2%' is not valid, at %3%") + % path % e.path % pos); + } + try { mkBool(v, pathExists(state.checkSourcePath(path))); } catch (SysError & e) { diff --git a/src/libstore/build.cc b/src/libstore/build.cc index dabd447f5..db7300c58 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1629,6 +1629,61 @@ void DerivationGoal::buildDone() being valid. */ registerOutputs(); + if (settings.postBuildHook != "") { + Activity act(*logger, lvlInfo, actPostBuildHook, + fmt("running post-build-hook '%s'", settings.postBuildHook), + Logger::Fields{drvPath}); + PushActivity pact(act.id); + auto outputPaths = drv->outputPaths(); + std::map hookEnvironment = getEnv(); + + hookEnvironment.emplace("DRV_PATH", drvPath); + hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", outputPaths))); + + RunOptions opts(settings.postBuildHook, {}); + opts.environment = hookEnvironment; + + struct LogSink : Sink { + Activity & act; + std::string currentLine; + + LogSink(Activity & act) : act(act) { } + + void operator() (const unsigned char * data, size_t len) override { + for (size_t i = 0; i < len; i++) { + auto c = data[i]; + + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() { + if (settings.verboseBuild) { + printError("post-build-hook: " + currentLine); + } else { + act.result(resPostBuildLogLine, currentLine); + } + currentLine.clear(); + } + + ~LogSink() { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } + } + }; + LogSink sink(act); + + opts.standardOut = &sink; + opts.mergeStderrToStdout = true; + runProgram2(opts); + } + if (buildMode == bmCheck) { done(BuildResult::Built); return; @@ -2734,7 +2789,13 @@ void DerivationGoal::runChild() on. */ if (fixedOutput) { ss.push_back("/etc/resolv.conf"); - ss.push_back("/etc/nsswitch.conf"); + + // Only use nss functions to resolve hosts and + // services. Don’t use it for anything else that may + // be configured for this system. This limits the + // potential impurities introduced in fixed outputs. + writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + ss.push_back("/etc/services"); ss.push_back("/etc/hosts"); if (pathExists("/var/run/nscd/socket")) @@ -3978,17 +4039,6 @@ void SubstitutionGoal::tryToRun() return; } - /* If the store path is already locked (probably by a - DerivationGoal), then put this goal to sleep. Note: we don't - acquire a lock here since that breaks addToStore(), so below we - handle an AlreadyLocked exception from addToStore(). The check - here is just an optimisation to prevent having to redo a - download due to a locked path. */ - if (pathIsLockedByMe(worker.store.toRealPath(storePath))) { - worker.waitForAWhile(shared_from_this()); - return; - } - maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); worker.updateProgress(); @@ -4028,12 +4078,6 @@ void SubstitutionGoal::finished() try { promise.get_future().get(); - } catch (AlreadyLocked & e) { - /* Probably a DerivationGoal is already building this store - path. Sleep for a while and try again. */ - state = &SubstitutionGoal::init; - worker.waitForAWhile(shared_from_this()); - return; } catch (std::exception & e) { printError(e.what()); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 26e2b0dca..366dbfb0a 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -29,7 +29,7 @@ static string gcRootsDir = "gcroots"; read. To be precise: when they try to create a new temporary root file, they will block until the garbage collector has finished / yielded the GC lock. */ -int LocalStore::openGCLock(LockType lockType) +AutoCloseFD LocalStore::openGCLock(LockType lockType) { Path fnGCLock = (format("%1%/%2%") % stateDir % gcLockName).str(); @@ -49,7 +49,7 @@ int LocalStore::openGCLock(LockType lockType) process that can open the file for reading can DoS the collector. */ - return fdGCLock.release(); + return fdGCLock; } @@ -221,26 +221,22 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) //FDPtr fd(new AutoCloseFD(openLockFile(path, false))); //if (*fd == -1) continue; - if (path != fnTempRoots) { - - /* Try to acquire a write lock without blocking. This can - only succeed if the owning process has died. In that case - we don't care about its temporary roots. */ - if (lockFile(fd->get(), ltWrite, false)) { - printError(format("removing stale temporary roots file '%1%'") % path); - unlink(path.c_str()); - writeFull(fd->get(), "d"); - continue; - } - - /* Acquire a read lock. This will prevent the owning process - from upgrading to a write lock, therefore it will block in - addTempRoot(). */ - debug(format("waiting for read lock on '%1%'") % path); - lockFile(fd->get(), ltRead, true); - + /* Try to acquire a write lock without blocking. This can + only succeed if the owning process has died. In that case + we don't care about its temporary roots. */ + if (lockFile(fd->get(), ltWrite, false)) { + printError(format("removing stale temporary roots file '%1%'") % path); + unlink(path.c_str()); + writeFull(fd->get(), "d"); + continue; } + /* Acquire a read lock. This will prevent the owning process + from upgrading to a write lock, therefore it will block in + addTempRoot(). */ + debug(format("waiting for read lock on '%1%'") % path); + lockFile(fd->get(), ltRead, true); + /* Read the entire file. */ string contents = readFile(fd->get()); @@ -444,17 +440,22 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) } #if !defined(__linux__) - try { - std::regex lsofRegex(R"(^n(/.*)$)"); - auto lsofLines = - tokenizeString>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); - for (const auto & line : lsofLines) { - std::smatch match; - if (std::regex_match(line, match, lsofRegex)) - unchecked[match[1]].emplace("{lsof}"); + // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail. + // See: https://github.com/NixOS/nix/issues/3011 + // Because of this we disable lsof when running the tests. + if (getEnv("_NIX_TEST_NO_LSOF") == "") { + try { + std::regex lsofRegex(R"(^n(/.*)$)"); + auto lsofLines = + tokenizeString>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); + for (const auto & line : lsofLines) { + std::smatch match; + if (std::regex_match(line, match, lsofRegex)) + unchecked[match[1]].emplace("{lsof}"); + } + } catch (ExecError & e) { + /* lsof not installed, lsof failed */ } - } catch (ExecError & e) { - /* lsof not installed, lsof failed */ } #endif @@ -866,7 +867,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) void LocalStore::autoGC(bool sync) { - auto getAvail = [this]() { + static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE", ""); + + auto getAvail = [this]() -> uint64_t { + if (!fakeFreeSpaceFile.empty()) + return std::stoll(readFile(fakeFreeSpaceFile)); + struct statvfs st; if (statvfs(realStoreDir.c_str(), &st)) throw SysError("getting filesystem info about '%s'", realStoreDir); @@ -887,7 +893,7 @@ void LocalStore::autoGC(bool sync) auto now = std::chrono::steady_clock::now(); - if (now < state->lastGCCheck + std::chrono::seconds(5)) return; + if (now < state->lastGCCheck + std::chrono::seconds(settings.minFreeCheckInterval)) return; auto avail = getAvail(); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 2aecebe3d..e845c29b0 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -315,6 +315,9 @@ public: "pre-build-hook", "A program to run just before a build to set derivation-specific build settings."}; + Setting postBuildHook{this, "", "post-build-hook", + "A program to run just after each succesful build."}; + Setting netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", "Path to the netrc file used to obtain usernames/passwords for downloads."}; @@ -342,6 +345,9 @@ public: Setting maxFree{this, std::numeric_limits::max(), "max-free", "Stop deleting garbage when free disk space is above the specified amount."}; + Setting minFreeCheckInterval{this, 5, "min-free-check-interval", + "Number of seconds between checking free disk space."}; + Setting pluginFiles{this, {}, "plugin-files", "Plugins to dynamically load at nix initialization time."}; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f39c73b23..a9399130c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1203,7 +1203,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) bool errors = false; - /* Acquire the global GC lock to prevent a garbage collection. */ + /* Acquire the global GC lock to get a consistent snapshot of + existing and valid paths. */ AutoCloseFD fdGCLock = openGCLock(ltWrite); PathSet store; @@ -1214,13 +1215,11 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) PathSet validPaths2 = queryAllValidPaths(), validPaths, done; + fdGCLock = -1; + for (auto & i : validPaths2) verifyPath(i, store, done, validPaths, repair, errors); - /* Release the GC lock so that checking content hashes (which can - take ages) doesn't block the GC or builds. */ - fdGCLock = -1; - /* Optionally, check the content hashes (slow). */ if (checkContents) { printInfo("checking hashes..."); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 6b655647b..af8b84bf5 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -263,7 +263,7 @@ private: bool isActiveTempFile(const GCState & state, const Path & path, const string & suffix); - int openGCLock(LockType lockType); + AutoCloseFD openGCLock(LockType lockType); void findRoots(const Path & path, unsigned char type, Roots & roots); diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 08d1efdbe..2635e3940 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -5,9 +5,10 @@ #include #include +#include #include #include -#include +#include namespace nix { @@ -40,17 +41,14 @@ void deleteLockFile(const Path & path, int fd) bool lockFile(int fd, LockType lockType, bool wait) { - struct flock lock; - if (lockType == ltRead) lock.l_type = F_RDLCK; - else if (lockType == ltWrite) lock.l_type = F_WRLCK; - else if (lockType == ltNone) lock.l_type = F_UNLCK; + int type; + if (lockType == ltRead) type = LOCK_SH; + else if (lockType == ltWrite) type = LOCK_EX; + else if (lockType == ltNone) type = LOCK_UN; else abort(); - lock.l_whence = SEEK_SET; - lock.l_start = 0; - lock.l_len = 0; /* entire file */ if (wait) { - while (fcntl(fd, F_SETLKW, &lock) != 0) { + while (flock(fd, type) != 0) { checkInterrupt(); if (errno != EINTR) throw SysError(format("acquiring/releasing lock")); @@ -58,9 +56,9 @@ bool lockFile(int fd, LockType lockType, bool wait) return false; } } else { - while (fcntl(fd, F_SETLK, &lock) != 0) { + while (flock(fd, type | LOCK_NB) != 0) { checkInterrupt(); - if (errno == EACCES || errno == EAGAIN) return false; + if (errno == EWOULDBLOCK) return false; if (errno != EINTR) throw SysError(format("acquiring/releasing lock")); } @@ -70,14 +68,6 @@ bool lockFile(int fd, LockType lockType, bool wait) } -/* This enables us to check whether are not already holding a lock on - a file ourselves. POSIX locks (fcntl) suck in this respect: if we - close a descriptor, the previous lock will be closed as well. And - there is no way to query whether we already have a lock (F_GETLK - only works on locks held by other processes). */ -static Sync lockedPaths_; - - PathLocks::PathLocks() : deletePaths(false) { @@ -91,7 +81,7 @@ PathLocks::PathLocks(const PathSet & paths, const string & waitMsg) } -bool PathLocks::lockPaths(const PathSet & _paths, +bool PathLocks::lockPaths(const PathSet & paths, const string & waitMsg, bool wait) { assert(fds.empty()); @@ -99,75 +89,54 @@ bool PathLocks::lockPaths(const PathSet & _paths, /* Note that `fds' is built incrementally so that the destructor will only release those locks that we have already acquired. */ - /* Sort the paths. This assures that locks are always acquired in - the same order, thus preventing deadlocks. */ - Paths paths(_paths.begin(), _paths.end()); - paths.sort(); - - /* Acquire the lock for each path. */ + /* Acquire the lock for each path in sorted order. This ensures + that locks are always acquired in the same order, thus + preventing deadlocks. */ for (auto & path : paths) { checkInterrupt(); Path lockPath = path + ".lock"; debug(format("locking path '%1%'") % path); - { - auto lockedPaths(lockedPaths_.lock()); - if (lockedPaths->count(lockPath)) { - if (!wait) return false; - throw AlreadyLocked("deadlock: trying to re-acquire self-held lock '%s'", lockPath); - } - lockedPaths->insert(lockPath); - } + AutoCloseFD fd; - try { + while (1) { - AutoCloseFD fd; + /* Open/create the lock file. */ + fd = openLockFile(lockPath, true); - while (1) { - - /* Open/create the lock file. */ - fd = openLockFile(lockPath, true); - - /* Acquire an exclusive lock. */ - if (!lockFile(fd.get(), ltWrite, false)) { - if (wait) { - if (waitMsg != "") printError(waitMsg); - lockFile(fd.get(), ltWrite, true); - } else { - /* Failed to lock this path; release all other - locks. */ - unlock(); - lockedPaths_.lock()->erase(lockPath); - return false; - } + /* Acquire an exclusive lock. */ + if (!lockFile(fd.get(), ltWrite, false)) { + if (wait) { + if (waitMsg != "") printError(waitMsg); + lockFile(fd.get(), ltWrite, true); + } else { + /* Failed to lock this path; release all other + locks. */ + unlock(); + return false; } - - debug(format("lock acquired on '%1%'") % lockPath); - - /* Check that the lock file hasn't become stale (i.e., - hasn't been unlinked). */ - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError(format("statting lock file '%1%'") % lockPath); - if (st.st_size != 0) - /* This lock file has been unlinked, so we're holding - a lock on a deleted file. This means that other - processes may create and acquire a lock on - `lockPath', and proceed. So we must retry. */ - debug(format("open lock file '%1%' has become stale") % lockPath); - else - break; } - /* Use borrow so that the descriptor isn't closed. */ - fds.push_back(FDPair(fd.release(), lockPath)); + debug(format("lock acquired on '%1%'") % lockPath); - } catch (...) { - lockedPaths_.lock()->erase(lockPath); - throw; + /* Check that the lock file hasn't become stale (i.e., + hasn't been unlinked). */ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError(format("statting lock file '%1%'") % lockPath); + if (st.st_size != 0) + /* This lock file has been unlinked, so we're holding + a lock on a deleted file. This means that other + processes may create and acquire a lock on + `lockPath', and proceed. So we must retry. */ + debug(format("open lock file '%1%' has become stale") % lockPath); + else + break; } + /* Use borrow so that the descriptor isn't closed. */ + fds.push_back(FDPair(fd.release(), lockPath)); } return true; @@ -189,8 +158,6 @@ void PathLocks::unlock() for (auto & i : fds) { if (deletePaths) deleteLockFile(i.second, i.first); - lockedPaths_.lock()->erase(i.second); - if (close(i.first) == -1) printError( format("error (ignored): cannot close lock file on '%1%'") % i.second); @@ -208,11 +175,4 @@ void PathLocks::setDeletion(bool deletePaths) } -bool pathIsLockedByMe(const Path & path) -{ - Path lockPath = path + ".lock"; - return lockedPaths_.lock()->count(lockPath); -} - - } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index db51f950a..411da0222 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -16,8 +16,6 @@ enum LockType { ltRead, ltWrite, ltNone }; bool lockFile(int fd, LockType lockType, bool wait); -MakeError(AlreadyLocked, Error); - class PathLocks { private: @@ -37,6 +35,4 @@ public: void setDeletion(bool deletePaths); }; -bool pathIsLockedByMe(const Path & path); - } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 5f2219445..5df03da74 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -26,6 +26,7 @@ typedef enum { actVerifyPaths = 107, actSubstitute = 108, actQueryPathInfo = 109, + actPostBuildHook = 110, } ActivityType; typedef enum { @@ -36,6 +37,7 @@ typedef enum { resSetPhase = 104, resProgress = 105, resSetExpected = 106, + resPostBuildLogLine = 107, } ResultType; typedef uint64_t ActivityId; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 92c8957ff..e353290d0 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -85,6 +85,15 @@ void clearEnv() unsetenv(name.first.c_str()); } +void replaceEnv(std::map newEnv) +{ + clearEnv(); + for (auto newEnvVar : newEnv) + { + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); + } +} + Path absPath(Path path, Path dir) { @@ -1044,10 +1053,22 @@ void runProgram2(const RunOptions & options) if (options.standardOut) out.create(); if (source) in.create(); + ProcessOptions processOptions; + // vfork implies that the environment of the main process and the fork will + // be shared (technically this is undefined, but in practice that's the + // case), so we can't use it if we alter the environment + if (options.environment) + processOptions.allowVfork = false; + /* Fork. */ Pid pid = startProcess([&]() { + if (options.environment) + replaceEnv(*options.environment); if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("dupping stdout"); + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) throw SysError("dupping stdin"); @@ -1074,7 +1095,7 @@ void runProgram2(const RunOptions & options) execv(options.program.c_str(), stringsToCharPtrs(args_).data()); throw SysError("executing '%1%'", options.program); - }); + }, processOptions); out.writeSide = -1; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index e05ef1e7d..1f3d1e2ae 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -276,12 +276,14 @@ struct RunOptions std::optional uid; std::optional gid; std::optional chdir; + std::optional> environment; Path program; bool searchPath = true; Strings args; std::optional input; Source * standardIn = nullptr; Sink * standardOut = nullptr; + bool mergeStderrToStdout = false; bool _killStderr = false; RunOptions(const Path & program, const Strings & args) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 56ed75dae..87b2e43f0 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -860,7 +860,10 @@ static void queryJSON(Globals & globals, vector & elems) for (auto & i : elems) { JSONObject pkgObj = topObj.object(i.attrPath); - pkgObj.attr("name", i.queryName()); + auto drvName = DrvName(i.queryName()); + pkgObj.attr("name", drvName.fullName); + pkgObj.attr("pname", drvName.name); + pkgObj.attr("version", drvName.version); pkgObj.attr("system", i.querySystem()); JSONObject metaObj = pkgObj.object("meta"); @@ -1026,10 +1029,14 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) else if (printAttrPath) columns.push_back(i.attrPath); - if (xmlOutput) - attrs["name"] = i.queryName(); - else if (printName) + if (xmlOutput) { + auto drvName = DrvName(i.queryName()); + attrs["name"] = drvName.fullName; + attrs["pname"] = drvName.name; + attrs["version"] = drvName.version; + } else if (printName) { columns.push_back(i.queryName()); + } if (compareVersions) { /* Compare this element against the versions of the diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index b1c1d87de..c0bcfb0c9 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -170,6 +170,14 @@ public: name, sub); } + if (type == actPostBuildHook) { + auto name = storePathToName(getS(fields, 0)); + if (hasSuffix(name, ".drv")) + name.resize(name.size() - 4); + i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name); + i->name = DrvName(name).name; + } + if (type == actQueryPathInfo) { auto name = storePathToName(getS(fields, 0)); i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1)); @@ -228,14 +236,18 @@ public: update(*state); } - else if (type == resBuildLogLine) { + else if (type == resBuildLogLine || type == resPostBuildLogLine) { auto lastLine = trim(getS(fields, 0)); if (!lastLine.empty()) { auto i = state->its.find(act); assert(i != state->its.end()); ActInfo info = *i->second; if (printBuildLogs) { - log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + "> " + ANSI_NORMAL + lastLine); + auto suffix = "> "; + if (type == resPostBuildLogLine) { + suffix = " (post)> "; + } + log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine); } else { state->activities.erase(i->second); info.lastLine = lastLine; diff --git a/tests/common.sh.in b/tests/common.sh.in index 6a523ca9d..15d7b1ef9 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -16,6 +16,7 @@ if [[ -n $NIX_STORE ]]; then export _NIX_TEST_NO_SANDBOX=1 fi export _NIX_IN_TEST=$TEST_ROOT/shared +export _NIX_TEST_NO_LSOF=1 export NIX_REMOTE=$NIX_REMOTE_ unset NIX_PATH export TEST_HOME=$TEST_ROOT/test-home diff --git a/tests/dependencies.nix b/tests/dependencies.nix index 687237add..eca4b2964 100644 --- a/tests/dependencies.nix +++ b/tests/dependencies.nix @@ -17,6 +17,7 @@ let { builder = ./dependencies.builder0.sh + "/FOOBAR/../."; input1 = input1 + "/."; input2 = "${input2}/."; + input1_drv = input1; meta.description = "Random test package"; }; diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh new file mode 100644 index 000000000..1e91282d0 --- /dev/null +++ b/tests/gc-auto.sh @@ -0,0 +1,59 @@ +source common.sh + +clearStore + +garbage1=$(nix add-to-store --name garbage1 ./tarball.sh) +garbage2=$(nix add-to-store --name garbage2 ./tarball.sh) +garbage3=$(nix add-to-store --name garbage3 ./tarball.sh) + +fake_free=$TEST_ROOT/fake-free +export _NIX_TEST_FREE_SPACE_FILE=$fake_free +echo 1100 > $fake_free + +expr=$(cat < \$out/bar + echo 1... + sleep 2 + echo 100 > $fake_free + echo 2... + sleep 2 + echo 3... + [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]] + ''; +} +EOF +) + +nix build -o $TEST_ROOT/result-A -L "($expr)" \ + --min-free 1000 --max-free 2000 --min-free-check-interval 1 & +pid=$! + +expr2=$(cat < \$out/bar + echo 1... + sleep 2 + echo 100 > $fake_free + echo 2... + sleep 2 + echo 3... + ''; +} +EOF +) + +nix build -o $TEST_ROOT/result-B -L "($expr2)" \ + --min-free 1000 --max-free 2000 --min-free-check-interval 1 + +wait "$pid" + +[[ foo = $(cat $TEST_ROOT/result-A/bar) ]] +[[ foo = $(cat $TEST_ROOT/result-B/bar) ]] diff --git a/tests/import-derivation.nix b/tests/import-derivation.nix index 91adcd288..44fa9a45d 100644 --- a/tests/import-derivation.nix +++ b/tests/import-derivation.nix @@ -10,7 +10,10 @@ let ''; }; - value = import bar; + value = + # Test that pathExists can check the existence of /nix/store paths + assert builtins.pathExists bar; + import bar; in diff --git a/tests/install-darwin.sh b/tests/install-darwin.sh index c99ce84ac..9933eba94 100755 --- a/tests/install-darwin.sh +++ b/tests/install-darwin.sh @@ -34,7 +34,7 @@ cleanup() { sudo rm -rf /etc/nix \ /nix \ /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels \ - "$USER/.nix-profile" "$USER/.nix-defexpr" "$USER/.nix-channels" + "$HOME/.nix-profile" "$HOME/.nix-defexpr" "$HOME/.nix-channels" } verify() { diff --git a/tests/local.mk b/tests/local.mk index a76d720d0..f4ef981fd 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -3,7 +3,9 @@ check: nix_tests = \ init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ - gc.sh gc-concurrent.sh \ + gc.sh \ + gc-concurrent.sh \ + gc-auto.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh check-refs.sh filter-source.sh \ remote-store.sh export.sh export-graph.sh \ @@ -27,6 +29,7 @@ nix_tests = \ plugins.sh \ search.sh \ nix-copy-ssh.sh \ + post-hook.sh \ flakes.sh # parallel.sh diff --git a/tests/post-hook.sh b/tests/post-hook.sh new file mode 100644 index 000000000..a02657215 --- /dev/null +++ b/tests/post-hook.sh @@ -0,0 +1,15 @@ +source common.sh + +clearStore + +export REMOTE_STORE=$TEST_ROOT/remote_store + +# Build the dependencies and push them to the remote store +nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh + +clearStore + +# Ensure that we the remote store contains both the runtime and buildtime +# closure of what we've just built +nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix +nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix input1_drv diff --git a/tests/push-to-store.sh b/tests/push-to-store.sh new file mode 100755 index 000000000..d97eb095d --- /dev/null +++ b/tests/push-to-store.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +echo Pushing "$@" to "$REMOTE_STORE" +echo -n "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs