Add build-dir setting, clean up default TMPDIR handling

This is a squash of upstream PRs #10303, #10312 and #10883.

fix: Treat empty TMPDIR as unset

Fixes an instance of

    nix: src/libutil/util.cc:139: nix::Path nix::canonPath(PathView, bool): Assertion `path != ""' failed.

... which I've been getting in one of my shells for some reason.
I have yet to find out why TMPDIR was empty, but it's no reason for
Nix to break.

(cherry picked from commit c3fb2aa1f9d1fa756dac38d3588c836c5a5395dc)

fix: Treat empty XDG_RUNTIME_DIR as unset

See preceding commit. Not observed in the wild, but is sensible
and consistent with TMPDIR behavior.

(cherry picked from commit b9e7f5aa2df3f0e223f5c44b8089cbf9b81be691)

local-derivation-goal.cc: Reuse defaultTempDir()

(cherry picked from commit fd31945742710984de22805ee8d97fbd83c3f8eb)

fix: remove usage of XDG_RUNTIME_DIR for TMP

(cherry picked from commit 1363f51bcb24ab9948b7b5093490a009947f7453)

tests/functional: Add count()

(cherry picked from commit 6221770c9de4d28137206bdcd1a67eea12e1e499)

Remove uncalled for message

(cherry picked from commit b1fe388d33530f0157dcf9f461348b61eda13228)

Add build-dir setting

(cherry picked from commit 8b16cced18925aa612049d08d5e78eccbf0530e4)
Change-Id: Ic7b75ff0b6a3b19e50a4ac8ff2d70f15c683c16a

doc/manual/rl-next/build-dir.md

@@ -0,0 +1,12 @@
+---
+synopsis: "Add a `build-dir` setting to set the backing directory for builds"
+cls: 1514
+credits: [roberth, tomberek]
+category: Improvements
+---
+
+`build-dir` can now be set in the Nix configuration to choose the backing directory for the build sandbox.
+This can be useful on systems with `/tmp` on tmpfs, or simply to relocate large builds to another disk.
+
+Also, `XDG_RUNTIME_DIR` is no longer considered when selecting the default temporary directory,
+as it's not intended to be used for large amounts of data.

src/libstore/build/local-derivation-goal.cc

@@ -495,7 +495,7 @@ void LocalDerivationGoal::startBuilder()
 
     /* Create a temporary directory where the build will take
        place. */
-    tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
+    tmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700);
 
     chownToBuilder(tmpDir);
 
@@ -2107,8 +2107,8 @@ void LocalDerivationGoal::runChild()
         bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
 
         /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
-           to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
+           to find temporary directories, so we want to open up a broader place for them to put their files, if needed. */
-        Path globalTmpDir = canonPath(getEnvNonEmpty("TMPDIR").value_or("/tmp"), true);
+        Path globalTmpDir = canonPath(defaultTempDir(), true);
 
         /* They don't like trailing slashes on subpath directives */
         if (globalTmpDir.back() == '/') globalTmpDir.pop_back();

src/libstore/globals.cc

@@ -429,7 +429,7 @@ void initLibStore() {
     /* On macOS, don't use the per-session TMPDIR (as set e.g. by
        sshd). This breaks build users because they don't have access
        to the TMPDIR, in particular in ‘nix-store --serve’. */
-    if (getEnv("TMPDIR").value_or("/tmp").starts_with("/var/folders/"))
+    if (defaultTempDir().starts_with("/var/folders/"))
         unsetenv("TMPDIR");
 #endif
 

src/libstore/globals.hh

@@ -582,16 +582,36 @@ public:
     Setting<std::string> sandboxShmSize{
         this, "50%", "sandbox-dev-shm-size",
         R"(
-          This option determines the maximum size of the `tmpfs` filesystem
+            *Linux only*
-          mounted on `/dev/shm` in Linux sandboxes. For the format, see the
+
-          description of the `size` option of `tmpfs` in mount(8). The default
+            This option determines the maximum size of the `tmpfs` filesystem
-          is `50%`.
+            mounted on `/dev/shm` in Linux sandboxes. For the format, see the
+            description of the `size` option of `tmpfs` in mount(8). The default
+            is `50%`.
         )"};
 
     Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir",
-        "The build directory inside the sandbox."};
+        R"(
+            *Linux only*
+
+            The build directory inside the sandbox.
+
+            This directory is backed by [`build-dir`](#conf-build-dir) on the host.
+        )"};
 #endif
 
+    Setting<std::optional<Path>> buildDir{this, std::nullopt, "build-dir",
+        R"(
+            The directory on the host, in which derivations' temporary build directories are created.
+
+            If not set, Nix will use the system temporary directory indicated by the `TMPDIR` environment variable.
+            Note that builds are often performed by the Nix daemon, so its `TMPDIR` is used, and not that of the Nix command line interface.
+
+            This is also the location where [`--keep-failed`](@docroot@/command-ref/opt-common.md#opt-keep-failed) leaves its files.
+
+            If Nix runs without sandbox, or if the platform does not support sandboxing with bind mounts (e.g. macOS), then the [`builder`](@docroot@/language/derivations.md#attr-builder)'s environment will contain this directory, instead of the virtual location [`sandbox-build-dir`](#conf-sandbox-build-dir).
+        )"};
+
     Setting<PathSet> allowedImpureHostPrefixes{this, {}, "allowed-impure-host-deps",
         "Which prefixes to allow derivations to ask for access to (primarily for Darwin)."};
 

src/libutil/file-system.cc

@@ -511,10 +511,14 @@ void AutoDelete::reset(const Path & p, bool recursive) {
 
 //////////////////////////////////////////////////////////////////////
 
+std::string defaultTempDir() {
+    return getEnvNonEmpty("TMPDIR").value_or("/tmp");
+}
+
 static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
     std::atomic<unsigned int> & counter)
 {
-    tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
+    tmpRoot = canonPath(tmpRoot.empty() ? defaultTempDir() : tmpRoot, true);
     if (includePid)
         return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++);
     else
@@ -554,7 +558,7 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix,
 
 std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
 {
-    Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
+    Path tmpl(defaultTempDir() + "/" + prefix + ".XXXXXX");
     // Strictly speaking, this is UB, but who cares...
     // FIXME: use O_TMPFILE.
     AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));

src/libutil/file-system.hh

@@ -258,6 +258,11 @@ Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
  */
 std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix");
 
+/**
+ * Return `TMPDIR`, or the default temporary directory if unset or empty.
+ */
+Path defaultTempDir();
+
 /**
  * Used in various places.
  */

src/nix-build/nix-build.cc

@@ -414,8 +414,7 @@ static void main_nix_build(int argc, char * * argv)
         // Set the environment.
         auto env = getEnv();
 
-        auto tmp = getEnv("TMPDIR");
+        auto tmp = defaultTempDir();
-        if (!tmp) tmp = getEnv("XDG_RUNTIME_DIR").value_or("/tmp");
 
         if (pure) {
             decltype(env) newEnv;
@@ -427,7 +426,7 @@ static void main_nix_build(int argc, char * * argv)
             env["__ETC_PROFILE_SOURCED"] = "1";
         }
 
-        env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = *tmp;
+        env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmp;
         env["NIX_STORE"] = store->storeDir;
         env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores);
 

tests/functional/check.sh

@@ -34,6 +34,21 @@ nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \
 [ "$status" = "100" ]
 if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi
 
+test_custom_build_dir() {
+  local customBuildDir="$TEST_ROOT/custom-build-dir"
+
+  # Nix does not create the parent directories, and perhaps it shouldn't try to
+  # decide the permissions of build-dir.
+  mkdir "$customBuildDir"
+  nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \
+      --no-out-link --keep-failed --option build-dir "$TEST_ROOT/custom-build-dir" 2> $TEST_ROOT/log || status=$?
+  [ "$status" = "100" ]
+  [[ 1 == "$(count "$customBuildDir/nix-build-"*)" ]]
+  local buildDir="$customBuildDir/nix-build-"*
+  grep $checkBuildId $buildDir/checkBuildId
+}
+test_custom_build_dir
+
 nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \
     --no-out-link 2> $TEST_ROOT/log
 checkBuildTempDirRemoved $TEST_ROOT/log

tests/functional/common/vars-and-functions.sh.in

@@ -274,6 +274,11 @@ grepQuietInverse() {
     ! grep "$@" > /dev/null
 }
 
+# Return the number of arguments
+count() {
+  echo $#
+}
+
 trap onError ERR
 
 fi # COMMON_VARS_AND_FUNCTIONS_SH_SOURCED