Merge remote-tracking branch 'origin/master' into flakes

doc/manual/advanced-topics/distributed-builds.xml

@@ -184,4 +184,7 @@ to be included. (This is the default.)</para>
 the option <link linkend='conf-builders-use-substitutes'><literal>builders-use-substitutes</literal></link>
 in your local <filename>nix.conf</filename>.</para>
 
+<para>To build only on remote builders and disable building on the local machine,
+you can use the option <option>--max-jobs 0</option>.</para>
+
 </chapter>

doc/manual/command-ref/opt-common.xml

@@ -107,14 +107,22 @@
 <varlistentry xml:id="opt-max-jobs"><term><option>--max-jobs</option> / <option>-j</option>
 <replaceable>number</replaceable></term>
 
-  <listitem><para>Sets the maximum number of build jobs that Nix will
+  <listitem>
+
+  <para>Sets the maximum number of build jobs that Nix will
   perform in parallel to the specified number.  Specify
   <literal>auto</literal> to use the number of CPUs in the system.
   The default is specified by the <link
   linkend='conf-max-jobs'><literal>max-jobs</literal></link>
   configuration setting, which itself defaults to
   <literal>1</literal>.  A higher value is useful on SMP systems or to
-  exploit I/O latency.</para></listitem>
+  exploit I/O latency.</para>
+
+  <para> Setting it to <literal>0</literal> disallows building on the local
+  machine, which is useful when you want builds to happen only on remote
+  builders.</para>
+
+  </listitem>
 
 </varlistentry>
 

doc/manual/expressions/builtins.xml

@@ -705,6 +705,19 @@ builtins.genList (x: x * x) 5
   </varlistentry>
 
 
+  <varlistentry xml:id='builtin-hashFile'>
+    <term><function>builtins.hashFile</function>
+    <replaceable>type</replaceable> <replaceable>p</replaceable></term>
+
+    <listitem><para>Return a base-16 representation of the
+    cryptographic hash of the file at path <replaceable>p</replaceable>.  The
+    hash algorithm specified by <replaceable>type</replaceable> must
+    be one of <literal>"md5"</literal>, <literal>"sha1"</literal>,
+    <literal>"sha256"</literal> or <literal>"sha512"</literal>.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry xml:id='builtin-head'>
     <term><function>builtins.head</function>
     <replaceable>list</replaceable></term>

doc/manual/packages/basic-package-mgmt.xml

@@ -24,11 +24,11 @@ symlinks to the files of the active applications.  </para>
 <para>Components are installed from a set of <emphasis>Nix
 expressions</emphasis> that tell Nix how to build those packages,
 including, if necessary, their dependencies.  There is a collection of
-Nix expressions called the Nix Package collection that contains
+Nix expressions called the Nixpkgs package collection that contains
 packages ranging from basic development stuff such as GCC and Glibc,
 to end-user applications like Mozilla Firefox.  (Nix is however not
-tied to the Nix Package collection; you could write your own Nix
+tied to the Nixpkgs package collection; you could write your own Nix
-expressions based on it, or completely new ones.)</para>
+expressions based on Nixpkgs, or completely new ones.)</para>
 
 <para>You can manually download the latest version of Nixpkgs from
 <link xlink:href='http://nixos.org/nixpkgs/download.html'/>. However,

maintainers/upload-release.pl

@@ -67,10 +67,10 @@ sub downloadFile {
     }
 
     my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die;
-    my $sha256_actual = `nix hash-file --type sha256 '$dstFile'`;
+    my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`;
     chomp $sha256_actual;
     if ($sha256_expected ne $sha256_actual) {
-        print STDERR "file $dstFile is corrupt\n";
+        print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
         exit 1;
     }
 

scripts/install-multi-user.sh

@@ -240,10 +240,16 @@ EOF
 }
 trap finish_fail EXIT
 
+channel_update_failed=0
 function finish_success {
     finish_cleanup
 
     ok "Alright! We're done!"
+    if [ "x$channel_update_failed" = x1 ]; then
+        echo ""
+        echo "But fetching the nixpkgs channel failed. (Are you offline?)"
+        echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"."
+    fi
     cat <<EOF
 
 Before Nix will work in your existing shells, you'll need to close
@@ -734,7 +740,9 @@ setup_default_profile() {
     # otherwise it will be lost in environments where sudo doesn't pass
     # all the environment variables by default.
     _sudo "to update the default channel in the default profile" \
-          HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs
+          HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \
+          || channel_update_failed=1
+
 }
 
 

scripts/install-nix-from-closure.sh

@@ -22,10 +22,12 @@ if [ -z "$HOME" ]; then
     exit 1
 fi
 
-# macOS support for 10.10 or higher
+# macOS support for 10.12.6 or higher
 if [ "$(uname -s)" = "Darwin" ]; then
-    if [ $(($(sw_vers -productVersion | cut -d '.' -f 2))) -lt 10 ]; then
+    macos_major=$(sw_vers -productVersion | cut -d '.' -f 2)
-        echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.10 or higher"
+    macos_minor=$(sw_vers -productVersion | cut -d '.' -f 3)
+    if [ "$macos_major" -lt 12 ] || ([ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]); then
+        echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher"
         exit 1
     fi
 fi
@@ -132,7 +134,10 @@ if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then
     $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
 fi
 if [ -z "$_NIX_INSTALLER_TEST" ]; then
-    $nix/bin/nix-channel --update nixpkgs
+    if ! $nix/bin/nix-channel --update nixpkgs; then
+        echo "Fetching the nixpkgs channel failed. (Are you offline?)"
+        echo "To try again later, run \"nix-channel --update nixpkgs\"."
+    fi
 fi
 
 added=

src/libexpr/eval.cc

@@ -1820,6 +1820,7 @@ void EvalState::printStats()
             gc.attr("totalBytes", totalBytes);
         }
 #endif
+
         if (countCalls) {
             {
                 auto obj = topObj.object("primops");
@@ -1855,6 +1856,11 @@ void EvalState::printStats()
                 }
             }
         }
+
+        if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") {
+            auto list = topObj.list("symbols");
+            symbols.dump([&](const std::string & s) { list.elem(s); });
+        }
     }
 }
 

src/libexpr/primops.cc

@@ -923,6 +923,20 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
     mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str());
 }
 
+/* Return the cryptographic hash of a file in base-16. */
+static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    string type = state.forceStringNoCtx(*args[0], pos);
+    HashType ht = parseHashType(type);
+    if (ht == htUnknown)
+      throw Error(format("unknown hash type '%1%', at %2%") % type % pos);
+
+    PathSet context; // discarded
+    Path p = state.coerceToPath(pos, *args[1], context);
+
+    mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context);
+}
+
 /* Read a directory (without . or ..) */
 static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
@@ -2202,6 +2216,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__readFile", 1, prim_readFile);
     addPrimOp("__readDir", 1, prim_readDir);
     addPrimOp("__findFile", 2, prim_findFile);
+    addPrimOp("__hashFile", 2, prim_hashFile);
 
     // Creating files
     addPrimOp("__toXML", 1, prim_toXML);

src/libexpr/symbol-table.hh

@@ -75,6 +75,13 @@ public:
     }
 
     size_t totalSize() const;
+
+    template<typename T>
+    void dump(T callback)
+    {
+        for (auto & s : symbols)
+            callback(s);
+    }
 };
 
 }

src/libstore/build.cc

@@ -803,6 +803,9 @@ private:
     /* Whether we're currently doing a chroot build. */
     bool useChroot = false;
 
+    /* Whether we need to perform hash rewriting if there are valid output paths. */
+    bool needsHashRewrite;
+
     Path chrootRootDir;
 
     /* RAII object to delete the chroot directory. */
@@ -994,6 +997,13 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut
     , wantedOutputs(wantedOutputs)
     , buildMode(buildMode)
 {
+#if __linux__
+    needsHashRewrite = !useChroot;
+#else
+    /* Darwin requires hash rewriting even when sandboxing is enabled. */
+    needsHashRewrite = true;
+#endif
+
     state = &DerivationGoal::getDerivation;
     name = (format("building of '%1%'") % drvPath).str();
     trace("created");
@@ -2073,7 +2083,7 @@ void DerivationGoal::startBuilder()
 #endif
     }
 
-    else {
+    if (needsHashRewrite) {
 
         if (pathExists(homeDir))
             throw Error(format("directory '%1%' exists; please remove it") % homeDir);
@@ -2500,17 +2510,17 @@ void setupSeccomp()
         seccomp_release(ctx);
     });
 
-    if (settings.thisSystem == "x86_64-linux" &&
+    if (nativeSystem == "x86_64-linux" &&
         seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
         throw SysError("unable to add 32-bit seccomp architecture");
 
-    if (settings.thisSystem == "x86_64-linux" &&
+    if (nativeSystem == "x86_64-linux" &&
         seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
         throw SysError("unable to add X32 seccomp architecture");
 
-    if (settings.thisSystem == "aarch64-linux" &&
+    if (nativeSystem == "aarch64-linux" &&
         seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
-        printError("unsable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes.");
+        printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
 
     /* Prevent builders from creating setuid/setgid binaries. */
     for (int perm : { S_ISUID, S_ISGID }) {
@@ -2873,6 +2883,10 @@ void DerivationGoal::runChild()
                 for (auto & i : missingPaths) {
                     sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str();
                 }
+                /* Also add redirected outputs to the chroot */
+                for (auto & i : redirectedOutputs) {
+                    sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.second.c_str()).str();
+                }
                 sandboxProfile += ")\n";
 
                 /* Our inputs (transitive dependencies and any impurities computed above)
@@ -3051,7 +3065,9 @@ void DerivationGoal::registerOutputs()
                         throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path);
             }
             if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path);
-        } else {
+        }
+
+        if (needsHashRewrite) {
             Path redirected = redirectedOutputs[path];
             if (buildMode == bmRepair
                 && redirectedBadOutputs.find(path) != redirectedBadOutputs.end()

src/libstore/gc.cc

@@ -326,10 +326,9 @@ void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
     findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
     findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
 
-    /* Add additional roots returned by the program specified by the
+    /* Add additional roots returned by different platforms-specific
-       NIX_ROOT_FINDER environment variable.  This is typically used
+       heuristics.  This is typically used to add running programs to
-       to add running programs to the set of roots (to prevent them
+       the set of roots (to prevent them from being garbage collected). */
-       from being garbage collected). */
     findRuntimeRoots(roots, censor);
 }
 

src/libutil/util.cc

@@ -38,6 +38,9 @@ extern char * * environ;
 namespace nix {
 
 
+const std::string nativeSystem = SYSTEM;
+
+
 BaseError & BaseError::addPrefix(const FormatOrString & fs)
 {
     prefix_ = fs.s + prefix_;

src/libutil/util.hh

@@ -30,6 +30,10 @@ struct Sink;
 struct Source;
 
 
+/* The system for which Nix is compiled. */
+extern const std::string nativeSystem;
+
+
 /* Return an environment variable. */
 string getEnv(const string & key, const string & def = "");
 

src/nix-build/nix-build.cc

@@ -274,19 +274,21 @@ static void _main(int argc, char * * argv)
         exprs = {state->parseStdin()};
     else
         for (auto i : left) {
-            auto absolute = i;
-            try {
-                absolute = canonPath(absPath(i), true);
-            } catch (Error e) {};
             if (fromArgs)
                 exprs.push_back(state->parseExprFromString(i, absPath(".")));
-            else if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?")))
+            else {
+                auto absolute = i;
+                try {
+                    absolute = canonPath(absPath(i), true);
+                } catch (Error e) {};
+                if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?")))
                 drvs.push_back(DrvInfo(*state, store, absolute));
             else
                 /* If we're in a #! script, interpret filenames
                    relative to the script. */
                 exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state,
                     inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i)))));
+            }
         }
 
     /* Evaluate them into derivations. */

src/nix/repl.cc

@@ -192,6 +192,14 @@ static int listPossibleCallback(char *s, char ***avp) {
   return ac;
 }
 
+namespace {
+    // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline.
+    volatile sig_atomic_t g_signal_received = 0;
+
+    void sigintHandler(int signo) {
+        g_signal_received = signo;
+    }
+}
 
 void NixRepl::mainLoop(const std::vector<std::string> & files)
 {
@@ -251,8 +259,40 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
 
 bool NixRepl::getLine(string & input, const std::string &prompt)
 {
+    struct sigaction act, old;
+    sigset_t savedSignalMask, set;
+
+    auto setupSignals = [&]() {
+        act.sa_handler = sigintHandler;
+        sigfillset(&act.sa_mask);
+        act.sa_flags = 0;
+        if (sigaction(SIGINT, &act, &old))
+            throw SysError("installing handler for SIGINT");
+
+        sigemptyset(&set);
+        sigaddset(&set, SIGINT);
+        if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask))
+            throw SysError("unblocking SIGINT");
+    };
+    auto restoreSignals = [&]() {
+        if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
+            throw SysError("restoring signals");
+
+        if (sigaction(SIGINT, &old, 0))
+            throw SysError("restoring handler for SIGINT");
+    };
+
+    setupSignals();
     char * s = readline(prompt.c_str());
     Finally doFree([&]() { free(s); });
+    restoreSignals();
+
+    if (g_signal_received) {
+        g_signal_received = 0;
+        input.clear();
+        return true;
+    }
+
     if (!s)
       return false;
     input += s;

tests/lang/eval-fail-hashfile-missing.nix

@@ -0,0 +1,5 @@
+let
+  paths = [ ./this-file-is-definitely-not-there-7392097 "/and/neither/is/this/37293620" ];
+in
+  toString (builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"]))
+

tests/lang/eval-okay-hash.exp

@@ -1 +0,0 @@
-[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]

tests/lang/eval-okay-hashfile.exp

@@ -0,0 +1 @@
+[ "d3b07384d113edec49eaa6238ad5ff00" "0f343b0931126a20f133d67c2b018a3b" "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15" "60cacbf3d72e1e7834203da608037b1bf83b40e8" "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c" "5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" "0cf9180a764aba863a67b6d72f0918bc131c6772642cb2dce5a34f0a702f9470ddc2bf125c12198b1995c233c34b4afd346c54a2334c350a948a51b6e8b4e6b6" "8efb4f73c5655351c444eb109230c556d39e2c7624e9c11abc9e3fb4b9b9254218cc5085b454a9698d085cfa92198491f07a723be4574adc70617b73eb0b6461" ]

tests/lang/eval-okay-hashfile.nix

@@ -0,0 +1,4 @@
+let
+  paths = [ ./data ./binary-data ];
+in
+  builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"])

tests/lang/eval-okay-hashstring.exp

@@ -0,0 +1 @@
+[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]