Merge remote-tracking branch 'upstream/master' into hash-always-has-type

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/test.yml

@@ -10,5 +10,5 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2
-    - uses: cachix/install-nix-action@v8
+    - uses: cachix/install-nix-action@v10
     - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings

doc/manual/installation/installing-binary.xml

@@ -97,7 +97,7 @@ $ rm -rf /nix
     installation on your system:
   </para>
 
-  <screen>sh &lt;(curl https://nixos.org/nix/install) --daemon</screen>
+  <screen>sh &lt;(curl -L https://nixos.org/nix/install) --daemon</screen>
 
   <para>
     The multi-user installation of Nix will create build users between
@@ -178,7 +178,7 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
     is a bit of a misnomer). To use this approach, just install Nix with:
   </para>
 
-  <screen>$ sh &lt;(curl https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume</screen>
+  <screen>$ sh &lt;(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume</screen>
 
   <para>
     If you don't like the sound of this, you'll want to weigh the
@@ -429,7 +429,7 @@ LABEL=Nix\040Store /nix apfs rw,nobrowse
   NixOS.org installation script:
 
   <screen>
-  sh &lt;(curl https://nixos.org/nix/install)
+  sh &lt;(curl -L https://nixos.org/nix/install)
 </screen>
   </para>
 

scripts/install-multi-user.sh

@@ -526,7 +526,7 @@ This script is going to call sudo a lot. Normally, it would show you
 exactly what commands it is running and why. However, the script is
 run in a headless fashion, like this:
 
-  $ curl https://nixos.org/nix/install | sh
+  $ curl -L https://nixos.org/nix/install | sh
 
 or maybe in a CI pipeline. Because of that, we're going to skip the
 verbose output in the interest of brevity.
@@ -534,7 +534,7 @@ verbose output in the interest of brevity.
 If you would like to
 see the output, try like this:
 
-  $ curl -o install-nix https://nixos.org/nix/install
+  $ curl -L -o install-nix https://nixos.org/nix/install
   $ sh ./install-nix
 
 EOF

scripts/install-nix-from-closure.sh

@@ -113,7 +113,7 @@ if [ "$(uname -s)" = "Darwin" ]; then
         (
             echo ""
             echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume."
-            echo "Use sh <(curl https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually."
+            echo "Use sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually."
             echo "See https://nixos.org/nix/manual/#sect-macos-installation"
             echo ""
         ) >&2

src/libstore/build.cc

@@ -1950,8 +1950,11 @@ void linkOrCopy(const Path & from, const Path & to)
         /* Hard-linking fails if we exceed the maximum link count on a
            file (e.g. 32000 of ext3), which is quite possible after a
            'nix-store --optimise'. FIXME: actually, why don't we just
-           bind-mount in this case? */
+           bind-mount in this case?
-        if (errno != EMLINK)
+           
+           It can also fail with EPERM in BeegFS v7 and earlier versions
+           which don't allow hard-links to other directories */
+        if (errno != EMLINK && errno != EPERM)
             throw SysError("linking '%s' to '%s'", to, from);
         copyPath(from, to);
     }

src/libstore/builtins/fetchurl.cc

@@ -58,13 +58,16 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         }
     };
 
+    /* We always have one output, and if it's a fixed-output derivation (as
+       checked below) it must be the only output */
+    auto & output = drv.outputs.begin()->second;
+
     /* Try the hashed mirrors first. */
-    if (getAttr("outputHashMode") == "flat")
+    if (output.hash && output.hash->method == FileIngestionMethod::Flat)
         for (auto hashedMirror : settings.hashedMirrors.get())
             try {
                 if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
-                auto ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
+                auto & h = output.hash->hash;
-                auto h = Hash(getAttr("outputHash"), ht);
                 fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
                 return;
             } catch (Error & e) {

src/libstore/derivations.cc

@@ -404,7 +404,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
 {
     auto path = store.parseStorePath(readString(in));
     auto hashAlgo = readString(in);
-    const auto hash = readString(in);
+    auto hash = readString(in);
 
     std::optional<FixedOutputHash> fsh;
     if (hashAlgo != "") {
@@ -413,7 +413,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
             method = FileIngestionMethod::Recursive;
             hashAlgo = string(hashAlgo, 2);
         }
-        const HashType hashType = parseHashType(hashAlgo);
+        auto hashType = parseHashType(hashAlgo);
         fsh = FixedOutputHash {
             .method = std::move(method),
             .hash = Hash(hash, hashType),
@@ -463,11 +463,16 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
 void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
 {
     out << drv.outputs.size();
-    for (auto & i : drv.outputs)
+    for (auto & i : drv.outputs) {
         out << i.first
-            << store.printStorePath(i.second.path)
+            << store.printStorePath(i.second.path);
-            << i.second.hash->printMethodAlgo()
+        if (i.second.hash) {
-            << i.second.hash->hash.to_string(Base16, false);
+            out << i.second.hash->printMethodAlgo()
+                << i.second.hash->hash.to_string(Base16, false);
+        } else {
+            out << "" << "";
+        }
+    }
     writeStorePaths(store, out, drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;
     out << drv.env.size();

src/libutil/hash.cc

@@ -104,6 +104,7 @@ static string printHash32(const Hash & hash)
 
 string printHash16or32(const Hash & hash)
 {
+    assert(hash.type);
     return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false);
 }
 
@@ -365,14 +366,15 @@ HashType parseHashType(std::string_view s)
 string printHashType(HashType ht)
 {
     switch (ht) {
-    case htMD5: return "md5"; break;
+    case htMD5: return "md5";
-    case htSHA1: return "sha1"; break;
+    case htSHA1: return "sha1";
-    case htSHA256: return "sha256"; break;
+    case htSHA256: return "sha256";
-    case htSHA512: return "sha512"; break;
+    case htSHA512: return "sha512";
+    default:
+        // illegal hash type enum value internally, as opposed to external input
+        // which should be validated with nice error message.
+        abort();
     }
-    // illegal hash type enum value internally, as opposed to external input
-    // which should be validated with nice error message.
-    abort();
 }
 
 }

src/libutil/hash.hh

@@ -10,7 +10,7 @@ namespace nix {
 MakeError(BadHash, Error);
 
 
-enum HashType : char { htMD5, htSHA1, htSHA256, htSHA512 };
+enum HashType : char { htMD5 = 42, htSHA1, htSHA256, htSHA512 };
 
 
 const int md5HashSize = 16;

tests/build-hook.nix

@@ -1,23 +1,39 @@
+{ busybox }:
+
 with import ./config.nix;
 
 let
 
+  mkDerivation = args:
+    derivation ({
+      inherit system;
+      builder = busybox;
+      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
+    } // removeAttrs args ["builder" "meta"])
+    // { meta = args.meta or {}; };
+
   input1 = mkDerivation {
-    name = "build-hook-input-1";
+    shell = busybox;
-    buildCommand = "mkdir $out; echo FOO > $out/foo";
+    name = "build-remote-input-1";
+    buildCommand = "echo FOO > $out";
     requiredSystemFeatures = ["foo"];
   };
 
   input2 = mkDerivation {
-    name = "build-hook-input-2";
+    shell = busybox;
-    buildCommand = "mkdir $out; echo BAR > $out/bar";
+    name = "build-remote-input-2";
+    buildCommand = "echo BAR > $out";
   };
 
 in
 
   mkDerivation {
-    name = "build-hook";
+    shell = busybox;
-    builder = ./dependencies.builder0.sh;
+    name = "build-remote";
-    input1 = " " + input1 + "/.";
+    buildCommand =
-    input2 = " ${input2}/.";
+      ''
+        read x < ${input1}
+        read y < ${input2}
+        echo $x$y > $out
+      '';
   }

tests/build-remote.sh

@@ -3,22 +3,29 @@ source common.sh
 clearStore
 
 if ! canUseSandbox; then exit; fi
-if [[ ! $SHELL =~ /nix/store ]]; then exit; fi
+if ! [[ $busybox =~ busybox ]]; then exit; fi
 
-chmod -R u+w $TEST_ROOT/store0 || true
+chmod -R u+w $TEST_ROOT/machine0 || true
-chmod -R u+w $TEST_ROOT/store1 || true
+chmod -R u+w $TEST_ROOT/machine1 || true
-rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1
+chmod -R u+w $TEST_ROOT/machine2 || true
+rm -rf $TEST_ROOT/machine0 $TEST_ROOT/machine1 $TEST_ROOT/machine2
+rm -f $TEST_ROOT/result
 
-nix build -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \
+unset NIX_STORE_DIR
-  --sandbox-paths /nix/store --sandbox-build-dir /build-tmp \
+unset NIX_STATE_DIR
-  --builders "$TEST_ROOT/store0; $TEST_ROOT/store1 - - 1 1 foo" \
+
+# Note: ssh://localhost bypasses ssh, directly invoking nix-store as a
+# child process. This allows us to test LegacySSHStore::buildDerivation().
+nix build -L -v -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \
+  --arg busybox $busybox \
+  --store $TEST_ROOT/machine0 \
+  --builders "ssh://localhost?remote-store=$TEST_ROOT/machine1; $TEST_ROOT/machine2 - - 1 1 foo" \
   --system-features foo
 
-outPath=$TEST_ROOT/result
+outPath=$(readlink -f $TEST_ROOT/result)
 
-cat $outPath/foobar | grep FOOBAR
+cat $TEST_ROOT/machine0/$outPath | grep FOOBAR
 
-# Ensure that input1 was built on store1 due to the required feature.
+# Ensure that input1 was built on store2 due to the required feature.
-p=$(readlink -f $outPath/input-2)
+(! nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-1.sh)
-(! nix path-info --store $TEST_ROOT/store0 --all | grep builder-build-hook-input-1.sh)
+nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-1.sh
-nix path-info --store $TEST_ROOT/store1 --all | grep builder-build-hook-input-1.sh

tests/common.sh.in

@@ -35,6 +35,7 @@ export xmllint="@xmllint@"
 export SHELL="@bash@"
 export PAGER=cat
 export HAVE_SODIUM="@HAVE_SODIUM@"
+export busybox="@sandbox_shell@"
 
 export version=@PACKAGE_VERSION@
 export system=@system@

tests/post-hook.sh

@@ -2,6 +2,8 @@ source common.sh
 
 clearStore
 
+rm -f $TEST_ROOT/result
+
 export REMOTE_STORE=$TEST_ROOT/remote_store
 
 # Build the dependencies and push them to the remote store

tests/recursive.sh

@@ -5,6 +5,8 @@ if [[ $(uname) != Linux ]]; then exit; fi
 
 clearStore
 
+rm -f $TEST_ROOT/result
+
 export unreachable=$(nix add-to-store ./recursive.sh)
 
 nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L '(

tests/structured-attrs.sh

@@ -2,6 +2,8 @@ source common.sh
 
 clearStore
 
+rm -f $TEST_ROOT/result
+
 nix-build structured-attrs.nix -A all -o $TEST_ROOT/result
 
 [[ $(cat $TEST_ROOT/result/foo) = bar ]]