Merge remote-tracking branch 'upstream/master' into hash-always-has-type

This commit is contained in:
John Ericson 2020-06-29 17:56:46 +00:00
commit 987a4a0be9
16 changed files with 96 additions and 47 deletions

6
.github/dependabot.yml vendored Normal file
View file

@ -0,0 +1,6 @@
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"

View file

@ -10,5 +10,5 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
- uses: cachix/install-nix-action@v8 - uses: cachix/install-nix-action@v10
- run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings

View file

@ -97,7 +97,7 @@ $ rm -rf /nix
installation on your system: installation on your system:
</para> </para>
<screen>sh &lt;(curl https://nixos.org/nix/install) --daemon</screen> <screen>sh &lt;(curl -L https://nixos.org/nix/install) --daemon</screen>
<para> <para>
The multi-user installation of Nix will create build users between The multi-user installation of Nix will create build users between
@ -178,7 +178,7 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
is a bit of a misnomer). To use this approach, just install Nix with: is a bit of a misnomer). To use this approach, just install Nix with:
</para> </para>
<screen>$ sh &lt;(curl https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume</screen> <screen>$ sh &lt;(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume</screen>
<para> <para>
If you don't like the sound of this, you'll want to weigh the If you don't like the sound of this, you'll want to weigh the
@ -429,7 +429,7 @@ LABEL=Nix\040Store /nix apfs rw,nobrowse
NixOS.org installation script: NixOS.org installation script:
<screen> <screen>
sh &lt;(curl https://nixos.org/nix/install) sh &lt;(curl -L https://nixos.org/nix/install)
</screen> </screen>
</para> </para>

View file

@ -526,7 +526,7 @@ This script is going to call sudo a lot. Normally, it would show you
exactly what commands it is running and why. However, the script is exactly what commands it is running and why. However, the script is
run in a headless fashion, like this: run in a headless fashion, like this:
$ curl https://nixos.org/nix/install | sh $ curl -L https://nixos.org/nix/install | sh
or maybe in a CI pipeline. Because of that, we're going to skip the or maybe in a CI pipeline. Because of that, we're going to skip the
verbose output in the interest of brevity. verbose output in the interest of brevity.
@ -534,7 +534,7 @@ verbose output in the interest of brevity.
If you would like to If you would like to
see the output, try like this: see the output, try like this:
$ curl -o install-nix https://nixos.org/nix/install $ curl -L -o install-nix https://nixos.org/nix/install
$ sh ./install-nix $ sh ./install-nix
EOF EOF

View file

@ -113,7 +113,7 @@ if [ "$(uname -s)" = "Darwin" ]; then
( (
echo "" echo ""
echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume." echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume."
echo "Use sh <(curl https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually." echo "Use sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually."
echo "See https://nixos.org/nix/manual/#sect-macos-installation" echo "See https://nixos.org/nix/manual/#sect-macos-installation"
echo "" echo ""
) >&2 ) >&2

View file

@ -1950,8 +1950,11 @@ void linkOrCopy(const Path & from, const Path & to)
/* Hard-linking fails if we exceed the maximum link count on a /* Hard-linking fails if we exceed the maximum link count on a
file (e.g. 32000 of ext3), which is quite possible after a file (e.g. 32000 of ext3), which is quite possible after a
'nix-store --optimise'. FIXME: actually, why don't we just 'nix-store --optimise'. FIXME: actually, why don't we just
bind-mount in this case? */ bind-mount in this case?
if (errno != EMLINK)
It can also fail with EPERM in BeegFS v7 and earlier versions
which don't allow hard-links to other directories */
if (errno != EMLINK && errno != EPERM)
throw SysError("linking '%s' to '%s'", to, from); throw SysError("linking '%s' to '%s'", to, from);
copyPath(from, to); copyPath(from, to);
} }

View file

@ -58,13 +58,16 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
} }
}; };
/* We always have one output, and if it's a fixed-output derivation (as
checked below) it must be the only output */
auto & output = drv.outputs.begin()->second;
/* Try the hashed mirrors first. */ /* Try the hashed mirrors first. */
if (getAttr("outputHashMode") == "flat") if (output.hash && output.hash->method == FileIngestionMethod::Flat)
for (auto hashedMirror : settings.hashedMirrors.get()) for (auto hashedMirror : settings.hashedMirrors.get())
try { try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
auto ht = parseHashTypeOpt(getAttr("outputHashAlgo")); auto & h = output.hash->hash;
auto h = Hash(getAttr("outputHash"), ht);
fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
return; return;
} catch (Error & e) { } catch (Error & e) {

View file

@ -404,7 +404,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
{ {
auto path = store.parseStorePath(readString(in)); auto path = store.parseStorePath(readString(in));
auto hashAlgo = readString(in); auto hashAlgo = readString(in);
const auto hash = readString(in); auto hash = readString(in);
std::optional<FixedOutputHash> fsh; std::optional<FixedOutputHash> fsh;
if (hashAlgo != "") { if (hashAlgo != "") {
@ -413,7 +413,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
method = FileIngestionMethod::Recursive; method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2); hashAlgo = string(hashAlgo, 2);
} }
const HashType hashType = parseHashType(hashAlgo); auto hashType = parseHashType(hashAlgo);
fsh = FixedOutputHash { fsh = FixedOutputHash {
.method = std::move(method), .method = std::move(method),
.hash = Hash(hash, hashType), .hash = Hash(hash, hashType),
@ -463,11 +463,16 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv) void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
{ {
out << drv.outputs.size(); out << drv.outputs.size();
for (auto & i : drv.outputs) for (auto & i : drv.outputs) {
out << i.first out << i.first
<< store.printStorePath(i.second.path) << store.printStorePath(i.second.path);
<< i.second.hash->printMethodAlgo() if (i.second.hash) {
out << i.second.hash->printMethodAlgo()
<< i.second.hash->hash.to_string(Base16, false); << i.second.hash->hash.to_string(Base16, false);
} else {
out << "" << "";
}
}
writeStorePaths(store, out, drv.inputSrcs); writeStorePaths(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args; out << drv.platform << drv.builder << drv.args;
out << drv.env.size(); out << drv.env.size();

View file

@ -104,6 +104,7 @@ static string printHash32(const Hash & hash)
string printHash16or32(const Hash & hash) string printHash16or32(const Hash & hash)
{ {
assert(hash.type);
return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false); return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false);
} }
@ -365,14 +366,15 @@ HashType parseHashType(std::string_view s)
string printHashType(HashType ht) string printHashType(HashType ht)
{ {
switch (ht) { switch (ht) {
case htMD5: return "md5"; break; case htMD5: return "md5";
case htSHA1: return "sha1"; break; case htSHA1: return "sha1";
case htSHA256: return "sha256"; break; case htSHA256: return "sha256";
case htSHA512: return "sha512"; break; case htSHA512: return "sha512";
} default:
// illegal hash type enum value internally, as opposed to external input // illegal hash type enum value internally, as opposed to external input
// which should be validated with nice error message. // which should be validated with nice error message.
abort(); abort();
}
} }
} }

View file

@ -10,7 +10,7 @@ namespace nix {
MakeError(BadHash, Error); MakeError(BadHash, Error);
enum HashType : char { htMD5, htSHA1, htSHA256, htSHA512 }; enum HashType : char { htMD5 = 42, htSHA1, htSHA256, htSHA512 };
const int md5HashSize = 16; const int md5HashSize = 16;

View file

@ -1,23 +1,39 @@
{ busybox }:
with import ./config.nix; with import ./config.nix;
let let
mkDerivation = args:
derivation ({
inherit system;
builder = busybox;
args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
} // removeAttrs args ["builder" "meta"])
// { meta = args.meta or {}; };
input1 = mkDerivation { input1 = mkDerivation {
name = "build-hook-input-1"; shell = busybox;
buildCommand = "mkdir $out; echo FOO > $out/foo"; name = "build-remote-input-1";
buildCommand = "echo FOO > $out";
requiredSystemFeatures = ["foo"]; requiredSystemFeatures = ["foo"];
}; };
input2 = mkDerivation { input2 = mkDerivation {
name = "build-hook-input-2"; shell = busybox;
buildCommand = "mkdir $out; echo BAR > $out/bar"; name = "build-remote-input-2";
buildCommand = "echo BAR > $out";
}; };
in in
mkDerivation { mkDerivation {
name = "build-hook"; shell = busybox;
builder = ./dependencies.builder0.sh; name = "build-remote";
input1 = " " + input1 + "/."; buildCommand =
input2 = " ${input2}/."; ''
read x < ${input1}
read y < ${input2}
echo $x$y > $out
'';
} }

View file

@ -3,22 +3,29 @@ source common.sh
clearStore clearStore
if ! canUseSandbox; then exit; fi if ! canUseSandbox; then exit; fi
if [[ ! $SHELL =~ /nix/store ]]; then exit; fi if ! [[ $busybox =~ busybox ]]; then exit; fi
chmod -R u+w $TEST_ROOT/store0 || true chmod -R u+w $TEST_ROOT/machine0 || true
chmod -R u+w $TEST_ROOT/store1 || true chmod -R u+w $TEST_ROOT/machine1 || true
rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1 chmod -R u+w $TEST_ROOT/machine2 || true
rm -rf $TEST_ROOT/machine0 $TEST_ROOT/machine1 $TEST_ROOT/machine2
rm -f $TEST_ROOT/result
nix build -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \ unset NIX_STORE_DIR
--sandbox-paths /nix/store --sandbox-build-dir /build-tmp \ unset NIX_STATE_DIR
--builders "$TEST_ROOT/store0; $TEST_ROOT/store1 - - 1 1 foo" \
# Note: ssh://localhost bypasses ssh, directly invoking nix-store as a
# child process. This allows us to test LegacySSHStore::buildDerivation().
nix build -L -v -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \
--arg busybox $busybox \
--store $TEST_ROOT/machine0 \
--builders "ssh://localhost?remote-store=$TEST_ROOT/machine1; $TEST_ROOT/machine2 - - 1 1 foo" \
--system-features foo --system-features foo
outPath=$TEST_ROOT/result outPath=$(readlink -f $TEST_ROOT/result)
cat $outPath/foobar | grep FOOBAR cat $TEST_ROOT/machine0/$outPath | grep FOOBAR
# Ensure that input1 was built on store1 due to the required feature. # Ensure that input1 was built on store2 due to the required feature.
p=$(readlink -f $outPath/input-2) (! nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-1.sh)
(! nix path-info --store $TEST_ROOT/store0 --all | grep builder-build-hook-input-1.sh) nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-1.sh
nix path-info --store $TEST_ROOT/store1 --all | grep builder-build-hook-input-1.sh

View file

@ -35,6 +35,7 @@ export xmllint="@xmllint@"
export SHELL="@bash@" export SHELL="@bash@"
export PAGER=cat export PAGER=cat
export HAVE_SODIUM="@HAVE_SODIUM@" export HAVE_SODIUM="@HAVE_SODIUM@"
export busybox="@sandbox_shell@"
export version=@PACKAGE_VERSION@ export version=@PACKAGE_VERSION@
export system=@system@ export system=@system@

View file

@ -2,6 +2,8 @@ source common.sh
clearStore clearStore
rm -f $TEST_ROOT/result
export REMOTE_STORE=$TEST_ROOT/remote_store export REMOTE_STORE=$TEST_ROOT/remote_store
# Build the dependencies and push them to the remote store # Build the dependencies and push them to the remote store

View file

@ -5,6 +5,8 @@ if [[ $(uname) != Linux ]]; then exit; fi
clearStore clearStore
rm -f $TEST_ROOT/result
export unreachable=$(nix add-to-store ./recursive.sh) export unreachable=$(nix add-to-store ./recursive.sh)
nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L '( nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L '(

View file

@ -2,6 +2,8 @@ source common.sh
clearStore clearStore
rm -f $TEST_ROOT/result
nix-build structured-attrs.nix -A all -o $TEST_ROOT/result nix-build structured-attrs.nix -A all -o $TEST_ROOT/result
[[ $(cat $TEST_ROOT/result/foo) = bar ]] [[ $(cat $TEST_ROOT/result/foo) = bar ]]