From f4a5913125cb6a8ccfba556a0d75b19c2a5cfa37 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 22 Jun 2020 15:17:20 +0000 Subject: [PATCH 1/9] hashed-mirrors: Use parsed derivation output rather than reconstructing it Now the derivation outputs are parsed up front, we can avoid a reparse by doing it. Also, this just feels a bit better as the `output*` env vars are more of a `libnixexpr` interface than `libnixstore` interface: ultimately, it's the derivation outputs that decide whether the derivation is fixed-output. Yes, hashed mirrors might go away with #3689, but this bit of code would be moved rather than deleted, so it's worth doing a cleanup anyways I think. --- src/libstore/builtins/fetchurl.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 1cfe4a46a..e630cf6f1 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -58,13 +58,16 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) } }; + /* We always have one output, and if it's a fixed-output derivation (as + checked below) it must be the only output */ + auto & output = drv.outputs.begin()->second; + /* Try the hashed mirrors first. */ - if (getAttr("outputHashMode") == "flat") + if (output.hash && output.hash->method == FileIngestionMethod::Flat) for (auto hashedMirror : settings.hashedMirrors.get()) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - auto ht = parseHashTypeOpt(getAttr("outputHashAlgo")); - auto h = Hash(getAttr("outputHash"), ht); + auto & h = output.hash->hash; fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false)); return; } catch (Error & e) { From 3a642187c33ed46d952d3a50a83b2576b704fab7 Mon Sep 17 00:00:00 2001 From: Rodrigo Date: Thu, 25 Jun 2020 12:03:26 +0200 Subject: [PATCH 2/9] Fall back to copyPath if link fails with EPERM BeeGFS doesn't allow hard-links and returns EPERM, so we fall back to copyPath. See https://github.com/NixOS/nix/issues/3748 --- src/libstore/build.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0c25897f8..9ac5fd923 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1950,8 +1950,11 @@ void linkOrCopy(const Path & from, const Path & to) /* Hard-linking fails if we exceed the maximum link count on a file (e.g. 32000 of ext3), which is quite possible after a 'nix-store --optimise'. FIXME: actually, why don't we just - bind-mount in this case? */ - if (errno != EMLINK) + bind-mount in this case? + + It can also fail with EPERM in BeegFS v7 and earlier versions + which don't allow hard-links to other directories */ + if (errno != EMLINK && errno != EPERM) throw SysError("linking '%s' to '%s'", to, from); copyPath(from, to); } From de2641ae99965ecf88e6f72e3969c0c7b7e7020a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 25 Jun 2020 15:50:30 +0200 Subject: [PATCH 3/9] Fix empty std::optional dereference in writeDerivation() https://hydra.nixos.org/build/123017579 --- src/libstore/derivations.cc | 17 +++++++++++------ src/libutil/hash.cc | 27 ++++++++++++++------------- src/libutil/hash.hh | 2 +- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 6c49075ba..42551ef6b 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -404,7 +404,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) { auto path = store.parseStorePath(readString(in)); auto hashAlgo = readString(in); - const auto hash = readString(in); + auto hash = readString(in); std::optional fsh; if (hashAlgo != "") { @@ -413,7 +413,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) method = FileIngestionMethod::Recursive; hashAlgo = string(hashAlgo, 2); } - const HashType hashType = parseHashType(hashAlgo); + auto hashType = parseHashType(hashAlgo); fsh = FixedOutputHash { .method = std::move(method), .hash = Hash(hash, hashType), @@ -463,11 +463,16 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv) { out << drv.outputs.size(); - for (auto & i : drv.outputs) + for (auto & i : drv.outputs) { out << i.first - << store.printStorePath(i.second.path) - << i.second.hash->printMethodAlgo() - << i.second.hash->hash.to_string(Base16, false); + << store.printStorePath(i.second.path); + if (i.second.hash) { + out << i.second.hash->printMethodAlgo() + << i.second.hash->hash.to_string(Base16, false); + } else { + out << "" << ""; + } + } writeStorePaths(store, out, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index c8fcdfed0..01fae3044 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -19,7 +19,7 @@ namespace nix { void Hash::init() { - if (!type) abort(); + assert(type); switch (*type) { case htMD5: hashSize = md5HashSize; break; case htSHA1: hashSize = sha1HashSize; break; @@ -101,15 +101,15 @@ static string printHash32(const Hash & hash) string printHash16or32(const Hash & hash) { + assert(hash.type); return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false); } -HashType assertInitHashType(const Hash & h) { - if (h.type) - return *h.type; - else - abort(); +HashType assertInitHashType(const Hash & h) +{ + assert(h.type); + return *h.type; } std::string Hash::to_string(Base base, bool includeType) const @@ -363,14 +363,15 @@ HashType parseHashType(const string & s) string printHashType(HashType ht) { switch (ht) { - case htMD5: return "md5"; break; - case htSHA1: return "sha1"; break; - case htSHA256: return "sha256"; break; - case htSHA512: return "sha512"; break; + case htMD5: return "md5"; + case htSHA1: return "sha1"; + case htSHA256: return "sha256"; + case htSHA512: return "sha512"; + default: + // illegal hash type enum value internally, as opposed to external input + // which should be validated with nice error message. + abort(); } - // illegal hash type enum value internally, as opposed to external input - // which should be validated with nice error message. - abort(); } } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 0d9916508..23259dced 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -10,7 +10,7 @@ namespace nix { MakeError(BadHash, Error); -enum HashType : char { htMD5, htSHA1, htSHA256, htSHA512 }; +enum HashType : char { htMD5 = 42, htSHA1, htSHA256, htSHA512 }; const int md5HashSize = 16; From b7ccf7ae2af3d7eaf3696358ecbbce5a2bcfe652 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 25 Jun 2020 18:26:34 +0200 Subject: [PATCH 4/9] build-remote.sh: Test LegacySSHStore --- tests/build-hook.nix | 32 ++++++++++++++++++++++++-------- tests/build-remote.sh | 33 ++++++++++++++++++++------------- tests/common.sh.in | 1 + tests/post-hook.sh | 2 ++ tests/recursive.sh | 2 ++ tests/structured-attrs.sh | 2 ++ 6 files changed, 51 insertions(+), 21 deletions(-) diff --git a/tests/build-hook.nix b/tests/build-hook.nix index 8c5ca8cd3..a19c10dde 100644 --- a/tests/build-hook.nix +++ b/tests/build-hook.nix @@ -1,23 +1,39 @@ +{ busybox }: + with import ./config.nix; let + mkDerivation = args: + derivation ({ + inherit system; + builder = busybox; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + } // removeAttrs args ["builder" "meta"]) + // { meta = args.meta or {}; }; + input1 = mkDerivation { - name = "build-hook-input-1"; - buildCommand = "mkdir $out; echo FOO > $out/foo"; + shell = busybox; + name = "build-remote-input-1"; + buildCommand = "echo FOO > $out"; requiredSystemFeatures = ["foo"]; }; input2 = mkDerivation { - name = "build-hook-input-2"; - buildCommand = "mkdir $out; echo BAR > $out/bar"; + shell = busybox; + name = "build-remote-input-2"; + buildCommand = "echo BAR > $out"; }; in mkDerivation { - name = "build-hook"; - builder = ./dependencies.builder0.sh; - input1 = " " + input1 + "/."; - input2 = " ${input2}/."; + shell = busybox; + name = "build-remote"; + buildCommand = + '' + read x < ${input1} + read y < ${input2} + echo $x$y > $out + ''; } diff --git a/tests/build-remote.sh b/tests/build-remote.sh index a550f4460..4dfb753e1 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -3,22 +3,29 @@ source common.sh clearStore if ! canUseSandbox; then exit; fi -if [[ ! $SHELL =~ /nix/store ]]; then exit; fi +if ! [[ $busybox =~ busybox ]]; then exit; fi -chmod -R u+w $TEST_ROOT/store0 || true -chmod -R u+w $TEST_ROOT/store1 || true -rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1 +chmod -R u+w $TEST_ROOT/machine0 || true +chmod -R u+w $TEST_ROOT/machine1 || true +chmod -R u+w $TEST_ROOT/machine2 || true +rm -rf $TEST_ROOT/machine0 $TEST_ROOT/machine1 $TEST_ROOT/machine2 +rm -f $TEST_ROOT/result -nix build -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \ - --sandbox-paths /nix/store --sandbox-build-dir /build-tmp \ - --builders "$TEST_ROOT/store0; $TEST_ROOT/store1 - - 1 1 foo" \ +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +# Note: ssh://localhost bypasses ssh, directly invoking nix-store as a +# child process. This allows us to test LegacySSHStore::buildDerivation(). +nix build -L -v -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \ + --arg busybox $busybox \ + --store $TEST_ROOT/machine0 \ + --builders "ssh://localhost?remote-store=$TEST_ROOT/machine1; $TEST_ROOT/machine2 - - 1 1 foo" \ --system-features foo -outPath=$TEST_ROOT/result +outPath=$(readlink -f $TEST_ROOT/result) -cat $outPath/foobar | grep FOOBAR +cat $TEST_ROOT/machine0/$outPath | grep FOOBAR -# Ensure that input1 was built on store1 due to the required feature. -p=$(readlink -f $outPath/input-2) -(! nix path-info --store $TEST_ROOT/store0 --all | grep builder-build-hook-input-1.sh) -nix path-info --store $TEST_ROOT/store1 --all | grep builder-build-hook-input-1.sh +# Ensure that input1 was built on store2 due to the required feature. +(! nix path-info --store $TEST_ROOT/machine1 --all | grep builder-build-remote-input-1.sh) +nix path-info --store $TEST_ROOT/machine2 --all | grep builder-build-remote-input-1.sh diff --git a/tests/common.sh.in b/tests/common.sh.in index dd7e61822..73fe77345 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -35,6 +35,7 @@ export xmllint="@xmllint@" export SHELL="@bash@" export PAGER=cat export HAVE_SODIUM="@HAVE_SODIUM@" +export busybox="@sandbox_shell@" export version=@PACKAGE_VERSION@ export system=@system@ diff --git a/tests/post-hook.sh b/tests/post-hook.sh index a02657215..aa3e6a574 100644 --- a/tests/post-hook.sh +++ b/tests/post-hook.sh @@ -2,6 +2,8 @@ source common.sh clearStore +rm -f $TEST_ROOT/result + export REMOTE_STORE=$TEST_ROOT/remote_store # Build the dependencies and push them to the remote store diff --git a/tests/recursive.sh b/tests/recursive.sh index 394ae5ddb..2d4f83895 100644 --- a/tests/recursive.sh +++ b/tests/recursive.sh @@ -5,6 +5,8 @@ if [[ $(uname) != Linux ]]; then exit; fi clearStore +rm -f $TEST_ROOT/result + export unreachable=$(nix add-to-store ./recursive.sh) nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L '( diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh index 646bdb876..dcfe6d580 100644 --- a/tests/structured-attrs.sh +++ b/tests/structured-attrs.sh @@ -2,6 +2,8 @@ source common.sh clearStore +rm -f $TEST_ROOT/result + nix-build structured-attrs.nix -A all -o $TEST_ROOT/result [[ $(cat $TEST_ROOT/result/foo) = bar ]] From 7af734bac138e80a2e689c140127443366e566d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 27 Jun 2020 20:35:33 +0100 Subject: [PATCH 5/9] dependabot: automatically keep github actions up-to-date --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..5ace4600a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" From 9937f4ed37a1120306103c6df16b48388aaeb896 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 28 Jun 2020 06:02:57 +0000 Subject: [PATCH 6/9] Bump cachix/install-nix-action from v8 to v10 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from v8 to v10. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v8...63cf434de4e4292c6960639d56c5dd550e789d77) Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7feefc855..7755466a0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,5 +10,5 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - - uses: cachix/install-nix-action@v8 + - uses: cachix/install-nix-action@v10 - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings From 2b834d48aae35c5050895fac540ca55387925d0f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Jun 2020 22:45:41 +0200 Subject: [PATCH 7/9] NAR parser: Fix missing name field check Discovered by @Kloenk. --- src/libutil/archive.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 6a8484705..51c88537e 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -262,7 +262,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path) names[name] = 0; } } else if (s == "node") { - if (s.empty()) throw badArchive("entry name missing"); + if (name.empty()) throw badArchive("entry name missing"); parse(sink, source, path + "/" + name); } else throw badArchive("unknown field " + s); From e72a16a339092359e58ac7626ea89d4182b26642 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 30 Jun 2020 11:00:51 -0600 Subject: [PATCH 8/9] check for a null symbol --- src/libutil/error.hh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 1e6102ce1..6982e30aa 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -67,7 +67,11 @@ struct ErrPos { { line = pos.line; column = pos.column; - file = pos.file; + // is file symbol null? + if (pos.file.set()) + file = pos.file; + else + file = ""; return *this; } From a0705e0dd152f2a19d096d02024723a5614c7726 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 30 Jun 2020 11:01:46 -0600 Subject: [PATCH 9/9] invalid pos check --- src/libutil/tests/logging.cc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index 6a6fb4ac3..6a58b9425 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -289,4 +289,22 @@ namespace nix { "what about this " ANSI_YELLOW "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL); } + + /* ---------------------------------------------------------------------------- + * ErrPos + * --------------------------------------------------------------------------*/ + + TEST(errpos, invalidPos) { + + // contains an invalid symbol, which we should not dereference! + Pos invalid; + + // constructing without access violation. + ErrPos ep(invalid); + + // assignment without access violation. + ep = invalid; + + } + }