diff --git a/mk/run_test.sh b/mk/run_test.sh new file mode 100755 index 000000000..6af5b070a --- /dev/null +++ b/mk/run_test.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +set -u + +red="" +green="" +yellow="" +normal="" + +post_run_msg="ran test $1..." +if [ -t 1 ]; then + red="" + green="" + yellow="" + normal="" +fi +(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null) +log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)" +status=$? +if [ $status -eq 0 ]; then + echo "$post_run_msg [${green}PASS$normal]" +elif [ $status -eq 99 ]; then + echo "$post_run_msg [${yellow}SKIP$normal]" +else + echo "$post_run_msg [${red}FAIL$normal]" + echo "$log" | sed 's/^/ /' + exit "$status" +fi diff --git a/mk/tests.mk b/mk/tests.mk index 70c30661b..e2258ede6 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -1,45 +1,12 @@ # Run program $1 as part of ‘make installcheck’. define run-install-test - installcheck: $1 + installcheck: $1.test - _installcheck-list += $1 + .PHONY: $1.test + $1.test: $1 tests/common.sh tests/init.sh + @env TEST_NAME=$1 TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 endef -# Color code from https://unix.stackexchange.com/a/10065 -installcheck: - @total=0; failed=0; \ - red=""; \ - green=""; \ - yellow=""; \ - normal=""; \ - if [ -t 1 ]; then \ - red=""; \ - green=""; \ - yellow=""; \ - normal=""; \ - fi; \ - for i in $(_installcheck-list); do \ - total=$$((total + 1)); \ - printf "running test $$i..."; \ - log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \ - status=$$?; \ - if [ $$status -eq 0 ]; then \ - echo " [$${green}PASS$$normal]"; \ - elif [ $$status -eq 99 ]; then \ - echo " [$${yellow}SKIP$$normal]"; \ - else \ - echo " [$${red}FAIL$$normal]"; \ - echo "$$log" | sed 's/^/ /'; \ - failed=$$((failed + 1)); \ - fi; \ - done; \ - if [ "$$failed" != 0 ]; then \ - echo "$${red}$$failed out of $$total tests failed $$normal"; \ - exit 1; \ - else \ - echo "$${green}All tests succeeded$$normal"; \ - fi - .PHONY: check installcheck diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 12e4b6c3b..55fc43cfb 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2753,8 +2753,8 @@ struct RestrictedStore : public LocalFSStore void queryReferrers(const StorePath & path, StorePathSet & referrers) override { } - StorePathSet queryDerivationOutputs(const StorePath & path) override - { throw Error("queryDerivationOutputs"); } + OutputPathMap queryDerivationOutputMap(const StorePath & path) override + { throw Error("queryDerivationOutputMap"); } std::optional queryPathFromHashPart(const std::string & hashPart) override { throw Error("queryPathFromHashPart"); } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index c2c8612d1..843ec8ca8 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -347,6 +347,15 @@ static void performOp(TunnelLogger * logger, ref store, break; } + case wopQueryDerivationOutputMap: { + auto path = store->parseStorePath(readString(from)); + logger->startWork(); + OutputPathMap outputs = store->queryDerivationOutputMap(path); + logger->stopWork(); + writeOutputPathMap(*store, to, outputs); + break; + } + case wopQueryDeriver: { auto path = store->parseStorePath(readString(from)); logger->startWork(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 659f80e77..84b47273d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -774,17 +774,20 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) } -StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path) +OutputPathMap LocalStore::queryDerivationOutputMap(const StorePath & path) { - return retrySQLite([&]() { + return retrySQLite([&]() { auto state(_state.lock()); auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() (queryValidPathId(*state, path))); - StorePathSet outputs; + OutputPathMap outputs; while (useQueryDerivationOutputs.next()) - outputs.insert(parseStorePath(useQueryDerivationOutputs.getStr(1))); + outputs.emplace( + useQueryDerivationOutputs.getStr(0), + parseStorePath(useQueryDerivationOutputs.getStr(1)) + ); return outputs; }); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ab9fbfbe4..17f6202bb 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -133,7 +133,7 @@ public: StorePathSet queryValidDerivers(const StorePath & path) override; - StorePathSet queryDerivationOutputs(const StorePath & path) override; + OutputPathMap queryDerivationOutputMap(const StorePath & path) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; diff --git a/src/libstore/path.hh b/src/libstore/path.hh index a07186323..b03a0f69d 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -62,6 +62,7 @@ public: typedef std::set StorePathSet; typedef std::vector StorePaths; +typedef std::map OutputPathMap; typedef std::map> StorePathCAMap; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 32c736d08..d636dd765 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -56,6 +56,29 @@ void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & } } +std::map readOutputPathMap(const Store & store, Source & from) +{ + std::map pathMap; + auto rawInput = readStrings(from); + if (rawInput.size() % 2) + throw Error("got an odd number of elements from the daemon when trying to read a output path map"); + auto curInput = rawInput.begin(); + while (curInput != rawInput.end()) { + auto thisKey = *curInput++; + auto thisValue = *curInput++; + pathMap.emplace(thisKey, store.parseStorePath(thisValue)); + } + return pathMap; +} + +void writeOutputPathMap(const Store & store, Sink & out, const std::map & pathMap) +{ + out << 2*pathMap.size(); + for (auto & i : pathMap) { + out << i.first; + out << store.printStorePath(i.second); + } +} /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) @@ -435,12 +458,24 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) { auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) { + return Store::queryDerivationOutputs(path); + } conn->to << wopQueryDerivationOutputs << printStorePath(path); conn.processStderr(); return readStorePaths(*this, conn->from); } +OutputPathMap RemoteStore::queryDerivationOutputMap(const StorePath & path) +{ + auto conn(getConnection()); + conn->to << wopQueryDerivationOutputMap << printStorePath(path); + conn.processStderr(); + return readOutputPathMap(*this, conn->from); + +} + std::optional RemoteStore::queryPathFromHashPart(const std::string & hashPart) { auto conn(getConnection()); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 07ed2b51a..290ffbfc8 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -51,6 +51,7 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; + OutputPathMap queryDerivationOutputMap(const StorePath & path) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c4fc1c861..189cf54bb 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -259,6 +259,16 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } +StorePathSet Store::queryDerivationOutputs(const StorePath & path) +{ + auto outputMap = this->queryDerivationOutputMap(path); + StorePathSet outputPaths; + for (auto & i: outputMap) { + outputPaths.emplace(std::move(i.second)); + } + return outputPaths; +} + bool Store::isValidPath(const StorePath & storePath) { std::string hashPart(storePath.hashPart()); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index c4a8d3890..d9d3fe185 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -422,8 +422,11 @@ public: virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; }; /* Query the outputs of the derivation denoted by `path'. */ - virtual StorePathSet queryDerivationOutputs(const StorePath & path) - { unsupported("queryDerivationOutputs"); } + virtual StorePathSet queryDerivationOutputs(const StorePath & path); + + /* Query the mapping outputName=>outputPath for the given derivation */ + virtual OutputPathMap queryDerivationOutputMap(const StorePath & path) + { unsupported("queryDerivationOutputMap"); } /* Query the full store path given the hash part of a valid store path, or empty if the path doesn't exist. */ diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 5a3e33c08..48fbb481f 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -30,7 +30,7 @@ typedef enum { wopSetOptions = 19, wopCollectGarbage = 20, wopQuerySubstitutablePathInfo = 21, - wopQueryDerivationOutputs = 22, + wopQueryDerivationOutputs = 22, // obsolete wopQueryAllValidPaths = 23, wopQueryFailedPaths = 24, wopClearFailedPaths = 25, @@ -49,6 +49,7 @@ typedef enum { wopNarFromPath = 38, wopAddToStoreNar = 39, wopQueryMissing = 40, + wopQueryDerivationOutputMap = 41, } WorkerOp; @@ -73,5 +74,6 @@ StorePathCAMap readStorePathCAMap(const Store & store, Source & from); void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths); +void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths); } diff --git a/tests/common.sh.in b/tests/common.sh.in index 73fe77345..c00ee58a1 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -1,6 +1,6 @@ set -e -export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test) +export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default} export NIX_STORE_DIR if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then # Maybe the build directory is symlinked. diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index de1e2cfe4..b282644ca 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -13,24 +13,32 @@ fake_free=$TEST_ROOT/fake-free export _NIX_TEST_FREE_SPACE_FILE=$fake_free echo 1100 > $fake_free +fifoLock=$TEST_ROOT/fifoLock +mkfifo "$fifoLock" + expr=$(cat < \$out/bar - echo 1... - sleep 2 - echo 200 > ${fake_free}.tmp1 + + # Pretend that we run out of space + echo 100 > ${fake_free}.tmp1 mv ${fake_free}.tmp1 $fake_free - echo 2... - sleep 2 - echo 3... - sleep 2 - echo 4... - [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]] + + # Wait for the GC to run + for i in {1..20}; do + echo ''\${i}... + if [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]]; then + exit 0 + fi + sleep 1 + done + exit 1 ''; } EOF @@ -43,15 +51,9 @@ with import ./config.nix; mkDerivation { set -x mkdir \$out echo foo > \$out/bar - echo 1... - sleep 2 - echo 200 > ${fake_free}.tmp2 - mv ${fake_free}.tmp2 $fake_free - echo 2... - sleep 2 - echo 3... - sleep 2 - echo 4... + + # Wait for the first build to finish + cat "$fifoLock" ''; } EOF @@ -59,12 +61,19 @@ EOF nix build -v -o $TEST_ROOT/result-A -L "($expr)" \ --min-free 1000 --max-free 2000 --min-free-check-interval 1 & -pid=$! +pid1=$! nix build -v -o $TEST_ROOT/result-B -L "($expr2)" \ - --min-free 1000 --max-free 2000 --min-free-check-interval 1 + --min-free 1000 --max-free 2000 --min-free-check-interval 1 & +pid2=$! -wait "$pid" +# Once the first build is done, unblock the second one. +# If the first build fails, we need to postpone the failure to still allow +# the second one to finish +wait "$pid1" || FIRSTBUILDSTATUS=$? +echo "unlock" > $fifoLock +( exit ${FIRSTBUILDSTATUS:-0} ) +wait "$pid2" [[ foo = $(cat $TEST_ROOT/result-A/bar) ]] [[ foo = $(cat $TEST_ROOT/result-B/bar) ]] diff --git a/tests/gc-concurrent.builder.sh b/tests/gc-concurrent.builder.sh index 0cd67df3a..bb6dcd4cf 100644 --- a/tests/gc-concurrent.builder.sh +++ b/tests/gc-concurrent.builder.sh @@ -1,7 +1,10 @@ +echo "Build started" > "$lockFifo" + mkdir $out echo $(cat $input1/foo)$(cat $input2/bar) > $out/foobar -sleep 10 +# Wait for someone to write on the fifo +cat "$lockFifo" # $out should not have been GC'ed while we were sleeping, but just in # case... diff --git a/tests/gc-concurrent.nix b/tests/gc-concurrent.nix index 21671ea2c..0aba1f983 100644 --- a/tests/gc-concurrent.nix +++ b/tests/gc-concurrent.nix @@ -1,5 +1,7 @@ with import ./config.nix; +{ lockFifo ? null }: + rec { input1 = mkDerivation { @@ -16,6 +18,7 @@ rec { name = "gc-concurrent"; builder = ./gc-concurrent.builder.sh; inherit input1 input2; + inherit lockFifo; }; test2 = mkDerivation { diff --git a/tests/gc-concurrent.sh b/tests/gc-concurrent.sh index d395930ca..2c6622c62 100644 --- a/tests/gc-concurrent.sh +++ b/tests/gc-concurrent.sh @@ -2,7 +2,10 @@ source common.sh clearStore -drvPath1=$(nix-instantiate gc-concurrent.nix -A test1) +lockFifo1=$TEST_ROOT/test1.fifo +mkfifo "$lockFifo1" + +drvPath1=$(nix-instantiate gc-concurrent.nix -A test1 --argstr lockFifo "$lockFifo1") outPath1=$(nix-store -q $drvPath1) drvPath2=$(nix-instantiate gc-concurrent.nix -A test2) @@ -22,19 +25,16 @@ ln -s $outPath3 "$NIX_STATE_DIR"/gcroots/foo2 nix-store -rvv "$drvPath1" & pid1=$! -# Start build #2 in the background after 10 seconds. -(sleep 10 && nix-store -rvv "$drvPath2") & -pid2=$! +# Wait for the build of $drvPath1 to start +cat $lockFifo1 # Run the garbage collector while the build is running. -sleep 6 nix-collect-garbage -# Wait for build #1/#2 to finish. +# Unlock the build of $drvPath1 +echo "" > $lockFifo1 echo waiting for pid $pid1 to finish... wait $pid1 -echo waiting for pid $pid2 to finish... -wait $pid2 # Check that the root of build #1 and its dependencies haven't been # deleted. The should not be deleted by the GC because they were @@ -42,8 +42,9 @@ wait $pid2 cat $outPath1/foobar cat $outPath1/input-2/bar -# Check that build #2 has succeeded. It should succeed because the -# derivation is a GC root. +# Check that the build build $drvPath2 succeeds. +# It should succeed because the derivation is a GC root. +nix-store -rvv "$drvPath2" cat $outPath2/foobar rm -f "$NIX_STATE_DIR"/gcroots/foo* diff --git a/tests/gc-concurrent2.builder.sh b/tests/gc-concurrent2.builder.sh index 4bfb33103..4f6c58b96 100644 --- a/tests/gc-concurrent2.builder.sh +++ b/tests/gc-concurrent2.builder.sh @@ -3,5 +3,3 @@ echo $(cat $input1/foo)$(cat $input2/bar)xyzzy > $out/foobar # Check that the GC hasn't deleted the lock on our output. test -e "$out.lock" - -sleep 6 diff --git a/tests/init.sh b/tests/init.sh index c62c4856a..0c2c0e170 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -18,6 +18,7 @@ build-users-group = keep-derivations = false sandbox = false experimental-features = nix-command flakes +gc-reserved-space = 0 include nix.conf.extra EOF