Merge remote-tracking branch 'origin/master' into substitute-other-storedir

mk/run_test.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -u
+
+red=""
+green=""
+yellow=""
+normal=""
+
+post_run_msg="ran test $1..."
+if [ -t 1 ]; then
+    red=""
+    green=""
+    yellow=""
+    normal=""
+fi
+(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
+log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
+status=$?
+if [ $status -eq 0 ]; then
+  echo "$post_run_msg [${green}PASS$normal]"
+elif [ $status -eq 99 ]; then
+  echo "$post_run_msg [${yellow}SKIP$normal]"
+else
+  echo "$post_run_msg [${red}FAIL$normal]"
+  echo "$log" | sed 's/^/    /'
+  exit "$status"
+fi

mk/tests.mk

@@ -1,45 +1,12 @@
 # Run program $1 as part of ‘make installcheck’.
 define run-install-test
 
-  installcheck: $1
+  installcheck: $1.test
 
-  _installcheck-list += $1
+  .PHONY: $1.test
+  $1.test: $1 tests/common.sh tests/init.sh
+	@env TEST_NAME=$1 TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1
 
 endef
 
-# Color code from https://unix.stackexchange.com/a/10065
-installcheck:
-	@total=0; failed=0; \
-	red=""; \
-	green=""; \
-	yellow=""; \
-	normal=""; \
-	if [ -t 1 ]; then \
-		red=""; \
-		green=""; \
-		yellow=""; \
-		normal=""; \
-	fi; \
-	for i in $(_installcheck-list); do \
-	  total=$$((total + 1)); \
-	  printf "running test $$i..."; \
-	  log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \
-	  status=$$?; \
-	  if [ $$status -eq 0 ]; then \
-	    echo " [$${green}PASS$$normal]"; \
-	  elif [ $$status -eq 99 ]; then \
-	    echo " [$${yellow}SKIP$$normal]"; \
-	  else \
-	    echo " [$${red}FAIL$$normal]"; \
-	    echo "$$log" | sed 's/^/    /'; \
-	    failed=$$((failed + 1)); \
-	  fi; \
-	done; \
-	if [ "$$failed" != 0 ]; then \
-	  echo "$${red}$$failed out of $$total tests failed $$normal"; \
-	  exit 1; \
-	else \
-		echo "$${green}All tests succeeded$$normal"; \
-	fi
-
 .PHONY: check installcheck

src/libstore/build.cc

@@ -2753,8 +2753,8 @@ struct RestrictedStore : public LocalFSStore
     void queryReferrers(const StorePath & path, StorePathSet & referrers) override
     { }
 
-    StorePathSet queryDerivationOutputs(const StorePath & path) override
+    OutputPathMap queryDerivationOutputMap(const StorePath & path) override
-    { throw Error("queryDerivationOutputs"); }
+    { throw Error("queryDerivationOutputMap"); }
 
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { throw Error("queryPathFromHashPart"); }

src/libstore/daemon.cc

@@ -347,6 +347,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
+    case wopQueryDerivationOutputMap: {
+        auto path = store->parseStorePath(readString(from));
+        logger->startWork();
+        OutputPathMap outputs = store->queryDerivationOutputMap(path);
+        logger->stopWork();
+        writeOutputPathMap(*store, to, outputs);
+        break;
+    }
+
     case wopQueryDeriver: {
         auto path = store->parseStorePath(readString(from));
         logger->startWork();

src/libstore/local-store.cc

@@ -774,17 +774,20 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
 }
 
 
-StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path)
+OutputPathMap LocalStore::queryDerivationOutputMap(const StorePath & path)
 {
-    return retrySQLite<StorePathSet>([&]() {
+    return retrySQLite<OutputPathMap>([&]() {
         auto state(_state.lock());
 
         auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
             (queryValidPathId(*state, path)));
 
-        StorePathSet outputs;
+        OutputPathMap outputs;
         while (useQueryDerivationOutputs.next())
-            outputs.insert(parseStorePath(useQueryDerivationOutputs.getStr(1)));
+            outputs.emplace(
+                useQueryDerivationOutputs.getStr(0),
+                parseStorePath(useQueryDerivationOutputs.getStr(1))
+            );
 
         return outputs;
     });

src/libstore/local-store.hh

@@ -133,7 +133,7 @@ public:
 
     StorePathSet queryValidDerivers(const StorePath & path) override;
 
-    StorePathSet queryDerivationOutputs(const StorePath & path) override;
+    OutputPathMap queryDerivationOutputMap(const StorePath & path) override;
 
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 

src/libstore/path.hh

@@ -62,6 +62,7 @@ public:
 
 typedef std::set<StorePath> StorePathSet;
 typedef std::vector<StorePath> StorePaths;
+typedef std::map<string, StorePath> OutputPathMap;
 
 typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
 

src/libstore/remote-store.cc

@@ -56,6 +56,29 @@ void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap &
     }
 }
 
+std::map<string, StorePath> readOutputPathMap(const Store & store, Source & from)
+{
+    std::map<string, StorePath> pathMap;
+    auto rawInput = readStrings<Strings>(from);
+    if (rawInput.size() % 2)
+        throw Error("got an odd number of elements from the daemon when trying to read a output path map");
+    auto curInput = rawInput.begin();
+    while (curInput != rawInput.end()) {
+        auto thisKey = *curInput++;
+        auto thisValue = *curInput++;
+        pathMap.emplace(thisKey, store.parseStorePath(thisValue));
+    }
+    return pathMap;
+}
+
+void writeOutputPathMap(const Store & store, Sink & out, const std::map<string, StorePath> & pathMap)
+{
+    out << 2*pathMap.size();
+    for (auto & i : pathMap) {
+        out << i.first;
+        out << store.printStorePath(i.second);
+    }
+}
 
 /* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
@@ -435,12 +458,24 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
 StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
 {
     auto conn(getConnection());
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) {
+        return Store::queryDerivationOutputs(path);
+    }
     conn->to << wopQueryDerivationOutputs << printStorePath(path);
     conn.processStderr();
     return readStorePaths<StorePathSet>(*this, conn->from);
 }
 
 
+OutputPathMap RemoteStore::queryDerivationOutputMap(const StorePath & path)
+{
+    auto conn(getConnection());
+    conn->to << wopQueryDerivationOutputMap << printStorePath(path);
+    conn.processStderr();
+    return readOutputPathMap(*this, conn->from);
+
+}
+
 std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
 {
     auto conn(getConnection());

src/libstore/remote-store.hh

@@ -51,6 +51,7 @@ public:
 
     StorePathSet queryDerivationOutputs(const StorePath & path) override;
 
+    OutputPathMap queryDerivationOutputMap(const StorePath & path) override;
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
 
     StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;

src/libstore/store-api.cc

@@ -259,6 +259,16 @@ bool Store::PathInfoCacheValue::isKnownNow()
     return std::chrono::steady_clock::now() < time_point + ttl;
 }
 
+StorePathSet Store::queryDerivationOutputs(const StorePath & path)
+{
+    auto outputMap = this->queryDerivationOutputMap(path);
+    StorePathSet outputPaths;
+    for (auto & i: outputMap) {
+        outputPaths.emplace(std::move(i.second));
+    }
+    return outputPaths;
+}
+
 bool Store::isValidPath(const StorePath & storePath)
 {
     std::string hashPart(storePath.hashPart());

src/libstore/store-api.hh

@@ -422,8 +422,11 @@ public:
     virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; };
 
     /* Query the outputs of the derivation denoted by `path'. */
-    virtual StorePathSet queryDerivationOutputs(const StorePath & path)
+    virtual StorePathSet queryDerivationOutputs(const StorePath & path);
-    { unsupported("queryDerivationOutputs"); }
+
+    /* Query the mapping outputName=>outputPath for the given derivation */
+    virtual OutputPathMap queryDerivationOutputMap(const StorePath & path)
+    { unsupported("queryDerivationOutputMap"); }
 
     /* Query the full store path given the hash part of a valid store
        path, or empty if the path doesn't exist. */

src/libstore/worker-protocol.hh

@@ -30,7 +30,7 @@ typedef enum {
     wopSetOptions = 19,
     wopCollectGarbage = 20,
     wopQuerySubstitutablePathInfo = 21,
-    wopQueryDerivationOutputs = 22,
+    wopQueryDerivationOutputs = 22, // obsolete
     wopQueryAllValidPaths = 23,
     wopQueryFailedPaths = 24,
     wopClearFailedPaths = 25,
@@ -49,6 +49,7 @@ typedef enum {
     wopNarFromPath = 38,
     wopAddToStoreNar = 39,
     wopQueryMissing = 40,
+    wopQueryDerivationOutputMap = 41,
 } WorkerOp;
 
 
@@ -73,5 +74,6 @@ StorePathCAMap readStorePathCAMap(const Store & store, Source & from);
 
 void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths);
 
+void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths);
 
 }

tests/common.sh.in

@@ -1,6 +1,6 @@
 set -e
 
-export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)
+export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default}
 export NIX_STORE_DIR
 if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then
     # Maybe the build directory is symlinked.

tests/gc-auto.sh

@@ -13,24 +13,32 @@ fake_free=$TEST_ROOT/fake-free
 export _NIX_TEST_FREE_SPACE_FILE=$fake_free
 echo 1100 > $fake_free
 
+fifoLock=$TEST_ROOT/fifoLock
+mkfifo "$fifoLock"
+
 expr=$(cat <<EOF
 with import ./config.nix; mkDerivation {
   name = "gc-A";
   buildCommand = ''
     set -x
     [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 3 ]]
+
     mkdir \$out
     echo foo > \$out/bar
-    echo 1...
+
-    sleep 2
+    # Pretend that we run out of space
-    echo 200 > ${fake_free}.tmp1
+    echo 100 > ${fake_free}.tmp1
     mv ${fake_free}.tmp1 $fake_free
-    echo 2...
+
-    sleep 2
+    # Wait for the GC to run
-    echo 3...
+    for i in {1..20}; do
-    sleep 2
+        echo ''\${i}...
-    echo 4...
+        if [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]]; then
-    [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]]
+            exit 0
+        fi
+        sleep 1
+    done
+    exit 1
   '';
 }
 EOF
@@ -43,15 +51,9 @@ with import ./config.nix; mkDerivation {
     set -x
     mkdir \$out
     echo foo > \$out/bar
-    echo 1...
+
-    sleep 2
+    # Wait for the first build to finish
-    echo 200 > ${fake_free}.tmp2
+    cat "$fifoLock"
-    mv ${fake_free}.tmp2 $fake_free
-    echo 2...
-    sleep 2
-    echo 3...
-    sleep 2
-    echo 4...
   '';
 }
 EOF
@@ -59,12 +61,19 @@ EOF
 
 nix build -v -o $TEST_ROOT/result-A -L "($expr)" \
     --min-free 1000 --max-free 2000 --min-free-check-interval 1 &
-pid=$!
+pid1=$!
 
 nix build -v -o $TEST_ROOT/result-B -L "($expr2)" \
-    --min-free 1000 --max-free 2000 --min-free-check-interval 1
+    --min-free 1000 --max-free 2000 --min-free-check-interval 1 &
+pid2=$!
 
-wait "$pid"
+# Once the first build is done, unblock the second one.
+# If the first build fails, we need to postpone the failure to still allow
+# the second one to finish
+wait "$pid1" || FIRSTBUILDSTATUS=$?
+echo "unlock" > $fifoLock
+( exit ${FIRSTBUILDSTATUS:-0} )
+wait "$pid2"
 
 [[ foo = $(cat $TEST_ROOT/result-A/bar) ]]
 [[ foo = $(cat $TEST_ROOT/result-B/bar) ]]

tests/gc-concurrent.builder.sh

@@ -1,7 +1,10 @@
+echo "Build started" > "$lockFifo"
+
 mkdir $out
 echo $(cat $input1/foo)$(cat $input2/bar) > $out/foobar
 
-sleep 10
+# Wait for someone to write on the fifo
+cat "$lockFifo"
 
 # $out should not have been GC'ed while we were sleeping, but just in
 # case...

tests/gc-concurrent.nix

@@ -1,5 +1,7 @@
 with import ./config.nix;
 
+{ lockFifo ? null }:
+
 rec {
 
   input1 = mkDerivation {
@@ -16,6 +18,7 @@ rec {
     name = "gc-concurrent";
     builder = ./gc-concurrent.builder.sh;
     inherit input1 input2;
+    inherit lockFifo;
   };
 
   test2 = mkDerivation {

tests/gc-concurrent.sh

@@ -2,7 +2,10 @@ source common.sh
 
 clearStore
 
-drvPath1=$(nix-instantiate gc-concurrent.nix -A test1)
+lockFifo1=$TEST_ROOT/test1.fifo
+mkfifo "$lockFifo1"
+
+drvPath1=$(nix-instantiate gc-concurrent.nix -A test1 --argstr lockFifo "$lockFifo1")
 outPath1=$(nix-store -q $drvPath1)
 
 drvPath2=$(nix-instantiate gc-concurrent.nix -A test2)
@@ -22,19 +25,16 @@ ln -s $outPath3 "$NIX_STATE_DIR"/gcroots/foo2
 nix-store -rvv "$drvPath1" &
 pid1=$!
 
-# Start build #2 in the background after 10 seconds.
+# Wait for the build of $drvPath1 to start
-(sleep 10 && nix-store -rvv "$drvPath2") &
+cat $lockFifo1
-pid2=$!
 
 # Run the garbage collector while the build is running.
-sleep 6
 nix-collect-garbage
 
-# Wait for build #1/#2 to finish.
+# Unlock the build of $drvPath1
+echo "" > $lockFifo1
 echo waiting for pid $pid1 to finish...
 wait $pid1
-echo waiting for pid $pid2 to finish...
-wait $pid2
 
 # Check that the root of build #1 and its dependencies haven't been
 # deleted.  The should not be deleted by the GC because they were
@@ -42,8 +42,9 @@ wait $pid2
 cat $outPath1/foobar
 cat $outPath1/input-2/bar
 
-# Check that build #2 has succeeded.  It should succeed because the
+# Check that the build build $drvPath2 succeeds.
-# derivation is a GC root.
+# It should succeed because the derivation is a GC root.
+nix-store -rvv "$drvPath2"
 cat $outPath2/foobar
 
 rm -f "$NIX_STATE_DIR"/gcroots/foo*

tests/gc-concurrent2.builder.sh

@@ -3,5 +3,3 @@ echo $(cat $input1/foo)$(cat $input2/bar)xyzzy > $out/foobar
 
 # Check that the GC hasn't deleted the lock on our output.
 test -e "$out.lock"
-
-sleep 6

tests/init.sh

@@ -18,6 +18,7 @@ build-users-group =
 keep-derivations = false
 sandbox = false
 experimental-features = nix-command flakes
+gc-reserved-space = 0
 include nix.conf.extra
 EOF