diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index a58b7733f..964c4017e 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -2,6 +2,8 @@
 #include "globals.hh"
 #include "nar-info-disk-cache.hh"
 
+#include <atomic>
+
 namespace nix {
 
 struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
@@ -50,7 +52,8 @@ protected:
         const std::string & mimeType) override
     {
         auto path2 = binaryCacheDir + "/" + path;
-        Path tmp = path2 + ".tmp." + std::to_string(getpid());
+        static std::atomic<int> counter{0};
+        Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter);
         AutoDelete del(tmp, false);
         StreamToSourceAdapter source(istream);
         writeFile(tmp, source);
diff --git a/tests/ca/substitute.sh b/tests/ca/substitute.sh
index 79a6ef8b1..b44fe499a 100644
--- a/tests/ca/substitute.sh
+++ b/tests/ca/substitute.sh
@@ -6,6 +6,8 @@ source common.sh
 
 sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
 
+rm -rf $TEST_ROOT/binary_cache
+
 export REMOTE_STORE=file://$TEST_ROOT/binary_cache
 
 buildDrvs () {
@@ -13,6 +15,7 @@ buildDrvs () {
 }
 
 # Populate the remote cache
+clearStore
 buildDrvs --post-build-hook ../push-to-store.sh
 
 # Restart the build on an empty store, ensuring that we don't build