diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 3cc2659fb..dba56c011 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -383,7 +383,8 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v)
     try {
         auto _ = state.realiseContext(context); // FIXME: Handle CA derivations
     } catch (InvalidPathError & e) {
-        state.error<EvalError>("cannot execute '%1%', since path '%2%' is not valid", program, e.path).atPos(pos).debugThrow();
+        e.addTrace(state.positions[pos], "while realising the context for builtins.exec");
+        throw;
     }
 
     auto output = runProgram(program, true, commandArgs);
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 3600eca60..7df55a32d 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -170,7 +170,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
             if (ref != info.path)
                 queryPathInfo(ref);
         } catch (InvalidPath &) {
-            throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid",
+            throw Error("cannot add '%s' to the binary cache because the reference '%s' does not exist",
                 printStorePath(info.path), printStorePath(ref));
         }
 
diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index b224fc3e9..56f13920c 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -23,7 +23,7 @@ struct LocalStoreAccessor : public FSAccessor
     {
         auto storePath = store->toStorePath(path).first;
         if (requireValidPath && !store->isValidPath(storePath))
-            throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
+            throw InvalidPath("path '%1%' does not exist in the store", store->printStorePath(storePath));
         return store->getRealStoreDir() + std::string(path, store->storeDir.size());
     }
 
@@ -81,7 +81,7 @@ ref<FSAccessor> LocalFSStore::getFSAccessor()
 void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)
 {
     if (!isValidPath(path))
-        throw Error("path '%s' is not valid", printStorePath(path));
+        throw Error("path '%s' does not exist in store", printStorePath(path));
     dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink);
 }
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index bae5fad7b..5bdf0362d 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -907,7 +907,7 @@ std::shared_ptr<const ValidPathInfo> LocalStore::queryPathInfoInternal(State & s
     try {
         narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1));
     } catch (BadHash & e) {
-        throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what());
+        throw BadStorePath("bad hash in store path '%s': %s", printStorePath(path), e.what());
     }
 
     auto info = std::make_shared<ValidPathInfo>(path, narHash);
@@ -957,8 +957,8 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
 uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path)
 {
     auto use(state.stmts->QueryPathInfo.use()(printStorePath(path)));
-    if (!use.next())
-        throw InvalidPath("path '%s' is not valid", printStorePath(path));
+    if (!use.next()) // TODO: I guess if SQLITE got corrupted..?
+        throw InvalidPath("path '%s' does not exist", printStorePath(path));
     return use.getInt(0);
 }
 
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index c8e20b3b5..8da8b9774 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -55,7 +55,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, boo
     auto [storePath, restPath] = store->toStorePath(path);
 
     if (requireValidPath && !store->isValidPath(storePath))
-        throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
+        throw InvalidPath("path '%1%' does not exist in remote store", store->printStorePath(storePath));
 
     auto i = nars.find(std::string(storePath.hashPart()));
     if (i != nars.end()) return {i->second, restPath};