Implement S3BinaryCacheStore::queryAllValidPaths()

This allows commands like "nix verify --all" or "nix path-info --all"
to work on S3 caches.

Unfortunately, this requires some ugly hackery: when querying the
contents of the bucket, we don't want to have to read every .narinfo
file. But the S3 bucket keys only include the hash part of each store
path, not the name part. So as a special exception
queryAllValidPaths() can now return store paths *without* the name
part, and queryPathInfo() accepts such store paths (returning a
ValidPathInfo object containing the full name).

src/libstore/binary-cache-store.cc

@@ -94,13 +94,15 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
 
     upsertFile(narInfoFile, narInfo->to_string());
 
+    auto hashPart = storePathToHash(narInfo->path);
+
     {
         auto state_(state.lock());
-        state_->pathInfoCache.upsert(narInfo->path, std::shared_ptr<NarInfo>(narInfo));
+        state_->pathInfoCache.upsert(hashPart, std::shared_ptr<NarInfo>(narInfo));
     }
 
     if (diskCache)
-        diskCache->upsertNarInfo(getUri(), std::shared_ptr<NarInfo>(narInfo));
+        diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
 
     stats.narInfoWrite++;
 }
@@ -197,8 +199,6 @@ std::shared_ptr<ValidPathInfo> BinaryCacheStore::queryPathInfoUncached(const Pat
     if (!data) return 0;
 
     auto narInfo = make_ref<NarInfo>(*data, narInfoFile);
-    if (narInfo->path != storePath)
-        throw Error(format("NAR info file for store path ‘%1%’ does not match ‘%2%’") % narInfo->path % storePath);
 
     stats.narInfoRead++;
 

src/libstore/local-store.cc

@@ -579,7 +579,7 @@ uint64_t LocalStore::addValidPath(State & state,
 
     {
         auto state_(Store::state.lock());
-        state_->pathInfoCache.upsert(info.path, std::make_shared<ValidPathInfo>(info));
+        state_->pathInfoCache.upsert(storePathToHash(info.path), std::make_shared<ValidPathInfo>(info));
     }
 
     return id;
@@ -1067,7 +1067,7 @@ void LocalStore::invalidatePath(State & state, const Path & path)
 
     {
         auto state_(Store::state.lock());
-        state_->pathInfoCache.erase(path);
+        state_->pathInfoCache.erase(storePathToHash(path));
     }
 }
 

src/libstore/nar-info-disk-cache.cc

@@ -20,7 +20,8 @@ create table if not exists BinaryCaches (
 
 create table if not exists NARs (
     cache            integer not null,
-    storePath        text not null,
+    hashPart         text not null,
+    namePart         text not null,
     url              text,
     compression      text,
     fileHash         text,
@@ -31,7 +32,7 @@ create table if not exists NARs (
     deriver          text,
     sigs             text,
     timestamp        integer not null,
-    primary key (cache, storePath),
+    primary key (cache, hashPart),
     foreign key (cache) references BinaryCaches(id) on delete cascade
 );
 
@@ -66,7 +67,7 @@ public:
     {
         auto state(_state.lock());
 
-        Path dbPath = getCacheDir() + "/nix/binary-cache-v3.sqlite";
+        Path dbPath = getCacheDir() + "/nix/binary-cache-v4.sqlite";
         createDirs(dirOf(dbPath));
 
         if (sqlite3_open_v2(dbPath.c_str(), &state->db.db,
@@ -92,11 +93,11 @@ public:
             "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ?");
 
         state->insertNAR.create(state->db,
-            "insert or replace into NARs(cache, storePath, url, compression, fileHash, fileSize, narHash, "
-            "narSize, refs, deriver, sigs, timestamp) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
+            "insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, "
+            "narSize, refs, deriver, sigs, timestamp) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
 
         state->queryNAR.create(state->db,
-            "select * from NARs where cache = ? and storePath = ?");
+            "select * from NARs where cache = ? and hashPart = ?");
 
         state->insertNARExistence.create(state->db,
             "insert or replace into NARExistence(cache, storePath, exist, timestamp) values (?, ?, ?, ?)");
@@ -141,13 +142,13 @@ public:
     }
 
     std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
-        const std::string & uri, const Path & storePath) override
+        const std::string & uri, const std::string & hashPart) override
     {
         auto state(_state.lock());
 
         auto queryNAR(state->queryNAR.use()
             (uriToInt(*state, uri))
-            (baseNameOf(storePath)));
+            (hashPart));
 
         if (!queryNAR.next())
             // FIXME: check NARExistence
@@ -157,26 +158,29 @@ public:
 
         // FIXME: implement TTL.
 
-        narInfo->path = storePath;
-        narInfo->url = queryNAR.getStr(2);
-        narInfo->compression = queryNAR.getStr(3);
-        if (!queryNAR.isNull(4))
-            narInfo->fileHash = parseHash(queryNAR.getStr(4));
-        narInfo->fileSize = queryNAR.getInt(5);
-        narInfo->narHash = parseHash(queryNAR.getStr(6));
-        narInfo->narSize = queryNAR.getInt(7);
-        for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
+        auto namePart = queryNAR.getStr(2);
+        narInfo->path = settings.nixStore + "/" +
+            hashPart + (namePart.empty() ? "" : "-" + namePart);
+        narInfo->url = queryNAR.getStr(3);
+        narInfo->compression = queryNAR.getStr(4);
+        if (!queryNAR.isNull(5))
+            narInfo->fileHash = parseHash(queryNAR.getStr(5));
+        narInfo->fileSize = queryNAR.getInt(6);
+        narInfo->narHash = parseHash(queryNAR.getStr(7));
+        narInfo->narSize = queryNAR.getInt(8);
+        for (auto & r : tokenizeString<Strings>(queryNAR.getStr(9), " "))
             narInfo->references.insert(settings.nixStore + "/" + r);
-        if (!queryNAR.isNull(9))
-            narInfo->deriver = settings.nixStore + "/" + queryNAR.getStr(9);
-        for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
+        if (!queryNAR.isNull(10))
+            narInfo->deriver = settings.nixStore + "/" + queryNAR.getStr(10);
+        for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(11), " "))
             narInfo->sigs.insert(sig);
 
         return {oValid, narInfo};
     }
 
     void upsertNarInfo(
-        const std::string & uri, std::shared_ptr<ValidPathInfo> info) override
+        const std::string & uri, const std::string & hashPart,
+        std::shared_ptr<ValidPathInfo> info) override
     {
         auto state(_state.lock());
 
@@ -184,9 +188,12 @@ public:
 
             auto narInfo = std::dynamic_pointer_cast<NarInfo>(info);
 
+            assert(hashPart == storePathToHash(info->path));
+
             state->insertNAR.use()
                 (uriToInt(*state, uri))
-                (baseNameOf(info->path))
+                (hashPart)
+                (storePathToName(info->path))
                 (narInfo ? narInfo->url : "", narInfo != 0)
                 (narInfo ? narInfo->compression : "", narInfo != 0)
                 (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash)

src/libstore/nar-info-disk-cache.hh

@@ -15,10 +15,11 @@ public:
     virtual bool cacheExists(const std::string & uri) = 0;
 
     virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
-        const std::string & uri, const Path & storePath) = 0;
+        const std::string & uri, const std::string & hashPart) = 0;
 
     virtual void upsertNarInfo(
-        const std::string & uri, std::shared_ptr<ValidPathInfo> narInfo) = 0;
+        const std::string & uri, const std::string & hashPart,
+        std::shared_ptr<ValidPathInfo> info) = 0;
 };
 
 /* Return a singleton cache object that can be used concurrently by

src/libstore/s3-binary-cache-store.cc

@@ -10,6 +10,7 @@
 #include <aws/s3/model/GetObjectRequest.h>
 #include <aws/s3/model/HeadObjectRequest.h>
 #include <aws/s3/model/PutObjectRequest.h>
+#include <aws/s3/model/ListObjectsRequest.h>
 
 namespace nix {
 
@@ -164,7 +165,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
     std::shared_ptr<std::string> getFile(const std::string & path)
     {
-        printMsg(lvlDebug, format("fetching ‘s3://%1%/%2%’...") % bucketName % path);
+        debug(format("fetching ‘s3://%1%/%2%’...") % bucketName % path);
 
         auto request =
             Aws::S3::Model::GetObjectRequest()
@@ -204,6 +205,38 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         }
     }
 
+    PathSet queryAllValidPaths() override
+    {
+        PathSet paths;
+        std::string marker;
+
+        do {
+            debug(format("listing bucket ‘s3://%s’ from key ‘%s’...") % bucketName % marker);
+
+            auto res = checkAws(format("AWS error listing bucket ‘%s’") % bucketName,
+                client->ListObjects(
+                    Aws::S3::Model::ListObjectsRequest()
+                    .WithBucket(bucketName)
+                    .WithDelimiter("/")
+                    .WithMarker(marker)));
+
+            auto & contents = res.GetContents();
+
+            debug(format("got %d keys, next marker ‘%s’")
+                % contents.size() % res.GetNextMarker());
+
+            for (auto object : contents) {
+                auto & key = object.GetKey();
+                if (!hasSuffix(key, ".narinfo")) continue;
+                paths.insert(settings.nixStore + "/" + std::string(key, 0, key.size() - 8));
+            }
+
+            marker = res.GetNextMarker();
+        } while (!marker.empty());
+
+        return paths;
+    }
+
 };
 
 static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr<Store> {

src/libstore/store-api.cc

@@ -63,13 +63,16 @@ Path followLinksToStorePath(const Path & path)
 string storePathToName(const Path & path)
 {
     assertStorePath(path);
-    return string(path, settings.nixStore.size() + storePathHashLen + 2);
+    auto l = settings.nixStore.size() + 1 + storePathHashLen;
+    assert(path.size() >= l);
+    return path.size() == l ? "" : string(path, l + 1);
 }
 
 
 string storePathToHash(const Path & path)
 {
     assertStorePath(path);
+    assert(path.size() >= settings.nixStore.size() + 1 + storePathHashLen);
     return string(path, settings.nixStore.size() + 1, storePathHashLen);
 }
 
@@ -234,9 +237,11 @@ std::string Store::getUri()
 
 bool Store::isValidPath(const Path & storePath)
 {
+    auto hashPart = storePathToHash(storePath);
+
     {
         auto state_(state.lock());
-        auto res = state_->pathInfoCache.get(storePath);
+        auto res = state_->pathInfoCache.get(hashPart);
         if (res) {
             stats.narInfoReadAverted++;
             return *res != 0;
@@ -244,10 +249,11 @@ bool Store::isValidPath(const Path & storePath)
     }
 
     if (diskCache) {
-        auto res = diskCache->lookupNarInfo(getUri(), storePath);
+        auto res = diskCache->lookupNarInfo(getUri(), hashPart);
         if (res.first != NarInfoDiskCache::oUnknown) {
+            stats.narInfoReadAverted++;
             auto state_(state.lock());
-            state_->pathInfoCache.upsert(storePath,
+            state_->pathInfoCache.upsert(hashPart,
                 res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
             return res.first == NarInfoDiskCache::oValid;
         }
@@ -261,9 +267,11 @@ bool Store::isValidPath(const Path & storePath)
 
 ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
 {
+    auto hashPart = storePathToHash(storePath);
+
     {
         auto state_(state.lock());
-        auto res = state_->pathInfoCache.get(storePath);
+        auto res = state_->pathInfoCache.get(hashPart);
         if (res) {
             stats.narInfoReadAverted++;
             if (!*res)
@@ -273,12 +281,14 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
     }
 
     if (diskCache) {
-        auto res = diskCache->lookupNarInfo(getUri(), storePath);
+        auto res = diskCache->lookupNarInfo(getUri(), hashPart);
         if (res.first != NarInfoDiskCache::oUnknown) {
+            stats.narInfoReadAverted++;
             auto state_(state.lock());
-            state_->pathInfoCache.upsert(storePath,
+            state_->pathInfoCache.upsert(hashPart,
                 res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
-            if (res.first == NarInfoDiskCache::oInvalid)
+            if (res.first == NarInfoDiskCache::oInvalid ||
+                (res.second->path != storePath && storePathToName(storePath) != ""))
                 throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
             return ref<ValidPathInfo>(res.second);
         }
@@ -287,14 +297,16 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
     auto info = queryPathInfoUncached(storePath);
 
     if (diskCache && info)
-        diskCache->upsertNarInfo(getUri(), info);
+        diskCache->upsertNarInfo(getUri(), hashPart, info);
 
     {
         auto state_(state.lock());
-        state_->pathInfoCache.upsert(storePath, info);
+        state_->pathInfoCache.upsert(hashPart, info);
     }
 
-    if (!info) {
+    if (!info
+        || (info->path != storePath && storePathToName(storePath) != ""))
+    {
         stats.narInfoMissing++;
         throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
     }

src/libstore/store-api.hh

@@ -181,7 +181,7 @@ protected:
 
     struct State
     {
-        LRUCache<Path, std::shared_ptr<ValidPathInfo>> pathInfoCache{64 * 1024};
+        LRUCache<std::string, std::shared_ptr<ValidPathInfo>> pathInfoCache{64 * 1024};
     };
 
     Sync<State> state;
@@ -206,10 +206,15 @@ public:
     /* Query which of the given paths is valid. */
     virtual PathSet queryValidPaths(const PathSet & paths) = 0;
 
-    /* Query the set of all valid paths. */
+    /* Query the set of all valid paths. Note that for some store
+       backends, the name part of store paths may be omitted
+       (i.e. you'll get /nix/store/<hash> rather than
+       /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
+       full store path. */
     virtual PathSet queryAllValidPaths() = 0;
 
-    /* Query information about a valid path. */
+    /* Query information about a valid path. It is permitted to omit
+       the name part of the store path. */
     ref<const ValidPathInfo> queryPathInfo(const Path & path);
 
 protected:

src/nix/path-info.cc

@@ -4,6 +4,7 @@
 #include "store-api.hh"
 
 #include <iomanip>
+#include <algorithm>
 
 using namespace nix;
 
@@ -48,14 +49,14 @@ struct CmdPathInfo : StorePathsCommand
         for (auto & storePath : storePaths)
             pathLen = std::max(pathLen, storePath.size());
 
-        for (auto & storePath : storePaths) {
-            if (!store->isValidPath(storePath))
-                throw Error(format("path ‘%s’ is not valid") % storePath);
+        for (auto storePath : storePaths) {
+            auto info = store->queryPathInfo(storePath);
+            storePath = info->path; // FIXME: screws up padding
 
-            std::cout << storePath << std::string(pathLen - storePath.size(), ' ');
+            std::cout << storePath << std::string(std::max(0, (int) pathLen - (int) storePath.size()), ' ');
 
             if (showSize) {
-                std::cout << '\t' << std::setw(11) << store->queryPathInfo(storePath)->narSize;
+                std::cout << '\t' << std::setw(11) << info->narSize;
             }
 
             if (showClosureSize) {

src/nix/verify.cc

@@ -98,7 +98,7 @@ struct CmdVerify : StorePathsCommand
                 if (!noContents) {
 
                     HashSink sink(info->narHash.type);
-                    store->narFromPath(storePath, sink);
+                    store->narFromPath(info->path, sink);
 
                     auto hash = sink.finish();
 
@@ -106,7 +106,7 @@ struct CmdVerify : StorePathsCommand
                         corrupted = 1;
                         printMsg(lvlError,
                             format("path ‘%s’ was modified! expected hash ‘%s’, got ‘%s’")
-                            % storePath % printHash(info->narHash) % printHash(hash.first));
+                            % info->path % printHash(info->narHash) % printHash(hash.first));
                     }
 
                 }
@@ -138,7 +138,7 @@ struct CmdVerify : StorePathsCommand
                         for (auto & store2 : substituters) {
                             if (validSigs >= actualSigsNeeded) break;
                             try {
-                                doSigs(store2->queryPathInfo(storePath)->sigs);
+                                doSigs(store2->queryPathInfo(info->path)->sigs);
                             } catch (InvalidPath &) {
                             } catch (Error & e) {
                                 printMsg(lvlError, format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what());
@@ -151,7 +151,7 @@ struct CmdVerify : StorePathsCommand
 
                     if (!good) {
                         untrusted++;
-                        printMsg(lvlError, format("path ‘%s’ is untrusted") % storePath);
+                        printMsg(lvlError, format("path ‘%s’ is untrusted") % info->path);
                     }
 
                 }