Cache path info lookups in SQLite

This re-implements the binary cache database in C++, allowing it to be
used by other Store backends, in particular the S3 backend.

src/libstore/binary-cache-store.cc

@@ -59,7 +59,7 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
     narInfo->narSize = nar.size();
     narInfo->narHash = hashString(htSHA256, nar);
 
-    if (info.narHash.type != htUnknown && info.narHash != narInfo->narHash)
+    if (info.narHash && info.narHash != narInfo->narHash)
         throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path);
 
     /* Compress the NAR. */
@@ -96,7 +96,6 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
     {
         auto state_(state.lock());
         state_->pathInfoCache.upsert(narInfo->path, std::shared_ptr<NarInfo>(narInfo));
-        stats.pathInfoCacheSize = state_->pathInfoCache.size();
     }
 
     stats.narInfoWrite++;

src/libstore/derivations.cc

@@ -290,7 +290,7 @@ Hash hashDerivationModulo(Store & store, Derivation drv)
     DerivationInputs inputs2;
     for (auto & i : drv.inputDrvs) {
         Hash h = drvHashes[i.first];
-        if (h.type == htUnknown) {
+        if (!h) {
             assert(store.isValidPath(i.first));
             Derivation drv2 = readDerivation(i.first);
             h = hashDerivationModulo(store, drv2);

src/libstore/download.cc

@@ -225,7 +225,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
 {
     auto url = resolveUri(url_);
 
-    Path cacheDir = getEnv("XDG_CACHE_HOME", getEnv("HOME", "") + "/.cache") + "/nix/tarballs";
+    Path cacheDir = getCacheDir() + "/nix/tarballs";
     createDirs(cacheDir);
 
     string urlHash = printHash32(hashString(htSHA256, url));

src/libstore/http-binary-cache-store.cc

@@ -1,6 +1,7 @@
 #include "binary-cache-store.hh"
 #include "download.hh"
 #include "globals.hh"
+#include "nar-info-disk-cache.hh"
 
 namespace nix {
 
@@ -24,13 +25,23 @@ public:
     {
         if (cacheUri.back() == '/')
             cacheUri.pop_back();
+
+        diskCache = getNarInfoDiskCache();
+    }
+
+    std::string getUri() override
+    {
+        return cacheUri;
     }
 
     void init() override
     {
         // FIXME: do this lazily?
+        if (!diskCache->cacheExists(cacheUri)) {
             if (!fileExists("nix-cache-info"))
                 throw Error(format("‘%s’ does not appear to be a binary cache") % cacheUri);
+            diskCache->createCache(cacheUri);
+        }
     }
 
 protected:

src/libstore/local-store.cc

@@ -580,7 +580,6 @@ uint64_t LocalStore::addValidPath(State & state,
     {
         auto state_(Store::state.lock());
         state_->pathInfoCache.upsert(info.path, std::make_shared<ValidPathInfo>(info));
-        stats.pathInfoCacheSize = state_->pathInfoCache.size();
     }
 
     return id;
@@ -1069,7 +1068,6 @@ void LocalStore::invalidatePath(State & state, const Path & path)
     {
         auto state_(Store::state.lock());
         state_->pathInfoCache.erase(path);
-        stats.pathInfoCacheSize = state_->pathInfoCache.size();
     }
 }
 

src/libstore/nar-info-disk-cache.cc

@@ -0,0 +1,217 @@
+#include "nar-info-disk-cache.hh"
+#include "sync.hh"
+#include "sqlite.hh"
+#include "globals.hh"
+
+#include <sqlite3.h>
+
+namespace nix {
+
+static const char * schema = R"sql(
+
+create table if not exists BinaryCaches (
+    id        integer primary key autoincrement not null,
+    url       text unique not null,
+    timestamp integer not null,
+    storeDir  text not null,
+    wantMassQuery integer not null,
+    priority  integer not null
+);
+
+create table if not exists NARs (
+    cache            integer not null,
+    storePath        text not null,
+    url              text,
+    compression      text,
+    fileHash         text,
+    fileSize         integer,
+    narHash          text,
+    narSize          integer,
+    refs             text,
+    deriver          text,
+    sigs             text,
+    timestamp        integer not null,
+    primary key (cache, storePath),
+    foreign key (cache) references BinaryCaches(id) on delete cascade
+);
+
+create table if not exists NARExistence (
+    cache            integer not null,
+    storePath        text not null,
+    exist            integer not null,
+    timestamp        integer not null,
+    primary key (cache, storePath),
+    foreign key (cache) references BinaryCaches(id) on delete cascade
+);
+
+)sql";
+
+class NarInfoDiskCacheImpl : public NarInfoDiskCache
+{
+public:
+
+    /* How long negative lookups are valid. */
+    const int ttlNegative = 3600;
+
+    struct State
+    {
+        SQLite db;
+        SQLiteStmt insertCache, queryCache, insertNAR, queryNAR, insertNARExistence, queryNARExistence;
+        std::map<std::string, int> caches;
+    };
+
+    Sync<State> _state;
+
+    NarInfoDiskCacheImpl()
+    {
+        auto state(_state.lock());
+
+        Path dbPath = getCacheDir() + "/nix/binary-cache-v3.sqlite";
+        createDirs(dirOf(dbPath));
+
+        if (sqlite3_open_v2(dbPath.c_str(), &state->db.db,
+                SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK)
+            throw Error(format("cannot open store cache ‘%s’") % dbPath);
+
+        if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK)
+            throwSQLiteError(state->db, "setting timeout");
+
+        // We can always reproduce the cache.
+        if (sqlite3_exec(state->db, "pragma synchronous = off", 0, 0, 0) != SQLITE_OK)
+            throwSQLiteError(state->db, "making database asynchronous");
+        if (sqlite3_exec(state->db, "pragma main.journal_mode = truncate", 0, 0, 0) != SQLITE_OK)
+            throwSQLiteError(state->db, "setting journal mode");
+
+        if (sqlite3_exec(state->db, schema, 0, 0, 0) != SQLITE_OK)
+            throwSQLiteError(state->db, "initialising database schema");
+
+        state->insertCache.create(state->db,
+            "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)");
+
+        state->queryCache.create(state->db,
+            "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ?");
+
+        state->insertNAR.create(state->db,
+            "insert or replace into NARs(cache, storePath, url, compression, fileHash, fileSize, narHash, "
+            "narSize, refs, deriver, sigs, timestamp) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
+
+        state->queryNAR.create(state->db,
+            "select * from NARs where cache = ? and storePath = ?");
+
+        state->insertNARExistence.create(state->db,
+            "insert or replace into NARExistence(cache, storePath, exist, timestamp) values (?, ?, ?, ?)");
+
+        state->queryNARExistence.create(state->db,
+            "select exist, timestamp from NARExistence where cache = ? and storePath = ?");
+    }
+
+    int uriToInt(State & state, const std::string & uri)
+    {
+        auto i = state.caches.find(uri);
+        if (i == state.caches.end()) abort();
+        return i->second;
+    }
+
+    void createCache(const std::string & uri) override
+    {
+        auto state(_state.lock());
+
+        // FIXME: race
+
+        state->insertCache.use()(uri)(time(0))(settings.nixStore)(1)(0).exec();
+        assert(sqlite3_changes(state->db) == 1);
+        state->caches[uri] = sqlite3_last_insert_rowid(state->db);
+    }
+
+    bool cacheExists(const std::string & uri) override
+    {
+        auto state(_state.lock());
+
+        auto i = state->caches.find(uri);
+        if (i != state->caches.end()) return true;
+
+        auto queryCache(state->queryCache.use()(uri));
+
+        if (queryCache.next()) {
+            state->caches[uri] = queryCache.getInt(0);
+            return true;
+        }
+
+        return false;
+    }
+
+    std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
+        const std::string & uri, const Path & storePath) override
+    {
+        auto state(_state.lock());
+
+        auto queryNAR(state->queryNAR.use()
+            (uriToInt(*state, uri))
+            (baseNameOf(storePath)));
+
+        if (!queryNAR.next())
+            // FIXME: check NARExistence
+            return {oUnknown, 0};
+
+        auto narInfo = make_ref<NarInfo>();
+
+        // FIXME: implement TTL.
+
+        narInfo->path = storePath;
+        narInfo->url = queryNAR.getStr(2);
+        narInfo->compression = queryNAR.getStr(3);
+        if (!queryNAR.isNull(4))
+            narInfo->fileHash = parseHash(queryNAR.getStr(4));
+        narInfo->fileSize = queryNAR.getInt(5);
+        narInfo->narHash = parseHash(queryNAR.getStr(6));
+        narInfo->narSize = queryNAR.getInt(7);
+        for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
+            narInfo->references.insert(settings.nixStore + "/" + r);
+        if (!queryNAR.isNull(9))
+            narInfo->deriver = settings.nixStore + "/" + queryNAR.getStr(9);
+        for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
+            narInfo->sigs.insert(sig);
+
+        return {oValid, narInfo};
+    }
+
+    void upsertNarInfo(
+        const std::string & uri, std::shared_ptr<ValidPathInfo> info) override
+    {
+        auto state(_state.lock());
+
+        if (info) {
+
+            auto narInfo = std::dynamic_pointer_cast<NarInfo>(info);
+
+            state->insertNAR.use()
+                (uriToInt(*state, uri))
+                (baseNameOf(info->path))
+                (narInfo ? narInfo->url : "", narInfo != 0)
+                (narInfo ? narInfo->compression : "", narInfo != 0)
+                (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash)
+                (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
+                (info->narHash.to_string())
+                (info->narSize)
+                (concatStringsSep(" ", info->shortRefs()))
+                (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "")
+                (concatStringsSep(" ", info->sigs))
+                (time(0)).exec();
+
+        } else {
+            // not implemented
+            abort();
+        }
+    }
+};
+
+ref<NarInfoDiskCache> getNarInfoDiskCache()
+{
+    static Sync<std::shared_ptr<NarInfoDiskCache>> cache;
+
+    auto cache_(cache.lock());
+    if (!*cache_) *cache_ = std::make_shared<NarInfoDiskCacheImpl>();
+    return ref<NarInfoDiskCache>(*cache_);
+}
+
+}

src/libstore/nar-info-disk-cache.hh

@@ -0,0 +1,28 @@
+#pragma once
+
+#include "ref.hh"
+#include "nar-info.hh"
+
+namespace nix {
+
+class NarInfoDiskCache
+{
+public:
+    typedef enum { oValid, oInvalid, oUnknown } Outcome;
+
+    virtual void createCache(const std::string & uri) = 0;
+
+    virtual bool cacheExists(const std::string & uri) = 0;
+
+    virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
+        const std::string & uri, const Path & storePath) = 0;
+
+    virtual void upsertNarInfo(
+        const std::string & uri, std::shared_ptr<ValidPathInfo> narInfo) = 0;
+};
+
+/* Return a singleton cache object that can be used concurrently by
+   multiple threads. */
+ref<NarInfoDiskCache> getNarInfoDiskCache();
+
+}

src/libstore/nar-info.cc

@@ -5,16 +5,16 @@ namespace nix {
 
 NarInfo::NarInfo(const std::string & s, const std::string & whence)
 {
-    auto corrupt = [&]() {
+    auto corrupt = [&]() [[noreturn]] {
         throw Error("NAR info file ‘%1%’ is corrupt");
     };
 
     auto parseHashField = [&](const string & s) {
-        string::size_type colon = s.find(':');
+        try {
-        if (colon == string::npos) corrupt();
+            return parseHash(s);
-        HashType ht = parseHashType(string(s, 0, colon));
+        } catch (BadHash &) {
-        if (ht == htUnknown) corrupt();
+            corrupt();
-        return parseHash16or32(ht, string(s, colon + 1));
+        }
     };
 
     size_t pos = 0;
@@ -103,12 +103,4 @@ std::string NarInfo::to_string() const
     return res;
 }
 
-Strings NarInfo::shortRefs() const
-{
-    Strings refs;
-    for (auto & r : references)
-        refs.push_back(baseNameOf(r));
-    return refs;
-}
-
 }

src/libstore/nar-info.hh

@@ -19,10 +19,6 @@ struct NarInfo : ValidPathInfo
     NarInfo(const std::string & s, const std::string & whence);
 
     std::string to_string() const;
-
-private:
-
-    Strings shortRefs() const;
 };
 
 }

src/libstore/remote-store.cc

@@ -489,7 +489,6 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
     {
         auto state_(Store::state.lock());
         state_->pathInfoCache.clear();
-        stats.pathInfoCacheSize = 0;
     }
 }
 

src/libstore/sqlite.cc

@@ -139,6 +139,11 @@ int64_t SQLiteStmt::Use::getInt(int col)
     return sqlite3_column_int64(stmt, col);
 }
 
+bool SQLiteStmt::Use::isNull(int col)
+{
+    return sqlite3_column_type(stmt, col) == SQLITE_NULL;
+}
+
 SQLiteTxn::SQLiteTxn(sqlite3 * db)
 {
     this->db = db;

src/libstore/sqlite.hh

@@ -58,6 +58,7 @@ struct SQLiteStmt
 
         std::string getStr(int col);
         int64_t getInt(int col);
+        bool isNull(int col);
     };
 
     Use use()

src/libstore/store-api.cc

@@ -2,6 +2,7 @@
 #include "globals.hh"
 #include "store-api.hh"
 #include "util.hh"
+#include "nar-info-disk-cache.hh"
 
 
 namespace nix {
@@ -225,6 +226,12 @@ Path computeStorePathForText(const string & name, const string & s,
 }
 
 
+std::string Store::getUri()
+{
+    return "";
+}
+
+
 bool Store::isValidPath(const Path & storePath)
 {
     {
@@ -236,7 +243,19 @@ bool Store::isValidPath(const Path & storePath)
         }
     }
 
+    if (diskCache) {
+        auto res = diskCache->lookupNarInfo(getUri(), storePath);
+        if (res.first != NarInfoDiskCache::oUnknown) {
+            auto state_(state.lock());
+            state_->pathInfoCache.upsert(storePath,
+                res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
+            return res.first == NarInfoDiskCache::oValid;
+        }
+    }
+
     return isValidPathUncached(storePath);
+
+    // FIXME: insert result into NARExistence table of diskCache.
 }
 
 
@@ -253,12 +272,26 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
         }
     }
 
+    if (diskCache) {
+        auto res = diskCache->lookupNarInfo(getUri(), storePath);
+        if (res.first != NarInfoDiskCache::oUnknown) {
+            auto state_(state.lock());
+            state_->pathInfoCache.upsert(storePath,
+                res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
+            if (res.first == NarInfoDiskCache::oInvalid)
+                throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
+            return ref<ValidPathInfo>(res.second);
+        }
+    }
+
     auto info = queryPathInfoUncached(storePath);
 
+    if (diskCache && info)
+        diskCache->upsertNarInfo(getUri(), info);
+
     {
         auto state_(state.lock());
         state_->pathInfoCache.upsert(storePath, info);
-        stats.pathInfoCacheSize = state_->pathInfoCache.size();
     }
 
     if (!info) {
@@ -303,6 +336,10 @@ string Store::makeValidityRegistration(const PathSet & paths,
 
 const Store::Stats & Store::getStats()
 {
+    {
+        auto state_(state.lock());
+        stats.pathInfoCacheSize = state_->pathInfoCache.size();
+    }
     return stats;
 }
 
@@ -356,7 +393,7 @@ void Store::exportPaths(const Paths & paths,
 
 std::string ValidPathInfo::fingerprint() const
 {
-    if (narSize == 0 || narHash.type == htUnknown)
+    if (narSize == 0 || !narHash)
         throw Error(format("cannot calculate fingerprint of path ‘%s’ because its size/hash is not known")
             % path);
     return
@@ -389,6 +426,15 @@ bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::str
 }
 
 
+Strings ValidPathInfo::shortRefs() const
+{
+    Strings refs;
+    for (auto & r : references)
+        refs.push_back(baseNameOf(r));
+    return refs;
+}
+
+
 }
 
 

src/libstore/store-api.hh

@@ -134,6 +134,8 @@ struct ValidPathInfo
     /* Verify a single signature. */
     bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
 
+    Strings shortRefs() const;
+
     virtual ~ValidPathInfo() { }
 };
 
@@ -170,6 +172,7 @@ struct BuildResult
 struct BasicDerivation;
 struct Derivation;
 class FSAccessor;
+class NarInfoDiskCache;
 
 
 class Store : public std::enable_shared_from_this<Store>
@@ -183,10 +186,14 @@ protected:
 
     Sync<State> state;
 
+    std::shared_ptr<NarInfoDiskCache> diskCache;
+
 public:
 
     virtual ~Store() { }
 
+    virtual std::string getUri();
+
     /* Check whether a path is valid. */
     bool isValidPath(const Path & path);
 

src/libutil/hash.cc

@@ -33,7 +33,7 @@ Hash::Hash(HashType type)
     else if (type == htSHA1) hashSize = sha1HashSize;
     else if (type == htSHA256) hashSize = sha256HashSize;
     else if (type == htSHA512) hashSize = sha512HashSize;
-    else throw Error("unknown hash type");
+    else abort();
     assert(hashSize <= maxHashSize);
     memset(hash, 0, maxHashSize);
 }
@@ -64,6 +64,12 @@ bool Hash::operator < (const Hash & h) const
 }
 
 
+std::string Hash::to_string(bool base32) const
+{
+    return printHashType(type) + ":" + (base32 ? printHash32(*this) : printHash(*this));
+}
+
+
 const string base16Chars = "0123456789abcdef";
 
 
@@ -78,15 +84,28 @@ string printHash(const Hash & hash)
 }
 
 
+Hash parseHash(const string & s)
+{
+    string::size_type colon = s.find(':');
+    if (colon == string::npos)
+        throw BadHash(format("invalid hash ‘%s’") % s);
+    string hts = string(s, 0, colon);
+    HashType ht = parseHashType(hts);
+    if (ht == htUnknown)
+        throw BadHash(format("unknown hash type ‘%s’") % hts);
+    return parseHash16or32(ht, string(s, colon + 1));
+}
+
+
 Hash parseHash(HashType ht, const string & s)
 {
     Hash hash(ht);
     if (s.length() != hash.hashSize * 2)
-        throw Error(format("invalid hash ‘%1%’") % s);
+        throw BadHash(format("invalid hash ‘%1%’") % s);
     for (unsigned int i = 0; i < hash.hashSize; i++) {
         string s2(s, i * 2, 2);
         if (!isxdigit(s2[0]) || !isxdigit(s2[1]))
-            throw Error(format("invalid hash ‘%1%’") % s);
+            throw BadHash(format("invalid hash ‘%1%’") % s);
         std::istringstream str(s2);
         int n;
         str >> std::hex >> n;
@@ -103,6 +122,7 @@ const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz";
 string printHash32(const Hash & hash)
 {
     size_t len = hash.base32Len();
+    assert(len);
 
     string s;
     s.reserve(len);
@@ -139,7 +159,7 @@ Hash parseHash32(HashType ht, const string & s)
         for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */
             if (base32Chars[digit] == c) break;
         if (digit >= 32)
-            throw Error(format("invalid base-32 hash ‘%1%’") % s);
+            throw BadHash(format("invalid base-32 hash ‘%1%’") % s);
         unsigned int b = n * 5;
         unsigned int i = b / 8;
         unsigned int j = b % 8;
@@ -161,7 +181,7 @@ Hash parseHash16or32(HashType ht, const string & s)
         /* base-32 representation */
         hash = parseHash32(ht, s);
     else
-        throw Error(format("hash ‘%1%’ has wrong length for hash type ‘%2%’")
+        throw BadHash(format("hash ‘%1%’ has wrong length for hash type ‘%2%’")
             % s % printHashType(ht));
     return hash;
 }
@@ -322,7 +342,7 @@ string printHashType(HashType ht)
     else if (ht == htSHA1) return "sha1";
     else if (ht == htSHA256) return "sha256";
     else if (ht == htSHA512) return "sha512";
-    else throw Error("cannot print unknown hash type");
+    else abort();
 }
 
 

src/libutil/hash.hh

@@ -7,6 +7,9 @@
 namespace nix {
 
 
+MakeError(BadHash, Error);
+
+
 enum HashType : char { htUnknown, htMD5, htSHA1, htSHA256, htSHA512 };
 
 
@@ -26,12 +29,15 @@ struct Hash
 
     HashType type;
 
-    /* Create an unusable hash object. */
+    /* Create an unset hash object. */
     Hash();
 
     /* Create a zero-filled hash object. */
     Hash(HashType type);
 
+    /* Check whether a hash is set. */
+    operator bool () const { return type != htUnknown; }
+
     /* Check whether two hash are equal. */
     bool operator == (const Hash & h2) const;
 
@@ -52,12 +58,16 @@ struct Hash
     {
         return (hashSize * 8 - 1) / 5 + 1;
     }
+
+    std::string to_string(bool base32 = true) const;
 };
 
 
 /* Convert a hash to a hexadecimal representation. */
 string printHash(const Hash & hash);
 
+Hash parseHash(const string & s);
+
 /* Parse a hexadecimal representation of a hash code. */
 Hash parseHash(HashType ht, const string & s);
 

src/libutil/util.cc

@@ -403,6 +403,18 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix,
 }
 
 
+Path getCacheDir()
+{
+    Path cacheDir = getEnv("XDG_CACHE_HOME");
+    if (cacheDir.empty()) {
+        Path homeDir = getEnv("HOME");
+        if (homeDir.empty()) throw Error("$XDG_CACHE_HOME and $HOME are not set");
+        cacheDir = homeDir + "/.cache";
+    }
+    return cacheDir;
+}
+
+
 Paths createDirs(const Path & path)
 {
     Paths created;

src/libutil/util.hh

@@ -102,6 +102,9 @@ void deletePath(const Path & path, unsigned long long & bytesFreed);
 Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
     bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
 
+/* Return the path to $XDG_CACHE_HOME/.cache. */
+Path getCacheDir();
+
 /* Create a directory and all its parents, if necessary.  Returns the
    list of created directories, in order of creation. */
 Paths createDirs(const Path & path);