Merge pull request #4781 from NixOS/locally_cache_the_remote_realisations

Add a realisations disk cache

src/libstore/binary-cache-store.cc

@@ -450,18 +450,43 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
 
 std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id)
 {
+    if (diskCache) {
+        auto [cacheOutcome, maybeCachedRealisation] =
+            diskCache->lookupRealisation(getUri(), id);
+        switch (cacheOutcome) {
+            case NarInfoDiskCache::oValid:
+                debug("Returning a cached realisation for %s", id.to_string());
+                return *maybeCachedRealisation;
+            case NarInfoDiskCache::oInvalid:
+                debug("Returning a cached missing realisation for %s", id.to_string());
+                return {};
+            case NarInfoDiskCache::oUnknown:
+                break;
+        }
+    }
+
     auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
     auto rawOutputInfo = getFile(outputInfoFilePath);
 
     if (rawOutputInfo) {
-        return {Realisation::fromJSON(
+        auto realisation = Realisation::fromJSON(
-            nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)};
+            nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath);
+
+        if (diskCache)
+            diskCache->upsertRealisation(
+                getUri(), realisation);
+
+        return {realisation};
     } else {
+        if (diskCache)
+            diskCache->upsertAbsentRealisation(getUri(), id);
         return std::nullopt;
     }
 }
 
 void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
+    if (diskCache)
+        diskCache->upsertRealisation(getUri(), info);
     auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi";
     upsertFile(filePath, info.toJSON().dump(), "application/json");
 }

src/libstore/nar-info-disk-cache.cc

@@ -4,6 +4,7 @@
 #include "globals.hh"
 
 #include <sqlite3.h>
+#include <nlohmann/json.hpp>
 
 namespace nix {
 
@@ -38,6 +39,15 @@ create table if not exists NARs (
     foreign key (cache) references BinaryCaches(id) on delete cascade
 );
 
+create table if not exists Realisations (
+    cache integer not null,
+    outputId text not null,
+    content blob, -- Json serialisation of the realisation, or null if the realisation is absent
+    timestamp        integer not null,
+    primary key (cache, outputId),
+    foreign key (cache) references BinaryCaches(id) on delete cascade
+);
+
 create table if not exists LastPurge (
     dummy            text primary key,
     value            integer
@@ -63,7 +73,9 @@ public:
     struct State
     {
         SQLite db;
-        SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, purgeCache;
+        SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR,
+            queryNAR, insertRealisation, insertMissingRealisation,
+            queryRealisation, purgeCache;
         std::map<std::string, Cache> caches;
     };
 
@@ -98,6 +110,26 @@ public:
         state->queryNAR.create(state->db,
             "select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))");
 
+        state->insertRealisation.create(state->db,
+            R"(
+                insert or replace into Realisations(cache, outputId, content, timestamp)
+                    values (?, ?, ?, ?)
+            )");
+
+        state->insertMissingRealisation.create(state->db,
+            R"(
+                insert or replace into Realisations(cache, outputId, timestamp)
+                    values (?, ?, ?)
+            )");
+
+        state->queryRealisation.create(state->db,
+            R"(
+                select content from Realisations
+                    where cache = ? and outputId = ?  and
+                        ((content is null and timestamp > ?) or
+                         (content is not null and timestamp > ?))
+            )");
+
         /* Periodically purge expired entries from the database. */
         retrySQLite<void>([&]() {
             auto now = time(0);
@@ -212,6 +244,38 @@ public:
         });
     }
 
+    std::pair<Outcome, std::shared_ptr<Realisation>> lookupRealisation(
+        const std::string & uri, const DrvOutput & id) override
+    {
+        return retrySQLite<std::pair<Outcome, std::shared_ptr<Realisation>>>(
+            [&]() -> std::pair<Outcome, std::shared_ptr<Realisation>> {
+            auto state(_state.lock());
+
+            auto & cache(getCache(*state, uri));
+
+            auto now = time(0);
+
+            auto queryRealisation(state->queryRealisation.use()
+                (cache.id)
+                (id.to_string())
+                (now - settings.ttlNegativeNarInfoCache)
+                (now - settings.ttlPositiveNarInfoCache));
+
+            if (!queryRealisation.next())
+                return {oUnknown, 0};
+
+            if (queryRealisation.isNull(0))
+                return {oInvalid, 0};
+
+            auto realisation =
+                std::make_shared<Realisation>(Realisation::fromJSON(
+                    nlohmann::json::parse(queryRealisation.getStr(0)),
+                    "Local disk cache"));
+
+            return {oValid, realisation};
+        });
+    }
+
     void upsertNarInfo(
         const std::string & uri, const std::string & hashPart,
         std::shared_ptr<const ValidPathInfo> info) override
@@ -251,6 +315,39 @@ public:
             }
         });
     }
+
+    void upsertRealisation(
+        const std::string & uri,
+        const Realisation & realisation) override
+    {
+        retrySQLite<void>([&]() {
+            auto state(_state.lock());
+
+            auto & cache(getCache(*state, uri));
+
+            state->insertRealisation.use()
+                (cache.id)
+                (realisation.id.to_string())
+                (realisation.toJSON().dump())
+                (time(0)).exec();
+        });
+
+    }
+
+    virtual void upsertAbsentRealisation(
+        const std::string & uri,
+        const DrvOutput & id) override
+    {
+        retrySQLite<void>([&]() {
+            auto state(_state.lock());
+
+            auto & cache(getCache(*state, uri));
+            state->insertMissingRealisation.use()
+                (cache.id)
+                (id.to_string())
+                (time(0)).exec();
+        });
+    }
 };
 
 ref<NarInfoDiskCache> getNarInfoDiskCache()

src/libstore/nar-info-disk-cache.hh

@@ -2,6 +2,7 @@
 
 #include "ref.hh"
 #include "nar-info.hh"
+#include "realisation.hh"
 
 namespace nix {
 
@@ -29,6 +30,15 @@ public:
     virtual void upsertNarInfo(
         const std::string & uri, const std::string & hashPart,
         std::shared_ptr<const ValidPathInfo> info) = 0;
+
+    virtual void upsertRealisation(
+        const std::string & uri,
+        const Realisation & realisation) = 0;
+    virtual void upsertAbsentRealisation(
+        const std::string & uri,
+        const DrvOutput & id) = 0;
+    virtual std::pair<Outcome, std::shared_ptr<Realisation>> lookupRealisation(
+        const std::string & uri, const DrvOutput & id) = 0;
 };
 
 /* Return a singleton cache object that can be used concurrently by

tests/ca/substitute.sh

@@ -45,3 +45,16 @@ if [[ -z "$(ls "$REMOTE_STORE_DIR/realisations")" ]]; then
     echo "Realisations not rebuilt"
     exit 1
 fi
+
+# Test the local realisation disk cache
+buildDrvs --post-build-hook ../push-to-store.sh
+clearStore
+# Add the realisations of rootCA to the cachecache
+clearCacheCache
+export _NIX_FORCE_HTTP=1
+buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0
+# Try rebuilding, but remove the realisations from the remote cache to force
+# using the cachecache
+clearStore
+rm $REMOTE_STORE_DIR/realisations/*
+buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0