Merge pull request #4781 from NixOS/locally_cache_the_remote_realisations

Add a realisations disk cache
This commit is contained in:
Eelco Dolstra 2021-05-10 20:37:57 +02:00 committed by GitHub
commit 7f9759b18d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 148 additions and 3 deletions

View file

@ -450,18 +450,43 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id) std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id)
{ {
if (diskCache) {
auto [cacheOutcome, maybeCachedRealisation] =
diskCache->lookupRealisation(getUri(), id);
switch (cacheOutcome) {
case NarInfoDiskCache::oValid:
debug("Returning a cached realisation for %s", id.to_string());
return *maybeCachedRealisation;
case NarInfoDiskCache::oInvalid:
debug("Returning a cached missing realisation for %s", id.to_string());
return {};
case NarInfoDiskCache::oUnknown:
break;
}
}
auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi"; auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
auto rawOutputInfo = getFile(outputInfoFilePath); auto rawOutputInfo = getFile(outputInfoFilePath);
if (rawOutputInfo) { if (rawOutputInfo) {
return {Realisation::fromJSON( auto realisation = Realisation::fromJSON(
nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)}; nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath);
if (diskCache)
diskCache->upsertRealisation(
getUri(), realisation);
return {realisation};
} else { } else {
if (diskCache)
diskCache->upsertAbsentRealisation(getUri(), id);
return std::nullopt; return std::nullopt;
} }
} }
void BinaryCacheStore::registerDrvOutput(const Realisation& info) { void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
if (diskCache)
diskCache->upsertRealisation(getUri(), info);
auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi"; auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi";
upsertFile(filePath, info.toJSON().dump(), "application/json"); upsertFile(filePath, info.toJSON().dump(), "application/json");
} }

View file

@ -4,6 +4,7 @@
#include "globals.hh" #include "globals.hh"
#include <sqlite3.h> #include <sqlite3.h>
#include <nlohmann/json.hpp>
namespace nix { namespace nix {
@ -38,6 +39,15 @@ create table if not exists NARs (
foreign key (cache) references BinaryCaches(id) on delete cascade foreign key (cache) references BinaryCaches(id) on delete cascade
); );
create table if not exists Realisations (
cache integer not null,
outputId text not null,
content blob, -- Json serialisation of the realisation, or null if the realisation is absent
timestamp integer not null,
primary key (cache, outputId),
foreign key (cache) references BinaryCaches(id) on delete cascade
);
create table if not exists LastPurge ( create table if not exists LastPurge (
dummy text primary key, dummy text primary key,
value integer value integer
@ -63,7 +73,9 @@ public:
struct State struct State
{ {
SQLite db; SQLite db;
SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, purgeCache; SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR,
queryNAR, insertRealisation, insertMissingRealisation,
queryRealisation, purgeCache;
std::map<std::string, Cache> caches; std::map<std::string, Cache> caches;
}; };
@ -98,6 +110,26 @@ public:
state->queryNAR.create(state->db, state->queryNAR.create(state->db,
"select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))"); "select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))");
state->insertRealisation.create(state->db,
R"(
insert or replace into Realisations(cache, outputId, content, timestamp)
values (?, ?, ?, ?)
)");
state->insertMissingRealisation.create(state->db,
R"(
insert or replace into Realisations(cache, outputId, timestamp)
values (?, ?, ?)
)");
state->queryRealisation.create(state->db,
R"(
select content from Realisations
where cache = ? and outputId = ? and
((content is null and timestamp > ?) or
(content is not null and timestamp > ?))
)");
/* Periodically purge expired entries from the database. */ /* Periodically purge expired entries from the database. */
retrySQLite<void>([&]() { retrySQLite<void>([&]() {
auto now = time(0); auto now = time(0);
@ -212,6 +244,38 @@ public:
}); });
} }
std::pair<Outcome, std::shared_ptr<Realisation>> lookupRealisation(
const std::string & uri, const DrvOutput & id) override
{
return retrySQLite<std::pair<Outcome, std::shared_ptr<Realisation>>>(
[&]() -> std::pair<Outcome, std::shared_ptr<Realisation>> {
auto state(_state.lock());
auto & cache(getCache(*state, uri));
auto now = time(0);
auto queryRealisation(state->queryRealisation.use()
(cache.id)
(id.to_string())
(now - settings.ttlNegativeNarInfoCache)
(now - settings.ttlPositiveNarInfoCache));
if (!queryRealisation.next())
return {oUnknown, 0};
if (queryRealisation.isNull(0))
return {oInvalid, 0};
auto realisation =
std::make_shared<Realisation>(Realisation::fromJSON(
nlohmann::json::parse(queryRealisation.getStr(0)),
"Local disk cache"));
return {oValid, realisation};
});
}
void upsertNarInfo( void upsertNarInfo(
const std::string & uri, const std::string & hashPart, const std::string & uri, const std::string & hashPart,
std::shared_ptr<const ValidPathInfo> info) override std::shared_ptr<const ValidPathInfo> info) override
@ -251,6 +315,39 @@ public:
} }
}); });
} }
void upsertRealisation(
const std::string & uri,
const Realisation & realisation) override
{
retrySQLite<void>([&]() {
auto state(_state.lock());
auto & cache(getCache(*state, uri));
state->insertRealisation.use()
(cache.id)
(realisation.id.to_string())
(realisation.toJSON().dump())
(time(0)).exec();
});
}
virtual void upsertAbsentRealisation(
const std::string & uri,
const DrvOutput & id) override
{
retrySQLite<void>([&]() {
auto state(_state.lock());
auto & cache(getCache(*state, uri));
state->insertMissingRealisation.use()
(cache.id)
(id.to_string())
(time(0)).exec();
});
}
}; };
ref<NarInfoDiskCache> getNarInfoDiskCache() ref<NarInfoDiskCache> getNarInfoDiskCache()

View file

@ -2,6 +2,7 @@
#include "ref.hh" #include "ref.hh"
#include "nar-info.hh" #include "nar-info.hh"
#include "realisation.hh"
namespace nix { namespace nix {
@ -29,6 +30,15 @@ public:
virtual void upsertNarInfo( virtual void upsertNarInfo(
const std::string & uri, const std::string & hashPart, const std::string & uri, const std::string & hashPart,
std::shared_ptr<const ValidPathInfo> info) = 0; std::shared_ptr<const ValidPathInfo> info) = 0;
virtual void upsertRealisation(
const std::string & uri,
const Realisation & realisation) = 0;
virtual void upsertAbsentRealisation(
const std::string & uri,
const DrvOutput & id) = 0;
virtual std::pair<Outcome, std::shared_ptr<Realisation>> lookupRealisation(
const std::string & uri, const DrvOutput & id) = 0;
}; };
/* Return a singleton cache object that can be used concurrently by /* Return a singleton cache object that can be used concurrently by

View file

@ -45,3 +45,16 @@ if [[ -z "$(ls "$REMOTE_STORE_DIR/realisations")" ]]; then
echo "Realisations not rebuilt" echo "Realisations not rebuilt"
exit 1 exit 1
fi fi
# Test the local realisation disk cache
buildDrvs --post-build-hook ../push-to-store.sh
clearStore
# Add the realisations of rootCA to the cachecache
clearCacheCache
export _NIX_FORCE_HTTP=1
buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0
# Try rebuilding, but remove the realisations from the remote cache to force
# using the cachecache
clearStore
rm $REMOTE_STORE_DIR/realisations/*
buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0