Add a realisations disk cache
Similar to the nar-info disk cache (and using the same db). This makes rebuilds muuch faster. - This works regardless of the ca-derivations experimental feature. I could modify the logic to not touch the db if the flag isn’t there, but given that this is a trash-able local cache, it doesn’t seem to be really worth it. - We could unify the `NARs` and `Realisation` tables to only have one generic kv table. This is left as an exercise to the reader. - I didn’t update the cache db version number as the new schema just adds a new table to the previous one, so the db will be transparently migrated and is backwards-compatible. Fix #4746
This commit is contained in:
regnat
parent
fe3a10a9b2
commit
b66234134f
|
|
@ -450,18 +450,43 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
|
|||
|
||||
std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id)
|
||||
{
|
||||
if (diskCache) {
|
||||
auto [cacheOutcome, maybeCachedRealisation] =
|
||||
diskCache->lookupRealisation(getUri(), id);
|
||||
switch (cacheOutcome) {
|
||||
case (NarInfoDiskCache::oValid):
|
||||
debug("Returning a cached realisation for %s", id.to_string());
|
||||
return *maybeCachedRealisation;
|
||||
case (NarInfoDiskCache::oInvalid):
|
||||
debug("Returning a cached missing realisation for %s", id.to_string());
|
||||
return {};
|
||||
case (NarInfoDiskCache::oUnknown):
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
|
||||
auto rawOutputInfo = getFile(outputInfoFilePath);
|
||||
|
||||
if (rawOutputInfo) {
|
||||
return {Realisation::fromJSON(
|
||||
nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)};
|
||||
auto realisation = Realisation::fromJSON(
|
||||
nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath);
|
||||
|
||||
if (diskCache)
|
||||
diskCache->upsertRealisation(
|
||||
getUri(), realisation);
|
||||
|
||||
return {realisation};
|
||||
} else {
|
||||
if (diskCache)
|
||||
diskCache->upsertAbsentRealisation(getUri(), id);
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
|
||||
if (diskCache)
|
||||
diskCache->upsertRealisation(getUri(), info);
|
||||
auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi";
|
||||
upsertFile(filePath, info.toJSON().dump(), "application/json");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
#include "globals.hh"
|
||||
|
||||
#include <sqlite3.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
@ -38,6 +39,16 @@ create table if not exists NARs (
|
|||
foreign key (cache) references BinaryCaches(id) on delete cascade
|
||||
);
|
||||
|
||||
create table if not exists Realisations (
|
||||
cache integer not null,
|
||||
outputId text not null,
|
||||
content blob, -- Json serialisation of the realisation, or empty if present is true
|
||||
timestamp integer not null,
|
||||
present integer not null,
|
||||
primary key (cache, outputId),
|
||||
foreign key (cache) references BinaryCaches(id) on delete cascade
|
||||
);
|
||||
|
||||
create table if not exists LastPurge (
|
||||
dummy text primary key,
|
||||
value integer
|
||||
|
|
@ -63,7 +74,9 @@ public:
|
|||
struct State
|
||||
{
|
||||
SQLite db;
|
||||
SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, purgeCache;
|
||||
SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR,
|
||||
queryNAR, insertRealisation, insertMissingRealisation,
|
||||
queryRealisation, purgeCache;
|
||||
std::map<std::string, Cache> caches;
|
||||
};
|
||||
|
||||
|
|
@ -98,6 +111,26 @@ public:
|
|||
state->queryNAR.create(state->db,
|
||||
"select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))");
|
||||
|
||||
state->insertRealisation.create(state->db,
|
||||
R"(
|
||||
insert or replace into Realisations(cache, outputId, content, timestamp, present)
|
||||
values (?, ?, ?, ?, 1)
|
||||
)");
|
||||
|
||||
state->insertMissingRealisation.create(state->db,
|
||||
R"(
|
||||
insert or replace into Realisations(cache, outputId, timestamp, present)
|
||||
values (?, ?, ?, 0)
|
||||
)");
|
||||
|
||||
state->queryRealisation.create(state->db,
|
||||
R"(
|
||||
select present, content from Realisations
|
||||
where cache = ? and outputId = ? and
|
||||
((present = 0 and timestamp > ?) or
|
||||
(present = 1 and timestamp > ?))
|
||||
)");
|
||||
|
||||
/* Periodically purge expired entries from the database. */
|
||||
retrySQLite<void>([&]() {
|
||||
auto now = time(0);
|
||||
|
|
@ -212,6 +245,38 @@ public:
|
|||
});
|
||||
}
|
||||
|
||||
std::pair<Outcome, std::shared_ptr<Realisation>> lookupRealisation(
|
||||
const std::string & uri, const DrvOutput & id) override
|
||||
{
|
||||
return retrySQLite<std::pair<Outcome, std::shared_ptr<Realisation>>>(
|
||||
[&]() -> std::pair<Outcome, std::shared_ptr<Realisation>> {
|
||||
auto state(_state.lock());
|
||||
|
||||
auto & cache(getCache(*state, uri));
|
||||
|
||||
auto now = time(0);
|
||||
|
||||
auto queryRealisation(state->queryRealisation.use()
|
||||
(cache.id)
|
||||
(id.to_string())
|
||||
(now - settings.ttlNegativeNarInfoCache)
|
||||
(now - settings.ttlPositiveNarInfoCache));
|
||||
|
||||
if (!queryRealisation.next())
|
||||
return {oUnknown, 0};
|
||||
|
||||
if (queryRealisation.getInt(0) == 0)
|
||||
return {oInvalid, 0};
|
||||
|
||||
auto realisation =
|
||||
std::make_shared<Realisation>(Realisation::fromJSON(
|
||||
nlohmann::json::parse(queryRealisation.getStr(1)),
|
||||
"Local disk cache"));
|
||||
|
||||
return {oValid, realisation};
|
||||
});
|
||||
}
|
||||
|
||||
void upsertNarInfo(
|
||||
const std::string & uri, const std::string & hashPart,
|
||||
std::shared_ptr<const ValidPathInfo> info) override
|
||||
|
|
@ -251,6 +316,39 @@ public:
|
|||
}
|
||||
});
|
||||
}
|
||||
|
||||
void upsertRealisation(
|
||||
const std::string & uri,
|
||||
const Realisation & realisation) override
|
||||
{
|
||||
retrySQLite<void>([&]() {
|
||||
auto state(_state.lock());
|
||||
|
||||
auto & cache(getCache(*state, uri));
|
||||
|
||||
state->insertRealisation.use()
|
||||
(cache.id)
|
||||
(realisation.id.to_string())
|
||||
(realisation.toJSON().dump())
|
||||
(time(0)).exec();
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
virtual void upsertAbsentRealisation(
|
||||
const std::string & uri,
|
||||
const DrvOutput & id) override
|
||||
{
|
||||
retrySQLite<void>([&]() {
|
||||
auto state(_state.lock());
|
||||
|
||||
auto & cache(getCache(*state, uri));
|
||||
state->insertMissingRealisation.use()
|
||||
(cache.id)
|
||||
(id.to_string())
|
||||
(time(0)).exec();
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
ref<NarInfoDiskCache> getNarInfoDiskCache()
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
#include "ref.hh"
|
||||
#include "nar-info.hh"
|
||||
#include "realisation.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
@ -29,6 +30,15 @@ public:
|
|||
virtual void upsertNarInfo(
|
||||
const std::string & uri, const std::string & hashPart,
|
||||
std::shared_ptr<const ValidPathInfo> info) = 0;
|
||||
|
||||
virtual void upsertRealisation(
|
||||
const std::string & uri,
|
||||
const Realisation & realisation) = 0;
|
||||
virtual void upsertAbsentRealisation(
|
||||
const std::string & uri,
|
||||
const DrvOutput & id) = 0;
|
||||
virtual std::pair<Outcome, std::shared_ptr<Realisation>> lookupRealisation(
|
||||
const std::string & uri, const DrvOutput & id) = 0;
|
||||
};
|
||||
|
||||
/* Return a singleton cache object that can be used concurrently by
|
||||
|
|
|
|||
|
|
@ -45,3 +45,16 @@ if [[ -z "$(ls "$REMOTE_STORE_DIR/realisations")" ]]; then
|
|||
echo "Realisations not rebuilt"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test the local realisation disk cache
|
||||
buildDrvs --post-build-hook ../push-to-store.sh
|
||||
clearStore
|
||||
# Add the realisations of rootCA to the cachecache
|
||||
clearCacheCache
|
||||
export _NIX_FORCE_HTTP=1
|
||||
buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0
|
||||
# Try rebuilding, but remove the realisations from the remote cache to force
|
||||
# using the cachecache
|
||||
clearStore
|
||||
rm $REMOTE_STORE_DIR/realisations/*
|
||||
buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0
|
||||
|
|
|
|||
Loading…
Reference in a new issue