From b66234134fadc720d6c177d62e6ce4c9f4093a89 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 6 May 2021 16:45:09 +0200 Subject: [PATCH 1/3] Add a realisations disk cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Similar to the nar-info disk cache (and using the same db). This makes rebuilds muuch faster. - This works regardless of the ca-derivations experimental feature. I could modify the logic to not touch the db if the flag isn’t there, but given that this is a trash-able local cache, it doesn’t seem to be really worth it. - We could unify the `NARs` and `Realisation` tables to only have one generic kv table. This is left as an exercise to the reader. - I didn’t update the cache db version number as the new schema just adds a new table to the previous one, so the db will be transparently migrated and is backwards-compatible. Fix #4746 --- src/libstore/binary-cache-store.cc | 29 +++++++- src/libstore/nar-info-disk-cache.cc | 100 +++++++++++++++++++++++++++- src/libstore/nar-info-disk-cache.hh | 10 +++ tests/ca/substitute.sh | 13 ++++ 4 files changed, 149 insertions(+), 3 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 09e1c254b..312630520 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -450,18 +450,43 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s std::optional BinaryCacheStore::queryRealisation(const DrvOutput & id) { + if (diskCache) { + auto [cacheOutcome, maybeCachedRealisation] = + diskCache->lookupRealisation(getUri(), id); + switch (cacheOutcome) { + case (NarInfoDiskCache::oValid): + debug("Returning a cached realisation for %s", id.to_string()); + return *maybeCachedRealisation; + case (NarInfoDiskCache::oInvalid): + debug("Returning a cached missing realisation for %s", id.to_string()); + return {}; + case (NarInfoDiskCache::oUnknown): + break; + } + } + auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi"; auto rawOutputInfo = getFile(outputInfoFilePath); if (rawOutputInfo) { - return {Realisation::fromJSON( - nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)}; + auto realisation = Realisation::fromJSON( + nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath); + + if (diskCache) + diskCache->upsertRealisation( + getUri(), realisation); + + return {realisation}; } else { + if (diskCache) + diskCache->upsertAbsentRealisation(getUri(), id); return std::nullopt; } } void BinaryCacheStore::registerDrvOutput(const Realisation& info) { + if (diskCache) + diskCache->upsertRealisation(getUri(), info); auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi"; upsertFile(filePath, info.toJSON().dump(), "application/json"); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 1d8d2d57e..84ce7972f 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -4,6 +4,7 @@ #include "globals.hh" #include +#include namespace nix { @@ -38,6 +39,16 @@ create table if not exists NARs ( foreign key (cache) references BinaryCaches(id) on delete cascade ); +create table if not exists Realisations ( + cache integer not null, + outputId text not null, + content blob, -- Json serialisation of the realisation, or empty if present is true + timestamp integer not null, + present integer not null, + primary key (cache, outputId), + foreign key (cache) references BinaryCaches(id) on delete cascade +); + create table if not exists LastPurge ( dummy text primary key, value integer @@ -63,7 +74,9 @@ public: struct State { SQLite db; - SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, purgeCache; + SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, + queryNAR, insertRealisation, insertMissingRealisation, + queryRealisation, purgeCache; std::map caches; }; @@ -98,6 +111,26 @@ public: state->queryNAR.create(state->db, "select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))"); + state->insertRealisation.create(state->db, + R"( + insert or replace into Realisations(cache, outputId, content, timestamp, present) + values (?, ?, ?, ?, 1) + )"); + + state->insertMissingRealisation.create(state->db, + R"( + insert or replace into Realisations(cache, outputId, timestamp, present) + values (?, ?, ?, 0) + )"); + + state->queryRealisation.create(state->db, + R"( + select present, content from Realisations + where cache = ? and outputId = ? and + ((present = 0 and timestamp > ?) or + (present = 1 and timestamp > ?)) + )"); + /* Periodically purge expired entries from the database. */ retrySQLite([&]() { auto now = time(0); @@ -212,6 +245,38 @@ public: }); } + std::pair> lookupRealisation( + const std::string & uri, const DrvOutput & id) override + { + return retrySQLite>>( + [&]() -> std::pair> { + auto state(_state.lock()); + + auto & cache(getCache(*state, uri)); + + auto now = time(0); + + auto queryRealisation(state->queryRealisation.use() + (cache.id) + (id.to_string()) + (now - settings.ttlNegativeNarInfoCache) + (now - settings.ttlPositiveNarInfoCache)); + + if (!queryRealisation.next()) + return {oUnknown, 0}; + + if (queryRealisation.getInt(0) == 0) + return {oInvalid, 0}; + + auto realisation = + std::make_shared(Realisation::fromJSON( + nlohmann::json::parse(queryRealisation.getStr(1)), + "Local disk cache")); + + return {oValid, realisation}; + }); + } + void upsertNarInfo( const std::string & uri, const std::string & hashPart, std::shared_ptr info) override @@ -251,6 +316,39 @@ public: } }); } + + void upsertRealisation( + const std::string & uri, + const Realisation & realisation) override + { + retrySQLite([&]() { + auto state(_state.lock()); + + auto & cache(getCache(*state, uri)); + + state->insertRealisation.use() + (cache.id) + (realisation.id.to_string()) + (realisation.toJSON().dump()) + (time(0)).exec(); + }); + + } + + virtual void upsertAbsentRealisation( + const std::string & uri, + const DrvOutput & id) override + { + retrySQLite([&]() { + auto state(_state.lock()); + + auto & cache(getCache(*state, uri)); + state->insertMissingRealisation.use() + (cache.id) + (id.to_string()) + (time(0)).exec(); + }); + } }; ref getNarInfoDiskCache() diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 04de2c5eb..2dcaa76a4 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -2,6 +2,7 @@ #include "ref.hh" #include "nar-info.hh" +#include "realisation.hh" namespace nix { @@ -29,6 +30,15 @@ public: virtual void upsertNarInfo( const std::string & uri, const std::string & hashPart, std::shared_ptr info) = 0; + + virtual void upsertRealisation( + const std::string & uri, + const Realisation & realisation) = 0; + virtual void upsertAbsentRealisation( + const std::string & uri, + const DrvOutput & id) = 0; + virtual std::pair> lookupRealisation( + const std::string & uri, const DrvOutput & id) = 0; }; /* Return a singleton cache object that can be used concurrently by diff --git a/tests/ca/substitute.sh b/tests/ca/substitute.sh index 737c851a5..dfc4ea68e 100644 --- a/tests/ca/substitute.sh +++ b/tests/ca/substitute.sh @@ -45,3 +45,16 @@ if [[ -z "$(ls "$REMOTE_STORE_DIR/realisations")" ]]; then echo "Realisations not rebuilt" exit 1 fi + +# Test the local realisation disk cache +buildDrvs --post-build-hook ../push-to-store.sh +clearStore +# Add the realisations of rootCA to the cachecache +clearCacheCache +export _NIX_FORCE_HTTP=1 +buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 +# Try rebuilding, but remove the realisations from the remote cache to force +# using the cachecache +clearStore +rm $REMOTE_STORE_DIR/realisations/* +buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 From ab96c1ee504674ddbb0a09796af87d898e2bf753 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 10 May 2021 17:36:49 +0200 Subject: [PATCH 2/3] Remove useless parents I never remember the exact syntax of the `switch` statement --- src/libstore/binary-cache-store.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 312630520..df401e6f4 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -454,13 +454,13 @@ std::optional BinaryCacheStore::queryRealisation(const DrvOut auto [cacheOutcome, maybeCachedRealisation] = diskCache->lookupRealisation(getUri(), id); switch (cacheOutcome) { - case (NarInfoDiskCache::oValid): + case NarInfoDiskCache::oValid: debug("Returning a cached realisation for %s", id.to_string()); return *maybeCachedRealisation; - case (NarInfoDiskCache::oInvalid): + case NarInfoDiskCache::oInvalid: debug("Returning a cached missing realisation for %s", id.to_string()); return {}; - case (NarInfoDiskCache::oUnknown): + case NarInfoDiskCache::oUnknown: break; } } From d5d19582ef24af3754f7a2675f43d6828c3a8638 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 10 May 2021 17:45:53 +0200 Subject: [PATCH 3/3] Simplify the realisations disk cache --- src/libstore/nar-info-disk-cache.cc | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 84ce7972f..9dd81ddfb 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -42,9 +42,8 @@ create table if not exists NARs ( create table if not exists Realisations ( cache integer not null, outputId text not null, - content blob, -- Json serialisation of the realisation, or empty if present is true + content blob, -- Json serialisation of the realisation, or null if the realisation is absent timestamp integer not null, - present integer not null, primary key (cache, outputId), foreign key (cache) references BinaryCaches(id) on delete cascade ); @@ -113,22 +112,22 @@ public: state->insertRealisation.create(state->db, R"( - insert or replace into Realisations(cache, outputId, content, timestamp, present) - values (?, ?, ?, ?, 1) + insert or replace into Realisations(cache, outputId, content, timestamp) + values (?, ?, ?, ?) )"); state->insertMissingRealisation.create(state->db, R"( - insert or replace into Realisations(cache, outputId, timestamp, present) - values (?, ?, ?, 0) + insert or replace into Realisations(cache, outputId, timestamp) + values (?, ?, ?) )"); state->queryRealisation.create(state->db, R"( - select present, content from Realisations + select content from Realisations where cache = ? and outputId = ? and - ((present = 0 and timestamp > ?) or - (present = 1 and timestamp > ?)) + ((content is null and timestamp > ?) or + (content is not null and timestamp > ?)) )"); /* Periodically purge expired entries from the database. */ @@ -265,12 +264,12 @@ public: if (!queryRealisation.next()) return {oUnknown, 0}; - if (queryRealisation.getInt(0) == 0) + if (queryRealisation.isNull(0)) return {oInvalid, 0}; auto realisation = std::make_shared(Realisation::fromJSON( - nlohmann::json::parse(queryRealisation.getStr(1)), + nlohmann::json::parse(queryRealisation.getStr(0)), "Local disk cache")); return {oValid, realisation};