From 2855c3d965bab5e88c4cd08dc3a92e705ff5ac9f Mon Sep 17 00:00:00 2001 From: AmineChikhaoui Date: Mon, 2 Apr 2018 12:45:18 +0100 Subject: [PATCH 1/5] Make the TTL for disk cache configurable, we can now completely disable disk cache lookup for example by doing: nix copy --from --option \ positive-disk-cache-ttl 0 Issues: #1885 #2035 --- src/libstore/globals.hh | 6 ++++++ src/libstore/nar-info-disk-cache.cc | 13 +++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 117404ec1..36c8b7357 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -310,6 +310,12 @@ public: "Disabled substituters that may be enabled via the substituters option by untrusted users.", {"trusted-binary-caches"}}; + Setting ttlNegativeDiskCache{this, 3600, "negative-disk-cache-ttl", + "The TTL in seconds for negative lookups in the disk cache."}; + + Setting ttlPositiveDiskCache{this, 30 * 24 * 3600, "positive-disk-cache-ttl", + "The TTL in seconds for positive lookups in the disk cache."}; + Setting trustedUsers{this, {"root"}, "trusted-users", "Which users or groups are trusted to ask the daemon to do unsafe things."}; diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 3c52303f0..1e143736f 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -1,6 +1,7 @@ #include "nar-info-disk-cache.hh" #include "sync.hh" #include "sqlite.hh" +#include "globals.hh" #include @@ -47,10 +48,6 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache { public: - /* How long negative and positive lookups are valid. */ - const int ttlNegative = 3600; - const int ttlPositive = 30 * 24 * 3600; - /* How often to purge expired entries from the cache. */ const int purgeInterval = 24 * 3600; @@ -116,8 +113,8 @@ public: SQLiteStmt(state->db, "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))") .use() - (now - ttlNegative) - (now - ttlPositive) + (now - settings.ttlNegativeDiskCache) + (now - settings.ttlPositiveDiskCache) .exec(); debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db)); @@ -186,8 +183,8 @@ public: auto queryNAR(state->queryNAR.use() (cache.id) (hashPart) - (now - ttlNegative) - (now - ttlPositive)); + (now - settings.ttlNegativeDiskCache) + (now - settings.ttlPositiveDiskCache)); if (!queryNAR.next()) return {oUnknown, 0}; From 86930ed414f9243ca75679b924bf26f5940eb8c9 Mon Sep 17 00:00:00 2001 From: AmineChikhaoui Date: Mon, 2 Apr 2018 17:41:49 +0100 Subject: [PATCH 2/5] add documentation for the local disk cache TTL config --- doc/manual/command-ref/conf-file.xml | 24 ++++++++++++++++++++++++ src/libstore/globals.hh | 14 ++++++++------ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 6638bf61e..1033d385e 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -788,6 +788,30 @@ password my-password + negative-disk-cache-ttl + + + + The TTL in seconds for negative lookups. If a store path is queried from a substituer but + was not found, there will be a negative lookup cached in the local disk cache database for the specified + duration. + + + + + + positive-disk-cache-ttl + + + + The TTL in seconds for positive lookups. If a store path is queried from a substituer, the result of + the query will be cached in the local disk cache database including some of the NAR metadata. Setting a TTL + for positive lookups can be useful in case of builds that aren't reproducible, in which case having a more + frequent cache invalidation would prevent hash mismatch issues. + + + + diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 36c8b7357..00147f615 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -310,15 +310,17 @@ public: "Disabled substituters that may be enabled via the substituters option by untrusted users.", {"trusted-binary-caches"}}; - Setting ttlNegativeDiskCache{this, 3600, "negative-disk-cache-ttl", - "The TTL in seconds for negative lookups in the disk cache."}; - - Setting ttlPositiveDiskCache{this, 30 * 24 * 3600, "positive-disk-cache-ttl", - "The TTL in seconds for positive lookups in the disk cache."}; - Setting trustedUsers{this, {"root"}, "trusted-users", "Which users or groups are trusted to ask the daemon to do unsafe things."}; + Setting ttlNegativeDiskCache{this, 3600, "negative-disk-cache-ttl", + "The TTL in seconds for negative lookups in the disk cache i.e binary cache lookups that " + "return an invalid path result"}; + + Setting ttlPositiveDiskCache{this, 30 * 24 * 3600, "positive-disk-cache-ttl", + "The TTL in seconds for positive lookups in the disk cache i.e binary cache lookups that " + "return a valid path result."}; + /* ?Who we trust to use the daemon in safe ways */ Setting allowedUsers{this, {"*"}, "allowed-users", "Which users or groups are allowed to connect to the daemon."}; From 33b08899d591125ad7900e7005e2b8862f31bba0 Mon Sep 17 00:00:00 2001 From: AmineChikhaoui Date: Tue, 3 Apr 2018 11:33:39 +0100 Subject: [PATCH 3/5] re-order the options using the alphabetical order and improve the example in the positive lookup case --- doc/manual/command-ref/conf-file.xml | 54 +++++++++++++++------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 1033d385e..a4d411d70 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -456,6 +456,18 @@ builtins.fetchurl { + negative-disk-cache-ttl + + + + The TTL in seconds for negative lookups. If a store path is + queried from a substituter but was not found, there will be a + negative lookup cached in the local disk cache database for the + specified duration. + + + + netrc-file @@ -511,6 +523,23 @@ password my-password + positive-disk-cache-ttl + + + + The TTL in seconds for positive lookups. If a store path is + queried from a substituter, the result of the query will be cached + in the local disk cache database including some of the NAR + metadata. The default TTL is a month, setting a shorter TTL for + positive lookups can be useful for binary caches that have + frequent garbage collection, in which case having a more frequent + cache invalidation would prevent trying to pull the path again and + failing with a hash mismatch if the build isn't reproducible. + + + + + pre-build-hook @@ -788,31 +817,6 @@ password my-password - negative-disk-cache-ttl - - - - The TTL in seconds for negative lookups. If a store path is queried from a substituer but - was not found, there will be a negative lookup cached in the local disk cache database for the specified - duration. - - - - - - positive-disk-cache-ttl - - - - The TTL in seconds for positive lookups. If a store path is queried from a substituer, the result of - the query will be cached in the local disk cache database including some of the NAR metadata. Setting a TTL - for positive lookups can be useful in case of builds that aren't reproducible, in which case having a more - frequent cache invalidation would prevent hash mismatch issues. - - - - - From 62d75ad3e18a6ff8973fb95e7fbc58802c477eac Mon Sep 17 00:00:00 2001 From: AmineChikhaoui Date: Fri, 6 Apr 2018 11:05:15 +0100 Subject: [PATCH 4/5] rename the options to mention it's a narinfo TTL as disk cache is used all over the place for other operations --- src/libstore/globals.hh | 4 ++-- src/libstore/nar-info-disk-cache.cc | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 00147f615..7430bbedb 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -313,11 +313,11 @@ public: Setting trustedUsers{this, {"root"}, "trusted-users", "Which users or groups are trusted to ask the daemon to do unsafe things."}; - Setting ttlNegativeDiskCache{this, 3600, "negative-disk-cache-ttl", + Setting ttlNegativeNarInfoCache{this, 3600, "narinfo-cache-negative-ttl", "The TTL in seconds for negative lookups in the disk cache i.e binary cache lookups that " "return an invalid path result"}; - Setting ttlPositiveDiskCache{this, 30 * 24 * 3600, "positive-disk-cache-ttl", + Setting ttlPositiveNarInfoCache{this, 30 * 24 * 3600, "narinfo-cache-positive-ttl", "The TTL in seconds for positive lookups in the disk cache i.e binary cache lookups that " "return a valid path result."}; diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 1e143736f..35403e5df 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -113,8 +113,8 @@ public: SQLiteStmt(state->db, "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))") .use() - (now - settings.ttlNegativeDiskCache) - (now - settings.ttlPositiveDiskCache) + (now - settings.ttlNegativeNarInfoCache) + (now - settings.ttlPositiveNarInfoCache) .exec(); debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db)); @@ -183,8 +183,8 @@ public: auto queryNAR(state->queryNAR.use() (cache.id) (hashPart) - (now - settings.ttlNegativeDiskCache) - (now - settings.ttlPositiveDiskCache)); + (now - settings.ttlNegativeNarInfoCache) + (now - settings.ttlPositiveNarInfoCache)); if (!queryNAR.next()) return {oUnknown, 0}; From e01b01c5799046a030f24d913ca4f4cbfa997a9c Mon Sep 17 00:00:00 2001 From: AmineChikhaoui Date: Fri, 6 Apr 2018 11:09:52 +0100 Subject: [PATCH 5/5] update/re-order the options docs --- doc/manual/command-ref/conf-file.xml | 38 ++++++++++++++-------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index a4d411d70..431c0e6d3 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -456,7 +456,7 @@ builtins.fetchurl { - negative-disk-cache-ttl + narinfo-cache-negative-ttl @@ -469,6 +469,24 @@ builtins.fetchurl { + narinfo-cache-positive-ttl + + + + The TTL in seconds for positive lookups. If a store path is + queried from a substituter, the result of the query will be cached + in the local disk cache database including some of the NAR + metadata. The default TTL is a month, setting a shorter TTL for + positive lookups can be useful for binary caches that have + frequent garbage collection, in which case having a more frequent + cache invalidation would prevent trying to pull the path again and + failing with a hash mismatch if the build isn't reproducible. + + + + + + netrc-file If set to an absolute path to a netrc @@ -523,24 +541,6 @@ password my-password - positive-disk-cache-ttl - - - - The TTL in seconds for positive lookups. If a store path is - queried from a substituter, the result of the query will be cached - in the local disk cache database including some of the NAR - metadata. The default TTL is a month, setting a shorter TTL for - positive lookups can be useful for binary caches that have - frequent garbage collection, in which case having a more frequent - cache invalidation would prevent trying to pull the path again and - failing with a hash mismatch if the build isn't reproducible. - - - - - - pre-build-hook