From f8abbdd4565542464f31f4dc203a9c3e091b3536 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Dec 2019 17:17:53 +0100 Subject: [PATCH] Add priority setting to stores This allows overriding the priority of substituters, e.g. $ nix-store --store ~/my-nix/ -r /nix/store/df3m4da96d84ljzxx4mygfshm1p0r2n3-geeqie-1.4 \ --substituters 'http://cache.nixos.org?priority=100 daemon?priority=10' Fixes #3264. --- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/binary-cache-store.hh | 9 --------- src/libstore/globals.cc | 2 +- src/libstore/http-binary-cache-store.cc | 7 +++++-- src/libstore/local-store.cc | 2 +- src/libstore/nar-info-disk-cache.cc | 16 ++++++++-------- src/libstore/nar-info-disk-cache.hh | 9 +++++++-- src/libstore/s3-binary-cache-store.cc | 9 +++++---- src/libstore/sqlite.hh | 4 ++-- src/libstore/store-api.cc | 2 +- src/libstore/store-api.hh | 11 ++++------- src/libutil/config.cc | 15 ++++++++++----- src/libutil/config.hh | 8 +++++--- 13 files changed, 51 insertions(+), 47 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index aaacf8281..717faec92 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -49,9 +49,9 @@ void BinaryCacheStore::init() throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'") % getUri() % value % storeDir); } else if (name == "WantMassQuery") { - wantMassQuery_ = value == "1"; + wantMassQuery.setDefault(value == "1" ? "true" : "false"); } else if (name == "Priority") { - string2Int(value, priority); + priority.setDefault(fmt("%d", std::stoi(value))); } } } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index fa2200ad8..aa13c1cb4 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -52,11 +52,6 @@ public: std::shared_ptr getFile(const std::string & path); -protected: - - bool wantMassQuery_ = false; - int priority = 50; - public: virtual void init(); @@ -79,8 +74,6 @@ public: std::optional queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } - bool wantMassQuery() override { return wantMassQuery_; } - void addToStore(const ValidPathInfo & info, const ref & nar, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) override; @@ -107,8 +100,6 @@ public: std::shared_ptr getBuildLog(const StorePath & path) override; - int getPriority() override { return priority; } - }; MakeError(NoSuchBinaryCacheFile, Error); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 6fedf3d56..cec85edca 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -127,7 +127,7 @@ template<> void BaseSetting::set(const std::string & str) else throw UsageError("option '%s' has invalid value '%s'", name, str); } -template<> std::string BaseSetting::to_string() +template<> std::string BaseSetting::to_string() const { if (value == smEnabled) return "true"; else if (value == smRelaxed) return "relaxed"; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 779f89e68..d4ae36662 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -42,13 +42,16 @@ public: void init() override { // FIXME: do this lazily? - if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) { + if (auto cacheInfo = diskCache->cacheExists(cacheUri)) { + wantMassQuery.setDefault(cacheInfo->wantMassQuery ? "true" : "false"); + priority.setDefault(fmt("%d", cacheInfo->priority)); + } else { try { BinaryCacheStore::init(); } catch (UploadToHTTP &) { throw Error("'%s' does not appear to be a binary cache", cacheUri); } - diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority); + diskCache->createCache(cacheUri, storeDir, wantMassQuery, priority); } } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c1380ef09..b254d766a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -840,7 +840,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) for (auto & sub : getDefaultSubstituters()) { if (remaining.empty()) break; if (sub->storeDir != storeDir) continue; - if (!sub->wantMassQuery()) continue; + if (!sub->wantMassQuery) continue; auto valid = sub->queryValidPaths(remaining); diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 3a2cde55f..907645d86 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -147,26 +147,26 @@ public: }); } - bool cacheExists(const std::string & uri, - bool & wantMassQuery, int & priority) override + std::optional cacheExists(const std::string & uri) override { - return retrySQLite([&]() { + return retrySQLite>([&]() -> std::optional { auto state(_state.lock()); auto i = state->caches.find(uri); if (i == state->caches.end()) { auto queryCache(state->queryCache.use()(uri)); - if (!queryCache.next()) return false; + if (!queryCache.next()) + return std::nullopt; state->caches.emplace(uri, Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)}); } auto & cache(getCache(*state, uri)); - wantMassQuery = cache.wantMassQuery; - priority = cache.priority; - - return true; + return CacheInfo { + .wantMassQuery = cache.wantMassQuery, + .priority = cache.priority + }; }); } diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 285873f7e..878acbb87 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -15,8 +15,13 @@ public: virtual void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) = 0; - virtual bool cacheExists(const std::string & uri, - bool & wantMassQuery, int & priority) = 0; + struct CacheInfo + { + bool wantMassQuery; + int priority; + }; + + virtual std::optional cacheExists(const std::string & uri) = 0; virtual std::pair> lookupNarInfo( const std::string & uri, const std::string & hashPart) = 0; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 58966a5b7..f2e4b63e0 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -205,11 +205,12 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore void init() override { - if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) { - + if (auto cacheInfo = diskCache->cacheExists(getUri())) { + wantMassQuery.setDefault(cacheInfo->wantMassQuery ? "true" : "false"); + priority.setDefault(fmt("%d", cacheInfo->priority)); + } else { BinaryCacheStore::init(); - - diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority); + diskCache->createCache(getUri(), storeDir, wantMassQuery, priority); } } diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 115679b84..bd012d9b9 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -99,8 +99,8 @@ void handleSQLiteBusy(const SQLiteBusy & e); /* Convenience function for retrying a SQLite transaction when the database is busy. */ -template -T retrySQLite(std::function fun) +template +T retrySQLite(F && fun) { while (true) { try { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1781e0c68..d8f6c22bc 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -975,7 +975,7 @@ std::list> getDefaultSubstituters() addStore(uri); stores.sort([](ref & a, ref & b) { - return a->getPriority() < b->getPriority(); + return a->priority < b->priority; }); return stores; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index d77d6bad8..743be9410 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -255,6 +255,10 @@ public: const Setting isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; + Setting priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"}; + + Setting wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; + protected: struct State @@ -422,8 +426,6 @@ public: virtual void querySubstitutablePathInfos(const StorePathSet & paths, SubstitutablePathInfos & infos) { return; }; - virtual bool wantMassQuery() { return false; } - /* Import a path into the store. */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, @@ -648,11 +650,6 @@ public: return 0; }; - /* Get the priority of the store, used to order substituters. In - particular, binary caches can specify a priority field in their - "nix-cache-info" file. Lower value means higher priority. */ - virtual int getPriority() { return 0; } - virtual Path toRealPath(const Path & storePath) { return storePath; diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 9023cb1bb..7551d97d1 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -154,6 +154,11 @@ AbstractSetting::AbstractSetting( { } +void AbstractSetting::setDefault(const std::string & str) +{ + if (!overriden) set(str); +} + void AbstractSetting::toJSON(JSONPlaceholder & out) { out.write(to_string()); @@ -185,7 +190,7 @@ template<> void BaseSetting::set(const std::string & str) value = str; } -template<> std::string BaseSetting::to_string() +template<> std::string BaseSetting::to_string() const { return value; } @@ -199,7 +204,7 @@ void BaseSetting::set(const std::string & str) } template -std::string BaseSetting::to_string() +std::string BaseSetting::to_string() const { static_assert(std::is_integral::value, "Integer required."); return std::to_string(value); @@ -215,7 +220,7 @@ template<> void BaseSetting::set(const std::string & str) throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str); } -template<> std::string BaseSetting::to_string() +template<> std::string BaseSetting::to_string() const { return value ? "true" : "false"; } @@ -239,7 +244,7 @@ template<> void BaseSetting::set(const std::string & str) value = tokenizeString(str); } -template<> std::string BaseSetting::to_string() +template<> std::string BaseSetting::to_string() const { return concatStringsSep(" ", value); } @@ -256,7 +261,7 @@ template<> void BaseSetting::set(const std::string & str) value = tokenizeString(str); } -template<> std::string BaseSetting::to_string() +template<> std::string BaseSetting::to_string() const { return concatStringsSep(" ", value); } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index d86c65ff0..7ea78fdaf 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -115,6 +115,8 @@ public: bool overriden = false; + void setDefault(const std::string & str); + protected: AbstractSetting( @@ -131,13 +133,13 @@ protected: virtual void set(const std::string & value) = 0; - virtual std::string to_string() = 0; + virtual std::string to_string() const = 0; virtual void toJSON(JSONPlaceholder & out); virtual void convertToArg(Args & args, const std::string & category); - bool isOverriden() { return overriden; } + bool isOverriden() const { return overriden; } }; /* A setting of type T. */ @@ -174,7 +176,7 @@ public: value = v; } - std::string to_string() override; + std::string to_string() const override; void convertToArg(Args & args, const std::string & category) override;