From 2040240e238a41c2eb799bf4dbf394fec297ac16 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 13 Apr 2017 15:55:38 +0200 Subject: [PATCH] Add a Config class to simplify adding configuration settings The typical use is to inherit Config and add Setting members: class MyClass : private Config { Setting foo{this, 123, "foo", "the number of foos to use"}; Setting bar{this, "blabla", "bar", "the name of the bar"}; MyClass() : Config(readConfigFile("/etc/my-app.conf")) { std::cout << foo << "\n"; // will print 123 unless overriden } }; Currently, this is used by Store and its subclasses for store parameters. You now get a warning if you specify a non-existant store parameter in a store URI. --- src/libstore/binary-cache-store.cc | 3 - src/libstore/binary-cache-store.hh | 10 +- src/libstore/build.cc | 8 +- src/libstore/legacy-ssh-store.cc | 10 +- src/libstore/local-fs-store.cc | 9 +- src/libstore/local-store.cc | 5 +- src/libstore/local-store.hh | 6 +- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 3 + src/libstore/s3-binary-cache-store.cc | 12 +- src/libstore/ssh-store.cc | 7 +- src/libstore/store-api.cc | 9 +- src/libstore/store-api.hh | 25 ++++- src/libutil/config.cc | 112 +++++++++++++++++++ src/libutil/config.hh | 151 ++++++++++++++++++++++++++ src/libutil/types.hh | 2 + 16 files changed, 334 insertions(+), 40 deletions(-) create mode 100644 src/libutil/config.cc create mode 100644 src/libutil/config.hh diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 25ad0d75b..b536c6c00 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -79,10 +79,7 @@ struct BinaryCacheStoreAccessor : public FSAccessor BinaryCacheStore::BinaryCacheStore(const Params & params) : Store(params) - , compression(get(params, "compression", "xz")) - , writeNARListing(get(params, "write-nar-listing", "0") == "1") { - auto secretKeyFile = get(params, "secret-key", ""); if (secretKeyFile != "") secretKey = std::unique_ptr(new SecretKey(readFile(secretKeyFile))); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index d42b1abd2..5c2d0acfd 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -13,14 +13,16 @@ struct NarInfo; class BinaryCacheStore : public Store { +public: + + const Setting compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; + const Setting writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; + const Setting secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; + private: std::unique_ptr secretKey; - std::string compression; - - bool writeNARListing; - protected: BinaryCacheStore(const Params & params); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 968e29112..d9c299d09 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3051,13 +3051,11 @@ Path DerivationGoal::openLogFile() string baseName = baseNameOf(drvPath); /* Create a log file. */ - Path dir = (format("%1%/%2%/%3%/") % worker.store.logDir % worker.store.drvsLogDir % string(baseName, 0, 2)).str(); + Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); createDirs(dir); - Path logFileName = (format("%1%/%2%%3%") - % dir - % string(baseName, 2) - % (settings.compressLog ? ".bz2" : "")).str(); + Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), + settings.compressLog ? ".bz2" : ""); fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); if (!fdLogFile) throw SysError(format("creating log file ‘%1%’") % logFileName); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 0e838846c..befc560bf 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -12,6 +12,10 @@ static std::string uriScheme = "ssh://"; struct LegacySSHStore : public Store { + const Setting maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"}; + const Setting sshKey{this, "", "ssh-key", "path to an SSH private key"}; + const Setting compress{this, false, "compress", "whether to compress the connection"}; + struct Connection { std::unique_ptr sshConn; @@ -29,16 +33,16 @@ struct LegacySSHStore : public Store : Store(params) , host(host) , connections(make_ref>( - std::max(1, std::stoi(get(params, "max-connections", "1"))), + std::max(1, (int) maxConnections), [this]() { return openConnection(); }, [](const ref & r) { return true; } )) , master( host, - get(params, "ssh-key", ""), + sshKey, // Use SSH master only if using more than 1 connection. connections->capacity() > 1, - get(params, "compress", "") == "true") + compress) { } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 57e1b8a09..bf247903c 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -9,9 +9,6 @@ namespace nix { LocalFSStore::LocalFSStore(const Params & params) : Store(params) - , rootDir(get(params, "root")) - , stateDir(canonPath(get(params, "state", rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir))) - , logDir(canonPath(get(params, "log", rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir))) { } @@ -88,6 +85,8 @@ void LocalFSStore::narFromPath(const Path & path, Sink & sink) const string LocalFSStore::drvsLogDir = "drvs"; + + std::shared_ptr LocalFSStore::getBuildLog(const Path & path_) { auto path(path_); @@ -110,8 +109,8 @@ std::shared_ptr LocalFSStore::getBuildLog(const Path & path_) Path logPath = j == 0 - ? (format("%1%/%2%/%3%/%4%") % logDir % drvsLogDir % string(baseName, 0, 2) % string(baseName, 2)).str() - : (format("%1%/%2%/%3%") % logDir % drvsLogDir % baseName).str(); + ? fmt("%s/%s/%s/%s", logDir, drvsLogDir, string(baseName, 0, 2), string(baseName, 2)) + : fmt("%s/%s/%s", logDir, drvsLogDir, baseName); Path logBz2Path = logPath + ".bz2"; if (pathExists(logPath)) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8610841d7..0ea897526 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -38,13 +38,14 @@ namespace nix { LocalStore::LocalStore(const Params & params) : Store(params) , LocalFSStore(params) - , realStoreDir(get(params, "real", rootDir != "" ? rootDir + "/nix/store" : storeDir)) + , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", + "physical path to the Nix store"} + , realStoreDir(realStoreDir_) , dbDir(stateDir + "/db") , linksDir(realStoreDir + "/.links") , reservedPath(dbDir + "/reserved") , schemaPath(dbDir + "/schema") , trashDir(realStoreDir + "/trash") - , requireSigs(trim(settings.get("signed-binary-caches", std::string("*"))) != "") // FIXME: rename option , publicKeys(getDefaultPublicKeys()) { auto state(_state.lock()); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 750da0c14..fec67ee7d 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -67,6 +67,8 @@ private: public: + PathSetting realStoreDir_; + const Path realStoreDir; const Path dbDir; const Path linksDir; @@ -76,7 +78,9 @@ public: private: - bool requireSigs; + Setting requireSigs{(Store*) this, + trim(settings.get("signed-binary-caches", std::string("*"))) != "", + "require-sigs", "whether store paths should have a trusted signature on import"}; PublicKeys publicKeys; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index c9c590787..e1df137e4 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -43,7 +43,7 @@ template Paths readStorePaths(Store & store, Source & from); RemoteStore::RemoteStore(const Params & params) : Store(params) , connections(make_ref>( - std::max(1, std::stoi(get(params, "max-connections", "1"))), + std::max(1, (int) maxConnections), [this]() { return openConnectionWrapper(); }, [](const ref & r) { return r->to.good() && r->from.good(); } )) diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index db8da7eaa..479cf3a79 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -22,6 +22,9 @@ class RemoteStore : public virtual Store { public: + const Setting maxConnections{(Store*) this, 1, + "max-connections", "maximum number of concurrent connections to the Nix daemon"}; + RemoteStore(const Params & params); /* Implementations of abstract store API methods. */ diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 3053f908c..245455296 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -125,22 +125,22 @@ S3Helper::DownloadResult S3Helper::getObject( struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore { + const Setting region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; + const Setting narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; + const Setting lsCompression{this, "", "ls-compression", "compression method for .ls files"}; + const Setting logCompression{this, "", "log-compression", "compression method for log/* files"}; + std::string bucketName; Stats stats; S3Helper s3Helper; - std::string narinfoCompression, lsCompression, logCompression; - S3BinaryCacheStoreImpl( const Params & params, const std::string & bucketName) : S3BinaryCacheStore(params) , bucketName(bucketName) - , s3Helper(get(params, "aws-region", Aws::Region::US_EAST_1)) - , narinfoCompression(get(params, "narinfo-compression", "")) - , lsCompression(get(params, "ls-compression", "")) - , logCompression(get(params, "log-compression", "")) + , s3Helper(region) { diskCache = getNarInfoDiskCache(); } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 2a81a8b1e..bb536fadf 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -14,16 +14,19 @@ class SSHStore : public RemoteStore { public: + const Setting sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"}; + const Setting compress{(Store*) this, false, "compress", "whether to compress the connection"}; + SSHStore(const std::string & host, const Params & params) : Store(params) , RemoteStore(params) , host(host) , master( host, - get(params, "ssh-key", ""), + sshKey, // Use SSH master only if using more than 1 connection. connections->capacity() > 1, - get(params, "compress", "") == "true") + compress) { } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 53c802044..cb62bdc0b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -241,8 +241,8 @@ Path Store::computeStorePathForText(const string & name, const string & s, Store::Store(const Params & params) - : storeDir(get(params, "store", settings.nixStore)) - , state({std::stoi(get(params, "path-info-cache-size", "65536"))}) + : Config(params) + , state({(size_t) pathInfoCacheSize}) { } @@ -718,7 +718,10 @@ ref openStore(const std::string & uri, const Store::Params & params) { for (auto fun : *RegisterStoreImplementation::implementations) { auto store = fun(uri, params); - if (store) return ref(store); + if (store) { + store->warnUnused(); + return ref(store); + } } throw Error(format("don't know how to open Nix store ‘%s’") % uri); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index c0a52145a..067309c9e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -6,6 +6,7 @@ #include "lru-cache.hh" #include "sync.hh" #include "globals.hh" +#include "config.hh" #include #include @@ -224,13 +225,17 @@ struct BuildResult }; -class Store : public std::enable_shared_from_this +class Store : public std::enable_shared_from_this, public Config { public: typedef std::map Params; - const Path storeDir; + const PathSetting storeDir_{this, false, settings.nixStore, + "store", "path to the Nix store"}; + const Path storeDir = storeDir_; + + const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"}; protected: @@ -585,9 +590,19 @@ protected: class LocalFSStore : public virtual Store { public: - const Path rootDir; - const Path stateDir; - const Path logDir; + + // FIXME: the (Store*) cast works around a bug in gcc that causes + // it to emit the call to the Option constructor. Clang works fine + // either way. + const PathSetting rootDir{(Store*) this, true, "", + "root", "directory prefixed to all other paths"}; + const PathSetting stateDir{(Store*) this, false, + rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, + "state", "directory where Nix will store state"}; + const PathSetting logDir{(Store*) this, false, + rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, + "log", "directory where Nix will store state"}; + const static string drvsLogDir; LocalFSStore(const Params & params); diff --git a/src/libutil/config.cc b/src/libutil/config.cc new file mode 100644 index 000000000..2f9f98860 --- /dev/null +++ b/src/libutil/config.cc @@ -0,0 +1,112 @@ +#include "config.hh" +#include "args.hh" + +namespace nix { + +void Config::set(const std::string & name, const std::string & value) +{ + auto i = _settings.find(name); + if (i == _settings.end()) + throw UsageError("unknown setting '%s'", name); + i->second.setting->set(value); +} + +void Config::add(AbstractSetting * setting) +{ + _settings.emplace(setting->name, Config::SettingData{false, setting}); + for (auto & alias : setting->aliases) + _settings.emplace(alias, Config::SettingData{true, setting}); + + bool set = false; + + auto i = initials.find(setting->name); + if (i != initials.end()) { + setting->set(i->second); + initials.erase(i); + set = true; + } + + for (auto & alias : setting->aliases) { + auto i = initials.find(alias); + if (i != initials.end()) { + if (set) + warn("setting '%s' is set, but it's an alias of '%s' which is also set", + alias, setting->name); + else { + setting->set(i->second); + initials.erase(i); + set = true; + } + } + } +} + +void Config::warnUnused() +{ + for (auto & i : initials) + warn("unknown setting '%s'", i.first); +} + +std::string Config::dump() +{ + std::string res; + for (auto & opt : _settings) + if (!opt.second.isAlias) + res += opt.first + " = " + opt.second.setting->to_string() + "\n"; + return res; +} + +AbstractSetting::AbstractSetting( + const std::string & name, + const std::string & description, + const std::set & aliases) + : name(name), description(description), aliases(aliases) +{ +} + +template<> void Setting::set(const std::string & str) +{ + value = str; +} + +template<> std::string Setting::to_string() +{ + return value; +} + +template<> void Setting::set(const std::string & str) +{ + try { + value = std::stoi(str); + } catch (...) { + throw UsageError("setting '%s' has invalid value '%s'", name, str); + } +} + +template<> std::string Setting::to_string() +{ + return std::to_string(value); +} + +template<> void Setting::set(const std::string & str) +{ + value = str == "true" || str == "1"; +} + +template<> std::string Setting::to_string() +{ + return value ? "true" : "false"; +} + +void PathSetting::set(const std::string & str) +{ + if (str == "") { + if (allowEmpty) + value = ""; + else + throw UsageError("setting '%s' cannot be empty", name); + } else + value = canonPath(str); +} + +} diff --git a/src/libutil/config.hh b/src/libutil/config.hh new file mode 100644 index 000000000..fb2d48e9c --- /dev/null +++ b/src/libutil/config.hh @@ -0,0 +1,151 @@ +#include +#include + +#include "types.hh" + +#pragma once + +namespace nix { + +class Args; +class AbstractSetting; + +/* A class to simplify providing configuration settings. The typical + use is to inherit Config and add Setting members: + + class MyClass : private Config + { + Setting foo{this, 123, "foo", "the number of foos to use"}; + Setting bar{this, "blabla", "bar", "the name of the bar"}; + + MyClass() : Config(readConfigFile("/etc/my-app.conf")) + { + std::cout << foo << "\n"; // will print 123 unless overriden + } + }; +*/ + +class Config +{ + friend class AbstractSetting; + + struct SettingData + { + bool isAlias = false; + AbstractSetting * setting; + }; + + std::map _settings; + + StringMap initials; + +public: + + Config(const StringMap & initials) + : initials(initials) + { } + + void set(const std::string & name, const std::string & value); + + void add(AbstractSetting * setting); + + void warnUnused(); + + std::string dump(); +}; + +class AbstractSetting +{ + friend class Config; + +public: + + const std::string name; + const std::string description; + const std::set aliases; + + int created = 123; + +protected: + + AbstractSetting( + const std::string & name, + const std::string & description, + const std::set & aliases); + + virtual ~AbstractSetting() + { + // Check against a gcc miscompilation causing our constructor + // not to run. + assert(created == 123); + } + + virtual void set(const std::string & value) = 0; + + virtual std::string to_string() = 0; +}; + +/* A setting of type T. */ +template +class Setting : public AbstractSetting +{ +protected: + + T value; + +public: + + Setting(Config * options, + const T & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : AbstractSetting(name, description, aliases) + , value(def) + { + options->add(this); + } + + operator const T &() const { return value; } + bool operator ==(const T & v2) const { return value == v2; } + bool operator !=(const T & v2) const { return value != v2; } + void operator =(const T & v) { value = v; } + + void set(const std::string & str) override; + + std::string to_string() override; +}; + +template +std::ostream & operator <<(std::ostream & str, const Setting & opt) +{ + str << (const T &) opt; + return str; +} + +/* A special setting for Paths. These are automatically canonicalised + (e.g. "/foo//bar/" becomes "/foo/bar"). */ +class PathSetting : public Setting +{ + bool allowEmpty; + +public: + + PathSetting(Config * options, + bool allowEmpty, + const Path & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : Setting(options, def, name, description, aliases) + , allowEmpty(allowEmpty) + { + set(value); + } + + void set(const std::string & str) override; + + Path operator +(const char * p) const { return value + p; } +}; + +} diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 97d79af9b..1429c2385 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -141,6 +142,7 @@ private: typedef list Strings; typedef set StringSet; +typedef std::map StringMap; /* Paths are just strings. */