kloenk/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

Merge branch 'document-store-options' of https://github.com/tweag/nix

Browse source
This commit is contained in:
Eelco Dolstra 2020-09-16 17:02:30 +02:00
parent 0066ef6c59 77a0e2c5be
commit 5080d4e7b2
30 changed files with 443 additions and 228 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/libexpr/flake/lockfile.hh
View file

@ -6,7 +6,7 @@
namespace nix {
class Store;
struct StorePath;
class StorePath;
}
namespace nix::flake {

2
src/libfetchers/fetchers.hh
View file

@ -23,7 +23,7 @@ struct InputScheme;
struct Input
{
friend class InputScheme;
friend struct InputScheme;
std::shared_ptr<InputScheme> scheme; // note: can be null
Attrs attrs;

3
src/libstore/binary-cache-store.cc
View file

@ -22,7 +22,8 @@
namespace nix {
BinaryCacheStore::BinaryCacheStore(const Params & params)
: Store(params)
: BinaryCacheStoreConfig(params)
, Store(params)
{
if (secretKeyFile != "")
secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));

22
src/libstore/binary-cache-store.hh
View file

@ -11,17 +11,21 @@ namespace nix {
struct NarInfo;
class BinaryCacheStore : public Store
struct BinaryCacheStoreConfig : virtual StoreConfig
{
public:
using StoreConfig::StoreConfig;
const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
const Setting<bool> writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
const Setting<bool> parallelCompression{this, false, "parallel-compression",
const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"};
const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
"enable multi-threading compression, available for xz only currently"};
};
class BinaryCacheStore : public Store, public virtual BinaryCacheStoreConfig
{
private:
@ -58,7 +62,7 @@ public:
public:
virtual void init();
virtual void init() override;
private:

9
src/libstore/build.cc
View file

@ -2872,18 +2872,23 @@ void DerivationGoal::writeStructuredAttrs()
chownToBuilder(tmpDir + "/.attrs.sh");
}
struct RestrictedStoreConfig : LocalFSStoreConfig
{
using LocalFSStoreConfig::LocalFSStoreConfig;
const std::string name() { return "Restricted Store"; }
};
/* A wrapper around LocalStore that only allows building/querying of
paths that are in the input closures of the build or were added via
recursive Nix calls. */
struct RestrictedStore : public LocalFSStore
struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig
{
ref<LocalStore> next;
DerivationGoal & goal;
RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
: Store(params), LocalFSStore(params), next(next), goal(goal)
: StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal)
{ }
Path getRealStoreDir() override

32
src/libstore/dummy-store.cc
View file

@ -2,17 +2,27 @@
namespace nix {
static std::string uriScheme = "dummy://";
struct DummyStoreConfig : virtual StoreConfig {
using StoreConfig::StoreConfig;
struct DummyStore : public Store
const std::string name() override { return "Dummy Store"; }
};
struct DummyStore : public Store, public virtual DummyStoreConfig
{
DummyStore(const Params & params)
: Store(params)
DummyStore(const std::string scheme, const std::string uri, const Params & params)
: DummyStore(params)
{ }
DummyStore(const Params & params)
: StoreConfig(params)
, Store(params)
{
}
string getUri() override
{
return uriScheme;
return *uriSchemes().begin();
}
void queryPathInfoUncached(const StorePath & path,
@ -21,6 +31,10 @@ struct DummyStore : public Store
callback(nullptr);
}
static std::set<std::string> uriSchemes() {
return {"dummy"};
}
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
{ unsupported("queryPathFromHashPart"); }
@ -48,12 +62,6 @@ struct DummyStore : public Store
{ unsupported("buildDerivation"); }
};
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
if (uri != uriScheme) return nullptr;
return std::make_shared<DummyStore>(params);
});
static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regStore;
}

5
src/libstore/globals.cc
View file

@ -162,11 +162,6 @@ template<> std::string BaseSetting<SandboxMode>::to_string() const
else abort();
}
template<> nlohmann::json BaseSetting<SandboxMode>::toJSON()
{
return AbstractSetting::toJSON();
}
template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category)
{
args.addFlag({

1
src/libstore/globals.hh
View file

@ -2,6 +2,7 @@
#include "types.hh"
#include "config.hh"
#include "abstractsettingtojson.hh"
#include "util.hh"
#include <map>

39
src/libstore/http-binary-cache-store.cc
View file

@ -7,7 +7,14 @@ namespace nix {
MakeError(UploadToHTTP, Error);
class HttpBinaryCacheStore : public BinaryCacheStore
struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
{
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
const std::string name() override { return "Http Binary Cache Store"; }
};
class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig
{
private:
@ -24,9 +31,12 @@ private:
public:
HttpBinaryCacheStore(
const Params & params, const Path & _cacheUri)
: BinaryCacheStore(params)
, cacheUri(_cacheUri)
const std::string & scheme,
const Path & _cacheUri,
const Params & params)
: StoreConfig(params)
, BinaryCacheStore(params)
, cacheUri(scheme + "://" + _cacheUri)
{
if (cacheUri.back() == '/')
cacheUri.pop_back();
@ -55,6 +65,13 @@ public:
}
}
static std::set<std::string> uriSchemes()
{
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
auto ret = std::set<std::string>({"http", "https"});
if (forceHttp) ret.insert("file");
return ret;
}
protected:
void maybeDisable()
@ -162,18 +179,6 @@ protected:
};
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
if (std::string(uri, 0, 7) != "http://" &&
std::string(uri, 0, 8) != "https://" &&
(!forceHttp || std::string(uri, 0, 7) != "file://"))
return 0;
auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
store->init();
return store;
});
static RegisterStoreImplementation<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig> regStore;
}

41
src/libstore/legacy-ssh-store.cc
View file

@ -9,18 +9,24 @@
namespace nix {
static std::string uriScheme = "ssh://";
struct LegacySSHStore : public Store
struct LegacySSHStoreConfig : virtual StoreConfig
{
const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"};
const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"};
const Setting<bool> compress{this, false, "compress", "whether to compress the connection"};
const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"};
using StoreConfig::StoreConfig;
const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"};
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
const std::string name() override { return "Legacy SSH Store"; }
};
struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
{
// Hack for getting remote build log output.
const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
// Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
// the documentation
const Setting<int> logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
struct Connection
{
@ -37,8 +43,11 @@ struct LegacySSHStore : public Store
SSHMaster master;
LegacySSHStore(const string & host, const Params & params)
: Store(params)
static std::set<std::string> uriSchemes() { return {"ssh"}; }
LegacySSHStore(const string & scheme, const string & host, const Params & params)
: StoreConfig(params)
, Store(params)
, host(host)
, connections(make_ref<Pool<Connection>>(
std::max(1, (int) maxConnections),
@ -84,7 +93,7 @@ struct LegacySSHStore : public Store
string getUri() override
{
return uriScheme + host;
return *uriSchemes().begin() + "://" + host;
}
void queryPathInfoUncached(const StorePath & path,
@ -325,12 +334,6 @@ public:
}
};
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
return std::make_shared<LegacySSHStore>(std::string(uri, uriScheme.size()), params);
});
static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regStore;
}

36
src/libstore/local-binary-cache-store.cc
View file

@ -4,7 +4,14 @@
namespace nix {
class LocalBinaryCacheStore : public BinaryCacheStore
struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
{
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
const std::string name() override { return "Local Binary Cache Store"; }
};
class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig
{
private:
@ -13,8 +20,11 @@ private:
public:
LocalBinaryCacheStore(
const Params & params, const Path & binaryCacheDir)
: BinaryCacheStore(params)
const std::string scheme,
const Path & binaryCacheDir,
const Params & params)
: StoreConfig(params)
, BinaryCacheStore(params)
, binaryCacheDir(binaryCacheDir)
{
}
@ -26,6 +36,8 @@ public:
return "file://" + binaryCacheDir;
}
static std::set<std::string> uriSchemes();
protected:
bool fileExists(const std::string & path) override;
@ -85,16 +97,14 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path)
return pathExists(binaryCacheDir + "/" + path);
}
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
std::set<std::string> LocalBinaryCacheStore::uriSchemes()
{
if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" ||
std::string(uri, 0, 7) != "file://")
return 0;
auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7));
store->init();
return store;
});
if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1")
return {};
else
return {"file"};
}
static RegisterStoreImplementation<LocalBinaryCacheStore, LocalBinaryCacheStoreConfig> regStore;
}

3
src/libstore/local-store.cc
View file

@ -42,7 +42,8 @@ namespace nix {
LocalStore::LocalStore(const Params & params)
: Store(params)
: StoreConfig(params)
, Store(params)
, LocalFSStore(params)
, realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
"physical path to the Nix store"}

17
src/libstore/local-store.hh
View file

@ -30,8 +30,19 @@ struct OptimiseStats
uint64_t blocksFreed = 0;
};
struct LocalStoreConfig : virtual LocalFSStoreConfig
{
using LocalFSStoreConfig::LocalFSStoreConfig;
class LocalStore : public LocalFSStore
Setting<bool> requireSigs{(StoreConfig*) this,
settings.requireSigs,
"require-sigs", "whether store paths should have a trusted signature on import"};
const std::string name() override { return "Local Store"; }
};
class LocalStore : public LocalFSStore, public virtual LocalStoreConfig
{
private:
@ -95,10 +106,6 @@ public:
private:
Setting<bool> requireSigs{(Store*) this,
settings.requireSigs,
"require-sigs", "whether store paths should have a trusted signature on import"};
const PublicKeys & getPublicKeys();
public:

25
src/libstore/remote-store.cc
View file

@ -94,6 +94,7 @@ void write(const Store & store, Sink & out, const std::optional<StorePath> & sto
/* TODO: Separate these store impls into different files, give them better names */
RemoteStore::RemoteStore(const Params & params)
: Store(params)
, RemoteStoreConfig(params)
, connections(make_ref<Pool<Connection>>(
std::max(1, (int) maxConnections),
[this]() { return openConnectionWrapper(); },
@ -123,19 +124,21 @@ ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper()
UDSRemoteStore::UDSRemoteStore(const Params & params)
: Store(params)
: StoreConfig(params)
, Store(params)
, LocalFSStore(params)
, RemoteStore(params)
{
}
UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params)
: Store(params)
, LocalFSStore(params)
, RemoteStore(params)
, path(socket_path)
UDSRemoteStore::UDSRemoteStore(
const std::string scheme,
std::string socket_path,
const Params & params)
: UDSRemoteStore(params)
{
path.emplace(socket_path);
}
@ -982,14 +985,6 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
return nullptr;
}
static std::string uriScheme = "unix://";
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params);
});
static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regStore;
}

42
src/libstore/remote-store.hh
View file

@ -16,19 +16,23 @@ struct FdSource;
template<typename T> class Pool;
struct ConnectionHandle;
struct RemoteStoreConfig : virtual StoreConfig
{
using StoreConfig::StoreConfig;
const Setting<int> maxConnections{(StoreConfig*) this, 1,
"max-connections", "maximum number of concurrent connections to the Nix daemon"};
const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, std::numeric_limits<unsigned int>::max(),
"max-connection-age", "number of seconds to reuse a connection"};
};
/* FIXME: RemoteStore is a misnomer - should be something like
DaemonStore. */
class RemoteStore : public virtual Store
class RemoteStore : public virtual Store, public virtual RemoteStoreConfig
{
public:
const Setting<int> maxConnections{(Store*) this, 1,
"max-connections", "maximum number of concurrent connections to the Nix daemon"};
const Setting<unsigned int> maxConnectionAge{(Store*) this, std::numeric_limits<unsigned int>::max(),
"max-connection-age", "number of seconds to reuse a connection"};
virtual bool sameMachine() = 0;
RemoteStore(const Params & params);
@ -141,15 +145,35 @@ private:
};
class UDSRemoteStore : public LocalFSStore, public RemoteStore
struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig
{
UDSRemoteStoreConfig(const Store::Params & params)
: StoreConfig(params)
, LocalFSStoreConfig(params)
, RemoteStoreConfig(params)
{
}
UDSRemoteStoreConfig()
: UDSRemoteStoreConfig(Store::Params({}))
{
}
const std::string name() override { return "Local Daemon Store"; }
};
class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig
{
public:
UDSRemoteStore(const Params & params);
UDSRemoteStore(std::string path, const Params & params);
UDSRemoteStore(const std::string scheme, std::string path, const Params & params);
std::string getUri() override;
static std::set<std::string> uriSchemes()
{ return {"unix"}; }
bool sameMachine() override
{ return true; }

45
src/libstore/s3-binary-cache-store.cc
View file

@ -172,20 +172,26 @@ S3Helper::FileTransferResult S3Helper::getObject(
return res;
}
struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
{
const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."};
const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
const Setting<std::string> profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."};
const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."};
const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"};
const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"};
const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"};
const Setting<bool> multipartUpload{
this, false, "multipart-upload", "whether to use multi-part uploads"};
(StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"};
const Setting<uint64_t> bufferSize{
this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
(StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
const std::string name() override { return "S3 Binary Cache Store"; }
};
struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig
{
std::string bucketName;
Stats stats;
@ -193,8 +199,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
S3Helper s3Helper;
S3BinaryCacheStoreImpl(
const Params & params, const std::string & bucketName)
: S3BinaryCacheStore(params)
const std::string & scheme,
const std::string & bucketName,
const Params & params)
: StoreConfig(params)
, S3BinaryCacheStore(params)
, bucketName(bucketName)
, s3Helper(profile, region, scheme, endpoint)
{
@ -426,17 +435,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
return paths;
}
static std::set<std::string> uriSchemes() { return {"s3"}; }
};
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
if (std::string(uri, 0, 5) != "s3://") return 0;
auto store = std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5));
store->init();
return store;
});
static RegisterStoreImplementation<S3BinaryCacheStoreImpl, S3BinaryCacheStoreConfig> regStore;
}

36
src/libstore/ssh-store.cc
View file

@ -8,19 +8,25 @@
namespace nix {
static std::string uriScheme = "ssh-ng://";
struct SSHStoreConfig : virtual RemoteStoreConfig
{
using RemoteStoreConfig::RemoteStoreConfig;
class SSHStore : public RemoteStore
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
const std::string name() override { return "SSH Store"; }
};
class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig
{
public:
const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"};
const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"};
const Setting<Path> remoteProgram{(Store*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
const Setting<std::string> remoteStore{(Store*) this, "", "remote-store", "URI of the store on the remote system"};
SSHStore(const std::string & host, const Params & params)
: Store(params)
SSHStore(const std::string & scheme, const std::string & host, const Params & params)
: StoreConfig(params)
, Store(params)
, RemoteStore(params)
, host(host)
, master(
@ -32,9 +38,11 @@ public:
{
}
static std::set<std::string> uriSchemes() { return {"ssh-ng"}; }
std::string getUri() override
{
return uriScheme + host;
return *uriSchemes().begin() + "://" + host;
}
bool sameMachine() override
@ -76,12 +84,6 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
return conn;
}
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params);
});
static RegisterStoreImplementation<SSHStore, SSHStoreConfig> regStore;
}

65
src/libstore/store-api.cc
View file

@ -346,7 +346,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
Store::Store(const Params & params)
: Config(params)
: StoreConfig(params)
, state({(size_t) pathInfoCacheSize})
{
}
@ -1009,7 +1009,6 @@ Derivation Store::readDerivation(const StorePath & drvPath)
}
}
}
@ -1019,9 +1018,6 @@ Derivation Store::readDerivation(const StorePath & drvPath)
namespace nix {
RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
{
@ -1035,24 +1031,6 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
return {uri, params};
}
ref<Store> openStore(const std::string & uri_,
const Store::Params & extraParams)
{
auto [uri, uriParams] = splitUriAndParams(uri_);
auto params = extraParams;
params.insert(uriParams.begin(), uriParams.end());
for (auto fun : *RegisterStoreImplementation::implementations) {
auto store = fun(uri, params);
if (store) {
store->warnUnknownSettings();
return ref<Store>(store);
}
}
throw Error("don't know how to open Nix store '%s'", uri);
}
static bool isNonUriPath(const std::string & spec) {
return
// is not a URL
@ -1080,10 +1058,7 @@ StoreType getStoreType(const std::string & uri, const std::string & stateDir)
}
}
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Params & params)
{
switch (getStoreType(uri, get(params, "state").value_or(settings.nixStateDir))) {
case tDaemon:
@ -1098,8 +1073,41 @@ static RegisterStoreImplementation regStore([](
default:
return nullptr;
}
});
}
ref<Store> openStore(const std::string & uri_,
const Store::Params & extraParams)
{
auto params = extraParams;
try {
auto parsedUri = parseURL(uri_);
params.insert(parsedUri.query.begin(), parsedUri.query.end());
auto baseURI = parsedUri.authority.value_or("") + parsedUri.path;
for (auto implem : *Implementations::registered) {
if (implem.uriSchemes.count(parsedUri.scheme)) {
auto store = implem.create(parsedUri.scheme, baseURI, params);
if (store) {
store->init();
store->warnUnknownSettings();
return ref<Store>(store);
}
}
}
}
catch (BadURL &) {
auto [uri, uriParams] = splitUriAndParams(uri_);
params.insert(uriParams.begin(), uriParams.end());
if (auto store = openFromNonUri(uri, params)) {
store->warnUnknownSettings();
return ref<Store>(store);
}
}
throw Error("don't know how to open Nix store '%s'", uri_);
}
std::list<ref<Store>> getDefaultSubstituters()
{
@ -1133,5 +1141,6 @@ std::list<ref<Store>> getDefaultSubstituters()
return stores;
}
std::vector<StoreFactory> * Implementations::registered = 0;
}

129
src/libstore/store-api.hh
View file

@ -24,6 +24,31 @@
namespace nix {
/**
* About the class hierarchy of the store implementations:
*
* Each store type `Foo` consists of two classes:
*
* 1. A class `FooConfig : virtual StoreConfig` that contains the configuration
* for the store
*
* It should only contain members of type `const Setting<T>` (or subclasses
* of it) and inherit the constructors of `StoreConfig`
* (`using StoreConfig::StoreConfig`).
*
* 2. A class `Foo : virtual Store, virtual FooConfig` that contains the
* implementation of the store.
*
* This class is expected to have a constructor `Foo(const Params & params)`
* that calls `StoreConfig(params)` (otherwise you're gonna encounter an
* `assertion failure` when trying to instantiate it).
*
* You can then register the new store using:
*
* ```
* cpp static RegisterStoreImplementation<Foo, FooConfig> regStore;
* ```
*/
MakeError(SubstError, Error);
MakeError(BuildError, Error); // denotes a permanent build failure
@ -33,6 +58,7 @@ MakeError(SubstituteGone, Error);
MakeError(SubstituterDisabled, Error);
MakeError(BadStorePath, Error);
MakeError(InvalidStoreURI, Error);
class FSAccessor;
class NarInfoDiskCache;
@ -144,12 +170,31 @@ struct BuildResult
}
};
class Store : public std::enable_shared_from_this<Store>, public Config
struct StoreConfig : public Config
{
public:
using Config::Config;
typedef std::map<std::string, std::string> Params;
/**
* When constructing a store implementation, we pass in a map `params` of
* parameters that's supposed to initialize the associated config.
* To do that, we must use the `StoreConfig(StringMap & params)`
* constructor, so we'd like to `delete` its default constructor to enforce
* it.
*
* However, actually deleting it means that all the subclasses of
* `StoreConfig` will have their default constructor deleted (because it's
* supposed to call the deleted default constructor of `StoreConfig`). But
* because we're always using virtual inheritance, the constructors of
* child classes will never implicitely call this one, so deleting it will
* be more painful than anything else.
*
* So we `assert(false)` here to ensure at runtime that the right
* constructor is always called without having to redefine a custom
* constructor for each `*Config` class.
*/
StoreConfig() { assert(false); }
virtual const std::string name() = 0;
const PathSetting storeDir_{this, false, settings.nixStore,
"store", "path to the Nix store"};
@ -167,6 +212,14 @@ public:
"system-features",
"Optional features that the system this store builds on implements (like \"kvm\")."};
};
class Store : public std::enable_shared_from_this<Store>, public virtual StoreConfig
{
public:
typedef std::map<std::string, std::string> Params;
protected:
struct PathInfoCacheValue {
@ -200,6 +253,11 @@ protected:
Store(const Params & params);
public:
/**
* Perform any necessary effectful operation to make the store up and
* running
*/
virtual void init() {};
virtual ~Store() { }
@ -626,22 +684,25 @@ protected:
};
class LocalFSStore : public virtual Store
struct LocalFSStoreConfig : virtual StoreConfig
{
public:
// FIXME: the (Store*) cast works around a bug in gcc that causes
using StoreConfig::StoreConfig;
// FIXME: the (StoreConfig*) cast works around a bug in gcc that causes
// it to omit the call to the Setting constructor. Clang works fine
// either way.
const PathSetting rootDir{(Store*) this, true, "",
const PathSetting rootDir{(StoreConfig*) this, true, "",
"root", "directory prefixed to all other paths"};
const PathSetting stateDir{(Store*) this, false,
const PathSetting stateDir{(StoreConfig*) this, false,
rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
"state", "directory where Nix will store state"};
const PathSetting logDir{(Store*) this, false,
const PathSetting logDir{(StoreConfig*) this, false,
rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
"log", "directory where Nix will store state"};
};
class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig
{
public:
const static string drvsLogDir;
@ -744,23 +805,43 @@ StoreType getStoreType(const std::string & uri = settings.storeUri.get(),
‘substituters’ option and various legacy options. */
std::list<ref<Store>> getDefaultSubstituters();
/* Store implementation registration. */
typedef std::function<std::shared_ptr<Store>(
const std::string & uri, const Store::Params & params)> OpenStore;
struct RegisterStoreImplementation
struct StoreFactory
{
typedef std::vector<OpenStore> Implementations;
static Implementations * implementations;
RegisterStoreImplementation(OpenStore fun)
std::set<std::string> uriSchemes;
std::function<std::shared_ptr<Store> (const std::string & scheme, const std::string & uri, const Store::Params & params)> create;
std::function<std::shared_ptr<StoreConfig> ()> getConfig;
};
struct Implementations
{
if (!implementations) implementations = new Implementations;
implementations->push_back(fun);
static std::vector<StoreFactory> * registered;
template<typename T, typename TConfig>
static void add()
{
if (!registered) registered = new std::vector<StoreFactory>();
StoreFactory factory{
.uriSchemes = T::uriSchemes(),
.create =
([](const std::string & scheme, const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{ return std::make_shared<T>(scheme, uri, params); }),
.getConfig =
([]()
-> std::shared_ptr<StoreConfig>
{ return std::make_shared<TConfig>(StringMap({})); })
};
registered->push_back(factory);
}
};
template<typename T, typename TConfig>
struct RegisterStoreImplementation
{
RegisterStoreImplementation()
{
Implementations::add<T, TConfig>();
}
};
/* Display a set of paths in human-readable form (i.e., between quotes

15
src/libutil/abstractsettingtojson.hh Normal file
View file

@ -0,0 +1,15 @@
#pragma once
#include <nlohmann/json.hpp>
#include "config.hh"
namespace nix {
template<typename T>
std::map<std::string, nlohmann::json> BaseSetting<T>::toJSONObject()
{
auto obj = AbstractSetting::toJSONObject();
obj.emplace("value", value);
obj.emplace("defaultValue", defaultValue);
return obj;
}
}

33
src/libutil/config.cc
View file

@ -1,5 +1,6 @@
#include "config.hh"
#include "args.hh"
#include "abstractsettingtojson.hh"
#include <nlohmann/json.hpp>
@ -137,11 +138,7 @@ nlohmann::json Config::toJSON()
auto res = nlohmann::json::object();
for (auto & s : _settings)
if (!s.second.isAlias) {
auto obj = nlohmann::json::object();
obj.emplace("description", s.second.setting->description);
obj.emplace("aliases", s.second.setting->aliases);
obj.emplace("value", s.second.setting->toJSON());
res.emplace(s.first, obj);
res.emplace(s.first, s.second.setting->toJSON());
}
return res;
}
@ -168,19 +165,21 @@ void AbstractSetting::setDefault(const std::string & str)
nlohmann::json AbstractSetting::toJSON()
{
return to_string();
return nlohmann::json(toJSONObject());
}
std::map<std::string, nlohmann::json> AbstractSetting::toJSONObject()
{
std::map<std::string, nlohmann::json> obj;
obj.emplace("description", description);
obj.emplace("aliases", aliases);
return obj;
}
void AbstractSetting::convertToArg(Args & args, const std::string & category)
{
}
template<typename T>
nlohmann::json BaseSetting<T>::toJSON()
{
return value;
}
template<typename T>
void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
{
@ -259,11 +258,6 @@ template<> std::string BaseSetting<Strings>::to_string() const
return concatStringsSep(" ", value);
}
template<> nlohmann::json BaseSetting<Strings>::toJSON()
{
return value;
}
template<> void BaseSetting<StringSet>::set(const std::string & str)
{
value = tokenizeString<StringSet>(str);
@ -274,11 +268,6 @@ template<> std::string BaseSetting<StringSet>::to_string() const
return concatStringsSep(" ", value);
}
template<> nlohmann::json BaseSetting<StringSet>::toJSON()
{
return value;
}
template class BaseSetting<int>;
template class BaseSetting<unsigned int>;
template class BaseSetting<long>;

8
src/libutil/config.hh
View file

@ -206,7 +206,9 @@ protected:
virtual std::string to_string() const = 0;
virtual nlohmann::json toJSON();
nlohmann::json toJSON();
virtual std::map<std::string, nlohmann::json> toJSONObject();
virtual void convertToArg(Args & args, const std::string & category);
@ -220,6 +222,7 @@ class BaseSetting : public AbstractSetting
protected:
T value;
const T defaultValue;
public:
@ -229,6 +232,7 @@ public:
const std::set<std::string> & aliases = {})
: AbstractSetting(name, description, aliases)
, value(def)
, defaultValue(def)
{ }
operator const T &() const { return value; }
@ -251,7 +255,7 @@ public:
void convertToArg(Args & args, const std::string & category) override;
nlohmann::json toJSON() override;
std::map<std::string, nlohmann::json> toJSONObject() override;
};
template<typename T>

2
src/libutil/tests/config.cc
View file

@ -161,7 +161,7 @@ namespace nix {
Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
setting.assign("value");
ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"description":"description\n","value":"value"}})#");
ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","value":"value"}})#");
}
TEST(Config, setSettingAlias) {

2
src/libutil/url.hh
View file

@ -31,7 +31,7 @@ ParsedURL parseURL(const std::string & url);
// URI stuff.
const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
const static std::string schemeRegex = "(?:[a-z+]+)";
const static std::string schemeRegex = "(?:[a-z+.-]+)";
const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])";
const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";

44
src/nix/describe-stores.cc Normal file
View file

@ -0,0 +1,44 @@
#include "command.hh"
#include "common-args.hh"
#include "shared.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
using namespace nix;
struct CmdDescribeStores : Command, MixJSON
{
std::string description() override
{
return "show registered store types and their available options";
}
Category category() override { return catUtility; }
void run() override
{
auto res = nlohmann::json::object();
for (auto & implem : *Implementations::registered) {
auto storeConfig = implem.getConfig();
auto storeName = storeConfig->name();
res[storeName] = storeConfig->toJSON();
}
if (json) {
std::cout << res;
} else {
for (auto & [storeName, storeConfig] : res.items()) {
std::cout << "## " << storeName << std::endl << std::endl;
for (auto & [optionName, optionDesc] : storeConfig.items()) {
std::cout << "### " << optionName << std::endl << std::endl;
std::cout << optionDesc["description"].get<std::string>() << std::endl;
std::cout << "default: " << optionDesc["defaultValue"] << std::endl <<std::endl;
if (!optionDesc["aliases"].empty())
std::cout << "aliases: " << optionDesc["aliases"] << std::endl << std::endl;
}
}
}
}
};
static auto r1 = registerCommand<CmdDescribeStores>("describe-stores");

2
src/nix/develop.cc
View file

@ -392,7 +392,7 @@ struct CmdDevelop : Common, MixEnvironment
auto bashInstallable = std::make_shared<InstallableFlake>(
state,
std::move(installable->nixpkgsFlakeRef()),
installable->nixpkgsFlakeRef(),
Strings{"bashInteractive"},
Strings{"legacyPackages." + settings.thisSystem.get() + "."},
lockFlags);

2
src/nix/diff-closures.cc
View file

@ -81,7 +81,7 @@ void printClosureDiff(
auto beforeSize = totalSize(beforeVersions);
auto afterSize = totalSize(afterVersions);
auto sizeDelta = (int64_t) afterSize - (int64_t) beforeSize;
auto showDelta = abs(sizeDelta) >= 8 * 1024;
auto showDelta = std::abs(sizeDelta) >= 8 * 1024;
std::set<std::string> removed, unchanged;
for (auto & [version, _] : beforeVersions)

2
src/nix/installables.hh
View file

@ -69,7 +69,7 @@ struct Installable
virtual FlakeRef nixpkgsFlakeRef() const
{
return std::move(FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}));
return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
}
};

8
tests/describe-stores.sh Normal file
View file

@ -0,0 +1,8 @@
source common.sh
# Query an arbitrary value in `nix describe-stores --json`'s output just to
# check that it has the right structure
[[ $(nix --experimental-features 'nix-command flakes' describe-stores --json | jq '.["SSH Store"]["compress"]["defaultValue"]') == false ]]
# Ensure that the output of `nix describe-stores` isn't empty
[[ -n $(nix --experimental-features 'nix-command flakes' describe-stores) ]]

1
tests/local.mk
View file

@ -32,6 +32,7 @@ nix_tests = \
post-hook.sh \
function-trace.sh \
recursive.sh \
describe-stores.sh \
flakes.sh \
content-addressed.sh
# parallel.sh