philiptaron/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

Improve store setting descriptions / Markdown formatting

Browse source
This commit is contained in:
Eelco Dolstra 2023-03-22 14:23:36 +01:00
parent da8903ecc3
commit 5691bac202
11 changed files with 167 additions and 81 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
doc/manual/src/package-management/s3-substituter.md
View file

@ -1,41 +1,11 @@
# Serving a Nix store via S3
Nix has built-in support for storing and fetching store paths from
Nix has [built-in support](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store)
for storing and fetching store paths from
Amazon S3 and S3-compatible services. This uses the same *binary*
cache mechanism that Nix usually uses to fetch prebuilt binaries from
[cache.nixos.org](https://cache.nixos.org/).
The following options can be specified as URL parameters to the S3 URL:
- `profile`\
The name of the AWS configuration profile to use. By default Nix
will use the `default` profile.
- `region`\
The region of the S3 bucket. `us–east-1` by default.
If your bucket is not in `us–east-1`, you should always explicitly
specify the region parameter.
- `endpoint`\
The URL to your S3-compatible service, for when not using Amazon S3.
Do not specify this value if you're using Amazon S3.
> **Note**
>
> This endpoint must support HTTPS and will use path-based
> addressing instead of virtual host based addressing.
- `scheme`\
The scheme used for S3 requests, `https` (default) or `http`. This
option allows you to disable HTTPS for binary caches which don't
support it.
> **Note**
>
> HTTPS should be used if the cache might contain sensitive
> information.
In this example we will use the bucket named `example-nix-cache`.
## Anonymous Reads to your S3-compatible binary cache

34
src/libstore/binary-cache-store.hh
View file

@ -16,17 +16,33 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
{
using StoreConfig::StoreConfig;
const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', 'gzip', 'zstd', or 'none')"};
const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"};
const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression",
"NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."};
const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing",
"Whether to write a JSON file that lists the files in each NAR."};
const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info",
R"(
Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to
fetch debug info on demand
)"};
const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key",
"Path to the secret key used to sign the binary cache."};
const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache",
"Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."};
const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
"enable multi-threading compression for NARs, available for xz and zstd only currently"};
"Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."};
const Setting<int> compressionLevel{(StoreConfig*) this, -1, "compression-level",
"specify 'preset level' of compression to be used with NARs: "
"meaning and accepted range of values depends on compression method selected, "
"other than -1 which we reserve to indicate Nix defaults should be used"};
R"(
The *preset level* to be used when compressing NARs.
The meaning and accepted values depend on the compression method selected.
`-1` specifies that the default compression level should be used.
)"};
};
class BinaryCacheStore : public virtual BinaryCacheStoreConfig,

14
src/libstore/legacy-ssh-store.cc
View file

@ -1,3 +1,4 @@
#include "ssh-store-config.hh"
#include "archive.hh"
#include "pool.hh"
#include "remote-store.hh"
@ -12,15 +13,12 @@
namespace nix {
struct LegacySSHStoreConfig : virtual StoreConfig
struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
{
using StoreConfig::StoreConfig;
const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"};
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"};
const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
using CommonSSHStoreConfig::CommonSSHStoreConfig;
const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections",
"Maximum number of concurrent SSH connections."};
const std::string name() override { return "SSH Store"; }

16
src/libstore/local-fs-store.hh
View file

@ -9,20 +9,28 @@ namespace nix {
struct LocalFSStoreConfig : virtual StoreConfig
{
using StoreConfig::StoreConfig;
// FIXME: the (StoreConfig*) cast works around a bug in gcc that causes
// it to omit the call to the Setting constructor. Clang works fine
// either way.
const PathSetting rootDir{(StoreConfig*) this, true, "",
"root", "directory prefixed to all other paths"};
"root",
"Directory prefixed to all other paths."};
const PathSetting stateDir{(StoreConfig*) this, false,
rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
"state", "directory where Nix will store state"};
"state",
"Directory where Nix will store state."};
const PathSetting logDir{(StoreConfig*) this, false,
rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
"log", "directory where Nix will store state"};
"log",
"directory where Nix will store log files."};
const PathSetting realStoreDir{(StoreConfig*) this, false,
rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
"physical path to the Nix store"};
"Physical path of the Nix store."};
};
class LocalFSStore : public virtual LocalFSStoreConfig,

3
src/libstore/local-store.hh
View file

@ -38,7 +38,8 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
Setting<bool> requireSigs{(StoreConfig*) this,
settings.requireSigs,
"require-sigs", "whether store paths should have a trusted signature on import"};
"require-sigs",
"Whether store paths copied into this store should have a trusted signature."};
const std::string name() override { return "Local Store"; }

10
src/libstore/remote-store.hh
View file

@ -22,11 +22,13 @@ struct RemoteStoreConfig : virtual StoreConfig
{
using StoreConfig::StoreConfig;
const Setting<int> maxConnections{(StoreConfig*) this, 1,
"max-connections", "maximum number of concurrent connections to the Nix daemon"};
const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections",
"Maximum number of concurrent connections to the Nix daemon."};
const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, std::numeric_limits<unsigned int>::max(),
"max-connection-age", "number of seconds to reuse a connection"};
const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this,
std::numeric_limits<unsigned int>::max(),
"max-connection-age",
"Maximum age of a connection before it is closed."};
};
/* FIXME: RemoteStore is a misnomer - should be something like

64
src/libstore/s3-binary-cache-store.cc
View file

@ -192,17 +192,63 @@ S3BinaryCacheStore::S3BinaryCacheStore(const Params & params)
struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
{
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
const Setting<std::string> profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."};
const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."};
const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"};
const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"};
const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"};
const Setting<std::string> profile{(StoreConfig*) this, "", "profile",
R"(
The name of the AWS configuration profile to use. By default
Nix will use the `default` profile.
)"};
const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region",
R"(
The region of the S3 bucket. If your bucket is not in
`us–east-1`, you should always explicitly specify the region
parameter.
)"};
const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme",
R"(
The scheme used for S3 requests, `https` (default) or `http`. This
option allows you to disable HTTPS for binary caches which don't
support it.
> **Note**
>
> HTTPS should be used if the cache might contain sensitive
> information.
)"};
const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint",
R"(
The URL of the endpoint of an S3-compatible service such as MinIO.
Do not specify this setting if you're using Amazon S3.
> **Note**
>
> This endpoint must support HTTPS and will use path-based
> addressing instead of virtual host based addressing.
)"};
const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression",
"Compression method for `.narinfo` files."};
const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression",
"Compression method for `.ls` files."};
const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression",
R"(
Compression method for `log/*` files. It is recommended to
use a compression method supported by most web browsers
(e.g. `brotli`).
)"};
const Setting<bool> multipartUpload{
(StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"};
(StoreConfig*) this, false, "multipart-upload",
"Whether to use multi-part uploads."};
const Setting<uint64_t> bufferSize{
(StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
(StoreConfig*) this, 5 * 1024 * 1024, "buffer-size",
"Size (in bytes) of each part in multi-part uploads."};
const std::string name() override { return "S3 Binary Cache Store"; }

29
src/libstore/ssh-store-config.hh Normal file
View file

@ -0,0 +1,29 @@
#include "store-api.hh"
namespace nix {
struct CommonSSHStoreConfig : virtual StoreConfig
{
using StoreConfig::StoreConfig;
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key",
"Path to the SSH private key used to authenticate to the remote machine."};
const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key",
"The public host key of the remote machine."};
const Setting<bool> compress{(StoreConfig*) this, false, "compress",
"Whether to enable SSH compression."};
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program",
"Path to the `nix-store` executable on the remote machine."};
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store",
R"(
[Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
to be used on the remote machine. The default is `auto`
(i.e. use the Nix daemon or `/nix/store` directly).
)"};
};
}

9
src/libstore/ssh-store.cc
View file

@ -1,3 +1,4 @@
#include "ssh-store-config.hh"
#include "store-api.hh"
#include "remote-store.hh"
#include "remote-fs-accessor.hh"
@ -8,16 +9,10 @@
namespace nix {
struct SSHStoreConfig : virtual RemoteStoreConfig
struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig
{
using RemoteStoreConfig::RemoteStoreConfig;
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"};
const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
const std::string name() override { return "Experimental SSH Store"; }
std::string doc() override

31
src/libstore/store-api.hh
View file

@ -107,16 +107,35 @@ struct StoreConfig : public Config
}
const PathSetting storeDir_{this, false, settings.nixStore,
"store", "path to the Nix store"};
"store",
R"(
Logical location of the Nix store, usually
`/nix/store`. Note that you can only copy store paths
between stores if they have the same `store` setting.
)"};
const Path storeDir = storeDir_;
const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"};
const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size",
"Size of the in-memory store path metadata cache."};
const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"};
const Setting<bool> isTrusted{this, false, "trusted",
R"(
Whether paths from this store can be used as substitutes
even if they are not signed by a key listed in the
[`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys)
setting.
)"};
Setting<int> priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"};
Setting<int> priority{this, 0, "priority",
R"(
Priority of this store when used as a substituter. A lower value means a higher priority.
)"};
Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"};
Setting<bool> wantMassQuery{this, false, "want-mass-query",
R"(
Whether this store (when used as a substituter) can be
queried efficiently for path validity.
)"};
Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
"system-features",
@ -130,8 +149,6 @@ public:
typedef std::map<std::string, std::string> Params;
protected:
struct PathInfoCacheValue {

4
src/nix/help-stores.md
View file

@ -2,6 +2,10 @@ R"(
Nix supports different types of stores. These are described below.
## Store URL format
TODO
@stores@
)"