Merge pull request #8084 from edolstra/store-docs

Auto-generate store documentation
This commit is contained in:
Eelco Dolstra 2023-03-27 15:46:18 +02:00 committed by GitHub
commit 237587bc0a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
38 changed files with 557 additions and 207 deletions

View file

@ -1,4 +1,4 @@
{ toplevel }: cliDumpStr:
with builtins; with builtins;
with import ./utils.nix; with import ./utils.nix;
@ -63,7 +63,10 @@ let
* [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description} * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description}
''; '';
maybeDocumentation = if details ? doc then details.doc else ""; maybeDocumentation =
if details ? doc
then replaceStrings ["@stores@"] [storeDocs] details.doc
else "";
maybeOptions = if details.flags == {} then "" else '' maybeOptions = if details.flags == {} then "" else ''
# Options # Options
@ -110,13 +113,13 @@ let
}; };
in [ cmd ] ++ concatMap subcommand (attrNames details.commands or {}); in [ cmd ] ++ concatMap subcommand (attrNames details.commands or {});
parsedToplevel = builtins.fromJSON toplevel; cliDump = builtins.fromJSON cliDumpStr;
manpages = processCommand { manpages = processCommand {
command = "nix"; command = "nix";
details = parsedToplevel; details = cliDump.args;
filename = "nix"; filename = "nix";
toplevel = parsedToplevel; toplevel = cliDump.args;
}; };
tableOfContents = let tableOfContents = let
@ -124,4 +127,18 @@ let
" - [${page.command}](command-ref/new-cli/${page.name})"; " - [${page.command}](command-ref/new-cli/${page.name})";
in concatStringsSep "\n" (map showEntry manpages) + "\n"; in concatStringsSep "\n" (map showEntry manpages) + "\n";
storeDocs =
let
showStore = name: { settings, doc }:
''
## ${name}
${doc}
**Settings**:
${showSettings { useAnchors = false; } settings}
'';
in concatStrings (attrValues (mapAttrs showStore cliDump.stores));
in (listToAttrs manpages) // { "SUMMARY.md" = tableOfContents; } in (listToAttrs manpages) // { "SUMMARY.md" = tableOfContents; }

View file

@ -1,41 +0,0 @@
let
inherit (builtins) attrNames concatStringsSep isAttrs isBool;
inherit (import ./utils.nix) concatStrings squash splitLines;
in
optionsInfo:
let
showOption = name:
let
inherit (optionsInfo.${name}) description documentDefault defaultValue aliases;
result = squash ''
- <span id="conf-${name}">[`${name}`](#conf-${name})</span>
${indent " " body}
'';
# separate body to cleanly handle indentation
body = ''
${description}
**Default:** ${showDefault documentDefault defaultValue}
${showAliases aliases}
'';
showDefault = documentDefault: defaultValue:
if documentDefault then
# a StringMap value type is specified as a string, but
# this shows the value type. The empty stringmap is `null` in
# JSON, but that converts to `{ }` here.
if defaultValue == "" || defaultValue == [] || isAttrs defaultValue
then "*empty*"
else if isBool defaultValue then
if defaultValue then "`true`" else "`false`"
else "`${toString defaultValue}`"
else "*machine-specific*";
showAliases = aliases:
if aliases == [] then "" else
"**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}";
indent = prefix: s:
concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s));
in result;
in concatStrings (map showOption (attrNames optionsInfo))

View file

@ -60,17 +60,17 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
@$(call process-includes,$@,$@) @$(call process-includes,$@,$@)
$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
@rm -rf $@ @rm -rf $@ $@.tmp
$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }' $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)'
@mv $@.tmp $@ @mv $@.tmp $@
$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; $(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
@mv $@.tmp $@ @mv $@.tmp $@
$(d)/nix.json: $(bindir)/nix $(d)/nix.json: $(bindir)/nix
$(trace-gen) $(dummy-env) $(bindir)/nix __dump-args > $@.tmp $(trace-gen) $(dummy-env) $(bindir)/nix __dump-cli > $@.tmp
@mv $@.tmp $@ @mv $@.tmp $@
$(d)/conf-file.json: $(bindir)/nix $(d)/conf-file.json: $(bindir)/nix

View file

@ -1,41 +1,11 @@
# Serving a Nix store via S3 # Serving a Nix store via S3
Nix has built-in support for storing and fetching store paths from Nix has [built-in support](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store)
for storing and fetching store paths from
Amazon S3 and S3-compatible services. This uses the same *binary* Amazon S3 and S3-compatible services. This uses the same *binary*
cache mechanism that Nix usually uses to fetch prebuilt binaries from cache mechanism that Nix usually uses to fetch prebuilt binaries from
[cache.nixos.org](https://cache.nixos.org/). [cache.nixos.org](https://cache.nixos.org/).
The following options can be specified as URL parameters to the S3 URL:
- `profile`\
The name of the AWS configuration profile to use. By default Nix
will use the `default` profile.
- `region`\
The region of the S3 bucket. `useast-1` by default.
If your bucket is not in `useast-1`, you should always explicitly
specify the region parameter.
- `endpoint`\
The URL to your S3-compatible service, for when not using Amazon S3.
Do not specify this value if you're using Amazon S3.
> **Note**
>
> This endpoint must support HTTPS and will use path-based
> addressing instead of virtual host based addressing.
- `scheme`\
The scheme used for S3 requests, `https` (default) or `http`. This
option allows you to disable HTTPS for binary caches which don't
support it.
> **Note**
>
> HTTPS should be used if the cache might contain sensitive
> information.
In this example we will use the bucket named `example-nix-cache`. In this example we will use the bucket named `example-nix-cache`.
## Anonymous Reads to your S3-compatible binary cache ## Anonymous Reads to your S3-compatible binary cache

View file

@ -27,3 +27,7 @@
$ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^* $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*
``` ```
provides information about each of its outputs. provides information about each of its outputs.
* The experimental command `nix describe-stores` has been removed.
* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md).

View file

@ -38,4 +38,46 @@ rec {
filterAttrs = pred: set: filterAttrs = pred: set:
listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set)); listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set));
showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value }:
let
result = squash ''
- ${if useAnchors
then ''<span id="conf-${name}">[`${name}`](#conf-${name})</span>''
else ''`${name}`''}
${indent " " body}
'';
# separate body to cleanly handle indentation
body = ''
${description}
**Default:** ${showDefault documentDefault defaultValue}
${showAliases aliases}
'';
showDefault = documentDefault: defaultValue:
if documentDefault then
# a StringMap value type is specified as a string, but
# this shows the value type. The empty stringmap is `null` in
# JSON, but that converts to `{ }` here.
if defaultValue == "" || defaultValue == [] || isAttrs defaultValue
then "*empty*"
else if isBool defaultValue then
if defaultValue then "`true`" else "`false`"
else "`${toString defaultValue}`"
else "*machine-specific*";
showAliases = aliases:
if aliases == [] then "" else
"**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}";
indent = prefix: s:
concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s));
in result;
showSettings = args: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting args) settingsInfo));
} }

View file

@ -18,6 +18,7 @@ class EvalState;
struct Pos; struct Pos;
class Store; class Store;
static constexpr Command::Category catHelp = -1;
static constexpr Command::Category catSecondary = 100; static constexpr Command::Category catSecondary = 100;
static constexpr Command::Category catUtility = 101; static constexpr Command::Category catUtility = 101;
static constexpr Command::Category catNixInstallation = 102; static constexpr Command::Category catNixInstallation = 102;

View file

@ -136,7 +136,11 @@ MixEvalArgs::MixEvalArgs()
addFlag({ addFlag({
.longName = "eval-store", .longName = "eval-store",
.description = "The Nix store to use for evaluations.", .description =
R"(
The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them.
)",
.category = category, .category = category,
.labels = {"store-url"}, .labels = {"store-url"},
.handler = {&evalStoreUrl}, .handler = {&evalStoreUrl},

View file

@ -16,17 +16,33 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
{ {
using StoreConfig::StoreConfig; using StoreConfig::StoreConfig;
const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', 'gzip', 'zstd', or 'none')"}; const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression",
const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."};
const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"}; const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing",
const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"}; "Whether to write a JSON file that lists the files in each NAR."};
const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info",
R"(
Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to
fetch debug info on demand
)"};
const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key",
"Path to the secret key used to sign the binary cache."};
const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache",
"Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."};
const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression", const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
"enable multi-threading compression for NARs, available for xz and zstd only currently"}; "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."};
const Setting<int> compressionLevel{(StoreConfig*) this, -1, "compression-level", const Setting<int> compressionLevel{(StoreConfig*) this, -1, "compression-level",
"specify 'preset level' of compression to be used with NARs: " R"(
"meaning and accepted range of values depends on compression method selected, " The *preset level* to be used when compressing NARs.
"other than -1 which we reserve to indicate Nix defaults should be used"}; The meaning and accepted values depend on the compression method selected.
`-1` specifies that the default compression level should be used.
)"};
}; };
class BinaryCacheStore : public virtual BinaryCacheStoreConfig, class BinaryCacheStore : public virtual BinaryCacheStoreConfig,

View file

@ -7,6 +7,13 @@ struct DummyStoreConfig : virtual StoreConfig {
using StoreConfig::StoreConfig; using StoreConfig::StoreConfig;
const std::string name() override { return "Dummy Store"; } const std::string name() override { return "Dummy Store"; }
std::string doc() override
{
return
#include "dummy-store.md"
;
}
}; };
struct DummyStore : public virtual DummyStoreConfig, public virtual Store struct DummyStore : public virtual DummyStoreConfig, public virtual Store

View file

@ -0,0 +1,13 @@
R"(
**Store URL format**: `dummy://`
This store type represents a store that contains no store paths and
cannot be written to. It's useful when you want to use the Nix
evaluator when no actual Nix store exists, e.g.
```console
# nix eval --store dummy:// --expr '1 + 2'
```
)"

View file

@ -98,7 +98,12 @@ public:
Path nixDaemonSocketFile; Path nixDaemonSocketFile;
Setting<std::string> storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", Setting<std::string> storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store",
"The default Nix store to use."}; R"(
The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
to use for most operations.
See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md)
for supported store types and settings.
)"};
Setting<bool> keepFailed{this, false, "keep-failed", Setting<bool> keepFailed{this, false, "keep-failed",
"Whether to keep temporary directories of failed builds."}; "Whether to keep temporary directories of failed builds."};
@ -679,8 +684,9 @@ public:
Strings{"https://cache.nixos.org/"}, Strings{"https://cache.nixos.org/"},
"substituters", "substituters",
R"( R"(
A list of URLs of substituters, separated by whitespace. Substituters A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
are tried based on their Priority value, which each substituter can set to be used as substituters, separated by whitespace.
Substituters are tried based on their Priority value, which each substituter can set
independently. Lower value means higher priority. independently. Lower value means higher priority.
The default is `https://cache.nixos.org`, with a Priority of 40. The default is `https://cache.nixos.org`, with a Priority of 40.
@ -698,7 +704,8 @@ public:
Setting<StringSet> trustedSubstituters{ Setting<StringSet> trustedSubstituters{
this, {}, "trusted-substituters", this, {}, "trusted-substituters",
R"( R"(
A list of URLs of substituters, separated by whitespace. These are A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format),
separated by whitespace. These are
not used by default, but can be enabled by users of the Nix daemon not used by default, but can be enabled by users of the Nix daemon
by specifying `--option substituters urls` on the command by specifying `--option substituters urls` on the command
line. Unprivileged users are only allowed to pass a subset of the line. Unprivileged users are only allowed to pass a subset of the

View file

@ -12,7 +12,14 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
{ {
using BinaryCacheStoreConfig::BinaryCacheStoreConfig; using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
const std::string name() override { return "Http Binary Cache Store"; } const std::string name() override { return "HTTP Binary Cache Store"; }
std::string doc() override
{
return
#include "http-binary-cache-store.md"
;
}
}; };
class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore

View file

@ -0,0 +1,8 @@
R"(
**Store URL format**: `http://...`, `https://...`
This store allows a binary cache to be accessed via the HTTP
protocol.
)"

View file

@ -1,3 +1,4 @@
#include "ssh-store-config.hh"
#include "archive.hh" #include "archive.hh"
#include "pool.hh" #include "pool.hh"
#include "remote-store.hh" #include "remote-store.hh"
@ -12,17 +13,24 @@
namespace nix { namespace nix {
struct LegacySSHStoreConfig : virtual StoreConfig struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
{ {
using StoreConfig::StoreConfig; using CommonSSHStoreConfig::CommonSSHStoreConfig;
const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"};
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"};
const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
const std::string name() override { return "Legacy SSH Store"; } const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program",
"Path to the `nix-store` executable on the remote machine."};
const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections",
"Maximum number of concurrent SSH connections."};
const std::string name() override { return "SSH Store"; }
std::string doc() override
{
return
#include "legacy-ssh-store.md"
;
}
}; };
struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store
@ -51,6 +59,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params) LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params)
: StoreConfig(params) : StoreConfig(params)
, CommonSSHStoreConfig(params)
, LegacySSHStoreConfig(params) , LegacySSHStoreConfig(params)
, Store(params) , Store(params)
, host(host) , host(host)

View file

@ -0,0 +1,8 @@
R"(
**Store URL format**: `ssh://[username@]hostname`
This store type allows limited access to a remote store on another
machine via SSH.
)"

View file

@ -11,6 +11,13 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
using BinaryCacheStoreConfig::BinaryCacheStoreConfig; using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
const std::string name() override { return "Local Binary Cache Store"; } const std::string name() override { return "Local Binary Cache Store"; }
std::string doc() override
{
return
#include "local-binary-cache-store.md"
;
}
}; };
class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore

View file

@ -0,0 +1,16 @@
R"(
**Store URL format**: `file://`*path*
This store allows reading and writing a binary cache stored in *path*
in the local filesystem. If *path* does not exist, it will be created.
For example, the following builds or downloads `nixpkgs#hello` into
the local store and then copies it to the binary cache in
`/tmp/binary-cache`:
```
# nix copy --to file:///tmp/binary-cache nixpkgs#hello
```
)"

View file

@ -9,20 +9,28 @@ namespace nix {
struct LocalFSStoreConfig : virtual StoreConfig struct LocalFSStoreConfig : virtual StoreConfig
{ {
using StoreConfig::StoreConfig; using StoreConfig::StoreConfig;
// FIXME: the (StoreConfig*) cast works around a bug in gcc that causes // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes
// it to omit the call to the Setting constructor. Clang works fine // it to omit the call to the Setting constructor. Clang works fine
// either way. // either way.
const PathSetting rootDir{(StoreConfig*) this, true, "", const PathSetting rootDir{(StoreConfig*) this, true, "",
"root", "directory prefixed to all other paths"}; "root",
"Directory prefixed to all other paths."};
const PathSetting stateDir{(StoreConfig*) this, false, const PathSetting stateDir{(StoreConfig*) this, false,
rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
"state", "directory where Nix will store state"}; "state",
"Directory where Nix will store state."};
const PathSetting logDir{(StoreConfig*) this, false, const PathSetting logDir{(StoreConfig*) this, false,
rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
"log", "directory where Nix will store state"}; "log",
"directory where Nix will store log files."};
const PathSetting realStoreDir{(StoreConfig*) this, false, const PathSetting realStoreDir{(StoreConfig*) this, false,
rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
"physical path to the Nix store"}; "Physical path of the Nix store."};
}; };
class LocalFSStore : public virtual LocalFSStoreConfig, class LocalFSStore : public virtual LocalFSStoreConfig,

View file

@ -44,6 +44,13 @@
namespace nix { namespace nix {
std::string LocalStoreConfig::doc()
{
return
#include "local-store.md"
;
}
struct LocalStore::State::Stmts { struct LocalStore::State::Stmts {
/* Some precompiled SQLite statements. */ /* Some precompiled SQLite statements. */
SQLiteStmt RegisterValidPath; SQLiteStmt RegisterValidPath;
@ -413,6 +420,13 @@ LocalStore::LocalStore(const Params & params)
} }
LocalStore::LocalStore(std::string scheme, std::string path, const Params & params)
: LocalStore(params)
{
throw UnimplementedError("LocalStore");
}
AutoCloseFD LocalStore::openGCLock() AutoCloseFD LocalStore::openGCLock()
{ {
Path fnGCLock = stateDir + "/gc.lock"; Path fnGCLock = stateDir + "/gc.lock";
@ -1950,5 +1964,6 @@ std::optional<std::string> LocalStore::getVersion()
return nixVersion; return nixVersion;
} }
static RegisterStoreImplementation<LocalStore, LocalStoreConfig> regLocalStore;
} // namespace nix } // namespace nix

View file

@ -38,11 +38,13 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
Setting<bool> requireSigs{(StoreConfig*) this, Setting<bool> requireSigs{(StoreConfig*) this,
settings.requireSigs, settings.requireSigs,
"require-sigs", "whether store paths should have a trusted signature on import"}; "require-sigs",
"Whether store paths copied into this store should have a trusted signature."};
const std::string name() override { return "Local Store"; } const std::string name() override { return "Local Store"; }
};
std::string doc() override;
};
class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore
{ {
@ -100,9 +102,13 @@ public:
/* Initialise the local store, upgrading the schema if /* Initialise the local store, upgrading the schema if
necessary. */ necessary. */
LocalStore(const Params & params); LocalStore(const Params & params);
LocalStore(std::string scheme, std::string path, const Params & params);
~LocalStore(); ~LocalStore();
static std::set<std::string> uriSchemes()
{ return {}; }
/* Implementations of abstract store API methods. */ /* Implementations of abstract store API methods. */
std::string getUri() override; std::string getUri() override;

View file

@ -0,0 +1,39 @@
R"(
**Store URL format**: `local`, *root*
This store type accesses a Nix store in the local filesystem directly
(i.e. not via the Nix daemon). *root* is an absolute path that is
prefixed to other directories such as the Nix store directory. The
store pseudo-URL `local` denotes a store that uses `/` as its root
directory.
A store that uses a *root* other than `/` is called a *chroot
store*. With such stores, the store directory is "logically" still
`/nix/store`, so programs stored in them can only be built and
executed by `chroot`-ing into *root*. Chroot stores only support
building and running on Linux when [`mount namespaces`](https://man7.org/linux/man-pages/man7/mount_namespaces.7.html) and [`user namespaces`](https://man7.org/linux/man-pages/man7/user_namespaces.7.html) are
enabled.
For example, the following uses `/tmp/root` as the chroot environment
to build or download `nixpkgs#hello` and then execute it:
```console
# nix run --store /tmp/root nixpkgs#hello
Hello, world!
```
Here, the "physical" store location is `/tmp/root/nix/store`, and
Nix's store metadata is in `/tmp/root/nix/var/nix/db`.
It is also possible, but not recommended, to change the "logical"
location of the Nix store from its default of `/nix/store`. This makes
it impossible to use default substituters such as
`https://cache.nixos.org/`, and thus you may have to build everything
locally. Here is an example:
```console
# nix build --store 'local?store=/tmp/my-nix/store&state=/tmp/my-nix/state&log=/tmp/my-nix/log' nixpkgs#hello
```
)"

View file

@ -22,11 +22,13 @@ struct RemoteStoreConfig : virtual StoreConfig
{ {
using StoreConfig::StoreConfig; using StoreConfig::StoreConfig;
const Setting<int> maxConnections{(StoreConfig*) this, 1, const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections",
"max-connections", "maximum number of concurrent connections to the Nix daemon"}; "Maximum number of concurrent connections to the Nix daemon."};
const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, std::numeric_limits<unsigned int>::max(), const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this,
"max-connection-age", "number of seconds to reuse a connection"}; std::numeric_limits<unsigned int>::max(),
"max-connection-age",
"Maximum age of a connection before it is closed."};
}; };
/* FIXME: RemoteStore is a misnomer - should be something like /* FIXME: RemoteStore is a misnomer - should be something like

View file

@ -192,19 +192,72 @@ S3BinaryCacheStore::S3BinaryCacheStore(const Params & params)
struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
{ {
using BinaryCacheStoreConfig::BinaryCacheStoreConfig; using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
const Setting<std::string> profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."};
const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; const Setting<std::string> profile{(StoreConfig*) this, "", "profile",
const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."}; R"(
const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; The name of the AWS configuration profile to use. By default
const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"}; Nix will use the `default` profile.
const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"}; )"};
const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"};
const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region",
R"(
The region of the S3 bucket. If your bucket is not in
`useast-1`, you should always explicitly specify the region
parameter.
)"};
const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme",
R"(
The scheme used for S3 requests, `https` (default) or `http`. This
option allows you to disable HTTPS for binary caches which don't
support it.
> **Note**
>
> HTTPS should be used if the cache might contain sensitive
> information.
)"};
const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint",
R"(
The URL of the endpoint of an S3-compatible service such as MinIO.
Do not specify this setting if you're using Amazon S3.
> **Note**
>
> This endpoint must support HTTPS and will use path-based
> addressing instead of virtual host based addressing.
)"};
const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression",
"Compression method for `.narinfo` files."};
const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression",
"Compression method for `.ls` files."};
const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression",
R"(
Compression method for `log/*` files. It is recommended to
use a compression method supported by most web browsers
(e.g. `brotli`).
)"};
const Setting<bool> multipartUpload{ const Setting<bool> multipartUpload{
(StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"}; (StoreConfig*) this, false, "multipart-upload",
"Whether to use multi-part uploads."};
const Setting<uint64_t> bufferSize{ const Setting<uint64_t> bufferSize{
(StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size",
"Size (in bytes) of each part in multi-part uploads."};
const std::string name() override { return "S3 Binary Cache Store"; } const std::string name() override { return "S3 Binary Cache Store"; }
std::string doc() override
{
return
#include "s3-binary-cache-store.md"
;
}
}; };
struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore

View file

@ -0,0 +1,8 @@
R"(
**Store URL format**: `s3://`*bucket-name*
This store allows reading and writing a binary cache stored in an AWS
S3 bucket.
)"

View file

@ -0,0 +1,26 @@
#include "store-api.hh"
namespace nix {
struct CommonSSHStoreConfig : virtual StoreConfig
{
using StoreConfig::StoreConfig;
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key",
"Path to the SSH private key used to authenticate to the remote machine."};
const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key",
"The public host key of the remote machine."};
const Setting<bool> compress{(StoreConfig*) this, false, "compress",
"Whether to enable SSH compression."};
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store",
R"(
[Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
to be used on the remote machine. The default is `auto`
(i.e. use the Nix daemon or `/nix/store` directly).
)"};
};
}

View file

@ -1,3 +1,4 @@
#include "ssh-store-config.hh"
#include "store-api.hh" #include "store-api.hh"
#include "remote-store.hh" #include "remote-store.hh"
#include "remote-fs-accessor.hh" #include "remote-fs-accessor.hh"
@ -8,17 +9,22 @@
namespace nix { namespace nix {
struct SSHStoreConfig : virtual RemoteStoreConfig struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig
{ {
using RemoteStoreConfig::RemoteStoreConfig; using RemoteStoreConfig::RemoteStoreConfig;
using CommonSSHStoreConfig::CommonSSHStoreConfig;
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program",
const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"}; "Path to the `nix-daemon` executable on the remote machine."};
const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
const std::string name() override { return "SSH Store"; } const std::string name() override { return "Experimental SSH Store"; }
std::string doc() override
{
return
#include "ssh-store.md"
;
}
}; };
class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore
@ -28,6 +34,7 @@ public:
SSHStore(const std::string & scheme, const std::string & host, const Params & params) SSHStore(const std::string & scheme, const std::string & host, const Params & params)
: StoreConfig(params) : StoreConfig(params)
, RemoteStoreConfig(params) , RemoteStoreConfig(params)
, CommonSSHStoreConfig(params)
, SSHStoreConfig(params) , SSHStoreConfig(params)
, Store(params) , Store(params)
, RemoteStore(params) , RemoteStore(params)

View file

@ -0,0 +1,8 @@
R"(
**Store URL format**: `ssh-ng://[username@]hostname`
Experimental store type that allows full access to a Nix store on a
remote machine.
)"

View file

@ -101,17 +101,41 @@ struct StoreConfig : public Config
virtual const std::string name() = 0; virtual const std::string name() = 0;
virtual std::string doc()
{
return "";
}
const PathSetting storeDir_{this, false, settings.nixStore, const PathSetting storeDir_{this, false, settings.nixStore,
"store", "path to the Nix store"}; "store",
R"(
Logical location of the Nix store, usually
`/nix/store`. Note that you can only copy store paths
between stores if they have the same `store` setting.
)"};
const Path storeDir = storeDir_; const Path storeDir = storeDir_;
const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"}; const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size",
"Size of the in-memory store path metadata cache."};
const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; const Setting<bool> isTrusted{this, false, "trusted",
R"(
Whether paths from this store can be used as substitutes
even if they are not signed by a key listed in the
[`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys)
setting.
)"};
Setting<int> priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"}; Setting<int> priority{this, 0, "priority",
R"(
Priority of this store when used as a substituter. A lower value means a higher priority.
)"};
Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; Setting<bool> wantMassQuery{this, false, "want-mass-query",
R"(
Whether this store (when used as a substituter) can be
queried efficiently for path validity.
)"};
Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(), Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
"system-features", "system-features",
@ -125,8 +149,6 @@ public:
typedef std::map<std::string, std::string> Params; typedef std::map<std::string, std::string> Params;
protected: protected:
struct PathInfoCacheValue { struct PathInfoCacheValue {

View file

@ -15,6 +15,13 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon
} }
const std::string name() override { return "Local Daemon Store"; } const std::string name() override { return "Local Daemon Store"; }
std::string doc() override
{
return
#include "uds-remote-store.md"
;
}
}; };
class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore

View file

@ -0,0 +1,9 @@
R"(
**Store URL format**: `daemon`, `unix://`*path*
This store type accesses a Nix store by talking to a Nix daemon
listening on the Unix domain socket *path*. The store pseudo-URL
`daemon` is equivalent to `unix:///nix/var/nix/daemon-socket/socket`.
)"

View file

@ -1,44 +0,0 @@
#include "command.hh"
#include "common-args.hh"
#include "shared.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
using namespace nix;
struct CmdDescribeStores : Command, MixJSON
{
std::string description() override
{
return "show registered store types and their available options";
}
Category category() override { return catUtility; }
void run() override
{
auto res = nlohmann::json::object();
for (auto & implem : *Implementations::registered) {
auto storeConfig = implem.getConfig();
auto storeName = storeConfig->name();
res[storeName] = storeConfig->toJSON();
}
if (json) {
logger->cout("%s", res);
} else {
for (auto & [storeName, storeConfig] : res.items()) {
std::cout << "## " << storeName << std::endl << std::endl;
for (auto & [optionName, optionDesc] : storeConfig.items()) {
std::cout << "### " << optionName << std::endl << std::endl;
std::cout << optionDesc["description"].get<std::string>() << std::endl;
std::cout << "default: " << optionDesc["defaultValue"] << std::endl <<std::endl;
if (!optionDesc["aliases"].empty())
std::cout << "aliases: " << optionDesc["aliases"] << std::endl << std::endl;
}
}
}
}
};
static auto rDescribeStore = registerCommand<CmdDescribeStores>("describe-stores");

46
src/nix/help-stores.md Normal file
View file

@ -0,0 +1,46 @@
R"(
Nix supports different types of stores. These are described below.
## Store URL format
Stores are specified using a URL-like syntax. For example, the command
```console
# nix path-info --store https://cache.nixos.org/ --json \
/nix/store/a7gvj343m05j2s32xcnwr35v31ynlypr-coreutils-9.1
```
fetches information about a store path in the HTTP binary cache
located at https://cache.nixos.org/, which is a type of store.
Store URLs can specify **store settings** using URL query strings,
i.e. by appending `?name1=value1&name2=value2&...` to the URL. For
instance,
```
--store ssh://machine.example.org?ssh-key=/path/to/my/key
```
tells Nix to access the store on a remote machine via the SSH
protocol, using `/path/to/my/key` as the SSH private key. The
supported settings for each store type are documented below.
The special store URL `auto` causes Nix to automatically select a
store as follows:
* Use the [local store](#local-store) `/nix/store` if `/nix/var/nix`
is writable by the current user.
* Otherwise, if `/nix/var/nix/daemon-socket/socket` exists, [connect
to the Nix daemon listening on that socket](#local-daemon-store).
* Otherwise, on Linux only, use the [local chroot store](#local-store)
`~/.local/share/nix/root`, which will be created automatically if it
does not exist.
* Otherwise, use the [local store](#local-store) `/nix/store`.
@stores@
)"

View file

@ -64,6 +64,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix") NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix")
{ {
categories.clear(); categories.clear();
categories[catHelp] = "Help commands";
categories[Command::catDefault] = "Main commands"; categories[Command::catDefault] = "Main commands";
categories[catSecondary] = "Infrequently used commands"; categories[catSecondary] = "Infrequently used commands";
categories[catUtility] = "Utility/scripting commands"; categories[catUtility] = "Utility/scripting commands";
@ -163,11 +164,29 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
{ {
commands = RegisterCommand::getCommandsFor({}); commands = RegisterCommand::getCommandsFor({});
} }
std::string dumpCli()
{
auto res = nlohmann::json::object();
res["args"] = toJSON();
auto stores = nlohmann::json::object();
for (auto & implem : *Implementations::registered) {
auto storeConfig = implem.getConfig();
auto storeName = storeConfig->name();
stores[storeName]["doc"] = storeConfig->doc();
stores[storeName]["settings"] = storeConfig->toJSON();
}
res["stores"] = std::move(stores);
return res.dump();
}
}; };
/* Render the help for the specified subcommand to stdout using /* Render the help for the specified subcommand to stdout using
lowdown. */ lowdown. */
static void showHelp(std::vector<std::string> subcommand, MultiCommand & toplevel) static void showHelp(std::vector<std::string> subcommand, NixArgs & toplevel)
{ {
auto mdName = subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand)); auto mdName = subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand));
@ -188,11 +207,11 @@ static void showHelp(std::vector<std::string> subcommand, MultiCommand & topleve
, "/"), , "/"),
*vUtils); *vUtils);
auto attrs = state.buildBindings(16); auto vDump = state.allocValue();
attrs.alloc("toplevel").mkString(toplevel.toJSON().dump()); vDump->mkString(toplevel.dumpCli());
auto vRes = state.allocValue(); auto vRes = state.allocValue();
state.callFunction(*vGenerateManpage, state.allocValue()->mkAttrs(attrs), *vRes, noPos); state.callFunction(*vGenerateManpage, *vDump, *vRes, noPos);
auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md")); auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md"));
if (!attr) if (!attr)
@ -204,6 +223,14 @@ static void showHelp(std::vector<std::string> subcommand, MultiCommand & topleve
std::cout << renderMarkdownToTerminal(markdown) << "\n"; std::cout << renderMarkdownToTerminal(markdown) << "\n";
} }
static NixArgs & getNixArgs(Command & cmd)
{
assert(cmd.parent);
MultiCommand * toplevel = cmd.parent;
while (toplevel->parent) toplevel = toplevel->parent;
return dynamic_cast<NixArgs &>(*toplevel);
}
struct CmdHelp : Command struct CmdHelp : Command
{ {
std::vector<std::string> subcommand; std::vector<std::string> subcommand;
@ -228,17 +255,43 @@ struct CmdHelp : Command
; ;
} }
Category category() override { return catHelp; }
void run() override void run() override
{ {
assert(parent); assert(parent);
MultiCommand * toplevel = parent; MultiCommand * toplevel = parent;
while (toplevel->parent) toplevel = toplevel->parent; while (toplevel->parent) toplevel = toplevel->parent;
showHelp(subcommand, *toplevel); showHelp(subcommand, getNixArgs(*this));
} }
}; };
static auto rCmdHelp = registerCommand<CmdHelp>("help"); static auto rCmdHelp = registerCommand<CmdHelp>("help");
struct CmdHelpStores : Command
{
std::string description() override
{
return "show help about store types and their settings";
}
std::string doc() override
{
return
#include "help-stores.md"
;
}
Category category() override { return catHelp; }
void run() override
{
showHelp({"help-stores"}, getNixArgs(*this));
}
};
static auto rCmdHelpStores = registerCommand<CmdHelpStores>("help-stores");
void mainWrapped(int argc, char * * argv) void mainWrapped(int argc, char * * argv)
{ {
savedArgv = argv; savedArgv = argv;
@ -290,8 +343,8 @@ void mainWrapped(int argc, char * * argv)
NixArgs args; NixArgs args;
if (argc == 2 && std::string(argv[1]) == "__dump-args") { if (argc == 2 && std::string(argv[1]) == "__dump-cli") {
logger->cout("%s", args.toJSON()); logger->cout(args.dumpCli());
return; return;
} }

View file

@ -220,8 +220,7 @@ operate are determined as follows:
# Nix stores # Nix stores
Most `nix` subcommands operate on a *Nix store*. Most `nix` subcommands operate on a *Nix store*. These are documented
in [`nix help-stores`](./nix3-help-stores.md).
TODO: list store types, options
)"" )""

View file

@ -1,8 +0,0 @@
source common.sh
# Query an arbitrary value in `nix describe-stores --json`'s output just to
# check that it has the right structure
[[ $(nix --experimental-features 'nix-command flakes' describe-stores --json | jq '.["SSH Store"]["compress"]["defaultValue"]') == false ]]
# Ensure that the output of `nix describe-stores` isn't empty
[[ -n $(nix --experimental-features 'nix-command flakes' describe-stores) ]]

View file

@ -116,7 +116,6 @@ nix_tests = \
db-migration.sh \ db-migration.sh \
bash-profile.sh \ bash-profile.sh \
pass-as-file.sh \ pass-as-file.sh \
describe-stores.sh \
nix-profile.sh \ nix-profile.sh \
suggestions.sh \ suggestions.sh \
store-ping.sh \ store-ping.sh \