Merge pull request #8084 from edolstra/store-docs

Auto-generate store documentation

doc/manual/generate-manpage.nix

@@ -1,4 +1,4 @@
-{ toplevel }:
+cliDumpStr:
 
 with builtins;
 with import ./utils.nix;
@@ -63,7 +63,10 @@ let
         * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description}
       '';
 
-      maybeDocumentation = if details ? doc then details.doc else "";
+      maybeDocumentation =
+        if details ? doc
+        then replaceStrings ["@stores@"] [storeDocs] details.doc
+        else "";
 
       maybeOptions = if details.flags == {} then "" else ''
         # Options
@@ -110,13 +113,13 @@ let
       };
     in [ cmd ] ++ concatMap subcommand (attrNames details.commands or {});
 
-  parsedToplevel = builtins.fromJSON toplevel;
+  cliDump = builtins.fromJSON cliDumpStr;
 
   manpages = processCommand {
     command = "nix";
-    details = parsedToplevel;
+    details = cliDump.args;
     filename = "nix";
-    toplevel = parsedToplevel;
+    toplevel = cliDump.args;
   };
 
   tableOfContents = let
@@ -124,4 +127,18 @@ let
       "    - [${page.command}](command-ref/new-cli/${page.name})";
     in concatStringsSep "\n" (map showEntry manpages) + "\n";
 
+  storeDocs =
+    let
+      showStore = name: { settings, doc }:
+        ''
+          ## ${name}
+
+          ${doc}
+
+          **Settings**:
+
+          ${showSettings { useAnchors = false; } settings}
+        '';
+    in concatStrings (attrValues (mapAttrs showStore cliDump.stores));
+
 in (listToAttrs manpages) // { "SUMMARY.md" = tableOfContents; }

doc/manual/generate-options.nix

@@ -1,41 +0,0 @@
-let
-  inherit (builtins) attrNames concatStringsSep isAttrs isBool;
-  inherit (import ./utils.nix) concatStrings squash splitLines;
-in
-
-optionsInfo:
-let
-  showOption = name:
-    let
-      inherit (optionsInfo.${name}) description documentDefault defaultValue aliases;
-      result = squash ''
-          - <span id="conf-${name}">[`${name}`](#conf-${name})</span>
-
-          ${indent "  " body}
-        '';
-      # separate body to cleanly handle indentation
-      body = ''
-          ${description}
-
-          **Default:** ${showDefault documentDefault defaultValue}
-
-          ${showAliases aliases}
-        '';
-      showDefault = documentDefault: defaultValue:
-        if documentDefault then
-          # a StringMap value type is specified as a string, but
-          # this shows the value type. The empty stringmap is `null` in
-          # JSON, but that converts to `{ }` here.
-          if defaultValue == "" || defaultValue == [] || isAttrs defaultValue
-            then "*empty*"
-            else if isBool defaultValue then
-              if defaultValue then "`true`" else "`false`"
-            else "`${toString defaultValue}`"
-        else "*machine-specific*";
-      showAliases = aliases:
-          if aliases == [] then "" else
-            "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}";
-      indent = prefix: s:
-        concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s));
-      in result;
-in concatStrings (map showOption (attrNames optionsInfo))

doc/manual/local.mk

@@ -60,17 +60,17 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
 	@$(call process-includes,$@,$@)
 
 $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
-	@rm -rf $@
+	@rm -rf $@ $@.tmp
-	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }'
+	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
+$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
-	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
+	$(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
 	@mv $@.tmp $@
 
 $(d)/nix.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-args > $@.tmp
+	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-cli > $@.tmp
 	@mv $@.tmp $@
 
 $(d)/conf-file.json: $(bindir)/nix

doc/manual/src/package-management/s3-substituter.md

@@ -1,41 +1,11 @@
 # Serving a Nix store via S3
 
-Nix has built-in support for storing and fetching store paths from
+Nix has [built-in support](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store)
+for storing and fetching store paths from
 Amazon S3 and S3-compatible services. This uses the same *binary*
 cache mechanism that Nix usually uses to fetch prebuilt binaries from
 [cache.nixos.org](https://cache.nixos.org/).
 
-The following options can be specified as URL parameters to the S3 URL:
-
-  - `profile`\
-    The name of the AWS configuration profile to use. By default Nix
-    will use the `default` profile.
-
-  - `region`\
-    The region of the S3 bucket. `us–east-1` by default.
-    
-    If your bucket is not in `us–east-1`, you should always explicitly
-    specify the region parameter.
-
-  - `endpoint`\
-    The URL to your S3-compatible service, for when not using Amazon S3.
-    Do not specify this value if you're using Amazon S3.
-    
-    > **Note**
-    > 
-    > This endpoint must support HTTPS and will use path-based
-    > addressing instead of virtual host based addressing.
-
-  - `scheme`\
-    The scheme used for S3 requests, `https` (default) or `http`. This
-    option allows you to disable HTTPS for binary caches which don't
-    support it.
-    
-    > **Note**
-    > 
-    > HTTPS should be used if the cache might contain sensitive
-    > information.
-
 In this example we will use the bucket named `example-nix-cache`.
 
 ## Anonymous Reads to your S3-compatible binary cache

doc/manual/src/release-notes/rl-next.md

@@ -27,3 +27,7 @@
   $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*
   ```
   provides information about each of its outputs.
+
+* The experimental command `nix describe-stores` has been removed.
+
+* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md).

doc/manual/utils.nix

@@ -38,4 +38,46 @@ rec {
 
   filterAttrs = pred: set:
     listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set));
+
+  showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value }:
+    let
+      result = squash ''
+          - ${if useAnchors
+              then ''<span id="conf-${name}">[`${name}`](#conf-${name})</span>''
+              else ''`${name}`''}
+
+          ${indent "  " body}
+        '';
+
+      # separate body to cleanly handle indentation
+      body = ''
+          ${description}
+
+          **Default:** ${showDefault documentDefault defaultValue}
+
+          ${showAliases aliases}
+        '';
+
+      showDefault = documentDefault: defaultValue:
+        if documentDefault then
+          # a StringMap value type is specified as a string, but
+          # this shows the value type. The empty stringmap is `null` in
+          # JSON, but that converts to `{ }` here.
+          if defaultValue == "" || defaultValue == [] || isAttrs defaultValue
+            then "*empty*"
+            else if isBool defaultValue then
+              if defaultValue then "`true`" else "`false`"
+            else "`${toString defaultValue}`"
+        else "*machine-specific*";
+
+      showAliases = aliases:
+          if aliases == [] then "" else
+            "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}";
+
+      indent = prefix: s:
+        concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s));
+
+    in result;
+
+  showSettings = args: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting args) settingsInfo));
 }

src/libcmd/command.hh

@@ -18,6 +18,7 @@ class EvalState;
 struct Pos;
 class Store;
 
+static constexpr Command::Category catHelp = -1;
 static constexpr Command::Category catSecondary = 100;
 static constexpr Command::Category catUtility = 101;
 static constexpr Command::Category catNixInstallation = 102;

src/libcmd/common-eval-args.cc

@@ -136,7 +136,11 @@ MixEvalArgs::MixEvalArgs()
 
     addFlag({
         .longName = "eval-store",
-        .description = "The Nix store to use for evaluations.",
+        .description =
+          R"(
+            The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+            to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them.
+          )",
         .category = category,
         .labels = {"store-url"},
         .handler = {&evalStoreUrl},

src/libstore/binary-cache-store.hh

@@ -16,17 +16,33 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
 {
     using StoreConfig::StoreConfig;
 
-    const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', 'gzip', 'zstd', or 'none')"};
+    const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression",
-    const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
+        "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."};
-    const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
+
-    const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"};
+    const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing",
-    const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"};
+        "Whether to write a JSON file that lists the files in each NAR."};
+
+    const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info",
+        R"(
+          Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to
+          fetch debug info on demand
+        )"};
+
+    const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key",
+        "Path to the secret key used to sign the binary cache."};
+
+    const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache",
+        "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."};
+
     const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
-        "enable multi-threading compression for NARs, available for xz and zstd only currently"};
+        "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."};
+
     const Setting<int> compressionLevel{(StoreConfig*) this, -1, "compression-level",
-        "specify 'preset level' of compression to be used with NARs: "
+        R"(
-        "meaning and accepted range of values depends on compression method selected, "
+          The *preset level* to be used when compressing NARs.
-        "other than -1 which we reserve to indicate Nix defaults should be used"};
+          The meaning and accepted values depend on the compression method selected.
+          `-1` specifies that the default compression level should be used.
+        )"};
 };
 
 class BinaryCacheStore : public virtual BinaryCacheStoreConfig,

src/libstore/dummy-store.cc

@@ -7,6 +7,13 @@ struct DummyStoreConfig : virtual StoreConfig {
     using StoreConfig::StoreConfig;
 
     const std::string name() override { return "Dummy Store"; }
+
+    std::string doc() override
+    {
+        return
+          #include "dummy-store.md"
+          ;
+    }
 };
 
 struct DummyStore : public virtual DummyStoreConfig, public virtual Store

src/libstore/dummy-store.md

@@ -0,0 +1,13 @@
+R"(
+
+**Store URL format**: `dummy://`
+
+This store type represents a store that contains no store paths and
+cannot be written to. It's useful when you want to use the Nix
+evaluator when no actual Nix store exists, e.g.
+
+```console
+# nix eval --store dummy:// --expr '1 + 2'
+```
+
+)"

src/libstore/globals.hh

@@ -98,7 +98,12 @@ public:
     Path nixDaemonSocketFile;
 
     Setting<std::string> storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store",
-        "The default Nix store to use."};
+        R"(
+          The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+          to use for most operations.
+          See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md)
+          for supported store types and settings.
+        )"};
 
     Setting<bool> keepFailed{this, false, "keep-failed",
         "Whether to keep temporary directories of failed builds."};
@@ -679,8 +684,9 @@ public:
         Strings{"https://cache.nixos.org/"},
         "substituters",
         R"(
-          A list of URLs of substituters, separated by whitespace. Substituters
+          A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
-          are tried based on their Priority value, which each substituter can set
+          to be used as substituters, separated by whitespace.
+          Substituters are tried based on their Priority value, which each substituter can set
           independently. Lower value means higher priority.
           The default is `https://cache.nixos.org`, with a Priority of 40.
 
@@ -698,7 +704,8 @@ public:
     Setting<StringSet> trustedSubstituters{
         this, {}, "trusted-substituters",
         R"(
-          A list of URLs of substituters, separated by whitespace. These are
+          A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format),
+          separated by whitespace. These are
           not used by default, but can be enabled by users of the Nix daemon
           by specifying `--option substituters urls` on the command
           line. Unprivileged users are only allowed to pass a subset of the

src/libstore/http-binary-cache-store.cc

@@ -12,7 +12,14 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
 {
     using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
 
-    const std::string name() override { return "Http Binary Cache Store"; }
+    const std::string name() override { return "HTTP Binary Cache Store"; }
+
+    std::string doc() override
+    {
+        return
+          #include "http-binary-cache-store.md"
+          ;
+    }
 };
 
 class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore

src/libstore/http-binary-cache-store.md

@@ -0,0 +1,8 @@
+R"(
+
+**Store URL format**: `http://...`, `https://...`
+
+This store allows a binary cache to be accessed via the HTTP
+protocol.
+
+)"

src/libstore/legacy-ssh-store.cc

@@ -1,3 +1,4 @@
+#include "ssh-store-config.hh"
 #include "archive.hh"
 #include "pool.hh"
 #include "remote-store.hh"
@@ -12,17 +13,24 @@
 
 namespace nix {
 
-struct LegacySSHStoreConfig : virtual StoreConfig
+struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
 {
-    using StoreConfig::StoreConfig;
+    using CommonSSHStoreConfig::CommonSSHStoreConfig;
-    const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"};
-    const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
-    const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"};
-    const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
-    const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
-    const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
 
-    const std::string name() override { return "Legacy SSH Store"; }
+    const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program",
+        "Path to the `nix-store` executable on the remote machine."};
+
+    const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections",
+        "Maximum number of concurrent SSH connections."};
+
+    const std::string name() override { return "SSH Store"; }
+
+    std::string doc() override
+    {
+        return
+          #include "legacy-ssh-store.md"
+          ;
+    }
 };
 
 struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store
@@ -51,6 +59,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
 
     LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params)
         : StoreConfig(params)
+        , CommonSSHStoreConfig(params)
         , LegacySSHStoreConfig(params)
         , Store(params)
         , host(host)

src/libstore/legacy-ssh-store.md

@@ -0,0 +1,8 @@
+R"(
+
+**Store URL format**: `ssh://[username@]hostname`
+
+This store type allows limited access to a remote store on another
+machine via SSH.
+
+)"

src/libstore/local-binary-cache-store.cc

@@ -11,6 +11,13 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
     using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
 
     const std::string name() override { return "Local Binary Cache Store"; }
+
+    std::string doc() override
+    {
+        return
+          #include "local-binary-cache-store.md"
+          ;
+    }
 };
 
 class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore

src/libstore/local-binary-cache-store.md

@@ -0,0 +1,16 @@
+R"(
+
+**Store URL format**: `file://`*path*
+
+This store allows reading and writing a binary cache stored in *path*
+in the local filesystem. If *path* does not exist, it will be created.
+
+For example, the following builds or downloads `nixpkgs#hello` into
+the local store and then copies it to the binary cache in
+`/tmp/binary-cache`:
+
+```
+# nix copy --to file:///tmp/binary-cache nixpkgs#hello
+```
+
+)"

src/libstore/local-fs-store.hh

@@ -9,20 +9,28 @@ namespace nix {
 struct LocalFSStoreConfig : virtual StoreConfig
 {
     using StoreConfig::StoreConfig;
+
     // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes
     // it to omit the call to the Setting constructor. Clang works fine
     // either way.
+
     const PathSetting rootDir{(StoreConfig*) this, true, "",
-        "root", "directory prefixed to all other paths"};
+        "root",
+        "Directory prefixed to all other paths."};
+
     const PathSetting stateDir{(StoreConfig*) this, false,
         rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
-        "state", "directory where Nix will store state"};
+        "state",
+        "Directory where Nix will store state."};
+
     const PathSetting logDir{(StoreConfig*) this, false,
         rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
-        "log", "directory where Nix will store state"};
+        "log",
+        "directory where Nix will store log files."};
+
     const PathSetting realStoreDir{(StoreConfig*) this, false,
         rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
-        "physical path to the Nix store"};
+        "Physical path of the Nix store."};
 };
 
 class LocalFSStore : public virtual LocalFSStoreConfig,

src/libstore/local-store.cc

@@ -44,6 +44,13 @@
 
 namespace nix {
 
+std::string LocalStoreConfig::doc()
+{
+    return
+        #include "local-store.md"
+        ;
+}
+
 struct LocalStore::State::Stmts {
     /* Some precompiled SQLite statements. */
     SQLiteStmt RegisterValidPath;
@@ -413,6 +420,13 @@ LocalStore::LocalStore(const Params & params)
 }
 
 
+LocalStore::LocalStore(std::string scheme, std::string path, const Params & params)
+    : LocalStore(params)
+{
+    throw UnimplementedError("LocalStore");
+}
+
+
 AutoCloseFD LocalStore::openGCLock()
 {
     Path fnGCLock = stateDir + "/gc.lock";
@@ -1950,5 +1964,6 @@ std::optional<std::string> LocalStore::getVersion()
     return nixVersion;
 }
 
+static RegisterStoreImplementation<LocalStore, LocalStoreConfig> regLocalStore;
 
 }  // namespace nix

src/libstore/local-store.hh

@@ -38,11 +38,13 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
 
     Setting<bool> requireSigs{(StoreConfig*) this,
         settings.requireSigs,
-        "require-sigs", "whether store paths should have a trusted signature on import"};
+        "require-sigs",
+        "Whether store paths copied into this store should have a trusted signature."};
 
     const std::string name() override { return "Local Store"; }
-};
 
+    std::string doc() override;
+};
 
 class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore
 {
@@ -100,9 +102,13 @@ public:
     /* Initialise the local store, upgrading the schema if
        necessary. */
     LocalStore(const Params & params);
+    LocalStore(std::string scheme, std::string path, const Params & params);
 
     ~LocalStore();
 
+    static std::set<std::string> uriSchemes()
+    { return {}; }
+
     /* Implementations of abstract store API methods. */
 
     std::string getUri() override;

src/libstore/local-store.md

@@ -0,0 +1,39 @@
+R"(
+
+**Store URL format**: `local`, *root*
+
+This store type accesses a Nix store in the local filesystem directly
+(i.e. not via the Nix daemon). *root* is an absolute path that is
+prefixed to other directories such as the Nix store directory. The
+store pseudo-URL `local` denotes a store that uses `/` as its root
+directory.
+
+A store that uses a *root* other than `/` is called a *chroot
+store*. With such stores, the store directory is "logically" still
+`/nix/store`, so programs stored in them can only be built and
+executed by `chroot`-ing into *root*. Chroot stores only support
+building and running on Linux when [`mount namespaces`](https://man7.org/linux/man-pages/man7/mount_namespaces.7.html) and [`user namespaces`](https://man7.org/linux/man-pages/man7/user_namespaces.7.html) are
+enabled.
+
+For example, the following uses `/tmp/root` as the chroot environment
+to build or download `nixpkgs#hello` and then execute it:
+
+```console
+# nix run --store /tmp/root nixpkgs#hello
+Hello, world!
+```
+
+Here, the "physical" store location is `/tmp/root/nix/store`, and
+Nix's store metadata is in `/tmp/root/nix/var/nix/db`.
+
+It is also possible, but not recommended, to change the "logical"
+location of the Nix store from its default of `/nix/store`. This makes
+it impossible to use default substituters such as
+`https://cache.nixos.org/`, and thus you may have to build everything
+locally. Here is an example:
+
+```console
+# nix build --store 'local?store=/tmp/my-nix/store&state=/tmp/my-nix/state&log=/tmp/my-nix/log' nixpkgs#hello
+```
+
+)"

src/libstore/remote-store.hh

@@ -22,11 +22,13 @@ struct RemoteStoreConfig : virtual StoreConfig
 {
     using StoreConfig::StoreConfig;
 
-    const Setting<int> maxConnections{(StoreConfig*) this, 1,
+    const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections",
-            "max-connections", "maximum number of concurrent connections to the Nix daemon"};
+        "Maximum number of concurrent connections to the Nix daemon."};
 
-    const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, std::numeric_limits<unsigned int>::max(),
+    const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this,
-            "max-connection-age", "number of seconds to reuse a connection"};
+        std::numeric_limits<unsigned int>::max(),
+        "max-connection-age",
+        "Maximum age of a connection before it is closed."};
 };
 
 /* FIXME: RemoteStore is a misnomer - should be something like

src/libstore/s3-binary-cache-store.cc

@@ -192,19 +192,72 @@ S3BinaryCacheStore::S3BinaryCacheStore(const Params & params)
 struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
 {
     using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
-    const Setting<std::string> profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."};
+
-    const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
+    const Setting<std::string> profile{(StoreConfig*) this, "", "profile",
-    const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."};
+        R"(
-    const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
+          The name of the AWS configuration profile to use. By default
-    const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"};
+          Nix will use the `default` profile.
-    const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"};
+        )"};
-    const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"};
+
+    const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region",
+        R"(
+          The region of the S3 bucket. If your bucket is not in
+          `us–east-1`, you should always explicitly specify the region
+          parameter.
+        )"};
+
+    const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme",
+        R"(
+          The scheme used for S3 requests, `https` (default) or `http`. This
+          option allows you to disable HTTPS for binary caches which don't
+          support it.
+
+          > **Note**
+          > 
+          > HTTPS should be used if the cache might contain sensitive
+          > information.
+        )"};
+
+    const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint",
+        R"(
+          The URL of the endpoint of an S3-compatible service such as MinIO.
+          Do not specify this setting if you're using Amazon S3.
+
+          > **Note**
+          > 
+          > This endpoint must support HTTPS and will use path-based
+          > addressing instead of virtual host based addressing.
+        )"};
+
+    const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression",
+        "Compression method for `.narinfo` files."};
+
+    const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression",
+        "Compression method for `.ls` files."};
+
+    const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression",
+        R"(
+          Compression method for `log/*` files. It is recommended to
+          use a compression method supported by most web browsers
+          (e.g. `brotli`).
+        )"};
+
     const Setting<bool> multipartUpload{
-        (StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"};
+        (StoreConfig*) this, false, "multipart-upload",
+        "Whether to use multi-part uploads."};
+
     const Setting<uint64_t> bufferSize{
-        (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
+        (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size",
+        "Size (in bytes) of each part in multi-part uploads."};
 
     const std::string name() override { return "S3 Binary Cache Store"; }
+
+    std::string doc() override
+    {
+        return
+          #include "s3-binary-cache-store.md"
+          ;
+    }
 };
 
 struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore

src/libstore/s3-binary-cache-store.md

@@ -0,0 +1,8 @@
+R"(
+
+**Store URL format**: `s3://`*bucket-name*
+
+This store allows reading and writing a binary cache stored in an AWS
+S3 bucket.
+
+)"

src/libstore/ssh-store-config.hh

@@ -0,0 +1,26 @@
+#include "store-api.hh"
+
+namespace nix {
+
+struct CommonSSHStoreConfig : virtual StoreConfig
+{
+    using StoreConfig::StoreConfig;
+
+    const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key",
+        "Path to the SSH private key used to authenticate to the remote machine."};
+
+    const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key",
+        "The public host key of the remote machine."};
+
+    const Setting<bool> compress{(StoreConfig*) this, false, "compress",
+        "Whether to enable SSH compression."};
+
+    const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store",
+        R"(
+          [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+          to be used on the remote machine. The default is `auto`
+          (i.e. use the Nix daemon or `/nix/store` directly).
+        )"};
+};
+
+}

src/libstore/ssh-store.cc

@@ -1,3 +1,4 @@
+#include "ssh-store-config.hh"
 #include "store-api.hh"
 #include "remote-store.hh"
 #include "remote-fs-accessor.hh"
@@ -8,17 +9,22 @@
 
 namespace nix {
 
-struct SSHStoreConfig : virtual RemoteStoreConfig
+struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig
 {
     using RemoteStoreConfig::RemoteStoreConfig;
+    using CommonSSHStoreConfig::CommonSSHStoreConfig;
 
-    const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
+    const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program",
-    const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"};
+        "Path to the `nix-daemon` executable on the remote machine."};
-    const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
-    const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
-    const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
 
-    const std::string name() override { return "SSH Store"; }
+    const std::string name() override { return "Experimental SSH Store"; }
+
+    std::string doc() override
+    {
+        return
+          #include "ssh-store.md"
+          ;
+    }
 };
 
 class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore
@@ -28,6 +34,7 @@ public:
     SSHStore(const std::string & scheme, const std::string & host, const Params & params)
         : StoreConfig(params)
         , RemoteStoreConfig(params)
+        , CommonSSHStoreConfig(params)
         , SSHStoreConfig(params)
         , Store(params)
         , RemoteStore(params)

src/libstore/ssh-store.md

@@ -0,0 +1,8 @@
+R"(
+
+**Store URL format**: `ssh-ng://[username@]hostname`
+
+Experimental store type that allows full access to a Nix store on a
+remote machine.
+
+)"

src/libstore/store-api.hh

@@ -101,17 +101,41 @@ struct StoreConfig : public Config
 
     virtual const std::string name() = 0;
 
+    virtual std::string doc()
+    {
+        return "";
+    }
+
     const PathSetting storeDir_{this, false, settings.nixStore,
-        "store", "path to the Nix store"};
+        "store",
+        R"(
+          Logical location of the Nix store, usually
+          `/nix/store`. Note that you can only copy store paths
+          between stores if they have the same `store` setting.
+        )"};
     const Path storeDir = storeDir_;
 
-    const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"};
+    const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size",
+        "Size of the in-memory store path metadata cache."};
 
-    const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"};
+    const Setting<bool> isTrusted{this, false, "trusted",
+        R"(
+          Whether paths from this store can be used as substitutes
+          even if they are not signed by a key listed in the
+          [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys)
+          setting.
+        )"};
 
-    Setting<int> priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"};
+    Setting<int> priority{this, 0, "priority",
+        R"(
+          Priority of this store when used as a substituter. A lower value means a higher priority.
+        )"};
 
-    Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"};
+    Setting<bool> wantMassQuery{this, false, "want-mass-query",
+        R"(
+          Whether this store (when used as a substituter) can be
+          queried efficiently for path validity.
+        )"};
 
     Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
         "system-features",
@@ -125,8 +149,6 @@ public:
 
     typedef std::map<std::string, std::string> Params;
 
-
-
 protected:
 
     struct PathInfoCacheValue {

src/libstore/uds-remote-store.hh

@@ -15,6 +15,13 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon
     }
 
     const std::string name() override { return "Local Daemon Store"; }
+
+    std::string doc() override
+    {
+        return
+          #include "uds-remote-store.md"
+          ;
+    }
 };
 
 class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore

src/libstore/uds-remote-store.md

@@ -0,0 +1,9 @@
+R"(
+
+**Store URL format**: `daemon`, `unix://`*path*
+
+This store type accesses a Nix store by talking to a Nix daemon
+listening on the Unix domain socket *path*. The store pseudo-URL
+`daemon` is equivalent to `unix:///nix/var/nix/daemon-socket/socket`.
+
+)"

src/nix/describe-stores.cc

@@ -1,44 +0,0 @@
-#include "command.hh"
-#include "common-args.hh"
-#include "shared.hh"
-#include "store-api.hh"
-
-#include <nlohmann/json.hpp>
-
-using namespace nix;
-
-struct CmdDescribeStores : Command, MixJSON
-{
-    std::string description() override
-    {
-        return "show registered store types and their available options";
-    }
-
-    Category category() override { return catUtility; }
-
-    void run() override
-    {
-        auto res = nlohmann::json::object();
-        for (auto & implem : *Implementations::registered) {
-            auto storeConfig = implem.getConfig();
-            auto storeName = storeConfig->name();
-            res[storeName] = storeConfig->toJSON();
-        }
-        if (json) {
-            logger->cout("%s", res);
-        } else {
-            for (auto & [storeName, storeConfig] : res.items()) {
-                std::cout << "## " << storeName << std::endl << std::endl;
-                for (auto & [optionName, optionDesc] : storeConfig.items()) {
-                    std::cout << "### " << optionName << std::endl << std::endl;
-                    std::cout << optionDesc["description"].get<std::string>() << std::endl;
-                    std::cout << "default: " << optionDesc["defaultValue"] << std::endl <<std::endl;
-                    if (!optionDesc["aliases"].empty())
-                        std::cout << "aliases: " << optionDesc["aliases"] << std::endl << std::endl;
-                }
-            }
-        }
-    }
-};
-
-static auto rDescribeStore = registerCommand<CmdDescribeStores>("describe-stores");

src/nix/help-stores.md

@@ -0,0 +1,46 @@
+R"(
+
+Nix supports different types of stores. These are described below.
+
+## Store URL format
+
+Stores are specified using a URL-like syntax. For example, the command
+
+```console
+# nix path-info --store https://cache.nixos.org/ --json \
+  /nix/store/a7gvj343m05j2s32xcnwr35v31ynlypr-coreutils-9.1
+```
+
+fetches information about a store path in the HTTP binary cache
+located at https://cache.nixos.org/, which is a type of store.
+
+Store URLs can specify **store settings** using URL query strings,
+i.e. by appending `?name1=value1&name2=value2&...` to the URL. For
+instance,
+
+```
+--store ssh://machine.example.org?ssh-key=/path/to/my/key
+```
+
+tells Nix to access the store on a remote machine via the SSH
+protocol, using `/path/to/my/key` as the SSH private key. The
+supported settings for each store type are documented below.
+
+The special store URL `auto` causes Nix to automatically select a
+store as follows:
+
+* Use the [local store](#local-store) `/nix/store` if `/nix/var/nix`
+  is writable by the current user.
+
+* Otherwise, if `/nix/var/nix/daemon-socket/socket` exists, [connect
+  to the Nix daemon listening on that socket](#local-daemon-store).
+
+* Otherwise, on Linux only, use the [local chroot store](#local-store)
+  `~/.local/share/nix/root`, which will be created automatically if it
+  does not exist.
+
+* Otherwise, use the [local store](#local-store) `/nix/store`.
+
+@stores@
+
+)"

src/nix/main.cc

@@ -64,6 +64,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
     NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix")
     {
         categories.clear();
+        categories[catHelp] = "Help commands";
         categories[Command::catDefault] = "Main commands";
         categories[catSecondary] = "Infrequently used commands";
         categories[catUtility] = "Utility/scripting commands";
@@ -163,11 +164,29 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
     {
         commands = RegisterCommand::getCommandsFor({});
     }
+
+    std::string dumpCli()
+    {
+        auto res = nlohmann::json::object();
+
+        res["args"] = toJSON();
+
+        auto stores = nlohmann::json::object();
+        for (auto & implem : *Implementations::registered) {
+            auto storeConfig = implem.getConfig();
+            auto storeName = storeConfig->name();
+            stores[storeName]["doc"] = storeConfig->doc();
+            stores[storeName]["settings"] = storeConfig->toJSON();
+        }
+        res["stores"] = std::move(stores);
+
+        return res.dump();
+    }
 };
 
 /* Render the help for the specified subcommand to stdout using
    lowdown. */
-static void showHelp(std::vector<std::string> subcommand, MultiCommand & toplevel)
+static void showHelp(std::vector<std::string> subcommand, NixArgs & toplevel)
 {
     auto mdName = subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand));
 
@@ -188,11 +207,11 @@ static void showHelp(std::vector<std::string> subcommand, MultiCommand & topleve
             , "/"),
         *vUtils);
 
-    auto attrs = state.buildBindings(16);
+    auto vDump = state.allocValue();
-    attrs.alloc("toplevel").mkString(toplevel.toJSON().dump());
+    vDump->mkString(toplevel.dumpCli());
 
     auto vRes = state.allocValue();
-    state.callFunction(*vGenerateManpage, state.allocValue()->mkAttrs(attrs), *vRes, noPos);
+    state.callFunction(*vGenerateManpage, *vDump, *vRes, noPos);
 
     auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md"));
     if (!attr)
@@ -204,6 +223,14 @@ static void showHelp(std::vector<std::string> subcommand, MultiCommand & topleve
     std::cout << renderMarkdownToTerminal(markdown) << "\n";
 }
 
+static NixArgs & getNixArgs(Command & cmd)
+{
+    assert(cmd.parent);
+    MultiCommand * toplevel = cmd.parent;
+    while (toplevel->parent) toplevel = toplevel->parent;
+    return dynamic_cast<NixArgs &>(*toplevel);
+}
+
 struct CmdHelp : Command
 {
     std::vector<std::string> subcommand;
@@ -228,17 +255,43 @@ struct CmdHelp : Command
           ;
     }
 
+    Category category() override { return catHelp; }
+
     void run() override
     {
         assert(parent);
         MultiCommand * toplevel = parent;
         while (toplevel->parent) toplevel = toplevel->parent;
-        showHelp(subcommand, *toplevel);
+        showHelp(subcommand, getNixArgs(*this));
     }
 };
 
 static auto rCmdHelp = registerCommand<CmdHelp>("help");
 
+struct CmdHelpStores : Command
+{
+    std::string description() override
+    {
+        return "show help about store types and their settings";
+    }
+
+    std::string doc() override
+    {
+        return
+          #include "help-stores.md"
+          ;
+    }
+
+    Category category() override { return catHelp; }
+
+    void run() override
+    {
+        showHelp({"help-stores"}, getNixArgs(*this));
+    }
+};
+
+static auto rCmdHelpStores = registerCommand<CmdHelpStores>("help-stores");
+
 void mainWrapped(int argc, char * * argv)
 {
     savedArgv = argv;
@@ -290,8 +343,8 @@ void mainWrapped(int argc, char * * argv)
 
     NixArgs args;
 
-    if (argc == 2 && std::string(argv[1]) == "__dump-args") {
+    if (argc == 2 && std::string(argv[1]) == "__dump-cli") {
-        logger->cout("%s", args.toJSON());
+        logger->cout(args.dumpCli());
         return;
     }
 

src/nix/nix.md

@@ -220,8 +220,7 @@ operate are determined as follows:
 
 # Nix stores
 
-Most `nix` subcommands operate on a *Nix store*.
+Most `nix` subcommands operate on a *Nix store*. These are documented
-
+in [`nix help-stores`](./nix3-help-stores.md).
-TODO: list store types, options
 
 )""

tests/describe-stores.sh

@@ -1,8 +0,0 @@
-source common.sh
-
-# Query an arbitrary value in `nix describe-stores --json`'s output just to
-# check that it has the right structure
-[[ $(nix --experimental-features 'nix-command flakes' describe-stores --json | jq '.["SSH Store"]["compress"]["defaultValue"]') == false ]]
-
-# Ensure that the output of `nix describe-stores` isn't empty
-[[ -n $(nix --experimental-features 'nix-command flakes' describe-stores) ]]

tests/local.mk

@@ -116,7 +116,6 @@ nix_tests = \
   db-migration.sh \
   bash-profile.sh \
   pass-as-file.sh \
-  describe-stores.sh \
   nix-profile.sh \
   suggestions.sh \
   store-ping.sh \