aleksanaa/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

Merge pull request #8084 from edolstra/store-docs

Browse source 
Auto-generate store documentation
This commit is contained in:
Eelco Dolstra 2023-03-27 15:46:18 +02:00 committed by GitHub
parent 1d539aa287 168b6021c5
commit 237587bc0a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
38 changed files with 557 additions and 207 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
doc/manual/generate-manpage.nix
View file

@ -1,4 +1,4 @@
{ toplevel }:
cliDumpStr:
with builtins;
with import ./utils.nix;
@ -63,7 +63,10 @@ let
* [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description}
'';
maybeDocumentation = if details ? doc then details.doc else "";
maybeDocumentation =
if details ? doc
then replaceStrings ["@stores@"] [storeDocs] details.doc
else "";
maybeOptions = if details.flags == {} then "" else ''
# Options
@ -110,13 +113,13 @@ let
};
in [ cmd ] ++ concatMap subcommand (attrNames details.commands or {});
parsedToplevel = builtins.fromJSON toplevel;
cliDump = builtins.fromJSON cliDumpStr;
manpages = processCommand {
command = "nix";
details = parsedToplevel;
details = cliDump.args;
filename = "nix";
toplevel = parsedToplevel;
toplevel = cliDump.args;
};
tableOfContents = let
@ -124,4 +127,18 @@ let
" - [${page.command}](command-ref/new-cli/${page.name})";
in concatStringsSep "\n" (map showEntry manpages) + "\n";
storeDocs =
let
showStore = name: { settings, doc }:
''
## ${name}
${doc}
**Settings**:
${showSettings { useAnchors = false; } settings}
'';
in concatStrings (attrValues (mapAttrs showStore cliDump.stores));
in (listToAttrs manpages) // { "SUMMARY.md" = tableOfContents; }

41
doc/manual/generate-options.nix
View file

@ -1,41 +0,0 @@
let
inherit (builtins) attrNames concatStringsSep isAttrs isBool;
inherit (import ./utils.nix) concatStrings squash splitLines;
in
optionsInfo:
let
showOption = name:
let
inherit (optionsInfo.${name}) description documentDefault defaultValue aliases;
result = squash ''
- <span id="conf-${name}">[`${name}`](#conf-${name})</span>
${indent " " body}
'';
# separate body to cleanly handle indentation
body = ''
${description}
**Default:** ${showDefault documentDefault defaultValue}
${showAliases aliases}
'';
showDefault = documentDefault: defaultValue:
if documentDefault then
# a StringMap value type is specified as a string, but
# this shows the value type. The empty stringmap is `null` in
# JSON, but that converts to `{ }` here.
if defaultValue == "" || defaultValue == [] || isAttrs defaultValue
then "*empty*"
else if isBool defaultValue then
if defaultValue then "`true`" else "`false`"
else "`${toString defaultValue}`"
else "*machine-specific*";
showAliases = aliases:
if aliases == [] then "" else
"**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}";
indent = prefix: s:
concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s));
in result;
in concatStrings (map showOption (attrNames optionsInfo))

10
doc/manual/local.mk
View file

@ -60,17 +60,17 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
@$(call process-includes,$@,$@)
$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
@rm -rf $@
$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }'
@rm -rf $@ $@.tmp
$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)'
@mv $@.tmp $@
$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
$(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
@mv $@.tmp $@
$(d)/nix.json: $(bindir)/nix
$(trace-gen) $(dummy-env) $(bindir)/nix __dump-args > $@.tmp
$(trace-gen) $(dummy-env) $(bindir)/nix __dump-cli > $@.tmp
@mv $@.tmp $@
$(d)/conf-file.json: $(bindir)/nix

34
doc/manual/src/package-management/s3-substituter.md
View file

@ -1,41 +1,11 @@
# Serving a Nix store via S3
Nix has built-in support for storing and fetching store paths from
Nix has [built-in support](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store)
for storing and fetching store paths from
Amazon S3 and S3-compatible services. This uses the same *binary*
cache mechanism that Nix usually uses to fetch prebuilt binaries from
[cache.nixos.org](https://cache.nixos.org/).
The following options can be specified as URL parameters to the S3 URL:
- `profile`\
The name of the AWS configuration profile to use. By default Nix
will use the `default` profile.
- `region`\
The region of the S3 bucket. `us–east-1` by default.
If your bucket is not in `us–east-1`, you should always explicitly
specify the region parameter.
- `endpoint`\
The URL to your S3-compatible service, for when not using Amazon S3.
Do not specify this value if you're using Amazon S3.
> **Note**
>
> This endpoint must support HTTPS and will use path-based
> addressing instead of virtual host based addressing.
- `scheme`\
The scheme used for S3 requests, `https` (default) or `http`. This
option allows you to disable HTTPS for binary caches which don't
support it.
> **Note**
>
> HTTPS should be used if the cache might contain sensitive
> information.
In this example we will use the bucket named `example-nix-cache`.
## Anonymous Reads to your S3-compatible binary cache

4
doc/manual/src/release-notes/rl-next.md
View file

@ -27,3 +27,7 @@
$ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*
```
provides information about each of its outputs.
* The experimental command `nix describe-stores` has been removed.
* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md).

42
doc/manual/utils.nix
View file

@ -38,4 +38,46 @@ rec {
filterAttrs = pred: set:
listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set));
showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value }:
let
result = squash ''
- ${if useAnchors
then ''<span id="conf-${name}">[`${name}`](#conf-${name})</span>''
else ''`${name}`''}
${indent " " body}
'';
# separate body to cleanly handle indentation
body = ''
${description}
**Default:** ${showDefault documentDefault defaultValue}
${showAliases aliases}
'';
showDefault = documentDefault: defaultValue:
if documentDefault then
# a StringMap value type is specified as a string, but
# this shows the value type. The empty stringmap is `null` in
# JSON, but that converts to `{ }` here.
if defaultValue == "" || defaultValue == [] || isAttrs defaultValue
then "*empty*"
else if isBool defaultValue then
if defaultValue then "`true`" else "`false`"
else "`${toString defaultValue}`"
else "*machine-specific*";
showAliases = aliases:
if aliases == [] then "" else
"**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}";
indent = prefix: s:
concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s));
in result;
showSettings = args: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting args) settingsInfo));
}

1
src/libcmd/command.hh
View file

@ -18,6 +18,7 @@ class EvalState;
struct Pos;
class Store;
static constexpr Command::Category catHelp = -1;
static constexpr Command::Category catSecondary = 100;
static constexpr Command::Category catUtility = 101;
static constexpr Command::Category catNixInstallation = 102;

6
src/libcmd/common-eval-args.cc
View file

@ -136,7 +136,11 @@ MixEvalArgs::MixEvalArgs()
addFlag({
.longName = "eval-store",
.description = "The Nix store to use for evaluations.",
.description =
R"(
The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them.
)",
.category = category,
.labels = {"store-url"},
.handler = {&evalStoreUrl},

34
src/libstore/binary-cache-store.hh
View file

@ -16,17 +16,33 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
{
using StoreConfig::StoreConfig;
const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', 'gzip', 'zstd', or 'none')"};
const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"};
const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression",
"NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."};
const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing",
"Whether to write a JSON file that lists the files in each NAR."};
const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info",
R"(
Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to
fetch debug info on demand
)"};
const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key",
"Path to the secret key used to sign the binary cache."};
const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache",
"Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."};
const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
"enable multi-threading compression for NARs, available for xz and zstd only currently"};
"Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."};
const Setting<int> compressionLevel{(StoreConfig*) this, -1, "compression-level",
"specify 'preset level' of compression to be used with NARs: "
"meaning and accepted range of values depends on compression method selected, "
"other than -1 which we reserve to indicate Nix defaults should be used"};
R"(
The *preset level* to be used when compressing NARs.
The meaning and accepted values depend on the compression method selected.
`-1` specifies that the default compression level should be used.
)"};
};
class BinaryCacheStore : public virtual BinaryCacheStoreConfig,

7
src/libstore/dummy-store.cc
View file

@ -7,6 +7,13 @@ struct DummyStoreConfig : virtual StoreConfig {
using StoreConfig::StoreConfig;
const std::string name() override { return "Dummy Store"; }
std::string doc() override
{
return
#include "dummy-store.md"
;
}
};
struct DummyStore : public virtual DummyStoreConfig, public virtual Store

13
src/libstore/dummy-store.md Normal file
View file

@ -0,0 +1,13 @@
R"(
**Store URL format**: `dummy://`
This store type represents a store that contains no store paths and
cannot be written to. It's useful when you want to use the Nix
evaluator when no actual Nix store exists, e.g.
```console
# nix eval --store dummy:// --expr '1 + 2'
```
)"

15
src/libstore/globals.hh
View file

@ -98,7 +98,12 @@ public:
Path nixDaemonSocketFile;
Setting<std::string> storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store",
"The default Nix store to use."};
R"(
The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
to use for most operations.
See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md)
for supported store types and settings.
)"};
Setting<bool> keepFailed{this, false, "keep-failed",
"Whether to keep temporary directories of failed builds."};
@ -679,8 +684,9 @@ public:
Strings{"https://cache.nixos.org/"},
"substituters",
R"(
A list of URLs of substituters, separated by whitespace. Substituters
are tried based on their Priority value, which each substituter can set
A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
to be used as substituters, separated by whitespace.
Substituters are tried based on their Priority value, which each substituter can set
independently. Lower value means higher priority.
The default is `https://cache.nixos.org`, with a Priority of 40.
@ -698,7 +704,8 @@ public:
Setting<StringSet> trustedSubstituters{
this, {}, "trusted-substituters",
R"(
A list of URLs of substituters, separated by whitespace. These are
A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format),
separated by whitespace. These are
not used by default, but can be enabled by users of the Nix daemon
by specifying `--option substituters urls` on the command
line. Unprivileged users are only allowed to pass a subset of the

9
src/libstore/http-binary-cache-store.cc
View file

@ -12,7 +12,14 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
{
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
const std::string name() override { return "Http Binary Cache Store"; }
const std::string name() override { return "HTTP Binary Cache Store"; }
std::string doc() override
{
return
#include "http-binary-cache-store.md"
;
}
};
class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore

8
src/libstore/http-binary-cache-store.md Normal file
View file

@ -0,0 +1,8 @@
R"(
**Store URL format**: `http://...`, `https://...`
This store allows a binary cache to be accessed via the HTTP
protocol.
)"

27
src/libstore/legacy-ssh-store.cc
View file

@ -1,3 +1,4 @@
#include "ssh-store-config.hh"
#include "archive.hh"
#include "pool.hh"
#include "remote-store.hh"
@ -12,17 +13,24 @@
namespace nix {
struct LegacySSHStoreConfig : virtual StoreConfig
struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
{
using StoreConfig::StoreConfig;
const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"};
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"};
const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
using CommonSSHStoreConfig::CommonSSHStoreConfig;
const std::string name() override { return "Legacy SSH Store"; }
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program",
"Path to the `nix-store` executable on the remote machine."};
const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections",
"Maximum number of concurrent SSH connections."};
const std::string name() override { return "SSH Store"; }
std::string doc() override
{
return
#include "legacy-ssh-store.md"
;
}
};
struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store
@ -51,6 +59,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params)
: StoreConfig(params)
, CommonSSHStoreConfig(params)
, LegacySSHStoreConfig(params)
, Store(params)
, host(host)

8
src/libstore/legacy-ssh-store.md Normal file
View file

@ -0,0 +1,8 @@
R"(
**Store URL format**: `ssh://[username@]hostname`
This store type allows limited access to a remote store on another
machine via SSH.
)"

7
src/libstore/local-binary-cache-store.cc
View file

@ -11,6 +11,13 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
const std::string name() override { return "Local Binary Cache Store"; }
std::string doc() override
{
return
#include "local-binary-cache-store.md"
;
}
};
class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore

16
src/libstore/local-binary-cache-store.md Normal file
View file

@ -0,0 +1,16 @@
R"(
**Store URL format**: `file://`*path*
This store allows reading and writing a binary cache stored in *path*
in the local filesystem. If *path* does not exist, it will be created.
For example, the following builds or downloads `nixpkgs#hello` into
the local store and then copies it to the binary cache in
`/tmp/binary-cache`:
```
# nix copy --to file:///tmp/binary-cache nixpkgs#hello
```
)"

16
src/libstore/local-fs-store.hh
View file

@ -9,20 +9,28 @@ namespace nix {
struct LocalFSStoreConfig : virtual StoreConfig
{
using StoreConfig::StoreConfig;
// FIXME: the (StoreConfig*) cast works around a bug in gcc that causes
// it to omit the call to the Setting constructor. Clang works fine
// either way.
const PathSetting rootDir{(StoreConfig*) this, true, "",
"root", "directory prefixed to all other paths"};
"root",
"Directory prefixed to all other paths."};
const PathSetting stateDir{(StoreConfig*) this, false,
rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir,
"state", "directory where Nix will store state"};
"state",
"Directory where Nix will store state."};
const PathSetting logDir{(StoreConfig*) this, false,
rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
"log", "directory where Nix will store state"};
"log",
"directory where Nix will store log files."};
const PathSetting realStoreDir{(StoreConfig*) this, false,
rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
"physical path to the Nix store"};
"Physical path of the Nix store."};
};
class LocalFSStore : public virtual LocalFSStoreConfig,

15
src/libstore/local-store.cc
View file

@ -44,6 +44,13 @@
namespace nix {
std::string LocalStoreConfig::doc()
{
return
#include "local-store.md"
;
}
struct LocalStore::State::Stmts {
/* Some precompiled SQLite statements. */
SQLiteStmt RegisterValidPath;
@ -413,6 +420,13 @@ LocalStore::LocalStore(const Params & params)
}
LocalStore::LocalStore(std::string scheme, std::string path, const Params & params)
: LocalStore(params)
{
throw UnimplementedError("LocalStore");
}
AutoCloseFD LocalStore::openGCLock()
{
Path fnGCLock = stateDir + "/gc.lock";
@ -1950,5 +1964,6 @@ std::optional<std::string> LocalStore::getVersion()
return nixVersion;
}
static RegisterStoreImplementation<LocalStore, LocalStoreConfig> regLocalStore;
} // namespace nix

10
src/libstore/local-store.hh
View file

@ -38,11 +38,13 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
Setting<bool> requireSigs{(StoreConfig*) this,
settings.requireSigs,
"require-sigs", "whether store paths should have a trusted signature on import"};
"require-sigs",
"Whether store paths copied into this store should have a trusted signature."};
const std::string name() override { return "Local Store"; }
};
std::string doc() override;
};
class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore
{
@ -100,9 +102,13 @@ public:
/* Initialise the local store, upgrading the schema if
necessary. */
LocalStore(const Params & params);
LocalStore(std::string scheme, std::string path, const Params & params);
~LocalStore();
static std::set<std::string> uriSchemes()
{ return {}; }
/* Implementations of abstract store API methods. */
std::string getUri() override;

39
src/libstore/local-store.md Normal file
View file

@ -0,0 +1,39 @@
R"(
**Store URL format**: `local`, *root*
This store type accesses a Nix store in the local filesystem directly
(i.e. not via the Nix daemon). *root* is an absolute path that is
prefixed to other directories such as the Nix store directory. The
store pseudo-URL `local` denotes a store that uses `/` as its root
directory.
A store that uses a *root* other than `/` is called a *chroot
store*. With such stores, the store directory is "logically" still
`/nix/store`, so programs stored in them can only be built and
executed by `chroot`-ing into *root*. Chroot stores only support
building and running on Linux when [`mount namespaces`](https://man7.org/linux/man-pages/man7/mount_namespaces.7.html) and [`user namespaces`](https://man7.org/linux/man-pages/man7/user_namespaces.7.html) are
enabled.
For example, the following uses `/tmp/root` as the chroot environment
to build or download `nixpkgs#hello` and then execute it:
```console
# nix run --store /tmp/root nixpkgs#hello
Hello, world!
```
Here, the "physical" store location is `/tmp/root/nix/store`, and
Nix's store metadata is in `/tmp/root/nix/var/nix/db`.
It is also possible, but not recommended, to change the "logical"
location of the Nix store from its default of `/nix/store`. This makes
it impossible to use default substituters such as
`https://cache.nixos.org/`, and thus you may have to build everything
locally. Here is an example:
```console
# nix build --store 'local?store=/tmp/my-nix/store&state=/tmp/my-nix/state&log=/tmp/my-nix/log' nixpkgs#hello
```
)"

10
src/libstore/remote-store.hh
View file

@ -22,11 +22,13 @@ struct RemoteStoreConfig : virtual StoreConfig
{
using StoreConfig::StoreConfig;
const Setting<int> maxConnections{(StoreConfig*) this, 1,
"max-connections", "maximum number of concurrent connections to the Nix daemon"};
const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections",
"Maximum number of concurrent connections to the Nix daemon."};
const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, std::numeric_limits<unsigned int>::max(),
"max-connection-age", "number of seconds to reuse a connection"};
const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this,
std::numeric_limits<unsigned int>::max(),
"max-connection-age",
"Maximum age of a connection before it is closed."};
};
/* FIXME: RemoteStore is a misnomer - should be something like

71
src/libstore/s3-binary-cache-store.cc
View file

@ -192,19 +192,72 @@ S3BinaryCacheStore::S3BinaryCacheStore(const Params & params)
struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
{
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
const Setting<std::string> profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."};
const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."};
const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"};
const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"};
const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"};
const Setting<std::string> profile{(StoreConfig*) this, "", "profile",
R"(
The name of the AWS configuration profile to use. By default
Nix will use the `default` profile.
)"};
const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region",
R"(
The region of the S3 bucket. If your bucket is not in
`us–east-1`, you should always explicitly specify the region
parameter.
)"};
const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme",
R"(
The scheme used for S3 requests, `https` (default) or `http`. This
option allows you to disable HTTPS for binary caches which don't
support it.
> **Note**
>
> HTTPS should be used if the cache might contain sensitive
> information.
)"};
const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint",
R"(
The URL of the endpoint of an S3-compatible service such as MinIO.
Do not specify this setting if you're using Amazon S3.
> **Note**
>
> This endpoint must support HTTPS and will use path-based
> addressing instead of virtual host based addressing.
)"};
const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression",
"Compression method for `.narinfo` files."};
const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression",
"Compression method for `.ls` files."};
const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression",
R"(
Compression method for `log/*` files. It is recommended to
use a compression method supported by most web browsers
(e.g. `brotli`).
)"};
const Setting<bool> multipartUpload{
(StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"};
(StoreConfig*) this, false, "multipart-upload",
"Whether to use multi-part uploads."};
const Setting<uint64_t> bufferSize{
(StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"};
(StoreConfig*) this, 5 * 1024 * 1024, "buffer-size",
"Size (in bytes) of each part in multi-part uploads."};
const std::string name() override { return "S3 Binary Cache Store"; }
std::string doc() override
{
return
#include "s3-binary-cache-store.md"
;
}
};
struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore

8
src/libstore/s3-binary-cache-store.md Normal file
View file

@ -0,0 +1,8 @@
R"(
**Store URL format**: `s3://`*bucket-name*
This store allows reading and writing a binary cache stored in an AWS
S3 bucket.
)"

26
src/libstore/ssh-store-config.hh Normal file
View file

@ -0,0 +1,26 @@
#include "store-api.hh"
namespace nix {
struct CommonSSHStoreConfig : virtual StoreConfig
{
using StoreConfig::StoreConfig;
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key",
"Path to the SSH private key used to authenticate to the remote machine."};
const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key",
"The public host key of the remote machine."};
const Setting<bool> compress{(StoreConfig*) this, false, "compress",
"Whether to enable SSH compression."};
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store",
R"(
[Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
to be used on the remote machine. The default is `auto`
(i.e. use the Nix daemon or `/nix/store` directly).
)"};
};
}

21
src/libstore/ssh-store.cc
View file

@ -1,3 +1,4 @@
#include "ssh-store-config.hh"
#include "store-api.hh"
#include "remote-store.hh"
#include "remote-fs-accessor.hh"
@ -8,17 +9,22 @@
namespace nix {
struct SSHStoreConfig : virtual RemoteStoreConfig
struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig
{
using RemoteStoreConfig::RemoteStoreConfig;
using CommonSSHStoreConfig::CommonSSHStoreConfig;
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"};
const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program",
"Path to the `nix-daemon` executable on the remote machine."};
const std::string name() override { return "SSH Store"; }
const std::string name() override { return "Experimental SSH Store"; }
std::string doc() override
{
return
#include "ssh-store.md"
;
}
};
class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore
@ -28,6 +34,7 @@ public:
SSHStore(const std::string & scheme, const std::string & host, const Params & params)
: StoreConfig(params)
, RemoteStoreConfig(params)
, CommonSSHStoreConfig(params)
, SSHStoreConfig(params)
, Store(params)
, RemoteStore(params)

8
src/libstore/ssh-store.md Normal file
View file

@ -0,0 +1,8 @@
R"(
**Store URL format**: `ssh-ng://[username@]hostname`
Experimental store type that allows full access to a Nix store on a
remote machine.
)"

36
src/libstore/store-api.hh
View file

@ -101,17 +101,41 @@ struct StoreConfig : public Config
virtual const std::string name() = 0;
virtual std::string doc()
{
return "";
}
const PathSetting storeDir_{this, false, settings.nixStore,
"store", "path to the Nix store"};
"store",
R"(
Logical location of the Nix store, usually
`/nix/store`. Note that you can only copy store paths
between stores if they have the same `store` setting.
)"};
const Path storeDir = storeDir_;
const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"};
const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size",
"Size of the in-memory store path metadata cache."};
const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"};
const Setting<bool> isTrusted{this, false, "trusted",
R"(
Whether paths from this store can be used as substitutes
even if they are not signed by a key listed in the
[`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys)
setting.
)"};
Setting<int> priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"};
Setting<int> priority{this, 0, "priority",
R"(
Priority of this store when used as a substituter. A lower value means a higher priority.
)"};
Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"};
Setting<bool> wantMassQuery{this, false, "want-mass-query",
R"(
Whether this store (when used as a substituter) can be
queried efficiently for path validity.
)"};
Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
"system-features",
@ -125,8 +149,6 @@ public:
typedef std::map<std::string, std::string> Params;
protected:
struct PathInfoCacheValue {

7
src/libstore/uds-remote-store.hh
View file

@ -15,6 +15,13 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon
}
const std::string name() override { return "Local Daemon Store"; }
std::string doc() override
{
return
#include "uds-remote-store.md"
;
}
};
class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore

9
src/libstore/uds-remote-store.md Normal file
View file

@ -0,0 +1,9 @@
R"(
**Store URL format**: `daemon`, `unix://`*path*
This store type accesses a Nix store by talking to a Nix daemon
listening on the Unix domain socket *path*. The store pseudo-URL
`daemon` is equivalent to `unix:///nix/var/nix/daemon-socket/socket`.
)"

44
src/nix/describe-stores.cc
View file

@ -1,44 +0,0 @@
#include "command.hh"
#include "common-args.hh"
#include "shared.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
using namespace nix;
struct CmdDescribeStores : Command, MixJSON
{
std::string description() override
{
return "show registered store types and their available options";
}
Category category() override { return catUtility; }
void run() override
{
auto res = nlohmann::json::object();
for (auto & implem : *Implementations::registered) {
auto storeConfig = implem.getConfig();
auto storeName = storeConfig->name();
res[storeName] = storeConfig->toJSON();
}
if (json) {
logger->cout("%s", res);
} else {
for (auto & [storeName, storeConfig] : res.items()) {
std::cout << "## " << storeName << std::endl << std::endl;
for (auto & [optionName, optionDesc] : storeConfig.items()) {
std::cout << "### " << optionName << std::endl << std::endl;
std::cout << optionDesc["description"].get<std::string>() << std::endl;
std::cout << "default: " << optionDesc["defaultValue"] << std::endl <<std::endl;
if (!optionDesc["aliases"].empty())
std::cout << "aliases: " << optionDesc["aliases"] << std::endl << std::endl;
}
}
}
}
};
static auto rDescribeStore = registerCommand<CmdDescribeStores>("describe-stores");

46
src/nix/help-stores.md Normal file
View file

@ -0,0 +1,46 @@
R"(
Nix supports different types of stores. These are described below.
## Store URL format
Stores are specified using a URL-like syntax. For example, the command
```console
# nix path-info --store https://cache.nixos.org/ --json \
/nix/store/a7gvj343m05j2s32xcnwr35v31ynlypr-coreutils-9.1
```
fetches information about a store path in the HTTP binary cache
located at https://cache.nixos.org/, which is a type of store.
Store URLs can specify **store settings** using URL query strings,
i.e. by appending `?name1=value1&name2=value2&...` to the URL. For
instance,
```
--store ssh://machine.example.org?ssh-key=/path/to/my/key
```
tells Nix to access the store on a remote machine via the SSH
protocol, using `/path/to/my/key` as the SSH private key. The
supported settings for each store type are documented below.
The special store URL `auto` causes Nix to automatically select a
store as follows:
* Use the [local store](#local-store) `/nix/store` if `/nix/var/nix`
is writable by the current user.
* Otherwise, if `/nix/var/nix/daemon-socket/socket` exists, [connect
to the Nix daemon listening on that socket](#local-daemon-store).
* Otherwise, on Linux only, use the [local chroot store](#local-store)
`~/.local/share/nix/root`, which will be created automatically if it
does not exist.
* Otherwise, use the [local store](#local-store) `/nix/store`.
@stores@
)"

67
src/nix/main.cc
View file

@ -64,6 +64,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix")
{
categories.clear();
categories[catHelp] = "Help commands";
categories[Command::catDefault] = "Main commands";
categories[catSecondary] = "Infrequently used commands";
categories[catUtility] = "Utility/scripting commands";
@ -163,11 +164,29 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
{
commands = RegisterCommand::getCommandsFor({});
}
std::string dumpCli()
{
auto res = nlohmann::json::object();
res["args"] = toJSON();
auto stores = nlohmann::json::object();
for (auto & implem : *Implementations::registered) {
auto storeConfig = implem.getConfig();
auto storeName = storeConfig->name();
stores[storeName]["doc"] = storeConfig->doc();
stores[storeName]["settings"] = storeConfig->toJSON();
}
res["stores"] = std::move(stores);
return res.dump();
}
};
/* Render the help for the specified subcommand to stdout using
lowdown. */
static void showHelp(std::vector<std::string> subcommand, MultiCommand & toplevel)
static void showHelp(std::vector<std::string> subcommand, NixArgs & toplevel)
{
auto mdName = subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand));
@ -188,11 +207,11 @@ static void showHelp(std::vector<std::string> subcommand, MultiCommand & topleve
, "/"),
*vUtils);
auto attrs = state.buildBindings(16);
attrs.alloc("toplevel").mkString(toplevel.toJSON().dump());
auto vDump = state.allocValue();
vDump->mkString(toplevel.dumpCli());
auto vRes = state.allocValue();
state.callFunction(*vGenerateManpage, state.allocValue()->mkAttrs(attrs), *vRes, noPos);
state.callFunction(*vGenerateManpage, *vDump, *vRes, noPos);
auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md"));
if (!attr)
@ -204,6 +223,14 @@ static void showHelp(std::vector<std::string> subcommand, MultiCommand & topleve
std::cout << renderMarkdownToTerminal(markdown) << "\n";
}
static NixArgs & getNixArgs(Command & cmd)
{
assert(cmd.parent);
MultiCommand * toplevel = cmd.parent;
while (toplevel->parent) toplevel = toplevel->parent;
return dynamic_cast<NixArgs &>(*toplevel);
}
struct CmdHelp : Command
{
std::vector<std::string> subcommand;
@ -228,17 +255,43 @@ struct CmdHelp : Command
;
}
Category category() override { return catHelp; }
void run() override
{
assert(parent);
MultiCommand * toplevel = parent;
while (toplevel->parent) toplevel = toplevel->parent;
showHelp(subcommand, *toplevel);
showHelp(subcommand, getNixArgs(*this));
}
};
static auto rCmdHelp = registerCommand<CmdHelp>("help");
struct CmdHelpStores : Command
{
std::string description() override
{
return "show help about store types and their settings";
}
std::string doc() override
{
return
#include "help-stores.md"
;
}
Category category() override { return catHelp; }
void run() override
{
showHelp({"help-stores"}, getNixArgs(*this));
}
};
static auto rCmdHelpStores = registerCommand<CmdHelpStores>("help-stores");
void mainWrapped(int argc, char * * argv)
{
savedArgv = argv;
@ -290,8 +343,8 @@ void mainWrapped(int argc, char * * argv)
NixArgs args;
if (argc == 2 && std::string(argv[1]) == "__dump-args") {
logger->cout("%s", args.toJSON());
if (argc == 2 && std::string(argv[1]) == "__dump-cli") {
logger->cout(args.dumpCli());
return;
}

5
src/nix/nix.md
View file

@ -220,8 +220,7 @@ operate are determined as follows:
# Nix stores
Most `nix` subcommands operate on a *Nix store*.
TODO: list store types, options
Most `nix` subcommands operate on a *Nix store*. These are documented
in [`nix help-stores`](./nix3-help-stores.md).
)""

8
tests/describe-stores.sh
View file

@ -1,8 +0,0 @@
source common.sh
# Query an arbitrary value in `nix describe-stores --json`'s output just to
# check that it has the right structure
[[ $(nix --experimental-features 'nix-command flakes' describe-stores --json | jq '.["SSH Store"]["compress"]["defaultValue"]') == false ]]
# Ensure that the output of `nix describe-stores` isn't empty
[[ -n $(nix --experimental-features 'nix-command flakes' describe-stores) ]]

1
tests/local.mk
View file

@ -116,7 +116,6 @@ nix_tests = \
db-migration.sh \
bash-profile.sh \
pass-as-file.sh \
describe-stores.sh \
nix-profile.sh \
suggestions.sh \
store-ping.sh \