Merge remote-tracking branch 'upstream/master' into trustless-remote-builder-simple

.github/labeler.yml

@@ -2,5 +2,22 @@
   - doc/manual/*
   - src/nix/**/*.md
 
+"store":
+  - src/libstore/store-api.*
+  - src/libstore/*-store.*
+
+"fetching":
+  - src/libfetchers/**/*
+
+"repl":
+  - src/libcmd/repl.*
+  - src/nix/repl.*
+
+"new-cli":
+  - src/nix/**/*
+
 "tests":
+  # Unit tests
+  - src/*/tests/**/*
+  # Functional and integration tests
   - tests/**/*

.gitignore

@@ -19,9 +19,12 @@ perl/Makefile.config
 /doc/manual/nix.json
 /doc/manual/conf-file.json
 /doc/manual/builtins.json
+/doc/manual/xp-features.json
 /doc/manual/src/SUMMARY.md
 /doc/manual/src/command-ref/new-cli
 /doc/manual/src/command-ref/conf-file.md
+/doc/manual/src/command-ref/experimental-features-shortlist.md
+/doc/manual/src/contributing/experimental-feature-descriptions.md
 /doc/manual/src/language/builtins.md
 
 # /scripts/

.version

@@ -1 +1 @@
-2.15.0
+2.16.0

configure.ac

@@ -289,13 +289,24 @@ PKG_CHECK_MODULES([GTEST], [gtest_main])
 
 
 # Look for rapidcheck.
+AC_ARG_VAR([RAPIDCHECK_HEADERS], [include path of gtest headers shipped by RAPIDCHECK])
 # No pkg-config yet, https://github.com/emil-e/rapidcheck/issues/302
 AC_LANG_PUSH(C++)
 AC_SUBST(RAPIDCHECK_HEADERS)
 [CXXFLAGS="-I $RAPIDCHECK_HEADERS $CXXFLAGS"]
+[LIBS="-lrapidcheck -lgtest $LIBS"]
 AC_CHECK_HEADERS([rapidcheck/gtest.h], [], [], [#include <gtest/gtest.h>])
-dnl No good for C++ libs with mangled symbols
+dnl AC_CHECK_LIB doesn't work for C++ libs with mangled symbols
-dnl AC_CHECK_LIB([rapidcheck], [])
+AC_LINK_IFELSE([
+    AC_LANG_PROGRAM([[
+      #include <gtest/gtest.h>
+      #include <rapidcheck/gtest.h>
+    ]], [[
+      return RUN_ALL_TESTS();
+    ]])
+  ],
+  [],
+  [AC_MSG_ERROR([librapidcheck is not found.])])
 AC_LANG_POP(C++)
 
 fi

doc/manual/generate-xp-features-shortlist.nix

@@ -0,0 +1,9 @@
+with builtins;
+with import ./utils.nix;
+
+let
+  showExperimentalFeature = name: doc:
+    ''
+      - [`${name}`](@docroot@/contributing/experimental-features.md#xp-feature-${name})
+    '';
+in xps: indent "  " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps)))

doc/manual/generate-xp-features.nix

@@ -0,0 +1,11 @@
+with builtins;
+with import ./utils.nix;
+
+let
+  showExperimentalFeature = name: doc:
+    squash ''
+      ## [`${name}`]{#xp-feature-${name}}
+
+      ${doc}
+    '';
+in xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps)))

doc/manual/local.mk

@@ -81,19 +81,20 @@ $(d)/%.8: $(d)/src/command-ref/%.md
 $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
 	@printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp
 	@cat $^ >> $^.tmp
+	@$(call process-includes,$^,$^.tmp)
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
+$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
 	@cp $< $@
 	@$(call process-includes,$@,$@)
 
-$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
+$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
+$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
 	@mv $@.tmp $@
@@ -106,6 +107,20 @@ $(d)/conf-file.json: $(bindir)/nix
 	$(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp
 	@mv $@.tmp $@
 
+$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix
+	@rm -rf $@ $@.tmp
+	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))'
+	@mv $@.tmp $@
+
+$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix
+	@rm -rf $@ $@.tmp
+	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))'
+	@mv $@.tmp $@
+
+$(d)/xp-features.json: $(bindir)/nix
+	$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp
+	@mv $@.tmp $@
+
 $(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
 	@cat doc/manual/src/language/builtins-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
@@ -145,7 +160,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 	done
 	@touch $@
 
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \

doc/manual/src/SUMMARY.md.in

@@ -99,6 +99,7 @@
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
   - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+  - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md)
   - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md)
   - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md)
   - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md)

doc/manual/src/contributing/experimental-features.md

@@ -89,3 +89,7 @@ However they serve different purposes:
   It is primarily an issue of *design* and *communication*, targeting the broader community.
 
 This means that experimental features and RFCs are orthogonal mechanisms, and can be used independently or together as needed.
+
+# Currently available experimental features
+
+{{#include ./experimental-feature-descriptions.md}}

doc/manual/src/release-notes/rl-2.15.md

@@ -0,0 +1,58 @@
+# Release 2.15 (2023-04-11)
+
+* Commands which take installables on the command line can now read them from the standard input if
+  passed the `--stdin` flag. This is primarily useful when you have a large amount of paths which
+  exceed the OS argument limit.
+
+* The `nix-hash` command now supports Base64 and SRI. Use the flags `--base64`
+  or `--sri` to specify the format of output hash as Base64 or SRI, and `--to-base64`
+  or `--to-sri` to convert a hash to Base64 or SRI format, respectively.
+
+  As the choice of hash formats is no longer binary, the `--base16` flag is also added
+  to explicitly specify the Base16 format, which is still the default.
+
+* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents.
+
+  The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation.
+  Using this is better and more clear than relying on the now-removed `.drv` special handling.
+
+  For example,
+  ```shell-session
+  $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv
+  ```
+
+  now gives info about the derivation itself, while
+
+  ```shell-session
+  $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*
+  ```
+  provides information about each of its outputs.
+
+* The experimental command `nix describe-stores` has been removed.
+
+* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
+
+* Documentation for operations of `nix-store` and `nix-env` are now available on separate pages of the manual.
+  They include all common options that can be specified and common environment variables that affect these commands.
+
+  These pages can be viewed offline with `man` using
+
+  * `man nix-store-<operation>` and `man nix-env-<operation>`
+  * `nix-store --help --<operation>` and `nix-env --help --<operation>`.
+
+* Nix when used as a client now checks whether the store (the server) trusts the client.
+  (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.)
+  This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores.
+
+  `nix store ping` and `nix doctor` now display this information.
+
+* The new command `nix derivation add` allows adding derivations to the store without involving the Nix language.
+  It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store.
+  It uses the same JSON layout as `nix derivation show`, and is its inverse.
+
+* `nix show-derivation` has been renamed to `nix derivation show`.
+  This matches `nix derivation add`, and avoids bloating the top-level namespace.
+  The old name is still kept as an alias for compatibility, however.
+
+* The `nix derivation {add,show}` JSON format now includes the derivation name as a top-level field.
+  This is useful in general, but especially necessary for the `add` direction, as otherwise we would need to pass in the name out of band for certain cases.

doc/manual/src/release-notes/rl-next.md

@@ -1,58 +1,2 @@
 # Release X.Y (202?-??-??)
 
-* Commands which take installables on the command line can now read them from the standard input if
-  passed the `--stdin` flag. This is primarily useful when you have a large amount of paths which
-  exceed the OS arg limit.
-
-* The `nix-hash` command now supports Base64 and SRI. Use the flags `--base64`
-  or `--sri` to specify the format of output hash as Base64 or SRI, and `--to-base64`
-  or `--to-sri` to convert a hash to Base64 or SRI format, respectively.
-
-  As the choice of hash formats is no longer binary, the `--base16` flag is also added
-  to explicitly specify the Base16 format, which is still the default.
-
-* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents.
-
-  The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation.
-  Using this is better and more clear than relying on the now-removed `.drv` special handling.
-
-  For example,
-  ```shell-session
-  $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv
-  ```
-
-  now gives info about the derivation itself, while
-
-  ```shell-session
-  $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*
-  ```
-  provides information about each of its outputs.
-
-* The experimental command `nix describe-stores` has been removed.
-
-* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
-
-* Documentation for operations of `nix-store` and `nix-env` are now available on separate pages of the manual.
-  They include all common options that can be specified and common environment variables that affect these commands.
-
-  These pages can be viewed offline with `man` using
-
-  * `man nix-store-<operation>` and `man nix-env-<operation>`
-  * `nix-store --help --<operation>` and `nix-env --help --<operation>`.
-
-* Nix when used as a client now checks whether the store (the server) trusts the client.
-  (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.)
-  This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores.
-
-  `nix store ping` and `nix doctor` now display this information.
-
-* A new command `nix derivation add` is created, to allow adding derivations to the store without involving the Nix language.
-  It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store.
-  It uses the same JSON layout as `nix show-derivation`, and is its inverse.
-
-* `nix show-derivation` has been renamed to `nix derivation show`.
-  This matches `nix derivation add`, and avoids bloating the top-level namespace.
-  The old name is still kept as an alias for compatibility, however.
-
-* The `nix derivation {add,show}` JSON format now includes the derivation name as a top-level field.
-  This is useful in general, but especially necessary for the `add` direction, as otherwise we would need to pass in the name out of band for certain cases.

doc/manual/utils.nix

@@ -5,6 +5,9 @@ rec {
 
   concatStrings = concatStringsSep "";
 
+  attrsToList = a:
+    map (name: { inherit name; value = a.${name}; }) (builtins.attrNames a);
+
   replaceStringsRec = from: to: string:
     # recursively replace occurrences of `from` with `to` within `string`
     # example:
@@ -74,10 +77,10 @@ rec {
           if aliases == [] then "" else
             "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}";
 
-      indent = prefix: s:
-        concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s));
-
     in result;
 
+  indent = prefix: s:
+    concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s));
+
   showSettings = args: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting args) settingsInfo));
 }

flake.nix

@@ -219,6 +219,7 @@
 
         enableParallelBuilding = true;
 
+        configureFlags = testConfigureFlags; # otherwise configure fails
         dontBuild = true;
         doInstallCheck = true;
 

src/libexpr/flake/lockfile.cc

@@ -234,6 +234,11 @@ bool LockFile::operator ==(const LockFile & other) const
     return toJSON() == other.toJSON();
 }
 
+bool LockFile::operator !=(const LockFile & other) const
+{
+    return !(*this == other);
+}
+
 InputPath parseInputPath(std::string_view s)
 {
     InputPath path;

src/libexpr/flake/lockfile.hh

@@ -73,6 +73,9 @@ struct LockFile
     std::optional<FlakeRef> isUnlocked() const;
 
     bool operator ==(const LockFile & other) const;
+    // Needed for old gcc versions that don't synthesize it (like gcc 8.2.2
+    // that is still the default on aarch64-linux)
+    bool operator !=(const LockFile & other) const;
 
     std::shared_ptr<Node> findInput(const InputPath & path);
 

src/libutil/config.cc

@@ -70,17 +70,10 @@ void AbstractConfig::reapplyUnknownSettings()
         set(s.first, s.second);
 }
 
-// Whether we should process the option. Excludes aliases, which are handled elsewhere, and disabled features.
-static bool applicable(const Config::SettingData & sd)
-{
-    return !sd.isAlias
-        && experimentalFeatureSettings.isEnabled(sd.setting->experimentalFeature);
-}
-
 void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly)
 {
     for (auto & opt : _settings)
-        if (applicable(opt.second) && (!overriddenOnly || opt.second.setting->overridden))
+        if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden))
             res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description});
 }
 
@@ -154,7 +147,7 @@ nlohmann::json Config::toJSON()
 {
     auto res = nlohmann::json::object();
     for (auto & s : _settings)
-        if (applicable(s.second))
+        if (!s.second.isAlias)
             res.emplace(s.first, s.second.setting->toJSON());
     return res;
 }
@@ -163,7 +156,7 @@ std::string Config::toKeyValue()
 {
     auto res = std::string();
     for (auto & s : _settings)
-        if (applicable(s.second))
+        if (s.second.isAlias)
             res += fmt("%s = %s\n", s.first, s.second.setting->to_string());
     return res;
 }
@@ -171,9 +164,6 @@ std::string Config::toKeyValue()
 void Config::convertToArgs(Args & args, const std::string & category)
 {
     for (auto & s : _settings) {
-        /* We do include args for settings gated on disabled
-           experimental-features. The args themselves however will also be
-           gated on any experimental feature the underlying setting is. */
         if (!s.second.isAlias)
             s.second.setting->convertToArg(args, category);
     }

src/libutil/config.hh

@@ -371,8 +371,23 @@ extern GlobalConfig globalConfig;
 
 struct ExperimentalFeatureSettings : Config {
 
-    Setting<std::set<ExperimentalFeature>> experimentalFeatures{this, {}, "experimental-features",
+    Setting<std::set<ExperimentalFeature>> experimentalFeatures{
-        "Experimental Nix features to enable."};
+        this, {}, "experimental-features",
+        R"(
+          Experimental features that are enabled.
+
+          Example:
+
+          ```
+          experimental-features = nix-command flakes
+          ```
+
+          The following experimental features are available:
+
+          {{#include experimental-features-shortlist.md}}
+
+          Experimental features are [further documented in the manual](@docroot@/contributing/experimental-features.md).
+        )"};
 
     /**
      * Check whether the given experimental feature is enabled.

src/libutil/experimental-features.cc

@@ -5,30 +5,218 @@
 
 namespace nix {
 
-std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
+struct ExperimentalFeatureDetails
-    { Xp::CaDerivations, "ca-derivations" },
+{
-    { Xp::ImpureDerivations, "impure-derivations" },
+    ExperimentalFeature tag;
-    { Xp::Flakes, "flakes" },
+    std::string_view name;
-    { Xp::NixCommand, "nix-command" },
+    std::string_view description;
-    { Xp::RecursiveNix, "recursive-nix" },
-    { Xp::NoUrlLiterals, "no-url-literals" },
-    { Xp::FetchClosure, "fetch-closure" },
-    { Xp::ReplFlake, "repl-flake" },
-    { Xp::AutoAllocateUids, "auto-allocate-uids" },
-    { Xp::Cgroups, "cgroups" },
-    { Xp::DiscardReferences, "discard-references" },
-    { Xp::NixTesting, "nix-testing" },
 };
 
+constexpr std::array<ExperimentalFeatureDetails, 12> xpFeatureDetails = {{
+    {
+        .tag = Xp::CaDerivations,
+        .name = "ca-derivations",
+        .description = R"(
+            Allow derivations to be content-addressed in order to prevent
+            rebuilds when changes to the derivation do not result in changes to
+            the derivation's output. See
+            [__contentAddressed](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed)
+            for details.
+        )",
+    },
+    {
+        .tag = Xp::ImpureDerivations,
+        .name = "impure-derivations",
+        .description = R"(
+            Allow derivations to produce non-fixed outputs by setting the
+            `__impure` derivation attribute to `true`. An impure derivation can
+            have differing outputs each time it is built.
+
+            Example:
+
+            ```
+            derivation {
+              name = "impure";
+              builder = /bin/sh;
+              __impure = true; # mark this derivation as impure
+              args = [ "-c" "read -n 10 random < /dev/random; echo $random > $out" ];
+              system = builtins.currentSystem;
+            }
+            ```
+
+            Each time this derivation is built, it can produce a different
+            output (as the builder outputs random bytes to `$out`).  Impure
+            derivations also have access to the network, and only fixed-output
+            or other impure derivations can rely on impure derivations. Finally,
+            an impure derivation cannot also be
+            [content-addressed](#xp-feature-ca-derivations).
+        )",
+    },
+    {
+        .tag = Xp::Flakes,
+        .name = "flakes",
+        .description = R"(
+            Enable flakes. See the manual entry for [`nix
+            flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details.
+        )",
+    },
+    {
+        .tag = Xp::NixCommand,
+        .name = "nix-command",
+        .description = R"(
+            Enable the new `nix` subcommands. See the manual on
+            [`nix`](@docroot@/command-ref/new-cli/nix.md) for details.
+        )",
+    },
+    {
+        .tag = Xp::RecursiveNix,
+        .name = "recursive-nix",
+        .description = R"(
+            Allow derivation builders to call Nix, and thus build derivations
+            recursively.
+
+            Example:
+
+            ```
+            with import <nixpkgs> {};
+
+            runCommand "foo"
+              {
+                 buildInputs = [ nix jq ];
+                 NIX_PATH = "nixpkgs=${<nixpkgs>}";
+              }
+              ''
+                hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "recursive-hello"; })')
+
+                mkdir -p $out/bin
+                ln -s $hello/bin/hello $out/bin/hello
+              ''
+            ```
+
+            An important restriction on recursive builders is disallowing
+            arbitrary substitutions. For example, running
+
+            ```
+            nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
+            ```
+
+            in the above `runCommand` script would be disallowed, as this could
+            lead to derivations with hidden dependencies or breaking
+            reproducibility by relying on the current state of the Nix store. An
+            exception would be if
+            `/nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10` were
+            already in the build inputs or built by a previous recursive Nix
+            call.
+        )",
+    },
+    {
+        .tag = Xp::NoUrlLiterals,
+        .name = "no-url-literals",
+        .description = R"(
+            Disallow unquoted URLs as part of the Nix language syntax. The Nix
+            language allows for URL literals, like so:
+
+            ```
+            $ nix repl
+            Welcome to Nix 2.15.0. Type :? for help.
+
+            nix-repl> http://foo
+            "http://foo"
+            ```
+
+            But enabling this experimental feature will cause the Nix parser to
+            throw an error when encountering a URL literal:
+
+            ```
+            $ nix repl --extra-experimental-features 'no-url-literals'
+            Welcome to Nix 2.15.0. Type :? for help.
+
+            nix-repl> http://foo
+            error: URL literals are disabled
+
+            at «string»:1:1:
+
+            1| http://foo
+             | ^
+
+            ```
+
+            While this is currently an experimental feature, unquoted URLs are
+            being deprecated and their usage is discouraged.
+
+            The reason is that, as opposed to path literals, URLs have no
+            special properties that distinguish them from regular strings, URLs
+            containing parameters have to be quoted anyway, and unquoted URLs
+            may confuse external tooling.
+        )",
+    },
+    {
+        .tag = Xp::FetchClosure,
+        .name = "fetch-closure",
+        .description = R"(
+            Enable the use of the [`fetchClosure`](@docroot@/language/builtins.md#builtins-fetchClosure) built-in function in the Nix language.
+        )",
+    },
+    {
+        .tag = Xp::ReplFlake,
+        .name = "repl-flake",
+        .description = R"(
+            Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands.
+        )",
+    },
+    {
+        .tag = Xp::AutoAllocateUids,
+        .name = "auto-allocate-uids",
+        .description = R"(
+            Allows Nix to automatically pick UIDs for builds, rather than creating
+            `nixbld*` user accounts. See the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting for details.
+        )",
+    },
+    {
+        .tag = Xp::Cgroups,
+        .name = "cgroups",
+        .description = R"(
+            Allows Nix to execute builds inside cgroups. See
+            the [`use-cgroups`](#conf-use-cgroups) setting for details.
+        )",
+    },
+    {
+        .tag = Xp::DiscardReferences,
+        .name = "discard-references",
+        .description = R"(
+            Allow the use of the [`unsafeDiscardReferences`](@docroot@/language/advanced-attributes.html#adv-attr-unsafeDiscardReferences) attribute in derivations
+            that use [structured attributes](@docroot@/language/advanced-attributes.html#adv-attr-structuredAttrs). This disables scanning of outputs for
+            runtime dependencies.
+        )",
+    },
+    {
+        .tag = Xp::NixTesting,
+        .name = "nix-testing",
+        .description = R"(
+            A "permanent" experimental feature for extra features we just need
+            for testing. Not actually an "experiment" in the sense of being
+            prospective functionality for regular users.
+        )",
+    },
+}};
+
+static_assert(
+    []() constexpr {
+        for (auto [index, feature] : enumerate(xpFeatureDetails))
+            if (index != (size_t)feature.tag)
+                return false;
+        return true;
+    }(),
+    "array order does not match enum tag order");
+
 const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)
 {
     using ReverseXpMap = std::map<std::string_view, ExperimentalFeature>;
 
-    static auto reverseXpMap = []()
+    static std::unique_ptr<ReverseXpMap> reverseXpMap = []() {
-    {
         auto reverseXpMap = std::make_unique<ReverseXpMap>();
-        for (auto & [feature, name] : stringifiedXpFeatures)
+        for (auto & xpFeature : xpFeatureDetails)
-            (*reverseXpMap)[name] = feature;
+            (*reverseXpMap)[xpFeature.name] = xpFeature.tag;
         return reverseXpMap;
     }();
 
@@ -38,20 +226,27 @@ const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::str
         return std::nullopt;
 }
 
-std::string_view showExperimentalFeature(const ExperimentalFeature feature)
+std::string_view showExperimentalFeature(const ExperimentalFeature tag)
 {
-    const auto ret = get(stringifiedXpFeatures, feature);
+    assert((size_t)tag < xpFeatureDetails.size());
-    assert(ret);
+    return xpFeatureDetails[(size_t)tag].name;
-    return *ret;
+}
+
+nlohmann::json documentExperimentalFeatures() 
+{
+    StringMap res;
+    for (auto & xpFeature : xpFeatureDetails)
+        res[std::string { xpFeature.name }] =
+            trim(stripIndentation(xpFeature.description));
+    return (nlohmann::json) res;
 }
 
 std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> & rawFeatures)
 {
     std::set<ExperimentalFeature> res;
-    for (auto & rawFeature : rawFeatures) {
+    for (auto & rawFeature : rawFeatures)
         if (auto feature = parseExperimentalFeature(rawFeature))
             res.insert(*feature);
-    }
     return res;
 }
 

src/libutil/experimental-features.hh

@@ -11,8 +11,9 @@ namespace nix {
 /**
  * The list of available experimental features.
  *
- * If you update this, don’t forget to also change the map defining their
+ * If you update this, don’t forget to also change the map defining
- * string representation in the corresponding `.cc` file.
+ * their string representation and documentation in the corresponding
+ * `.cc` file as well.
  */
 enum struct ExperimentalFeature
 {
@@ -27,11 +28,6 @@ enum struct ExperimentalFeature
     AutoAllocateUids,
     Cgroups,
     DiscardReferences,
-
-    /**
-     * A "permanent" experimental feature for extra features we just
-     * need for testing.
-     **/
     NixTesting,
 };
 
@@ -40,26 +36,52 @@ enum struct ExperimentalFeature
  */
 using Xp = ExperimentalFeature;
 
+/**
+ * Parse an experimental feature (enum value) from its name. Experimental
+ * feature flag names are hyphenated and do not contain spaces.
+ */
 const std::optional<ExperimentalFeature> parseExperimentalFeature(
         const std::string_view & name);
+
+/**
+ * Show the name of an experimental feature. This is the opposite of
+ * parseExperimentalFeature().
+ */
 std::string_view showExperimentalFeature(const ExperimentalFeature);
 
+/**
+ * Compute the documentation of all experimental features.
+ *
+ * See `doc/manual` for how this information is used.
+ */
+nlohmann::json documentExperimentalFeatures();
+
+/**
+ * Shorthand for `str << showExperimentalFeature(feature)`.
+ */
 std::ostream & operator<<(
         std::ostream & str,
         const ExperimentalFeature & feature);
 
 /**
- * Parse a set of strings to the corresponding set of experimental features,
+ * Parse a set of strings to the corresponding set of experimental
- * ignoring (but warning for) any unkwown feature.
+ * features, ignoring (but warning for) any unknown feature.
  */
 std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> &);
 
+/**
+ * An experimental feature was required for some (experimental)
+ * operation, but was not enabled.
+ */
 class MissingExperimentalFeature : public Error
 {
 public:
+    /**
+     * The experimental feature that was required but not enabled.
+     */
     ExperimentalFeature missingFeature;
 
-    MissingExperimentalFeature(ExperimentalFeature);
+    MissingExperimentalFeature(ExperimentalFeature missingFeature);
 };
 
 /**

src/libutil/util.hh

@@ -916,16 +916,16 @@ constexpr auto enumerate(T && iterable)
     {
         size_t i;
         TIter iter;
-        bool operator != (const iterator & other) const { return iter != other.iter; }
+        constexpr bool operator != (const iterator & other) const { return iter != other.iter; }
-        void operator ++ () { ++i; ++iter; }
+        constexpr void operator ++ () { ++i; ++iter; }
-        auto operator * () const { return std::tie(i, *iter); }
+        constexpr auto operator * () const { return std::tie(i, *iter); }
     };
 
     struct iterable_wrapper
     {
         T iterable;
-        auto begin() { return iterator{ 0, std::begin(iterable) }; }
+        constexpr auto begin() { return iterator{ 0, std::begin(iterable) }; }
-        auto end() { return iterator{ 0, std::end(iterable) }; }
+        constexpr auto end() { return iterator{ 0, std::end(iterable) }; }
     };
 
     return iterable_wrapper{ std::forward<T>(iterable) };

src/nix/main.cc

@@ -375,6 +375,11 @@ void mainWrapped(int argc, char * * argv)
         return;
     }
 
+    if (argc == 2 && std::string(argv[1]) == "__dump-xp-features") {
+        logger->cout(documentExperimentalFeatures().dump());
+        return;
+    }
+
     Finally printCompletions([&]()
     {
         if (completions) {

tests/experimental-features.sh

@@ -1,25 +1,27 @@
 source common.sh
 
-# Without flakes, flake options should not show up
+# Skipping these two for now, because we actually *do* want flags and
-# With flakes, flake options should show up
+# config settings to always show up in the manual, just be marked
-
+# experimental. Will reenable once the manual generation takes advantage
-function both_ways {
+# of the JSON metadata on this.
-    nix --experimental-features 'nix-command' "$@" | grepQuietInverse flake
+#
-    nix --experimental-features 'nix-command flakes' "$@" | grepQuiet flake
+# # Without flakes, flake options should not show up
-
+# # With flakes, flake options should show up
-    # Also, the order should not matter
+#
-    nix "$@" --experimental-features 'nix-command' | grepQuietInverse flake
+# function grep_both_ways {
-    nix "$@" --experimental-features 'nix-command flakes' | grepQuiet flake
+#     nix --experimental-features 'nix-command' "$@" | grepQuietInverse flake
-}
+#     nix --experimental-features 'nix-command flakes' "$@" | grepQuiet flake
-
+#
-# Simple case, the configuration effects the running command
+#     # Also, the order should not matter
-both_ways show-config
+#     nix "$@" --experimental-features 'nix-command' | grepQuietInverse flake
-
+#     nix "$@" --experimental-features 'nix-command flakes' | grepQuiet flake
-# Skipping for now, because we actually *do* want these to show up in
+# }
-# the manual, just be marked experimental. Will reenable once the manual
+#
-# generation takes advantage of the JSON metadata on this.
+# # Simple case, the configuration effects the running command
-
+# grep_both_ways show-config
-# both_ways store gc --help
+#
+# # Medium case, the configuration effects --help
+# grep_both_ways store gc --help
 
 expect 1 nix --experimental-features 'nix-command' show-config --flake-registry 'https://no'
 nix --experimental-features 'nix-command flakes' show-config --flake-registry 'https://no'

tests/nix-profile.sh

@@ -144,6 +144,7 @@ expect 1 nix profile install $flake2Dir
 diff -u <(
     nix --offline profile install $flake2Dir 2>&1 1> /dev/null \
         | grep -vE "^warning: " \
+        | grep -vE "^error \(ignored\): " \
         || true
 ) <(cat << EOF
 error: An existing package already provides the following file: