Merge pull request #7798 from peeley/list-experimental-features

Documentation: list experimental features in manual

.gitignore

@@ -19,9 +19,12 @@ perl/Makefile.config
 /doc/manual/nix.json
 /doc/manual/conf-file.json
 /doc/manual/builtins.json
+/doc/manual/xp-features.json
 /doc/manual/src/SUMMARY.md
 /doc/manual/src/command-ref/new-cli
 /doc/manual/src/command-ref/conf-file.md
+/doc/manual/src/command-ref/experimental-features-shortlist.md
+/doc/manual/src/contributing/experimental-feature-descriptions.md
 /doc/manual/src/language/builtins.md
 
 # /scripts/

doc/manual/generate-xp-features-shortlist.nix

@@ -0,0 +1,9 @@
+with builtins;
+with import ./utils.nix;
+
+let
+  showExperimentalFeature = name: doc:
+    ''
+      - [`${name}`](@docroot@/contributing/experimental-features.md#xp-feature-${name})
+    '';
+in xps: indent "  " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps)))

doc/manual/generate-xp-features.nix

@@ -0,0 +1,11 @@
+with builtins;
+with import ./utils.nix;
+
+let
+  showExperimentalFeature = name: doc:
+    squash ''
+      ## [`${name}`]{#xp-feature-${name}}
+
+      ${doc}
+    '';
+in xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps)))

doc/manual/local.mk

@@ -81,19 +81,20 @@ $(d)/%.8: $(d)/src/command-ref/%.md
 $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
 	@printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp
 	@cat $^ >> $^.tmp
+	@$(call process-includes,$^,$^.tmp)
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
+$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
 	@cp $< $@
 	@$(call process-includes,$@,$@)
 
-$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
+$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
+$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
 	@mv $@.tmp $@
@@ -106,6 +107,20 @@ $(d)/conf-file.json: $(bindir)/nix
 	$(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp
 	@mv $@.tmp $@
 
+$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix
+	@rm -rf $@ $@.tmp
+	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))'
+	@mv $@.tmp $@
+
+$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix
+	@rm -rf $@ $@.tmp
+	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))'
+	@mv $@.tmp $@
+
+$(d)/xp-features.json: $(bindir)/nix
+	$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp
+	@mv $@.tmp $@
+
 $(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
 	@cat doc/manual/src/language/builtins-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
@@ -145,7 +160,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 	done
 	@touch $@
 
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \

doc/manual/src/contributing/experimental-features.md

@@ -89,3 +89,7 @@ However they serve different purposes:
   It is primarily an issue of *design* and *communication*, targeting the broader community.
 
 This means that experimental features and RFCs are orthogonal mechanisms, and can be used independently or together as needed.
+
+# Currently available experimental features
+
+{{#include ./experimental-feature-descriptions.md}}

doc/manual/utils.nix

@@ -5,6 +5,9 @@ rec {
 
   concatStrings = concatStringsSep "";
 
+  attrsToList = a:
+    map (name: { inherit name; value = a.${name}; }) (builtins.attrNames a);
+
   replaceStringsRec = from: to: string:
     # recursively replace occurrences of `from` with `to` within `string`
     # example:
@@ -74,10 +77,10 @@ rec {
           if aliases == [] then "" else
             "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}";
 
+    in result;
+
   indent = prefix: s:
     concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s));
 
-    in result;
-
   showSettings = args: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting args) settingsInfo));
 }

src/libutil/config.hh

@@ -371,8 +371,23 @@ extern GlobalConfig globalConfig;
 
 struct ExperimentalFeatureSettings : Config {
 
-    Setting<std::set<ExperimentalFeature>> experimentalFeatures{this, {}, "experimental-features",
+    Setting<std::set<ExperimentalFeature>> experimentalFeatures{
-        "Experimental Nix features to enable."};
+        this, {}, "experimental-features",
+        R"(
+          Experimental features that are enabled.
+
+          Example:
+
+          ```
+          experimental-features = nix-command flakes
+          ```
+
+          The following experimental features are available:
+
+          {{#include experimental-features-shortlist.md}}
+
+          Experimental features are [further documented in the manual](@docroot@/contributing/experimental-features.md).
+        )"};
 
     /**
      * Check whether the given experimental feature is enabled.

src/libutil/experimental-features.cc

@@ -5,29 +5,209 @@
 
 namespace nix {
 
-std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
+struct ExperimentalFeatureDetails
-    { Xp::CaDerivations, "ca-derivations" },
+{
-    { Xp::ImpureDerivations, "impure-derivations" },
+    ExperimentalFeature tag;
-    { Xp::Flakes, "flakes" },
+    std::string_view name;
-    { Xp::NixCommand, "nix-command" },
+    std::string_view description;
-    { Xp::RecursiveNix, "recursive-nix" },
-    { Xp::NoUrlLiterals, "no-url-literals" },
-    { Xp::FetchClosure, "fetch-closure" },
-    { Xp::ReplFlake, "repl-flake" },
-    { Xp::AutoAllocateUids, "auto-allocate-uids" },
-    { Xp::Cgroups, "cgroups" },
-    { Xp::DiscardReferences, "discard-references" },
 };
 
+constexpr std::array<ExperimentalFeatureDetails, 11> xpFeatureDetails = {{
+    {
+        .tag = Xp::CaDerivations,
+        .name = "ca-derivations",
+        .description = R"(
+            Allow derivations to be content-addressed in order to prevent
+            rebuilds when changes to the derivation do not result in changes to
+            the derivation's output. See
+            [__contentAddressed](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed)
+            for details.
+        )",
+    },
+    {
+        .tag = Xp::ImpureDerivations,
+        .name = "impure-derivations",
+        .description = R"(
+            Allow derivations to produce non-fixed outputs by setting the
+            `__impure` derivation attribute to `true`. An impure derivation can
+            have differing outputs each time it is built.
+
+            Example:
+
+            ```
+            derivation {
+              name = "impure";
+              builder = /bin/sh;
+              __impure = true; # mark this derivation as impure
+              args = [ "-c" "read -n 10 random < /dev/random; echo $random > $out" ];
+              system = builtins.currentSystem;
+            }
+            ```
+
+            Each time this derivation is built, it can produce a different
+            output (as the builder outputs random bytes to `$out`).  Impure
+            derivations also have access to the network, and only fixed-output
+            or other impure derivations can rely on impure derivations. Finally,
+            an impure derivation cannot also be
+            [content-addressed](#xp-feature-ca-derivations).
+        )",
+    },
+    {
+        .tag = Xp::Flakes,
+        .name = "flakes",
+        .description = R"(
+            Enable flakes. See the manual entry for [`nix
+            flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details.
+        )",
+    },
+    {
+        .tag = Xp::NixCommand,
+        .name = "nix-command",
+        .description = R"(
+            Enable the new `nix` subcommands. See the manual on
+            [`nix`](@docroot@/command-ref/new-cli/nix.md) for details.
+        )",
+    },
+    {
+        .tag = Xp::RecursiveNix,
+        .name = "recursive-nix",
+        .description = R"(
+            Allow derivation builders to call Nix, and thus build derivations
+            recursively.
+
+            Example:
+
+            ```
+            with import <nixpkgs> {};
+
+            runCommand "foo"
+              {
+                 buildInputs = [ nix jq ];
+                 NIX_PATH = "nixpkgs=${<nixpkgs>}";
+              }
+              ''
+                hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "recursive-hello"; })')
+
+                mkdir -p $out/bin
+                ln -s $hello/bin/hello $out/bin/hello
+              ''
+            ```
+
+            An important restriction on recursive builders is disallowing
+            arbitrary substitutions. For example, running
+
+            ```
+            nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
+            ```
+
+            in the above `runCommand` script would be disallowed, as this could
+            lead to derivations with hidden dependencies or breaking
+            reproducibility by relying on the current state of the Nix store. An
+            exception would be if
+            `/nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10` were
+            already in the build inputs or built by a previous recursive Nix
+            call.
+        )",
+    },
+    {
+        .tag = Xp::NoUrlLiterals,
+        .name = "no-url-literals",
+        .description = R"(
+            Disallow unquoted URLs as part of the Nix language syntax. The Nix
+            language allows for URL literals, like so:
+
+            ```
+            $ nix repl
+            Welcome to Nix 2.15.0. Type :? for help.
+
+            nix-repl> http://foo
+            "http://foo"
+            ```
+
+            But enabling this experimental feature will cause the Nix parser to
+            throw an error when encountering a URL literal:
+
+            ```
+            $ nix repl --extra-experimental-features 'no-url-literals'
+            Welcome to Nix 2.15.0. Type :? for help.
+
+            nix-repl> http://foo
+            error: URL literals are disabled
+
+            at «string»:1:1:
+
+            1| http://foo
+             | ^
+
+            ```
+
+            While this is currently an experimental feature, unquoted URLs are
+            being deprecated and their usage is discouraged.
+
+            The reason is that, as opposed to path literals, URLs have no
+            special properties that distinguish them from regular strings, URLs
+            containing parameters have to be quoted anyway, and unquoted URLs
+            may confuse external tooling.
+        )",
+    },
+    {
+        .tag = Xp::FetchClosure,
+        .name = "fetch-closure",
+        .description = R"(
+            Enable the use of the [`fetchClosure`](@docroot@/language/builtins.md#builtins-fetchClosure) built-in function in the Nix language.
+        )",
+    },
+    {
+        .tag = Xp::ReplFlake,
+        .name = "repl-flake",
+        .description = R"(
+            Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands.
+        )",
+    },
+    {
+        .tag = Xp::AutoAllocateUids,
+        .name = "auto-allocate-uids",
+        .description = R"(
+            Allows Nix to automatically pick UIDs for builds, rather than creating
+            `nixbld*` user accounts. See the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting for details.
+        )",
+    },
+    {
+        .tag = Xp::Cgroups,
+        .name = "cgroups",
+        .description = R"(
+            Allows Nix to execute builds inside cgroups. See
+            the [`use-cgroups`](#conf-use-cgroups) setting for details.
+        )",
+    },
+    {
+        .tag = Xp::DiscardReferences,
+        .name = "discard-references",
+        .description = R"(
+            Allow the use of the [`unsafeDiscardReferences`](@docroot@/language/advanced-attributes.html#adv-attr-unsafeDiscardReferences) attribute in derivations
+            that use [structured attributes](@docroot@/language/advanced-attributes.html#adv-attr-structuredAttrs). This disables scanning of outputs for
+            runtime dependencies.
+        )",
+    },
+}};
+
+static_assert(
+    []() constexpr {
+        for (auto [index, feature] : enumerate(xpFeatureDetails))
+            if (index != (size_t)feature.tag)
+                return false;
+        return true;
+    }(),
+    "array order does not match enum tag order");
+
 const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)
 {
     using ReverseXpMap = std::map<std::string_view, ExperimentalFeature>;
 
-    static auto reverseXpMap = []()
+    static std::unique_ptr<ReverseXpMap> reverseXpMap = []() {
-    {
         auto reverseXpMap = std::make_unique<ReverseXpMap>();
-        for (auto & [feature, name] : stringifiedXpFeatures)
+        for (auto & xpFeature : xpFeatureDetails)
-            (*reverseXpMap)[name] = feature;
+            (*reverseXpMap)[xpFeature.name] = xpFeature.tag;
         return reverseXpMap;
     }();
 
@@ -37,20 +217,27 @@ const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::str
         return std::nullopt;
 }
 
-std::string_view showExperimentalFeature(const ExperimentalFeature feature)
+std::string_view showExperimentalFeature(const ExperimentalFeature tag)
 {
-    const auto ret = get(stringifiedXpFeatures, feature);
+    assert((size_t)tag < xpFeatureDetails.size());
-    assert(ret);
+    return xpFeatureDetails[(size_t)tag].name;
-    return *ret;
+}
+
+nlohmann::json documentExperimentalFeatures() 
+{
+    StringMap res;
+    for (auto & xpFeature : xpFeatureDetails)
+        res[std::string { xpFeature.name }] =
+            trim(stripIndentation(xpFeature.description));
+    return (nlohmann::json) res;
 }
 
 std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> & rawFeatures)
 {
     std::set<ExperimentalFeature> res;
-    for (auto & rawFeature : rawFeatures) {
+    for (auto & rawFeature : rawFeatures)
         if (auto feature = parseExperimentalFeature(rawFeature))
             res.insert(*feature);
-    }
     return res;
 }
 

src/libutil/experimental-features.hh

@@ -11,8 +11,9 @@ namespace nix {
 /**
  * The list of available experimental features.
  *
- * If you update this, don’t forget to also change the map defining their
+ * If you update this, don’t forget to also change the map defining
- * string representation in the corresponding `.cc` file.
+ * their string representation and documentation in the corresponding
+ * `.cc` file as well.
  */
 enum struct ExperimentalFeature
 {
@@ -34,26 +35,52 @@ enum struct ExperimentalFeature
  */
 using Xp = ExperimentalFeature;
 
+/**
+ * Parse an experimental feature (enum value) from its name. Experimental
+ * feature flag names are hyphenated and do not contain spaces.
+ */
 const std::optional<ExperimentalFeature> parseExperimentalFeature(
         const std::string_view & name);
+
+/**
+ * Show the name of an experimental feature. This is the opposite of
+ * parseExperimentalFeature().
+ */
 std::string_view showExperimentalFeature(const ExperimentalFeature);
 
+/**
+ * Compute the documentation of all experimental features.
+ *
+ * See `doc/manual` for how this information is used.
+ */
+nlohmann::json documentExperimentalFeatures();
+
+/**
+ * Shorthand for `str << showExperimentalFeature(feature)`.
+ */
 std::ostream & operator<<(
         std::ostream & str,
         const ExperimentalFeature & feature);
 
 /**
- * Parse a set of strings to the corresponding set of experimental features,
+ * Parse a set of strings to the corresponding set of experimental
- * ignoring (but warning for) any unkwown feature.
+ * features, ignoring (but warning for) any unknown feature.
  */
 std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> &);
 
+/**
+ * An experimental feature was required for some (experimental)
+ * operation, but was not enabled.
+ */
 class MissingExperimentalFeature : public Error
 {
 public:
+    /**
+     * The experimental feature that was required but not enabled.
+     */
     ExperimentalFeature missingFeature;
 
-    MissingExperimentalFeature(ExperimentalFeature);
+    MissingExperimentalFeature(ExperimentalFeature missingFeature);
 };
 
 /**

src/libutil/util.hh

@@ -916,16 +916,16 @@ constexpr auto enumerate(T && iterable)
     {
         size_t i;
         TIter iter;
-        bool operator != (const iterator & other) const { return iter != other.iter; }
+        constexpr bool operator != (const iterator & other) const { return iter != other.iter; }
-        void operator ++ () { ++i; ++iter; }
+        constexpr void operator ++ () { ++i; ++iter; }
-        auto operator * () const { return std::tie(i, *iter); }
+        constexpr auto operator * () const { return std::tie(i, *iter); }
     };
 
     struct iterable_wrapper
     {
         T iterable;
-        auto begin() { return iterator{ 0, std::begin(iterable) }; }
+        constexpr auto begin() { return iterator{ 0, std::begin(iterable) }; }
-        auto end() { return iterator{ 0, std::end(iterable) }; }
+        constexpr auto end() { return iterator{ 0, std::end(iterable) }; }
     };
 
     return iterable_wrapper{ std::forward<T>(iterable) };

src/nix/main.cc

@@ -375,6 +375,11 @@ void mainWrapped(int argc, char * * argv)
         return;
     }
 
+    if (argc == 2 && std::string(argv[1]) == "__dump-xp-features") {
+        logger->cout(documentExperimentalFeatures().dump());
+        return;
+    }
+
     Finally printCompletions([&]()
     {
         if (completions) {