From 21fc0ddce55a3caa4cc920c8394d1f9f218661ed Mon Sep 17 00:00:00 2001 From: Alois Wohlschlager Date: Sun, 8 Sep 2024 19:01:46 +0200 Subject: [PATCH] libutil: generate experimental and deprecated features from data Currently, a bunch of documentation is generated by embedding parts of it in the nix executable, getting it out again by running it, and then postprocessing the output. This is bad, since it creates a pointless dependency of the documentation on the executable, and also makes documentation generation impossible when cross-compiling. Instead, both the code and the documentation should be generated from data, see https://git.lix.systems/lix-project/lix/issues/292 . Here we start applying this approach to the experimental and deprecated features, which are done in one go since the technical implementation is very similar. Of course, the actual benefits are not realised yet, since the offending pattern is used in several more places. These will be fixed later. Change-Id: I4c802052cc7e865c61119a34b8f1063c4decc9cb --- .gitignore | 2 +- doc/manual/generate-features-shortlist.nix | 14 -- doc/manual/generate-features.nix | 18 -- doc/manual/meson.build | 12 +- doc/manual/src/command-ref/meson.build | 34 --- .../src/contributing/deprecated-features.md | 2 +- .../src/contributing/experimental-features.md | 2 +- doc/manual/src/contributing/hacking.md | 14 ++ doc/manual/src/contributing/meson.build | 28 --- meson/clang-tidy/build_required_targets.py | 3 +- package.nix | 2 + .../build_experimental_features.py | 58 +++++ src/code-generation/common.py | 50 ++++ src/libutil/config.hh | 4 +- src/libutil/deprecated-features.cc | 33 +-- src/libutil/deprecated-features.hh | 9 +- .../deprecated-features/ancient-let.md | 7 + .../deprecated-features/rec-set-overrides.md | 7 + .../deprecated-features/url-literals.md | 5 + src/libutil/experimental-features.cc | 231 +----------------- src/libutil/experimental-features.hh | 22 +- .../auto-allocate-uids.md | 6 + .../experimental-features/ca-derivations.md | 9 + src/libutil/experimental-features/cgroups.md | 6 + .../daemon-trust-override.md | 8 + .../dynamic-derivations.md | 11 + .../experimental-features/fetch-closure.md | 5 + src/libutil/experimental-features/flakes.md | 6 + .../impure-derivations.md | 28 +++ .../experimental-features/nix-command.md | 6 + .../experimental-features/no-url-literals.md | 39 +++ .../parse-toml-timestamps.md | 5 + .../experimental-features/pipe-operator.md | 8 + .../read-only-local-store.md | 5 + .../experimental-features/recursive-nix.md | 39 +++ .../experimental-features/repl-automation.md | 5 + .../experimental-features/repl-flake.md | 5 + src/libutil/meson.build | 98 ++++++++ 38 files changed, 452 insertions(+), 394 deletions(-) delete mode 100644 doc/manual/generate-features-shortlist.nix delete mode 100644 doc/manual/generate-features.nix delete mode 100644 doc/manual/src/contributing/meson.build create mode 100644 src/code-generation/build_experimental_features.py create mode 100644 src/code-generation/common.py create mode 100644 src/libutil/deprecated-features/ancient-let.md create mode 100644 src/libutil/deprecated-features/rec-set-overrides.md create mode 100644 src/libutil/deprecated-features/url-literals.md create mode 100644 src/libutil/experimental-features/auto-allocate-uids.md create mode 100644 src/libutil/experimental-features/ca-derivations.md create mode 100644 src/libutil/experimental-features/cgroups.md create mode 100644 src/libutil/experimental-features/daemon-trust-override.md create mode 100644 src/libutil/experimental-features/dynamic-derivations.md create mode 100644 src/libutil/experimental-features/fetch-closure.md create mode 100644 src/libutil/experimental-features/flakes.md create mode 100644 src/libutil/experimental-features/impure-derivations.md create mode 100644 src/libutil/experimental-features/nix-command.md create mode 100644 src/libutil/experimental-features/no-url-literals.md create mode 100644 src/libutil/experimental-features/parse-toml-timestamps.md create mode 100644 src/libutil/experimental-features/pipe-operator.md create mode 100644 src/libutil/experimental-features/read-only-local-store.md create mode 100644 src/libutil/experimental-features/recursive-nix.md create mode 100644 src/libutil/experimental-features/repl-automation.md create mode 100644 src/libutil/experimental-features/repl-flake.md diff --git a/.gitignore b/.gitignore index 15c871218..146d433a1 100644 --- a/.gitignore +++ b/.gitignore @@ -37,5 +37,5 @@ buildtime.bin # Rust build files when using Cargo (not actually supported for building but it spews the files anyway) /target/ -# Python compiled files from the test suite +# Python compiled files from the code generators and test suite *.pyc diff --git a/doc/manual/generate-features-shortlist.nix b/doc/manual/generate-features-shortlist.nix deleted file mode 100644 index 055698d64..000000000 --- a/doc/manual/generate-features-shortlist.nix +++ /dev/null @@ -1,14 +0,0 @@ -# Usually "experimental" or "deprecated" -kind: -# "xp" or "dp" -kindShort: - -with builtins; -with import ./utils.nix; - -let - showExperimentalFeature = name: doc: '' - - [`${name}`](@docroot@/contributing/${kind}-features.md#${kindShort}-feature-${name}) - ''; -in -xps: indent " " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/generate-features.nix b/doc/manual/generate-features.nix deleted file mode 100644 index 4a12ccdce..000000000 --- a/doc/manual/generate-features.nix +++ /dev/null @@ -1,18 +0,0 @@ -# Usually "experimental" or "deprecated" -_kind: -# "xp" or "dp" -kindShort: - -with builtins; -with import ./utils.nix; - -let - showFeature = - name: doc: - squash '' - ## [`${name}`]{#${kindShort}-feature-${name}} - - ${doc} - ''; -in -xps: (concatStringsSep "\n" (attrValues (mapAttrs showFeature xps))) diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 35d94740c..02383f6bb 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -48,14 +48,6 @@ nix_conf_file_md = custom_target( output : 'conf-file.md', ) -nix_exp_features_json = custom_target( - command : [ nix, '__dump-xp-features' ], - capture : true, - output : 'xp-features.json', - # FIXME: put the actual lib targets in here? meson have introspection challenge 2024 though. - build_always_stale : true, -) - language_json = custom_target( command: [nix, '__dump-language'], output : 'language.json', @@ -80,10 +72,8 @@ generate_manual_deps = files( # Generates builtins.md and builtin-constants.md. subdir('src/language') -# Generates new-cli pages, {experimental,deprecated}-features-shortlist.md, and conf-file.md. +# Generates new-cli pages and conf-file.md. subdir('src/command-ref') -# Generates {experimental,deprecated}-feature-descriptions.md. -subdir('src/contributing') # Generates rl-next-generated.md. subdir('src/release-notes') diff --git a/doc/manual/src/command-ref/meson.build b/doc/manual/src/command-ref/meson.build index f66ee0d39..6acc79f4e 100644 --- a/doc/manual/src/command-ref/meson.build +++ b/doc/manual/src/command-ref/meson.build @@ -1,37 +1,3 @@ -experimental_features_shortlist_md = custom_target( - command : nix_eval_for_docs + [ - '--expr', - 'import @INPUT0@ "experimental" "xp" (builtins.fromJSON (builtins.readFile @INPUT1@))', - ], - input : [ - '../../generate-features-shortlist.nix', - nix_exp_features_json, - ], - capture : true, - output : 'experimental-features-shortlist.md', - env : nix_env_for_docs, -) - -dp_features_json = custom_target( - command : [nix, '__dump-dp-features'], - capture : true, - output : 'dp-features.json', -) - -deprecated_features_shortlist_md = custom_target( - command : nix_eval_for_docs + [ - '--expr', - 'import @INPUT0@ "deprecated" "dp" (builtins.fromJSON (builtins.readFile @INPUT1@))', - ], - input : [ - '../../generate-features-shortlist.nix', - dp_features_json, - ], - capture : true, - output : 'deprecated-features-shortlist.md', - env : nix_env_for_docs, -) - # Intermediate step for manpage generation. # This splorks the output of generate-manpage.nix as JSON, # which gets written as a directory tree below. diff --git a/doc/manual/src/contributing/deprecated-features.md b/doc/manual/src/contributing/deprecated-features.md index 7536944f7..ef37a1d98 100644 --- a/doc/manual/src/contributing/deprecated-features.md +++ b/doc/manual/src/contributing/deprecated-features.md @@ -34,4 +34,4 @@ However, we do not live in such an ideal world, and currently this goal is so fa # Currently available deprecated features -{{#include @generated@/contributing/deprecated-feature-descriptions.md}} +{{#include @generated@/../../../src/libutil/deprecated-feature-descriptions.md}} diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/contributing/experimental-features.md index c27ba5da0..d5f6f2526 100644 --- a/doc/manual/src/contributing/experimental-features.md +++ b/doc/manual/src/contributing/experimental-features.md @@ -99,4 +99,4 @@ This means that experimental features and RFCs are orthogonal mechanisms, and ca # Currently available experimental features -{{#include @generated@/contributing/experimental-feature-descriptions.md}} +{{#include @generated@/../../../src/libutil/experimental-feature-descriptions.md}} diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 37c96a5c0..d245e20d1 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -399,3 +399,17 @@ The following properties are supported: Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly. + +## Adding experimental or deprecated features + +Experimental and deprecated features are generally referenced both in the code and in the documentation. +To prevent duplication or divergence, they are defined in data files, and a script generates the necessary glue. + +The data file format is similar to the release notes: it consists of a YAML metadata header, followed by the documentation in Markdown format. +The following metadata properties are supported: +* `name` (required): user-facing name of the feature, to be used in `nix.conf` options and on the command line. + This should also be the stem of the file name (with extension `md`). +* `internalName` (required): identifier used to refer to the feature inside the C++ code. + +Experimental feature data files should live in `src/libutil/experimental-features`, and deprecated features in `src/libutil/deprecated-features`. +They must be listed in the `experimental_feature_definitions` or `deprecated_feature_definitions` lists in `src/libutil/meson.build` respectively to be considered by the build system. diff --git a/doc/manual/src/contributing/meson.build b/doc/manual/src/contributing/meson.build deleted file mode 100644 index da88ca146..000000000 --- a/doc/manual/src/contributing/meson.build +++ /dev/null @@ -1,28 +0,0 @@ -# Intermediate step for experimental-feature-descriptions.md. -# This splorks the output of generate-xp-features.nix as JSON, -# which gets written as a directory tree below. -experimental_feature_descriptions_md = custom_target( - command : nix_eval_for_docs + [ - '--expr', - 'import @INPUT0@ "experimental" "xp" (builtins.fromJSON (builtins.readFile @INPUT1@))', - ], - input : [ - '../../generate-features.nix', - nix_exp_features_json, - ], - capture : true, - output : 'experimental-feature-descriptions.md', -) - -deprecated_feature_descriptions_md = custom_target( - command : nix_eval_for_docs + [ - '--expr', - 'import @INPUT0@ "deprecated" "dp" (builtins.fromJSON (builtins.readFile @INPUT1@))', - ], - input : [ - '../../generate-features.nix', - dp_features_json, - ], - capture : true, - output : 'deprecated-feature-descriptions.md', -) diff --git a/meson/clang-tidy/build_required_targets.py b/meson/clang-tidy/build_required_targets.py index 5c0e9641e..f26aa24a5 100755 --- a/meson/clang-tidy/build_required_targets.py +++ b/meson/clang-tidy/build_required_targets.py @@ -14,7 +14,8 @@ def main(): args = ap.parse_args() - targets = [t for t in get_targets_of_rule(args.build_root, 'CUSTOM_COMMAND') if t.endswith('gen.hh')] + targets = [t for t in get_targets_of_rule(args.build_root, 'CUSTOM_COMMAND') if t.endswith('.gen.hh')] + targets += [t for t in get_targets_of_rule(args.build_root, 'CUSTOM_COMMAND') if t.endswith('.gen.inc')] ninja_build(args.build_root, targets) if __name__ == '__main__': diff --git a/package.nix b/package.nix index 5c66b8446..297bc01fc 100644 --- a/package.nix +++ b/package.nix @@ -245,9 +245,11 @@ stdenv.mkDerivation (finalAttrs: { nativeBuildInputs = [ + # python3.withPackages does not splice properly, see https://github.com/NixOS/nixpkgs/issues/305858 (python3.pythonOnBuildForHost.withPackages (p: [ p.pytest p.pytest-xdist + p.python-frontmatter ])) meson ninja diff --git a/src/code-generation/build_experimental_features.py b/src/code-generation/build_experimental_features.py new file mode 100644 index 000000000..8768d826e --- /dev/null +++ b/src/code-generation/build_experimental_features.py @@ -0,0 +1,58 @@ +from typing import NamedTuple + +from common import cxx_literal, generate_file, load_data + +KNOWN_KEYS = set([ + 'name', + 'internalName', +]) + +class ExperimentalFeature(NamedTuple): + name: str + internal_name: str + description: str + + def parse(datum): + unknown_keys = set(datum.keys()) - KNOWN_KEYS + if unknown_keys: + raise ValueError('unknown keys', unknown_keys) + return ExperimentalFeature( + name = datum['name'], + internal_name = datum['internalName'], + description = datum.content, + ) + +def main(): + import argparse + + ap = argparse.ArgumentParser() + ap.add_argument('--deprecated', action='store_true', help='Generate deprecated features') + ap.add_argument('--header', help='Path of the declaration header to generate') + ap.add_argument('--impl-header', help='Path of the implementation header to generate') + ap.add_argument('--descriptions', help='Path of the description file to generate') + ap.add_argument('--shortlist', help='Path of the shortlist file to generate') + ap.add_argument('defs', help='Experimental feature definition files', nargs='+') + args = ap.parse_args() + + features = load_data(args.defs, ExperimentalFeature.parse) + + generate_file(args.header, features, lambda feature: feature.name, lambda feature: + f' {feature.internal_name},\n') + generate_file(args.impl_header, features, lambda feature: feature.name, lambda feature: + f''' {{ + .tag = {"Dep" if args.deprecated else "Xp"}::{feature.internal_name}, + .name = {cxx_literal(feature.name)}, + .description = {cxx_literal(feature.description)}, + }}, +''') + generate_file(args.descriptions, features, lambda feature: feature.name, lambda feature: + f'''## [`{feature.name}`]{{#{"dp" if args.deprecated else "xp"}-feature-{feature.name}}} + +{feature.description} + +''') + generate_file(args.shortlist, features, lambda feature: feature.name, lambda feature: + f' - [`{feature.name}`](@docroot@/contributing/{"deprecated" if args.deprecated else "experimental"}-features.md#{"dp" if args.deprecated else "xp"}-feature-{feature.name})\n') + +if __name__ == '__main__': + main() diff --git a/src/code-generation/common.py b/src/code-generation/common.py new file mode 100644 index 000000000..228359c78 --- /dev/null +++ b/src/code-generation/common.py @@ -0,0 +1,50 @@ +import frontmatter +import pathlib +from collections import defaultdict + +def cxx_escape_character(c): + if ord(c) >= 0x20 and ord(c) < 0x7f and c != '"' and c != '?' and c != '\\': + return c + elif c == '\t': + return r'\t' + elif c == '\n': + return r'\n' + elif c == '\r': + return r'\r' + elif c == '"': + return r'\"' + elif c == '?': + return r'\?' + elif c == '\\': + return r'\\' + elif ord(c) <= 0xffff: + return str.format(r'\u{:04x}', ord(c)) + else: + return str.format(r'\U{:08x}', ord(c)) + +def cxx_literal(v): + if isinstance(v, str): + return ''.join(['"', *(cxx_escape_character(c) for c in v), '"']) + else: + raise NotImplementedError(f'cannot represent {repr(v)} in C++') + +def load_data(defs, parse_function): + data = [] + for path in defs: + try: + datum = frontmatter.load(path) + data.append((path, parse_function(datum))) + except Exception as e: + e.add_note(f'in {path}') + raise + return data + +def generate_file(path, data, sort_key_function, generate_function): + if path is not None: + with open(path, 'w') as out: + for path, datum in sorted(data, key=lambda pathAndDatum: sort_key_function(pathAndDatum[1])): + try: + out.write(generate_function(datum)) + except Exception as e: + e.add_note(f'in {path}') + raise diff --git a/src/libutil/config.hh b/src/libutil/config.hh index b3dcc122f..eadc71926 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -410,7 +410,7 @@ struct FeatureSettings : Config { The following experimental features are available: - {{#include @generated@/command-ref/experimental-features-shortlist.md}} + {{#include @generated@/../../../src/libutil/experimental-features-shortlist.md}} Experimental features are [further documented in the manual](@docroot@/contributing/experimental-features.md). )"}; @@ -450,7 +450,7 @@ struct FeatureSettings : Config { The following deprecated feature features can be re-activated: - {{#include @generated@/command-ref/deprecated-features-shortlist.md}} + {{#include @generated@/../../../src/libutil/deprecated-features-shortlist.md}} Deprecated features are [further documented in the manual](@docroot@/contributing/deprecated-features.md). )"}; diff --git a/src/libutil/deprecated-features.cc b/src/libutil/deprecated-features.cc index 4de4c8ec7..65a003cd4 100644 --- a/src/libutil/deprecated-features.cc +++ b/src/libutil/deprecated-features.cc @@ -20,38 +20,13 @@ struct DeprecatedFeatureDetails * counter will be incremented once instead of twice, causing a build * failure. * - * By instead defining this instead as 1 + the bottom deprecated - * feature, we either have no issue at all if few features are not added - * at the end of the list, or a proper merge conflict if they are. + * By instead defining this instead as a dummy bottom deprecated + * feature, we do not have this issue. */ -constexpr size_t numDepFeatures = 1 + static_cast(Dep::UrlLiterals); +constexpr size_t numDepFeatures = static_cast(Dep::NumDepFeatures); constexpr std::array depFeatureDetails = {{ - { - .tag = Dep::RecSetOverrides, - .name = "rec-set-overrides", - .description = R"( - Allow `__overrides` in recursive attribute sets. - - Use fix point functions (e.g. `lib.fix` in Nixpkgs) instead. - )", - }, - { - .tag = Dep::AncientLet, - .name = "ancient-let", - .description = R"( - Allow the ancient `let { body = …; … }` syntax. - - Use the `let … in` syntax instead. - )", - }, - { - .tag = Dep::UrlLiterals, - .name = "url-literals", - .description = R"( - Allow unquoted URLs as part of the Nix language syntax. - )", - }, + #include "deprecated-features-impl.gen.inc" }}; static_assert( diff --git a/src/libutil/deprecated-features.hh b/src/libutil/deprecated-features.hh index bdff1bcdb..cd425e8f7 100644 --- a/src/libutil/deprecated-features.hh +++ b/src/libutil/deprecated-features.hh @@ -9,18 +9,13 @@ namespace nix { /** * The list of available deprecated features. * - * If you update this, don’t forget to also change the map defining - * their string representation and documentation in the corresponding - * `.cc` file as well. - * * Reminder: New deprecated features should start out with a warning without throwing an error. * See the developer documentation for details. */ enum struct DeprecatedFeature { - RecSetOverrides, - AncientLet, - UrlLiterals, + #include "deprecated-features.gen.inc" + NumDepFeatures, // number of available deprecated features, do not use }; enum struct DeprecatedFeatures {}; diff --git a/src/libutil/deprecated-features/ancient-let.md b/src/libutil/deprecated-features/ancient-let.md new file mode 100644 index 000000000..ced9934bb --- /dev/null +++ b/src/libutil/deprecated-features/ancient-let.md @@ -0,0 +1,7 @@ +--- +name: ancient-let +internalName: AncientLet +--- +Allow the ancient `let { body = …; … }` syntax. + +Use the `let … in` syntax instead. diff --git a/src/libutil/deprecated-features/rec-set-overrides.md b/src/libutil/deprecated-features/rec-set-overrides.md new file mode 100644 index 000000000..37d7003b1 --- /dev/null +++ b/src/libutil/deprecated-features/rec-set-overrides.md @@ -0,0 +1,7 @@ +--- +name: rec-set-overrides +internalName: RecSetOverrides +--- +Allow `__overrides` in recursive attribute sets. + +Use fix point functions (e.g. `lib.fix` in Nixpkgs) instead. diff --git a/src/libutil/deprecated-features/url-literals.md b/src/libutil/deprecated-features/url-literals.md new file mode 100644 index 000000000..99232913f --- /dev/null +++ b/src/libutil/deprecated-features/url-literals.md @@ -0,0 +1,5 @@ +--- +name: url-literals +internalName: UrlLiterals +--- +Allow unquoted URLs as part of the Nix language syntax. diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index bb7a95a24..da3e6c2cd 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -20,236 +20,13 @@ struct ExperimentalFeatureDetails * counter will be incremented once instead of twice, causing a build * failure. * - * By instead defining this instead as 1 + the bottom experimental - * feature, we either have no issue at all if few features are not added - * at the end of the list, or a proper merge conflict if they are. + * By instead defining this instead as a dummy bottom experimental + * feature, we do not get this issue. */ -constexpr size_t numXpFeatures = 1 + static_cast(Xp::ReplAutomation); +constexpr size_t numXpFeatures = static_cast(Xp::NumXpFeatures); constexpr std::array xpFeatureDetails = {{ - { - .tag = Xp::CaDerivations, - .name = "ca-derivations", - .description = R"( - Allow derivations to be content-addressed in order to prevent - rebuilds when changes to the derivation do not result in changes to - the derivation's output. See - [__contentAddressed](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed) - for details. - )", - }, - { - .tag = Xp::ImpureDerivations, - .name = "impure-derivations", - .description = R"( - Allow derivations to produce non-fixed outputs by setting the - `__impure` derivation attribute to `true`. An impure derivation can - have differing outputs each time it is built. - - Example: - - ``` - derivation { - name = "impure"; - builder = /bin/sh; - __impure = true; # mark this derivation as impure - args = [ "-c" "read -n 10 random < /dev/random; echo $random > $out" ]; - system = builtins.currentSystem; - } - ``` - - Each time this derivation is built, it can produce a different - output (as the builder outputs random bytes to `$out`). Impure - derivations also have access to the network, and only fixed-output - or other impure derivations can rely on impure derivations. Finally, - an impure derivation cannot also be - [content-addressed](#xp-feature-ca-derivations). - - This is a more explicit alternative to using [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime). - )", - }, - { - .tag = Xp::Flakes, - .name = "flakes", - .description = R"( - Enable flakes. See the manual entry for [`nix - flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details. - )", - }, - { - .tag = Xp::NixCommand, - .name = "nix-command", - .description = R"( - Enable the new `nix` subcommands. See the manual on - [`nix`](@docroot@/command-ref/new-cli/nix.md) for details. - )", - }, - { - .tag = Xp::RecursiveNix, - .name = "recursive-nix", - .description = R"( - Allow derivation builders to call Nix, and thus build derivations - recursively. - - Example: - - ``` - with import {}; - - runCommand "foo" - { - buildInputs = [ nix jq ]; - NIX_PATH = "nixpkgs=${}"; - } - '' - hello=$(nix-build -E '(import {}).hello.overrideDerivation (args: { name = "recursive-hello"; })') - - mkdir -p $out/bin - ln -s $hello/bin/hello $out/bin/hello - '' - ``` - - An important restriction on recursive builders is disallowing - arbitrary substitutions. For example, running - - ``` - nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10 - ``` - - in the above `runCommand` script would be disallowed, as this could - lead to derivations with hidden dependencies or breaking - reproducibility by relying on the current state of the Nix store. An - exception would be if - `/nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10` were - already in the build inputs or built by a previous recursive Nix - call. - )", - }, - { - .tag = Xp::NoUrlLiterals, - .name = "no-url-literals", - .description = R"( - Disallow unquoted URLs as part of the Nix language syntax. The Nix - language allows for URL literals, like so: - - ``` - $ nix repl - Welcome to Nix 2.15.0. Type :? for help. - - nix-repl> http://foo - "http://foo" - ``` - - But enabling this experimental feature will cause the Nix parser to - throw an error when encountering a URL literal: - - ``` - $ nix repl --extra-experimental-features 'no-url-literals' - Welcome to Nix 2.15.0. Type :? for help. - - nix-repl> http://foo - error: URL literals are disabled - - at «string»:1:1: - - 1| http://foo - | ^ - - ``` - - While this is currently an experimental feature, unquoted URLs are - being deprecated and their usage is discouraged. - - The reason is that, as opposed to path literals, URLs have no - special properties that distinguish them from regular strings, URLs - containing parameters have to be quoted anyway, and unquoted URLs - may confuse external tooling. - )", - }, - { - .tag = Xp::PipeOperator, - .name = "pipe-operator", - .description = R"( - Enable new operators for function application to "pipe" arguments through a chain of functions similar to `lib.pipe`. - This implementation is based on Nix [RFC 148](https://github.com/NixOS/rfcs/pull/148). - - Tracking issue: https://git.lix.systems/lix-project/lix/issues/438 - )", - }, - { - .tag = Xp::FetchClosure, - .name = "fetch-closure", - .description = R"( - Enable the use of the [`fetchClosure`](@docroot@/language/builtins.md#builtins-fetchClosure) built-in function in the Nix language. - )", - }, - { - .tag = Xp::ReplFlake, - .name = "repl-flake", - .description = R"( - Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands. - )", - }, - { - .tag = Xp::AutoAllocateUids, - .name = "auto-allocate-uids", - .description = R"( - Allows Nix to automatically pick UIDs for builds, rather than creating - `nixbld*` user accounts. See the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting for details. - )", - }, - { - .tag = Xp::Cgroups, - .name = "cgroups", - .description = R"( - Allows Nix to execute builds inside cgroups. See - the [`use-cgroups`](#conf-use-cgroups) setting for details. - )", - }, - { - .tag = Xp::DaemonTrustOverride, - .name = "daemon-trust-override", - .description = R"( - Allow forcing trusting or not trusting clients with - `nix-daemon`. This is useful for testing, but possibly also - useful for various experiments with `nix-daemon --stdio` - networking. - )", - }, - { - .tag = Xp::DynamicDerivations, - .name = "dynamic-derivations", - .description = R"( - Allow the use of a few things related to dynamic derivations: - - - "text hashing" derivation outputs, so we can build .drv - files. - - - dependencies in derivations on the outputs of - derivations that are themselves derivations outputs. - )", - }, - { - .tag = Xp::ParseTomlTimestamps, - .name = "parse-toml-timestamps", - .description = R"( - Allow parsing of timestamps in builtins.fromTOML. - )", - }, - { - .tag = Xp::ReadOnlyLocalStore, - .name = "read-only-local-store", - .description = R"( - Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. - )", - }, - { - .tag = Xp::ReplAutomation, - .name = "repl-automation", - .description = R"( - Makes the repl not use editline, print ENQ (U+0005) when ready for a command, and take commands followed by newline. - )", - }, + #include "experimental-features-impl.gen.inc" }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index c45f05d78..3241ed955 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -8,29 +8,11 @@ namespace nix { /** * The list of available experimental features. - * - * If you update this, don’t forget to also change the map defining - * their string representation and documentation in the corresponding - * `.cc` file as well. */ enum struct ExperimentalFeature { - CaDerivations, - ImpureDerivations, - Flakes, - NixCommand, - RecursiveNix, - NoUrlLiterals, - PipeOperator, - FetchClosure, - ReplFlake, - AutoAllocateUids, - Cgroups, - DaemonTrustOverride, - DynamicDerivations, - ParseTomlTimestamps, - ReadOnlyLocalStore, - ReplAutomation, + #include "experimental-features.gen.inc" + NumXpFeatures, // number of available experimental features, do not use }; enum struct ExperimentalFeatures {}; diff --git a/src/libutil/experimental-features/auto-allocate-uids.md b/src/libutil/experimental-features/auto-allocate-uids.md new file mode 100644 index 000000000..af7a2b68d --- /dev/null +++ b/src/libutil/experimental-features/auto-allocate-uids.md @@ -0,0 +1,6 @@ +--- +name: auto-allocate-uids +internalName: AutoAllocateUids +--- +Allows Nix to automatically pick UIDs for builds, rather than creating +`nixbld*` user accounts. See the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting for details. diff --git a/src/libutil/experimental-features/ca-derivations.md b/src/libutil/experimental-features/ca-derivations.md new file mode 100644 index 000000000..423666c54 --- /dev/null +++ b/src/libutil/experimental-features/ca-derivations.md @@ -0,0 +1,9 @@ +--- +name: ca-derivations +internalName: CaDerivations +--- +Allow derivations to be content-addressed in order to prevent +rebuilds when changes to the derivation do not result in changes to +the derivation's output. See +[__contentAddressed](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed) +for details. diff --git a/src/libutil/experimental-features/cgroups.md b/src/libutil/experimental-features/cgroups.md new file mode 100644 index 000000000..66e64fde9 --- /dev/null +++ b/src/libutil/experimental-features/cgroups.md @@ -0,0 +1,6 @@ +--- +name: cgroups +internalName: Cgroups +--- +Allows Nix to execute builds inside cgroups. See +the [`use-cgroups`](#conf-use-cgroups) setting for details. diff --git a/src/libutil/experimental-features/daemon-trust-override.md b/src/libutil/experimental-features/daemon-trust-override.md new file mode 100644 index 000000000..e87f31dd3 --- /dev/null +++ b/src/libutil/experimental-features/daemon-trust-override.md @@ -0,0 +1,8 @@ +--- +name: daemon-trust-override +internalName: DaemonTrustOverride +--- +Allow forcing trusting or not trusting clients with +`nix-daemon`. This is useful for testing, but possibly also +useful for various experiments with `nix-daemon --stdio` +networking. diff --git a/src/libutil/experimental-features/dynamic-derivations.md b/src/libutil/experimental-features/dynamic-derivations.md new file mode 100644 index 000000000..79201bcc3 --- /dev/null +++ b/src/libutil/experimental-features/dynamic-derivations.md @@ -0,0 +1,11 @@ +--- +name: dynamic-derivations +internalName: DynamicDerivations +--- +Allow the use of a few things related to dynamic derivations: + + - "text hashing" derivation outputs, so we can build .drv + files. + + - dependencies in derivations on the outputs of + derivations that are themselves derivations outputs. diff --git a/src/libutil/experimental-features/fetch-closure.md b/src/libutil/experimental-features/fetch-closure.md new file mode 100644 index 000000000..45692f030 --- /dev/null +++ b/src/libutil/experimental-features/fetch-closure.md @@ -0,0 +1,5 @@ +--- +name: fetch-closure +internalName: FetchClosure +--- +Enable the use of the [`fetchClosure`](@docroot@/language/builtins.md#builtins-fetchClosure) built-in function in the Nix language. diff --git a/src/libutil/experimental-features/flakes.md b/src/libutil/experimental-features/flakes.md new file mode 100644 index 000000000..3cb1c74a0 --- /dev/null +++ b/src/libutil/experimental-features/flakes.md @@ -0,0 +1,6 @@ +--- +name: flakes +internalName: Flakes +--- +Enable flakes. See the manual entry for [`nix +flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details. diff --git a/src/libutil/experimental-features/impure-derivations.md b/src/libutil/experimental-features/impure-derivations.md new file mode 100644 index 000000000..f2ab9f5f2 --- /dev/null +++ b/src/libutil/experimental-features/impure-derivations.md @@ -0,0 +1,28 @@ +--- +name: impure-derivations +internalName: ImpureDerivations +--- +Allow derivations to produce non-fixed outputs by setting the +`__impure` derivation attribute to `true`. An impure derivation can +have differing outputs each time it is built. + +Example: + +``` +derivation { + name = "impure"; + builder = /bin/sh; + __impure = true; # mark this derivation as impure + args = [ "-c" "read -n 10 random < /dev/random; echo $random > $out" ]; + system = builtins.currentSystem; +} +``` + +Each time this derivation is built, it can produce a different +output (as the builder outputs random bytes to `$out`). Impure +derivations also have access to the network, and only fixed-output +or other impure derivations can rely on impure derivations. Finally, +an impure derivation cannot also be +[content-addressed](#xp-feature-ca-derivations). + +This is a more explicit alternative to using [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime). diff --git a/src/libutil/experimental-features/nix-command.md b/src/libutil/experimental-features/nix-command.md new file mode 100644 index 000000000..7cfe6a97f --- /dev/null +++ b/src/libutil/experimental-features/nix-command.md @@ -0,0 +1,6 @@ +--- +name: nix-command +internalName: NixCommand +--- +Enable the new `nix` subcommands. See the manual on +[`nix`](@docroot@/command-ref/new-cli/nix.md) for details. diff --git a/src/libutil/experimental-features/no-url-literals.md b/src/libutil/experimental-features/no-url-literals.md new file mode 100644 index 000000000..2c183900e --- /dev/null +++ b/src/libutil/experimental-features/no-url-literals.md @@ -0,0 +1,39 @@ +--- +name: no-url-literals +internalName: NoUrlLiterals +--- +Disallow unquoted URLs as part of the Nix language syntax. The Nix +language allows for URL literals, like so: + +``` +$ nix repl +Welcome to Nix 2.15.0. Type :? for help. + +nix-repl> http://foo +"http://foo" +``` + +But enabling this experimental feature will cause the Nix parser to +throw an error when encountering a URL literal: + +``` +$ nix repl --extra-experimental-features 'no-url-literals' +Welcome to Nix 2.15.0. Type :? for help. + +nix-repl> http://foo +error: URL literals are disabled + +at «string»:1:1: + +1| http://foo + | ^ + +``` + +While this is currently an experimental feature, unquoted URLs are +being deprecated and their usage is discouraged. + +The reason is that, as opposed to path literals, URLs have no +special properties that distinguish them from regular strings, URLs +containing parameters have to be quoted anyway, and unquoted URLs +may confuse external tooling. diff --git a/src/libutil/experimental-features/parse-toml-timestamps.md b/src/libutil/experimental-features/parse-toml-timestamps.md new file mode 100644 index 000000000..9dbfb70d0 --- /dev/null +++ b/src/libutil/experimental-features/parse-toml-timestamps.md @@ -0,0 +1,5 @@ +--- +name: parse-toml-timestamps +internalName: ParseTomlTimestamps +--- +Allow parsing of timestamps in builtins.fromTOML. diff --git a/src/libutil/experimental-features/pipe-operator.md b/src/libutil/experimental-features/pipe-operator.md new file mode 100644 index 000000000..c9f26ffa3 --- /dev/null +++ b/src/libutil/experimental-features/pipe-operator.md @@ -0,0 +1,8 @@ +--- +name: pipe-operator +internalName: PipeOperator +--- +Enable new operators for function application to "pipe" arguments through a chain of functions similar to `lib.pipe`. +This implementation is based on Nix [RFC 148](https://github.com/NixOS/rfcs/pull/148). + +Tracking issue: https://git.lix.systems/lix-project/lix/issues/438 diff --git a/src/libutil/experimental-features/read-only-local-store.md b/src/libutil/experimental-features/read-only-local-store.md new file mode 100644 index 000000000..700618765 --- /dev/null +++ b/src/libutil/experimental-features/read-only-local-store.md @@ -0,0 +1,5 @@ +--- +name: read-only-local-store +internalName: ReadOnlyLocalStore +--- +Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. diff --git a/src/libutil/experimental-features/recursive-nix.md b/src/libutil/experimental-features/recursive-nix.md new file mode 100644 index 000000000..fd67ed5aa --- /dev/null +++ b/src/libutil/experimental-features/recursive-nix.md @@ -0,0 +1,39 @@ +--- +name: recursive-nix +internalName: RecursiveNix +--- +Allow derivation builders to call Nix, and thus build derivations +recursively. + +Example: + +``` +with import {}; + +runCommand "foo" + { + buildInputs = [ nix jq ]; + NIX_PATH = "nixpkgs=${}"; + } + '' + hello=$(nix-build -E '(import {}).hello.overrideDerivation (args: { name = "recursive-hello"; })') + + mkdir -p $out/bin + ln -s $hello/bin/hello $out/bin/hello + '' +``` + +An important restriction on recursive builders is disallowing +arbitrary substitutions. For example, running + +``` +nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10 +``` + +in the above `runCommand` script would be disallowed, as this could +lead to derivations with hidden dependencies or breaking +reproducibility by relying on the current state of the Nix store. An +exception would be if +`/nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10` were +already in the build inputs or built by a previous recursive Nix +call. diff --git a/src/libutil/experimental-features/repl-automation.md b/src/libutil/experimental-features/repl-automation.md new file mode 100644 index 000000000..c21cd4478 --- /dev/null +++ b/src/libutil/experimental-features/repl-automation.md @@ -0,0 +1,5 @@ +--- +name: repl-automation +internalName: ReplAutomation +--- +Makes the repl not use editline, print ENQ (U+0005) when ready for a command, and take commands followed by newline. diff --git a/src/libutil/experimental-features/repl-flake.md b/src/libutil/experimental-features/repl-flake.md new file mode 100644 index 000000000..76d910bbf --- /dev/null +++ b/src/libutil/experimental-features/repl-flake.md @@ -0,0 +1,5 @@ +--- +name: repl-flake +internalName: ReplFlake +--- +Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands. diff --git a/src/libutil/meson.build b/src/libutil/meson.build index afca4e021..ca75df2d5 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -130,9 +130,99 @@ libutil_headers = files( 'xml-writer.hh', ) +experimental_feature_definitions = files( + 'experimental-features/auto-allocate-uids.md', + 'experimental-features/ca-derivations.md', + 'experimental-features/cgroups.md', + 'experimental-features/daemon-trust-override.md', + 'experimental-features/dynamic-derivations.md', + 'experimental-features/fetch-closure.md', + 'experimental-features/flakes.md', + 'experimental-features/impure-derivations.md', + 'experimental-features/nix-command.md', + 'experimental-features/no-url-literals.md', + 'experimental-features/parse-toml-timestamps.md', + 'experimental-features/pipe-operator.md', + 'experimental-features/read-only-local-store.md', + 'experimental-features/recursive-nix.md', + 'experimental-features/repl-automation.md', + 'experimental-features/repl-flake.md', +) + +deprecated_feature_definitions = files( + 'deprecated-features/ancient-let.md', + 'deprecated-features/rec-set-overrides.md', + 'deprecated-features/url-literals.md', +) + +experimental_features_gen = custom_target( + command : [ + python.full_path(), + '@SOURCE_ROOT@/src/code-generation/build_experimental_features.py', + '--header', '@OUTPUT0@', + '--impl-header', '@OUTPUT1@', + '--shortlist', '@OUTPUT2@', + '--descriptions', '@OUTPUT3@', + '@INPUT@', + ], + input : experimental_feature_definitions, + output : [ + 'experimental-features.gen.inc', + 'experimental-features-impl.gen.inc', + 'experimental-features-shortlist.md', + 'experimental-feature-descriptions.md', + ], + install : true, + install_dir: [ + includedir / 'lix/libutil', + false, + false, + false, + ], +) +experimental_features_header = experimental_features_gen[0] +experimental_features_impl_header = experimental_features_gen[1] +experimental_features_shortlist_md = experimental_features_gen[2] +experimental_feature_descriptions_md = experimental_features_gen[3] + +deprecated_features_gen = custom_target( + command : [ + python.full_path(), + '@SOURCE_ROOT@/src/code-generation/build_experimental_features.py', + '--deprecated', + '--header', '@OUTPUT0@', + '--impl-header', '@OUTPUT1@', + '--shortlist', '@OUTPUT2@', + '--descriptions', '@OUTPUT3@', + '@INPUT@', + ], + input : deprecated_feature_definitions, + output : [ + 'deprecated-features.gen.inc', + 'deprecated-features-impl.gen.inc', + 'deprecated-features-shortlist.md', + 'deprecated-feature-descriptions.md', + ], + install : true, + install_dir: [ + includedir / 'lix/libutil', + false, + false, + false + ], +) +deprecated_features_header = deprecated_features_gen[0] +deprecated_features_impl_header = deprecated_features_gen[1] +deprecated_features_shortlist_md = deprecated_features_gen[2] +deprecated_feature_descriptions_md = deprecated_features_gen[3] + libutil = library( 'lixutil', libutil_sources, + experimental_features_header, + experimental_features_impl_header, + deprecated_features_header, + deprecated_features_impl_header, dependencies : [ aws_sdk, aws_s3, @@ -169,6 +259,10 @@ configure_file( # Used by libstore and libfetchers. liblixutil = declare_dependency( include_directories : include_directories('.'), + sources : [ + experimental_features_header, + deprecated_features_header, + ], link_with : libutil ) @@ -176,6 +270,10 @@ liblixutil = declare_dependency( if is_static liblixutil_mstatic = declare_dependency( include_directories : include_directories('.'), + sources : [ + experimental_features_header, + deprecated_features_header, + ], link_whole : libutil, ) else