diff --git a/.gitignore b/.gitignore index 15c871218..146d433a1 100644 --- a/.gitignore +++ b/.gitignore @@ -37,5 +37,5 @@ buildtime.bin # Rust build files when using Cargo (not actually supported for building but it spews the files anyway) /target/ -# Python compiled files from the test suite +# Python compiled files from the code generators and test suite *.pyc diff --git a/doc/manual/generate-features-shortlist.nix b/doc/manual/generate-features-shortlist.nix deleted file mode 100644 index 055698d64..000000000 --- a/doc/manual/generate-features-shortlist.nix +++ /dev/null @@ -1,14 +0,0 @@ -# Usually "experimental" or "deprecated" -kind: -# "xp" or "dp" -kindShort: - -with builtins; -with import ./utils.nix; - -let - showExperimentalFeature = name: doc: '' - - [`${name}`](@docroot@/contributing/${kind}-features.md#${kindShort}-feature-${name}) - ''; -in -xps: indent " " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/generate-features.nix b/doc/manual/generate-features.nix deleted file mode 100644 index 4a12ccdce..000000000 --- a/doc/manual/generate-features.nix +++ /dev/null @@ -1,18 +0,0 @@ -# Usually "experimental" or "deprecated" -_kind: -# "xp" or "dp" -kindShort: - -with builtins; -with import ./utils.nix; - -let - showFeature = - name: doc: - squash '' - ## [`${name}`]{#${kindShort}-feature-${name}} - - ${doc} - ''; -in -xps: (concatStringsSep "\n" (attrValues (mapAttrs showFeature xps))) diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 35d94740c..02383f6bb 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -48,14 +48,6 @@ nix_conf_file_md = custom_target( output : 'conf-file.md', ) -nix_exp_features_json = custom_target( - command : [ nix, '__dump-xp-features' ], - capture : true, - output : 'xp-features.json', - # FIXME: put the actual lib targets in here? meson have introspection challenge 2024 though. - build_always_stale : true, -) - language_json = custom_target( command: [nix, '__dump-language'], output : 'language.json', @@ -80,10 +72,8 @@ generate_manual_deps = files( # Generates builtins.md and builtin-constants.md. subdir('src/language') -# Generates new-cli pages, {experimental,deprecated}-features-shortlist.md, and conf-file.md. +# Generates new-cli pages and conf-file.md. subdir('src/command-ref') -# Generates {experimental,deprecated}-feature-descriptions.md. -subdir('src/contributing') # Generates rl-next-generated.md. subdir('src/release-notes') diff --git a/doc/manual/src/command-ref/meson.build b/doc/manual/src/command-ref/meson.build index f66ee0d39..6acc79f4e 100644 --- a/doc/manual/src/command-ref/meson.build +++ b/doc/manual/src/command-ref/meson.build @@ -1,37 +1,3 @@ -experimental_features_shortlist_md = custom_target( - command : nix_eval_for_docs + [ - '--expr', - 'import @INPUT0@ "experimental" "xp" (builtins.fromJSON (builtins.readFile @INPUT1@))', - ], - input : [ - '../../generate-features-shortlist.nix', - nix_exp_features_json, - ], - capture : true, - output : 'experimental-features-shortlist.md', - env : nix_env_for_docs, -) - -dp_features_json = custom_target( - command : [nix, '__dump-dp-features'], - capture : true, - output : 'dp-features.json', -) - -deprecated_features_shortlist_md = custom_target( - command : nix_eval_for_docs + [ - '--expr', - 'import @INPUT0@ "deprecated" "dp" (builtins.fromJSON (builtins.readFile @INPUT1@))', - ], - input : [ - '../../generate-features-shortlist.nix', - dp_features_json, - ], - capture : true, - output : 'deprecated-features-shortlist.md', - env : nix_env_for_docs, -) - # Intermediate step for manpage generation. # This splorks the output of generate-manpage.nix as JSON, # which gets written as a directory tree below. diff --git a/doc/manual/src/contributing/deprecated-features.md b/doc/manual/src/contributing/deprecated-features.md index 7536944f7..ef37a1d98 100644 --- a/doc/manual/src/contributing/deprecated-features.md +++ b/doc/manual/src/contributing/deprecated-features.md @@ -34,4 +34,4 @@ However, we do not live in such an ideal world, and currently this goal is so fa # Currently available deprecated features -{{#include @generated@/contributing/deprecated-feature-descriptions.md}} +{{#include @generated@/../../../src/libutil/deprecated-feature-descriptions.md}} diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/contributing/experimental-features.md index c27ba5da0..d5f6f2526 100644 --- a/doc/manual/src/contributing/experimental-features.md +++ b/doc/manual/src/contributing/experimental-features.md @@ -99,4 +99,4 @@ This means that experimental features and RFCs are orthogonal mechanisms, and ca # Currently available experimental features -{{#include @generated@/contributing/experimental-feature-descriptions.md}} +{{#include @generated@/../../../src/libutil/experimental-feature-descriptions.md}} diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 37c96a5c0..d245e20d1 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -399,3 +399,17 @@ The following properties are supported: Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly. + +## Adding experimental or deprecated features + +Experimental and deprecated features are generally referenced both in the code and in the documentation. +To prevent duplication or divergence, they are defined in data files, and a script generates the necessary glue. + +The data file format is similar to the release notes: it consists of a YAML metadata header, followed by the documentation in Markdown format. +The following metadata properties are supported: +* `name` (required): user-facing name of the feature, to be used in `nix.conf` options and on the command line. + This should also be the stem of the file name (with extension `md`). +* `internalName` (required): identifier used to refer to the feature inside the C++ code. + +Experimental feature data files should live in `src/libutil/experimental-features`, and deprecated features in `src/libutil/deprecated-features`. +They must be listed in the `experimental_feature_definitions` or `deprecated_feature_definitions` lists in `src/libutil/meson.build` respectively to be considered by the build system. diff --git a/doc/manual/src/contributing/meson.build b/doc/manual/src/contributing/meson.build deleted file mode 100644 index da88ca146..000000000 --- a/doc/manual/src/contributing/meson.build +++ /dev/null @@ -1,28 +0,0 @@ -# Intermediate step for experimental-feature-descriptions.md. -# This splorks the output of generate-xp-features.nix as JSON, -# which gets written as a directory tree below. -experimental_feature_descriptions_md = custom_target( - command : nix_eval_for_docs + [ - '--expr', - 'import @INPUT0@ "experimental" "xp" (builtins.fromJSON (builtins.readFile @INPUT1@))', - ], - input : [ - '../../generate-features.nix', - nix_exp_features_json, - ], - capture : true, - output : 'experimental-feature-descriptions.md', -) - -deprecated_feature_descriptions_md = custom_target( - command : nix_eval_for_docs + [ - '--expr', - 'import @INPUT0@ "deprecated" "dp" (builtins.fromJSON (builtins.readFile @INPUT1@))', - ], - input : [ - '../../generate-features.nix', - dp_features_json, - ], - capture : true, - output : 'deprecated-feature-descriptions.md', -) diff --git a/meson/clang-tidy/build_required_targets.py b/meson/clang-tidy/build_required_targets.py index 5c0e9641e..f26aa24a5 100755 --- a/meson/clang-tidy/build_required_targets.py +++ b/meson/clang-tidy/build_required_targets.py @@ -14,7 +14,8 @@ def main(): args = ap.parse_args() - targets = [t for t in get_targets_of_rule(args.build_root, 'CUSTOM_COMMAND') if t.endswith('gen.hh')] + targets = [t for t in get_targets_of_rule(args.build_root, 'CUSTOM_COMMAND') if t.endswith('.gen.hh')] + targets += [t for t in get_targets_of_rule(args.build_root, 'CUSTOM_COMMAND') if t.endswith('.gen.inc')] ninja_build(args.build_root, targets) if __name__ == '__main__': diff --git a/package.nix b/package.nix index 5c66b8446..297bc01fc 100644 --- a/package.nix +++ b/package.nix @@ -245,9 +245,11 @@ stdenv.mkDerivation (finalAttrs: { nativeBuildInputs = [ + # python3.withPackages does not splice properly, see https://github.com/NixOS/nixpkgs/issues/305858 (python3.pythonOnBuildForHost.withPackages (p: [ p.pytest p.pytest-xdist + p.python-frontmatter ])) meson ninja diff --git a/src/code-generation/build_experimental_features.py b/src/code-generation/build_experimental_features.py new file mode 100644 index 000000000..8768d826e --- /dev/null +++ b/src/code-generation/build_experimental_features.py @@ -0,0 +1,58 @@ +from typing import NamedTuple + +from common import cxx_literal, generate_file, load_data + +KNOWN_KEYS = set([ + 'name', + 'internalName', +]) + +class ExperimentalFeature(NamedTuple): + name: str + internal_name: str + description: str + + def parse(datum): + unknown_keys = set(datum.keys()) - KNOWN_KEYS + if unknown_keys: + raise ValueError('unknown keys', unknown_keys) + return ExperimentalFeature( + name = datum['name'], + internal_name = datum['internalName'], + description = datum.content, + ) + +def main(): + import argparse + + ap = argparse.ArgumentParser() + ap.add_argument('--deprecated', action='store_true', help='Generate deprecated features') + ap.add_argument('--header', help='Path of the declaration header to generate') + ap.add_argument('--impl-header', help='Path of the implementation header to generate') + ap.add_argument('--descriptions', help='Path of the description file to generate') + ap.add_argument('--shortlist', help='Path of the shortlist file to generate') + ap.add_argument('defs', help='Experimental feature definition files', nargs='+') + args = ap.parse_args() + + features = load_data(args.defs, ExperimentalFeature.parse) + + generate_file(args.header, features, lambda feature: feature.name, lambda feature: + f' {feature.internal_name},\n') + generate_file(args.impl_header, features, lambda feature: feature.name, lambda feature: + f''' {{ + .tag = {"Dep" if args.deprecated else "Xp"}::{feature.internal_name}, + .name = {cxx_literal(feature.name)}, + .description = {cxx_literal(feature.description)}, + }}, +''') + generate_file(args.descriptions, features, lambda feature: feature.name, lambda feature: + f'''## [`{feature.name}`]{{#{"dp" if args.deprecated else "xp"}-feature-{feature.name}}} + +{feature.description} + +''') + generate_file(args.shortlist, features, lambda feature: feature.name, lambda feature: + f' - [`{feature.name}`](@docroot@/contributing/{"deprecated" if args.deprecated else "experimental"}-features.md#{"dp" if args.deprecated else "xp"}-feature-{feature.name})\n') + +if __name__ == '__main__': + main() diff --git a/src/code-generation/common.py b/src/code-generation/common.py new file mode 100644 index 000000000..228359c78 --- /dev/null +++ b/src/code-generation/common.py @@ -0,0 +1,50 @@ +import frontmatter +import pathlib +from collections import defaultdict + +def cxx_escape_character(c): + if ord(c) >= 0x20 and ord(c) < 0x7f and c != '"' and c != '?' and c != '\\': + return c + elif c == '\t': + return r'\t' + elif c == '\n': + return r'\n' + elif c == '\r': + return r'\r' + elif c == '"': + return r'\"' + elif c == '?': + return r'\?' + elif c == '\\': + return r'\\' + elif ord(c) <= 0xffff: + return str.format(r'\u{:04x}', ord(c)) + else: + return str.format(r'\U{:08x}', ord(c)) + +def cxx_literal(v): + if isinstance(v, str): + return ''.join(['"', *(cxx_escape_character(c) for c in v), '"']) + else: + raise NotImplementedError(f'cannot represent {repr(v)} in C++') + +def load_data(defs, parse_function): + data = [] + for path in defs: + try: + datum = frontmatter.load(path) + data.append((path, parse_function(datum))) + except Exception as e: + e.add_note(f'in {path}') + raise + return data + +def generate_file(path, data, sort_key_function, generate_function): + if path is not None: + with open(path, 'w') as out: + for path, datum in sorted(data, key=lambda pathAndDatum: sort_key_function(pathAndDatum[1])): + try: + out.write(generate_function(datum)) + except Exception as e: + e.add_note(f'in {path}') + raise diff --git a/src/libutil/config.hh b/src/libutil/config.hh index b3dcc122f..eadc71926 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -410,7 +410,7 @@ struct FeatureSettings : Config { The following experimental features are available: - {{#include @generated@/command-ref/experimental-features-shortlist.md}} + {{#include @generated@/../../../src/libutil/experimental-features-shortlist.md}} Experimental features are [further documented in the manual](@docroot@/contributing/experimental-features.md). )"}; @@ -450,7 +450,7 @@ struct FeatureSettings : Config { The following deprecated feature features can be re-activated: - {{#include @generated@/command-ref/deprecated-features-shortlist.md}} + {{#include @generated@/../../../src/libutil/deprecated-features-shortlist.md}} Deprecated features are [further documented in the manual](@docroot@/contributing/deprecated-features.md). )"}; diff --git a/src/libutil/deprecated-features.cc b/src/libutil/deprecated-features.cc index 4de4c8ec7..65a003cd4 100644 --- a/src/libutil/deprecated-features.cc +++ b/src/libutil/deprecated-features.cc @@ -20,38 +20,13 @@ struct DeprecatedFeatureDetails * counter will be incremented once instead of twice, causing a build * failure. * - * By instead defining this instead as 1 + the bottom deprecated - * feature, we either have no issue at all if few features are not added - * at the end of the list, or a proper merge conflict if they are. + * By instead defining this instead as a dummy bottom deprecated + * feature, we do not have this issue. */ -constexpr size_t numDepFeatures = 1 + static_cast(Dep::UrlLiterals); +constexpr size_t numDepFeatures = static_cast(Dep::NumDepFeatures); constexpr std::array depFeatureDetails = {{ - { - .tag = Dep::RecSetOverrides, - .name = "rec-set-overrides", - .description = R"( - Allow `__overrides` in recursive attribute sets. - - Use fix point functions (e.g. `lib.fix` in Nixpkgs) instead. - )", - }, - { - .tag = Dep::AncientLet, - .name = "ancient-let", - .description = R"( - Allow the ancient `let { body = …; … }` syntax. - - Use the `let … in` syntax instead. - )", - }, - { - .tag = Dep::UrlLiterals, - .name = "url-literals", - .description = R"( - Allow unquoted URLs as part of the Nix language syntax. - )", - }, + #include "deprecated-features-impl.gen.inc" }}; static_assert( diff --git a/src/libutil/deprecated-features.hh b/src/libutil/deprecated-features.hh index bdff1bcdb..cd425e8f7 100644 --- a/src/libutil/deprecated-features.hh +++ b/src/libutil/deprecated-features.hh @@ -9,18 +9,13 @@ namespace nix { /** * The list of available deprecated features. * - * If you update this, don’t forget to also change the map defining - * their string representation and documentation in the corresponding - * `.cc` file as well. - * * Reminder: New deprecated features should start out with a warning without throwing an error. * See the developer documentation for details. */ enum struct DeprecatedFeature { - RecSetOverrides, - AncientLet, - UrlLiterals, + #include "deprecated-features.gen.inc" + NumDepFeatures, // number of available deprecated features, do not use }; enum struct DeprecatedFeatures {}; diff --git a/src/libutil/deprecated-features/ancient-let.md b/src/libutil/deprecated-features/ancient-let.md new file mode 100644 index 000000000..ced9934bb --- /dev/null +++ b/src/libutil/deprecated-features/ancient-let.md @@ -0,0 +1,7 @@ +--- +name: ancient-let +internalName: AncientLet +--- +Allow the ancient `let { body = …; … }` syntax. + +Use the `let … in` syntax instead. diff --git a/src/libutil/deprecated-features/rec-set-overrides.md b/src/libutil/deprecated-features/rec-set-overrides.md new file mode 100644 index 000000000..37d7003b1 --- /dev/null +++ b/src/libutil/deprecated-features/rec-set-overrides.md @@ -0,0 +1,7 @@ +--- +name: rec-set-overrides +internalName: RecSetOverrides +--- +Allow `__overrides` in recursive attribute sets. + +Use fix point functions (e.g. `lib.fix` in Nixpkgs) instead. diff --git a/src/libutil/deprecated-features/url-literals.md b/src/libutil/deprecated-features/url-literals.md new file mode 100644 index 000000000..99232913f --- /dev/null +++ b/src/libutil/deprecated-features/url-literals.md @@ -0,0 +1,5 @@ +--- +name: url-literals +internalName: UrlLiterals +--- +Allow unquoted URLs as part of the Nix language syntax. diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index bb7a95a24..da3e6c2cd 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -20,236 +20,13 @@ struct ExperimentalFeatureDetails * counter will be incremented once instead of twice, causing a build * failure. * - * By instead defining this instead as 1 + the bottom experimental - * feature, we either have no issue at all if few features are not added - * at the end of the list, or a proper merge conflict if they are. + * By instead defining this instead as a dummy bottom experimental + * feature, we do not get this issue. */ -constexpr size_t numXpFeatures = 1 + static_cast(Xp::ReplAutomation); +constexpr size_t numXpFeatures = static_cast(Xp::NumXpFeatures); constexpr std::array xpFeatureDetails = {{ - { - .tag = Xp::CaDerivations, - .name = "ca-derivations", - .description = R"( - Allow derivations to be content-addressed in order to prevent - rebuilds when changes to the derivation do not result in changes to - the derivation's output. See - [__contentAddressed](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed) - for details. - )", - }, - { - .tag = Xp::ImpureDerivations, - .name = "impure-derivations", - .description = R"( - Allow derivations to produce non-fixed outputs by setting the - `__impure` derivation attribute to `true`. An impure derivation can - have differing outputs each time it is built. - - Example: - - ``` - derivation { - name = "impure"; - builder = /bin/sh; - __impure = true; # mark this derivation as impure - args = [ "-c" "read -n 10 random < /dev/random; echo $random > $out" ]; - system = builtins.currentSystem; - } - ``` - - Each time this derivation is built, it can produce a different - output (as the builder outputs random bytes to `$out`). Impure - derivations also have access to the network, and only fixed-output - or other impure derivations can rely on impure derivations. Finally, - an impure derivation cannot also be - [content-addressed](#xp-feature-ca-derivations). - - This is a more explicit alternative to using [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime). - )", - }, - { - .tag = Xp::Flakes, - .name = "flakes", - .description = R"( - Enable flakes. See the manual entry for [`nix - flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details. - )", - }, - { - .tag = Xp::NixCommand, - .name = "nix-command", - .description = R"( - Enable the new `nix` subcommands. See the manual on - [`nix`](@docroot@/command-ref/new-cli/nix.md) for details. - )", - }, - { - .tag = Xp::RecursiveNix, - .name = "recursive-nix", - .description = R"( - Allow derivation builders to call Nix, and thus build derivations - recursively. - - Example: - - ``` - with import {}; - - runCommand "foo" - { - buildInputs = [ nix jq ]; - NIX_PATH = "nixpkgs=${}"; - } - '' - hello=$(nix-build -E '(import {}).hello.overrideDerivation (args: { name = "recursive-hello"; })') - - mkdir -p $out/bin - ln -s $hello/bin/hello $out/bin/hello - '' - ``` - - An important restriction on recursive builders is disallowing - arbitrary substitutions. For example, running - - ``` - nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10 - ``` - - in the above `runCommand` script would be disallowed, as this could - lead to derivations with hidden dependencies or breaking - reproducibility by relying on the current state of the Nix store. An - exception would be if - `/nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10` were - already in the build inputs or built by a previous recursive Nix - call. - )", - }, - { - .tag = Xp::NoUrlLiterals, - .name = "no-url-literals", - .description = R"( - Disallow unquoted URLs as part of the Nix language syntax. The Nix - language allows for URL literals, like so: - - ``` - $ nix repl - Welcome to Nix 2.15.0. Type :? for help. - - nix-repl> http://foo - "http://foo" - ``` - - But enabling this experimental feature will cause the Nix parser to - throw an error when encountering a URL literal: - - ``` - $ nix repl --extra-experimental-features 'no-url-literals' - Welcome to Nix 2.15.0. Type :? for help. - - nix-repl> http://foo - error: URL literals are disabled - - at «string»:1:1: - - 1| http://foo - | ^ - - ``` - - While this is currently an experimental feature, unquoted URLs are - being deprecated and their usage is discouraged. - - The reason is that, as opposed to path literals, URLs have no - special properties that distinguish them from regular strings, URLs - containing parameters have to be quoted anyway, and unquoted URLs - may confuse external tooling. - )", - }, - { - .tag = Xp::PipeOperator, - .name = "pipe-operator", - .description = R"( - Enable new operators for function application to "pipe" arguments through a chain of functions similar to `lib.pipe`. - This implementation is based on Nix [RFC 148](https://github.com/NixOS/rfcs/pull/148). - - Tracking issue: https://git.lix.systems/lix-project/lix/issues/438 - )", - }, - { - .tag = Xp::FetchClosure, - .name = "fetch-closure", - .description = R"( - Enable the use of the [`fetchClosure`](@docroot@/language/builtins.md#builtins-fetchClosure) built-in function in the Nix language. - )", - }, - { - .tag = Xp::ReplFlake, - .name = "repl-flake", - .description = R"( - Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands. - )", - }, - { - .tag = Xp::AutoAllocateUids, - .name = "auto-allocate-uids", - .description = R"( - Allows Nix to automatically pick UIDs for builds, rather than creating - `nixbld*` user accounts. See the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting for details. - )", - }, - { - .tag = Xp::Cgroups, - .name = "cgroups", - .description = R"( - Allows Nix to execute builds inside cgroups. See - the [`use-cgroups`](#conf-use-cgroups) setting for details. - )", - }, - { - .tag = Xp::DaemonTrustOverride, - .name = "daemon-trust-override", - .description = R"( - Allow forcing trusting or not trusting clients with - `nix-daemon`. This is useful for testing, but possibly also - useful for various experiments with `nix-daemon --stdio` - networking. - )", - }, - { - .tag = Xp::DynamicDerivations, - .name = "dynamic-derivations", - .description = R"( - Allow the use of a few things related to dynamic derivations: - - - "text hashing" derivation outputs, so we can build .drv - files. - - - dependencies in derivations on the outputs of - derivations that are themselves derivations outputs. - )", - }, - { - .tag = Xp::ParseTomlTimestamps, - .name = "parse-toml-timestamps", - .description = R"( - Allow parsing of timestamps in builtins.fromTOML. - )", - }, - { - .tag = Xp::ReadOnlyLocalStore, - .name = "read-only-local-store", - .description = R"( - Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. - )", - }, - { - .tag = Xp::ReplAutomation, - .name = "repl-automation", - .description = R"( - Makes the repl not use editline, print ENQ (U+0005) when ready for a command, and take commands followed by newline. - )", - }, + #include "experimental-features-impl.gen.inc" }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index c45f05d78..3241ed955 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -8,29 +8,11 @@ namespace nix { /** * The list of available experimental features. - * - * If you update this, don’t forget to also change the map defining - * their string representation and documentation in the corresponding - * `.cc` file as well. */ enum struct ExperimentalFeature { - CaDerivations, - ImpureDerivations, - Flakes, - NixCommand, - RecursiveNix, - NoUrlLiterals, - PipeOperator, - FetchClosure, - ReplFlake, - AutoAllocateUids, - Cgroups, - DaemonTrustOverride, - DynamicDerivations, - ParseTomlTimestamps, - ReadOnlyLocalStore, - ReplAutomation, + #include "experimental-features.gen.inc" + NumXpFeatures, // number of available experimental features, do not use }; enum struct ExperimentalFeatures {}; diff --git a/src/libutil/experimental-features/auto-allocate-uids.md b/src/libutil/experimental-features/auto-allocate-uids.md new file mode 100644 index 000000000..af7a2b68d --- /dev/null +++ b/src/libutil/experimental-features/auto-allocate-uids.md @@ -0,0 +1,6 @@ +--- +name: auto-allocate-uids +internalName: AutoAllocateUids +--- +Allows Nix to automatically pick UIDs for builds, rather than creating +`nixbld*` user accounts. See the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting for details. diff --git a/src/libutil/experimental-features/ca-derivations.md b/src/libutil/experimental-features/ca-derivations.md new file mode 100644 index 000000000..423666c54 --- /dev/null +++ b/src/libutil/experimental-features/ca-derivations.md @@ -0,0 +1,9 @@ +--- +name: ca-derivations +internalName: CaDerivations +--- +Allow derivations to be content-addressed in order to prevent +rebuilds when changes to the derivation do not result in changes to +the derivation's output. See +[__contentAddressed](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed) +for details. diff --git a/src/libutil/experimental-features/cgroups.md b/src/libutil/experimental-features/cgroups.md new file mode 100644 index 000000000..66e64fde9 --- /dev/null +++ b/src/libutil/experimental-features/cgroups.md @@ -0,0 +1,6 @@ +--- +name: cgroups +internalName: Cgroups +--- +Allows Nix to execute builds inside cgroups. See +the [`use-cgroups`](#conf-use-cgroups) setting for details. diff --git a/src/libutil/experimental-features/daemon-trust-override.md b/src/libutil/experimental-features/daemon-trust-override.md new file mode 100644 index 000000000..e87f31dd3 --- /dev/null +++ b/src/libutil/experimental-features/daemon-trust-override.md @@ -0,0 +1,8 @@ +--- +name: daemon-trust-override +internalName: DaemonTrustOverride +--- +Allow forcing trusting or not trusting clients with +`nix-daemon`. This is useful for testing, but possibly also +useful for various experiments with `nix-daemon --stdio` +networking. diff --git a/src/libutil/experimental-features/dynamic-derivations.md b/src/libutil/experimental-features/dynamic-derivations.md new file mode 100644 index 000000000..79201bcc3 --- /dev/null +++ b/src/libutil/experimental-features/dynamic-derivations.md @@ -0,0 +1,11 @@ +--- +name: dynamic-derivations +internalName: DynamicDerivations +--- +Allow the use of a few things related to dynamic derivations: + + - "text hashing" derivation outputs, so we can build .drv + files. + + - dependencies in derivations on the outputs of + derivations that are themselves derivations outputs. diff --git a/src/libutil/experimental-features/fetch-closure.md b/src/libutil/experimental-features/fetch-closure.md new file mode 100644 index 000000000..45692f030 --- /dev/null +++ b/src/libutil/experimental-features/fetch-closure.md @@ -0,0 +1,5 @@ +--- +name: fetch-closure +internalName: FetchClosure +--- +Enable the use of the [`fetchClosure`](@docroot@/language/builtins.md#builtins-fetchClosure) built-in function in the Nix language. diff --git a/src/libutil/experimental-features/flakes.md b/src/libutil/experimental-features/flakes.md new file mode 100644 index 000000000..3cb1c74a0 --- /dev/null +++ b/src/libutil/experimental-features/flakes.md @@ -0,0 +1,6 @@ +--- +name: flakes +internalName: Flakes +--- +Enable flakes. See the manual entry for [`nix +flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details. diff --git a/src/libutil/experimental-features/impure-derivations.md b/src/libutil/experimental-features/impure-derivations.md new file mode 100644 index 000000000..f2ab9f5f2 --- /dev/null +++ b/src/libutil/experimental-features/impure-derivations.md @@ -0,0 +1,28 @@ +--- +name: impure-derivations +internalName: ImpureDerivations +--- +Allow derivations to produce non-fixed outputs by setting the +`__impure` derivation attribute to `true`. An impure derivation can +have differing outputs each time it is built. + +Example: + +``` +derivation { + name = "impure"; + builder = /bin/sh; + __impure = true; # mark this derivation as impure + args = [ "-c" "read -n 10 random < /dev/random; echo $random > $out" ]; + system = builtins.currentSystem; +} +``` + +Each time this derivation is built, it can produce a different +output (as the builder outputs random bytes to `$out`). Impure +derivations also have access to the network, and only fixed-output +or other impure derivations can rely on impure derivations. Finally, +an impure derivation cannot also be +[content-addressed](#xp-feature-ca-derivations). + +This is a more explicit alternative to using [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime). diff --git a/src/libutil/experimental-features/nix-command.md b/src/libutil/experimental-features/nix-command.md new file mode 100644 index 000000000..7cfe6a97f --- /dev/null +++ b/src/libutil/experimental-features/nix-command.md @@ -0,0 +1,6 @@ +--- +name: nix-command +internalName: NixCommand +--- +Enable the new `nix` subcommands. See the manual on +[`nix`](@docroot@/command-ref/new-cli/nix.md) for details. diff --git a/src/libutil/experimental-features/no-url-literals.md b/src/libutil/experimental-features/no-url-literals.md new file mode 100644 index 000000000..2c183900e --- /dev/null +++ b/src/libutil/experimental-features/no-url-literals.md @@ -0,0 +1,39 @@ +--- +name: no-url-literals +internalName: NoUrlLiterals +--- +Disallow unquoted URLs as part of the Nix language syntax. The Nix +language allows for URL literals, like so: + +``` +$ nix repl +Welcome to Nix 2.15.0. Type :? for help. + +nix-repl> http://foo +"http://foo" +``` + +But enabling this experimental feature will cause the Nix parser to +throw an error when encountering a URL literal: + +``` +$ nix repl --extra-experimental-features 'no-url-literals' +Welcome to Nix 2.15.0. Type :? for help. + +nix-repl> http://foo +error: URL literals are disabled + +at «string»:1:1: + +1| http://foo + | ^ + +``` + +While this is currently an experimental feature, unquoted URLs are +being deprecated and their usage is discouraged. + +The reason is that, as opposed to path literals, URLs have no +special properties that distinguish them from regular strings, URLs +containing parameters have to be quoted anyway, and unquoted URLs +may confuse external tooling. diff --git a/src/libutil/experimental-features/parse-toml-timestamps.md b/src/libutil/experimental-features/parse-toml-timestamps.md new file mode 100644 index 000000000..9dbfb70d0 --- /dev/null +++ b/src/libutil/experimental-features/parse-toml-timestamps.md @@ -0,0 +1,5 @@ +--- +name: parse-toml-timestamps +internalName: ParseTomlTimestamps +--- +Allow parsing of timestamps in builtins.fromTOML. diff --git a/src/libutil/experimental-features/pipe-operator.md b/src/libutil/experimental-features/pipe-operator.md new file mode 100644 index 000000000..c9f26ffa3 --- /dev/null +++ b/src/libutil/experimental-features/pipe-operator.md @@ -0,0 +1,8 @@ +--- +name: pipe-operator +internalName: PipeOperator +--- +Enable new operators for function application to "pipe" arguments through a chain of functions similar to `lib.pipe`. +This implementation is based on Nix [RFC 148](https://github.com/NixOS/rfcs/pull/148). + +Tracking issue: https://git.lix.systems/lix-project/lix/issues/438 diff --git a/src/libutil/experimental-features/read-only-local-store.md b/src/libutil/experimental-features/read-only-local-store.md new file mode 100644 index 000000000..700618765 --- /dev/null +++ b/src/libutil/experimental-features/read-only-local-store.md @@ -0,0 +1,5 @@ +--- +name: read-only-local-store +internalName: ReadOnlyLocalStore +--- +Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. diff --git a/src/libutil/experimental-features/recursive-nix.md b/src/libutil/experimental-features/recursive-nix.md new file mode 100644 index 000000000..fd67ed5aa --- /dev/null +++ b/src/libutil/experimental-features/recursive-nix.md @@ -0,0 +1,39 @@ +--- +name: recursive-nix +internalName: RecursiveNix +--- +Allow derivation builders to call Nix, and thus build derivations +recursively. + +Example: + +``` +with import {}; + +runCommand "foo" + { + buildInputs = [ nix jq ]; + NIX_PATH = "nixpkgs=${}"; + } + '' + hello=$(nix-build -E '(import {}).hello.overrideDerivation (args: { name = "recursive-hello"; })') + + mkdir -p $out/bin + ln -s $hello/bin/hello $out/bin/hello + '' +``` + +An important restriction on recursive builders is disallowing +arbitrary substitutions. For example, running + +``` +nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10 +``` + +in the above `runCommand` script would be disallowed, as this could +lead to derivations with hidden dependencies or breaking +reproducibility by relying on the current state of the Nix store. An +exception would be if +`/nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10` were +already in the build inputs or built by a previous recursive Nix +call. diff --git a/src/libutil/experimental-features/repl-automation.md b/src/libutil/experimental-features/repl-automation.md new file mode 100644 index 000000000..c21cd4478 --- /dev/null +++ b/src/libutil/experimental-features/repl-automation.md @@ -0,0 +1,5 @@ +--- +name: repl-automation +internalName: ReplAutomation +--- +Makes the repl not use editline, print ENQ (U+0005) when ready for a command, and take commands followed by newline. diff --git a/src/libutil/experimental-features/repl-flake.md b/src/libutil/experimental-features/repl-flake.md new file mode 100644 index 000000000..76d910bbf --- /dev/null +++ b/src/libutil/experimental-features/repl-flake.md @@ -0,0 +1,5 @@ +--- +name: repl-flake +internalName: ReplFlake +--- +Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands. diff --git a/src/libutil/meson.build b/src/libutil/meson.build index afca4e021..ca75df2d5 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -130,9 +130,99 @@ libutil_headers = files( 'xml-writer.hh', ) +experimental_feature_definitions = files( + 'experimental-features/auto-allocate-uids.md', + 'experimental-features/ca-derivations.md', + 'experimental-features/cgroups.md', + 'experimental-features/daemon-trust-override.md', + 'experimental-features/dynamic-derivations.md', + 'experimental-features/fetch-closure.md', + 'experimental-features/flakes.md', + 'experimental-features/impure-derivations.md', + 'experimental-features/nix-command.md', + 'experimental-features/no-url-literals.md', + 'experimental-features/parse-toml-timestamps.md', + 'experimental-features/pipe-operator.md', + 'experimental-features/read-only-local-store.md', + 'experimental-features/recursive-nix.md', + 'experimental-features/repl-automation.md', + 'experimental-features/repl-flake.md', +) + +deprecated_feature_definitions = files( + 'deprecated-features/ancient-let.md', + 'deprecated-features/rec-set-overrides.md', + 'deprecated-features/url-literals.md', +) + +experimental_features_gen = custom_target( + command : [ + python.full_path(), + '@SOURCE_ROOT@/src/code-generation/build_experimental_features.py', + '--header', '@OUTPUT0@', + '--impl-header', '@OUTPUT1@', + '--shortlist', '@OUTPUT2@', + '--descriptions', '@OUTPUT3@', + '@INPUT@', + ], + input : experimental_feature_definitions, + output : [ + 'experimental-features.gen.inc', + 'experimental-features-impl.gen.inc', + 'experimental-features-shortlist.md', + 'experimental-feature-descriptions.md', + ], + install : true, + install_dir: [ + includedir / 'lix/libutil', + false, + false, + false, + ], +) +experimental_features_header = experimental_features_gen[0] +experimental_features_impl_header = experimental_features_gen[1] +experimental_features_shortlist_md = experimental_features_gen[2] +experimental_feature_descriptions_md = experimental_features_gen[3] + +deprecated_features_gen = custom_target( + command : [ + python.full_path(), + '@SOURCE_ROOT@/src/code-generation/build_experimental_features.py', + '--deprecated', + '--header', '@OUTPUT0@', + '--impl-header', '@OUTPUT1@', + '--shortlist', '@OUTPUT2@', + '--descriptions', '@OUTPUT3@', + '@INPUT@', + ], + input : deprecated_feature_definitions, + output : [ + 'deprecated-features.gen.inc', + 'deprecated-features-impl.gen.inc', + 'deprecated-features-shortlist.md', + 'deprecated-feature-descriptions.md', + ], + install : true, + install_dir: [ + includedir / 'lix/libutil', + false, + false, + false + ], +) +deprecated_features_header = deprecated_features_gen[0] +deprecated_features_impl_header = deprecated_features_gen[1] +deprecated_features_shortlist_md = deprecated_features_gen[2] +deprecated_feature_descriptions_md = deprecated_features_gen[3] + libutil = library( 'lixutil', libutil_sources, + experimental_features_header, + experimental_features_impl_header, + deprecated_features_header, + deprecated_features_impl_header, dependencies : [ aws_sdk, aws_s3, @@ -169,6 +259,10 @@ configure_file( # Used by libstore and libfetchers. liblixutil = declare_dependency( include_directories : include_directories('.'), + sources : [ + experimental_features_header, + deprecated_features_header, + ], link_with : libutil ) @@ -176,6 +270,10 @@ liblixutil = declare_dependency( if is_static liblixutil_mstatic = declare_dependency( include_directories : include_directories('.'), + sources : [ + experimental_features_header, + deprecated_features_header, + ], link_whole : libutil, ) else