docs: redo content generation for mdbook and manual

manpages can be rendered using the markdown output of mdbook, the rest
of the manual can generated out of the main doc/manual source tree. we
still use lowdown to actually render manpages instead of eg mdbook-man
because lowdown does generate reasonably good manpages (though that is
also somewhat debatable, but they're a lot better than mdbook-man).

doing this not only lets us drastically simplify the lowdown pipeline,
but also remove all custom {{#include}} handling since now mdbook does
all of it, even for the manpage builds. even the lowdown wrapper isn't
entirely necessary because lowdown can take all wrapper arguments with
command line flags rather than bits of input file content.

This also implements running mdbook in Meson, in order to generate the
manpages. The mdbook outputs are also installed in the usual location.

Co-authored-by: Qyriad <qyriad@qyriad.me>

Change-Id: I60193f9fd0f15d48872f071af35855cda2a0f40b

.gitignore

@@ -21,12 +21,8 @@ perl/Makefile.config
 /doc/manual/conf-file.json
 /doc/manual/language.json
 /doc/manual/xp-features.json
-/doc/manual/src/command-ref/new-cli
-/doc/manual/src/command-ref/conf-file.md
 /doc/manual/src/command-ref/experimental-features-shortlist.md
 /doc/manual/src/contributing/experimental-feature-descriptions.md
-/doc/manual/src/language/builtins.md
-/doc/manual/src/language/builtin-constants.md
 /doc/manual/src/release-notes/rl-next-generated.md
 
 # /scripts/

doc/manual/book.toml

@@ -7,20 +7,22 @@ additional-js = ["redirects.js"]
 edit-url-template = "https://github.com/NixOS/nix/tree/master/doc/manual/{path}"
 git-repository-url = "https://github.com/NixOS/nix"
 
-# Handles replacing @docroot@ with a path to ./src relative to that markdown file.
+# Handles replacing @docroot@ with a path to ./src relative to that markdown file,
-[preprocessor.docroot]
+# {{#include handlebars}}, and the @generated@ syntax used within these. it mostly
-renderers = ["html", "linkcheck"]
+# but not entirely replaces the links preprocessor (which we cannot simply use due
-command = "python3 doc/manual/docroot.py"
+# to @generated@ files living in a different directory to make meson happy). we do
-# I would have thought that @docroot@ replacement had to be done *before*
+# not want to disable the links preprocessor entirely though because that requires
-# the link preprocessor gets its hands on this book, but nope it's actually
+# disabling *all* built-in preprocessors and selectively reenabling those we want.
-# the opposite.
+[preprocessor.substitute]
-after = ["links"]
+command = "python3 doc/manual/substitute.py"
-before = ["anchors"]
+before = ["anchors", "links"]
 
 [preprocessor.anchors]
 renderers = ["html"]
 command = "jq --from-file doc/manual/anchors.jq"
 
+[output.markdown]
+
 [output.linkcheck]
 # no Internet during the build (in the sandbox)
 follow-web-links = false

doc/manual/docroot.py

@@ -1,84 +0,0 @@
-#!/usr/bin/env python3
-
-from pathlib import Path
-import json
-import os, os.path
-import sys
-
-name = 'process-docroot.py'
-
-def log(*args, **kwargs):
-    kwargs['file'] = sys.stderr
-    return print(f'{name}:', *args, **kwargs)
-
-def replace_docroot(relative_md_path: Path, content: str, book_root: Path):
-    assert not relative_md_path.is_absolute(), f'{relative_md_path=} from mdbook should be relative'
-
-    md_path_abs = book_root / relative_md_path
-    docroot_abs = md_path_abs.parent
-    assert docroot_abs.is_dir(), f'supposed docroot {docroot_abs} is not a directory (cwd={os.getcwd()})'
-
-    # The paths mdbook gives us are relative to the directory with book.toml.
-    # @docroot@ wants to be replaced with the path relative to `src/`.
-    docroot_rel = os.path.relpath(book_root / 'src', start=docroot_abs)
-
-    return content.replace('@docroot@', docroot_rel)
-
-def recursive_replace(data, book_root):
-    match data:
-        case {'sections': sections}:
-            return data | dict(
-                sections = [recursive_replace(section, book_root) for section in sections],
-            )
-        case {'Chapter': chapter}:
-            # Path to the .md file for this chapter, relative to book_root.
-            path_to_chapter = Path('src') / chapter['path']
-            chapter_content = chapter['content']
-
-            return data | dict(
-                Chapter = chapter | dict(
-                    content = replace_docroot(path_to_chapter, chapter_content, book_root),
-                    sub_items = [recursive_replace(sub_item, book_root) for sub_item in chapter['sub_items']],
-                ),
-            )
-
-        case rest:
-            assert False, f'should have been called on a dict, not {type(rest)=}\n\t{rest=}'
-
-def main():
-
-    if len(sys.argv) > 1 and sys.argv[1] == 'supports':
-        log('confirming to mdbook that we support their stuff')
-        return 0
-
-    # mdbook communicates with us over stdin and stdout.
-    # It splorks us a JSON array, the first element describing the context,
-    # the second element describing the book itself,
-    # and then expects us to send it the modified book JSON over stdout.
-
-    context, book = json.load(sys.stdin)
-
-    # book_root is *not* @docroot@. @docroot@ gets replaced with a relative path to `./src/`.
-    # book_root is the directory where book.toml, aka `src`'s parent.
-    book_root = Path(context['root'])
-    assert book_root.exists(), f'{book_root=} does not exist'
-    assert book_root.joinpath('book.toml').is_file(), f'{book_root / "book.toml"} is not a file'
-
-    log('replacing all occurrences of @docroot@ with a relative path')
-
-    # Find @docroot@ in all parts of our recursive book structure.
-    replaced_content = recursive_replace(book, book_root)
-
-    replaced_content_str = json.dumps(replaced_content)
-
-    # Give mdbook our changes.
-    print(replaced_content_str)
-
-    log('done!')
-
-try:
-    sys.exit(main())
-except AssertionError as e:
-    print(f'{name}: INTERNAL ERROR in mdbook preprocessor', file=sys.stderr)
-    print(f'this is a bug in {name}')
-    raise

doc/manual/generate-deps.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+import glob
+import sys
+
+# meson expects makefile-style dependency declarations, i.e.
+#
+#   target: dependency...
+#
+# meson seems to pass depfiles straight on to ninja even though
+# it also parses the file itself (or at least has code to do so
+# in its tree), so we must live by ninja's rules: only slashes,
+# spaces and octothorpes can be escaped, anything else is taken
+# literally. since the rules for these aren't even the same for
+# all three we will just fail when we encounter any of them (if
+# asserts are off for some reason the depfile will likely point
+# to nonexistant paths, making everything phony and thus fine.)
+for path in glob.glob(sys.argv[1] + '/**', recursive=True):
+    assert '\\' not in path
+    assert ' ' not in path
+    assert '#' not in path
+    print("ignored:", path)

doc/manual/local.mk

@@ -17,14 +17,14 @@ man-pages := $(foreach n, \
 	nix-hash.1 nix-copy-closure.1 \
 	nix.conf.5 nix-daemon.8 \
 	nix-profiles.5 \
-, $(d)/$(n))
+, doc/manual/generated/in/$(n))
 
 # man pages for subcommands
 # convert from `$(d)/src/command-ref/nix-{1}/{2}.md` to `$(d)/nix-{1}-{2}.1`
 # FIXME: unify with how nix3-cli man pages are generated
 man-pages += $(foreach subcommand, \
 	$(filter-out %opt-common.md %env-common.md, $(wildcard $(d)/src/command-ref/nix-*/*.md)), \
-	$(d)/$(subst /,-,$(subst $(d)/src/command-ref/,,$(subst .md,.1,$(subcommand)))))
+	doc/manual/generated/in/$(subst /,-,$(subst $(d)/src/command-ref/,,$(subst .md,.1,$(subcommand)))))
 
 clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
 
@@ -39,77 +39,91 @@ dummy-env = env -i \
 
 nix-eval = $(dummy-env) $(doc_nix) eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw
 
-$(d)/nix-env-%.1: $(d)/src/command-ref/nix-env/%.md
+doc/manual/generated/in/nix-env-%.1: doc/manual/generated/out
 	$(trace-gen) doc/manual/render-manpage.sh \
-		--out-no-smarty "$(subst nix-env-,nix-env --,$$(basename "$@" .1))" 1 $^ $^.tmp $@
+		--out-no-smarty "$(subst nix-env-,nix-env --,$$(basename "$@" .1))" 1 \
+		doc/manual/generated/out/markdown/command-ref/nix-env/$*.md \
+		$@
 
-$(d)/nix-store-%.1: $(d)/src/command-ref/nix-store/%.md
+doc/manual/generated/in/nix-store-%.1: doc/manual/generated/out
 	$(trace-gen) doc/manual/render-manpage.sh \
-		--out-no-smarty "$(subst nix-store-,nix-store --,$$(basename "$@" .1))" 1 $^ $^.tmp $@
+		--out-no-smarty "$(subst nix-store-,nix-store --,$$(basename "$@" .1))" 1 \
+		doc/manual/generated/out/markdown/command-ref/nix-store/$*.md \
+		$@
 
 
-$(d)/%.1: $(d)/src/command-ref/%.md
+doc/manual/generated/in/%.1: doc/manual/generated/out
-	$(trace-gen) doc/manual/render-manpage.sh "$$(basename $@ .1)" 1 $^ $^.tmp $@
+	$(trace-gen) doc/manual/render-manpage.sh "$$(basename $@ .1)" 1 \
+		doc/manual/generated/out/markdown/command-ref/$*.md \
+		$@
 
-$(d)/%.8: $(d)/src/command-ref/%.md
+doc/manual/generated/in/%.8: doc/manual/generated/out
-	$(trace-gen) doc/manual/render-manpage.sh "$$(basename $@ .8)" 8 $^ $^.tmp $@
+	$(trace-gen) doc/manual/render-manpage.sh "$$(basename $@ .8)" 8 \
+		doc/manual/generated/out/markdown/command-ref/$*.md \
+		$@
 
-$(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
+doc/manual/generated/in/nix.conf.5: doc/manual/generated/out
-	$(trace-gen) doc/manual/render-manpage.sh "$$(basename $@ .5)" 5 $^ $^.tmp $@
+	$(trace-gen) doc/manual/render-manpage.sh "$$(basename $@ .5)" 5 \
+		doc/manual/generated/out/markdown/command-ref/conf-file.md \
+		$@
 
-$(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md
+doc/manual/generated/in/nix-profiles.5: doc/manual/generated/out
-	$(trace-gen) doc/manual/render-manpage.sh "$$(basename $@ .5)" 5 $^ $^.tmp $@
+	$(trace-gen) doc/manual/render-manpage.sh "$$(basename $@ .5)" 5 \
+		doc/manual/generated/out/markdown/command-ref/files/profiles.md \
+		$@
 
-$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(doc_nix)
+doc/manual/generated/in/command-ref/new-cli: doc/manual/generated/in/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(doc_nix)
+	@mkdir -p doc/manual/generated/in/command-ref
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(doc_nix)
+doc/manual/generated/in/command-ref/conf-file.md: doc/manual/generated/in/conf-file.json $(d)/utils.nix doc/manual/generated/in/command-ref/experimental-features-shortlist.md $(doc_nix)
-	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
+	@mkdir -p doc/manual/generated/in/command-ref
-	$(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { inlineHTML = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
+	$(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { inlineHTML = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@
-	@mv $@.tmp $@
 
-$(d)/nix.json: $(doc_nix)
+doc/manual/generated/in/nix.json: $(doc_nix)
+	@mkdir -p doc/manual/generated/in
 	$(trace-gen) $(dummy-env) $(doc_nix) __dump-cli > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/conf-file.json: $(doc_nix)
+doc/manual/generated/in/conf-file.json: $(doc_nix)
+	@mkdir -p doc/manual/generated/in
 	$(trace-gen) $(dummy-env) $(doc_nix) show-config --json --experimental-features nix-command > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix)
+doc/manual/generated/in/contributing/experimental-feature-descriptions.md: doc/manual/generated/in/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix)
+	@mkdir -p doc/manual/generated/in/contributing
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(doc_nix)
+doc/manual/generated/in/command-ref/experimental-features-shortlist.md: doc/manual/generated/in/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(doc_nix)
+	@mkdir -p doc/manual/generated/in/command-ref
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))'
 	@mv $@.tmp $@
 
-$(d)/xp-features.json: $(doc_nix)
+doc/manual/generated/in/xp-features.json: $(doc_nix)
 	$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(doc_nix) __dump-xp-features > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(doc_nix)
+doc/manual/generated/in/language/builtins.md: doc/manual/generated/in/language.json $(d)/generate-builtins.nix $(doc_nix)
-	@cat doc/manual/src/language/builtins-prefix.md > $@.tmp
+	@mkdir -p doc/manual/generated/in/language
-	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@.tmp;
+	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@
-	@cat doc/manual/src/language/builtins-suffix.md >> $@.tmp
-	@mv $@.tmp $@
 
-$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(doc_nix)
+doc/manual/generated/in/language/builtin-constants.md: doc/manual/generated/in/language.json $(d)/generate-builtin-constants.nix $(doc_nix)
-	@cat doc/manual/src/language/builtin-constants-prefix.md > $@.tmp
+	@mkdir -p doc/manual/generated/in/language
-	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@.tmp;
+	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@
-	@cat doc/manual/src/language/builtin-constants-suffix.md >> $@.tmp
-	@mv $@.tmp $@
 
-$(d)/language.json: $(doc_nix)
+doc/manual/generated/in/language.json: $(doc_nix)
+	@mkdir -p doc/manual/generated/in
 	$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(doc_nix) __dump-language > $@.tmp
 	@mv $@.tmp $@
 
 # Generate "Upcoming release" notes (or clear it and remove from menu)
-$(d)/src/release-notes/rl-next-generated.md: $(d)/rl-next $(d)/rl-next/*
+doc/manual/generated/in/release-notes/rl-next-generated.md: $(d)/rl-next $(d)/rl-next/*
+	@mkdir -p doc/manual/generated/in/release-notes
 	@if type -p build-release-notes > /dev/null; then \
 		echo "  GEN   " $@; \
 		build-release-notes doc/manual/rl-next > $@; \
@@ -134,9 +148,9 @@ $(mandir)/man1/nix3-manpages: doc/manual/generated/man1/nix3-manpages
 	@mkdir -p $(DESTDIR)$$(dirname $@)
 	$(trace-install) install -m 0644 $$(dirname $<)/* $(DESTDIR)$$(dirname $@)
 
-doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
+doc/manual/generated/man1/nix3-manpages: doc/manual/generated/out
 	@mkdir -p $(DESTDIR)$$(dirname $@)
-	$(trace-gen) for i in doc/manual/src/command-ref/new-cli/*.md; do \
+	$(trace-gen) for i in doc/manual/generated/out/markdown/command-ref/new-cli/*.md; do \
 		name=$$(basename $$i .md); \
 		tmpFile=$$(mktemp); \
 		if [[ $$name = SUMMARY ]]; then continue; fi; \
@@ -147,11 +161,14 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 	done
 	@touch $@
 
-doc/manual/generated/out: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next-generated.md $(d)/docroot.py
+doc/manual/generated/out: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md doc/manual/generated/in/command-ref/new-cli doc/manual/generated/in/command-ref/experimental-features-shortlist.md doc/manual/generated/in/contributing/experimental-feature-descriptions.md doc/manual/generated/in/command-ref/conf-file.md doc/manual/generated/in/language/builtins.md doc/manual/generated/in/language/builtin-constants.md doc/manual/generated/in/release-notes/rl-next-generated.md $(d)/substitute.py
 	@rm -rf $@
 	$(trace-gen) \
-		RUST_LOG=warn mdbook build doc/manual -d generated/out 2>&1 \
+		MDBOOK_SUBSTITUTE_SEARCH=doc/manual/generated/in \
+		RUST_LOG=warn \
+		mdbook build doc/manual -d generated/out 2>&1 \
 			| { grep -Fv "because fragment resolution isn't implemented" || :; }
+	@find $@ -iname meson.build -delete
 
 $(docdir)/manual/index.html: doc/manual/generated/out
 	@mkdir -p $(DESTDIR)$(docdir)

doc/manual/meson.build

@@ -15,7 +15,7 @@ nix_eval_for_docs_common = nix_for_docs + [
 ]
 nix_eval_for_docs = nix_eval_for_docs_common + '--raw'
 
-nix_conf_file_json = custom_target(
+conf_file_json = custom_target(
   command : nix_for_docs + [ 'show-config', '--json' ],
   capture : true,
   output : 'conf-file.json',
@@ -30,7 +30,7 @@ nix_conf_file_md_body = custom_target(
   capture : true,
   input : [
     'utils.nix',
-    nix_conf_file_json,
+    conf_file_json,
   ],
   output : 'conf-file.md.body',
   env : nix_env_for_docs,
@@ -40,7 +40,7 @@ nix_conf_file_md = custom_target(
   command : [ 'cat', '@INPUT@' ],
   capture : true,
   input : [
-    'src/command-ref/conf-file-prefix.md',
+    'src/command-ref/conf-file.md',
     nix_conf_file_md_body,
   ],
   output : 'conf-file.md',
@@ -51,20 +51,83 @@ nix_exp_features_json = custom_target(
   capture : true,
   output : 'xp-features.json',
 )
-nix_exp_feature_shortlist = custom_target(
+
-  command : nix_eval_for_docs + [
+language_json = custom_target(
-    '--expr',
+  command: [nix, '__dump-language'],
-    'import @INPUT0@ (builtins.fromJSON (builtins.readFile @INPUT1@))',
+  output : 'language.json',
-  ],
-  input : [
-    'generate-xp-features-shortlist.nix',
-    nix_exp_features_json,
-  ],
-  output : 'experimental-features-shortlist.md',
   capture : true,
   env : nix_env_for_docs,
 )
 
+nix3_cli_json = custom_target(
+  command : [ nix, '__dump-cli' ],
+  capture : true,
+  output : 'nix.json',
+)
+
+generate_manual_deps = files(
+  'generate-deps.py',
+)
+
+# Generates builtins.md and builtin-constants.md.
+subdir('src/language')
+# Generates new-cli pages, experimental-features-shortlist.md, and conf-file.md.
+subdir('src/command-ref')
+# Generates experimental-feature-descriptions.md.
+subdir('src/contributing')
+# Generates rl-next-generated.md.
+subdir('src/release-notes')
+
+manual = custom_target(
+  'manual',
+  command : [
+    bash,
+    '-euo', 'pipefail',
+    '-c',
+    '''
+        @0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
+        cd @SOURCE_ROOT@
+        @1@ build doc/manual -d @2@ | { grep -Fv "because fragment resolution isn't implemented" || :; }
+        rm -rf @2@/manual
+        mv @2@/html @2@/manual
+        find @2@/manual -iname meson.build -delete
+    '''.format(
+      python.full_path(),
+      mdbook.full_path(),
+      meson.current_build_dir(),
+    ),
+  ],
+  input : [
+    generate_manual_deps,
+    'book.toml',
+    'anchors.jq',
+    'custom.css',
+    nix3_cli_files,
+    experimental_features_shortlist_md,
+    experimental_feature_descriptions_md,
+    conf_file_md,
+    builtins_md,
+    builtin_constants_md,
+    rl_next_generated,
+  ],
+  output : [
+    'manual',
+    'markdown',
+  ],
+  depfile : 'manual.d',
+  env : {
+    'RUST_LOG': 'info',
+    'MDBOOK_SUBSTITUTE_SEARCH': meson.current_build_dir() / 'src',
+  },
+)
+manual_html = manual[0]
+manual_md = manual[1]
+
+install_subdir(
+  manual_html.full_path(),
+  install_dir : datadir / 'doc/nix',
+)
+
 nix_nested_manpages = [
   [ 'nix-env',
     [
@@ -109,17 +172,20 @@ nix_nested_manpages = [
 
 foreach command : nix_nested_manpages
   foreach page : command[1]
+    title = command[0] + ' --' + page
+    section = '1'
     custom_target(
       command : [
         './render-manpage.sh',
         '--out-no-smarty',
-        command[0] + ' --' + page,
+        title,
-        '1',
+        section,
-        '@INPUT@',
+        '@INPUT0@/command-ref' / command[0] / (page + '.md'),
-        '@OUTPUT@.tmp',
+        '@OUTPUT0@',
-        '@OUTPUT@',
+      ],
+      input : [
+        manual_md,
       ],
-      input : 'src/command-ref' / command[0] / (page + '.md'),
       output : command[0] + '-' + page + '.1',
       install : true,
       install_dir : mandir / 'man1',
@@ -127,43 +193,6 @@ foreach command : nix_nested_manpages
   endforeach
 endforeach
 
-nix3_cli_json = custom_target(
-  command : [ nix, '__dump-cli' ],
-  capture : true,
-  output : 'nix.json',
-)
-
-# Intermediate step for manpage generation.
-# This splorks the output of generate-manpage.nix as JSON,
-# which gets written as a directory tree below.
-nix3_cli_files_json = custom_target(
-  command : nix_eval_for_docs_common + [
-    '--json',
-    '--expr',
-    'import @INPUT0@ true (builtins.readFile @INPUT1@)',
-  ],
-  input : [
-    'generate-manpage.nix',
-    nix3_cli_json,
-  ],
-  capture : true,
-  output : 'new-cli.json',
-  env : nix_env_for_docs,
-)
-nix3_cli_files = custom_target(
-  command : [
-    python,
-    '@INPUT0@',
-    '-i', '@INPUT1@',
-    '-o', '@OUTPUT@',
-  ],
-  input : [
-    'json-to-tree.py',
-    nix3_cli_files_json,
-  ],
-  output : 'new-cli',
-)
-
 nix3_manpages = [
   'nix3-build',
   'nix3-bundle',
@@ -254,16 +283,20 @@ nix3_manpages = [
 ]
 
 foreach page : nix3_manpages
+  section = '1'
   custom_target(
     command : [
-      './render-manpage.sh',
+      bash,
+      '@INPUT0@',
       page,
-      '1',
+      section,
-      '@INPUT0@/' + page + '.md',
+      '@INPUT1@/command-ref/new-cli/@0@.md'.format(page),
-      '@OUTPUT@.tmp',
       '@OUTPUT@',
     ],
-    input : nix3_cli_files,
+    input : [
+      'render-manpage.sh',
+      manual_md,
+    ],
     output : page + '.1',
     install : true,
     install_dir : mandir / 'man1',
@@ -281,24 +314,30 @@ nix_manpages = [
   [ 'nix-channel', 1 ],
   [ 'nix-hash', 1 ],
   [ 'nix-copy-closure', 1 ],
-  [ 'nix.conf', 5, nix_conf_file_md, nix_exp_feature_shortlist ],
+  [ 'nix.conf', 5, 'conf-file.md' ],
   [ 'nix-daemon', 8 ],
-  [ 'nix-profiles', 5, 'src/command-ref/files/profiles.md', nix_exp_feature_shortlist ],
+  [ 'nix-profiles', 5, 'files/profiles.md' ],
 ]
 
 foreach entry : nix_manpages
+  title = entry[0]
+  # nix.conf.5 and nix-profiles.5 are based off of conf-file.md and files/profiles.md,
+  # rather than a stem identical to its mdbook source.
+  # Therefore we use an optional third element of this array to override the name pattern
+  md_file = entry.get(2, title + '.md')
+  section = entry[1].to_string()
   custom_target(
     command : [
-      './render-manpage.sh',
+      bash,
-      entry[0],
-      entry[1].to_string(),
       '@INPUT0@',
-      '@OUTPUT@.tmp',
+      title,
+      section,
+      '@INPUT1@/command-ref/@0@'.format(md_file),
       '@OUTPUT@',
-      meson.current_build_dir(),
     ],
     input : [
-      entry.get(2, 'src/command-ref' / (entry[0] + '.md')),
+      'render-manpage.sh',
+      manual_md,
       entry.get(3, []),
     ],
     output : '@0@.@1@'.format(entry[0], entry[1]),

doc/manual/render-manpage.sh

@@ -9,14 +9,17 @@ if [ "$1" = --out-no-smarty ]; then
     shift
 fi
 
+[ "$#" = 4 ] || {
+    echo "wrong number of args passed" >&2
+    exit 1
+}
+
 title="$1"
 section="$2"
 infile="$3"
-tmpfile="$4"
+outfile="$4"
-outfile="$5"
 
-printf "Title: %s\n\n" "$title" > "$tmpfile"
+(
-cat "$infile" >> "$tmpfile"
+    printf "Title: %s\n\n" "$title"
-"$(dirname "$0")"/process-includes.sh "$infile" "$tmpfile"
+    cat "$infile"
-lowdown -sT man --nroff-nolinks $lowdown_args -M section="$section" "$tmpfile" -o "$outfile"
+) | lowdown -sT man --nroff-nolinks $lowdown_args -M section="$section" -o "$outfile"
-rm "$tmpfile"

doc/manual/src/command-ref/conf-file.md

@@ -67,3 +67,5 @@ Configuration options can be set on the command line, overriding the values set
 The `extra-` prefix is supported for settings that take a list of items (e.g. `--extra-trusted users alice` or `--option extra-trusted-users alice`).
 
 # Available settings
+
+{{#include @generated@/command-ref/conf-file.md}}

doc/manual/src/command-ref/meson.build

@@ -0,0 +1,65 @@
+xp_features_json = custom_target(
+  command : [nix, '__dump-xp-features'],
+  capture : true,
+  output : 'xp-features.json',
+)
+
+experimental_features_shortlist_md = custom_target(
+  command : nix_eval_for_docs + [
+    '--expr',
+    'import @INPUT0@ (builtins.fromJSON (builtins.readFile @INPUT1@))',
+  ],
+  input : [
+    '../../generate-xp-features-shortlist.nix',
+    xp_features_json,
+  ],
+  capture : true,
+  output : 'experimental-features-shortlist.md',
+  env : nix_env_for_docs,
+)
+
+# Intermediate step for manpage generation.
+# This splorks the output of generate-manpage.nix as JSON,
+# which gets written as a directory tree below.
+nix3_cli_files_json = custom_target(
+  command : nix_eval_for_docs_common + [
+    '--json',
+    '--expr',
+    'import @INPUT0@ true (builtins.readFile @INPUT1@)',
+  ],
+  input : [
+    '../../generate-manpage.nix',
+    nix3_cli_json,
+  ],
+  capture : true,
+  output : 'new-cli.json',
+  env : nix_env_for_docs,
+)
+nix3_cli_files = custom_target(
+  command : [
+    python,
+    '@INPUT0@',
+    '-i', '@INPUT1@',
+    '-o', '@OUTPUT@',
+  ],
+  input : [
+    '../../json-to-tree.py',
+    nix3_cli_files_json,
+  ],
+  output : 'new-cli',
+)
+
+conf_file_md = custom_target(
+  command : [
+    nix_eval_for_docs,
+    '--expr',
+    '(import @INPUT0@).showSettings { inlineHTML = true; } (builtins.fromJSON (builtins.readFile @INPUT1@))',
+  ],
+  capture : true,
+  input : [
+    '../../utils.nix',
+    conf_file_json,
+    experimental_features_shortlist_md,
+  ],
+  output : 'conf-file.md',
+)

doc/manual/src/command-ref/new-cli/nix.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix.md}}

doc/manual/src/command-ref/new-cli/nix3-build.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-build.md}}

doc/manual/src/command-ref/new-cli/nix3-bundle.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-bundle.md}}

doc/manual/src/command-ref/new-cli/nix3-copy.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-copy.md}}

doc/manual/src/command-ref/new-cli/nix3-daemon.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-daemon.md}}

doc/manual/src/command-ref/new-cli/nix3-derivation-add.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-derivation-add.md}}

doc/manual/src/command-ref/new-cli/nix3-derivation-show.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-derivation-show.md}}

doc/manual/src/command-ref/new-cli/nix3-derivation.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-derivation.md}}

doc/manual/src/command-ref/new-cli/nix3-develop.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-develop.md}}

doc/manual/src/command-ref/new-cli/nix3-doctor.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-doctor.md}}

doc/manual/src/command-ref/new-cli/nix3-edit.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-edit.md}}

doc/manual/src/command-ref/new-cli/nix3-eval.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-eval.md}}

doc/manual/src/command-ref/new-cli/nix3-flake-archive.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-flake-archive.md}}

doc/manual/src/command-ref/new-cli/nix3-flake-check.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-flake-check.md}}

doc/manual/src/command-ref/new-cli/nix3-flake-clone.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-flake-clone.md}}

doc/manual/src/command-ref/new-cli/nix3-flake-info.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-flake-info.md}}

doc/manual/src/command-ref/new-cli/nix3-flake-init.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-flake-init.md}}

doc/manual/src/command-ref/new-cli/nix3-flake-lock.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-flake-lock.md}}

doc/manual/src/command-ref/new-cli/nix3-flake-metadata.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-flake-metadata.md}}

doc/manual/src/command-ref/new-cli/nix3-flake-new.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-flake-new.md}}

doc/manual/src/command-ref/new-cli/nix3-flake-prefetch.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-flake-prefetch.md}}

doc/manual/src/command-ref/new-cli/nix3-flake-show.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-flake-show.md}}

doc/manual/src/command-ref/new-cli/nix3-flake-update.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-flake-update.md}}

doc/manual/src/command-ref/new-cli/nix3-flake.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-flake.md}}

doc/manual/src/command-ref/new-cli/nix3-fmt.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-fmt.md}}

doc/manual/src/command-ref/new-cli/nix3-hash-file.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-hash-file.md}}

doc/manual/src/command-ref/new-cli/nix3-hash-path.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-hash-path.md}}

doc/manual/src/command-ref/new-cli/nix3-hash-to-base16.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-hash-to-base16.md}}

doc/manual/src/command-ref/new-cli/nix3-hash-to-base32.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-hash-to-base32.md}}

doc/manual/src/command-ref/new-cli/nix3-hash-to-base64.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-hash-to-base64.md}}

doc/manual/src/command-ref/new-cli/nix3-hash-to-sri.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-hash-to-sri.md}}

doc/manual/src/command-ref/new-cli/nix3-hash.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-hash.md}}

doc/manual/src/command-ref/new-cli/nix3-help-stores.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-help-stores.md}}

doc/manual/src/command-ref/new-cli/nix3-help.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-help.md}}

doc/manual/src/command-ref/new-cli/nix3-key-convert-secret-to-public.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-key-convert-secret-to-public.md}}

doc/manual/src/command-ref/new-cli/nix3-key-generate-secret.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-key-generate-secret.md}}

doc/manual/src/command-ref/new-cli/nix3-key.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-key.md}}

doc/manual/src/command-ref/new-cli/nix3-log.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-log.md}}

doc/manual/src/command-ref/new-cli/nix3-nar-cat.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-nar-cat.md}}

doc/manual/src/command-ref/new-cli/nix3-nar-dump-path.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-nar-dump-path.md}}

doc/manual/src/command-ref/new-cli/nix3-nar-ls.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-nar-ls.md}}

doc/manual/src/command-ref/new-cli/nix3-nar.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-nar.md}}

doc/manual/src/command-ref/new-cli/nix3-path-info.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-path-info.md}}

doc/manual/src/command-ref/new-cli/nix3-print-dev-env.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-print-dev-env.md}}

doc/manual/src/command-ref/new-cli/nix3-profile-diff-closures.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-profile-diff-closures.md}}

doc/manual/src/command-ref/new-cli/nix3-profile-history.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-profile-history.md}}

doc/manual/src/command-ref/new-cli/nix3-profile-install.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-profile-install.md}}

doc/manual/src/command-ref/new-cli/nix3-profile-list.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-profile-list.md}}

doc/manual/src/command-ref/new-cli/nix3-profile-remove.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-profile-remove.md}}

doc/manual/src/command-ref/new-cli/nix3-profile-rollback.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-profile-rollback.md}}

doc/manual/src/command-ref/new-cli/nix3-profile-upgrade.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-profile-upgrade.md}}

doc/manual/src/command-ref/new-cli/nix3-profile-wipe-history.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-profile-wipe-history.md}}

doc/manual/src/command-ref/new-cli/nix3-profile.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-profile.md}}

doc/manual/src/command-ref/new-cli/nix3-realisation-info.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-realisation-info.md}}

doc/manual/src/command-ref/new-cli/nix3-realisation.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-realisation.md}}

doc/manual/src/command-ref/new-cli/nix3-registry-add.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-registry-add.md}}

doc/manual/src/command-ref/new-cli/nix3-registry-list.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-registry-list.md}}

doc/manual/src/command-ref/new-cli/nix3-registry-pin.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-registry-pin.md}}

doc/manual/src/command-ref/new-cli/nix3-registry-remove.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-registry-remove.md}}

doc/manual/src/command-ref/new-cli/nix3-registry.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-registry.md}}

doc/manual/src/command-ref/new-cli/nix3-repl.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-repl.md}}

doc/manual/src/command-ref/new-cli/nix3-run.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-run.md}}

doc/manual/src/command-ref/new-cli/nix3-search.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-search.md}}

doc/manual/src/command-ref/new-cli/nix3-shell.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-shell.md}}

doc/manual/src/command-ref/new-cli/nix3-show-config.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-show-config.md}}

doc/manual/src/command-ref/new-cli/nix3-store-add-file.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store-add-file.md}}

doc/manual/src/command-ref/new-cli/nix3-store-add-path.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store-add-path.md}}

doc/manual/src/command-ref/new-cli/nix3-store-cat.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store-cat.md}}

doc/manual/src/command-ref/new-cli/nix3-store-copy-log.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store-copy-log.md}}

doc/manual/src/command-ref/new-cli/nix3-store-copy-sigs.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store-copy-sigs.md}}

doc/manual/src/command-ref/new-cli/nix3-store-delete.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store-delete.md}}

doc/manual/src/command-ref/new-cli/nix3-store-diff-closures.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store-diff-closures.md}}

doc/manual/src/command-ref/new-cli/nix3-store-dump-path.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store-dump-path.md}}

doc/manual/src/command-ref/new-cli/nix3-store-gc.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store-gc.md}}

doc/manual/src/command-ref/new-cli/nix3-store-ls.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store-ls.md}}

doc/manual/src/command-ref/new-cli/nix3-store-make-content-addressed.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store-make-content-addressed.md}}

doc/manual/src/command-ref/new-cli/nix3-store-optimise.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store-optimise.md}}

doc/manual/src/command-ref/new-cli/nix3-store-path-from-hash-part.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store-path-from-hash-part.md}}

doc/manual/src/command-ref/new-cli/nix3-store-ping.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store-ping.md}}

doc/manual/src/command-ref/new-cli/nix3-store-prefetch-file.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store-prefetch-file.md}}

doc/manual/src/command-ref/new-cli/nix3-store-repair.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store-repair.md}}

doc/manual/src/command-ref/new-cli/nix3-store-sign.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store-sign.md}}

doc/manual/src/command-ref/new-cli/nix3-store-verify.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store-verify.md}}

doc/manual/src/command-ref/new-cli/nix3-store.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-store.md}}

doc/manual/src/command-ref/new-cli/nix3-upgrade-nix.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-upgrade-nix.md}}

doc/manual/src/command-ref/new-cli/nix3-why-depends.md

@@ -0,0 +1 @@
+{{#include @generated@/command-ref/new-cli/nix3-why-depends.md}}

doc/manual/src/contributing/experimental-features.md

@@ -92,4 +92,4 @@ This means that experimental features and RFCs are orthogonal mechanisms, and ca
 
 # Currently available experimental features
 
-{{#include ./experimental-feature-descriptions.md}}
+{{#include @generated@/contributing/experimental-feature-descriptions.md}}

doc/manual/src/contributing/meson.build

@@ -0,0 +1,15 @@
+# Intermediate step for experimental-feature-descriptions.md.
+# This splorks the output of generate-xp-features.nix as JSON,
+# which gets written as a directory tree below.
+experimental_feature_descriptions_md = custom_target(
+  command : nix_eval_for_docs + [
+    '--expr',
+    'import @INPUT0@ (builtins.fromJSON (builtins.readFile @INPUT1@))',
+  ],
+  input : [
+    '../../generate-xp-features.nix',
+    xp_features_json,
+  ],
+  capture : true,
+  output : 'experimental-feature-descriptions.md',
+)

doc/manual/src/language/builtin-constants-suffix.md

@@ -1 +0,0 @@
-</dl>

doc/manual/src/language/builtin-constants.md

@@ -3,3 +3,7 @@
 These constants are built into the Nix language evaluator:
 
 <dl>
+
+{{#include @generated@/language/builtin-constants.md}}
+
+</dl>

doc/manual/src/language/builtins-suffix.md

@@ -1 +0,0 @@
-</dl>