From be10c09d2350019bbf4075c5e22ddb1f97d3dad0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Jan 2023 08:53:29 +0100 Subject: [PATCH 1/8] manual: Check links mdbook-linkcheck is not consistent about its warning setting. It disables some warnings, but not the warnings about lack of fragment checking support; hence the extra filtering. --- doc/manual/book.toml | 9 +++++++++ doc/manual/local.mk | 5 ++++- flake.nix | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/doc/manual/book.toml b/doc/manual/book.toml index 46ced7ff7..73fb7e75e 100644 --- a/doc/manual/book.toml +++ b/doc/manual/book.toml @@ -10,3 +10,12 @@ git-repository-url = "https://github.com/NixOS/nix" [preprocessor.anchors] renderers = ["html"] command = "jq --from-file doc/manual/anchors.jq" + +[output.linkcheck] +# no Internet during the build (in the sandbox) +follow-web-links = false + +# mdbook-linkcheck does not understand [foo]{#bar} style links, resulting in +# excessive "Potential incomplete link" warnings. No other kind of warning was +# produced at the time of writing. +warning-policy = "ignore" diff --git a/doc/manual/local.mk b/doc/manual/local.mk index c0f69e00f..2a32f1a63 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -102,6 +102,9 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli @touch $@ $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md - $(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual + $(trace-gen) \ + set -euo pipefail; \ + RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual 2>&1 \ + | { grep -Fv "because fragment resolution isn't implemented" || :; } endif diff --git a/flake.nix b/flake.nix index 652695989..68011a16b 100644 --- a/flake.nix +++ b/flake.nix @@ -96,6 +96,7 @@ buildPackages.flex (lib.getBin buildPackages.lowdown-nix) buildPackages.mdbook + buildPackages.mdbook-linkcheck buildPackages.autoconf-archive buildPackages.autoreconfHook buildPackages.pkg-config From 34a1e0d29b55ff4f3fc7c923a3f03a65c3ff79a3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Jan 2023 10:09:08 +0100 Subject: [PATCH 2/8] doc/manual: Introduce @docroot@ as a stable base for includable snippets This way the links are clearly within the manual (ie not absolute paths), while allowing snippets to reference the documentation root reliably, regardless of at which base url they're included. --- doc/manual/local.mk | 10 +++++++--- src/libcmd/common-eval-args.cc | 4 ++-- src/libexpr/primops.cc | 2 +- src/libstore/globals.hh | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 2a32f1a63..fdcd131de 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -50,11 +50,14 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix @rm -rf $@ - $(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }' + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }' + $(trace-gen) sed -i $@.tmp/*.md -e 's^@docroot@^../..^g' + @mv $@.tmp $@ $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' \ + | sed -e 's^@docroot@^..^g'>> $@.tmp @mv $@.tmp $@ $(d)/nix.json: $(bindir)/nix @@ -67,7 +70,8 @@ $(d)/conf-file.json: $(bindir)/nix $(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix @cat doc/manual/src/language/builtins-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' \ + | sed -e 's^@docroot@^..^g' >> $@.tmp @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 0e321e5e4..908127b4d 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -34,8 +34,8 @@ MixEvalArgs::MixEvalArgs() .shortName = 'I', .description = R"( Add *path* to the Nix search path. The Nix search path is - initialized from the colon-separated [`NIX_PATH`](./env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](../language/values.md#type-path) enclosed in angle + initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment + variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle brackets (i.e., ``). For instance, passing diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9ef91cbc5..5519642b1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1917,7 +1917,7 @@ static RegisterPrimOp primop_toFile({ ``` Note that `${configFile}` is a - [string interpolation](language/values.md#type-string), so the result of the + [string interpolation](@docroot@/language/values.md#type-string), so the result of the expression `configFile` (i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be spliced into the resulting string. diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index f026c8808..7111def92 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -676,7 +676,7 @@ public: - the store object is signed by one of the [`trusted-public-keys`](#conf-trusted-public-keys) - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list - the [`require-sigs`](#conf-require-sigs) option has been set to `false` - - the store object is [output-addressed](glossary.md#gloss-output-addressed-store-object) + - the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object) )", {"binary-caches"}}; From e79f93571862f62447b7c95d10e797369a0be880 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Jan 2023 10:11:06 +0100 Subject: [PATCH 3/8] doc/manual: Fix broken internal links The targets I could find. --- doc/manual/src/architecture/architecture.md | 2 +- doc/manual/src/command-ref/env-common.md | 2 +- doc/manual/src/command-ref/nix-copy-closure.md | 2 +- src/libexpr/primops.cc | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 2d1b26558..e51958052 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -68,7 +68,7 @@ It can also execute build plans to produce new data, which are made available to A build plan itself is a series of *build tasks*, together with their build inputs. > **Important** -> A build task in Nix is called [derivation](../glossary#gloss-derivation). +> A build task in Nix is called [derivation](../glossary.md#gloss-derivation). Each build task has a special build input executed as *build instructions* in order to perform the build. The result of a build task can be input to another build task. diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index 5845bdc43..bb85a6b07 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -11,7 +11,7 @@ Most Nix commands interpret the following environment variables: expressions using [paths](../language/values.md#type-path) enclosed in angle brackets (i.e., ``), e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the - [`-I` option](./opt-common#opt-I). + [`-I` option](./opt-common.md#opt-I). - [`NIX_IGNORE_SYMLINK_STORE`]{#env-NIX_IGNORE_SYMLINK_STORE}\ Normally, the Nix store directory (typically `/nix/store`) is not diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md index 83e8a2936..cd8e351bb 100644 --- a/doc/manual/src/command-ref/nix-copy-closure.md +++ b/doc/manual/src/command-ref/nix-copy-closure.md @@ -49,7 +49,7 @@ authentication, you can avoid typing the passphrase with `ssh-agent`. - `--include-outputs`\ Also copy the outputs of [store derivation]s included in the closure. - [store derivation]: ../../glossary.md#gloss-store-derivation + [store derivation]: ../glossary.md#gloss-store-derivation - `--use-substitutes` / `-s`\ Attempt to download missing paths on the target machine using Nix’s diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5519642b1..23ddd607c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -240,6 +240,7 @@ static RegisterPrimOp primop_scopedImport(RegisterPrimOp::Info { static RegisterPrimOp primop_import({ .name = "import", .args = {"path"}, + // TODO turn "normal path values" into link below .doc = R"( Load, parse and return the Nix expression in the file *path*. If *path* is a directory, the file ` default.nix ` in that directory @@ -253,7 +254,7 @@ static RegisterPrimOp primop_import({ > > Unlike some languages, `import` is a regular function in Nix. > Paths using the angle bracket syntax (e.g., `import` *\*) - > are [normal path values](language-values.md). + > are normal path values. A Nix expression loaded by `import` must not contain any *free variables* (identifiers that are not defined in the Nix expression @@ -1872,8 +1873,7 @@ static RegisterPrimOp primop_toFile({ path. The file has suffix *name*. This file can be used as an input to derivations. One application is to write builders “inline”. For instance, the following Nix expression combines the - [Nix expression for GNU Hello](expression-syntax.md) and its - [build script](build-script.md) into one file: + Nix expression for GNU Hello and its build script into one file: ```nix { stdenv, fetchurl, perl }: From d5c8289f1e39397fcf562fa750719e9e86569041 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Jan 2023 10:29:05 +0100 Subject: [PATCH 4/8] doc/manual: Document hacking on the manual links --- doc/manual/src/contributing/hacking.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index c9da1962f..cd3788efc 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -249,3 +249,22 @@ search/replaced in it for each new build. The installer now supports a `--tarball-url-prefix` flag which _may_ have solved this need? --> + +### Checking the manual links + +The build checks for broken internal links, but this happens late in the process, +so `nix build .` is not suitable for iterating. To check the manual incrementally, run: + +```console +make html -j $NIX_BUILD_CORES +``` + +When iterating on the makefile, run: + +```console +rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make html -j $NIX_BUILD_CORES +``` + +If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. + +`mdbook-linkcheck` does not implement fragment checking yet. From fd2af69e600fdf0e06c996df607d560b222f7a38 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Jan 2023 11:04:52 +0100 Subject: [PATCH 5/8] doc/manual: Move the html files back where they were before ... before the link checking "output" was added, bumping the html output into a subdirectory. --- doc/manual/local.mk | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index fdcd131de..7639f510a 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -108,7 +108,10 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(trace-gen) \ set -euo pipefail; \ - RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual 2>&1 \ + RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \ | { grep -Fv "because fragment resolution isn't implemented" || :; } + @rm -rf $(DESTDIR)$(docdir)/manual + @mv $(DESTDIR)$(docdir)/manual.tmp/html $(DESTDIR)$(docdir)/manual + @rm -rf $(DESTDIR)$(docdir)/manual.tmp endif From fefa3a49ce67998923af0b6c927cb9cea1f4ceaa Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Jan 2023 11:21:21 +0100 Subject: [PATCH 6/8] doc/manual: Apply suggestions from code review Co-authored-by: Valentin Gagarin --- doc/manual/src/contributing/hacking.md | 16 +++++++++++----- src/libexpr/primops.cc | 2 +- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index cd3788efc..fe24294f8 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -250,16 +250,19 @@ The installer now supports a `--tarball-url-prefix` flag which _may_ have solved this need? --> -### Checking the manual links +### Checking links in the manual -The build checks for broken internal links, but this happens late in the process, -so `nix build .` is not suitable for iterating. To check the manual incrementally, run: +The build checks for broken internal links. +This happens late in the process, so `nix build` is not suitable for iterating. +To build the manual incrementally, run: ```console make html -j $NIX_BUILD_CORES ``` -When iterating on the makefile, run: +In order to reflect changes to the [Makefile], clear all generated files before re-building: + +[Makefile]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk ```console rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make html -j $NIX_BUILD_CORES @@ -267,4 +270,7 @@ rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/comman If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. -`mdbook-linkcheck` does not implement fragment checking yet. +[`mdbook-linkcheck`] does not implement checking [URI fragments] yet. + +[`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck +[URI fragments]: https://en.m.wikipedia.org/wiki/URI_fragment diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 23ddd607c..fc1524599 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -254,7 +254,7 @@ static RegisterPrimOp primop_import({ > > Unlike some languages, `import` is a regular function in Nix. > Paths using the angle bracket syntax (e.g., `import` *\*) - > are normal path values. + > are normal [path values](@docroot@/language/values.md#type-path). A Nix expression loaded by `import` must not contain any *free variables* (identifiers that are not defined in the Nix expression From da4d4feacf76a829ee7929c3d69f6f3d6ba86f1f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Jan 2023 11:36:01 +0100 Subject: [PATCH 7/8] doc/manual/hacking: Document @docroot@ variable --- doc/manual/local.mk | 3 +++ doc/manual/src/contributing/hacking.md | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 7639f510a..190f0258a 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -51,11 +51,13 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix @rm -rf $@ $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }' + # @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable $(trace-gen) sed -i $@.tmp/*.md -e 's^@docroot@^../..^g' @mv $@.tmp $@ $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp + # @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' \ | sed -e 's^@docroot@^..^g'>> $@.tmp @mv $@.tmp $@ @@ -70,6 +72,7 @@ $(d)/conf-file.json: $(bindir)/nix $(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix @cat doc/manual/src/language/builtins-prefix.md > $@.tmp + # @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' \ | sed -e 's^@docroot@^..^g' >> $@.tmp @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index fe24294f8..b9b327fba 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -268,9 +268,15 @@ In order to reflect changes to the [Makefile], clear all generated files before rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make html -j $NIX_BUILD_CORES ``` -If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. - [`mdbook-linkcheck`] does not implement checking [URI fragments] yet. [`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck [URI fragments]: https://en.m.wikipedia.org/wiki/URI_fragment + +#### `@docroot@` variable + +`@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own. + +If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. + +If the `@docroot@` literal appears in an error message from the `mdbook-linkcheck` tool, the `@docroot@` replacement needs to be applied to the (probably) generated source file that mentions it. See existing `@docroot@` logic in the [Makefile]. Regular markdown files that are not copied to other markdown files do have a base path of their own and they can use relative paths instead of `@docroot@`. From 6ae4d762d0a1b208eaa47e9bd8960571ac908af7 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 3 Jan 2023 13:52:17 +0100 Subject: [PATCH 8/8] doc/manual/src/contributing/hacking.md: Apply suggestion Co-authored-by: Valentin Gagarin --- doc/manual/src/contributing/hacking.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index b9b327fba..aeb0d41b3 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -279,4 +279,6 @@ rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/comman If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. -If the `@docroot@` literal appears in an error message from the `mdbook-linkcheck` tool, the `@docroot@` replacement needs to be applied to the (probably) generated source file that mentions it. See existing `@docroot@` logic in the [Makefile]. Regular markdown files that are not copied to other markdown files do have a base path of their own and they can use relative paths instead of `@docroot@`. +If the `@docroot@` literal appears in an error message from the `mdbook-linkcheck` tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it. +See existing `@docroot@` logic in the [Makefile]. +Regular markdown files used for the manual have a base path of their own and they can use relative paths instead of `@docroot@`.