build: build lix-doc with Meson! 🎉

lix-doc is now built with Meson, with lix-doc's dependencies built as
Meson subprojects, either fetched on demand with .wrap files, or fetched
in advance by Nix with importCargoLock. It even builds statically.

Fixes #256.

Co-authored-by: Lunaphied <lunaphied@lunaphied.me>
Co-authored-by: Jade Lovelace <lix@jade.fyi>

Change-Id: I3a4731ff13278e7117e0316bc0d7169e85f5eb0c

lix-doc/meson.build

@@ -0,0 +1,52 @@
+# Until Meson 1.5¹, we can't just give Meson a Cargo.lock file and be done with it.
+# Meson will *detect* what dependencies are needed from Cargo files; it just won't
+# fetch them. The Meson 1.5 feature essentially internally translates Cargo.lock entries
+# to .wrap files, and that translation is incredibly straightforward, so let's just
+# use a simple Python script to generate the .wrap files ourselves while we wait for
+# Meson 1.5. Weirdly, it seems Meson will only detect dependencies from other
+# dependency() calls, so we have to specify lix-doc's two top-level dependencies,
+# rnix and rowan, manually, and then their dependencies will be recursively translated
+# into more dependency() calls.
+#
+# When Meson translates a Cargo dependency, the string passed to `dependency()` follows
+# a fixed format, which is important as the .wrap files' basenames must match the string
+# passed to `dependency()` exactly.
+# In Meson 1.4, this format is `$packageName-rs`. Meson 1.5 changes this to
+# `$packageName-$shortenedVersionString-rs`, because of course it does, but we'll cross
+# that bridge when we get there...
+#
+# [1]: https://github.com/mesonbuild/meson/commit/9b8378985dbdc0112d11893dd42b33b7bc8d1e62
+
+run_command(
+  python,
+  meson.project_source_root() / 'meson/cargo-lock-to-wraps.py',
+  meson.current_source_dir() / 'Cargo.lock',
+  meson.project_source_root() / 'subprojects',
+  check : true,
+)
+
+# The external crate rowan has an ambiguous pointer comparison warning, which
+# we don't want to fail our whole build if werror is on.
+subproject('rowan-rs', default_options : ['werror=false'])
+
+rnix = dependency('rnix-rs')
+rowan = dependency('rowan-rs')
+
+lix_doc = static_library(
+  'lix_doc',
+  sources : files('src/lib.rs'),
+  rust_abi : 'c',
+  dependencies : [
+    rowan,
+    rnix,
+  ],
+  # If an installed static library depends on this target, then Meson will force
+  # that to link with `-Wl,--whole-archive`, unless we also install this target.
+  # `-Wl,--whole-archive` can cause some Problems when linking multiple nested
+  # static libraries, so let's just install the damn thing.
+  install : true,
+)
+
+liblix_doc = declare_dependency(
+  link_with : lix_doc,
+)

lix-doc/package.nix

@@ -1,8 +0,0 @@
-{ rustPlatform, lib }:
-
-rustPlatform.buildRustPackage {
-  name = "lix-doc";
-
-  cargoLock.lockFile = ./Cargo.lock;
-  src = lib.cleanSource ./.;
-}

meson.build

@@ -30,6 +30,14 @@
 # FIXME: This hack should be removed when https://git.lix.systems/lix-project/lix/issues/359
 # is fixed.
 #
+# lix-doc is built with Meson in lix-doc/meson.build, and linked into libcmd in
+# src/libcmd/meson.build. When building outside the Nix sandbox, Meson will use the .wrap
+# files in subprojects/ to download and extract the dependency crates into subprojects/.
+# When building inside the Nix sandbox, Lix's derivation in package.nix uses a
+# fixed-output derivation to fetch those crates in advance instead, and then symlinks
+# them into subprojects/ with the same names that Meson uses when downloading them
+# itself -- perfect for --wrap-mode=nodownload, which mesonConfigurePhase uses.
+#
 # Unit tests are setup in tests/unit/meson.build, under the test suite "check".
 #
 # Functional tests are a bit more complicated. Generally they're defined in
@@ -38,10 +46,11 @@
 # be placed in specific directories' meson.build files to create the right directory tree
 # in the build directory.
 
-project('lix', 'cpp',
+project('lix', 'cpp', 'rust',
   version : run_command('bash', '-c', 'echo -n $(jq -r .version < ./version.json)$VERSION_SUFFIX', check : true).stdout().strip(),
   default_options : [
     'cpp_std=c++2a',
+    'rust_std=2021',
     # TODO(Qyriad): increase the warning level
     'warning_level=1',
     'debug=true',
@@ -322,13 +331,6 @@ pegtl = dependency(
 
 nlohmann_json = dependency('nlohmann_json', required : true, include_type : 'system')
 
-# lix-doc is a Rust project provided via buildInputs and unfortunately doesn't have any way to be detected.
-# Just declare it manually to resolve this.
-#
-# FIXME: build this with meson in the future after we drop Make (with which we
-# *absolutely* are not going to make it work)
-lix_doc = declare_dependency(link_args : [ '-llix_doc' ])
-
 if is_freebsd
   libprocstat = declare_dependency(link_args : [ '-lprocstat' ])
 endif
@@ -552,6 +554,7 @@ if is_darwin
   )
 endif
 
+subdir('lix-doc')
 subdir('src')
 subdir('scripts')
 subdir('misc')

meson/cargo-lock-to-wraps.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+
+import argparse
+import tomllib
+import sys
+
+DOWNLOAD_URI_FORMAT = 'https://crates.io/api/v1/crates/{crate}/{version}/download'
+
+WRAP_TEMPLATE = """
+[wrap-file]
+method = cargo
+directory = {crate}-{version}
+source_url = {url}
+source_filename = {crate}-{version}.tar.gz
+source_hash = {hash}
+""".lstrip()
+
+parser = argparse.ArgumentParser()
+parser.add_argument('lockfile', help='path to the Cargo lockfile to generate wraps from')
+parser.add_argument('outdir', help="the 'subprojects' directory to write .wrap files to")
+
+args = parser.parse_args()
+
+with open(args.lockfile, 'rb') as f:
+    lock_toml = tomllib.load(f)
+
+for dependency in lock_toml['package']:
+    try:
+        hash = dependency['checksum']
+    except KeyError:
+        # The base package, e.g. lix-doc, won't have a checksum, and conveniently
+        # the base package is also not something we want a wrap file for.
+        # Doesn't that work out nicely?
+        continue
+
+    crate = dependency['name']
+    version = dependency['version']
+
+    url = DOWNLOAD_URI_FORMAT.format(crate=crate, version=version)
+
+    wrap_text = WRAP_TEMPLATE.format(crate=crate, version=version, url=url, hash=hash)
+    with open(f'{args.outdir}/{crate}-rs.wrap', 'w') as f:
+        f.write(wrap_text)

meson/clang-tidy/clean_compdb.py

@@ -30,7 +30,14 @@ def process_compdb(compdb: list[dict]) -> list[dict]:
         item['command'] = shlex.join(munch_command(shlex.split(item['command'])))
         return item
 
-    return [chomp(x) for x in compdb if not x['file'].endswith('precompiled-headers.hh')]
+    def cmdfilter(item: dict) -> bool:
+        file = item['file']
+        return (
+            not file.endswith('precompiled-headers.hh')
+            and not file.endswith('.rs')
+        )
+
+    return [chomp(x) for x in compdb if cmdfilter(x)]
 
 
 def main():

package.nix

@@ -41,6 +41,8 @@
   pkg-config,
   python3,
   rapidcheck,
+  rustPlatform,
+  rustc,
   sqlite,
   toml11,
   util-linuxMinimal ? utillinuxMinimal,
@@ -49,9 +51,6 @@
 
   busybox-sandbox-shell,
 
-  # internal fork of nix-doc providing :doc in the repl
-  lix-doc ? __forDefaults.lix-doc,
-
   pname ? "lix",
   versionSuffix ? "",
   officialRelease ? __forDefaults.versionJson.official_release,
@@ -83,7 +82,6 @@
       configureFlags = prev.configureFlags or [ ] ++ [ (lib.enableFeature true "sigstop") ];
     });
 
-    lix-doc = callPackage ./lix-doc/package.nix { };
     build-release-notes = callPackage ./maintainers/build-release-notes.nix { };
   },
 }:
@@ -137,6 +135,7 @@ let
     ./meson.build
     ./meson.options
     ./meson
+    ./lix-doc
     ./scripts/meson.build
     ./subprojects
   ]);
@@ -219,6 +218,7 @@ stdenv.mkDerivation (finalAttrs: {
       meson
       ninja
       cmake
+      rustc
     ]
     ++ [
       (lib.getBin lowdown)
@@ -258,7 +258,6 @@ stdenv.mkDerivation (finalAttrs: {
       lowdown
       libsodium
       toml11
-      lix-doc
       pegtl
     ]
     ++ lib.optionals hostPlatform.isLinux [
@@ -290,6 +289,8 @@ stdenv.mkDerivation (finalAttrs: {
     BOOST_LIBRARYDIR = "${lib.getLib boost}/lib";
   };
 
+  cargoDeps = rustPlatform.importCargoLock { lockFile = ./lix-doc/Cargo.lock; };
+
   preConfigure =
     lib.optionalString (!finalAttrs.dontBuild && !hostPlatform.isStatic) ''
       # Copy libboost_context so we don't get all of Boost in our closure.
@@ -311,6 +312,17 @@ stdenv.mkDerivation (finalAttrs: {
       install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
     ''
     + ''
+      # Copy the Cargo dependencies to where Meson expects them to be, so we
+      # can seamlessly use Meson's subproject wraps, but just do the download
+      # ahead of time. Luckily for us, importCargoLock-downloaded crates use
+      # the exact naming scheme Meson expects!
+      # The directory from importCargoLock does contain a lockfile, which we
+      # don't need, but all the crate directories start with a word character,
+      # then have a hyphen, and then a sequence of digits or periods for the
+      # version.
+      find "$cargoDeps" -type l -regex '.*/\w.+-[0-9.]+$' -exec \
+          ln -sv "{}" "$PWD/subprojects/" ";"
+
       # Fix up /usr/bin/env shebangs relied on by the build
       patchShebangs --build tests/ doc/manual/
     '';

src/libcmd/meson.build

@@ -50,7 +50,7 @@ libcmd = library(
     editline,
     lowdown,
     nlohmann_json,
-    lix_doc
+    liblix_doc,
   ],
   cpp_pch : cpp_pch,
   install : true,

subprojects/.gitignore

@@ -0,0 +1,2 @@
+# Downloaded wrapped Rust projects
+*-*.*.*

subprojects/autocfg-rs.wrap

@@ -0,0 +1,6 @@
+[wrap-file]
+method = cargo
+directory = autocfg-1.1.0
+source_url = https://crates.io/api/v1/crates/autocfg/1.1.0/download
+source_filename = autocfg-1.1.0.tar.gz
+source_hash = d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa

subprojects/countme-rs.wrap

@@ -0,0 +1,6 @@
+[wrap-file]
+method = cargo
+directory = countme-3.0.1
+source_url = https://crates.io/api/v1/crates/countme/3.0.1/download
+source_filename = countme-3.0.1.tar.gz
+source_hash = 7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636

subprojects/dissimilar-rs.wrap

@@ -0,0 +1,6 @@
+[wrap-file]
+method = cargo
+directory = dissimilar-1.0.7
+source_url = https://crates.io/api/v1/crates/dissimilar/1.0.7/download
+source_filename = dissimilar-1.0.7.tar.gz
+source_hash = 86e3bdc80eee6e16b2b6b0f87fbc98c04bee3455e35174c0de1a125d0688c632

subprojects/expect-test-rs.wrap

@@ -0,0 +1,6 @@
+[wrap-file]
+method = cargo
+directory = expect-test-1.4.1
+source_url = https://crates.io/api/v1/crates/expect-test/1.4.1/download
+source_filename = expect-test-1.4.1.tar.gz
+source_hash = 30d9eafeadd538e68fb28016364c9732d78e420b9ff8853fa5e4058861e9f8d3

subprojects/hashbrown-rs.wrap

@@ -0,0 +1,6 @@
+[wrap-file]
+method = cargo
+directory = hashbrown-0.14.5
+source_url = https://crates.io/api/v1/crates/hashbrown/0.14.5/download
+source_filename = hashbrown-0.14.5.tar.gz
+source_hash = e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1

subprojects/memoffset-rs.wrap

@@ -0,0 +1,6 @@
+[wrap-file]
+method = cargo
+directory = memoffset-0.9.1
+source_url = https://crates.io/api/v1/crates/memoffset/0.9.1/download
+source_filename = memoffset-0.9.1.tar.gz
+source_hash = 488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a

subprojects/once_cell-rs.wrap

@@ -0,0 +1,6 @@
+[wrap-file]
+method = cargo
+directory = once_cell-1.19.0
+source_url = https://crates.io/api/v1/crates/once_cell/1.19.0/download
+source_filename = once_cell-1.19.0.tar.gz
+source_hash = 3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92

subprojects/rnix-rs.wrap

@@ -0,0 +1,6 @@
+[wrap-file]
+method = cargo
+directory = rnix-0.11.0
+source_url = https://crates.io/api/v1/crates/rnix/0.11.0/download
+source_filename = rnix-0.11.0.tar.gz
+source_hash = bb35cedbeb70e0ccabef2a31bcff0aebd114f19566086300b8f42c725fc2cb5f

subprojects/rowan-rs.wrap

@@ -0,0 +1,6 @@
+[wrap-file]
+method = cargo
+directory = rowan-0.15.15
+source_url = https://crates.io/api/v1/crates/rowan/0.15.15/download
+source_filename = rowan-0.15.15.tar.gz
+source_hash = 32a58fa8a7ccff2aec4f39cc45bf5f985cec7125ab271cf681c279fd00192b49

subprojects/rustc-hash-rs.wrap

@@ -0,0 +1,6 @@
+[wrap-file]
+method = cargo
+directory = rustc-hash-1.1.0
+source_url = https://crates.io/api/v1/crates/rustc-hash/1.1.0/download
+source_filename = rustc-hash-1.1.0.tar.gz
+source_hash = 08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2

subprojects/text-size-rs.wrap

@@ -0,0 +1,6 @@
+[wrap-file]
+method = cargo
+directory = text-size-1.1.1
+source_url = https://crates.io/api/v1/crates/text-size/1.1.1/download
+source_filename = text-size-1.1.1.tar.gz
+source_hash = f18aa187839b2bdb1ad2fa35ead8c4c2976b64e4363c386d45ac0f7ee85c9233