meson: flip the switch!!

This commit makes Meson the default buildsystem for Lix.
The Make buildsystem is now deprecated and will be removed soon, but has
not yet, which will be done in a later commit when all seems good. The
mesonBuild jobs have been removed, and have not been replaced with
equivalent jobs to ensure the Make buildsystem still works.

The full, new commands in a development shell are:

$ meson setup ./build "--prefix=$out" $mesonFlags

(A simple `meson setup ./build` will also build, but will do a different
thing, not having the settings from package.nix applied.)

$ meson compile -C build
$ meson test -C build --suite=check
$ meson install -C build
$ meson test -C build --suite=installcheck

(Check and installcheck may both be done after install, allowing you to
omit the --suite argument entirely, but this is the order package.nix
runs them in.)

If tests fail and Meson helpfully has no output for why, use the
`--print-error-logs` option to `meson test`. Why this is not the default
I cannot explain.

If you change a setting in the buildsystem, most cases will automatically
regenerate the Meson configuration, but some cases, like trying to build
a specific target whose name is new to the buildsystem (e.g.
`meson compile -C build src/libmelt/libmelt.dylib`, when
`libmelt.dylib` did not exist as a target the last time the buildsystem
was generated), then you can reconfigure using new settings but existing
options, and only recompiling stuff affected by the changes:

$ meson setup --reconfigure build

Note that changes to the default values in `meson.options` or in the
`default_options :` argument to project() are NOT propagated with
`--reconfigure`.

If you want a totally clean build, you can use:

$ meson setup --wipe build

That will work regardless of if `./build` exists or not.

Specific, named targets may be addressed in `meson build -C build <target>`
with "target ID" if there is one, which is the first string argument
passed to target functions that have one, and unrelated to the variable
name, e.g.:

libexpr_dylib = library('nixexpr', …)

can be addressed with:

$ meson compile -C build nixexpr

All targets may be addressed as their output, relative to the build
directory, e.g.:

$ meson compile -C build src/libexpr/libnixexpr.so

But Meson does not consider intermediate files like object files
targets. To build a specific object file, use Ninja directly and
specify the output file relative to the build directory:

$ ninja -C build src/libexpr/libnixexpr.so.p/nixexpr.cc.o

To inspect the canonical source of truth on what the state of the
buildsystem configuration is, use:

$ meson introspect

Have fun!

Change-Id: Ia3e7b1e6fae26daf3162e655b4ded611a5cd57ad

doc/internal-api/doxygen.cfg.in

@@ -14,6 +14,8 @@ PROJECT_NAME           = "Nix"
 
 PROJECT_NUMBER         = @PACKAGE_VERSION@
 
+OUTPUT_DIRECTORY       = @docdir@
+
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
 # quick idea about the purpose of the project. Keep the description short.

doc/internal-api/meson.build

@@ -0,0 +1,33 @@
+doxygen_cfg = configure_file(
+  input : 'doxygen.cfg.in',
+  output : 'doxygen.cfg',
+  configuration : {
+    'PACKAGE_VERSION': meson.project_version(),
+    'RAPIDCHECK_HEADERS': rapidcheck_meson.get_variable('includedir'),
+    'docdir' : meson.current_build_dir(),
+  },
+)
+
+internal_api_docs = custom_target(
+  'internal-api-docs',
+  command : [
+    bash,
+    # Meson can you please just give us a `workdir` argument to custom targets...
+    '-c',
+    # We have to prefix the doxygen_cfg path with the project build root
+    # because of the cd in front.
+    'cd @0@ && @1@ @2@/@INPUT0@'.format(
+      meson.project_source_root(),
+      doxygen.full_path(),
+      meson.project_build_root(),
+    ),
+  ],
+  input : [
+    doxygen_cfg,
+  ],
+  output : 'html',
+  install : true,
+  install_dir : datadir / 'doc/nix/internal-api',
+)
+
+alias_target('internal-api-html', internal_api_docs)

flake.nix

@@ -26,7 +26,7 @@
     let
       inherit (nixpkgs) lib;
 
-      officialRelease = true;
+      officialRelease = false;
 
       # Set to true to build the release notes for the next release.
       buildUnreleasedNotes = false;
@@ -196,24 +196,6 @@
           }
         );
 
-        # FIXME(Qyriad): remove this when the migration to Meson has been completed.
-        # NOTE: mesonBuildClang depends on mesonBuild depends on build to avoid OOMs
-        # on aarch64 builders caused by too many parallel compiler/linker processes.
-        mesonBuild = forAllSystems (
-          system:
-          (self.packages.${system}.nix.override { buildWithMeson = true; }).overrideAttrs (prev: {
-            buildInputs = prev.buildInputs ++ [ self.packages.${system}.nix ];
-          })
-        );
-        mesonBuildClang = forAllSystems (
-          system:
-          (nixpkgsFor.${system}.stdenvs.clangStdenvPackages.nix.override { buildWithMeson = true; })
-          .overrideAttrs
-            (prev: {
-              buildInputs = prev.buildInputs ++ [ self.hydraJobs.mesonBuild.${system} ];
-            })
-        );
-
         # Perl bindings for various platforms.
         perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings);
 
@@ -366,9 +348,6 @@
       checks = forAllSystems (
         system:
         {
-          # FIXME(Qyriad): remove this when the migration to Meson has been completed.
-          mesonBuild = self.hydraJobs.mesonBuild.${system};
-          mesonBuildClang = self.hydraJobs.mesonBuildClang.${system};
           binaryTarball = self.hydraJobs.binaryTarball.${system};
           perlBindings = self.hydraJobs.perlBindings.${system};
           nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system};

meson.build

@@ -79,7 +79,25 @@ if not fs.is_absolute(sysconfdir)
   sysconfdir = '/' / sysconfdir
 endif
 
+# All of this has to go before the rest of the dependency checking,
+# so that internal-api-docs can be built with -Denable-build=false
+
 enable_docs = get_option('enable-docs')
+enable_internal_api_docs = get_option('internal-api-docs')
+
+doxygen = find_program('doxygen', required : enable_internal_api_docs)
+bash = find_program('bash')
+
+rapidcheck_meson = dependency('rapidcheck', required : enable_internal_api_docs)
+
+if enable_internal_api_docs.enabled()
+  message('subdiring()')
+  subdir('doc/internal-api')
+endif
+
+if not get_option('enable-build')
+  subdir_done()
+endif
 
 enable_tests = get_option('enable-tests')
 
@@ -223,6 +241,8 @@ lowdown = dependency('lowdown', version : '>=0.9.0', required : true)
 deps += lowdown
 
 # HACK(Qyriad): rapidcheck's pkg-config doesn't include the libs lol
+# Note: technically we 'check' for rapidcheck twice, for the internal-api-docs handling above,
+# but Meson will cache the result of the first one, and the required : arguments are different.
 rapidcheck_meson = dependency('rapidcheck', required : enable_tests)
 rapidcheck = declare_dependency(dependencies : rapidcheck_meson, link_args : ['-lrapidcheck'])
 deps += rapidcheck
@@ -252,7 +272,6 @@ deps += lix_doc
 #
 # Build-time tools
 #
-bash = find_program('bash')
 coreutils = find_program('coreutils')
 dot = find_program('dot', required : false)
 pymod = import('python')
@@ -265,6 +284,16 @@ endif
 # Used to workaround https://github.com/mesonbuild/meson/issues/2320 in src/nix/meson.build.
 installcmd = find_program('install')
 
+enable_embedded_sandbox_shell = get_option('enable-embedded-sandbox-shell')
+if enable_embedded_sandbox_shell
+  # This one goes in config.h
+  # The path to busybox is passed as a -D flag when compiling libstore.
+  # Idk why, ask the old buildsystem.
+  configdata += {
+    'HAVE_EMBEDDED_SANDBOX_SHELL': 1,
+  }
+endif
+
 sandbox_shell = get_option('sandbox-shell')
 # Consider it required if we're on Linux and the user explicitly specified a non-default value.
 sandbox_shell_required = sandbox_shell != 'busybox' and host_machine.system() == 'linux'
@@ -394,6 +423,21 @@ if get_option('profile-build').require(meson.get_compiler('cpp').get_id() == 'cl
   add_project_arguments('-ftime-trace', language: 'cpp')
 endif
 
+if cxx.get_id() in ['clang', 'gcc']
+  add_project_arguments([
+      # Meson uses out of source builds, conventionally usually in a subdirectory
+      # of the source tree (e.g. meson setup ./build). This means that unlike in
+      # the previous Make buildsystem, all compilation sources are passed as a relative
+      # parent, e.g. `cc -o src/libexpr/nixexpr.cc.o ../src/libexpr/nixexpr.cc`.
+      # These paths show up when debugging, and in asserts, which look both look strange
+      # and confuse debuggers.
+      # So let's just tell GCC and Clang that ../src really means src.
+      '-ffile-prefix-map=../src=src',
+    ],
+    language : 'cpp',
+  )
+endif
+
 subdir('src')
 subdir('scripts')
 subdir('misc')

meson.options

@@ -1,9 +1,14 @@
 # vim: filetype=meson
+
+option('enable-build', type : 'boolean', value : true,
+  description : 'Set to false to not actually build. Only really makes sense with -Dinternal-api-docs=true',
+)
+
 option('gc', type : 'feature',
   description : 'enable garbage collection in the Nix expression evaluator (requires Boehm GC)',
 )
-# TODO(Qyriad): is this feature maintained?
-option('embedded-sandbox-shell', type : 'feature',
+
+option('enable-embedded-sandbox-shell', type : 'boolean', value : false,
   description : 'include the sandbox shell in the Nix binary',
 )
 
@@ -51,6 +56,10 @@ option('enable-docs', type : 'boolean', value : true,
   description : 'whether to build documentation',
 )
 
+option('internal-api-docs', type : 'feature', value : 'auto',
+  description : 'whether to build internal API documentation (requires doxygen)',
+)
+
 # A relative path means it gets appended to prefix.
 option('profile-dir', type : 'string', value : 'etc/profile.d',
   description : 'the path to install shell profile files',

package.nix

@@ -62,7 +62,7 @@
 
   # FIXME(Qyriad): build Lix using Meson instead of autoconf and make.
   # This flag will be removed when the migration to Meson is complete.
-  buildWithMeson ? false,
+  buildWithMeson ? true,
 
   # Not a real argument, just the only way to approximate let-binding some
   # stuff for argument defaults.
@@ -178,9 +178,16 @@ stdenv.mkDerivation (finalAttrs: {
   dontBuild = false;
 
   # FIXME(Qyriad): see if this is still needed once the migration to Meson is completed.
-  mesonFlags = lib.optionals (buildWithMeson && stdenv.hostPlatform.isLinux) [
-    "-Dsandbox-shell=${lib.getBin busybox-sandbox-shell}/bin/busybox"
-  ];
+  mesonFlags =
+    lib.optionals (buildWithMeson && stdenv.hostPlatform.isLinux) [
+      "-Dsandbox-shell=${lib.getBin busybox-sandbox-shell}/bin/busybox"
+    ]
+    ++ lib.optional stdenv.hostPlatform.isStatic "-Denable-embedded-sandbox-shell=true"
+    ++ lib.optional (finalAttrs.dontBuild) "-Denable-build=false"
+    # mesonConfigurePhase automatically passes -Dauto_features=enabled,
+    # so we must explicitly enable or disable features that we are not passing
+    # dependencies for.
+    ++ lib.singleton (lib.mesonEnable "internal-api-docs" internalApiDocs);
 
   # We only include CMake so that Meson can locate toml11, which only ships CMake dependency metadata.
   dontUseCmakeConfigure = true;
@@ -209,7 +216,7 @@ stdenv.mkDerivation (finalAttrs: {
     ]
     ++ lib.optional stdenv.hostPlatform.isLinux util-linuxMinimal
     ++ lib.optional (!officialRelease && buildUnreleasedNotes) build-release-notes
-    ++ lib.optional internalApiDocs doxygen
+    ++ lib.optional (internalApiDocs || forDevShell) doxygen
     ++ lib.optionals buildWithMeson [
       meson
       ninja
@@ -236,6 +243,7 @@ stdenv.mkDerivation (finalAttrs: {
       libseccomp
       busybox-sandbox-shell
     ]
+    ++ lib.optional internalApiDocs rapidcheck
     ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
     # There have been issues building these dependencies
     ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform) aws-sdk-cpp-nix
@@ -313,6 +321,13 @@ stdenv.mkDerivation (finalAttrs: {
 
   installFlags = "sysconfdir=$(out)/etc";
 
+  # Make sure the internal API docs are already built, because mesonInstallPhase
+  # won't let us build them there. They would normally be built in buildPhase,
+  # but the internal API docs are conventionally built with doBuild = false.
+  preInstall = lib.optional (buildWithMeson && internalApiDocs) ''
+    meson ''${mesonBuildFlags:-} compile "$installTargets"
+  '';
+
   postInstall =
     lib.optionalString (!finalAttrs.dontBuild) ''
       mkdir -p $doc/nix-support

src/libstore/meson.build

@@ -10,6 +10,24 @@ foreach header : [ 'schema.sql', 'ca-specific-schema.sql' ]
   )
 endforeach
 
+if enable_embedded_sandbox_shell
+  hexdump = find_program('hexdump', required : true)
+  embedded_sandbox_shell_gen = custom_target(
+    'embedded-sandbox-shell.gen.hh',
+    command : [
+      hexdump,
+      '-v',
+      '-e',
+      '1/1 "0x%x," "\n"'
+    ],
+    input : busybox.full_path(),
+    output : 'embedded-sandbox-shell.gen.hh',
+    capture : true,
+    feed : true,
+  )
+  libstore_generated_headers += embedded_sandbox_shell_gen
+endif
+
 libstore_sources = files(
   'binary-cache-store.cc',
   'build-result.cc',
@@ -156,6 +174,12 @@ cpp_str_defines = {
   'NIX_MAN_DIR':   mandir,
 }
 
+if busybox.found()
+  cpp_str_defines += {
+    'SANDBOX_SHELL': busybox.full_path()
+  }
+endif
+
 cpp_args = []
 
 foreach name, value : cpp_str_defines

tests/nixos/github-flakes.nix

@@ -160,6 +160,10 @@ in
     def cat_log():
          github.succeed("cat /var/log/httpd/*.log >&2")
 
+    client.succeed("systemctl start network-online.target")
+    github.succeed("systemctl start network-online.target")
+    client.wait_for_unit("network-online.target")
+    github.wait_for_unit("network-online.target")
     github.wait_for_unit("httpd.service")
 
     client.succeed("curl -v https://github.com/ >&2")

tests/nixos/sourcehut-flakes.nix

@@ -121,6 +121,10 @@ in
 
       start_all()
 
+      client.succeed("systemctl start network-online.target")
+      sourcehut.succeed("systemctl start network-online.target")
+      client.wait_for_unit("network-online.target")
+      sourcehut.wait_for_unit("network-online.target")
       sourcehut.wait_for_unit("httpd.service")
 
       client.succeed("curl -v https://git.sr.ht/ >&2")