From 904be6beaa376f7f3c0e8b6ee13f3b63280cba2b Mon Sep 17 00:00:00 2001 From: Qyriad Date: Mon, 22 Apr 2024 15:39:22 -0600 Subject: [PATCH 1/2] meson: fix cross compilation This should fix cross compilation in the base case, but this is difficult to test as cross compilation is broken in many different places right now. This should bring Meson back up to cross parity with the Make buildsystem though. Change-Id: If09be8142d1fc975a82b994143ff35be1297dad8 --- meson.build | 42 +++++++++++++++++++++++--------- package.nix | 52 ++++++++++++++++++++++++++++++++++++---- src/libstore/meson.build | 2 +- 3 files changed, 79 insertions(+), 17 deletions(-) diff --git a/meson.build b/meson.build index 2128ec6e2..8c990e66e 100644 --- a/meson.build +++ b/meson.build @@ -85,8 +85,8 @@ endif enable_docs = get_option('enable-docs') enable_internal_api_docs = get_option('internal-api-docs') -doxygen = find_program('doxygen', required : enable_internal_api_docs) -bash = find_program('bash') +doxygen = find_program('doxygen', required : enable_internal_api_docs, native : true) +bash = find_program('bash', native : true) rapidcheck_meson = dependency('rapidcheck', required : enable_internal_api_docs) @@ -114,6 +114,25 @@ endif cxx = meson.get_compiler('cpp') +# Translate some historical and Mesony CPU names to Lixy CPU names. +# FIXME(Qyriad): the 32-bit x86 code is not tested right now, because cross compilation for Lix +# to those architectures is currently broken for other reasons, namely: +# - nixos-23.11's x86_64-linux -> i686-linux glibc does not build (also applies to cppnix) +# - nixpkgs-unstable (as of 2024/04)'s boehmgc is not compatible with our patches +# It's also broken in cppnix, though. +host_cpu = host_machine.cpu_family() +if host_cpu in ['x86', 'i686', 'i386'] + # Meson considers 32-bit x86 CPUs to be "x86", and does not consider 64-bit + # x86 CPUs to be "x86" (instead using "x86_64", which needs no translation). + host_cpu = 'i686' +elif host_cpu == 'amd64' + # This should not be needed under normal circumstances, but someone could pass a --cross-file + # that sets the cpu_family to this. + host_cpu = 'x86_64' +elif host_cpu in ['armv6', 'armv7'] + host_cpu += 'l' +endif + host_system = host_machine.cpu_family() + '-' + host_machine.system() message('canonical Nix system name:', host_system) @@ -181,6 +200,7 @@ openssl = dependency('libcrypto', 'openssl', required : true) deps += openssl aws_sdk = dependency('aws-cpp-sdk-core', required : false) +aws_sdk_transfer = dependency('aws-cpp-sdk-transfer', required : false) if aws_sdk.found() # The AWS pkg-config adds -std=c++11. # https://github.com/aws/aws-sdk-cpp/issues/2673 @@ -198,7 +218,7 @@ if aws_sdk.found() 'AWS_VERSION_MINOR': s[1].to_int(), 'AWS_VERSION_PATCH': s[2].to_int(), } - aws_sdk_transfer = dependency('aws-cpp-sdk-transfer', required : true).partial_dependency( + aws_sdk_transfer = aws_sdk_transfer.partial_dependency( compile_args : false, includes : true, link_args : true, @@ -255,7 +275,7 @@ gtest = [ ] deps += gtest -toml11 = dependency('toml11', version : '>=3.7.0', required : true) +toml11 = dependency('toml11', version : '>=3.7.0', required : true, method : 'cmake') deps += toml11 nlohmann_json = dependency('nlohmann_json', required : true) @@ -272,17 +292,17 @@ deps += lix_doc # # Build-time tools # -coreutils = find_program('coreutils') -dot = find_program('dot', required : false) +coreutils = find_program('coreutils', native : true) +dot = find_program('dot', required : false, native : true) pymod = import('python') python = pymod.find_installation('python3') if enable_docs - mdbook = find_program('mdbook') + mdbook = find_program('mdbook', native : true) endif # Used to workaround https://github.com/mesonbuild/meson/issues/2320 in src/nix/meson.build. -installcmd = find_program('install') +installcmd = find_program('install', native : true) enable_embedded_sandbox_shell = get_option('enable-embedded-sandbox-shell') if enable_embedded_sandbox_shell @@ -307,9 +327,9 @@ endif # FIXME(Qyriad): the autoconf system checks that busybox has the "standalone" feature, indicating # that busybox sh won't run busybox applets as builtins (which would break our sandbox). -lsof = find_program('lsof') -bison = find_program('bison') -flex = find_program('flex') +lsof = find_program('lsof', native : true) +bison = find_program('bison', native : true) +flex = find_program('flex', native : true) # This is how Nix does generated headers... # other instances of header generation use a very similar command. diff --git a/package.nix b/package.nix index 9a2e08038..6b9fbf6d2 100644 --- a/package.nix +++ b/package.nix @@ -100,6 +100,43 @@ let testConfigureFlags = [ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" ]; + # Reimplementation of Nixpkgs' Meson cross file, with some additions to make + # it actually work. + mesonCrossFile = + let + cpuFamily = + platform: + with platform; + if isAarch32 then + "arm" + else if isx86_32 then + "x86" + else + platform.uname.processor; + in + builtins.toFile "lix-cross-file.conf" '' + [properties] + bindgen_clang_arguments = ['-target', '${stdenv.targetPlatform.config}'] + # Meson is convinced that if !buildPlatform.canExecute hostPlatform then we cannot + # build anything at all, which is not at all correct. If we can't execute the host + # platform, we'll just disable tests and doc gen. + needs_exe_wrapper = false + + [host_machine] + system = '${stdenv.targetPlatform.parsed.kernel.name}' + cpu_family = '${cpuFamily stdenv.targetPlatform}' + cpu = '${stdenv.targetPlatform.parsed.cpu.name}' + endian = ${if stdenv.targetPlatform.isLittleEndian then "'little'" else "'big'"} + + [binaries] + llvm-config = 'llvm-config-native' + rust = ['rustc', '--target', '${stdenv.targetPlatform.rust.rustcTargetSpec}'] + # Meson refuses to consider any CMake binary during cross compilation if it's + # not explicitly specified here, in the cross file. + # https://github.com/mesonbuild/meson/blob/0ed78cf6fa6d87c0738f67ae43525e661b50a8a2/mesonbuild/cmake/executor.py#L72 + cmake = 'cmake' + ''; + # The internal API docs need these for the build, but if we're not building # Nix itself, then these don't need to be propagated. maybePropagatedInputs = [ @@ -184,10 +221,15 @@ stdenv.mkDerivation (finalAttrs: { ] ++ lib.optional stdenv.hostPlatform.isStatic "-Denable-embedded-sandbox-shell=true" ++ lib.optional (finalAttrs.dontBuild) "-Denable-build=false" - # mesonConfigurePhase automatically passes -Dauto_features=enabled, - # so we must explicitly enable or disable features that we are not passing - # dependencies for. - ++ lib.singleton (lib.mesonEnable "internal-api-docs" internalApiDocs); + ++ [ + # mesonConfigurePhase automatically passes -Dauto_features=enabled, + # so we must explicitly enable or disable features that we are not passing + # dependencies for. + (lib.mesonEnable "internal-api-docs" internalApiDocs) + (lib.mesonBool "enable-tests" finalAttrs.doCheck) + (lib.mesonBool "enable-docs" canRunInstalled) + ] + ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "--cross-file=${mesonCrossFile}"; # We only include CMake so that Meson can locate toml11, which only ships CMake dependency metadata. dontUseCmakeConfigure = true; @@ -315,7 +357,7 @@ stdenv.mkDerivation (finalAttrs: { makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - doCheck = true; + doCheck = canRunInstalled; mesonCheckFlags = lib.optionals (buildWithMeson || forDevShell) [ "--suite=check" ]; diff --git a/src/libstore/meson.build b/src/libstore/meson.build index e1c6c267a..a6d7e6d56 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -11,7 +11,7 @@ foreach header : [ 'schema.sql', 'ca-specific-schema.sql' ] endforeach if enable_embedded_sandbox_shell - hexdump = find_program('hexdump', required : true) + hexdump = find_program('hexdump', required : true, native : true) embedded_sandbox_shell_gen = custom_target( 'embedded-sandbox-shell.gen.hh', command : [ From 77df8acc43a7ee30d5058e68d9ea8f1bae389006 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Wed, 17 Apr 2024 20:07:38 -0600 Subject: [PATCH 2/2] meson: flip the switch!! MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit makes Meson the default buildsystem for Lix. The Make buildsystem is now deprecated and will be removed soon, but has not yet, which will be done in a later commit when all seems good. The mesonBuild jobs have been removed, and have not been replaced with equivalent jobs to ensure the Make buildsystem still works. The full, new commands in a development shell are: $ meson setup ./build "--prefix=$out" $mesonFlags (A simple `meson setup ./build` will also build, but will do a different thing, not having the settings from package.nix applied.) $ meson compile -C build $ meson test -C build --suite=check $ meson install -C build $ meson test -C build --suite=installcheck (Check and installcheck may both be done after install, allowing you to omit the --suite argument entirely, but this is the order package.nix runs them in.) If tests fail and Meson helpfully has no output for why, use the `--print-error-logs` option to `meson test`. Why this is not the default I cannot explain. If you change a setting in the buildsystem, most cases will automatically regenerate the Meson configuration, but some cases, like trying to build a specific target whose name is new to the buildsystem (e.g. `meson compile -C build src/libmelt/libmelt.dylib`, when `libmelt.dylib` did not exist as a target the last time the buildsystem was generated), then you can reconfigure using new settings but existing options, and only recompiling stuff affected by the changes: $ meson setup --reconfigure build Note that changes to the default values in `meson.options` or in the `default_options :` argument to project() are NOT propagated with `--reconfigure`. If you want a totally clean build, you can use: $ meson setup --wipe build That will work regardless of if `./build` exists or not. Specific, named targets may be addressed in `meson build -C build ` with "target ID" if there is one, which is the first string argument passed to target functions that have one, and unrelated to the variable name, e.g.: libexpr_dylib = library('nixexpr', …) can be addressed with: $ meson compile -C build nixexpr All targets may be addressed as their output, relative to the build directory, e.g.: $ meson compile -C build src/libexpr/libnixexpr.so But Meson does not consider intermediate files like object files targets. To build a specific object file, use Ninja directly and specify the output file relative to the build directory: $ ninja -C build src/libexpr/libnixexpr.so.p/nixexpr.cc.o To inspect the canonical source of truth on what the state of the buildsystem configuration is, use: $ meson introspect Have fun! Change-Id: Ia3e7b1e6fae26daf3162e655b4ded611a5cd57ad --- flake.nix | 22 ---------------------- package.nix | 2 +- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/flake.nix b/flake.nix index bb03b16ab..e8526f5c4 100644 --- a/flake.nix +++ b/flake.nix @@ -196,24 +196,6 @@ } ); - # FIXME(Qyriad): remove this when the migration to Meson has been completed. - # NOTE: mesonBuildClang depends on mesonBuild depends on build to avoid OOMs - # on aarch64 builders caused by too many parallel compiler/linker processes. - mesonBuild = forAllSystems ( - system: - (self.packages.${system}.nix.override { buildWithMeson = true; }).overrideAttrs (prev: { - buildInputs = prev.buildInputs ++ [ self.packages.${system}.nix ]; - }) - ); - mesonBuildClang = forAllSystems ( - system: - (nixpkgsFor.${system}.stdenvs.clangStdenvPackages.nix.override { buildWithMeson = true; }) - .overrideAttrs - (prev: { - buildInputs = prev.buildInputs ++ [ self.hydraJobs.mesonBuild.${system} ]; - }) - ); - # Perl bindings for various platforms. perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings); @@ -237,7 +219,6 @@ inherit (pkgs) build-release-notes; internalApiDocs = true; busybox-sandbox-shell = pkgs.busybox-sandbox-shell; - buildWithMeson = true; }; in nix.overrideAttrs (prev: { @@ -367,9 +348,6 @@ checks = forAllSystems ( system: { - # FIXME(Qyriad): remove this when the migration to Meson has been completed. - mesonBuild = self.hydraJobs.mesonBuild.${system}; - mesonBuildClang = self.hydraJobs.mesonBuildClang.${system}; binaryTarball = self.hydraJobs.binaryTarball.${system}; perlBindings = self.hydraJobs.perlBindings.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; diff --git a/package.nix b/package.nix index 6b9fbf6d2..63372caa1 100644 --- a/package.nix +++ b/package.nix @@ -62,7 +62,7 @@ # FIXME(Qyriad): build Lix using Meson instead of autoconf and make. # This flag will be removed when the migration to Meson is complete. - buildWithMeson ? false, + buildWithMeson ? true, # Not a real argument, just the only way to approximate let-binding some # stuff for argument defaults.