From 15380b4c6e54aa60d50154ea3d45510750ddda14 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Fri, 1 Mar 2024 13:15:44 -0700 Subject: [PATCH] package: refactor Nix out of flake.nix and into package.nix This series takes a somewhat different approach from the flake rework done in NixOS/nix. The package.nix here does not provide callPackage options for all the various settings in the build, and instead the other places Nix derivations are used (like internal-api-docs) will .overrideAttrs the normal Nix package derivation. This more closely matches how these things were structured originally, and results in less churn and more atomicity in these changes. In the future, package.nix likely will migrate to have more build options in the callPackage arguments, but we are also planning to rewrite the build system anyway. Change-Id: I170c4e5a4184bab62e1fd75e56db876d4ff116cf --- flake.nix | 216 +++++++++++++++--------------------------------- package.nix | 232 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 299 insertions(+), 149 deletions(-) create mode 100644 package.nix diff --git a/flake.nix b/flake.nix index 3e5023cb6..a3a395b1c 100644 --- a/flake.nix +++ b/flake.nix @@ -118,11 +118,34 @@ cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); - commonDeps = - { pkgs - , isStatic ? pkgs.stdenv.hostPlatform.isStatic - }: - with pkgs; rec { + commonDeps = { + pkgs, + isStatic ? pkgs.stdenv.hostPlatform.isStatic + }: let + inherit (pkgs) stdenv buildPackages + busybox curl bzip2 xz brotli editline openssl sqlite libarchive boost + libseccomp libsodium libcpuid gtest rapidcheck aws-sdk-cpp boehmgc nlohmann_json + lowdown; + changelog-d = pkgs.buildPackages.callPackage ./misc/changelog-d.nix { }; + boehmgc-nix = (boehmgc.override { + enableLargeConfig = true; + }).overrideAttrs (o: { + patches = (o.patches or [ ]) ++ [ + ./boehmgc-coroutine-sp-fallback.diff + + # https://github.com/ivmai/bdwgc/pull/586 + ./boehmgc-traceable_allocator-public.diff + ]; + }); + in rec { + calledPackage = pkgs.callPackage ./package.nix { + inherit stdenv versionSuffix fileset changelog-d officialRelease buildUnreleasedNotes lowdown; + boehmgc = boehmgc-nix; + busybox-sandbox-shell = sh; + }; + + inherit boehmgc-nix; + # Use "busybox-sandbox-shell" if present, # if not (legacy) fallback and hope it's sufficient. sh = pkgs.busybox-sandbox-shell or (busybox.override { @@ -166,45 +189,12 @@ "--enable-internal-api-docs" ]; - changelog-d = pkgs.buildPackages.callPackage ./misc/changelog-d.nix { }; + inherit changelog-d; + nativeBuildDeps = calledPackage.nativeBuildInputs; - nativeBuildDeps = - [ - buildPackages.bison - buildPackages.flex - (lib.getBin buildPackages.lowdown) - buildPackages.mdbook - buildPackages.mdbook-linkcheck - buildPackages.autoconf-archive - buildPackages.autoreconfHook - buildPackages.pkg-config + buildDeps = calledPackage.buildInputs; - # Tests - buildPackages.git - buildPackages.mercurial # FIXME: remove? only needed for tests - buildPackages.jq # Also for custom mdBook preprocessor. - ] - ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)] - # Official releases don't have rl-next, so we don't need to compile a changelog - ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d - ; - - buildDeps = - [ curl - bzip2 xz brotli editline - openssl sqlite - libarchive - boost - lowdown - libsodium - ] - ++ lib.optionals stdenv.isLinux [libseccomp] - ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; - - checkDeps = [ - gtest - rapidcheck - ]; + checkDeps = calledPackage.finalAttrs.passthru._checkInputs; internalApiDocsDeps = [ buildPackages.doxygen @@ -216,20 +206,7 @@ customMemoryManagement = false; }); - propagatedDeps = - [ ((boehmgc.override { - enableLargeConfig = true; - }).overrideAttrs(o: { - patches = (o.patches or []) ++ [ - ./boehmgc-coroutine-sp-fallback.diff - - # https://github.com/ivmai/bdwgc/pull/586 - ./boehmgc-traceable_allocator-public.diff - ]; - }) - ) - nlohmann_json - ]; + propagatedDeps = calledPackage.propagatedBuildInputs; }; installScriptFor = systems: @@ -387,109 +364,50 @@ ''; overlayFor = getStdenv: final: prev: - let currentStdenv = getStdenv final; in - { - nixStable = prev.nix; - - nix = - with final; - with commonDeps { + let + currentStdenv = getStdenv final; + comDeps = with final; commonDeps { inherit pkgs; inherit (currentStdenv.hostPlatform) isStatic; }; - let - canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform; - in currentStdenv.mkDerivation (finalAttrs: { - name = "nix-${version}"; - inherit version; + in { + nixStable = prev.nix; - src = nixSrc; - VERSION_SUFFIX = versionSuffix; + # Forward from the previous stage as we don’t want it to pick the lowdown override + nixUnstable = prev.nixUnstable; - outputs = [ "out" "dev" "doc" ]; + inherit (comDeps) boehmgc-nix; - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps - # There have been issues building these dependencies - ++ lib.optionals (currentStdenv.hostPlatform == currentStdenv.buildPlatform) awsDeps - ++ lib.optionals finalAttrs.doCheck checkDeps; + default-busybox-sandbox-shell = final.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y - propagatedBuildInputs = propagatedDeps; + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y - disallowedReferences = [ boost ]; - - preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic) - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - ${lib.optionalString currentStdenv.hostPlatform.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - ''} - ${lib.optionalString currentStdenv.hostPlatform.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true - done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - ''} - ''; - - configureFlags = configureFlags ++ - [ "--sysconfdir=/etc" ] ++ - lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" ++ - [ (lib.enableFeature finalAttrs.doCheck "tests") ] ++ - lib.optionals finalAttrs.doCheck testConfigureFlags ++ - lib.optional (!canRunInstalled) "--disable-doc-gen"; - - enableParallelBuilding = true; - - makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - - doCheck = true; - - installFlags = "sysconfdir=$(out)/etc"; - - postInstall = '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - ${lib.optionalString currentStdenv.hostPlatform.isStatic '' - mkdir -p $out/nix-support - echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - ''} - ${lib.optionalString currentStdenv.isDarwin '' - install_name_tool \ - -change ${boost}/lib/libboost_context.dylib \ - $out/lib/libboost_context.dylib \ - $out/lib/libnixutil.dylib - ''} + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y ''; + }; - doInstallCheck = finalAttrs.doCheck; - installCheckFlags = "sysconfdir=$(out)/etc"; - installCheckTarget = "installcheck"; # work around buggy detection in stdenv - - preInstallCheck = lib.optionalString stdenv.hostPlatform.isDarwin '' - export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES - ''; - - separateDebugInfo = !currentStdenv.hostPlatform.isStatic; - - strictDeps = true; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - - passthru.perl-bindings = final.callPackage ./perl { - inherit fileset; - stdenv = currentStdenv; - }; - - meta.platforms = lib.platforms.unix; - }); + nix = final.callPackage ./package.nix { + inherit versionSuffix fileset; + stdenv = currentStdenv; + boehmgc = final.boehmgc-nix; + busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; + }; }; in { diff --git a/package.nix b/package.nix new file mode 100644 index 000000000..941d961a6 --- /dev/null +++ b/package.nix @@ -0,0 +1,232 @@ +{ + pkgs, + lib, + stdenv, + autoconf-archive, + autoreconfHook, + aws-sdk-cpp, + boehmgc, + nlohmann_json, + bison, + changelog-d, + boost, + brotli, + bzip2, + curl, + editline, + fileset, + flex, + git, + gtest, + jq, + libarchive, + libcpuid, + libseccomp, + libsodium, + lowdown, + mdbook, + mdbook-linkcheck, + mercurial, + openssl, + pkg-config, + rapidcheck, + sqlite, + util-linuxMinimal ? utillinuxMinimal, + utillinuxMinimal ? null, + xz, + + busybox-sandbox-shell ? null, + + pname ? "nix", + versionSuffix ? "", + officialRelease ? true, + # Set to true to build the release notes for the next release. + buildUnreleasedNotes ? false, + + # Not a real argument, just the only way to approximate let-binding some + # stuff for argument defaults. + __forDefaults ? { + canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + }, +}: let + inherit (__forDefaults) canRunInstalled; + + version = lib.fileContents ./.version + versionSuffix; + + # .gitignore has already been processed, so any changes in it are irrelevant + # at this point. It is not represented verbatim for test purposes because + # that would interfere with repo semantics. + baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.; + src = fileset.toSource { + root = ./.; + fileset = fileset.intersection baseFiles (fileset.unions [ + ./.version + ./boehmgc-coroutine-sp-fallback.diff + ./configure.ac + ./doc + ./local.mk + ./m4 + ./Makefile + ./Makefile.config.in + ./misc + ./mk + ./precompiled-headers.h + ./src + ./tests/functional + ./tests/unit + ./unit-test-data + ./COPYING + ./scripts/local.mk + (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) + # TODO: do we really need README.md? It doesn't seem used in the build. + ./README.md + ]); + }; + + aws-sdk-cpp-nix = aws-sdk-cpp.override { + apis = [ "s3" "transfer" ]; + customMemoryManagement = false; + }; + + testConfigureFlags = [ + "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" + ]; + +in stdenv.mkDerivation (finalAttrs: { + name = "nix-${version}"; + inherit version; + + inherit src; + + VERSION_SUFFIX = versionSuffix; + + outputs = [ "out" "dev" "doc" ]; + + nativeBuildInputs = [ + bison + flex + (lib.getBin lowdown) + mdbook + mdbook-linkcheck + autoconf-archive + autoreconfHook + pkg-config + + # Tests + git + mercurial + jq + ] ++ lib.optional stdenv.hostPlatform.isLinux util-linuxMinimal + ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d; + + buildInputs = [ + curl + bzip2 + xz + brotli + editline + openssl + sqlite + libarchive + boost + lowdown + libsodium + ] + ++ lib.optionals stdenv.isLinux [ libseccomp ] + ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid + # There have been issues building these dependencies + ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform) aws-sdk-cpp-nix + # FIXME(Qyriad): This is how the flake.nix version does it, but this is cursed. + ++ lib.optionals (finalAttrs.doCheck) finalAttrs.passthru._checkInputs + ; + + passthru._checkInputs = [ + gtest + rapidcheck + ]; + + propagatedBuildInputs = [ + boehmgc + nlohmann_json + ]; + + disallowedReferences = [ + boost + ]; + + preConfigure = lib.optionalString (! stdenv.hostPlatform.isStatic) '' + # Copy libboost_context so we don't get all of Boost in our closure. + # https://github.com/NixOS/nixpkgs/issues/45462 + mkdir -p $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + rm -f $out/lib/*.a + ${lib.optionalString stdenv.hostPlatform.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + ''} + ${lib.optionalString stdenv.hostPlatform.isDarwin '' + for LIB in $out/lib/*.dylib; do + chmod u+w $LIB + install_name_tool -id $LIB $LIB + install_name_tool -delete_rpath ${boost}/lib/ $LIB || true + done + install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib + ''} + ''; + + configureFlags = lib.optionals stdenv.isLinux [ + "--with-boost=${boost}/lib" + "--with-sandbox-shell=${busybox-sandbox-shell}/bin/busybox" + ] ++ lib.optionals (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) [ + "LDFLAGS=-fuse-ld=gold" + ] ++ [ "--sysconfdir=/etc" ] + ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" + ++ [ (lib.enableFeature finalAttrs.doCheck "tests") ] + ++ lib.optionals finalAttrs.doCheck testConfigureFlags + ++ lib.optional (!canRunInstalled) "--disable-doc-gen" + ; + + enableParallelBuilding = true; + + makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; + + doCheck = true; + + installFlags = "sysconfdir=$(out)/etc"; + + postInstall = '' + mkdir -p $doc/nix-support + echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products + ${lib.optionalString stdenv.hostPlatform.isStatic '' + mkdir -p $out/nix-support + echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products + ''} + ${lib.optionalString stdenv.isDarwin '' + install_name_tool \ + -change ${boost}/lib/libboost_context.dylib \ + $out/lib/libboost_context.dylib \ + $out/lib/libnixutil.dylib + ''} + ''; + + doInstallCheck = finalAttrs.doCheck; + installCheckFlags = "sysconfdir=$(out)/etc"; + installCheckTarget = "installcheck"; # work around buggy detection in stdenv + + preInstallCheck = lib.optionalString stdenv.hostPlatform.isDarwin '' + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + ''; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta.platforms = lib.platforms.unix; + + passthru.finalAttrs = finalAttrs; + passthru.perl-bindings = pkgs.callPackage ./perl { + inherit fileset stdenv; + }; +})