package: refactor Nix out of flake.nix and into package.nix

This series takes a somewhat different approach from the flake rework
done in NixOS/nix. The package.nix here does not provide callPackage
options for all the various settings in the build, and instead the other
places Nix derivations are used (like internal-api-docs) will .overrideAttrs
the normal Nix package derivation. This more closely matches how these
things were structured originally, and results in less churn and more
atomicity in these changes.

In the future, package.nix likely will migrate to have more build
options in the callPackage arguments, but we are also planning to
rewrite the build system anyway.

Change-Id: I170c4e5a4184bab62e1fd75e56db876d4ff116cf
This commit is contained in:
Qyriad 2024-03-01 13:15:44 -07:00
parent b221a14f0a
commit f8efdea4a2
2 changed files with 300 additions and 149 deletions

217
flake.nix
View file

@ -118,11 +118,34 @@
cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv");
}); });
commonDeps = commonDeps = {
{ pkgs pkgs,
, isStatic ? pkgs.stdenv.hostPlatform.isStatic isStatic ? pkgs.stdenv.hostPlatform.isStatic
}: }: let
with pkgs; rec { inherit (pkgs) stdenv buildPackages
busybox curl bzip2 xz brotli editline openssl sqlite libarchive boost
libseccomp libsodium libcpuid gtest rapidcheck aws-sdk-cpp boehmgc nlohmann_json
lowdown;
changelog-d = pkgs.buildPackages.callPackage ./misc/changelog-d.nix { };
boehmgc-nix = (boehmgc.override {
enableLargeConfig = true;
}).overrideAttrs (o: {
patches = (o.patches or [ ]) ++ [
./boehmgc-coroutine-sp-fallback.diff
# https://github.com/ivmai/bdwgc/pull/586
./boehmgc-traceable_allocator-public.diff
];
});
in rec {
calledPackage = pkgs.callPackage ./package.nix {
inherit stdenv versionSuffix fileset changelog-d officialRelease buildUnreleasedNotes lowdown;
boehmgc = boehmgc-nix;
busybox-sandbox-shell = sh;
};
inherit boehmgc-nix;
# Use "busybox-sandbox-shell" if present, # Use "busybox-sandbox-shell" if present,
# if not (legacy) fallback and hope it's sufficient. # if not (legacy) fallback and hope it's sufficient.
sh = pkgs.busybox-sandbox-shell or (busybox.override { sh = pkgs.busybox-sandbox-shell or (busybox.override {
@ -166,45 +189,12 @@
"--enable-internal-api-docs" "--enable-internal-api-docs"
]; ];
changelog-d = pkgs.buildPackages.callPackage ./misc/changelog-d.nix { }; inherit changelog-d;
nativeBuildDeps = calledPackage.nativeBuildInputs;
nativeBuildDeps = buildDeps = calledPackage.buildInputs;
[
buildPackages.bison
buildPackages.flex
(lib.getBin buildPackages.lowdown)
buildPackages.mdbook
buildPackages.mdbook-linkcheck
buildPackages.autoconf-archive
buildPackages.autoreconfHook
buildPackages.pkg-config
# Tests checkDeps = calledPackage.finalAttrs.passthru._checkInputs;
buildPackages.git
buildPackages.mercurial # FIXME: remove? only needed for tests
buildPackages.jq # Also for custom mdBook preprocessor.
]
++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]
# Official releases don't have rl-next, so we don't need to compile a changelog
++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d
;
buildDeps =
[ curl
bzip2 xz brotli editline
openssl sqlite
libarchive
boost
lowdown
libsodium
]
++ lib.optionals stdenv.isLinux [libseccomp]
++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid;
checkDeps = [
gtest
rapidcheck
];
internalApiDocsDeps = [ internalApiDocsDeps = [
buildPackages.doxygen buildPackages.doxygen
@ -216,20 +206,7 @@
customMemoryManagement = false; customMemoryManagement = false;
}); });
propagatedDeps = propagatedDeps = calledPackage.propagatedBuildInputs;
[ ((boehmgc.override {
enableLargeConfig = true;
}).overrideAttrs(o: {
patches = (o.patches or []) ++ [
./boehmgc-coroutine-sp-fallback.diff
# https://github.com/ivmai/bdwgc/pull/586
./boehmgc-traceable_allocator-public.diff
];
})
)
nlohmann_json
];
}; };
installScriptFor = systems: installScriptFor = systems:
@ -387,109 +364,51 @@
''; '';
overlayFor = getStdenv: final: prev: overlayFor = getStdenv: final: prev:
let currentStdenv = getStdenv final; in let
{ currentStdenv = getStdenv final;
nixStable = prev.nix; comDeps = with final; commonDeps {
nix =
with final;
with commonDeps {
inherit pkgs; inherit pkgs;
inherit (currentStdenv.hostPlatform) isStatic; inherit (currentStdenv.hostPlatform) isStatic;
}; };
let inherit (final) stdenv boost;
canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform; in {
in currentStdenv.mkDerivation (finalAttrs: { nixStable = prev.nix;
name = "nix-${version}";
inherit version;
src = nixSrc; # Forward from the previous stage as we dont want it to pick the lowdown override
VERSION_SUFFIX = versionSuffix; nixUnstable = prev.nixUnstable;
outputs = [ "out" "dev" "doc" ]; inherit (comDeps) boehmgc-nix;
nativeBuildInputs = nativeBuildDeps; default-busybox-sandbox-shell = final.busybox.override {
buildInputs = buildDeps useMusl = true;
# There have been issues building these dependencies enableStatic = true;
++ lib.optionals (currentStdenv.hostPlatform == currentStdenv.buildPlatform) awsDeps enableMinimal = true;
++ lib.optionals finalAttrs.doCheck checkDeps; extraConfig = ''
CONFIG_FEATURE_FANCY_ECHO y
CONFIG_FEATURE_SH_MATH y
CONFIG_FEATURE_SH_MATH_64 y
propagatedBuildInputs = propagatedDeps; CONFIG_ASH y
CONFIG_ASH_OPTIMIZE_FOR_SIZE y
disallowedReferences = [ boost ]; CONFIG_ASH_ALIAS y
CONFIG_ASH_BASH_COMPAT y
preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic) CONFIG_ASH_CMDCMD y
'' CONFIG_ASH_ECHO y
# Copy libboost_context so we don't get all of Boost in our closure. CONFIG_ASH_GETOPTS y
# https://github.com/NixOS/nixpkgs/issues/45462 CONFIG_ASH_INTERNAL_GLOB y
mkdir -p $out/lib CONFIG_ASH_JOB_CONTROL y
cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib CONFIG_ASH_PRINTF y
rm -f $out/lib/*.a CONFIG_ASH_TEST y
${lib.optionalString currentStdenv.hostPlatform.isLinux ''
chmod u+w $out/lib/*.so.*
patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
''}
${lib.optionalString currentStdenv.hostPlatform.isDarwin ''
for LIB in $out/lib/*.dylib; do
chmod u+w $LIB
install_name_tool -id $LIB $LIB
install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
done
install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
''}
'';
configureFlags = configureFlags ++
[ "--sysconfdir=/etc" ] ++
lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" ++
[ (lib.enableFeature finalAttrs.doCheck "tests") ] ++
lib.optionals finalAttrs.doCheck testConfigureFlags ++
lib.optional (!canRunInstalled) "--disable-doc-gen";
enableParallelBuilding = true;
makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
doCheck = true;
installFlags = "sysconfdir=$(out)/etc";
postInstall = ''
mkdir -p $doc/nix-support
echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
${lib.optionalString currentStdenv.hostPlatform.isStatic ''
mkdir -p $out/nix-support
echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
''}
${lib.optionalString currentStdenv.isDarwin ''
install_name_tool \
-change ${boost}/lib/libboost_context.dylib \
$out/lib/libboost_context.dylib \
$out/lib/libnixutil.dylib
''}
''; '';
};
doInstallCheck = finalAttrs.doCheck; nix = final.callPackage ./package.nix {
installCheckFlags = "sysconfdir=$(out)/etc"; inherit (final) stdenv;
installCheckTarget = "installcheck"; # work around buggy detection in stdenv inherit versionSuffix fileset;
boehmgc = final.boehmgc-nix;
preInstallCheck = lib.optionalString stdenv.hostPlatform.isDarwin '' busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES };
'';
separateDebugInfo = !currentStdenv.hostPlatform.isStatic;
strictDeps = true;
hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
passthru.perl-bindings = final.callPackage ./perl {
inherit fileset;
stdenv = currentStdenv;
};
meta.platforms = lib.platforms.unix;
});
}; };
in { in {

232
package.nix Normal file
View file

@ -0,0 +1,232 @@
{
pkgs,
lib,
stdenv,
autoconf-archive,
autoreconfHook,
aws-sdk-cpp,
boehmgc,
nlohmann_json,
bison,
changelog-d,
boost,
brotli,
bzip2,
curl,
editline,
fileset,
flex,
git,
gtest,
jq,
libarchive,
libcpuid,
libseccomp,
libsodium,
lowdown,
mdbook,
mdbook-linkcheck,
mercurial,
openssl,
pkg-config,
rapidcheck,
sqlite,
util-linuxMinimal ? utillinuxMinimal,
utillinuxMinimal ? null,
xz,
busybox-sandbox-shell ? null,
pname ? "nix",
versionSuffix ? "",
officialRelease ? true,
# Set to true to build the release notes for the next release.
buildUnreleasedNotes ? false,
# Not a real argument, just the only way to approximate let-binding some
# stuff for argument defaults.
__forDefaults ? {
canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
},
}: let
inherit (__forDefaults) canRunInstalled;
version = lib.fileContents ./.version + versionSuffix;
# .gitignore has already been processed, so any changes in it are irrelevant
# at this point. It is not represented verbatim for test purposes because
# that would interfere with repo semantics.
baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.;
src = fileset.toSource {
root = ./.;
fileset = fileset.intersection baseFiles (fileset.unions [
./.version
./boehmgc-coroutine-sp-fallback.diff
./configure.ac
./doc
./local.mk
./m4
./Makefile
./Makefile.config.in
./misc
./mk
./precompiled-headers.h
./src
./tests/functional
./tests/unit
./unit-test-data
./COPYING
./scripts/local.mk
(fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
# TODO: do we really need README.md? It doesn't seem used in the build.
./README.md
]);
};
aws-sdk-cpp-nix = aws-sdk-cpp.override {
apis = [ "s3" "transfer" ];
customMemoryManagement = false;
};
testConfigureFlags = [
"RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include"
];
in stdenv.mkDerivation (finalAttrs: {
name = "nix-${version}";
inherit version;
inherit src;
VERSION_SUFFIX = versionSuffix;
outputs = [ "out" "dev" "doc" ];
nativeBuildInputs = [
bison
flex
(lib.getBin lowdown)
mdbook
mdbook-linkcheck
autoconf-archive
autoreconfHook
pkg-config
# Tests
git
mercurial
jq
] ++ lib.optional stdenv.hostPlatform.isLinux util-linuxMinimal
++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d;
buildInputs = [
curl
bzip2
xz
brotli
editline
openssl
sqlite
libarchive
boost
lowdown
libsodium
]
++ lib.optionals stdenv.isLinux [ libseccomp ]
++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
# There have been issues building these dependencies
++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform) aws-sdk-cpp-nix
# FIXME(Qyriad): This is how the flake.nix version does it, but this is cursed.
++ lib.optionals (finalAttrs.doCheck) finalAttrs.passthru._checkInputs
;
passthru._checkInputs = [
gtest
rapidcheck
];
propagatedBuildInputs = [
boehmgc
nlohmann_json
];
disallowedReferences = [
boost
];
preConfigure = lib.optionalString (! stdenv.hostPlatform.isStatic) ''
# Copy libboost_context so we don't get all of Boost in our closure.
# https://github.com/NixOS/nixpkgs/issues/45462
mkdir -p $out/lib
cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
rm -f $out/lib/*.a
${lib.optionalString stdenv.hostPlatform.isLinux ''
chmod u+w $out/lib/*.so.*
patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
''}
${lib.optionalString stdenv.hostPlatform.isDarwin ''
for LIB in $out/lib/*.dylib; do
chmod u+w $LIB
install_name_tool -id $LIB $LIB
install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
done
install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
''}
'';
configureFlags = lib.optionals stdenv.isLinux [
"--with-boost=${boost}/lib"
"--with-sandbox-shell=${busybox-sandbox-shell}/bin/busybox"
] ++ lib.optionals (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) [
"LDFLAGS=-fuse-ld=gold"
] ++ [ "--sysconfdir=/etc" ]
++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell"
++ [ (lib.enableFeature finalAttrs.doCheck "tests") ]
++ lib.optionals finalAttrs.doCheck testConfigureFlags
++ lib.optional (!canRunInstalled) "--disable-doc-gen"
;
enableParallelBuilding = true;
makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
doCheck = true;
installFlags = "sysconfdir=$(out)/etc";
postInstall = ''
mkdir -p $doc/nix-support
echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
${lib.optionalString stdenv.hostPlatform.isStatic ''
mkdir -p $out/nix-support
echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
''}
${lib.optionalString stdenv.isDarwin ''
install_name_tool \
-change ${boost}/lib/libboost_context.dylib \
$out/lib/libboost_context.dylib \
$out/lib/libnixutil.dylib
''}
'';
doInstallCheck = finalAttrs.doCheck;
installCheckFlags = "sysconfdir=$(out)/etc";
installCheckTarget = "installcheck"; # work around buggy detection in stdenv
preInstallCheck = lib.optionalString stdenv.hostPlatform.isDarwin ''
export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
'';
separateDebugInfo = !stdenv.hostPlatform.isStatic;
strictDeps = true;
hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
meta.platforms = lib.platforms.unix;
passthru.finalAttrs = finalAttrs;
passthru.perl-bindings = pkgs.callPackage ./perl {
inherit fileset stdenv;
};
})