From 15380b4c6e54aa60d50154ea3d45510750ddda14 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Fri, 1 Mar 2024 13:15:44 -0700 Subject: [PATCH 1/6] package: refactor Nix out of flake.nix and into package.nix This series takes a somewhat different approach from the flake rework done in NixOS/nix. The package.nix here does not provide callPackage options for all the various settings in the build, and instead the other places Nix derivations are used (like internal-api-docs) will .overrideAttrs the normal Nix package derivation. This more closely matches how these things were structured originally, and results in less churn and more atomicity in these changes. In the future, package.nix likely will migrate to have more build options in the callPackage arguments, but we are also planning to rewrite the build system anyway. Change-Id: I170c4e5a4184bab62e1fd75e56db876d4ff116cf --- flake.nix | 216 +++++++++++++++--------------------------------- package.nix | 232 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 299 insertions(+), 149 deletions(-) create mode 100644 package.nix diff --git a/flake.nix b/flake.nix index 3e5023cb6..a3a395b1c 100644 --- a/flake.nix +++ b/flake.nix @@ -118,11 +118,34 @@ cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); - commonDeps = - { pkgs - , isStatic ? pkgs.stdenv.hostPlatform.isStatic - }: - with pkgs; rec { + commonDeps = { + pkgs, + isStatic ? pkgs.stdenv.hostPlatform.isStatic + }: let + inherit (pkgs) stdenv buildPackages + busybox curl bzip2 xz brotli editline openssl sqlite libarchive boost + libseccomp libsodium libcpuid gtest rapidcheck aws-sdk-cpp boehmgc nlohmann_json + lowdown; + changelog-d = pkgs.buildPackages.callPackage ./misc/changelog-d.nix { }; + boehmgc-nix = (boehmgc.override { + enableLargeConfig = true; + }).overrideAttrs (o: { + patches = (o.patches or [ ]) ++ [ + ./boehmgc-coroutine-sp-fallback.diff + + # https://github.com/ivmai/bdwgc/pull/586 + ./boehmgc-traceable_allocator-public.diff + ]; + }); + in rec { + calledPackage = pkgs.callPackage ./package.nix { + inherit stdenv versionSuffix fileset changelog-d officialRelease buildUnreleasedNotes lowdown; + boehmgc = boehmgc-nix; + busybox-sandbox-shell = sh; + }; + + inherit boehmgc-nix; + # Use "busybox-sandbox-shell" if present, # if not (legacy) fallback and hope it's sufficient. sh = pkgs.busybox-sandbox-shell or (busybox.override { @@ -166,45 +189,12 @@ "--enable-internal-api-docs" ]; - changelog-d = pkgs.buildPackages.callPackage ./misc/changelog-d.nix { }; + inherit changelog-d; + nativeBuildDeps = calledPackage.nativeBuildInputs; - nativeBuildDeps = - [ - buildPackages.bison - buildPackages.flex - (lib.getBin buildPackages.lowdown) - buildPackages.mdbook - buildPackages.mdbook-linkcheck - buildPackages.autoconf-archive - buildPackages.autoreconfHook - buildPackages.pkg-config + buildDeps = calledPackage.buildInputs; - # Tests - buildPackages.git - buildPackages.mercurial # FIXME: remove? only needed for tests - buildPackages.jq # Also for custom mdBook preprocessor. - ] - ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)] - # Official releases don't have rl-next, so we don't need to compile a changelog - ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d - ; - - buildDeps = - [ curl - bzip2 xz brotli editline - openssl sqlite - libarchive - boost - lowdown - libsodium - ] - ++ lib.optionals stdenv.isLinux [libseccomp] - ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; - - checkDeps = [ - gtest - rapidcheck - ]; + checkDeps = calledPackage.finalAttrs.passthru._checkInputs; internalApiDocsDeps = [ buildPackages.doxygen @@ -216,20 +206,7 @@ customMemoryManagement = false; }); - propagatedDeps = - [ ((boehmgc.override { - enableLargeConfig = true; - }).overrideAttrs(o: { - patches = (o.patches or []) ++ [ - ./boehmgc-coroutine-sp-fallback.diff - - # https://github.com/ivmai/bdwgc/pull/586 - ./boehmgc-traceable_allocator-public.diff - ]; - }) - ) - nlohmann_json - ]; + propagatedDeps = calledPackage.propagatedBuildInputs; }; installScriptFor = systems: @@ -387,109 +364,50 @@ ''; overlayFor = getStdenv: final: prev: - let currentStdenv = getStdenv final; in - { - nixStable = prev.nix; - - nix = - with final; - with commonDeps { + let + currentStdenv = getStdenv final; + comDeps = with final; commonDeps { inherit pkgs; inherit (currentStdenv.hostPlatform) isStatic; }; - let - canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform; - in currentStdenv.mkDerivation (finalAttrs: { - name = "nix-${version}"; - inherit version; + in { + nixStable = prev.nix; - src = nixSrc; - VERSION_SUFFIX = versionSuffix; + # Forward from the previous stage as we don’t want it to pick the lowdown override + nixUnstable = prev.nixUnstable; - outputs = [ "out" "dev" "doc" ]; + inherit (comDeps) boehmgc-nix; - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps - # There have been issues building these dependencies - ++ lib.optionals (currentStdenv.hostPlatform == currentStdenv.buildPlatform) awsDeps - ++ lib.optionals finalAttrs.doCheck checkDeps; + default-busybox-sandbox-shell = final.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y - propagatedBuildInputs = propagatedDeps; + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y - disallowedReferences = [ boost ]; - - preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic) - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - ${lib.optionalString currentStdenv.hostPlatform.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - ''} - ${lib.optionalString currentStdenv.hostPlatform.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true - done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - ''} - ''; - - configureFlags = configureFlags ++ - [ "--sysconfdir=/etc" ] ++ - lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" ++ - [ (lib.enableFeature finalAttrs.doCheck "tests") ] ++ - lib.optionals finalAttrs.doCheck testConfigureFlags ++ - lib.optional (!canRunInstalled) "--disable-doc-gen"; - - enableParallelBuilding = true; - - makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - - doCheck = true; - - installFlags = "sysconfdir=$(out)/etc"; - - postInstall = '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - ${lib.optionalString currentStdenv.hostPlatform.isStatic '' - mkdir -p $out/nix-support - echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - ''} - ${lib.optionalString currentStdenv.isDarwin '' - install_name_tool \ - -change ${boost}/lib/libboost_context.dylib \ - $out/lib/libboost_context.dylib \ - $out/lib/libnixutil.dylib - ''} + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y ''; + }; - doInstallCheck = finalAttrs.doCheck; - installCheckFlags = "sysconfdir=$(out)/etc"; - installCheckTarget = "installcheck"; # work around buggy detection in stdenv - - preInstallCheck = lib.optionalString stdenv.hostPlatform.isDarwin '' - export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES - ''; - - separateDebugInfo = !currentStdenv.hostPlatform.isStatic; - - strictDeps = true; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - - passthru.perl-bindings = final.callPackage ./perl { - inherit fileset; - stdenv = currentStdenv; - }; - - meta.platforms = lib.platforms.unix; - }); + nix = final.callPackage ./package.nix { + inherit versionSuffix fileset; + stdenv = currentStdenv; + boehmgc = final.boehmgc-nix; + busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; + }; }; in { diff --git a/package.nix b/package.nix new file mode 100644 index 000000000..941d961a6 --- /dev/null +++ b/package.nix @@ -0,0 +1,232 @@ +{ + pkgs, + lib, + stdenv, + autoconf-archive, + autoreconfHook, + aws-sdk-cpp, + boehmgc, + nlohmann_json, + bison, + changelog-d, + boost, + brotli, + bzip2, + curl, + editline, + fileset, + flex, + git, + gtest, + jq, + libarchive, + libcpuid, + libseccomp, + libsodium, + lowdown, + mdbook, + mdbook-linkcheck, + mercurial, + openssl, + pkg-config, + rapidcheck, + sqlite, + util-linuxMinimal ? utillinuxMinimal, + utillinuxMinimal ? null, + xz, + + busybox-sandbox-shell ? null, + + pname ? "nix", + versionSuffix ? "", + officialRelease ? true, + # Set to true to build the release notes for the next release. + buildUnreleasedNotes ? false, + + # Not a real argument, just the only way to approximate let-binding some + # stuff for argument defaults. + __forDefaults ? { + canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + }, +}: let + inherit (__forDefaults) canRunInstalled; + + version = lib.fileContents ./.version + versionSuffix; + + # .gitignore has already been processed, so any changes in it are irrelevant + # at this point. It is not represented verbatim for test purposes because + # that would interfere with repo semantics. + baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.; + src = fileset.toSource { + root = ./.; + fileset = fileset.intersection baseFiles (fileset.unions [ + ./.version + ./boehmgc-coroutine-sp-fallback.diff + ./configure.ac + ./doc + ./local.mk + ./m4 + ./Makefile + ./Makefile.config.in + ./misc + ./mk + ./precompiled-headers.h + ./src + ./tests/functional + ./tests/unit + ./unit-test-data + ./COPYING + ./scripts/local.mk + (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) + # TODO: do we really need README.md? It doesn't seem used in the build. + ./README.md + ]); + }; + + aws-sdk-cpp-nix = aws-sdk-cpp.override { + apis = [ "s3" "transfer" ]; + customMemoryManagement = false; + }; + + testConfigureFlags = [ + "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" + ]; + +in stdenv.mkDerivation (finalAttrs: { + name = "nix-${version}"; + inherit version; + + inherit src; + + VERSION_SUFFIX = versionSuffix; + + outputs = [ "out" "dev" "doc" ]; + + nativeBuildInputs = [ + bison + flex + (lib.getBin lowdown) + mdbook + mdbook-linkcheck + autoconf-archive + autoreconfHook + pkg-config + + # Tests + git + mercurial + jq + ] ++ lib.optional stdenv.hostPlatform.isLinux util-linuxMinimal + ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d; + + buildInputs = [ + curl + bzip2 + xz + brotli + editline + openssl + sqlite + libarchive + boost + lowdown + libsodium + ] + ++ lib.optionals stdenv.isLinux [ libseccomp ] + ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid + # There have been issues building these dependencies + ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform) aws-sdk-cpp-nix + # FIXME(Qyriad): This is how the flake.nix version does it, but this is cursed. + ++ lib.optionals (finalAttrs.doCheck) finalAttrs.passthru._checkInputs + ; + + passthru._checkInputs = [ + gtest + rapidcheck + ]; + + propagatedBuildInputs = [ + boehmgc + nlohmann_json + ]; + + disallowedReferences = [ + boost + ]; + + preConfigure = lib.optionalString (! stdenv.hostPlatform.isStatic) '' + # Copy libboost_context so we don't get all of Boost in our closure. + # https://github.com/NixOS/nixpkgs/issues/45462 + mkdir -p $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + rm -f $out/lib/*.a + ${lib.optionalString stdenv.hostPlatform.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + ''} + ${lib.optionalString stdenv.hostPlatform.isDarwin '' + for LIB in $out/lib/*.dylib; do + chmod u+w $LIB + install_name_tool -id $LIB $LIB + install_name_tool -delete_rpath ${boost}/lib/ $LIB || true + done + install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib + ''} + ''; + + configureFlags = lib.optionals stdenv.isLinux [ + "--with-boost=${boost}/lib" + "--with-sandbox-shell=${busybox-sandbox-shell}/bin/busybox" + ] ++ lib.optionals (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) [ + "LDFLAGS=-fuse-ld=gold" + ] ++ [ "--sysconfdir=/etc" ] + ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" + ++ [ (lib.enableFeature finalAttrs.doCheck "tests") ] + ++ lib.optionals finalAttrs.doCheck testConfigureFlags + ++ lib.optional (!canRunInstalled) "--disable-doc-gen" + ; + + enableParallelBuilding = true; + + makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; + + doCheck = true; + + installFlags = "sysconfdir=$(out)/etc"; + + postInstall = '' + mkdir -p $doc/nix-support + echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products + ${lib.optionalString stdenv.hostPlatform.isStatic '' + mkdir -p $out/nix-support + echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products + ''} + ${lib.optionalString stdenv.isDarwin '' + install_name_tool \ + -change ${boost}/lib/libboost_context.dylib \ + $out/lib/libboost_context.dylib \ + $out/lib/libnixutil.dylib + ''} + ''; + + doInstallCheck = finalAttrs.doCheck; + installCheckFlags = "sysconfdir=$(out)/etc"; + installCheckTarget = "installcheck"; # work around buggy detection in stdenv + + preInstallCheck = lib.optionalString stdenv.hostPlatform.isDarwin '' + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + ''; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta.platforms = lib.platforms.unix; + + passthru.finalAttrs = finalAttrs; + passthru.perl-bindings = pkgs.callPackage ./perl { + inherit fileset stdenv; + }; +}) From 875b76d0c7ab596ad26830a80ab63dc2869e83db Mon Sep 17 00:00:00 2001 From: Qyriad Date: Tue, 5 Mar 2024 13:51:49 -0700 Subject: [PATCH 2/6] package: use pname, version, and dontBuild (first change with diff hash) The src fileset, preConfigure, and separateDebugInfo also respond to doBuild if its overridden This commit is logically just a continuation of the previous commit's refactor, but exists separately to delineate when the core Nix derivation hash changed (this commit). Change-Id: I67a61bc9608d91b6a833ebc5c3894b2d2e694050 --- package.nix | 114 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 65 insertions(+), 49 deletions(-) diff --git a/package.nix b/package.nix index 941d961a6..f809ae67b 100644 --- a/package.nix +++ b/package.nix @@ -53,36 +53,6 @@ version = lib.fileContents ./.version + versionSuffix; - # .gitignore has already been processed, so any changes in it are irrelevant - # at this point. It is not represented verbatim for test purposes because - # that would interfere with repo semantics. - baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.; - src = fileset.toSource { - root = ./.; - fileset = fileset.intersection baseFiles (fileset.unions [ - ./.version - ./boehmgc-coroutine-sp-fallback.diff - ./configure.ac - ./doc - ./local.mk - ./m4 - ./Makefile - ./Makefile.config.in - ./misc - ./mk - ./precompiled-headers.h - ./src - ./tests/functional - ./tests/unit - ./unit-test-data - ./COPYING - ./scripts/local.mk - (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) - # TODO: do we really need README.md? It doesn't seem used in the build. - ./README.md - ]); - }; - aws-sdk-cpp-nix = aws-sdk-cpp.override { apis = [ "s3" "transfer" ]; customMemoryManagement = false; @@ -92,19 +62,64 @@ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" ]; -in stdenv.mkDerivation (finalAttrs: { - name = "nix-${version}"; - inherit version; + # .gitignore has already been processed, so any changes in it are irrelevant + # at this point. It is not represented verbatim for test purposes because + # that would interfere with repo semantics. + baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.; - inherit src; + configureFiles = fileset.unions [ + ./.version + ./configure.ac + ./m4 + # TODO: do we really need README.md? It doesn't seem used in the build. + ./README.md + ]; + + topLevelBuildFiles = fileset.unions [ + ./local.mk + ./Makefile + ./Makefile.config.in + ./mk + ]; + + functionalTestFiles = fileset.unions [ + ./tests/functional + ./tests/unit + (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) + ]; + +in stdenv.mkDerivation (finalAttrs: { + inherit pname version; + + src = fileset.toSource { + root = ./.; + fileset = fileset.intersection baseFiles (fileset.unions ([ + configureFiles + topLevelBuildFiles + functionalTestFiles + ./unit-test-data + ] ++ lib.optionals (!finalAttrs.dontBuild) [ + ./boehmgc-coroutine-sp-fallback.diff + ./doc + ./misc + ./precompiled-headers.h + ./src + ./COPYING + ./scripts/local.mk + ])); + }; VERSION_SUFFIX = versionSuffix; - outputs = [ "out" "dev" "doc" ]; + outputs = [ "out" ] + ++ lib.optionals (!finalAttrs.dontBuild) [ "dev" "doc" ]; + + dontBuild = false; nativeBuildInputs = [ bison flex + ] ++ [ (lib.getBin lowdown) mdbook mdbook-linkcheck @@ -145,6 +160,9 @@ in stdenv.mkDerivation (finalAttrs: { rapidcheck ]; + # FIXME(Qyriad): remove at the end of refactoring. + checkInputs = finalAttrs.passthru._checkInputs; + propagatedBuildInputs = [ boehmgc nlohmann_json @@ -154,24 +172,22 @@ in stdenv.mkDerivation (finalAttrs: { boost ]; - preConfigure = lib.optionalString (! stdenv.hostPlatform.isStatic) '' + preConfigure = lib.optionalString (!finalAttrs.dontBuild && !stdenv.hostPlatform.isStatic) '' # Copy libboost_context so we don't get all of Boost in our closure. # https://github.com/NixOS/nixpkgs/issues/45462 mkdir -p $out/lib cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib rm -f $out/lib/*.a - ${lib.optionalString stdenv.hostPlatform.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - ''} - ${lib.optionalString stdenv.hostPlatform.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true - done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - ''} + '' + lib.optionalString stdenv.hostPlatform.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + '' + lib.optionalString stdenv.hostPlatform.isDarwin '' + for LIB in $out/lib/*.dylib; do + chmod u+w $LIB + install_name_tool -id $LIB $LIB + install_name_tool -delete_rpath ${boost}/lib/ $LIB || true + done + install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib ''; configureFlags = lib.optionals stdenv.isLinux [ @@ -217,7 +233,7 @@ in stdenv.mkDerivation (finalAttrs: { export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES ''; - separateDebugInfo = !stdenv.hostPlatform.isStatic; + separateDebugInfo = !stdenv.hostPlatform.isStatic && !finalAttrs.dontBuild; strictDeps = true; From 4ad3446311de5fc7a1f254dcda7634126d87435c Mon Sep 17 00:00:00 2001 From: Qyriad Date: Tue, 5 Mar 2024 16:26:46 -0700 Subject: [PATCH 3/6] package: migrate testNixVersions Change-Id: I71845f8a6d7b77c3617d055e726ed4a28cd05fa3 --- flake.nix | 51 ++++++++++++++++++++------------------------------- package.nix | 4 ++-- 2 files changed, 22 insertions(+), 33 deletions(-) diff --git a/flake.nix b/flake.nix index a3a395b1c..e547846f8 100644 --- a/flake.nix +++ b/flake.nix @@ -243,51 +243,40 @@ echo "file installer $out/install" >> $out/nix-support/hydra-build-products ''; - testNixVersions = pkgs: client: daemon: with commonDeps { inherit pkgs; }; with pkgs.lib; pkgs.stdenv.mkDerivation { + testNixVersions = pkgs: client: daemon: let + nix = pkgs.callPackage ./package.nix { + pname = + "nix-tests" + + lib.optionalString + (lib.versionAtLeast daemon.version "2.4pre20211005" && + lib.versionAtLeast client.version "2.4pre20211005") + "-${client.version}-against-${daemon.version}"; + + inherit fileset; + }; + in nix.overrideAttrs (prevAttrs: { NIX_DAEMON_PACKAGE = daemon; NIX_CLIENT_PACKAGE = client; - name = - "nix-tests" - + optionalString - (versionAtLeast daemon.version "2.4pre20211005" && - versionAtLeast client.version "2.4pre20211005") - "-${client.version}-against-${daemon.version}"; - inherit version; - src = fileset.toSource { - root = ./.; - fileset = fileset.intersection baseFiles (fileset.unions [ - configureFiles - topLevelBuildFiles - functionalTestFiles - ]); - }; - - VERSION_SUFFIX = versionSuffix; - - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ awsDeps ++ checkDeps; - propagatedBuildInputs = propagatedDeps; - - enableParallelBuilding = true; - - configureFlags = - testConfigureFlags # otherwise configure fails - ++ [ "--disable-build" ]; dontBuild = true; doInstallCheck = true; + configureFlags = prevAttrs.configureFlags ++ [ + # We don't need the actual build here. + "--disable-build" + ]; + installPhase = '' mkdir -p $out ''; - installCheckPhase = (optionalString pkgs.stdenv.hostPlatform.isDarwin '' + installCheckPhase = lib.optionalString pkgs.stdenv.hostPlatform.isDarwin '' export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES - '') + '' + '' + '' mkdir -p src/nix-channel make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES ''; - }; + }); binaryTarball = nix: pkgs: let diff --git a/package.nix b/package.nix index f809ae67b..a6933b0ab 100644 --- a/package.nix +++ b/package.nix @@ -178,10 +178,10 @@ in stdenv.mkDerivation (finalAttrs: { mkdir -p $out/lib cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib rm -f $out/lib/*.a - '' + lib.optionalString stdenv.hostPlatform.isLinux '' + '' + lib.optionalString (!finalAttrs.dontBuild && stdenv.hostPlatform.isLinux) '' chmod u+w $out/lib/*.so.* patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - '' + lib.optionalString stdenv.hostPlatform.isDarwin '' + '' + lib.optionalString (!finalAttrs.dontBuild && stdenv.hostPlatform.isDarwin) '' for LIB in $out/lib/*.dylib; do chmod u+w $LIB install_name_tool -id $LIB $LIB From b072c069b741d5939baf0350d53392197da3b2d7 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Fri, 8 Mar 2024 21:09:11 -0700 Subject: [PATCH 4/6] package: migrate internal-api-docs Change-Id: I344d73a412c2c6e4bb2eb14bd4859056324f1ba7 --- flake.nix | 41 ++++++++++++++++++----------------------- package.nix | 53 +++++++++++++++++++++++++++++++++-------------------- 2 files changed, 51 insertions(+), 43 deletions(-) diff --git a/flake.nix b/flake.nix index e547846f8..f991e4d46 100644 --- a/flake.nix +++ b/flake.nix @@ -421,31 +421,26 @@ dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = - with nixpkgsFor.x86_64-linux.native; - with commonDeps { inherit pkgs; }; + internal-api-docs = let + nixpkgs = nixpkgsFor.x86_64-linux.native; + inherit (nixpkgs) pkgs; + comDeps = commonDeps { inherit pkgs; }; - stdenv.mkDerivation { - pname = "nix-internal-api-docs"; - inherit version; - - src = nixSrc; - - configureFlags = testConfigureFlags ++ internalApiDocsConfigureFlags; - - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ propagatedDeps - ++ awsDeps ++ checkDeps ++ internalApiDocsDeps; - - dontBuild = true; - - installTargets = [ "internal-api-html" ]; - - postInstall = '' - mkdir -p $out/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products - ''; + nix = nixpkgs.pkgs.callPackage ./package.nix { + inherit versionSuffix fileset officialRelease buildUnreleasedNotes; + inherit (comDeps) changelog-d; + internalApiDocs = true; + boehmgc = comDeps.boehmgc-nix; + busybox-sandbox-shell = comDeps.sh; }; + in + nix.overrideAttrs (prev: { + # This Hydra job is just for the internal API docs. + # We don't need the build artifacts here. + dontBuild = true; + doCheck = false; + doInstallCheck = false; + }); # System tests. tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { diff --git a/package.nix b/package.nix index a6933b0ab..73e887d8d 100644 --- a/package.nix +++ b/package.nix @@ -13,6 +13,7 @@ brotli, bzip2, curl, + doxygen, editline, fileset, flex, @@ -42,6 +43,7 @@ officialRelease ? true, # Set to true to build the release notes for the next release. buildUnreleasedNotes ? false, + internalApiDocs ? false, # Not a real argument, just the only way to approximate let-binding some # stuff for argument defaults. @@ -62,6 +64,13 @@ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" ]; + # The internal API docs need these for the build, but if we're not building + # Nix itself, then these don't need to be propagated. + maybePropagatedInputs = [ + boehmgc + nlohmann_json + ]; + # .gitignore has already been processed, so any changes in it are irrelevant # at this point. It is not represented verbatim for test purposes because # that would interfere with repo semantics. @@ -98,7 +107,7 @@ in stdenv.mkDerivation (finalAttrs: { topLevelBuildFiles functionalTestFiles ./unit-test-data - ] ++ lib.optionals (!finalAttrs.dontBuild) [ + ] ++ lib.optionals (!finalAttrs.dontBuild || internalApiDocs) [ ./boehmgc-coroutine-sp-fallback.diff ./doc ./misc @@ -132,7 +141,9 @@ in stdenv.mkDerivation (finalAttrs: { mercurial jq ] ++ lib.optional stdenv.hostPlatform.isLinux util-linuxMinimal - ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d; + ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d + ++ lib.optional internalApiDocs doxygen + ; buildInputs = [ curl @@ -153,6 +164,7 @@ in stdenv.mkDerivation (finalAttrs: { ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform) aws-sdk-cpp-nix # FIXME(Qyriad): This is how the flake.nix version does it, but this is cursed. ++ lib.optionals (finalAttrs.doCheck) finalAttrs.passthru._checkInputs + ++ lib.optionals (finalAttrs.dontBuild) maybePropagatedInputs ; passthru._checkInputs = [ @@ -163,10 +175,7 @@ in stdenv.mkDerivation (finalAttrs: { # FIXME(Qyriad): remove at the end of refactoring. checkInputs = finalAttrs.passthru._checkInputs; - propagatedBuildInputs = [ - boehmgc - nlohmann_json - ]; + propagatedBuildInputs = lib.optionals (!finalAttrs.dontBuild) maybePropagatedInputs; disallowedReferences = [ boost @@ -198,10 +207,13 @@ in stdenv.mkDerivation (finalAttrs: { ] ++ [ "--sysconfdir=/etc" ] ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" ++ [ (lib.enableFeature finalAttrs.doCheck "tests") ] - ++ lib.optionals finalAttrs.doCheck testConfigureFlags + ++ lib.optionals (finalAttrs.doCheck || internalApiDocs) testConfigureFlags ++ lib.optional (!canRunInstalled) "--disable-doc-gen" + ++ [ (lib.enableFeature internalApiDocs "internal-api-docs") ] ; + installTargets = lib.optional internalApiDocs "internal-api-html"; + enableParallelBuilding = true; makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; @@ -210,19 +222,20 @@ in stdenv.mkDerivation (finalAttrs: { installFlags = "sysconfdir=$(out)/etc"; - postInstall = '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - ${lib.optionalString stdenv.hostPlatform.isStatic '' - mkdir -p $out/nix-support - echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - ''} - ${lib.optionalString stdenv.isDarwin '' - install_name_tool \ - -change ${boost}/lib/libboost_context.dylib \ - $out/lib/libboost_context.dylib \ - $out/lib/libnixutil.dylib - ''} + postInstall = lib.optionalString (!finalAttrs.dontBuild) '' + mkdir -p $doc/nix-support + echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products + '' + lib.optionalString stdenv.hostPlatform.isStatic '' + mkdir -p $out/nix-support + echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products + '' + lib.optionalString stdenv.isDarwin '' + install_name_tool \ + -change ${boost}/lib/libboost_context.dylib \ + $out/lib/libboost_context.dylib \ + $out/lib/libnixutil.dylib + '' + lib.optionalString internalApiDocs '' + mkdir -p $out/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> "$out/nix-support/hydra-build-products" ''; doInstallCheck = finalAttrs.doCheck; From 529a01ade2b5cd19f0013e1cf030d59dd88532fe Mon Sep 17 00:00:00 2001 From: Qyriad Date: Sat, 9 Mar 2024 00:44:49 -0700 Subject: [PATCH 5/6] package: migrate devShells Change-Id: Ic63721667edd4bef79aa699a0de8411639e5159b --- flake.nix | 43 ++++++++++++++++--------------------------- 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/flake.nix b/flake.nix index f991e4d46..d3033ca05 100644 --- a/flake.nix +++ b/flake.nix @@ -531,36 +531,25 @@ devShells = let makeShell = pkgs: stdenv: let - canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + nix = pkgs.callPackage ./package.nix { + inherit stdenv versionSuffix fileset; + boehmgc = pkgs.boehmgc-nix; + busybox-sandbox-shell = pkgs.busybox-sandbox-shell or pkgs.default-busybox-sandbox; + }; in - with commonDeps { inherit pkgs; }; - stdenv.mkDerivation { - name = "nix"; + nix.overrideAttrs (prev: { + nativeBuildInputs = prev.nativeBuildInputs + ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear + ++ lib.optional + (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) + pkgs.buildPackages.clang-tools; - outputs = [ "out" "dev" "doc" ]; + src = null; - nativeBuildInputs = nativeBuildDeps - ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear - ++ lib.optional - (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) - pkgs.buildPackages.clang-tools - # We want changelog-d in the shell even if the current build doesn't need it - ++ lib.optional (officialRelease || ! buildUnreleasedNotes) changelog-d - ; + installFlags = "sysconfdir=$(out)/etc"; + strictDeps = false; - buildInputs = buildDeps ++ propagatedDeps - ++ awsDeps ++ checkDeps ++ internalApiDocsDeps; - - configureFlags = configureFlags - ++ testConfigureFlags ++ internalApiDocsConfigureFlags - ++ lib.optional (!canRunInstalled) "--disable-doc-gen"; - - enableParallelBuilding = true; - - installFlags = "sysconfdir=$(out)/etc"; - - shellHook = - '' + shellHook = '' PATH=$prefix/bin:$PATH unset PYTHONPATH export MANPATH=$out/share/man:$MANPATH @@ -568,7 +557,7 @@ # Make bash completion work. XDG_DATA_DIRS+=:$out/share ''; - }; + }); in forAllSystems (system: let From 36a8d151e10d5d7e37eb9a7102c8ebf2455e4a57 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Sat, 9 Mar 2024 01:22:06 -0700 Subject: [PATCH 6/6] package: cleanup of all intermediaries Change-Id: I0da5182de6b01c192cfcba407959d659d70c6dc9 --- flake.nix | 158 ++++++---------------------------------------------- package.nix | 18 +++--- 2 files changed, 25 insertions(+), 151 deletions(-) diff --git a/flake.nix b/flake.nix index d3033ca05..8fde05179 100644 --- a/flake.nix +++ b/flake.nix @@ -48,50 +48,6 @@ }) stdenvs); - baseFiles = - # .gitignore has already been processed, so any changes in it are irrelevant - # at this point. It is not represented verbatim for test purposes because - # that would interfere with repo semantics. - fileset.fileFilter (f: f.name != ".gitignore") ./.; - - configureFiles = fileset.unions [ - ./.version - ./configure.ac - ./m4 - # TODO: do we really need README.md? It doesn't seem used in the build. - ./README.md - ]; - - topLevelBuildFiles = fileset.unions [ - ./local.mk - ./Makefile - ./Makefile.config.in - ./mk - ]; - - functionalTestFiles = fileset.unions [ - ./tests/functional - ./tests/unit - (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) - ]; - - nixSrc = fileset.toSource { - root = ./.; - fileset = fileset.intersection baseFiles (fileset.unions [ - configureFiles - topLevelBuildFiles - ./boehmgc-coroutine-sp-fallback.diff - ./doc - ./misc - ./precompiled-headers.h - ./src - ./unit-test-data - ./COPYING - ./scripts/local.mk - functionalTestFiles - ]); - }; - # Memoize nixpkgs for different platforms for efficiency. nixpkgsFor = forAllSystems (system: let @@ -118,97 +74,6 @@ cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); - commonDeps = { - pkgs, - isStatic ? pkgs.stdenv.hostPlatform.isStatic - }: let - inherit (pkgs) stdenv buildPackages - busybox curl bzip2 xz brotli editline openssl sqlite libarchive boost - libseccomp libsodium libcpuid gtest rapidcheck aws-sdk-cpp boehmgc nlohmann_json - lowdown; - changelog-d = pkgs.buildPackages.callPackage ./misc/changelog-d.nix { }; - boehmgc-nix = (boehmgc.override { - enableLargeConfig = true; - }).overrideAttrs (o: { - patches = (o.patches or [ ]) ++ [ - ./boehmgc-coroutine-sp-fallback.diff - - # https://github.com/ivmai/bdwgc/pull/586 - ./boehmgc-traceable_allocator-public.diff - ]; - }); - in rec { - calledPackage = pkgs.callPackage ./package.nix { - inherit stdenv versionSuffix fileset changelog-d officialRelease buildUnreleasedNotes lowdown; - boehmgc = boehmgc-nix; - busybox-sandbox-shell = sh; - }; - - inherit boehmgc-nix; - - # Use "busybox-sandbox-shell" if present, - # if not (legacy) fallback and hope it's sufficient. - sh = pkgs.busybox-sandbox-shell or (busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y - - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }); - - configureFlags = - lib.optionals stdenv.isLinux [ - "--with-boost=${boost}/lib" - "--with-sandbox-shell=${sh}/bin/busybox" - ] - ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [ - "LDFLAGS=-fuse-ld=gold" - ]; - - testConfigureFlags = [ - "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" - ]; - - internalApiDocsConfigureFlags = [ - "--enable-internal-api-docs" - ]; - - inherit changelog-d; - nativeBuildDeps = calledPackage.nativeBuildInputs; - - buildDeps = calledPackage.buildInputs; - - checkDeps = calledPackage.finalAttrs.passthru._checkInputs; - - internalApiDocsDeps = [ - buildPackages.doxygen - ]; - - awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }); - - propagatedDeps = calledPackage.propagatedBuildInputs; - }; - installScriptFor = systems: with nixpkgsFor.x86_64-linux.native; runCommand "installer-script" @@ -365,7 +230,17 @@ # Forward from the previous stage as we don’t want it to pick the lowdown override nixUnstable = prev.nixUnstable; - inherit (comDeps) boehmgc-nix; + changelog-d = final.buildPackages.callPackage ./misc/changelog-d.nix { }; + boehmgc-nix = (final.boehmgc.override { + enableLargeConfig = true; + }).overrideAttrs (o: { + patches = (o.patches or [ ]) ++ [ + ./boehmgc-coroutine-sp-fallback.diff + + # https://github.com/ivmai/bdwgc/pull/586 + ./boehmgc-traceable_allocator-public.diff + ]; + }); default-busybox-sandbox-shell = final.busybox.override { useMusl = true; @@ -424,14 +299,13 @@ internal-api-docs = let nixpkgs = nixpkgsFor.x86_64-linux.native; inherit (nixpkgs) pkgs; - comDeps = commonDeps { inherit pkgs; }; - nix = nixpkgs.pkgs.callPackage ./package.nix { + nix = pkgs.callPackage ./package.nix { inherit versionSuffix fileset officialRelease buildUnreleasedNotes; - inherit (comDeps) changelog-d; + inherit (pkgs) changelog-d; internalApiDocs = true; - boehmgc = comDeps.boehmgc-nix; - busybox-sandbox-shell = comDeps.sh; + boehmgc = pkgs.boehmgc-nix; + busybox-sandbox-shell = pkgs.busybox-sandbox-shell; }; in nix.overrideAttrs (prev: { @@ -490,7 +364,7 @@ rl-next = let pkgs = nixpkgsFor.${system}.native; in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' - LANG=C.UTF-8 ${(commonDeps { inherit pkgs; }).changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out + LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out ''; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; diff --git a/package.nix b/package.nix index 73e887d8d..06d644627 100644 --- a/package.nix +++ b/package.nix @@ -36,7 +36,7 @@ utillinuxMinimal ? null, xz, - busybox-sandbox-shell ? null, + busybox-sandbox-shell, pname ? "nix", versionSuffix ? "", @@ -162,19 +162,14 @@ in stdenv.mkDerivation (finalAttrs: { ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform) aws-sdk-cpp-nix - # FIXME(Qyriad): This is how the flake.nix version does it, but this is cursed. - ++ lib.optionals (finalAttrs.doCheck) finalAttrs.passthru._checkInputs ++ lib.optionals (finalAttrs.dontBuild) maybePropagatedInputs ; - passthru._checkInputs = [ + checkInputs = [ gtest rapidcheck ]; - # FIXME(Qyriad): remove at the end of refactoring. - checkInputs = finalAttrs.passthru._checkInputs; - propagatedBuildInputs = lib.optionals (!finalAttrs.dontBuild) maybePropagatedInputs; disallowedReferences = [ @@ -197,6 +192,13 @@ in stdenv.mkDerivation (finalAttrs: { install_name_tool -delete_rpath ${boost}/lib/ $LIB || true done install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib + '' + '' + # Workaround https://github.com/NixOS/nixpkgs/issues/294890. + if [[ -n "''${doCheck:-}" ]]; then + appendToVar configureFlags "--enable-tests" + else + appendToVar configureFlags "--disable-tests" + fi ''; configureFlags = lib.optionals stdenv.isLinux [ @@ -206,7 +208,6 @@ in stdenv.mkDerivation (finalAttrs: { "LDFLAGS=-fuse-ld=gold" ] ++ [ "--sysconfdir=/etc" ] ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" - ++ [ (lib.enableFeature finalAttrs.doCheck "tests") ] ++ lib.optionals (finalAttrs.doCheck || internalApiDocs) testConfigureFlags ++ lib.optional (!canRunInstalled) "--disable-doc-gen" ++ [ (lib.enableFeature internalApiDocs "internal-api-docs") ] @@ -254,7 +255,6 @@ in stdenv.mkDerivation (finalAttrs: { meta.platforms = lib.platforms.unix; - passthru.finalAttrs = finalAttrs; passthru.perl-bindings = pkgs.callPackage ./perl { inherit fileset stdenv; };