puck/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

Nix's own flake: Dedup and memoize more

Browse source 
- `nixpkgsFor` does all of native, static, cross, and the different stdenvs.

- The main Nix derivation is no longer duplicated for static.

- DRY nixpkgs.lib and lib.genAttrs calls.
This commit is contained in:
John Ericson 2022-03-02 02:40:18 +00:00 committed by zimbatm
parent 9a3f66d9d9
commit d7a4f08d42
No known key found for this signature in database
GPG key ID: 71BAF6D40C1D63D7
3 changed files with 112 additions and 184 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
doc/manual/src/contributing/hacking.md
View file

@ -30,7 +30,7 @@ This shell also adds `./outputs/bin/nix` to your `$PATH` so you can run `nix` im
To get a shell with one of the other [supported compilation environments](#compilation-environments): To get a shell with one of the other [supported compilation environments](#compilation-environments):
```console ```console
$ nix develop .#clang11Stdenv $ nix develop .#native-clang11StdenvPackages
``` ```
> **Note** > **Note**
@ -77,12 +77,12 @@ $ nix-shell
To get a shell with one of the other [supported compilation environments](#compilation-environments): To get a shell with one of the other [supported compilation environments](#compilation-environments):
```console ```console
$ nix-shell -A devShells.x86_64-linux.clang11Stdenv $ nix-shell -A devShells.x86_64-linux.native-clang11StdenvPackages
``` ```
> **Note** > **Note**
> >
> You can use `ccacheStdenv` to drastically improve rebuild time. > You can use `native-ccacheStdenvPackages` to drastically improve rebuild time.
> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. > By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`.
To build Nix itself in this shell: To build Nix itself in this shell:

288
flake.nix
View file

@ -8,10 +8,11 @@
outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src }: outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src }:
let let
inherit (nixpkgs) lib;
officialRelease = false; officialRelease = false;
version = nixpkgs.lib.fileContents ./.version + versionSuffix; version = lib.fileContents ./.version + versionSuffix;
versionSuffix = versionSuffix =
if officialRelease if officialRelease
then "" then ""
@ -25,34 +26,36 @@
stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ];
forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); forAllSystems = lib.genAttrs systems;
forAllSystemsAndStdenvs = f: forAllSystems (system:
nixpkgs.lib.listToAttrs forAllCrossSystems = lib.genAttrs crossSystems;
(map
(n: forAllStdenvs = f:
nixpkgs.lib.nameValuePair "${n}Packages" ( lib.listToAttrs
f system n (map
)) stdenvs (stdenvName: {
) name = "${stdenvName}Packages";
); value = f stdenvName;
})
stdenvs);
forAllStdenvs = f: nixpkgs.lib.genAttrs stdenvs (stdenv: f stdenv);
# Memoize nixpkgs for different platforms for efficiency. # Memoize nixpkgs for different platforms for efficiency.
nixpkgsFor = nixpkgsFor = forAllSystems
let stdenvsPackages = forAllSystemsAndStdenvs (system: let
(system: stdenv: make-pkgs = crossSystem: stdenv: import nixpkgs {
import nixpkgs { inherit system crossSystem;
inherit system; overlays = [
overlays = [ (overlayFor (p: p.${stdenv}))
(overlayFor (p: p.${stdenv})) ];
]; };
} stdenvs = forAllStdenvs (make-pkgs null);
); native = stdenvs.stdenvPackages;
in in {
# Add the `stdenvPackages` at toplevel, both because these are the ones inherit stdenvs native;
# we want most of the time and for backwards compatibility static = native.pkgsStatic;
forAllSystems (system: stdenvsPackages.${system} // stdenvsPackages.${system}.stdenvPackages); cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv");
});
commonDeps = { pkgs, isStatic ? false }: with pkgs; rec { commonDeps = { pkgs, isStatic ? false }: with pkgs; rec {
# Use "busybox-sandbox-shell" if present, # Use "busybox-sandbox-shell" if present,
@ -144,7 +147,7 @@
}; };
installScriptFor = systems: installScriptFor = systems:
with nixpkgsFor.x86_64-linux; with nixpkgsFor.x86_64-linux.native;
runCommand "installer-script" runCommand "installer-script"
{ buildInputs = [ nix ]; { buildInputs = [ nix ];
} }
@ -208,8 +211,9 @@
installCheckPhase = "make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES"; installCheckPhase = "make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES";
}; };
binaryTarball = buildPackages: nix: pkgs: binaryTarball = nix: pkgs:
let let
inherit (pkgs) buildPackages;
inherit (pkgs) cacert; inherit (pkgs) cacert;
installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; }; installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; };
in in
@ -289,7 +293,9 @@
# Forward from the previous stage as we don’t want it to pick the lowdown override # Forward from the previous stage as we don’t want it to pick the lowdown override
nixUnstable = prev.nixUnstable; nixUnstable = prev.nixUnstable;
nix = with final; with commonDeps { inherit pkgs; }; currentStdenv.mkDerivation { nix = with final; with commonDeps { inherit pkgs; }; let
canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform;
in currentStdenv.mkDerivation {
name = "nix-${version}"; name = "nix-${version}";
inherit version; inherit version;
@ -300,24 +306,26 @@
outputs = [ "out" "dev" "doc" ]; outputs = [ "out" "dev" "doc" ];
nativeBuildInputs = nativeBuildDeps; nativeBuildInputs = nativeBuildDeps;
buildInputs = buildDeps ++ awsDeps; buildInputs = buildDeps
# There have been issues building these dependencies
++ lib.optionals (currentStdenv.hostPlatform == currentStdenv.buildPlatform) awsDeps;
propagatedBuildInputs = propagatedDeps; propagatedBuildInputs = propagatedDeps;
disallowedReferences = [ boost ]; disallowedReferences = [ boost ];
preConfigure = preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic)
'' ''
# Copy libboost_context so we don't get all of Boost in our closure. # Copy libboost_context so we don't get all of Boost in our closure.
# https://github.com/NixOS/nixpkgs/issues/45462 # https://github.com/NixOS/nixpkgs/issues/45462
mkdir -p $out/lib mkdir -p $out/lib
cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
rm -f $out/lib/*.a rm -f $out/lib/*.a
${lib.optionalString currentStdenv.isLinux '' ${lib.optionalString currentStdenv.hostPlatform.isLinux ''
chmod u+w $out/lib/*.so.* chmod u+w $out/lib/*.so.*
patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
''} ''}
${lib.optionalString currentStdenv.isDarwin '' ${lib.optionalString currentStdenv.hostPlatform.isDarwin ''
for LIB in $out/lib/*.dylib; do for LIB in $out/lib/*.dylib; do
chmod u+w $LIB chmod u+w $LIB
install_name_tool -id $LIB $LIB install_name_tool -id $LIB $LIB
@ -328,7 +336,9 @@
''; '';
configureFlags = configureFlags ++ configureFlags = configureFlags ++
[ "--sysconfdir=/etc" ]; [ "--sysconfdir=/etc" ] ++
lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" ++
lib.optional (!canRunInstalled) "--disable-doc-gen";
enableParallelBuilding = true; enableParallelBuilding = true;
@ -356,6 +366,8 @@
strictDeps = true; strictDeps = true;
hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
passthru.perl-bindings = with final; perl.pkgs.toPerlModule (currentStdenv.mkDerivation { passthru.perl-bindings = with final; perl.pkgs.toPerlModule (currentStdenv.mkDerivation {
name = "nix-perl-${version}"; name = "nix-perl-${version}";
@ -388,7 +400,7 @@
postUnpack = "sourceRoot=$sourceRoot/perl"; postUnpack = "sourceRoot=$sourceRoot/perl";
}); });
meta.platforms = systems; meta.platforms = lib.platforms.unix;
}; };
lowdown-nix = with final; currentStdenv.mkDerivation rec { lowdown-nix = with final; currentStdenv.mkDerivation rec {
@ -414,14 +426,15 @@
# https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests
runNixOSTestFor = system: test: nixos-lib.runTest { runNixOSTestFor = system: test: nixos-lib.runTest {
imports = [ test ]; imports = [ test ];
hostPkgs = nixpkgsFor.${system}; hostPkgs = nixpkgsFor.${system}.native;
defaults = { defaults = {
nixpkgs.pkgs = nixpkgsFor.${system}; nixpkgs.pkgs = nixpkgsFor.${system}.native;
}; };
_module.args.nixpkgs = nixpkgs; _module.args.nixpkgs = nixpkgs;
}; };
in { in {
inherit nixpkgsFor;
# A Nixpkgs overlay that overrides the 'nix' and # A Nixpkgs overlay that overrides the 'nix' and
# 'nix.perl-bindings' packages. # 'nix.perl-bindings' packages.
@ -430,32 +443,28 @@
hydraJobs = { hydraJobs = {
# Binary package for various platforms. # Binary package for various platforms.
build = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix); build = forAllSystems (system: self.packages.${system}.nix);
buildStatic = nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static);
buildCross = nixpkgs.lib.genAttrs crossSystems (crossSystem: buildCross = forAllCrossSystems (crossSystem:
nixpkgs.lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}")); lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}"));
buildNoGc = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.overrideAttrs (a: { configureFlags = (a.configureFlags or []) ++ ["--enable-gc=no"];})); buildNoGc = forAllSystems (system: self.packages.${system}.nix.overrideAttrs (a: { configureFlags = (a.configureFlags or []) ++ ["--enable-gc=no"];}));
# Perl bindings for various platforms. # Perl bindings for various platforms.
perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings); perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings);
# Binary tarball for various platforms, containing a Nix store # Binary tarball for various platforms, containing a Nix store
# with the closure of 'nix' package, and the second half of # with the closure of 'nix' package, and the second half of
# the installation script. # the installation script.
binaryTarball = nixpkgs.lib.genAttrs systems (system: binaryTarball nixpkgsFor.${system} nixpkgsFor.${system}.nix nixpkgsFor.${system}); binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native);
binaryTarballCross = nixpkgs.lib.genAttrs ["x86_64-linux"] (system: builtins.listToAttrs (map (crossSystem: { binaryTarballCross = lib.genAttrs ["x86_64-linux"] (system:
name = crossSystem; forAllCrossSystems (crossSystem:
value = let binaryTarball
nixpkgsCross = import nixpkgs { self.packages.${system}."nix-${crossSystem}"
inherit system crossSystem; nixpkgsFor.${system}.cross.${crossSystem}));
overlays = [ self.overlays.default ];
};
in binaryTarball nixpkgsFor.${system} self.packages.${system}."nix-${crossSystem}" nixpkgsCross;
}) crossSystems));
# The first half of the installation script. This is uploaded # The first half of the installation script. This is uploaded
# to https://nixos.org/nix/install. It downloads the binary # to https://nixos.org/nix/install. It downloads the binary
@ -465,11 +474,11 @@
installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"]; installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"];
# docker image with Nix inside # docker image with Nix inside
dockerImage = nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);
# Line coverage analysis. # Line coverage analysis.
coverage = coverage =
with nixpkgsFor.x86_64-linux; with nixpkgsFor.x86_64-linux.native;
with commonDeps { inherit pkgs; }; with commonDeps { inherit pkgs; };
releaseTools.coverageAnalysis { releaseTools.coverageAnalysis {
@ -513,7 +522,7 @@
tests.containers = runNixOSTestFor "x86_64-linux" ./tests/nixos/containers/containers.nix; tests.containers = runNixOSTestFor "x86_64-linux" ./tests/nixos/containers/containers.nix;
tests.setuid = nixpkgs.lib.genAttrs tests.setuid = lib.genAttrs
["i686-linux" "x86_64-linux"] ["i686-linux" "x86_64-linux"]
(system: runNixOSTestFor system ./tests/nixos/setuid.nix); (system: runNixOSTestFor system ./tests/nixos/setuid.nix);
@ -521,7 +530,7 @@
# Make sure that nix-env still produces the exact same result # Make sure that nix-env still produces the exact same result
# on a particular version of Nixpkgs. # on a particular version of Nixpkgs.
tests.evalNixpkgs = tests.evalNixpkgs =
with nixpkgsFor.x86_64-linux; with nixpkgsFor.x86_64-linux.native;
runCommand "eval-nixos" { buildInputs = [ nix ]; } runCommand "eval-nixos" { buildInputs = [ nix ]; }
'' ''
type -p nix-env type -p nix-env
@ -532,18 +541,18 @@
''; '';
tests.nixpkgsLibTests = tests.nixpkgsLibTests =
nixpkgs.lib.genAttrs systems (system: forAllSystems (system:
import (nixpkgs + "/lib/tests/release.nix") import (nixpkgs + "/lib/tests/release.nix")
{ pkgs = nixpkgsFor.${system}; } { pkgs = nixpkgsFor.${system}.native; }
); );
metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" {
pkgs = nixpkgsFor.x86_64-linux; pkgs = nixpkgsFor.x86_64-linux.native;
nixpkgs = nixpkgs-regression; nixpkgs = nixpkgs-regression;
}; };
installTests = forAllSystems (system: installTests = forAllSystems (system:
let pkgs = nixpkgsFor.${system}; in let pkgs = nixpkgsFor.${system}.native; in
pkgs.runCommand "install-tests" { pkgs.runCommand "install-tests" {
againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
againstCurrentUnstable = againstCurrentUnstable =
@ -568,67 +577,18 @@
perlBindings = self.hydraJobs.perlBindings.${system}; perlBindings = self.hydraJobs.perlBindings.${system};
installTests = self.hydraJobs.installTests.${system}; installTests = self.hydraJobs.installTests.${system};
nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system};
} // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems)) { } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
dockerImage = self.hydraJobs.dockerImage.${system}; dockerImage = self.hydraJobs.dockerImage.${system};
}); });
packages = forAllSystems (system: rec { packages = forAllSystems (system: rec {
inherit (nixpkgsFor.${system}) nix; inherit (nixpkgsFor.${system}.native) nix;
default = nix; default = nix;
} // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) { } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) {
nix-static = let nix-static = nixpkgsFor.${system}.static.nix;
nixpkgs = nixpkgsFor.${system}.pkgsStatic;
in with commonDeps { pkgs = nixpkgs; isStatic = true; }; nixpkgs.stdenv.mkDerivation {
name = "nix-${version}";
src = self;
VERSION_SUFFIX = versionSuffix;
outputs = [ "out" "dev" "doc" ];
nativeBuildInputs = nativeBuildDeps;
buildInputs = buildDeps ++ propagatedDeps;
# Work around pkgsStatic disabling all tests.
# Remove in NixOS 22.11, see https://github.com/NixOS/nixpkgs/pull/140271.
preHook =
''
doCheck=1
doInstallCheck=1
'';
configureFlags =
configureFlags ++
[ "--sysconfdir=/etc"
"--enable-embedded-sandbox-shell"
];
enableParallelBuilding = true;
makeFlags = "profiledir=$(out)/etc/profile.d";
installFlags = "sysconfdir=$(out)/etc";
postInstall = ''
mkdir -p $doc/nix-support
echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
mkdir -p $out/nix-support
echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
'';
installCheckFlags = "sysconfdir=$(out)/etc";
stripAllList = ["bin"];
strictDeps = true;
hardeningDisable = [ "pie" ];
};
dockerImage = dockerImage =
let let
pkgs = nixpkgsFor.${system}; pkgs = nixpkgsFor.${system}.native;
image = import ./docker.nix { inherit pkgs; tag = version; }; image = import ./docker.nix { inherit pkgs; tag = version; };
in in
pkgs.runCommand pkgs.runCommand
@ -640,73 +600,30 @@
ln -s ${image} $image ln -s ${image} $image
echo "file binary-dist $image" >> $out/nix-support/hydra-build-products echo "file binary-dist $image" >> $out/nix-support/hydra-build-products
''; '';
} } // builtins.listToAttrs (map
(crossSystem: {
name = "nix-${crossSystem}";
value = nixpkgsFor.${system}.cross.${crossSystem}.nix;
})
crossSystems)
// builtins.listToAttrs (map
(stdenvName: {
name = "nix-${stdenvName}";
value = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nix;
})
stdenvs)));
// builtins.listToAttrs (map (crossSystem: { devShells = let
name = "nix-${crossSystem}"; makeShell = pkgs: stdenv:
value = let
nixpkgsCross = import nixpkgs {
inherit system crossSystem;
overlays = [ self.overlays.default ];
};
inherit (nixpkgsCross) lib;
in with commonDeps { pkgs = nixpkgsCross; }; nixpkgsCross.stdenv.mkDerivation {
name = "nix-${version}";
src = self;
VERSION_SUFFIX = versionSuffix;
outputs = [ "out" "dev" "doc" ];
nativeBuildInputs = nativeBuildDeps;
buildInputs = buildDeps ++ propagatedDeps;
configureFlags = [
"CXXFLAGS=-I${lib.getDev nixpkgsCross.rapidcheck}/extras/gtest/include"
"--sysconfdir=/etc"
"--disable-doc-gen"
];
enableParallelBuilding = true;
makeFlags = "profiledir=$(out)/etc/profile.d";
doCheck = true;
installFlags = "sysconfdir=$(out)/etc";
postInstall = ''
mkdir -p $doc/nix-support
echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
mkdir -p $out/nix-support
echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
'';
doInstallCheck = true;
installCheckFlags = "sysconfdir=$(out)/etc";
};
}) (if system == "x86_64-linux" then crossSystems else [])))
// (builtins.listToAttrs (map (stdenvName:
nixpkgsFor.${system}.lib.nameValuePair
"nix-${stdenvName}"
nixpkgsFor.${system}."${stdenvName}Packages".nix
) stdenvs)));
devShells = forAllSystems (system:
forAllStdenvs (stdenv:
with nixpkgsFor.${system};
with commonDeps { inherit pkgs; }; with commonDeps { inherit pkgs; };
nixpkgsFor.${system}.${stdenv}.mkDerivation { stdenv.mkDerivation {
name = "nix"; name = "nix";
outputs = [ "out" "dev" "doc" ]; outputs = [ "out" "dev" "doc" ];
nativeBuildInputs = nativeBuildDeps nativeBuildInputs = nativeBuildDeps
++ (lib.optionals ++ (lib.optionals stdenv.cc.isClang [ pkgs.bear pkgs.clang-tools ]);
nixpkgsFor.${system}.${stdenv}.cc.isClang
[ bear clang-tools ]);
buildInputs = buildDeps ++ propagatedDeps ++ awsDeps; buildInputs = buildDeps ++ propagatedDeps ++ awsDeps;
inherit configureFlags; inherit configureFlags;
@ -724,10 +641,21 @@
# Make bash completion work. # Make bash completion work.
XDG_DATA_DIRS+=:$out/share XDG_DATA_DIRS+=:$out/share
''; '';
} };
) in
// { default = self.devShells.${system}.stdenv; } forAllSystems (system:
); let
makeShells = prefix: pkgs:
lib.mapAttrs'
(k: v: lib.nameValuePair "${prefix}-${k}" v)
(forAllStdenvs (stdenvName: makeShell pkgs pkgs.${stdenvName}));
in
(makeShells "native" nixpkgsFor.${system}.native) //
(makeShells "static" nixpkgsFor.${system}.static) //
(forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) //
{
default = self.devShells.${system}.native-stdenvPackages;
}
);
}; };
} }

2
tests/installer/default.nix
View file

@ -120,7 +120,7 @@ let
makeTest = imageName: testName: makeTest = imageName: testName:
let image = images.${imageName}; in let image = images.${imageName}; in
with nixpkgsFor.${image.system}; with nixpkgsFor.${image.system}.native;
runCommand runCommand
"installer-test-${imageName}-${testName}" "installer-test-${imageName}-${testName}"
{ buildInputs = [ qemu_kvm openssh ]; { buildInputs = [ qemu_kvm openssh ];