lix/package.nix

617 lines
20 KiB
Nix
Raw Permalink Normal View History

{
pkgs,
lib,
stdenv,
aws-sdk-cpp,
# If the patched version of Boehm isn't passed, then patch it based off of
# pkgs.boehmgc. This allows `callPackage`ing this file without needing to
# to implement behavior that this package flat out doesn't build without
# anyway, but also allows easily overriding the patch logic.
boehmgc-nix ? __forDefaults.boehmgc-nix,
boehmgc,
nlohmann_json,
build-release-notes ? __forDefaults.build-release-notes,
boost,
brotli,
bzip2,
callPackage,
capnproto-lix ? __forDefaults.capnproto-lix,
cmake,
curl,
doxygen,
editline-lix ? __forDefaults.editline-lix,
editline,
git,
gtest,
jq,
libarchive,
libcpuid,
libseccomp,
libsodium,
lix-clang-tidy ? null,
llvmPackages,
lsof,
# FIXME: remove default after dropping NixOS 24.05
lowdown-unsandboxed ? lowdown,
lowdown,
mdbook,
mdbook-linkcheck,
mercurial,
meson,
ninja,
ncurses,
openssl,
pegtl,
pkg-config,
python3,
rapidcheck,
rustPlatform,
rustc,
sqlite,
toml11,
util-linuxMinimal ? utillinuxMinimal,
utillinuxMinimal ? null,
xz,
busybox-sandbox-shell,
pname ? "lix",
versionSuffix ? "",
officialRelease ? __forDefaults.versionJson.official_release,
# Set to true to build the release notes for the next release.
buildUnreleasedNotes ? true,
internalApiDocs ? false,
# Support garbage collection in the evaluator.
enableGC ? sanitize == null || !builtins.elem "address" sanitize,
# List of Meson sanitize options. Accepts values of b_sanitize, e.g.
# "address", "undefined", "thread".
# Enabling the "address" sanitizer will disable garbage collection in the evaluator.
sanitize ? null,
# Turn compiler warnings into errors.
werror ? false,
lintInsteadOfBuild ? false,
# Not a real argument, just the only way to approximate let-binding some
# stuff for argument defaults.
__forDefaults ? {
canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
versionJson = builtins.fromJSON (builtins.readFile ./version.json);
boehmgc-nix = boehmgc.override { enableLargeConfig = true; };
editline-lix = editline.overrideAttrs (prev: {
patches = (prev.patches or [ ]) ++ [
# Recognize `Alt-Left` and `Alt-Right` for navigating by words in more
# terminals/shells/platforms.
#
# See: https://github.com/troglobit/editline/pull/70
./nix-support/editline.patch
];
configureFlags = (prev.configureFlags or [ ]) ++ [
# Enable SIGSTOP (Ctrl-Z) behavior.
(lib.enableFeature true "sigstop")
# Enable ANSI arrow keys.
(lib.enableFeature true "arrow-keys")
# Use termcap library to query terminal size.
(lib.enableFeature (ncurses != null) "termcap")
];
buildInputs = (prev.buildInputs or [ ]) ++ [ ncurses ];
});
build-release-notes = callPackage ./maintainers/build-release-notes.nix { };
# needs derivation patching to add debuginfo and coroutine library support
# !! must build this with clang as it is affected by the gcc coroutine bugs
capnproto-lix = callPackage ./misc/capnproto.nix { inherit stdenv; };
},
}:
# gcc miscompiles coroutines at least until 13.2, possibly longer
assert stdenv.cc.isClang;
let
inherit (__forDefaults) canRunInstalled;
inherit (lib) fileset;
inherit (stdenv) hostPlatform buildPlatform;
version = __forDefaults.versionJson.version + versionSuffix;
aws-sdk-cpp-nix =
if aws-sdk-cpp == null then
null
else
aws-sdk-cpp.override {
apis = [
"s3"
"transfer"
];
customMemoryManagement = false;
};
# Reimplementation of Nixpkgs' Meson cross file, with some additions to make
# it actually work.
mesonCrossFile = builtins.toFile "lix-cross-file.conf" ''
[properties]
# Meson is convinced that if !buildPlatform.canExecute hostPlatform then we cannot
# build anything at all, which is not at all correct. If we can't execute the host
# platform, we'll just disable tests and doc gen.
needs_exe_wrapper = false
[binaries]
# Meson refuses to consider any CMake binary during cross compilation if it's
# not explicitly specified here, in the cross file.
# https://github.com/mesonbuild/meson/blob/0ed78cf6fa6d87c0738f67ae43525e661b50a8a2/mesonbuild/cmake/executor.py#L72
cmake = 'cmake'
'';
# The internal API docs need these for the build, but if we're not building
# Nix itself, then these don't need to be propagated.
maybePropagatedInputs = lib.optional enableGC boehmgc-nix ++ [ nlohmann_json ];
# .gitignore has already been processed, so any changes in it are irrelevant
# at this point. It is not represented verbatim for test purposes because
# that would interfere with repo semantics.
baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.;
configureFiles = fileset.unions [ ./version.json ];
topLevelBuildFiles = fileset.unions ([
./meson.build
./meson.options
./meson
./scripts/meson.build
./subprojects
# Required for meson to generate Cargo wraps
./Cargo.lock
]);
functionalTestFiles = fileset.unions [
./tests/functional
./tests/functional2
./tests/unit
(fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
];
in
assert (lintInsteadOfBuild -> lix-clang-tidy != null);
stdenv.mkDerivation (finalAttrs: {
inherit pname version;
src = fileset.toSource {
root = ./.;
fileset = fileset.intersection baseFiles (
fileset.unions (
[
configureFiles
topLevelBuildFiles
functionalTestFiles
]
++ lib.optionals (!finalAttrs.dontBuild || internalApiDocs || lintInsteadOfBuild) [
./doc
./misc
./src
./COPYING
]
++ lib.optionals lintInsteadOfBuild [ ./.clang-tidy ]
)
);
};
VERSION_SUFFIX = versionSuffix;
outputs =
[ "out" ]
++ lib.optionals (!finalAttrs.dontBuild) [
"dev"
"doc"
];
dontBuild = lintInsteadOfBuild;
mesonFlags =
let
sanitizeOpts = lib.optional (
sanitize != null
) "-Db_sanitize=${builtins.concatStringsSep "," sanitize}";
in
lib.optionals hostPlatform.isLinux [
# You'd think meson could just find this in PATH, but busybox is in buildInputs,
# which don't actually get added to PATH. And buildInputs is correct over
# nativeBuildInputs since this should be a busybox executable on the host.
"-Dsandbox-shell=${lib.getExe' busybox-sandbox-shell "busybox"}"
]
++ lib.optional hostPlatform.isStatic "-Denable-embedded-sandbox-shell=true"
++ lib.optional (finalAttrs.dontBuild && !lintInsteadOfBuild) "-Denable-build=false"
++ lib.optional lintInsteadOfBuild "-Dlix-clang-tidy-checks-path=${lix-clang-tidy}/lib/liblix-clang-tidy.so"
++ [
# mesonConfigurePhase automatically passes -Dauto_features=enabled,
# so we must explicitly enable or disable features that we are not passing
# dependencies for.
(lib.mesonEnable "gc" enableGC)
(lib.mesonEnable "internal-api-docs" internalApiDocs)
(lib.mesonBool "enable-tests" (finalAttrs.finalPackage.doCheck || lintInsteadOfBuild))
(lib.mesonBool "enable-docs" canRunInstalled)
(lib.mesonBool "werror" werror)
]
++ lib.optional (hostPlatform != buildPlatform) "--cross-file=${mesonCrossFile}"
++ sanitizeOpts;
# We only include CMake so that Meson can locate toml11, which only ships CMake dependency metadata.
dontUseCmakeConfigure = true;
nativeBuildInputs =
[
python3
python3.pkgs.pytest
python3.pkgs.pytest-xdist
meson
ninja
cmake
rustc
capnproto-lix
]
++ [
(lib.getBin lowdown-unsandboxed)
mdbook
mdbook-linkcheck
]
++ [
pkg-config
# Tests
git
mercurial
jq
lsof
]
++ lib.optional hostPlatform.isLinux util-linuxMinimal
++ lib.optional (!officialRelease && buildUnreleasedNotes) build-release-notes
++ lib.optional internalApiDocs doxygen
++ lib.optionals lintInsteadOfBuild [
# required for a wrapped clang-tidy
llvmPackages.clang-tools
# load-bearing order (just as below); the actual stdenv wrapped clang
# needs to precede the unwrapped clang in PATH such that calling `clang`
# can compile things.
stdenv.cc
# required for run-clang-tidy
llvmPackages.clang-unwrapped
];
buildInputs =
[
curl
bzip2
xz
brotli
editline-lix
openssl
sqlite
libarchive
boost
lowdown
libsodium
toml11
libexpr: rewrite the parser with pegtl instead of flex/bison this gives about 20% performance improvements on pure parsing. obviously it will be less on full eval, but depending on how much parsing is to be done (e.g. including hackage-packages.nix or not) it's more like 4%-10%. this has been tested (with thousands of core hours of fuzzing) to ensure that the ASTs produced by the new parser are exactly the same as the old one would have produced. error messages will change (sometimes by a lot) and are not yet perfect, but we would rather leave this as is for later. test results for running only the parser (excluding the variable binding code) in a tight loop with inputs and parameters as given are promising: - 40% faster on lix's package.nix at 10000 iterations - 1.3% faster on nixpkgs all-packages.nix at 1000 iterations - equivalent on all of nixpkgs concatenated at 100 iterations (excluding invalid files, each file surrounded with parens) more realistic benchmarks are somewhere in between the extremes, parsing once again getting the largest uplift. other realistic workloads improve by a few percentage points as well, notably system builds are 4% faster. Benchmarks summary (from ./bench/summarize.jq bench/bench-*.json) old/bin/nix --extra-experimental-features 'nix-command flakes' eval -f bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix mean: 0.408s ± 0.025s user: 0.355s | system: 0.033s median: 0.389s range: 0.388s ... 0.442s relative: 1 new/bin/nix --extra-experimental-features 'nix-command flakes' eval -f bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix mean: 0.332s ± 0.024s user: 0.279s | system: 0.033s median: 0.314s range: 0.313s ... 0.361s relative: 0.814 --- old/bin/nix --extra-experimental-features 'nix-command flakes' eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system' mean: 6.133s ± 0.022s user: 5.395s | system: 0.437s median: 6.128s range: 6.099s ... 6.183s relative: 1 new/bin/nix --extra-experimental-features 'nix-command flakes' eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system' mean: 5.925s ± 0.025s user: 5.176s | system: 0.456s median: 5.934s range: 5.861s ... 5.943s relative: 0.966 --- GC_INITIAL_HEAP_SIZE=10g old/bin/nix eval --extra-experimental-features 'nix-command flakes' --raw --impure --expr 'with import <nixpkgs/nixos> {}; system' mean: 4.503s ± 0.027s user: 3.731s | system: 0.547s median: 4.499s range: 4.478s ... 4.541s relative: 1 GC_INITIAL_HEAP_SIZE=10g new/bin/nix eval --extra-experimental-features 'nix-command flakes' --raw --impure --expr 'with import <nixpkgs/nixos> {}; system' mean: 4.285s ± 0.031s user: 3.504s | system: 0.571s median: 4.281s range: 4.221s ... 4.328s relative: 0.951 --- old/bin/nix --extra-experimental-features 'nix-command flakes' search --no-eval-cache github:nixos/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870 hello mean: 16.475s ± 0.07s user: 14.088s | system: 1.572s median: 16.495s range: 16.351s ... 16.536s relative: 1 new/bin/nix --extra-experimental-features 'nix-command flakes' search --no-eval-cache github:nixos/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870 hello mean: 15.973s ± 0.013s user: 13.558s | system: 1.615s median: 15.973s range: 15.946s ... 15.99s relative: 0.97 --- Change-Id: Ie66ec2d045dec964632c6541e25f8f0797319ee2
2024-06-16 21:10:09 +00:00
pegtl
capnproto-lix
]
++ lib.optionals hostPlatform.isLinux [
libseccomp
busybox-sandbox-shell
]
++ lib.optional internalApiDocs rapidcheck
++ lib.optional hostPlatform.isx86_64 libcpuid
# There have been issues building these dependencies
++ lib.optional (hostPlatform.canExecute buildPlatform) aws-sdk-cpp-nix
++ lib.optionals (finalAttrs.dontBuild) maybePropagatedInputs
# I am so sorry. This is because checkInputs are required to pass
# configure, but we don't actually want to *run* the checks here.
++ lib.optionals lintInsteadOfBuild finalAttrs.checkInputs;
checkInputs = [
gtest
rapidcheck
];
propagatedBuildInputs = lib.optionals (!finalAttrs.dontBuild) maybePropagatedInputs;
disallowedReferences = [ boost ];
# Needed for Meson to find Boost.
# https://github.com/NixOS/nixpkgs/issues/86131.
env = {
BOOST_INCLUDEDIR = "${lib.getDev boost}/include";
BOOST_LIBRARYDIR = "${lib.getLib boost}/lib";
# Meson allows referencing a /usr/share/cargo/registry shaped thing for subproject sources.
# Turns out the Nix-generated Cargo dependencies are named the same as they
# would be in a Cargo registry cache.
MESON_PACKAGE_CACHE_DIR = finalAttrs.cargoDeps;
};
cargoDeps = rustPlatform.importCargoLock { lockFile = ./Cargo.lock; };
preConfigure =
lib.optionalString (!finalAttrs.dontBuild && !hostPlatform.isStatic) ''
# Copy libboost_context so we don't get all of Boost in our closure.
# https://github.com/NixOS/nixpkgs/issues/45462
mkdir -p $out/lib
cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
rm -f $out/lib/*.a
''
+ lib.optionalString (!finalAttrs.dontBuild && hostPlatform.isLinux && !hostPlatform.isStatic) ''
chmod u+w $out/lib/*.so.*
patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
''
+ lib.optionalString (!finalAttrs.dontBuild && hostPlatform.isDarwin) ''
for LIB in $out/lib/*.dylib; do
chmod u+w $LIB
install_name_tool -id $LIB $LIB
install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
done
install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
''
+ ''
# Fix up /usr/bin/env shebangs relied on by the build
patchShebangs --build tests/ doc/manual/
'';
mesonBuildType = "debugoptimized";
installTargets = lib.optional internalApiDocs "internal-api-html";
enableParallelBuilding = true;
doCheck = canRunInstalled && !lintInsteadOfBuild;
mesonCheckFlags = [
"--suite=check"
"--print-errorlogs"
];
# the tests access localhost.
__darwinAllowLocalNetworking = true;
# Make sure the internal API docs are already built, because mesonInstallPhase
# won't let us build them there. They would normally be built in buildPhase,
# but the internal API docs are conventionally built with doBuild = false.
preInstall =
(lib.optionalString internalApiDocs ''
meson ''${mesonBuildFlags:-} compile "$installTargets"
'')
# evil, but like above, we do not want to run an actual build phase
+ lib.optionalString lintInsteadOfBuild ''
ninja clang-tidy
'';
installPhase = lib.optionalString lintInsteadOfBuild ''
runHook preInstall
touch $out
runHook postInstall
'';
postInstall =
lib.optionalString (!finalAttrs.dontBuild) ''
mkdir -p $doc/nix-support
echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
''
+ lib.optionalString hostPlatform.isStatic ''
mkdir -p $out/nix-support
echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
''
+ lib.optionalString stdenv.isDarwin ''
for lib in liblixutil.dylib liblixexpr.dylib; do
install_name_tool \
-change "${lib.getLib boost}/lib/libboost_context.dylib" \
"$out/lib/libboost_context.dylib" \
"$out/lib/$lib"
done
''
+ lib.optionalString internalApiDocs ''
mkdir -p $out/nix-support
echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> "$out/nix-support/hydra-build-products"
'';
doInstallCheck = finalAttrs.doCheck;
mesonInstallCheckFlags = [
"--suite=installcheck"
"--print-errorlogs"
];
installCheckPhase = ''
runHook preInstallCheck
flagsArray=($mesonInstallCheckFlags "''${mesonInstallCheckFlagsArray[@]}")
meson test --no-rebuild "''${flagsArray[@]}"
runHook postInstallCheck
'';
separateDebugInfo = !hostPlatform.isStatic && !finalAttrs.dontBuild;
strictDeps = true;
Build with traps on signed overflow This is UB, we should not be doing it, and we can cheaply turn it into crashes reliably. We would much rather have crashes than the program doing something silly. Benchmarks, but i wonder if they are nonsense because they get identical times across compilers?! | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| | `result-clang/bin/nix eval -f ../nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix` | 375.5 ± 24.0 | 353.8 | 408.8 | 1.00 | | `result-gcc/bin/nix eval -f ../nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix` | 407.9 ± 26.0 | 385.1 | 449.5 | 1.09 ± 0.10 | | `result-clangsan/bin/nix eval -f ../nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix` | 382.2 ± 26.6 | 354.9 | 419.0 | 1.02 ± 0.10 | | `result-gccsan/bin/nix eval -f ../nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix` | 408.6 ± 24.6 | 384.5 | 441.9 | 1.09 ± 0.10 | | Command | Mean [s] | Min [s] | Max [s] | Relative | |:---|---:|---:|---:|---:| | `result-clang/bin/nix search --no-eval-cache github:nixos/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870 hello` | 17.199 ± 0.167 | 16.930 | 17.499 | 1.01 ± 0.01 | | `result-gcc/bin/nix search --no-eval-cache github:nixos/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870 hello` | 17.409 ± 0.126 | 17.242 | 17.633 | 1.02 ± 0.01 | | `result-clangsan/bin/nix search --no-eval-cache github:nixos/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870 hello` | 17.080 ± 0.137 | 16.879 | 17.350 | 1.00 | | `result-gccsan/bin/nix search --no-eval-cache github:nixos/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870 hello` | 17.396 ± 0.160 | 17.131 | 17.660 | 1.02 ± 0.01 | | Command | Mean [s] | Min [s] | Max [s] | Relative | |:---|---:|---:|---:|---:| | `result-clang/bin/nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'` | 6.267 ± 0.069 | 6.197 | 6.415 | 1.02 ± 0.01 | | `result-gcc/bin/nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'` | 6.232 ± 0.045 | 6.180 | 6.311 | 1.01 ± 0.01 | | `result-clangsan/bin/nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'` | 6.162 ± 0.020 | 6.133 | 6.196 | 1.00 | | `result-gccsan/bin/nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'` | 6.229 ± 0.031 | 6.199 | 6.289 | 1.01 ± 0.01 | | Command | Mean [s] | Min [s] | Max [s] | Relative | |:---|---:|---:|---:|---:| | `GC_INITIAL_HEAP_SIZE=10g result-clang/bin/nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'` | 4.683 ± 0.044 | 4.630 | 4.761 | 1.00 | | `GC_INITIAL_HEAP_SIZE=10g result-gcc/bin/nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'` | 4.750 ± 0.041 | 4.680 | 4.812 | 1.01 ± 0.01 | | `GC_INITIAL_HEAP_SIZE=10g result-clangsan/bin/nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'` | 4.703 ± 0.040 | 4.640 | 4.760 | 1.00 ± 0.01 | | `GC_INITIAL_HEAP_SIZE=10g result-gccsan/bin/nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'` | 4.766 ± 0.037 | 4.727 | 4.844 | 1.02 ± 0.01 | Change-Id: I616ca3eab670317587d47b41870d8ac963c019ae
2024-03-25 06:45:25 +00:00
# strictoverflow is disabled because we trap on signed overflow instead
hardeningDisable = [ "strictoverflow" ] ++ lib.optional hostPlatform.isStatic "pie";
meta = {
mainProgram = "nix";
platforms = lib.platforms.unix;
};
# Export the patched version of boehmgc.
# flake.nix exports that into its overlay.
libstore/local-derivation-goal: prohibit creating setuid/setgid binaries With Linux kernel >=6.6 & glibc 2.39 a `fchmodat2(2)` is available that isn't filtered away by the libseccomp sandbox. Being able to use this to bypass that restriction has surprising results for some builds such as lxc[1]: > With kernel ≥6.6 and glibc 2.39, lxc's install phase uses fchmodat2, > which slips through https://github.com/NixOS/nix/blob/9b88e5284608116b7db0dbd3d5dd7a33b90d52d7/src/libstore/build/local-derivation-goal.cc#L1650-L1663. > The fixupPhase then uses fchmodat, which fails. > With older kernel or glibc, setting the suid bit fails in the > install phase, which is not treated as fatal, and then the > fixup phase does not try to set it again. Please note that there are still ways to bypass this sandbox[2] and this is mostly a fix for the breaking builds. This change works by creating a syscall filter for the `fchmodat2` syscall (number 452 on most systems). The problem is that glibc 2.39 is needed to have the correct syscall number available via `__NR_fchmodat2` / `__SNR_fchmodat2`, but this flake is still on nixpkgs 23.11. To have this change everywhere and not dependent on the glibc this package is built against, I added a header "fchmodat2-compat.hh" that sets the syscall number based on the architecture. On most platforms its 452 according to glibc with a few exceptions: $ rg --pcre2 'define __NR_fchmodat2 (?!452)' sysdeps/unix/sysv/linux/x86_64/x32/arch-syscall.h 58:#define __NR_fchmodat2 1073742276 sysdeps/unix/sysv/linux/mips/mips64/n32/arch-syscall.h 67:#define __NR_fchmodat2 6452 sysdeps/unix/sysv/linux/mips/mips64/n64/arch-syscall.h 62:#define __NR_fchmodat2 5452 sysdeps/unix/sysv/linux/mips/mips32/arch-syscall.h 70:#define __NR_fchmodat2 4452 sysdeps/unix/sysv/linux/alpha/arch-syscall.h 59:#define __NR_fchmodat2 562 I added a small regression-test to the setuid integration-test that attempts to set the suid bit on a file using the fchmodat2 syscall. I confirmed that the test fails without the change in local-derivation-goal. Additionally, we require libseccomp 2.5.5 or greater now: as it turns out, libseccomp maintains an internal syscall table and validates each rule against it. This means that when using libseccomp 2.5.4 or older, one may pass `452` as syscall number against it, but since it doesn't exist in the internal structure, `libseccomp` will refuse to create a filter for that. This happens with nixpkgs-23.11, i.e. on stable NixOS and when building Lix against the project's flake. To work around that * a backport of libseccomp 2.5.5 on upstream nixpkgs has been scheduled[3]. * the package now uses libseccomp 2.5.5 on its own already. This is to provide a quick fix since the correct fix for 23.11 is still a staging cycle away. We still need the compat header though since `SCMP_SYS(fchmodat2)` internally transforms this into `__SNR_fchmodat2` which points to `__NR_fchmodat2` from glibc 2.39, so it wouldn't build on glibc 2.38. The updated syscall table from libseccomp 2.5.5 is NOT used for that step, but used later, so we need both, our compat header and their syscall table 🤷 Relevant PRs in CppNix: * https://github.com/NixOS/nix/pull/10591 * https://github.com/NixOS/nix/pull/10501 [1] https://github.com/NixOS/nixpkgs/issues/300635#issuecomment-2031073804 [2] https://github.com/NixOS/nixpkgs/issues/300635#issuecomment-2030844251 [3] https://github.com/NixOS/nixpkgs/pull/306070 (cherry picked from commit ba6804518772e6afb403dd55478365d4b863c854) Change-Id: I6921ab5a363188c6bff617750d00bb517276b7fe
2024-04-14 12:10:23 +00:00
passthru = {
inherit (__forDefaults)
boehmgc-nix
editline-lix
build-release-notes
pegtl
capnproto-lix
;
# The collection of dependency logic for this derivation is complicated enough that
# it's easier to parameterize the devShell off an already called package.nix.
mkDevShell =
{
mkShell,
bashInteractive,
clangbuildanalyzer,
doxygen,
glibcLocales,
just,
nixfmt-rfc-style,
[resubmit] flake: update nixpkgs pin 23.11->24.05 (+ boehmgc compat changes) -- message from cl/1418 -- The boehmgc changes are bundled into this commit because doing otherwise would require an annoying dance of "adding compatibility for < 8.2.6 and >= 8.2.6" then updating the pin then removing the (now unneeded) compatibility. It doesn't seem worth the trouble to me given the low complexity of said changes. Rebased coroutine-sp-fallback.diff patch taken from https://github.com/NixOS/nixpkgs/pull/317227 -- jade resubmit changes -- This is a resubmission of https://gerrit.lix.systems/c/lix/+/1418, which was reverted in https://gerrit.lix.systems/c/lix/+/1432 for breaking CI evaluation without being detected. I have run `nix flake check -Lv` on this one before submission and it passes on my machine and crucially without eval errors, so the CI result should be accurate. It seems like someone renamed forbiddenDependenciesRegex to forbiddenDependenciesRegexes in nixpkgs and also changed the type incompatibly. That's pretty silly, but at least it's just an eval error. Also, `xonsh` regressed the availability of `xonsh-unwrapped`, but it was fixed by us in https://github.com/NixOS/nixpkgs/pull/317636, which is now in our channel, so we update nixpkgs compared to the original iteration of this to simply get that. We originally had a regression related to some reorganization of the nixpkgs lib test suite in which there was broken parameter passing. This, too, we got quickfixed in nixpkgs, so we don't need any changes for it: https://github.com/NixOS/nixpkgs/pull/317772 Related: https://gerrit.lix.systems/c/lix/+/1428 Fixes: https://git.lix.systems/lix-project/lix/issues/385 Change-Id: I26d41ea826fec900ebcad0f82a727feb6bcd28f3
2024-06-08 14:57:08 +00:00
skopeo,
xonsh,
# Lix specific packages
pre-commit-checks,
contribNotice,
libstore/build: use an allowlist approach to syscall filtering Previously, system call filtering (to prevent builders from storing files with setuid/setgid permission bits or extended attributes) was performed using a blocklist. While this looks simple at first, it actually carries significant security and maintainability risks: after all, the kernel may add new syscalls to achieve the same functionality one is trying to block, and it can even be hard to actually add the syscall to the blocklist when building against a C library that doesn't know about it yet. For a recent demonstration of this happening in practice to Nix, see the introduction of fchmodat2 [0] [1]. The allowlist approach does not share the same drawback. While it does require a rather large list of harmless syscalls to be maintained in the codebase, failing to update this list (and roll out the update to all users) in time has rather benign effects; at worst, very recent programs that already rely on new syscalls will fail with an error the same way they would on a slightly older kernel that doesn't support them yet. Most importantly, no unintended new ways of performing dangerous operations will be silently allowed. Another possible drawback is reduced system call performance due to the larger filter created by the allowlist requiring more computation [2]. However, this issue has not convincingly been demonstrated yet in practice, for example in systemd or various browsers. To the contrary, it has been measured that the the actual filter constructed here has approximately the same overhead as a very simple filter blocking only one system call. This commit tries to keep the behavior as close to unchanged as possible. The system call list is in line with libseccomp 2.5.5 and glibc 2.39, which are the latest versions at the point of writing. Since libseccomp 2.5.5 is already a requirement and the distributions shipping this together with older versions of glibc are mostly not a thing any more, this should not lead to more build failures any more. [0] https://github.com/NixOS/nixpkgs/issues/300635 [1] https://github.com/NixOS/nix/issues/10424 [2] https://github.com/flatpak/flatpak/pull/4462#issuecomment-1061690607 Change-Id: I541be3ea9b249bcceddfed6a5a13ac10b11e16ad
2024-05-26 08:35:03 +00:00
check-syscalls,
# debuggers
gdb,
rr,
}:
let
glibcFix = lib.optionalAttrs (buildPlatform.isLinux && glibcLocales != null) {
# Required to make non-NixOS Linux not complain about missing locale files during configure in a dev shell
LOCALE_ARCHIVE = "${lib.getLib pkgs.glibcLocales}/lib/locale/locale-archive";
};
pythonPackages = (
p: [
# FIXME: these have to be added twice due to the nix shell using a
# wrapped python instead of build inputs for its python inputs
p.pytest
p.pytest-xdist
p.yapf
p.python-frontmatter
p.requests
p.xdg-base-dirs
p.packaging
[resubmit] flake: update nixpkgs pin 23.11->24.05 (+ boehmgc compat changes) -- message from cl/1418 -- The boehmgc changes are bundled into this commit because doing otherwise would require an annoying dance of "adding compatibility for < 8.2.6 and >= 8.2.6" then updating the pin then removing the (now unneeded) compatibility. It doesn't seem worth the trouble to me given the low complexity of said changes. Rebased coroutine-sp-fallback.diff patch taken from https://github.com/NixOS/nixpkgs/pull/317227 -- jade resubmit changes -- This is a resubmission of https://gerrit.lix.systems/c/lix/+/1418, which was reverted in https://gerrit.lix.systems/c/lix/+/1432 for breaking CI evaluation without being detected. I have run `nix flake check -Lv` on this one before submission and it passes on my machine and crucially without eval errors, so the CI result should be accurate. It seems like someone renamed forbiddenDependenciesRegex to forbiddenDependenciesRegexes in nixpkgs and also changed the type incompatibly. That's pretty silly, but at least it's just an eval error. Also, `xonsh` regressed the availability of `xonsh-unwrapped`, but it was fixed by us in https://github.com/NixOS/nixpkgs/pull/317636, which is now in our channel, so we update nixpkgs compared to the original iteration of this to simply get that. We originally had a regression related to some reorganization of the nixpkgs lib test suite in which there was broken parameter passing. This, too, we got quickfixed in nixpkgs, so we don't need any changes for it: https://github.com/NixOS/nixpkgs/pull/317772 Related: https://gerrit.lix.systems/c/lix/+/1428 Fixes: https://git.lix.systems/lix-project/lix/issues/385 Change-Id: I26d41ea826fec900ebcad0f82a727feb6bcd28f3
2024-06-08 14:57:08 +00:00
(p.toPythonModule xonsh.passthru.unwrapped)
]
);
pythonEnv = python3.withPackages pythonPackages;
# pkgs.mkShell uses pkgs.stdenv by default, regardless of inputsFrom.
actualMkShell = mkShell.override { inherit stdenv; };
in
actualMkShell (
glibcFix
// {
name = "lix-shell-env";
# finalPackage is necessary to propagate stuff that is set by mkDerivation itself,
# like doCheck.
inputsFrom = [ finalAttrs.finalPackage ];
# For Meson to find Boost.
env = finalAttrs.env;
mesonFlags =
# I guess this is necessary because mesonFlags to mkDerivation doesn't propagate in inputsFrom,
# which only propagates stuff set in hooks? idk.
finalAttrs.mesonFlags
# Clangd breaks when GCC is using precompiled headers, so for the devshell specifically
# we make precompiled C++ stdlib conditional on using Clang.
# https://git.lix.systems/lix-project/lix/issues/374
++ [ (lib.mesonBool "enable-pch-std" stdenv.cc.isClang) ];
packages =
lib.optional (stdenv.cc.isClang && hostPlatform == buildPlatform) llvmPackages.clang-tools
++ [
# Why are we providing a bashInteractive? Well, when you run
# `bash` from inside `nix develop`, say, because you are using it
# via direnv, you will by default get bash (unusable edition).
bashInteractive
libstore/build: use an allowlist approach to syscall filtering Previously, system call filtering (to prevent builders from storing files with setuid/setgid permission bits or extended attributes) was performed using a blocklist. While this looks simple at first, it actually carries significant security and maintainability risks: after all, the kernel may add new syscalls to achieve the same functionality one is trying to block, and it can even be hard to actually add the syscall to the blocklist when building against a C library that doesn't know about it yet. For a recent demonstration of this happening in practice to Nix, see the introduction of fchmodat2 [0] [1]. The allowlist approach does not share the same drawback. While it does require a rather large list of harmless syscalls to be maintained in the codebase, failing to update this list (and roll out the update to all users) in time has rather benign effects; at worst, very recent programs that already rely on new syscalls will fail with an error the same way they would on a slightly older kernel that doesn't support them yet. Most importantly, no unintended new ways of performing dangerous operations will be silently allowed. Another possible drawback is reduced system call performance due to the larger filter created by the allowlist requiring more computation [2]. However, this issue has not convincingly been demonstrated yet in practice, for example in systemd or various browsers. To the contrary, it has been measured that the the actual filter constructed here has approximately the same overhead as a very simple filter blocking only one system call. This commit tries to keep the behavior as close to unchanged as possible. The system call list is in line with libseccomp 2.5.5 and glibc 2.39, which are the latest versions at the point of writing. Since libseccomp 2.5.5 is already a requirement and the distributions shipping this together with older versions of glibc are mostly not a thing any more, this should not lead to more build failures any more. [0] https://github.com/NixOS/nixpkgs/issues/300635 [1] https://github.com/NixOS/nix/issues/10424 [2] https://github.com/flatpak/flatpak/pull/4462#issuecomment-1061690607 Change-Id: I541be3ea9b249bcceddfed6a5a13ac10b11e16ad
2024-05-26 08:35:03 +00:00
check-syscalls
pythonEnv
# docker image tool
skopeo
just
nixfmt-rfc-style
# Included above when internalApiDocs is true, but we set that to
# false intentionally to save dev build time.
# To build them in a dev shell, you can set -Dinternal-api-docs=enabled when configuring.
doxygen
# Load-bearing order. Must come before clang-unwrapped below, but after clang_tools above.
stdenv.cc
]
++ [
pkgs.rust-analyzer
pkgs.cargo
pkgs.rustc
pkgs.rustfmt
pkgs.rustPlatform.rustLibSrc
pkgs.rustPlatform.rustcSrc
]
++ lib.optionals stdenv.cc.isClang [
# Required for clang-tidy checks.
llvmPackages.llvm
llvmPackages.clang-unwrapped.dev
]
++ lib.optional (pre-commit-checks ? enabledPackages) pre-commit-checks.enabledPackages
++ lib.optional (lib.meta.availableOn buildPlatform clangbuildanalyzer) clangbuildanalyzer
++ lib.optional (!stdenv.isDarwin) gdb
++ lib.optional (lib.meta.availableOn buildPlatform rr) rr
++ finalAttrs.checkInputs;
shellHook = ''
# don't re-run the hook in (other) nested nix-shells
function lixShellHook() {
# n.b. how the heck does this become -env-env? well, `nix develop` does it:
# https://git.lix.systems/lix-project/lix/src/commit/7575db522e9008685c4009423398f6900a16bcce/src/nix/develop.cc#L240-L241
# this is, of course, absurd.
if [[ $name != lix-shell-env && $name != lix-shell-env-env ]]; then
return
fi
PATH=$prefix/bin''${PATH:+:''${PATH}}
unset PYTHONPATH
export MANPATH=$out/share/man:''${MANPATH:-}
# Make bash completion work.
XDG_DATA_DIRS+=:$out/share
if [[ ! -f ./.this-is-lix ]]; then
echo "Dev shell not started from inside a Lix repo, skipping repo setup" >&2
return
fi
${lib.optionalString (pre-commit-checks ? shellHook) pre-commit-checks.shellHook}
# Allow `touch .nocontribmsg` to turn this notice off.
if ! [[ -f .nocontribmsg ]]; then
cat ${contribNotice}
fi
# Install the Gerrit commit-msg hook.
# (git common dir is the main .git, including for worktrees)
if gitcommondir=$(git rev-parse --git-common-dir 2>/dev/null) && [[ ! -f "$gitcommondir/hooks/commit-msg" ]]; then
echo 'Installing Gerrit commit-msg hook (adds Change-Id to commit messages)' >&2
mkdir -p "$gitcommondir/hooks"
curl -s -Lo "$gitcommondir/hooks/commit-msg" https://gerrit.lix.systems/tools/hooks/commit-msg
chmod u+x "$gitcommondir/hooks/commit-msg"
fi
unset gitcommondir
}
lixShellHook
'';
}
);
perl-bindings = pkgs.callPackage ./perl { inherit fileset stdenv; };
binaryTarball = pkgs.callPackage ./nix-support/binary-tarball.nix {
nix = finalAttrs.finalPackage;
};
libstore/local-derivation-goal: prohibit creating setuid/setgid binaries With Linux kernel >=6.6 & glibc 2.39 a `fchmodat2(2)` is available that isn't filtered away by the libseccomp sandbox. Being able to use this to bypass that restriction has surprising results for some builds such as lxc[1]: > With kernel ≥6.6 and glibc 2.39, lxc's install phase uses fchmodat2, > which slips through https://github.com/NixOS/nix/blob/9b88e5284608116b7db0dbd3d5dd7a33b90d52d7/src/libstore/build/local-derivation-goal.cc#L1650-L1663. > The fixupPhase then uses fchmodat, which fails. > With older kernel or glibc, setting the suid bit fails in the > install phase, which is not treated as fatal, and then the > fixup phase does not try to set it again. Please note that there are still ways to bypass this sandbox[2] and this is mostly a fix for the breaking builds. This change works by creating a syscall filter for the `fchmodat2` syscall (number 452 on most systems). The problem is that glibc 2.39 is needed to have the correct syscall number available via `__NR_fchmodat2` / `__SNR_fchmodat2`, but this flake is still on nixpkgs 23.11. To have this change everywhere and not dependent on the glibc this package is built against, I added a header "fchmodat2-compat.hh" that sets the syscall number based on the architecture. On most platforms its 452 according to glibc with a few exceptions: $ rg --pcre2 'define __NR_fchmodat2 (?!452)' sysdeps/unix/sysv/linux/x86_64/x32/arch-syscall.h 58:#define __NR_fchmodat2 1073742276 sysdeps/unix/sysv/linux/mips/mips64/n32/arch-syscall.h 67:#define __NR_fchmodat2 6452 sysdeps/unix/sysv/linux/mips/mips64/n64/arch-syscall.h 62:#define __NR_fchmodat2 5452 sysdeps/unix/sysv/linux/mips/mips32/arch-syscall.h 70:#define __NR_fchmodat2 4452 sysdeps/unix/sysv/linux/alpha/arch-syscall.h 59:#define __NR_fchmodat2 562 I added a small regression-test to the setuid integration-test that attempts to set the suid bit on a file using the fchmodat2 syscall. I confirmed that the test fails without the change in local-derivation-goal. Additionally, we require libseccomp 2.5.5 or greater now: as it turns out, libseccomp maintains an internal syscall table and validates each rule against it. This means that when using libseccomp 2.5.4 or older, one may pass `452` as syscall number against it, but since it doesn't exist in the internal structure, `libseccomp` will refuse to create a filter for that. This happens with nixpkgs-23.11, i.e. on stable NixOS and when building Lix against the project's flake. To work around that * a backport of libseccomp 2.5.5 on upstream nixpkgs has been scheduled[3]. * the package now uses libseccomp 2.5.5 on its own already. This is to provide a quick fix since the correct fix for 23.11 is still a staging cycle away. We still need the compat header though since `SCMP_SYS(fchmodat2)` internally transforms this into `__SNR_fchmodat2` which points to `__NR_fchmodat2` from glibc 2.39, so it wouldn't build on glibc 2.38. The updated syscall table from libseccomp 2.5.5 is NOT used for that step, but used later, so we need both, our compat header and their syscall table 🤷 Relevant PRs in CppNix: * https://github.com/NixOS/nix/pull/10591 * https://github.com/NixOS/nix/pull/10501 [1] https://github.com/NixOS/nixpkgs/issues/300635#issuecomment-2031073804 [2] https://github.com/NixOS/nixpkgs/issues/300635#issuecomment-2030844251 [3] https://github.com/NixOS/nixpkgs/pull/306070 (cherry picked from commit ba6804518772e6afb403dd55478365d4b863c854) Change-Id: I6921ab5a363188c6bff617750d00bb517276b7fe
2024-04-14 12:10:23 +00:00
};
})