From d8e452a91b4440a049c03dec118c439fa773e323 Mon Sep 17 00:00:00 2001 From: stuebinm Date: Tue, 28 May 2024 15:58:13 +0200 Subject: [PATCH 01/10] document context-dependent keywords Documents some of the weirdness of __curPos and the or keyword. This does not fit well into any existing section for either of them, though the use of or as a quasi-operator is mentioned in the section on operators. Addresses https://git.lix.systems/lix-project/lix/issues/353 Change-Id: I7c906c8368843dca6944e8b22573b6d201cd9a76 --- doc/manual/src/language/builtin-constants.md | 11 ++++ doc/manual/src/language/constructs.md | 67 ++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/doc/manual/src/language/builtin-constants.md b/doc/manual/src/language/builtin-constants.md index 74e87146f..970bf6c5a 100644 --- a/doc/manual/src/language/builtin-constants.md +++ b/doc/manual/src/language/builtin-constants.md @@ -7,3 +7,14 @@ These constants are built into the Nix language evaluator: {{#include @generated@/language/builtin-constants.md}} + +## Things which might be mistaken for constants + +
+
__curPos
+
+ +This is not a constant but a [context-dependent keyword](@docroot@/language/constructs.md#keywords-__curPos) + +
+
diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md index 2a527770a..67fd0126d 100644 --- a/doc/manual/src/language/constructs.md +++ b/doc/manual/src/language/constructs.md @@ -380,3 +380,70 @@ let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... Comments can be single-line, started with a `#` character, or inline/multi-line, enclosed within `/* ... */`. + +## Context-dependent keywords + +
+
+ __curPos +
+
+ +A quasi-constant which will be replaced with an attribute set describing +the location where `__curPos` was used, with attributes `file`, `line`, +and `column`. For example, `import ./file.nix` will result in + +```nix +{ + column = 1; + file = "/path/to/some/file.nix"; + line = 1; +} +``` + +assuming `file.nix` contains nothing but `__curPos`. + +In context without a source file (such as `nix-repl`), it will always +be replaced with `null`: + +```nix-repl +nix-repl> __curPos +null +``` + +While it may vaguely look like a builtin, this is a very different beast +that is handled directly by the parser. It thus cannot be shadowed, +bound to a different name, and is also not available under +[`builtins`](@docroot@/language/builtin-constants.md#builtins-builtins). + +```nix-repl +nix-repl> let __curPos = "no"; in __curPos +null +``` + +Despite this `__curPos`, much like `or`, may still be used as an identifier, +it is only treated specially when it appears as an unqualified name: + +```nix-repl +nix-repl> { __curPos = 1; }.__curPos +1 +``` + +
+ +
+ or +
+
+ +`or` is used in [Attribute selection](@docroot@/language/operators.html#attribute-selection), +where it is a keyword. + +However, it is not a keyword in some other contexts, and can be used as +a binding name in attribute sets, let-bindings, non-initial function +application position, and as a label in attribute paths. + +Its use as anything other than a keyword is discouraged. + +
+
From e6e5cacabeab0959d092aaf4957932a0b6f4b78e Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 30 May 2024 13:41:31 -0700 Subject: [PATCH 02/10] shellHook: make it actually run When we changed this in I91cb6eb6668f3a8eace36ecbdb01eb367861d77b to not run in nested shells, we didn't predict that `nix develop` would do something ridiculous and append -env to things silently. `nix-shell` of course does not do this, so we need to tolerate both. Change-Id: Ibe7cf546823d7358ebc0414ecbe154e3e3194f45 --- package.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/package.nix b/package.nix index 6251b28ba..283a590c1 100644 --- a/package.nix +++ b/package.nix @@ -449,7 +449,10 @@ stdenv.mkDerivation (finalAttrs: { shellHook = '' # don't re-run the hook in (other) nested nix-shells function lixShellHook() { - if [[ $name != lix-shell-env ]]; then + # n.b. how the heck does this become -env-env? well, `nix develop` does it: + # https://git.lix.systems/lix-project/lix/src/commit/7575db522e9008685c4009423398f6900a16bcce/src/nix/develop.cc#L240-L241 + # this is, of course, absurd. + if [[ $name != lix-shell-env && $name != lix-shell-env-env ]]; then return; fi From 0f99ed43f15aba7805ebe6d757a2d98418596205 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 30 May 2024 13:44:01 -0700 Subject: [PATCH 03/10] build-time: remove 20% more by PCH'ing C++ stdlib It seems like someone implemented precompiled headers a long time ago and then it never got ported to meson or maybe didn't work at all. This is, however, blessedly easy to simply implement. I went looking for `#define` that could affect the result of precompiling the headers, and as far as I can tell we aren't doing any of that, so this should truly just be free build time savings. Previous state: Compilation (551 times): Parsing (frontend): 1302.1 s Codegen & opts (backend): 956.3 s New state: **** Time summary: Compilation (567 times): Parsing (frontend): 1123.0 s Codegen & opts (backend): 1078.1 s I wonder if the "regression" in codegen time is just doing the PCH operation a few times, because meson does it per-target. Change-Id: I664366b8069bab4851308b3a7571bea97ac64022 --- misc/pre-commit.nix | 9 +++++---- package.nix | 1 - src/libcmd/meson.build | 1 + src/libexpr/meson.build | 1 + src/libfetchers/meson.build | 1 + src/libmain/meson.build | 1 + src/libstore/meson.build | 1 + src/libutil/meson.build | 1 + src/nix/meson.build | 1 + precompiled-headers.h => src/pch/precompiled-headers.hh | 0 tests/unit/meson.build | 6 ++++++ 11 files changed, 18 insertions(+), 5 deletions(-) rename precompiled-headers.h => src/pch/precompiled-headers.hh (100%) diff --git a/misc/pre-commit.nix b/misc/pre-commit.nix index ed2b152a3..4f54141b3 100644 --- a/misc/pre-commit.nix +++ b/misc/pre-commit.nix @@ -87,11 +87,12 @@ pre-commit-run { "file" "header" ]; - # generated files; these will never actually be seen by this - # check, and are left here as documentation excludes = [ - "(parser|lexer)-tab\\.hh$" - "\\.gen\\.hh$" + ''^src/pch/.*$'' + # generated files; these will never actually be seen by this + # check, and are left here as documentation + ''(parser|lexer)-tab\.hh$'' + ''\.gen\.hh$'' ]; entry = lib.getExe pkgs.check-headers; }; diff --git a/package.nix b/package.nix index 283a590c1..43b709023 100644 --- a/package.nix +++ b/package.nix @@ -169,7 +169,6 @@ stdenv.mkDerivation (finalAttrs: { ./boehmgc-coroutine-sp-fallback.diff ./doc ./misc - ./precompiled-headers.h ./src ./COPYING ] diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 4da1b496b..4dc0714c8 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -54,6 +54,7 @@ libcmd = library( nlohmann_json, lix_doc ], + cpp_pch : ['../pch/precompiled-headers.hh'], install : true, # FIXME(Qyriad): is this right? install_rpath : libdir, diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index fda6fde2c..9a18c7ab3 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -145,6 +145,7 @@ libexpr = library( include_directories : [ '../libmain', ], + cpp_pch : ['../pch/precompiled-headers.hh'], install : true, # FIXME(Qyriad): is this right? install_rpath : libdir, diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index dbb85b84c..365bcd4a7 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -30,6 +30,7 @@ libfetchers = library( liblixutil, nlohmann_json, ], + cpp_pch : ['../pch/precompiled-headers.hh'], install : true, # FIXME(Qyriad): is this right? install_rpath : libdir, diff --git a/src/libmain/meson.build b/src/libmain/meson.build index b17247a9d..075aa83b2 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -20,6 +20,7 @@ libmain = library( liblixutil, liblixstore, ], + cpp_pch : ['../pch/precompiled-headers.hh'], install : true, # FIXME(Qyriad): is this right? install_rpath : libdir, diff --git a/src/libstore/meson.build b/src/libstore/meson.build index f776e9621..65ecacc20 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -220,6 +220,7 @@ libstore = library( nlohmann_json, ], cpp_args : cpp_args, + cpp_pch : ['../pch/precompiled-headers.hh'], install : true, # FIXME(Qyriad): is this right? install_rpath : libdir, diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 96450fbe2..cfdd0e52c 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -129,6 +129,7 @@ libutil = library( openssl, nlohmann_json, ], + cpp_pch : ['../pch/precompiled-headers.hh'], implicit_include_directories : true, install : true, ) diff --git a/src/nix/meson.build b/src/nix/meson.build index e41399b5d..45303641f 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -89,6 +89,7 @@ nix = executable( boehm, nlohmann_json, ], + cpp_pch : ['../pch/precompiled-headers.hh'], install : true, # FIXME(Qyriad): is this right? install_rpath : libdir, diff --git a/precompiled-headers.h b/src/pch/precompiled-headers.hh similarity index 100% rename from precompiled-headers.h rename to src/pch/precompiled-headers.hh diff --git a/tests/unit/meson.build b/tests/unit/meson.build index 35a11a5d3..d8d4a00d1 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -68,6 +68,7 @@ libutil_tester = executable( liblixutil_test_support, nlohmann_json, ], + cpp_pch : ['../../src/pch/precompiled-headers.hh'], ) test( @@ -102,6 +103,7 @@ libstore_test_support = library( include_directories : include_directories( 'libstore-support', ), + cpp_pch : ['../../src/pch/precompiled-headers.hh'], ) liblixstore_test_support = declare_dependency( include_directories : include_directories('libstore-support'), @@ -135,6 +137,7 @@ libstore_tester = executable( gtest, nlohmann_json, ], + cpp_pch : ['../../src/pch/precompiled-headers.hh'], ) test( @@ -166,6 +169,7 @@ libexpr_test_support = library( include_directories : include_directories( 'libexpr-support', ), + cpp_pch : ['../../src/pch/precompiled-headers.hh'], ) liblixexpr_test_support = declare_dependency( include_directories : include_directories('libexpr-support'), @@ -199,6 +203,7 @@ libexpr_tester = executable( gtest, nlohmann_json, ], + cpp_pch : ['../../src/pch/precompiled-headers.hh'], ) test( @@ -225,6 +230,7 @@ libcmd_tester = executable( gtest, boost, ], + cpp_pch : ['../../src/pch/precompiled-headers.hh'], ) test( From a6b33cb3b22f98c7e40ddc47fd5a09c905abd9e6 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 30 May 2024 14:21:00 -0700 Subject: [PATCH 04/10] gitignore: delete 90% of it *laughs in meson putting it all in build/* Change-Id: Ifcb0d3104cf9e64c4de91c3a92828899a209d00d --- .gitignore | 129 ----------------------------------------------------- 1 file changed, 129 deletions(-) diff --git a/.gitignore b/.gitignore index 4d921d97f..816a8e4b1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,128 +1,5 @@ -Makefile.config -perl/Makefile.config - -# / -/aclocal.m4 -/autom4te.cache -/precompiled-headers.h.gch -/config.* -/configure -/stamp-h1 -/svn-revision -/libtool -/config - -# /doc/manual/ -/doc/manual/*.1 -/doc/manual/*.5 -/doc/manual/*.8 -/doc/manual/generated/* -/doc/manual/nix.json -/doc/manual/conf-file.json -/doc/manual/language.json -/doc/manual/xp-features.json -/doc/manual/src/command-ref/experimental-features-shortlist.md -/doc/manual/src/contributing/experimental-feature-descriptions.md -/doc/manual/src/release-notes/rl-next-generated.md - -# /scripts/ -/scripts/nix-profile.sh -/scripts/nix-profile-daemon.sh -/scripts/nix-profile.fish -/scripts/nix-profile-daemon.fish - -# /src/libexpr/ -/src/libexpr/lexer-tab.cc -/src/libexpr/lexer-tab.hh -/src/libexpr/parser-tab.cc -/src/libexpr/parser-tab.hh -/src/libexpr/parser-tab.output -/src/libexpr/nix.tbl -/src/libexpr/tests -/tests/unit/libexpr/libnixexpr-tests - -# /src/libstore/ -*.gen.* -/src/libstore/tests -/tests/unit/libstore/libnixstore-tests - -# /src/libutil/ -/src/libutil/tests -/tests/unit/libutil/libnixutil-tests - -/src/nix/nix - -/src/nix/doc - -# /src/nix-env/ -/src/nix-env/nix-env - -# /src/nix-instantiate/ -/src/nix-instantiate/nix-instantiate - -# /src/nix-store/ -/src/nix-store/nix-store - -/src/nix-prefetch-url/nix-prefetch-url - -/src/nix-collect-garbage/nix-collect-garbage - -# /src/nix-channel/ -/src/nix-channel/nix-channel - -# /src/nix-build/ -/src/nix-build/nix-build - -/src/nix-copy-closure/nix-copy-closure - -/src/error-demo/error-demo - -/src/build-remote/build-remote - -# /tests/functional/ -/tests/functional/test-tmp -/tests/functional/common/vars-and-functions.sh -/tests/functional/result* -/tests/functional/restricted-innocent -/tests/functional/shell -/tests/functional/shell.drv -/tests/functional/config.nix -/tests/functional/ca/config.nix -/tests/functional/dyn-drv/config.nix -/tests/functional/repl-result-out -/tests/functional/debugger-test-out -/tests/functional/test-libstoreconsumer/test-libstoreconsumer - -# /tests/functional/lang/ -/tests/functional/lang/*.out -/tests/functional/lang/*.out.xml -/tests/functional/lang/*.err -/tests/functional/lang/*.ast - -/perl/lib/Nix/Config.pm -/perl/lib/Nix/Store.cc - -/misc/systemd/nix-daemon.service -/misc/systemd/nix-daemon.socket -/misc/systemd/nix-daemon.conf -/misc/upstart/nix-daemon.conf - -/src/resolve-system-dependencies/resolve-system-dependencies - outputs/ -*.a -*.o -*.o.tmp -*.so -*.dylib -*.dll -*.exe -*.dep -*~ -*.pc -*.plist - # GNU Global GPATH GRTAGS @@ -132,17 +9,11 @@ GTAGS # ccls /.ccls-cache -# auto-generated compilation database -compile_commands.json - -nix-rust/target - result result-* .vscode/ .direnv/ -.envrc.local # clangd and possibly more .cache/ From cf756fdf3c1a804af726703a12ed2990ad6c2639 Mon Sep 17 00:00:00 2001 From: Alois Wohlschlager Date: Wed, 29 May 2024 18:16:18 +0200 Subject: [PATCH 05/10] libstore/build: copy ca-certificates too In b469c6509ba616da6df8a27e4ccb205a877c66c9, the ca-certificates file was missed. It should be copied too so that we don't end up bind-mounting a broken symlink. Change-Id: Ic9b292d602eb94b0e78f77f2a27a19d24665783c --- src/libstore/build/local-derivation-goal.cc | 8 ++++++-- tests/functional/linux-sandbox.sh | 5 +++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 99468d420..7066f5c93 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1847,8 +1847,12 @@ void LocalDerivationGoal::runChild() copyFile(path, chrootRootDir + path, { .followSymlinks = true }); } - if (settings.caFile != "") - pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); + if (settings.caFile != "" && pathExists(settings.caFile)) { + // For the same reasons as above, copy the CA certificates file too. + // It should be even less likely to change during the build than resolv.conf. + createDirs(chrootRootDir + "/etc/ssl/certs"); + copyFile(settings.caFile, chrootRootDir + "/etc/ssl/certs/ca-certificates.crt", { .followSymlinks = true }); + } } for (auto & i : ss) pathsInChroot.emplace(i, i); diff --git a/tests/functional/linux-sandbox.sh b/tests/functional/linux-sandbox.sh index 04209277b..82f363a09 100644 --- a/tests/functional/linux-sandbox.sh +++ b/tests/functional/linux-sandbox.sh @@ -60,7 +60,9 @@ testCert () { nocert=$TEST_ROOT/no-cert-file.pem cert=$TEST_ROOT/some-cert-file.pem +certsymlink=$TEST_ROOT/cert-symlink.pem echo -n "CERT_CONTENT" > $cert +ln -s $cert $certsymlink # No cert in sandbox when not a fixed-output derivation testCert missing normal "$cert" @@ -74,5 +76,8 @@ testCert missing fixed-output "$nocert" # Cert in sandbox when ssl-cert-file is set to an existing file testCert present fixed-output "$cert" +# Cert in sandbox when ssl-cert-file is set to a symlink +testCert present fixed-output "$certsymlink" + # Symlinks should be added in the sandbox directly and not followed nix-sandbox-build symlink-derivation.nix From ac78c1dcd501233339f6a2f2e67651c2eeac6498 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Mon, 27 May 2024 15:16:15 -0600 Subject: [PATCH 06/10] libutil: fix args assert being thrown on Darwin in nix-eval-jobs This is because a dynamic_cast of a (n-e-j) MyArgs returns nullptr even though MyArgs has virtual nix::RootArgs as a parent. class MyArgs : virtual public nix::MixEvalArgs, virtual public nix::MixCommonArgs, virtual nix::RootArgs { ... }; So this should work right?? But it does not. We found out that it's caused by -fvisibility=hidden in n-e-j, but honestly this code was bad anyway. The trivial solution is to simply stop relying on RTTI working properly here, which is probably better OO architecture anyway. However, I am not 100% confident *this* is sound, since we have this horrible hierarchy: Args (defines getRoot) / | \ RootArgs MixCommonArgs MixEvalArgs (overrides) I am not confident that this is guaranteed to resolve from Args always in the case of this override. Assertion failed: (res), function getRoot, file src/libutil/args.cc, line 67. 6MyArgsProcess 60503 stopped * thread #1, queue = 'com.apple.main-thread', stop reason = hit program assert frame #4: 0x0000000100b1a41c liblixutil.dylib`nix::Args::processArgs(std::__1::list, std::__1::allocator>, std::__1::allocator, std::__1::allocator>>> const&, bool) [inlined] nix::Args::getRoot(this=0x00000001000d0688) at args.cc:67:5 [opt] 64 std::cout << typeid(*p).name(); 65 66 auto * res = dynamic_cast(p); -> 67 assert(res); 68 return *res; 69 } 70 Target 0: (nix-eval-jobs) stopped. (lldb) p this (MyArgs *) 0x00000001000d0688 (lldb) p *this (nix::Args) { longFlags = size=180 { ... } shortFlags = size=4 { ... } expectedArgs = size=1 { ... } processedArgs = size=0 {} hiddenCategories = size=1 { [0] = "Options to override configuration settings" } parent = nullptr } We also found that if we did this: class [[gnu::visibility("default")]] RootArgs : virtual public Args it would work properly (???!). This is of course, very strange, because objdump -Ct output on liblixexpr.dylib is identical both with and without it. Possibly related: https://www.qt.io/blog/quality-assurance/one-way-dynamic_cast-across-library-boundaries-can-fail-and-how-to-fix-it Fixes: https://git.lix.systems/lix-project/nix-eval-jobs/issues/2 Change-Id: I6b9ed968ed56420a9c4d2dffd18999d78c2761bd --- src/libutil/args.cc | 2 +- src/libutil/args.hh | 7 +++++++ src/libutil/args/root.hh | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 655b3e82f..1342e7c6a 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -64,7 +64,7 @@ RootArgs & Args::getRoot() while (p->parent) p = p->parent; - auto * res = dynamic_cast(p); + auto res = p->asRootArgs(); assert(res); return *res; } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 35a5238c0..71f8f88fa 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -244,6 +244,13 @@ protected: */ virtual void initialFlagsProcessed() {} + /** + * Returns this Args as a RootArgs if it is one, or \ref std::nullopt otherwise. + */ + virtual std::optional> asRootArgs() { + return std::nullopt; + } + public: void addFlag(Flag && flag); diff --git a/src/libutil/args/root.hh b/src/libutil/args/root.hh index f8124eaff..499ee6df4 100644 --- a/src/libutil/args/root.hh +++ b/src/libutil/args/root.hh @@ -65,6 +65,10 @@ protected: */ std::set flagExperimentalFeatures; + virtual std::optional> asRootArgs() override { + return *this; + } + private: std::optional needsCompletion(std::string_view s); From d4b7e6baca5b036cb624eec3a7cbb863349ceda3 Mon Sep 17 00:00:00 2001 From: Lunaphied Date: Wed, 3 Apr 2024 20:45:47 -0600 Subject: [PATCH 07/10] build-remote: truncate+hash store URI used in lockfile paths Fixes: https://git.lix.systems/lix-project/lix/issues/157 Fixes: https://git.lix.systems/lix-project/lix/issues/221 Previously the entire escaped store URI was included. This would cause build failures if a very long or deeply nested path was being used in the store. Now, we use the first 48 characters of the URL (escaped), then 16 bytes of hash of the entire URL. This should never collide and limits the length of the file name to a bit over 64, which is fine. Change-Id: Ic1ba690a94e83749567c2c29460b8d1bcf2ac413 --- src/build-remote/build-remote.cc | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index f7a159829..fb90403a0 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #if __APPLE__ @@ -20,9 +21,9 @@ #include "local-store.hh" #include "legacy.hh" #include "experimental-features.hh" +#include "hash.hh" using namespace nix; -using std::cin; static void handleAlarm(int sig) { } @@ -35,9 +36,19 @@ std::string escapeUri(std::string uri) static std::string currentLoad; +static std::string makeLockFilename(const std::string & storeUri) { + // We include 48 bytes of escaped URI to give an idea of what the lock + // is on, then 16 bytes of hash to disambiguate. + // This avoids issues with the escaped URI being very long and causing + // path too long errors, while also avoiding any possibility of collision + // caused by simple truncation. + auto hash = hashString(HashType::htSHA256, storeUri).to_string(Base::Base32, false); + return escapeUri(storeUri).substr(0, 48) + "-" + hash.substr(0, 16); +} + static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) { - return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); + return openLockFile(fmt("%s/%s-%d", currentLoad, makeLockFilename(m.storeUri), slot), true); } static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { @@ -263,7 +274,9 @@ connected: auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + auto lockFileName = currentLoad + "/" + makeLockFilename(storeUri) + ".upload-lock"; + + AutoCloseFD uploadLock = openLockFile(lockFileName, true); { Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); From 713cd7e9e7012a11d2347e9b1a5dee9b4d3c62c0 Mon Sep 17 00:00:00 2001 From: annalee <150648636+a-n-n-a-l-e-e@users.noreply.github.com> Date: Sat, 23 Mar 2024 06:41:33 +0000 Subject: [PATCH 08/10] truncate WAL files on exit Fix for https://github.com/NixOS/nix/issues/10300 https://github.com/lix-project/lix/commit/18a26202737a74f216d285d92bd4a84761788026 enabled persistent WAL files that will never get truncated. to fix this, journal_size_limit is set to 2^40, which results in the WAL files being truncated to 0 on exit, as well as limiting the WAL files to 2^40 bytes following a checkpoint. this aligns lix with the nix change: https://github.com/NixOS/nix/pull/10301 https://www.sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpersistwal https://www.sqlite.org/pragma.html#pragma_journal_size_limit https://github.com/sqlite/sqlite/blob/ed517a708284b6e00b6ae5f1e3f702bbfcbd32ed/src/wal.c#L2518 PR-Link: https://github.com/lix-project/lix/pull/9 Co-Authored-By: paparodeo <170618376+paparodeo@users.noreply.github.com> Change-Id: I90ec1a467c92c582ff8c07dd363a4cf789782214 --- src/libstore/local-store.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7bcbe3298..d92fafa1b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -550,7 +550,12 @@ void LocalStore::openDB(State & state, bool create) if (mode == "wal" ) { /* persist the WAL files when the DB connection is closed. * This allows for read-only connections without any write permissions - * on the state directory to succeed on a closed database. */ + * on the state directory to succeed on a closed database. Setting the + * journal_size_limit to 2^40 bytes results in the WAL files getting + * truncated to 0 on exit and limits the on disk size of the WAL files + * to 2^40 bytes following a checkpoint */ + if (sqlite3_exec(db, "pragma main.journal_size_limit = 1099511627776;", 0, 0, 0) != SQLITE_OK) + SQLiteError::throw_(db, "setting journal_size_limit"); int enable = 1; if (sqlite3_file_control(db, NULL, SQLITE_FCNTL_PERSIST_WAL, &enable) != SQLITE_OK) SQLiteError::throw_(db, "setting persistent WAL mode"); From e54d4c9381492d44a69469a0e719fea50e9914f3 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Thu, 30 May 2024 17:10:10 -0600 Subject: [PATCH 09/10] build: fix static linking with a hack This causes libstore, libexpr, libfetchers, and libutil to be linked with -Wl,--whole-archive to executables, when building statically. libstore for the store backends, libexpr for the primops, libfetchers for the fetcher backends I assume(?), and libutil for the nix::logger initializer (which notably shows in pre-main constructors when HOME is not owned by the user. cursed.). This workaround should be removed when #359 is fixed. Fixes #306. Change-Id: Ie9ef0154e09a6ed97920ee8ab23810ca5e2de84c --- meson.build | 15 +++++++++++++++ src/libexpr/meson.build | 10 ++++++++++ src/libfetchers/meson.build | 10 ++++++++++ src/libstore/meson.build | 10 ++++++++++ src/libutil/meson.build | 10 ++++++++++ src/nix/meson.build | 8 ++++---- tests/unit/meson.build | 14 +++++++------- 7 files changed, 66 insertions(+), 11 deletions(-) diff --git a/meson.build b/meson.build index 9db764f00..b98b1fb15 100644 --- a/meson.build +++ b/meson.build @@ -17,6 +17,19 @@ # # Finally, src/nix/meson.build defines the Nix command itself, relying on all prior meson files. # +# libstore, libexpr, and libfetchers have some special handling to make static builds work. +# Their use static constructors for dynamic registration of primops, store backends, etc +# gets borked during static link. We can't simply wholesale apply `link_whole :` either, +# because these libraries get linked multiple times since Lix's components are transitively +# dependent. So instead, each of those libraries have two dependency objects: +# liblix{store,expr,fetchers,util} and liblix{store,expr,fetchers,util}_mstatic ("maybe static"). +# The _mstatic versions should be used in the `dependencies :` arguments to ALL EXECUTABLES +# but executables ONLY. When we are not building statically (default_library != 'static'), +# they are equivalent. When we are building statically, the _mstatic version will be +# `link_whole :` rather than `link_with :`. +# FIXME: This hack should be removed when https://git.lix.systems/lix-project/lix/issues/359 +# is fixed. +# # Unit tests are setup in tests/unit/meson.build, under the test suite "check". # # Functional tests are a bit more complicated. Generally they're defined in @@ -79,6 +92,8 @@ if not fs.is_absolute(sysconfdir) sysconfdir = '/' / sysconfdir endif +is_static = get_option('default_library') == 'static' + # All of this has to go before the rest of the dependency checking, # so that internal-api-docs can be built with -Denable-build=false diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 9a18c7ab3..e60a85b5d 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -162,6 +162,16 @@ liblixexpr = declare_dependency( link_with : libexpr, ) +# FIXME: remove when https://git.lix.systems/lix-project/lix/issues/359 is fixed. +if is_static + liblixexpr_mstatic = declare_dependency( + include_directories : include_directories('.'), + link_whole : libexpr, + ) +else + liblixexpr_mstatic = liblixexpr +endif + # FIXME: not using the pkg-config module because it creates way too many deps # while meson migration is in progress, and we want to not include boost here configure_file( diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index 365bcd4a7..ee38b6cda 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -56,3 +56,13 @@ liblixfetchers = declare_dependency( include_directories : include_directories('.'), link_with : libfetchers, ) + +# FIXME: remove when https://git.lix.systems/lix-project/lix/issues/359 is fixed. +if is_static + liblixfetchers_mstatic = declare_dependency( + include_directories : include_directories('.'), + link_whole : libfetchers, + ) +else + liblixfetchers_mstatic = liblixfetchers +endif diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 65ecacc20..7a129d22c 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -234,6 +234,16 @@ liblixstore = declare_dependency( link_with : libstore, ) +# FIXME: remove when https://git.lix.systems/lix-project/lix/issues/359 is fixed. +if is_static + liblixstore_mstatic = declare_dependency( + include_directories : include_directories('.'), + link_whole : libstore, + ) +else + liblixstore_mstatic = liblixstore +endif + # FIXME: not using the pkg-config module because it creates way too many deps # while meson migration is in progress, and we want to not include boost here configure_file( diff --git a/src/libutil/meson.build b/src/libutil/meson.build index cfdd0e52c..8c3377e76 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -155,3 +155,13 @@ liblixutil = declare_dependency( include_directories : include_directories('.'), link_with : libutil ) + +# FIXME: remove when https://git.lix.systems/lix-project/lix/issues/359 is fixed. +if is_static + liblixutil_mstatic = declare_dependency( + include_directories : include_directories('.'), + link_whole : libutil, + ) +else + liblixutil_mstatic = liblixutil +endif diff --git a/src/nix/meson.build b/src/nix/meson.build index 45303641f..8115a3d08 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -81,10 +81,10 @@ nix = executable( nix2_commands_sources, dependencies : [ liblixcmd, - liblixutil, - liblixstore, - liblixexpr, - liblixfetchers, + liblixutil_mstatic, + liblixstore_mstatic, + liblixexpr_mstatic, + liblixfetchers_mstatic, liblixmain, boehm, nlohmann_json, diff --git a/tests/unit/meson.build b/tests/unit/meson.build index d8d4a00d1..06aff4626 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -64,7 +64,7 @@ libutil_tester = executable( gtest, boehm, liblixutil, - liblixexpr, + liblixexpr_mstatic, liblixutil_test_support, nlohmann_json, ], @@ -131,7 +131,7 @@ libstore_tester = executable( dependencies : [ liblixstore_test_support, liblixutil_test_support, - liblixstore, + liblixstore_mstatic, liblixutil, rapidcheck, gtest, @@ -194,10 +194,10 @@ libexpr_tester = executable( dependencies : [ liblixexpr_test_support, liblixstore_test_support, - liblixstore, + liblixstore_mstatic, liblixutil, - liblixexpr, - liblixfetchers, + liblixexpr_mstatic, + liblixfetchers_mstatic, rapidcheck, boehm, gtest, @@ -225,8 +225,8 @@ libcmd_tester = executable( liblixcmd, liblixutil, liblixmain, - liblixexpr, - liblixstore, + liblixexpr_mstatic, + liblixstore_mstatic, gtest, boost, ], From 8f5c8a8d758cec1a2d6320f3fba06ecdfef828e6 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Thu, 30 May 2024 18:16:01 -0600 Subject: [PATCH 10/10] flake: CI a *configure* test for static builds Adding an entire second Lix build to test the static version is too expensive for our current CI setup, but we can at least run the up to the configure phase, which is a good premise. Change-Id: I61e1ded7dd79698a9bb0f9a1d113b8b9963dfd44 --- flake.nix | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 14ee8e1ad..397cf0eb8 100644 --- a/flake.nix +++ b/flake.nix @@ -140,10 +140,11 @@ config.permittedInsecurePackages = [ "nix-2.13.6" ]; }; stdenvs = forAllStdenvs (make-pkgs null); + static-stdenvs = forAllStdenvs (name: (make-pkgs null name).pkgsStatic); native = stdenvs.stdenvPackages; in { - inherit stdenvs native; + inherit stdenvs native static-stdenvs; static = native.pkgsStatic; cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); } @@ -288,6 +289,22 @@ ); }; + # Running a static build in CI for every CL is too expensive for us right now... + # But we can at least make sure it configures successfuly, which is cheap and where + # a good chunk of issues show up. + configure-static = forAvailableSystems ( + system: + let + staticConfigure = self.packages.${system}.nix-static.overrideAttrs { + dontBuild = true; + installPhase = '' + mkdir -p "$out" + ''; + }; + in + lib.optionalAttrs (self.packages.${system} ? nix-static) staticConfigure + ); + pre-commit = forAvailableSystems ( system: let @@ -316,6 +333,9 @@ // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; } + // (lib.optionalAttrs (self.hydraJobs.configure-static ? "${system}")) { + configure-static = self.hydraJobs.configure-static.${system}; + } ); packages = forAllSystems (