[resubmit] flake: update nixpkgs pin 23.11->24.05 (+ boehmgc compat changes)

-- message from cl/1418 --

The boehmgc changes are bundled into this commit because doing otherwise
would require an annoying dance of "adding compatibility for < 8.2.6 and
>= 8.2.6" then updating the pin then removing the (now unneeded)
compatibility. It doesn't seem worth the trouble to me given the low
complexity of said changes.

Rebased coroutine-sp-fallback.diff patch taken from https://github.com/NixOS/nixpkgs/pull/317227

-- jade resubmit changes --

This is a resubmission of https://gerrit.lix.systems/c/lix/+/1418, which
was reverted in https://gerrit.lix.systems/c/lix/+/1432 for breaking CI
evaluation without being detected.

I have run `nix flake check -Lv` on this one before submission and it
passes on my machine and crucially without eval errors, so the CI result
should be accurate.

It seems like someone renamed forbiddenDependenciesRegex to
forbiddenDependenciesRegexes in nixpkgs and also changed the type
incompatibly. That's pretty silly, but at least it's just an eval error.

Also, `xonsh` regressed the availability of `xonsh-unwrapped`, but it
was fixed by us in https://github.com/NixOS/nixpkgs/pull/317636, which
is now in our channel, so we update nixpkgs compared to the original
iteration of this to simply get that.

We originally had a regression related to some reorganization of the
nixpkgs lib test suite in which there was broken parameter passing.
This, too, we got quickfixed in nixpkgs, so we don't need any changes
for it: https://github.com/NixOS/nixpkgs/pull/317772

Related: https://gerrit.lix.systems/c/lix/+/1428
Fixes: lix-project/lix#385

Change-Id: I26d41ea826fec900ebcad0f82a727feb6bcd28f3

boehmgc-coroutine-sp-fallback.diff

@@ -1,56 +1,8 @@
-diff --git a/darwin_stop_world.c b/darwin_stop_world.c
-index 0468aaec..b348d869 100644
---- a/darwin_stop_world.c
-+++ b/darwin_stop_world.c
-@@ -356,6 +356,7 @@ GC_INNER void GC_push_all_stacks(void)
-   int nthreads = 0;
-   word total_size = 0;
-   mach_msg_type_number_t listcount = (mach_msg_type_number_t)THREAD_TABLE_SZ;
-+  size_t stack_limit;
-   if (!EXPECT(GC_thr_initialized, TRUE))
-     GC_thr_init();
-
-@@ -411,6 +412,19 @@ GC_INNER void GC_push_all_stacks(void)
-             GC_push_all_stack_sections(lo, hi, p->traced_stack_sect);
-           }
-           if (altstack_lo) {
-+            // When a thread goes into a coroutine, we lose its original sp until
-+            // control flow returns to the thread.
-+            // While in the coroutine, the sp points outside the thread stack,
-+            // so we can detect this and push the entire thread stack instead,
-+            // as an approximation.
-+            // We assume that the coroutine has similarly added its entire stack.
-+            // This could be made accurate by cooperating with the application
-+            // via new functions and/or callbacks.
-+            stack_limit = pthread_get_stacksize_np(p->id);
-+            if (altstack_lo >= altstack_hi || altstack_lo < altstack_hi - stack_limit) { // sp outside stack
-+              altstack_lo = altstack_hi - stack_limit;
-+            }
-+
-             total_size += altstack_hi - altstack_lo;
-             GC_push_all_stack(altstack_lo, altstack_hi);
-           }
-diff --git a/include/gc.h b/include/gc.h
-index edab6c22..f2c61282 100644
---- a/include/gc.h
-+++ b/include/gc.h
-@@ -2172,6 +2172,11 @@ GC_API void GC_CALL GC_win32_free_heap(void);
-         (*GC_amiga_allocwrapper_do)(a,GC_malloc_atomic_ignore_off_page)
- #endif /* _AMIGA && !GC_AMIGA_MAKINGLIB */
-
-+#if !__APPLE__
-+/* Patch doesn't work on apple */
-+#define NIX_BOEHM_PATCH_VERSION 1
-+#endif
-+
- #ifdef __cplusplus
-   } /* extern "C" */
- #endif
 diff --git a/pthread_stop_world.c b/pthread_stop_world.c
-index b5d71e62..aed7b0bf 100644
+index 2b45489..0e6d8ef 100644
 --- a/pthread_stop_world.c
 +++ b/pthread_stop_world.c
-@@ -768,6 +768,8 @@ STATIC void GC_restart_handler(int sig)
+@@ -776,6 +776,8 @@ STATIC void GC_restart_handler(int sig)
  /* world is stopped.  Should not fail if it isn't.                      */
  GC_INNER void GC_push_all_stacks(void)
  {
@@ -59,20 +11,23 @@ index b5d71e62..aed7b0bf 100644
      GC_bool found_me = FALSE;
      size_t nthreads = 0;
      int i;
-@@ -851,6 +853,37 @@ GC_INNER void GC_push_all_stacks(void)
+@@ -868,6 +870,40 @@ GC_INNER void GC_push_all_stacks(void)
              hi = p->altstack + p->altstack_size;
+ #         endif
            /* FIXME: Need to scan the normal stack too, but how ? */
-           /* FIXME: Assume stack grows down */
 +        } else {
 +          #ifdef HAVE_PTHREAD_ATTR_GET_NP
-+          if (!pthread_attr_init(&pattr)
++          if (pthread_attr_init(&pattr) != 0) {
-+              || !pthread_attr_get_np(p->id, &pattr))
++            ABORT("GC_push_all_stacks: pthread_attr_init failed!");
-+#else /* HAVE_PTHREAD_GETATTR_NP */
++          }
-+          if (pthread_getattr_np(p->id, &pattr))
++          if (pthread_attr_get_np(p->id, &pattr) != 0) {
-+#endif
++            ABORT("GC_push_all_stacks: pthread_attr_get_np failed!");
-+          {
++          }
++          #else
++          if (pthread_getattr_np(p->id, &pattr)) {
 +            ABORT("GC_push_all_stacks: pthread_getattr_np failed!");
 +          }
++          #endif
 +          if (pthread_attr_getstacksize(&pattr, &stack_limit)) {
 +            ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!");
 +          }
@@ -95,5 +50,5 @@ index b5d71e62..aed7b0bf 100644
 +          #error "STACK_GROWS_UP not supported in boost_coroutine2 (as of june 2021), so we don't support it in Nix."
 +          #endif
          }
-         GC_push_all_stack_sections(lo, hi, traced_stack_sect);
+ #       ifdef STACKPTR_CORRECTOR_AVAILABLE
- #       ifdef STACK_GROWS_UP
+           if (GC_sp_corrector != 0)

boehmgc-traceable_allocator-public.diff

@@ -1,12 +0,0 @@
-diff --git a/include/gc_allocator.h b/include/gc_allocator.h
-index 597c7f13..587286be 100644
---- a/include/gc_allocator.h
-+++ b/include/gc_allocator.h
-@@ -312,6 +312,7 @@ public:
-
- template<>
- class traceable_allocator<void> {
-+public:
-   typedef size_t      size_type;
-   typedef ptrdiff_t   difference_type;
-   typedef void*       pointer;

flake.lock

@@ -34,16 +34,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1715123187,
+        "lastModified": 1718111384,
-        "narHash": "sha256-0czuu757t53lK6uWeo1a5/jJbCd9t4sOtLDFpts60DM=",
+        "narHash": "sha256-7tSst0S5FOmcgvNtfy6cjZX5w8CabCVAfAeCkhY4OVg=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "0c592f9a288bdf764b6f24c757277c0e49757a46",
+        "rev": "a508a44af0c1b1b57785c34d8b54783536273eeb",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.11-small",
+        "ref": "nixos-24.05-small",
         "repo": "nixpkgs",
         "type": "github"
       }

flake.nix

@@ -2,7 +2,7 @@
   description = "The purely functional package manager";
 
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11-small";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05-small";
     nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
     pre-commit-hooks = {
       url = "github:cachix/git-hooks.nix";
@@ -87,10 +87,10 @@
       crossSystems = [
         "armv6l-linux"
         "armv7l-linux"
-        # FIXME: doesn't evaluate, plausibly fixed in >=24.05, so recheck when
+        # FIXME: still broken in 24.05: fails to build rustc(??) due to missing -lstdc++ dep
-        # we update to 24.05
+        # "x86_64-freebsd"
-        # "x86_64-freebsd13"
+        # FIXME: broken dev shell due to python
-        "x86_64-netbsd"
+        # "x86_64-netbsd"
       ];
 
       stdenvs = [
@@ -282,9 +282,20 @@
 
           nixpkgsLibTests = forAllSystems (
             system:
-            import (nixpkgs + "/lib/tests/release.nix") {
+            let
+              inherit (self.packages.${system}) nix;
               pkgs = nixpkgsFor.${system}.native;
-              nixVersions = [ self.packages.${system}.nix ];
+              testWithNix = import (nixpkgs + "/lib/tests/test-with-nix.nix") { inherit pkgs lib nix; };
+            in
+            pkgs.symlinkJoin {
+              name = "nixpkgs-lib-tests";
+              paths =
+                [ testWithNix ]
+                # FIXME: This is disabled on darwin due to a nixpkgs bug https://github.com/NixOS/nixpkgs/issues/319147
+                # After that is fixed, it should be restored to use lib/tests/release.nix as before, rather than this reimplementation.
+                ++ lib.optionals pkgs.stdenv.isLinux [
+                  (import (nixpkgs + "/pkgs/test/release") { inherit pkgs lib nix; })
+                ];
             }
           );
         };

meson.build

@@ -188,7 +188,7 @@ configdata = { }
 # Dependencies
 #
 
-boehm = dependency('bdw-gc', required : get_option('gc'))
+boehm = dependency('bdw-gc', required : get_option('gc'), version : '>=8.2.6')
 configdata += {
   'HAVE_BOEHMGC': boehm.found().to_int(),
 }

package.nix

@@ -39,12 +39,10 @@
   pkg-config,
   python3,
   rapidcheck,
-  skopeo,
   sqlite,
   toml11,
   util-linuxMinimal ? utillinuxMinimal,
   utillinuxMinimal ? null,
-  xonsh-unwrapped,
   xz,
 
   busybox-sandbox-shell,
@@ -71,8 +69,6 @@
         # `boehmgc-nix` then this will almost certainly have duplicate patches, which means
         # the patches won't apply and we'll get a build failure.
         ./boehmgc-coroutine-sp-fallback.diff
-        # https://github.com/ivmai/bdwgc/pull/586
-        ./boehmgc-traceable_allocator-public.diff
       ];
     };
 
@@ -402,6 +398,8 @@ stdenv.mkDerivation (finalAttrs: {
         llvmPackages,
         clangbuildanalyzer,
         contribNotice,
+        skopeo,
+        xonsh,
       }:
       let
         glibcFix = lib.optionalAttrs (buildPlatform.isLinux && glibcLocales != null) {
@@ -419,11 +417,9 @@ stdenv.mkDerivation (finalAttrs: {
             p.python-frontmatter
             p.requests
             p.xdg-base-dirs
-            (p.toPythonModule xonsh-unwrapped)
+            (p.toPythonModule xonsh.passthru.unwrapped)
           ]
         );
-        # FIXME: This will explode when we switch to 24.05 if we don't backport
-        # https://github.com/NixOS/nixpkgs/pull/317636 first
         pythonEnv = python3.withPackages pythonPackages;
 
         # pkgs.mkShell uses pkgs.stdenv by default, regardless of inputsFrom.

tests/nixos/default.nix

@@ -33,7 +33,7 @@ let
   checkOverrideNixVersion = { pkgs, lib, ... }: {
     # pkgs.nix: The new Nix in this repo
     # We disallow it, to make sure we don't accidentally use it.
-    system.forbiddenDependenciesRegex = lib.strings.escapeRegex "nix-${pkgs.nix.version}";
+    system.forbiddenDependenciesRegexes = [ (lib.strings.escapeRegex "nix-${pkgs.nix.version}") ];
   };
 in
 
@@ -53,15 +53,6 @@ in
     };
   };
 
-  remoteBuilds_remote_2_13 = runNixOSTestFor "x86_64-linux" ({ lib, pkgs, ... }: {
-    name = "remoteBuilds_remote_2_13";
-    imports = [ ./remote-builds.nix ];
-    builders.config = { lib, pkgs, ... }: {
-      imports = [ checkOverrideNixVersion ];
-      nix.package = lib.mkForce pkgs.nixVersions.nix_2_13;
-    };
-  });
-
   remoteBuilds_remote_2_18 = runNixOSTestFor "x86_64-linux" ({ lib, pkgs, ... }: {
     name = "remoteBuilds_remote_2_18";
     imports = [ ./remote-builds.nix ];
@@ -82,15 +73,6 @@ in
     };
   });
 
-  remoteBuilds_local_2_13 = runNixOSTestFor "x86_64-linux" ({ lib, pkgs, ... }: {
-    name = "remoteBuilds_local_2_13";
-    imports = [ ./remote-builds.nix ];
-    nodes.client = { lib, pkgs, ... }: {
-      imports = [ checkOverrideNixVersion ];
-      nix.package = lib.mkForce pkgs.nixVersions.nix_2_13;
-    };
-  });
-
   remoteBuilds_local_2_18 = runNixOSTestFor "x86_64-linux" ({ lib, pkgs, ... }: {
     name = "remoteBuilds_local_2_18";
     imports = [ ./remote-builds.nix ];
@@ -115,15 +97,6 @@ in
     };
   };
 
-  remoteBuildsSshNg_remote_2_13 = runNixOSTestFor "x86_64-linux" {
-    name = "remoteBuildsSshNg_remote_2_13";
-    imports = [ ./remote-builds-ssh-ng.nix ];
-    builders.config = { lib, pkgs, ... }: {
-      imports = [ checkOverrideNixVersion ];
-      nix.package = lib.mkForce pkgs.nixVersions.nix_2_13;
-    };
-  };
-
   remoteBuildsSshNg_remote_2_18 = runNixOSTestFor "x86_64-linux" {
     name = "remoteBuildsSshNg_remote_2_18";
     imports = [ ./remote-builds-ssh-ng.nix ];
@@ -146,15 +119,6 @@ in
     };
   });
 
-  remoteBuildsSshNg_local_2_13 = runNixOSTestFor "x86_64-linux" ({ lib, pkgs, ... }: {
-    name = "remoteBuildsSshNg_local_2_13";
-    imports = [ ./remote-builds-ssh-ng.nix ];
-    nodes.client = { lib, pkgs, ... }: {
-      imports = [ checkOverrideNixVersion ];
-      nix.package = lib.mkForce pkgs.nixVersions.nix_2_13;
-    };
-  });
-
   # TODO: (nixpkgs update) remoteBuildsSshNg_local_2_18 = ...
   */
 

tests/nixos/nix-upgrade-nix.nix

@@ -11,7 +11,7 @@ let
   lix = pkgs.nix;
   lixVersion = lib.getVersion lix;
 
-  newNix = pkgs.nixVersions.unstable;
+  newNix = pkgs.nixVersions.latest;
   newNixVersion = lib.getVersion newNix;
 
 in {
@@ -20,7 +20,7 @@ in {
   nodes = {
     machine = { config, lib, pkgs, ... }: {
       virtualisation.writableStore = true;
-      virtualisation.additionalPaths = [ pkgs.hello.drvPath ];
+      virtualisation.additionalPaths = [ pkgs.hello.drvPath newNix ];
       nix.settings.substituters = lib.mkForce [ ];
       nix.settings.experimental-features = [ "nix-command" "flakes" ];
       services.getty.autologinUser = "root";