Merge pull request 'overlay: fix bug where prev should have been final' (#38) from fabianhjr/nixos-module:fix-bug-between-prev-final into main

Reviewed-on: lix-project/nixos-module#38

README.md

@@ -1,5 +1,79 @@
 # Lix NixOS module
 
-See the [beta guide][beta-guide] for details on how to use this:
+See the [official installation guide][install-guide] for details on usage of
+release versions.
+
+[install-guide]: https://lix.systems/install/
+
+See the [beta guide][beta-guide] for a setup guide on how to use HEAD:
 
 [beta-guide]: https://wiki.lix.systems/link/1
+
+## What does this do?
+
+This is an overlay wrapped in a NixOS module that will replace CppNix with Lix
+in nixpkgs. This is useful so that `nixos-rebuild`, `nix-direnv` and similar
+will use Lix.
+
+Optionally, it can build Lix from source.
+
+## Versioning with Lix
+
+The version of this overlay should match the major Lix version, *and* if
+running HEAD, it should be the latest available version of the overlay.
+Changes breaking the overlay are shamelessly done on HEAD, and we expect people
+doing such changes to have prepared corresponding fix commits to make to the
+overlay immediately after submitting their changes. If your build randomly
+broke when updating HEAD, try updating your overlay.
+
+The version of Lix pinned in this flake's `flake.lock` is a version of Lix
+that is expected to work, however if running HEAD, it can be assumed to work
+with HEAD as well if both `lix-nixos-module` and `lix` are the latest version.
+
+## Common pitfalls
+
+Various flake frameworks such as flake-parts and snowfall (and possibly Colmena
+in the future if they do a similar optimization) manage overlays separately
+from NixOS, since they provide `pkgs` pre-imported to NixOS. This saves a
+couple of seconds of evaluation time and resources, but it means that the NixOS
+option `nixpkgs.overlays` **is completely ignored** on these frameworks.
+
+If you are using such a framework, add `overlays.default` to the overlays list
+for said framework.
+
+## Flake structure and usage
+
+The flake here has two inputs of note:
+- `nixpkgs`, *which is unused for most people*. It is purely used for `checks`
+  in developing `lix-nixos-module` itself.
+
+  The installation instructions make it `follows` to make `flake.lock` less
+  confusing, but it is nonetheless unused.
+- `lix`, which determines the version of Lix to do source builds for, if doing
+  source builds.
+
+These are the most relevant outputs for most people:
+
+- `nixosModules.lixFromNixpkgs` - uses Lix from nixpkgs and installs the
+  overlay to use Lix on a NixOS system. This is only useful for a stable
+  version of Lix, and cannot be used for running HEAD.
+- `nixosModules.default` - uses Lix from source and installs the overlay to use
+  Lix on a NixOS system.
+- `overlays.lixFromNixpkgs` - overlay to use Lix from nixpkgs in place of Nix.
+- `overlays.default` - overlay to use Lix from source in place of Nix.
+
+## Non-flake usage
+
+Import `module.nix` or `overlay.nix` as desired, with the arguments `lix`
+(derivation-like attribute set with the Lix sources, or `null` to use Lix from
+nixpkgs) and `versionSuffix` (optional string).
+
+It's desirable to also include a `versionSuffix` like the following while
+building HEAD from source, to have `nix --version` include date and commit
+information. To get such metadata, it depends on which pinning system is in
+use, but `builtins.fetchGit` will provide the necessary metadata for the
+following to work:
+
+```
+versionSuffix = "pre${builtins.substring 0 8 lix.lastModifiedDate}-${lix.shortRev or lix.dirtyShortRev}";
+```

flake.lock

@@ -5,11 +5,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1710146030,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "lastModified": 1726560853,
+        "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
         "type": "github"
       },
       "original": {
@@ -36,24 +36,24 @@
     "lix": {
       "flake": false,
       "locked": {
-        "lastModified": 1720626042,
-        "narHash": "sha256-w8JAk9Z3Fmkyway0VCjy/PtoBC6bGQVhNfTzFA98Pg8=",
-        "rev": "4bbdb2f5564b9b42bcaf0e1eec28325300f31c72",
+        "lastModified": 1727712632,
+        "narHash": "sha256-7kaZRZCWG8PmxwIDTVKjJo3VtAGDun7mqfAMlucQbdQ=",
+        "rev": "a16ceb9411c57993d811c6bebb517742fe3d34e3",
         "type": "tarball",
-        "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/4bbdb2f5564b9b42bcaf0e1eec28325300f31c72.tar.gz"
+        "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/a16ceb9411c57993d811c6bebb517742fe3d34e3.tar.gz?rev=a16ceb9411c57993d811c6bebb517742fe3d34e3"
       },
       "original": {
         "type": "tarball",
-        "url": "https://git.lix.systems/lix-project/lix/archive/release-2.90.tar.gz"
+        "url": "https://git.lix.systems/lix-project/lix/archive/main.tar.gz"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1720418205,
-        "narHash": "sha256-cPJoFPXU44GlhWg4pUk9oUPqurPlCFZ11ZQPk21GTPU=",
+        "lastModified": 1727634051,
+        "narHash": "sha256-S5kVU7U82LfpEukbn/ihcyNt2+EvG7Z5unsKW9H/yFA=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "655a58a72a6601292512670343087c2d75d859c1",
+        "rev": "06cf0e1da4208d3766d898b7fdab6513366d45b9",
         "type": "github"
       },
       "original": {

flake.nix

@@ -1,18 +1,35 @@
 {
   inputs.nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
   inputs.lix = {
-    url = "https://git.lix.systems/lix-project/lix/archive/release-2.90.tar.gz";
+    url = "https://git.lix.systems/lix-project/lix/archive/main.tar.gz";
     flake = false;
   };
   inputs.flake-utils.url = "github:numtide/flake-utils";
   inputs.flakey-profile.url = "github:lf-/flakey-profile";
 
   outputs = inputs@{ self, nixpkgs, lix, flake-utils, flakey-profile, ... }:
-    let versionSuffix = "pre${builtins.substring 0 8 lix.lastModifiedDate}-${lix.shortRev or lix.dirtyShortRev}";
-    in {
+    let
+      lixVersionJson = builtins.fromJSON (builtins.readFile (lix + "/version.json"));
+      versionSuffix = nixpkgs.lib.optionalString (!lixVersionJson.official_release)
+        "-pre${builtins.substring 0 8 lix.lastModifiedDate}-${lix.shortRev or lix.dirtyShortRev}";
+    in
+    {
       inherit inputs;
-      nixosModules.default = import ./module.nix { inherit lix versionSuffix; };
-      overlays.default = import ./overlay.nix { inherit lix versionSuffix; };
+      nixosModules = {
+        # Use a locally built Lix
+        default = import ./module.nix { inherit lix versionSuffix; };
+
+        # Use Lix from nixpkgs
+        lixFromNixpkgs = import ./module.nix { lix = null; };
+      };
+
+      overlays = {
+        # Use a locally built Lix
+        default = import ./overlay.nix { inherit lix versionSuffix; };
+
+        # Use Lix from nixpkgs
+        lixFromNixpkgs = import ./overlay.nix { lix = null; };
+      };
     } // flake-utils.lib.eachDefaultSystem (system:
       let
         pkgs = import nixpkgs {
@@ -30,17 +47,19 @@
       {
         inherit pkgs;
         packages = {
-          default = pkgs.nixVersions.nix_2_18;
+          default = pkgs.nix;
           inherit (pkgs) nix-doc nix-eval-jobs;
         };
 
         packages.system-profile = import ./system-profile.nix { inherit pkgs flakey-profile; };
 
-        nixosTests = pkgs.recurseIntoAttrs (pkgs.callPackage ./test-nixos.nix { lix-module = self.nixosModules.default; });
+        nixosTests = pkgs.recurseIntoAttrs (pkgs.callPackage ./test-nixos.nix { inherit pkgs; lix-module = self.nixosModules.default; });
 
         checks = {
           inherit (self.packages.${system}) default nix-eval-jobs;
+          inherit (pkgs) nixos-option;
         } // lib.optionalAttrs (lib.elem system linux64BitSystems) {
+          # wrongMajor intentionally not included here since it is expected to fail
           inherit (self.nixosTests.${system}) it-builds;
         };
       });

overlay.nix

@@ -1,19 +1,14 @@
 { lix, versionSuffix ? "" }:
 final: prev:
 let
-  # This is kind of scary to not override the nix version to pretend to be
-  # 2.18 since nixpkgs can introduce new breakage in its Nix unstable CLI
-  # usage.
-  # https://github.com/nixos/nixpkgs/blob/6afb255d976f85f3359e4929abd6f5149c323a02/nixos/modules/config/nix.nix#L121
-  lixPkg = final.callPackage (lix + "/package.nix") ({
-    versionSuffix = "-${versionSuffix}";
-    # FIXME: do this more sensibly for future releases
-    # https://git.lix.systems/lix-project/lix/issues/406
-    officialRelease = false;
+  lixPackageFromSource = final.callPackage (lix + "/package.nix") ({
+    inherit versionSuffix;
+    stdenv = final.clangStdenv;
   });
 
-  # These packages depend on Nix features that Lix does not support
-  overridelist_2_18 = [
+  # These packages should receive CppNix since they may link to it or otherwise
+  # cause problems (or even just silly mass-rebuilds) if we give them Lix
+  overridelist_upstream = [
     "attic-client"
     "devenv"
     "nix-du"
@@ -23,17 +18,14 @@ let
     "nurl"
     "prefetch-yarn-deps" # force these onto upstream so we are not regularly rebuilding electron
   ];
-  override_2_18 = prev.lib.genAttrs overridelist_2_18 (
-    name: prev.${name}.override {
-      nix = final.nixVersions.nix_2_18_upstream;
-    });
 
   inherit (prev) lib;
 
-  prefetch-npm-deps-args = lib.functionArgs prev.prefetch-npm-deps.override;
-
+  csi = builtins.fromJSON ''"\u001b"'';
+  orange = "${csi}[35;1m";
+  normal = "${csi}[0m";
   warning = ''
-    warning: You have the lix overlay included into a nixpkgs import twice,
+    ${orange}warning${normal}: You have the lix overlay included into a nixpkgs import twice,
     perhaps due to the NixOS module being included twice, or because of using
     pkgs.nixos and also including it in imports, or perhaps some unknown
     machinations of a complicated flake library.
@@ -46,30 +38,76 @@ let
     delete that.
     P.P.S. This Lix has super catgirl powers.
   '';
+  wrongMajorWarning = ''
+    ${orange}warning${normal}: This Lix NixOS module is being built against a Lix with a
+    major version (got ${lixPackageToUse.version}) other than the one the
+    module was designed for (expecting ${supportedLixMajor}). Some downstream
+    packages like nix-eval-jobs may be broken by this. Consider using a
+    matching version of the Lix NixOS module to the version of Lix you are
+    using.
+  '';
 
   maybeWarnDuplicate = x: if final.lix-overlay-present > 1 then builtins.trace warning x else x;
 
-  overlay = override_2_18 // {
+  versionJson = builtins.fromJSON (builtins.readFile ./version.json);
+  supportedLixMajor = lib.versions.majorMinor versionJson.version;
+  lixPackageToUse = if lix != null then lixPackageFromSource else prev.lix;
+  # Especially if using Lix from nixpkgs, it is plausible that the overlay
+  # could be used against the wrong Lix major version and cause confusing build
+  # errors. This is a simple safeguard to put in at least something that might be seen.
+  maybeWarnWrongMajor = x: if !(lib.hasPrefix supportedLixMajor lixPackageToUse.version) then builtins.trace wrongMajorWarning x else x;
+
+  # It is not enough to *just* throw whatever the default nix version is at
+  # anything in the "don't give lix" list, we have to *also* ensure that we
+  # give whatever upstream version as specified in the callPackage invocation.
+  #
+  # Unfortunately I don't think there is any actual way to directly query that,
+  # so we instead do something extremely evil and guess which version it
+  # probably was. This code is not generalizable to arbitrary derivations, so
+  # it will hopefully not make us cry, at least.
+  useCppNixOverlay =
+    lib.genAttrs overridelist_upstream (
+      name:
+      if (lib.functionArgs prev.${name}.override ? "nix") then
+        let
+          # Get the two common inputs of a derivation/package.
+          inputs = prev.${name}.buildInputs ++ prev.${name}.nativeBuildInputs;
+          nixDependency = lib.findFirst
+            (drv: (drv.pname or "") == "nix")
+            final.nixVersions.stable_upstream # default to stable nix if nix is not an input
+            inputs;
+          nixMajor = lib.versions.major (nixDependency.version or "");
+          nixMinor = lib.versions.minor (nixDependency.version or "");
+          nixAttr = "nix_${nixMajor}_${nixMinor}";
+          finalNix = final.nixVersions.${nixAttr};
+        in
+        prev.${name}.override {
+          nix = finalNix;
+        }
+      else prev.${name}
+    );
+
+  overlay = useCppNixOverlay // {
     lix-overlay-present = 1;
     # used for things that one wouldn't necessarily want to update, but we
     # nevertheless shove it in the overlay and fixed-point it in case one *does*
     # want to do that.
     lix-sources = import ./pins.nix;
 
-    nixVersions = prev.nixVersions // rec {
-      # FIXME: do something less scuffed
-      nix_2_18 = maybeWarnDuplicate lixPkg;
-      stable = nix_2_18;
-      nix_2_18_upstream = prev.nixVersions.nix_2_18;
+    lix = maybeWarnWrongMajor (maybeWarnDuplicate lixPackageToUse);
+
+    nixVersions = prev.nixVersions // {
+      stable = final.lix;
+      stable_upstream = prev.nixVersions.stable;
     };
 
     nix-eval-jobs = (prev.nix-eval-jobs.override {
       # lix
-      nix = final.nixVersions.nix_2_18;
+      nix = final.lix;
     }).overrideAttrs (old:
       let src = final.lix-sources.nix-eval-jobs;
       in {
-        version = "2.90.0-lix-${builtins.substring 0 7 src.rev}";
+        version = "2.92.0-lix-${builtins.substring 0 7 src.rev}";
 
         # FIXME: should this be patches instead?
         inherit src;
@@ -80,14 +118,6 @@ let
       }
     );
 
-    # support both having and missing https://github.com/NixOS/nixpkgs/pull/304913
-    prefetch-npm-deps =
-      if (prefetch-npm-deps-args ? nix) || (prefetch-npm-deps-args == {})
-      then prev.prefetch-npm-deps.override {
-        nix = final.nixVersions.nix_2_18_upstream;
-      }
-      else prev.prefetch-npm-deps;
-
     nix-doc = prev.callPackage ./nix-doc/package.nix { withPlugin = false; };
   };
 in

pins.json

@@ -1 +1 @@
-{"nix-eval-jobs": {"kind": "tarball", "rev": "9c23772cf25e0d891bef70b7bcb7df36239672a5", "nar_hash": "sha256-oT273pDmYzzI7ACAFUOcsxtT6y34V5KF7VBSqTza7j8=", "locked_url": "https://git.lix.systems/api/v1/repos/lix-project/nix-eval-jobs/archive/9c23772cf25e0d891bef70b7bcb7df36239672a5.tar.gz?rev=9c23772cf25e0d891bef70b7bcb7df36239672a5", "url": "https://git.lix.systems/lix-project/nix-eval-jobs/archive/release-2.90.tar.gz"}}
+{"nix-eval-jobs": {"kind": "tarball", "rev": "f8869bdcca7c1d5aaf37de3da3a4176811279a57", "nar_hash": "sha256-F/RvI9chHywnckEqHO1ggjzCayknhDnnl2kNnnVXpWg=", "locked_url": "https://git.lix.systems/api/v1/repos/lix-project/nix-eval-jobs/archive/f8869bdcca7c1d5aaf37de3da3a4176811279a57.tar.gz?rev=f8869bdcca7c1d5aaf37de3da3a4176811279a57", "url": "https://git.lix.systems/lix-project/nix-eval-jobs/archive/main.tar.gz"}}

test-nixos.nix

@@ -1,5 +1,8 @@
-{ nixos, lix-module }:
+{ pkgs, nixos, lix-module }:
 let
+  pkgs' = import pkgs.path {
+    inherit (pkgs) system;
+  };
   configs = {
     it-builds = nixos ({ ... }: {
       imports = [ lix-module ];
@@ -9,10 +12,21 @@ let
       system.stateVersion = "24.05";
     });
 
+    # Intentionally provoke the wrong major version.
+    # Does assume that the module is one major ahead of the release; the main
+    # purpose here is a manual testing fixture.
+    wrongMajor = pkgs'.nixos ({ ... }: {
+      imports = [ (import ./module.nix { lix = null; }) ];
+      documentation.enable = false;
+      fileSystems."/".device = "ignore-root-device";
+      boot.loader.grub.enable = false;
+      system.stateVersion = "24.05";
+    });
   };
 in
 {
   inherit configs;
 
   it-builds = configs.it-builds.config.system.build.toplevel;
+  wrongMajor = configs.wrongMajor.config.system.build.toplevel;
 }

update_pins.py

@@ -147,7 +147,7 @@ def lock_git(url, branch) -> GitPinSerialized:
 
 PINS = {
     'nix-eval-jobs':
-    TarballPinSpec('https://git.lix.systems/lix-project/nix-eval-jobs/archive/release-2.90.tar.gz')
+    TarballPinSpec('https://git.lix.systems/lix-project/nix-eval-jobs/archive/main.tar.gz')
 }
 
 

version.json

@@ -0,0 +1,3 @@
+{
+    "version": "2.92.0-dev"
+}