Merge pull request 'overlay: support nix-doc integration into Lix' (#11) from feature/nix-doc into main

Reviewed-on: #11

README.md

@@ -1,5 +0,0 @@
-# Lix NixOS module
-
-See the [beta guide][beta-guide] for details on how to use this:
-
-[beta-guide]: https://wiki.lix.systems/link/1

default.nix

@@ -0,0 +1,10 @@
+(import
+  (
+    let lock = builtins.fromJSON (builtins.readFile ./flake.lock); in
+    fetchTarball {
+      url = lock.nodes.flake-compat.locked.url or "https://git.lix.systems/lix-project/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
+      sha256 = lock.nodes.flake-compat.locked.narHash;
+    }
+  )
+  { src = ./.; }
+).defaultNix

flake.lock

@@ -1,15 +1,30 @@
 {
   "nodes": {
+    "flake-compat": {
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "ref": "refs/heads/master",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "revCount": 57,
+        "type": "git",
+        "url": "ssh://git@git.lix.systems/lix-project/flake-compat"
+      },
+      "original": {
+        "type": "git",
+        "url": "ssh://git@git.lix.systems/lix-project/flake-compat"
+      }
+    },
     "flake-utils": {
       "inputs": {
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1710146030,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "lastModified": 1709126324,
+        "narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "rev": "d465f4819400de7c8d874d50b982301f28a84605",
         "type": "github"
       },
       "original": {
@@ -20,11 +35,11 @@
     },
     "flakey-profile": {
       "locked": {
-        "lastModified": 1712898590,
-        "narHash": "sha256-FhGIEU93VHAChKEXx905TSiPZKga69bWl1VB37FK//I=",
+        "lastModified": 1711325813,
+        "narHash": "sha256-ygJR5VikyCfK0CUJHboOKJVr6s9HQ1RXcvFEFnv+KIk=",
         "owner": "lf-",
         "repo": "flakey-profile",
-        "rev": "243c903fd8eadc0f63d205665a92d4df91d42d9d",
+        "rev": "3b32c4a71f89b874fe0be2dc125eacb9c3473204",
         "type": "github"
       },
       "original": {
@@ -36,24 +51,26 @@
     "lix": {
       "flake": false,
       "locked": {
-        "lastModified": 1718767907,
-        "narHash": "sha256-gpd+mGQxqVHw2kO6rSPQel8TkChHh9UpqxjsmQi0QJM=",
-        "rev": "85f282ef572577899b3d80ba8def1b920a386218",
-        "type": "tarball",
-        "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/85f282ef572577899b3d80ba8def1b920a386218.tar.gz"
+        "lastModified": 1712180067,
+        "narHash": "sha256-XU32CuKt0wXB8Ws5P3pZFMlYIMVx9om1GycYpITAwZ0=",
+        "ref": "refs/heads/main",
+        "rev": "89c31953c58d8089c85ae7ab43a5c59bb0f9f69b",
+        "revCount": 15311,
+        "type": "git",
+        "url": "ssh://git@git.lix.systems/lix-project/lix.git"
       },
       "original": {
-        "type": "tarball",
-        "url": "https://git.lix.systems/lix-project/lix/archive/main.tar.gz"
+        "type": "git",
+        "url": "ssh://git@git.lix.systems/lix-project/lix.git"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1718530797,
-        "narHash": "sha256-pup6cYwtgvzDpvpSCFh1TEUjw2zkNpk8iolbKnyFmmU=",
+        "lastModified": 1709703039,
+        "narHash": "sha256-6hqgQ8OK6gsMu1VtcGKBxKQInRLHtzulDo9Z5jxHEFY=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "b60ebf54c15553b393d144357375ea956f89e9a9",
+        "rev": "9df3e30ce24fd28c7b3e2de0d986769db5d6225d",
         "type": "github"
       },
       "original": {
@@ -65,6 +82,7 @@
     },
     "root": {
       "inputs": {
+        "flake-compat": "flake-compat",
         "flake-utils": "flake-utils",
         "flakey-profile": "flakey-profile",
         "lix": "lix",

flake.nix

@@ -1,14 +1,15 @@
 {
   inputs.nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
   inputs.lix = {
-    url = "https://git.lix.systems/lix-project/lix/archive/main.tar.gz";
+    url = "git+ssh://git@git.lix.systems/lix-project/lix.git";
     flake = false;
   };
   inputs.flake-utils.url = "github:numtide/flake-utils";
+  inputs.flake-compat.url = "git+ssh://git@git.lix.systems/lix-project/flake-compat";
   inputs.flakey-profile.url = "github:lf-/flakey-profile";
 
   outputs = inputs@{ self, nixpkgs, lix, flake-utils, flakey-profile, ... }:
-    let versionSuffix = "pre${builtins.substring 0 8 lix.lastModifiedDate}-${lix.shortRev or lix.dirtyShortRev}";
+    let versionSuffix = "pre${builtins.substring 0 8 lix.lastModifiedDate}-${lix.shortRev}";
     in {
       inherit inputs;
       nixosModules.default = import ./module.nix { inherit lix versionSuffix; };
@@ -19,29 +20,12 @@
           inherit system;
           overlays = [ self.overlays.default ];
         };
-
-        linux64BitSystems = [
-          "x86_64-linux"
-          "aarch64-linux"
-        ];
-
-        inherit (pkgs) lib;
       in
       {
         inherit pkgs;
-        packages = {
-          default = pkgs.nixVersions.nix_2_18;
-          inherit (pkgs) nix-doc nix-eval-jobs;
-        };
+        packages.default = pkgs.nixVersions.nix_2_18;
+        packages.nix-doc = pkgs.nix-doc;
 
         packages.system-profile = import ./system-profile.nix { inherit pkgs flakey-profile; };
-
-        nixosTests = pkgs.recurseIntoAttrs (pkgs.callPackage ./test-nixos.nix { lix-module = self.nixosModules.default; });
-
-        checks = {
-          inherit (self.packages.${system}) default nix-eval-jobs;
-        } // lib.optionalAttrs (lib.elem system linux64BitSystems) {
-          inherit (self.nixosTests.${system}) it-builds;
-        };
       });
 }

overlay.nix

@@ -1,99 +1,73 @@
 { lix, versionSuffix ? "" }:
 final: prev:
 let
-  # This is kind of scary to not override the nix version to pretend to be
-  # 2.18 since nixpkgs can introduce new breakage in its Nix unstable CLI
-  # usage.
-  # https://github.com/nixos/nixpkgs/blob/6afb255d976f85f3359e4929abd6f5149c323a02/nixos/modules/config/nix.nix#L121
-  lixPkg = final.callPackage (lix + "/package.nix") ({
-    versionSuffix = "-${versionSuffix}";
-    # FIXME: do this more sensibly for future releases
-    # https://git.lix.systems/lix-project/lix/issues/406
-    officialRelease = false;
-  });
+  boehmgc-patched = ((final.boehmgc.override {
+    enableLargeConfig = true;
+  }).overrideAttrs (o: {
+    patches = (o.patches or [ ]) ++ [
+      # for clown reasons this version is newer than the one in lix, we should
+      # fix this and update our nixpkgs pin
+      (prev.path + "/pkgs/tools/package-management/nix/patches/boehmgc-coroutine-sp-fallback.patch")
 
-  # These packages depend on Nix features that Lix does not support
-  overridelist_2_18 = [
-    "attic-client"
-    "devenv"
-    "nix-du"
-    "nix-init"
-    "nix-prefetch-git"
-    "nixos-option"
-    "nurl"
-    "prefetch-yarn-deps" # force these onto upstream so we are not regularly rebuilding electron
-  ];
-  override_2_18 = prev.lib.genAttrs overridelist_2_18 (
-    name: prev.${name}.override {
-      nix = final.nixVersions.nix_2_18_upstream;
-    });
+      # https://github.com/ivmai/bdwgc/pull/586
+      (builtins.path { path = lix + "/boehmgc-traceable_allocator-public.diff"; name = "boehmgc-traceable_allocator-public.patch"; })
+    ];
+  })
+  );
 
-  inherit (prev) lib;
-
-  prefetch-npm-deps-args = lib.functionArgs prev.prefetch-npm-deps.override;
-
-  warning = ''
-    warning: You have the lix overlay included into a nixpkgs import twice,
-    perhaps due to the NixOS module being included twice, or because of using
-    pkgs.nixos and also including it in imports, or perhaps some unknown
-    machinations of a complicated flake library.
-    This is completely harmless since we have no-op'd the second one if you are
-    seeing this message, but it would be a small style improvement to fix
-    it :)
-    P.S. If you had some hack to fix nixos-option build failures in your
-    configuration, that was caused by including an older version of the lix
-    overlay twice, which is now mitigated if you see this message, so you can
-    delete that.
-    P.P.S. This Lix has super catgirl powers.
-  '';
-
-  maybeWarnDuplicate = x: if final.lix-overlay-present > 1 then builtins.trace warning x else x;
-
-  overlay = override_2_18 // {
-    lix-overlay-present = 1;
-    # used for things that one wouldn't necessarily want to update, but we
-    # nevertheless shove it in the overlay and fixed-point it in case one *does*
-    # want to do that.
-    lix-sources = import ./pins.nix;
-
-    nixVersions = prev.nixVersions // rec {
-      # FIXME: do something less scuffed
-      nix_2_18 = maybeWarnDuplicate lixPkg;
-      stable = nix_2_18;
-      nix_2_18_upstream = prev.nixVersions.nix_2_18;
-    };
-
-    nix-eval-jobs = (prev.nix-eval-jobs.override {
-      # lix
-      nix = final.nixVersions.nix_2_18;
-    }).overrideAttrs (old:
-      let src = final.lix-sources.nix-eval-jobs;
-      in {
-        version = "2.90.0-lix-${builtins.substring 0 7 src.rev}";
-
-        # FIXME: should this be patches instead?
-        inherit src;
-
-        mesonBuildType = "debugoptimized";
-
-        ninjaFlags = old.ninjaFlags or [ ] ++ [ "-v" ];
-      }
-    );
-
-    # support both having and missing https://github.com/NixOS/nixpkgs/pull/304913
-    prefetch-npm-deps =
-      if (prefetch-npm-deps-args ? nix) || (prefetch-npm-deps-args == {})
-      then prev.prefetch-npm-deps.override {
-        nix = final.nixVersions.nix_2_18_upstream;
-      }
-      else prev.prefetch-npm-deps;
-
-    nix-doc = prev.callPackage ./nix-doc/package.nix { withPlugin = false; };
-
-    pegtl = prev.callPackage ./pegtl.nix { };
-  };
+  # Internal nix-doc used by Lix.
+  lix-doc = final.callPackage (lix + "/nix-doc/package.nix") { };
 in
-  # Make the overlay idempotent, since flakes passing nixos modules around by
-  # value and many other things make it way too easy to include the overlay
-  # twice
-  if (prev ? lix-overlay-present) then { lix-overlay-present = 2; } else overlay
+{
+  nixVersions = prev.nixVersions // rec {
+    # FIXME: do something less scuffed
+    nix_2_18 = (prev.nixVersions.nix_2_18.override { boehmgc = boehmgc-patched; }).overrideAttrs (old: {
+      src = lix;
+      # FIXME: fake version so that nixpkgs will not try to use nix config >_>
+      version = "2.18.3-lix${versionSuffix}";
+      VERSION_SUFFIX = "-lix${versionSuffix}";
+
+      # We only include CMake so that Meson can locate toml11, which only ships CMake dependency metadata.
+      dontUseCmakeConfigure = true;
+
+      patches = [ ];
+      buildInputs = old.buildInputs or [ ] ++ [
+        final.toml11
+        lix-doc
+      ];
+      nativeBuildInputs = old.nativeBuildInputs or [ ] ++ [
+        final.buildPackages.cmake
+        # FIXME: we don't know why this was not being picked up properly when
+        # included in nativeCheckInputs.
+        final.buildPackages.git
+      ];
+    });
+    stable = nix_2_18;
+    nix_2_18_upstream = prev.nixVersions.nix_2_18;
+  };
+
+  # force these onto upstream so we are not regularly rebuilding electron
+  prefetch-yarn-deps = prev.prefetch-yarn-deps.override {
+    nix = final.nixVersions.nix_2_18_upstream;
+  };
+  prefetch-npm-deps = prev.prefetch-npm-deps.override {
+    nix = final.nixVersions.nix_2_18_upstream;
+  };
+  nix-prefetch-git = prev.nix-prefetch-git.override {
+    nix = final.nixVersions.nix_2_18_upstream;
+  };
+
+  nixos-option = prev.nixos-option.override {
+    nix = final.nixVersions.nix_2_18_upstream;
+  };
+
+  nix-doc = prev.callPackage ./nix-doc/package.nix { withPlugin = false; };
+
+  nix-init = prev.nix-init.override {
+    nix = final.nixVersions.nix_2_18_upstream;
+  };
+
+  nurl = prev.nurl.override {
+    nix = final.nixVersions.nix_2_18_upstream;
+  };
+}

pegtl.nix

@@ -1,23 +0,0 @@
-{
-  stdenv,
-  cmake,
-  ninja,
-  fetchFromGitHub,
-}:
-
-stdenv.mkDerivation {
-  pname = "pegtl";
-  version = "3.2.7";
-
-  src = fetchFromGitHub {
-    repo = "PEGTL";
-    owner = "taocpp";
-    rev = "refs/tags/3.2.7";
-    hash = "sha256-IV5YNGE4EWVrmg2Sia/rcU8jCuiBynQGJM6n3DCWTQU=";
-  };
-
-  nativeBuildInputs = [
-    cmake
-    ninja
-  ];
-}

pins.json

@@ -1 +0,0 @@
-{"nix-eval-jobs": {"kind": "tarball", "rev": "f8869bdcca7c1d5aaf37de3da3a4176811279a57", "nar_hash": "sha256-F/RvI9chHywnckEqHO1ggjzCayknhDnnl2kNnnVXpWg=", "locked_url": "https://git.lix.systems/api/v1/repos/lix-project/nix-eval-jobs/archive/f8869bdcca7c1d5aaf37de3da3a4176811279a57.tar.gz?rev=f8869bdcca7c1d5aaf37de3da3a4176811279a57", "url": "https://git.lix.systems/lix-project/nix-eval-jobs/archive/main.tar.gz"}}

pins.nix

@@ -1,26 +0,0 @@
-# this is a custom pinning tool, written because npins doesn't have narHash
-# compatible output for git inputs, and also doesn't support the Nix immutable
-# tarball protocol
-let
-  pins = builtins.fromJSON (builtins.readFile ./pins.json);
-  fetchPin = args@{ kind, ... }:
-    if kind == "git" then
-      builtins.fetchGit
-        {
-          name = "source";
-          url = args.url;
-          ref = args.ref;
-          rev = args.rev;
-          narHash = args.nar_hash;
-        }
-    else if kind == "tarball" then
-      args // {
-        outPath = builtins.fetchTarball {
-          name = "source";
-          url = args.locked_url;
-          sha256 = args.nar_hash;
-        };
-      }
-    else builtins.throw "unsupported input type ${kind}";
-in
-builtins.mapAttrs (_: fetchPin) pins

test-nixos.nix

@@ -1,18 +0,0 @@
-{ nixos, lix-module }:
-let
-  configs = {
-    it-builds = nixos ({ ... }: {
-      imports = [ lix-module ];
-      documentation.enable = false;
-      fileSystems."/".device = "ignore-root-device";
-      boot.loader.grub.enable = false;
-      system.stateVersion = "24.05";
-    });
-
-  };
-in
-{
-  inherit configs;
-
-  it-builds = configs.it-builds.config.system.build.toplevel;
-}

update_pins.py

@@ -1,165 +0,0 @@
-#!/usr/bin/env nix-shell
-#!nix-shell -i python3 -p 'python3.withPackages (ps: [ ps.requests ])'
-"""
-Updates pins in this repo to their latest version.
-
-This is a custom pinning tool, written because npins doesn't have narHash
-compatible output for git inputs (it is not SRI), and also doesn't support the
-Nix immutable tarball protocol which we would like to use when we become public.
-"""
-import subprocess
-import tempfile
-from pathlib import Path
-import re
-import dataclasses
-from typing import Literal
-import urllib.parse
-import json
-
-
-# https://stackoverflow.com/a/51286749
-class DataclassJSONEncoder(json.JSONEncoder):
-
-    def default(self, o):
-        if dataclasses.is_dataclass(o):
-            return dataclasses.asdict(o)
-        return super().default(o)
-
-
-@dataclasses.dataclass
-class PinSerialized:
-    kind: str
-    rev: str | None
-    nar_hash: str
-
-
-@dataclasses.dataclass
-class GitPinSerialized(PinSerialized):
-    kind: Literal['git']
-    url: str
-    rev: str
-    ref: str
-
-
-@dataclasses.dataclass
-class TarballPinSerialized(PinSerialized):
-    kind: Literal['tarball']
-    locked_url: str
-    url: str
-
-
-class PinSpec:
-
-    def do_pin(self) -> dict[str, str]:
-        raise ValueError('unimplemented')
-
-
-@dataclasses.dataclass
-class GitPinSpec(PinSpec):
-    url: str
-    branch: str
-
-    def do_pin(self) -> GitPinSerialized:
-        return lock_git(self.url, self.branch)
-
-
-@dataclasses.dataclass
-class TarballPinSpec(PinSpec):
-    url: str
-
-    def do_pin(self) -> TarballPinSerialized:
-        return lock_tarball(self.url)
-
-
-@dataclasses.dataclass
-class LinkHeader:
-    url: str
-    rev: str | None
-
-
-LINK_HEADER_RE = re.compile(r'<(?P<url>.*)>; rel="immutable"')
-
-
-def parse_link_header(header) -> LinkHeader | None:
-    matched = LINK_HEADER_RE.match(header)
-    if not matched:
-        return None
-
-    url = matched.group('url')
-    parsed_url = urllib.parse.urlparse(url)
-    parsed_qs = urllib.parse.parse_qs(parsed_url.query)
-
-    return LinkHeader(url=url, rev=next(iter(parsed_qs.get('rev', [])), None))
-
-
-def lock_tarball(url) -> TarballPinSerialized:
-    """
-    Prefetches a tarball using the Nix immutable tarball protocol
-    """
-    import requests
-    resp = requests.get(url)
-    with tempfile.TemporaryDirectory() as td:
-        td = Path(td)
-        proc = subprocess.Popen(["tar", "-C", td, "-xvzf", "-"],
-                                stdin=subprocess.PIPE)
-        assert proc.stdin
-        for chunk in resp.iter_content(64 * 1024):
-            proc.stdin.write(chunk)
-        proc.stdin.close()
-        if proc.wait() != 0:
-            raise RuntimeError("untarring failed")
-
-        children = list(td.iterdir())
-        # FIXME: allow different tarball structures
-        assert len(children) == 1
-
-        child = children[0].rename(children[0].parent.joinpath('source'))
-        sri_hash = subprocess.check_output(
-            ["nix-hash", "--type", "sha256", "--sri", child]).decode().strip()
-        path = subprocess.check_output(
-            ["nix-store", "--add-fixed", "--recursive", "sha256",
-             child]).decode().strip()
-
-    link_info = parse_link_header(resp.headers['Link'])
-
-    print(sri_hash, path)
-    return TarballPinSerialized(kind='tarball',
-                                nar_hash=sri_hash,
-                                locked_url=link_info.url if link_info else url,
-                                rev=link_info.rev if link_info else None,
-                                url=url)
-
-
-def lock_git(url, branch) -> GitPinSerialized:
-    url_escaped = json.dumps(url)
-    ref_escaped = json.dumps(branch)
-    data = json.loads(
-        subprocess.check_output([
-            "nix", "eval", "--impure", "--json", "--expr",
-            f"builtins.removeAttrs (builtins.fetchGit {{ url = {url_escaped}; ref = {ref_escaped}; }}) [ \"outPath\" ]"
-        ]).strip())
-    return GitPinSerialized(kind='git',
-                            url=url,
-                            rev=data['rev'],
-                            ref=branch,
-                            nar_hash=data['narHash'])
-
-
-PINS = {
-    'nix-eval-jobs':
-    TarballPinSpec('https://git.lix.systems/lix-project/nix-eval-jobs/archive/main.tar.gz')
-}
-
-
-def main():
-    output = {}
-    for (name, pin) in PINS.items():
-        output[name] = pin.do_pin()
-
-    print(output)
-    with open('pins.json', 'w') as fh:
-        json.dump(output, fh, cls=DataclassJSONEncoder)
-
-
-if __name__ == '__main__':
-    main()