update nix-eval-jobs for 2.91-dev

lix-project/nix-eval-jobs#7
https://gerrit.lix.systems/c/lix/+/1241/5

.gitignore

@@ -0,0 +1,2 @@
+result
+result-*

LICENSE

@@ -0,0 +1,19 @@
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is furnished
+to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice (including the next
+paragraph) shall be included in all copies or substantial portions of the
+Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -0,0 +1,5 @@
+# Lix NixOS module
+
+See the [beta guide][beta-guide] for details on how to use this:
+
+[beta-guide]: https://wiki.lix.systems/link/1

flake.lock

@@ -5,11 +5,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1709126324,
-        "narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=",
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "d465f4819400de7c8d874d50b982301f28a84605",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
         "type": "github"
       },
       "original": {
@@ -18,28 +18,42 @@
         "type": "github"
       }
     },
+    "flakey-profile": {
+      "locked": {
+        "lastModified": 1712898590,
+        "narHash": "sha256-FhGIEU93VHAChKEXx905TSiPZKga69bWl1VB37FK//I=",
+        "owner": "lf-",
+        "repo": "flakey-profile",
+        "rev": "243c903fd8eadc0f63d205665a92d4df91d42d9d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "lf-",
+        "repo": "flakey-profile",
+        "type": "github"
+      }
+    },
     "lix": {
+      "flake": false,
       "locked": {
-        "lastModified": 1709811453,
-        "narHash": "sha256-UOyTAdUAJv80w3s6rrgYB5tQjp43u5Zp2dlM94uud04=",
-        "ref": "refs/heads/main",
-        "rev": "8a268359b06471b463a8f3fc46c83a1fced8ab75",
-        "revCount": 15077,
-        "type": "git",
-        "url": "ssh://gerrit.lix.systems:2022/lix"
+        "lastModified": 1718767907,
+        "narHash": "sha256-gpd+mGQxqVHw2kO6rSPQel8TkChHh9UpqxjsmQi0QJM=",
+        "rev": "85f282ef572577899b3d80ba8def1b920a386218",
+        "type": "tarball",
+        "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/85f282ef572577899b3d80ba8def1b920a386218.tar.gz"
       },
       "original": {
-        "type": "git",
-        "url": "ssh://gerrit.lix.systems:2022/lix"
+        "type": "tarball",
+        "url": "https://git.lix.systems/lix-project/lix/archive/main.tar.gz"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1709703039,
-        "narHash": "sha256-6hqgQ8OK6gsMu1VtcGKBxKQInRLHtzulDo9Z5jxHEFY=",
+        "lastModified": 1718530797,
+        "narHash": "sha256-pup6cYwtgvzDpvpSCFh1TEUjw2zkNpk8iolbKnyFmmU=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "9df3e30ce24fd28c7b3e2de0d986769db5d6225d",
+        "rev": "b60ebf54c15553b393d144357375ea956f89e9a9",
         "type": "github"
       },
       "original": {
@@ -52,6 +66,7 @@
     "root": {
       "inputs": {
         "flake-utils": "flake-utils",
+        "flakey-profile": "flakey-profile",
         "lix": "lix",
         "nixpkgs": "nixpkgs"
       }

flake.nix

@@ -1,25 +1,47 @@
 {
-  # fixme: use the forgejo address
   inputs.nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
   inputs.lix = {
-    url = "git+ssh://gerrit.lix.systems:2022/lix";
+    url = "https://git.lix.systems/lix-project/lix/archive/main.tar.gz";
     flake = false;
   };
   inputs.flake-utils.url = "github:numtide/flake-utils";
+  inputs.flakey-profile.url = "github:lf-/flakey-profile";
 
-  outputs = { self, nixpkgs, lix, flake-utils }: {
-    nixosModules.default = import ./module.nix { inherit lix; };
-    overlays.default = import ./overlay.nix { inherit lix; };
-  } // flake-utils.lib.eachDefaultSystem (system:
-    let
-      pkgs = import nixpkgs {
-        inherit system;
-        overlays = [ self.overlays.default ];
-      };
-    in
-    {
-      inherit pkgs;
-      packages.default = pkgs.nixVersions.nix_2_18;
-      packages.nix-doc = pkgs.nix-doc;
-    });
+  outputs = inputs@{ self, nixpkgs, lix, flake-utils, flakey-profile, ... }:
+    let versionSuffix = "pre${builtins.substring 0 8 lix.lastModifiedDate}-${lix.shortRev or lix.dirtyShortRev}";
+    in {
+      inherit inputs;
+      nixosModules.default = import ./module.nix { inherit lix versionSuffix; };
+      overlays.default = import ./overlay.nix { inherit lix versionSuffix; };
+    } // flake-utils.lib.eachDefaultSystem (system:
+      let
+        pkgs = import nixpkgs {
+          inherit system;
+          overlays = [ self.overlays.default ];
+        };
+
+        linux64BitSystems = [
+          "x86_64-linux"
+          "aarch64-linux"
+        ];
+
+        inherit (pkgs) lib;
+      in
+      {
+        inherit pkgs;
+        packages = {
+          default = pkgs.nixVersions.nix_2_18;
+          inherit (pkgs) nix-doc nix-eval-jobs;
+        };
+
+        packages.system-profile = import ./system-profile.nix { inherit pkgs flakey-profile; };
+
+        nixosTests = pkgs.recurseIntoAttrs (pkgs.callPackage ./test-nixos.nix { lix-module = self.nixosModules.default; });
+
+        checks = {
+          inherit (self.packages.${system}) default nix-eval-jobs;
+        } // lib.optionalAttrs (lib.elem system linux64BitSystems) {
+          inherit (self.nixosTests.${system}) it-builds;
+        };
+      });
 }

module.nix

@@ -1,5 +1,5 @@
-{ lix }:
+{ lix, versionSuffix ? "" }:
 { pkgs, config, ... }:
 {
-  nixpkgs.overlays = [ (import ./overlay.nix { inherit lix; }) ];
+  nixpkgs.overlays = [ (import ./overlay.nix { inherit lix versionSuffix; }) ];
 }

nix-doc/package.nix

@@ -0,0 +1,64 @@
+# Temporary replacement of the nix-doc package with
+# https://github.com/NixOS/nixpkgs/pull/296523 so that we can have working Lix
+# with nix-doc on 23.11 and 24.05-pre
+#
+# Can be removed when that commit is in 23.11 and 24.05-pre, or 24.05 is
+# released with the commit.
+{ lib
+, stdenv
+, rustPlatform
+, fetchFromGitHub
+, boost
+, nix
+, pkg-config
+# Whether to build the nix-doc plugin for Nix
+, withPlugin ? true
+}:
+
+let
+  packageFlags = [ "-p" "nix-doc" ] ++ lib.optionals withPlugin [ "-p" "nix-doc-plugin" ];
+in
+rustPlatform.buildRustPackage rec {
+  pname = "nix-doc";
+  version = "0.6.5";
+
+  src = fetchFromGitHub {
+    rev = "v${version}";
+    owner = "lf-";
+    repo = "nix-doc";
+    sha256 = "sha256-9cuNzq+CBA2jz0LkZb7lh/WISIlKklfovGBAbSo1Mgk=";
+  };
+
+  doCheck = true;
+  buildInputs = lib.optionals withPlugin [ boost nix ];
+
+  nativeBuildInputs = lib.optionals withPlugin [ pkg-config nix ];
+
+  cargoBuildFlags = packageFlags;
+  cargoTestFlags = packageFlags;
+
+  # Packaging support for making the nix-doc plugin load cleanly as a no-op on
+  # the wrong Nix version (disabling bindnow permits loading libraries
+  # requiring unavailable symbols if they are unreached)
+  hardeningDisable = lib.optionals withPlugin [ "bindnow" ];
+  # Due to a Rust bug, setting -Z relro-level to anything including "off" on
+  # macOS will cause link errors
+  env = lib.optionalAttrs (withPlugin && stdenv.isLinux) {
+    # nix-doc does not use nightly features, however, there is no other way to
+    # set relro-level
+    RUSTC_BOOTSTRAP = 1;
+    RUSTFLAGS = "-Z relro-level=partial";
+  };
+
+  cargoSha256 = "sha256-CHagzXTG9AfrFd3WmHanQ+YddMgmVxSuB8vK98A1Mlw=";
+
+  meta = with lib; {
+    description = "An interactive Nix documentation tool";
+    longDescription = "An interactive Nix documentation tool providing a CLI for function search, a Nix plugin for docs in the REPL, and a ctags implementation for Nix script";
+    homepage = "https://github.com/lf-/nix-doc";
+    license = licenses.lgpl3Plus;
+    maintainers = [ maintainers.lf- ];
+    platforms = platforms.unix;
+    mainProgram = "nix-doc";
+  };
+}

overlay.nix

@@ -1,47 +1,99 @@
-{ lix }:
+{ lix, versionSuffix ? "" }:
 final: prev:
 let
-  boehmgc-patched = ((final.boehmgc.override {
-    enableLargeConfig = true;
-  }).overrideAttrs (o: {
-    patches = (o.patches or [ ]) ++ [
-      # for clown reasons this version is newer than the one in lix, we should
-      # fix this and update our nixpkgs pin
-      (prev.path + "/pkgs/tools/package-management/nix/patches/boehmgc-coroutine-sp-fallback.patch")
-
-      # https://github.com/ivmai/bdwgc/pull/586
-      (lix + "/boehmgc-traceable_allocator-public.diff")
-    ];
-  })
-  );
-in
-{
-  nixVersions = prev.nixVersions // rec {
-    # FIXME: do something less scuffed
-    nix_2_18 = (prev.nixVersions.nix_2_18.override { boehmgc = boehmgc-patched; }).overrideAttrs (old: {
-      src = lix;
-      version = "2.18.3-lix";
-      VERSION_SUFFIX = "-lix";
-
-      patches = [ ];
-    });
-    stable = nix_2_18;
-    nix_2_18_upstream = prev.nixVersions.nix_2_18;
-  };
-
-  # force these onto upstream so we are not regularly rebuilding electron
-  prefetch-yarn-deps = prev.prefetch-yarn-deps.override {
-    nix = final.nixVersions.nix_2_18_upstream;
-  };
-  prefetch-npm-deps = prev.prefetch-npm-deps.override {
-    nix = final.nixVersions.nix_2_18_upstream;
-  };
-  nix-prefetch-git = prev.nix-prefetch-git.override {
-    nix = final.nixVersions.nix_2_18_upstream;
-  };
-
-  nix-doc = prev.nix-doc.overrideAttrs (old: {
-    # for the purposes of nix C++ API for nix-doc, lix is Nix 2.20
-    NIX_CFLAGS_COMPILE = [ "-DNIX_2_20_0" ];
+  # This is kind of scary to not override the nix version to pretend to be
+  # 2.18 since nixpkgs can introduce new breakage in its Nix unstable CLI
+  # usage.
+  # https://github.com/nixos/nixpkgs/blob/6afb255d976f85f3359e4929abd6f5149c323a02/nixos/modules/config/nix.nix#L121
+  lixPkg = final.callPackage (lix + "/package.nix") ({
+    versionSuffix = "-${versionSuffix}";
+    # FIXME: do this more sensibly for future releases
+    # https://git.lix.systems/lix-project/lix/issues/406
+    officialRelease = false;
   });
-}
+
+  # These packages depend on Nix features that Lix does not support
+  overridelist_2_18 = [
+    "attic-client"
+    "devenv"
+    "nix-du"
+    "nix-init"
+    "nix-prefetch-git"
+    "nixos-option"
+    "nurl"
+    "prefetch-yarn-deps" # force these onto upstream so we are not regularly rebuilding electron
+  ];
+  override_2_18 = prev.lib.genAttrs overridelist_2_18 (
+    name: prev.${name}.override {
+      nix = final.nixVersions.nix_2_18_upstream;
+    });
+
+  inherit (prev) lib;
+
+  prefetch-npm-deps-args = lib.functionArgs prev.prefetch-npm-deps.override;
+
+  warning = ''
+    warning: You have the lix overlay included into a nixpkgs import twice,
+    perhaps due to the NixOS module being included twice, or because of using
+    pkgs.nixos and also including it in imports, or perhaps some unknown
+    machinations of a complicated flake library.
+    This is completely harmless since we have no-op'd the second one if you are
+    seeing this message, but it would be a small style improvement to fix
+    it :)
+    P.S. If you had some hack to fix nixos-option build failures in your
+    configuration, that was caused by including an older version of the lix
+    overlay twice, which is now mitigated if you see this message, so you can
+    delete that.
+    P.P.S. This Lix has super catgirl powers.
+  '';
+
+  maybeWarnDuplicate = x: if final.lix-overlay-present > 1 then builtins.trace warning x else x;
+
+  overlay = override_2_18 // {
+    lix-overlay-present = 1;
+    # used for things that one wouldn't necessarily want to update, but we
+    # nevertheless shove it in the overlay and fixed-point it in case one *does*
+    # want to do that.
+    lix-sources = import ./pins.nix;
+
+    nixVersions = prev.nixVersions // rec {
+      # FIXME: do something less scuffed
+      nix_2_18 = maybeWarnDuplicate lixPkg;
+      stable = nix_2_18;
+      nix_2_18_upstream = prev.nixVersions.nix_2_18;
+    };
+
+    nix-eval-jobs = (prev.nix-eval-jobs.override {
+      # lix
+      nix = final.nixVersions.nix_2_18;
+    }).overrideAttrs (old:
+      let src = final.lix-sources.nix-eval-jobs;
+      in {
+        version = "2.90.0-lix-${builtins.substring 0 7 src.rev}";
+
+        # FIXME: should this be patches instead?
+        inherit src;
+
+        mesonBuildType = "debugoptimized";
+
+        ninjaFlags = old.ninjaFlags or [ ] ++ [ "-v" ];
+      }
+    );
+
+    # support both having and missing https://github.com/NixOS/nixpkgs/pull/304913
+    prefetch-npm-deps =
+      if (prefetch-npm-deps-args ? nix) || (prefetch-npm-deps-args == {})
+      then prev.prefetch-npm-deps.override {
+        nix = final.nixVersions.nix_2_18_upstream;
+      }
+      else prev.prefetch-npm-deps;
+
+    nix-doc = prev.callPackage ./nix-doc/package.nix { withPlugin = false; };
+
+    pegtl = prev.callPackage ./pegtl.nix { };
+  };
+in
+  # Make the overlay idempotent, since flakes passing nixos modules around by
+  # value and many other things make it way too easy to include the overlay
+  # twice
+  if (prev ? lix-overlay-present) then { lix-overlay-present = 2; } else overlay

pegtl.nix

@@ -0,0 +1,23 @@
+{
+  stdenv,
+  cmake,
+  ninja,
+  fetchFromGitHub,
+}:
+
+stdenv.mkDerivation {
+  pname = "pegtl";
+  version = "3.2.7";
+
+  src = fetchFromGitHub {
+    repo = "PEGTL";
+    owner = "taocpp";
+    rev = "refs/tags/3.2.7";
+    hash = "sha256-IV5YNGE4EWVrmg2Sia/rcU8jCuiBynQGJM6n3DCWTQU=";
+  };
+
+  nativeBuildInputs = [
+    cmake
+    ninja
+  ];
+}

pins.json

@@ -0,0 +1 @@
+{"nix-eval-jobs": {"kind": "tarball", "rev": "f8869bdcca7c1d5aaf37de3da3a4176811279a57", "nar_hash": "sha256-F/RvI9chHywnckEqHO1ggjzCayknhDnnl2kNnnVXpWg=", "locked_url": "https://git.lix.systems/api/v1/repos/lix-project/nix-eval-jobs/archive/f8869bdcca7c1d5aaf37de3da3a4176811279a57.tar.gz?rev=f8869bdcca7c1d5aaf37de3da3a4176811279a57", "url": "https://git.lix.systems/lix-project/nix-eval-jobs/archive/main.tar.gz"}}

pins.nix

@@ -0,0 +1,26 @@
+# this is a custom pinning tool, written because npins doesn't have narHash
+# compatible output for git inputs, and also doesn't support the Nix immutable
+# tarball protocol
+let
+  pins = builtins.fromJSON (builtins.readFile ./pins.json);
+  fetchPin = args@{ kind, ... }:
+    if kind == "git" then
+      builtins.fetchGit
+        {
+          name = "source";
+          url = args.url;
+          ref = args.ref;
+          rev = args.rev;
+          narHash = args.nar_hash;
+        }
+    else if kind == "tarball" then
+      args // {
+        outPath = builtins.fetchTarball {
+          name = "source";
+          url = args.locked_url;
+          sha256 = args.nar_hash;
+        };
+      }
+    else builtins.throw "unsupported input type ${kind}";
+in
+builtins.mapAttrs (_: fetchPin) pins

system-profile.nix

@@ -0,0 +1,10 @@
+{ pkgs, flakey-profile }:
+flakey-profile.lib.mkProfile {
+  inherit pkgs;
+  paths = with pkgs; [
+    cacert
+    nix
+  ];
+  name = "system-profile";
+  extraSwitchArgs = [ "--profile" "/nix/var/nix/profiles/default" ];
+}

test-nixos.nix

@@ -0,0 +1,18 @@
+{ nixos, lix-module }:
+let
+  configs = {
+    it-builds = nixos ({ ... }: {
+      imports = [ lix-module ];
+      documentation.enable = false;
+      fileSystems."/".device = "ignore-root-device";
+      boot.loader.grub.enable = false;
+      system.stateVersion = "24.05";
+    });
+
+  };
+in
+{
+  inherit configs;
+
+  it-builds = configs.it-builds.config.system.build.toplevel;
+}

update_pins.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p 'python3.withPackages (ps: [ ps.requests ])'
+"""
+Updates pins in this repo to their latest version.
+
+This is a custom pinning tool, written because npins doesn't have narHash
+compatible output for git inputs (it is not SRI), and also doesn't support the
+Nix immutable tarball protocol which we would like to use when we become public.
+"""
+import subprocess
+import tempfile
+from pathlib import Path
+import re
+import dataclasses
+from typing import Literal
+import urllib.parse
+import json
+
+
+# https://stackoverflow.com/a/51286749
+class DataclassJSONEncoder(json.JSONEncoder):
+
+    def default(self, o):
+        if dataclasses.is_dataclass(o):
+            return dataclasses.asdict(o)
+        return super().default(o)
+
+
+@dataclasses.dataclass
+class PinSerialized:
+    kind: str
+    rev: str | None
+    nar_hash: str
+
+
+@dataclasses.dataclass
+class GitPinSerialized(PinSerialized):
+    kind: Literal['git']
+    url: str
+    rev: str
+    ref: str
+
+
+@dataclasses.dataclass
+class TarballPinSerialized(PinSerialized):
+    kind: Literal['tarball']
+    locked_url: str
+    url: str
+
+
+class PinSpec:
+
+    def do_pin(self) -> dict[str, str]:
+        raise ValueError('unimplemented')
+
+
+@dataclasses.dataclass
+class GitPinSpec(PinSpec):
+    url: str
+    branch: str
+
+    def do_pin(self) -> GitPinSerialized:
+        return lock_git(self.url, self.branch)
+
+
+@dataclasses.dataclass
+class TarballPinSpec(PinSpec):
+    url: str
+
+    def do_pin(self) -> TarballPinSerialized:
+        return lock_tarball(self.url)
+
+
+@dataclasses.dataclass
+class LinkHeader:
+    url: str
+    rev: str | None
+
+
+LINK_HEADER_RE = re.compile(r'<(?P<url>.*)>; rel="immutable"')
+
+
+def parse_link_header(header) -> LinkHeader | None:
+    matched = LINK_HEADER_RE.match(header)
+    if not matched:
+        return None
+
+    url = matched.group('url')
+    parsed_url = urllib.parse.urlparse(url)
+    parsed_qs = urllib.parse.parse_qs(parsed_url.query)
+
+    return LinkHeader(url=url, rev=next(iter(parsed_qs.get('rev', [])), None))
+
+
+def lock_tarball(url) -> TarballPinSerialized:
+    """
+    Prefetches a tarball using the Nix immutable tarball protocol
+    """
+    import requests
+    resp = requests.get(url)
+    with tempfile.TemporaryDirectory() as td:
+        td = Path(td)
+        proc = subprocess.Popen(["tar", "-C", td, "-xvzf", "-"],
+                                stdin=subprocess.PIPE)
+        assert proc.stdin
+        for chunk in resp.iter_content(64 * 1024):
+            proc.stdin.write(chunk)
+        proc.stdin.close()
+        if proc.wait() != 0:
+            raise RuntimeError("untarring failed")
+
+        children = list(td.iterdir())
+        # FIXME: allow different tarball structures
+        assert len(children) == 1
+
+        child = children[0].rename(children[0].parent.joinpath('source'))
+        sri_hash = subprocess.check_output(
+            ["nix-hash", "--type", "sha256", "--sri", child]).decode().strip()
+        path = subprocess.check_output(
+            ["nix-store", "--add-fixed", "--recursive", "sha256",
+             child]).decode().strip()
+
+    link_info = parse_link_header(resp.headers['Link'])
+
+    print(sri_hash, path)
+    return TarballPinSerialized(kind='tarball',
+                                nar_hash=sri_hash,
+                                locked_url=link_info.url if link_info else url,
+                                rev=link_info.rev if link_info else None,
+                                url=url)
+
+
+def lock_git(url, branch) -> GitPinSerialized:
+    url_escaped = json.dumps(url)
+    ref_escaped = json.dumps(branch)
+    data = json.loads(
+        subprocess.check_output([
+            "nix", "eval", "--impure", "--json", "--expr",
+            f"builtins.removeAttrs (builtins.fetchGit {{ url = {url_escaped}; ref = {ref_escaped}; }}) [ \"outPath\" ]"
+        ]).strip())
+    return GitPinSerialized(kind='git',
+                            url=url,
+                            rev=data['rev'],
+                            ref=branch,
+                            nar_hash=data['narHash'])
+
+
+PINS = {
+    'nix-eval-jobs':
+    TarballPinSpec('https://git.lix.systems/lix-project/nix-eval-jobs/archive/main.tar.gz')
+}
+
+
+def main():
+    output = {}
+    for (name, pin) in PINS.items():
+        output[name] = pin.do_pin()
+
+    print(output)
+    with open('pins.json', 'w') as fh:
+        json.dump(output, fh, cls=DataclassJSONEncoder)
+
+
+if __name__ == '__main__':
+    main()