Delete npins

This fixes a bug where nixos-rebuild as root cannot build a nixos
configuration since it cannot fetch the sources of our fork of
nix-eval-jobs for lack of ssh key. It doesn't hit cache *in the store*
for lack of `narHash` being specified, which cannot be specified with
npins since npins doesn't output SRI hashes for git inputs.

Altogether a very silly situation. Clearly we just need another pinning
tool.

npins/default.nix

@@ -1,47 +0,0 @@
-# Generated by npins. Do not modify; will be overwritten regularly
-let
-  data = builtins.fromJSON (builtins.readFile ./sources.json);
-  version = data.version;
-
-  mkSource = spec:
-    assert spec ? type; let
-      path =
-        if spec.type == "Git" then mkGitSource spec
-        else if spec.type == "GitRelease" then mkGitSource spec
-        else if spec.type == "PyPi" then mkPyPiSource spec
-        else if spec.type == "Channel" then mkChannelSource spec
-        else builtins.throw "Unknown source type ${spec.type}";
-    in
-    spec // { outPath = path; };
-
-  mkGitSource = { repository, revision, url ? null, hash, ... }:
-    assert repository ? type;
-    # At the moment, either it is a plain git repository (which has an url), or it is a GitHub/GitLab repository
-    # In the latter case, there we will always be an url to the tarball
-    if url != null then
-      (builtins.fetchTarball {
-        inherit url;
-        sha256 = hash; # FIXME: check nix version & use SRI hashes
-      })
-    else assert repository.type == "Git"; builtins.fetchGit {
-      url = repository.url;
-      rev = revision;
-      # hash = hash;
-    };
-
-  mkPyPiSource = { url, hash, ... }:
-    builtins.fetchurl {
-      inherit url;
-      sha256 = hash;
-    };
-
-  mkChannelSource = { url, hash, ... }:
-    builtins.fetchTarball {
-      inherit url;
-      sha256 = hash;
-    };
-in
-if version == 3 then
-  builtins.mapAttrs (_: mkSource) data.pins
-else
-  throw "Unsupported format version ${toString version} in sources.json. Try running `npins upgrade`"

npins/sources.json

@@ -1,16 +0,0 @@
-{
-  "pins": {
-    "nix-eval-jobs": {
-      "type": "Git",
-      "repository": {
-        "type": "Git",
-        "url": "git+ssh://git@git.lix.systems/lix-project/nix-eval-jobs"
-      },
-      "branch": "main",
-      "revision": "793841a9b7b689e37c9a7902710aab2bd6a833d5",
-      "url": null,
-      "hash": "1pkb7glscd6dkfjf7x1cj51l21k09wjw5mzm32j37yrln5f20il5"
-    }
-  },
-  "version": 3
-}

overlay.nix

@@ -31,7 +31,7 @@ in
   # used for things that one wouldn't necessarily want to update, but we
   # nevertheless shove it in the overlay and fixed-point it in case one *does*
   # want to do that.
-  lix-sources = import ./npins;
+  lix-sources = import ./pins.nix;
 
   nixVersions = prev.nixVersions // rec {
     # FIXME: do something less scuffed

pins.json

@@ -0,0 +1 @@
+{"nix-eval-jobs": {"kind": "git", "rev": "793841a9b7b689e37c9a7902710aab2bd6a833d5", "nar_hash": "sha256-hUYgXLE0+zOkGPXXwiVPYAZBQ5Es9OOkm800puk7a94=", "url": "git@git.lix.systems:lix-project/nix-eval-jobs", "ref": "main"}}

pins.nix

@@ -0,0 +1,22 @@
+# this is a custom pinning tool, written because npins doesn't have narHash
+# compatible output for git inputs, and also doesn't support the Nix immutable
+# tarball protocol
+let
+  pins = builtins.fromJSON (builtins.readFile ./pins.json);
+  fetchPin = args@{ kind, ... }:
+    if kind == "git" then
+      builtins.fetchGit
+        {
+          url = args.url;
+          ref = args.ref;
+          rev = args.rev;
+          narHash = args.nar_hash;
+        }
+    else if kind == "tarball" then
+      builtins.fetchTarball
+        {
+          url = args.locked_url;
+          sha256 = args.nar_hash;
+        } else builtins.throw "unsupported input type ${kind}";
+in
+builtins.mapAttrs (_: fetchPin) pins

update_pins.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python
+"""
+Updates pins in this repo to their latest version.
+
+This is a custom pinning tool, written because npins doesn't have narHash
+compatible output for git inputs (it is not SRI), and also doesn't support the
+Nix immutable tarball protocol which we would like to use when we become public.
+"""
+import subprocess
+import tempfile
+from pathlib import Path
+import re
+import dataclasses
+from typing import Literal
+import urllib.parse
+import json
+
+
+# https://stackoverflow.com/a/51286749
+class DataclassJSONEncoder(json.JSONEncoder):
+
+    def default(self, o):
+        if dataclasses.is_dataclass(o):
+            return dataclasses.asdict(o)
+        return super().default(o)
+
+
+@dataclasses.dataclass
+class PinSerialized:
+    kind: str
+    rev: str | None
+    nar_hash: str
+
+
+@dataclasses.dataclass
+class GitPinSerialized(PinSerialized):
+    kind: Literal['git']
+    url: str
+    rev: str
+    ref: str
+
+
+@dataclasses.dataclass
+class TarballPinSerialized(PinSerialized):
+    kind: Literal['tarball']
+    locked_url: str
+    url: str
+
+
+class PinSpec:
+
+    def do_pin(self) -> dict[str, str]:
+        raise ValueError('unimplemented')
+
+
+@dataclasses.dataclass
+class GitPinSpec(PinSpec):
+    url: str
+    branch: str
+
+    def do_pin(self) -> GitPinSerialized:
+        return lock_git(self.url, self.branch)
+
+
+@dataclasses.dataclass
+class TarballPinSpec(PinSpec):
+    url: str
+
+    def do_pin(self) -> TarballPinSerialized:
+        return lock_tarball(self.url)
+
+
+@dataclasses.dataclass
+class LinkHeader:
+    url: str
+    rev: str | None
+
+
+LINK_HEADER_RE = re.compile(r'<(?P<url>.*)>; rel="immutable"')
+
+
+def parse_link_header(header) -> LinkHeader | None:
+    matched = LINK_HEADER_RE.match(header)
+    if not matched:
+        return None
+
+    url = matched.group('url')
+    parsed_url = urllib.parse.urlparse(url)
+    parsed_qs = urllib.parse.parse_qs(parsed_url.query)
+
+    return LinkHeader(url=url, rev=next(iter(parsed_qs.get('rev', [])), None))
+
+
+def lock_tarball(url) -> TarballPinSerialized:
+    """
+    Prefetches a tarball using the Nix immutable tarball protocol
+    """
+    import requests
+    resp = requests.get(url)
+    with tempfile.TemporaryDirectory() as td:
+        td = Path(td)
+        proc = subprocess.Popen(["tar", "-C", td, "-xvzf", "-"],
+                                stdin=subprocess.PIPE)
+        assert proc.stdin
+        for chunk in resp.iter_content(64 * 1024):
+            proc.stdin.write(chunk)
+        proc.stdin.close()
+        if proc.wait() != 0:
+            raise RuntimeError("untarring failed")
+
+        children = list(td.iterdir())
+        # FIXME: allow different tarball structures
+        assert len(children) == 1
+
+        child = children[0].rename(children[0].parent.joinpath('source'))
+        sri_hash = subprocess.check_output(
+            ["nix-hash", "--type", "sha256", "--sri", child]).decode().strip()
+        path = subprocess.check_output(
+            ["nix-store", "--add-fixed", "--recursive", "sha256",
+             child]).decode().strip()
+
+    link_info = parse_link_header(resp.headers['Link'])
+
+    print(sri_hash, path)
+    return TarballPinSerialized(kind='tarball',
+                                nar_hash=sri_hash,
+                                locked_url=link_info.url if link_info else url,
+                                rev=link_info.rev if link_info else None,
+                                url=url)
+
+
+def lock_git(url, branch) -> GitPinSerialized:
+    url_escaped = json.dumps(url)
+    ref_escaped = json.dumps(branch)
+    data = json.loads(
+        subprocess.check_output([
+            "nix", "eval", "--impure", "--json", "--expr",
+            f"builtins.removeAttrs (builtins.fetchGit {{ url = {url_escaped}; ref = {ref_escaped}; }}) [ \"outPath\" ]"
+        ]).strip())
+    return GitPinSerialized(kind='git',
+                            url=url,
+                            rev=data['rev'],
+                            ref=branch,
+                            nar_hash=data['narHash'])
+
+
+PINS = {
+    'nix-eval-jobs':
+    GitPinSpec('git@git.lix.systems:lix-project/nix-eval-jobs', 'main')
+}
+
+
+def main():
+    output = {}
+    for (name, pin) in PINS.items():
+        output[name] = pin.do_pin()
+
+    print(output)
+    with open('pins.json', 'w') as fh:
+        json.dump(output, fh, cls=DataclassJSONEncoder)
+
+
+if __name__ == '__main__':
+    main()