From 491e0f076f0f4a62f4d91d7724bb8d4663535d05 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Sat, 4 May 2024 17:03:57 -0700 Subject: [PATCH] Delete npins This fixes a bug where nixos-rebuild as root cannot build a nixos configuration since it cannot fetch the sources of our fork of nix-eval-jobs for lack of ssh key. It doesn't hit cache *in the store* for lack of `narHash` being specified, which cannot be specified with npins since npins doesn't output SRI hashes for git inputs. Altogether a very silly situation. Clearly we just need another pinning tool. --- npins/default.nix | 47 ------------- npins/sources.json | 16 ----- overlay.nix | 2 +- pins.json | 1 + pins.nix | 22 ++++++ update_pins.py | 164 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 188 insertions(+), 64 deletions(-) delete mode 100644 npins/default.nix delete mode 100644 npins/sources.json create mode 100644 pins.json create mode 100644 pins.nix create mode 100755 update_pins.py diff --git a/npins/default.nix b/npins/default.nix deleted file mode 100644 index 4a7c372..0000000 --- a/npins/default.nix +++ /dev/null @@ -1,47 +0,0 @@ -# Generated by npins. Do not modify; will be overwritten regularly -let - data = builtins.fromJSON (builtins.readFile ./sources.json); - version = data.version; - - mkSource = spec: - assert spec ? type; let - path = - if spec.type == "Git" then mkGitSource spec - else if spec.type == "GitRelease" then mkGitSource spec - else if spec.type == "PyPi" then mkPyPiSource spec - else if spec.type == "Channel" then mkChannelSource spec - else builtins.throw "Unknown source type ${spec.type}"; - in - spec // { outPath = path; }; - - mkGitSource = { repository, revision, url ? null, hash, ... }: - assert repository ? type; - # At the moment, either it is a plain git repository (which has an url), or it is a GitHub/GitLab repository - # In the latter case, there we will always be an url to the tarball - if url != null then - (builtins.fetchTarball { - inherit url; - sha256 = hash; # FIXME: check nix version & use SRI hashes - }) - else assert repository.type == "Git"; builtins.fetchGit { - url = repository.url; - rev = revision; - # hash = hash; - }; - - mkPyPiSource = { url, hash, ... }: - builtins.fetchurl { - inherit url; - sha256 = hash; - }; - - mkChannelSource = { url, hash, ... }: - builtins.fetchTarball { - inherit url; - sha256 = hash; - }; -in -if version == 3 then - builtins.mapAttrs (_: mkSource) data.pins -else - throw "Unsupported format version ${toString version} in sources.json. Try running `npins upgrade`" diff --git a/npins/sources.json b/npins/sources.json deleted file mode 100644 index 4b9f152..0000000 --- a/npins/sources.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "pins": { - "nix-eval-jobs": { - "type": "Git", - "repository": { - "type": "Git", - "url": "git+ssh://git@git.lix.systems/lix-project/nix-eval-jobs" - }, - "branch": "main", - "revision": "793841a9b7b689e37c9a7902710aab2bd6a833d5", - "url": null, - "hash": "1pkb7glscd6dkfjf7x1cj51l21k09wjw5mzm32j37yrln5f20il5" - } - }, - "version": 3 -} diff --git a/overlay.nix b/overlay.nix index ec8a422..4d8a326 100644 --- a/overlay.nix +++ b/overlay.nix @@ -31,7 +31,7 @@ in # used for things that one wouldn't necessarily want to update, but we # nevertheless shove it in the overlay and fixed-point it in case one *does* # want to do that. - lix-sources = import ./npins; + lix-sources = import ./pins.nix; nixVersions = prev.nixVersions // rec { # FIXME: do something less scuffed diff --git a/pins.json b/pins.json new file mode 100644 index 0000000..0e54d88 --- /dev/null +++ b/pins.json @@ -0,0 +1 @@ +{"nix-eval-jobs": {"kind": "git", "rev": "793841a9b7b689e37c9a7902710aab2bd6a833d5", "nar_hash": "sha256-hUYgXLE0+zOkGPXXwiVPYAZBQ5Es9OOkm800puk7a94=", "url": "git@git.lix.systems:lix-project/nix-eval-jobs", "ref": "main"}} \ No newline at end of file diff --git a/pins.nix b/pins.nix new file mode 100644 index 0000000..78ef4eb --- /dev/null +++ b/pins.nix @@ -0,0 +1,22 @@ +# this is a custom pinning tool, written because npins doesn't have narHash +# compatible output for git inputs, and also doesn't support the Nix immutable +# tarball protocol +let + pins = builtins.fromJSON (builtins.readFile ./pins.json); + fetchPin = args@{ kind, ... }: + if kind == "git" then + builtins.fetchGit + { + url = args.url; + ref = args.ref; + rev = args.rev; + narHash = args.nar_hash; + } + else if kind == "tarball" then + builtins.fetchTarball + { + url = args.locked_url; + sha256 = args.nar_hash; + } else builtins.throw "unsupported input type ${kind}"; +in +builtins.mapAttrs (_: fetchPin) pins diff --git a/update_pins.py b/update_pins.py new file mode 100755 index 0000000..4f68b57 --- /dev/null +++ b/update_pins.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python +""" +Updates pins in this repo to their latest version. + +This is a custom pinning tool, written because npins doesn't have narHash +compatible output for git inputs (it is not SRI), and also doesn't support the +Nix immutable tarball protocol which we would like to use when we become public. +""" +import subprocess +import tempfile +from pathlib import Path +import re +import dataclasses +from typing import Literal +import urllib.parse +import json + + +# https://stackoverflow.com/a/51286749 +class DataclassJSONEncoder(json.JSONEncoder): + + def default(self, o): + if dataclasses.is_dataclass(o): + return dataclasses.asdict(o) + return super().default(o) + + +@dataclasses.dataclass +class PinSerialized: + kind: str + rev: str | None + nar_hash: str + + +@dataclasses.dataclass +class GitPinSerialized(PinSerialized): + kind: Literal['git'] + url: str + rev: str + ref: str + + +@dataclasses.dataclass +class TarballPinSerialized(PinSerialized): + kind: Literal['tarball'] + locked_url: str + url: str + + +class PinSpec: + + def do_pin(self) -> dict[str, str]: + raise ValueError('unimplemented') + + +@dataclasses.dataclass +class GitPinSpec(PinSpec): + url: str + branch: str + + def do_pin(self) -> GitPinSerialized: + return lock_git(self.url, self.branch) + + +@dataclasses.dataclass +class TarballPinSpec(PinSpec): + url: str + + def do_pin(self) -> TarballPinSerialized: + return lock_tarball(self.url) + + +@dataclasses.dataclass +class LinkHeader: + url: str + rev: str | None + + +LINK_HEADER_RE = re.compile(r'<(?P.*)>; rel="immutable"') + + +def parse_link_header(header) -> LinkHeader | None: + matched = LINK_HEADER_RE.match(header) + if not matched: + return None + + url = matched.group('url') + parsed_url = urllib.parse.urlparse(url) + parsed_qs = urllib.parse.parse_qs(parsed_url.query) + + return LinkHeader(url=url, rev=next(iter(parsed_qs.get('rev', [])), None)) + + +def lock_tarball(url) -> TarballPinSerialized: + """ + Prefetches a tarball using the Nix immutable tarball protocol + """ + import requests + resp = requests.get(url) + with tempfile.TemporaryDirectory() as td: + td = Path(td) + proc = subprocess.Popen(["tar", "-C", td, "-xvzf", "-"], + stdin=subprocess.PIPE) + assert proc.stdin + for chunk in resp.iter_content(64 * 1024): + proc.stdin.write(chunk) + proc.stdin.close() + if proc.wait() != 0: + raise RuntimeError("untarring failed") + + children = list(td.iterdir()) + # FIXME: allow different tarball structures + assert len(children) == 1 + + child = children[0].rename(children[0].parent.joinpath('source')) + sri_hash = subprocess.check_output( + ["nix-hash", "--type", "sha256", "--sri", child]).decode().strip() + path = subprocess.check_output( + ["nix-store", "--add-fixed", "--recursive", "sha256", + child]).decode().strip() + + link_info = parse_link_header(resp.headers['Link']) + + print(sri_hash, path) + return TarballPinSerialized(kind='tarball', + nar_hash=sri_hash, + locked_url=link_info.url if link_info else url, + rev=link_info.rev if link_info else None, + url=url) + + +def lock_git(url, branch) -> GitPinSerialized: + url_escaped = json.dumps(url) + ref_escaped = json.dumps(branch) + data = json.loads( + subprocess.check_output([ + "nix", "eval", "--impure", "--json", "--expr", + f"builtins.removeAttrs (builtins.fetchGit {{ url = {url_escaped}; ref = {ref_escaped}; }}) [ \"outPath\" ]" + ]).strip()) + return GitPinSerialized(kind='git', + url=url, + rev=data['rev'], + ref=branch, + nar_hash=data['narHash']) + + +PINS = { + 'nix-eval-jobs': + GitPinSpec('git@git.lix.systems:lix-project/nix-eval-jobs', 'main') +} + + +def main(): + output = {} + for (name, pin) in PINS.items(): + output[name] = pin.do_pin() + + print(output) + with open('pins.json', 'w') as fh: + json.dump(output, fh, cls=DataclassJSONEncoder) + + +if __name__ == '__main__': + main()