Delete npins

This fixes a bug where nixos-rebuild as root cannot build a nixos
configuration since it cannot fetch the sources of our fork of
nix-eval-jobs for lack of ssh key. It doesn't hit cache *in the store*
for lack of `narHash` being specified, which cannot be specified with
npins since npins doesn't output SRI hashes for git inputs.

Altogether a very silly situation. Clearly we just need another pinning
tool.
This commit is contained in:
jade 2024-05-04 17:03:57 -07:00
parent 71df0f4a4a
commit 491e0f076f
6 changed files with 188 additions and 64 deletions

View file

@ -1,47 +0,0 @@
# Generated by npins. Do not modify; will be overwritten regularly
let
data = builtins.fromJSON (builtins.readFile ./sources.json);
version = data.version;
mkSource = spec:
assert spec ? type; let
path =
if spec.type == "Git" then mkGitSource spec
else if spec.type == "GitRelease" then mkGitSource spec
else if spec.type == "PyPi" then mkPyPiSource spec
else if spec.type == "Channel" then mkChannelSource spec
else builtins.throw "Unknown source type ${spec.type}";
in
spec // { outPath = path; };
mkGitSource = { repository, revision, url ? null, hash, ... }:
assert repository ? type;
# At the moment, either it is a plain git repository (which has an url), or it is a GitHub/GitLab repository
# In the latter case, there we will always be an url to the tarball
if url != null then
(builtins.fetchTarball {
inherit url;
sha256 = hash; # FIXME: check nix version & use SRI hashes
})
else assert repository.type == "Git"; builtins.fetchGit {
url = repository.url;
rev = revision;
# hash = hash;
};
mkPyPiSource = { url, hash, ... }:
builtins.fetchurl {
inherit url;
sha256 = hash;
};
mkChannelSource = { url, hash, ... }:
builtins.fetchTarball {
inherit url;
sha256 = hash;
};
in
if version == 3 then
builtins.mapAttrs (_: mkSource) data.pins
else
throw "Unsupported format version ${toString version} in sources.json. Try running `npins upgrade`"

View file

@ -1,16 +0,0 @@
{
"pins": {
"nix-eval-jobs": {
"type": "Git",
"repository": {
"type": "Git",
"url": "git+ssh://git@git.lix.systems/lix-project/nix-eval-jobs"
},
"branch": "main",
"revision": "793841a9b7b689e37c9a7902710aab2bd6a833d5",
"url": null,
"hash": "1pkb7glscd6dkfjf7x1cj51l21k09wjw5mzm32j37yrln5f20il5"
}
},
"version": 3
}

View file

@ -31,7 +31,7 @@ in
# used for things that one wouldn't necessarily want to update, but we # used for things that one wouldn't necessarily want to update, but we
# nevertheless shove it in the overlay and fixed-point it in case one *does* # nevertheless shove it in the overlay and fixed-point it in case one *does*
# want to do that. # want to do that.
lix-sources = import ./npins; lix-sources = import ./pins.nix;
nixVersions = prev.nixVersions // rec { nixVersions = prev.nixVersions // rec {
# FIXME: do something less scuffed # FIXME: do something less scuffed

1
pins.json Normal file
View file

@ -0,0 +1 @@
{"nix-eval-jobs": {"kind": "git", "rev": "793841a9b7b689e37c9a7902710aab2bd6a833d5", "nar_hash": "sha256-hUYgXLE0+zOkGPXXwiVPYAZBQ5Es9OOkm800puk7a94=", "url": "git@git.lix.systems:lix-project/nix-eval-jobs", "ref": "main"}}

22
pins.nix Normal file
View file

@ -0,0 +1,22 @@
# this is a custom pinning tool, written because npins doesn't have narHash
# compatible output for git inputs, and also doesn't support the Nix immutable
# tarball protocol
let
pins = builtins.fromJSON (builtins.readFile ./pins.json);
fetchPin = args@{ kind, ... }:
if kind == "git" then
builtins.fetchGit
{
url = args.url;
ref = args.ref;
rev = args.rev;
narHash = args.nar_hash;
}
else if kind == "tarball" then
builtins.fetchTarball
{
url = args.locked_url;
sha256 = args.nar_hash;
} else builtins.throw "unsupported input type ${kind}";
in
builtins.mapAttrs (_: fetchPin) pins

164
update_pins.py Executable file
View file

@ -0,0 +1,164 @@
#!/usr/bin/env python
"""
Updates pins in this repo to their latest version.
This is a custom pinning tool, written because npins doesn't have narHash
compatible output for git inputs (it is not SRI), and also doesn't support the
Nix immutable tarball protocol which we would like to use when we become public.
"""
import subprocess
import tempfile
from pathlib import Path
import re
import dataclasses
from typing import Literal
import urllib.parse
import json
# https://stackoverflow.com/a/51286749
class DataclassJSONEncoder(json.JSONEncoder):
def default(self, o):
if dataclasses.is_dataclass(o):
return dataclasses.asdict(o)
return super().default(o)
@dataclasses.dataclass
class PinSerialized:
kind: str
rev: str | None
nar_hash: str
@dataclasses.dataclass
class GitPinSerialized(PinSerialized):
kind: Literal['git']
url: str
rev: str
ref: str
@dataclasses.dataclass
class TarballPinSerialized(PinSerialized):
kind: Literal['tarball']
locked_url: str
url: str
class PinSpec:
def do_pin(self) -> dict[str, str]:
raise ValueError('unimplemented')
@dataclasses.dataclass
class GitPinSpec(PinSpec):
url: str
branch: str
def do_pin(self) -> GitPinSerialized:
return lock_git(self.url, self.branch)
@dataclasses.dataclass
class TarballPinSpec(PinSpec):
url: str
def do_pin(self) -> TarballPinSerialized:
return lock_tarball(self.url)
@dataclasses.dataclass
class LinkHeader:
url: str
rev: str | None
LINK_HEADER_RE = re.compile(r'<(?P<url>.*)>; rel="immutable"')
def parse_link_header(header) -> LinkHeader | None:
matched = LINK_HEADER_RE.match(header)
if not matched:
return None
url = matched.group('url')
parsed_url = urllib.parse.urlparse(url)
parsed_qs = urllib.parse.parse_qs(parsed_url.query)
return LinkHeader(url=url, rev=next(iter(parsed_qs.get('rev', [])), None))
def lock_tarball(url) -> TarballPinSerialized:
"""
Prefetches a tarball using the Nix immutable tarball protocol
"""
import requests
resp = requests.get(url)
with tempfile.TemporaryDirectory() as td:
td = Path(td)
proc = subprocess.Popen(["tar", "-C", td, "-xvzf", "-"],
stdin=subprocess.PIPE)
assert proc.stdin
for chunk in resp.iter_content(64 * 1024):
proc.stdin.write(chunk)
proc.stdin.close()
if proc.wait() != 0:
raise RuntimeError("untarring failed")
children = list(td.iterdir())
# FIXME: allow different tarball structures
assert len(children) == 1
child = children[0].rename(children[0].parent.joinpath('source'))
sri_hash = subprocess.check_output(
["nix-hash", "--type", "sha256", "--sri", child]).decode().strip()
path = subprocess.check_output(
["nix-store", "--add-fixed", "--recursive", "sha256",
child]).decode().strip()
link_info = parse_link_header(resp.headers['Link'])
print(sri_hash, path)
return TarballPinSerialized(kind='tarball',
nar_hash=sri_hash,
locked_url=link_info.url if link_info else url,
rev=link_info.rev if link_info else None,
url=url)
def lock_git(url, branch) -> GitPinSerialized:
url_escaped = json.dumps(url)
ref_escaped = json.dumps(branch)
data = json.loads(
subprocess.check_output([
"nix", "eval", "--impure", "--json", "--expr",
f"builtins.removeAttrs (builtins.fetchGit {{ url = {url_escaped}; ref = {ref_escaped}; }}) [ \"outPath\" ]"
]).strip())
return GitPinSerialized(kind='git',
url=url,
rev=data['rev'],
ref=branch,
nar_hash=data['narHash'])
PINS = {
'nix-eval-jobs':
GitPinSpec('git@git.lix.systems:lix-project/nix-eval-jobs', 'main')
}
def main():
output = {}
for (name, pin) in PINS.items():
output[name] = pin.do_pin()
print(output)
with open('pins.json', 'w') as fh:
json.dump(output, fh, cls=DataclassJSONEncoder)
if __name__ == '__main__':
main()