From 72e8f94081290784e2d115cfbf09375b53d8cbe9 Mon Sep 17 00:00:00 2001 From: Gabriel Fontes Date: Wed, 6 Oct 2021 13:32:46 -0300 Subject: [PATCH 1/2] add sourcehut input scheme --- src/libfetchers/github.cc | 91 ++++++++++++++++++++++++++++++++++++++- src/nix/flake.md | 32 ++++++++++++++ 2 files changed, 122 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 1c539b80e..e101aa3d6 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -8,6 +8,7 @@ #include #include +#include namespace nix::fetchers { @@ -17,7 +18,7 @@ struct DownloadUrl Headers headers; }; -// A github or gitlab host +// A github, gitlab, or sourcehut host const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check std::regex hostRegex(hostRegexS, std::regex::ECMAScript); @@ -348,7 +349,95 @@ struct GitLabInputScheme : GitArchiveInputScheme } }; +struct SourceHutInputScheme : GitArchiveInputScheme +{ + std::string type() override { return "sourcehut"; } + + std::optional> accessHeaderFromToken(const std::string & token) const override + { + // SourceHut supports both PAT and OAuth2. See + // https://man.sr.ht/meta.sr.ht/oauth.md + return std::pair("Authorization", fmt("Bearer %s", token)); + // Note: This currently serves no purpose, as this kind of authorization + // does not allow for downloading tarballs on sourcehut private repos. + // Once it is implemented, however, should work as expected. + } + + Hash getRevFromRef(nix::ref store, const Input & input) const override + { + // TODO: In the future, when the sourcehut graphql API is implemented for mercurial + // and with anonymous access, this method should use it instead. + + auto ref = *input.getRef(); + + auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); + auto base_url = fmt("https://%s/%s/%s", + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")); + + Headers headers = makeHeadersWithAuthTokens(host); + + std::string ref_uri; + if (ref == "HEAD") { + auto file = store->toRealPath( + downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers).storePath); + std::ifstream is(file); + std::string line; + getline(is, line); + + auto ref_index = line.find("ref: "); + if (ref_index == std::string::npos) { + throw BadURL("in '%d', couldn't resolve HEAD ref '%d'", input.to_string(), ref); + } + + ref_uri = line.substr(ref_index+5, line.length()-1); + } else + ref_uri = fmt("refs/heads/%s", ref); + + auto file = store->toRealPath( + downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath); + std::ifstream is(file); + + std::string line; + std::string id; + while(getline(is, line)) { + auto index = line.find(ref_uri); + if (index != std::string::npos) { + id = line.substr(0, index-1); + break; + } + } + + if(id.empty()) + throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref); + + auto rev = Hash::parseAny(id, htSHA1); + debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev()); + return rev; + } + + DownloadUrl getDownloadUrl(const Input & input) const override + { + auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); + auto url = fmt("https://%s/%s/%s/archive/%s.tar.gz", + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + input.getRev()->to_string(Base16, false)); + + Headers headers = makeHeadersWithAuthTokens(host); + return DownloadUrl { url, headers }; + } + + void clone(const Input & input, const Path & destDir) override + { + auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); + Input::fromURL(fmt("git+https://%s/%s/%s", + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + .applyOverrides(input.getRef(), input.getRev()) + .clone(destDir); + } +}; + static auto rGitHubInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); static auto rGitLabInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto rSourceHutInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/nix/flake.md b/src/nix/flake.md index accddd436..f539a7c28 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -209,6 +209,38 @@ Currently the `type` attribute can be one of the following: * `github:edolstra/dwarffs/unstable` * `github:edolstra/dwarffs/d3f2baba8f425779026c6ec04021b2e927f61e31` +* `sourcehut`: Similar to `github`, is a more efficient way to fetch + SourceHut repositories. The following attributes are required: + + * `owner`: The owner of the repository (including leading `~`). + + * `repo`: The name of the repository. + + Like `github`, these are downloaded as tarball archives. + + The URL syntax for `sourcehut` flakes is: + + `sourcehut:/(/)?(\?)?` + + `` works the same as `github`. Either a branch or tag name + (`ref`), or a commit hash (`rev`) can be specified. + + Since SourceHut allows for self-hosting, you can specify `host` as + a parameter, to point to any instances other than `git.sr.ht`. + + Currently, `ref` name resolution only works for Git repositories. + You can refer to Mercurial repositories by simply changing `host` to + `hg.sr.ht` (or any other Mercurial instance). With the caveat + that you must explicitly specify a commit hash (`rev`). + + Some examples: + + * `sourcehut:~misterio/nix-colors` + * `sourcehut:~misterio/nix-colors/main` + * `sourcehut:~misterio/nix-colors?host=git.example.org` + * `sourcehut:~misterio/nix-colors/182b4b8709b8ffe4e9774a4c5d6877bf6bb9a21c` + * `sourcehut:~misterio/nix-colors/21c1a380a6915d890d408e9f22203436a35bb2de?host=hg.sr.ht` + * `indirect`: Indirections through the flake registry. These have the form From 770f3af31df552c47ac6bc38d386d10f8ff1c856 Mon Sep 17 00:00:00 2001 From: Gabriel Fontes Date: Wed, 23 Feb 2022 11:58:09 -0300 Subject: [PATCH 2/2] add sourcehut integration test --- flake.nix | 6 ++ tests/sourcehut-flakes.nix | 156 +++++++++++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 tests/sourcehut-flakes.nix diff --git a/flake.nix b/flake.nix index ae9305144..87b00edf4 100644 --- a/flake.nix +++ b/flake.nix @@ -501,6 +501,12 @@ inherit (self) overlay; }); + tests.sourcehutFlakes = (import ./tests/sourcehut-flakes.nix rec { + system = "x86_64-linux"; + inherit nixpkgs; + inherit (self) overlay; + }); + tests.setuid = nixpkgs.lib.genAttrs ["i686-linux" "x86_64-linux"] (system: diff --git a/tests/sourcehut-flakes.nix b/tests/sourcehut-flakes.nix new file mode 100644 index 000000000..d1d89d149 --- /dev/null +++ b/tests/sourcehut-flakes.nix @@ -0,0 +1,156 @@ +{ nixpkgs, system, overlay }: + +with import (nixpkgs + "/nixos/lib/testing-python.nix") +{ + inherit system; + extraConfigurations = [{ nixpkgs.overlays = [ overlay ]; }]; +}; + +let + # Generate a fake root CA and a fake git.sr.ht certificate. + cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; } + '' + mkdir -p $out + + openssl genrsa -out ca.key 2048 + openssl req -new -x509 -days 36500 -key ca.key \ + -subj "/C=NL/ST=Denial/L=Springfield/O=Dis/CN=Root CA" -out $out/ca.crt + + openssl req -newkey rsa:2048 -nodes -keyout $out/server.key \ + -subj "/C=CN/ST=Denial/L=Springfield/O=Dis/CN=git.sr.ht" -out server.csr + openssl x509 -req -extfile <(printf "subjectAltName=DNS:git.sr.ht") \ + -days 36500 -in server.csr -CA $out/ca.crt -CAkey ca.key -CAcreateserial -out $out/server.crt + ''; + + registry = pkgs.writeTextFile { + name = "registry"; + text = '' + { + "flakes": [ + { + "from": { + "type": "indirect", + "id": "nixpkgs" + }, + "to": { + "type": "sourcehut", + "owner": "~NixOS", + "repo": "nixpkgs" + } + } + ], + "version": 2 + } + ''; + destination = "/flake-registry.json"; + }; + + nixpkgs-repo = pkgs.runCommand "nixpkgs-flake" { } + '' + dir=NixOS-nixpkgs-${nixpkgs.shortRev} + cp -prd ${nixpkgs} $dir + + # Set the correct timestamp in the tarball. + find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- + + mkdir -p $out/archive + tar cfz $out/archive/${nixpkgs.rev}.tar.gz $dir --hard-dereference + + echo 'ref: refs/heads/master' > $out/HEAD + + mkdir -p $out/info + echo '${nixpkgs.rev} refs/heads/master' > $out/info/refs + ''; + +in + +makeTest ( + + { + name = "sourcehut-flakes"; + + nodes = + { + # Impersonate git.sr.ht + sourcehut = + { config, pkgs, ... }: + { + networking.firewall.allowedTCPPorts = [ 80 443 ]; + + services.httpd.enable = true; + services.httpd.adminAddr = "foo@example.org"; + services.httpd.extraConfig = '' + ErrorLog syslog:local6 + ''; + services.httpd.virtualHosts."git.sr.ht" = + { + forceSSL = true; + sslServerKey = "${cert}/server.key"; + sslServerCert = "${cert}/server.crt"; + servedDirs = + [ + { + urlPath = "/~NixOS/nixpkgs"; + dir = nixpkgs-repo; + } + { + urlPath = "/~NixOS/flake-registry/blob/master"; + dir = registry; + } + ]; + }; + }; + + client = + { config, lib, pkgs, nodes, ... }: + { + virtualisation.writableStore = true; + virtualisation.diskSize = 2048; + virtualisation.pathsInNixDB = [ pkgs.hello pkgs.fuse ]; + virtualisation.memorySize = 4096; + nix.binaryCaches = lib.mkForce [ ]; + nix.extraOptions = '' + experimental-features = nix-command flakes + flake-registry = https://git.sr.ht/~NixOS/flake-registry/blob/master/flake-registry.json + ''; + environment.systemPackages = [ pkgs.jq ]; + networking.hosts.${(builtins.head nodes.sourcehut.config.networking.interfaces.eth1.ipv4.addresses).address} = + [ "git.sr.ht" ]; + security.pki.certificateFiles = [ "${cert}/ca.crt" ]; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import json + import time + + start_all() + + sourcehut.wait_for_unit("httpd.service") + + client.succeed("curl -v https://git.sr.ht/ >&2") + client.succeed("nix registry list | grep nixpkgs") + + rev = client.succeed("nix flake info nixpkgs --json | jq -r .revision") + assert rev.strip() == "${nixpkgs.rev}", "revision mismatch" + + client.succeed("nix registry pin nixpkgs") + + client.succeed("nix flake info nixpkgs --tarball-ttl 0 >&2") + + # Shut down the web server. The flake should be cached on the client. + sourcehut.succeed("systemctl stop httpd.service") + + info = json.loads(client.succeed("nix flake info nixpkgs --json")) + date = time.strftime("%Y%m%d%H%M%S", time.gmtime(info['lastModified'])) + assert date == "${nixpkgs.lastModifiedDate}", "time mismatch" + + client.succeed("nix build nixpkgs#hello") + + # The build shouldn't fail even with --tarball-ttl 0 (the server + # being down should not be a fatal error). + client.succeed("nix build nixpkgs#fuse --tarball-ttl 0") + ''; + + })