Merge pull request #5342 from Misterio77/add-sourcehut

Add support for sourcehut input scheme
This commit is contained in:
Théophane Hufschmitt 2022-02-24 09:09:21 +01:00 committed by GitHub
commit 6524eb4b77
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 284 additions and 1 deletions

View file

@ -501,6 +501,12 @@
inherit (self) overlay;
});
tests.sourcehutFlakes = (import ./tests/sourcehut-flakes.nix rec {
system = "x86_64-linux";
inherit nixpkgs;
inherit (self) overlay;
});
tests.setuid = nixpkgs.lib.genAttrs
["i686-linux" "x86_64-linux"]
(system:

View file

@ -8,6 +8,7 @@
#include <optional>
#include <nlohmann/json.hpp>
#include <fstream>
namespace nix::fetchers {
@ -17,7 +18,7 @@ struct DownloadUrl
Headers headers;
};
// A github or gitlab host
// A github, gitlab, or sourcehut host
const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
std::regex hostRegex(hostRegexS, std::regex::ECMAScript);
@ -345,7 +346,95 @@ struct GitLabInputScheme : GitArchiveInputScheme
}
};
struct SourceHutInputScheme : GitArchiveInputScheme
{
std::string type() override { return "sourcehut"; }
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{
// SourceHut supports both PAT and OAuth2. See
// https://man.sr.ht/meta.sr.ht/oauth.md
return std::pair<std::string, std::string>("Authorization", fmt("Bearer %s", token));
// Note: This currently serves no purpose, as this kind of authorization
// does not allow for downloading tarballs on sourcehut private repos.
// Once it is implemented, however, should work as expected.
}
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
// TODO: In the future, when the sourcehut graphql API is implemented for mercurial
// and with anonymous access, this method should use it instead.
auto ref = *input.getRef();
auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
auto base_url = fmt("https://%s/%s/%s",
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"));
Headers headers = makeHeadersWithAuthTokens(host);
std::string ref_uri;
if (ref == "HEAD") {
auto file = store->toRealPath(
downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers).storePath);
std::ifstream is(file);
std::string line;
getline(is, line);
auto ref_index = line.find("ref: ");
if (ref_index == std::string::npos) {
throw BadURL("in '%d', couldn't resolve HEAD ref '%d'", input.to_string(), ref);
}
ref_uri = line.substr(ref_index+5, line.length()-1);
} else
ref_uri = fmt("refs/heads/%s", ref);
auto file = store->toRealPath(
downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath);
std::ifstream is(file);
std::string line;
std::string id;
while(getline(is, line)) {
auto index = line.find(ref_uri);
if (index != std::string::npos) {
id = line.substr(0, index-1);
break;
}
}
if(id.empty())
throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref);
auto rev = Hash::parseAny(id, htSHA1);
debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev());
return rev;
}
DownloadUrl getDownloadUrl(const Input & input) const override
{
auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
auto url = fmt("https://%s/%s/%s/archive/%s.tar.gz",
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false));
Headers headers = makeHeadersWithAuthTokens(host);
return DownloadUrl { url, headers };
}
void clone(const Input & input, const Path & destDir) override
{
auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
Input::fromURL(fmt("git+https://%s/%s/%s",
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef(), input.getRev())
.clone(destDir);
}
};
static auto rGitHubInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
static auto rGitLabInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitLabInputScheme>()); });
static auto rSourceHutInputScheme = OnStartup([] { registerInputScheme(std::make_unique<SourceHutInputScheme>()); });
}

View file

@ -209,6 +209,38 @@ Currently the `type` attribute can be one of the following:
* `github:edolstra/dwarffs/unstable`
* `github:edolstra/dwarffs/d3f2baba8f425779026c6ec04021b2e927f61e31`
* `sourcehut`: Similar to `github`, is a more efficient way to fetch
SourceHut repositories. The following attributes are required:
* `owner`: The owner of the repository (including leading `~`).
* `repo`: The name of the repository.
Like `github`, these are downloaded as tarball archives.
The URL syntax for `sourcehut` flakes is:
`sourcehut:<owner>/<repo>(/<rev-or-ref>)?(\?<params>)?`
`<rev-or-ref>` works the same as `github`. Either a branch or tag name
(`ref`), or a commit hash (`rev`) can be specified.
Since SourceHut allows for self-hosting, you can specify `host` as
a parameter, to point to any instances other than `git.sr.ht`.
Currently, `ref` name resolution only works for Git repositories.
You can refer to Mercurial repositories by simply changing `host` to
`hg.sr.ht` (or any other Mercurial instance). With the caveat
that you must explicitly specify a commit hash (`rev`).
Some examples:
* `sourcehut:~misterio/nix-colors`
* `sourcehut:~misterio/nix-colors/main`
* `sourcehut:~misterio/nix-colors?host=git.example.org`
* `sourcehut:~misterio/nix-colors/182b4b8709b8ffe4e9774a4c5d6877bf6bb9a21c`
* `sourcehut:~misterio/nix-colors/21c1a380a6915d890d408e9f22203436a35bb2de?host=hg.sr.ht`
* `indirect`: Indirections through the flake registry. These have the
form

156
tests/sourcehut-flakes.nix Normal file
View file

@ -0,0 +1,156 @@
{ nixpkgs, system, overlay }:
with import (nixpkgs + "/nixos/lib/testing-python.nix")
{
inherit system;
extraConfigurations = [{ nixpkgs.overlays = [ overlay ]; }];
};
let
# Generate a fake root CA and a fake git.sr.ht certificate.
cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; }
''
mkdir -p $out
openssl genrsa -out ca.key 2048
openssl req -new -x509 -days 36500 -key ca.key \
-subj "/C=NL/ST=Denial/L=Springfield/O=Dis/CN=Root CA" -out $out/ca.crt
openssl req -newkey rsa:2048 -nodes -keyout $out/server.key \
-subj "/C=CN/ST=Denial/L=Springfield/O=Dis/CN=git.sr.ht" -out server.csr
openssl x509 -req -extfile <(printf "subjectAltName=DNS:git.sr.ht") \
-days 36500 -in server.csr -CA $out/ca.crt -CAkey ca.key -CAcreateserial -out $out/server.crt
'';
registry = pkgs.writeTextFile {
name = "registry";
text = ''
{
"flakes": [
{
"from": {
"type": "indirect",
"id": "nixpkgs"
},
"to": {
"type": "sourcehut",
"owner": "~NixOS",
"repo": "nixpkgs"
}
}
],
"version": 2
}
'';
destination = "/flake-registry.json";
};
nixpkgs-repo = pkgs.runCommand "nixpkgs-flake" { }
''
dir=NixOS-nixpkgs-${nixpkgs.shortRev}
cp -prd ${nixpkgs} $dir
# Set the correct timestamp in the tarball.
find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} --
mkdir -p $out/archive
tar cfz $out/archive/${nixpkgs.rev}.tar.gz $dir --hard-dereference
echo 'ref: refs/heads/master' > $out/HEAD
mkdir -p $out/info
echo '${nixpkgs.rev} refs/heads/master' > $out/info/refs
'';
in
makeTest (
{
name = "sourcehut-flakes";
nodes =
{
# Impersonate git.sr.ht
sourcehut =
{ config, pkgs, ... }:
{
networking.firewall.allowedTCPPorts = [ 80 443 ];
services.httpd.enable = true;
services.httpd.adminAddr = "foo@example.org";
services.httpd.extraConfig = ''
ErrorLog syslog:local6
'';
services.httpd.virtualHosts."git.sr.ht" =
{
forceSSL = true;
sslServerKey = "${cert}/server.key";
sslServerCert = "${cert}/server.crt";
servedDirs =
[
{
urlPath = "/~NixOS/nixpkgs";
dir = nixpkgs-repo;
}
{
urlPath = "/~NixOS/flake-registry/blob/master";
dir = registry;
}
];
};
};
client =
{ config, lib, pkgs, nodes, ... }:
{
virtualisation.writableStore = true;
virtualisation.diskSize = 2048;
virtualisation.pathsInNixDB = [ pkgs.hello pkgs.fuse ];
virtualisation.memorySize = 4096;
nix.binaryCaches = lib.mkForce [ ];
nix.extraOptions = ''
experimental-features = nix-command flakes
flake-registry = https://git.sr.ht/~NixOS/flake-registry/blob/master/flake-registry.json
'';
environment.systemPackages = [ pkgs.jq ];
networking.hosts.${(builtins.head nodes.sourcehut.config.networking.interfaces.eth1.ipv4.addresses).address} =
[ "git.sr.ht" ];
security.pki.certificateFiles = [ "${cert}/ca.crt" ];
};
};
testScript = { nodes }: ''
# fmt: off
import json
import time
start_all()
sourcehut.wait_for_unit("httpd.service")
client.succeed("curl -v https://git.sr.ht/ >&2")
client.succeed("nix registry list | grep nixpkgs")
rev = client.succeed("nix flake info nixpkgs --json | jq -r .revision")
assert rev.strip() == "${nixpkgs.rev}", "revision mismatch"
client.succeed("nix registry pin nixpkgs")
client.succeed("nix flake info nixpkgs --tarball-ttl 0 >&2")
# Shut down the web server. The flake should be cached on the client.
sourcehut.succeed("systemctl stop httpd.service")
info = json.loads(client.succeed("nix flake info nixpkgs --json"))
date = time.strftime("%Y%m%d%H%M%S", time.gmtime(info['lastModified']))
assert date == "${nixpkgs.lastModifiedDate}", "time mismatch"
client.succeed("nix build nixpkgs#hello")
# The build shouldn't fail even with --tarball-ttl 0 (the server
# being down should not be a fatal error).
client.succeed("nix build nixpkgs#fuse --tarball-ttl 0")
'';
})