Merge pull request #7039 from Mic92/libfetchers-variant

libfetchers: avoid api.github.com ratelimit if no github token is set
This commit is contained in:
Théophane Hufschmitt 2022-10-31 11:48:51 +01:00 committed by GitHub
commit 90ed041677
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 90 additions and 27 deletions

View file

@ -262,17 +262,20 @@ struct GitHubInputScheme : GitArchiveInputScheme
DownloadUrl getDownloadUrl(const Input & input) const override DownloadUrl getDownloadUrl(const Input & input) const override
{ {
// FIXME: use regular /archive URLs instead? api.github.com
// might have stricter rate limits.
auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
auto url = fmt( Headers headers = makeHeadersWithAuthTokens(host);
host == "github.com" // If we have no auth headers then we default to the public archive
? "https://api.%s/repos/%s/%s/tarball/%s" // urls so we do not run into rate limits.
: "https://%s/api/v3/repos/%s/%s/tarball/%s", const auto urlFmt =
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), host != "github.com"
? "https://%s/api/v3/repos/%s/%s/tarball/%s"
: headers.empty()
? "https://%s/%s/%s/archive/%s.tar.gz"
: "https://api.%s/repos/%s/%s/tarball/%s";
const auto url = fmt(urlFmt, host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false)); input.getRev()->to_string(Base16, false));
Headers headers = makeHeadersWithAuthTokens(host);
return DownloadUrl { url, headers }; return DownloadUrl { url, headers };
} }

View file

@ -7,7 +7,7 @@ with import (nixpkgs + "/nixos/lib/testing-python.nix") {
let let
# Generate a fake root CA and a fake api.github.com / channels.nixos.org certificate. # Generate a fake root CA and a fake api.github.com / github.com / channels.nixos.org certificate.
cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; } cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; }
'' ''
mkdir -p $out mkdir -p $out
@ -18,7 +18,7 @@ let
openssl req -newkey rsa:2048 -nodes -keyout $out/server.key \ openssl req -newkey rsa:2048 -nodes -keyout $out/server.key \
-subj "/C=CN/ST=Denial/L=Springfield/O=Dis/CN=github.com" -out server.csr -subj "/C=CN/ST=Denial/L=Springfield/O=Dis/CN=github.com" -out server.csr
openssl x509 -req -extfile <(printf "subjectAltName=DNS:api.github.com,DNS:channels.nixos.org") \ openssl x509 -req -extfile <(printf "subjectAltName=DNS:api.github.com,DNS:github.com,DNS:channels.nixos.org") \
-days 36500 -in server.csr -CA $out/ca.crt -CAkey ca.key -CAcreateserial -out $out/server.crt -days 36500 -in server.csr -CA $out/ca.crt -CAkey ca.key -CAcreateserial -out $out/server.crt
''; '';
@ -37,6 +37,17 @@ let
"owner": "NixOS", "owner": "NixOS",
"repo": "nixpkgs" "repo": "nixpkgs"
} }
},
{
"from": {
"type": "indirect",
"id": "private-flake"
},
"to": {
"type": "github",
"owner": "fancy-enterprise",
"repo": "private-flake"
}
} }
], ],
"version": 2 "version": 2
@ -45,20 +56,40 @@ let
destination = "/flake-registry.json"; destination = "/flake-registry.json";
}; };
api = pkgs.runCommand "nixpkgs-flake" {} private-flake-rev = "9f1dd0df5b54a7dc75b618034482ed42ce34383d";
private-flake-api = pkgs.runCommand "private-flake" {}
'' ''
mkdir -p $out/tarball mkdir -p $out/{commits,tarball}
# Setup https://docs.github.com/en/rest/commits/commits#get-a-commit
echo '{"sha": "${private-flake-rev}"}' > $out/commits/HEAD
# Setup tarball download via API
dir=private-flake
mkdir $dir
echo '{ outputs = {...}: {}; }' > $dir/flake.nix
tar cfz $out/tarball/${private-flake-rev} $dir --hard-dereference
'';
nixpkgs-api = pkgs.runCommand "nixpkgs-flake" {}
''
mkdir -p $out/commits
# Setup https://docs.github.com/en/rest/commits/commits#get-a-commit
echo '{"sha": "${nixpkgs.rev}"}' > $out/commits/HEAD
'';
archive = pkgs.runCommand "nixpkgs-flake" {}
''
mkdir -p $out/archive
dir=NixOS-nixpkgs-${nixpkgs.shortRev} dir=NixOS-nixpkgs-${nixpkgs.shortRev}
cp -prd ${nixpkgs} $dir cp -prd ${nixpkgs} $dir
# Set the correct timestamp in the tarball. # Set the correct timestamp in the tarball.
find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} --
tar cfz $out/tarball/${nixpkgs.rev} $dir --hard-dereference tar cfz $out/archive/${nixpkgs.rev}.tar.gz $dir --hard-dereference
mkdir -p $out/commits
echo '{"sha": "${nixpkgs.rev}"}' > $out/commits/HEAD
''; '';
in in
makeTest ( makeTest (
@ -93,7 +124,20 @@ makeTest (
sslServerCert = "${cert}/server.crt"; sslServerCert = "${cert}/server.crt";
servedDirs = servedDirs =
[ { urlPath = "/repos/NixOS/nixpkgs"; [ { urlPath = "/repos/NixOS/nixpkgs";
dir = api; dir = nixpkgs-api;
}
{ urlPath = "/repos/fancy-enterprise/private-flake";
dir = private-flake-api;
}
];
};
services.httpd.virtualHosts."github.com" =
{ forceSSL = true;
sslServerKey = "${cert}/server.key";
sslServerCert = "${cert}/server.crt";
servedDirs =
[ { urlPath = "/NixOS/nixpkgs";
dir = archive;
} }
]; ];
}; };
@ -107,9 +151,8 @@ makeTest (
virtualisation.memorySize = 4096; virtualisation.memorySize = 4096;
nix.binaryCaches = lib.mkForce [ ]; nix.binaryCaches = lib.mkForce [ ];
nix.extraOptions = "experimental-features = nix-command flakes"; nix.extraOptions = "experimental-features = nix-command flakes";
environment.systemPackages = [ pkgs.jq ];
networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} = networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} =
[ "channels.nixos.org" "api.github.com" ]; [ "channels.nixos.org" "api.github.com" "github.com" ];
security.pki.certificateFiles = [ "${cert}/ca.crt" ]; security.pki.certificateFiles = [ "${cert}/ca.crt" ];
}; };
}; };
@ -121,22 +164,39 @@ makeTest (
start_all() start_all()
def cat_log():
github.succeed("cat /var/log/httpd/*.log >&2")
github.wait_for_unit("httpd.service") github.wait_for_unit("httpd.service")
client.succeed("curl -v https://api.github.com/ >&2") client.succeed("curl -v https://github.com/ >&2")
client.succeed("nix registry list | grep nixpkgs") out = client.succeed("nix registry list")
print(out)
assert "github:NixOS/nixpkgs" in out, "nixpkgs flake not found"
assert "github:fancy-enterprise/private-flake" in out, "private flake not found"
cat_log()
rev = client.succeed("nix flake info nixpkgs --json | jq -r .revision") # If no github access token is provided, nix should use the public archive url...
assert rev.strip() == "${nixpkgs.rev}", "revision mismatch" out = client.succeed("nix flake metadata nixpkgs --json")
print(out)
info = json.loads(out)
assert info["revision"] == "${nixpkgs.rev}", f"revision mismatch: {info['revision']} != ${nixpkgs.rev}"
cat_log()
# ... otherwise it should use the API
out = client.succeed("nix flake metadata private-flake --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0")
print(out)
info = json.loads(out)
assert info["revision"] == "${private-flake-rev}", f"revision mismatch: {info['revision']} != ${private-flake-rev}"
cat_log()
client.succeed("nix registry pin nixpkgs") client.succeed("nix registry pin nixpkgs")
client.succeed("nix flake metadata nixpkgs --tarball-ttl 0 >&2")
client.succeed("nix flake info nixpkgs --tarball-ttl 0 >&2")
# Shut down the web server. The flake should be cached on the client. # Shut down the web server. The flake should be cached on the client.
github.succeed("systemctl stop httpd.service") github.succeed("systemctl stop httpd.service")
info = json.loads(client.succeed("nix flake info nixpkgs --json")) info = json.loads(client.succeed("nix flake metadata nixpkgs --json"))
date = time.strftime("%Y%m%d%H%M%S", time.gmtime(info['lastModified'])) date = time.strftime("%Y%m%d%H%M%S", time.gmtime(info['lastModified']))
assert date == "${nixpkgs.lastModifiedDate}", "time mismatch" assert date == "${nixpkgs.lastModifiedDate}", "time mismatch"