From d9abce4ad4b6888183271c0a4051981dee5fffe3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Mon, 12 Sep 2022 22:50:18 +0200
Subject: [PATCH 1/3] libfetchers: avoid api.github.com ratelimit if no github
 token is set

If we don't have any github token, we won't be able to fetch private
repos, but we are also more likely to run into API limits since
we don't have a token. To mitigate this only ever use the github api
if we actually have a token.
---
 src/libfetchers/github.cc | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index a491d82a6..2115ce2f5 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -262,17 +262,20 @@ struct GitHubInputScheme : GitArchiveInputScheme
 
     DownloadUrl getDownloadUrl(const Input & input) const override
     {
-        // FIXME: use regular /archive URLs instead? api.github.com
-        // might have stricter rate limits.
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
-        auto url = fmt(
-            host == "github.com"
-            ? "https://api.%s/repos/%s/%s/tarball/%s"
-            : "https://%s/api/v3/repos/%s/%s/tarball/%s",
-            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+        Headers headers = makeHeadersWithAuthTokens(host);
+        // If we have no auth headers then we default to the public archive
+        // urls so we do not run into rate limits.
+        const auto urlFmt =
+            host != "github.com"
+            ? "https://%s/api/v3/repos/%s/%s/tarball/%s"
+            : headers.empty()
+            ? "https://%s/%s/%s/archive/%s.tar.gz"
+            : "https://api.%s/repos/%s/%s/tarball/%s";
+
+        const auto url = fmt(urlFmt, host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 
-        Headers headers = makeHeadersWithAuthTokens(host);
         return DownloadUrl { url, headers };
     }
 

From 9f1dd0df5b54a7dc75b618034482ed42ce34383d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= <sandro.jaeckel@gmail.com>
Date: Sat, 29 Oct 2022 21:51:29 +0200
Subject: [PATCH 2/3] Update test after api.github.com ratelimit avoidance

---
 tests/github-flakes.nix | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/tests/github-flakes.nix b/tests/github-flakes.nix
index fc481c7e3..1b60a9f9a 100644
--- a/tests/github-flakes.nix
+++ b/tests/github-flakes.nix
@@ -7,7 +7,7 @@ with import (nixpkgs + "/nixos/lib/testing-python.nix") {
 
 let
 
-  # Generate a fake root CA and a fake api.github.com / channels.nixos.org certificate.
+  # Generate a fake root CA and a fake api.github.com / github.com / channels.nixos.org certificate.
   cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; }
     ''
       mkdir -p $out
@@ -18,7 +18,7 @@ let
 
       openssl req -newkey rsa:2048 -nodes -keyout $out/server.key \
         -subj "/C=CN/ST=Denial/L=Springfield/O=Dis/CN=github.com" -out server.csr
-      openssl x509 -req -extfile <(printf "subjectAltName=DNS:api.github.com,DNS:channels.nixos.org") \
+      openssl x509 -req -extfile <(printf "subjectAltName=DNS:api.github.com,DNS:github.com,DNS:channels.nixos.org") \
         -days 36500 -in server.csr -CA $out/ca.crt -CAkey ca.key -CAcreateserial -out $out/server.crt
     '';
 
@@ -47,18 +47,20 @@ let
 
   api = pkgs.runCommand "nixpkgs-flake" {}
     ''
-      mkdir -p $out/tarball
+      mkdir -p $out/commits
+      echo '{"sha": "${nixpkgs.rev}"}' > $out/commits/HEAD
+    '';
+
+  archive = pkgs.runCommand "nixpkgs-flake" {}
+    ''
+      mkdir -p $out/archive
 
       dir=NixOS-nixpkgs-${nixpkgs.shortRev}
       cp -prd ${nixpkgs} $dir
       # Set the correct timestamp in the tarball.
       find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} --
-      tar cfz $out/tarball/${nixpkgs.rev} $dir --hard-dereference
-
-      mkdir -p $out/commits
-      echo '{"sha": "${nixpkgs.rev}"}' > $out/commits/HEAD
+      tar cfz $out/archive/${nixpkgs.rev}.tar.gz $dir --hard-dereference
     '';
-
 in
 
 makeTest (
@@ -97,6 +99,16 @@ makeTest (
                   }
                 ];
             };
+          services.httpd.virtualHosts."github.com" =
+            { forceSSL = true;
+              sslServerKey = "${cert}/server.key";
+              sslServerCert = "${cert}/server.crt";
+              servedDirs =
+                [ { urlPath = "/NixOS/nixpkgs";
+                    dir = archive;
+                  }
+                ];
+            };
         };
 
       client =
@@ -109,7 +121,7 @@ makeTest (
           nix.extraOptions = "experimental-features = nix-command flakes";
           environment.systemPackages = [ pkgs.jq ];
           networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} =
-            [ "channels.nixos.org" "api.github.com" ];
+            [ "channels.nixos.org" "api.github.com" "github.com" ];
           security.pki.certificateFiles = [ "${cert}/ca.crt" ];
         };
     };
@@ -123,7 +135,7 @@ makeTest (
 
     github.wait_for_unit("httpd.service")
 
-    client.succeed("curl -v https://api.github.com/ >&2")
+    client.succeed("curl -v https://github.com/ >&2")
     client.succeed("nix registry list | grep nixpkgs")
 
     rev = client.succeed("nix flake info nixpkgs --json | jq -r .revision")

From e00761af73f68fa7e6b833b4800a17e93f715097 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Sun, 30 Oct 2022 07:10:58 +0100
Subject: [PATCH 3/3] Also test github flakes when access tokens are provided

---
 tests/github-flakes.nix | 66 +++++++++++++++++++++++++++++++++++------
 1 file changed, 57 insertions(+), 9 deletions(-)

diff --git a/tests/github-flakes.nix b/tests/github-flakes.nix
index 1b60a9f9a..43a4f1432 100644
--- a/tests/github-flakes.nix
+++ b/tests/github-flakes.nix
@@ -37,6 +37,17 @@ let
               "owner": "NixOS",
               "repo": "nixpkgs"
             }
+          },
+          {
+            "from": {
+              "type": "indirect",
+              "id": "private-flake"
+            },
+            "to": {
+              "type": "github",
+              "owner": "fancy-enterprise",
+              "repo": "private-flake"
+            }
           }
         ],
         "version": 2
@@ -45,9 +56,27 @@ let
     destination = "/flake-registry.json";
   };
 
-  api = pkgs.runCommand "nixpkgs-flake" {}
+  private-flake-rev = "9f1dd0df5b54a7dc75b618034482ed42ce34383d";
+
+  private-flake-api = pkgs.runCommand "private-flake" {}
+    ''
+      mkdir -p $out/{commits,tarball}
+
+      # Setup https://docs.github.com/en/rest/commits/commits#get-a-commit
+      echo '{"sha": "${private-flake-rev}"}' > $out/commits/HEAD
+
+      # Setup tarball download via API
+      dir=private-flake
+      mkdir $dir
+      echo '{ outputs = {...}: {}; }' > $dir/flake.nix
+      tar cfz $out/tarball/${private-flake-rev} $dir --hard-dereference
+    '';
+
+  nixpkgs-api = pkgs.runCommand "nixpkgs-flake" {}
     ''
       mkdir -p $out/commits
+
+      # Setup https://docs.github.com/en/rest/commits/commits#get-a-commit
       echo '{"sha": "${nixpkgs.rev}"}' > $out/commits/HEAD
     '';
 
@@ -95,7 +124,10 @@ makeTest (
               sslServerCert = "${cert}/server.crt";
               servedDirs =
                 [ { urlPath = "/repos/NixOS/nixpkgs";
-                    dir = api;
+                    dir = nixpkgs-api;
+                  }
+                  { urlPath = "/repos/fancy-enterprise/private-flake";
+                    dir = private-flake-api;
                   }
                 ];
             };
@@ -119,7 +151,6 @@ makeTest (
           virtualisation.memorySize = 4096;
           nix.binaryCaches = lib.mkForce [ ];
           nix.extraOptions = "experimental-features = nix-command flakes";
-          environment.systemPackages = [ pkgs.jq ];
           networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} =
             [ "channels.nixos.org" "api.github.com" "github.com" ];
           security.pki.certificateFiles = [ "${cert}/ca.crt" ];
@@ -133,22 +164,39 @@ makeTest (
 
     start_all()
 
+    def cat_log():
+         github.succeed("cat /var/log/httpd/*.log >&2")
+
     github.wait_for_unit("httpd.service")
 
     client.succeed("curl -v https://github.com/ >&2")
-    client.succeed("nix registry list | grep nixpkgs")
+    out = client.succeed("nix registry list")
+    print(out)
+    assert "github:NixOS/nixpkgs" in out, "nixpkgs flake not found"
+    assert "github:fancy-enterprise/private-flake" in out, "private flake not found"
+    cat_log()
 
-    rev = client.succeed("nix flake info nixpkgs --json | jq -r .revision")
-    assert rev.strip() == "${nixpkgs.rev}", "revision mismatch"
+    # If no github access token is provided, nix should use the public archive url...
+    out = client.succeed("nix flake metadata nixpkgs --json")
+    print(out)
+    info = json.loads(out)
+    assert info["revision"] == "${nixpkgs.rev}", f"revision mismatch: {info['revision']} != ${nixpkgs.rev}"
+    cat_log()
+
+    # ... otherwise it should use the API
+    out = client.succeed("nix flake metadata private-flake --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0")
+    print(out)
+    info = json.loads(out)
+    assert info["revision"] == "${private-flake-rev}", f"revision mismatch: {info['revision']} != ${private-flake-rev}"
+    cat_log()
 
     client.succeed("nix registry pin nixpkgs")
-
-    client.succeed("nix flake info nixpkgs --tarball-ttl 0 >&2")
+    client.succeed("nix flake metadata nixpkgs --tarball-ttl 0 >&2")
 
     # Shut down the web server. The flake should be cached on the client.
     github.succeed("systemctl stop httpd.service")
 
-    info = json.loads(client.succeed("nix flake info nixpkgs --json"))
+    info = json.loads(client.succeed("nix flake metadata nixpkgs --json"))
     date = time.strftime("%Y%m%d%H%M%S", time.gmtime(info['lastModified']))
     assert date == "${nixpkgs.lastModifiedDate}", "time mismatch"