Merge branch 'access-tokens' of https://github.com/kquick/nix

doc/manual/generate-options.nix

@@ -13,7 +13,12 @@ concatStrings (map
       then "*empty*"
       else if isBool option.value
       then (if option.value then "`true`" else "`false`")
-      else "`" + toString option.value + "`") + "\n\n"
+      else
+        # n.b. a StringMap value type is specified as a string, but
+        # this shows the value type.  The empty stringmap is "null" in
+        # JSON, but that converts to "{ }" here.
+        (if isAttrs option.value then "`\"\"`"
+         else "`" + toString option.value + "`")) + "\n\n"
     + (if option.aliases != []
        then "    **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n"
        else "")

src/libfetchers/github.cc

@@ -6,6 +6,7 @@
 #include "types.hh"
 #include "url-parts.hh"
 
+#include <optional>
 #include <nlohmann/json.hpp>
 
 namespace nix::fetchers {
@@ -13,7 +14,10 @@ namespace nix::fetchers {
 struct DownloadUrl
 {
     std::string url;
-    std::optional<std::pair<std::string, std::string>> access_token_header;
+    Headers headers;
+
+    DownloadUrl(const std::string & url, const Headers & headers)
+        : url(url), headers(headers) { }
 };
 
 // A github or gitlab host
@@ -24,7 +28,7 @@ struct GitArchiveInputScheme : InputScheme
 {
     virtual std::string type() = 0;
 
-    virtual std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const = 0;
+    virtual std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const = 0;
 
     std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
@@ -139,6 +143,27 @@ struct GitArchiveInputScheme : InputScheme
         return input;
     }
 
+    std::optional<std::string> getAccessToken(const std::string &host) const {
+        auto tokens = settings.accessTokens.get();
+        auto pat = tokens.find(host);
+        if (pat == tokens.end())
+            return std::nullopt;
+        return pat->second;
+    }
+
+    Headers makeHeadersWithAuthTokens(const std::string & host) const {
+        Headers headers;
+        auto accessToken = getAccessToken(host);
+        if (accessToken) {
+            auto hdr = accessHeaderFromToken(*accessToken);
+            if (hdr)
+                headers.push_back(*hdr);
+            else
+                warn("Unrecognized access token for host '%s'", host);
+        }
+        return headers;
+    }
+
     virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
 
     virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
@@ -170,12 +195,7 @@ struct GitArchiveInputScheme : InputScheme
 
         auto url = getDownloadUrl(input);
 
-        Headers headers;
+        auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
-        if (url.access_token_header) {
-            headers.push_back(*url.access_token_header);
-        }
-
-        auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, headers);
 
         input.attrs.insert_or_assign("lastModified", lastModified);
 
@@ -197,20 +217,23 @@ struct GitHubInputScheme : GitArchiveInputScheme
 {
     std::string type() override { return "github"; }
 
-    std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const {
+    std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const {
+        // Github supports PAT/OAuth2 tokens and HTTP Basic
+        // Authentication.  The former simply specifies the token, the
+        // latter can use the token as the password.  Only the first
+        // is used here. See
+        // https://developer.github.com/v3/#authentication and
+        // https://docs.github.com/en/developers/apps/authorizing-oath-apps
         return std::pair<std::string, std::string>("Authorization", fmt("token %s", token));
     }
 
     Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
     {
-        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
         auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
-            host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
 
-        Headers headers;
+        Headers headers = makeHeadersWithAuthTokens(host);
-        std::string accessToken = settings.githubAccessToken.get();
-        if (accessToken != "")
-            headers.push_back(accessHeaderFromToken(accessToken));
 
         auto json = nlohmann::json::parse(
             readFile(
@@ -225,25 +248,20 @@ struct GitHubInputScheme : GitArchiveInputScheme
     {
         // FIXME: use regular /archive URLs instead? api.github.com
         // might have stricter rate limits.
-        auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
         auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
-            host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 
-        std::string accessToken = settings.githubAccessToken.get();
+        Headers headers = makeHeadersWithAuthTokens(host);
-        if (accessToken != "") {
+        return DownloadUrl(url, headers);
-            auto auth_header = accessHeaderFromToken(accessToken);
-            return DownloadUrl { url, auth_header };
-        } else {
-            return DownloadUrl { url };
-        }
     }
 
     void clone(const Input & input, const Path & destDir) override
     {
-        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
         Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
-                host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
+                host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
             .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
             .clone(destDir);
     }
@@ -253,20 +271,32 @@ struct GitLabInputScheme : GitArchiveInputScheme
 {
     std::string type() override { return "gitlab"; }
 
-    std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const {
+    std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const {
-        return std::pair<std::string, std::string>("Authorization", fmt("Bearer %s", token));
+        // Gitlab supports 4 kinds of authorization, two of which are
+        // relevant here: OAuth2 and PAT (Private Access Token).  The
+        // user can indicate which token is used by specifying the
+        // token as <TYPE>:<VALUE>, where type is "OAuth2" or "PAT".
+        // If the <TYPE> is unrecognized, this will fall back to
+        // treating this simply has <HDRNAME>:<HDRVAL>.  See
+        // https://docs.gitlab.com/12.10/ee/api/README.html#authentication
+        auto fldsplit = token.find_first_of(':');
+        // n.b. C++20 would allow: if (token.starts_with("OAuth2:")) ...
+        if ("OAuth2" == token.substr(0, fldsplit))
+            return std::make_pair("Authorization", fmt("Bearer %s", token.substr(fldsplit+1)));
+        if ("PAT" == token.substr(0, fldsplit))
+            return std::make_pair("Private-token", token.substr(fldsplit+1));
+        warn("Unrecognized GitLab token type %s",  token.substr(0, fldsplit));
+        return std::nullopt;
     }
 
     Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
     {
-        auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
+        // See rate limiting note below
         auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
-            host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
 
-        Headers headers;
+        Headers headers = makeHeadersWithAuthTokens(host);
-        std::string accessToken = settings.gitlabAccessToken.get();
-        if (accessToken != "")
-            headers.push_back(accessHeaderFromToken(accessToken));
 
         auto json = nlohmann::json::parse(
             readFile(
@@ -279,28 +309,26 @@ struct GitLabInputScheme : GitArchiveInputScheme
 
     DownloadUrl getDownloadUrl(const Input & input) const override
     {
-        // FIXME: This endpoint has a rate limit threshold of 5 requests per minute
+        // This endpoint has a rate limit threshold that may be
-        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+        // server-specific and vary based whether the user is
+        // authenticated via an accessToken or not, but the usual rate
+        // is 10 reqs/sec/ip-addr.  See
+        // https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
         auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s",
-            host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 
-        std::string accessToken = settings.gitlabAccessToken.get();
+        Headers headers = makeHeadersWithAuthTokens(host);
-        if (accessToken != "") {
+        return DownloadUrl(url, headers);
-            auto auth_header = accessHeaderFromToken(accessToken);
-            return DownloadUrl { url, auth_header };
-        } else {
-            return DownloadUrl { url };
-        }
-
     }
 
     void clone(const Input & input, const Path & destDir) override
     {
-        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
         // FIXME: get username somewhere
         Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
-                host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
+                host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
             .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
             .clone(destDir);
     }

src/libstore/globals.hh

@@ -859,11 +859,54 @@ public:
           are loaded as plugins (non-recursively).
         )"};
 
-    Setting<std::string> githubAccessToken{this, "", "github-access-token",
+    Setting<StringMap> accessTokens{this, {}, "access-tokens",
-        "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes."};
+        R"(
+          Access tokens used to access protected GitHub, GitLab, or
+          other locations requiring token-based authentication.
 
-    Setting<std::string> gitlabAccessToken{this, "", "gitlab-access-token",
+          Access tokens are specified as a string made up of
-        "GitLab access token to get access to GitLab data through the GitLab API for gitlab:<..> flakes."};
+          space-separated `host=token` values.  The specific token
+          used is selected by matching the `host` portion against the
+          "host" specification of the input. The actual use of the
+          `token` value is determined by the type of resource being
+          accessed:
+
+          * Github: the token value is the OAUTH-TOKEN string obtained
+            as the Personal Access Token from the Github server (see
+            https://docs.github.com/en/developers/apps/authorizing-oath-apps).
+
+          * Gitlab: the token value is either the OAuth2 token or the
+            Personal Access Token (these are different types tokens
+            for gitlab, see
+            https://docs.gitlab.com/12.10/ee/api/README.html#authentication).
+            The `token` value should be `type:tokenstring` where
+            `type` is either `OAuth2` or `PAT` to indicate which type
+            of token is being specified.
+
+          Example `~/.config/nix/nix.conf`:
+
+          ```
+          access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk"
+          ```
+
+          Example `~/code/flake.nix`:
+
+          ```nix
+          input.foo = {
+            type="gitlab";
+            host="gitlab.mycompany.com";
+            owner="mycompany";
+            repo="pro";
+          };
+          ```
+
+          This example specifies three tokens, one each for accessing
+          github.com, gitlab.mycompany.com, and sourceforge.net.
+
+          The `input.foo` uses the "gitlab" fetcher, which might
+          requires specifying the token type along with the token
+          value.
+          )"};
 
     Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
         "Experimental Nix features to enable."};

src/libutil/config.cc

@@ -268,6 +268,26 @@ template<> std::string BaseSetting<StringSet>::to_string() const
     return concatStringsSep(" ", value);
 }
 
+template<> void BaseSetting<StringMap>::set(const std::string & str)
+{
+    auto kvpairs = tokenizeString<Strings>(str);
+    for (auto & s : kvpairs)
+    {
+        auto eq = s.find_first_of('=');
+        if (std::string::npos != eq)
+            value.emplace(std::string(s, 0, eq), std::string(s, eq + 1));
+        // else ignored
+    }
+}
+
+template<> std::string BaseSetting<StringMap>::to_string() const
+{
+    Strings kvstrs;
+    std::transform(value.begin(), value.end(), back_inserter(kvstrs),
+        [&](auto kvpair){ return kvpair.first + "=" + kvpair.second; });
+    return concatStringsSep(" ", kvstrs);
+}
+
 template class BaseSetting<int>;
 template class BaseSetting<unsigned int>;
 template class BaseSetting<long>;
@@ -278,6 +298,7 @@ template class BaseSetting<bool>;
 template class BaseSetting<std::string>;
 template class BaseSetting<Strings>;
 template class BaseSetting<StringSet>;
+template class BaseSetting<StringMap>;
 
 void PathSetting::set(const std::string & str)
 {