From 8fba2a8b54283ea1cf56ae75faf4ced5f3e8e4a1 Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Thu, 24 Sep 2020 22:49:44 -0700 Subject: [PATCH] Update to use access-tokens configuration for github/gitlab access. This change provides support for using access tokens with other instances of GitHub and GitLab beyond just github.com and gitlab.com (especially company-specific or foundation-specific instances). This change also provides the ability to specify the type of access token being used, where different types may have different handling, based on the forge type. --- src/libfetchers/github.cc | 100 ++++++++++++++++++++++---------------- src/libstore/globals.hh | 50 ++++++++++++++++++- 2 files changed, 107 insertions(+), 43 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index eb758bc5f..0e0655367 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -5,6 +5,7 @@ #include "store-api.hh" #include "types.hh" +#include #include namespace nix::fetchers { @@ -12,13 +13,10 @@ namespace nix::fetchers { struct DownloadUrl { std::string url; - std::optional> access_token_header; + Headers headers; - DownloadUrl(const std::string & url) - : url(url) { } - - DownloadUrl(const std::string & url, const std::pair & access_token_header) - : url(url), access_token_header(access_token_header) { } + DownloadUrl(const std::string & url, const Headers & headers) + : url(url), headers(headers) { } }; // A github or gitlab url @@ -29,7 +27,7 @@ struct GitArchiveInputScheme : InputScheme { virtual std::string type() = 0; - virtual std::pair accessHeaderFromToken(const std::string & token) const = 0; + virtual std::optional > accessHeaderFromToken(const std::string & token) const = 0; std::optional inputFromURL(const ParsedURL & url) override { @@ -144,6 +142,27 @@ struct GitArchiveInputScheme : InputScheme return input; } + std::optional getAccessToken(const std::string &host) const { + auto tokens = settings.accessTokens.get(); + auto pat = tokens.find(host); + if (pat == tokens.end()) + return std::nullopt; + return pat->second; + } + + Headers makeHeadersWithAuthTokens(const std::string & host) const { + Headers headers; + auto accessToken = getAccessToken(host); + if (accessToken) { + auto hdr = accessHeaderFromToken(*accessToken); + if (hdr) + headers.push_back(*hdr); + else + warn("Unrecognized access token for host '%s'", host); + } + return headers; + } + virtual Hash getRevFromRef(nix::ref store, const Input & input) const = 0; virtual DownloadUrl getDownloadUrl(const Input & input) const = 0; @@ -175,12 +194,7 @@ struct GitArchiveInputScheme : InputScheme auto url = getDownloadUrl(input); - Headers headers; - if (url.access_token_header) { - headers.push_back(*url.access_token_header); - } - - auto [tree, lastModified] = downloadTarball(store, url.url, headers, "source", true); + auto [tree, lastModified] = downloadTarball(store, url.url, url.headers, "source", true); input.attrs.insert_or_assign("lastModified", lastModified); @@ -202,7 +216,13 @@ struct GitHubInputScheme : GitArchiveInputScheme { std::string type() override { return "github"; } - std::pair accessHeaderFromToken(const std::string & token) const { + std::optional > accessHeaderFromToken(const std::string & token) const { + // Github supports PAT/OAuth2 tokens and HTTP Basic + // Authentication. The former simply specifies the token, the + // latter can use the token as the password. Only the first + // is used here. See + // https://developer.github.com/v3/#authentication and + // https://docs.github.com/en/developers/apps/authorizing-oath-apps return std::pair("Authorization", fmt("token %s", token)); } @@ -212,10 +232,7 @@ struct GitHubInputScheme : GitArchiveInputScheme auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); - Headers headers; - std::string accessToken = settings.githubAccessToken.get(); - if (accessToken != "") - headers.push_back(accessHeaderFromToken(accessToken)); + Headers headers = makeHeadersWithAuthTokens(host); auto json = nlohmann::json::parse( readFile( @@ -235,13 +252,8 @@ struct GitHubInputScheme : GitArchiveInputScheme host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); - std::string accessToken = settings.githubAccessToken.get(); - if (accessToken != "") { - auto auth_header = accessHeaderFromToken(accessToken); - return DownloadUrl(url, auth_header); - } else { - return DownloadUrl(url); - } + Headers headers = makeHeadersWithAuthTokens(host); + return DownloadUrl(url, headers); } void clone(const Input & input, const Path & destDir) override @@ -258,20 +270,32 @@ struct GitLabInputScheme : GitArchiveInputScheme { std::string type() override { return "gitlab"; } - std::pair accessHeaderFromToken(const std::string & token) const { - return std::pair("Authorization", fmt("Bearer %s", token)); + std::optional > accessHeaderFromToken(const std::string & token) const { + // Gitlab supports 4 kinds of authorization, two of which are + // relevant here: OAuth2 and PAT (Private Access Token). The + // user can indicate which token is used by specifying the + // token as :, where type is "OAuth2" or "PAT". + // If the is unrecognized, this will fall back to + // treating this simply has :. See + // https://docs.gitlab.com/12.10/ee/api/README.html#authentication + auto fldsplit = token.find_first_of(':'); + // n.b. C++20 would allow: if (token.starts_with("OAuth2:")) ... + if ("OAuth2" == token.substr(0, fldsplit)) + return std::make_pair("Authorization", fmt("Bearer %s", token.substr(fldsplit+1))); + if ("PAT" == token.substr(0, fldsplit)) + return std::make_pair("Private-token", token.substr(fldsplit+1)); + warn("Unrecognized GitLab token type %s", token.substr(0, fldsplit)); + return std::nullopt; } Hash getRevFromRef(nix::ref store, const Input & input) const override { - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); + auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); + // See rate limiting note below auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s", - host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); - Headers headers; - std::string accessToken = settings.gitlabAccessToken.get(); - if (accessToken != "") - headers.push_back(accessHeaderFromToken(accessToken)); + Headers headers = makeHeadersWithAuthTokens(host); auto json = nlohmann::json::parse( readFile( @@ -294,14 +318,8 @@ struct GitLabInputScheme : GitArchiveInputScheme host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); - std::string accessToken = settings.gitlabAccessToken.get(); - if (accessToken != "") { - auto auth_header = accessHeaderFromToken(accessToken); - return DownloadUrl(url, auth_header); - } else { - return DownloadUrl(url); - } - + Headers headers = makeHeadersWithAuthTokens(host); + return DownloadUrl(url, headers); } void clone(const Input & input, const Path & destDir) override diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index b2e7610ee..646422399 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -863,8 +863,54 @@ public: Setting githubAccessToken{this, "", "github-access-token", "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes."}; - Setting gitlabAccessToken{this, "", "gitlab-access-token", - "GitLab access token to get access to GitLab data through the GitLab API for gitlab:<..> flakes."}; + Setting accessTokens{this, {}, "access-tokens", + R"( + Access tokens used to access protected GitHub, GitLab, or + other locations requiring token-based authentication. + + Access tokens are specified as a string made up of + space-separated `host=token` values. The specific token + used is selected by matching the `host` portion against the + "host" specification of the input. The actual use of the + `token` value is determined by the type of resource being + accessed: + + * Github: the token value is the OAUTH-TOKEN string obtained + as the Personal Access Token from the Github server (see + https://docs.github.com/en/developers/apps/authorizing-oath-apps). + + * Gitlab: the token value is either the OAuth2 token or the + Personal Access Token (these are different types tokens + for gitlab, see + https://docs.gitlab.com/12.10/ee/api/README.html#authentication). + The `token` value should be `type:tokenstring` where + `type` is either `OAuth2` or `PAT` to indicate which type + of token is being specified. + + Example `~/.config/nix/nix.conf`: + + ``` + personal-access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk" + ``` + + Example `~/code/flake.nix`: + + ```nix + input.foo = { + type="gitlab"; + host="gitlab.mycompany.com"; + owner="mycompany"; + repo="pro"; + }; + ``` + + This example specifies three tokens, one each for accessing + github.com, gitlab.mycompany.com, and sourceforge.net. + + The `input.foo` uses the "gitlab" fetcher, which might + requires specifying the token type along with the token + value. + )"}; Setting experimentalFeatures{this, {}, "experimental-features", "Experimental Nix features to enable."};