lix-project/lix
18
45
Fork 24
Code Issues 315 Code Review (Gerrit) Projects 1 Releases Wiki Activity

Update to use access-tokens configuration for github/gitlab access.

Browse source 
This change provides support for using access tokens with other
instances of GitHub and GitLab beyond just github.com and
gitlab.com (especially company-specific or foundation-specific
instances).

This change also provides the ability to specify the type of access
token being used, where different types may have different handling,
based on the forge type.
This commit is contained in:
Kevin Quick 2020-09-24 22:49:44 -07:00
parent c2f48cfcee
commit 8fba2a8b54
No known key found for this signature in database
GPG key ID: E6D7733599CC0A21
2 changed files with 107 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

100
src/libfetchers/github.cc
View file

@ -5,6 +5,7 @@
#include "store-api.hh"
#include "types.hh"
#include <optional>
#include <nlohmann/json.hpp>
namespace nix::fetchers {
@ -12,13 +13,10 @@ namespace nix::fetchers {
struct DownloadUrl
{
std::string url;
std::optional<std::pair<std::string, std::string>> access_token_header;
Headers headers;
DownloadUrl(const std::string & url)
: url(url) { }
DownloadUrl(const std::string & url, const std::pair<std::string, std::string> & access_token_header)
: url(url), access_token_header(access_token_header) { }
DownloadUrl(const std::string & url, const Headers & headers)
: url(url), headers(headers) { }
};
// A github or gitlab url
@ -29,7 +27,7 @@ struct GitArchiveInputScheme : InputScheme
{
virtual std::string type() = 0;
virtual std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const = 0;
virtual std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const = 0;
std::optional<Input> inputFromURL(const ParsedURL & url) override
{
@ -144,6 +142,27 @@ struct GitArchiveInputScheme : InputScheme
return input;
}
std::optional<std::string> getAccessToken(const std::string &host) const {
auto tokens = settings.accessTokens.get();
auto pat = tokens.find(host);
if (pat == tokens.end())
return std::nullopt;
return pat->second;
}
Headers makeHeadersWithAuthTokens(const std::string & host) const {
Headers headers;
auto accessToken = getAccessToken(host);
if (accessToken) {
auto hdr = accessHeaderFromToken(*accessToken);
if (hdr)
headers.push_back(*hdr);
else
warn("Unrecognized access token for host '%s'", host);
}
return headers;
}
virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
@ -175,12 +194,7 @@ struct GitArchiveInputScheme : InputScheme
auto url = getDownloadUrl(input);
Headers headers;
if (url.access_token_header) {
headers.push_back(*url.access_token_header);
}
auto [tree, lastModified] = downloadTarball(store, url.url, headers, "source", true);
auto [tree, lastModified] = downloadTarball(store, url.url, url.headers, "source", true);
input.attrs.insert_or_assign("lastModified", lastModified);
@ -202,7 +216,13 @@ struct GitHubInputScheme : GitArchiveInputScheme
{
std::string type() override { return "github"; }
std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const {
std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const {
// Github supports PAT/OAuth2 tokens and HTTP Basic
// Authentication. The former simply specifies the token, the
// latter can use the token as the password. Only the first
// is used here. See
// https://developer.github.com/v3/#authentication and
// https://docs.github.com/en/developers/apps/authorizing-oath-apps
return std::pair<std::string, std::string>("Authorization", fmt("token %s", token));
}
@ -212,10 +232,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
Headers headers;
std::string accessToken = settings.githubAccessToken.get();
if (accessToken != "")
headers.push_back(accessHeaderFromToken(accessToken));
Headers headers = makeHeadersWithAuthTokens(host);
auto json = nlohmann::json::parse(
readFile(
@ -235,13 +252,8 @@ struct GitHubInputScheme : GitArchiveInputScheme
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false));
std::string accessToken = settings.githubAccessToken.get();
if (accessToken != "") {
auto auth_header = accessHeaderFromToken(accessToken);
return DownloadUrl(url, auth_header);
} else {
return DownloadUrl(url);
}
Headers headers = makeHeadersWithAuthTokens(host);
return DownloadUrl(url, headers);
}
void clone(const Input & input, const Path & destDir) override
@ -258,20 +270,32 @@ struct GitLabInputScheme : GitArchiveInputScheme
{
std::string type() override { return "gitlab"; }
std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const {
return std::pair<std::string, std::string>("Authorization", fmt("Bearer %s", token));
std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const {
// Gitlab supports 4 kinds of authorization, two of which are
// relevant here: OAuth2 and PAT (Private Access Token). The
// user can indicate which token is used by specifying the
// token as <TYPE>:<VALUE>, where type is "OAuth2" or "PAT".
// If the <TYPE> is unrecognized, this will fall back to
// treating this simply has <HDRNAME>:<HDRVAL>. See
// https://docs.gitlab.com/12.10/ee/api/README.html#authentication
auto fldsplit = token.find_first_of(':');
// n.b. C++20 would allow: if (token.starts_with("OAuth2:")) ...
if ("OAuth2" == token.substr(0, fldsplit))
return std::make_pair("Authorization", fmt("Bearer %s", token.substr(fldsplit+1)));
if ("PAT" == token.substr(0, fldsplit))
return std::make_pair("Private-token", token.substr(fldsplit+1));
warn("Unrecognized GitLab token type %s", token.substr(0, fldsplit));
return std::nullopt;
}
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// See rate limiting note below
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
Headers headers;
std::string accessToken = settings.gitlabAccessToken.get();
if (accessToken != "")
headers.push_back(accessHeaderFromToken(accessToken));
Headers headers = makeHeadersWithAuthTokens(host);
auto json = nlohmann::json::parse(
readFile(
@ -294,14 +318,8 @@ struct GitLabInputScheme : GitArchiveInputScheme
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false));
std::string accessToken = settings.gitlabAccessToken.get();
if (accessToken != "") {
auto auth_header = accessHeaderFromToken(accessToken);
return DownloadUrl(url, auth_header);
} else {
return DownloadUrl(url);
}
Headers headers = makeHeadersWithAuthTokens(host);
return DownloadUrl(url, headers);
}
void clone(const Input & input, const Path & destDir) override

50
src/libstore/globals.hh
View file

@ -863,8 +863,54 @@ public:
Setting<std::string> githubAccessToken{this, "", "github-access-token",
"GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes."};
Setting<std::string> gitlabAccessToken{this, "", "gitlab-access-token",
"GitLab access token to get access to GitLab data through the GitLab API for gitlab:<..> flakes."};
Setting<StringMap> accessTokens{this, {}, "access-tokens",
R"(
Access tokens used to access protected GitHub, GitLab, or
other locations requiring token-based authentication.
Access tokens are specified as a string made up of
space-separated `host=token` values. The specific token
used is selected by matching the `host` portion against the
"host" specification of the input. The actual use of the
`token` value is determined by the type of resource being
accessed:
* Github: the token value is the OAUTH-TOKEN string obtained
as the Personal Access Token from the Github server (see
https://docs.github.com/en/developers/apps/authorizing-oath-apps).
* Gitlab: the token value is either the OAuth2 token or the
Personal Access Token (these are different types tokens
for gitlab, see
https://docs.gitlab.com/12.10/ee/api/README.html#authentication).
The `token` value should be `type:tokenstring` where
`type` is either `OAuth2` or `PAT` to indicate which type
of token is being specified.
Example `~/.config/nix/nix.conf`:
```
personal-access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk"
```
Example `~/code/flake.nix`:
```nix
input.foo = {
type="gitlab";
host="gitlab.mycompany.com";
owner="mycompany";
repo="pro";
};
```
This example specifies three tokens, one each for accessing
github.com, gitlab.mycompany.com, and sourceforge.net.
The `input.foo` uses the "gitlab" fetcher, which might
requires specifying the token type along with the token
value.
)"};
Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
"Experimental Nix features to enable."};