Merge branch 'access-tokens' of https://github.com/kquick/nix

This commit is contained in:
Eelco Dolstra 2020-09-30 11:35:15 +02:00
commit 002ce8449d
4 changed files with 149 additions and 52 deletions

View file

@ -13,7 +13,12 @@ concatStrings (map
then "*empty*" then "*empty*"
else if isBool option.value else if isBool option.value
then (if option.value then "`true`" else "`false`") then (if option.value then "`true`" else "`false`")
else "`" + toString option.value + "`") + "\n\n" else
# n.b. a StringMap value type is specified as a string, but
# this shows the value type. The empty stringmap is "null" in
# JSON, but that converts to "{ }" here.
(if isAttrs option.value then "`\"\"`"
else "`" + toString option.value + "`")) + "\n\n"
+ (if option.aliases != [] + (if option.aliases != []
then " **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n" then " **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n"
else "") else "")

View file

@ -6,6 +6,7 @@
#include "types.hh" #include "types.hh"
#include "url-parts.hh" #include "url-parts.hh"
#include <optional>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
namespace nix::fetchers { namespace nix::fetchers {
@ -13,7 +14,10 @@ namespace nix::fetchers {
struct DownloadUrl struct DownloadUrl
{ {
std::string url; std::string url;
std::optional<std::pair<std::string, std::string>> access_token_header; Headers headers;
DownloadUrl(const std::string & url, const Headers & headers)
: url(url), headers(headers) { }
}; };
// A github or gitlab host // A github or gitlab host
@ -24,7 +28,7 @@ struct GitArchiveInputScheme : InputScheme
{ {
virtual std::string type() = 0; virtual std::string type() = 0;
virtual std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const = 0; virtual std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const = 0;
std::optional<Input> inputFromURL(const ParsedURL & url) override std::optional<Input> inputFromURL(const ParsedURL & url) override
{ {
@ -139,6 +143,27 @@ struct GitArchiveInputScheme : InputScheme
return input; return input;
} }
std::optional<std::string> getAccessToken(const std::string &host) const {
auto tokens = settings.accessTokens.get();
auto pat = tokens.find(host);
if (pat == tokens.end())
return std::nullopt;
return pat->second;
}
Headers makeHeadersWithAuthTokens(const std::string & host) const {
Headers headers;
auto accessToken = getAccessToken(host);
if (accessToken) {
auto hdr = accessHeaderFromToken(*accessToken);
if (hdr)
headers.push_back(*hdr);
else
warn("Unrecognized access token for host '%s'", host);
}
return headers;
}
virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0; virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
virtual DownloadUrl getDownloadUrl(const Input & input) const = 0; virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
@ -170,12 +195,7 @@ struct GitArchiveInputScheme : InputScheme
auto url = getDownloadUrl(input); auto url = getDownloadUrl(input);
Headers headers; auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
if (url.access_token_header) {
headers.push_back(*url.access_token_header);
}
auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, headers);
input.attrs.insert_or_assign("lastModified", lastModified); input.attrs.insert_or_assign("lastModified", lastModified);
@ -197,20 +217,23 @@ struct GitHubInputScheme : GitArchiveInputScheme
{ {
std::string type() override { return "github"; } std::string type() override { return "github"; }
std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const { std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const {
// Github supports PAT/OAuth2 tokens and HTTP Basic
// Authentication. The former simply specifies the token, the
// latter can use the token as the password. Only the first
// is used here. See
// https://developer.github.com/v3/#authentication and
// https://docs.github.com/en/developers/apps/authorizing-oath-apps
return std::pair<std::string, std::string>("Authorization", fmt("token %s", token)); return std::pair<std::string, std::string>("Authorization", fmt("token %s", token));
} }
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{ {
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
Headers headers; Headers headers = makeHeadersWithAuthTokens(host);
std::string accessToken = settings.githubAccessToken.get();
if (accessToken != "")
headers.push_back(accessHeaderFromToken(accessToken));
auto json = nlohmann::json::parse( auto json = nlohmann::json::parse(
readFile( readFile(
@ -225,25 +248,20 @@ struct GitHubInputScheme : GitArchiveInputScheme
{ {
// FIXME: use regular /archive URLs instead? api.github.com // FIXME: use regular /archive URLs instead? api.github.com
// might have stricter rate limits. // might have stricter rate limits.
auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false)); input.getRev()->to_string(Base16, false));
std::string accessToken = settings.githubAccessToken.get(); Headers headers = makeHeadersWithAuthTokens(host);
if (accessToken != "") { return DownloadUrl(url, headers);
auto auth_header = accessHeaderFromToken(accessToken);
return DownloadUrl { url, auth_header };
} else {
return DownloadUrl { url };
}
} }
void clone(const Input & input, const Path & destDir) override void clone(const Input & input, const Path & destDir) override
{ {
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git", Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef().value_or("HEAD"), input.getRev()) .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
.clone(destDir); .clone(destDir);
} }
@ -253,20 +271,32 @@ struct GitLabInputScheme : GitArchiveInputScheme
{ {
std::string type() override { return "gitlab"; } std::string type() override { return "gitlab"; }
std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const { std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const {
return std::pair<std::string, std::string>("Authorization", fmt("Bearer %s", token)); // Gitlab supports 4 kinds of authorization, two of which are
// relevant here: OAuth2 and PAT (Private Access Token). The
// user can indicate which token is used by specifying the
// token as <TYPE>:<VALUE>, where type is "OAuth2" or "PAT".
// If the <TYPE> is unrecognized, this will fall back to
// treating this simply has <HDRNAME>:<HDRVAL>. See
// https://docs.gitlab.com/12.10/ee/api/README.html#authentication
auto fldsplit = token.find_first_of(':');
// n.b. C++20 would allow: if (token.starts_with("OAuth2:")) ...
if ("OAuth2" == token.substr(0, fldsplit))
return std::make_pair("Authorization", fmt("Bearer %s", token.substr(fldsplit+1)));
if ("PAT" == token.substr(0, fldsplit))
return std::make_pair("Private-token", token.substr(fldsplit+1));
warn("Unrecognized GitLab token type %s", token.substr(0, fldsplit));
return std::nullopt;
} }
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{ {
auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// See rate limiting note below
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s", auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
Headers headers; Headers headers = makeHeadersWithAuthTokens(host);
std::string accessToken = settings.gitlabAccessToken.get();
if (accessToken != "")
headers.push_back(accessHeaderFromToken(accessToken));
auto json = nlohmann::json::parse( auto json = nlohmann::json::parse(
readFile( readFile(
@ -279,28 +309,26 @@ struct GitLabInputScheme : GitArchiveInputScheme
DownloadUrl getDownloadUrl(const Input & input) const override DownloadUrl getDownloadUrl(const Input & input) const override
{ {
// FIXME: This endpoint has a rate limit threshold of 5 requests per minute // This endpoint has a rate limit threshold that may be
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); // server-specific and vary based whether the user is
// authenticated via an accessToken or not, but the usual rate
// is 10 reqs/sec/ip-addr. See
// https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s",
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false)); input.getRev()->to_string(Base16, false));
std::string accessToken = settings.gitlabAccessToken.get(); Headers headers = makeHeadersWithAuthTokens(host);
if (accessToken != "") { return DownloadUrl(url, headers);
auto auth_header = accessHeaderFromToken(accessToken);
return DownloadUrl { url, auth_header };
} else {
return DownloadUrl { url };
}
} }
void clone(const Input & input, const Path & destDir) override void clone(const Input & input, const Path & destDir) override
{ {
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// FIXME: get username somewhere // FIXME: get username somewhere
Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git", Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef().value_or("HEAD"), input.getRev()) .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
.clone(destDir); .clone(destDir);
} }

View file

@ -859,11 +859,54 @@ public:
are loaded as plugins (non-recursively). are loaded as plugins (non-recursively).
)"}; )"};
Setting<std::string> githubAccessToken{this, "", "github-access-token", Setting<StringMap> accessTokens{this, {}, "access-tokens",
"GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes."}; R"(
Access tokens used to access protected GitHub, GitLab, or
other locations requiring token-based authentication.
Setting<std::string> gitlabAccessToken{this, "", "gitlab-access-token", Access tokens are specified as a string made up of
"GitLab access token to get access to GitLab data through the GitLab API for gitlab:<..> flakes."}; space-separated `host=token` values. The specific token
used is selected by matching the `host` portion against the
"host" specification of the input. The actual use of the
`token` value is determined by the type of resource being
accessed:
* Github: the token value is the OAUTH-TOKEN string obtained
as the Personal Access Token from the Github server (see
https://docs.github.com/en/developers/apps/authorizing-oath-apps).
* Gitlab: the token value is either the OAuth2 token or the
Personal Access Token (these are different types tokens
for gitlab, see
https://docs.gitlab.com/12.10/ee/api/README.html#authentication).
The `token` value should be `type:tokenstring` where
`type` is either `OAuth2` or `PAT` to indicate which type
of token is being specified.
Example `~/.config/nix/nix.conf`:
```
access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk"
```
Example `~/code/flake.nix`:
```nix
input.foo = {
type="gitlab";
host="gitlab.mycompany.com";
owner="mycompany";
repo="pro";
};
```
This example specifies three tokens, one each for accessing
github.com, gitlab.mycompany.com, and sourceforge.net.
The `input.foo` uses the "gitlab" fetcher, which might
requires specifying the token type along with the token
value.
)"};
Setting<Strings> experimentalFeatures{this, {}, "experimental-features", Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
"Experimental Nix features to enable."}; "Experimental Nix features to enable."};

View file

@ -268,6 +268,26 @@ template<> std::string BaseSetting<StringSet>::to_string() const
return concatStringsSep(" ", value); return concatStringsSep(" ", value);
} }
template<> void BaseSetting<StringMap>::set(const std::string & str)
{
auto kvpairs = tokenizeString<Strings>(str);
for (auto & s : kvpairs)
{
auto eq = s.find_first_of('=');
if (std::string::npos != eq)
value.emplace(std::string(s, 0, eq), std::string(s, eq + 1));
// else ignored
}
}
template<> std::string BaseSetting<StringMap>::to_string() const
{
Strings kvstrs;
std::transform(value.begin(), value.end(), back_inserter(kvstrs),
[&](auto kvpair){ return kvpair.first + "=" + kvpair.second; });
return concatStringsSep(" ", kvstrs);
}
template class BaseSetting<int>; template class BaseSetting<int>;
template class BaseSetting<unsigned int>; template class BaseSetting<unsigned int>;
template class BaseSetting<long>; template class BaseSetting<long>;
@ -278,6 +298,7 @@ template class BaseSetting<bool>;
template class BaseSetting<std::string>; template class BaseSetting<std::string>;
template class BaseSetting<Strings>; template class BaseSetting<Strings>;
template class BaseSetting<StringSet>; template class BaseSetting<StringSet>;
template class BaseSetting<StringMap>;
void PathSetting::set(const std::string & str) void PathSetting::set(const std::string & str)
{ {