forked from lix-project/lix
Merge branch 'access-tokens' of https://github.com/kquick/nix
This commit is contained in:
Eelco Dolstra
commit
002ce8449d
4 changed files with 149 additions and 52 deletions
|
|
@ -13,7 +13,12 @@ concatStrings (map
|
|||
then "*empty*"
|
||||
else if isBool option.value
|
||||
then (if option.value then "`true`" else "`false`")
|
||||
else "`" + toString option.value + "`") + "\n\n"
|
||||
else
|
||||
# n.b. a StringMap value type is specified as a string, but
|
||||
# this shows the value type. The empty stringmap is "null" in
|
||||
# JSON, but that converts to "{ }" here.
|
||||
(if isAttrs option.value then "`\"\"`"
|
||||
else "`" + toString option.value + "`")) + "\n\n"
|
||||
+ (if option.aliases != []
|
||||
then " **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n"
|
||||
else "")
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#include "types.hh"
|
||||
#include "url-parts.hh"
|
||||
|
||||
#include <optional>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
|
@ -13,7 +14,10 @@ namespace nix::fetchers {
|
|||
struct DownloadUrl
|
||||
{
|
||||
std::string url;
|
||||
std::optional<std::pair<std::string, std::string>> access_token_header;
|
||||
Headers headers;
|
||||
|
||||
DownloadUrl(const std::string & url, const Headers & headers)
|
||||
: url(url), headers(headers) { }
|
||||
};
|
||||
|
||||
// A github or gitlab host
|
||||
|
|
@ -24,7 +28,7 @@ struct GitArchiveInputScheme : InputScheme
|
|||
{
|
||||
virtual std::string type() = 0;
|
||||
|
||||
virtual std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const = 0;
|
||||
virtual std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const = 0;
|
||||
|
||||
std::optional<Input> inputFromURL(const ParsedURL & url) override
|
||||
{
|
||||
|
|
@ -139,6 +143,27 @@ struct GitArchiveInputScheme : InputScheme
|
|||
return input;
|
||||
}
|
||||
|
||||
std::optional<std::string> getAccessToken(const std::string &host) const {
|
||||
auto tokens = settings.accessTokens.get();
|
||||
auto pat = tokens.find(host);
|
||||
if (pat == tokens.end())
|
||||
return std::nullopt;
|
||||
return pat->second;
|
||||
}
|
||||
|
||||
Headers makeHeadersWithAuthTokens(const std::string & host) const {
|
||||
Headers headers;
|
||||
auto accessToken = getAccessToken(host);
|
||||
if (accessToken) {
|
||||
auto hdr = accessHeaderFromToken(*accessToken);
|
||||
if (hdr)
|
||||
headers.push_back(*hdr);
|
||||
else
|
||||
warn("Unrecognized access token for host '%s'", host);
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
|
||||
|
||||
virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
|
||||
|
|
@ -170,12 +195,7 @@ struct GitArchiveInputScheme : InputScheme
|
|||
|
||||
auto url = getDownloadUrl(input);
|
||||
|
||||
Headers headers;
|
||||
if (url.access_token_header) {
|
||||
headers.push_back(*url.access_token_header);
|
||||
}
|
||||
|
||||
auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, headers);
|
||||
auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
|
||||
|
||||
input.attrs.insert_or_assign("lastModified", lastModified);
|
||||
|
||||
|
|
@ -197,20 +217,23 @@ struct GitHubInputScheme : GitArchiveInputScheme
|
|||
{
|
||||
std::string type() override { return "github"; }
|
||||
|
||||
std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const {
|
||||
std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const {
|
||||
// Github supports PAT/OAuth2 tokens and HTTP Basic
|
||||
// Authentication. The former simply specifies the token, the
|
||||
// latter can use the token as the password. Only the first
|
||||
// is used here. See
|
||||
// https://developer.github.com/v3/#authentication and
|
||||
// https://docs.github.com/en/developers/apps/authorizing-oath-apps
|
||||
return std::pair<std::string, std::string>("Authorization", fmt("token %s", token));
|
||||
}
|
||||
|
||||
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
|
||||
{
|
||||
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
|
||||
auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
|
||||
auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
|
||||
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
|
||||
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
|
||||
|
||||
Headers headers;
|
||||
std::string accessToken = settings.githubAccessToken.get();
|
||||
if (accessToken != "")
|
||||
headers.push_back(accessHeaderFromToken(accessToken));
|
||||
Headers headers = makeHeadersWithAuthTokens(host);
|
||||
|
||||
auto json = nlohmann::json::parse(
|
||||
readFile(
|
||||
|
|
@ -225,25 +248,20 @@ struct GitHubInputScheme : GitArchiveInputScheme
|
|||
{
|
||||
// FIXME: use regular /archive URLs instead? api.github.com
|
||||
// might have stricter rate limits.
|
||||
auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
|
||||
auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
|
||||
auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
|
||||
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
|
||||
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
|
||||
input.getRev()->to_string(Base16, false));
|
||||
|
||||
std::string accessToken = settings.githubAccessToken.get();
|
||||
if (accessToken != "") {
|
||||
auto auth_header = accessHeaderFromToken(accessToken);
|
||||
return DownloadUrl { url, auth_header };
|
||||
} else {
|
||||
return DownloadUrl { url };
|
||||
}
|
||||
Headers headers = makeHeadersWithAuthTokens(host);
|
||||
return DownloadUrl(url, headers);
|
||||
}
|
||||
|
||||
void clone(const Input & input, const Path & destDir) override
|
||||
{
|
||||
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
|
||||
auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
|
||||
Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
|
||||
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
|
||||
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
|
||||
.applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
|
||||
.clone(destDir);
|
||||
}
|
||||
|
|
@ -253,20 +271,32 @@ struct GitLabInputScheme : GitArchiveInputScheme
|
|||
{
|
||||
std::string type() override { return "gitlab"; }
|
||||
|
||||
std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const {
|
||||
return std::pair<std::string, std::string>("Authorization", fmt("Bearer %s", token));
|
||||
std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const {
|
||||
// Gitlab supports 4 kinds of authorization, two of which are
|
||||
// relevant here: OAuth2 and PAT (Private Access Token). The
|
||||
// user can indicate which token is used by specifying the
|
||||
// token as <TYPE>:<VALUE>, where type is "OAuth2" or "PAT".
|
||||
// If the <TYPE> is unrecognized, this will fall back to
|
||||
// treating this simply has <HDRNAME>:<HDRVAL>. See
|
||||
// https://docs.gitlab.com/12.10/ee/api/README.html#authentication
|
||||
auto fldsplit = token.find_first_of(':');
|
||||
// n.b. C++20 would allow: if (token.starts_with("OAuth2:")) ...
|
||||
if ("OAuth2" == token.substr(0, fldsplit))
|
||||
return std::make_pair("Authorization", fmt("Bearer %s", token.substr(fldsplit+1)));
|
||||
if ("PAT" == token.substr(0, fldsplit))
|
||||
return std::make_pair("Private-token", token.substr(fldsplit+1));
|
||||
warn("Unrecognized GitLab token type %s", token.substr(0, fldsplit));
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
|
||||
{
|
||||
auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
|
||||
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
|
||||
// See rate limiting note below
|
||||
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
|
||||
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
|
||||
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
|
||||
|
||||
Headers headers;
|
||||
std::string accessToken = settings.gitlabAccessToken.get();
|
||||
if (accessToken != "")
|
||||
headers.push_back(accessHeaderFromToken(accessToken));
|
||||
Headers headers = makeHeadersWithAuthTokens(host);
|
||||
|
||||
auto json = nlohmann::json::parse(
|
||||
readFile(
|
||||
|
|
@ -279,28 +309,26 @@ struct GitLabInputScheme : GitArchiveInputScheme
|
|||
|
||||
DownloadUrl getDownloadUrl(const Input & input) const override
|
||||
{
|
||||
// FIXME: This endpoint has a rate limit threshold of 5 requests per minute
|
||||
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
|
||||
// This endpoint has a rate limit threshold that may be
|
||||
// server-specific and vary based whether the user is
|
||||
// authenticated via an accessToken or not, but the usual rate
|
||||
// is 10 reqs/sec/ip-addr. See
|
||||
// https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits
|
||||
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
|
||||
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s",
|
||||
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
|
||||
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
|
||||
input.getRev()->to_string(Base16, false));
|
||||
|
||||
std::string accessToken = settings.gitlabAccessToken.get();
|
||||
if (accessToken != "") {
|
||||
auto auth_header = accessHeaderFromToken(accessToken);
|
||||
return DownloadUrl { url, auth_header };
|
||||
} else {
|
||||
return DownloadUrl { url };
|
||||
}
|
||||
|
||||
Headers headers = makeHeadersWithAuthTokens(host);
|
||||
return DownloadUrl(url, headers);
|
||||
}
|
||||
|
||||
void clone(const Input & input, const Path & destDir) override
|
||||
{
|
||||
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
|
||||
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
|
||||
// FIXME: get username somewhere
|
||||
Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
|
||||
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
|
||||
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
|
||||
.applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
|
||||
.clone(destDir);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -859,11 +859,54 @@ public:
|
|||
are loaded as plugins (non-recursively).
|
||||
)"};
|
||||
|
||||
Setting<std::string> githubAccessToken{this, "", "github-access-token",
|
||||
"GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes."};
|
||||
Setting<StringMap> accessTokens{this, {}, "access-tokens",
|
||||
R"(
|
||||
Access tokens used to access protected GitHub, GitLab, or
|
||||
other locations requiring token-based authentication.
|
||||
|
||||
Setting<std::string> gitlabAccessToken{this, "", "gitlab-access-token",
|
||||
"GitLab access token to get access to GitLab data through the GitLab API for gitlab:<..> flakes."};
|
||||
Access tokens are specified as a string made up of
|
||||
space-separated `host=token` values. The specific token
|
||||
used is selected by matching the `host` portion against the
|
||||
"host" specification of the input. The actual use of the
|
||||
`token` value is determined by the type of resource being
|
||||
accessed:
|
||||
|
||||
* Github: the token value is the OAUTH-TOKEN string obtained
|
||||
as the Personal Access Token from the Github server (see
|
||||
https://docs.github.com/en/developers/apps/authorizing-oath-apps).
|
||||
|
||||
* Gitlab: the token value is either the OAuth2 token or the
|
||||
Personal Access Token (these are different types tokens
|
||||
for gitlab, see
|
||||
https://docs.gitlab.com/12.10/ee/api/README.html#authentication).
|
||||
The `token` value should be `type:tokenstring` where
|
||||
`type` is either `OAuth2` or `PAT` to indicate which type
|
||||
of token is being specified.
|
||||
|
||||
Example `~/.config/nix/nix.conf`:
|
||||
|
||||
```
|
||||
access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk"
|
||||
```
|
||||
|
||||
Example `~/code/flake.nix`:
|
||||
|
||||
```nix
|
||||
input.foo = {
|
||||
type="gitlab";
|
||||
host="gitlab.mycompany.com";
|
||||
owner="mycompany";
|
||||
repo="pro";
|
||||
};
|
||||
```
|
||||
|
||||
This example specifies three tokens, one each for accessing
|
||||
github.com, gitlab.mycompany.com, and sourceforge.net.
|
||||
|
||||
The `input.foo` uses the "gitlab" fetcher, which might
|
||||
requires specifying the token type along with the token
|
||||
value.
|
||||
)"};
|
||||
|
||||
Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
|
||||
"Experimental Nix features to enable."};
|
||||
|
|
|
|||
|
|
@ -268,6 +268,26 @@ template<> std::string BaseSetting<StringSet>::to_string() const
|
|||
return concatStringsSep(" ", value);
|
||||
}
|
||||
|
||||
template<> void BaseSetting<StringMap>::set(const std::string & str)
|
||||
{
|
||||
auto kvpairs = tokenizeString<Strings>(str);
|
||||
for (auto & s : kvpairs)
|
||||
{
|
||||
auto eq = s.find_first_of('=');
|
||||
if (std::string::npos != eq)
|
||||
value.emplace(std::string(s, 0, eq), std::string(s, eq + 1));
|
||||
// else ignored
|
||||
}
|
||||
}
|
||||
|
||||
template<> std::string BaseSetting<StringMap>::to_string() const
|
||||
{
|
||||
Strings kvstrs;
|
||||
std::transform(value.begin(), value.end(), back_inserter(kvstrs),
|
||||
[&](auto kvpair){ return kvpair.first + "=" + kvpair.second; });
|
||||
return concatStringsSep(" ", kvstrs);
|
||||
}
|
||||
|
||||
template class BaseSetting<int>;
|
||||
template class BaseSetting<unsigned int>;
|
||||
template class BaseSetting<long>;
|
||||
|
|
@ -278,6 +298,7 @@ template class BaseSetting<bool>;
|
|||
template class BaseSetting<std::string>;
|
||||
template class BaseSetting<Strings>;
|
||||
template class BaseSetting<StringSet>;
|
||||
template class BaseSetting<StringMap>;
|
||||
|
||||
void PathSetting::set(const std::string & str)
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in a new issue