From 83fec38fc93922192ada7c0409fec76578ef8dfb Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Thu, 24 Sep 2020 22:41:24 -0700 Subject: [PATCH 1/7] Update document generation for empty json object values. --- doc/manual/generate-options.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/manual/generate-options.nix b/doc/manual/generate-options.nix index 7afe279c3..3c31a4eec 100644 --- a/doc/manual/generate-options.nix +++ b/doc/manual/generate-options.nix @@ -13,7 +13,12 @@ concatStrings (map then "*empty*" else if isBool option.value then (if option.value then "`true`" else "`false`") - else "`" + toString option.value + "`") + "\n\n" + else + # n.b. a StringMap value type is specified as a string, but + # this shows the value type. The empty stringmap is "null" in + # JSON, but that converts to "{ }" here. + (if isAttrs option.value then "`\"\"`" + else "`" + toString option.value + "`")) + "\n\n" + (if option.aliases != [] then " **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n" else "") From a439e9488df6c13d0e44dd4816df98487d69f4c6 Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Thu, 24 Sep 2020 22:42:59 -0700 Subject: [PATCH 2/7] Support StringMap configuration settings. Allows Configuration values that are space-separated key=value pairs. --- src/libutil/config.cc | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 309d23b40..b14feb10d 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -268,6 +268,26 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } +template<> void BaseSetting::set(const std::string & str) +{ + auto kvpairs = tokenizeString(str); + for (auto & s : kvpairs) + { + auto eq = s.find_first_of('='); + if (std::string::npos != eq) + value.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); + // else ignored + } +} + +template<> std::string BaseSetting::to_string() const +{ + Strings kvstrs; + std::transform(value.begin(), value.end(), back_inserter(kvstrs), + [&](auto kvpair){ return kvpair.first + "=" + kvpair.second; }); + return concatStringsSep(" ", kvstrs); +} + template class BaseSetting; template class BaseSetting; template class BaseSetting; @@ -278,6 +298,7 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting; template class BaseSetting; +template class BaseSetting; void PathSetting::set(const std::string & str) { From c2f48cfcee501dd15690245d481d154444456f66 Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Thu, 24 Sep 2020 22:46:03 -0700 Subject: [PATCH 3/7] Complete conversion of "url" to "host" with associated variable renaming. Completes the change begun in commit 56f1e0d to consistently use the "host" attribute for "github" and "gitlab" inputs instead of a "url" attribute. --- src/libfetchers/github.cc | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index d8d0351b9..eb758bc5f 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -65,9 +65,9 @@ struct GitArchiveInputScheme : InputScheme throw BadURL("URL '%s' contains multiple branch/tag names", url.url); ref = value; } - else if (name == "url") { + else if (name == "host") { if (!std::regex_match(value, urlRegex)) - throw BadURL("URL '%s' contains an invalid instance url", url.url); + throw BadURL("URL '%s' contains an invalid instance host", url.url); host_url = value; } // FIXME: barf on unsupported attributes @@ -82,7 +82,7 @@ struct GitArchiveInputScheme : InputScheme input.attrs.insert_or_assign("repo", path[1]); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); if (ref) input.attrs.insert_or_assign("ref", *ref); - if (host_url) input.attrs.insert_or_assign("url", *host_url); + if (host_url) input.attrs.insert_or_assign("host", *host_url); return input; } @@ -92,7 +92,7 @@ struct GitArchiveInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != type()) return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified") + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host") throw Error("unsupported input attribute '%s'", name); getStrAttr(attrs, "owner"); @@ -208,9 +208,9 @@ struct GitHubInputScheme : GitArchiveInputScheme Hash getRevFromRef(nix::ref store, const Input & input) const override { - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com"); + auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check - host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); Headers headers; std::string accessToken = settings.githubAccessToken.get(); @@ -230,9 +230,9 @@ struct GitHubInputScheme : GitArchiveInputScheme { // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com"); + auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances - host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); std::string accessToken = settings.githubAccessToken.get(); @@ -246,9 +246,9 @@ struct GitHubInputScheme : GitArchiveInputScheme void clone(const Input & input, const Path & destDir) override { - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com"); + auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git", - host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef().value_or("HEAD"), input.getRev()) .clone(destDir); } @@ -284,10 +284,14 @@ struct GitLabInputScheme : GitArchiveInputScheme DownloadUrl getDownloadUrl(const Input & input) const override { - // FIXME: This endpoint has a rate limit threshold of 5 requests per minute - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); + // This endpoint has a rate limit threshold that may be + // server-specific and vary based whether the user is + // authenticated via an accessToken or not, but the usual rate + // is 10 reqs/sec/ip-addr. See + // https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits + auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", - host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); std::string accessToken = settings.gitlabAccessToken.get(); @@ -302,10 +306,10 @@ struct GitLabInputScheme : GitArchiveInputScheme void clone(const Input & input, const Path & destDir) override { - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); + auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); // FIXME: get username somewhere Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git", - host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef().value_or("HEAD"), input.getRev()) .clone(destDir); } From 8fba2a8b54283ea1cf56ae75faf4ced5f3e8e4a1 Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Thu, 24 Sep 2020 22:49:44 -0700 Subject: [PATCH 4/7] Update to use access-tokens configuration for github/gitlab access. This change provides support for using access tokens with other instances of GitHub and GitLab beyond just github.com and gitlab.com (especially company-specific or foundation-specific instances). This change also provides the ability to specify the type of access token being used, where different types may have different handling, based on the forge type. --- src/libfetchers/github.cc | 100 ++++++++++++++++++++++---------------- src/libstore/globals.hh | 50 ++++++++++++++++++- 2 files changed, 107 insertions(+), 43 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index eb758bc5f..0e0655367 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -5,6 +5,7 @@ #include "store-api.hh" #include "types.hh" +#include #include namespace nix::fetchers { @@ -12,13 +13,10 @@ namespace nix::fetchers { struct DownloadUrl { std::string url; - std::optional> access_token_header; + Headers headers; - DownloadUrl(const std::string & url) - : url(url) { } - - DownloadUrl(const std::string & url, const std::pair & access_token_header) - : url(url), access_token_header(access_token_header) { } + DownloadUrl(const std::string & url, const Headers & headers) + : url(url), headers(headers) { } }; // A github or gitlab url @@ -29,7 +27,7 @@ struct GitArchiveInputScheme : InputScheme { virtual std::string type() = 0; - virtual std::pair accessHeaderFromToken(const std::string & token) const = 0; + virtual std::optional > accessHeaderFromToken(const std::string & token) const = 0; std::optional inputFromURL(const ParsedURL & url) override { @@ -144,6 +142,27 @@ struct GitArchiveInputScheme : InputScheme return input; } + std::optional getAccessToken(const std::string &host) const { + auto tokens = settings.accessTokens.get(); + auto pat = tokens.find(host); + if (pat == tokens.end()) + return std::nullopt; + return pat->second; + } + + Headers makeHeadersWithAuthTokens(const std::string & host) const { + Headers headers; + auto accessToken = getAccessToken(host); + if (accessToken) { + auto hdr = accessHeaderFromToken(*accessToken); + if (hdr) + headers.push_back(*hdr); + else + warn("Unrecognized access token for host '%s'", host); + } + return headers; + } + virtual Hash getRevFromRef(nix::ref store, const Input & input) const = 0; virtual DownloadUrl getDownloadUrl(const Input & input) const = 0; @@ -175,12 +194,7 @@ struct GitArchiveInputScheme : InputScheme auto url = getDownloadUrl(input); - Headers headers; - if (url.access_token_header) { - headers.push_back(*url.access_token_header); - } - - auto [tree, lastModified] = downloadTarball(store, url.url, headers, "source", true); + auto [tree, lastModified] = downloadTarball(store, url.url, url.headers, "source", true); input.attrs.insert_or_assign("lastModified", lastModified); @@ -202,7 +216,13 @@ struct GitHubInputScheme : GitArchiveInputScheme { std::string type() override { return "github"; } - std::pair accessHeaderFromToken(const std::string & token) const { + std::optional > accessHeaderFromToken(const std::string & token) const { + // Github supports PAT/OAuth2 tokens and HTTP Basic + // Authentication. The former simply specifies the token, the + // latter can use the token as the password. Only the first + // is used here. See + // https://developer.github.com/v3/#authentication and + // https://docs.github.com/en/developers/apps/authorizing-oath-apps return std::pair("Authorization", fmt("token %s", token)); } @@ -212,10 +232,7 @@ struct GitHubInputScheme : GitArchiveInputScheme auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); - Headers headers; - std::string accessToken = settings.githubAccessToken.get(); - if (accessToken != "") - headers.push_back(accessHeaderFromToken(accessToken)); + Headers headers = makeHeadersWithAuthTokens(host); auto json = nlohmann::json::parse( readFile( @@ -235,13 +252,8 @@ struct GitHubInputScheme : GitArchiveInputScheme host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); - std::string accessToken = settings.githubAccessToken.get(); - if (accessToken != "") { - auto auth_header = accessHeaderFromToken(accessToken); - return DownloadUrl(url, auth_header); - } else { - return DownloadUrl(url); - } + Headers headers = makeHeadersWithAuthTokens(host); + return DownloadUrl(url, headers); } void clone(const Input & input, const Path & destDir) override @@ -258,20 +270,32 @@ struct GitLabInputScheme : GitArchiveInputScheme { std::string type() override { return "gitlab"; } - std::pair accessHeaderFromToken(const std::string & token) const { - return std::pair("Authorization", fmt("Bearer %s", token)); + std::optional > accessHeaderFromToken(const std::string & token) const { + // Gitlab supports 4 kinds of authorization, two of which are + // relevant here: OAuth2 and PAT (Private Access Token). The + // user can indicate which token is used by specifying the + // token as :, where type is "OAuth2" or "PAT". + // If the is unrecognized, this will fall back to + // treating this simply has :. See + // https://docs.gitlab.com/12.10/ee/api/README.html#authentication + auto fldsplit = token.find_first_of(':'); + // n.b. C++20 would allow: if (token.starts_with("OAuth2:")) ... + if ("OAuth2" == token.substr(0, fldsplit)) + return std::make_pair("Authorization", fmt("Bearer %s", token.substr(fldsplit+1))); + if ("PAT" == token.substr(0, fldsplit)) + return std::make_pair("Private-token", token.substr(fldsplit+1)); + warn("Unrecognized GitLab token type %s", token.substr(0, fldsplit)); + return std::nullopt; } Hash getRevFromRef(nix::ref store, const Input & input) const override { - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); + auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); + // See rate limiting note below auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s", - host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); + host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); - Headers headers; - std::string accessToken = settings.gitlabAccessToken.get(); - if (accessToken != "") - headers.push_back(accessHeaderFromToken(accessToken)); + Headers headers = makeHeadersWithAuthTokens(host); auto json = nlohmann::json::parse( readFile( @@ -294,14 +318,8 @@ struct GitLabInputScheme : GitArchiveInputScheme host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); - std::string accessToken = settings.gitlabAccessToken.get(); - if (accessToken != "") { - auto auth_header = accessHeaderFromToken(accessToken); - return DownloadUrl(url, auth_header); - } else { - return DownloadUrl(url); - } - + Headers headers = makeHeadersWithAuthTokens(host); + return DownloadUrl(url, headers); } void clone(const Input & input, const Path & destDir) override diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index b2e7610ee..646422399 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -863,8 +863,54 @@ public: Setting githubAccessToken{this, "", "github-access-token", "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes."}; - Setting gitlabAccessToken{this, "", "gitlab-access-token", - "GitLab access token to get access to GitLab data through the GitLab API for gitlab:<..> flakes."}; + Setting accessTokens{this, {}, "access-tokens", + R"( + Access tokens used to access protected GitHub, GitLab, or + other locations requiring token-based authentication. + + Access tokens are specified as a string made up of + space-separated `host=token` values. The specific token + used is selected by matching the `host` portion against the + "host" specification of the input. The actual use of the + `token` value is determined by the type of resource being + accessed: + + * Github: the token value is the OAUTH-TOKEN string obtained + as the Personal Access Token from the Github server (see + https://docs.github.com/en/developers/apps/authorizing-oath-apps). + + * Gitlab: the token value is either the OAuth2 token or the + Personal Access Token (these are different types tokens + for gitlab, see + https://docs.gitlab.com/12.10/ee/api/README.html#authentication). + The `token` value should be `type:tokenstring` where + `type` is either `OAuth2` or `PAT` to indicate which type + of token is being specified. + + Example `~/.config/nix/nix.conf`: + + ``` + personal-access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk" + ``` + + Example `~/code/flake.nix`: + + ```nix + input.foo = { + type="gitlab"; + host="gitlab.mycompany.com"; + owner="mycompany"; + repo="pro"; + }; + ``` + + This example specifies three tokens, one each for accessing + github.com, gitlab.mycompany.com, and sourceforge.net. + + The `input.foo` uses the "gitlab" fetcher, which might + requires specifying the token type along with the token + value. + )"}; Setting experimentalFeatures{this, {}, "experimental-features", "Experimental Nix features to enable."}; From ef2a14be190f7162e85e9bdd44dd45bd9ddfe391 Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Fri, 25 Sep 2020 08:08:27 -0700 Subject: [PATCH 5/7] Fix reference to older name for access-tokens config value. --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 646422399..959ebe360 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -890,7 +890,7 @@ public: Example `~/.config/nix/nix.conf`: ``` - personal-access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk" + access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk" ``` Example `~/code/flake.nix`: From 5a35cc29bffc88b88f883dfcdd1bb251eab53ecd Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Fri, 25 Sep 2020 08:09:56 -0700 Subject: [PATCH 6/7] Re-add support for github-access-token, but mark as deprecated. --- src/libfetchers/github.cc | 10 ++++++++++ src/libstore/globals.hh | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 0e0655367..443644639 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -146,7 +146,17 @@ struct GitArchiveInputScheme : InputScheme auto tokens = settings.accessTokens.get(); auto pat = tokens.find(host); if (pat == tokens.end()) + { + if ("github.com" == host) + { + auto oldcfg = settings.githubAccessToken.get(); + if (!oldcfg.empty()) { + warn("using deprecated 'github-access-token' config value; please use 'access-tokens' instead"); + return oldcfg; + } + } return std::nullopt; + } return pat->second; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 959ebe360..bd36ffc17 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -861,7 +861,7 @@ public: )"}; Setting githubAccessToken{this, "", "github-access-token", - "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes."}; + "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes (deprecated, please use 'access-tokens' instead)."}; Setting accessTokens{this, {}, "access-tokens", R"( From 5e7838512e2b8de3c8fe271b8beae5ca9e1efaf9 Mon Sep 17 00:00:00 2001 From: Kevin Quick Date: Tue, 29 Sep 2020 16:20:54 -0700 Subject: [PATCH 7/7] Remove github-access-token in favor of access-token. --- src/libfetchers/github.cc | 10 ---------- src/libstore/globals.hh | 3 --- 2 files changed, 13 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 142b8b87c..8286edf75 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -147,17 +147,7 @@ struct GitArchiveInputScheme : InputScheme auto tokens = settings.accessTokens.get(); auto pat = tokens.find(host); if (pat == tokens.end()) - { - if ("github.com" == host) - { - auto oldcfg = settings.githubAccessToken.get(); - if (!oldcfg.empty()) { - warn("using deprecated 'github-access-token' config value; please use 'access-tokens' instead"); - return oldcfg; - } - } return std::nullopt; - } return pat->second; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 3b8ccadf3..0f0c0fe6f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -859,9 +859,6 @@ public: are loaded as plugins (non-recursively). )"}; - Setting githubAccessToken{this, "", "github-access-token", - "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes (deprecated, please use 'access-tokens' instead)."}; - Setting accessTokens{this, {}, "access-tokens", R"( Access tokens used to access protected GitHub, GitLab, or