Merge pull request #2703 from pradd/support_s3_region_param
Support parameters in S3 URLs
This commit is contained in:
commit
ebd4d50e6e
|
@ -614,6 +614,22 @@ struct CurlDownloader : public Downloader
|
||||||
writeFull(wakeupPipe.writeSide.get(), " ");
|
writeFull(wakeupPipe.writeSide.get(), " ");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#ifdef ENABLE_S3
|
||||||
|
std::tuple<std::string, std::string, Store::Params> parseS3Uri(std::string uri)
|
||||||
|
{
|
||||||
|
auto [path, params] = splitUriAndParams(uri);
|
||||||
|
|
||||||
|
auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix
|
||||||
|
if (slash == std::string::npos)
|
||||||
|
throw nix::Error("bad S3 URI '%s'", path);
|
||||||
|
|
||||||
|
std::string bucketName(path, 5, slash - 5);
|
||||||
|
std::string key(path, slash + 1);
|
||||||
|
|
||||||
|
return {bucketName, key, params};
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
void enqueueDownload(const DownloadRequest & request,
|
void enqueueDownload(const DownloadRequest & request,
|
||||||
Callback<DownloadResult> callback) override
|
Callback<DownloadResult> callback) override
|
||||||
{
|
{
|
||||||
|
@ -622,12 +638,15 @@ struct CurlDownloader : public Downloader
|
||||||
// FIXME: do this on a worker thread
|
// FIXME: do this on a worker thread
|
||||||
try {
|
try {
|
||||||
#ifdef ENABLE_S3
|
#ifdef ENABLE_S3
|
||||||
S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable
|
auto [bucketName, key, params] = parseS3Uri(request.uri);
|
||||||
auto slash = request.uri.find('/', 5);
|
|
||||||
if (slash == std::string::npos)
|
std::string profile = get(params, "profile", "");
|
||||||
throw nix::Error("bad S3 URI '%s'", request.uri);
|
std::string region = get(params, "region", Aws::Region::US_EAST_1);
|
||||||
std::string bucketName(request.uri, 5, slash - 5);
|
std::string scheme = get(params, "scheme", "");
|
||||||
std::string key(request.uri, slash + 1);
|
std::string endpoint = get(params, "endpoint", "");
|
||||||
|
|
||||||
|
S3Helper s3Helper(profile, region, scheme, endpoint);
|
||||||
|
|
||||||
// FIXME: implement ETag
|
// FIXME: implement ETag
|
||||||
auto s3Res = s3Helper.getObject(bucketName, key);
|
auto s3Res = s3Helper.getObject(bucketName, key);
|
||||||
DownloadResult res;
|
DownloadResult res;
|
||||||
|
|
|
@ -842,12 +842,11 @@ namespace nix {
|
||||||
|
|
||||||
RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
|
RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
|
||||||
|
|
||||||
|
/* Split URI into protocol+hierarchy part and its parameter set. */
|
||||||
ref<Store> openStore(const std::string & uri_,
|
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
|
||||||
const Store::Params & extraParams)
|
|
||||||
{
|
{
|
||||||
auto uri(uri_);
|
auto uri(uri_);
|
||||||
Store::Params params(extraParams);
|
Store::Params params;
|
||||||
auto q = uri.find('?');
|
auto q = uri.find('?');
|
||||||
if (q != std::string::npos) {
|
if (q != std::string::npos) {
|
||||||
for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
|
for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
|
||||||
|
@ -873,6 +872,15 @@ ref<Store> openStore(const std::string & uri_,
|
||||||
}
|
}
|
||||||
uri = uri_.substr(0, q);
|
uri = uri_.substr(0, q);
|
||||||
}
|
}
|
||||||
|
return {uri, params};
|
||||||
|
}
|
||||||
|
|
||||||
|
ref<Store> openStore(const std::string & uri_,
|
||||||
|
const Store::Params & extraParams)
|
||||||
|
{
|
||||||
|
auto [uri, uriParams] = splitUriAndParams(uri_);
|
||||||
|
auto params = extraParams;
|
||||||
|
params.insert(uriParams.begin(), uriParams.end());
|
||||||
|
|
||||||
for (auto fun : *RegisterStoreImplementation::implementations) {
|
for (auto fun : *RegisterStoreImplementation::implementations) {
|
||||||
auto store = fun(uri, params);
|
auto store = fun(uri, params);
|
||||||
|
|
|
@ -798,4 +798,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str,
|
||||||
for paths created by makeFixedOutputPath() / addToStore(). */
|
for paths created by makeFixedOutputPath() / addToStore(). */
|
||||||
std::string makeFixedOutputCA(bool recursive, const Hash & hash);
|
std::string makeFixedOutputCA(bool recursive, const Hash & hash);
|
||||||
|
|
||||||
|
|
||||||
|
/* Split URI into protocol+hierarchy part and its parameter set. */
|
||||||
|
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue