lix-project/lix
20
55
Fork 31
Code Issues 346 Code Review (Gerrit) Projects 3 Releases Wiki Activity

Merge pull request #2703 from pradd/support_s3_region_param

Browse source 
Support parameters in S3 URLs
This commit is contained in:
Eelco Dolstra 2019-02-26 22:59:29 +08:00 committed by GitHub
parent 6bfb082ea2 06d6335987
commit ebd4d50e6e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 41 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
src/libstore/download.cc
View file

@ -614,6 +614,22 @@ struct CurlDownloader : public Downloader
writeFull(wakeupPipe.writeSide.get(), " ");
}
#ifdef ENABLE_S3
std::tuple<std::string, std::string, Store::Params> parseS3Uri(std::string uri)
{
auto [path, params] = splitUriAndParams(uri);
auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix
if (slash == std::string::npos)
throw nix::Error("bad S3 URI '%s'", path);
std::string bucketName(path, 5, slash - 5);
std::string key(path, slash + 1);
return {bucketName, key, params};
}
#endif
void enqueueDownload(const DownloadRequest & request,
Callback<DownloadResult> callback) override
{
@ -622,12 +638,15 @@ struct CurlDownloader : public Downloader
// FIXME: do this on a worker thread
try {
#ifdef ENABLE_S3
S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable
auto slash = request.uri.find('/', 5);
if (slash == std::string::npos)
throw nix::Error("bad S3 URI '%s'", request.uri);
std::string bucketName(request.uri, 5, slash - 5);
std::string key(request.uri, slash + 1);
auto [bucketName, key, params] = parseS3Uri(request.uri);
std::string profile = get(params, "profile", "");
std::string region = get(params, "region", Aws::Region::US_EAST_1);
std::string scheme = get(params, "scheme", "");
std::string endpoint = get(params, "endpoint", "");
S3Helper s3Helper(profile, region, scheme, endpoint);
// FIXME: implement ETag
auto s3Res = s3Helper.getObject(bucketName, key);
DownloadResult res;

16
src/libstore/store-api.cc
View file

@ -842,12 +842,11 @@ namespace nix {
RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
ref<Store> openStore(const std::string & uri_,
const Store::Params & extraParams)
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
{
auto uri(uri_);
Store::Params params(extraParams);
Store::Params params;
auto q = uri.find('?');
if (q != std::string::npos) {
for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
@ -873,6 +872,15 @@ ref<Store> openStore(const std::string & uri_,
}
uri = uri_.substr(0, q);
}
return {uri, params};
}
ref<Store> openStore(const std::string & uri_,
const Store::Params & extraParams)
{
auto [uri, uriParams] = splitUriAndParams(uri_);
auto params = extraParams;
params.insert(uriParams.begin(), uriParams.end());
for (auto fun : *RegisterStoreImplementation::implementations) {
auto store = fun(uri, params);

4
src/libstore/store-api.hh
View file

@ -798,4 +798,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str,
for paths created by makeFixedOutputPath() / addToStore(). */
std::string makeFixedOutputCA(bool recursive, const Hash & hash);
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
}