From 07f992a74b64f4376d5b415d0042babc924772f3 Mon Sep 17 00:00:00 2001 From: Dzmitry Zaitsau Date: Thu, 21 Feb 2019 11:44:25 +0100 Subject: [PATCH 1/4] Extract and expose splitUriAndParams function which splits a URL into localtor and parameter parts --- src/libstore/store-api.cc | 16 ++++++++++++---- src/libstore/store-api.hh | 4 ++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 913a11121..c13ff1156 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -842,12 +842,11 @@ namespace nix { RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; - -ref openStore(const std::string & uri_, - const Store::Params & extraParams) +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair splitUriAndParams(const std::string & uri_) { auto uri(uri_); - Store::Params params(extraParams); + Store::Params params; auto q = uri.find('?'); if (q != std::string::npos) { for (auto s : tokenizeString(uri.substr(q + 1), "&")) { @@ -873,6 +872,15 @@ ref openStore(const std::string & uri_, } uri = uri_.substr(0, q); } + return {uri, params}; +} + +ref openStore(const std::string & uri_, + const Store::Params & extraParams) +{ + auto [uri, uriParams] = splitUriAndParams(uri_); + auto params = extraParams; + params.insert(uriParams.begin(), uriParams.end()); for (auto fun : *RegisterStoreImplementation::implementations) { auto store = fun(uri, params); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f504735e0..ad0f8df11 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -798,4 +798,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, for paths created by makeFixedOutputPath() / addToStore(). */ std::string makeFixedOutputCA(bool recursive, const Hash & hash); + +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair splitUriAndParams(const std::string & uri); + } From 56c18c67d98078dbed1d05ac68663cc52d2cb543 Mon Sep 17 00:00:00 2001 From: Dzmitry Zaitsau Date: Mon, 25 Feb 2019 17:56:52 +0100 Subject: [PATCH 2/4] Extend S3 URL parsing with parameters extraction --- src/libstore/download.cc | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 467f570bb..8bc496515 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -614,6 +614,22 @@ struct CurlDownloader : public Downloader writeFull(wakeupPipe.writeSide.get(), " "); } +#ifdef ENABLE_S3 + std::tuple parseS3Uri(std::string uri) + { + auto [path, params] = splitUriAndParams(uri); + + auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix + if (slash == std::string::npos) + throw nix::Error("bad S3 URI '%s'", path); + + std::string bucketName(path, 5, slash - 5); + std::string key(path, slash + 1); + + return {bucketName, key, params}; + } +#endif + void enqueueDownload(const DownloadRequest & request, Callback callback) override { @@ -622,12 +638,8 @@ struct CurlDownloader : public Downloader // FIXME: do this on a worker thread try { #ifdef ENABLE_S3 + auto [bucketName, key, params] = parseS3Uri(request.uri); S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable - auto slash = request.uri.find('/', 5); - if (slash == std::string::npos) - throw nix::Error("bad S3 URI '%s'", request.uri); - std::string bucketName(request.uri, 5, slash - 5); - std::string key(request.uri, slash + 1); // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); DownloadResult res; From ac200c3678c0f569cd962d8bbd22bb74b223d748 Mon Sep 17 00:00:00 2001 From: Dzmitry Zaitsau Date: Thu, 21 Feb 2019 14:26:50 +0100 Subject: [PATCH 3/4] Apply param values on S3Helper initialization --- src/libstore/download.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 8bc496515..f1666d293 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -639,7 +639,14 @@ struct CurlDownloader : public Downloader try { #ifdef ENABLE_S3 auto [bucketName, key, params] = parseS3Uri(request.uri); - S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable + + std::string profile = get(params, "profile", ""); + std::string region = get(params, "region", Aws::Region::US_EAST_1); + std::string scheme = get(params, "scheme", ""); + std::string endpoint = get(params, "endpoint", ""); + + S3Helper s3Helper(profile, region, scheme, endpoint); + // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); DownloadResult res; From 06d633598727763c54b4b049dbc213106474d10c Mon Sep 17 00:00:00 2001 From: Dzmitry Zaitsau Date: Tue, 26 Feb 2019 11:04:18 +0100 Subject: [PATCH 4/4] fix indentation --- src/libstore/download.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index f1666d293..22382ab1d 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -620,8 +620,8 @@ struct CurlDownloader : public Downloader auto [path, params] = splitUriAndParams(uri); auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix - if (slash == std::string::npos) - throw nix::Error("bad S3 URI '%s'", path); + if (slash == std::string::npos) + throw nix::Error("bad S3 URI '%s'", path); std::string bucketName(path, 5, slash - 5); std::string key(path, slash + 1);