From 49a53c1d3f77ff33e14a95d004a6ce0720293997 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 31 Jul 2018 16:45:49 -0400 Subject: [PATCH] s3 binary cache: support specifying an endpoint Works for uploading and not downloading. --- doc/manual/packages/s3-substituter.xml | 17 +++++++++++++++++ src/libstore/download.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 14 +++++++++----- src/libstore/s3.hh | 4 ++-- src/nix/copy.cc | 4 ++++ 5 files changed, 33 insertions(+), 8 deletions(-) diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml index 9fe137d72..bcd91cfdb 100644 --- a/doc/manual/packages/s3-substituter.xml +++ b/doc/manual/packages/s3-substituter.xml @@ -118,12 +118,29 @@ fetch prebuilt binaries from cache.nixos.org. + + endpoint + + + The URL to your S3-compatible service, for when not using + Amazon S3. Do not specify this value if you're using Amazon + S3. + + This endpoint must support HTTPS and will use + path-based addressing instead of virtual host based + addressing. + + Uploading with non-default credential profile for Amazon S3 nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload + Uploading to an S3-Compatible Binary Cache + nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload&endpoint=minio.example.com + + The user writing to the bucket will need to perform the following actions against the bucket: diff --git a/src/libstore/download.cc b/src/libstore/download.cc index ecec0f205..f80c9e45b 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -598,7 +598,7 @@ struct CurlDownloader : public Downloader // FIXME: do this on a worker thread try { #ifdef ENABLE_S3 - S3Helper s3Helper("", Aws::Region::US_EAST_1); // FIXME: make configurable + S3Helper s3Helper("", Aws::Region::US_EAST_1, ""); // FIXME: make configurable auto slash = request.uri.find('/', 5); if (slash == std::string::npos) throw nix::Error("bad S3 URI '%s'", request.uri); diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 26144ccb4..2f18e3f38 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -84,8 +84,8 @@ static void initAWS() }); } -S3Helper::S3Helper(const std::string & profile, const std::string & region) - : config(makeConfig(region)) +S3Helper::S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint) + : config(makeConfig(region, endpoint)) , client(make_ref( profile == "" ? std::dynamic_pointer_cast( @@ -99,7 +99,7 @@ S3Helper::S3Helper(const std::string & profile, const std::string & region) #else Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, #endif - false)) + endpoint.empty())) { } @@ -116,11 +116,14 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy } }; -ref S3Helper::makeConfig(const string & region) +ref S3Helper::makeConfig(const string & region, const string & endpoint) { initAWS(); auto res = make_ref(); res->region = region; + if (!endpoint.empty()) { + res->endpointOverride = endpoint; + } res->requestTimeoutMs = 600 * 1000; res->retryStrategy = std::make_shared(); res->caFile = settings.caFile; @@ -170,6 +173,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore { const Setting profile{this, "", "profile", "The name of the AWS configuration profile to use."}; const Setting region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; + const Setting endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; const Setting narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; const Setting lsCompression{this, "", "ls-compression", "compression method for .ls files"}; const Setting logCompression{this, "", "log-compression", "compression method for log/* files"}; @@ -186,7 +190,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore const Params & params, const std::string & bucketName) : S3BinaryCacheStore(params) , bucketName(bucketName) - , s3Helper(profile, region) + , s3Helper(profile, region, endpoint) { diskCache = getNarInfoDiskCache(); } diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index 4f9964003..95d612b66 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -14,9 +14,9 @@ struct S3Helper ref config; ref client; - S3Helper(const std::string & profile, const std::string & region); + S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint); - ref makeConfig(const std::string & region); + ref makeConfig(const std::string & region, const std::string & endpoint); struct DownloadResult { diff --git a/src/nix/copy.cc b/src/nix/copy.cc index e4e6c3e30..91711c8b4 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -72,6 +72,10 @@ struct CmdCopy : StorePathsCommand "To populate the current folder build output to a S3 binary cache:", "nix copy --to s3://my-bucket?region=eu-west-1" }, + Example{ + "To populate the current folder build output to an S3-compatible binary cache:", + "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com" + }, #endif }; }