Merge pull request #2566 from LnL7/s3-scheme

s3: make scheme configurable
This commit is contained in:
Eelco Dolstra 2018-12-09 12:59:13 +01:00 committed by GitHub
commit 18ecd087ae
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 24 additions and 8 deletions

View file

@ -51,6 +51,18 @@ the S3 URL:</para>
addressing.</para></note> addressing.</para></note>
</listitem> </listitem>
</varlistentry> </varlistentry>
<varlistentry><term><literal>scheme</literal></term>
<listitem>
<para>
The scheme used for S3 requests, <literal>https</literal>
(default) or <literal>http</literal>. This option allows you to
disable HTTPS for binary caches which don't support it.
</para>
<note><para>HTTPS should be used if the cache might contain
sensitive information.</para></note>
</listitem>
</varlistentry>
</variablelist> </variablelist>
<para>In this example we will use the bucket named <para>In this example we will use the bucket named
@ -165,7 +177,7 @@ the S3 URL:</para>
</example> </example>
<example><title>Uploading to an S3-Compatible Binary Cache</title> <example><title>Uploading to an S3-Compatible Binary Cache</title>
<para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&amp;endpoint=minio.example.com' nixpkgs.hello</command></para> <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&amp;scheme=https&amp;endpoint=minio.example.com' nixpkgs.hello</command></para>
</example> </example>
</section> </section>
</section> </section>

View file

@ -622,7 +622,7 @@ struct CurlDownloader : public Downloader
// FIXME: do this on a worker thread // FIXME: do this on a worker thread
try { try {
#ifdef ENABLE_S3 #ifdef ENABLE_S3
S3Helper s3Helper("", Aws::Region::US_EAST_1, ""); // FIXME: make configurable S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable
auto slash = request.uri.find('/', 5); auto slash = request.uri.find('/', 5);
if (slash == std::string::npos) if (slash == std::string::npos)
throw nix::Error("bad S3 URI '%s'", request.uri); throw nix::Error("bad S3 URI '%s'", request.uri);

View file

@ -82,8 +82,8 @@ static void initAWS()
}); });
} }
S3Helper::S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint) S3Helper::S3Helper(const string & profile, const string & region, const string & scheme, const string & endpoint)
: config(makeConfig(region, endpoint)) : config(makeConfig(region, scheme, endpoint))
, client(make_ref<Aws::S3::S3Client>( , client(make_ref<Aws::S3::S3Client>(
profile == "" profile == ""
? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>( ? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
@ -114,11 +114,14 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
} }
}; };
ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & endpoint) ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & scheme, const string & endpoint)
{ {
initAWS(); initAWS();
auto res = make_ref<Aws::Client::ClientConfiguration>(); auto res = make_ref<Aws::Client::ClientConfiguration>();
res->region = region; res->region = region;
if (!scheme.empty()) {
res->scheme = Aws::Http::SchemeMapper::FromString(scheme.c_str());
}
if (!endpoint.empty()) { if (!endpoint.empty()) {
res->endpointOverride = endpoint; res->endpointOverride = endpoint;
} }
@ -169,6 +172,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
{ {
const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."}; const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."};
const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"}; const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
@ -188,7 +192,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
const Params & params, const std::string & bucketName) const Params & params, const std::string & bucketName)
: S3BinaryCacheStore(params) : S3BinaryCacheStore(params)
, bucketName(bucketName) , bucketName(bucketName)
, s3Helper(profile, region, endpoint) , s3Helper(profile, region, scheme, endpoint)
{ {
diskCache = getNarInfoDiskCache(); diskCache = getNarInfoDiskCache();
} }

View file

@ -14,9 +14,9 @@ struct S3Helper
ref<Aws::Client::ClientConfiguration> config; ref<Aws::Client::ClientConfiguration> config;
ref<Aws::S3::S3Client> client; ref<Aws::S3::S3Client> client;
S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint); S3Helper(const std::string & profile, const std::string & region, const std::string & scheme, const std::string & endpoint);
ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & endpoint); ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint);
struct DownloadResult struct DownloadResult
{ {