s3 binary cache: support specifying an endpoint

Works for uploading and not downloading.
This commit is contained in:
Graham Christensen 2018-07-31 16:45:49 -04:00
parent 3193f5ff3e
commit 49a53c1d3f
5 changed files with 33 additions and 8 deletions

View file

@ -118,12 +118,29 @@ fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para>
</para>
</listitem>
</varlistentry>
<varlistentry><term><literal>endpoint</literal></term>
<listitem>
<para>
The URL to your S3-compatible service, for when not using
Amazon S3. Do not specify this value if you're using Amazon
S3.
</para>
<note><para>This endpoint must support HTTPS and will use
path-based addressing instead of virtual host based
addressing.</para></note>
</listitem>
</varlistentry>
</variablelist>
<example><title>Uploading with non-default credential profile for Amazon S3</title>
<para><command>nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload</command></para>
</example>
<example><title>Uploading to an S3-Compatible Binary Cache</title>
<para><command>nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload&amp;endpoint=minio.example.com</command></para>
</example>
<para>The user writing to the bucket will need to perform the
following actions against the bucket:</para>

View file

@ -598,7 +598,7 @@ struct CurlDownloader : public Downloader
// FIXME: do this on a worker thread
try {
#ifdef ENABLE_S3
S3Helper s3Helper("", Aws::Region::US_EAST_1); // FIXME: make configurable
S3Helper s3Helper("", Aws::Region::US_EAST_1, ""); // FIXME: make configurable
auto slash = request.uri.find('/', 5);
if (slash == std::string::npos)
throw nix::Error("bad S3 URI '%s'", request.uri);

View file

@ -84,8 +84,8 @@ static void initAWS()
});
}
S3Helper::S3Helper(const std::string & profile, const std::string & region)
: config(makeConfig(region))
S3Helper::S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint)
: config(makeConfig(region, endpoint))
, client(make_ref<Aws::S3::S3Client>(
profile == ""
? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
@ -99,7 +99,7 @@ S3Helper::S3Helper(const std::string & profile, const std::string & region)
#else
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
#endif
false))
endpoint.empty()))
{
}
@ -116,11 +116,14 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
}
};
ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region)
ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & endpoint)
{
initAWS();
auto res = make_ref<Aws::Client::ClientConfiguration>();
res->region = region;
if (!endpoint.empty()) {
res->endpointOverride = endpoint;
}
res->requestTimeoutMs = 600 * 1000;
res->retryStrategy = std::make_shared<RetryStrategy>();
res->caFile = settings.caFile;
@ -170,6 +173,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
{
const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."};
const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"};
@ -186,7 +190,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
const Params & params, const std::string & bucketName)
: S3BinaryCacheStore(params)
, bucketName(bucketName)
, s3Helper(profile, region)
, s3Helper(profile, region, endpoint)
{
diskCache = getNarInfoDiskCache();
}

View file

@ -14,9 +14,9 @@ struct S3Helper
ref<Aws::Client::ClientConfiguration> config;
ref<Aws::S3::S3Client> client;
S3Helper(const std::string & profile, const std::string & region);
S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint);
ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region);
ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & endpoint);
struct DownloadResult
{

View file

@ -72,6 +72,10 @@ struct CmdCopy : StorePathsCommand
"To populate the current folder build output to a S3 binary cache:",
"nix copy --to s3://my-bucket?region=eu-west-1"
},
Example{
"To populate the current folder build output to an S3-compatible binary cache:",
"nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com"
},
#endif
};
}