S3BinaryCacheStore: Share TransferManager
TransferManager allocates a lot of memory (50 MiB by default), and it might leak but I'm not sure about that. In any case it was causing OOMs in hydra-queue-runner. So allocate only one TransferManager per S3BinaryCacheStore. Hopefully fixes https://github.com/NixOS/hydra/issues/586.
This commit is contained in:
parent
c87f4b9324
commit
ada4e90267
|
@ -275,6 +275,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::shared_ptr<TransferManager> transferManager;
|
||||||
|
std::once_flag transferManagerCreated;
|
||||||
|
|
||||||
void uploadFile(const std::string & path, const std::string & data,
|
void uploadFile(const std::string & path, const std::string & data,
|
||||||
const std::string & mimeType,
|
const std::string & mimeType,
|
||||||
const std::string & contentEncoding)
|
const std::string & contentEncoding)
|
||||||
|
@ -286,6 +289,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
||||||
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
|
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
|
||||||
executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads);
|
executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads);
|
||||||
|
|
||||||
|
std::call_once(transferManagerCreated, [&]() {
|
||||||
|
|
||||||
TransferManagerConfiguration transferConfig(executor.get());
|
TransferManagerConfiguration transferConfig(executor.get());
|
||||||
|
|
||||||
transferConfig.s3Client = s3Helper.client;
|
transferConfig.s3Client = s3Helper.client;
|
||||||
|
@ -294,7 +299,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
||||||
transferConfig.uploadProgressCallback =
|
transferConfig.uploadProgressCallback =
|
||||||
[&](const TransferManager *transferManager,
|
[&](const TransferManager *transferManager,
|
||||||
const std::shared_ptr<const TransferHandle>
|
const std::shared_ptr<const TransferHandle>
|
||||||
&transferHandle) {
|
&transferHandle)
|
||||||
|
{
|
||||||
//FIXME: find a way to properly abort the multipart upload.
|
//FIXME: find a way to properly abort the multipart upload.
|
||||||
//checkInterrupt();
|
//checkInterrupt();
|
||||||
debug("upload progress ('%s'): '%d' of '%d' bytes",
|
debug("upload progress ('%s'): '%d' of '%d' bytes",
|
||||||
|
@ -303,8 +309,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
||||||
transferHandle->GetBytesTotalSize());
|
transferHandle->GetBytesTotalSize());
|
||||||
};
|
};
|
||||||
|
|
||||||
std::shared_ptr<TransferManager> transferManager =
|
transferManager = TransferManager::Create(transferConfig);
|
||||||
TransferManager::Create(transferConfig);
|
});
|
||||||
|
|
||||||
auto now1 = std::chrono::steady_clock::now();
|
auto now1 = std::chrono::steady_clock::now();
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue