From ada4e902672e4d82f47fa260adcc450d9a85d41f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 Aug 2018 20:44:18 +0200 Subject: [PATCH] S3BinaryCacheStore: Share TransferManager TransferManager allocates a lot of memory (50 MiB by default), and it might leak but I'm not sure about that. In any case it was causing OOMs in hydra-queue-runner. So allocate only one TransferManager per S3BinaryCacheStore. Hopefully fixes https://github.com/NixOS/hydra/issues/586. --- src/libstore/s3-binary-cache-store.cc | 38 ++++++++++++++++----------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index ef41e413f..7711388f0 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -275,6 +275,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore return true; } + std::shared_ptr transferManager; + std::once_flag transferManagerCreated; + void uploadFile(const std::string & path, const std::string & data, const std::string & mimeType, const std::string & contentEncoding) @@ -286,25 +289,28 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore static std::shared_ptr executor = std::make_shared(maxThreads); - TransferManagerConfiguration transferConfig(executor.get()); + std::call_once(transferManagerCreated, [&]() { - transferConfig.s3Client = s3Helper.client; - transferConfig.bufferSize = bufferSize; + TransferManagerConfiguration transferConfig(executor.get()); - transferConfig.uploadProgressCallback = - [&](const TransferManager *transferManager, - const std::shared_ptr - &transferHandle) { - //FIXME: find a way to properly abort the multipart upload. - //checkInterrupt(); - debug("upload progress ('%s'): '%d' of '%d' bytes", - path, - transferHandle->GetBytesTransferred(), - transferHandle->GetBytesTotalSize()); - }; + transferConfig.s3Client = s3Helper.client; + transferConfig.bufferSize = bufferSize; - std::shared_ptr transferManager = - TransferManager::Create(transferConfig); + transferConfig.uploadProgressCallback = + [&](const TransferManager *transferManager, + const std::shared_ptr + &transferHandle) + { + //FIXME: find a way to properly abort the multipart upload. + //checkInterrupt(); + debug("upload progress ('%s'): '%d' of '%d' bytes", + path, + transferHandle->GetBytesTransferred(), + transferHandle->GetBytesTotalSize()); + }; + + transferManager = TransferManager::Create(transferConfig); + }); auto now1 = std::chrono::steady_clock::now();