libstore: use curl-builtin read callback

when no custom READFUNCTION is set curl will fall back to using its
READDATA as a FILE pointer and calling fread on it to retrieve data
to send to the peer. we can wrap our upload data in a FILE pointer.
doing so also allows for upload retries, but we don't do that here.

Change-Id: I0509f936229dd978e89135862dacc8a3937cd4de
This commit is contained in:
eldritch horrors 2024-11-12 14:01:46 +01:00
parent 5be7956592
commit 7d7e16ecec

View file

@ -7,6 +7,7 @@
#include "strings.hh" #include "strings.hh"
#include <cstddef> #include <cstddef>
#include <cstdio>
#include <kj/encoding.h> #include <kj/encoding.h>
#if ENABLE_S3 #if ENABLE_S3
@ -45,7 +46,7 @@ struct curlFileTransfer : public FileTransfer
std::string uri; std::string uri;
FileTransferResult result; FileTransferResult result;
Activity act; Activity act;
std::optional<std::string_view> uploadData; std::unique_ptr<FILE, decltype([](FILE * f) { fclose(f); })> uploadData;
std::string downloadData; std::string downloadData;
enum { enum {
/// nothing has been transferred yet /// nothing has been transferred yet
@ -98,7 +99,6 @@ struct curlFileTransfer : public FileTransfer
, act(*logger, lvlTalkative, actFileTransfer, , act(*logger, lvlTalkative, actFileTransfer,
fmt(uploadData ? "uploading '%s'" : "downloading '%s'", uri), fmt(uploadData ? "uploading '%s'" : "downloading '%s'", uri),
{uri}, parentAct) {uri}, parentAct)
, uploadData(uploadData)
, doneCallback([cb{std::move(doneCallback)}] (std::exception_ptr ex) { , doneCallback([cb{std::move(doneCallback)}] (std::exception_ptr ex) {
cb(ex); cb(ex);
}) })
@ -150,9 +150,9 @@ struct curlFileTransfer : public FileTransfer
curl_easy_setopt(req, CURLOPT_NOBODY, 1); curl_easy_setopt(req, CURLOPT_NOBODY, 1);
if (uploadData) { if (uploadData) {
this->uploadData.reset(fmemopen(const_cast<char *>(uploadData->data()), uploadData->size(), "r"));
curl_easy_setopt(req, CURLOPT_UPLOAD, 1L); curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper); curl_easy_setopt(req, CURLOPT_READDATA, this->uploadData.get());
curl_easy_setopt(req, CURLOPT_READDATA, this);
curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) uploadData->length()); curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) uploadData->length());
} }
@ -327,26 +327,6 @@ struct curlFileTransfer : public FileTransfer
return 0; return 0;
} }
size_t readOffset = 0;
size_t readCallback(char *buffer, size_t size, size_t nitems)
{
if (readOffset == uploadData->length())
return 0;
auto count = std::min(size * nitems, uploadData->length() - readOffset);
assert(count);
// Lint: this is turning a string into a byte array to hand to
// curl, which is fine.
// NOLINTNEXTLINE(bugprone-not-null-terminated-result)
memcpy(buffer, uploadData->data() + readOffset, count);
readOffset += count;
return count;
}
static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
{
return static_cast<TransferItem *>(userp)->readCallback(buffer, size, nitems);
}
void finish(CURLcode code) void finish(CURLcode code)
{ {
auto httpStatus = getHTTPStatus(); auto httpStatus = getHTTPStatus();