libstore: use curl-builtin read callback
when no custom READFUNCTION is set curl will fall back to using its
READDATA as a FILE pointer and calling fread on it to retrieve data
to send to the peer. we can wrap our upload data in a FILE pointer.
doing so also allows for upload retries, but we don't do that here.
Change-Id: I0509f936229dd978e89135862dacc8a3937cd4de
This commit is contained in:
parent
5be7956592
commit
7d7e16ecec
|
@ -7,6 +7,7 @@
|
||||||
#include "strings.hh"
|
#include "strings.hh"
|
||||||
#include <cstddef>
|
#include <cstddef>
|
||||||
|
|
||||||
|
#include <cstdio>
|
||||||
#include <kj/encoding.h>
|
#include <kj/encoding.h>
|
||||||
|
|
||||||
#if ENABLE_S3
|
#if ENABLE_S3
|
||||||
|
@ -45,7 +46,7 @@ struct curlFileTransfer : public FileTransfer
|
||||||
std::string uri;
|
std::string uri;
|
||||||
FileTransferResult result;
|
FileTransferResult result;
|
||||||
Activity act;
|
Activity act;
|
||||||
std::optional<std::string_view> uploadData;
|
std::unique_ptr<FILE, decltype([](FILE * f) { fclose(f); })> uploadData;
|
||||||
std::string downloadData;
|
std::string downloadData;
|
||||||
enum {
|
enum {
|
||||||
/// nothing has been transferred yet
|
/// nothing has been transferred yet
|
||||||
|
@ -98,7 +99,6 @@ struct curlFileTransfer : public FileTransfer
|
||||||
, act(*logger, lvlTalkative, actFileTransfer,
|
, act(*logger, lvlTalkative, actFileTransfer,
|
||||||
fmt(uploadData ? "uploading '%s'" : "downloading '%s'", uri),
|
fmt(uploadData ? "uploading '%s'" : "downloading '%s'", uri),
|
||||||
{uri}, parentAct)
|
{uri}, parentAct)
|
||||||
, uploadData(uploadData)
|
|
||||||
, doneCallback([cb{std::move(doneCallback)}] (std::exception_ptr ex) {
|
, doneCallback([cb{std::move(doneCallback)}] (std::exception_ptr ex) {
|
||||||
cb(ex);
|
cb(ex);
|
||||||
})
|
})
|
||||||
|
@ -150,9 +150,9 @@ struct curlFileTransfer : public FileTransfer
|
||||||
curl_easy_setopt(req, CURLOPT_NOBODY, 1);
|
curl_easy_setopt(req, CURLOPT_NOBODY, 1);
|
||||||
|
|
||||||
if (uploadData) {
|
if (uploadData) {
|
||||||
|
this->uploadData.reset(fmemopen(const_cast<char *>(uploadData->data()), uploadData->size(), "r"));
|
||||||
curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
|
curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
|
||||||
curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper);
|
curl_easy_setopt(req, CURLOPT_READDATA, this->uploadData.get());
|
||||||
curl_easy_setopt(req, CURLOPT_READDATA, this);
|
|
||||||
curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) uploadData->length());
|
curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) uploadData->length());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -327,26 +327,6 @@ struct curlFileTransfer : public FileTransfer
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t readOffset = 0;
|
|
||||||
size_t readCallback(char *buffer, size_t size, size_t nitems)
|
|
||||||
{
|
|
||||||
if (readOffset == uploadData->length())
|
|
||||||
return 0;
|
|
||||||
auto count = std::min(size * nitems, uploadData->length() - readOffset);
|
|
||||||
assert(count);
|
|
||||||
// Lint: this is turning a string into a byte array to hand to
|
|
||||||
// curl, which is fine.
|
|
||||||
// NOLINTNEXTLINE(bugprone-not-null-terminated-result)
|
|
||||||
memcpy(buffer, uploadData->data() + readOffset, count);
|
|
||||||
readOffset += count;
|
|
||||||
return count;
|
|
||||||
}
|
|
||||||
|
|
||||||
static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
|
|
||||||
{
|
|
||||||
return static_cast<TransferItem *>(userp)->readCallback(buffer, size, nitems);
|
|
||||||
}
|
|
||||||
|
|
||||||
void finish(CURLcode code)
|
void finish(CURLcode code)
|
||||||
{
|
{
|
||||||
auto httpStatus = getHTTPStatus();
|
auto httpStatus = getHTTPStatus();
|
||||||
|
|
Loading…
Reference in a new issue