lix-project/lix
20
57
Fork 31
Code Issues 349 Code Review (Gerrit) Projects 3 Releases Wiki Activity

libstore: use curl-builtin read callback

Browse source 
when no custom READFUNCTION is set curl will fall back to using its
READDATA as a FILE pointer and calling fread on it to retrieve data
to send to the peer. we can wrap our upload data in a FILE pointer.
doing so also allows for upload retries, but we don't do that here.

Change-Id: I0509f936229dd978e89135862dacc8a3937cd4de
This commit is contained in:
eldritch horrors 2024-11-12 14:01:46 +01:00
parent 5be7956592
commit 7d7e16ecec
1 changed files with 4 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
src/libstore/filetransfer.cc
View file

@ -7,6 +7,7 @@
#include "strings.hh"
#include <cstddef>
#include <cstdio>
#include <kj/encoding.h>
#if ENABLE_S3
@ -45,7 +46,7 @@ struct curlFileTransfer : public FileTransfer
std::string uri;
FileTransferResult result;
Activity act;
std::optional<std::string_view> uploadData;
std::unique_ptr<FILE, decltype([](FILE * f) { fclose(f); })> uploadData;
std::string downloadData;
enum {
/// nothing has been transferred yet
@ -98,7 +99,6 @@ struct curlFileTransfer : public FileTransfer
, act(*logger, lvlTalkative, actFileTransfer,
fmt(uploadData ? "uploading '%s'" : "downloading '%s'", uri),
{uri}, parentAct)
, uploadData(uploadData)
, doneCallback([cb{std::move(doneCallback)}] (std::exception_ptr ex) {
cb(ex);
})
@ -150,9 +150,9 @@ struct curlFileTransfer : public FileTransfer
curl_easy_setopt(req, CURLOPT_NOBODY, 1);
if (uploadData) {
this->uploadData.reset(fmemopen(const_cast<char *>(uploadData->data()), uploadData->size(), "r"));
curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper);
curl_easy_setopt(req, CURLOPT_READDATA, this);
curl_easy_setopt(req, CURLOPT_READDATA, this->uploadData.get());
curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) uploadData->length());
}
@ -327,26 +327,6 @@ struct curlFileTransfer : public FileTransfer
return 0;
}
size_t readOffset = 0;
size_t readCallback(char *buffer, size_t size, size_t nitems)
{
if (readOffset == uploadData->length())
return 0;
auto count = std::min(size * nitems, uploadData->length() - readOffset);
assert(count);
// Lint: this is turning a string into a byte array to hand to
// curl, which is fine.
// NOLINTNEXTLINE(bugprone-not-null-terminated-result)
memcpy(buffer, uploadData->data() + readOffset, count);
readOffset += count;
return count;
}
static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
{
return static_cast<TransferItem *>(userp)->readCallback(buffer, size, nitems);
}
void finish(CURLcode code)
{
auto httpStatus = getHTTPStatus();