lix-project/lix
20
57
Fork 31
Code Issues 347 Code Review (Gerrit) Projects 3 Releases Wiki Activity

libstore: use curl content-encoding support, not our own

Browse source 
let's use the automatic decoding functions curl provides instead of
implementing them ourselves for the dubious ability to support both
xz and bzip2 encodings as well, neither of which anything will send

Change-Id: I3edfebeb596a0e9d5c986efca9270501c996f2dd
This commit is contained in:
eldritch horrors 2024-10-28 18:59:43 +01:00
parent 8c567c0424
commit 10488f7431
2 changed files with 22 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
doc/manual/rl-next/content-encodings.md Normal file
View file

@ -0,0 +1,15 @@
---
synopsis: "Drop support for `xz` and `bzip2` Content-Encoding"
category: Miscellany
cls: [2134]
credits: horrors
---
Lix no longer supports the non-standard HTTP Content-Encoding values `xz` and `bzip2`.
We do not expect this to cause any problems in practice since these encodings *aren't*
standard, and any server delivering them anyway without being asked to is already well
and truly set on the path of causing inexplicable client breakages.
Lix's ability to decompress files compressed with `xz` or `bzip2` is unaffected. We're
only bringing Lix more in line with the HTTP standard; all post-transfer data handling
remains as it was before.

58
src/libstore/filetransfer.cc
View file

@ -4,7 +4,6 @@
#include "store-api.hh"
#include "s3.hh"
#include "signals.hh"
#include "compression.hh"
#include "strings.hh"
#include <cstddef>
@ -78,8 +77,6 @@ struct curlFileTransfer : public FileTransfer
struct curl_slist * requestHeaders = 0;
std::string encoding;
bool acceptRanges = false;
curl_off_t writtenToSink = 0;
@ -127,7 +124,6 @@ struct curlFileTransfer : public FileTransfer
if (req == nullptr) {
throw FileTransferError(Misc, {}, "could not allocate curl handle");
}
requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
for (auto it = headers.begin(); it != headers.end(); ++it){
requestHeaders = curl_slist_append(requestHeaders, fmt("%s: %s", it->first, it->second).c_str());
}
@ -244,14 +240,6 @@ struct curlFileTransfer : public FileTransfer
result.etag = std::move(etag);
}
else if (name == "content-encoding") {
auto encoding = trim(line.substr(i + 1));
if (!this->encoding.empty() && this->encoding != encoding) {
throwChangedTarget("encoding", this->encoding, encoding);
}
this->encoding = std::move(encoding);
}
else if (name == "accept-ranges" && toLower(trim(line.substr(i + 1))) == "bytes")
acceptRanges = true;
@ -340,6 +328,7 @@ struct curlFileTransfer : public FileTransfer
curl_easy_setopt(req, CURLOPT_URL, uri.c_str());
curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(req, CURLOPT_ACCEPT_ENCODING, ""); // all of them!
curl_easy_setopt(req, CURLOPT_MAXREDIRS, 10);
curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
curl_easy_setopt(req, CURLOPT_USERAGENT,
@ -405,14 +394,6 @@ struct curlFileTransfer : public FileTransfer
debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes",
verb(), uri, code, httpStatus, bodySize);
// this has to happen here until we can return an actual future.
// wrapping user `callback`s instead is not possible because the
// Callback api expects std::functions, and copying Callbacks is
// not possible due the promises they hold.
if (code == CURLE_OK && !dataCallback && downloadData.length() > 0) {
downloadData = decompress(encoding, downloadData);
}
if (callbackException)
failEx(callbackException);
@ -500,7 +481,7 @@ struct curlFileTransfer : public FileTransfer
&& attempt < tries
&& (!this->dataCallback
|| writtenToSink == 0
|| (acceptRanges && encoding.empty())))
|| acceptRanges))
{
int ms = fileTransfer.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937));
if (writtenToSink)
@ -819,7 +800,7 @@ struct curlFileTransfer : public FileTransfer
struct State {
bool done = false, failed = false;
std::exception_ptr exc;
std::string data, encoding;
std::string data;
std::condition_variable avail, request;
};
@ -853,10 +834,6 @@ struct curlFileTransfer : public FileTransfer
state.wait_for(state->request, std::chrono::seconds(10));
}
if (state->encoding.empty()) {
state->encoding = transfer.encoding;
}
/* Append data to the buffer and wake up the calling
thread. */
state->data.append(data);
@ -866,15 +843,15 @@ struct curlFileTransfer : public FileTransfer
false
);
struct InnerSource : Source
struct DownloadSource : Source
{
const std::shared_ptr<Sync<State>> _state;
std::string chunk;
std::string_view buffered;
explicit InnerSource(const std::shared_ptr<Sync<State>> & state) : _state(state) {}
explicit DownloadSource(const std::shared_ptr<Sync<State>> & state) : _state(state) {}
~InnerSource()
~DownloadSource()
{
// wake up the download thread if it's still going and have it abort
auto state(_state->lock());
@ -936,30 +913,9 @@ struct curlFileTransfer : public FileTransfer
}
};
struct DownloadSource : Source
{
InnerSource inner;
std::unique_ptr<Source> decompressor;
explicit DownloadSource(const std::shared_ptr<Sync<State>> & state) : inner(state) {}
size_t read(char * data, size_t len) override
{
checkInterrupt();
if (!decompressor) {
auto state(inner._state->lock());
inner.awaitData(state);
decompressor = makeDecompressionSource(state->encoding, inner);
}
return decompressor->read(data, len);
}
};
auto source = make_box_ptr<DownloadSource>(_state);
auto lock(_state->lock());
source->inner.awaitData(lock);
source->awaitData(lock);
return source;
}
};