lix-project/lix
20
57
Fork 31
Code Issues 349 Code Review (Gerrit) Projects 3 Releases Wiki Activity

libstore: reunify all file transfer methods again

Browse source 
with the api cleaned up we can suddenly reunify uploads, downloads, and
existence checks through curl in the same wrapper function. uploads and
existence checks simply don't use the result source, and given that all
transfers (or at least *most* transfers to date) go through the network
the few extra allocations do not hurt us at all. even for file:// calls
the overhead won't much matter as going to disk and back *is* expensive

Change-Id: I4f9ca6681a8fc303377b4cf4c63e3363ae32c18b
This commit is contained in:
eldritch horrors 2024-10-28 18:59:43 +01:00
parent d65838a900
commit c83b13eafd
1 changed files with 49 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

128
src/libstore/filetransfer.cc
View file

@ -702,64 +702,19 @@ struct curlFileTransfer : public FileTransfer
void upload(const std::string & uri, std::string data, const Headers & headers) override
{
enqueueFileTransfer(uri, headers, std::move(data), false).get();
enqueueFileTransfer(uri, headers, std::move(data), false);
}
std::future<std::pair<FileTransferResult, std::string>> enqueueFileTransfer(
std::pair<FileTransferResult, box_ptr<Source>> enqueueFileTransfer(
const std::string & uri,
const Headers & headers,
std::optional<std::string> data,
bool noBody
)
{
struct State {
std::string data;
};
auto _state = std::make_shared<Sync<State>>();
auto [meta, done] = enqueueFileTransfer(
uri,
headers,
[](std::exception_ptr ex) {
if (ex) {
std::rethrow_exception(ex);
}
},
[_state](std::string_view data) {
_state->lock()->data.append(data);
},
std::move(data),
noBody
);
return std::async(
std::launch::deferred,
[_state, _meta{std::move(meta)}, done{std::move(done)}]() mutable {
auto meta = _meta.get();
done.get();
auto state(_state->lock());
return std::pair(std::move(meta), std::move(state->data));
}
);
}
std::pair<std::future<FileTransferResult>, std::future<void>> enqueueFileTransfer(
const std::string & uri,
const Headers & headers,
std::invocable<std::exception_ptr> auto doneCallback,
std::function<void(std::string_view data)> dataCallback,
std::optional<std::string> data,
bool noBody
)
{
/* Ugly hack to support s3:// URIs. */
if (uri.starts_with("s3://")) {
// FIXME: do this on a worker thread
return {
std::async(
std::launch::deferred,
[uri, dataCallback] {
#if ENABLE_S3
auto [bucketName, key, params] = parseS3Uri(uri);
@ -775,49 +730,14 @@ struct curlFileTransfer : public FileTransfer
FileTransferResult res;
if (!s3Res.data)
throw FileTransferError(NotFound, "S3 object '%s' does not exist", uri);
dataCallback(*s3Res.data);
return res;
return {res, make_box_ptr<StringSource>(std::move(*s3Res.data))};
#else
throw nix::Error(
"cannot download '%s' because Lix is not built with S3 support", uri
);
#endif
}
),
std::async(std::launch::deferred, []{}),
};
}
auto item = enqueueItem(std::make_shared<TransferItem>(
*this,
uri,
headers,
getCurActivity(),
std::move(doneCallback),
std::move(dataCallback),
std::move(data),
noBody
));
return {item->metadataPromise.get_future(), item->doneCallback.get_future()};
}
bool exists(const std::string & uri, const Headers & headers) override
{
try {
enqueueFileTransfer(uri, headers, std::nullopt, true).get();
return true;
} catch (FileTransferError & e) {
/* S3 buckets return 403 if a file doesn't exist and the
bucket is unlistable, so treat 403 as 404. */
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
return false;
throw;
}
}
std::pair<FileTransferResult, box_ptr<Source>>
download(const std::string & uri, const Headers & headers) override
{
struct State {
bool done = false, failed = false;
std::exception_ptr exc;
@ -827,9 +747,11 @@ struct curlFileTransfer : public FileTransfer
auto _state = std::make_shared<Sync<State>>();
auto [metadataFuture, _done] = enqueueFileTransfer(
auto item = enqueueItem(std::make_shared<TransferItem>(
*this,
uri,
headers,
getCurActivity(),
[_state](std::exception_ptr ex) {
auto state(_state->lock());
state->done = true;
@ -860,19 +782,19 @@ struct curlFileTransfer : public FileTransfer
state->data.append(data);
state->avail.notify_one();
},
std::nullopt,
false
);
std::move(data),
noBody
));
struct DownloadSource : Source
struct TransferSource : Source
{
const std::shared_ptr<Sync<State>> _state;
std::string chunk;
std::string_view buffered;
explicit DownloadSource(const std::shared_ptr<Sync<State>> & state) : _state(state) {}
explicit TransferSource(const std::shared_ptr<Sync<State>> & state) : _state(state) {}
~DownloadSource()
~TransferSource()
{
// wake up the download thread if it's still going and have it abort
auto state(_state->lock());
@ -927,19 +849,39 @@ struct curlFileTransfer : public FileTransfer
}
if (total == 0) {
throw EndOfFile("download finished");
throw EndOfFile("transfer finished");
}
return total;
}
};
auto metadata = metadataFuture.get();
auto source = make_box_ptr<DownloadSource>(_state);
auto metadata = item->metadataPromise.get_future().get();
auto source = make_box_ptr<TransferSource>(_state);
auto lock(_state->lock());
source->awaitData(lock);
return {std::move(metadata), std::move(source)};
}
bool exists(const std::string & uri, const Headers & headers) override
{
try {
enqueueFileTransfer(uri, headers, std::nullopt, true);
return true;
} catch (FileTransferError & e) {
/* S3 buckets return 403 if a file doesn't exist and the
bucket is unlistable, so treat 403 as 404. */
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
return false;
throw;
}
}
std::pair<FileTransferResult, box_ptr<Source>>
download(const std::string & uri, const Headers & headers) override
{
return enqueueFileTransfer(uri, headers, std::nullopt, false);
}
};
ref<curlFileTransfer> makeCurlFileTransfer(std::optional<unsigned int> baseRetryTimeMs)