From 5587dbdcf078586dcc5b9c54af614c1915e9da0a Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Sat, 11 May 2024 00:22:21 +0200 Subject: [PATCH] libstore: make BinaryCacheStore::getFile return a source this lets us remove the last true remaining uses of makeDecompressionSink. Change-Id: I146ca2bbe1a9ae9a367117a7b8a304b23a63e5e2 --- src/libstore/binary-cache-store.cc | 15 ++++++++------- src/libstore/binary-cache-store.hh | 2 +- src/libstore/http-binary-cache-store.cc | 4 ++-- src/libstore/local-binary-cache-store.cc | 4 ++-- src/libstore/s3-binary-cache-store.cc | 8 ++++++-- src/libutil/file-system.cc | 4 +++- 6 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 2519a2830..27ea621db 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -67,16 +67,18 @@ void BinaryCacheStore::upsertFile(const std::string & path, upsertFile(path, std::make_shared(std::move(data)), mimeType); } -void BinaryCacheStore::getFile(const std::string & path, Sink & sink) +box_ptr BinaryCacheStore::getFile(const std::string & path) { - sink(*getFileContents(path)); + return make_box_ptr([](std::string data) -> Generator { + co_yield std::span{data.data(), data.size()}; + }(std::move(*getFileContents(path)))); } std::optional BinaryCacheStore::getFileContents(const std::string & path) { StringSink sink; try { - getFile(path, sink); + return getFile(path)->drain(); } catch (NoSuchBinaryCacheFile &) { return std::nullopt; } @@ -334,16 +336,15 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) LengthSink narSize; TeeSink tee { sink, narSize }; - auto decompressor = makeDecompressionSink(info->compression, tee); try { - getFile(info->url, *decompressor); + auto file = getFile(info->url); + auto decompressor = makeDecompressionSource(info->compression, *file); + decompressor->drainInto(tee); } catch (NoSuchBinaryCacheFile & e) { throw SubstituteGone(std::move(e.info())); } - decompressor->finish(); - stats.narRead++; //stats.narReadCompressedBytes += nar->size(); // FIXME stats.narReadBytes += narSize.length; diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index cd963fbf9..6c1f1348c 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -83,7 +83,7 @@ public: /** * Dump the contents of the specified file to a sink. */ - virtual void getFile(const std::string & path, Sink & sink); + virtual box_ptr getFile(const std::string & path); virtual std::optional getFileContents(const std::string & path); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 06297e2eb..7ceea716a 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -150,12 +150,12 @@ protected: } - void getFile(const std::string & path, Sink & sink) override + box_ptr getFile(const std::string & path) override { checkEnabled(); auto request(makeRequest(path)); try { - getFileTransfer()->download(std::move(request))->drainInto(sink); + return getFileTransfer()->download(std::move(request)); } catch (FileTransferError & e) { if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 5ba5bd802..99dd0e0f6 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -68,10 +68,10 @@ protected: del.cancel(); } - void getFile(const std::string & path, Sink & sink) override + box_ptr getFile(const std::string & path) override { try { - sink << readFileSource(binaryCacheDir + "/" + path); + return make_box_ptr(readFileSource(binaryCacheDir + "/" + path)); } catch (SysError & e) { if (e.errNo == ENOENT) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 4aed791ce..4683f00f9 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -455,7 +455,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual uploadFile(path, istream, mimeType, ""); } - void getFile(const std::string & path, Sink & sink) override + box_ptr getFile(const std::string & path) override { stats.get++; @@ -469,7 +469,11 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", bucketName, path, res.data->size(), res.durationMs); - sink(*res.data); + return make_box_ptr( + [](std::string data) -> Generator { + co_yield std::span{data.data(), data.size()}; + }(std::move(*res.data)) + ); } else throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); } diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 13c2b27eb..278e5187c 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -294,7 +294,9 @@ Generator readFileSource(const Path & path) AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; if (!fd) throw SysError("opening file '%s'", path); - co_yield drainFDSource(fd.get()); + return [](AutoCloseFD fd) -> Generator { + co_yield drainFDSource(fd.get()); + }(std::move(fd)); }