libstore: make BinaryCacheStore::getFile return a source

this lets us remove the last true remaining uses of
makeDecompressionSink.

Change-Id: I146ca2bbe1a9ae9a367117a7b8a304b23a63e5e2
This commit is contained in:
eldritch horrors 2024-05-11 00:22:21 +02:00
parent df8851f286
commit 5587dbdcf0
6 changed files with 22 additions and 15 deletions

View file

@ -67,16 +67,18 @@ void BinaryCacheStore::upsertFile(const std::string & path,
upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType); upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
} }
void BinaryCacheStore::getFile(const std::string & path, Sink & sink) box_ptr<Source> BinaryCacheStore::getFile(const std::string & path)
{ {
sink(*getFileContents(path)); return make_box_ptr<GeneratorSource>([](std::string data) -> Generator<Bytes> {
co_yield std::span{data.data(), data.size()};
}(std::move(*getFileContents(path))));
} }
std::optional<std::string> BinaryCacheStore::getFileContents(const std::string & path) std::optional<std::string> BinaryCacheStore::getFileContents(const std::string & path)
{ {
StringSink sink; StringSink sink;
try { try {
getFile(path, sink); return getFile(path)->drain();
} catch (NoSuchBinaryCacheFile &) { } catch (NoSuchBinaryCacheFile &) {
return std::nullopt; return std::nullopt;
} }
@ -334,16 +336,15 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
LengthSink narSize; LengthSink narSize;
TeeSink tee { sink, narSize }; TeeSink tee { sink, narSize };
auto decompressor = makeDecompressionSink(info->compression, tee);
try { try {
getFile(info->url, *decompressor); auto file = getFile(info->url);
auto decompressor = makeDecompressionSource(info->compression, *file);
decompressor->drainInto(tee);
} catch (NoSuchBinaryCacheFile & e) { } catch (NoSuchBinaryCacheFile & e) {
throw SubstituteGone(std::move(e.info())); throw SubstituteGone(std::move(e.info()));
} }
decompressor->finish();
stats.narRead++; stats.narRead++;
//stats.narReadCompressedBytes += nar->size(); // FIXME //stats.narReadCompressedBytes += nar->size(); // FIXME
stats.narReadBytes += narSize.length; stats.narReadBytes += narSize.length;

View file

@ -83,7 +83,7 @@ public:
/** /**
* Dump the contents of the specified file to a sink. * Dump the contents of the specified file to a sink.
*/ */
virtual void getFile(const std::string & path, Sink & sink); virtual box_ptr<Source> getFile(const std::string & path);
virtual std::optional<std::string> getFileContents(const std::string & path); virtual std::optional<std::string> getFileContents(const std::string & path);

View file

@ -150,12 +150,12 @@ protected:
} }
void getFile(const std::string & path, Sink & sink) override box_ptr<Source> getFile(const std::string & path) override
{ {
checkEnabled(); checkEnabled();
auto request(makeRequest(path)); auto request(makeRequest(path));
try { try {
getFileTransfer()->download(std::move(request))->drainInto(sink); return getFileTransfer()->download(std::move(request));
} catch (FileTransferError & e) { } catch (FileTransferError & e) {
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());

View file

@ -68,10 +68,10 @@ protected:
del.cancel(); del.cancel();
} }
void getFile(const std::string & path, Sink & sink) override box_ptr<Source> getFile(const std::string & path) override
{ {
try { try {
sink << readFileSource(binaryCacheDir + "/" + path); return make_box_ptr<GeneratorSource>(readFileSource(binaryCacheDir + "/" + path));
} catch (SysError & e) { } catch (SysError & e) {
if (e.errNo == ENOENT) if (e.errNo == ENOENT)
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path);

View file

@ -455,7 +455,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
uploadFile(path, istream, mimeType, ""); uploadFile(path, istream, mimeType, "");
} }
void getFile(const std::string & path, Sink & sink) override box_ptr<Source> getFile(const std::string & path) override
{ {
stats.get++; stats.get++;
@ -469,7 +469,11 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
bucketName, path, res.data->size(), res.durationMs); bucketName, path, res.data->size(), res.durationMs);
sink(*res.data); return make_box_ptr<GeneratorSource>(
[](std::string data) -> Generator<Bytes> {
co_yield std::span{data.data(), data.size()};
}(std::move(*res.data))
);
} else } else
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
} }

View file

@ -294,7 +294,9 @@ Generator<Bytes> readFileSource(const Path & path)
AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; AutoCloseFD fd{open(path.c_str(), O_RDONLY | O_CLOEXEC)};
if (!fd) if (!fd)
throw SysError("opening file '%s'", path); throw SysError("opening file '%s'", path);
return [](AutoCloseFD fd) -> Generator<Bytes> {
co_yield drainFDSource(fd.get()); co_yield drainFDSource(fd.get());
}(std::move(fd));
} }