From 4a2c948943e02af7829a758104044c72b49f9b64 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 May 2018 12:54:30 +0200 Subject: [PATCH] Fix bzip2 compression of files > 4 GiB Bzip2's 'avail_in' parameter is declared as an unsigned int, so assigning a size_t length to it led to silent truncation. Fixes #2111. --- src/libutil/compression.cc | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 81cb5e98c..e1782f8c4 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -368,8 +368,21 @@ struct BzipSink : CompressionSink } void write(const unsigned char * data, size_t len) override + { + /* Bzip2's 'avail_in' parameter is an unsigned int, so we need + to split the input into chunks of at most 4 GiB. */ + while (len) { + auto n = std::min((size_t) std::numeric_limits::max(), len); + writeInternal(data, n); + data += n; + len -= n; + } + } + + void writeInternal(const unsigned char * data, size_t len) { assert(!finished); + assert(len <= std::numeric_limits::max()); strm.next_in = (char *) data; strm.avail_in = len; @@ -475,8 +488,6 @@ struct BrotliSink : CompressionSink void write(const unsigned char * data, size_t len) override { - assert(!finished); - // Don't feed brotli too much at once const size_t CHUNK_SIZE = sizeof(outbuf) << 2; while (len) { @@ -486,7 +497,7 @@ struct BrotliSink : CompressionSink len -= n; } } - private: + void writeInternal(const unsigned char * data, size_t len) { assert(!finished);