From 5a082ad15a0f46dc1cbfd8aa5cb3ad9d94b5f178 Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Fri, 9 Feb 2018 20:57:31 -0600 Subject: [PATCH 1/3] configure.ac: check if lzma has MT support, fix deb build/etc. --- configure.ac | 2 ++ src/libutil/compression.cc | 2 ++ 2 files changed, 4 insertions(+) diff --git a/configure.ac b/configure.ac index 83b2346d0..c7498fb52 100644 --- a/configure.ac +++ b/configure.ac @@ -175,6 +175,8 @@ AC_SUBST(HAVE_SODIUM, [$have_sodium]) # Look for liblzma, a required dependency. PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"]) +AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt], + [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])]) # Look for libbrotli{enc,dec}, optional dependencies diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index ed15761b3..c509472b3 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -192,6 +192,7 @@ struct XzSink : CompressionSink XzSink(Sink & nextSink, const bool parallel) : nextSink(nextSink) { lzma_ret ret; +#ifdef HAVE_LZMA_MT if (parallel) { lzma_mt mt_options = {}; mt_options.flags = 0; @@ -208,6 +209,7 @@ struct XzSink : CompressionSink ret = lzma_stream_encoder_mt( &strm, &mt_options); } else +#endif ret = lzma_easy_encoder( &strm, 6, LZMA_CHECK_CRC64); From a0bdc96726b15b7f529156bccd60d0f8dd5544f3 Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Sun, 11 Feb 2018 12:47:42 -0600 Subject: [PATCH 2/3] compression: print warning if parallel requested but not supported --- src/libutil/compression.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index c509472b3..0b0ff1102 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,6 +1,7 @@ #include "compression.hh" #include "util.hh" #include "finally.hh" +#include "logging.hh" #include #include @@ -192,8 +193,8 @@ struct XzSink : CompressionSink XzSink(Sink & nextSink, const bool parallel) : nextSink(nextSink) { lzma_ret ret; -#ifdef HAVE_LZMA_MT if (parallel) { +#ifdef HAVE_LZMA_MT lzma_mt mt_options = {}; mt_options.flags = 0; mt_options.timeout = 300; // Using the same setting as the xz cmd line @@ -209,6 +210,9 @@ struct XzSink : CompressionSink ret = lzma_stream_encoder_mt( &strm, &mt_options); } else +#else + printMsg(lvlError, "Warning: parallel XZ compression requested but not supported, falling back to single-threaded compression"); + } #endif ret = lzma_easy_encoder( &strm, 6, LZMA_CHECK_CRC64); @@ -471,6 +475,9 @@ struct BrotliSink : CompressionSink ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel) { + if (parallel && method != "xz") + printMsg(lvlError, format("Warning: parallel compression requested but not supported for method '%1%', falling back to single-threaded compression") % method); + if (method == "none") return make_ref(nextSink); else if (method == "xz") From c6209030c424424ebd51283326d5e5df68a48533 Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Sun, 11 Feb 2018 13:23:31 -0600 Subject: [PATCH 3/3] compression: make parallel sink separate class --- src/libutil/compression.cc | 63 ++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 0b0ff1102..470c925ed 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -190,33 +190,9 @@ struct XzSink : CompressionSink lzma_stream strm = LZMA_STREAM_INIT; bool finished = false; - XzSink(Sink & nextSink, const bool parallel) : nextSink(nextSink) - { - lzma_ret ret; - if (parallel) { -#ifdef HAVE_LZMA_MT - lzma_mt mt_options = {}; - mt_options.flags = 0; - mt_options.timeout = 300; // Using the same setting as the xz cmd line - mt_options.preset = LZMA_PRESET_DEFAULT; - mt_options.filters = NULL; - mt_options.check = LZMA_CHECK_CRC64; - mt_options.threads = lzma_cputhreads(); - mt_options.block_size = 0; - if (mt_options.threads == 0) - mt_options.threads = 1; - // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the - // number of threads. - ret = lzma_stream_encoder_mt( - &strm, &mt_options); - } else -#else - printMsg(lvlError, "Warning: parallel XZ compression requested but not supported, falling back to single-threaded compression"); - } -#endif - ret = lzma_easy_encoder( - &strm, 6, LZMA_CHECK_CRC64); - + template + XzSink(Sink & nextSink, F&& initEncoder) : nextSink(nextSink) { + lzma_ret ret = initEncoder(); if (ret != LZMA_OK) throw CompressionError("unable to initialise lzma encoder"); // FIXME: apply the x86 BCJ filter? @@ -224,6 +200,9 @@ struct XzSink : CompressionSink strm.next_out = outbuf; strm.avail_out = sizeof(outbuf); } + XzSink(Sink & nextSink) : XzSink(nextSink, [this]() { + return lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64); + }) {} ~XzSink() { @@ -277,6 +256,27 @@ struct XzSink : CompressionSink } }; +#ifdef HAVE_LZMA_MT +struct ParallelXzSink : public XzSink +{ + ParallelXzSink(Sink &nextSink) : XzSink(nextSink, [this]() { + lzma_mt mt_options = {}; + mt_options.flags = 0; + mt_options.timeout = 300; // Using the same setting as the xz cmd line + mt_options.preset = LZMA_PRESET_DEFAULT; + mt_options.filters = NULL; + mt_options.check = LZMA_CHECK_CRC64; + mt_options.threads = lzma_cputhreads(); + mt_options.block_size = 0; + if (mt_options.threads == 0) + mt_options.threads = 1; + // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the + // number of threads. + return lzma_stream_encoder_mt(&strm, &mt_options); + }) {} +}; +#endif + struct BzipSink : CompressionSink { Sink & nextSink; @@ -475,13 +475,18 @@ struct BrotliSink : CompressionSink ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel) { - if (parallel && method != "xz") + if (parallel) { +#ifdef HAVE_LZMA_MT + if (method == "xz") + return make_ref(nextSink); +#endif printMsg(lvlError, format("Warning: parallel compression requested but not supported for method '%1%', falling back to single-threaded compression") % method); + } if (method == "none") return make_ref(nextSink); else if (method == "xz") - return make_ref(nextSink, parallel); + return make_ref(nextSink); else if (method == "bzip2") return make_ref(nextSink); else if (method == "br")