lix-project/lix
20
57
Fork 31
Code Issues 348 Code Review (Gerrit) Projects 3 Releases Wiki Activity

Merge pull request #1848 from AmineChikhaoui/parallel-xz

Browse source 
support multi threaded xz encoder
This commit is contained in:
Eelco Dolstra 2018-02-09 15:03:25 +01:00 committed by GitHub
parent ad97a21834 0685a6480a
commit 3d2d207aad
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 30 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/libstore/binary-cache-store.cc
View file

@ -149,7 +149,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
/* Compress the NAR. */ /* Compress the NAR. */
narInfo->compression = compression; narInfo->compression = compression;
auto now1 = std::chrono::steady_clock::now(); auto now1 = std::chrono::steady_clock::now();
auto narCompressed = compress(compression, *nar); auto narCompressed = compress(compression, *nar, parallelCompression);
auto now2 = std::chrono::steady_clock::now(); auto now2 = std::chrono::steady_clock::now();
narInfo->fileHash = hashString(htSHA256, *narCompressed); narInfo->fileHash = hashString(htSHA256, *narCompressed);
narInfo->fileSize = narCompressed->size(); narInfo->fileSize = narCompressed->size();

2
src/libstore/binary-cache-store.hh
View file

@ -19,6 +19,8 @@ public:
const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
const Setting<bool> parallelCompression{this, false, "parallel-compression",
"enable multi-threading compression, available for xz only currently"};
private: private:

32
src/libutil/compression.cc
View file

@ -151,10 +151,10 @@ static ref<std::string> decompressBrotli(const std::string & in)
#endif // HAVE_BROTLI #endif // HAVE_BROTLI
} }
ref<std::string> compress(const std::string & method, const std::string & in) ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel)
{ {
StringSink ssink; StringSink ssink;
auto sink = makeCompressionSink(method, ssink); auto sink = makeCompressionSink(method, ssink, parallel);
(*sink)(in); (*sink)(in);
sink->finish(); sink->finish();
return ssink.s; return ssink.s;
@ -189,10 +189,28 @@ struct XzSink : CompressionSink
lzma_stream strm = LZMA_STREAM_INIT; lzma_stream strm = LZMA_STREAM_INIT;
bool finished = false; bool finished = false;
XzSink(Sink & nextSink) : nextSink(nextSink) XzSink(Sink & nextSink, const bool parallel) : nextSink(nextSink)
{ {
lzma_ret ret = lzma_easy_encoder( lzma_ret ret;
&strm, 6, LZMA_CHECK_CRC64); if (parallel) {
lzma_mt mt_options = {};
mt_options.flags = 0;
mt_options.timeout = 300; // Using the same setting as the xz cmd line
mt_options.preset = LZMA_PRESET_DEFAULT;
mt_options.filters = NULL;
mt_options.check = LZMA_CHECK_CRC64;
mt_options.threads = lzma_cputhreads();
mt_options.block_size = 0;
if (mt_options.threads == 0)
mt_options.threads = 1;
// FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
// number of threads.
ret = lzma_stream_encoder_mt(
&strm, &mt_options);
} else
ret = lzma_easy_encoder(
&strm, 6, LZMA_CHECK_CRC64);
if (ret != LZMA_OK) if (ret != LZMA_OK)
throw CompressionError("unable to initialise lzma encoder"); throw CompressionError("unable to initialise lzma encoder");
// FIXME: apply the x86 BCJ filter? // FIXME: apply the x86 BCJ filter?
@ -449,12 +467,12 @@ struct BrotliSink : CompressionSink
}; };
#endif // HAVE_BROTLI #endif // HAVE_BROTLI
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink) ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
{ {
if (method == "none") if (method == "none")
return make_ref<NoneSink>(nextSink); return make_ref<NoneSink>(nextSink);
else if (method == "xz") else if (method == "xz")
return make_ref<XzSink>(nextSink); return make_ref<XzSink>(nextSink, parallel);
else if (method == "bzip2") else if (method == "bzip2")
return make_ref<BzipSink>(nextSink); return make_ref<BzipSink>(nextSink);
else if (method == "br") else if (method == "br")

4
src/libutil/compression.hh
View file

@ -8,7 +8,7 @@
namespace nix { namespace nix {
ref<std::string> compress(const std::string & method, const std::string & in); ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
ref<std::string> decompress(const std::string & method, const std::string & in); ref<std::string> decompress(const std::string & method, const std::string & in);
@ -17,7 +17,7 @@ struct CompressionSink : BufferedSink
virtual void finish() = 0; virtual void finish() = 0;
}; };
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink); ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false);
MakeError(UnknownCompressionMethod, Error); MakeError(UnknownCompressionMethod, Error);