Merge branch 'feature/comp-level' of https://github.com/tomberek/nix

This commit is contained in:
Eelco Dolstra 2021-10-13 10:45:44 +02:00
commit abd685d373
6 changed files with 47 additions and 12 deletions

View file

@ -149,7 +149,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
{ {
FdSink fileSink(fdTemp.get()); FdSink fileSink(fdTemp.get());
TeeSink teeSinkCompressed { fileSink, fileHashSink }; TeeSink teeSinkCompressed { fileSink, fileHashSink };
auto compressionSink = makeCompressionSink(compression, teeSinkCompressed); auto compressionSink = makeCompressionSink(compression, teeSinkCompressed, parallelCompression, compressionLevel);
TeeSink teeSinkUncompressed { *compressionSink, narHashSink }; TeeSink teeSinkUncompressed { *compressionSink, narHashSink };
TeeSource teeSource { narSource, teeSinkUncompressed }; TeeSource teeSource { narSource, teeSinkUncompressed };
narAccessor = makeNarAccessor(teeSource); narAccessor = makeNarAccessor(teeSource);

View file

@ -15,13 +15,17 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
{ {
using StoreConfig::StoreConfig; using StoreConfig::StoreConfig;
const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', 'gzip', 'zstd', or 'none')"};
const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"}; const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"}; const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"};
const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression", const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
"enable multi-threading compression, available for xz only currently"}; "enable multi-threading compression for NARs, available for xz and zstd only currently"};
const Setting<int> compressionLevel{(StoreConfig*) this, -1, "compression-level",
"specify 'preset level' of compression to be used with NARs: "
"meaning and accepted range of values depends on compression method selected, "
"other than -1 which we reserve to indicate Nix defaults should be used"};
}; };
class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store

View file

@ -16,6 +16,8 @@
namespace nix { namespace nix {
static const int COMPRESSION_LEVEL_DEFAULT = -1;
// Don't feed brotli too much at once. // Don't feed brotli too much at once.
struct ChunkedCompressionSink : CompressionSink struct ChunkedCompressionSink : CompressionSink
{ {
@ -65,14 +67,17 @@ struct ArchiveCompressionSink : CompressionSink
Sink & nextSink; Sink & nextSink;
struct archive * archive; struct archive * archive;
ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel) : nextSink(nextSink) { ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel, int _level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink) {
archive = archive_write_new(); archive = archive_write_new();
if (!archive) throw Error("failed to initialize libarchive"); if (!archive) throw Error("failed to initialize libarchive");
check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)"); check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)");
check(archive_write_set_format_raw(archive)); check(archive_write_set_format_raw(archive));
if (format == "xz" && parallel) { if (parallel) {
check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0")); check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0"));
} }
if (_level != COMPRESSION_LEVEL_DEFAULT){
check(archive_write_set_filter_option(archive, format.c_str(), "compression-level", std::to_string(_level).c_str()));
}
// disable internal buffering // disable internal buffering
check(archive_write_set_bytes_per_block(archive, 0)); check(archive_write_set_bytes_per_block(archive, 0));
// disable output padding // disable output padding
@ -126,7 +131,10 @@ private:
struct NoneSink : CompressionSink struct NoneSink : CompressionSink
{ {
Sink & nextSink; Sink & nextSink;
NoneSink(Sink & nextSink) : nextSink(nextSink) { } NoneSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink) {
if (level != COMPRESSION_LEVEL_DEFAULT)
printError("Warning: requested compression level '%d' not supported by compression method 'none'", level);
}
void finish() override { flush(); } void finish() override { flush(); }
void write(std::string_view data) override { nextSink(data); } void write(std::string_view data) override { nextSink(data); }
}; };
@ -257,13 +265,13 @@ struct BrotliCompressionSink : ChunkedCompressionSink
} }
}; };
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel) ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level)
{ {
std::vector<std::string> la_supports = { std::vector<std::string> la_supports = {
"bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd" "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"
}; };
if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) { if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) {
return make_ref<ArchiveCompressionSink>(nextSink, method, parallel); return make_ref<ArchiveCompressionSink>(nextSink, method, parallel, level);
} }
if (method == "none") if (method == "none")
return make_ref<NoneSink>(nextSink); return make_ref<NoneSink>(nextSink);
@ -273,10 +281,10 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next
throw UnknownCompressionMethod("unknown compression method '%s'", method); throw UnknownCompressionMethod("unknown compression method '%s'", method);
} }
ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel) ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel, int level)
{ {
StringSink ssink; StringSink ssink;
auto sink = makeCompressionSink(method, ssink, parallel); auto sink = makeCompressionSink(method, ssink, parallel, level);
(*sink)(in); (*sink)(in);
sink->finish(); sink->finish();
return ssink.s; return ssink.s;

View file

@ -19,9 +19,9 @@ ref<std::string> decompress(const std::string & method, const std::string & in);
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink); std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false); ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false, int level = -1);
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false); ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1);
MakeError(UnknownCompressionMethod, Error); MakeError(UnknownCompressionMethod, Error);

View file

@ -0,0 +1,22 @@
source common.sh
clearStore
clearCache
outPath=$(nix-build dependencies.nix --no-out-link)
cacheURI="file://$cacheDir?compression=xz&compression-level=0"
nix copy --to $cacheURI $outPath
FILESIZES=$(cat ${cacheDir}/*.narinfo | awk '/FileSize: /{sum+=$2}END{print sum}')
clearCache
cacheURI="file://$cacheDir?compression=xz&compression-level=5"
nix copy --to $cacheURI $outPath
FILESIZES2=$(cat ${cacheDir}/*.narinfo | awk '/FileSize: /{sum+=$2}END{print sum}')
[[ $FILESIZES -gt $FILESIZES2 ]]

View file

@ -33,6 +33,7 @@ nix_tests = \
shell.sh \ shell.sh \
brotli.sh \ brotli.sh \
zstd.sh \ zstd.sh \
compression-levels.sh \
pure-eval.sh \ pure-eval.sh \
check.sh \ check.sh \
plugins.sh \ plugins.sh \