From 4dde0b0562115525f76943653357410835910a99 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 29 Apr 2016 17:43:37 +0200 Subject: [PATCH] BinaryCacheStore: Support bzip2 compression --- src/libstore/binary-cache-store.cc | 5 +- src/libutil/compression.cc | 168 +++++++++++++++++++++-------- src/libutil/local.mk | 2 +- 3 files changed, 130 insertions(+), 45 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index ff717a5dc..411d10130 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -87,7 +87,10 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info, ref n % duration); /* Atomically write the NAR file. */ - narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar.xz"; + narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar" + + (compression == "xz" ? ".xz" : + compression == "bzip2" ? ".bz2" : + ""); if (!fileExists(narInfo->url)) { stats.narWrite++; upsertFile(narInfo->url, *narCompressed); diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index e8a820d30..4d15d2acd 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,51 +1,88 @@ #include "compression.hh" #include "util.hh" +#include "finally.hh" #include +#include #include +#include namespace nix { -/* RAII wrapper around lzma_stream. */ -struct LzmaStream -{ - lzma_stream strm; - LzmaStream() : strm(LZMA_STREAM_INIT) { }; - ~LzmaStream() { lzma_end(&strm); }; - lzma_stream & operator()() { return strm; } -}; - static ref compressXZ(const std::string & in) { - LzmaStream strm; + lzma_stream strm; // FIXME: apply the x86 BCJ filter? lzma_ret ret = lzma_easy_encoder( - &strm(), 6, LZMA_CHECK_CRC64); + &strm, 6, LZMA_CHECK_CRC64); if (ret != LZMA_OK) throw Error("unable to initialise lzma encoder"); + Finally free([&]() { lzma_end(&strm); }); + lzma_action action = LZMA_RUN; uint8_t outbuf[BUFSIZ]; ref res = make_ref(); - strm().next_in = (uint8_t *) in.c_str(); - strm().avail_in = in.size(); - strm().next_out = outbuf; - strm().avail_out = sizeof(outbuf); + strm.next_in = (uint8_t *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); while (true) { checkInterrupt(); - if (strm().avail_in == 0) + if (strm.avail_in == 0) action = LZMA_FINISH; - lzma_ret ret = lzma_code(&strm(), action); + lzma_ret ret = lzma_code(&strm, action); - if (strm().avail_out == 0 || ret == LZMA_STREAM_END) { - res->append((char *) outbuf, sizeof(outbuf) - strm().avail_out); - strm().next_out = outbuf; - strm().avail_out = sizeof(outbuf); + if (strm.avail_out == 0 || ret == LZMA_STREAM_END) { + res->append((char *) outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + if (ret == LZMA_STREAM_END) + return res; + + if (ret != LZMA_OK) + throw Error("error while compressing xz file"); + } +} + +static ref decompressXZ(const std::string & in) +{ + lzma_stream strm; + + lzma_ret ret = lzma_stream_decoder( + &strm, UINT64_MAX, LZMA_CONCATENATED); + if (ret != LZMA_OK) + throw Error("unable to initialise lzma decoder"); + + Finally free([&]() { lzma_end(&strm); }); + + lzma_action action = LZMA_RUN; + uint8_t outbuf[BUFSIZ]; + ref res = make_ref(); + strm.next_in = (uint8_t *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + + while (true) { + checkInterrupt(); + + if (strm.avail_in == 0) + action = LZMA_FINISH; + + lzma_ret ret = lzma_code(&strm, action); + + if (strm.avail_out == 0 || ret == LZMA_STREAM_END) { + res->append((char *) outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); } if (ret == LZMA_STREAM_END) @@ -56,42 +93,83 @@ static ref compressXZ(const std::string & in) } } -static ref decompressXZ(const std::string & in) +static ref compressBzip2(const std::string & in) { - LzmaStream strm; + bz_stream strm; + memset(&strm, 0, sizeof(strm)); - lzma_ret ret = lzma_stream_decoder( - &strm(), UINT64_MAX, LZMA_CONCATENATED); - if (ret != LZMA_OK) - throw Error("unable to initialise lzma decoder"); + int ret = BZ2_bzCompressInit(&strm, 9, 0, 30); + if (ret != BZ_OK) + throw Error("unable to initialise bzip2 encoder"); - lzma_action action = LZMA_RUN; - uint8_t outbuf[BUFSIZ]; + Finally free([&]() { BZ2_bzCompressEnd(&strm); }); + + int action = BZ_RUN; + char outbuf[BUFSIZ]; ref res = make_ref(); - strm().next_in = (uint8_t *) in.c_str(); - strm().avail_in = in.size(); - strm().next_out = outbuf; - strm().avail_out = sizeof(outbuf); + strm.next_in = (char *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); while (true) { checkInterrupt(); - if (strm().avail_in == 0) - action = LZMA_FINISH; + if (strm.avail_in == 0) + action = BZ_FINISH; - lzma_ret ret = lzma_code(&strm(), action); + int ret = BZ2_bzCompress(&strm, action); - if (strm().avail_out == 0 || ret == LZMA_STREAM_END) { - res->append((char *) outbuf, sizeof(outbuf) - strm().avail_out); - strm().next_out = outbuf; - strm().avail_out = sizeof(outbuf); + if (strm.avail_out == 0 || ret == BZ_STREAM_END) { + res->append(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); } - if (ret == LZMA_STREAM_END) + if (ret == BZ_STREAM_END) return res; - if (ret != LZMA_OK) - throw Error("error while decompressing xz file"); + if (ret != BZ_OK && ret != BZ_FINISH_OK) + Error("error while compressing bzip2 file"); + } + + return res; +} + +static ref decompressBzip2(const std::string & in) +{ + bz_stream strm; + memset(&strm, 0, sizeof(strm)); + + int ret = BZ2_bzDecompressInit(&strm, 0, 0); + if (ret != BZ_OK) + throw Error("unable to initialise bzip2 decoder"); + + Finally free([&]() { BZ2_bzDecompressEnd(&strm); }); + + char outbuf[BUFSIZ]; + ref res = make_ref(); + strm.next_in = (char *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + + while (true) { + checkInterrupt(); + + int ret = BZ2_bzDecompress(&strm); + + if (strm.avail_out == 0 || ret == BZ_STREAM_END) { + res->append(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + if (ret == BZ_STREAM_END) + return res; + + if (ret != BZ_OK) + throw Error("error while decompressing bzip2 file"); } } @@ -101,6 +179,8 @@ ref compress(const std::string & method, ref in) return in; else if (method == "xz") return compressXZ(*in); + else if (method == "bzip2") + return compressBzip2(*in); else throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); } @@ -111,6 +191,8 @@ ref decompress(const std::string & method, ref in) return in; else if (method == "xz") return decompressXZ(*in); + else if (method == "bzip2") + return decompressBzip2(*in); else throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); } diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 2e5d2672e..98cad00d6 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,6 +6,6 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -libutil_LDFLAGS = -llzma -pthread $(OPENSSL_LIBS) +libutil_LDFLAGS = -llzma -lbz2 -pthread $(OPENSSL_LIBS) libutil_LIBS = libformat