lix-project/lix
20
57
Fork 31
Code Issues 349 Code Review (Gerrit) Projects 3 Releases Wiki Activity

BinaryCacheStore: Support bzip2 compression

Browse source
This commit is contained in:
Eelco Dolstra 2016-04-29 17:43:37 +02:00
parent 6963de2091
commit 4dde0b0562
3 changed files with 130 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
src/libstore/binary-cache-store.cc
View file

@ -87,7 +87,10 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info, ref<std::string> n
% duration); % duration);
/* Atomically write the NAR file. */ /* Atomically write the NAR file. */
narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar.xz"; narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
"");
if (!fileExists(narInfo->url)) { if (!fileExists(narInfo->url)) {
stats.narWrite++; stats.narWrite++;
upsertFile(narInfo->url, *narCompressed); upsertFile(narInfo->url, *narCompressed);

168
src/libutil/compression.cc
View file

@ -1,51 +1,88 @@
#include "compression.hh" #include "compression.hh"
#include "util.hh" #include "util.hh"
#include "finally.hh"
#include <lzma.h> #include <lzma.h>
#include <bzlib.h>
#include <cstdio> #include <cstdio>
#include <cstring>
namespace nix { namespace nix {
/* RAII wrapper around lzma_stream. */
struct LzmaStream
{
lzma_stream strm;
LzmaStream() : strm(LZMA_STREAM_INIT) { };
~LzmaStream() { lzma_end(&strm); };
lzma_stream & operator()() { return strm; }
};
static ref<std::string> compressXZ(const std::string & in) static ref<std::string> compressXZ(const std::string & in)
{ {
LzmaStream strm; lzma_stream strm;
// FIXME: apply the x86 BCJ filter? // FIXME: apply the x86 BCJ filter?
lzma_ret ret = lzma_easy_encoder( lzma_ret ret = lzma_easy_encoder(
&strm(), 6, LZMA_CHECK_CRC64); &strm, 6, LZMA_CHECK_CRC64);
if (ret != LZMA_OK) if (ret != LZMA_OK)
throw Error("unable to initialise lzma encoder"); throw Error("unable to initialise lzma encoder");
Finally free([&]() { lzma_end(&strm); });
lzma_action action = LZMA_RUN; lzma_action action = LZMA_RUN;
uint8_t outbuf[BUFSIZ]; uint8_t outbuf[BUFSIZ];
ref<std::string> res = make_ref<std::string>(); ref<std::string> res = make_ref<std::string>();
strm().next_in = (uint8_t *) in.c_str(); strm.next_in = (uint8_t *) in.c_str();
strm().avail_in = in.size(); strm.avail_in = in.size();
strm().next_out = outbuf; strm.next_out = outbuf;
strm().avail_out = sizeof(outbuf); strm.avail_out = sizeof(outbuf);
while (true) { while (true) {
checkInterrupt(); checkInterrupt();
if (strm().avail_in == 0) if (strm.avail_in == 0)
action = LZMA_FINISH; action = LZMA_FINISH;
lzma_ret ret = lzma_code(&strm(), action); lzma_ret ret = lzma_code(&strm, action);
if (strm().avail_out == 0 || ret == LZMA_STREAM_END) { if (strm.avail_out == 0 || ret == LZMA_STREAM_END) {
res->append((char *) outbuf, sizeof(outbuf) - strm().avail_out); res->append((char *) outbuf, sizeof(outbuf) - strm.avail_out);
strm().next_out = outbuf; strm.next_out = outbuf;
strm().avail_out = sizeof(outbuf); strm.avail_out = sizeof(outbuf);
}
if (ret == LZMA_STREAM_END)
return res;
if (ret != LZMA_OK)
throw Error("error while compressing xz file");
}
}
static ref<std::string> decompressXZ(const std::string & in)
{
lzma_stream strm;
lzma_ret ret = lzma_stream_decoder(
&strm, UINT64_MAX, LZMA_CONCATENATED);
if (ret != LZMA_OK)
throw Error("unable to initialise lzma decoder");
Finally free([&]() { lzma_end(&strm); });
lzma_action action = LZMA_RUN;
uint8_t outbuf[BUFSIZ];
ref<std::string> res = make_ref<std::string>();
strm.next_in = (uint8_t *) in.c_str();
strm.avail_in = in.size();
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
while (true) {
checkInterrupt();
if (strm.avail_in == 0)
action = LZMA_FINISH;
lzma_ret ret = lzma_code(&strm, action);
if (strm.avail_out == 0 || ret == LZMA_STREAM_END) {
res->append((char *) outbuf, sizeof(outbuf) - strm.avail_out);
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
} }
if (ret == LZMA_STREAM_END) if (ret == LZMA_STREAM_END)
@ -56,42 +93,83 @@ static ref<std::string> compressXZ(const std::string & in)
} }
} }
static ref<std::string> decompressXZ(const std::string & in) static ref<std::string> compressBzip2(const std::string & in)
{ {
LzmaStream strm; bz_stream strm;
memset(&strm, 0, sizeof(strm));
lzma_ret ret = lzma_stream_decoder( int ret = BZ2_bzCompressInit(&strm, 9, 0, 30);
&strm(), UINT64_MAX, LZMA_CONCATENATED); if (ret != BZ_OK)
if (ret != LZMA_OK) throw Error("unable to initialise bzip2 encoder");
throw Error("unable to initialise lzma decoder");
lzma_action action = LZMA_RUN; Finally free([&]() { BZ2_bzCompressEnd(&strm); });
uint8_t outbuf[BUFSIZ];
int action = BZ_RUN;
char outbuf[BUFSIZ];
ref<std::string> res = make_ref<std::string>(); ref<std::string> res = make_ref<std::string>();
strm().next_in = (uint8_t *) in.c_str(); strm.next_in = (char *) in.c_str();
strm().avail_in = in.size(); strm.avail_in = in.size();
strm().next_out = outbuf; strm.next_out = outbuf;
strm().avail_out = sizeof(outbuf); strm.avail_out = sizeof(outbuf);
while (true) { while (true) {
checkInterrupt(); checkInterrupt();
if (strm().avail_in == 0) if (strm.avail_in == 0)
action = LZMA_FINISH; action = BZ_FINISH;
lzma_ret ret = lzma_code(&strm(), action); int ret = BZ2_bzCompress(&strm, action);
if (strm().avail_out == 0 || ret == LZMA_STREAM_END) { if (strm.avail_out == 0 || ret == BZ_STREAM_END) {
res->append((char *) outbuf, sizeof(outbuf) - strm().avail_out); res->append(outbuf, sizeof(outbuf) - strm.avail_out);
strm().next_out = outbuf; strm.next_out = outbuf;
strm().avail_out = sizeof(outbuf); strm.avail_out = sizeof(outbuf);
} }
if (ret == LZMA_STREAM_END) if (ret == BZ_STREAM_END)
return res; return res;
if (ret != LZMA_OK) if (ret != BZ_OK && ret != BZ_FINISH_OK)
throw Error("error while decompressing xz file"); Error("error while compressing bzip2 file");
}
return res;
}
static ref<std::string> decompressBzip2(const std::string & in)
{
bz_stream strm;
memset(&strm, 0, sizeof(strm));
int ret = BZ2_bzDecompressInit(&strm, 0, 0);
if (ret != BZ_OK)
throw Error("unable to initialise bzip2 decoder");
Finally free([&]() { BZ2_bzDecompressEnd(&strm); });
char outbuf[BUFSIZ];
ref<std::string> res = make_ref<std::string>();
strm.next_in = (char *) in.c_str();
strm.avail_in = in.size();
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
while (true) {
checkInterrupt();
int ret = BZ2_bzDecompress(&strm);
if (strm.avail_out == 0 || ret == BZ_STREAM_END) {
res->append(outbuf, sizeof(outbuf) - strm.avail_out);
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
if (ret == BZ_STREAM_END)
return res;
if (ret != BZ_OK)
throw Error("error while decompressing bzip2 file");
} }
} }
@ -101,6 +179,8 @@ ref<std::string> compress(const std::string & method, ref<std::string> in)
return in; return in;
else if (method == "xz") else if (method == "xz")
return compressXZ(*in); return compressXZ(*in);
else if (method == "bzip2")
return compressBzip2(*in);
else else
throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method);
} }
@ -111,6 +191,8 @@ ref<std::string> decompress(const std::string & method, ref<std::string> in)
return in; return in;
else if (method == "xz") else if (method == "xz")
return decompressXZ(*in); return decompressXZ(*in);
else if (method == "bzip2")
return decompressBzip2(*in);
else else
throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method);
} }

2
src/libutil/local.mk
View file

@ -6,6 +6,6 @@ libutil_DIR := $(d)
libutil_SOURCES := $(wildcard $(d)/*.cc) libutil_SOURCES := $(wildcard $(d)/*.cc)
libutil_LDFLAGS = -llzma -pthread $(OPENSSL_LIBS) libutil_LDFLAGS = -llzma -lbz2 -pthread $(OPENSSL_LIBS)
libutil_LIBS = libformat libutil_LIBS = libformat