Initial gzip support

Closes #3256
This commit is contained in:
Tom Bereknyei 2019-12-13 03:29:33 -05:00
parent f800d450b7
commit c6295a3afd
5 changed files with 88 additions and 1 deletions

View file

@ -209,6 +209,11 @@ PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt], AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
[AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])]) [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
# Look for zlib, a required dependency.
PKG_CHECK_MODULES([ZLIB], [zlib], [CXXFLAGS="$ZLIB_CFLAGS $CXXFLAGS"])
AC_CHECK_HEADER([zlib.h],[:],[AC_MSG_ERROR([could not find the zlib.h header])])
LDFLAGS="-lz $LDFLAGS"
# Look for libbrotli{enc,dec}. # Look for libbrotli{enc,dec}.
PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]) PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])

View file

@ -47,7 +47,7 @@ rec {
buildDeps = buildDeps =
[ curl [ curl
bzip2 xz brotli editline bzip2 xz brotli zlib editline
openssl pkgconfig sqlite boehmgc openssl pkgconfig sqlite boehmgc
boost boost
nlohmann_json nlohmann_json

View file

@ -11,6 +11,8 @@
#include <brotli/decode.h> #include <brotli/decode.h>
#include <brotli/encode.h> #include <brotli/encode.h>
#include <zlib.h>
#include <iostream> #include <iostream>
namespace nix { namespace nix {
@ -42,6 +44,66 @@ struct NoneSink : CompressionSink
void write(const unsigned char * data, size_t len) override { nextSink(data, len); } void write(const unsigned char * data, size_t len) override { nextSink(data, len); }
}; };
struct GzipDecompressionSink : CompressionSink
{
Sink & nextSink;
z_stream strm;
bool finished = false;
uint8_t outbuf[BUFSIZ];
GzipDecompressionSink(Sink & nextSink) : nextSink(nextSink)
{
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.avail_in = 0;
strm.next_in = Z_NULL;
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
// Enable gzip and zlib decoding (+32) with 15 windowBits
int ret = inflateInit2(&strm,15+32);
if (ret != Z_OK)
throw CompressionError("unable to initialise gzip encoder");
}
~GzipDecompressionSink()
{
inflateEnd(&strm);
}
void finish() override
{
CompressionSink::flush();
write(nullptr, 0);
}
void write(const unsigned char * data, size_t len) override
{
assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
strm.next_in = (Bytef *) data;
strm.avail_in = len;
while (!finished && (!data || strm.avail_in)) {
checkInterrupt();
int ret = inflate(&strm,Z_SYNC_FLUSH);
if (ret != Z_OK && ret != Z_STREAM_END)
throw CompressionError("error while decompressing gzip file: %d: %d: %d",ret, len, strm.avail_in);
finished = ret == Z_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
strm.next_out = (Bytef *) outbuf;
strm.avail_out = sizeof(outbuf);
}
}
}
};
struct XzDecompressionSink : CompressionSink struct XzDecompressionSink : CompressionSink
{ {
Sink & nextSink; Sink & nextSink;
@ -215,6 +277,8 @@ ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & ne
return make_ref<XzDecompressionSink>(nextSink); return make_ref<XzDecompressionSink>(nextSink);
else if (method == "bzip2") else if (method == "bzip2")
return make_ref<BzipDecompressionSink>(nextSink); return make_ref<BzipDecompressionSink>(nextSink);
else if (method == "gzip")
return make_ref<GzipDecompressionSink>(nextSink);
else if (method == "br") else if (method == "br")
return make_ref<BrotliDecompressionSink>(nextSink); return make_ref<BrotliDecompressionSink>(nextSink);
else else

View file

@ -24,6 +24,7 @@ void unpackTarfile(const Path & tarFile, const Path & destDir,
auto decompressor = auto decompressor =
// FIXME: add .gz support // FIXME: add .gz support
hasSuffix(*baseName, ".bz2") ? makeDecompressionSink("bzip2", sink) : hasSuffix(*baseName, ".bz2") ? makeDecompressionSink("bzip2", sink) :
hasSuffix(*baseName, ".gz") ? makeDecompressionSink("gzip", sink) :
hasSuffix(*baseName, ".xz") ? makeDecompressionSink("xz", sink) : hasSuffix(*baseName, ".xz") ? makeDecompressionSink("xz", sink) :
makeDecompressionSink("none", sink); makeDecompressionSink("none", sink);
readFile(tarFile, *decompressor); readFile(tarFile, *decompressor);

View file

@ -26,3 +26,20 @@ nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-ta
(! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball.tar.xz) (! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball.tar.xz)
nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball.tar.xz -I fnord=. nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball.tar.xz -I fnord=.
tarball=$TEST_ROOT/tarball.tar.gz
(cd $TEST_ROOT && tar c tarball) | gzip > $tarball
nix-env -f file://$tarball -qa --out-path | grep -q dependencies
nix-build -o $TEST_ROOT/result file://$tarball
nix-build -o $TEST_ROOT/result '<foo>' -I foo=file://$tarball
nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)"
nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar.gz
nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball.tar.gz
(! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball.tar.gz)
nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball.tar.gz -I fnord=.