qyriad/lix
1
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

Drop libbz2 / zlib / lzma dependency + style fixes

Browse source
This commit is contained in:
Eelco Dolstra 2021-04-15 13:51:00 +02:00
parent 6fb7582413
commit 15f4d4fd43
8 changed files with 69 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Makefile.config.in
View file

@ -15,7 +15,6 @@ LDFLAGS = @LDFLAGS@
LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
LIBCURL_LIBS = @LIBCURL_LIBS@ LIBCURL_LIBS = @LIBCURL_LIBS@
LIBLZMA_LIBS = @LIBLZMA_LIBS@
OPENSSL_LIBS = @OPENSSL_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@
LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@ LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@
PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_NAME = @PACKAGE_NAME@

15
configure.ac
View file

@ -172,11 +172,6 @@ fi
PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
# Look for libbz2, a required dependency.
AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
[AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://sourceware.org/bzip2/.])])
AC_CHECK_HEADERS([bzlib.h], [true],
[AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://sourceware.org/bzip2/.])])
# Checks for libarchive # Checks for libarchive
PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"]) PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"])
# Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed # Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed
@ -205,16 +200,6 @@ PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLA
# Look for libsodium, an optional dependency. # Look for libsodium, an optional dependency.
PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
# Look for liblzma, a required dependency.
PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
[AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
# Look for zlib, a required dependency.
PKG_CHECK_MODULES([ZLIB], [zlib], [CXXFLAGS="$ZLIB_CFLAGS $CXXFLAGS"])
AC_CHECK_HEADER([zlib.h],[:],[AC_MSG_ERROR([could not find the zlib.h header])])
LDFLAGS="-lz $LDFLAGS"
# Look for libbrotli{enc,dec}. # Look for libbrotli{enc,dec}.
PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]) PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])

2
flake.nix
View file

@ -82,7 +82,7 @@
buildDeps = buildDeps =
[ curl [ curl
bzip2 xz brotli zlib editline bzip2 xz brotli editline
openssl sqlite openssl sqlite
libarchive libarchive
boost boost

60
src/libutil/compression.cc
View file

@ -4,8 +4,6 @@
#include "finally.hh" #include "finally.hh"
#include "logging.hh" #include "logging.hh"
#include <lzma.h>
#include <bzlib.h>
#include <archive.h> #include <archive.h>
#include <archive_entry.h> #include <archive_entry.h>
#include <cstdio> #include <cstdio>
@ -45,12 +43,13 @@ struct ArchiveDecompressionSource : Source
ArchiveDecompressionSource(Source & src) : src(src) {} ArchiveDecompressionSource(Source & src) : src(src) {}
~ArchiveDecompressionSource() override {} ~ArchiveDecompressionSource() override {}
size_t read(char * data, size_t len) override { size_t read(char * data, size_t len) override {
struct archive_entry* ae; struct archive_entry * ae;
if (!archive) { if (!archive) {
archive = std::make_unique<TarArchive>(src, true); archive = std::make_unique<TarArchive>(src, true);
this->archive->check(archive_read_next_header(this->archive->archive, &ae), "Failed to read header (%s)"); this->archive->check(archive_read_next_header(this->archive->archive, &ae),
"failed to read header (%s)");
if (archive_filter_count(this->archive->archive) < 2) { if (archive_filter_count(this->archive->archive) < 2) {
throw CompressionError("Input compression not recognized."); throw CompressionError("input compression not recognized");
} }
} }
ssize_t result = archive_read_data(this->archive->archive, data, len); ssize_t result = archive_read_data(this->archive->archive, data, len);
@ -58,18 +57,20 @@ struct ArchiveDecompressionSource : Source
if (result == 0) { if (result == 0) {
throw EndOfFile("reached end of compressed file"); throw EndOfFile("reached end of compressed file");
} }
this->archive->check(result, "Failed to read compressed data (%s)"); this->archive->check(result, "failed to read compressed data (%s)");
return result; return result;
} }
}; };
struct ArchiveCompressionSink : CompressionSink struct ArchiveCompressionSink : CompressionSink
{ {
Sink & nextSink; Sink & nextSink;
struct archive* archive; struct archive * archive;
ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel) : nextSink(nextSink) { ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel) : nextSink(nextSink) {
archive = archive_write_new(); archive = archive_write_new();
if (!archive) throw Error("failed to initialize libarchive"); if (!archive) throw Error("failed to initialize libarchive");
check(archive_write_add_filter_by_name(archive, format.c_str()), "Couldn't initialize compression (%s)"); check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)");
check(archive_write_set_format_raw(archive)); check(archive_write_set_format_raw(archive));
if (format == "xz" && parallel) { if (format == "xz" && parallel) {
check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0")); check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0"));
@ -80,34 +81,46 @@ struct ArchiveCompressionSink : CompressionSink
check(archive_write_set_bytes_in_last_block(archive, 1)); check(archive_write_set_bytes_in_last_block(archive, 1));
open(); open();
} }
~ArchiveCompressionSink() override {
~ArchiveCompressionSink() override
{
if (archive) archive_write_free(archive); if (archive) archive_write_free(archive);
} }
void finish() override {
void finish() override
{
flush(); flush();
check(archive_write_close(archive)); check(archive_write_close(archive));
} }
void check(int err, const char *reason="Failed to compress (%s)") {
void check(int err, const std::string & reason = "failed to compress (%s)")
{
if (err == ARCHIVE_EOF) if (err == ARCHIVE_EOF)
throw EndOfFile("reached end of archive"); throw EndOfFile("reached end of archive");
else if (err != ARCHIVE_OK) else if (err != ARCHIVE_OK)
throw Error(reason, archive_error_string(this->archive)); throw Error(reason, archive_error_string(this->archive));
} }
void write(std::string_view data) override {
void write(std::string_view data) override
{
ssize_t result = archive_write_data(archive, data.data(), data.length()); ssize_t result = archive_write_data(archive, data.data(), data.length());
if (result <= 0) check(result); if (result <= 0) check(result);
} }
private: private:
void open() { void open()
check(archive_write_open(archive, this, NULL, ArchiveCompressionSink::callback_write, NULL)); {
struct archive_entry *ae = archive_entry_new(); check(archive_write_open(archive, this, nullptr, ArchiveCompressionSink::callback_write, nullptr));
auto ae = archive_entry_new();
archive_entry_set_filetype(ae, AE_IFREG); archive_entry_set_filetype(ae, AE_IFREG);
check(archive_write_header(archive, ae)); check(archive_write_header(archive, ae));
archive_entry_free(ae); archive_entry_free(ae);
} }
static ssize_t callback_write(struct archive *archive, void *_self, const void *buffer, size_t length) {
ArchiveCompressionSink *self = (ArchiveCompressionSink *)_self; static ssize_t callback_write(struct archive * archive, void * _self, const void * buffer, size_t length)
self->nextSink({(const char*)buffer, length}); {
auto self = (ArchiveCompressionSink *) _self;
self->nextSink({(const char *) buffer, length});
return length; return length;
} }
}; };
@ -193,17 +206,17 @@ std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Si
else if (method == "br") else if (method == "br")
return std::make_unique<BrotliDecompressionSink>(nextSink); return std::make_unique<BrotliDecompressionSink>(nextSink);
else else
return sourceToSink([&](Source & source) { return sourceToSink([&](Source & source) {
auto decompressionSource = makeDecompressionSource(source); auto decompressionSource = makeDecompressionSource(source);
decompressionSource->drainInto(nextSink); decompressionSource->drainInto(nextSink);
}); });
} }
struct BrotliCompressionSink : ChunkedCompressionSink struct BrotliCompressionSink : ChunkedCompressionSink
{ {
Sink & nextSink; Sink & nextSink;
uint8_t outbuf[BUFSIZ]; uint8_t outbuf[BUFSIZ];
BrotliEncoderState *state; BrotliEncoderState * state;
bool finished = false; bool finished = false;
BrotliCompressionSink(Sink & nextSink) : nextSink(nextSink) BrotliCompressionSink(Sink & nextSink) : nextSink(nextSink)
@ -251,6 +264,7 @@ struct BrotliCompressionSink : ChunkedCompressionSink
} }
} }
}; };
std::unique_ptr<Source> makeDecompressionSource(Source & prev) { std::unique_ptr<Source> makeDecompressionSource(Source & prev) {
return std::unique_ptr<Source>(new ArchiveDecompressionSource(prev)); return std::unique_ptr<Source>(new ArchiveDecompressionSource(prev));
} }

6
src/libutil/compression.hh
View file

@ -10,9 +10,9 @@ namespace nix {
struct CompressionSink : BufferedSink, FinishSink struct CompressionSink : BufferedSink, FinishSink
{ {
using BufferedSink::operator (); using BufferedSink::operator ();
using BufferedSink::write; using BufferedSink::write;
using FinishSink::finish; using FinishSink::finish;
}; };
std::unique_ptr<Source> makeDecompressionSource(Source & prev); std::unique_ptr<Source> makeDecompressionSource(Source & prev);

2
src/libutil/local.mk
View file

@ -6,7 +6,7 @@ libutil_DIR := $(d)
libutil_SOURCES := $(wildcard $(d)/*.cc) libutil_SOURCES := $(wildcard $(d)/*.cc)
libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context libutil_LDFLAGS = -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
ifeq ($(HAVE_LIBCPUID), 1) ifeq ($(HAVE_LIBCPUID), 1)
libutil_LDFLAGS += -lcpuid libutil_LDFLAGS += -lcpuid

37
src/libutil/tarfile.cc
View file

@ -5,38 +5,41 @@
#include "tarfile.hh" #include "tarfile.hh"
namespace nix { namespace nix {
static int callback_open(struct archive *, void *self) {
static int callback_open(struct archive *, void * self)
{
return ARCHIVE_OK; return ARCHIVE_OK;
} }
static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer) { static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer)
TarArchive *self = (TarArchive *)_self; {
auto self = (TarArchive *) _self;
*buffer = self->buffer.data(); *buffer = self->buffer.data();
try { try {
return self->source->read((char *) self->buffer.data(), 4096); return self->source->read((char *) self->buffer.data(), 4096);
} catch (EndOfFile &) { } catch (EndOfFile &) {
return 0; return 0;
} catch (std::exception &err) { } catch (std::exception & err) {
archive_set_error(archive, EIO, "Source threw exception: %s", err.what()); archive_set_error(archive, EIO, "Source threw exception: %s", err.what());
return -1; return -1;
} }
} }
static int callback_close(struct archive *, void *self) { static int callback_close(struct archive *, void * self)
{
return ARCHIVE_OK; return ARCHIVE_OK;
} }
void TarArchive::check(int err, const char *reason) void TarArchive::check(int err, const std::string & reason)
{ {
if (err == ARCHIVE_EOF) if (err == ARCHIVE_EOF)
throw EndOfFile("reached end of archive"); throw EndOfFile("reached end of archive");
else if (err != ARCHIVE_OK) else if (err != ARCHIVE_OK)
throw Error(reason, archive_error_string(this->archive)); throw Error(reason, archive_error_string(this->archive));
} }
TarArchive::TarArchive(Source& source, bool raw) : buffer(4096) TarArchive::TarArchive(Source & source, bool raw) : buffer(4096)
{ {
this->archive = archive_read_new(); this->archive = archive_read_new();
this->source = &source; this->source = &source;
@ -53,7 +56,7 @@ TarArchive::TarArchive(Source& source, bool raw) : buffer(4096)
} }
TarArchive::TarArchive(const Path &path) TarArchive::TarArchive(const Path & path)
{ {
this->archive = archive_read_new(); this->archive = archive_read_new();
@ -62,11 +65,13 @@ TarArchive::TarArchive(const Path &path)
check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s"); check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
} }
void TarArchive::close() { void TarArchive::close()
{
check(archive_read_close(this->archive), "Failed to close archive (%s)"); check(archive_read_close(this->archive), "Failed to close archive (%s)");
} }
TarArchive::~TarArchive() { TarArchive::~TarArchive()
{
if (this->archive) archive_read_free(this->archive); if (this->archive) archive_read_free(this->archive);
} }

12
src/libutil/tarfile.hh
View file

@ -4,18 +4,18 @@
namespace nix { namespace nix {
struct TarArchive { struct TarArchive {
struct archive *archive; struct archive * archive;
Source *source; Source * source;
std::vector<unsigned char> buffer; std::vector<unsigned char> buffer;
void check(int err, const char *reason = "Failed to extract archive (%s)"); void check(int err, const std::string & reason = "failed to extract archive (%s)");
TarArchive(Source& source, bool raw = false); TarArchive(Source & source, bool raw = false);
TarArchive(const Path &path); TarArchive(const Path & path);
// disable copy constructor // disable copy constructor
TarArchive(const TarArchive&) = delete; TarArchive(const TarArchive &) = delete;
void close(); void close();