Merge branch 'libarchive-decompress' of https://github.com/serokell/nix

This commit is contained in:
Eelco Dolstra 2021-04-15 13:39:04 +02:00
commit 6fb7582413
10 changed files with 273 additions and 378 deletions

View file

@ -179,6 +179,9 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "zstd" ? ".zst" :
compression == "lzip" ? ".lzip" :
compression == "lz4" ? ".lz4" :
compression == "br" ? ".br" :
"");

View file

@ -148,7 +148,7 @@ struct curlFileTransfer : public FileTransfer
}
LambdaSink finalSink;
std::shared_ptr<CompressionSink> decompressionSink;
std::shared_ptr<FinishSink> decompressionSink;
std::optional<StringSink> errorSink;
std::exception_ptr writeException;

View file

@ -1,10 +1,13 @@
#include "compression.hh"
#include "tarfile.hh"
#include "util.hh"
#include "finally.hh"
#include "logging.hh"
#include <lzma.h>
#include <bzlib.h>
#include <archive.h>
#include <archive_entry.h>
#include <cstdio>
#include <cstring>
@ -35,6 +38,80 @@ struct ChunkedCompressionSink : CompressionSink
virtual void writeInternal(std::string_view data) = 0;
};
struct ArchiveDecompressionSource : Source
{
std::unique_ptr<TarArchive> archive = 0;
Source & src;
ArchiveDecompressionSource(Source & src) : src(src) {}
~ArchiveDecompressionSource() override {}
size_t read(char * data, size_t len) override {
struct archive_entry* ae;
if (!archive) {
archive = std::make_unique<TarArchive>(src, true);
this->archive->check(archive_read_next_header(this->archive->archive, &ae), "Failed to read header (%s)");
if (archive_filter_count(this->archive->archive) < 2) {
throw CompressionError("Input compression not recognized.");
}
}
ssize_t result = archive_read_data(this->archive->archive, data, len);
if (result > 0) return result;
if (result == 0) {
throw EndOfFile("reached end of compressed file");
}
this->archive->check(result, "Failed to read compressed data (%s)");
return result;
}
};
struct ArchiveCompressionSink : CompressionSink
{
Sink & nextSink;
struct archive* archive;
ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel) : nextSink(nextSink) {
archive = archive_write_new();
if (!archive) throw Error("failed to initialize libarchive");
check(archive_write_add_filter_by_name(archive, format.c_str()), "Couldn't initialize compression (%s)");
check(archive_write_set_format_raw(archive));
if (format == "xz" && parallel) {
check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0"));
}
// disable internal buffering
check(archive_write_set_bytes_per_block(archive, 0));
// disable output padding
check(archive_write_set_bytes_in_last_block(archive, 1));
open();
}
~ArchiveCompressionSink() override {
if (archive) archive_write_free(archive);
}
void finish() override {
flush();
check(archive_write_close(archive));
}
void check(int err, const char *reason="Failed to compress (%s)") {
if (err == ARCHIVE_EOF)
throw EndOfFile("reached end of archive");
else if (err != ARCHIVE_OK)
throw Error(reason, archive_error_string(this->archive));
}
void write(std::string_view data) override {
ssize_t result = archive_write_data(archive, data.data(), data.length());
if (result <= 0) check(result);
}
private:
void open() {
check(archive_write_open(archive, this, NULL, ArchiveCompressionSink::callback_write, NULL));
struct archive_entry *ae = archive_entry_new();
archive_entry_set_filetype(ae, AE_IFREG);
check(archive_write_header(archive, ae));
archive_entry_free(ae);
}
static ssize_t callback_write(struct archive *archive, void *_self, const void *buffer, size_t length) {
ArchiveCompressionSink *self = (ArchiveCompressionSink *)_self;
self->nextSink({(const char*)buffer, length});
return length;
}
};
struct NoneSink : CompressionSink
{
Sink & nextSink;
@ -43,171 +120,6 @@ struct NoneSink : CompressionSink
void write(std::string_view data) override { nextSink(data); }
};
struct GzipDecompressionSink : CompressionSink
{
Sink & nextSink;
z_stream strm;
bool finished = false;
uint8_t outbuf[BUFSIZ];
GzipDecompressionSink(Sink & nextSink) : nextSink(nextSink)
{
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.avail_in = 0;
strm.next_in = Z_NULL;
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
// Enable gzip and zlib decoding (+32) with 15 windowBits
int ret = inflateInit2(&strm,15+32);
if (ret != Z_OK)
throw CompressionError("unable to initialise gzip encoder");
}
~GzipDecompressionSink()
{
inflateEnd(&strm);
}
void finish() override
{
CompressionSink::flush();
write({});
}
void write(std::string_view data) override
{
assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
strm.next_in = (Bytef *) data.data();
strm.avail_in = data.size();
while (!finished && (!data.data() || strm.avail_in)) {
checkInterrupt();
int ret = inflate(&strm,Z_SYNC_FLUSH);
if (ret != Z_OK && ret != Z_STREAM_END)
throw CompressionError("error while decompressing gzip file: %d (%d, %d)",
zError(ret), data.size(), strm.avail_in);
finished = ret == Z_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = (Bytef *) outbuf;
strm.avail_out = sizeof(outbuf);
}
}
}
};
struct XzDecompressionSink : CompressionSink
{
Sink & nextSink;
uint8_t outbuf[BUFSIZ];
lzma_stream strm = LZMA_STREAM_INIT;
bool finished = false;
XzDecompressionSink(Sink & nextSink) : nextSink(nextSink)
{
lzma_ret ret = lzma_stream_decoder(
&strm, UINT64_MAX, LZMA_CONCATENATED);
if (ret != LZMA_OK)
throw CompressionError("unable to initialise lzma decoder");
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
~XzDecompressionSink()
{
lzma_end(&strm);
}
void finish() override
{
CompressionSink::flush();
write({});
}
void write(std::string_view data) override
{
strm.next_in = (const unsigned char *) data.data();
strm.avail_in = data.size();
while (!finished && (!data.data() || strm.avail_in)) {
checkInterrupt();
lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH);
if (ret != LZMA_OK && ret != LZMA_STREAM_END)
throw CompressionError("error %d while decompressing xz file", ret);
finished = ret == LZMA_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
}
}
};
struct BzipDecompressionSink : ChunkedCompressionSink
{
Sink & nextSink;
bz_stream strm;
bool finished = false;
BzipDecompressionSink(Sink & nextSink) : nextSink(nextSink)
{
memset(&strm, 0, sizeof(strm));
int ret = BZ2_bzDecompressInit(&strm, 0, 0);
if (ret != BZ_OK)
throw CompressionError("unable to initialise bzip2 decoder");
strm.next_out = (char *) outbuf;
strm.avail_out = sizeof(outbuf);
}
~BzipDecompressionSink()
{
BZ2_bzDecompressEnd(&strm);
}
void finish() override
{
flush();
write({});
}
void writeInternal(std::string_view data) override
{
assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
strm.next_in = (char *) data.data();
strm.avail_in = data.size();
while (strm.avail_in) {
checkInterrupt();
int ret = BZ2_bzDecompress(&strm);
if (ret != BZ_OK && ret != BZ_STREAM_END)
throw CompressionError("error while decompressing bzip2 file");
finished = ret == BZ_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = (char *) outbuf;
strm.avail_out = sizeof(outbuf);
}
}
}
};
struct BrotliDecompressionSink : ChunkedCompressionSink
{
Sink & nextSink;
@ -261,161 +173,32 @@ struct BrotliDecompressionSink : ChunkedCompressionSink
ref<std::string> decompress(const std::string & method, const std::string & in)
{
if (method == "br") {
StringSink ssink;
auto sink = makeDecompressionSink(method, ssink);
(*sink)(in);
sink->finish();
return ssink.s;
} else {
StringSource ssrc(in);
auto src = makeDecompressionSource(ssrc);
return make_ref<std::string>(src->drain());
}
}
ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
{
if (method == "none" || method == "")
return make_ref<NoneSink>(nextSink);
else if (method == "xz")
return make_ref<XzDecompressionSink>(nextSink);
else if (method == "bzip2")
return make_ref<BzipDecompressionSink>(nextSink);
else if (method == "gzip")
return make_ref<GzipDecompressionSink>(nextSink);
return std::make_unique<NoneSink>(nextSink);
else if (method == "br")
return make_ref<BrotliDecompressionSink>(nextSink);
return std::make_unique<BrotliDecompressionSink>(nextSink);
else
throw UnknownCompressionMethod("unknown compression method '%s'", method);
return sourceToSink([&](Source & source) {
auto decompressionSource = makeDecompressionSource(source);
decompressionSource->drainInto(nextSink);
});
}
struct XzCompressionSink : CompressionSink
{
Sink & nextSink;
uint8_t outbuf[BUFSIZ];
lzma_stream strm = LZMA_STREAM_INIT;
bool finished = false;
XzCompressionSink(Sink & nextSink, bool parallel) : nextSink(nextSink)
{
lzma_ret ret;
bool done = false;
if (parallel) {
#ifdef HAVE_LZMA_MT
lzma_mt mt_options = {};
mt_options.flags = 0;
mt_options.timeout = 300; // Using the same setting as the xz cmd line
mt_options.preset = LZMA_PRESET_DEFAULT;
mt_options.filters = NULL;
mt_options.check = LZMA_CHECK_CRC64;
mt_options.threads = lzma_cputhreads();
mt_options.block_size = 0;
if (mt_options.threads == 0)
mt_options.threads = 1;
// FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
// number of threads.
ret = lzma_stream_encoder_mt(&strm, &mt_options);
done = true;
#else
printMsg(lvlError, "warning: parallel XZ compression requested but not supported, falling back to single-threaded compression");
#endif
}
if (!done)
ret = lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64);
if (ret != LZMA_OK)
throw CompressionError("unable to initialise lzma encoder");
// FIXME: apply the x86 BCJ filter?
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
~XzCompressionSink()
{
lzma_end(&strm);
}
void finish() override
{
CompressionSink::flush();
write({});
}
void write(std::string_view data) override
{
strm.next_in = (const unsigned char *) data.data();
strm.avail_in = data.size();
while (!finished && (!data.data() || strm.avail_in)) {
checkInterrupt();
lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH);
if (ret != LZMA_OK && ret != LZMA_STREAM_END)
throw CompressionError("error %d while compressing xz file", ret);
finished = ret == LZMA_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
}
}
};
struct BzipCompressionSink : ChunkedCompressionSink
{
Sink & nextSink;
bz_stream strm;
bool finished = false;
BzipCompressionSink(Sink & nextSink) : nextSink(nextSink)
{
memset(&strm, 0, sizeof(strm));
int ret = BZ2_bzCompressInit(&strm, 9, 0, 30);
if (ret != BZ_OK)
throw CompressionError("unable to initialise bzip2 encoder");
strm.next_out = (char *) outbuf;
strm.avail_out = sizeof(outbuf);
}
~BzipCompressionSink()
{
BZ2_bzCompressEnd(&strm);
}
void finish() override
{
flush();
writeInternal({});
}
void writeInternal(std::string_view data) override
{
assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
strm.next_in = (char *) data.data();
strm.avail_in = data.size();
while (!finished && (!data.data() || strm.avail_in)) {
checkInterrupt();
int ret = BZ2_bzCompress(&strm, data.data() ? BZ_RUN : BZ_FINISH);
if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END)
throw CompressionError("error %d while compressing bzip2 file", ret);
finished = ret == BZ_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = (char *) outbuf;
strm.avail_out = sizeof(outbuf);
}
}
}
};
struct BrotliCompressionSink : ChunkedCompressionSink
{
Sink & nextSink;
@ -468,15 +251,20 @@ struct BrotliCompressionSink : ChunkedCompressionSink
}
}
};
std::unique_ptr<Source> makeDecompressionSource(Source & prev) {
return std::unique_ptr<Source>(new ArchiveDecompressionSource(prev));
}
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
{
std::vector<std::string> la_supports = {
"bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"
};
if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) {
return make_ref<ArchiveCompressionSink>(nextSink, method, parallel);
}
if (method == "none")
return make_ref<NoneSink>(nextSink);
else if (method == "xz")
return make_ref<XzCompressionSink>(nextSink, parallel);
else if (method == "bzip2")
return make_ref<BzipCompressionSink>(nextSink);
else if (method == "br")
return make_ref<BrotliCompressionSink>(nextSink);
else

View file

@ -8,14 +8,18 @@
namespace nix {
struct CompressionSink : BufferedSink
struct CompressionSink : BufferedSink, FinishSink
{
virtual void finish() = 0;
using BufferedSink::operator ();
using BufferedSink::write;
using FinishSink::finish;
};
std::unique_ptr<Source> makeDecompressionSource(Source & prev);
ref<std::string> decompress(const std::string & method, const std::string & in);
ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);

View file

@ -201,6 +201,61 @@ static DefaultStackAllocator defaultAllocatorSingleton;
StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton;
std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun)
{
struct SourceToSink : FinishSink
{
typedef boost::coroutines2::coroutine<bool> coro_t;
std::function<void(Source &)> fun;
std::optional<coro_t::push_type> coro;
SourceToSink(std::function<void(Source &)> fun) : fun(fun)
{
}
std::string_view cur;
void operator () (std::string_view in) override
{
if (in.empty()) return;
cur = in;
if (!coro)
coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) {
LambdaSource source([&](char *out, size_t out_len) {
if (cur.empty()) {
yield();
if (yield.get()) {
return (size_t)0;
}
}
size_t n = std::min(cur.size(), out_len);
memcpy(out, cur.data(), n);
cur.remove_prefix(n);
return n;
});
fun(source);
});
if (!*coro) { abort(); }
if (!cur.empty()) (*coro)(false);
}
void finish() {
if (!coro) return;
if (!*coro) abort();
(*coro)(true);
if (*coro) abort();
}
};
return std::make_unique<SourceToSink>(fun);
}
std::unique_ptr<Source> sinkToSource(
std::function<void(Sink &)> fun,
std::function<void()> eof)
@ -212,7 +267,6 @@ std::unique_ptr<Source> sinkToSource(
std::function<void(Sink &)> fun;
std::function<void()> eof;
std::optional<coro_t::pull_type> coro;
bool started = false;
SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof)
: fun(fun), eof(eof)

View file

@ -25,6 +25,13 @@ struct NullSink : Sink
{ }
};
struct FinishSink : virtual Sink
{
virtual void finish() = 0;
};
/* A buffered abstract sink. Warning: a BufferedSink should not be
used from multiple threads concurrently. */
struct BufferedSink : virtual Sink
@ -281,6 +288,7 @@ struct ChainSource : Source
size_t read(char * data, size_t len) override;
};
std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun);
/* Convert a function that feeds data into a Sink into a Source. The
Source executes the function as a coroutine. */

View file

@ -2,83 +2,73 @@
#include <archive_entry.h>
#include "serialise.hh"
#include "tarfile.hh"
namespace nix {
struct TarArchive {
struct archive * archive;
Source * source;
std::vector<unsigned char> buffer;
void check(int err, const char * reason = "failed to extract archive: %s")
{
if (err == ARCHIVE_EOF)
throw EndOfFile("reached end of archive");
else if (err != ARCHIVE_OK)
throw Error(reason, archive_error_string(this->archive));
}
TarArchive(Source & source) : buffer(4096)
{
this->archive = archive_read_new();
this->source = &source;
archive_read_support_filter_all(archive);
archive_read_support_format_all(archive);
check(archive_read_open(archive,
(void *)this,
TarArchive::callback_open,
TarArchive::callback_read,
TarArchive::callback_close),
"failed to open archive: %s");
}
TarArchive(const Path & path)
{
this->archive = archive_read_new();
archive_read_support_filter_all(archive);
archive_read_support_format_all(archive);
check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
}
TarArchive(const TarArchive &) = delete;
void close()
{
check(archive_read_close(archive), "failed to close archive: %s");
}
~TarArchive()
{
if (this->archive) archive_read_free(this->archive);
}
private:
static int callback_open(struct archive *, void * self) {
static int callback_open(struct archive *, void *self) {
return ARCHIVE_OK;
}
}
static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer)
{
auto self = (TarArchive *)_self;
static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer) {
TarArchive *self = (TarArchive *)_self;
*buffer = self->buffer.data();
try {
return self->source->read((char *) self->buffer.data(), 4096);
} catch (EndOfFile &) {
return 0;
} catch (std::exception & err) {
archive_set_error(archive, EIO, "source threw exception: %s", err.what());
} catch (std::exception &err) {
archive_set_error(archive, EIO, "Source threw exception: %s", err.what());
return -1;
}
}
static int callback_close(struct archive *, void *self) {
return ARCHIVE_OK;
}
void TarArchive::check(int err, const char *reason)
{
if (err == ARCHIVE_EOF)
throw EndOfFile("reached end of archive");
else if (err != ARCHIVE_OK)
throw Error(reason, archive_error_string(this->archive));
}
static int callback_close(struct archive *, void * self) {
return ARCHIVE_OK;
TarArchive::TarArchive(Source& source, bool raw) : buffer(4096)
{
this->archive = archive_read_new();
this->source = &source;
if (!raw) {
archive_read_support_filter_all(archive);
archive_read_support_format_all(archive);
} else {
archive_read_support_filter_all(archive);
archive_read_support_format_raw(archive);
archive_read_support_format_empty(archive);
}
};
check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)");
}
TarArchive::TarArchive(const Path &path)
{
this->archive = archive_read_new();
archive_read_support_filter_all(archive);
archive_read_support_format_all(archive);
check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
}
void TarArchive::close() {
check(archive_read_close(this->archive), "Failed to close archive (%s)");
}
TarArchive::~TarArchive() {
if (this->archive) archive_read_free(this->archive);
}
static void extract_archive(TarArchive & archive, const Path & destDir)
{

View file

@ -1,7 +1,26 @@
#include "serialise.hh"
#include <archive.h>
namespace nix {
struct TarArchive {
struct archive *archive;
Source *source;
std::vector<unsigned char> buffer;
void check(int err, const char *reason = "Failed to extract archive (%s)");
TarArchive(Source& source, bool raw = false);
TarArchive(const Path &path);
// disable copy constructor
TarArchive(const TarArchive&) = delete;
void close();
~TarArchive();
};
void unpackTarfile(Source & source, const Path & destDir);
void unpackTarfile(const Path & tarFile, const Path & destDir);

View file

@ -30,6 +30,7 @@ nix_tests = \
signing.sh \
shell.sh \
brotli.sh \
zstd.sh \
pure-eval.sh \
check.sh \
plugins.sh \

28
tests/zstd.sh Normal file
View file

@ -0,0 +1,28 @@
source common.sh
clearStore
clearCache
cacheURI="file://$cacheDir?compression=zstd"
outPath=$(nix-build dependencies.nix --no-out-link)
nix copy --to $cacheURI $outPath
HASH=$(nix hash path $outPath)
clearStore
clearCacheCache
nix copy --from $cacheURI $outPath --no-check-sigs
if ls $cacheDir/nar/*.zst &> /dev/null; then
echo "files do exist"
else
echo "nars do not exist"
exit 1
fi
HASH2=$(nix hash path $outPath)
[[ $HASH = $HASH2 ]]