forked from lix-project/lix
Merge branch 'libarchive' of https://github.com/yorickvP/nix
This commit is contained in:
commit
ee235e764c
|
@ -18,6 +18,7 @@ SODIUM_LIBS = @SODIUM_LIBS@
|
|||
LIBLZMA_LIBS = @LIBLZMA_LIBS@
|
||||
SQLITE3_LIBS = @SQLITE3_LIBS@
|
||||
LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
|
||||
LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
|
||||
EDITLINE_LIBS = @EDITLINE_LIBS@
|
||||
bash = @bash@
|
||||
bindir = @bindir@
|
||||
|
|
|
@ -178,6 +178,8 @@ AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
|
|||
[AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
|
||||
AC_CHECK_HEADERS([bzlib.h], [true],
|
||||
[AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
|
||||
# Checks for libarchive
|
||||
PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"])
|
||||
|
||||
# Look for SQLite, a required dependency.
|
||||
PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"])
|
||||
|
|
34
nix-rust/Cargo.lock
generated
34
nix-rust/Cargo.lock
generated
|
@ -54,17 +54,6 @@ dependencies = [
|
|||
"bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "filetime"
|
||||
version = "0.2.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fnv"
|
||||
version = "1.0.6"
|
||||
|
@ -109,7 +98,6 @@ dependencies = [
|
|||
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
@ -319,17 +307,6 @@ dependencies = [
|
|||
"wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tar"
|
||||
version = "0.4.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.1.0"
|
||||
|
@ -375,14 +352,6 @@ name = "winapi-x86_64-pc-windows-gnu"
|
|||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "xattr"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[metadata]
|
||||
"checksum assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7deb0a829ca7bcfaf5da70b073a8d128619259a7be8216a355e23f00763059e5"
|
||||
"checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
|
||||
|
@ -393,7 +362,6 @@ dependencies = [
|
|||
"checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
|
||||
"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
|
||||
"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
|
||||
"checksum filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1ff6d4dab0aa0c8e6346d46052e93b13a16cf847b54ed357087c35011048cc7d"
|
||||
"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
|
||||
"checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
|
||||
"checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407"
|
||||
|
@ -423,11 +391,9 @@ dependencies = [
|
|||
"checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716"
|
||||
"checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e"
|
||||
"checksum rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3dd93264e10c577503e926bd1430193eeb5d21b059148910082245309b424fae"
|
||||
"checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3"
|
||||
"checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
|
||||
"checksum wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
|
||||
"checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d"
|
||||
"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6"
|
||||
"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
"checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c"
|
||||
|
|
|
@ -9,7 +9,6 @@ name = "nixrust"
|
|||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
tar = "0.4"
|
||||
libc = "0.2"
|
||||
#futures-preview = { version = "=0.3.0-alpha.19" }
|
||||
#hyper = "0.13.0-alpha.4"
|
||||
|
|
|
@ -1,22 +1,4 @@
|
|||
use super::{
|
||||
error,
|
||||
foreign::{self},
|
||||
store::path,
|
||||
store::StorePath,
|
||||
util,
|
||||
};
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn unpack_tarfile(
|
||||
source: foreign::Source,
|
||||
dest_dir: &str,
|
||||
out: *mut Result<(), error::CppException>,
|
||||
) {
|
||||
out.write(
|
||||
util::tarfile::unpack_tarfile(source, std::path::Path::new(dest_dir))
|
||||
.map_err(|err| err.into()),
|
||||
);
|
||||
}
|
||||
use super::{error, store::path, store::StorePath, util};
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ffi_String_new(s: &str, out: *mut String) {
|
||||
|
|
|
@ -1,14 +0,0 @@
|
|||
/// A wrapper around Nix's Source class that provides the Read trait.
|
||||
#[repr(C)]
|
||||
pub struct Source {
|
||||
fun: extern "C" fn(this: *mut libc::c_void, data: &mut [u8]) -> usize,
|
||||
this: *mut libc::c_void,
|
||||
}
|
||||
|
||||
impl std::io::Read for Source {
|
||||
fn read(&mut self, buf: &mut [u8]) -> std::result::Result<usize, std::io::Error> {
|
||||
let n = (self.fun)(self.this, buf);
|
||||
assert!(n <= buf.len());
|
||||
Ok(n)
|
||||
}
|
||||
}
|
|
@ -12,7 +12,6 @@ extern crate proptest;
|
|||
#[cfg(not(test))]
|
||||
mod c;
|
||||
mod error;
|
||||
mod foreign;
|
||||
#[cfg(unused)]
|
||||
mod nar;
|
||||
mod store;
|
||||
|
|
|
@ -1,2 +1 @@
|
|||
pub mod base32;
|
||||
pub mod tarfile;
|
||||
|
|
|
@ -1,56 +0,0 @@
|
|||
use crate::{foreign::Source, Error};
|
||||
use std::fs;
|
||||
use std::io;
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
use std::path::{Component, Path};
|
||||
use tar::Archive;
|
||||
|
||||
pub fn unpack_tarfile(source: Source, dest_dir: &Path) -> Result<(), Error> {
|
||||
fs::create_dir_all(dest_dir)?;
|
||||
|
||||
let mut tar = Archive::new(source);
|
||||
|
||||
for file in tar.entries()? {
|
||||
let mut file = file?;
|
||||
|
||||
let path = file.path()?;
|
||||
|
||||
for i in path.components() {
|
||||
if let Component::Prefix(_) | Component::RootDir | Component::ParentDir = i {
|
||||
return Err(Error::BadTarFileMemberName(
|
||||
file.path()?.to_str().unwrap().to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let dest_file = dest_dir.join(path);
|
||||
|
||||
fs::create_dir_all(dest_file.parent().unwrap())?;
|
||||
|
||||
match file.header().entry_type() {
|
||||
tar::EntryType::Directory => {
|
||||
fs::create_dir(dest_file)?;
|
||||
}
|
||||
tar::EntryType::Regular => {
|
||||
let mode = if file.header().mode()? & (libc::S_IXUSR as u32) == 0 {
|
||||
0o666
|
||||
} else {
|
||||
0o777
|
||||
};
|
||||
let mut f = fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.write(true)
|
||||
.mode(mode)
|
||||
.open(dest_file)?;
|
||||
io::copy(&mut file, &mut f)?;
|
||||
}
|
||||
tar::EntryType::Symlink => {
|
||||
std::os::unix::fs::symlink(file.header().link_name()?.unwrap(), dest_file)?;
|
||||
}
|
||||
tar::EntryType::XGlobalHeader | tar::EntryType::XHeader => {}
|
||||
t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
|
@ -49,6 +49,7 @@ rec {
|
|||
[ curl
|
||||
bzip2 xz brotli zlib editline
|
||||
openssl pkgconfig sqlite boehmgc
|
||||
libarchive
|
||||
boost
|
||||
nlohmann_json
|
||||
rustc cargo
|
||||
|
|
|
@ -908,7 +908,7 @@ CachedDownloadResult Downloader::downloadCached(
|
|||
printInfo("unpacking '%s'...", url);
|
||||
Path tmpDir = createTempDir();
|
||||
AutoDelete autoDelete(tmpDir, true);
|
||||
unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir, std::string(baseNameOf(url)));
|
||||
unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir);
|
||||
auto members = readDirectory(tmpDir);
|
||||
if (members.size() != 1)
|
||||
throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
|
||||
|
|
|
@ -6,6 +6,6 @@ libutil_DIR := $(d)
|
|||
|
||||
libutil_SOURCES := $(wildcard $(d)/*.cc)
|
||||
|
||||
libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context
|
||||
libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
|
||||
|
||||
libutil_LIBS = libnixrust
|
||||
|
|
|
@ -19,15 +19,4 @@ std::ostream & operator << (std::ostream & str, const String & s)
|
|||
return str;
|
||||
}
|
||||
|
||||
size_t Source::sourceWrapper(void * _this, rust::Slice<uint8_t> data)
|
||||
{
|
||||
try {
|
||||
// FIXME: how to propagate exceptions?
|
||||
auto n = ((nix::Source *) _this)->read((unsigned char *) data.ptr, data.size);
|
||||
return n;
|
||||
} catch (...) {
|
||||
abort();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -131,18 +131,6 @@ struct String : Vec<char, ffi_String_drop>
|
|||
|
||||
std::ostream & operator << (std::ostream & str, const String & s);
|
||||
|
||||
struct Source
|
||||
{
|
||||
size_t (*fun)(void * source_this, rust::Slice<uint8_t> data);
|
||||
nix::Source * _this;
|
||||
|
||||
Source(nix::Source & _this)
|
||||
: fun(sourceWrapper), _this(&_this)
|
||||
{}
|
||||
|
||||
static size_t sourceWrapper(void * _this, rust::Slice<uint8_t> data);
|
||||
};
|
||||
|
||||
/* C++ representation of Rust's Result<T, CppException>. */
|
||||
template<typename T>
|
||||
struct Result
|
||||
|
|
|
@ -1,38 +1,132 @@
|
|||
#include "rust-ffi.hh"
|
||||
#include "compression.hh"
|
||||
#include <archive.h>
|
||||
#include <archive_entry.h>
|
||||
|
||||
extern "C" {
|
||||
rust::Result<std::tuple<>> *
|
||||
unpack_tarfile(rust::Source source, rust::StringSlice dest_dir, rust::Result<std::tuple<>> & out);
|
||||
}
|
||||
#include "serialise.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct TarArchive {
|
||||
struct archive *archive;
|
||||
Source *source;
|
||||
std::vector<unsigned char> buffer;
|
||||
|
||||
void check(int err, const char *reason = "Failed to extract archive (%s)") {
|
||||
if (err == ARCHIVE_EOF)
|
||||
throw EndOfFile("reached end of archive");
|
||||
else if (err != ARCHIVE_OK)
|
||||
throw Error(reason, archive_error_string(this->archive));
|
||||
}
|
||||
|
||||
TarArchive(Source& source) : buffer(4096) {
|
||||
this->archive = archive_read_new();
|
||||
this->source = &source;
|
||||
|
||||
archive_read_support_filter_all(archive);
|
||||
archive_read_support_format_all(archive);
|
||||
check(archive_read_open(archive, (void *)this, TarArchive::callback_open, TarArchive::callback_read, TarArchive::callback_close), "Failed to open archive (%s)");
|
||||
}
|
||||
|
||||
TarArchive(const Path &path) {
|
||||
this->archive = archive_read_new();
|
||||
|
||||
archive_read_support_filter_all(archive);
|
||||
archive_read_support_format_all(archive);
|
||||
check(archive_read_open_filename(archive, path.c_str(), 16384), "Failed to open archive (%s)");
|
||||
}
|
||||
|
||||
// disable copy constructor
|
||||
TarArchive(const TarArchive&) = delete;
|
||||
|
||||
void close() {
|
||||
check(archive_read_close(archive), "Failed to close archive (%s)");
|
||||
}
|
||||
|
||||
~TarArchive() {
|
||||
if (this->archive) archive_read_free(this->archive);
|
||||
}
|
||||
|
||||
private:
|
||||
static int callback_open(struct archive *, void *self) {
|
||||
return ARCHIVE_OK;
|
||||
}
|
||||
|
||||
static ssize_t callback_read(struct archive *archive, void *_self, const void **buffer) {
|
||||
TarArchive *self = (TarArchive *)_self;
|
||||
*buffer = self->buffer.data();
|
||||
|
||||
try {
|
||||
return self->source->read(self->buffer.data(), 4096);
|
||||
} catch (EndOfFile &) {
|
||||
return 0;
|
||||
} catch (std::exception &err) {
|
||||
archive_set_error(archive, EIO, "Source threw exception: %s", err.what());
|
||||
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
static int callback_close(struct archive *, void *self) {
|
||||
return ARCHIVE_OK;
|
||||
}
|
||||
};
|
||||
|
||||
struct PushD {
|
||||
char * oldDir;
|
||||
|
||||
PushD(const std::string &newDir) {
|
||||
oldDir = getcwd(0, 0);
|
||||
if (!oldDir) throw SysError("getcwd");
|
||||
int r = chdir(newDir.c_str());
|
||||
if (r != 0) throw SysError("changing directory to tar output path");
|
||||
}
|
||||
|
||||
~PushD() {
|
||||
int r = chdir(oldDir);
|
||||
free(oldDir);
|
||||
if (r != 0)
|
||||
std::cerr << "warning: failed to change directory back after tar extraction";
|
||||
/* can't throw out of a destructor */
|
||||
}
|
||||
};
|
||||
|
||||
static void extract_archive(TarArchive &archive, const Path & destDir) {
|
||||
// need to chdir back *after* archive closing
|
||||
PushD newDir(destDir);
|
||||
struct archive_entry *entry;
|
||||
int flags = ARCHIVE_EXTRACT_FFLAGS
|
||||
| ARCHIVE_EXTRACT_PERM
|
||||
| ARCHIVE_EXTRACT_SECURE_SYMLINKS
|
||||
| ARCHIVE_EXTRACT_SECURE_NODOTDOT
|
||||
| ARCHIVE_EXTRACT_SECURE_NOABSOLUTEPATHS;
|
||||
|
||||
for(;;) {
|
||||
int r = archive_read_next_header(archive.archive, &entry);
|
||||
if (r == ARCHIVE_EOF) break;
|
||||
else if (r == ARCHIVE_WARN)
|
||||
std::cerr << "warning: " << archive_error_string(archive.archive) << std::endl;
|
||||
else
|
||||
archive.check(r);
|
||||
|
||||
archive.check(archive_read_extract(archive.archive, entry, flags));
|
||||
}
|
||||
|
||||
archive.close();
|
||||
}
|
||||
|
||||
void unpackTarfile(Source & source, const Path & destDir)
|
||||
{
|
||||
rust::Source source2(source);
|
||||
rust::Result<std::tuple<>> res;
|
||||
unpack_tarfile(source2, destDir, res);
|
||||
res.unwrap();
|
||||
auto archive = TarArchive(source);
|
||||
|
||||
createDirs(destDir);
|
||||
extract_archive(archive, destDir);
|
||||
}
|
||||
|
||||
void unpackTarfile(const Path & tarFile, const Path & destDir,
|
||||
std::optional<std::string> baseName)
|
||||
void unpackTarfile(const Path & tarFile, const Path & destDir)
|
||||
{
|
||||
if (!baseName) baseName = std::string(baseNameOf(tarFile));
|
||||
auto archive = TarArchive(tarFile);
|
||||
|
||||
auto source = sinkToSource([&](Sink & sink) {
|
||||
// FIXME: look at first few bytes to determine compression type.
|
||||
auto decompressor =
|
||||
hasSuffix(*baseName, ".bz2") ? makeDecompressionSink("bzip2", sink) :
|
||||
hasSuffix(*baseName, ".gz") ? makeDecompressionSink("gzip", sink) :
|
||||
hasSuffix(*baseName, ".xz") ? makeDecompressionSink("xz", sink) :
|
||||
makeDecompressionSink("none", sink);
|
||||
readFile(tarFile, *decompressor);
|
||||
decompressor->finish();
|
||||
});
|
||||
|
||||
unpackTarfile(*source, destDir);
|
||||
createDirs(destDir);
|
||||
extract_archive(archive, destDir);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -4,7 +4,6 @@ namespace nix {
|
|||
|
||||
void unpackTarfile(Source & source, const Path & destDir);
|
||||
|
||||
void unpackTarfile(const Path & tarFile, const Path & destDir,
|
||||
std::optional<std::string> baseName = {});
|
||||
void unpackTarfile(const Path & tarFile, const Path & destDir);
|
||||
|
||||
}
|
||||
|
|
|
@ -190,10 +190,7 @@ static int _main(int argc, char * * argv)
|
|||
printInfo("unpacking...");
|
||||
Path unpacked = (Path) tmpDir + "/unpacked";
|
||||
createDirs(unpacked);
|
||||
if (hasSuffix(baseNameOf(uri), ".zip"))
|
||||
runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked});
|
||||
else
|
||||
unpackTarfile(tmpFile, unpacked, std::string(baseNameOf(uri)));
|
||||
unpackTarfile(tmpFile, unpacked);
|
||||
|
||||
/* If the archive unpacks to a single file/directory, then use
|
||||
that as the top-level. */
|
||||
|
|
Loading…
Reference in a new issue