Merge remote-tracking branch 'origin/master' into flakes

This commit is contained in:
Eelco Dolstra 2019-12-16 20:17:21 +01:00
commit ad6b738ed8
18 changed files with 207 additions and 82 deletions

View file

@ -210,6 +210,11 @@ PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
[AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
# Look for zlib, a required dependency.
PKG_CHECK_MODULES([ZLIB], [zlib], [CXXFLAGS="$ZLIB_CFLAGS $CXXFLAGS"])
AC_CHECK_HEADER([zlib.h],[:],[AC_MSG_ERROR([could not find the zlib.h header])])
LDFLAGS="-lz $LDFLAGS"
# Look for libbrotli{enc,dec}.
PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])

View file

@ -68,7 +68,7 @@
buildDeps =
[ curl
bzip2 xz brotli editline
bzip2 xz brotli zlib editline
openssl pkgconfig sqlite boehmgc
boost
(nlohmann_json.override { multipleHeaders = true; })

View file

@ -1,17 +1,21 @@
use super::{
error,
foreign::{self, CBox},
foreign::{self},
store::path,
store::StorePath,
util,
};
#[no_mangle]
pub extern "C" fn unpack_tarfile(
pub unsafe extern "C" fn unpack_tarfile(
source: foreign::Source,
dest_dir: &str,
) -> CBox<Result<(), error::CppException>> {
CBox::new(util::tarfile::unpack_tarfile(source, dest_dir).map_err(|err| err.into()))
out: *mut Result<(), error::CppException>,
) {
out.write(
util::tarfile::unpack_tarfile(source, std::path::Path::new(dest_dir))
.map_err(|err| err.into()),
);
}
#[no_mangle]

View file

@ -22,7 +22,9 @@ pub enum Error {
#[cfg(unused)]
HttpError(hyper::error::Error),
Misc(String),
#[cfg(not(test))]
Foreign(CppException),
BadTarFileMemberName(String),
}
impl From<std::io::Error> for Error {
@ -62,26 +64,51 @@ impl fmt::Display for Error {
Error::IOError(err) => write!(f, "I/O error: {}", err),
#[cfg(unused)]
Error::HttpError(err) => write!(f, "HTTP error: {}", err),
#[cfg(not(test))]
Error::Foreign(_) => write!(f, "<C++ exception>"), // FIXME
Error::Misc(s) => write!(f, "{}", s),
Error::BadTarFileMemberName(s) => {
write!(f, "tar archive contains illegal file name '{}'", s)
}
}
}
}
#[cfg(not(test))]
impl From<Error> for CppException {
fn from(err: Error) -> Self {
match err {
Error::Foreign(ex) => ex,
_ => unsafe { make_error(&err.to_string()) },
_ => CppException::new(&err.to_string()),
}
}
}
#[cfg(not(test))]
#[repr(C)]
#[derive(Debug)]
pub struct CppException(*const libc::c_void); // == std::exception_ptr*
#[cfg(not(test))]
impl CppException {
fn new(s: &str) -> Self {
Self(unsafe { make_error(s) })
}
}
#[cfg(not(test))]
impl Drop for CppException {
fn drop(&mut self) {
unsafe {
destroy_error(self.0);
}
}
}
#[cfg(not(test))]
extern "C" {
#[allow(improper_ctypes)] // YOLO
fn make_error(s: &str) -> CppException;
fn make_error(s: &str) -> *const libc::c_void;
fn destroy_error(exc: *const libc::c_void);
}

View file

@ -12,22 +12,3 @@ impl std::io::Read for Source {
Ok(n)
}
}
pub struct CBox<T> {
pub ptr: *mut libc::c_void,
phantom: std::marker::PhantomData<T>,
}
impl<T> CBox<T> {
pub fn new(t: T) -> Self {
unsafe {
let size = std::mem::size_of::<T>();
let ptr = libc::malloc(size);
*(ptr as *mut T) = t; // FIXME: probably UB
Self {
ptr,
phantom: std::marker::PhantomData,
}
}
}
}

View file

@ -9,6 +9,7 @@ extern crate assert_matches;
#[macro_use]
extern crate proptest;
#[cfg(not(test))]
mod c;
mod error;
mod foreign;

View file

@ -2,18 +2,28 @@ use crate::{foreign::Source, Error};
use std::fs;
use std::io;
use std::os::unix::fs::OpenOptionsExt;
use std::path::Path;
use std::path::{Component, Path};
use tar::Archive;
pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> {
let dest_dir = Path::new(dest_dir);
pub fn unpack_tarfile(source: Source, dest_dir: &Path) -> Result<(), Error> {
fs::create_dir_all(dest_dir)?;
let mut tar = Archive::new(source);
for file in tar.entries()? {
let mut file = file?;
let dest_file = dest_dir.join(file.path()?);
let path = file.path()?;
for i in path.components() {
if let Component::Prefix(_) | Component::RootDir | Component::ParentDir = i {
return Err(Error::BadTarFileMemberName(
file.path()?.to_str().unwrap().to_string(),
));
}
}
let dest_file = dest_dir.join(path);
fs::create_dir_all(dest_file.parent().unwrap())?;

View file

@ -576,12 +576,17 @@ Path resolveExprPath(Path path)
{
assert(path[0] == '/');
unsigned int followCount = 0, maxFollow = 1024;
/* If `path' is a symlink, follow it. This is so that relative
path references work. */
struct stat st;
while (true) {
// Basic cycle/depth limit to avoid infinite loops.
if (++followCount >= maxFollow)
throw Error("too many symbolic links encountered while traversing the path '%s'", path);
if (lstat(path.c_str(), &st))
throw SysError(format("getting status of '%1%'") % path);
throw SysError("getting status of '%s'", path);
if (!S_ISLNK(st.st_mode)) break;
path = absPath(readLink(path), dirOf(path));
}

View file

@ -13,7 +13,9 @@ ifneq ($(OS), FreeBSD)
libstore_LDFLAGS += -ldl
endif
ifeq ($(OS), Darwin)
libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb
endif
$(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox)))

View file

@ -62,7 +62,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
args.push_back(command);
execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
throw SysError("executing '%s' on '%s'", command, host);
// could not exec ssh/bash
throw SysError("unable to execute '%s'", args.front());
});
@ -108,7 +109,7 @@ Path SSHMaster::startMaster()
addCommonSSHOpts(args);
execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
throw SysError("starting SSH master");
throw SysError("unable to execute '%s'", args.front());
});
out.writeSide = -1;

View file

@ -11,6 +11,8 @@
#include <brotli/decode.h>
#include <brotli/encode.h>
#include <zlib.h>
#include <iostream>
namespace nix {
@ -42,6 +44,66 @@ struct NoneSink : CompressionSink
void write(const unsigned char * data, size_t len) override { nextSink(data, len); }
};
struct GzipDecompressionSink : CompressionSink
{
Sink & nextSink;
z_stream strm;
bool finished = false;
uint8_t outbuf[BUFSIZ];
GzipDecompressionSink(Sink & nextSink) : nextSink(nextSink)
{
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.avail_in = 0;
strm.next_in = Z_NULL;
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
// Enable gzip and zlib decoding (+32) with 15 windowBits
int ret = inflateInit2(&strm,15+32);
if (ret != Z_OK)
throw CompressionError("unable to initialise gzip encoder");
}
~GzipDecompressionSink()
{
inflateEnd(&strm);
}
void finish() override
{
CompressionSink::flush();
write(nullptr, 0);
}
void write(const unsigned char * data, size_t len) override
{
assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
strm.next_in = (Bytef *) data;
strm.avail_in = len;
while (!finished && (!data || strm.avail_in)) {
checkInterrupt();
int ret = inflate(&strm,Z_SYNC_FLUSH);
if (ret != Z_OK && ret != Z_STREAM_END)
throw CompressionError("error while decompressing gzip file: %d (%d, %d)",
zError(ret), len, strm.avail_in);
finished = ret == Z_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
strm.next_out = (Bytef *) outbuf;
strm.avail_out = sizeof(outbuf);
}
}
}
};
struct XzDecompressionSink : CompressionSink
{
Sink & nextSink;
@ -215,6 +277,8 @@ ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & ne
return make_ref<XzDecompressionSink>(nextSink);
else if (method == "bzip2")
return make_ref<BzipDecompressionSink>(nextSink);
else if (method == "gzip")
return make_ref<GzipDecompressionSink>(nextSink);
else if (method == "br")
return make_ref<BrotliDecompressionSink>(nextSink);
else

View file

@ -3,10 +3,14 @@
extern "C" std::exception_ptr * make_error(rust::StringSlice s)
{
// FIXME: leak
return new std::exception_ptr(std::make_exception_ptr(nix::Error(std::string(s.ptr, s.size))));
}
extern "C" void destroy_error(std::exception_ptr * ex)
{
free(ex);
}
namespace rust {
std::ostream & operator << (std::ostream & str, const String & s)
@ -15,4 +19,15 @@ std::ostream & operator << (std::ostream & str, const String & s)
return str;
}
size_t Source::sourceWrapper(void * _this, rust::Slice<uint8_t> data)
{
try {
// FIXME: how to propagate exceptions?
auto n = ((nix::Source *) _this)->read((unsigned char *) data.ptr, data.size);
return n;
} catch (...) {
abort();
}
}
}

View file

@ -140,64 +140,58 @@ struct Source
: fun(sourceWrapper), _this(&_this)
{}
// FIXME: how to propagate exceptions?
static size_t sourceWrapper(void * _this, rust::Slice<uint8_t> data)
{
auto n = ((nix::Source *) _this)->read((unsigned char *) data.ptr, data.size);
return n;
}
static size_t sourceWrapper(void * _this, rust::Slice<uint8_t> data);
};
/* C++ representation of Rust's Result<T, CppException>. */
template<typename T>
struct Result
{
unsigned int tag;
enum { Ok = 0, Err = 1, Uninit = 2 } tag;
union {
T data;
std::exception_ptr * exc;
};
Result() : tag(Uninit) { }; // FIXME: remove
Result(const Result &) = delete;
Result(Result && other)
: tag(other.tag)
{
other.tag = Uninit;
if (tag == Ok)
data = std::move(other.data);
else if (tag == Err)
exc = other.exc;
}
~Result()
{
if (tag == 0)
if (tag == Ok)
data.~T();
else if (tag == 1)
// FIXME: don't leak exc
else if (tag == Err)
free(exc);
else if (tag == Uninit)
;
else
abort();
}
/* Rethrow the wrapped exception or return the wrapped value. */
T unwrap()
{
if (tag == 0)
if (tag == Ok) {
tag = Uninit;
return std::move(data);
else if (tag == 1)
}
else if (tag == Err)
std::rethrow_exception(*exc);
else
abort();
}
};
template<typename T>
struct CBox
{
T * ptr;
T * operator ->()
{
return ptr;
}
CBox(T * ptr) : ptr(ptr) { }
CBox(const CBox &) = delete;
CBox(CBox &&) = delete;
~CBox()
{
free(ptr);
}
};
}

View file

@ -3,7 +3,7 @@
extern "C" {
rust::Result<std::tuple<>> *
unpack_tarfile(rust::Source source, rust::StringSlice dest_dir);
unpack_tarfile(rust::Source source, rust::StringSlice dest_dir, rust::Result<std::tuple<>> & out);
}
namespace nix {
@ -11,7 +11,9 @@ namespace nix {
void unpackTarfile(Source & source, const Path & destDir)
{
rust::Source source2(source);
rust::CBox(unpack_tarfile(source2, destDir))->unwrap();
rust::Result<std::tuple<>> res;
unpack_tarfile(source2, destDir, res);
res.unwrap();
}
void unpackTarfile(const Path & tarFile, const Path & destDir,
@ -22,8 +24,8 @@ void unpackTarfile(const Path & tarFile, const Path & destDir,
auto source = sinkToSource([&](Sink & sink) {
// FIXME: look at first few bytes to determine compression type.
auto decompressor =
// FIXME: add .gz support
hasSuffix(*baseName, ".bz2") ? makeDecompressionSink("bzip2", sink) :
hasSuffix(*baseName, ".gz") ? makeDecompressionSink("gzip", sink) :
hasSuffix(*baseName, ".xz") ? makeDecompressionSink("xz", sink) :
makeDecompressionSink("none", sink);
readFile(tarFile, *decompressor);

View file

@ -106,7 +106,7 @@ static void _main(int argc, char * * argv)
// Heuristic to see if we're invoked as a shebang script, namely,
// if we have at least one argument, it's the name of an
// executable file, and it starts with "#!".
if (runEnv && argc > 1 && !std::regex_search(std::string(baseNameOf(argv[1])), std::regex("nix-shell"))) {
if (runEnv && argc > 1) {
script = argv[1];
try {
auto lines = tokenizeString<Strings>(readFile(script), "\n");

View file

@ -155,7 +155,7 @@ public:
if (type == actBuild) {
auto name = storePathToName(getS(fields, 0));
if (hasSuffix(name, ".drv"))
name = name.substr(name.size() - 4);
name = name.substr(0, name.size() - 4);
i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name);
auto machineName = getS(fields, 1);
if (machineName != "")
@ -180,7 +180,7 @@ public:
if (type == actPostBuildHook) {
auto name = storePathToName(getS(fields, 0));
if (hasSuffix(name, ".drv"))
name = name.substr(name.size() - 4);
name = name.substr(0, name.size() - 4);
i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name);
i->name = DrvName(name).name;
}

BIN
tests/bad.tar.xz Normal file

Binary file not shown.

View file

@ -10,8 +10,12 @@ mkdir -p $tarroot
cp dependencies.nix $tarroot/default.nix
cp config.nix dependencies.builder*.sh $tarroot/
tarball=$TEST_ROOT/tarball.tar.xz
(cd $TEST_ROOT && tar c tarball) | xz > $tarball
test_tarball() {
local ext="$1"
local compressor="$2"
tarball=$TEST_ROOT/tarball.tar$ext
(cd $TEST_ROOT && tar c tarball) | $compressor > $tarball
nix-env -f file://$tarball -qa --out-path | grep -q dependencies
@ -21,8 +25,18 @@ nix-build -o $TEST_ROOT/result '<foo>' -I foo=file://$tarball
nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)"
nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar.xz
nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball.tar.xz
(! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball.tar.xz)
nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext
nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball$ext
(! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball$ext)
nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball.tar.xz -I fnord=.
nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball$ext -I fnord=.
}
test_tarball '' cat
test_tarball .xz xz
test_tarball .gz gzip
rm -rf $TEST_ROOT/tmp
mkdir -p $TEST_ROOT/tmp
(! TMPDIR=$TEST_ROOT/tmp XDG_RUNTIME_DIR=$TEST_ROOT/tmp nix-env -f file://$(pwd)/bad.tar.xz -qa --out-path)
(! [ -e $TEST_ROOT/tmp/bad ])