Merge remote-tracking branch 'origin/master' into flakes

configure.ac

@@ -210,6 +210,11 @@ PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
 AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
   [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
 
+# Look for zlib, a required dependency.
+PKG_CHECK_MODULES([ZLIB], [zlib], [CXXFLAGS="$ZLIB_CFLAGS $CXXFLAGS"])
+AC_CHECK_HEADER([zlib.h],[:],[AC_MSG_ERROR([could not find the zlib.h header])])
+LDFLAGS="-lz $LDFLAGS"
+
 # Look for libbrotli{enc,dec}.
 PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])
 

flake.nix

@@ -68,7 +68,7 @@
 
         buildDeps =
           [ curl
-            bzip2 xz brotli editline
+            bzip2 xz brotli zlib editline
             openssl pkgconfig sqlite boehmgc
             boost
             (nlohmann_json.override { multipleHeaders = true; })

nix-rust/src/c.rs

@@ -1,17 +1,21 @@
 use super::{
     error,
-    foreign::{self, CBox},
+    foreign::{self},
     store::path,
     store::StorePath,
     util,
 };
 
 #[no_mangle]
-pub extern "C" fn unpack_tarfile(
+pub unsafe extern "C" fn unpack_tarfile(
     source: foreign::Source,
     dest_dir: &str,
-) -> CBox<Result<(), error::CppException>> {
+    out: *mut Result<(), error::CppException>,
-    CBox::new(util::tarfile::unpack_tarfile(source, dest_dir).map_err(|err| err.into()))
+) {
+    out.write(
+        util::tarfile::unpack_tarfile(source, std::path::Path::new(dest_dir))
+            .map_err(|err| err.into()),
+    );
 }
 
 #[no_mangle]

nix-rust/src/error.rs

@@ -22,7 +22,9 @@ pub enum Error {
     #[cfg(unused)]
     HttpError(hyper::error::Error),
     Misc(String),
+    #[cfg(not(test))]
     Foreign(CppException),
+    BadTarFileMemberName(String),
 }
 
 impl From<std::io::Error> for Error {
@@ -62,26 +64,51 @@ impl fmt::Display for Error {
             Error::IOError(err) => write!(f, "I/O error: {}", err),
             #[cfg(unused)]
             Error::HttpError(err) => write!(f, "HTTP error: {}", err),
+            #[cfg(not(test))]
             Error::Foreign(_) => write!(f, "<C++ exception>"), // FIXME
             Error::Misc(s) => write!(f, "{}", s),
+            Error::BadTarFileMemberName(s) => {
+                write!(f, "tar archive contains illegal file name '{}'", s)
+            }
         }
     }
 }
 
+#[cfg(not(test))]
 impl From<Error> for CppException {
     fn from(err: Error) -> Self {
         match err {
             Error::Foreign(ex) => ex,
-            _ => unsafe { make_error(&err.to_string()) },
+            _ => CppException::new(&err.to_string()),
         }
     }
 }
 
+#[cfg(not(test))]
 #[repr(C)]
 #[derive(Debug)]
 pub struct CppException(*const libc::c_void); // == std::exception_ptr*
 
+#[cfg(not(test))]
+impl CppException {
+    fn new(s: &str) -> Self {
+        Self(unsafe { make_error(s) })
+    }
+}
+
+#[cfg(not(test))]
+impl Drop for CppException {
+    fn drop(&mut self) {
+        unsafe {
+            destroy_error(self.0);
+        }
+    }
+}
+
+#[cfg(not(test))]
 extern "C" {
     #[allow(improper_ctypes)] // YOLO
-    fn make_error(s: &str) -> CppException;
+    fn make_error(s: &str) -> *const libc::c_void;
+
+    fn destroy_error(exc: *const libc::c_void);
 }

nix-rust/src/foreign.rs

@@ -12,22 +12,3 @@ impl std::io::Read for Source {
         Ok(n)
     }
 }
-
-pub struct CBox<T> {
-    pub ptr: *mut libc::c_void,
-    phantom: std::marker::PhantomData<T>,
-}
-
-impl<T> CBox<T> {
-    pub fn new(t: T) -> Self {
-        unsafe {
-            let size = std::mem::size_of::<T>();
-            let ptr = libc::malloc(size);
-            *(ptr as *mut T) = t; // FIXME: probably UB
-            Self {
-                ptr,
-                phantom: std::marker::PhantomData,
-            }
-        }
-    }
-}

nix-rust/src/lib.rs

@@ -9,6 +9,7 @@ extern crate assert_matches;
 #[macro_use]
 extern crate proptest;
 
+#[cfg(not(test))]
 mod c;
 mod error;
 mod foreign;

nix-rust/src/util/tarfile.rs

@@ -2,18 +2,28 @@ use crate::{foreign::Source, Error};
 use std::fs;
 use std::io;
 use std::os::unix::fs::OpenOptionsExt;
-use std::path::Path;
+use std::path::{Component, Path};
 use tar::Archive;
 
-pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> {
+pub fn unpack_tarfile(source: Source, dest_dir: &Path) -> Result<(), Error> {
-    let dest_dir = Path::new(dest_dir);
+    fs::create_dir_all(dest_dir)?;
 
     let mut tar = Archive::new(source);
 
     for file in tar.entries()? {
         let mut file = file?;
 
-        let dest_file = dest_dir.join(file.path()?);
+        let path = file.path()?;
+
+        for i in path.components() {
+            if let Component::Prefix(_) | Component::RootDir | Component::ParentDir = i {
+                return Err(Error::BadTarFileMemberName(
+                    file.path()?.to_str().unwrap().to_string(),
+                ));
+            }
+        }
+
+        let dest_file = dest_dir.join(path);
 
         fs::create_dir_all(dest_file.parent().unwrap())?;
 

src/libexpr/parser.y

@@ -576,12 +576,17 @@ Path resolveExprPath(Path path)
 {
     assert(path[0] == '/');
 
+    unsigned int followCount = 0, maxFollow = 1024;
+
     /* If `path' is a symlink, follow it.  This is so that relative
        path references work. */
     struct stat st;
     while (true) {
+        // Basic cycle/depth limit to avoid infinite loops.
+        if (++followCount >= maxFollow)
+            throw Error("too many symbolic links encountered while traversing the path '%s'", path);
         if (lstat(path.c_str(), &st))
-            throw SysError(format("getting status of '%1%'") % path);
+            throw SysError("getting status of '%s'", path);
         if (!S_ISLNK(st.st_mode)) break;
         path = absPath(readLink(path), dirOf(path));
     }

src/libstore/local.mk

@@ -13,7 +13,9 @@ ifneq ($(OS), FreeBSD)
  libstore_LDFLAGS += -ldl
 endif
 
+ifeq ($(OS), Darwin)
 libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb
+endif
 
 $(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox)))
 

src/libstore/ssh.cc

@@ -62,7 +62,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
         args.push_back(command);
         execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
 
-        throw SysError("executing '%s' on '%s'", command, host);
+        // could not exec ssh/bash
+        throw SysError("unable to execute '%s'", args.front());
     });
 
 
@@ -108,7 +109,7 @@ Path SSHMaster::startMaster()
         addCommonSSHOpts(args);
         execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
 
-        throw SysError("starting SSH master");
+        throw SysError("unable to execute '%s'", args.front());
     });
 
     out.writeSide = -1;

src/libutil/compression.cc

@@ -11,6 +11,8 @@
 #include <brotli/decode.h>
 #include <brotli/encode.h>
 
+#include <zlib.h>
+
 #include <iostream>
 
 namespace nix {
@@ -42,6 +44,66 @@ struct NoneSink : CompressionSink
     void write(const unsigned char * data, size_t len) override { nextSink(data, len); }
 };
 
+struct GzipDecompressionSink : CompressionSink
+{
+    Sink & nextSink;
+    z_stream strm;
+    bool finished = false;
+    uint8_t outbuf[BUFSIZ];
+
+    GzipDecompressionSink(Sink & nextSink) : nextSink(nextSink)
+    {
+        strm.zalloc = Z_NULL;
+        strm.zfree = Z_NULL;
+        strm.opaque = Z_NULL;
+        strm.avail_in = 0;
+        strm.next_in = Z_NULL;
+        strm.next_out = outbuf;
+        strm.avail_out = sizeof(outbuf);
+
+        // Enable gzip and zlib decoding (+32) with 15 windowBits
+        int ret = inflateInit2(&strm,15+32);
+        if (ret != Z_OK)
+            throw CompressionError("unable to initialise gzip encoder");
+    }
+
+    ~GzipDecompressionSink()
+    {
+        inflateEnd(&strm);
+    }
+
+    void finish() override
+    {
+        CompressionSink::flush();
+        write(nullptr, 0);
+    }
+
+    void write(const unsigned char * data, size_t len) override
+    {
+        assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
+
+        strm.next_in = (Bytef *) data;
+        strm.avail_in = len;
+
+        while (!finished && (!data || strm.avail_in)) {
+            checkInterrupt();
+
+            int ret = inflate(&strm,Z_SYNC_FLUSH);
+            if (ret != Z_OK && ret != Z_STREAM_END)
+                throw CompressionError("error while decompressing gzip file: %d (%d, %d)",
+                    zError(ret), len, strm.avail_in);
+
+            finished = ret == Z_STREAM_END;
+
+            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
+                nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
+                strm.next_out = (Bytef *) outbuf;
+                strm.avail_out = sizeof(outbuf);
+            }
+        }
+    }
+};
+
 struct XzDecompressionSink : CompressionSink
 {
     Sink & nextSink;
@@ -215,6 +277,8 @@ ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & ne
         return make_ref<XzDecompressionSink>(nextSink);
     else if (method == "bzip2")
         return make_ref<BzipDecompressionSink>(nextSink);
+    else if (method == "gzip")
+        return make_ref<GzipDecompressionSink>(nextSink);
     else if (method == "br")
         return make_ref<BrotliDecompressionSink>(nextSink);
     else

src/libutil/rust-ffi.cc

@@ -3,10 +3,14 @@
 
 extern "C" std::exception_ptr * make_error(rust::StringSlice s)
 {
-    // FIXME: leak
     return new std::exception_ptr(std::make_exception_ptr(nix::Error(std::string(s.ptr, s.size))));
 }
 
+extern "C" void destroy_error(std::exception_ptr * ex)
+{
+    free(ex);
+}
+
 namespace rust {
 
 std::ostream & operator << (std::ostream & str, const String & s)
@@ -15,4 +19,15 @@ std::ostream & operator << (std::ostream & str, const String & s)
     return str;
 }
 
+size_t Source::sourceWrapper(void * _this, rust::Slice<uint8_t> data)
+{
+    try {
+        // FIXME: how to propagate exceptions?
+        auto n = ((nix::Source *) _this)->read((unsigned char *) data.ptr, data.size);
+        return n;
+    } catch (...) {
+        abort();
+    }
+}
+
 }

src/libutil/rust-ffi.hh

@@ -140,64 +140,58 @@ struct Source
         : fun(sourceWrapper), _this(&_this)
     {}
 
-    // FIXME: how to propagate exceptions?
+    static size_t sourceWrapper(void * _this, rust::Slice<uint8_t> data);
-    static size_t sourceWrapper(void * _this, rust::Slice<uint8_t> data)
-    {
-        auto n = ((nix::Source *) _this)->read((unsigned char *) data.ptr, data.size);
-        return n;
-    }
 };
 
 /* C++ representation of Rust's Result<T, CppException>. */
 template<typename T>
 struct Result
 {
-    unsigned int tag;
+    enum { Ok = 0, Err = 1, Uninit = 2 } tag;
 
     union {
         T data;
         std::exception_ptr * exc;
     };
 
+    Result() : tag(Uninit) { }; // FIXME: remove
+
+    Result(const Result &) = delete;
+
+    Result(Result && other)
+        : tag(other.tag)
+    {
+        other.tag = Uninit;
+        if (tag == Ok)
+            data = std::move(other.data);
+        else if (tag == Err)
+            exc = other.exc;
+    }
+
     ~Result()
     {
-        if (tag == 0)
+        if (tag == Ok)
             data.~T();
-        else if (tag == 1)
+        else if (tag == Err)
-            // FIXME: don't leak exc
+            free(exc);
+        else if (tag == Uninit)
             ;
+        else
+            abort();
     }
 
     /* Rethrow the wrapped exception or return the wrapped value. */
     T unwrap()
     {
-        if (tag == 0)
+        if (tag == Ok) {
+            tag = Uninit;
             return std::move(data);
-        else if (tag == 1)
+        }
+        else if (tag == Err)
             std::rethrow_exception(*exc);
         else
             abort();
     }
 };
 
-template<typename T>
-struct CBox
-{
-    T * ptr;
-
-    T * operator ->()
-    {
-        return ptr;
-    }
-
-    CBox(T * ptr) : ptr(ptr) { }
-    CBox(const CBox &) = delete;
-    CBox(CBox &&) = delete;
-
-    ~CBox()
-    {
-        free(ptr);
-    }
-};
-
 }

src/libutil/tarfile.cc

@@ -3,7 +3,7 @@
 
 extern "C" {
     rust::Result<std::tuple<>> *
-    unpack_tarfile(rust::Source source, rust::StringSlice dest_dir);
+    unpack_tarfile(rust::Source source, rust::StringSlice dest_dir, rust::Result<std::tuple<>> & out);
 }
 
 namespace nix {
@@ -11,7 +11,9 @@ namespace nix {
 void unpackTarfile(Source & source, const Path & destDir)
 {
     rust::Source source2(source);
-    rust::CBox(unpack_tarfile(source2, destDir))->unwrap();
+    rust::Result<std::tuple<>> res;
+    unpack_tarfile(source2, destDir, res);
+    res.unwrap();
 }
 
 void unpackTarfile(const Path & tarFile, const Path & destDir,
@@ -22,8 +24,8 @@ void unpackTarfile(const Path & tarFile, const Path & destDir,
     auto source = sinkToSource([&](Sink & sink) {
         // FIXME: look at first few bytes to determine compression type.
         auto decompressor =
-            // FIXME: add .gz support
             hasSuffix(*baseName, ".bz2") ? makeDecompressionSink("bzip2", sink) :
+            hasSuffix(*baseName, ".gz") ? makeDecompressionSink("gzip", sink) :
             hasSuffix(*baseName, ".xz") ? makeDecompressionSink("xz", sink) :
             makeDecompressionSink("none", sink);
         readFile(tarFile, *decompressor);

src/nix-build/nix-build.cc

@@ -106,7 +106,7 @@ static void _main(int argc, char * * argv)
     // Heuristic to see if we're invoked as a shebang script, namely,
     // if we have at least one argument, it's the name of an
     // executable file, and it starts with "#!".
-    if (runEnv && argc > 1 && !std::regex_search(std::string(baseNameOf(argv[1])), std::regex("nix-shell"))) {
+    if (runEnv && argc > 1) {
         script = argv[1];
         try {
             auto lines = tokenizeString<Strings>(readFile(script), "\n");

src/nix/progress-bar.cc

@@ -155,7 +155,7 @@ public:
         if (type == actBuild) {
             auto name = storePathToName(getS(fields, 0));
             if (hasSuffix(name, ".drv"))
-                name = name.substr(name.size() - 4);
+                name = name.substr(0, name.size() - 4);
             i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name);
             auto machineName = getS(fields, 1);
             if (machineName != "")
@@ -180,7 +180,7 @@ public:
         if (type == actPostBuildHook) {
             auto name = storePathToName(getS(fields, 0));
             if (hasSuffix(name, ".drv"))
-                name = name.substr(name.size() - 4);
+                name = name.substr(0, name.size() - 4);
             i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name);
             i->name = DrvName(name).name;
         }

tests/tarball.sh

@@ -10,19 +10,33 @@ mkdir -p $tarroot
 cp dependencies.nix $tarroot/default.nix
 cp config.nix dependencies.builder*.sh $tarroot/
 
-tarball=$TEST_ROOT/tarball.tar.xz
+test_tarball() {
-(cd $TEST_ROOT && tar c tarball) | xz > $tarball
+    local ext="$1"
+    local compressor="$2"
 
-nix-env -f file://$tarball -qa --out-path | grep -q dependencies
+    tarball=$TEST_ROOT/tarball.tar$ext
+    (cd $TEST_ROOT && tar c tarball) | $compressor > $tarball
 
-nix-build -o $TEST_ROOT/result file://$tarball
+    nix-env -f file://$tarball -qa --out-path | grep -q dependencies
 
-nix-build -o $TEST_ROOT/result '<foo>' -I foo=file://$tarball
+    nix-build -o $TEST_ROOT/result file://$tarball
 
-nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)"
+    nix-build -o $TEST_ROOT/result '<foo>' -I foo=file://$tarball
 
-nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar.xz
+    nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)"
-nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball.tar.xz
-(! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball.tar.xz)
 
-nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball.tar.xz -I fnord=.
+    nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext
+    nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball$ext
+    (! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball$ext)
+
+    nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball$ext -I fnord=.
+}
+
+test_tarball '' cat
+test_tarball .xz xz
+test_tarball .gz gzip
+
+rm -rf $TEST_ROOT/tmp
+mkdir -p $TEST_ROOT/tmp
+(! TMPDIR=$TEST_ROOT/tmp XDG_RUNTIME_DIR=$TEST_ROOT/tmp nix-env -f file://$(pwd)/bad.tar.xz -qa --out-path)
+(! [ -e $TEST_ROOT/tmp/bad ])