From 2df9a972fc1f361020ece1a21721379d090dd0ae Mon Sep 17 00:00:00 2001 From: Jude Taylor Date: Sat, 13 Aug 2016 11:36:22 -0700 Subject: [PATCH 01/11] resolve-system-dependencies: implement in C++ --- Makefile | 1 + src/resolve-system-dependencies/local.mk | 7 + .../resolve-system-dependencies.cc | 201 ++++++++++++++++++ 3 files changed, 209 insertions(+) create mode 100644 src/resolve-system-dependencies/local.mk create mode 100644 src/resolve-system-dependencies/resolve-system-dependencies.cc diff --git a/Makefile b/Makefile index 11b3309f8..6b2d97cdd 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ makefiles = \ src/nix-collect-garbage/local.mk \ src/nix-prefetch-url/local.mk \ src/buildenv/local.mk \ + src/resolve-system-dependencies/local.mk \ perl/local.mk \ scripts/local.mk \ corepkgs/local.mk \ diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk new file mode 100644 index 000000000..4e1379ae8 --- /dev/null +++ b/src/resolve-system-dependencies/local.mk @@ -0,0 +1,7 @@ +programs += resolve-system-dependencies + +resolve-system-dependencies_DIR := $(d) + +resolve-system-dependencies_LIBS := libstore libmain libutil libformat + +resolve-system-dependencies_SOURCES := $(d)/resolve-system-dependencies.cc diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc new file mode 100644 index 000000000..80dc6cf1a --- /dev/null +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -0,0 +1,201 @@ +#include "derivations.hh" +#include "globals.hh" +#include "shared.hh" +#include "store-api.hh" +#include +#include +#include +#include +#include +#include + +using namespace nix; + +typedef std::map> SetMap; + +static auto cacheDir = Path{}; + +Path resolveCacheFile(const Path & lib) { + Path lib2 = Path(lib); + std::replace(lib2.begin(), lib2.end(), '/', '%'); + return cacheDir + "/" + lib2; +} + +std::set readCacheFile(const Path & file) { + return tokenizeString>(readFile(file), "\n"); +} + +void writeCacheFile(const Path & file, std::set & deps) { + std::ofstream fp; + fp.open(file); + for (auto & d : deps) { + fp << d << "\n"; + } + fp.close(); +} + +std::string find_dylib_name(FILE *obj_file, struct load_command cmd) { + fpos_t pos; + fgetpos(obj_file, &pos); + struct dylib_command dylc; + dylc.cmd = cmd.cmd; + dylc.cmdsize = cmd.cmdsize; + fread(&dylc.dylib, sizeof(struct dylib), 1, obj_file); + + char *dylib_name = (char*)calloc(cmd.cmdsize, sizeof(char)); + fseek(obj_file, + // offset is calculated from the beginning of the load command, which is two + // uint32_t's backwards + dylc.dylib.name.offset - (sizeof(uint32_t) * 2) + pos, + SEEK_SET); + fread(dylib_name, sizeof(char), cmd.cmdsize, obj_file); + fseek(obj_file, pos, SEEK_SET); + return std::string(dylib_name); +} + +bool seek_mach64_blob(FILE *obj_file, enum NXByteOrder end) { + struct fat_header head; + fread(&head, sizeof(struct fat_header), 1, obj_file); + swap_fat_header(&head, end); + for(uint32_t narches = 0; narches < head.nfat_arch; narches++) { + struct fat_arch arch; + fread(&arch, sizeof(struct fat_arch), 1, obj_file); + swap_fat_arch(&arch, 1, end); + if(arch.cputype == CPU_TYPE_X86_64) { + fseek(obj_file, arch.offset, SEEK_SET); + return true; + } + } + return false; +} + +std::set runResolver(const Path & filename) { + FILE *obj_file = fopen(filename.c_str(), "rb"); + uint32_t magic; + fread(&magic, sizeof(uint32_t), 1, obj_file); + fseek(obj_file, 0, SEEK_SET); + enum NXByteOrder endianness; + if(magic == 0xBEBAFECA) { + endianness = NX_BigEndian; + if(!seek_mach64_blob(obj_file, endianness)) { + std::cerr << "Could not find any mach64 blobs in file " << filename << ", continuing..." << std::endl; + return std::set(); + } + } + struct mach_header_64 header; + fread(&header, sizeof(struct mach_header_64), 1, obj_file); + if(!(header.magic == MH_MAGIC_64 || header.magic == MH_CIGAM_64)) { + std::cerr << "Not a mach-o object file: " << filename << std::endl; + return std::set(); + } + std::set libs; + for(uint32_t i = 0; i < header.ncmds; i++) { + struct load_command cmd; + fread(&cmd.cmd, sizeof(uint32_t), 1, obj_file); + fread(&cmd.cmdsize, sizeof(uint32_t), 1, obj_file); + switch(cmd.cmd) { + case LC_LOAD_DYLIB: + case LC_REEXPORT_DYLIB: + libs.insert(find_dylib_name(obj_file, cmd)); + break; + } + fseek(obj_file, cmd.cmdsize - (sizeof(uint32_t) * 2), SEEK_CUR); + } + fclose(obj_file); + libs.erase(filename); + return libs; +} + +bool isSymlink(const Path & path) { + struct stat st; + if(lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path ‘%1%’") % path); + + return S_ISLNK(st.st_mode); +} + +Path resolveSymlink(const Path & path) { + char buf[PATH_MAX]; + ssize_t len = readlink(path.c_str(), buf, sizeof(buf) - 1); + if(len != -1) { + buf[len] = 0; + return Path(buf); + } else { + throw SysError(format("readlink('%1%')") % path); + } +} + +std::set resolve_tree(const Path & path, PathSet & deps) { + std::set results; + if(deps.find(path) != deps.end()) { + return std::set(); + } + deps.insert(path); + for (auto & lib : runResolver(path)) { + results.insert(lib); + for (auto & p : resolve_tree(lib, deps)) { + results.insert(p); + } + } + return results; +} + +std::set get_path(const Path & path) { + Path cacheFile = resolveCacheFile(path); + if(pathExists(cacheFile)) { + return readCacheFile(cacheFile); + } + + std::set deps; + std::set paths; + paths.insert(path); + + Path next_path = Path(path); + while(isSymlink(next_path)) { + next_path = resolveSymlink(next_path); + paths.insert(next_path); + } + + for(auto & t : resolve_tree(next_path, deps)) { + paths.insert(t); + } + + writeCacheFile(cacheFile, paths); + + return paths; +} + +int main(int argc, char ** argv) { + return handleExceptions(argv[0], [&]() { + initNix(); + + struct utsname _uname; + + uname(&_uname); + + cacheDir = (format("%1%/dependency-maps/%2%-%3%-%4%") + % settings.nixStateDir + % _uname.machine + % _uname.sysname + % _uname.release).str(); + + auto store = openStore(); + + auto drv = store->derivationFromPath(Path(argv[1])); + Strings impurePaths = tokenizeString(get(drv.env, "__impureHostDeps")); + + std::set all_paths; + + for (auto & path : impurePaths) { + for(auto & p : get_path(path)) { + all_paths.insert(p); + } + } + + std::cout << "extra-chroot-dirs" << std::endl; + for(auto & path : all_paths) { + std::cout << path << std::endl; + } + std::cout << std::endl; + }); +} From 596e4a5693fc710ee47126d19910951b22d91018 Mon Sep 17 00:00:00 2001 From: Jude Taylor Date: Sat, 13 Aug 2016 15:27:49 -0700 Subject: [PATCH 02/11] remove old traces of resolve-system-dependencies --- scripts/local.mk | 7 -- scripts/resolve-system-dependencies.pl.in | 122 ---------------------- src/libstore/globals.cc | 2 +- src/resolve-system-dependencies/local.mk | 2 + 4 files changed, 3 insertions(+), 130 deletions(-) delete mode 100755 scripts/resolve-system-dependencies.pl.in diff --git a/scripts/local.mk b/scripts/local.mk index fef256451..9852d8298 100644 --- a/scripts/local.mk +++ b/scripts/local.mk @@ -11,19 +11,12 @@ nix_noinst_scripts := \ $(d)/nix-profile.sh \ $(d)/nix-reduce-build -ifeq ($(OS), Darwin) - nix_noinst_scripts += $(d)/resolve-system-dependencies.pl -endif - noinst-scripts += $(nix_noinst_scripts) profiledir = $(sysconfdir)/profile.d $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644)) $(eval $(call install-program-in, $(d)/build-remote.pl, $(libexecdir)/nix)) -ifeq ($(OS), Darwin) - $(eval $(call install-program-in, $(d)/resolve-system-dependencies.pl, $(libexecdir)/nix)) -endif $(eval $(call install-symlink, nix-build, $(bindir)/nix-shell)) clean-files += $(nix_bin_scripts) $(nix_noinst_scripts) diff --git a/scripts/resolve-system-dependencies.pl.in b/scripts/resolve-system-dependencies.pl.in deleted file mode 100755 index a20f0dc02..000000000 --- a/scripts/resolve-system-dependencies.pl.in +++ /dev/null @@ -1,122 +0,0 @@ -#! @perl@ -w @perlFlags@ - -use utf8; -use strict; -use warnings; -use Cwd qw(realpath); -use Errno; -use File::Basename qw(dirname); -use File::Path qw(make_path); -use File::Spec::Functions qw(catfile); -use List::Util qw(reduce); -use IPC::Open3; -use Nix::Config; -use Nix::Store qw(derivationFromPath); -use POSIX qw(uname); -use Storable qw(lock_retrieve lock_store); - -my ($sysname, undef, $version, undef, $machine) = uname; -$sysname =~ /Darwin/ or die "This tool is only meant to be used on Darwin systems."; - -my $cache = "$Nix::Config::stateDir/dependency-maps/$machine-$sysname-$version.map"; - -make_path dirname($cache); - -our $DEPS; -eval { - $DEPS = lock_retrieve($cache); -}; - -if($!{ENOENT}) { - lock_store {}, $cache; - $DEPS = {}; -} elsif($@) { - die "Unable to obtain a lock on dependency-map file $cache: $@"; -} - -sub mkset(@) { - my %set; - @set{@_} = (); - \%set -} - -sub union($$) { - my ($set1, $set2) = @_; - my %new = (%$set1, %$set2); - \%new -} - -sub cache_filepath($) { - my $fp = shift; - $fp =~ s/-/--/g; - $fp =~ s/\//-/g; - $fp =~ s/^-//g; - catfile $cache, $fp -} - -sub resolve_tree { - sub resolve_tree_inner { - my ($lib, $TREE) = @_; - return if (defined $TREE->{$lib}); - $TREE->{$lib} = mkset(@{cache_get($lib)}); - foreach my $dep (keys %{$TREE->{$lib}}) { - resolve_tree_inner($dep, $TREE); - } - values %$TREE - } - - reduce { union($a, $b) } {}, resolve_tree_inner(@_) -} - -sub cache_get { - my $key = shift; - if (defined $DEPS->{$key}) { - $DEPS->{$key} - } else { - cache_insert($key); - cache_get($key) - } -} - -sub cache_insert($) { - my $key = shift; - print STDERR "Finding dependencies for $key...\n"; - my @deps = find_deps($key); - $DEPS->{$key} = \@deps; -} - -sub find_deps($) { - my $lib = shift; - my($chld_in, $chld_out, $chld_err); - my $pid = open3($chld_in, $chld_out, $chld_err, "@otool@", "-L", "-arch", "x86_64", $lib); - waitpid($pid, 0); - my $line = readline $chld_out; - if($? == 0 and $line !~ /not an object file/) { - my @libs; - while(<$chld_out>) { - my $dep = (split /\s+/)[1]; - push @libs, $dep unless $dep eq $lib or $dep =~ /\@rpath/; - } - @libs - } elsif (-l $lib) { - (realpath($lib)) - } else { - () - } -} - -if (defined $ARGV[0]) { - my $deps = derivationFromPath($ARGV[0])->{"env"}->{"__impureHostDeps"}; - if (defined $deps) { - my @files = split(/\s+/, $deps); - my $depcache = {}; - my $depset = reduce { union($a, $b) } (map { resolve_tree($_, $depcache) } @files); - print "extra-chroot-dirs\n"; - print join("\n", keys %$depset); - print "\n"; - } - lock_store($DEPS, $cache); -} else { - print STDERR "Usage: $0 path/to/derivation.drv\n"; - exit 1 -} diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 7bf48be37..ecf81e8eb 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -76,7 +76,7 @@ void Settings::processEnvironment() // should be set with the other config options, but depends on nixLibexecDir #ifdef __APPLE__ - preBuildHook = nixLibexecDir + "/nix/resolve-system-dependencies.pl"; + preBuildHook = nixLibexecDir + "/nix/resolve-system-dependencies"; #endif } diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk index 4e1379ae8..abf4edaff 100644 --- a/src/resolve-system-dependencies/local.mk +++ b/src/resolve-system-dependencies/local.mk @@ -2,6 +2,8 @@ programs += resolve-system-dependencies resolve-system-dependencies_DIR := $(d) +resolve-system-dependencies_INSTALL_DIR := $(libexecdir)/nix + resolve-system-dependencies_LIBS := libstore libmain libutil libformat resolve-system-dependencies_SOURCES := $(d)/resolve-system-dependencies.cc From 5b01f5cbb2d53d186e5ed5d901840148ed7f3567 Mon Sep 17 00:00:00 2001 From: Jude Taylor Date: Sat, 13 Aug 2016 15:30:35 -0700 Subject: [PATCH 03/11] remove otool check --- configure.ac | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/configure.ac b/configure.ac index a9e6b4313..1ae0e782d 100644 --- a/configure.ac +++ b/configure.ac @@ -246,23 +246,6 @@ AC_MSG_RESULT(yes) AC_SUBST(perlFlags) -# Check for otool, an optional dependency on Darwin. -AC_PATH_PROG(otool, otool) -AC_MSG_CHECKING([that otool works]) -case $host_os in - darwin*) - if test -z "$otool" || ! $otool --version 2>/dev/null; then - AC_MSG_RESULT(no) - AC_MSG_ERROR([Can't get version from otool; do you need to install developer tools?]) - fi - AC_MSG_RESULT(yes) - ;; - *) - AC_MSG_RESULT(not needed) - ;; -esac - - # Whether to build the Perl bindings AC_MSG_CHECKING([whether to build the Perl bindings]) AC_ARG_ENABLE(perl-bindings, AC_HELP_STRING([--enable-perl-bindings], From f37b6fd07e499a2d566be2b0b564facf042a0480 Mon Sep 17 00:00:00 2001 From: Jude Taylor Date: Sun, 14 Aug 2016 18:54:40 -0700 Subject: [PATCH 04/11] add a missing load command --- src/resolve-system-dependencies/resolve-system-dependencies.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index 80dc6cf1a..d432781d4 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -95,6 +95,7 @@ std::set runResolver(const Path & filename) { fread(&cmd.cmdsize, sizeof(uint32_t), 1, obj_file); switch(cmd.cmd) { case LC_LOAD_DYLIB: + case LC_LOAD_UPWARD_DYLIB: case LC_REEXPORT_DYLIB: libs.insert(find_dylib_name(obj_file, cmd)); break; From 60f4b25d7d235e971bc795cf95324cf201314b19 Mon Sep 17 00:00:00 2001 From: Jude Taylor Date: Sun, 14 Aug 2016 19:10:38 -0700 Subject: [PATCH 05/11] make inclusion conditional --- src/resolve-system-dependencies/local.mk | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk index abf4edaff..8792a4a25 100644 --- a/src/resolve-system-dependencies/local.mk +++ b/src/resolve-system-dependencies/local.mk @@ -1,4 +1,6 @@ -programs += resolve-system-dependencies +ifeq ($(OS), Darwin) + programs += resolve-system-dependencies +endif resolve-system-dependencies_DIR := $(d) From adf0216d98146f925659e8a5f83a9c653ae78b22 Mon Sep 17 00:00:00 2001 From: Jude Taylor Date: Mon, 15 Aug 2016 10:43:14 -0700 Subject: [PATCH 06/11] code review comments --- .../resolve-system-dependencies.cc | 249 +++++++++--------- 1 file changed, 123 insertions(+), 126 deletions(-) diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index d432781d4..641069c2c 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -11,14 +11,11 @@ using namespace nix; -typedef std::map> SetMap; - static auto cacheDir = Path{}; -Path resolveCacheFile(const Path & lib) { - Path lib2 = Path(lib); - std::replace(lib2.begin(), lib2.end(), '/', '%'); - return cacheDir + "/" + lib2; +Path resolveCacheFile(Path lib) { + std::replace(lib.begin(), lib.end(), '/', '%'); + return cacheDir + "/" + lib; } std::set readCacheFile(const Path & file) { @@ -34,169 +31,169 @@ void writeCacheFile(const Path & file, std::set & deps) { fp.close(); } -std::string find_dylib_name(FILE *obj_file, struct load_command cmd) { - fpos_t pos; - fgetpos(obj_file, &pos); - struct dylib_command dylc; - dylc.cmd = cmd.cmd; - dylc.cmdsize = cmd.cmdsize; - fread(&dylc.dylib, sizeof(struct dylib), 1, obj_file); +std::string findDylibName(FILE *obj_file, struct load_command cmd) { + fpos_t pos; + fgetpos(obj_file, &pos); + struct dylib_command dylc; + dylc.cmd = cmd.cmd; + dylc.cmdsize = cmd.cmdsize; + fread(&dylc.dylib, sizeof(struct dylib), 1, obj_file); - char *dylib_name = (char*)calloc(cmd.cmdsize, sizeof(char)); - fseek(obj_file, - // offset is calculated from the beginning of the load command, which is two - // uint32_t's backwards - dylc.dylib.name.offset - (sizeof(uint32_t) * 2) + pos, - SEEK_SET); - fread(dylib_name, sizeof(char), cmd.cmdsize, obj_file); - fseek(obj_file, pos, SEEK_SET); - return std::string(dylib_name); + char *dylib_name = (char*)calloc(cmd.cmdsize, sizeof(char)); + fseek(obj_file, + // offset is calculated from the beginning of the load command, which is two + // uint32_t's backwards + dylc.dylib.name.offset - (sizeof(uint32_t) * 2) + pos, + SEEK_SET); + fread(dylib_name, sizeof(char), cmd.cmdsize, obj_file); + fseek(obj_file, pos, SEEK_SET); + return std::string(dylib_name); } -bool seek_mach64_blob(FILE *obj_file, enum NXByteOrder end) { - struct fat_header head; - fread(&head, sizeof(struct fat_header), 1, obj_file); - swap_fat_header(&head, end); - for(uint32_t narches = 0; narches < head.nfat_arch; narches++) { - struct fat_arch arch; - fread(&arch, sizeof(struct fat_arch), 1, obj_file); - swap_fat_arch(&arch, 1, end); - if(arch.cputype == CPU_TYPE_X86_64) { - fseek(obj_file, arch.offset, SEEK_SET); - return true; +bool seekMach64Blob(FILE *obj_file, enum NXByteOrder end) { + struct fat_header head; + fread(&head, sizeof(struct fat_header), 1, obj_file); + swap_fat_header(&head, end); + for(uint32_t narches = 0; narches < head.nfat_arch; narches++) { + struct fat_arch arch; + fread(&arch, sizeof(struct fat_arch), 1, obj_file); + swap_fat_arch(&arch, 1, end); + if(arch.cputype == CPU_TYPE_X86_64) { + fseek(obj_file, arch.offset, SEEK_SET); + return true; + } } - } - return false; + return false; } std::set runResolver(const Path & filename) { - FILE *obj_file = fopen(filename.c_str(), "rb"); - uint32_t magic; - fread(&magic, sizeof(uint32_t), 1, obj_file); - fseek(obj_file, 0, SEEK_SET); - enum NXByteOrder endianness; - if(magic == 0xBEBAFECA) { - endianness = NX_BigEndian; - if(!seek_mach64_blob(obj_file, endianness)) { - std::cerr << "Could not find any mach64 blobs in file " << filename << ", continuing..." << std::endl; - return std::set(); + FILE *obj_file = fopen(filename.c_str(), "rb"); + uint32_t magic; + fread(&magic, sizeof(uint32_t), 1, obj_file); + fseek(obj_file, 0, SEEK_SET); + enum NXByteOrder endianness; + if(magic == 0xBEBAFECA) { + endianness = NX_BigEndian; + if(!seekMach64Blob(obj_file, endianness)) { + std::cerr << "Could not find any mach64 blobs in file " << filename << ", continuing..." << std::endl; + return std::set(); + } } - } - struct mach_header_64 header; - fread(&header, sizeof(struct mach_header_64), 1, obj_file); - if(!(header.magic == MH_MAGIC_64 || header.magic == MH_CIGAM_64)) { - std::cerr << "Not a mach-o object file: " << filename << std::endl; - return std::set(); - } - std::set libs; - for(uint32_t i = 0; i < header.ncmds; i++) { - struct load_command cmd; - fread(&cmd.cmd, sizeof(uint32_t), 1, obj_file); - fread(&cmd.cmdsize, sizeof(uint32_t), 1, obj_file); - switch(cmd.cmd) { - case LC_LOAD_DYLIB: - case LC_LOAD_UPWARD_DYLIB: - case LC_REEXPORT_DYLIB: - libs.insert(find_dylib_name(obj_file, cmd)); - break; + struct mach_header_64 header; + fread(&header, sizeof(struct mach_header_64), 1, obj_file); + if(!(header.magic == MH_MAGIC_64 || header.magic == MH_CIGAM_64)) { + std::cerr << "Not a mach-o object file: " << filename << std::endl; + return std::set(); } - fseek(obj_file, cmd.cmdsize - (sizeof(uint32_t) * 2), SEEK_CUR); - } - fclose(obj_file); - libs.erase(filename); - return libs; + std::set libs; + for(uint32_t i = 0; i < header.ncmds; i++) { + struct load_command cmd; + fread(&cmd.cmd, sizeof(cmd.cmd), 1, obj_file); + fread(&cmd.cmdsize, sizeof(cmd.cmdsize), 1, obj_file); + switch(cmd.cmd) { + case LC_LOAD_DYLIB: + case LC_LOAD_UPWARD_DYLIB: + case LC_REEXPORT_DYLIB: + libs.insert(findDylibName(obj_file, cmd)); + break; + } + fseek(obj_file, cmd.cmdsize - (sizeof(uint32_t) * 2), SEEK_CUR); + } + fclose(obj_file); + libs.erase(filename); + return libs; } bool isSymlink(const Path & path) { - struct stat st; - if(lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path ‘%1%’") % path); + struct stat st; + if(lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path ‘%1%’") % path); - return S_ISLNK(st.st_mode); + return S_ISLNK(st.st_mode); } Path resolveSymlink(const Path & path) { char buf[PATH_MAX]; ssize_t len = readlink(path.c_str(), buf, sizeof(buf) - 1); if(len != -1) { - buf[len] = 0; - return Path(buf); + buf[len] = 0; + return Path(buf); } else { - throw SysError(format("readlink('%1%')") % path); + throw SysError(format("readlink('%1%')") % path); } } -std::set resolve_tree(const Path & path, PathSet & deps) { - std::set results; - if(deps.find(path) != deps.end()) { - return std::set(); - } - deps.insert(path); - for (auto & lib : runResolver(path)) { - results.insert(lib); - for (auto & p : resolve_tree(lib, deps)) { - results.insert(p); +std::set resolveTree(const Path & path, PathSet & deps) { + std::set results; + if(deps.find(path) != deps.end()) { + return std::set(); } - } - return results; + deps.insert(path); + for (auto & lib : runResolver(path)) { + results.insert(lib); + for (auto & p : resolveTree(lib, deps)) { + results.insert(p); + } + } + return results; } -std::set get_path(const Path & path) { - Path cacheFile = resolveCacheFile(path); - if(pathExists(cacheFile)) { - return readCacheFile(cacheFile); - } +std::set getPath(const Path & path) { + Path cacheFile = resolveCacheFile(path); + if(pathExists(cacheFile)) { + return readCacheFile(cacheFile); + } - std::set deps; - std::set paths; - paths.insert(path); + std::set deps; + std::set paths; + paths.insert(path); - Path next_path = Path(path); - while(isSymlink(next_path)) { - next_path = resolveSymlink(next_path); - paths.insert(next_path); - } + Path next_path = Path(path); + while(isSymlink(next_path)) { + next_path = resolveSymlink(next_path); + paths.insert(next_path); + } - for(auto & t : resolve_tree(next_path, deps)) { - paths.insert(t); - } + for(auto & t : resolveTree(next_path, deps)) { + paths.insert(t); + } - writeCacheFile(cacheFile, paths); + writeCacheFile(cacheFile, paths); - return paths; + return paths; } int main(int argc, char ** argv) { return handleExceptions(argv[0], [&]() { - initNix(); + initNix(); - struct utsname _uname; + struct utsname _uname; - uname(&_uname); + uname(&_uname); - cacheDir = (format("%1%/dependency-maps/%2%-%3%-%4%") - % settings.nixStateDir - % _uname.machine - % _uname.sysname - % _uname.release).str(); + cacheDir = (format("%1%/dependency-maps/%2%-%3%-%4%") + % settings.nixStateDir + % _uname.machine + % _uname.sysname + % _uname.release).str(); - auto store = openStore(); + auto store = openStore(); - auto drv = store->derivationFromPath(Path(argv[1])); - Strings impurePaths = tokenizeString(get(drv.env, "__impureHostDeps")); + auto drv = store->derivationFromPath(Path(argv[1])); + Strings impurePaths = tokenizeString(get(drv.env, "__impureHostDeps")); - std::set all_paths; + std::set all_paths; - for (auto & path : impurePaths) { - for(auto & p : get_path(path)) { - all_paths.insert(p); + for (auto & path : impurePaths) { + for(auto & p : getPath(path)) { + all_paths.insert(p); + } } - } - std::cout << "extra-chroot-dirs" << std::endl; - for(auto & path : all_paths) { - std::cout << path << std::endl; - } - std::cout << std::endl; + std::cout << "extra-chroot-dirs" << std::endl; + for(auto & path : all_paths) { + std::cout << path << std::endl; + } + std::cout << std::endl; }); } From 7b006122aee221f1ec69b6439bf86e79190491c1 Mon Sep 17 00:00:00 2001 From: Jude Taylor Date: Mon, 15 Aug 2016 10:46:21 -0700 Subject: [PATCH 07/11] ensure presence of directories --- .../resolve-system-dependencies.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index 641069c2c..4b2f55f09 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -171,12 +171,17 @@ int main(int argc, char ** argv) { uname(&_uname); - cacheDir = (format("%1%/dependency-maps/%2%-%3%-%4%") - % settings.nixStateDir + auto cacheParentDir = (format("%1%/dependency-maps") % settings.nixStateDir).str(); + + cacheDir = (format("%1%/%2%-%3%-%4%") + % cacheParentDir % _uname.machine % _uname.sysname % _uname.release).str(); + mkdir(cacheParentDir.c_str(), 0755); + mkdir(cacheDir.c_str(), 0755); + auto store = openStore(); auto drv = store->derivationFromPath(Path(argv[1])); From 90516c5a7be5457813313d89ecf8ae9d3c0d1708 Mon Sep 17 00:00:00 2001 From: Jude Taylor Date: Tue, 16 Aug 2016 12:45:55 -0700 Subject: [PATCH 08/11] switch to mmap strategy --- .../resolve-system-dependencies.cc | 149 ++++++++---------- 1 file changed, 68 insertions(+), 81 deletions(-) diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index 4b2f55f09..216be1210 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -6,9 +6,13 @@ #include #include #include +#include +#include #include #include +#define DO_SWAP(x, y) ((x) ? OSSwapInt32(y) : (y)) + using namespace nix; static auto cacheDir = Path{}; @@ -31,76 +35,59 @@ void writeCacheFile(const Path & file, std::set & deps) { fp.close(); } -std::string findDylibName(FILE *obj_file, struct load_command cmd) { - fpos_t pos; - fgetpos(obj_file, &pos); - struct dylib_command dylc; - dylc.cmd = cmd.cmd; - dylc.cmdsize = cmd.cmdsize; - fread(&dylc.dylib, sizeof(struct dylib), 1, obj_file); +std::string findDylibName(bool should_swap, ptrdiff_t dylib_command_start) { + struct dylib_command *dylc = (struct dylib_command*)dylib_command_start; - char *dylib_name = (char*)calloc(cmd.cmdsize, sizeof(char)); - fseek(obj_file, - // offset is calculated from the beginning of the load command, which is two - // uint32_t's backwards - dylc.dylib.name.offset - (sizeof(uint32_t) * 2) + pos, - SEEK_SET); - fread(dylib_name, sizeof(char), cmd.cmdsize, obj_file); - fseek(obj_file, pos, SEEK_SET); - return std::string(dylib_name); -} - -bool seekMach64Blob(FILE *obj_file, enum NXByteOrder end) { - struct fat_header head; - fread(&head, sizeof(struct fat_header), 1, obj_file); - swap_fat_header(&head, end); - for(uint32_t narches = 0; narches < head.nfat_arch; narches++) { - struct fat_arch arch; - fread(&arch, sizeof(struct fat_arch), 1, obj_file); - swap_fat_arch(&arch, 1, end); - if(arch.cputype == CPU_TYPE_X86_64) { - fseek(obj_file, arch.offset, SEEK_SET); - return true; - } - } - return false; + return std::string((char*)(dylib_command_start + DO_SWAP(should_swap, dylc->dylib.name.offset))); } std::set runResolver(const Path & filename) { - FILE *obj_file = fopen(filename.c_str(), "rb"); - uint32_t magic; - fread(&magic, sizeof(uint32_t), 1, obj_file); - fseek(obj_file, 0, SEEK_SET); - enum NXByteOrder endianness; - if(magic == 0xBEBAFECA) { - endianness = NX_BigEndian; - if(!seekMach64Blob(obj_file, endianness)) { - std::cerr << "Could not find any mach64 blobs in file " << filename << ", continuing..." << std::endl; + int fd = open(filename.c_str(), O_RDONLY); + struct stat s; + fstat(fd, &s); + void *obj = mmap(NULL, s.st_size, PROT_READ, MAP_SHARED, fd, 0); + + ptrdiff_t mach64_offset = 0; + + uint32_t magic = ((struct mach_header_64*)obj)->magic; + if(magic == FAT_CIGAM || magic == FAT_MAGIC) { + bool should_swap = magic == FAT_CIGAM; + uint32_t narches = DO_SWAP(should_swap, ((struct fat_header*)obj)->nfat_arch); + + for(uint32_t iter = 0; iter < narches; iter++) { + ptrdiff_t header_offset = (ptrdiff_t)obj + sizeof(struct fat_header); + struct fat_arch* arch = (struct fat_arch*)header_offset; + if(DO_SWAP(should_swap, arch->cputype) == CPU_TYPE_X86_64) { + mach64_offset = (ptrdiff_t)DO_SWAP(should_swap, arch->offset); + break; + } + } + if (mach64_offset == 0) { + printMsg(lvlError, format("Could not find any mach64 blobs in file ‘%1%’, continuing...") % filename); return std::set(); } + } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) { + mach64_offset = 0; } - struct mach_header_64 header; - fread(&header, sizeof(struct mach_header_64), 1, obj_file); - if(!(header.magic == MH_MAGIC_64 || header.magic == MH_CIGAM_64)) { - std::cerr << "Not a mach-o object file: " << filename << std::endl; - return std::set(); - } + + struct mach_header_64 *m_header = (struct mach_header_64 *)((ptrdiff_t)obj + mach64_offset); + + bool should_swap = magic == MH_CIGAM_64; + ptrdiff_t cmd_offset = (ptrdiff_t)m_header + sizeof(struct mach_header_64); + std::set libs; - for(uint32_t i = 0; i < header.ncmds; i++) { - struct load_command cmd; - fread(&cmd.cmd, sizeof(cmd.cmd), 1, obj_file); - fread(&cmd.cmdsize, sizeof(cmd.cmdsize), 1, obj_file); - switch(cmd.cmd) { - case LC_LOAD_DYLIB: + for(uint32_t i = 0; i < DO_SWAP(should_swap, m_header->ncmds); i++) { + struct load_command *cmd = (struct load_command*)cmd_offset; + switch(DO_SWAP(should_swap, cmd->cmd)) { case LC_LOAD_UPWARD_DYLIB: + case LC_LOAD_DYLIB: case LC_REEXPORT_DYLIB: - libs.insert(findDylibName(obj_file, cmd)); + libs.insert(findDylibName(should_swap, cmd_offset)); break; } - fseek(obj_file, cmd.cmdsize - (sizeof(uint32_t) * 2), SEEK_CUR); + cmd_offset += DO_SWAP(should_swap, cmd->cmdsize); } - fclose(obj_file); - libs.erase(filename); + return libs; } @@ -165,40 +152,40 @@ std::set getPath(const Path & path) { int main(int argc, char ** argv) { return handleExceptions(argv[0], [&]() { - initNix(); + initNix(); - struct utsname _uname; + struct utsname _uname; - uname(&_uname); + uname(&_uname); - auto cacheParentDir = (format("%1%/dependency-maps") % settings.nixStateDir).str(); + auto cacheParentDir = (format("%1%/dependency-maps") % settings.nixStateDir).str(); - cacheDir = (format("%1%/%2%-%3%-%4%") - % cacheParentDir - % _uname.machine - % _uname.sysname - % _uname.release).str(); + cacheDir = (format("%1%/%2%-%3%-%4%") + % cacheParentDir + % _uname.machine + % _uname.sysname + % _uname.release).str(); - mkdir(cacheParentDir.c_str(), 0755); - mkdir(cacheDir.c_str(), 0755); + mkdir(cacheParentDir.c_str(), 0755); + mkdir(cacheDir.c_str(), 0755); - auto store = openStore(); + auto store = openStore(); - auto drv = store->derivationFromPath(Path(argv[1])); - Strings impurePaths = tokenizeString(get(drv.env, "__impureHostDeps")); + auto drv = store->derivationFromPath(Path(argv[1])); + Strings impurePaths = tokenizeString(get(drv.env, "__impureHostDeps")); - std::set all_paths; + std::set all_paths; - for (auto & path : impurePaths) { - for(auto & p : getPath(path)) { - all_paths.insert(p); - } + for (auto & path : impurePaths) { + for(auto & p : getPath(path)) { + all_paths.insert(p); } + } - std::cout << "extra-chroot-dirs" << std::endl; - for(auto & path : all_paths) { - std::cout << path << std::endl; - } - std::cout << std::endl; + std::cout << "extra-chroot-dirs" << std::endl; + for(auto & path : all_paths) { + std::cout << path << std::endl; + } + std::cout << std::endl; }); } From cfb77d6e5ba03f40743ba17eef20266909ac5640 Mon Sep 17 00:00:00 2001 From: Jude Taylor Date: Tue, 16 Aug 2016 12:52:49 -0700 Subject: [PATCH 09/11] account for unknown magic numbers --- src/resolve-system-dependencies/resolve-system-dependencies.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index 216be1210..7aad8e1c5 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -68,6 +68,9 @@ std::set runResolver(const Path & filename) { } } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) { mach64_offset = 0; + } else { + printMsg(lvlError, format("Object file has unknown magic number ‘%1%’, skipping it...") % magic); + return std::set(); } struct mach_header_64 *m_header = (struct mach_header_64 *)((ptrdiff_t)obj + mach64_offset); From ca0bce28512bfa8029cc072bb7154b4b6c50c0a6 Mon Sep 17 00:00:00 2001 From: Jude Taylor Date: Tue, 16 Aug 2016 14:09:57 -0700 Subject: [PATCH 10/11] remove double casting --- .../resolve-system-dependencies.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index 7aad8e1c5..8502444ba 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -37,7 +37,6 @@ void writeCacheFile(const Path & file, std::set & deps) { std::string findDylibName(bool should_swap, ptrdiff_t dylib_command_start) { struct dylib_command *dylc = (struct dylib_command*)dylib_command_start; - return std::string((char*)(dylib_command_start + DO_SWAP(should_swap, dylc->dylib.name.offset))); } @@ -73,10 +72,11 @@ std::set runResolver(const Path & filename) { return std::set(); } - struct mach_header_64 *m_header = (struct mach_header_64 *)((ptrdiff_t)obj + mach64_offset); + ptrdiff_t mach_header_offset = (ptrdiff_t)obj + mach64_offset; + struct mach_header_64 *m_header = (struct mach_header_64 *)mach_header_offset; bool should_swap = magic == MH_CIGAM_64; - ptrdiff_t cmd_offset = (ptrdiff_t)m_header + sizeof(struct mach_header_64); + ptrdiff_t cmd_offset = mach_header_offset + sizeof(struct mach_header_64); std::set libs; for(uint32_t i = 0; i < DO_SWAP(should_swap, m_header->ncmds); i++) { From 50c3b5df3204a465338fcb2b000c4bd08b0b7335 Mon Sep 17 00:00:00 2001 From: Jude Taylor Date: Wed, 17 Aug 2016 10:24:11 -0700 Subject: [PATCH 11/11] iterate through fat_headers correctly --- src/resolve-system-dependencies/resolve-system-dependencies.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index 8502444ba..a5f0cd7b3 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -54,7 +54,7 @@ std::set runResolver(const Path & filename) { uint32_t narches = DO_SWAP(should_swap, ((struct fat_header*)obj)->nfat_arch); for(uint32_t iter = 0; iter < narches; iter++) { - ptrdiff_t header_offset = (ptrdiff_t)obj + sizeof(struct fat_header); + ptrdiff_t header_offset = (ptrdiff_t)obj + sizeof(struct fat_header) * (iter + 1); struct fat_arch* arch = (struct fat_arch*)header_offset; if(DO_SWAP(should_swap, arch->cputype) == CPU_TYPE_X86_64) { mach64_offset = (ptrdiff_t)DO_SWAP(should_swap, arch->offset);