From 9b1824ecbd222b4bdc8fa2b6f345dc55ef4872d0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 3 Dec 2020 15:35:38 -0600 Subject: [PATCH 1/3] Add extraPlatforms for Rosetta 2 macOS macOS systems with ARM64 can utilize a translation layer at /Library/Apple/usr/libexec/oah to run x86_64 binaries. This change makes Nix recognize that and it to "extra-platforms". Note that there are two cases here since Nix could be built for either x86_64 or aarch64. In either case, we can switch to the other architecture. Unfortunately there is not a good way to prevent aarch64 binaries from being run in x86_64 contexts or vice versa - programs can always execute programs for the other architecture. --- src/libstore/globals.cc | 22 ++++++++++++++++++++++ src/libstore/globals.hh | 4 +++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index f38601d6d..59c49af8a 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -131,6 +131,28 @@ StringSet Settings::getDefaultSystemFeatures() return features; } +StringSet Settings::getDefaultExtraPlatforms() +{ + if (std::string{SYSTEM} == "x86_64-linux" && !isWSL1()) + return StringSet{"i686-linux"}; +#if __APPLE__ + // Rosetta 2 emulation layer can run x86_64 binaries on aarch64 + // machines. Note that we can’t force processes from executing + // x86_64 in aarch64 environments or vice versa since they can + // always exec with their own binary preferences. + else if (pathExists("/Library/Apple/usr/libexec/oah")) { + if (std::string{SYSTEM} == "x86_64-darwin") + return StringSet{"aarch64-darwin"}; + else if (std::string{SYSTEM} == "aarch64-darwin") + return StringSet{"x86_64-darwin"}; + else + return StringSet{}; + } +#endif + else + return StringSet{}; +} + bool Settings::isExperimentalFeatureEnabled(const std::string & name) { auto & f = experimentalFeatures.get(); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 4655ca058..8666a7d28 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -34,6 +34,8 @@ class Settings : public Config { StringSet getDefaultSystemFeatures(); + StringSet getDefaultExtraPlatforms(); + bool isWSL1(); public: @@ -545,7 +547,7 @@ public: Setting extraPlatforms{ this, - std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{}, + getDefaultExtraPlatforms(), "extra-platforms", R"( Platforms other than the native one which this machine is capable of From 4b9acf4e21a834276b7d061942e7b5d3692662b6 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 3 Dec 2020 15:41:59 -0600 Subject: [PATCH 2/3] Use posix_spawn_setbinpref_np to advise which architecture to run When running universal binaries like /bin/bash, Darwin XNU will choose which architecture of the binary to use based on "binary preferences". This change sets that to the current platform for aarch64 and x86_64 builds. In addition it now uses posix_spawn instead of the usual execve. Note, that this does not prevent the other architecture from being run, just advises which to use. Unfortunately, posix_spawnattr_setbinpref_np does not appear to be inherited by child processes in x86_64 Rosetta 2 translations, meaning that this will not always work as expected. For example: { arm = derivation { name = "test"; system = "aarch64-darwin"; builder = "/bin/bash"; args = [ "-e" (builtins.toFile "test" '' set -x /usr/sbin/sysctl sysctl.proc_translated /usr/sbin/sysctl sysctl.proc_native [ "$(/usr/bin/arch)" = arm64 ] /usr/bin/touch $out '') ]; }; rosetta = derivation { name = "test"; system = "x86_64-darwin"; builder = "/bin/bash"; args = [ "-e" (builtins.toFile "test" '' set -x /usr/sbin/sysctl sysctl.proc_translated /usr/sbin/sysctl sysctl.proc_native [ "$(/usr/bin/arch)" = i386 ] echo It works! /usr/bin/touch $out '') ]; }; } `arm' fails on x86_64-compiled Nix, but `arm' and `rosetta' succeed on aarch64-compiled Nix. I suspect there is a way to fix this since: $ /usr/bin/arch -arch x86_64 /bin/bash \ -c '/usr/bin/arch -arch arm64e /bin/bash -c /usr/bin/arch' arm64 seems to work correctly. We may need to wait for Apple to update system_cmds in opensource.apple.com to find out how though. --- src/libstore/build/derivation-goal.cc | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 1db85bd37..f370fd82d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -50,6 +50,10 @@ #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) #endif +#if __APPLE__ +#include +#endif + #include #include @@ -2844,7 +2848,27 @@ void DerivationGoal::runChild() } } +#if __APPLE__ + posix_spawnattr_t attrp; + + if (posix_spawnattr_init(&attrp)) + throw SysError("failed to initialize builder"); + + if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) + throw SysError("failed to initialize builder"); + + if (drv->platform == "aarch64-darwin") { + cpu_type_t cpu = CPU_TYPE_ARM64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } else if (drv->platform == "x86_64-darwin") { + cpu_type_t cpu = CPU_TYPE_X86_64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } + + posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#else execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#endif throw SysError("executing '%1%'", drv->builder); From 3c9b7029ba88e8b831f2054c085ab1fc55c31673 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 4 Dec 2020 13:26:53 -0600 Subject: [PATCH 3/3] Use com.apple.oahd.plist for rosetta 2 detection --- src/libstore/globals.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 59c49af8a..ad66ef8a8 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -140,7 +140,7 @@ StringSet Settings::getDefaultExtraPlatforms() // machines. Note that we can’t force processes from executing // x86_64 in aarch64 environments or vice versa since they can // always exec with their own binary preferences. - else if (pathExists("/Library/Apple/usr/libexec/oah")) { + else if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) { if (std::string{SYSTEM} == "x86_64-darwin") return StringSet{"aarch64-darwin"}; else if (std::string{SYSTEM} == "aarch64-darwin")