Improve dlopen() on Apple Silicon

- Introduce MAP_JIT which is zero on other platforms - Invent __jit_begin() and __jit_end() which wrap Apple's APIs - Runtime dispatch to sys_icache_invalidate() in __clear_cache()
2025-07-15 23:29:56 +00:00 · 2023-11-17 02:33:14 -08:00 · 2023-11-17 02:33:14 -08:00 · 529cb4817c
commit 529cb4817c
parent 7a9e176ecf
20 changed files with 120 additions and 117 deletions
--- a/libc/calls/munmap-sysv.c
+++ b/libc/calls/munmap-sysv.c
@ -21,6 +21,7 @@
 #include "libc/intrin/describeflags.internal.h"
 #include "libc/intrin/directmap.internal.h"
 #include "libc/intrin/strace.internal.h"
+#include "libc/runtime/syslib.internal.h"

 /**
 * Unmaps memory directly with system.
@ -32,10 +33,12 @@
 */
 int sys_munmap(void *p, size_t n) {
  int rc;
-  if (!IsMetal()) {
-    rc = __sys_munmap(p, n);
-  } else {
+  if (IsXnuSilicon()) {
+    rc = _sysret(__syslib->__munmap(p, n));
+  } else if (IsMetal()) {
    rc = sys_munmap_metal(p, n);
+  } else {
+    rc = __sys_munmap(p, n);
  }
  KERNTRACE("sys_munmap(%p /* %s */, %'zu) → %d", p,
            DescribeFrame((intptr_t)p >> 16), n, rc);
--- a/libc/calls/syscall-sysv.internal.h
+++ b/libc/calls/syscall-sysv.internal.h
@ -20,6 +20,7 @@ i32 __sys_fcntl_cp(i32, i32, ...);
 i32 __sys_fstat(i32, void *);
 i32 __sys_fstatat(i32, const char *, void *, i32);
 i32 __sys_gettid(i64 *);
+i32 __sys_mprotect(void *, u64, i32);
 i32 __sys_munmap(void *, u64);
 i32 __sys_openat(i32, const char *, i32, u32);
 i32 __sys_openat_nc(i32, const char *, i32, u32);
--- a/libc/dlopen/dlopen.c
+++ b/libc/dlopen/dlopen.c
@ -424,12 +424,12 @@ static char *dlerror_set(const char *str) {
  return dlerror_buf;
 }

-static char *foreign_alloc_block(void) {
+static dontinline char *foreign_alloc_block(void) {
  char *p = 0;
  size_t sz = 65536;
  if (!IsWindows()) {
    p = __sys_mmap(0, sz, PROT_READ | PROT_WRITE | PROT_EXEC,
-                   MAP_PRIVATE | MAP_ANONYMOUS, -1, 0, 0);
+                   MAP_PRIVATE | MAP_ANONYMOUS | MAP_JIT, -1, 0, 0);
    if (p == MAP_FAILED) {
      p = 0;
    }
@ -448,7 +448,7 @@ static char *foreign_alloc_block(void) {
  return p;
 }

-static void *foreign_alloc(size_t n) {
+static dontinline void *foreign_alloc(size_t n) {
  void *res;
  static char *block;
  static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
@ -520,11 +520,14 @@ static void *foreign_thunk_sysv(void *func) {
  *p++ = 0xff;
  *p++ = 0xe2;
 #elif defined(__aarch64__)
-  if (!(p = code = foreign_alloc(36))) return 0;  // 16 + 16 + 4 = 36
-  p = movimm(p, 5, (uintptr_t)func);
-  p = movimm(p, 10, (uintptr_t)foreign_tramp);
-  *(uint32_t *)p = 0xd61f0140;  // br x10
-  __clear_cache(code, p + 4);
+  __jit_begin();
+  if ((p = code = foreign_alloc(36))) {
+    p = movimm(p, 5, (uintptr_t)func);
+    p = movimm(p, 10, (uintptr_t)foreign_tramp);
+    *(uint32_t *)p = 0xd61f0140;  // br x10
+    __clear_cache(code, p + 4);
+  }
+  __jit_end();
 #else
 #error "unsupported architecture"
 #endif
--- a/libc/dlopen/dlopen.mk
+++ b/libc/dlopen/dlopen.mk
@ -33,8 +33,7 @@ LIBC_DLOPEN_A_DIRECTDEPS =						\
 	LIBC_RUNTIME							\
 	LIBC_SYSV							\
 	LIBC_SYSV_CALLS							\
-	LIBC_STR							\
-	THIRD_PARTY_COMPILER_RT
+	LIBC_STR

 LIBC_DLOPEN_A_DEPS :=							\
 	$(call uniq,$(foreach x,$(LIBC_DLOPEN_A_DIRECTDEPS),$($(x))))
--- a/libc/intrin/directmap.c
+++ b/libc/intrin/directmap.c
@ -17,12 +17,14 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/calls/syscall-sysv.internal.h"
+#include "libc/dce.h"
 #include "libc/errno.h"
 #include "libc/intrin/describeflags.internal.h"
 #include "libc/intrin/directmap.internal.h"
 #include "libc/intrin/strace.internal.h"
 #include "libc/nt/runtime.h"
 #include "libc/runtime/memtrack.internal.h"
+#include "libc/runtime/syslib.internal.h"

 /**
 * Obtains memory mapping directly from system.
@ -37,7 +39,11 @@
 struct DirectMap sys_mmap(void *addr, size_t size, int prot, int flags, int fd,
                          int64_t off) {
  struct DirectMap d;
-  if (!IsWindows() && !IsMetal()) {
+  if (IsXnuSilicon()) {
+    long p = _sysret(__syslib->__mmap(addr, size, prot, flags, fd, off));
+    d.maphandle = kNtInvalidHandleValue;
+    d.addr = (void *)p;
+  } else if (!IsWindows() && !IsMetal()) {
    d.addr = __sys_mmap(addr, size, prot, flags, fd, off, off);
    d.maphandle = kNtInvalidHandleValue;
  } else if (IsMetal()) {
--- a/libc/intrin/mprotect-sysv.c
+++ b/libc/intrin/mprotect-sysv.c
@ -0,0 +1,29 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2023 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/syscall-sysv.internal.h"
+#include "libc/dce.h"
+#include "libc/runtime/syslib.internal.h"
+
+int sys_mprotect(void *data, size_t size, int prot) {
+  if (IsXnuSilicon()) {
+    return _sysret(__syslib->__mprotect(data, size, prot));
+  } else {
+    return __sys_mprotect(data, size, prot);
+  }
+}
--- a/libc/runtime/jit.c
+++ b/libc/runtime/jit.c
@ -0,0 +1,37 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2023 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/dce.h"
+#include "libc/runtime/runtime.h"
+#include "libc/runtime/syslib.internal.h"
+
+void __jit_begin(void) {
+  if (IsXnuSilicon()) {
+    if (__syslib->__pthread_jit_write_protect_supported_np()) {
+      __syslib->__pthread_jit_write_protect_np(false);
+    }
+  }
+}
+
+void __jit_end(void) {
+  if (IsXnuSilicon()) {
+    if (__syslib->__pthread_jit_write_protect_supported_np()) {
+      __syslib->__pthread_jit_write_protect_np(true);
+    }
+  }
+}
--- a/libc/runtime/runtime.h
+++ b/libc/runtime/runtime.h
@ -115,6 +115,8 @@ bool32 _isheap(void *);
 /* code morphing */
 void __morph_begin(void);
 void __morph_end(void);
+void __jit_begin(void);
+void __jit_end(void);
 void __clear_cache(void *, void *);
 /* portability */
 int NtGetVersion(void) pureconst;
--- a/libc/runtime/runtime.mk
+++ b/libc/runtime/runtime.mk
@ -45,6 +45,7 @@ LIBC_RUNTIME_A_DIRECTDEPS =				\
 	LIBC_STR					\
 	LIBC_SYSV					\
 	LIBC_SYSV_CALLS					\
+	THIRD_PARTY_COMPILER_RT				\
 	THIRD_PARTY_NSYNC				\
 	THIRD_PARTY_PUFF				\
 	THIRD_PARTY_XED
--- a/libc/runtime/syslib.internal.h
+++ b/libc/runtime/syslib.internal.h
@ -12,7 +12,7 @@ COSMOPOLITAN_C_START_
 */

 #define SYSLIB_MAGIC   ('s' | 'l' << 8 | 'i' << 16 | 'b' << 24)
-#define SYSLIB_VERSION 6
+#define SYSLIB_VERSION 8

 typedef uint64_t dispatch_time_t;
 typedef uint64_t dispatch_semaphore_t;
--- a/libc/sysv/calls/__sys_mprotect.S
+++ b/libc/sysv/calls/__sys_mprotect.S
@ -0,0 +1,2 @@
+#include "libc/sysv/macros.internal.h"
+.scall __sys_mprotect,0x04a04a04a204a00a,226,74,globl,hidden
--- a/libc/sysv/calls/sys_mprotect.S
+++ b/libc/sysv/calls/sys_mprotect.S
@ -1,2 +0,0 @@
-#include "libc/sysv/macros.internal.h"
-.scall sys_mprotect,0x04a04a04a204a00a,226,74,globl,hidden
--- a/libc/sysv/consts.sh
+++ b/libc/sysv/consts.sh
@ -234,6 +234,7 @@ syscon	mmap	MAP_INHERIT				-1			-1			-1			-1			-1			-1			0x00000080		-1			# make
 syscon	mmap	MAP_HASSEMAPHORE			0			0			0x00000200		0x00000200		0x00000200		0			0x00000200		0			# does it matter on x86?
 syscon	mmap	MAP_NOSYNC				0			0			0			0			0x00000800		0			0			0			# flush to physical media only when necessary rather than gratuitously; be sure to use write() rather than ftruncate() with this!
 syscon	mmap	MAP_CONCEAL				0			0			0			0			0x00020000		0x00008000		0x00008000		0			# omit from core dumps; MAP_NOCORE on FreeBSD
+syscon	mmap	MAP_JIT					0			0			0			0x00000800		0			0			0			0			# omit from core dumps; MAP_NOCORE on FreeBSD
 syscon	compat	MAP_NOCORE				0			0			0			0			0x00020000		0x00008000		0x00008000		0			# use MAP_CONCEAL
 syscon	compat	MAP_ANON				0x00000020		0x00000020		0x00001000		0x00001000		0x00001000		0x00001000		0x00001000		0x00000020		# bsd consensus; faked nt
 syscon	compat	MAP_EXECUTABLE				0x00001000		0x00001000		0			0			0			0			0			0			# ignored
--- a/libc/sysv/consts/MAP_JIT.S
+++ b/libc/sysv/consts/MAP_JIT.S
@ -0,0 +1,2 @@
+#include "libc/sysv/consts/syscon.internal.h"
+.syscon mmap,MAP_JIT,0,0,0,0x00000800,0,0,0,0
--- a/libc/sysv/consts/RLIMIT_AS.S
+++ b/libc/sysv/consts/RLIMIT_AS.S
@ -1,2 +1,2 @@
 #include "libc/sysv/consts/syscon.internal.h"
-.syscon rlimit,RLIMIT_AS,9,9,5,5,10,2,10,0
+.syscon rlimit,RLIMIT_AS,9	,9,5,5,10,2,10,0
--- a/libc/sysv/consts/map.h
+++ b/libc/sysv/consts/map.h
@ -14,6 +14,7 @@ extern const int MAP_FIXED;
 extern const int MAP_FIXED_NOREPLACE;
 extern const int MAP_HASSEMAPHORE;
 extern const int MAP_INHERIT;
+extern const int MAP_JIT;
 extern const int MAP_LOCKED;
 extern const int MAP_NONBLOCK;
 extern const int MAP_NORESERVE;
@ -43,5 +44,4 @@ COSMOPOLITAN_C_END_
 #define MAP_ANON   MAP_ANONYMOUS
 #define MAP_NOCORE MAP_CONCEAL

-
 #endif /* COSMOPOLITAN_LIBC_SYSV_CONSTS_MAP_H_ */
--- a/libc/sysv/syscalls.sh
+++ b/libc/sysv/syscalls.sh
@ -45,7 +45,7 @@ scall	sys_ppoll		0xfff86da21ffff90f	0x849	globl hidden # consider INTON/INTOFF t
 scall	sys_lseek		0x0c70a61de20c7008	0x03e	globl hidden # netbsd:evilpad, OpenBSD 7.3+
 scall	__sys_mmap		0x0c50311dd20c5009	0x0de	globl hidden # netbsd:pad, OpenBSD 7.3+
 scall	sys_msync		0x915900841284181a	0x8e3	globl hidden
-scall	sys_mprotect		0x04a04a04a204a00a	0x0e2	globl hidden
+scall	__sys_mprotect		0x04a04a04a204a00a	0x0e2	globl hidden
 scall	__sys_munmap		0x049049049204900b	0x0d7	globl hidden
 scall	sys_sigaction		0x15402e1a0202e00d	0x086	globl hidden # rt_sigaction on Lunix; __sigaction_sigtramp() on NetBSD
 scall	__sys_sigprocmask	0x125030154214900e	0x087	globl hidden # a.k.a. rt_sigprocmask, openbsd:byvalue, a.k.a. pthread_sigmask