Improve dlopen() on Apple Silicon

- Introduce MAP_JIT which is zero on other platforms
- Invent __jit_begin() and __jit_end() which wrap Apple's APIs
- Runtime dispatch to sys_icache_invalidate() in __clear_cache()
This commit is contained in:
Justine Tunney 2023-11-17 02:33:14 -08:00
parent 7a9e176ecf
commit 529cb4817c
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
20 changed files with 120 additions and 117 deletions

View file

@ -21,6 +21,7 @@
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/runtime/syslib.internal.h"
/**
* Unmaps memory directly with system.
@ -32,10 +33,12 @@
*/
int sys_munmap(void *p, size_t n) {
int rc;
if (!IsMetal()) {
rc = __sys_munmap(p, n);
} else {
if (IsXnuSilicon()) {
rc = _sysret(__syslib->__munmap(p, n));
} else if (IsMetal()) {
rc = sys_munmap_metal(p, n);
} else {
rc = __sys_munmap(p, n);
}
KERNTRACE("sys_munmap(%p /* %s */, %'zu) → %d", p,
DescribeFrame((intptr_t)p >> 16), n, rc);

View file

@ -20,6 +20,7 @@ i32 __sys_fcntl_cp(i32, i32, ...);
i32 __sys_fstat(i32, void *);
i32 __sys_fstatat(i32, const char *, void *, i32);
i32 __sys_gettid(i64 *);
i32 __sys_mprotect(void *, u64, i32);
i32 __sys_munmap(void *, u64);
i32 __sys_openat(i32, const char *, i32, u32);
i32 __sys_openat_nc(i32, const char *, i32, u32);

View file

@ -424,12 +424,12 @@ static char *dlerror_set(const char *str) {
return dlerror_buf;
}
static char *foreign_alloc_block(void) {
static dontinline char *foreign_alloc_block(void) {
char *p = 0;
size_t sz = 65536;
if (!IsWindows()) {
p = __sys_mmap(0, sz, PROT_READ | PROT_WRITE | PROT_EXEC,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0, 0);
MAP_PRIVATE | MAP_ANONYMOUS | MAP_JIT, -1, 0, 0);
if (p == MAP_FAILED) {
p = 0;
}
@ -448,7 +448,7 @@ static char *foreign_alloc_block(void) {
return p;
}
static void *foreign_alloc(size_t n) {
static dontinline void *foreign_alloc(size_t n) {
void *res;
static char *block;
static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
@ -520,11 +520,14 @@ static void *foreign_thunk_sysv(void *func) {
*p++ = 0xff;
*p++ = 0xe2;
#elif defined(__aarch64__)
if (!(p = code = foreign_alloc(36))) return 0; // 16 + 16 + 4 = 36
p = movimm(p, 5, (uintptr_t)func);
p = movimm(p, 10, (uintptr_t)foreign_tramp);
*(uint32_t *)p = 0xd61f0140; // br x10
__clear_cache(code, p + 4);
__jit_begin();
if ((p = code = foreign_alloc(36))) {
p = movimm(p, 5, (uintptr_t)func);
p = movimm(p, 10, (uintptr_t)foreign_tramp);
*(uint32_t *)p = 0xd61f0140; // br x10
__clear_cache(code, p + 4);
}
__jit_end();
#else
#error "unsupported architecture"
#endif

View file

@ -33,8 +33,7 @@ LIBC_DLOPEN_A_DIRECTDEPS = \
LIBC_RUNTIME \
LIBC_SYSV \
LIBC_SYSV_CALLS \
LIBC_STR \
THIRD_PARTY_COMPILER_RT
LIBC_STR
LIBC_DLOPEN_A_DEPS := \
$(call uniq,$(foreach x,$(LIBC_DLOPEN_A_DIRECTDEPS),$($(x))))

View file

@ -17,12 +17,14 @@
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/syslib.internal.h"
/**
* Obtains memory mapping directly from system.
@ -37,7 +39,11 @@
struct DirectMap sys_mmap(void *addr, size_t size, int prot, int flags, int fd,
int64_t off) {
struct DirectMap d;
if (!IsWindows() && !IsMetal()) {
if (IsXnuSilicon()) {
long p = _sysret(__syslib->__mmap(addr, size, prot, flags, fd, off));
d.maphandle = kNtInvalidHandleValue;
d.addr = (void *)p;
} else if (!IsWindows() && !IsMetal()) {
d.addr = __sys_mmap(addr, size, prot, flags, fd, off, off);
d.maphandle = kNtInvalidHandleValue;
} else if (IsMetal()) {

View file

@ -0,0 +1,29 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi
Copyright 2023 Justine Alexandra Roberts Tunney
Permission to use, copy, modify, and/or distribute this software for
any purpose with or without fee is hereby granted, provided that the
above copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/runtime/syslib.internal.h"
int sys_mprotect(void *data, size_t size, int prot) {
if (IsXnuSilicon()) {
return _sysret(__syslib->__mprotect(data, size, prot));
} else {
return __sys_mprotect(data, size, prot);
}
}

37
libc/runtime/jit.c Normal file
View file

@ -0,0 +1,37 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi
Copyright 2023 Justine Alexandra Roberts Tunney
Permission to use, copy, modify, and/or distribute this software for
any purpose with or without fee is hereby granted, provided that the
above copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/dce.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/syslib.internal.h"
void __jit_begin(void) {
if (IsXnuSilicon()) {
if (__syslib->__pthread_jit_write_protect_supported_np()) {
__syslib->__pthread_jit_write_protect_np(false);
}
}
}
void __jit_end(void) {
if (IsXnuSilicon()) {
if (__syslib->__pthread_jit_write_protect_supported_np()) {
__syslib->__pthread_jit_write_protect_np(true);
}
}
}

View file

@ -115,6 +115,8 @@ bool32 _isheap(void *);
/* code morphing */
void __morph_begin(void);
void __morph_end(void);
void __jit_begin(void);
void __jit_end(void);
void __clear_cache(void *, void *);
/* portability */
int NtGetVersion(void) pureconst;

View file

@ -45,6 +45,7 @@ LIBC_RUNTIME_A_DIRECTDEPS = \
LIBC_STR \
LIBC_SYSV \
LIBC_SYSV_CALLS \
THIRD_PARTY_COMPILER_RT \
THIRD_PARTY_NSYNC \
THIRD_PARTY_PUFF \
THIRD_PARTY_XED

View file

@ -12,7 +12,7 @@ COSMOPOLITAN_C_START_
*/
#define SYSLIB_MAGIC ('s' | 'l' << 8 | 'i' << 16 | 'b' << 24)
#define SYSLIB_VERSION 6
#define SYSLIB_VERSION 8
typedef uint64_t dispatch_time_t;
typedef uint64_t dispatch_semaphore_t;

View file

@ -0,0 +1,2 @@
#include "libc/sysv/macros.internal.h"
.scall __sys_mprotect,0x04a04a04a204a00a,226,74,globl,hidden

View file

@ -1,2 +0,0 @@
#include "libc/sysv/macros.internal.h"
.scall sys_mprotect,0x04a04a04a204a00a,226,74,globl,hidden

View file

@ -234,6 +234,7 @@ syscon mmap MAP_INHERIT -1 -1 -1 -1 -1 -1 0x00000080 -1 # make
syscon mmap MAP_HASSEMAPHORE 0 0 0x00000200 0x00000200 0x00000200 0 0x00000200 0 # does it matter on x86?
syscon mmap MAP_NOSYNC 0 0 0 0 0x00000800 0 0 0 # flush to physical media only when necessary rather than gratuitously; be sure to use write() rather than ftruncate() with this!
syscon mmap MAP_CONCEAL 0 0 0 0 0x00020000 0x00008000 0x00008000 0 # omit from core dumps; MAP_NOCORE on FreeBSD
syscon mmap MAP_JIT 0 0 0 0x00000800 0 0 0 0 # omit from core dumps; MAP_NOCORE on FreeBSD
syscon compat MAP_NOCORE 0 0 0 0 0x00020000 0x00008000 0x00008000 0 # use MAP_CONCEAL
syscon compat MAP_ANON 0x00000020 0x00000020 0x00001000 0x00001000 0x00001000 0x00001000 0x00001000 0x00000020 # bsd consensus; faked nt
syscon compat MAP_EXECUTABLE 0x00001000 0x00001000 0 0 0 0 0 0 # ignored

View file

@ -0,0 +1,2 @@
#include "libc/sysv/consts/syscon.internal.h"
.syscon mmap,MAP_JIT,0,0,0,0x00000800,0,0,0,0

View file

@ -1,2 +1,2 @@
#include "libc/sysv/consts/syscon.internal.h"
.syscon rlimit,RLIMIT_AS,9,9,5,5,10,2,10,0
.syscon rlimit,RLIMIT_AS,9 ,9,5,5,10,2,10,0

View file

@ -14,6 +14,7 @@ extern const int MAP_FIXED;
extern const int MAP_FIXED_NOREPLACE;
extern const int MAP_HASSEMAPHORE;
extern const int MAP_INHERIT;
extern const int MAP_JIT;
extern const int MAP_LOCKED;
extern const int MAP_NONBLOCK;
extern const int MAP_NORESERVE;
@ -43,5 +44,4 @@ COSMOPOLITAN_C_END_
#define MAP_ANON MAP_ANONYMOUS
#define MAP_NOCORE MAP_CONCEAL
#endif /* COSMOPOLITAN_LIBC_SYSV_CONSTS_MAP_H_ */

View file

@ -45,7 +45,7 @@ scall sys_ppoll 0xfff86da21ffff90f 0x849 globl hidden # consider INTON/INTOFF t
scall sys_lseek 0x0c70a61de20c7008 0x03e globl hidden # netbsd:evilpad, OpenBSD 7.3+
scall __sys_mmap 0x0c50311dd20c5009 0x0de globl hidden # netbsd:pad, OpenBSD 7.3+
scall sys_msync 0x915900841284181a 0x8e3 globl hidden
scall sys_mprotect 0x04a04a04a204a00a 0x0e2 globl hidden
scall __sys_mprotect 0x04a04a04a204a00a 0x0e2 globl hidden
scall __sys_munmap 0x049049049204900b 0x0d7 globl hidden
scall sys_sigaction 0x15402e1a0202e00d 0x086 globl hidden # rt_sigaction on Lunix; __sigaction_sigtramp() on NetBSD
scall __sys_sigprocmask 0x125030154214900e 0x087 globl hidden # a.k.a. rt_sigprocmask, openbsd:byvalue, a.k.a. pthread_sigmask