mirror of
https://github.com/jart/cosmopolitan.git
synced 2025-07-12 05:59:10 +00:00
Make _Thread_local work across platforms
We now rewrite the binary image at runtime on Windows and XNU to change mov %fs:0,%reg instructions to use %gs instead. There's also simpler threading API introduced by this change and it's called _spawn() and _join(), which has replaced most clone() usage.
This commit is contained in:
Justine Tunney
parent
e4d6e263d4
commit
5f4f6b0e69
51 changed files with 808 additions and 1043 deletions
|
|
@ -26,7 +26,7 @@
|
|||
// @param rdx is ptid
|
||||
// @param rcx is ctid
|
||||
// @param r8 is tls
|
||||
// @param r9 is func
|
||||
// @param r9 is func(void*,int)→int
|
||||
// @param 8(rsp) is arg
|
||||
// @return tid of child on success, or -1 w/ errno
|
||||
sys_clone_linux:
|
||||
|
|
@ -48,8 +48,9 @@ sys_clone_linux:
|
|||
jmp 0b
|
||||
2: xor %ebp,%ebp # child thread
|
||||
mov %rbx,%rdi # arg
|
||||
call *%r9 # func(arg)
|
||||
xchg %eax,%edi # func(arg) → exitcode
|
||||
mov (%r10),%esi # tid
|
||||
call *%r9 # func(arg,tid)
|
||||
xchg %eax,%edi # func(arg,tid) → exitcode
|
||||
mov $60,%eax # __NR_exit(exitcode)
|
||||
syscall
|
||||
.endfn sys_clone_linux,globl,hidden
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@
|
|||
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
|
||||
│ PERFORMANCE OF THIS SOFTWARE. │
|
||||
╚─────────────────────────────────────────────────────────────────────────────*/
|
||||
#include "libc/assert.h"
|
||||
#include "libc/calls/calls.h"
|
||||
#include "libc/calls/strace.internal.h"
|
||||
#include "libc/calls/struct/ucontext-netbsd.internal.h"
|
||||
|
|
@ -24,7 +23,6 @@
|
|||
#include "libc/dce.h"
|
||||
#include "libc/errno.h"
|
||||
#include "libc/intrin/asan.internal.h"
|
||||
#include "libc/intrin/kprintf.h"
|
||||
#include "libc/intrin/spinlock.h"
|
||||
#include "libc/limits.h"
|
||||
#include "libc/macros.internal.h"
|
||||
|
|
@ -68,19 +66,23 @@ struct CloneArgs {
|
|||
};
|
||||
union {
|
||||
char lock;
|
||||
void *pstack;
|
||||
void *oldrsp;
|
||||
};
|
||||
int *ptid;
|
||||
int *ctid;
|
||||
int *ztid;
|
||||
char *tls;
|
||||
int (*func)(void *);
|
||||
int (*func)(void *, int);
|
||||
void *arg;
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// THE NEW TECHNOLOGY
|
||||
|
||||
int WinThreadLaunch(void *arg, int (*func)(void *), intptr_t rsp);
|
||||
int WinThreadLaunch(void *arg, // rdi
|
||||
int tid, // rsi
|
||||
int (*func)(void *, int), // rdx
|
||||
intptr_t rsp); // rcx
|
||||
|
||||
// we can't log this function because:
|
||||
// 1. windows owns the backtrace pointer right now
|
||||
|
|
@ -90,16 +92,20 @@ int WinThreadLaunch(void *arg, int (*func)(void *), intptr_t rsp);
|
|||
// 2. windows owns the stack memory right now
|
||||
// we need win32 raw imports because:
|
||||
// 1. generated thunks are function logged
|
||||
noasan noinstrument static textwindows wontreturn void WinThreadEntry(
|
||||
int rdi, int rsi, int rdx, struct CloneArgs *wt) {
|
||||
noasan noinstrument static textwindows wontreturn void //
|
||||
WinThreadEntry(int rdi, // rcx
|
||||
int rsi, // rdx
|
||||
int rdx, // r8
|
||||
struct CloneArgs *wt) { // r9
|
||||
int rc;
|
||||
if (wt->tls) {
|
||||
asm("mov\t%1,%%gs:%0"
|
||||
: "=m"(*((long *)0x1480 + __tls_index))
|
||||
: "r"(wt->tls));
|
||||
}
|
||||
*wt->ptid = wt->tid;
|
||||
*wt->ctid = wt->tid;
|
||||
rc = WinThreadLaunch(wt->arg, wt->func, (intptr_t)wt & -16);
|
||||
rc = WinThreadLaunch(wt->arg, wt->tid, wt->func, (intptr_t)wt & -16);
|
||||
// we can now clear ctid directly since we're no longer using our own
|
||||
// stack memory, which can now be safely free'd by the parent thread.
|
||||
*wt->ztid = 0;
|
||||
|
|
@ -109,14 +115,16 @@ noasan noinstrument static textwindows wontreturn void WinThreadEntry(
|
|||
unreachable;
|
||||
}
|
||||
|
||||
static textwindows int CloneWindows(int (*func)(void *), char *stk,
|
||||
static textwindows int CloneWindows(int (*func)(void *, int), char *stk,
|
||||
size_t stksz, int flags, void *arg,
|
||||
void *tls, size_t tlssz, int *ctid) {
|
||||
void *tls, size_t tlssz, int *ptid,
|
||||
int *ctid) {
|
||||
int64_t h;
|
||||
struct CloneArgs *wt;
|
||||
wt = (struct CloneArgs *)(((intptr_t)(stk + stksz) -
|
||||
sizeof(struct CloneArgs)) &
|
||||
-alignof(struct CloneArgs));
|
||||
wt->ptid = flags & CLONE_PARENT_SETTID ? ptid : &wt->tid;
|
||||
wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
|
||||
wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid;
|
||||
wt->func = func;
|
||||
|
|
@ -133,8 +141,12 @@ static textwindows int CloneWindows(int (*func)(void *), char *stk,
|
|||
////////////////////////////////////////////////////////////////////////////////
|
||||
// XNU'S NOT UNIX
|
||||
|
||||
void XnuThreadThunk(void *pthread, int machport, void *(*func)(void *),
|
||||
void *arg, intptr_t *stack, unsigned xnuflags);
|
||||
void XnuThreadThunk(void *pthread, // rdi
|
||||
int machport, // rsi
|
||||
void *(*func)(void *), // rdx
|
||||
void *arg, // rcx
|
||||
intptr_t *stack, // r8
|
||||
unsigned xnuflags); // r9
|
||||
asm("XnuThreadThunk:\n\t"
|
||||
"xor\t%ebp,%ebp\n\t"
|
||||
"mov\t%r8,%rsp\n\t"
|
||||
|
|
@ -145,11 +157,18 @@ asm("XnuThreadThunk:\n\t"
|
|||
__attribute__((__used__, __no_reorder__))
|
||||
|
||||
static wontreturn void
|
||||
XnuThreadMain(void *pthread, int tid, int (*func)(void *arg), void *arg,
|
||||
struct CloneArgs *wt, unsigned xnuflags) {
|
||||
XnuThreadMain(void *pthread, // rdi
|
||||
int tid, // rsi
|
||||
int (*func)(void *arg, int tid), // rdx
|
||||
void *arg, // rcx
|
||||
struct CloneArgs *wt, // r8
|
||||
unsigned xnuflags) { // r9
|
||||
int ax;
|
||||
wt->tid = tid;
|
||||
*wt->ptid = tid;
|
||||
*wt->ctid = tid;
|
||||
_spunlock(&wt->lock);
|
||||
|
||||
if (wt->tls) {
|
||||
// XNU uses the same 0x30 offset as the WIN32 TIB x64. They told the
|
||||
// Go team at Google that they Apply stands by our ability to use it
|
||||
|
|
@ -159,10 +178,9 @@ XnuThreadMain(void *pthread, int tid, int (*func)(void *arg), void *arg,
|
|||
: "0"(__NR_thread_fast_set_cthread_self), "D"(wt->tls - 0x30)
|
||||
: "rcx", "r11", "memory", "cc");
|
||||
}
|
||||
if (wt->ctid) {
|
||||
*wt->ctid = tid;
|
||||
}
|
||||
func(arg);
|
||||
|
||||
func(arg, tid);
|
||||
|
||||
// we no longer use the stack after this point
|
||||
// %rax = int bsdthread_terminate(%rdi = void *stackaddr,
|
||||
// %rsi = size_t freesize,
|
||||
|
|
@ -179,7 +197,7 @@ XnuThreadMain(void *pthread, int tid, int (*func)(void *arg), void *arg,
|
|||
}
|
||||
|
||||
static int CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags,
|
||||
void *arg, void *tls, size_t tlssz, int *ctid) {
|
||||
void *arg, void *tls, size_t tlssz, int *ptid, int *ctid) {
|
||||
int rc;
|
||||
bool failed;
|
||||
static bool once;
|
||||
|
|
@ -198,6 +216,7 @@ static int CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags,
|
|||
wt = (struct CloneArgs *)(((intptr_t)(stk + stksz) -
|
||||
sizeof(struct CloneArgs)) &
|
||||
-alignof(struct CloneArgs));
|
||||
wt->ptid = flags & CLONE_PARENT_SETTID ? ptid : &wt->tid;
|
||||
wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
|
||||
wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid;
|
||||
wt->tls = flags & CLONE_SETTLS ? tls : 0;
|
||||
|
|
@ -215,8 +234,9 @@ static int CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags,
|
|||
|
||||
static wontreturn void FreebsdThreadMain(void *p) {
|
||||
struct CloneArgs *wt = p;
|
||||
*wt->ptid = wt->tid;
|
||||
*wt->ctid = wt->tid;
|
||||
wt->func(wt->arg);
|
||||
wt->func(wt->arg, wt->tid);
|
||||
// we no longer use the stack after this point
|
||||
// void thr_exit(%rdi = long *state);
|
||||
asm volatile("movl\t$0,%0\n\t" // *wt->ztid = 0
|
||||
|
|
@ -227,8 +247,9 @@ static wontreturn void FreebsdThreadMain(void *p) {
|
|||
unreachable;
|
||||
}
|
||||
|
||||
static int CloneFreebsd(int (*func)(void *), char *stk, size_t stksz, int flags,
|
||||
void *arg, void *tls, size_t tlssz, int *ctid) {
|
||||
static int CloneFreebsd(int (*func)(void *, int), char *stk, size_t stksz,
|
||||
int flags, void *arg, void *tls, size_t tlssz,
|
||||
int *ptid, int *ctid) {
|
||||
int ax;
|
||||
bool failed;
|
||||
int64_t tid;
|
||||
|
|
@ -236,6 +257,7 @@ static int CloneFreebsd(int (*func)(void *), char *stk, size_t stksz, int flags,
|
|||
wt = (struct CloneArgs *)(((intptr_t)(stk + stksz) -
|
||||
sizeof(struct CloneArgs)) &
|
||||
-alignof(struct CloneArgs));
|
||||
wt->ptid = flags & CLONE_PARENT_SETTID ? ptid : &wt->tid;
|
||||
wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
|
||||
wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid;
|
||||
wt->tls = tls;
|
||||
|
|
@ -267,7 +289,9 @@ static int CloneFreebsd(int (*func)(void *), char *stk, size_t stksz, int flags,
|
|||
|
||||
static wontreturn void OpenbsdThreadMain(void *p) {
|
||||
struct CloneArgs *wt = p;
|
||||
wt->func(wt->arg);
|
||||
*wt->ptid = wt->tid;
|
||||
*wt->ctid = wt->tid;
|
||||
wt->func(wt->arg, wt->tid);
|
||||
// we no longer use the stack after this point. however openbsd
|
||||
// validates the rsp register too so a race condition can still
|
||||
// happen if the parent tries to free the stack. we'll solve it
|
||||
|
|
@ -279,13 +303,14 @@ static wontreturn void OpenbsdThreadMain(void *p) {
|
|||
"movl\t$0,(%%rdi)\n\t" // *wt->ztid = 0
|
||||
"syscall" // __threxit()
|
||||
: "=m"(*wt->ztid)
|
||||
: "a"(302), "m"(wt->pstack), "D"(wt->ztid)
|
||||
: "a"(302), "m"(wt->oldrsp), "D"(wt->ztid)
|
||||
: "rcx", "r11", "memory");
|
||||
unreachable;
|
||||
}
|
||||
|
||||
static int CloneOpenbsd(int (*func)(void *), char *stk, size_t stksz, int flags,
|
||||
void *arg, void *tls, size_t tlssz, int *ctid) {
|
||||
static int CloneOpenbsd(int (*func)(void *, int), char *stk, size_t stksz,
|
||||
int flags, void *arg, void *tls, size_t tlssz,
|
||||
int *ptid, int *ctid) {
|
||||
int tid;
|
||||
intptr_t sp;
|
||||
struct __tfork *tf;
|
||||
|
|
@ -297,13 +322,15 @@ static int CloneOpenbsd(int (*func)(void *), char *stk, size_t stksz, int flags,
|
|||
sp -= sizeof(struct CloneArgs);
|
||||
sp &= -MAX(16, alignof(struct CloneArgs));
|
||||
wt = (struct CloneArgs *)sp;
|
||||
wt->ptid = flags & CLONE_PARENT_SETTID ? ptid : &wt->tid;
|
||||
wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
|
||||
wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid;
|
||||
wt->pstack = __builtin_frame_address(0);
|
||||
wt->oldrsp = __builtin_frame_address(0);
|
||||
wt->arg = arg;
|
||||
wt->func = func;
|
||||
tf->tf_stack = (char *)wt - 8;
|
||||
tf->tf_tcb = flags & CLONE_SETTLS ? tls : 0;
|
||||
tf->tf_tid = flags & CLONE_CHILD_SETTID ? ctid : 0;
|
||||
tf->tf_tid = &wt->tid;
|
||||
if ((tid = __tfork_thread(tf, sizeof(*tf), OpenbsdThreadMain, wt)) < 0) {
|
||||
errno = -tid;
|
||||
tid = -1;
|
||||
|
|
@ -314,11 +341,17 @@ static int CloneOpenbsd(int (*func)(void *), char *stk, size_t stksz, int flags,
|
|||
////////////////////////////////////////////////////////////////////////////////
|
||||
// NET BESIYATA DISHMAYA
|
||||
|
||||
static wontreturn void NetbsdThreadMain(void *arg, int (*func)(void *arg),
|
||||
int *tid, int *ctid, int *ztid) {
|
||||
static wontreturn void NetbsdThreadMain(void *arg, // rdi
|
||||
int (*func)(void *, int), // rsi
|
||||
int *tid, // rdx
|
||||
int *ctid, // rcx
|
||||
int *ztid, // r8
|
||||
int *ptid) { // r9
|
||||
int ax, dx;
|
||||
*ctid = *tid;
|
||||
func(arg);
|
||||
ax = *tid;
|
||||
*ptid = ax;
|
||||
*ctid = ax;
|
||||
func(arg, ax);
|
||||
// we no longer use the stack after this point
|
||||
// %eax = int __lwp_exit(void);
|
||||
asm volatile("movl\t$0,%2\n\t" // *wt->ztid = 0
|
||||
|
|
@ -330,8 +363,9 @@ static wontreturn void NetbsdThreadMain(void *arg, int (*func)(void *arg),
|
|||
unreachable;
|
||||
}
|
||||
|
||||
static int CloneNetbsd(int (*func)(void *), char *stk, size_t stksz, int flags,
|
||||
void *arg, void *tls, size_t tlssz, int *ctid) {
|
||||
static int CloneNetbsd(int (*func)(void *, int), char *stk, size_t stksz,
|
||||
int flags, void *arg, void *tls, size_t tlssz, int *ptid,
|
||||
int *ctid) {
|
||||
// NetBSD has its own clone() and it works, but it's technically a
|
||||
// second-class API, intended to help Linux folks migrate to this.
|
||||
bool failed;
|
||||
|
|
@ -341,7 +375,6 @@ static int CloneNetbsd(int (*func)(void *), char *stk, size_t stksz, int flags,
|
|||
static int broken;
|
||||
struct ucontext_netbsd *ctx;
|
||||
static struct ucontext_netbsd netbsd_clone_template;
|
||||
_Static_assert(sizeof(struct ucontext_netbsd) == 784, "fix assembly");
|
||||
|
||||
// memoize arbitrary valid processor state structure
|
||||
if (!once) {
|
||||
|
|
@ -360,7 +393,7 @@ static int CloneNetbsd(int (*func)(void *), char *stk, size_t stksz, int flags,
|
|||
}
|
||||
sp = (intptr_t)(stk + stksz);
|
||||
|
||||
// allocate memory for child tid
|
||||
// allocate memory for tid
|
||||
sp -= sizeof(int);
|
||||
sp = sp & -alignof(int);
|
||||
tid = (int *)sp;
|
||||
|
|
@ -388,6 +421,7 @@ static int CloneNetbsd(int (*func)(void *), char *stk, size_t stksz, int flags,
|
|||
ctx->uc_mcontext.rdx = (intptr_t)tid;
|
||||
ctx->uc_mcontext.rcx = (intptr_t)(flags & CLONE_CHILD_SETTID ? ctid : tid);
|
||||
ctx->uc_mcontext.r8 = (intptr_t)(flags & CLONE_CHILD_CLEARTID ? ctid : tid);
|
||||
ctx->uc_mcontext.r9 = (intptr_t)(flags & CLONE_PARENT_SETTID ? ptid : tid);
|
||||
ctx->uc_flags |= _UC_STACK;
|
||||
ctx->uc_stack.ss_sp = stk;
|
||||
ctx->uc_stack.ss_size = stksz;
|
||||
|
|
@ -413,8 +447,28 @@ static int CloneNetbsd(int (*func)(void *), char *stk, size_t stksz, int flags,
|
|||
////////////////////////////////////////////////////////////////////////////////
|
||||
// GNU/SYSTEMD
|
||||
|
||||
int sys_clone_linux(int flags, char *stk, int *ptid, int *ctid, void *tls,
|
||||
int (*func)(void *), void *arg);
|
||||
int sys_clone_linux(int flags, // rdi
|
||||
long sp, // rsi
|
||||
int *ptid, // rdx
|
||||
int *ctid, // rcx
|
||||
void *tls, // r8
|
||||
void *func, // r9
|
||||
void *arg); // 8(rsp)
|
||||
|
||||
static int CloneLinux(int (*func)(void *arg, int tid), char *stk, size_t stksz,
|
||||
int flags, void *arg, void *tls, size_t tlssz, int *ptid,
|
||||
int *ctid) {
|
||||
long sp;
|
||||
sp = (intptr_t)(stk + stksz);
|
||||
if (~flags & CLONE_CHILD_SETTID) {
|
||||
flags |= CLONE_CHILD_SETTID;
|
||||
sp -= sizeof(int);
|
||||
sp = sp & -alignof(int);
|
||||
ctid = (int *)sp;
|
||||
}
|
||||
sp = sp & -16; // align the stack
|
||||
return sys_clone_linux(flags, sp, ptid, ctid, tls, func, arg);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// COSMOPOLITAN
|
||||
|
|
@ -461,36 +515,69 @@ int sys_clone_linux(int flags, char *stk, int *ptid, int *ctid, void *tls,
|
|||
* other calls like getpid() may return incorrect values.
|
||||
*
|
||||
* @param func is your callback function, which this wrapper requires
|
||||
* not be null, otherwise EINVAL is raised
|
||||
* not be null, otherwise EINVAL is raised. It is passed two args
|
||||
* within the child thread: (1) the caller-supplied `arg` and (2)
|
||||
* the new tid is always passed in the second arg for convenience
|
||||
*
|
||||
* @param stk points to the bottom of a caller allocated stack, which
|
||||
* must be allocated via mmap() using the MAP_STACK flag, or else
|
||||
* you won't get optimal performance and it won't work on OpenBSD
|
||||
*
|
||||
* @param stksz is the size of that stack in bytes, we recommend that
|
||||
* that this be set to GetStackSize() or else memory safety tools
|
||||
* like kprintf() can't do as good and quick of a job; this value
|
||||
* must be 16-aligned plus it must be at least 4192 bytes in size
|
||||
* and it's advised to have the bottom-most page, be a guard page
|
||||
* @param flags should have:
|
||||
* - `CLONE_THREAD|CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND`
|
||||
* and you may optionally bitwise or any of the following:
|
||||
* - `CLONE_CHILD_SETTID` is needed too if you use `ctid` which
|
||||
* is part of the memory the child owns and it'll be set right
|
||||
* before the callback function is invoked
|
||||
* - `CLONE_CHILD_CLEARTID` causes `*ctid = 0` upon termination
|
||||
* which can be used to implement join so that the parent may
|
||||
* safely free the stack memory that the child is using
|
||||
* - `CLONE_PARENT_SETTID` is needed too if you use `ptid` and this
|
||||
* is guaranteed to happen before clone() returns
|
||||
* - `CLONE_SETTLS` is needed too if you set `tls`. You may get this
|
||||
* value from the thread by calling __get_tls(). There are a few
|
||||
* layout expectations imposed by your C library. Those are all
|
||||
* documented by __initialize_tls() which initializes the parts of
|
||||
* the first 64 bytes of tls memory that libc cares about. This
|
||||
* flag will transition the C runtime to the `__tls_enabled` state
|
||||
* automatically. If it's used for one thread, then it must be
|
||||
* used for all threads. The first time it's used, it must be used
|
||||
* from the main thread.
|
||||
* @param arg will be passed to your callback
|
||||
*
|
||||
* @param flags which SHOULD always have all of these flags:
|
||||
*
|
||||
* - `CLONE_THREAD`
|
||||
* - `CLONE_VM`
|
||||
* - `CLONE_FS`
|
||||
* - `CLONE_FILES`
|
||||
* - `CLONE_SIGHAND`
|
||||
*
|
||||
* This system call wrapper is intended for threads, and as such, we
|
||||
* won't polyfill Linux's ability to simulate unrelated calls (e.g.
|
||||
* fork, vfork) via clone() on other platforms. Please just call
|
||||
* fork() and vfork() when that's what you want.
|
||||
*
|
||||
* Your `flags` may also optionally also additionally bitwise-OR any
|
||||
* combination of the following additional flags:
|
||||
*
|
||||
* - `CLONE_PARENT_SETTID` must be specified if you intend to set
|
||||
* the `ptid` argument, which is guaranteed to be updated with the
|
||||
* child tid BEFORE BOTH clone() returns and `func` is invoked
|
||||
*
|
||||
* - `CLONE_CHILD_SETTID` must be specified if you intend to set the
|
||||
* `ctid` argument, which is guaranteed to be updated with the
|
||||
* child tid before `func` is called, however we CAN NOT guarantee
|
||||
* this will happen BEFORE clone() returns
|
||||
*
|
||||
* - `CLONE_CHILD_CLEARTID` causes `*ctid = 0` upon child thread
|
||||
* termination. This is used to implement join so that the parent
|
||||
* may know when it's safe to free the child's stack memory, and
|
||||
* as such, is guaranteed to happen AFTER the child thread has
|
||||
* either terminated or has finished using its stack memory
|
||||
*
|
||||
* - `CLONE_SETTLS` is needed if you intend to specify the `tls`
|
||||
* argument, which provides a fast-path solution for changing the
|
||||
* appropriate TLS segment register within the child thread. The
|
||||
* child thread may then obtain a reference to the TIB address you
|
||||
* supplied, by calling __get_tls(). Your C library holds certain
|
||||
* expectations about the layout of your Thread Information Block
|
||||
* (TIB), which are all documented by __initialize_tls(). That
|
||||
* function can be used to initialize the first positive 64 bytes
|
||||
* of your TLS allocation, which is the memory Cosmopolitan Libc
|
||||
* wants for itself (and negative addresses are reserved by the
|
||||
* GNU Linker). Using this flag will transition the C runtime to a
|
||||
* `__tls_enabled` state automatically. If you use TLS for just
|
||||
* one thread, then you must be specify TLS for ALL THREADS. It's
|
||||
* a good idea to do that since TLS can offer considerable (i.e.
|
||||
* multiple orders of a magnitude) performance improvement for
|
||||
* TID-dependent C library services, e.g. recursive mutexes.
|
||||
*
|
||||
* @param arg is passed as an argument to `func` in the child thread
|
||||
* @param tls may be used to set the thread local storage segment;
|
||||
* this parameter is ignored if `CLONE_SETTLS` is not set
|
||||
* @param tlssz is the size of tls in bytes which must be at least 64
|
||||
|
|
@ -499,8 +586,8 @@ int sys_clone_linux(int flags, char *stk, int *ptid, int *ctid, void *tls,
|
|||
* @return tid of child on success, or -1 w/ errno
|
||||
* @threadsafe
|
||||
*/
|
||||
int clone(int (*func)(void *), void *stk, size_t stksz, int flags, void *arg,
|
||||
int *ptid, void *tls, size_t tlssz, int *ctid) {
|
||||
int clone(void *func, void *stk, size_t stksz, int flags, void *arg, int *ptid,
|
||||
void *tls, size_t tlssz, int *ctid) {
|
||||
int rc;
|
||||
struct CloneArgs *wt;
|
||||
|
||||
|
|
@ -529,8 +616,7 @@ int clone(int (*func)(void *), void *stk, size_t stksz, int flags, void *arg,
|
|||
!__asan_is_valid(ctid, sizeof(*ctid))))) {
|
||||
rc = efault();
|
||||
} else if (IsLinux()) {
|
||||
rc =
|
||||
sys_clone_linux(flags, (char *)stk + stksz, ptid, ctid, tls, func, arg);
|
||||
rc = CloneLinux(func, stk, stksz, flags, arg, tls, tlssz, ptid, ctid);
|
||||
} else if (!IsTiny() &&
|
||||
(flags & ~(CLONE_SETTLS | CLONE_PARENT_SETTID |
|
||||
CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID)) !=
|
||||
|
|
@ -539,19 +625,20 @@ int clone(int (*func)(void *), void *stk, size_t stksz, int flags, void *arg,
|
|||
STRACE("clone flag unsupported on this platform");
|
||||
rc = einval();
|
||||
} else if (IsXnu()) {
|
||||
rc = CloneXnu(func, stk, stksz, flags, arg, tls, tlssz, ctid);
|
||||
rc = CloneXnu(func, stk, stksz, flags, arg, tls, tlssz, ptid, ctid);
|
||||
} else if (IsFreebsd()) {
|
||||
rc = CloneFreebsd(func, stk, stksz, flags, arg, tls, tlssz, ctid);
|
||||
rc = CloneFreebsd(func, stk, stksz, flags, arg, tls, tlssz, ptid, ctid);
|
||||
} else if (IsNetbsd()) {
|
||||
rc = CloneNetbsd(func, stk, stksz, flags, arg, tls, tlssz, ctid);
|
||||
rc = CloneNetbsd(func, stk, stksz, flags, arg, tls, tlssz, ptid, ctid);
|
||||
} else if (IsOpenbsd()) {
|
||||
rc = CloneOpenbsd(func, stk, stksz, flags, arg, tls, tlssz, ctid);
|
||||
rc = CloneOpenbsd(func, stk, stksz, flags, arg, tls, tlssz, ptid, ctid);
|
||||
} else if (IsWindows()) {
|
||||
rc = CloneWindows(func, stk, stksz, flags, arg, tls, tlssz, ctid);
|
||||
rc = CloneWindows(func, stk, stksz, flags, arg, tls, tlssz, ptid, ctid);
|
||||
} else {
|
||||
rc = enosys();
|
||||
}
|
||||
|
||||
// TODO(jart): do we need it?
|
||||
if (rc != -1 && (flags & CLONE_PARENT_SETTID)) {
|
||||
*ptid = rc;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ static struct SymbolTable *GetSymbolTableFromZip(struct Zipos *zipos) {
|
|||
lf = GetZipCfileOffset(zipos->map + cf);
|
||||
size = GetZipLfileUncompressedSize(zipos->map + lf);
|
||||
size2 = ROUNDUP(size, FRAMESIZE);
|
||||
if ((res = mapanon(size2))) {
|
||||
if ((res = _mapanon(size2))) {
|
||||
switch (ZIP_LFILE_COMPRESSIONMETHOD(zipos->map + lf)) {
|
||||
case kZipCompressionNone:
|
||||
memcpy(res, (void *)ZIP_LFILE_CONTENT(zipos->map + lf), size);
|
||||
|
|
|
|||
|
|
@ -54,13 +54,16 @@
|
|||
* }
|
||||
*
|
||||
* That is performed automatically for unit test executables.
|
||||
*
|
||||
* @return memory map address on success, or null w/ errrno
|
||||
*/
|
||||
noasan void *mapanon(size_t size) {
|
||||
void *_mapanon(size_t size) {
|
||||
/* asan runtime depends on this function */
|
||||
void *m;
|
||||
m = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||
if (m == MAP_FAILED && weaken(__oom_hook)) {
|
||||
weaken(__oom_hook)(size);
|
||||
return 0;
|
||||
}
|
||||
return m;
|
||||
}
|
||||
|
|
|
|||
41
libc/runtime/mapstack.c
Normal file
41
libc/runtime/mapstack.c
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
|
||||
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
|
||||
╞══════════════════════════════════════════════════════════════════════════════╡
|
||||
│ Copyright 2022 Justine Alexandra Roberts Tunney │
|
||||
│ │
|
||||
│ Permission to use, copy, modify, and/or distribute this software for │
|
||||
│ any purpose with or without fee is hereby granted, provided that the │
|
||||
│ above copyright notice and this permission notice appear in all copies. │
|
||||
│ │
|
||||
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
|
||||
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
|
||||
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
|
||||
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
|
||||
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
|
||||
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
|
||||
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
|
||||
│ PERFORMANCE OF THIS SOFTWARE. │
|
||||
╚─────────────────────────────────────────────────────────────────────────────*/
|
||||
#include "libc/runtime/runtime.h"
|
||||
#include "libc/runtime/stack.h"
|
||||
#include "libc/sysv/consts/map.h"
|
||||
#include "libc/sysv/consts/prot.h"
|
||||
|
||||
/**
|
||||
* Allocates stack.
|
||||
*
|
||||
* @return stack bottom address on success, or null w/ errrno
|
||||
*/
|
||||
void *_mapstack(void) {
|
||||
return mmap(0, GetStackSize(), PROT_READ | PROT_WRITE,
|
||||
MAP_STACK | MAP_ANONYMOUS, -1, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Frees stack.
|
||||
*
|
||||
* @param stk was allocated by _mapstack()
|
||||
*/
|
||||
int _freestack(void *stk) {
|
||||
return munmap(stk, GetStackSize());
|
||||
}
|
||||
|
|
@ -6,7 +6,7 @@ COSMOPOLITAN_C_START_
|
|||
│ cosmopolitan § runtime ─╬─│┼
|
||||
╚────────────────────────────────────────────────────────────────────────────│*/
|
||||
|
||||
typedef long jmp_buf[8] forcealign(CACHELINE);
|
||||
typedef long jmp_buf[8];
|
||||
|
||||
extern char **environ; /* CRT */
|
||||
extern int __argc; /* CRT */
|
||||
|
|
@ -45,8 +45,10 @@ extern size_t __virtualmax;
|
|||
extern bool __isworker;
|
||||
|
||||
void mcount(void);
|
||||
int _freestack(void *);
|
||||
unsigned long getauxval(unsigned long);
|
||||
void *mapanon(size_t) attributeallocsize((1));
|
||||
void *_mapanon(size_t) attributeallocsize((1)) mallocesque;
|
||||
void *_mapstack(void) returnsaligned((FRAMESIZE)) mallocesque;
|
||||
int setjmp(jmp_buf) libcesque returnstwice paramsnonnull();
|
||||
void longjmp(jmp_buf, int) libcesque wontreturn paramsnonnull();
|
||||
axdx_t setlongerjmp(jmp_buf) libcesque returnstwice paramsnonnull();
|
||||
|
|
|
|||
|
|
@ -16,29 +16,206 @@
|
|||
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
|
||||
│ PERFORMANCE OF THIS SOFTWARE. │
|
||||
╚─────────────────────────────────────────────────────────────────────────────*/
|
||||
#include "libc/assert.h"
|
||||
#include "libc/bits/bits.h"
|
||||
#include "libc/calls/calls.h"
|
||||
#include "libc/calls/syscall-sysv.internal.h"
|
||||
#include "libc/dce.h"
|
||||
#include "libc/errno.h"
|
||||
#include "libc/intrin/kprintf.h"
|
||||
#include "libc/macros.internal.h"
|
||||
#include "libc/nexgen32e/threaded.h"
|
||||
#include "libc/nt/thread.h"
|
||||
#include "libc/nt/thunk/msabi.h"
|
||||
#include "libc/runtime/internal.h"
|
||||
#include "libc/runtime/runtime.h"
|
||||
#include "libc/str/str.h"
|
||||
#include "libc/sysv/consts/nrlinux.h"
|
||||
#include "libc/thread/thread.h"
|
||||
#include "third_party/xed/x86.h"
|
||||
|
||||
#define __NR_sysarch 0x000000a5 // freebsd+netbsd
|
||||
#define AMD64_SET_GSBASE 131 // freebsd
|
||||
#define AMD64_SET_FSBASE 129 // freebsd
|
||||
#define X86_SET_GSBASE 16 // netbsd
|
||||
#define X86_SET_FSBASE 17 // netbsd
|
||||
|
||||
#define __NR___set_tcb 0x00000149
|
||||
#define __NR__lwp_setprivate 0x0000013d
|
||||
#define __NR_thread_fast_set_cthread_self 0x03000003
|
||||
|
||||
#define _TLSZ ((intptr_t)_tls_size)
|
||||
#define _TLDZ ((intptr_t)_tdata_size)
|
||||
#define _TIBZ sizeof(struct cthread_descriptor_t)
|
||||
|
||||
static char tibdefault[64];
|
||||
extern int __threadcalls_end[];
|
||||
extern int __threadcalls_start[];
|
||||
extern unsigned char __get_tls_nt_rax[];
|
||||
__msabi extern typeof(TlsAlloc) *const __imp_TlsAlloc;
|
||||
|
||||
void __enable_tls(void) {
|
||||
__initialize_tls(tibdefault);
|
||||
*(int *)((char *)tibdefault + 0x38) = sys_gettid();
|
||||
*(int *)((char *)tibdefault + 0x3c) = __errno;
|
||||
__install_tls(tibdefault);
|
||||
privileged void __enable_tls(void) {
|
||||
assert(!__threaded);
|
||||
assert(!__tls_enabled);
|
||||
|
||||
// allocate tls memory for main process
|
||||
//
|
||||
// %fs Linux/BSDs
|
||||
// │
|
||||
// _Thread_local │ __get_tls()
|
||||
// ┌───┬──────────┬──────────┼───┐
|
||||
// │pad│ .tdata │ .tbss │tib│
|
||||
// └───┴──────────┴──────────┼───┘
|
||||
// │
|
||||
// Windows/Mac %gs
|
||||
//
|
||||
size_t siz;
|
||||
cthread_t tib;
|
||||
char *mem, *tls;
|
||||
siz = ROUNDUP(_TLSZ + _TIBZ, FRAMESIZE);
|
||||
mem = _mapanon(siz);
|
||||
tib = (cthread_t)(mem + siz - _TIBZ);
|
||||
tls = mem + siz - _TIBZ - _TLSZ;
|
||||
tib->self = tib;
|
||||
tib->self2 = tib;
|
||||
tib->err = __errno;
|
||||
tib->tid = sys_gettid();
|
||||
memmove(tls, _tdata_start, _TLDZ);
|
||||
|
||||
// ask the operating system to change the x86 segment register
|
||||
int ax, dx;
|
||||
if (IsWindows()) {
|
||||
__tls_index = __imp_TlsAlloc();
|
||||
asm("mov\t%1,%%gs:%0" : "=m"(*((long *)0x1480 + __tls_index)) : "r"(tib));
|
||||
} else if (IsFreebsd()) {
|
||||
asm volatile("syscall"
|
||||
: "=a"(ax)
|
||||
: "0"(__NR_sysarch), "D"(AMD64_SET_FSBASE), "S"(tib)
|
||||
: "rcx", "r11", "memory", "cc");
|
||||
} else if (IsNetbsd()) {
|
||||
asm volatile("syscall"
|
||||
: "=a"(ax), "=d"(dx)
|
||||
: "0"(__NR_sysarch), "D"(X86_SET_FSBASE), "S"(tib)
|
||||
: "rcx", "r11", "memory", "cc");
|
||||
} else if (IsXnu()) {
|
||||
asm volatile("syscall"
|
||||
: "=a"(ax)
|
||||
: "0"(__NR_thread_fast_set_cthread_self),
|
||||
"D"((intptr_t)tib - 0x30)
|
||||
: "rcx", "r11", "memory", "cc");
|
||||
} else if (IsOpenbsd()) {
|
||||
asm volatile("syscall"
|
||||
: "=a"(ax)
|
||||
: "0"(__NR___set_tcb), "D"(tib)
|
||||
: "rcx", "r11", "memory", "cc");
|
||||
} else {
|
||||
asm volatile("syscall"
|
||||
: "=a"(ax)
|
||||
: "0"(__NR_linux_arch_prctl), "D"(ARCH_SET_FS), "S"(tib)
|
||||
: "rcx", "r11", "memory");
|
||||
}
|
||||
|
||||
/*
|
||||
* We need to rewrite SysV _Thread_local code. You MUST use the
|
||||
* -mno-tls-direct-seg-refs flag which generates code like this
|
||||
*
|
||||
* 64 48 8b 0R4 25 00 00 00 00 mov %fs:0,%R
|
||||
*
|
||||
* Which on Mac we can replace with this:
|
||||
*
|
||||
* 65 48 8b 0R4 25 30 00 00 00 mov %gs:0x30,%R
|
||||
*
|
||||
* Whereas on Windows we'll replace it with this:
|
||||
*
|
||||
* 0f 1f 40 00 fatnop4
|
||||
* e8 xx xx xx xx call __get_tls_nt_%R
|
||||
*
|
||||
* Since we have no idea where the TLS instructions exist in the
|
||||
* binary, we need to disassemble the whole program image. This'll
|
||||
* potentially take a few milliseconds for some larger programs.
|
||||
*
|
||||
* TODO(jart): compute probability this is just overkill
|
||||
*/
|
||||
if (IsWindows() || IsXnu()) {
|
||||
int n, reg, dis;
|
||||
unsigned char *p;
|
||||
struct XedDecodedInst xedd;
|
||||
__morph_begin();
|
||||
|
||||
// The most expensive part of this process is we need to compute the
|
||||
// byte length of each instruction in our program. We'll use Intel's
|
||||
// disassembler for this purpose.
|
||||
for (p = _ereal; p < __privileged_start; p += n) {
|
||||
xed_decoded_inst_zero_set_mode(&xedd, XED_MACHINE_MODE_LONG_64);
|
||||
if (!xed_instruction_length_decode(&xedd, p, 15)) {
|
||||
|
||||
// We now know p[0] is most likely the first byte of an x86 op.
|
||||
// Let's check and see if it's the GCC linear TIB address load.
|
||||
// We hope and pray GCC won't generate TLS stores to %r8..%r15.
|
||||
if (xedd.length == 9 && //
|
||||
0144 == p[0] && // fs
|
||||
0110 == p[1] && // rex.w (64-bit operand size)
|
||||
0213 == p[2] && // mov reg/mem → reg (word-sized)
|
||||
0004 == (p[3] & 0307) && // mod/rm (4,reg,0) means sib → reg
|
||||
0045 == p[4] && // sib (5,4,0) → (rbp,rsp,0) → disp32
|
||||
0000 == p[5] && // displacement (von Neumann endian)
|
||||
0000 == p[6] && // displacement
|
||||
0000 == p[7] && // displacement
|
||||
0000 == p[8]) { // displacement
|
||||
|
||||
// Apple is quite straightforward to patch. We basically
|
||||
// just change the segment register, and the linear slot
|
||||
if (IsXnu()) {
|
||||
p[0] = 0145; // this changes gs segment to fs segment
|
||||
p[5] = 0x30; // tib slot index for tib linear address
|
||||
}
|
||||
|
||||
// Windows is kind of complicated. We need to replace the
|
||||
// segment mov instruction with a function call, that (a)
|
||||
// won't clobber registers, and (b) has a return register
|
||||
// that's the same as the mov destination. When setting
|
||||
// function displacement, &CALL+5+DISP must equal &FUNC.
|
||||
else {
|
||||
reg = (p[3] & 070) >> 3;
|
||||
dis = (__get_tls_nt_rax + reg * 18) - (p + 9);
|
||||
p[0] = 0017; // map1
|
||||
p[1] = 0037; // nopl (onl if reg=0)
|
||||
p[2] = 0100; // mod/rm (%rax)+disp8
|
||||
p[3] = 0000; // displacement
|
||||
p[4] = 0350; // call
|
||||
p[5] = (dis & 0x000000ff) >> 000; // displacement
|
||||
p[6] = (dis & 0x0000ff00) >> 010; // displacement
|
||||
p[7] = (dis & 0x00ff0000) >> 020; // displacement
|
||||
p[8] = (dis & 0xff000000) >> 030; // displacement
|
||||
}
|
||||
}
|
||||
|
||||
// Move to the next instruction.
|
||||
n = xedd.length;
|
||||
} else {
|
||||
// If Xed failed to decode the instruction, then we'll just plow
|
||||
// through memory one byte at a time until Xed's morale improves
|
||||
n = 1;
|
||||
}
|
||||
}
|
||||
|
||||
__morph_end();
|
||||
}
|
||||
|
||||
// we are now allowed to use tls
|
||||
__tls_enabled = true;
|
||||
}
|
||||
|
||||
privileged void __enable_threads(void) {
|
||||
assert(!__threaded);
|
||||
__threaded = gettid();
|
||||
__morph_begin();
|
||||
/*
|
||||
* _NOPL("__threadcalls", func)
|
||||
*
|
||||
* The big ugly macro above is used by Cosmopolitan Libc to unser
|
||||
* locking primitive (e.g. flockfile, funlockfile) have zero impact on
|
||||
* performance and binary size when threads aren't actually in play.
|
||||
*
|
||||
* we have this
|
||||
*
|
||||
* 0f 1f 05 b1 19 00 00 nopl func(%rip)
|
||||
|
|
@ -46,8 +223,10 @@ privileged void __enable_threads(void) {
|
|||
* we're going to turn it into this
|
||||
*
|
||||
* 67 67 e8 b1 19 00 00 addr32 addr32 call func
|
||||
*
|
||||
* This is cheap and fast because the big ugly macro stored in the
|
||||
* binary the offsets of all the instructions we need to change.
|
||||
*/
|
||||
__morph_begin();
|
||||
for (int *p = __threadcalls_start; p < __threadcalls_end; ++p) {
|
||||
_base[*p + 0] = 0x67;
|
||||
_base[*p + 1] = 0x67;
|
||||
|
|
|
|||
|
|
@ -26,8 +26,9 @@
|
|||
// runtime facilities.
|
||||
//
|
||||
// @param %rdi is arg
|
||||
// @param %rsi is func
|
||||
// @param %rdx is stack
|
||||
// @param %rsi is tid
|
||||
// @param %rdx is func
|
||||
// @param %rcx is stack
|
||||
// @return %rax is exit code
|
||||
// @see clone()
|
||||
WinThreadLaunch:
|
||||
|
|
@ -35,9 +36,9 @@ WinThreadLaunch:
|
|||
push %r15
|
||||
mov %rbp,%r15
|
||||
mov %rsp,%rbx
|
||||
mov %rdx,%rsp
|
||||
mov %rcx,%rsp
|
||||
xor %rbp,%rbp
|
||||
call *%rsi
|
||||
call *%rdx
|
||||
mov %r15,%rbp
|
||||
mov %rbx,%rsp
|
||||
pop %r15
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue