mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-07-30 14:30:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve threading support further

Browse source
This commit is contained in:
Justine Tunney 2022-05-17 04:14:28 -07:00
parent 8bfb70ca3f
commit ce71677156
61 changed files with 882 additions and 747 deletions
Download patch file Download diff file Expand all files Collapse all files

39
libc/intrin/asan.c
View file

@ -27,9 +27,11 @@
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/asancodes.h"
#include "libc/intrin/cmpxchg.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/lockcmpxchg.h"
#include "libc/intrin/nomultics.internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/internal.h"
#include "libc/log/libfatal.internal.h"
@ -154,7 +156,8 @@ struct ReportOriginHeap {
int z;
};
bool __asan_noreentry;
static int __asan_noreentry;
_Alignas(64) static int __asan_lock;
static struct AsanMorgue __asan_morgue;
#define __asan_unreachable() \
@ -835,30 +838,27 @@ dontdiscard __asan_die_f *__asan_report_memory_fault(void *addr, int size,
}
void *__asan_morgue_add(void *p) {
int i;
void *r;
int i, j;
for (;;) {
i = __asan_morgue.i;
j = (i + 1) & (ARRAYLEN(__asan_morgue.p) - 1);
if (_lockcmpxchg(&__asan_morgue.i, i, j)) {
r = __asan_morgue.p[i];
__asan_morgue.p[i] = p;
return r;
}
}
_spinlock_optimistic(&__asan_lock);
i = __asan_morgue.i++ & (ARRAYLEN(__asan_morgue.p) - 1);
r = __asan_morgue.p[i];
__asan_morgue.p[i] = p;
_spunlock(&__asan_lock);
return r;
}
static void __asan_morgue_flush(void) {
int i;
void *p;
_spinlock_optimistic(&__asan_lock);
for (i = 0; i < ARRAYLEN(__asan_morgue.p); ++i) {
p = __asan_morgue.p[i];
if (_lockcmpxchg(__asan_morgue.p + i, p, 0)) {
if (weaken(dlfree)) {
weaken(dlfree)(p);
}
if (weaken(dlfree)) {
weaken(dlfree)(__asan_morgue.p[i]);
}
__asan_morgue.p[i] = 0;
}
_spunlock(&__asan_lock);
}
static size_t __asan_user_size(size_t n) {
@ -1197,12 +1197,13 @@ void __asan_evil(uint8_t *addr, int size, const char *s1, const char *s2) {
struct AsanTrace tr;
__asan_rawtrace(&tr, __builtin_frame_address(0));
kprintf(
"WARNING: ASAN error during %s bad %d byte %s at %x bt %x %x %x %x %x\n",
"WARNING: ASAN %s %s bad %d byte %s at %x bt %x %x %x %x %x\n",
__asan_noreentry == gettid() ? "error during" : "multi-threaded crash",
s1, size, s2, addr, tr.p[0], tr.p[1], tr.p[2], tr.p[3], tr.p[4], tr.p[5]);
}
void __asan_report_load(uint8_t *addr, int size) {
if (_lockcmpxchg(&__asan_noreentry, false, true)) {
if (_lockcmpxchg(&__asan_noreentry, 0, gettid())) {
if (!__vforked) {
__asan_report_memory_fault(addr, size, "load")();
__asan_unreachable();
@ -1215,7 +1216,7 @@ void __asan_report_load(uint8_t *addr, int size) {
}
void __asan_report_store(uint8_t *addr, int size) {
if (_lockcmpxchg(&__asan_noreentry, false, true)) {
if (_lockcmpxchg(&__asan_noreentry, 0, gettid())) {
if (!__vforked) {
__asan_report_memory_fault(addr, size, "store")();
__asan_unreachable();

2
libc/intrin/asan.internal.h
View file

@ -16,8 +16,6 @@ struct AsanFault {
const signed char *shadow;
};
extern bool __asan_noreentry;
void __asan_unpoison(long, long);
void __asan_poison(long, long, signed char);
void __asan_verify(const void *, size_t);

6
libc/intrin/cxaatexit.c
View file

@ -19,6 +19,7 @@
#include "libc/assert.h"
#include "libc/bits/weaken.h"
#include "libc/calls/strace.internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/macros.internal.h"
#include "libc/mem/mem.h"
#include "libc/nexgen32e/bsr.h"
@ -28,6 +29,8 @@
STATIC_YOINK("__cxa_finalize");
static int __cxa_lock;
/**
* Adds global destructor.
*
@ -47,6 +50,7 @@ noasan int __cxa_atexit(void *fp, void *arg, void *pred) {
unsigned i;
struct CxaAtexitBlock *b, *b2;
_Static_assert(ATEXIT_MAX == CHAR_BIT * sizeof(b->mask), "");
_spinlock(&__cxa_lock);
b = __cxa_blocks.p;
if (!b) b = __cxa_blocks.p = &__cxa_blocks.root;
if (!~b->mask) {
@ -55,6 +59,7 @@ noasan int __cxa_atexit(void *fp, void *arg, void *pred) {
b2->next = b;
__cxa_blocks.p = b = b2;
} else {
_spunlock(&__cxa_lock);
return enomem();
}
}
@ -64,5 +69,6 @@ noasan int __cxa_atexit(void *fp, void *arg, void *pred) {
b->p[i].fp = fp;
b->p[i].arg = arg;
b->p[i].pred = pred;
_spunlock(&__cxa_lock);
return 0;
}

3
libc/intrin/exit1.greg.c
View file

@ -19,7 +19,6 @@
#include "libc/calls/strace.internal.h"
#include "libc/dce.h"
#include "libc/intrin/setjmp.internal.h"
#include "libc/intrin/winthread.internal.h"
#include "libc/nt/thread.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/nr.h"
@ -34,7 +33,7 @@
*/
privileged wontreturn void _Exit1(int rc) {
struct WinThread *wt;
/* STRACE("_Exit1(%d)", rc); */
STRACE("_Exit1(%d)", rc);
if (!IsWindows() && !IsMetal()) {
register long r10 asm("r10") = 0;
asm volatile("syscall"

2
libc/intrin/g_fds.c
View file

@ -25,7 +25,7 @@
STATIC_YOINK("_init_g_fds");
struct Fds g_fds;
_Alignas(64) char __fds_lock;
_Alignas(64) int __fds_lock;
textstartup void InitializeFileDescriptors(void) {
struct Fds *fds;

2
libc/intrin/gettid.greg.c
View file

@ -18,7 +18,7 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/intrin/tls.h"
#include "libc/intrin/threaded.h"
#include "libc/nt/thread.h"
/**

2
libc/intrin/kprintf.greg.c
View file

@ -31,7 +31,7 @@
#include "libc/intrin/lockcmpxchg.h"
#include "libc/intrin/nomultics.internal.h"
#include "libc/intrin/spinlock.h"
#include "libc/intrin/threaded.internal.h"
#include "libc/intrin/threaded.h"
#include "libc/limits.h"
#include "libc/log/internal.h"
#include "libc/macros.internal.h"

8
libc/intrin/releasefd.c
View file

@ -20,11 +20,15 @@
#include "libc/intrin/spinlock.h"
#include "libc/macros.internal.h"
void __releasefd(int fd) {
_spinlock(&__fds_lock);
void __releasefd_unlocked(int fd) {
if (0 <= fd && fd < g_fds.n) {
g_fds.p[fd].kind = 0;
g_fds.f = MIN(fd, g_fds.f);
}
}
void __releasefd(int fd) {
_spinlock(&__fds_lock);
__releasefd_unlocked(fd);
_spunlock(&__fds_lock);
}

52
libc/intrin/spinlock.h
View file

@ -2,16 +2,26 @@
#define COSMOPOLITAN_LIBC_INTRIN_SPINLOCK_H_
#include "libc/bits/weaken.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/lockcmpxchg.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/log.h"
#include "libc/runtime/symbols.internal.h"
#if IsModeDbg() && !defined(_SPINLOCK_DEBUG)
#define _SPINLOCK_DEBUG
#endif
#if defined(_SPINLOCK_DEBUG)
#define _spinlock(lock) _spinlock_debug(lock)
#define _spinlock(lock) _spinlock_ndebug(lock)
#define _spinlock_ndebug(lock) _spinlock_optimistic(lock)
#elif defined(TINY)
#define _spinlock(lock) _spinlock_tiny(lock)
#define _spinlock(lock) _spinlock_tiny(lock)
#define _spinlock_ndebug(lock) _spinlock_tiny(lock)
#else
#define _spinlock(lock) _spinlock_optimistic(lock)
#define _spinlock(lock) _spinlock_optimistic(lock)
#define _spinlock_ndebug(lock) _spinlock_optimistic(lock)
#endif
#define _spunlock(lock) __atomic_clear(lock, __ATOMIC_RELAXED)
@ -44,22 +54,26 @@
} \
} while (0)
#define _spinlock_debug(lock) \
do { \
typeof(*(lock)) me, owner; \
me = gettid(); \
if (_trylock(lock)) { \
__atomic_load(lock, &owner, __ATOMIC_RELAXED); \
if (owner == me) { \
kprintf("%s:%d: warning: possible spinlock re-entry in %s()\n", \
__FILE__, __LINE__, __FUNCTION__); \
if (weaken(ShowBacktrace)) { \
weaken(ShowBacktrace)(2, 0); \
} \
} \
_spinlock_optimistic(lock); \
} \
*lock = me; \
#define _spinlock_debug(lock) \
do { \
typeof(*(lock)) me, owner; \
unsigned long warntries = 10000000; \
me = gettid(); \
if (!_lockcmpxchg(lock, 0, me)) { \
__atomic_load(lock, &owner, __ATOMIC_RELAXED); \
if (owner == me) { \
kprintf("%s:%d: warning: possible re-entry on %s in %s()\n", __FILE__, \
__LINE__, #lock, __FUNCTION__); \
} \
while (!_lockcmpxchg(lock, 0, me)) { \
if (!--warntries) { \
warntries = -1; \
kprintf("%s:%d: warning: possible deadlock on %s in %s()\n", \
__FILE__, __LINE__, #lock, __FUNCTION__); \
} \
__builtin_ia32_pause(); \
} \
} \
} while (0)
#endif /* COSMOPOLITAN_LIBC_INTRIN_SPINLOCK_H_ */

16
libc/intrin/threaded.h Normal file
View file

@ -0,0 +1,16 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_THREADED_H_
#define COSMOPOLITAN_LIBC_INTRIN_THREADED_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
extern bool __threaded;
extern bool __tls_enabled;
extern unsigned __tls_index;
void *__initialize_tls(char[hasatleast 64]);
void __install_tls(char[hasatleast 64]);
char *__get_tls(void);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_INTRIN_THREADED_H_ */

11
libc/intrin/threaded.internal.h
View file

@ -1,11 +0,0 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_THREADED_INTERNAL_H_
#define COSMOPOLITAN_LIBC_INTRIN_THREADED_INTERNAL_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
extern bool __hastls;
extern bool __threaded;
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_INTRIN_THREADED_INTERNAL_H_ */

111
libc/intrin/tls.greg.c
View file

@ -17,76 +17,67 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/intrin/tls.h"
#include "libc/errno.h"
#include "libc/intrin/threaded.h"
#include "libc/nt/thread.h"
#include "libc/nt/thunk/msabi.h"
#include "libc/sysv/consts/nrlinux.h"
__msabi extern typeof(TlsFree) *const __imp_TlsFree;
__msabi extern typeof(TlsAlloc) *const __imp_TlsAlloc;
__msabi extern typeof(TlsGetValue) *const __imp_TlsGetValue;
__msabi extern typeof(TlsSetValue) *const __imp_TlsSetValue;
#define __NR_sysarch 0x000000a5
#define __NR___set_tcb 0x00000149
#define __NR__lwp_setprivate 0x0000013d
#define __NR_thread_fast_set_cthread_self 0x03000003
/**
* Assigns thread-local storage slot.
*
* This function may for instance be called at startup and the result
* can be assigned to a global static variable; from then on, all the
* threads in your application may pass that value to TlsGetValue, to
* retrieve their thread-local values.
*
* @return index on success, or -1u w/ errno
* @threadsafe
* Initializes thread information block.
*/
uint32_t TlsAlloc(void) {
return __imp_TlsAlloc();
privileged void *__initialize_tls(char tib[hasatleast 64]) {
*(intptr_t *)tib = (intptr_t)tib;
*(intptr_t *)(tib + 0x30) = (intptr_t)tib;
*(int *)(tib + 0x3c) = __errno;
return tib;
}
/**
* Releases thread-local storage slot.
* @threadsafe
* Installs thread information block on main process.
*/
bool32 TlsFree(uint32_t dwTlsIndex) {
return __imp_TlsFree(dwTlsIndex);
}
/**
* Sets value to thread-local storage slot.
*
* @param dwTlsIndex is something returned by TlsAlloc()
* @return true if successful, otherwise false
* @threadsafe
*/
bool32 TlsSetValue(uint32_t dwTlsIndex, void *lpTlsValue) {
assert(IsWindows());
if (dwTlsIndex < 64) {
asm("mov\t%1,%%gs:%0"
: "=m"(*((long *)0x1480 + dwTlsIndex))
: "r"(lpTlsValue));
return true;
privileged void __install_tls(char tib[hasatleast 64]) {
int ax, dx;
uint64_t magic;
unsigned char *p;
if (IsWindows()) {
if (!__tls_index) {
__tls_index = TlsAlloc();
}
asm("mov\t%1,%%gs:%0" : "=m"(*((long *)0x1480 + __tls_index)) : "r"(tib));
} else if (IsFreebsd()) {
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR_sysarch), "D"(129), "S"(tib)
: "rcx", "r11", "memory", "cc");
} else if (IsXnu()) {
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR_thread_fast_set_cthread_self),
"D"((intptr_t)tib - 0x30)
: "rcx", "r11", "memory", "cc");
} else if (IsOpenbsd()) {
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR___set_tcb), "D"(tib)
: "rcx", "r11", "memory", "cc");
} else if (IsNetbsd()) {
asm volatile("syscall"
: "=a"(ax), "=d"(dx)
: "0"(__NR__lwp_setprivate), "D"(tib)
: "rcx", "r11", "memory", "cc");
} else {
return __imp_TlsSetValue(dwTlsIndex, lpTlsValue);
}
}
/**
* Retrieves value from thread-local storage slot.
*
* @param dwTlsIndex is something returned by TlsAlloc()
* @return true if successful, otherwise false
* @threadsafe
*/
void *TlsGetValue(uint32_t dwTlsIndex) {
void *lpTlsValue;
assert(IsWindows());
if (dwTlsIndex < 64) {
asm("mov\t%%gs:%1,%0"
: "=r"(lpTlsValue)
: "m"(*((long *)0x1480 + dwTlsIndex)));
return lpTlsValue;
// // this could also be written as...
// asm("movq\t%%gs:0x30,%0" : "=a"(tib));
// return (void *)tib[0x1480 / 8 + dwTlsIndex];
} else {
return __imp_TlsGetValue(dwTlsIndex);
asm volatile("syscall"
: "=a"(ax)
: "0"(__NR_linux_arch_prctl), "D"(ARCH_SET_FS), "S"(tib)
: "rcx", "r11", "memory");
}
__tls_enabled = true;
}

13
libc/intrin/tls.h
View file

@ -1,13 +0,0 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_TLS_H_
#define COSMOPOLITAN_LIBC_INTRIN_TLS_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
uint32_t TlsAlloc(void);
bool32 TlsFree(uint32_t);
bool32 TlsSetValue(uint32_t, void *);
void *TlsGetValue(uint32_t);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_INTRIN_TLS_H_ */

38
libc/intrin/winthread.c
View file

@ -1,38 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/dce.h"
#include "libc/intrin/tls.h"
#include "libc/intrin/winthread.internal.h"
/**
* @fileoverview TLS slot for clone() win32 polyfill.
*/
int __winthread;
static textstartup void __winthread_init(void) {
if (IsWindows()) {
__winthread = TlsAlloc();
TlsSetValue(__winthread, 0);
}
}
const void *const __winthread_ctor[] initarray = {
__winthread_init,
};

24
libc/intrin/winthread.internal.h
View file

@ -1,24 +0,0 @@
#ifndef COSMOPOLITAN_LIBC_RUNTIME_WINTHREAD_INTERNAL_H_
#define COSMOPOLITAN_LIBC_RUNTIME_WINTHREAD_INTERNAL_H_
#include "libc/intrin/tls.h"
#include "libc/runtime/runtime.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
struct WinThread {
uint32_t tid;
int flags;
int *ctid;
int (*func)(void *);
void *arg;
};
extern int __winthread;
static inline struct WinThread *GetWinThread(void) {
return TlsGetValue(__winthread);
}
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_RUNTIME_WINTHREAD_INTERNAL_H_ */