mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-07-16 23:50:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Make improvements to locking

Browse source 
This change makes pthread_mutex_lock() as fast as _spinlock() by
default. Thread instability issues on NetBSD have been resolved.
Improvements made to gdtoa thread code. Crash reporting will now
synchronize between threads in a slightly better way.
This commit is contained in:
Justine Tunney 2022-06-19 01:13:03 -07:00
parent 25041b8026
commit d5312b60f7
60 changed files with 890 additions and 629 deletions
Download patch file Download diff file Expand all files Collapse all files

40
libc/intrin/assertfail.greg.c
View file

@ -18,34 +18,44 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/bits/weaken.h"
#include "libc/calls/strace.internal.h"
#include "libc/intrin/cmpxchg.h"
#include "libc/calls/calls.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/lockcmpxchg.h"
#include "libc/log/log.h"
#include "libc/intrin/lockcmpxchgp.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/symbols.internal.h"
/**
* Handles failure of assert() macro.
*/
relegated wontreturn void __assert_fail(const char *expr, const char *file,
int line) {
int rc;
static bool noreentry;
int me, owner;
static int sync;
--__strace;
--__ftrace;
kprintf("%s:%d: assert(%s) failed\n", file, line, expr);
if (_lockcmpxchg(&noreentry, false, true)) {
if (weaken(__die)) {
weaken(__die)();
owner = 0;
me = gettid();
kprintf("%s:%d: assert(%s) failed (tid %d)\n", file, line, expr, me);
if (_lockcmpxchgp(&sync, &owner, me)) {
__restore_tty();
if (weaken(ShowBacktrace)) {
weaken(ShowBacktrace)(2, __builtin_frame_address(0));
} else if (weaken(PrintBacktraceUsingSymbols) && weaken(GetSymbolTable)) {
weaken(PrintBacktraceUsingSymbols)(2, __builtin_frame_address(0),
weaken(GetSymbolTable)());
} else {
kprintf("can't backtrace b/c `__die` not linked\n");
kprintf("can't backtrace b/c `ShowCrashReports` not linked\n");
}
rc = 23;
__restorewintty();
_Exit(23);
} else if (owner == me) {
kprintf("assert failed while failing\n");
__restorewintty();
_Exit(24);
} else {
rc = 24;
_Exit1(25);
}
__restorewintty();
_Exit(rc);
}

1
libc/intrin/exit1.greg.c
View file

@ -31,7 +31,6 @@
* @noreturn
*/
privileged wontreturn void _Exit1(int rc) {
jmp_buf *jb;
struct WinThread *wt;
STRACE("_Exit1(%d)", rc);
if (!IsWindows() && !IsMetal()) {

2
libc/intrin/g_fds.c
View file

@ -30,10 +30,12 @@ struct Fds g_fds;
static pthread_mutex_t __fds_lock_obj;
void(__fds_lock)(void) {
__fds_lock_obj.attr = PTHREAD_MUTEX_RECURSIVE;
pthread_mutex_lock(&__fds_lock_obj);
}
void(__fds_unlock)(void) {
__fds_lock_obj.attr = PTHREAD_MUTEX_RECURSIVE;
pthread_mutex_unlock(&__fds_lock_obj);
}

2
libc/intrin/mmi.c
View file

@ -22,7 +22,7 @@
STATIC_YOINK("_init__mmi");
struct MemoryIntervals _mmi;
static pthread_mutex_t __mmi_lock_obj;
pthread_mutex_t __mmi_lock_obj; // recursive :'(
void(__mmi_lock)(void) {
pthread_mutex_lock(&__mmi_lock_obj);

2
libc/intrin/mmi.init.S
View file

@ -16,9 +16,11 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/pthread.h"
#include "libc/macros.internal.h"
.init.start 200,_init__mmi
movb $OPEN_MAX,_mmi+8
movl $_mmi+24,_mmi+16
movb $PTHREAD_MUTEX_RECURSIVE,__mmi_lock_obj(%rip)
.init.end 200,_init__mmi

40
libc/intrin/pthread.h
View file

@ -1,19 +1,21 @@
#ifndef COSMOPOLITAN_LIBC_RUNTIME_PTHREAD_H_
#define COSMOPOLITAN_LIBC_RUNTIME_PTHREAD_H_
#include "libc/bits/atomic.h"
#include "libc/calls/struct/timespec.h"
#include "libc/intrin/kprintf.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
#include "libc/dce.h"
#define PTHREAD_ONCE_INIT 0
#define PTHREAD_MUTEX_DEFAULT PTHREAD_MUTEX_NORMAL
#define PTHREAD_MUTEX_RECURSIVE 0
#define PTHREAD_MUTEX_NORMAL 1
#define PTHREAD_MUTEX_NORMAL 0
#define PTHREAD_MUTEX_RECURSIVE 1
#define PTHREAD_MUTEX_ERRORCHECK 2
#define PTHREAD_MUTEX_STALLED 0
#define PTHREAD_MUTEX_ROBUST 1
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
/* clang-format off */
#define PTHREAD_MUTEX_INITIALIZER {PTHREAD_MUTEX_DEFAULT}
#define PTHREAD_RWLOCK_INITIALIZER {{{0}}}
@ -66,8 +68,8 @@ typedef struct {
} __u;
} pthread_rwlock_t;
wontreturn void pthread_exit(void *);
pureconst pthread_t pthread_self(void);
void pthread_exit(void *) wontreturn;
pthread_t pthread_self(void) pureconst;
int pthread_create(pthread_t *, const pthread_attr_t *, void *(*)(void *),
void *);
int pthread_yield(void);
@ -104,6 +106,30 @@ int pthread_rwlock_trywrlock(pthread_rwlock_t *);
int pthread_rwlock_timedwrlock(pthread_rwlock_t *, const struct timespec *);
int pthread_rwlock_unlock(pthread_rwlock_t *);
#define pthread_mutexattr_init(pAttr) ((pAttr)->attr = PTHREAD_MUTEX_DEFAULT, 0)
#define pthread_mutexattr_destroy(pAttr) ((pAttr)->attr = 0)
#define pthread_mutexattr_gettype(pAttr, pType) (*(pType) = (pAttr)->attr, 0)
#define pthread_mutexattr_settype(pAttr, type) ((pAttr)->attr = type, 0)
#ifdef __GNUC__
#define pthread_mutex_lock(mutex) \
(((mutex)->attr == PTHREAD_MUTEX_NORMAL && \
!atomic_load_explicit(&(mutex)->lock, memory_order_relaxed) && \
!atomic_exchange(&(mutex)->lock, 1)) \
? 0 \
: pthread_mutex_lock(mutex))
#define pthread_mutex_unlock(mutex) \
((mutex)->attr == PTHREAD_MUTEX_NORMAL \
? (atomic_store_explicit(&(mutex)->lock, 0, memory_order_relaxed), \
(IsLinux() && \
atomic_load_explicit(&(mutex)->waits, memory_order_relaxed) && \
_pthread_mutex_wake(mutex)), \
0) \
: pthread_mutex_unlock(mutex))
#endif
int _pthread_mutex_wake(pthread_mutex_t *) hidden;
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_RUNTIME_PTHREAD_H_ */

79
libc/intrin/pthread_mutex_lock.c
View file

@ -21,41 +21,68 @@
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/pthread.h"
#include "libc/intrin/spinlock.h"
#include "libc/linux/futex.h"
#include "libc/nexgen32e/threaded.h"
static int pthread_mutex_lock_spin(pthread_mutex_t *mutex, int tries) {
volatile int i;
if (tries < 7) {
for (i = 0; i != 1 << tries; i++) {
}
tries++;
} else if (IsLinux()) {
atomic_fetch_add(&mutex->waits, 1);
LinuxFutexWait(&mutex->lock, 1, 0);
atomic_fetch_sub(&mutex->waits, 1);
} else {
sched_yield();
}
return tries;
}
/**
* Locks mutex.
*
* _spinlock() l: 181,570c 58,646ns
* mutex normal l: 297,965c 96,241ns
* mutex recursive l: 1,112,166c 359,223ns
* mutex errorcheck l: 1,449,723c 468,252ns
*
* @return 0 on success, or error number on failure
*/
int pthread_mutex_lock(pthread_mutex_t *mutex) {
volatile int i;
int(pthread_mutex_lock)(pthread_mutex_t *mutex) {
int me, owner, tries;
for (tries = 0, me = gettid();;) {
owner = atomic_load_explicit(&mutex->lock, memory_order_relaxed);
if (!owner && atomic_compare_exchange_weak_explicit(
&mutex->lock, &owner, me, memory_order_acquire,
memory_order_relaxed)) {
break;
} else if (owner == me) {
if (mutex->attr != PTHREAD_MUTEX_ERRORCHECK) {
break;
} else {
return EDEADLK;
switch (mutex->attr) {
case PTHREAD_MUTEX_NORMAL:
for (tries = 0;;) {
if (!atomic_load_explicit(&mutex->lock, memory_order_relaxed) &&
!atomic_exchange_explicit(&mutex->lock, 1, memory_order_acquire)) {
break;
}
tries = pthread_mutex_lock_spin(mutex, tries);
}
}
if (tries < 7) {
for (i = 0; i != 1 << tries; i++) {
return 0;
case PTHREAD_MUTEX_RECURSIVE:
case PTHREAD_MUTEX_ERRORCHECK:
for (tries = 0, me = gettid();;) {
owner = atomic_load_explicit(&mutex->lock, memory_order_relaxed);
if (!owner && atomic_compare_exchange_weak_explicit(
&mutex->lock, &owner, me, memory_order_acquire,
memory_order_relaxed)) {
break;
} else if (owner == me) {
if (mutex->attr != PTHREAD_MUTEX_ERRORCHECK) {
break;
} else {
return EDEADLK;
}
}
tries = pthread_mutex_lock_spin(mutex, tries);
}
tries++;
} else if (IsLinux()) {
atomic_fetch_add(&mutex->waits, 1);
LinuxFutexWait(&mutex->lock, owner, 0);
atomic_fetch_sub(&mutex->waits, 1);
} else {
sched_yield();
}
++mutex->reent;
return 0;
default:
return EINVAL;
}
++mutex->reent;
return 0;
}

36
libc/intrin/pthread_mutex_unlock.c
View file

@ -18,28 +18,34 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/bits/atomic.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/pthread.h"
#include "libc/linux/futex.h"
#include "libc/nexgen32e/threaded.h"
/**
* Releases mutex.
* @return 0 on success or error number on failure
* @raises EPERM if in error check mode and not owned by caller
*/
int pthread_mutex_unlock(pthread_mutex_t *mutex) {
int owner;
if (mutex->attr == PTHREAD_MUTEX_ERRORCHECK && mutex->lock != gettid()) {
return EPERM;
int(pthread_mutex_unlock)(pthread_mutex_t *mutex) {
int me, owner;
switch (mutex->attr) {
case PTHREAD_MUTEX_ERRORCHECK:
me = gettid();
owner = atomic_load_explicit(&mutex->lock, memory_order_relaxed);
if (owner != me) return EPERM;
// fallthrough
case PTHREAD_MUTEX_RECURSIVE:
if (--mutex->reent) return 0;
// fallthrough
case PTHREAD_MUTEX_NORMAL:
atomic_store_explicit(&mutex->lock, 0, memory_order_relaxed);
if (IsLinux() &&
atomic_load_explicit(&mutex->waits, memory_order_relaxed)) {
_pthread_mutex_wake(mutex);
}
return 0;
default:
return EINVAL;
}
if (!--mutex->reent) {
atomic_store_explicit(&mutex->lock, 0, memory_order_relaxed);
if (IsLinux() &&
atomic_load_explicit(&mutex->waits, memory_order_acquire)) {
LinuxFutexWake(&mutex->lock, 1);
}
}
return 0;
}

26
libc/intrin/pthread_mutex_wake.c Normal file
View file

@ -0,0 +1,26 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/bits/atomic.h"
#include "libc/dce.h"
#include "libc/intrin/pthread.h"
#include "libc/linux/futex.h"
int _pthread_mutex_wake(pthread_mutex_t *mutex) {
return LinuxFutexWake(&mutex->lock, 1);
}

5
libc/intrin/pthread_mutexattr_destroy.c
View file

@ -17,13 +17,12 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/pthread.h"
#include "libc/str/str.h"
/**
* Destroys mutex attr.
* @return 0 on success, or error number on failure
*/
int pthread_mutexattr_destroy(pthread_mutexattr_t *attr) {
bzero(attr, sizeof(*attr));
int(pthread_mutexattr_destroy)(pthread_mutexattr_t *attr) {
attr->attr = 0;
return 0;
}

2
libc/intrin/pthread_mutexattr_gettype.c
View file

@ -29,7 +29,7 @@
* - `PTHREAD_MUTEX_ERRORCHECK`
* @return 0 on success, or error on failure
*/
int pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *type) {
int(pthread_mutexattr_gettype)(const pthread_mutexattr_t *attr, int *type) {
*type = attr->attr;
return 0;
}

4
libc/intrin/pthread_mutexattr_init.c
View file

@ -17,14 +17,12 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/pthread.h"
#include "libc/str/str.h"
/**
* Initializes mutex attr.
* @return 0 on success, or error number on failure
*/
int pthread_mutexattr_init(pthread_mutexattr_t *attr) {
bzero(attr, sizeof(*attr));
int(pthread_mutexattr_init)(pthread_mutexattr_t *attr) {
attr->attr = PTHREAD_MUTEX_DEFAULT;
return 0;
}

2
libc/intrin/pthread_mutexattr_settype.c
View file

@ -30,7 +30,7 @@
* @return 0 on success, or error on failure
* @raises EINVAL if `type` is invalid
*/
int pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type) {
int(pthread_mutexattr_settype)(pthread_mutexattr_t *attr, int type) {
switch (type) {
case PTHREAD_MUTEX_NORMAL:
case PTHREAD_MUTEX_RECURSIVE:

68
libc/intrin/restoretty.c Normal file
View file

@ -0,0 +1,68 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/calls/struct/metatermios.internal.h"
#include "libc/calls/struct/termios.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/calls/termios.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/log/color.internal.h"
#include "libc/log/internal.h"
#include "libc/log/libfatal.internal.h"
#include "libc/runtime/internal.h"
#include "libc/sysv/consts/termios.h"
/**
* @fileoverview Terminal Restoration Helper for System Five.
*
* This is used by the crash reporting functions, e.g. __die(), to help
* ensure the terminal is in an unborked state after a crash happens.
*/
#define RESET_COLOR "\e[0m"
#define SHOW_CURSOR "\e[?25h"
#define DISABLE_MOUSE "\e[?1000;1002;1015;1006l"
#define ANSI_RESTORE RESET_COLOR SHOW_CURSOR DISABLE_MOUSE
static bool __isrestorable;
static union metatermios __oldtermios;
static textstartup void __oldtermios_init() {
int e;
e = errno;
if (sys_ioctl(0, TCGETS, &__oldtermios) != -1) {
__isrestorable = true;
}
errno = e;
}
const void *const __oldtermios_ctor[] initarray = {
__oldtermios_init,
};
void __restore_tty(void) {
int e;
if (__isrestorable && !__isworker && !__nocolor) {
e = errno;
sys_write(0, ANSI_RESTORE, __strlen(ANSI_RESTORE));
sys_ioctl(0, TCSETSF, &__oldtermios);
errno = e;
}
}

8
libc/intrin/spinlock.h
View file

@ -1,5 +1,7 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_SPINLOCK_H_
#define COSMOPOLITAN_LIBC_INTRIN_SPINLOCK_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
/*───────────────────────────────────────────────────────────────────────────│─╗
│ cosmopolitan § spinlocks ─╬─│┼
╚────────────────────────────────────────────────────────────────────────────│─╝
@ -11,7 +13,7 @@
#define _spinlock(lock) _spinlock_cooperative(lock)
#endif
#define _spunlock(lock) __atomic_store_n(lock, 0, __ATOMIC_RELAXED)
#define _spunlock(lock) (__atomic_store_n(lock, 0, __ATOMIC_RELAXED), 0)
#define _seizelock(lock, value) \
({ \
@ -69,6 +71,8 @@
#define _trylock(lock) __atomic_test_and_set(lock, __ATOMIC_SEQ_CST)
void _spinlock_yield(void);
int _spinlock_yield(void);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_INTRIN_SPINLOCK_H_ */