diff --git a/libc/intrin/pthread_mutex_lock.c b/libc/intrin/pthread_mutex_lock.c index 99d4d9ba2..0b96949de 100644 --- a/libc/intrin/pthread_mutex_lock.c +++ b/libc/intrin/pthread_mutex_lock.c @@ -37,6 +37,7 @@ static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex) { // get current state of lock word = atomic_load_explicit(&mutex->_word, memory_order_relaxed); +#if PTHREAD_USE_NSYNC // use fancy nsync mutex if possible if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL && // MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && // @@ -44,6 +45,7 @@ static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex) { _weaken(nsync_mu_lock)((nsync_mu *)mutex); return 0; } +#endif // implement barebones normal mutexes if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) { diff --git a/libc/intrin/pthread_mutex_trylock.c b/libc/intrin/pthread_mutex_trylock.c index 21513656a..a793eed34 100644 --- a/libc/intrin/pthread_mutex_trylock.c +++ b/libc/intrin/pthread_mutex_trylock.c @@ -46,6 +46,7 @@ errno_t pthread_mutex_trylock(pthread_mutex_t *mutex) { // get current state of lock word = atomic_load_explicit(&mutex->_word, memory_order_relaxed); +#if PTHREAD_USE_NSYNC // delegate to *NSYNC if possible if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL && MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && // @@ -56,6 +57,7 @@ errno_t pthread_mutex_trylock(pthread_mutex_t *mutex) { return EBUSY; } } +#endif // handle normal mutexes if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) { diff --git a/libc/intrin/pthread_mutex_unlock.c b/libc/intrin/pthread_mutex_unlock.c index 652896475..4d796d9a4 100644 --- a/libc/intrin/pthread_mutex_unlock.c +++ b/libc/intrin/pthread_mutex_unlock.c @@ -45,6 +45,7 @@ errno_t pthread_mutex_unlock(pthread_mutex_t *mutex) { // get current state of lock word = atomic_load_explicit(&mutex->_word, memory_order_relaxed); +#if PTHREAD_USE_NSYNC // use fancy nsync mutex if possible if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL && // MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && // @@ -52,6 +53,7 @@ errno_t pthread_mutex_unlock(pthread_mutex_t *mutex) { _weaken(nsync_mu_unlock)((nsync_mu *)mutex); return 0; } +#endif // implement barebones normal mutexes if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) { diff --git a/libc/intrin/pthread_yield_np.c b/libc/intrin/pthread_yield_np.c index dd209e6be..fefc3f283 100644 --- a/libc/intrin/pthread_yield_np.c +++ b/libc/intrin/pthread_yield_np.c @@ -22,6 +22,8 @@ #include "libc/runtime/syslib.internal.h" #include "libc/thread/thread.h" +void sys_sched_yield(void); + /** * Yields current thread's remaining timeslice to operating system. * @@ -30,13 +32,16 @@ int pthread_yield_np(void) { if (IsXnuSilicon()) { __syslib->__pthread_yield_np(); - } else if (IsOpenbsd()) { - pthread_pause_np(); // sched_yield() is punishingly slow on OpenBSD + } else if (IsOpenbsd() || IsNetbsd()) { + // sched_yield() is punishingly slow on OpenBSD + // it's ruinously slow it'll destroy everything + pthread_pause_np(); } else { - sched_yield(); + sys_sched_yield(); } return 0; } __weak_reference(pthread_yield_np, thrd_yield); +__weak_reference(pthread_yield_np, sched_yield); __weak_reference(pthread_yield_np, pthread_yield); diff --git a/libc/intrin/sched_yield.S b/libc/intrin/sys_sched_yield.S similarity index 98% rename from libc/intrin/sched_yield.S rename to libc/intrin/sys_sched_yield.S index 42433fa6d..eab709511 100644 --- a/libc/intrin/sched_yield.S +++ b/libc/intrin/sys_sched_yield.S @@ -19,12 +19,11 @@ #include "libc/dce.h" #include "libc/sysv/consts/nr.h" #include "libc/macros.internal.h" -.privileged // Relinquishes scheduled quantum. // // @return 0 on success, or -1 w/ errno -sched_yield: +sys_sched_yield: #ifdef __x86_64__ push %rbp mov %rsp,%rbp @@ -107,5 +106,5 @@ sched_yield: #else #error "arch unsupported" #endif - .endfn sched_yield,globl + .endfn sys_sched_yield,globl .previous diff --git a/libc/thread/pthread_cond_broadcast.c b/libc/thread/pthread_cond_broadcast.c index df68a2ec1..b757c867c 100644 --- a/libc/thread/pthread_cond_broadcast.c +++ b/libc/thread/pthread_cond_broadcast.c @@ -1,7 +1,7 @@ /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ │ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ ╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ │ │ │ Permission to use, copy, modify, and/or distribute this software for │ │ any purpose with or without fee is hereby granted, provided that the │ @@ -16,8 +16,11 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/atomic.h" +#include "libc/limits.h" #include "libc/thread/thread.h" #include "third_party/nsync/cv.h" +#include "third_party/nsync/futex.internal.h" /** * Wakes all threads waiting on condition, e.g. @@ -37,6 +40,19 @@ * @see pthread_cond_wait */ errno_t pthread_cond_broadcast(pthread_cond_t *cond) { - nsync_cv_broadcast((nsync_cv *)cond); + +#if PTHREAD_USE_NSYNC + // favor *NSYNC if this is a process private condition variable + // if using Mike Burrows' code isn't possible, use a naive impl + if (!cond->_pshared) { + nsync_cv_broadcast((nsync_cv *)cond); + return 0; + } +#endif + + // roll forward the monotonic sequence + atomic_fetch_add_explicit(&cond->_sequence, 1, memory_order_acq_rel); + if (atomic_load_explicit(&cond->_waiters, memory_order_acquire)) + nsync_futex_wake_((atomic_int *)&cond->_sequence, INT_MAX, cond->_pshared); return 0; } diff --git a/libc/thread/pthread_cond_destroy.c b/libc/thread/pthread_cond_destroy.c index 3dbd8971b..0b77c88f2 100644 --- a/libc/thread/pthread_cond_destroy.c +++ b/libc/thread/pthread_cond_destroy.c @@ -16,8 +16,11 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/errno.h" +#include "libc/intrin/atomic.h" #include "libc/str/str.h" #include "libc/thread/thread.h" +#include "third_party/nsync/cv.h" /** * Destroys condition. @@ -26,6 +29,22 @@ * @raise EINVAL if threads are still waiting on condition */ errno_t pthread_cond_destroy(pthread_cond_t *cond) { + + // check if there's active waiters +#if PTHREAD_USE_NSYNC + if (cond->_pshared) { + if (((nsync_cv *)cond)->waiters) + return EINVAL; + } else { + if (atomic_load_explicit(&cond->_waiters, memory_order_relaxed)) + return EINVAL; + } +#else + if (atomic_load_explicit(&cond->_waiters, memory_order_relaxed)) + return EINVAL; +#endif + + // destroy object memset(cond, -1, sizeof(*cond)); return 0; } diff --git a/libc/thread/pthread_cond_init.c b/libc/thread/pthread_cond_init.c index 915e6bc6f..8f6fbe298 100644 --- a/libc/thread/pthread_cond_init.c +++ b/libc/thread/pthread_cond_init.c @@ -27,5 +27,7 @@ errno_t pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr) { *cond = (pthread_cond_t){0}; + if (attr) + cond->_pshared = *attr; return 0; } diff --git a/libc/thread/pthread_cond_signal.c b/libc/thread/pthread_cond_signal.c index e490bb402..e2a615df0 100644 --- a/libc/thread/pthread_cond_signal.c +++ b/libc/thread/pthread_cond_signal.c @@ -16,8 +16,10 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/atomic.h" #include "libc/thread/thread.h" #include "third_party/nsync/cv.h" +#include "third_party/nsync/futex.internal.h" /** * Wakes at least one thread waiting on condition, e.g. @@ -37,6 +39,19 @@ * @see pthread_cond_wait */ errno_t pthread_cond_signal(pthread_cond_t *cond) { - nsync_cv_signal((nsync_cv *)cond); + +#if PTHREAD_USE_NSYNC + // favor *NSYNC if this is a process private condition variable + // if using Mike Burrows' code isn't possible, use a naive impl + if (!cond->_pshared) { + nsync_cv_signal((nsync_cv *)cond); + return 0; + } +#endif + + // roll forward the monotonic sequence + atomic_fetch_add_explicit(&cond->_sequence, 1, memory_order_acq_rel); + if (atomic_load_explicit(&cond->_waiters, memory_order_acquire)) + nsync_futex_wake_((atomic_int *)&cond->_sequence, 1, cond->_pshared); return 0; } diff --git a/libc/thread/pthread_cond_timedwait.c b/libc/thread/pthread_cond_timedwait.c index 6d964449b..84e5fe7ef 100644 --- a/libc/thread/pthread_cond_timedwait.c +++ b/libc/thread/pthread_cond_timedwait.c @@ -16,14 +16,61 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" +#include "libc/calls/cp.internal.h" #include "libc/errno.h" #include "libc/thread/lock.h" +#include "libc/thread/posixthread.internal.h" #include "libc/thread/thread.h" #include "libc/thread/thread2.h" #include "third_party/nsync/common.internal.h" #include "third_party/nsync/cv.h" +#include "third_party/nsync/futex.internal.h" #include "third_party/nsync/time.h" +struct PthreadWait { + pthread_cond_t *cond; + pthread_mutex_t *mutex; +}; + +static void pthread_cond_leave(void *arg) { + struct PthreadWait *wait = (struct PthreadWait *)arg; + if (pthread_mutex_lock(wait->mutex)) + __builtin_trap(); + atomic_fetch_sub_explicit(&wait->cond->_waiters, 1, memory_order_acq_rel); +} + +static errno_t pthread_cond_timedwait_impl(pthread_cond_t *cond, + pthread_mutex_t *mutex, + const struct timespec *abstime) { + + // this is a cancelation point + // check the cancelation status before we begin waiting + if (pthread_testcancel_np() == ECANCELED) + return ECANCELED; + + // get original monotonic sequence while lock is held + uint32_t seq1 = atomic_load_explicit(&cond->_sequence, memory_order_relaxed); + + // start waiting on condition variable + atomic_fetch_add_explicit(&cond->_waiters, 1, memory_order_acq_rel); + if (pthread_mutex_unlock(mutex)) + __builtin_trap(); + + // wait for sequence change, timeout, or cancelation + int rc; + struct PthreadWait waiter = {cond, mutex}; + pthread_cleanup_push(pthread_cond_leave, &waiter); + rc = nsync_futex_wait_((atomic_int *)&cond->_sequence, seq1, cond->_pshared, + abstime); + pthread_cleanup_pop(true); + if (rc == -EAGAIN) + rc = 0; + + // turn linux syscall status into posix errno + return -rc; +} + /** * Waits for condition with optional time limit, e.g. * @@ -49,11 +96,39 @@ */ errno_t pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, const struct timespec *abstime) { + + // validate arguments + struct PosixThread *pt; + if (!(pt = _pthread_self())) + return EINVAL; if (abstime && !(0 <= abstime->tv_nsec && abstime->tv_nsec < 1000000000)) return EINVAL; - if (MUTEX_TYPE(mutex->_word) != PTHREAD_MUTEX_NORMAL) - nsync_panic_("pthread cond needs normal mutex\n"); - return nsync_cv_wait_with_deadline( - (nsync_cv *)cond, (nsync_mu *)mutex, - abstime ? *abstime : nsync_time_no_deadline, 0); + + // look at the mutex argument + uint64_t muword = atomic_load_explicit(&mutex->_word, memory_order_relaxed); + + // check that mutex is held by caller + if (MUTEX_TYPE(muword) == PTHREAD_MUTEX_ERRORCHECK && + MUTEX_OWNER(muword) != gettid()) + return EPERM; + + // if condition variable is shared then mutex must be too + if (cond->_pshared) + if (MUTEX_PSHARED(muword) != PTHREAD_PROCESS_SHARED) + return EINVAL; + +#if PTHREAD_USE_NSYNC + // favor *NSYNC if this is a process private condition variable + // if using Mike Burrows' code isn't possible, use a naive impl + if (!cond->_pshared) + return nsync_cv_wait_with_deadline( + (nsync_cv *)cond, (nsync_mu *)mutex, + abstime ? *abstime : nsync_time_no_deadline, 0); +#endif + + errno_t err; + BEGIN_CANCELATION_POINT; + err = pthread_cond_timedwait_impl(cond, mutex, abstime); + END_CANCELATION_POINT; + return err; } diff --git a/libc/thread/sem_open.c b/libc/thread/sem_open.c index 9a3e5b2ce..d708ef7e4 100644 --- a/libc/thread/sem_open.c +++ b/libc/thread/sem_open.c @@ -183,7 +183,8 @@ sem_t *sem_open(const char *name, int oflag, ...) { #if 0 if (IsXnuSilicon()) { long kernel; - if (!(sem = calloc(1, sizeof(sem_t)))) return SEM_FAILED; + if (!(sem = calloc(1, sizeof(sem_t)))) + return SEM_FAILED; sem->sem_magic = SEM_MAGIC_KERNEL; kernel = _sysret(__syslib->__sem_open(name, oflag, mode, value)); if (kernel == -1) { diff --git a/libc/thread/sem_timedwait.c b/libc/thread/sem_timedwait.c index 2ffcbf5ee..bd2e5d9d9 100644 --- a/libc/thread/sem_timedwait.c +++ b/libc/thread/sem_timedwait.c @@ -81,13 +81,15 @@ int sem_timedwait(sem_t *sem, const struct timespec *abstime) { return ecanceled(); } rc = _sysret(__syslib->__sem_trywait(sem->sem_kernel)); - if (!rc) return 0; + if (!rc) + return 0; if (errno == EINTR && // _weaken(pthread_testcancel_np) && // _weaken(pthread_testcancel_np)()) { return ecanceled(); } - if (errno != EAGAIN) return -1; + if (errno != EAGAIN) + return -1; errno = e; struct timespec now = timespec_real(); if (timespec_cmp(*abstime, now) >= 0) { diff --git a/libc/thread/thread.h b/libc/thread/thread.h index af8ecd60c..406c19ff8 100644 --- a/libc/thread/thread.h +++ b/libc/thread/thread.h @@ -3,6 +3,7 @@ #define PTHREAD_KEYS_MAX 46 #define PTHREAD_STACK_MIN 65536 +#define PTHREAD_USE_NSYNC 1 #define PTHREAD_DESTRUCTOR_ITERATIONS 4 #define PTHREAD_BARRIER_SERIAL_THREAD 31337 @@ -74,7 +75,15 @@ typedef struct pthread_mutexattr_s { } pthread_mutexattr_t; typedef struct pthread_cond_s { - void *_nsync[2]; + union { + void *_align; + struct { + uint32_t _nsync; + char _pshared; + }; + }; + _Atomic(uint32_t) _sequence; + _Atomic(uint32_t) _waiters; } pthread_cond_t; typedef struct pthread_rwlock_s { diff --git a/test/posix/mutex_async_signal_safety_test.c b/test/posix/mutex_async_signal_safety_test.c index d9da7b6bc..5102ab2fb 100644 --- a/test/posix/mutex_async_signal_safety_test.c +++ b/test/posix/mutex_async_signal_safety_test.c @@ -66,7 +66,7 @@ int main() { if (ready) break; - for (int i = 0; i < 1000; ++i) { + for (int i = 0; i < 100; ++i) { if (pthread_kill(th, SIGUSR1)) _Exit(11); if (pthread_kill(th, SIGUSR1))