From 3c61a541bd510e0121fc23020c9a66870f86d5cc Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Mon, 2 Sep 2024 23:37:50 -0700 Subject: [PATCH] Introduce pthread_condattr_setclock() This is one of the few POSIX APIs that was missing. It lets you choose a monotonic clock for your condition variables. This might improve perf on some platforms. It might also grant more flexibility with NTP configs. I know Qt is one project that believes it needs this. To introduce this, I needed to change some the *NSYNC APIs, to support passing a clock param. There's also new benchmarks, demonstrating Cosmopolitan's supremacy over many libc implementations when it comes to mutex performance. Cygwin has an alarmingly bad pthread_mutex_t implementation. It is so bad that they would have been significantly better off if they'd used naive spinlocks. --- libc/calls/clock_gettime.c | 32 ++++++- libc/calls/clock_nanosleep.c | 27 ++++-- libc/intrin/maps.h | 1 + libc/intrin/mmap.c | 31 +++++-- libc/intrin/pthread_mutex_lock.c | 6 +- libc/intrin/sys_umtx_timedwait_uint.c | 4 +- libc/proc/proc.c | 9 +- libc/sysv/consts.sh | 3 +- libc/sysv/consts/CLOCK_MONOTONIC_RAW.S | 2 +- libc/sysv/consts/CLOCK_MONOTONIC_RAW_APPROX.S | 2 + libc/sysv/consts/clock.h | 19 +++- libc/thread/freebsd.internal.h | 3 +- libc/thread/itimer.c | 17 ++-- libc/thread/pthread_barrier_wait.c | 2 +- libc/thread/pthread_cond_init.c | 9 +- libc/thread/pthread_cond_timedwait.c | 11 ++- libc/thread/pthread_condattr_getclock.c | 32 +++++++ libc/thread/pthread_condattr_getpshared.c | 2 +- libc/thread/pthread_condattr_init.c | 2 +- libc/thread/pthread_condattr_setclock.c | 38 ++++++++ libc/thread/pthread_condattr_setpshared.c | 2 +- libc/thread/pthread_timedjoin_np.c | 4 +- libc/thread/sem_timedwait.c | 3 +- libc/thread/thread.h | 13 ++- test/libc/thread/footek_test.c | 28 ++++-- test/libc/thread/nsync_test.c | 7 +- .../libc/thread/pthread_cond_timedwait_test.c | 64 ++++++++++++++ test/libc/thread/setitimer_test.c | 1 - third_party/lua/lunix.c | 3 +- third_party/nsync/README.cosmo | 2 + third_party/nsync/common.internal.h | 2 +- third_party/nsync/cv.h | 4 +- third_party/nsync/futex.c | 88 +++++++++---------- third_party/nsync/futex.internal.h | 2 +- third_party/nsync/mem/nsync_counter.c | 3 +- third_party/nsync/mem/nsync_cv.c | 12 +-- third_party/nsync/mem/nsync_mu_wait.c | 6 +- third_party/nsync/mem/nsync_note.c | 3 +- third_party/nsync/mem/nsync_once.c | 3 +- third_party/nsync/mem/nsync_sem_wait.c | 6 +- third_party/nsync/mem/nsync_wait.c | 4 +- third_party/nsync/mu_semaphore.c | 8 +- third_party/nsync/mu_semaphore.h | 2 +- third_party/nsync/mu_semaphore.internal.h | 6 +- third_party/nsync/mu_semaphore_futex.c | 16 ++-- third_party/nsync/mu_semaphore_gcd.c | 9 +- third_party/nsync/mu_semaphore_sem.c | 15 +++- third_party/nsync/mu_wait.h | 2 +- .../testing/cv_mu_timeout_stress_test_.c | 6 +- third_party/nsync/testing/cv_test.c | 11 +-- .../nsync/testing/cv_wait_example_test.c | 4 +- third_party/nsync/testing/mu_test.c | 4 +- third_party/nsync/testing/pingpong_test.c | 5 +- third_party/nsync/waiter.h | 2 +- third_party/openmp/kmp_lock.cpp | 2 +- 55 files changed, 449 insertions(+), 155 deletions(-) create mode 100644 libc/sysv/consts/CLOCK_MONOTONIC_RAW_APPROX.S create mode 100644 libc/thread/pthread_condattr_getclock.c create mode 100644 libc/thread/pthread_condattr_setclock.c create mode 100644 test/libc/thread/pthread_cond_timedwait_test.c diff --git a/libc/calls/clock_gettime.c b/libc/calls/clock_gettime.c index 2aac56c98..5d9d150f5 100644 --- a/libc/calls/clock_gettime.c +++ b/libc/calls/clock_gettime.c @@ -24,6 +24,7 @@ #include "libc/intrin/describeflags.h" #include "libc/intrin/strace.h" #include "libc/runtime/syslib.internal.h" +#include "libc/sysv/consts/clock.h" #ifdef __aarch64__ #define CGT_VDSO __vdsosym("LINUX_2.6.39", "__kernel_clock_gettime") @@ -58,14 +59,43 @@ static int __clock_gettime_init(int clockid, struct timespec *ts) { return cgt(clockid, ts); } +static int clock_gettime_impl(int clock, struct timespec *ts) { + int rc; + if (!IsLinux()) + return __clock_gettime(clock, ts); +TryAgain: + + // Ensure fallback for old Linux sticks. + if (clock == 4 /* CLOCK_MONOTONIC_RAW */) + clock = CLOCK_MONOTONIC_RAW; + + // Call appropriate implementation. + rc = __clock_gettime(clock, ts); + + // CLOCK_MONOTONIC_RAW is Linux 2.6.28+ so not available on RHEL5 + if (rc == -EINVAL && clock == 4 /* CLOCK_MONOTONIC_RAW */) { + CLOCK_MONOTONIC_RAW = CLOCK_MONOTONIC; + CLOCK_MONOTONIC_RAW_APPROX = CLOCK_MONOTONIC; + goto TryAgain; + } + + return rc; +} + /** * Returns nanosecond time. * * @param clock supports the following values across OSes: * - `CLOCK_REALTIME` * - `CLOCK_MONOTONIC` + * - `CLOCK_MONOTONIC_RAW` + * - `CLOCK_MONOTONIC_RAW_APPROX` + * - `CLOCK_REALTIME_FAST` * - `CLOCK_REALTIME_COARSE` + * - `CLOCK_REALTIME_PRECISE` + * - `CLOCK_MONOTONIC_FAST` * - `CLOCK_MONOTONIC_COARSE` + * - `CLOCK_MONOTONIC_PRECISE` * - `CLOCK_THREAD_CPUTIME_ID` * - `CLOCK_PROCESS_CPUTIME_ID` * @param ts is where the result is stored (or null to do clock check) @@ -80,7 +110,7 @@ static int __clock_gettime_init(int clockid, struct timespec *ts) { */ int clock_gettime(int clock, struct timespec *ts) { // threads on win32 stacks call this so we can't asan check *ts - int rc = __clock_gettime(clock, ts); + int rc = clock_gettime_impl(clock, ts); if (rc) { errno = -rc; rc = -1; diff --git a/libc/calls/clock_nanosleep.c b/libc/calls/clock_nanosleep.c index 20a6b03ee..d912ce41c 100644 --- a/libc/calls/clock_nanosleep.c +++ b/libc/calls/clock_nanosleep.c @@ -19,6 +19,7 @@ #include "libc/calls/struct/timespec.h" #include "libc/dce.h" #include "libc/errno.h" +#include "libc/sysv/consts/clock.h" #include "libc/sysv/consts/timer.h" /** @@ -79,18 +80,32 @@ errno_t clock_nanosleep(int clock, int flags, // const struct timespec *req, // struct timespec *rem) { - if (IsMetal()) { + if (IsMetal()) return ENOSYS; - } if (clock == 127 || // (flags & ~TIMER_ABSTIME) || // req->tv_sec < 0 || // - !(0 <= req->tv_nsec && req->tv_nsec <= 999999999)) { + !(0 <= req->tv_nsec && req->tv_nsec <= 999999999)) return EINVAL; + int rc; + errno_t err, old = errno; + +TryAgain: + // Ensure fallback for old Linux sticks. + if (IsLinux() && clock == 4 /* CLOCK_MONOTONIC_RAW */) + clock = CLOCK_MONOTONIC_RAW; + + rc = sys_clock_nanosleep(clock, flags, req, rem); + + // CLOCK_MONOTONIC_RAW is Linux 2.6.28+ so not available on RHEL5 + if (IsLinux() && rc && errno == EINVAL && + clock == 4 /* CLOCK_MONOTONIC_RAW */) { + CLOCK_MONOTONIC_RAW = CLOCK_MONOTONIC; + CLOCK_MONOTONIC_RAW_APPROX = CLOCK_MONOTONIC; + goto TryAgain; } - errno_t old = errno; - int rc = sys_clock_nanosleep(clock, flags, req, rem); - errno_t err = !rc ? 0 : errno; + + err = !rc ? 0 : errno; errno = old; return err; } diff --git a/libc/intrin/maps.h b/libc/intrin/maps.h index c0b0a911d..33048ca03 100644 --- a/libc/intrin/maps.h +++ b/libc/intrin/maps.h @@ -53,6 +53,7 @@ void *__maps_pickaddr(size_t); void __maps_add(struct Map *); void __maps_free(struct Map *); void __maps_insert(struct Map *); +bool __maps_track(char *, size_t); struct Map *__maps_alloc(void); struct Map *__maps_floor(const char *); void __maps_stack(char *, int, int, size_t, int, intptr_t); diff --git a/libc/intrin/mmap.c b/libc/intrin/mmap.c index 8be86f963..64a4c49c6 100644 --- a/libc/intrin/mmap.c +++ b/libc/intrin/mmap.c @@ -305,6 +305,28 @@ void __maps_insert(struct Map *map) { __maps_check(); } +static void __maps_track_insert(struct Map *map, char *addr, size_t size, + uintptr_t map_handle) { + map->addr = addr; + map->size = size; + map->prot = PROT_READ | PROT_WRITE; + map->flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NOFORK; + map->hand = map_handle; + __maps_lock(); + __maps_insert(map); + __maps_unlock(); +} + +bool __maps_track(char *addr, size_t size) { + struct Map *map; + do { + if (!(map = __maps_alloc())) + return false; + } while (map == MAPS_RETRY); + __maps_track_insert(map, addr, size, -1); + return true; +} + struct Map *__maps_alloc(void) { struct Map *map; uintptr_t tip = atomic_load_explicit(&__maps.freed, memory_order_relaxed); @@ -321,14 +343,7 @@ struct Map *__maps_alloc(void) { if (sys.addr == MAP_FAILED) return 0; map = sys.addr; - map->addr = sys.addr; - map->size = gransz; - map->prot = PROT_READ | PROT_WRITE; - map->flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NOFORK; - map->hand = sys.maphandle; - __maps_lock(); - __maps_insert(map); - __maps_unlock(); + __maps_track_insert(map, sys.addr, gransz, sys.maphandle); for (int i = 1; i < gransz / sizeof(struct Map); ++i) __maps_free(map + i); return MAPS_RETRY; diff --git a/libc/intrin/pthread_mutex_lock.c b/libc/intrin/pthread_mutex_lock.c index c1d0459ff..a71202200 100644 --- a/libc/intrin/pthread_mutex_lock.c +++ b/libc/intrin/pthread_mutex_lock.c @@ -57,12 +57,12 @@ static void pthread_mutex_lock_drepper(atomic_int *futex, char pshare) { LOCKTRACE("acquiring pthread_mutex_lock_drepper(%t)...", futex); if (word == 1) word = atomic_exchange_explicit(futex, 2, memory_order_acquire); + BLOCK_CANCELATION; while (word > 0) { - BLOCK_CANCELATION; - _weaken(nsync_futex_wait_)(futex, 2, pshare, 0); - ALLOW_CANCELATION; + _weaken(nsync_futex_wait_)(futex, 2, pshare, 0, 0); word = atomic_exchange_explicit(futex, 2, memory_order_acquire); } + ALLOW_CANCELATION; } static errno_t pthread_mutex_lock_recursive(pthread_mutex_t *mutex, diff --git a/libc/intrin/sys_umtx_timedwait_uint.c b/libc/intrin/sys_umtx_timedwait_uint.c index 9bcc7c595..f82c2898c 100644 --- a/libc/intrin/sys_umtx_timedwait_uint.c +++ b/libc/intrin/sys_umtx_timedwait_uint.c @@ -23,7 +23,7 @@ int sys_umtx_timedwait_uint_cp(atomic_int *, int, int, size_t, struct _umtx_time *) asm("sys_futex_cp"); -int sys_umtx_timedwait_uint(atomic_int *p, int expect, bool pshare, +int sys_umtx_timedwait_uint(atomic_int *p, int expect, bool pshare, int clock, const struct timespec *abstime) { int op; size_t size; @@ -32,7 +32,7 @@ int sys_umtx_timedwait_uint(atomic_int *p, int expect, bool pshare, tm_p = 0; size = 0; } else { - timo._clockid = CLOCK_REALTIME; + timo._clockid = clock; timo._flags = UMTX_ABSTIME; timo._timeout = *abstime; tm_p = &timo; diff --git a/libc/proc/proc.c b/libc/proc/proc.c index 4814c1e43..bdada4c46 100644 --- a/libc/proc/proc.c +++ b/libc/proc/proc.c @@ -27,6 +27,7 @@ #include "libc/errno.h" #include "libc/fmt/wintime.internal.h" #include "libc/intrin/dll.h" +#include "libc/intrin/maps.h" #include "libc/intrin/strace.h" #include "libc/intrin/weaken.h" #include "libc/mem/leaks.h" @@ -60,6 +61,8 @@ * @fileoverview Windows Subprocess Management. */ +#define STACK_SIZE 65536 + struct Procs __proc; static textwindows void __proc_stats(int64_t h, struct rusage *ru) { @@ -130,7 +133,11 @@ textwindows int __proc_harvest(struct Proc *pr, bool iswait4) { static textwindows dontinstrument uint32_t __proc_worker(void *arg) { struct CosmoTib tls; + char *sp = __builtin_frame_address(0); __bootstrap_tls(&tls, __builtin_frame_address(0)); + __maps_track( + (char *)(((uintptr_t)sp + __pagesize - 1) & -__pagesize) - STACK_SIZE, + STACK_SIZE); for (;;) { // assemble a group of processes to wait on. if more than 64 @@ -238,7 +245,7 @@ static textwindows dontinstrument uint32_t __proc_worker(void *arg) { static textwindows void __proc_setup(void) { __proc.onbirth = CreateEvent(0, 0, 0, 0); // auto reset __proc.haszombies = CreateEvent(0, 1, 0, 0); // manual reset - __proc.thread = CreateThread(0, 65536, __proc_worker, 0, + __proc.thread = CreateThread(0, STACK_SIZE, __proc_worker, 0, kNtStackSizeParamIsAReservation, 0); } diff --git a/libc/sysv/consts.sh b/libc/sysv/consts.sh index b6ff0405a..9729faccc 100755 --- a/libc/sysv/consts.sh +++ b/libc/sysv/consts.sh @@ -577,10 +577,11 @@ syscon clock CLOCK_REALTIME_PRECISE 0 0 0 0 9 0 0 0 # syscon clock CLOCK_REALTIME_FAST 0 0 0 0 10 0 0 0 # syscon clock CLOCK_REALTIME_COARSE 5 5 0 0 10 0 0 2 # Linux 2.6.32+; bsd consensus; not available on RHEL5 syscon clock CLOCK_MONOTONIC 1 1 6 6 4 3 3 1 # XNU/NT faked; could move backwards if NTP introduces negative leap second +syscon clock CLOCK_MONOTONIC_RAW 4 4 4 4 4 3 3 1 # actually monotonic; not subject to NTP adjustments; Linux 2.6.28+; XNU/NT/FreeBSD/OpenBSD faked; not available on RHEL5 (will fallback to CLOCK_MONOTONIC) +syscon clock CLOCK_MONOTONIC_RAW_APPROX 4 4 5 5 4 3 3 1 # goes faster on xnu, otherwise faked syscon clock CLOCK_MONOTONIC_PRECISE 1 1 6 6 11 3 3 1 # syscon clock CLOCK_MONOTONIC_FAST 1 1 6 6 12 3 3 1 # syscon clock CLOCK_MONOTONIC_COARSE 6 6 5 5 12 3 3 1 # Linux 2.6.32+; bsd consensus; not available on RHEL5 -syscon clock CLOCK_MONOTONIC_RAW 4 4 4 4 127 127 127 127 # actually monotonic; not subject to NTP adjustments; Linux 2.6.28+; XNU/NT/FreeBSD/OpenBSD faked; not available on RHEL5 syscon clock CLOCK_PROCESS_CPUTIME_ID 2 2 12 12 15 2 0x40000000 4 # NetBSD lets you bitwise a PID into clockid_t syscon clock CLOCK_THREAD_CPUTIME_ID 3 3 16 16 14 4 0x20000000 5 # syscon clock CLOCK_PROF 127 127 127 127 2 127 2 127 # diff --git a/libc/sysv/consts/CLOCK_MONOTONIC_RAW.S b/libc/sysv/consts/CLOCK_MONOTONIC_RAW.S index 1c158565e..133098e2f 100644 --- a/libc/sysv/consts/CLOCK_MONOTONIC_RAW.S +++ b/libc/sysv/consts/CLOCK_MONOTONIC_RAW.S @@ -1,2 +1,2 @@ #include "libc/sysv/consts/syscon.internal.h" -.syscon clock,CLOCK_MONOTONIC_RAW,4,4,4,4,127,127,127,127 +.syscon clock,CLOCK_MONOTONIC_RAW,4,4,4,4,4,3,3,1 diff --git a/libc/sysv/consts/CLOCK_MONOTONIC_RAW_APPROX.S b/libc/sysv/consts/CLOCK_MONOTONIC_RAW_APPROX.S new file mode 100644 index 000000000..5f81cd8bf --- /dev/null +++ b/libc/sysv/consts/CLOCK_MONOTONIC_RAW_APPROX.S @@ -0,0 +1,2 @@ +#include "libc/sysv/consts/syscon.internal.h" +.syscon clock,CLOCK_MONOTONIC_RAW_APPROX,4,4,5,5,4,3,3,1 diff --git a/libc/sysv/consts/clock.h b/libc/sysv/consts/clock.h index 4b6e7b193..8aa3b7b81 100644 --- a/libc/sysv/consts/clock.h +++ b/libc/sysv/consts/clock.h @@ -8,7 +8,8 @@ extern const int CLOCK_MONOTONIC; extern const int CLOCK_MONOTONIC_COARSE; extern const int CLOCK_MONOTONIC_FAST; extern const int CLOCK_MONOTONIC_PRECISE; -extern const int CLOCK_MONOTONIC_RAW; +extern int CLOCK_MONOTONIC_RAW; +extern int CLOCK_MONOTONIC_RAW_APPROX; extern const int CLOCK_PROCESS_CPUTIME_ID; extern const int CLOCK_PROF; extern const int CLOCK_REALTIME_ALARM; @@ -24,9 +25,19 @@ extern const int CLOCK_UPTIME_PRECISE; COSMOPOLITAN_C_END_ -#define CLOCK_REALTIME 0 -#define CLOCK_MONOTONIC CLOCK_MONOTONIC -#define CLOCK_PROCESS_CPUTIME_ID CLOCK_PROCESS_CPUTIME_ID +#define CLOCK_REALTIME 0 +#define CLOCK_REALTIME_FAST CLOCK_REALTIME_FAST +#define CLOCK_REALTIME_PRECISE CLOCK_REALTIME_PRECISE +#define CLOCK_REALTIME_COARSE CLOCK_REALTIME_COARSE + +#define CLOCK_MONOTONIC CLOCK_MONOTONIC +#define CLOCK_MONOTONIC_RAW CLOCK_MONOTONIC_RAW +#define CLOCK_MONOTONIC_RAW_APPROX CLOCK_MONOTONIC_RAW_APPROX +#define CLOCK_MONOTONIC_FAST CLOCK_MONOTONIC_FAST +#define CLOCK_MONOTONIC_PRECISE CLOCK_MONOTONIC_PRECISE +#define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC_COARSE + #define CLOCK_THREAD_CPUTIME_ID CLOCK_THREAD_CPUTIME_ID +#define CLOCK_PROCESS_CPUTIME_ID CLOCK_PROCESS_CPUTIME_ID #endif /* COSMOPOLITAN_LIBC_SYSV_CONSTS_CLOCK_H_ */ diff --git a/libc/thread/freebsd.internal.h b/libc/thread/freebsd.internal.h index 97a7c9e06..f7fec2ed5 100644 --- a/libc/thread/freebsd.internal.h +++ b/libc/thread/freebsd.internal.h @@ -47,7 +47,8 @@ struct _umtx_time { uint32_t _clockid; }; -int sys_umtx_timedwait_uint(_Atomic(int) *, int, bool, const struct timespec *); +int sys_umtx_timedwait_uint(_Atomic(int) *, int, bool, int, + const struct timespec *); COSMOPOLITAN_C_END_ #endif /* COSMOPOLITAN_LIBC_THREAD_FREEBSD_INTERNAL_H_ */ diff --git a/libc/thread/itimer.c b/libc/thread/itimer.c index 91a55580a..15db1893d 100644 --- a/libc/thread/itimer.c +++ b/libc/thread/itimer.c @@ -24,10 +24,12 @@ #include "libc/calls/struct/sigset.internal.h" #include "libc/calls/struct/timeval.h" #include "libc/cosmo.h" +#include "libc/intrin/maps.h" #include "libc/intrin/strace.h" #include "libc/nt/enum/processcreationflags.h" #include "libc/nt/thread.h" #include "libc/str/str.h" +#include "libc/sysv/consts/clock.h" #include "libc/sysv/consts/sicode.h" #include "libc/sysv/consts/sig.h" #include "libc/sysv/errfuns.h" @@ -36,11 +38,17 @@ #include "third_party/nsync/mu.h" #ifdef __x86_64__ +#define STACK_SIZE 65536 + struct IntervalTimer __itimer; static textwindows dontinstrument uint32_t __itimer_worker(void *arg) { struct CosmoTib tls; - __bootstrap_tls(&tls, __builtin_frame_address(0)); + char *sp = __builtin_frame_address(0); + __bootstrap_tls(&tls, sp); + __maps_track( + (char *)(((uintptr_t)sp + __pagesize - 1) & -__pagesize) - STACK_SIZE, + STACK_SIZE); for (;;) { bool dosignal = false; struct timeval now, waituntil; @@ -66,11 +74,10 @@ static textwindows dontinstrument uint32_t __itimer_worker(void *arg) { } } nsync_mu_unlock(&__itimer.lock); - if (dosignal) { + if (dosignal) __sig_generate(SIGALRM, SI_TIMER); - } nsync_mu_lock(&__itimer.lock); - nsync_cv_wait_with_deadline(&__itimer.cond, &__itimer.lock, + nsync_cv_wait_with_deadline(&__itimer.cond, &__itimer.lock, CLOCK_REALTIME, timeval_totimespec(waituntil), 0); nsync_mu_unlock(&__itimer.lock); } @@ -78,7 +85,7 @@ static textwindows dontinstrument uint32_t __itimer_worker(void *arg) { } static textwindows void __itimer_setup(void) { - __itimer.thread = CreateThread(0, 65536, __itimer_worker, 0, + __itimer.thread = CreateThread(0, STACK_SIZE, __itimer_worker, 0, kNtStackSizeParamIsAReservation, 0); } diff --git a/libc/thread/pthread_barrier_wait.c b/libc/thread/pthread_barrier_wait.c index 27f64773e..a4d44bf5e 100644 --- a/libc/thread/pthread_barrier_wait.c +++ b/libc/thread/pthread_barrier_wait.c @@ -61,7 +61,7 @@ errno_t pthread_barrier_wait(pthread_barrier_t *barrier) { // wait for everyone else to arrive at barrier BLOCK_CANCELATION; while ((n = atomic_load_explicit(&barrier->_waiters, memory_order_acquire))) - nsync_futex_wait_(&barrier->_waiters, n, barrier->_pshared, 0); + nsync_futex_wait_(&barrier->_waiters, n, barrier->_pshared, 0, 0); ALLOW_CANCELATION; return 0; diff --git a/libc/thread/pthread_cond_init.c b/libc/thread/pthread_cond_init.c index 8f6fbe298..150ef78a4 100644 --- a/libc/thread/pthread_cond_init.c +++ b/libc/thread/pthread_cond_init.c @@ -16,10 +16,11 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/sysv/consts/clock.h" #include "libc/thread/thread.h" /** - * Initializes condition. + * Initializes condition variable. * * @param attr may be null * @return 0 on success, or error number on failure @@ -27,7 +28,9 @@ errno_t pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr) { *cond = (pthread_cond_t){0}; - if (attr) - cond->_pshared = *attr; + if (attr) { + cond->_pshared = attr->_pshared; + cond->_clock = attr->_clock; + } return 0; } diff --git a/libc/thread/pthread_cond_timedwait.c b/libc/thread/pthread_cond_timedwait.c index 53c33cf5a..96f0cf8a5 100644 --- a/libc/thread/pthread_cond_timedwait.c +++ b/libc/thread/pthread_cond_timedwait.c @@ -20,6 +20,7 @@ #include "libc/calls/cp.internal.h" #include "libc/dce.h" #include "libc/errno.h" +#include "libc/sysv/consts/clock.h" #include "libc/thread/lock.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/thread.h" @@ -63,7 +64,7 @@ static errno_t pthread_cond_timedwait_impl(pthread_cond_t *cond, struct PthreadWait waiter = {cond, mutex}; pthread_cleanup_push(pthread_cond_leave, &waiter); rc = nsync_futex_wait_((atomic_int *)&cond->_sequence, seq1, cond->_pshared, - abstime); + cond->_clock, abstime); pthread_cleanup_pop(true); if (rc == -EAGAIN) rc = 0; @@ -82,8 +83,10 @@ static errno_t pthread_cond_timedwait_impl(pthread_cond_t *cond, * } * * @param mutex needs to be held by thread when calling this function - * @param abstime may be null to wait indefinitely and should contain - * some arbitrary interval added to a `CLOCK_REALTIME` timestamp + * @param abstime is an absolute timestamp, which may be null to wait + * forever; it's relative to `clock_gettime(CLOCK_REALTIME)` by + * default; pthread_condattr_setclock() may be used to customize + * which system clock is used * @return 0 on success, or errno on error * @raise ETIMEDOUT if `abstime` was specified and the current time * exceeded its value @@ -125,7 +128,7 @@ errno_t pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, // if using Mike Burrows' code isn't possible, use a naive impl if (!cond->_pshared && !IsXnuSilicon()) { err = nsync_cv_wait_with_deadline( - (nsync_cv *)cond, (nsync_mu *)mutex, + (nsync_cv *)cond, (nsync_mu *)mutex, cond->_clock, abstime ? *abstime : nsync_time_no_deadline, 0); } else { err = pthread_cond_timedwait_impl(cond, mutex, abstime); diff --git a/libc/thread/pthread_condattr_getclock.c b/libc/thread/pthread_condattr_getclock.c new file mode 100644 index 000000000..8e0616e9c --- /dev/null +++ b/libc/thread/pthread_condattr_getclock.c @@ -0,0 +1,32 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/thread/thread.h" + +/** + * Gets clock on condition variable attributes. + * + * @param clock will be set to one of + * - `CLOCK_REALTIME` (default) + * - `CLOCK_MONOTONIC` + * @return 0 on success, or error on failure + */ +int pthread_condattr_getclock(const pthread_condattr_t *attr, int *clock) { + *clock = attr->_clock; + return 0; +} diff --git a/libc/thread/pthread_condattr_getpshared.c b/libc/thread/pthread_condattr_getpshared.c index 81963c4c3..e1b5dc7fc 100644 --- a/libc/thread/pthread_condattr_getpshared.c +++ b/libc/thread/pthread_condattr_getpshared.c @@ -28,6 +28,6 @@ */ errno_t pthread_condattr_getpshared(const pthread_condattr_t *attr, int *pshared) { - *pshared = *attr; + *pshared = attr->_pshared; return 0; } diff --git a/libc/thread/pthread_condattr_init.c b/libc/thread/pthread_condattr_init.c index 575912210..cd377f087 100644 --- a/libc/thread/pthread_condattr_init.c +++ b/libc/thread/pthread_condattr_init.c @@ -24,6 +24,6 @@ * @return 0 on success, or error on failure */ errno_t pthread_condattr_init(pthread_condattr_t *attr) { - *attr = 0; + *attr = (pthread_condattr_t){0}; return 0; } diff --git a/libc/thread/pthread_condattr_setclock.c b/libc/thread/pthread_condattr_setclock.c new file mode 100644 index 000000000..232267ef5 --- /dev/null +++ b/libc/thread/pthread_condattr_setclock.c @@ -0,0 +1,38 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/errno.h" +#include "libc/sysv/consts/clock.h" +#include "libc/thread/thread.h" + +/** + * Sets clock for condition variable. + * + * @param clock can be one of + * - `CLOCK_REALTIME` (default) + * - `CLOCK_MONOTONIC` + * @return 0 on success, or error on failure + * @raises EINVAL if `clock` is invalid + */ +int pthread_condattr_setclock(pthread_condattr_t *attr, int clock) { + if (clock != CLOCK_REALTIME && // + clock != CLOCK_MONOTONIC) + return EINVAL; + attr->_clock = clock; + return 0; +} diff --git a/libc/thread/pthread_condattr_setpshared.c b/libc/thread/pthread_condattr_setpshared.c index 8d59b2fa7..f109fe518 100644 --- a/libc/thread/pthread_condattr_setpshared.c +++ b/libc/thread/pthread_condattr_setpshared.c @@ -32,7 +32,7 @@ errno_t pthread_condattr_setpshared(pthread_condattr_t *attr, int pshared) { switch (pshared) { case PTHREAD_PROCESS_SHARED: case PTHREAD_PROCESS_PRIVATE: - *attr = pshared; + attr->_pshared = pshared; return 0; default: return EINVAL; diff --git a/libc/thread/pthread_timedjoin_np.c b/libc/thread/pthread_timedjoin_np.c index b6b5c2e8a..2ba356567 100644 --- a/libc/thread/pthread_timedjoin_np.c +++ b/libc/thread/pthread_timedjoin_np.c @@ -26,6 +26,7 @@ #include "libc/intrin/describeflags.h" #include "libc/intrin/dll.h" #include "libc/intrin/strace.h" +#include "libc/sysv/consts/clock.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/thread2.h" #include "libc/thread/tls.h" @@ -74,7 +75,8 @@ static errno_t _pthread_wait(atomic_int *ctid, struct timespec *abstime) { if (!(err = pthread_testcancel_np())) { BEGIN_CANCELATION_POINT; while ((x = atomic_load_explicit(ctid, memory_order_acquire))) { - e = nsync_futex_wait_(ctid, x, !IsWindows() && !IsXnu(), abstime); + e = nsync_futex_wait_(ctid, x, !IsWindows() && !IsXnu(), CLOCK_REALTIME, + abstime); if (e == -ECANCELED) { err = ECANCELED; break; diff --git a/libc/thread/sem_timedwait.c b/libc/thread/sem_timedwait.c index bd2e5d9d9..7ac0f3acc 100644 --- a/libc/thread/sem_timedwait.c +++ b/libc/thread/sem_timedwait.c @@ -28,6 +28,7 @@ #include "libc/intrin/weaken.h" #include "libc/limits.h" #include "libc/runtime/syslib.internal.h" +#include "libc/sysv/consts/clock.h" #include "libc/sysv/errfuns.h" #include "libc/thread/semaphore.h" #include "libc/thread/thread.h" @@ -121,7 +122,7 @@ int sem_timedwait(sem_t *sem, const struct timespec *abstime) { do { if (!(v = atomic_load_explicit(&sem->sem_value, memory_order_relaxed))) { - rc = nsync_futex_wait_(&sem->sem_value, v, true, abstime); + rc = nsync_futex_wait_(&sem->sem_value, v, true, CLOCK_REALTIME, abstime); if (rc == -EINTR || rc == -ECANCELED) { errno = -rc; rc = -1; diff --git a/libc/thread/thread.h b/libc/thread/thread.h index 2e4448f2e..fd70f30fb 100644 --- a/libc/thread/thread.h +++ b/libc/thread/thread.h @@ -56,7 +56,6 @@ COSMOPOLITAN_C_START_ typedef uintptr_t pthread_t; typedef int pthread_id_np_t; -typedef char pthread_condattr_t; typedef char pthread_rwlockattr_t; typedef char pthread_barrierattr_t; typedef unsigned pthread_key_t; @@ -83,12 +82,18 @@ typedef struct pthread_mutexattr_s { unsigned _word; } pthread_mutexattr_t; +typedef struct pthread_condattr_s { + char _pshared; + char _clock; +} pthread_condattr_t; + typedef struct pthread_cond_s { union { void *_align; struct { uint32_t _nsync; char _pshared; + char _clock; }; }; _PTHREAD_ATOMIC(uint32_t) _sequence; @@ -165,10 +170,12 @@ int pthread_cond_destroy(pthread_cond_t *) libcesque paramsnonnull(); int pthread_cond_init(pthread_cond_t *, const pthread_condattr_t *) libcesque paramsnonnull((1)); int pthread_cond_signal(pthread_cond_t *) libcesque paramsnonnull(); int pthread_cond_wait(pthread_cond_t *, pthread_mutex_t *) libcesque paramsnonnull(); -int pthread_condattr_destroy(pthread_condattr_t *) libcesque paramsnonnull(); -int pthread_condattr_getpshared(const pthread_condattr_t *, int *) libcesque paramsnonnull(); int pthread_condattr_init(pthread_condattr_t *) libcesque paramsnonnull(); +int pthread_condattr_destroy(pthread_condattr_t *) libcesque paramsnonnull(); int pthread_condattr_setpshared(pthread_condattr_t *, int) libcesque paramsnonnull(); +int pthread_condattr_getpshared(const pthread_condattr_t *, int *) libcesque paramsnonnull(); +int pthread_condattr_setclock(pthread_condattr_t *, int) libcesque paramsnonnull(); +int pthread_condattr_getclock(const pthread_condattr_t *, int *) libcesque paramsnonnull(); int pthread_create(pthread_t *, const pthread_attr_t *, void *(*)(void *), void *) dontthrow paramsnonnull((1)); int pthread_detach(pthread_t) libcesque; int pthread_equal(pthread_t, pthread_t) libcesque; diff --git a/test/libc/thread/footek_test.c b/test/libc/thread/footek_test.c index 43a72feae..bb72b93e3 100644 --- a/test/libc/thread/footek_test.c +++ b/test/libc/thread/footek_test.c @@ -1,9 +1,9 @@ -// config #define USE POSIX #define ITERATIONS 50000 #define THREADS 10 +// #define ITERATIONS 100000 +// #define THREADS 30 -// USE may be #define SPIN 1 #define FUTEX 2 #define POSIX 3 @@ -26,6 +26,7 @@ #include #include static inline long nsync_futex_wait_(atomic_int *uaddr, int val, char pshare, + int clock, const struct timespec *timeout) { return syscall(SYS_futex, uaddr, pshare ? FUTEX_WAIT : FUTEX_WAIT_PRIVATE, val, timeout, NULL, 0); @@ -144,25 +145,40 @@ static inline long nsync_futex_wake_(atomic_int *uaddr, int num_to_wake, // 216,236 us user // 127,344 us sys // -// footek_test on freebsd.test. 613 µs 2'120 µs 133'272 µs +// footek_test on freebsd.test. (cosmo) // 126,803 us real // 3,100 us user // 176,744 us sys // +// footek_test on freebsd.test. (freebsd libc) +// 219,073 us real +// 158,103 us user +// 1,146,252 us sys +// // footek_test on netbsd.test. 350 µs 3'570 µs 262'186 µs // 199,882 us real // 138,178 us user // 329,501 us sys // -// footek_test on openbsd.test. 454 µs 2'185 µs 153'258 µs +// footek_test on openbsd.test. (cosmo) // 138,619 us real // 30,000 us user // 110,000 us sys // -// footek_test on win10.test. 233 µs 6'133 µs 260'812 µs +// footek_test on openbsd.test. (openbsd libc) +// 385,431 us real +// 80,000 us user +// 1,350,000 us sys +// +// footek_test on win10.test. (cosmo) // 156,382 us real // 312,500 us user // 31,250 us sys +// +// footek_test on win10.test. (cygwin) +// 9,334,610 us real +// 1,562,000 us user +// 6,093,000 us sys // arm fleet // with spin lock @@ -261,7 +277,7 @@ void lock(atomic_int *futex) { while (word > 0) { pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs); #if USE == FUTEX - nsync_futex_wait_(futex, 2, 0, 0); + nsync_futex_wait_(futex, 2, 0, 0, 0); #endif pthread_setcancelstate(cs, 0); word = atomic_exchange_explicit(futex, 2, memory_order_acquire); diff --git a/test/libc/thread/nsync_test.c b/test/libc/thread/nsync_test.c index d781c5243..ac2d64514 100644 --- a/test/libc/thread/nsync_test.c +++ b/test/libc/thread/nsync_test.c @@ -17,6 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/errno.h" +#include "libc/sysv/consts/clock.h" #include "libc/testlib/testlib.h" #include "libc/thread/thread.h" #include "third_party/nsync/cv.h" @@ -34,7 +35,8 @@ int Put(long v, nsync_time abs_deadline) { int err, added = 0, wake = 0; nsync_mu_lock(&mu); while (count == limit) { - if ((err = nsync_cv_wait_with_deadline(&non_full, &mu, abs_deadline, 0))) { + if ((err = nsync_cv_wait_with_deadline(&non_full, &mu, CLOCK_REALTIME, + abs_deadline, 0))) { ASSERT_EQ(ETIMEDOUT, err); ASSERT_NE(0, nsync_time_cmp(nsync_time_no_deadline, abs_deadline)); } @@ -59,7 +61,8 @@ long Get(nsync_time abs_deadline) { long err, v = 0; nsync_mu_lock(&mu); while (!count) { - if ((err = nsync_cv_wait_with_deadline(&non_empty, &mu, abs_deadline, 0))) { + if ((err = nsync_cv_wait_with_deadline(&non_empty, &mu, CLOCK_REALTIME, + abs_deadline, 0))) { ASSERT_EQ(ETIMEDOUT, err); ASSERT_NE(0, nsync_time_cmp(nsync_time_no_deadline, abs_deadline)); } diff --git a/test/libc/thread/pthread_cond_timedwait_test.c b/test/libc/thread/pthread_cond_timedwait_test.c new file mode 100644 index 000000000..57ed46e38 --- /dev/null +++ b/test/libc/thread/pthread_cond_timedwait_test.c @@ -0,0 +1,64 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/struct/timespec.h" +#include "libc/errno.h" +#include "libc/sysv/consts/clock.h" +#include "libc/testlib/testlib.h" +#include "libc/thread/thread.h" +#include "libc/thread/thread2.h" + +TEST(pthread_cond_timedwait, real) { + pthread_cond_t cv; + pthread_mutex_t mu; + pthread_condattr_t ca; + ASSERT_EQ(0, pthread_condattr_init(&ca)); + ASSERT_EQ(0, pthread_condattr_setclock(&ca, CLOCK_REALTIME)); + ASSERT_EQ(0, pthread_cond_init(&cv, &ca)); + ASSERT_EQ(0, pthread_condattr_destroy(&ca)); + ASSERT_EQ(0, pthread_mutex_init(&mu, 0)); + ASSERT_EQ(0, pthread_mutex_lock(&mu)); + struct timespec start = timespec_real(); + struct timespec deadline = timespec_add(start, timespec_frommillis(100)); + ASSERT_EQ(ETIMEDOUT, pthread_cond_timedwait(&cv, &mu, &deadline)); + struct timespec end = timespec_real(); + ASSERT_GE(timespec_tomillis(timespec_sub(end, start)), 100); + ASSERT_EQ(0, pthread_mutex_unlock(&mu)); + ASSERT_EQ(0, pthread_mutex_destroy(&mu)); + ASSERT_EQ(0, pthread_cond_destroy(&cv)); +} + +TEST(pthread_cond_timedwait, mono) { + pthread_cond_t cv; + pthread_mutex_t mu; + pthread_condattr_t ca; + ASSERT_EQ(0, pthread_condattr_init(&ca)); + ASSERT_EQ(0, pthread_condattr_setclock(&ca, CLOCK_MONOTONIC)); + ASSERT_EQ(0, pthread_cond_init(&cv, &ca)); + ASSERT_EQ(0, pthread_condattr_destroy(&ca)); + ASSERT_EQ(0, pthread_mutex_init(&mu, 0)); + ASSERT_EQ(0, pthread_mutex_lock(&mu)); + struct timespec start = timespec_mono(); + struct timespec deadline = timespec_add(start, timespec_frommillis(100)); + ASSERT_EQ(ETIMEDOUT, pthread_cond_timedwait(&cv, &mu, &deadline)); + struct timespec end = timespec_mono(); + ASSERT_GE(timespec_tomillis(timespec_sub(end, start)), 100); + ASSERT_EQ(0, pthread_mutex_unlock(&mu)); + ASSERT_EQ(0, pthread_mutex_destroy(&mu)); + ASSERT_EQ(0, pthread_cond_destroy(&cv)); +} diff --git a/test/libc/thread/setitimer_test.c b/test/libc/thread/setitimer_test.c index 6ab5d42d0..d63c65e5f 100644 --- a/test/libc/thread/setitimer_test.c +++ b/test/libc/thread/setitimer_test.c @@ -26,7 +26,6 @@ #include "libc/calls/ucontext.h" #include "libc/dce.h" #include "libc/errno.h" -#include "libc/intrin/kprintf.h" #include "libc/limits.h" #include "libc/runtime/runtime.h" #include "libc/sysv/consts/itimer.h" diff --git a/third_party/lua/lunix.c b/third_party/lua/lunix.c index 8105cc313..ccaf4cbe6 100644 --- a/third_party/lua/lunix.c +++ b/third_party/lua/lunix.c @@ -110,6 +110,7 @@ #include "third_party/lua/lua.h" #include "third_party/lua/luaconf.h" #include "third_party/nsync/futex.internal.h" +#include "libc/sysv/consts/clock.h" #include "tool/net/luacheck.h" #define DNS_NAME_MAX 253 @@ -2855,7 +2856,7 @@ static int LuaUnixMemoryWait(lua_State *L) { } BEGIN_CANCELATION_POINT; rc = nsync_futex_wait_((atomic_int *)GetWord(L), expect, - PTHREAD_PROCESS_SHARED, deadline); + PTHREAD_PROCESS_SHARED, CLOCK_REALTIME, deadline); END_CANCELATION_POINT; if (rc < 0) errno = -rc, rc = -1; return SysretInteger(L, "futex_wait", olderr, rc); diff --git a/third_party/nsync/README.cosmo b/third_party/nsync/README.cosmo index 415a659fb..044532305 100644 --- a/third_party/nsync/README.cosmo +++ b/third_party/nsync/README.cosmo @@ -17,6 +17,8 @@ LOCAL CHANGES - Fix nsync_mu_unlock() on Apple Silicon + - Add clock parameter to many NSYNC wait APIs + - Time APIs were so good that they're now in libc - Double linked list API was so good that it's now in libc diff --git a/third_party/nsync/common.internal.h b/third_party/nsync/common.internal.h index be42db19e..ebc1a7d64 100644 --- a/third_party/nsync/common.internal.h +++ b/third_party/nsync/common.internal.h @@ -266,7 +266,7 @@ void nsync_mu_unlock_slow_(nsync_mu *mu, lock_type *l_type); struct Dll *nsync_remove_from_mu_queue_(struct Dll *mu_queue, struct Dll *e); void nsync_maybe_merge_conditions_(struct Dll *p, struct Dll *n); nsync_time nsync_note_notified_deadline_(nsync_note n); -int nsync_sem_wait_with_cancel_(waiter *w, nsync_time abs_deadline, +int nsync_sem_wait_with_cancel_(waiter *w, int clock, nsync_time abs_deadline, nsync_note cancel_note); COSMOPOLITAN_C_END_ diff --git a/third_party/nsync/cv.h b/third_party/nsync/cv.h index 4209a7909..a02b587b8 100644 --- a/third_party/nsync/cv.h +++ b/third_party/nsync/cv.h @@ -144,7 +144,7 @@ int nsync_cv_wait(nsync_cv *cv, nsync_mu *mu); mostly in tests and trivial examples than they are in real programmes. */ int nsync_cv_wait_with_deadline(nsync_cv *cv, nsync_mu *mu, - nsync_time abs_deadline, + int clock, nsync_time abs_deadline, struct nsync_note_s_ *cancel_note); /* Like nsync_cv_wait_with_deadline(), but allow an arbitrary lock *v to be @@ -152,7 +152,7 @@ int nsync_cv_wait_with_deadline(nsync_cv *cv, nsync_mu *mu, int nsync_cv_wait_with_deadline_generic(nsync_cv *cv, void *mu, void (*lock)(void *), void (*unlock)(void *), - nsync_time abs_deadline, + int clock, nsync_time abs_deadline, struct nsync_note_s_ *cancel_note); COSMOPOLITAN_C_END_ diff --git a/third_party/nsync/futex.c b/third_party/nsync/futex.c index ce34af900..7f3d63da8 100644 --- a/third_party/nsync/futex.c +++ b/third_party/nsync/futex.c @@ -52,7 +52,6 @@ #include "third_party/nsync/atomic.h" #include "third_party/nsync/common.internal.h" #include "third_party/nsync/futex.internal.h" -#include "libc/intrin/kprintf.h" #include "third_party/nsync/time.h" #define FUTEX_WAIT_BITS_ FUTEX_BITSET_MATCH_ANY @@ -65,6 +64,7 @@ static struct NsyncFutex { atomic_uint once; int FUTEX_WAIT_; int FUTEX_PRIVATE_FLAG_; + int FUTEX_CLOCK_REALTIME_; bool is_supported; bool timeout_is_relative; } nsync_futex_; @@ -92,9 +92,8 @@ static void nsync_futex_init_ (void) { return; } - if (!(nsync_futex_.is_supported = IsLinux () || IsOpenbsd ())) { + if (!(nsync_futex_.is_supported = IsLinux () || IsOpenbsd ())) return; - } // In our testing, we found that the monotonic clock on various // popular systems (such as Linux, and some BSD variants) was no @@ -111,16 +110,11 @@ static void nsync_futex_init_ (void) { if (IsLinux () && _futex (&x, FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME, 1, 0, 0, FUTEX_BITSET_MATCH_ANY) == -EAGAIN) { - nsync_futex_.FUTEX_WAIT_ = - FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME; - nsync_futex_.FUTEX_PRIVATE_FLAG_ = FUTEX_PRIVATE_FLAG; - } else if (!IsTiny () && IsLinux () && - _futex (&x, FUTEX_WAIT_BITSET, 1, 0, 0, - FUTEX_BITSET_MATCH_ANY) == -EAGAIN) { nsync_futex_.FUTEX_WAIT_ = FUTEX_WAIT_BITSET; nsync_futex_.FUTEX_PRIVATE_FLAG_ = FUTEX_PRIVATE_FLAG; + nsync_futex_.FUTEX_CLOCK_REALTIME_ = FUTEX_CLOCK_REALTIME; } else if (IsOpenbsd () || - (!IsTiny () && IsLinux () && + (IsLinux () && !_futex_wake (&x, FUTEX_WAKE_PRIVATE, 1))) { nsync_futex_.FUTEX_WAIT_ = FUTEX_WAIT; nsync_futex_.FUTEX_PRIVATE_FLAG_ = FUTEX_PRIVATE_FLAG; @@ -132,24 +126,24 @@ static void nsync_futex_init_ (void) { errno = e; } -static int nsync_futex_polyfill_ (atomic_int *w, int expect, struct timespec *abstime) { +static int nsync_futex_polyfill_ (atomic_int *w, int expect, int clock, struct timespec *abstime) { for (;;) { - if (atomic_load_explicit (w, memory_order_acquire) != expect) { + if (atomic_load_explicit (w, memory_order_acquire) != expect) return 0; - } if (_weaken (pthread_testcancel_np) && - _weaken (pthread_testcancel_np) ()) { + _weaken (pthread_testcancel_np) ()) return -ECANCELED; - } - if (abstime && timespec_cmp (timespec_real (), *abstime) >= 0) { + struct timespec now; + if (clock_gettime (clock, &now)) + return -EINVAL; + if (abstime && timespec_cmp (now, *abstime) >= 0) return -ETIMEDOUT; - } pthread_yield_np (); } } static int nsync_futex_wait_win32_ (atomic_int *w, int expect, char pshare, - const struct timespec *timeout, + int clock, const struct timespec *timeout, struct PosixThread *pt, sigset_t waitmask) { #ifdef __x86_64__ @@ -164,23 +158,20 @@ static int nsync_futex_wait_win32_ (atomic_int *w, int expect, char pshare, } for (;;) { - now = timespec_real (); - if (timespec_cmp (now, deadline) >= 0) { - return etimedout(); - } + if (clock_gettime (clock, &now)) + return einval (); + if (timespec_cmp (now, deadline) >= 0) + return etimedout (); wait = timespec_sub (deadline, now); - if (atomic_load_explicit (w, memory_order_acquire) != expect) { + if (atomic_load_explicit (w, memory_order_acquire) != expect) return 0; - } if (pt) { - if (_check_cancel () == -1) { + if (_check_cancel () == -1) return -1; /* ECANCELED */ - } if ((sig = __sig_get (waitmask))) { __sig_relay (sig, SI_KERNEL, waitmask); - if (_check_cancel () == -1) { + if (_check_cancel () == -1) return -1; /* ECANCELED */ - } return eintr (); } pt->pt_blkmask = waitmask; @@ -192,9 +183,8 @@ static int nsync_futex_wait_win32_ (atomic_int *w, int expect, char pshare, atomic_store_explicit (&pt->pt_blocker, 0, memory_order_release); if (ok && atomic_load_explicit (w, memory_order_acquire) == expect && (sig = __sig_get (waitmask))) { __sig_relay (sig, SI_KERNEL, waitmask); - if (_check_cancel () == -1) { + if (_check_cancel () == -1) return -1; /* ECANCELED */ - } return eintr (); } } @@ -209,33 +199,41 @@ static int nsync_futex_wait_win32_ (atomic_int *w, int expect, char pshare, #endif /* __x86_64__ */ } -static struct timespec *nsync_futex_timeout_ (struct timespec *memory, - const struct timespec *abstime) { +static int nsync_futex_fix_timeout_ (struct timespec *memory, int clock, + const struct timespec *abstime, + struct timespec **result) { struct timespec now; if (!abstime) { + *result = 0; return 0; } else if (!nsync_futex_.timeout_is_relative) { *memory = *abstime; - return memory; + *result = memory; + return 0; } else { - now = timespec_real (); + if (clock_gettime (clock, &now)) + return -EINVAL; *memory = timespec_subz (*abstime, now); - return memory; + *result = memory; + return 0; } } -int nsync_futex_wait_ (atomic_int *w, int expect, char pshare, const struct timespec *abstime) { +int nsync_futex_wait_ (atomic_int *w, int expect, char pshare, + int clock, const struct timespec *abstime) { int e, rc, op; struct CosmoTib *tib; struct PosixThread *pt; - struct timespec tsmem, *timeout; + struct timespec tsmem; + struct timespec *timeout = 0; cosmo_once (&nsync_futex_.once, nsync_futex_init_); op = nsync_futex_.FUTEX_WAIT_; - if (pshare == PTHREAD_PROCESS_PRIVATE) { + if (pshare == PTHREAD_PROCESS_PRIVATE) op |= nsync_futex_.FUTEX_PRIVATE_FLAG_; - } + if (clock == CLOCK_REALTIME) + op |= nsync_futex_.FUTEX_CLOCK_REALTIME_; if (abstime && timespec_cmp (*abstime, timespec_zero) <= 0) { rc = -ETIMEDOUT; @@ -247,7 +245,8 @@ int nsync_futex_wait_ (atomic_int *w, int expect, char pshare, const struct time goto Finished; } - timeout = nsync_futex_timeout_ (&tsmem, abstime); + if ((rc = nsync_futex_fix_timeout_ (&tsmem, clock, abstime, &timeout))) + goto Finished; LOCKTRACE ("futex(%t [%d], %s, %#x, %s) → ...", w, atomic_load_explicit (w, memory_order_relaxed), @@ -263,7 +262,7 @@ int nsync_futex_wait_ (atomic_int *w, int expect, char pshare, const struct time // Windows 8 futexes don't support multiple processes :( if (pshare) goto Polyfill; sigset_t m = __sig_block (); - rc = nsync_futex_wait_win32_ (w, expect, pshare, timeout, pt, m); + rc = nsync_futex_wait_win32_ (w, expect, pshare, clock, timeout, pt, m); __sig_unblock (m); } else if (IsXnu ()) { uint32_t op, us; @@ -280,7 +279,7 @@ int nsync_futex_wait_ (atomic_int *w, int expect, char pshare, const struct time rc = ulock_wait (op, w, expect, us); if (rc > 0) rc = 0; // don't care about #waiters } else if (IsFreebsd ()) { - rc = sys_umtx_timedwait_uint (w, expect, pshare, timeout); + rc = sys_umtx_timedwait_uint (w, expect, pshare, clock, timeout); } else { if (IsOpenbsd()) { // OpenBSD 6.8 futex() returns errors as @@ -313,7 +312,7 @@ int nsync_futex_wait_ (atomic_int *w, int expect, char pshare, const struct time } } else { Polyfill: - rc = nsync_futex_polyfill_ (w, expect, timeout); + rc = nsync_futex_polyfill_ (w, expect, clock, timeout); } Finished: @@ -334,9 +333,8 @@ int nsync_futex_wake_ (atomic_int *w, int count, char pshare) { cosmo_once (&nsync_futex_.once, nsync_futex_init_); op = FUTEX_WAKE; - if (pshare == PTHREAD_PROCESS_PRIVATE) { + if (pshare == PTHREAD_PROCESS_PRIVATE) op |= nsync_futex_.FUTEX_PRIVATE_FLAG_; - } if (nsync_futex_.is_supported) { if (IsWindows ()) { diff --git a/third_party/nsync/futex.internal.h b/third_party/nsync/futex.internal.h index c572a1595..f555224ff 100644 --- a/third_party/nsync/futex.internal.h +++ b/third_party/nsync/futex.internal.h @@ -11,7 +11,7 @@ COSMOPOLITAN_C_START_ #endif int nsync_futex_wake_(_FUTEX_ATOMIC(int) *, int, char); -int nsync_futex_wait_(_FUTEX_ATOMIC(int) *, int, char, const struct timespec *); +int nsync_futex_wait_(_FUTEX_ATOMIC(int) *, int, char, int, const struct timespec *); COSMOPOLITAN_C_END_ #endif /* NSYNC_FUTEX_INTERNAL_H_ */ diff --git a/third_party/nsync/mem/nsync_counter.c b/third_party/nsync/mem/nsync_counter.c index c508797fc..d8b58841b 100644 --- a/third_party/nsync/mem/nsync_counter.c +++ b/third_party/nsync/mem/nsync_counter.c @@ -25,6 +25,7 @@ #include "third_party/nsync/mu_semaphore.h" #include "third_party/nsync/races.internal.h" #include "third_party/nsync/wait_s.internal.h" +#include "libc/sysv/consts/clock.h" #include "third_party/nsync/waiter.h" __static_yoink("nsync_notice"); @@ -100,7 +101,7 @@ uint32_t nsync_counter_wait (nsync_counter c, nsync_time abs_deadline) { uint32_t result = 0; waitable.v = c; waitable.funcs = &nsync_counter_waitable_funcs; - if (nsync_wait_n (NULL, NULL, NULL, abs_deadline, 1, &pwaitable) != 0) { + if (nsync_wait_n (NULL, NULL, NULL, CLOCK_REALTIME, abs_deadline, 1, &pwaitable) != 0) { IGNORE_RACES_START (); result = ATM_LOAD_ACQ (&c->value); IGNORE_RACES_END (); diff --git a/third_party/nsync/mem/nsync_cv.c b/third_party/nsync/mem/nsync_cv.c index a85fb67d2..9e798d4eb 100644 --- a/third_party/nsync/mem/nsync_cv.c +++ b/third_party/nsync/mem/nsync_cv.c @@ -175,6 +175,7 @@ struct nsync_cv_wait_with_deadline_s { void *pmu; void (*lock) (void *); nsync_mu *cv_mu; + int clock; nsync_time abs_deadline; nsync_note cancel_note; waiter *w; @@ -187,7 +188,7 @@ static int nsync_cv_wait_with_deadline_impl_ (struct nsync_cv_wait_with_deadline IGNORE_RACES_START (); while (ATM_LOAD_ACQ (&c->w->nw.waiting) != 0) { /* acquire load */ if (c->sem_outcome == 0) { - c->sem_outcome = nsync_sem_wait_with_cancel_ (c->w, c->abs_deadline, c->cancel_note); + c->sem_outcome = nsync_sem_wait_with_cancel_ (c->w, c->clock, c->abs_deadline, c->cancel_note); } if (c->sem_outcome != 0 && ATM_LOAD (&c->w->nw.waiting) != 0) { /* A timeout or cancellation occurred, and no wakeup. @@ -278,13 +279,14 @@ static void nsync_cv_wait_with_deadline_unwind_ (void *arg) { programmes. */ int nsync_cv_wait_with_deadline_generic (nsync_cv *pcv, void *pmu, void (*lock) (void *), void (*unlock) (void *), - nsync_time abs_deadline, + int clock, nsync_time abs_deadline, nsync_note cancel_note) { int outcome; struct nsync_cv_wait_with_deadline_s c; IGNORE_RACES_START (); c.w = nsync_waiter_new_ (); + c.clock = clock; c.abs_deadline = abs_deadline; c.cancel_note = cancel_note; c.cv_mu = NULL; @@ -470,10 +472,10 @@ void nsync_cv_broadcast (nsync_cv *pcv) { /* Wait with deadline, using an nsync_mu. */ errno_t nsync_cv_wait_with_deadline (nsync_cv *pcv, nsync_mu *pmu, - nsync_time abs_deadline, + int clock, nsync_time abs_deadline, nsync_note cancel_note) { return (nsync_cv_wait_with_deadline_generic (pcv, pmu, &void_mu_lock, - &void_mu_unlock, + &void_mu_unlock, clock, abs_deadline, cancel_note)); } @@ -486,7 +488,7 @@ errno_t nsync_cv_wait_with_deadline (nsync_cv *pcv, nsync_mu *pmu, ECANCELED may be returned if calling POSIX thread is cancelled only when the PTHREAD_CANCEL_MASKED mode is in play. */ errno_t nsync_cv_wait (nsync_cv *pcv, nsync_mu *pmu) { - return nsync_cv_wait_with_deadline (pcv, pmu, nsync_time_no_deadline, NULL); + return nsync_cv_wait_with_deadline (pcv, pmu, 0, nsync_time_no_deadline, NULL); } static nsync_time cv_ready_time (void *v, struct nsync_waiter_s *nw) { diff --git a/third_party/nsync/mem/nsync_mu_wait.c b/third_party/nsync/mem/nsync_mu_wait.c index e46492aa9..785823c5c 100644 --- a/third_party/nsync/mem/nsync_mu_wait.c +++ b/third_party/nsync/mem/nsync_mu_wait.c @@ -141,7 +141,7 @@ int nsync_mu_wait_with_deadline (nsync_mu *mu, int (*condition) (const void *condition_arg), const void *condition_arg, int (*condition_arg_eq) (const void *a, const void *b), - nsync_time abs_deadline, nsync_note cancel_note) { + int clock, nsync_time abs_deadline, nsync_note cancel_note) { lock_type *l_type; int first_wait; int condition_is_true; @@ -231,7 +231,7 @@ int nsync_mu_wait_with_deadline (nsync_mu *mu, have_lock = 0; while (ATM_LOAD_ACQ (&w->nw.waiting) != 0) { /* acquire load */ if (sem_outcome == 0) { - sem_outcome = nsync_sem_wait_with_cancel_ (w, abs_deadline, + sem_outcome = nsync_sem_wait_with_cancel_ (w, clock, abs_deadline, cancel_note); if (sem_outcome != 0 && ATM_LOAD (&w->nw.waiting) != 0) { /* A timeout or cancellation occurred, and no wakeup. @@ -280,7 +280,7 @@ void nsync_mu_wait (nsync_mu *mu, int (*condition) (const void *condition_arg), const void *condition_arg, int (*condition_arg_eq) (const void *a, const void *b)) { if (nsync_mu_wait_with_deadline (mu, condition, condition_arg, condition_arg_eq, - nsync_time_no_deadline, NULL) != 0) { + 0, nsync_time_no_deadline, NULL) != 0) { nsync_panic_ ("nsync_mu_wait woke but condition not true\n"); } } diff --git a/third_party/nsync/mem/nsync_note.c b/third_party/nsync/mem/nsync_note.c index bdf8e9ad0..a4349f28e 100644 --- a/third_party/nsync/mem/nsync_note.c +++ b/third_party/nsync/mem/nsync_note.c @@ -24,6 +24,7 @@ #include "third_party/nsync/mu_wait.h" #include "third_party/nsync/races.internal.h" #include "third_party/nsync/wait_s.internal.h" +#include "libc/sysv/consts/clock.h" #include "third_party/nsync/waiter.h" __static_yoink("nsync_notice"); @@ -247,7 +248,7 @@ int nsync_note_wait (nsync_note n, nsync_time abs_deadline) { struct nsync_waitable_s *pwaitable = &waitable; waitable.v = n; waitable.funcs = &nsync_note_waitable_funcs; - return (nsync_wait_n (NULL, NULL, NULL, abs_deadline, 1, &pwaitable) == 0); + return (nsync_wait_n (NULL, NULL, NULL, CLOCK_REALTIME, abs_deadline, 1, &pwaitable) == 0); } nsync_time nsync_note_expiry (nsync_note n) { diff --git a/third_party/nsync/mem/nsync_once.c b/third_party/nsync/mem/nsync_once.c index 873766b99..780be803a 100644 --- a/third_party/nsync/mem/nsync_once.c +++ b/third_party/nsync/mem/nsync_once.c @@ -22,6 +22,7 @@ #include "third_party/nsync/once.h" #include "third_party/nsync/races.internal.h" #include "libc/thread/thread.h" +#include "libc/sysv/consts/clock.h" #include "third_party/nsync/wait_s.internal.h" __static_yoink("nsync_notice"); @@ -91,7 +92,7 @@ static void nsync_run_once_impl (nsync_once *once, struct once_sync_s *s, attempts += 10; } deadline = nsync_time_add (nsync_time_now (), nsync_time_ms (attempts)); - nsync_cv_wait_with_deadline (&s->once_cv, &s->once_mu, deadline, NULL); + nsync_cv_wait_with_deadline (&s->once_cv, &s->once_mu, CLOCK_REALTIME, deadline, NULL); } else { attempts = pthread_delay_np (once, attempts); } diff --git a/third_party/nsync/mem/nsync_sem_wait.c b/third_party/nsync/mem/nsync_sem_wait.c index 62507d686..9ee5044c5 100644 --- a/third_party/nsync/mem/nsync_sem_wait.c +++ b/third_party/nsync/mem/nsync_sem_wait.c @@ -29,11 +29,11 @@ __static_yoink("nsync_notice"); w->sem is non-zero----decrement it and return 0. abs_deadline expires---return ETIMEDOUT. cancel_note is non-NULL and *cancel_note becomes notified---return ECANCELED. */ -int nsync_sem_wait_with_cancel_ (waiter *w, nsync_time abs_deadline, +int nsync_sem_wait_with_cancel_ (waiter *w, int clock, nsync_time abs_deadline, nsync_note cancel_note) { int sem_outcome; if (cancel_note == NULL) { - sem_outcome = nsync_mu_semaphore_p_with_deadline (&w->sem, abs_deadline); + sem_outcome = nsync_mu_semaphore_p_with_deadline (&w->sem, clock, abs_deadline); } else { nsync_time cancel_time; cancel_time = nsync_note_notified_deadline_ (cancel_note); @@ -58,7 +58,7 @@ int nsync_sem_wait_with_cancel_ (waiter *w, nsync_time abs_deadline, } nsync_mu_unlock (&cancel_note->note_mu); sem_outcome = nsync_mu_semaphore_p_with_deadline (&w->sem, - local_abs_deadline); + clock, local_abs_deadline); if (sem_outcome == ETIMEDOUT && !deadline_is_nearer) { sem_outcome = ECANCELED; nsync_note_notify (cancel_note); diff --git a/third_party/nsync/mem/nsync_wait.c b/third_party/nsync/mem/nsync_wait.c index 9d8e95b7d..6cbebd3e1 100644 --- a/third_party/nsync/mem/nsync_wait.c +++ b/third_party/nsync/mem/nsync_wait.c @@ -28,7 +28,7 @@ __static_yoink("nsync_notice"); int nsync_wait_n (void *mu, void (*lock) (void *), void (*unlock) (void *), - nsync_time abs_deadline, + int clock, nsync_time abs_deadline, int count, struct nsync_waitable_s *waitable[]) { int ready; IGNORE_RACES_START (); @@ -77,7 +77,7 @@ int nsync_wait_n (void *mu, void (*lock) (void *), void (*unlock) (void *), } } while (nsync_time_cmp (min_ntime, nsync_time_zero) > 0 && nsync_mu_semaphore_p_with_deadline (&w->sem, - min_ntime) == 0); + clock, min_ntime) == 0); } /* An attempt was made above to enqueue waitable[0..i-1]. diff --git a/third_party/nsync/mu_semaphore.c b/third_party/nsync/mu_semaphore.c index 274b4e75b..1ab8951af 100644 --- a/third_party/nsync/mu_semaphore.c +++ b/third_party/nsync/mu_semaphore.c @@ -54,15 +54,15 @@ errno_t nsync_mu_semaphore_p (nsync_semaphore *s) { while additionally supporting a time parameter specifying at what point in the future ETIMEDOUT should be returned, if neither cancelation, or semaphore release happens. */ -errno_t nsync_mu_semaphore_p_with_deadline (nsync_semaphore *s, nsync_time abs_deadline) { +errno_t nsync_mu_semaphore_p_with_deadline (nsync_semaphore *s, int clock, nsync_time abs_deadline) { errno_t err; BEGIN_CANCELATION_POINT; if (NSYNC_USE_GRAND_CENTRAL && IsXnuSilicon ()) { - err = nsync_mu_semaphore_p_with_deadline_gcd (s, abs_deadline); + err = nsync_mu_semaphore_p_with_deadline_gcd (s, clock, abs_deadline); } else if (IsNetbsd ()) { - err = nsync_mu_semaphore_p_with_deadline_sem (s, abs_deadline); + err = nsync_mu_semaphore_p_with_deadline_sem (s, clock, abs_deadline); } else { - err = nsync_mu_semaphore_p_with_deadline_futex (s, abs_deadline); + err = nsync_mu_semaphore_p_with_deadline_futex (s, clock, abs_deadline); } END_CANCELATION_POINT; return err; diff --git a/third_party/nsync/mu_semaphore.h b/third_party/nsync/mu_semaphore.h index 992d4849f..634d9fea4 100644 --- a/third_party/nsync/mu_semaphore.h +++ b/third_party/nsync/mu_semaphore.h @@ -16,7 +16,7 @@ errno_t nsync_mu_semaphore_p(nsync_semaphore *s); /* Wait until one of: the count of *s is non-zero, in which case decrement *s and return 0; or abs_deadline expires, in which case return ETIMEDOUT. */ -errno_t nsync_mu_semaphore_p_with_deadline(nsync_semaphore *s, +errno_t nsync_mu_semaphore_p_with_deadline(nsync_semaphore *s, int clock, nsync_time abs_deadline); /* Ensure that the count of *s is at least 1. */ diff --git a/third_party/nsync/mu_semaphore.internal.h b/third_party/nsync/mu_semaphore.internal.h index 8795fe349..b74291223 100755 --- a/third_party/nsync/mu_semaphore.internal.h +++ b/third_party/nsync/mu_semaphore.internal.h @@ -20,17 +20,17 @@ COSMOPOLITAN_C_START_ bool nsync_mu_semaphore_init_futex(nsync_semaphore *); errno_t nsync_mu_semaphore_p_futex(nsync_semaphore *); -errno_t nsync_mu_semaphore_p_with_deadline_futex(nsync_semaphore *, nsync_time); +errno_t nsync_mu_semaphore_p_with_deadline_futex(nsync_semaphore *, int, nsync_time); void nsync_mu_semaphore_v_futex(nsync_semaphore *); bool nsync_mu_semaphore_init_sem(nsync_semaphore *); errno_t nsync_mu_semaphore_p_sem(nsync_semaphore *); -errno_t nsync_mu_semaphore_p_with_deadline_sem(nsync_semaphore *, nsync_time); +errno_t nsync_mu_semaphore_p_with_deadline_sem(nsync_semaphore *, int, nsync_time); void nsync_mu_semaphore_v_sem(nsync_semaphore *); bool nsync_mu_semaphore_init_gcd(nsync_semaphore *); errno_t nsync_mu_semaphore_p_gcd(nsync_semaphore *); -errno_t nsync_mu_semaphore_p_with_deadline_gcd(nsync_semaphore *, nsync_time); +errno_t nsync_mu_semaphore_p_with_deadline_gcd(nsync_semaphore *, int, nsync_time); void nsync_mu_semaphore_v_gcd(nsync_semaphore *); COSMOPOLITAN_C_END_ diff --git a/third_party/nsync/mu_semaphore_futex.c b/third_party/nsync/mu_semaphore_futex.c index a4e605a6e..863773da1 100644 --- a/third_party/nsync/mu_semaphore_futex.c +++ b/third_party/nsync/mu_semaphore_futex.c @@ -63,7 +63,7 @@ errno_t nsync_mu_semaphore_p_futex (nsync_semaphore *s) { int futex_result; futex_result = -nsync_futex_wait_ ( (atomic_int *)&f->i, i, - PTHREAD_PROCESS_PRIVATE, 0); + PTHREAD_PROCESS_PRIVATE, 0, 0); ASSERT (futex_result == 0 || futex_result == EINTR || futex_result == EAGAIN || @@ -81,7 +81,7 @@ errno_t nsync_mu_semaphore_p_futex (nsync_semaphore *s) { while additionally supporting a time parameter specifying at what point in the future ETIMEDOUT should be returned, if neither cancellation, or semaphore release happens. */ -errno_t nsync_mu_semaphore_p_with_deadline_futex (nsync_semaphore *s, nsync_time abs_deadline) { +errno_t nsync_mu_semaphore_p_with_deadline_futex (nsync_semaphore *s, int clock, nsync_time abs_deadline) { struct futex *f = (struct futex *)s; int i; int result = 0; @@ -98,7 +98,8 @@ errno_t nsync_mu_semaphore_p_with_deadline_futex (nsync_semaphore *s, nsync_time ts = &ts_buf; } futex_result = nsync_futex_wait_ ((atomic_int *)&f->i, i, - PTHREAD_PROCESS_PRIVATE, ts); + PTHREAD_PROCESS_PRIVATE, + clock, ts); ASSERT (futex_result == 0 || futex_result == -EINTR || futex_result == -EAGAIN || @@ -106,9 +107,12 @@ errno_t nsync_mu_semaphore_p_with_deadline_futex (nsync_semaphore *s, nsync_time futex_result == -ETIMEDOUT || futex_result == -EWOULDBLOCK); /* Some systems don't wait as long as they are told. */ - if (futex_result == -ETIMEDOUT && - nsync_time_cmp (abs_deadline, nsync_time_now ()) <= 0) { - result = ETIMEDOUT; + if (futex_result == -ETIMEDOUT) { + nsync_time now; + if (clock_gettime (clock, &now)) + result = EINVAL; + if (nsync_time_cmp (now, abs_deadline) >= 0) + result = ETIMEDOUT; } if (futex_result == -ECANCELED) { result = ECANCELED; diff --git a/third_party/nsync/mu_semaphore_gcd.c b/third_party/nsync/mu_semaphore_gcd.c index 088b50414..c8722adfe 100644 --- a/third_party/nsync/mu_semaphore_gcd.c +++ b/third_party/nsync/mu_semaphore_gcd.c @@ -94,14 +94,14 @@ bool nsync_mu_semaphore_init_gcd (nsync_semaphore *s) { they're enabled in MASKED mode, this function may return ECANCELED. Otherwise, cancellation will occur by unwinding cleanup handlers pushed to the stack. */ errno_t nsync_mu_semaphore_p_gcd (nsync_semaphore *s) { - return nsync_mu_semaphore_p_with_deadline_gcd (s, nsync_time_no_deadline); + return nsync_mu_semaphore_p_with_deadline_gcd (s, 0, nsync_time_no_deadline); } /* Like nsync_mu_semaphore_p() this waits for the count of *s to exceed 0, while additionally supporting a time parameter specifying at what point in the future ETIMEDOUT should be returned, if neither cancellation, or semaphore release happens. */ -errno_t nsync_mu_semaphore_p_with_deadline_gcd (nsync_semaphore *s, +errno_t nsync_mu_semaphore_p_with_deadline_gcd (nsync_semaphore *s, int clock, nsync_time abs_deadline) { errno_t result = 0; struct PosixThread *pt; @@ -117,7 +117,10 @@ errno_t nsync_mu_semaphore_p_with_deadline_gcd (nsync_semaphore *s, result = ECANCELED; break; } - now = timespec_real(); + if (clock_gettime (clock, &now)) { + result = EINVAL; + break; + } if (timespec_cmp (now, abs_deadline) >= 0) { result = ETIMEDOUT; break; diff --git a/third_party/nsync/mu_semaphore_sem.c b/third_party/nsync/mu_semaphore_sem.c index b821dd08c..2876c902d 100644 --- a/third_party/nsync/mu_semaphore_sem.c +++ b/third_party/nsync/mu_semaphore_sem.c @@ -33,6 +33,7 @@ #include "third_party/nsync/mu_semaphore.h" #include "libc/intrin/atomic.h" #include "libc/atomic.h" +#include "libc/sysv/consts/clock.h" #include "third_party/nsync/time.h" /** @@ -126,10 +127,22 @@ errno_t nsync_mu_semaphore_p_sem (nsync_semaphore *s) { while additionally supporting a time parameter specifying at what point in the future ETIMEDOUT should be returned, if neither cancellation, or semaphore release happens. */ -errno_t nsync_mu_semaphore_p_with_deadline_sem (nsync_semaphore *s, nsync_time abs_deadline) { +errno_t nsync_mu_semaphore_p_with_deadline_sem (nsync_semaphore *s, int clock, + nsync_time abs_deadline) { int e, rc; errno_t result; struct sem *f = (struct sem *) s; + + // convert monotonic back to realtime just for netbsd + if (clock && nsync_time_cmp (abs_deadline, nsync_time_no_deadline)) { + struct timespec now, delta; + if (clock_gettime (clock, &now)) + return EINVAL; + delta = timespec_subz (abs_deadline, now); + clock_gettime (CLOCK_REALTIME, &now); + abs_deadline = timespec_add (now, delta); + } + e = errno; rc = sys_sem_timedwait (f->id, &abs_deadline); STRACE ("sem_timedwait(%ld, %s) → %d% m", f->id, diff --git a/third_party/nsync/mu_wait.h b/third_party/nsync/mu_wait.h index 3ee9d9793..d17a10c9d 100644 --- a/third_party/nsync/mu_wait.h +++ b/third_party/nsync/mu_wait.h @@ -97,7 +97,7 @@ void nsync_mu_wait(nsync_mu *mu, int (*condition)(const void *condition_arg), int nsync_mu_wait_with_deadline( nsync_mu *mu, int (*condition)(const void *condition_arg), const void *condition_arg, - int (*condition_arg_eq)(const void *a, const void *b), + int (*condition_arg_eq)(const void *a, const void *b), int clock, nsync_time abs_deadline, struct nsync_note_s_ *cancel_note); /* Unlock *mu, which must be held in write mode, and wake waiters, if diff --git a/third_party/nsync/testing/cv_mu_timeout_stress_test_.c b/third_party/nsync/testing/cv_mu_timeout_stress_test_.c index 302ec90f5..0b6982768 100644 --- a/third_party/nsync/testing/cv_mu_timeout_stress_test_.c +++ b/third_party/nsync/testing/cv_mu_timeout_stress_test_.c @@ -22,6 +22,7 @@ #include "third_party/nsync/mu_wait.h" #include "third_party/nsync/testing/closure.h" #include "third_party/nsync/testing/smprintf.h" +#include "libc/sysv/consts/clock.h" #include "third_party/nsync/testing/testing.h" /* A cv_stress_data represents the data used by the threads of the tests below. */ @@ -79,7 +80,7 @@ static void cv_stress_inc_loop (cv_stress_data *s, uintmax_t count_imod4) { nsync_time_us (rand () % STRESS_MAX_DELAY_MICROS)); while (nsync_cv_wait_with_deadline ( &s->count_is_imod4[count_imod4], - &s->mu, abs_deadline, NULL) != 0 && + &s->mu, CLOCK_REALTIME, abs_deadline, NULL) != 0 && (s->count&3) != count_imod4) { nsync_mu_assert_held (&s->mu); s->timeouts++; @@ -130,7 +131,8 @@ static void cv_stress_reader_loop (cv_stress_data *s, uintmax_t count_imod4) { abs_deadline = nsync_time_add (nsync_time_now (), nsync_time_us (rand () % STRESS_MAX_DELAY_MICROS)); while (nsync_cv_wait_with_deadline (&s->count_is_imod4[count_imod4], - &s->mu, abs_deadline, NULL) != 0 && + &s->mu, CLOCK_REALTIME, + abs_deadline, NULL) != 0 && (s->count&3) != count_imod4 && s->refs != 0) { nsync_mu_rassert_held (&s->mu); diff --git a/third_party/nsync/testing/cv_test.c b/third_party/nsync/testing/cv_test.c index d31413d4f..9677b2083 100644 --- a/third_party/nsync/testing/cv_test.c +++ b/third_party/nsync/testing/cv_test.c @@ -29,6 +29,7 @@ #include "third_party/nsync/testing/smprintf.h" #include "third_party/nsync/testing/testing.h" #include "third_party/nsync/testing/time_extra.h" +#include "libc/sysv/consts/clock.h" #include "third_party/nsync/time.h" /* --------------------------- */ @@ -63,7 +64,7 @@ static int cv_queue_put (cv_queue *q, void *v, nsync_time abs_deadline) { int wake = 0; nsync_mu_lock (&q->mu); while (q->count == q->limit && - nsync_cv_wait_with_deadline (&q->non_full, &q->mu, abs_deadline, NULL) == 0) { + nsync_cv_wait_with_deadline (&q->non_full, &q->mu, CLOCK_REALTIME, abs_deadline, NULL) == 0) { } if (q->count != q->limit) { int i = q->pos + q->count; @@ -91,7 +92,7 @@ static void *cv_queue_get (cv_queue *q, nsync_time abs_deadline) { void *v = NULL; nsync_mu_lock (&q->mu); while (q->count == 0 && - nsync_cv_wait_with_deadline (&q->non_empty, &q->mu, abs_deadline, NULL) == 0) { + nsync_cv_wait_with_deadline (&q->non_empty, &q->mu, CLOCK_REALTIME, abs_deadline, NULL) == 0) { } if (q->count != 0) { v = q->data[q->pos]; @@ -237,7 +238,7 @@ static void test_cv_deadline (testing t) { nsync_time expected_end_time; start_time = nsync_time_now (); expected_end_time = nsync_time_add (start_time, nsync_time_ms (87)); - if (nsync_cv_wait_with_deadline (&cv, &mu, expected_end_time, + if (nsync_cv_wait_with_deadline (&cv, &mu, CLOCK_REALTIME, expected_end_time, NULL) != ETIMEDOUT) { TEST_FATAL (t, ("nsync_cv_wait() returned non-expired for a timeout")); } @@ -289,7 +290,7 @@ static void test_cv_cancel (testing t) { cancel = nsync_note_new (NULL, expected_end_time); - x = nsync_cv_wait_with_deadline (&cv, &mu, future_time, cancel); + x = nsync_cv_wait_with_deadline (&cv, &mu, CLOCK_REALTIME, future_time, cancel); if (x != ECANCELED) { TEST_FATAL (t, ("nsync_cv_wait() returned non-cancelled (%d) for " "a cancellation; expected %d", @@ -308,7 +309,7 @@ static void test_cv_cancel (testing t) { /* Check that an already cancelled wait returns immediately. */ start_time = nsync_time_now (); - x = nsync_cv_wait_with_deadline (&cv, &mu, nsync_time_no_deadline, cancel); + x = nsync_cv_wait_with_deadline (&cv, &mu, CLOCK_REALTIME, nsync_time_no_deadline, cancel); if (x != ECANCELED) { TEST_FATAL (t, ("nsync_cv_wait() returned non-cancelled (%d) for " "a cancellation; expected %d", diff --git a/third_party/nsync/testing/cv_wait_example_test.c b/third_party/nsync/testing/cv_wait_example_test.c index 20f7ac000..6368e526b 100644 --- a/third_party/nsync/testing/cv_wait_example_test.c +++ b/third_party/nsync/testing/cv_wait_example_test.c @@ -24,6 +24,7 @@ #include "third_party/nsync/testing/closure.h" #include "third_party/nsync/testing/smprintf.h" #include "third_party/nsync/testing/testing.h" +#include "libc/sysv/consts/clock.h" #include "third_party/nsync/testing/time_extra.h" /* Example use of CV.wait(): A priority queue of strings whose @@ -74,7 +75,8 @@ static const char *string_priority_queue_cv_remove_with_deadline (string_priorit const char *s = NULL; nsync_mu_lock (&q->mu); while (A_LEN (&q->heap) == 0 && - nsync_cv_wait_with_deadline (&q->non_empty, &q->mu, abs_deadline, NULL) == 0) { + nsync_cv_wait_with_deadline (&q->non_empty, &q->mu, CLOCK_REALTIME, + abs_deadline, NULL) == 0) { } alen = A_LEN (&q->heap); if (alen != 0) { diff --git a/third_party/nsync/testing/mu_test.c b/third_party/nsync/testing/mu_test.c index 5229e6957..4b4d6289c 100644 --- a/third_party/nsync/testing/mu_test.c +++ b/third_party/nsync/testing/mu_test.c @@ -24,6 +24,7 @@ #include "third_party/nsync/testing/closure.h" #include "third_party/nsync/testing/smprintf.h" #include "third_party/nsync/testing/testing.h" +#include "libc/sysv/consts/clock.h" #include "third_party/nsync/testing/time_extra.h" /* The state shared between the threads in each of the tests below. */ @@ -67,6 +68,7 @@ static void test_data_wait_for_all_threads (test_data *td) { while (td->finished_threads != td->n_threads) { nsync_cv_wait_with_deadline_generic (&td->done, td->mu_in_use, td->lock, td->unlock, + CLOCK_REALTIME, nsync_time_no_deadline, NULL); } (*td->unlock) (td->mu_in_use); @@ -303,7 +305,7 @@ static int counter_wait_for_zero_with_deadline (counter *c, nsync_time abs_deadl int value; nsync_mu_rlock (&c->mu); while (c->value != 0 && - nsync_cv_wait_with_deadline (&c->cv, &c->mu, abs_deadline, NULL) == 0) { + nsync_cv_wait_with_deadline (&c->cv, &c->mu, CLOCK_REALTIME, abs_deadline, NULL) == 0) { } value = c->value; nsync_mu_runlock (&c->mu); diff --git a/third_party/nsync/testing/pingpong_test.c b/third_party/nsync/testing/pingpong_test.c index 5d653bb95..00d1fa4cc 100644 --- a/third_party/nsync/testing/pingpong_test.c +++ b/third_party/nsync/testing/pingpong_test.c @@ -107,6 +107,7 @@ static void mutex_cv_ping_pong (ping_pong *pp, int parity) { nsync_cv_wait_with_deadline_generic (&pp->cv[parity], &pp->mutex, &void_pthread_mutex_lock, &void_pthread_mutex_unlock, + CLOCK_REALTIME, nsync_time_no_deadline, NULL); } pp->i++; @@ -163,7 +164,8 @@ static void mu_cv_unexpired_deadline_ping_pong (ping_pong *pp, int parity) { while (pp->i < pp->limit) { while ((pp->i & 1) == parity) { nsync_cv_wait_with_deadline (&pp->cv[parity], &pp->mu, - deadline_in1hour, NULL); + CLOCK_REALTIME, deadline_in1hour, + NULL); } pp->i++; nsync_cv_signal (&pp->cv[1 - parity]); @@ -318,6 +320,7 @@ static void rw_mutex_cv_ping_pong (ping_pong *pp, int parity) { nsync_cv_wait_with_deadline_generic (&pp->cv[parity], &pp->rwmutex, &void_pthread_rwlock_wrlock, &void_pthread_rwlock_unlock, + CLOCK_REALTIME, nsync_time_no_deadline, NULL); } pp->i++; diff --git a/third_party/nsync/waiter.h b/third_party/nsync/waiter.h index b1eeba29f..4bf9b801c 100644 --- a/third_party/nsync/waiter.h +++ b/third_party/nsync/waiter.h @@ -102,7 +102,7 @@ struct nsync_waitable_s { mu/lock/unlock are used to acquire and release the relevant locks whan waiting on condition variables. */ int nsync_wait_n(void *mu, void (*lock)(void *), void (*unlock)(void *), - nsync_time abs_deadline, int count, + int clock, nsync_time abs_deadline, int count, struct nsync_waitable_s *waitable[]); /* A "struct nsync_waitable_s" implementation must implement these diff --git a/third_party/openmp/kmp_lock.cpp b/third_party/openmp/kmp_lock.cpp index 95e734536..cc7be24da 100644 --- a/third_party/openmp/kmp_lock.cpp +++ b/third_party/openmp/kmp_lock.cpp @@ -380,7 +380,7 @@ __kmp_acquire_futex_lock_timed_template(kmp_futex_lock_t *lck, kmp_int32 gtid) { long rc; #ifdef __COSMOPOLITAN__ - if ((rc = nsync_futex_wait_((int *)&(lck->lk.poll), poll_val, false, NULL)) != 0) { + if ((rc = nsync_futex_wait_((int *)&(lck->lk.poll), poll_val, false, 0, NULL)) != 0) { #else if ((rc = syscall(__NR_futex, (int *)&(lck->lk.poll), FUTEX_WAIT, poll_val, NULL, NULL, 0)) != 0) {