From 7549a5755eefecd9d82ee13c1e879d3ef6cd8255 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Sun, 2 Oct 2022 11:56:27 -0700 Subject: [PATCH] Support futexes on FreeBSD --- libc/runtime/clone.c | 21 +++++----- libc/sysv/calls/sys_umtx_op.s | 2 + libc/sysv/consts.sh | 4 +- libc/sysv/consts/FUTEX_PRIVATE_FLAG.s | 2 +- libc/sysv/consts/FUTEX_WAKE.s | 2 +- libc/sysv/syscalls.sh | 3 +- libc/thread/freebsd.internal.h | 13 +++--- libc/thread/pthread_cond_timedwait.c | 4 +- libc/thread/wait0.c | 3 ++ third_party/nsync/futex.c | 58 ++++++++++++++++++++++----- 10 files changed, 81 insertions(+), 31 deletions(-) create mode 100644 libc/sysv/calls/sys_umtx_op.s diff --git a/libc/runtime/clone.c b/libc/runtime/clone.c index d90e64fbd..cebcedf18 100644 --- a/libc/runtime/clone.c +++ b/libc/runtime/clone.c @@ -112,7 +112,7 @@ WinThreadEntry(int rdi, // rcx // since we didn't indirect this function through NT2SYSV() it's not // safe to simply return, and as such, we just call ExitThread(). __imp_ExitThread(rc); - unreachable; + notpossible; } static textwindows int CloneWindows(int (*func)(void *, int), char *stk, @@ -192,7 +192,7 @@ XnuThreadMain(void *pthread, // rdi : "=m"(*wt->ztid) : "a"(0x2000000 | 361), "D"(0), "S"(0), "d"(0) : "rcx", "r10", "r11", "memory"); - unreachable; + notpossible; } static int CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags, @@ -239,12 +239,15 @@ static wontreturn void FreebsdThreadMain(void *p) { wt->func(wt->arg, wt->tid); // we no longer use the stack after this point // void thr_exit(%rdi = long *state); - asm volatile("movl\t$0,%0\n\t" // *wt->ztid = 0 - "syscall" // thr_exit() + asm volatile("movl\t$0,%0\n\t" // *wt->ztid = 0 + "syscall\n\t" // _umtx_op() + "movl\t$431,%%eax\n\t" // thr_exit() + "xor\t%%edi,%%edi\n\t" + "syscall" : "=m"(*wt->ztid) - : "a"(431), "D"(0) - : "rcx", "r11", "memory"); - unreachable; + : "a"(454), "D"(wt->ztid), "S"(UMTX_OP_WAKE) + : "rcx", "r8", "r9", "r10", "r11", "memory"); + notpossible; } static int CloneFreebsd(int (*func)(void *, int), char *stk, size_t stksz, @@ -318,7 +321,7 @@ noasan static wontreturn void OpenbsdThreadMain(void *p) { : "a"(83), "m"(oldrsp), "D"(wt->ztid), "S"(FUTEX_WAKE), "d"(INT_MAX) : "rcx", "r11", "memory"); - unreachable; + notpossible; } static int CloneOpenbsd(int (*func)(void *, int), char *stk, size_t stksz, @@ -372,7 +375,7 @@ static wontreturn void NetbsdThreadMain(void *arg, // rdi : "=a"(ax), "=d"(dx), "=m"(*ztid) : "0"(310) : "rcx", "r11", "memory"); - unreachable; + notpossible; } static int CloneNetbsd(int (*func)(void *, int), char *stk, size_t stksz, diff --git a/libc/sysv/calls/sys_umtx_op.s b/libc/sysv/calls/sys_umtx_op.s new file mode 100644 index 000000000..acb267c33 --- /dev/null +++ b/libc/sysv/calls/sys_umtx_op.s @@ -0,0 +1,2 @@ +.include "o/libc/sysv/macros.internal.inc" +.scall sys_umtx_op,0xffffff1c6fffffff,globl diff --git a/libc/sysv/consts.sh b/libc/sysv/consts.sh index 0850d4bcd..3351a3062 100755 --- a/libc/sysv/consts.sh +++ b/libc/sysv/consts.sh @@ -1316,9 +1316,9 @@ syscon rusage RUSAGE_BOTH -2 99 99 99 99 99 # woop # # group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon futex FUTEX_WAIT 0 0 0 1 0 0 -syscon futex FUTEX_WAKE 1 0 0 2 0 1 +syscon futex FUTEX_WAKE 1 0 1 2 0 1 syscon futex FUTEX_REQUEUE 3 0 0 3 0 0 -syscon futex FUTEX_PRIVATE_FLAG 128 0 0 128 0 0 +syscon futex FUTEX_PRIVATE_FLAG 128 0 128 128 0 0 # lio_listio() magnums # diff --git a/libc/sysv/consts/FUTEX_PRIVATE_FLAG.s b/libc/sysv/consts/FUTEX_PRIVATE_FLAG.s index 3d21804a5..debac8648 100644 --- a/libc/sysv/consts/FUTEX_PRIVATE_FLAG.s +++ b/libc/sysv/consts/FUTEX_PRIVATE_FLAG.s @@ -1,2 +1,2 @@ .include "o/libc/sysv/consts/syscon.internal.inc" -.syscon futex,FUTEX_PRIVATE_FLAG,128,0,0,128,0,0 +.syscon futex,FUTEX_PRIVATE_FLAG,128,0,128,128,0,0 diff --git a/libc/sysv/consts/FUTEX_WAKE.s b/libc/sysv/consts/FUTEX_WAKE.s index e9ed1e63e..740ca398a 100644 --- a/libc/sysv/consts/FUTEX_WAKE.s +++ b/libc/sysv/consts/FUTEX_WAKE.s @@ -1,2 +1,2 @@ .include "o/libc/sysv/consts/syscon.internal.inc" -.syscon futex,FUTEX_WAKE,1,0,0,2,0,1 +.syscon futex,FUTEX_WAKE,1,0,1,2,0,1 diff --git a/libc/sysv/syscalls.sh b/libc/sysv/syscalls.sh index 0619c452c..7d9ce94b8 100755 --- a/libc/sysv/syscalls.sh +++ b/libc/sysv/syscalls.sh @@ -18,6 +18,7 @@ ╚────────────────────────────────────────────────────────────────'>/dev/null #*/ dir=libc/sysv/calls . libc/sysv/gen.sh +#include "libc/calls/syscall-sysv.internal.h" # The Fifth Bell System Interface, Community Edition ┌─────────────────────────┐ # » so many numbers │ legend │ @@ -641,7 +642,7 @@ scall sys_bsdthread_register 0xfffffffff216efff globl hidden #scall write_nocancel 0xfffffffff218dfff globl #scall writev_nocancel 0xfffffffff219cfff globl #──────────────────────────FREEBSD─────────────────────────── -#scall _umtx_op 0xffffff1c6fffffff globl +scall sys_umtx_op 0xffffff1c6fffffff globl #scall abort2 0xffffff1cffffffff globl #scall afs3_syscall 0xffffff179fffffff globl #scall bindat 0xffffff21afffffff globl diff --git a/libc/thread/freebsd.internal.h b/libc/thread/freebsd.internal.h index b6e71a92d..d29c132f2 100644 --- a/libc/thread/freebsd.internal.h +++ b/libc/thread/freebsd.internal.h @@ -1,8 +1,8 @@ #ifndef COSMOPOLITAN_LIBC_THREAD_FREEBSD_INTERNAL_H_ #define COSMOPOLITAN_LIBC_THREAD_FREEBSD_INTERNAL_H_ -#include "libc/intrin/asmflag.h" #include "libc/calls/struct/timespec.h" #include "libc/errno.h" +#include "libc/intrin/asmflag.h" /** * @fileoverview FreeBSD Threading @@ -11,9 +11,12 @@ * maximum legal range is PID_MAX + 2 (100001) through INT_MAX */ -#define UMTX_OP_MUTEX_WAIT 17 -#define UMTX_OP_MUTEX_WAKE 18 -#define UMTX_ABSTIME 1 +#define UMTX_OP_WAIT 2 +#define UMTX_OP_WAIT_UINT 11 +#define UMTX_OP_WAIT_UINT_PRIVATE 15 +#define UMTX_OP_WAKE 3 +#define UMTX_OP_WAKE_PRIVATE 16 +#define UMTX_ABSTIME 1 #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ @@ -42,7 +45,7 @@ struct _umtx_time { uint32_t _clockid; }; -int _umtx_op(void *, int, unsigned long, void *, void *); +int sys_umtx_op(void *, int, unsigned long, void *, void *); COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ diff --git a/libc/thread/pthread_cond_timedwait.c b/libc/thread/pthread_cond_timedwait.c index 672685c3a..77da43c44 100644 --- a/libc/thread/pthread_cond_timedwait.c +++ b/libc/thread/pthread_cond_timedwait.c @@ -27,14 +27,14 @@ * Waits for condition with optional time limit, e.g. * * struct timespec ts; // one second timeout - * ts = _timespec_add(_timespec_mono(), _timespec_frommillis(1000)); + * ts = _timespec_add(_timespec_real(), _timespec_frommillis(1000)); * if (pthread_cond_timedwait(cond, mutex, &ts) == ETIMEDOUT) { * // handle timeout... * } * * @param mutex needs to be held by thread when calling this function * @param abstime may be null to wait indefinitely and should contain - * some arbitrary interval added to a `CLOCK_MONOTONIC` timestamp + * some arbitrary interval added to a `CLOCK_REALTIME` timestamp * @return 0 on success, or errno on error * @raise ETIMEDOUT if `abstime` was specified and the current time * exceeded its value diff --git a/libc/thread/wait0.c b/libc/thread/wait0.c index ea2a885ae..2c4bdf124 100644 --- a/libc/thread/wait0.c +++ b/libc/thread/wait0.c @@ -29,6 +29,7 @@ #include "libc/nt/runtime.h" #include "libc/nt/synchronization.h" #include "libc/sysv/consts/futex.h" +#include "libc/thread/freebsd.internal.h" #include "libc/thread/wait0.internal.h" int _futex(atomic_int *, int, int, const struct timespec *); @@ -64,6 +65,8 @@ static void _wait0_futex(const atomic_int *a, int e) { } else { rc = -GetLastError(); } + } else if (IsFreebsd()) { + rc = sys_umtx_op(a, UMTX_OP_WAIT_UINT, 0, 0, 0); } else { rc = _futex(a, op, e, 0); if (IsOpenbsd() && rc > 0) { diff --git a/third_party/nsync/futex.c b/third_party/nsync/futex.c index d7b300e52..5000ccc3d 100644 --- a/third_party/nsync/futex.c +++ b/third_party/nsync/futex.c @@ -25,7 +25,9 @@ #include "libc/limits.h" #include "libc/nt/runtime.h" #include "libc/nt/synchronization.h" +#include "libc/sysv/consts/clock.h" #include "libc/sysv/consts/futex.h" +#include "libc/thread/freebsd.internal.h" #include "libc/thread/thread.h" #include "third_party/nsync/common.internal.h" #include "third_party/nsync/futex.internal.h" @@ -52,6 +54,14 @@ __attribute__((__constructor__)) static void nsync_futex_init_ (void) { return; } + if (IsFreebsd ()) { + FUTEX_IS_SUPPORTED = true; + FUTEX_WAIT_ = FUTEX_WAIT; + FUTEX_PRIVATE_FLAG_ = FUTEX_PRIVATE_FLAG; + FUTEX_TIMEOUT_IS_ABSOLUTE = true; + return; + } + if (!(FUTEX_IS_SUPPORTED = IsLinux () || IsOpenbsd ())) { // we're using sched_yield() so let's // avoid needless clock_gettime calls @@ -75,14 +85,14 @@ __attribute__((__constructor__)) static void nsync_futex_init_ (void) { FUTEX_WAIT_ = FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME; FUTEX_PRIVATE_FLAG_ = FUTEX_PRIVATE_FLAG; FUTEX_TIMEOUT_IS_ABSOLUTE = true; - } else if (IsLinux () && + } else if (!IsTiny () && IsLinux () && _futex (&x, FUTEX_WAIT_BITSET, 1, 0, 0, FUTEX_BITSET_MATCH_ANY) == -EAGAIN) { FUTEX_WAIT_ = FUTEX_WAIT_BITSET; FUTEX_PRIVATE_FLAG_ = FUTEX_PRIVATE_FLAG; FUTEX_TIMEOUT_IS_ABSOLUTE = true; } else if (IsOpenbsd () || - (IsLinux () && + (!IsTiny () && IsLinux () && !_futex (&x, FUTEX_WAKE_PRIVATE, 1, 0, 0, 0))) { FUTEX_WAIT_ = FUTEX_WAIT; FUTEX_PRIVATE_FLAG_ = FUTEX_PRIVATE_FLAG; @@ -92,8 +102,9 @@ __attribute__((__constructor__)) static void nsync_futex_init_ (void) { } int nsync_futex_wait_ (int *p, int expect, char pshare, struct timespec *timeout) { - int rc, op; + size_t size; uint32_t ms; + int rc, op, fop; if (!FUTEX_IS_SUPPORTED) { nsync_yield_ (); @@ -105,6 +116,10 @@ int nsync_futex_wait_ (int *p, int expect, char pshare, struct timespec *timeout } op = FUTEX_WAIT_; + if (pshare == PTHREAD_PROCESS_PRIVATE) { + op |= FUTEX_PRIVATE_FLAG_; + } + if (NSYNC_FUTEX_WIN32 && IsWindows ()) { if (timeout) { ms = _timespec_tomillis (*timeout); @@ -116,10 +131,25 @@ int nsync_futex_wait_ (int *p, int expect, char pshare, struct timespec *timeout } else { rc = -GetLastError (); } - } else { - if (pshare == PTHREAD_PROCESS_PRIVATE) { - op |= FUTEX_PRIVATE_FLAG_; + } else if (IsFreebsd ()) { + struct _umtx_time *put, ut; + if (!timeout) { + put = 0; + size = 0; + } else { + ut._flags = UMTX_ABSTIME; + ut._clockid = CLOCK_REALTIME; + ut._timeout = *timeout; + put = &ut; + size = sizeof(ut); } + if (pshare) { + fop = UMTX_OP_WAIT_UINT; + } else { + fop = UMTX_OP_WAIT_UINT_PRIVATE; + } + rc = sys_umtx_op (p, fop, 0, &size, put); + } else { rc = _futex (p, op, expect, timeout, 0, FUTEX_WAIT_BITS_); if (IsOpenbsd() && rc > 0) { // [jart] openbsd does this without setting carry flag @@ -136,7 +166,7 @@ int nsync_futex_wait_ (int *p, int expect, char pshare, struct timespec *timeout } int nsync_futex_wake_ (int *p, int count, char pshare) { - int rc, op; + int rc, op, fop; int wake (void *, int, int) asm ("_futex"); ASSERT (count == 1 || count == INT_MAX); @@ -147,6 +177,10 @@ int nsync_futex_wake_ (int *p, int count, char pshare) { } op = FUTEX_WAKE; + if (pshare == PTHREAD_PROCESS_PRIVATE) { + op |= FUTEX_PRIVATE_FLAG_; + } + if (NSYNC_FUTEX_WIN32 && IsWindows ()) { if (count == 1) { WakeByAddressSingle (p); @@ -154,10 +188,14 @@ int nsync_futex_wake_ (int *p, int count, char pshare) { WakeByAddressAll (p); } rc = 0; - } else { - if (pshare == PTHREAD_PROCESS_PRIVATE) { - op |= FUTEX_PRIVATE_FLAG_; + } else if (IsFreebsd ()) { + if (pshare) { + fop = UMTX_OP_WAKE; + } else { + fop = UMTX_OP_WAKE_PRIVATE; } + rc = sys_umtx_op (p, fop, count, 0, 0); + } else { rc = wake (p, op, count); }