From b5cb71ab84e1978fdda72fa29679979c414f109a Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Sun, 11 Sep 2022 11:02:07 -0700 Subject: [PATCH] Use *NSYNC for POSIX threads locking APIs Condition variables, barriers, and r/w locks now work very well. --- Makefile | 3 + ape/loader.c | 28 +- examples/examples.mk | 1 + examples/greenbean.c | 17 +- libc/atomic.h | 1 + .../futex_wait.c => calls/_timespec_cmp.c} | 36 +- libc/calls/_timespec_real.c | 3 +- libc/calls/_timespec_sub.c | 14 +- libc/calls/siglock.c | 2 +- libc/calls/struct/timespec.h | 1 + libc/intrin/asan.c | 35 +- libc/intrin/asan.internal.h | 2 + libc/intrin/describeflags.internal.h | 3 +- libc/intrin/describefutexop.c | 49 ++- libc/intrin/extend.c | 8 +- libc/intrin/ftrace.c | 3 +- libc/intrin/futex.internal.h | 12 - libc/intrin/g_fds.c | 4 +- libc/intrin/kprintf.greg.c | 2 + libc/intrin/leaky.S | 39 +++ libc/intrin/leaky.internal.h | 16 + libc/{fmt => intrin}/lengthuint64.c | 0 libc/intrin/memtrack.greg.c | 9 +- libc/intrin/mmi.init.S | 2 +- libc/intrin/mmi_lock.c | 4 +- .../printmemoryintervals.c | 0 libc/{thread => intrin}/pthread_getspecific.c | 2 - libc/{thread => intrin}/pthread_key_create.c | 9 +- libc/{thread => intrin}/pthread_key_delete.c | 8 +- .../{thread => intrin}/pthread_key_destruct.c | 7 + libc/{thread => intrin}/pthread_keys.c | 3 + libc/intrin/pthread_mutex_init.c | 4 +- libc/intrin/pthread_mutex_lock.c | 40 +-- libc/intrin/pthread_mutex_trylock.c | 25 +- libc/intrin/pthread_mutex_unlock.c | 30 +- libc/intrin/pthread_once.c | 25 +- libc/{thread => intrin}/pthread_setspecific.c | 0 libc/intrin/wantcrashreports.c | 21 ++ libc/log/internal.h | 1 + libc/log/leaks.c | 11 +- libc/log/showcrashreports.c | 1 + libc/nt/kernel32/CreateSemaphoreW.s | 10 + libc/nt/master.sh | 2 +- libc/nt/synchronization.h | 4 + libc/runtime/memtrack64.txt | 2 +- libc/stdio/fdopen.c | 4 +- libc/stdio/fmemopen.c | 4 +- libc/stdio/stderr-init.S | 2 +- libc/stdio/stdin-init.S | 2 +- libc/stdio/stdout-init.S | 2 +- libc/sysv/consts.sh | 12 +- libc/sysv/consts/FUTEX_PRIVATE_FLAG.s | 2 +- libc/sysv/consts/FUTEX_REQUEUE.s | 2 +- libc/sysv/consts/FUTEX_WAKE.s | 2 +- libc/sysv/consts/SOMAXCONN.s | 2 +- libc/sysv/consts/__NR_futex.s | 2 +- libc/sysv/consts/futex.h | 14 +- libc/sysv/consts/ss.h | 2 +- libc/sysv/strace.greg.c | 3 +- libc/sysv/syscall.S | 57 +++- libc/testlib/globals.c | 5 +- libc/testlib/testmain.c | 4 +- libc/thread/README.md | 5 + libc/thread/atomic.h | 103 ------ libc/thread/clone.c | 2 +- libc/thread/common.h | 293 ---------------- libc/thread/compiler.h | 24 -- libc/thread/cputype.h | 21 -- libc/thread/dll.h | 78 ----- libc/thread/headers.h | 27 -- libc/thread/nsync.h | 28 -- libc/thread/nsync_atomic.h | 67 ---- libc/thread/{counter.c => nsync_counter.c} | 62 ++-- libc/thread/nsync_counter.h | 64 ---- libc/thread/nsync_cpp.h | 46 --- libc/thread/{cv.c => nsync_cv.c} | 59 ++-- libc/thread/nsync_cv.h | 150 --------- libc/thread/{debug.c => nsync_debug.c} | 57 ++-- libc/thread/nsync_debug.h | 55 --- libc/thread/nsync_mu.h | 115 ------- libc/thread/{mu_wait.c => nsync_mu_wait.c} | 55 +-- libc/thread/nsync_mu_wait.h | 129 ------- libc/thread/{note.c => nsync_note.c} | 61 ++-- libc/thread/nsync_note.h | 68 ---- libc/thread/{once.c => nsync_once.c} | 57 ++-- libc/thread/nsync_once.h | 51 --- libc/thread/nsync_panic.c | 42 --- libc/thread/nsync_sem_wait.c | 85 +++++ libc/thread/nsync_sem_wait_no_note.c | 33 ++ libc/thread/nsync_semaphore_futex.c | 132 -------- libc/thread/nsync_time.h | 62 ---- libc/thread/nsync_time_init.h | 21 -- libc/thread/nsync_time_internal.h | 215 ------------ libc/thread/nsync_wait.c | 108 ++++++ libc/thread/nsync_waiter.h | 153 --------- libc/thread/per_thread_waiter.c | 49 --- libc/thread/platform.h | 72 ---- libc/thread/posixthread.internal.h | 2 + libc/thread/pthread_barrier_destroy.c | 12 +- libc/thread/pthread_barrier_init.c | 18 +- libc/thread/pthread_barrier_wait.c | 27 +- libc/thread/pthread_barrierattr_getpshared.c | 4 +- libc/thread/pthread_barrierattr_setpshared.c | 5 +- libc/thread/pthread_cond_broadcast.c | 70 +--- libc/thread/pthread_cond_destroy.c | 8 +- libc/thread/pthread_cond_init.c | 2 +- .../pthread_cond_signal.c} | 46 ++- libc/thread/pthread_cond_timedwait.c | 49 +-- libc/thread/pthread_cond_wait.c | 9 +- libc/thread/pthread_condattr_getpshared.c | 2 +- libc/thread/pthread_condattr_setpshared.c | 5 +- libc/thread/pthread_create.c | 30 +- libc/thread/pthread_mutex_destroy.c | 6 - libc/thread/pthread_mutexattr_getpshared.c | 4 +- libc/thread/pthread_mutexattr_gettype.c | 2 +- libc/thread/pthread_mutexattr_setpshared.c | 4 +- libc/thread/pthread_mutexattr_settype.c | 2 +- libc/thread/pthread_rwlock_destroy.c | 8 +- libc/thread/pthread_rwlock_init.c | 2 +- libc/thread/pthread_rwlock_rdlock.c | 34 +- libc/thread/pthread_rwlock_unlock.c | 33 +- libc/thread/pthread_rwlock_wrlock.c | 34 +- libc/thread/pthread_rwlockattr_getpshared.c | 4 +- libc/thread/pthread_rwlockattr_setpshared.c | 5 +- libc/thread/sem.h | 47 --- libc/thread/sem_wait.c | 82 ----- libc/thread/sem_wait_no_note.c | 37 -- libc/thread/spawn.c | 3 +- libc/thread/start_thread.c | 41 --- libc/thread/thread.h | 76 ++--- libc/thread/thread.mk | 18 +- libc/thread/time_internal.c | 34 -- libc/thread/time_rep.c | 96 ------ libc/thread/wait.c | 104 ------ libc/{intrin => thread}/wait0.c | 28 +- libc/{intrin => thread}/wait0.internal.h | 0 libc/thread/wait_internal.h | 40 --- test/libc/calls/pledge_test.c | 2 +- test/libc/calls/reservefd_test.c | 4 +- test/libc/intrin/pthread_mutex_lock2_test.c | 84 ++++- test/libc/intrin/pthread_mutex_lock_test.c | 8 +- test/libc/intrin/pthread_once_test.c | 10 +- test/libc/intrin/rand64_test.c | 5 +- test/libc/intrin/test.mk | 3 +- test/libc/nexgen32e/stackrw_test.c | 9 +- test/libc/stdio/dtoa_test.c | 2 +- test/libc/thread/clone_test.c | 8 +- test/libc/thread/nsync_test.c | 119 +++++++ test/libc/thread/pthread_barrier_wait_test.c | 6 +- .../libc/thread/pthread_cond_broadcast_test.c | 123 ------- test/libc/thread/pthread_cond_signal_test.c | 118 +++++++ test/libc/thread/pthread_rwlock_rdlock_test.c | 16 +- test/libc/thread/pthread_setname_np_test.c | 3 +- test/libc/thread/spawn_test.c | 3 +- test/libc/thread/test.mk | 3 +- test/libc/x/makedirs_test.c | 3 +- test/libc/zipos/open_test.c | 6 +- test/tool/plinko/plinko_test.c | 1 + third_party/dlmalloc/dlmalloc.mk | 3 +- third_party/dlmalloc/locks.inc | 232 +------------ third_party/nsync/LICENSE.txt | 202 +++++++++++ third_party/nsync/README.md | 21 ++ third_party/nsync/atomic.h | 15 + third_party/nsync/atomic.internal.h | 113 +++++++ {libc/thread => third_party/nsync}/common.c | 114 +++---- third_party/nsync/common.internal.h | 318 ++++++++++++++++++ third_party/nsync/counter.h | 43 +++ third_party/nsync/cv.h | 157 +++++++++ third_party/nsync/debug.h | 39 +++ {libc/thread => third_party/nsync}/dll.c | 46 +-- third_party/nsync/dll.h | 69 ++++ third_party/nsync/futex.c | 124 +++++++ third_party/nsync/futex.internal.h | 15 + third_party/nsync/malloc.c | 49 +++ third_party/nsync/malloc.internal.h | 10 + {libc/thread => third_party/nsync}/mu.c | 56 +-- third_party/nsync/mu.h | 103 ++++++ third_party/nsync/mu_semaphore.c | 56 +++ third_party/nsync/mu_semaphore.h | 28 ++ third_party/nsync/mu_semaphore.internal.h | 20 ++ third_party/nsync/mu_semaphore_futex.c | 124 +++++++ third_party/nsync/mu_semaphore_win32.c | 84 +++++ third_party/nsync/mu_wait.h | 118 +++++++ third_party/nsync/note.h | 51 +++ third_party/nsync/nsync.mk | 58 ++++ third_party/nsync/once.h | 37 ++ third_party/nsync/panic.c | 30 ++ third_party/nsync/time.c | 54 +++ third_party/nsync/time.h | 53 +++ third_party/nsync/wait_s.internal.h | 26 ++ third_party/nsync/waiter.h | 138 ++++++++ third_party/nsync/waiter_per_thread.c | 56 +++ third_party/nsync/yield.c | 27 ++ third_party/third_party.mk | 1 + tool/build/mkdeps.c | 6 +- tool/build/pstrace.c | 10 +- tool/net/redbean.c | 7 +- 197 files changed, 3734 insertions(+), 3817 deletions(-) rename libc/{intrin/futex_wait.c => calls/_timespec_cmp.c} (63%) delete mode 100644 libc/intrin/futex.internal.h create mode 100644 libc/intrin/leaky.S create mode 100644 libc/intrin/leaky.internal.h rename libc/{fmt => intrin}/lengthuint64.c (100%) rename libc/{runtime => intrin}/printmemoryintervals.c (100%) rename libc/{thread => intrin}/pthread_getspecific.c (97%) rename libc/{thread => intrin}/pthread_key_create.c (94%) rename libc/{thread => intrin}/pthread_key_delete.c (93%) rename libc/{thread => intrin}/pthread_key_destruct.c (89%) rename libc/{thread => intrin}/pthread_keys.c (96%) rename libc/{thread => intrin}/pthread_setspecific.c (100%) create mode 100644 libc/intrin/wantcrashreports.c create mode 100644 libc/thread/README.md delete mode 100644 libc/thread/atomic.h delete mode 100644 libc/thread/common.h delete mode 100644 libc/thread/compiler.h delete mode 100644 libc/thread/cputype.h delete mode 100644 libc/thread/dll.h delete mode 100644 libc/thread/headers.h delete mode 100644 libc/thread/nsync.h delete mode 100644 libc/thread/nsync_atomic.h rename libc/thread/{counter.c => nsync_counter.c} (60%) delete mode 100644 libc/thread/nsync_counter.h delete mode 100644 libc/thread/nsync_cpp.h rename libc/thread/{cv.c => nsync_cv.c} (89%) delete mode 100644 libc/thread/nsync_cv.h rename libc/thread/{debug.c => nsync_debug.c} (83%) delete mode 100644 libc/thread/nsync_debug.h delete mode 100644 libc/thread/nsync_mu.h rename libc/thread/{mu_wait.c => nsync_mu_wait.c} (85%) delete mode 100644 libc/thread/nsync_mu_wait.h rename libc/thread/{note.c => nsync_note.c} (80%) delete mode 100644 libc/thread/nsync_note.h rename libc/thread/{once.c => nsync_once.c} (62%) delete mode 100644 libc/thread/nsync_once.h delete mode 100644 libc/thread/nsync_panic.c create mode 100644 libc/thread/nsync_sem_wait.c create mode 100644 libc/thread/nsync_sem_wait_no_note.c delete mode 100644 libc/thread/nsync_semaphore_futex.c delete mode 100644 libc/thread/nsync_time.h delete mode 100644 libc/thread/nsync_time_init.h delete mode 100644 libc/thread/nsync_time_internal.h create mode 100644 libc/thread/nsync_wait.c delete mode 100644 libc/thread/nsync_waiter.h delete mode 100644 libc/thread/per_thread_waiter.c delete mode 100644 libc/thread/platform.h rename libc/{intrin/futex_wake.c => thread/pthread_cond_signal.c} (69%) delete mode 100644 libc/thread/sem.h delete mode 100644 libc/thread/sem_wait.c delete mode 100644 libc/thread/sem_wait_no_note.c delete mode 100644 libc/thread/start_thread.c delete mode 100644 libc/thread/time_internal.c delete mode 100644 libc/thread/time_rep.c delete mode 100644 libc/thread/wait.c rename libc/{intrin => thread}/wait0.c (76%) rename libc/{intrin => thread}/wait0.internal.h (100%) delete mode 100644 libc/thread/wait_internal.h create mode 100644 test/libc/thread/nsync_test.c delete mode 100644 test/libc/thread/pthread_cond_broadcast_test.c create mode 100644 test/libc/thread/pthread_cond_signal_test.c create mode 100644 third_party/nsync/LICENSE.txt create mode 100644 third_party/nsync/README.md create mode 100644 third_party/nsync/atomic.h create mode 100644 third_party/nsync/atomic.internal.h rename {libc/thread => third_party/nsync}/common.c (75%) create mode 100644 third_party/nsync/common.internal.h create mode 100644 third_party/nsync/counter.h create mode 100644 third_party/nsync/cv.h create mode 100644 third_party/nsync/debug.h rename {libc/thread => third_party/nsync}/dll.c (67%) create mode 100644 third_party/nsync/dll.h create mode 100644 third_party/nsync/futex.c create mode 100644 third_party/nsync/futex.internal.h create mode 100644 third_party/nsync/malloc.c create mode 100644 third_party/nsync/malloc.internal.h rename {libc/thread => third_party/nsync}/mu.c (90%) create mode 100644 third_party/nsync/mu.h create mode 100644 third_party/nsync/mu_semaphore.c create mode 100644 third_party/nsync/mu_semaphore.h create mode 100644 third_party/nsync/mu_semaphore.internal.h create mode 100644 third_party/nsync/mu_semaphore_futex.c create mode 100644 third_party/nsync/mu_semaphore_win32.c create mode 100644 third_party/nsync/mu_wait.h create mode 100644 third_party/nsync/note.h create mode 100644 third_party/nsync/nsync.mk create mode 100644 third_party/nsync/once.h create mode 100644 third_party/nsync/panic.c create mode 100644 third_party/nsync/time.c create mode 100644 third_party/nsync/time.h create mode 100644 third_party/nsync/wait_s.internal.h create mode 100644 third_party/nsync/waiter.h create mode 100644 third_party/nsync/waiter_per_thread.c create mode 100644 third_party/nsync/yield.c diff --git a/Makefile b/Makefile index 73fed5bd3..a8c92725b 100644 --- a/Makefile +++ b/Makefile @@ -122,6 +122,7 @@ include libc/vga/vga.mk #─┘ include libc/calls/calls.mk #─┐ include libc/runtime/runtime.mk # ├──SYSTEMS RUNTIME include libc/crt/crt.mk # │ You can issue system calls +include third_party/nsync/nsync.mk # │ include third_party/dlmalloc/dlmalloc.mk #─┘ include libc/mem/mem.mk #─┐ include libc/zipos/zipos.mk # ├──DYNAMIC RUNTIME @@ -299,6 +300,7 @@ COSMOPOLITAN_OBJECTS = \ LIBC_MEM \ THIRD_PARTY_DLMALLOC \ LIBC_RUNTIME \ + THIRD_PARTY_NSYNC \ LIBC_ELF \ LIBC_CALLS \ LIBC_SYSV_CALLS \ @@ -338,6 +340,7 @@ COSMOPOLITAN_HEADERS = \ LIBC_RUNTIME \ LIBC_SOCK \ LIBC_STDIO \ + THIRD_PARTY_NSYNC \ THIRD_PARTY_XED \ LIBC_STR \ LIBC_SYSV \ diff --git a/ape/loader.c b/ape/loader.c index cc0600b25..1e0ae6950 100644 --- a/ape/loader.c +++ b/ape/loader.c @@ -179,7 +179,7 @@ struct ElfPhdr { extern char ehdr[]; extern char _end[]; -static void *syscall; +static void *syscall_; static char relocated; static struct PathSearcher ps; extern char __syscall_loader[]; @@ -275,7 +275,7 @@ __attribute__((__noreturn__)) static void Exit(int rc, int os) { asm volatile("call\t*%2" : /* no outputs */ : "a"((IsLinux() ? 60 : 1) | (IsXnu() ? 0x2000000 : 0)), "D"(rc), - "rm"(syscall) + "rm"(syscall_) : "memory"); __builtin_unreachable(); } @@ -285,7 +285,7 @@ static void Close(int fd, int os) { asm volatile("call\t*%4" : "=a"(ax), "=D"(di) : "0"((IsLinux() ? 3 : 6) | (IsXnu() ? 0x2000000 : 0)), "1"(fd), - "rm"(syscall) + "rm"(syscall_) : "rcx", "rdx", "rsi", "r8", "r9", "r10", "r11", "memory", "cc"); } @@ -295,7 +295,7 @@ static int Read(int fd, void *data, int size, int os) { asm volatile("call\t*%8" : "=a"(ax), "=D"(di), "=S"(si), "=d"(dx) : "0"((IsLinux() ? 0 : 3) | (IsXnu() ? 0x2000000 : 0)), "1"(fd), - "2"(data), "3"(size), "rm"(syscall) + "2"(data), "3"(size), "rm"(syscall_) : "rcx", "r8", "r9", "r10", "r11", "memory"); return ax; } @@ -306,7 +306,7 @@ static void Write(int fd, const void *data, int size, int os) { asm volatile("call\t*%8" : "=a"(ax), "=D"(di), "=S"(si), "=d"(dx) : "0"((IsLinux() ? 1 : 4) | (IsXnu() ? 0x2000000 : 0)), "1"(fd), - "2"(data), "3"(size), "rm"(syscall) + "2"(data), "3"(size), "rm"(syscall_) : "rcx", "r8", "r9", "r10", "r11", "memory", "cc"); } @@ -315,7 +315,7 @@ static void Execve(const char *prog, char **argv, char **envp, int os) { asm volatile("call\t*%8" : "=a"(ax), "=D"(di), "=S"(si), "=d"(dx) : "0"(59 | (IsXnu() ? 0x2000000 : 0)), "1"(prog), "2"(argv), - "3"(envp), "rm"(syscall) + "3"(envp), "rm"(syscall_) : "rcx", "r8", "r9", "r10", "r11", "memory", "cc"); } @@ -325,7 +325,7 @@ static int Access(const char *path, int mode, int os) { asm volatile("call\t*%7" : "=a"(ax), "=D"(di), "=S"(si), "=d"(dx) : "0"((IsLinux() ? 21 : 33) | (IsXnu() ? 0x2000000 : 0)), - "1"(path), "2"(mode), "rm"(syscall) + "1"(path), "2"(mode), "rm"(syscall_) : "rcx", "r8", "r9", "r10", "r11", "memory", "cc"); return ax; } @@ -338,7 +338,7 @@ static int Msyscall(long p, long n, int os) { } else { asm volatile("call\t*%6" : "=a"(ax), "=D"(di), "=S"(si) - : "0"(37), "1"(p), "2"(n), "rm"(syscall) + : "0"(37), "1"(p), "2"(n), "rm"(syscall_) : "rcx", "rdx", "r8", "r9", "r10", "r11", "memory", "cc"); return ax; } @@ -350,7 +350,7 @@ static int Open(const char *path, int flags, int mode, int os) { asm volatile("call\t*%8" : "=a"(ax), "=D"(di), "=S"(si), "=d"(dx) : "0"((IsLinux() ? 2 : 5) | (IsXnu() ? 0x2000000 : 0)), - "1"(path), "2"(flags), "3"(mode), "rm"(syscall) + "1"(path), "2"(flags), "3"(mode), "rm"(syscall_) : "rcx", "r8", "r9", "r10", "r11", "memory", "cc"); return ax; } @@ -369,7 +369,7 @@ __attribute__((__noinline__)) static long Mmap(long addr, long size, int prot, "pop\t%%r9" : "=a"(ax), "=D"(di), "=S"(si), "=d"(dx), "+r"(flags_), "+r"(fd_), "+r"(off_) - : "rm"(syscall), + : "rm"(syscall_), "0"((IsLinux() ? 9 : IsFreebsd() ? 477 : 197) | @@ -589,7 +589,7 @@ __attribute__((__noreturn__)) static void Spawn(int os, const char *exe, int fd, // since it probably means a userspace program executed this loader // and passed us a custom syscall function earlier. if (Msyscall(code, codesize, os) != -1) { - syscall = 0; + syscall_ = 0; } #if TROUBLESHOOT @@ -600,7 +600,7 @@ __attribute__((__noreturn__)) static void Spawn(int os, const char *exe, int fd, // to extend the behavior of this loader in the future. we don't need // to clear the xmm registers since the ape loader should be compiled // with the -mgeneral-regs-only flag. - register void *r8 asm("r8") = syscall; + register void *r8 asm("r8") = syscall_; asm volatile("xor\t%%eax,%%eax\n\t" "xor\t%%r9d,%%r9d\n\t" "xor\t%%r10d,%%r10d\n\t" @@ -660,9 +660,9 @@ __attribute__((__noreturn__)) void ApeLoader(long di, long *sp, char dl, // get syscall function pointer if (handoff && handoff->systemcall) { - syscall = handoff->systemcall; + syscall_ = handoff->systemcall; } else { - syscall = __syscall_loader; + syscall_ = __syscall_loader; } if (handoff) { diff --git a/examples/examples.mk b/examples/examples.mk index 1f1134f09..d18fe533d 100644 --- a/examples/examples.mk +++ b/examples/examples.mk @@ -80,6 +80,7 @@ EXAMPLES_DIRECTDEPS = \ THIRD_PARTY_LUA \ THIRD_PARTY_MBEDTLS \ THIRD_PARTY_MUSL \ + THIRD_PARTY_NSYNC \ THIRD_PARTY_QUICKJS \ THIRD_PARTY_STB \ THIRD_PARTY_XED \ diff --git a/examples/greenbean.c b/examples/greenbean.c index 379884ee3..484fa361c 100644 --- a/examples/greenbean.c +++ b/examples/greenbean.c @@ -8,6 +8,7 @@ ╚─────────────────────────────────────────────────────────────────*/ #endif #include "libc/assert.h" +#include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/calls/struct/sigaction.h" #include "libc/calls/struct/sigset.h" @@ -18,14 +19,11 @@ #include "libc/fmt/itoa.h" #include "libc/intrin/atomic.h" #include "libc/intrin/kprintf.h" -#include "libc/thread/thread.h" -#include "libc/intrin/wait0.internal.h" #include "libc/limits.h" #include "libc/log/check.h" #include "libc/log/log.h" #include "libc/macros.internal.h" #include "libc/mem/mem.h" -#include "libc/thread/tls.h" #include "libc/runtime/internal.h" #include "libc/runtime/runtime.h" #include "libc/runtime/stack.h" @@ -48,6 +46,9 @@ #include "libc/sysv/consts/sol.h" #include "libc/sysv/consts/tcp.h" #include "libc/thread/spawn.h" +#include "libc/thread/thread.h" +#include "libc/thread/tls.h" +#include "libc/thread/wait0.internal.h" #include "libc/time/struct/tm.h" #include "libc/time/time.h" #include "net/http/http.h" @@ -104,11 +105,11 @@ "Cache-Control: private; max-age=0\r\n" int threads; -_Atomic(int) workers; -_Atomic(int) messages; -_Atomic(int) listening; -_Atomic(int) connections; -_Atomic(int) closingtime; +atomic_int workers; +atomic_int messages; +atomic_int listening; +atomic_int connections; +atomic_int closingtime; const char *volatile status; void *Worker(void *id) { diff --git a/libc/atomic.h b/libc/atomic.h index dd8f831f8..f132e5ed2 100644 --- a/libc/atomic.h +++ b/libc/atomic.h @@ -1,5 +1,6 @@ #ifndef COSMOPOLITAN_LIBC_ATOMIC_H_ #define COSMOPOLITAN_LIBC_ATOMIC_H_ +#include "libc/inttypes.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ diff --git a/libc/intrin/futex_wait.c b/libc/calls/_timespec_cmp.c similarity index 63% rename from libc/intrin/futex_wait.c rename to libc/calls/_timespec_cmp.c index 0f7fbb904..46b9bf122 100644 --- a/libc/intrin/futex_wait.c +++ b/libc/calls/_timespec_cmp.c @@ -16,35 +16,15 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/strace.internal.h" #include "libc/calls/struct/timespec.h" -#include "libc/calls/struct/timespec.internal.h" -#include "libc/dce.h" -#include "libc/errno.h" -#include "libc/intrin/describeflags.internal.h" -#include "libc/intrin/futex.internal.h" -#include "libc/thread/thread.h" -#include "libc/sysv/consts/futex.h" -int _futex(void *, int, int, struct timespec *) hidden; - -int _futex_wait(void *addr, int expect, char pshared, - struct timespec *timeout) { - int op, ax, pf; - if (IsLinux() || IsOpenbsd()) { - pf = pshared == PTHREAD_PROCESS_PRIVATE ? FUTEX_PRIVATE_FLAG : 0; - op = FUTEX_WAIT | pf; - ax = _futex(addr, op, expect, timeout); - if (SupportsLinux() && pf && ax == -ENOSYS) { - // RHEL5 doesn't support FUTEX_PRIVATE_FLAG - op = FUTEX_WAIT; - ax = _futex(addr, op, expect, timeout); - } - if (IsOpenbsd() && ax > 0) ax = -ax; // yes openbsd does this w/o cf - STRACE("futex(%t, %s, %d, %s) → %s", addr, DescribeFutexOp(op), expect, - DescribeTimespec(0, timeout), DescribeFutexResult(ax)); - return ax; - } else { - return pthread_yield(); +/** + * Compares two nanosecond timestamps. + */ +int _timespec_cmp(struct timespec a, struct timespec b) { + int cmp; + if (!(cmp = (a.tv_sec > b.tv_sec) - (a.tv_sec < b.tv_sec))) { + cmp = (a.tv_nsec > b.tv_nsec) - (a.tv_nsec < b.tv_nsec); } + return cmp; } diff --git a/libc/calls/_timespec_real.c b/libc/calls/_timespec_real.c index 805c4b9d6..80912dc1f 100644 --- a/libc/calls/_timespec_real.c +++ b/libc/calls/_timespec_real.c @@ -16,7 +16,6 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" #include "libc/calls/struct/timespec.h" #include "libc/sysv/consts/clock.h" #include "libc/sysv/consts/nr.h" @@ -26,6 +25,6 @@ struct timespec _timespec_real(void) { int ax, dx; struct timespec ts; ax = clock_gettime(CLOCK_REALTIME_FAST, &ts); - assert(!ax); + if (ax) notpossible; return ts; } diff --git a/libc/calls/_timespec_sub.c b/libc/calls/_timespec_sub.c index 5ba0d0fb4..0f1fe4c35 100644 --- a/libc/calls/_timespec_sub.c +++ b/libc/calls/_timespec_sub.c @@ -21,12 +21,12 @@ /** * Subtracts two nanosecond timestamps. */ -struct timespec _timespec_sub(struct timespec x, struct timespec y) { - x.tv_sec -= y.tv_sec; - x.tv_nsec -= y.tv_nsec; - if (x.tv_nsec < 0) { - x.tv_nsec += 1000000000; - x.tv_sec -= 1; +struct timespec _timespec_sub(struct timespec a, struct timespec b) { + a.tv_sec -= b.tv_sec; + if (a.tv_nsec < b.tv_nsec) { + a.tv_nsec += 1000000000; + a.tv_sec--; } - return x; + a.tv_nsec -= b.tv_nsec; + return a; } diff --git a/libc/calls/siglock.c b/libc/calls/siglock.c index 84cf89582..9d3632dea 100644 --- a/libc/calls/siglock.c +++ b/libc/calls/siglock.c @@ -30,5 +30,5 @@ void(__sig_unlock)(void) { } __attribute__((__constructor__)) static void init(void) { - __sig_lock_obj.type = PTHREAD_MUTEX_RECURSIVE; + __sig_lock_obj._type = PTHREAD_MUTEX_RECURSIVE; } diff --git a/libc/calls/struct/timespec.h b/libc/calls/struct/timespec.h index 4c555e637..3bc0cb042 100644 --- a/libc/calls/struct/timespec.h +++ b/libc/calls/struct/timespec.h @@ -17,6 +17,7 @@ int utimensat(int, const char *, const struct timespec[2], int); int timespec_get(struct timespec *, int); int timespec_getres(struct timespec *, int); +int _timespec_cmp(struct timespec, struct timespec) pureconst; bool _timespec_eq(struct timespec, struct timespec) pureconst; bool _timespec_gte(struct timespec, struct timespec) pureconst; int64_t _timespec_tomicros(struct timespec) pureconst; diff --git a/libc/intrin/asan.c b/libc/intrin/asan.c index 9c3887746..dc0315568 100644 --- a/libc/intrin/asan.c +++ b/libc/intrin/asan.c @@ -25,6 +25,7 @@ #include "libc/intrin/asan.internal.h" #include "libc/intrin/asancodes.h" #include "libc/intrin/kprintf.h" +#include "libc/intrin/leaky.internal.h" #include "libc/intrin/likely.h" #include "libc/intrin/lockcmpxchg.h" #include "libc/intrin/nomultics.internal.h" @@ -170,7 +171,7 @@ struct ReportOriginHeap { }; static int __asan_noreentry; -static pthread_mutex_t __asan_lock; +static pthread_spinlock_t __asan_lock; static struct AsanMorgue __asan_morgue; #define __asan_unreachable() \ @@ -867,25 +868,25 @@ dontdiscard __asan_die_f *__asan_report_memory_fault(void *addr, int size, void *__asan_morgue_add(void *p) { int i; void *r; - if (__threaded) pthread_mutex_lock(&__asan_lock); + if (__threaded) pthread_spin_lock(&__asan_lock); i = __asan_morgue.i++ & (ARRAYLEN(__asan_morgue.p) - 1); r = __asan_morgue.p[i]; __asan_morgue.p[i] = p; - if (__threaded) pthread_mutex_unlock(&__asan_lock); + if (__threaded) pthread_spin_unlock(&__asan_lock); return r; } static void __asan_morgue_flush(void) { int i; void *p; - if (__threaded) pthread_mutex_lock(&__asan_lock); + if (__threaded) pthread_spin_lock(&__asan_lock); for (i = 0; i < ARRAYLEN(__asan_morgue.p); ++i) { if (__asan_morgue.p[i] && weaken(dlfree)) { weaken(dlfree)(__asan_morgue.p[i]); } __asan_morgue.p[i] = 0; } - if (__threaded) pthread_mutex_unlock(&__asan_lock); + if (__threaded) pthread_spin_unlock(&__asan_lock); } static size_t __asan_user_size(size_t n) { @@ -1052,6 +1053,30 @@ int __asan_print_trace(void *p) { return 0; } +// Returns true if `p` was allocated by an IGNORE_LEAKS(function). +int __asan_is_leaky(void *p) { + int sym; + size_t c, i, n; + intptr_t f, *l; + struct AsanExtra *e; + struct SymbolTable *st; + if (!weaken(GetSymbolTable)) notpossible; + if (!(e = __asan_get_extra(p, &c))) return 0; + if (!__asan_read48(e->size, &n)) return 0; + if (!__asan_is_mapped((((intptr_t)p >> 3) + 0x7fff8000) >> 16)) return 0; + if (!(st = GetSymbolTable())) return 0; + for (i = 0; i < ARRAYLEN(e->bt.p) && e->bt.p[i]; ++i) { + if ((sym = weaken(__get_symbol)(st, e->bt.p[i])) == -1) continue; + f = st->addr_base + st->symbols[sym].x; + for (l = _leaky_start; l < _leaky_end; ++l) { + if (f == *l) { + return 1; + } + } + } + return 0; +} + static void __asan_deallocate(char *p, long kind) { size_t c, n; struct AsanExtra *e; diff --git a/libc/intrin/asan.internal.h b/libc/intrin/asan.internal.h index 5dc92372f..eee7b4797 100644 --- a/libc/intrin/asan.internal.h +++ b/libc/intrin/asan.internal.h @@ -3,6 +3,7 @@ #include "libc/calls/struct/iovec.h" #include "libc/intrin/asancodes.h" #include "libc/macros.internal.h" +#include "libc/thread/thread.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ @@ -29,6 +30,7 @@ struct AsanFault __asan_check(const void *, long) nosideeffect; void __asan_free(void *); void *__asan_malloc(size_t); +int __asan_is_leaky(void *); int __asan_malloc_trim(size_t); int __asan_print_trace(void *); void *__asan_calloc(size_t, size_t); diff --git a/libc/intrin/describeflags.internal.h b/libc/intrin/describeflags.internal.h index 0bc065de1..afb87b598 100644 --- a/libc/intrin/describeflags.internal.h +++ b/libc/intrin/describeflags.internal.h @@ -17,7 +17,7 @@ const char *DescribeCapability(char[20], int); const char *DescribeClockName(char[32], int); const char *DescribeDirfd(char[12], int); const char *DescribeFrame(char[32], int); -const char *DescribeFutexOp(int); +const char *DescribeFutexOp(char[64], int); const char *DescribeFutexResult(char[12], int); const char *DescribeHow(char[12], int); const char *DescribeMapFlags(char[64], int); @@ -60,6 +60,7 @@ const char *DescribeWhence(char[12], int); #define DescribeClockName(x) DescribeClockName(alloca(32), x) #define DescribeDirfd(x) DescribeDirfd(alloca(12), x) #define DescribeFrame(x) DescribeFrame(alloca(32), x) +#define DescribeFutexOp(x) DescribeFutexOp(alloca(64), x) #define DescribeFutexResult(x) DescribeFutexResult(alloca(12), x) #define DescribeHow(x) DescribeHow(alloca(12), x) #define DescribeMapFlags(x) DescribeMapFlags(alloca(64), x) diff --git a/libc/intrin/describefutexop.c b/libc/intrin/describefutexop.c index f956658d0..80993d8d7 100644 --- a/libc/intrin/describefutexop.c +++ b/libc/intrin/describefutexop.c @@ -16,16 +16,47 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/fmt/itoa.h" #include "libc/intrin/describeflags.internal.h" +#include "libc/str/str.h" #include "libc/sysv/consts/futex.h" -const char *DescribeFutexOp(int x) { - if (x == FUTEX_WAIT) return "FUTEX_WAIT"; - if (x == FUTEX_WAKE) return "FUTEX_WAKE"; - if (x == FUTEX_REQUEUE) return "FUTEX_REQUEUE"; - // order matters (the private bit might be zero) - if (x == FUTEX_WAIT_PRIVATE) return "FUTEX_WAIT_PRIVATE"; - if (x == FUTEX_WAKE_PRIVATE) return "FUTEX_WAKE_PRIVATE"; - if (x == FUTEX_REQUEUE_PRIVATE) return "FUTEX_REQUEUE_PRIVATE"; - return "FUTEX_???"; +const char *(DescribeFutexOp)(char buf[64], int x) { + + bool priv = false; + if (x & FUTEX_PRIVATE_FLAG) { + priv = true; + x &= ~FUTEX_PRIVATE_FLAG; + } + + bool real = false; + if (x & FUTEX_CLOCK_REALTIME) { + real = true; + x &= ~FUTEX_CLOCK_REALTIME; + } + + char *p = buf; + + if (x == FUTEX_WAIT) { + p = stpcpy(p, "FUTEX_WAIT"); + } else if (x == FUTEX_WAKE) { + p = stpcpy(p, "FUTEX_WAKE"); + } else if (x == FUTEX_REQUEUE) { + p = stpcpy(p, "FUTEX_REQUEUE"); + } else if (x == FUTEX_WAIT_BITSET) { + p = stpcpy(p, "FUTEX_WAIT_BITSET"); + } else { + p = stpcpy(p, "FUTEX_"); + p = FormatUint32(p, x); + } + + if (priv) { + p = stpcpy(p, "_PRIVATE"); + } + + if (real) { + p = stpcpy(p, "|FUTEX_CLOCK_REALTIME"); + } + + return buf; } diff --git a/libc/intrin/extend.c b/libc/intrin/extend.c index 11da2f91f..3a24354f4 100644 --- a/libc/intrin/extend.c +++ b/libc/intrin/extend.c @@ -37,11 +37,12 @@ static void _mapframe(void *p) { if ((dm = sys_mmap(p, G, prot, flags, -1, 0)).addr != p) { notpossible; } - if (TrackMemoryInterval(&_mmi, (uintptr_t)p >> 16, - ((uintptr_t)p + G - 1) >> 16, dm.maphandle, prot, - flags, false, false, 0, G)) { + __mmi_lock(); + if (TrackMemoryInterval(&_mmi, (uintptr_t)p >> 16, (uintptr_t)p >> 16, + dm.maphandle, prot, flags, false, false, 0, G)) { notpossible; } + __mmi_unlock(); } /** @@ -80,5 +81,6 @@ noasan void *_extend(void *p, size_t n, void *e, intptr_t h) { *SHADOW(q) = 0; } } + asm("mfence"); return q; } diff --git a/libc/intrin/ftrace.c b/libc/intrin/ftrace.c index 202fe746c..e63f11d68 100644 --- a/libc/intrin/ftrace.c +++ b/libc/intrin/ftrace.c @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/atomic.h" #include "libc/runtime/runtime.h" /** @@ -39,4 +40,4 @@ * though under normal circumstances, `__ftrace` should only be either * zero or one. */ -_Atomic(int) __ftrace; +atomic_int __ftrace; diff --git a/libc/intrin/futex.internal.h b/libc/intrin/futex.internal.h deleted file mode 100644 index 732c10891..000000000 --- a/libc/intrin/futex.internal.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef COSMOPOLITAN_LIBC_INTRIN_FUTEX_INTERNAL_H_ -#define COSMOPOLITAN_LIBC_INTRIN_FUTEX_INTERNAL_H_ -#include "libc/calls/struct/timespec.h" -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -int _futex_wait(void *, int, char, struct timespec *) hidden; -int _futex_wake(void *, int, char) hidden; - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* COSMOPOLITAN_LIBC_INTRIN_FUTEX_INTERNAL_H_ */ diff --git a/libc/intrin/g_fds.c b/libc/intrin/g_fds.c index c0e187cbc..f3c0cfe8b 100644 --- a/libc/intrin/g_fds.c +++ b/libc/intrin/g_fds.c @@ -19,11 +19,11 @@ #include "libc/calls/extend.internal.h" #include "libc/calls/internal.h" #include "libc/calls/state.internal.h" -#include "libc/thread/thread.h" #include "libc/intrin/pushpop.h" #include "libc/intrin/weaken.h" #include "libc/nt/runtime.h" #include "libc/sysv/consts/o.h" +#include "libc/thread/thread.h" STATIC_YOINK("_init_g_fds"); @@ -41,7 +41,7 @@ static textwindows dontinline void SetupWinStd(struct Fds *fds, int i, int x) { textstartup void InitializeFileDescriptors(void) { struct Fds *fds; - __fds_lock_obj.type = PTHREAD_MUTEX_RECURSIVE; + __fds_lock_obj._type = PTHREAD_MUTEX_RECURSIVE; fds = VEIL("r", &g_fds); fds->p = fds->e = (void *)0x6fe000040000; fds->n = 4; diff --git a/libc/intrin/kprintf.greg.c b/libc/intrin/kprintf.greg.c index bd8db531d..fd072467d 100644 --- a/libc/intrin/kprintf.greg.c +++ b/libc/intrin/kprintf.greg.c @@ -24,6 +24,8 @@ #include "libc/errno.h" #include "libc/fmt/divmod10.internal.h" #include "libc/fmt/fmt.h" +#include "libc/intrin/asan.internal.h" +#include "libc/intrin/asancodes.h" #include "libc/intrin/bits.h" #include "libc/intrin/cmpxchg.h" #include "libc/intrin/kprintf.h" diff --git a/libc/intrin/leaky.S b/libc/intrin/leaky.S new file mode 100644 index 000000000..d79d752d2 --- /dev/null +++ b/libc/intrin/leaky.S @@ -0,0 +1,39 @@ +/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│ +│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/macros.internal.h" + +// Decentralized section for leaky functions. + .section .piro.relo.sort.leaky.1,"aw",@nobits + .type _leaky_start,@object + .type _leaky_end,@object + .globl _leaky_start,_leaky_end + .hidden _leaky_start,_leaky_end + .byte 0 + .align __SIZEOF_POINTER__ + .underrun +_leaky_start: + .previous/* + ... + decentralized content + ... + */.section .piro.relo.sort.leaky.3,"aw",@nobits +_leaky_end: + .quad 0 + .overrun + .previous diff --git a/libc/intrin/leaky.internal.h b/libc/intrin/leaky.internal.h new file mode 100644 index 000000000..225671a9a --- /dev/null +++ b/libc/intrin/leaky.internal.h @@ -0,0 +1,16 @@ +#ifndef COSMOPOLITAN_LIBC_INTRIN_LEAKY_INTERNAL_H_ +#define COSMOPOLITAN_LIBC_INTRIN_LEAKY_INTERNAL_H_ +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +#define IGNORE_LEAKS(FUNC) \ + STATIC_YOINK("_leaky_start"); \ + void *_leaky_##FUNC[] _Section(".piro.relo.sort.leaky.2." #FUNC \ + ",\"aw\",@init_array #") = {FUNC} + +extern intptr_t _leaky_end[] __attribute__((__weak__)); +extern intptr_t _leaky_start[] __attribute__((__weak__)); + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* COSMOPOLITAN_LIBC_INTRIN_LEAKY_INTERNAL_H_ */ diff --git a/libc/fmt/lengthuint64.c b/libc/intrin/lengthuint64.c similarity index 100% rename from libc/fmt/lengthuint64.c rename to libc/intrin/lengthuint64.c diff --git a/libc/intrin/memtrack.greg.c b/libc/intrin/memtrack.greg.c index b35066329..4f5a2482a 100644 --- a/libc/intrin/memtrack.greg.c +++ b/libc/intrin/memtrack.greg.c @@ -17,14 +17,14 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" -#include "libc/intrin/bits.h" -#include "libc/intrin/likely.h" -#include "libc/intrin/weaken.h" #include "libc/calls/calls.h" #include "libc/calls/strace.internal.h" #include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/asan.internal.h" +#include "libc/intrin/bits.h" +#include "libc/intrin/likely.h" +#include "libc/intrin/weaken.h" #include "libc/log/libfatal.internal.h" #include "libc/log/log.h" #include "libc/macros.internal.h" @@ -201,6 +201,9 @@ int TrackMemoryInterval(struct MemoryIntervals *mm, int x, int y, long h, unsigned i; #if IsModeDbg() assert(y >= x); + if (!AreMemoryIntervalsOk(mm)) { + PrintMemoryIntervals(2, mm); + } assert(AreMemoryIntervalsOk(mm)); #endif diff --git a/libc/intrin/mmi.init.S b/libc/intrin/mmi.init.S index 1f0a77bc8..5910d3c0f 100644 --- a/libc/intrin/mmi.init.S +++ b/libc/intrin/mmi.init.S @@ -22,5 +22,5 @@ .init.start 200,_init__mmi movb $OPEN_MAX,_mmi+8 movl $_mmi+24,_mmi+16 - movb $PTHREAD_MUTEX_RECURSIVE,__mmi_lock_obj(%rip) + movb $PTHREAD_MUTEX_RECURSIVE,__mmi_lock_obj+16(%rip) .init.end 200,_init__mmi diff --git a/libc/intrin/mmi_lock.c b/libc/intrin/mmi_lock.c index 3c4e4c672..0bcbed171 100644 --- a/libc/intrin/mmi_lock.c +++ b/libc/intrin/mmi_lock.c @@ -16,8 +16,10 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" #include "libc/runtime/memtrack.internal.h" +#include "libc/thread/thread.h" + +// this lock currently needs to be (1) recursive and (2) not nsync extern pthread_mutex_t __mmi_lock_obj; diff --git a/libc/runtime/printmemoryintervals.c b/libc/intrin/printmemoryintervals.c similarity index 100% rename from libc/runtime/printmemoryintervals.c rename to libc/intrin/printmemoryintervals.c diff --git a/libc/thread/pthread_getspecific.c b/libc/intrin/pthread_getspecific.c similarity index 97% rename from libc/thread/pthread_getspecific.c rename to libc/intrin/pthread_getspecific.c index 399350e7f..404dcb1fd 100644 --- a/libc/thread/pthread_getspecific.c +++ b/libc/intrin/pthread_getspecific.c @@ -16,10 +16,8 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/errno.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/thread.h" -#include "libc/thread/tls.h" /** * Gets value of TLS slot for current thread. diff --git a/libc/thread/pthread_key_create.c b/libc/intrin/pthread_key_create.c similarity index 94% rename from libc/thread/pthread_key_create.c rename to libc/intrin/pthread_key_create.c index 5da4b9358..e7c7a9c98 100644 --- a/libc/thread/pthread_key_create.c +++ b/libc/intrin/pthread_key_create.c @@ -27,17 +27,20 @@ * Allocates TLS slot. */ int pthread_key_create(pthread_key_t *key, pthread_key_dtor dtor) { - int i, j; + int i, j, rc = EAGAIN; + pthread_spin_lock(&_pthread_keys_lock); for (i = 0; i < (PTHREAD_KEYS_MAX + 63) / 64; ++i) { if (~_pthread_key_usage[i]) { j = bsrl(~_pthread_key_usage[i]); _pthread_key_usage[i] |= 1ul << j; _pthread_key_dtor[i * 64 + j] = dtor; *key = i * 64 + j; - return 0; + rc = 0; + break; } } - return EAGAIN; + pthread_spin_unlock(&_pthread_keys_lock); + return rc; } static textexit void _pthread_key_atexit(void) { diff --git a/libc/thread/pthread_key_delete.c b/libc/intrin/pthread_key_delete.c similarity index 93% rename from libc/thread/pthread_key_delete.c rename to libc/intrin/pthread_key_delete.c index af05dda8e..b78120c21 100644 --- a/libc/thread/pthread_key_delete.c +++ b/libc/intrin/pthread_key_delete.c @@ -24,11 +24,15 @@ * Deletes TLS slot. */ int pthread_key_delete(pthread_key_t key) { + int rc; + pthread_spin_lock(&_pthread_keys_lock); if (key < PTHREAD_KEYS_MAX) { _pthread_key_usage[key / 64] &= ~(1ul << (key % 64)); _pthread_key_dtor[key] = 0; - return 0; + rc = 0; } else { - return EINVAL; + rc = EINVAL; } + pthread_spin_unlock(&_pthread_keys_lock); + return rc; } diff --git a/libc/thread/pthread_key_destruct.c b/libc/intrin/pthread_key_destruct.c similarity index 89% rename from libc/thread/pthread_key_destruct.c rename to libc/intrin/pthread_key_destruct.c index 60c914e05..cfad22162 100644 --- a/libc/thread/pthread_key_destruct.c +++ b/libc/intrin/pthread_key_destruct.c @@ -18,12 +18,16 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/nexgen32e/bsr.h" #include "libc/thread/posixthread.internal.h" +#include "libc/thread/thread.h" +#include "libc/thread/tls.h" void _pthread_key_destruct(void *key[PTHREAD_KEYS_MAX]) { int i, j; uint64_t x; void *value; pthread_key_dtor dtor; + if (!__tls_enabled) return; + pthread_spin_lock(&_pthread_keys_lock); if (!key) key = _pthread_keys; StartOver: for (i = 0; i < (PTHREAD_KEYS_MAX + 63) / 64; ++i) { @@ -32,10 +36,13 @@ StartOver: j = bsrl(x); if ((value = key[i * 64 + j]) && (dtor = _pthread_key_dtor[i * 64 + j])) { key[i * 64 + j] = 0; + pthread_spin_unlock(&_pthread_keys_lock); dtor(value); + pthread_spin_lock(&_pthread_keys_lock); goto StartOver; } x &= ~(1ul << j); } } + pthread_spin_unlock(&_pthread_keys_lock); } diff --git a/libc/thread/pthread_keys.c b/libc/intrin/pthread_keys.c similarity index 96% rename from libc/thread/pthread_keys.c rename to libc/intrin/pthread_keys.c index 5be1cf30f..69be8bafd 100644 --- a/libc/thread/pthread_keys.c +++ b/libc/intrin/pthread_keys.c @@ -17,6 +17,9 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/thread/posixthread.internal.h" +#include "libc/thread/thread.h" + +pthread_spinlock_t _pthread_keys_lock; // tls value slots for pthread keys api _Thread_local void *_pthread_keys[PTHREAD_KEYS_MAX]; diff --git a/libc/intrin/pthread_mutex_init.c b/libc/intrin/pthread_mutex_init.c index 242aff95e..f7fda57cd 100644 --- a/libc/intrin/pthread_mutex_init.c +++ b/libc/intrin/pthread_mutex_init.c @@ -27,8 +27,8 @@ int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr) { *mutex = (pthread_mutex_t){ - attr ? attr->type : 0, - attr ? attr->pshared : 0, + ._type = attr ? attr->_type : 0, + ._pshared = attr ? attr->_pshared : 0, }; return 0; } diff --git a/libc/intrin/pthread_mutex_lock.c b/libc/intrin/pthread_mutex_lock.c index 01fb50c41..2d4d04818 100644 --- a/libc/intrin/pthread_mutex_lock.c +++ b/libc/intrin/pthread_mutex_lock.c @@ -16,12 +16,14 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" #include "libc/calls/calls.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" -#include "libc/intrin/futex.internal.h" +#include "libc/intrin/likely.h" +#include "libc/intrin/weaken.h" #include "libc/thread/thread.h" +#include "libc/thread/tls.h" +#include "third_party/nsync/mu.h" /** * Locks mutex. @@ -60,28 +62,18 @@ int pthread_mutex_lock(pthread_mutex_t *mutex) { int c, d, t; - if (mutex->type == PTHREAD_MUTEX_NORMAL) { - // From Futexes Are Tricky Version 1.1 § Mutex, Take 3; - // Ulrich Drepper, Red Hat Incorporated, June 27, 2004. - c = 0; - if (!atomic_compare_exchange_strong_explicit( - &mutex->lock, &c, 1, memory_order_acquire, memory_order_relaxed)) { - if (c != 2) { - c = atomic_exchange_explicit(&mutex->lock, 2, memory_order_acquire); - } - while (c) { - _futex_wait(&mutex->lock, 2, mutex->pshared, 0); - c = atomic_exchange_explicit(&mutex->lock, 2, memory_order_acquire); - } - } + if (LIKELY(__tls_enabled && // + mutex->_type == PTHREAD_MUTEX_NORMAL && // + mutex->_pshared == PTHREAD_PROCESS_PRIVATE && // + weaken(nsync_mu_lock))) { + weaken(nsync_mu_lock)((nsync_mu *)mutex); return 0; } t = gettid(); - if (mutex->type == PTHREAD_MUTEX_ERRORCHECK) { - c = atomic_load_explicit(&mutex->lock, memory_order_relaxed); + if (mutex->_type == PTHREAD_MUTEX_ERRORCHECK) { + c = atomic_load_explicit(&mutex->_lock, memory_order_relaxed); if ((c & 0x000fffff) == t) { - assert(!"deadlock"); return EDEADLK; } } @@ -90,29 +82,29 @@ int pthread_mutex_lock(pthread_mutex_t *mutex) { c = 0; d = 0x10100000 | t; if (atomic_compare_exchange_weak_explicit( - &mutex->lock, &c, d, memory_order_acquire, memory_order_relaxed)) { + &mutex->_lock, &c, d, memory_order_acquire, memory_order_relaxed)) { break; } else { if ((c & 0x000fffff) == t) { if ((c & 0x0ff00000) < 0x0ff00000) { - c = atomic_fetch_add_explicit(&mutex->lock, 0x00100000, + c = atomic_fetch_add_explicit(&mutex->_lock, 0x00100000, memory_order_relaxed); break; } else { - assert(!"recurse"); return EAGAIN; } } if ((c & 0xf0000000) == 0x10000000) { d = 0x20000000 | c; - if (atomic_compare_exchange_weak_explicit(&mutex->lock, &c, d, + if (atomic_compare_exchange_weak_explicit(&mutex->_lock, &c, d, memory_order_acquire, memory_order_relaxed)) { c = d; } } if ((c & 0xf0000000) == 0x20000000) { - _futex_wait(&mutex->lock, c, mutex->pshared, 0); + // _futex_wait(&mutex->_lock, c, mutex->_pshared, 0); + pthread_yield(); } } } diff --git a/libc/intrin/pthread_mutex_trylock.c b/libc/intrin/pthread_mutex_trylock.c index e162cb521..9621115e0 100644 --- a/libc/intrin/pthread_mutex_trylock.c +++ b/libc/intrin/pthread_mutex_trylock.c @@ -20,7 +20,11 @@ #include "libc/calls/calls.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" +#include "libc/intrin/likely.h" +#include "libc/intrin/weaken.h" #include "libc/thread/thread.h" +#include "libc/thread/tls.h" +#include "third_party/nsync/mu.h" /** * Locks mutex if it isn't locked already. @@ -35,10 +39,21 @@ int pthread_mutex_trylock(pthread_mutex_t *mutex) { int c, d, t; - if (mutex->type == PTHREAD_MUTEX_NORMAL) { + if (LIKELY(__tls_enabled && // + mutex->_type == PTHREAD_MUTEX_NORMAL && // + mutex->_pshared == PTHREAD_PROCESS_PRIVATE && // + weaken(nsync_mu_trylock))) { + if (weaken(nsync_mu_trylock)((nsync_mu *)mutex)) { + return 0; + } else { + return EBUSY; + } + } + + if (mutex->_type == PTHREAD_MUTEX_NORMAL) { c = 0; if (atomic_compare_exchange_strong_explicit( - &mutex->lock, &c, 1, memory_order_acquire, memory_order_relaxed)) { + &mutex->_lock, &c, 1, memory_order_acquire, memory_order_relaxed)) { return 0; } else { return EBUSY; @@ -49,14 +64,14 @@ int pthread_mutex_trylock(pthread_mutex_t *mutex) { t = gettid(); d = 0x10100000 | t; if (!atomic_compare_exchange_strong_explicit( - &mutex->lock, &c, d, memory_order_acquire, memory_order_relaxed)) { - if ((c & 0x000fffff) != t || mutex->type == PTHREAD_MUTEX_ERRORCHECK) { + &mutex->_lock, &c, d, memory_order_acquire, memory_order_relaxed)) { + if ((c & 0x000fffff) != t || mutex->_type == PTHREAD_MUTEX_ERRORCHECK) { return EBUSY; } if ((c & 0x0ff00000) == 0x0ff00000) { return EAGAIN; } - atomic_fetch_add_explicit(&mutex->lock, 0x00100000, memory_order_relaxed); + atomic_fetch_add_explicit(&mutex->_lock, 0x00100000, memory_order_relaxed); } return 0; diff --git a/libc/intrin/pthread_mutex_unlock.c b/libc/intrin/pthread_mutex_unlock.c index ab0da0806..d0e65b732 100644 --- a/libc/intrin/pthread_mutex_unlock.c +++ b/libc/intrin/pthread_mutex_unlock.c @@ -16,12 +16,14 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" #include "libc/calls/calls.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" -#include "libc/intrin/futex.internal.h" +#include "libc/intrin/likely.h" +#include "libc/intrin/weaken.h" #include "libc/thread/thread.h" +#include "libc/thread/tls.h" +#include "third_party/nsync/mu.h" /** * Releases mutex. @@ -32,30 +34,28 @@ int pthread_mutex_unlock(pthread_mutex_t *mutex) { int c, t; - if (mutex->type == PTHREAD_MUTEX_NORMAL) { - // From Futexes Are Tricky Version 1.1 § Mutex, Take 3; - // Ulrich Drepper, Red Hat Incorporated, June 27, 2004. - if ((c = atomic_fetch_sub(&mutex->lock, 1)) != 1) { - atomic_store_explicit(&mutex->lock, 0, memory_order_release); - _futex_wake(&mutex->lock, 1, mutex->pshared); - } + if (LIKELY(__tls_enabled && // + mutex->_type == PTHREAD_MUTEX_NORMAL && // + mutex->_pshared == PTHREAD_PROCESS_PRIVATE && // + weaken(nsync_mu_unlock))) { + weaken(nsync_mu_unlock)((nsync_mu *)mutex); return 0; } t = gettid(); - if (mutex->type == PTHREAD_MUTEX_ERRORCHECK) { - c = atomic_load_explicit(&mutex->lock, memory_order_relaxed); + if (mutex->_type == PTHREAD_MUTEX_ERRORCHECK) { + c = atomic_load_explicit(&mutex->_lock, memory_order_relaxed); if ((c & 0x000fffff) != t) { - assert(!"permlock"); return EPERM; } } - c = atomic_fetch_sub(&mutex->lock, 0x00100000); + c = atomic_fetch_sub(&mutex->_lock, 0x00100000); if ((c & 0x0ff00000) == 0x00100000) { - atomic_store_explicit(&mutex->lock, 0, memory_order_release); + atomic_store_explicit(&mutex->_lock, 0, memory_order_release); if ((c & 0xf0000000) == 0x20000000) { - _futex_wake(&mutex->lock, 1, mutex->pshared); + // _futex_wake(&mutex->_lock, 1, mutex->_pshared); + pthread_yield(); } } diff --git a/libc/intrin/pthread_once.c b/libc/intrin/pthread_once.c index 7ca91dbae..dc8ce6fb7 100644 --- a/libc/intrin/pthread_once.c +++ b/libc/intrin/pthread_once.c @@ -16,11 +16,12 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" #include "libc/calls/calls.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" +#include "libc/intrin/weaken.h" #include "libc/thread/thread.h" +#include "third_party/nsync/once.h" #define INIT 0 #define CALLING 1 @@ -44,28 +45,30 @@ * @return 0 on success, or errno on error */ int pthread_once(pthread_once_t *once, void init(void)) { - char old; - switch ((old = atomic_load_explicit(&once->lock, memory_order_relaxed))) { + uint32_t old; + if (weaken(nsync_run_once)) { + weaken(nsync_run_once)((nsync_once *)once, init); + return 0; + } + switch ((old = atomic_load_explicit(&once->_lock, memory_order_relaxed))) { case INIT: - if (atomic_compare_exchange_strong_explicit(&once->lock, &old, CALLING, + if (atomic_compare_exchange_strong_explicit(&once->_lock, &old, CALLING, memory_order_acquire, memory_order_relaxed)) { init(); - atomic_store(&once->lock, FINISHED); - break; + atomic_store(&once->_lock, FINISHED); + return 0; } // fallthrough case CALLING: do { pthread_yield(); - } while (atomic_load_explicit(&once->lock, memory_order_relaxed) == + } while (atomic_load_explicit(&once->_lock, memory_order_relaxed) == CALLING); - break; + return 0; case FINISHED: - break; + return 0; default: - assert(!"bad once"); return EINVAL; } - return 0; } diff --git a/libc/thread/pthread_setspecific.c b/libc/intrin/pthread_setspecific.c similarity index 100% rename from libc/thread/pthread_setspecific.c rename to libc/intrin/pthread_setspecific.c diff --git a/libc/intrin/wantcrashreports.c b/libc/intrin/wantcrashreports.c new file mode 100644 index 000000000..8b2d88935 --- /dev/null +++ b/libc/intrin/wantcrashreports.c @@ -0,0 +1,21 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/log/internal.h" + +bool _wantcrashreports; diff --git a/libc/log/internal.h b/libc/log/internal.h index 51fd64b46..0d7480f73 100644 --- a/libc/log/internal.h +++ b/libc/log/internal.h @@ -7,6 +7,7 @@ COSMOPOLITAN_C_START_ extern hidden bool __nocolor; extern hidden int kCrashSigs[8]; +extern hidden bool _wantcrashreports; extern hidden bool g_isrunningundermake; void __start_fatal(const char *, int) hidden; diff --git a/libc/log/leaks.c b/libc/log/leaks.c index 2b9d0c70c..4ec7371f4 100644 --- a/libc/log/leaks.c +++ b/libc/log/leaks.c @@ -16,9 +16,10 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/intrin/bits.h" #include "libc/calls/strace.internal.h" +#include "libc/dce.h" #include "libc/intrin/asan.internal.h" +#include "libc/intrin/bits.h" #include "libc/intrin/kprintf.h" #include "libc/intrin/lockcmpxchg.h" #include "libc/mem/mem.h" @@ -37,7 +38,7 @@ static bool hasleaks; static noasan void CheckLeak(void *x, void *y, size_t n, void *a) { if (n) { if (IsAsan()) { - if (__asan_get_heap_size(x)) { + if (__asan_get_heap_size(x) && !__asan_is_leaky(x)) { hasleaks = true; } } else { @@ -53,9 +54,10 @@ static noasan void OnMemory(void *x, void *y, size_t n, void *a) { if (i < MAXLEAKS) { ++i; kprintf("%p %,lu bytes [dlmalloc]", x, n); - if (IsAsan()) { - __asan_print_trace(x); + if (__asan_is_leaky(x)) { + kprintf(" [leaky]"); } + __asan_print_trace(x); kprintf("\n"); } else if (i == MAXLEAKS) { ++i; @@ -79,6 +81,7 @@ static noasan bool HasLeaks(void) { */ noasan void CheckForMemoryLeaks(void) { struct mallinfo mi; + if (!IsAsan()) return; // we need traces to exclude leaky if (!_lockcmpxchg(&once, false, true)) { kprintf("CheckForMemoryLeaks() may only be called once\n"); exit(1); diff --git a/libc/log/showcrashreports.c b/libc/log/showcrashreports.c index c460ea04b..66615ef89 100644 --- a/libc/log/showcrashreports.c +++ b/libc/log/showcrashreports.c @@ -105,6 +105,7 @@ static void FreeSigAltStack(void *p) { void ShowCrashReports(void) { char *sp; struct sigaltstack ss; + _wantcrashreports = true; /* : showcrashreports.c, oncrashthunks.S, oncrash.c */ kCrashSigs[0] = SIGQUIT; /* ctrl+\ aka ctrl+break */ kCrashSigs[1] = SIGFPE; /* 1 / 0 */ diff --git a/libc/nt/kernel32/CreateSemaphoreW.s b/libc/nt/kernel32/CreateSemaphoreW.s index bcac9a2e9..febaeb9b6 100644 --- a/libc/nt/kernel32/CreateSemaphoreW.s +++ b/libc/nt/kernel32/CreateSemaphoreW.s @@ -1,2 +1,12 @@ .include "o/libc/nt/codegen.inc" .imp kernel32,__imp_CreateSemaphoreW,CreateSemaphoreW,0 + + .text.windows +CreateSemaphore: + push %rbp + mov %rsp,%rbp + .profilable + mov __imp_CreateSemaphoreW(%rip),%rax + jmp __sysv2nt + .endfn CreateSemaphore,globl + .previous diff --git a/libc/nt/master.sh b/libc/nt/master.sh index 29a438991..ae8625595 100755 --- a/libc/nt/master.sh +++ b/libc/nt/master.sh @@ -166,7 +166,7 @@ imp 'CreateMutexEx' CreateMutexExW kernel32 0 imp 'CreatePrivateNamespace' CreatePrivateNamespaceW kernel32 0 imp 'CreateRemoteThread' CreateRemoteThread kernel32 0 imp 'CreateRemoteThreadEx' CreateRemoteThreadEx kernel32 0 -imp 'CreateSemaphore' CreateSemaphoreW kernel32 0 +imp 'CreateSemaphore' CreateSemaphoreW kernel32 0 4 imp 'CreateSemaphoreEx' CreateSemaphoreExW kernel32 0 imp 'CreateSymbolicLinkTransacted' CreateSymbolicLinkTransactedW kernel32 238 imp 'CreateTapePartition' CreateTapePartition kernel32 240 diff --git a/libc/nt/synchronization.h b/libc/nt/synchronization.h index cdf052d95..5c0fe55eb 100644 --- a/libc/nt/synchronization.h +++ b/libc/nt/synchronization.h @@ -82,6 +82,10 @@ bool32 SetWaitableTimer(int64_t hTimer, const int64_t *lpDueTimeAsFtOrNegRela, int32_t opt_lPeriodMs, NtTimerapcroutine opt_callback, void *lpArgToCallback, bool32 fUnsleepSystem); +int64_t CreateSemaphore(struct NtSecurityAttributes *opt_lpSemaphoreAttributes, + uint32_t lInitialCount, uint32_t lMaximumCount, + const char16_t *opt_lpName); + int32_t SetEvent(int64_t hEvent); int32_t ResetEvent(int64_t hEvent); int32_t PulseEvent(int64_t hEvent); diff --git a/libc/runtime/memtrack64.txt b/libc/runtime/memtrack64.txt index 9b090ce99..a16347005 100644 --- a/libc/runtime/memtrack64.txt +++ b/libc/runtime/memtrack64.txt @@ -1808,7 +1808,7 @@ 6f900000-6f9fffff 64gb free 6fa00000-6fafffff 64gb free 6fb00000-6fbfffff 64gb free -6fc00000-6fcfffff 64gb free +6fc00004-6fcfffff 64gb nsync 6fd00000-6fdfffff 64gb zipos 6fe00004-6feffffc 64gb g_fds 6ff00000-6ffffffd 64gb free diff --git a/libc/stdio/fdopen.c b/libc/stdio/fdopen.c index bcbb82033..de8875d62 100644 --- a/libc/stdio/fdopen.c +++ b/libc/stdio/fdopen.c @@ -17,12 +17,12 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" -#include "libc/thread/thread.h" #include "libc/mem/mem.h" #include "libc/stdio/internal.h" #include "libc/stdio/stdio.h" #include "libc/sysv/consts/o.h" #include "libc/sysv/errfuns.h" +#include "libc/thread/thread.h" /** * Allocates stream object for already-opened file descriptor. @@ -38,7 +38,7 @@ FILE *fdopen(int fd, const char *mode) { f->fd = fd; f->bufmode = ischardev(fd) ? _IOLBF : _IOFBF; f->iomode = fopenflags(mode); - f->lock.type = PTHREAD_MUTEX_RECURSIVE; + f->lock._type = PTHREAD_MUTEX_RECURSIVE; f->size = BUFSIZ; if ((f->buf = malloc(f->size))) { if ((f->iomode & O_ACCMODE) != O_RDONLY) { diff --git a/libc/stdio/fmemopen.c b/libc/stdio/fmemopen.c index 148850be3..3183982a3 100644 --- a/libc/stdio/fmemopen.c +++ b/libc/stdio/fmemopen.c @@ -16,12 +16,12 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" #include "libc/mem/mem.h" #include "libc/stdio/stdio.h" #include "libc/str/str.h" #include "libc/sysv/consts/o.h" #include "libc/sysv/errfuns.h" +#include "libc/thread/thread.h" /** * Opens buffer as stream. @@ -54,7 +54,7 @@ FILE *fmemopen(void *buf, size_t size, const char *mode) { f->end = size; f->size = size; f->iomode = fopenflags(mode); - f->lock.type = PTHREAD_MUTEX_RECURSIVE; + f->lock._type = PTHREAD_MUTEX_RECURSIVE; if (f->iomode & O_APPEND) { if ((p = memchr(buf, '\0', size))) { f->beg = p - (char *)buf; diff --git a/libc/stdio/stderr-init.S b/libc/stdio/stderr-init.S index 7dc295db4..24b3ec928 100644 --- a/libc/stdio/stderr-init.S +++ b/libc/stdio/stderr-init.S @@ -34,6 +34,6 @@ ezlea __stderr_buf,cx mov %rcx,0x18(%rax) #→ f.buf movl $BUFSIZ,0x20(%rax) #→ f.size - movb $PTHREAD_MUTEX_RECURSIVE,0x38(%rax) #→ f.lock.attr + movb $PTHREAD_MUTEX_RECURSIVE,0x38+16(%rax) #→ f.lock.attr mov %rax,stderr(%rip) .init.end 400,_init_stderr,globl,hidden diff --git a/libc/stdio/stdin-init.S b/libc/stdio/stdin-init.S index fa7eda46f..86051c810 100644 --- a/libc/stdio/stdin-init.S +++ b/libc/stdio/stdin-init.S @@ -30,6 +30,6 @@ ezlea __stdin_buf,cx mov %rcx,0x18(%rax) #→ f.buf movl $BUFSIZ,0x20(%rax) #→ f.size - movb $PTHREAD_MUTEX_RECURSIVE,0x38(%rax) #→ f.lock.attr + movb $PTHREAD_MUTEX_RECURSIVE,0x38+16(%rax) #→ f.lock.attr mov %rax,stdin(%rip) .init.end 400,_init_stdin,globl,hidden diff --git a/libc/stdio/stdout-init.S b/libc/stdio/stdout-init.S index f43249810..1711b69b6 100644 --- a/libc/stdio/stdout-init.S +++ b/libc/stdio/stdout-init.S @@ -32,6 +32,6 @@ ezlea __stdout_buf,cx mov %rcx,0x18(%rax) #→ f.buf movl $BUFSIZ,0x20(%rax) #→ f.size - movb $PTHREAD_MUTEX_RECURSIVE,0x38(%rax) #→ f.lock.attr + movb $PTHREAD_MUTEX_RECURSIVE,0x38+16(%rax) #→ f.lock.attr mov %rax,stdout(%rip) .init.end 400,_init_stdout,globl,hidden diff --git a/libc/sysv/consts.sh b/libc/sysv/consts.sh index e548083d7..3df60aca6 100755 --- a/libc/sysv/consts.sh +++ b/libc/sysv/consts.sh @@ -579,7 +579,7 @@ syscon sicode SYS_USER_DISPATCH 2 -1 -1 -1 -1 -1 # SIGSYS; syscall # sigaltstack() values # # group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary -syscon ss SIGSTKSZ 8192 131072 34816 28672 28672 8192 # overlaid with STACKSIZE; you need to #undef SIGSTKSZ to access this symbol +syscon ss SIGSTKSZ 8192 131072 34816 28672 28672 8192 # overlaid with FRAMESIZE; you need to #undef SIGSTKSZ to access this symbol syscon ss MINSIGSTKSZ 2048 32768 2048 12288 8192 2048 # overlaid with 32768; you need to #undef MINSIGSTKSZ to access this symbol syscon ss SS_ONSTACK 1 1 1 1 1 1 # unix consensus syscon ss SS_DISABLE 2 4 4 4 4 2 # bsd consensus @@ -1316,9 +1316,9 @@ syscon rusage RUSAGE_BOTH -2 99 99 99 99 99 # woop # # group name GNU/Systemd XNU's Not UNIX! FreeBSD OpenBSD NetBSD The New Technology Commentary syscon futex FUTEX_WAIT 0 0 0 1 0 0 -syscon futex FUTEX_WAKE 1 0 0 2 1 0 -syscon futex FUTEX_REQUEUE 3 0 0 3 3 0 -syscon futex FUTEX_PRIVATE_FLAG 128 0 0 128 128 0 +syscon futex FUTEX_WAKE 1 0 0 2 0 0 +syscon futex FUTEX_REQUEUE 3 0 0 3 0 0 +syscon futex FUTEX_PRIVATE_FLAG 128 0 0 128 0 0 # lio_listio() magnums # @@ -1336,7 +1336,7 @@ syscon sched SCHED_OTHER 0 127 2 127 0 127 # standard round-robin syscon sched SCHED_FIFO 1 127 1 127 1 127 # [real-time] first-in, first-out policy syscon sched SCHED_RR 2 127 3 127 2 127 # [real-time] round-robin policy syscon sched SCHED_BATCH 3 127 2 127 0 127 # for "batch" style execution of processes; polyfilled as SCHED_OTHER on non-Linux -𝔰𝔶𝔰𝔠𝔬𝔫 sched SCHED_IDLE 5 127 2 127 0 127 # for running very low priority background jobs; polyfilled as SCHED_OTHER on non-Linux +syscon sched SCHED_IDLE 5 127 2 127 0 127 # for running very low priority background jobs; polyfilled as SCHED_OTHER on non-Linux syscon sched SCHED_DEADLINE 6 127 127 127 127 127 # can only be set by sched_setattr() syscon sched SCHED_RESET_ON_FORK 0x40000000 0 0 0 0 0 # Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork(); no-op on non-Linux @@ -1847,7 +1847,7 @@ syscon nr __NR_kill 0x003e 0x2000025 0x0025 0x007a 0x025 0xfff syscon nr __NR_killpg 0xfff 0xfff 0x0092 0xfff 0xfff 0xfff syscon nr __NR_clone 0x0038 0xfff 0xfff 0xfff 0x11f 0xfff syscon nr __NR_tkill 0x00c8 0xfff 0xfff 0xfff 0xfff 0xfff -syscon nr __NR_futex 0x00ca 0xfff 0xfff 0x0053 0x0a6 0xfff +syscon nr __NR_futex 0x00ca 0xfff 0xfff 0x0053 0xfff 0xfff syscon nr __NR_set_robust_list 0x0111 0xfff 0xfff 0xfff 0x0a7 0xfff syscon nr __NR_get_robust_list 0x0112 0xfff 0xfff 0xfff 0x0a8 0xfff syscon nr __NR_uname 0x003f 0xfff 0x00a4 0xfff 0xfff 0xfff diff --git a/libc/sysv/consts/FUTEX_PRIVATE_FLAG.s b/libc/sysv/consts/FUTEX_PRIVATE_FLAG.s index da3dd6143..3d21804a5 100644 --- a/libc/sysv/consts/FUTEX_PRIVATE_FLAG.s +++ b/libc/sysv/consts/FUTEX_PRIVATE_FLAG.s @@ -1,2 +1,2 @@ .include "o/libc/sysv/consts/syscon.internal.inc" -.syscon futex,FUTEX_PRIVATE_FLAG,128,0,0,128,128,0 +.syscon futex,FUTEX_PRIVATE_FLAG,128,0,0,128,0,0 diff --git a/libc/sysv/consts/FUTEX_REQUEUE.s b/libc/sysv/consts/FUTEX_REQUEUE.s index b0a8baa21..e25e5dffb 100644 --- a/libc/sysv/consts/FUTEX_REQUEUE.s +++ b/libc/sysv/consts/FUTEX_REQUEUE.s @@ -1,2 +1,2 @@ .include "o/libc/sysv/consts/syscon.internal.inc" -.syscon futex,FUTEX_REQUEUE,3,0,0,3,3,0 +.syscon futex,FUTEX_REQUEUE,3,0,0,3,0,0 diff --git a/libc/sysv/consts/FUTEX_WAKE.s b/libc/sysv/consts/FUTEX_WAKE.s index 2d6184318..acdc80afa 100644 --- a/libc/sysv/consts/FUTEX_WAKE.s +++ b/libc/sysv/consts/FUTEX_WAKE.s @@ -1,2 +1,2 @@ .include "o/libc/sysv/consts/syscon.internal.inc" -.syscon futex,FUTEX_WAKE,1,0,0,2,1,0 +.syscon futex,FUTEX_WAKE,1,0,0,2,0,0 diff --git a/libc/sysv/consts/SOMAXCONN.s b/libc/sysv/consts/SOMAXCONN.s index 5c6a6ef77..f711831ce 100644 --- a/libc/sysv/consts/SOMAXCONN.s +++ b/libc/sysv/consts/SOMAXCONN.s @@ -1,2 +1,2 @@ .include "o/libc/sysv/consts/syscon.internal.inc" -.syscon misc,SOMAXCONN,0x80,0x80,0x80,0x80,0x80,0x7fffffff +.syscon limits,SOMAXCONN,4096,128,128,128,128,2147483647 diff --git a/libc/sysv/consts/__NR_futex.s b/libc/sysv/consts/__NR_futex.s index bbcc0b744..a7bd77d79 100644 --- a/libc/sysv/consts/__NR_futex.s +++ b/libc/sysv/consts/__NR_futex.s @@ -1,2 +1,2 @@ .include "o/libc/sysv/consts/syscon.internal.inc" -.syscon nr,__NR_futex,0x00ca,0xfff,0xfff,0x0053,0x0a6,0xfff +.syscon nr,__NR_futex,0x00ca,0xfff,0xfff,0x0053,0xfff,0xfff diff --git a/libc/sysv/consts/futex.h b/libc/sysv/consts/futex.h index f6b12cf2b..91641a3d7 100644 --- a/libc/sysv/consts/futex.h +++ b/libc/sysv/consts/futex.h @@ -2,21 +2,25 @@ #define COSMOPOLITAN_LIBC_SYSV_CONSTS_FUTEX_H_ #include "libc/runtime/symbolic.h" -#define FUTEX_WAIT SYMBOLIC(FUTEX_WAIT) -#define FUTEX_WAKE SYMBOLIC(FUTEX_WAKE) -#define FUTEX_REQUEUE SYMBOLIC(FUTEX_REQUEUE) -#define FUTEX_PRIVATE_FLAG SYMBOLIC(FUTEX_PRIVATE_FLAG) +#define FUTEX_WAIT SYMBOLIC(FUTEX_WAIT) +#define FUTEX_WAKE SYMBOLIC(FUTEX_WAKE) +#define FUTEX_REQUEUE SYMBOLIC(FUTEX_REQUEUE) +#define FUTEX_PRIVATE_FLAG 128 + #define FUTEX_WAIT_PRIVATE (FUTEX_WAIT | FUTEX_PRIVATE_FLAG) #define FUTEX_WAKE_PRIVATE (FUTEX_WAKE | FUTEX_PRIVATE_FLAG) #define FUTEX_REQUEUE_PRIVATE (FUTEX_REQUEUE | FUTEX_PRIVATE_FLAG) +#define FUTEX_WAIT_BITSET 9 +#define FUTEX_CLOCK_REALTIME 256 +#define FUTEX_BITSET_MATCH_ANY 0xffffffff + #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ extern const int FUTEX_WAIT; extern const int FUTEX_WAKE; extern const int FUTEX_REQUEUE; -extern const int FUTEX_PRIVATE_FLAG; COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ diff --git a/libc/sysv/consts/ss.h b/libc/sysv/consts/ss.h index 68d135fae..9f05e231c 100644 --- a/libc/sysv/consts/ss.h +++ b/libc/sysv/consts/ss.h @@ -10,7 +10,7 @@ extern const int SS_DISABLE; COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#define SIGSTKSZ STACKSIZE +#define SIGSTKSZ FRAMESIZE #define MINSIGSTKSZ 32768 #define SS_ONSTACK 1 #define SS_DISABLE SS_DISABLE diff --git a/libc/sysv/strace.greg.c b/libc/sysv/strace.greg.c index 0822d65cb..302fc69c1 100644 --- a/libc/sysv/strace.greg.c +++ b/libc/sysv/strace.greg.c @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/atomic.h" #include "libc/runtime/runtime.h" /** @@ -40,4 +41,4 @@ * under normal circumstances, `__strace` should only be either zero or * one. */ -_Atomic(int) __strace; +atomic_int __strace; diff --git a/libc/sysv/syscall.S b/libc/sysv/syscall.S index 275a656da..6cffdcf8a 100644 --- a/libc/sysv/syscall.S +++ b/libc/sysv/syscall.S @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/dce.h" #include "libc/macros.internal.h" .privileged @@ -32,12 +33,52 @@ // @param %rsi,%rdx,%rcx,%r8,%r9 may supply parameters 1 through 5 // @param sixth is optionally pushed on the stack before call // @return %rax has result, or -1 w/ errno on failure -syscall:mov %rdi,%rax - mov %rsi,%rdi - mov %rdx,%rsi - mov %rcx,%rdx - mov %r8,%rcx # ← intended - mov %r9,%r8 - mov 8(%rsp),%r9 - jmp *__systemfive(%rip) +syscall: + push %rbp + mov %rsp,%rbp + .profilable + +// slide arguments into their right places + mov %rdi,%rax # nr + mov %rsi,%rdi # arg 1 + mov %rdx,%rsi # arg 2 + mov %rcx,%rdx # arg 3 + mov %r8,%rcx # arg 4 + mov %r9,%r8 # arg 5 + mov 16(%rbp),%r9 # arg 6 + push 32(%rbp) # arg 8 + push 24(%rbp) # arg 7 + +// convert from consts.sh to syscalls.sh encoding + push %rcx + mov __hostos(%rip),%cl + test $LINUX,%cl + jnz 2f +1: test $FREEBSD,%cl + jz 1f + shl $4*7,%rax + jmp 2f +1: test $OPENBSD,%cl + jz 1f + shl $4*10,%rax + jmp 2f +1: test $NETBSD,%cl + jz 1f + shl $4*13,%rax + jmp 2f +1: test $XNU,%cl + jz 2f + mov %eax,%ecx + and $0x0f000000,%ecx + and $0x00000fff,%eax + shl $4*3,%eax + or %ecx,%eax +2: pop %rcx + +// trigger the system call + call *__systemfive(%rip) + +// clean up stack and return + leave + ret .endfn syscall,globl diff --git a/libc/testlib/globals.c b/libc/testlib/globals.c index c6563c30d..e844ce117 100644 --- a/libc/testlib/globals.c +++ b/libc/testlib/globals.c @@ -16,8 +16,9 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/atomic.h" #include "libc/testlib/testlib.h" char g_fixturename[256]; -_Atomic(unsigned) g_testlib_ran; -_Atomic(unsigned) g_testlib_failed; +atomic_uint g_testlib_ran; +atomic_uint g_testlib_failed; diff --git a/libc/testlib/testmain.c b/libc/testlib/testmain.c index af8ed85fa..3f9d9a310 100644 --- a/libc/testlib/testmain.c +++ b/libc/testlib/testmain.c @@ -182,13 +182,13 @@ noasan int main(int argc, char *argv[]) { testlib_runalltests(); if (!g_testlib_failed && runbenchmarks_ && weaken(testlib_runallbenchmarks)) { weaken(testlib_runallbenchmarks)(); - if (!g_testlib_failed) { + if (IsAsan() && !g_testlib_failed) { CheckForMemoryLeaks(); } if (!g_testlib_failed && IsRunningUnderMake()) { return 254; // compile.com considers this 0 and propagates output } - } else if (!g_testlib_failed) { + } else if (IsAsan() && !g_testlib_failed) { CheckForMemoryLeaks(); } diff --git a/libc/thread/README.md b/libc/thread/README.md new file mode 100644 index 000000000..99c6a61d4 --- /dev/null +++ b/libc/thread/README.md @@ -0,0 +1,5 @@ +# Cosmpolitan POSIX Threads Library + +Cosmopolitan Libc implements threading as it is written in The Open +Group Base Specifications Issue 7, 2018 edition IEEE Std 1003.1-2017 +(Revision of IEEE Std 1003.1-2008) in addition to GNU extensions. diff --git a/libc/thread/atomic.h b/libc/thread/atomic.h deleted file mode 100644 index c1219f280..000000000 --- a/libc/thread/atomic.h +++ /dev/null @@ -1,103 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_PLATFORM_C11_ATOMIC_H_ -#define NSYNC_PLATFORM_C11_ATOMIC_H_ - -/* Atomic operations on nsync_atomic_uint32_ quantities - CAS, load, and store. - - Normally, these are used only on nsync_atomic_uint32_ values, but on Linux they may be - invoked on int values, because futexes operate on int values. A - compile-time check in the futex code ensures that both int and - nsync_atomic_uint32_ are 32 bits. - - Memory barriers: - Operations with the suffixes _ACQ and _RELACQ ensure that the operation - appears to complete before other memory operations subsequently performed by - the same thread, as seen by other threads. (In the case of ATM_CAS_ACQ, - this applies only if the operation returns a non-zero value.) - - Operations with the suffixes _REL and _RELACQ ensure that the operation - appears to complete after other memory operations previously performed by - the same thread, as seen by other threads. (In the case of ATM_CAS_REL, - this applies only if the operation returns a non-zero value.) - - // Atomically, - // int ATM_CAS (nsync_atomic_uint32_ *p, uint32_t old_value, uint32_t new_value) { - // if (*p == old_value) { - // *p = new_value; - // return (some-non-zero-value); - // } else { - // return (0); - // } - // } - // *_ACQ, *_REL, *_RELACQ variants are available, - // with the barrier semantics described above. - int ATM_CAS (nsync_atomic_uint32_ *p, uint32_t old_value, uint32_t new_value); - - // Atomically, - // uint32_t ATM_LOAD (nsync_atomic_uint32_ *p) { return (*p); } - // A *_ACQ variant is available, - // with the barrier semantics described above. - uint32_t ATM_LOAD (nsync_atomic_uint32_ *p); - - // Atomically, - // void ATM_STORE (nsync_atomic_uint32_ *p, uint32_t value) { *p = value; } - // A *_REL variant is available, - // with the barrier semantics described above. - void ATM_STORE (nsync_atomic_uint32_ *p, uint32_t value); - */ - -#include "libc/thread/compiler.h" -#include "libc/intrin/atomic.h" -#include "libc/thread/nsync_atomic.h" - -NSYNC_CPP_START_ - -static __inline__ int atm_cas_nomb_u32_ (nsync_atomic_uint32_ *p, uint32_t o, uint32_t n) { - return (atomic_compare_exchange_strong_explicit (NSYNC_ATOMIC_UINT32_PTR_ (p), &o, n, - memory_order_relaxed, memory_order_relaxed)); -} -static __inline__ int atm_cas_acq_u32_ (nsync_atomic_uint32_ *p, uint32_t o, uint32_t n) { - return (atomic_compare_exchange_strong_explicit (NSYNC_ATOMIC_UINT32_PTR_ (p), &o, n, - memory_order_acquire, memory_order_relaxed)); -} -static __inline__ int atm_cas_rel_u32_ (nsync_atomic_uint32_ *p, uint32_t o, uint32_t n) { - return (atomic_compare_exchange_strong_explicit (NSYNC_ATOMIC_UINT32_PTR_ (p), &o, n, - memory_order_release, memory_order_relaxed)); -} -static __inline__ int atm_cas_relacq_u32_ (nsync_atomic_uint32_ *p, uint32_t o, uint32_t n) { - return (atomic_compare_exchange_strong_explicit (NSYNC_ATOMIC_UINT32_PTR_ (p), &o, n, - memory_order_acq_rel, memory_order_relaxed)); -} - -#define ATM_CAS_HELPER_(barrier, p, o, n) (atm_cas_##barrier##_u32_ ((p), (o), (n))) - -#define ATM_CAS(p,o,n) ATM_CAS_HELPER_ (nomb, (p), (o), (n)) -#define ATM_CAS_ACQ(p,o,n) ATM_CAS_HELPER_ (acq, (p), (o), (n)) -#define ATM_CAS_REL(p,o,n) ATM_CAS_HELPER_ (rel, (p), (o), (n)) -#define ATM_CAS_RELACQ(p,o,n) ATM_CAS_HELPER_ (relacq, (p), (o), (n)) - -/* Need a cast to remove "const" from some uses. */ -#define ATM_LOAD(p) (atomic_load_explicit ((nsync_atomic_uint32_ *) NSYNC_ATOMIC_UINT32_PTR_ (p), memory_order_relaxed)) -#define ATM_LOAD_ACQ(p) (atomic_load_explicit ((nsync_atomic_uint32_ *) NSYNC_ATOMIC_UINT32_PTR_ (p), memory_order_acquire)) - -#define ATM_STORE(p,v) (atomic_store_explicit (NSYNC_ATOMIC_UINT32_PTR_ (p), (v), memory_order_relaxed)) -#define ATM_STORE_REL(p,v) (atomic_store_explicit (NSYNC_ATOMIC_UINT32_PTR_ (p), (v), memory_order_release)) - -NSYNC_CPP_END_ - -#endif /*NSYNC_PLATFORM_C11_ATOMIC_H_*/ diff --git a/libc/thread/clone.c b/libc/thread/clone.c index 866eb4429..37f1bce6b 100644 --- a/libc/thread/clone.c +++ b/libc/thread/clone.c @@ -215,7 +215,7 @@ static int CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags, wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid; wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid; wt->tls = flags & CLONE_SETTLS ? tls : 0; - wt->lock.lock = 1; + wt->lock._lock = 1; if ((rc = bsdthread_create(fn, arg, wt, 0, PTHREAD_START_CUSTOM_XNU)) != -1) { pthread_spin_lock(&wt->lock); rc = wt->tid; diff --git a/libc/thread/common.h b/libc/thread/common.h deleted file mode 100644 index 1c672f54c..000000000 --- a/libc/thread/common.h +++ /dev/null @@ -1,293 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_INTERNAL_COMMON_H_ -#define NSYNC_INTERNAL_COMMON_H_ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/platform.h" -#include "libc/thread/nsync_atomic.h" -#include "libc/thread/sem.h" -#include "libc/thread/nsync_waiter.h" -#include "libc/thread/dll.h" -#include "libc/thread/nsync_mu.h" -#include "libc/thread/nsync_cv.h" -#include "libc/thread/nsync_note.h" -#include "libc/thread/wait_internal.h" - -/* Annotations for race detectors. */ -#if defined(__has_feature) && !defined(__SANITIZE_THREAD__) -#if __has_feature(thread_sanitizer) /* used by clang */ -#define __SANITIZE_THREAD__ 1 /* GCC uses this; fake it in clang */ -#endif -#endif -#if defined(__SANITIZE_THREAD__) -NSYNC_C_START_ -void AnnotateIgnoreWritesBegin(const char* file, int line); -void AnnotateIgnoreWritesEnd(const char* file, int line); -void AnnotateIgnoreReadsBegin(const char* file, int line); -void AnnotateIgnoreReadsEnd(const char* file, int line); -NSYNC_C_END_ -#define IGNORE_RACES_START() \ - do { \ - AnnotateIgnoreReadsBegin(__FILE__, __LINE__); \ - AnnotateIgnoreWritesBegin(__FILE__, __LINE__); \ - } while (0) -#define IGNORE_RACES_END() \ - do { \ - AnnotateIgnoreWritesEnd(__FILE__, __LINE__); \ - AnnotateIgnoreReadsEnd(__FILE__, __LINE__); \ - } while (0) -#else -#define IGNORE_RACES_START() -#define IGNORE_RACES_END() -#endif - -#ifndef NSYNC_DEBUG -#define NSYNC_DEBUG 0 -#endif - -NSYNC_CPP_START_ - -/* Yield the CPU. Platform specific. */ -void nsync_yield_ (void); - -/* Retrieve the per-thread cache of the waiter object. Platform specific. */ -void *nsync_per_thread_waiter_ (void (*dest) (void *)); - -/* Set the per-thread cache of the waiter object. Platform specific. */ -void nsync_set_per_thread_waiter_ (void *v, void (*dest) (void *)); - -/* Used in spinloops to delay resumption of the loop. - Usage: - unsigned attempts = 0; - while (try_something) { - attempts = nsync_spin_delay_ (attempts); - } */ -unsigned nsync_spin_delay_ (unsigned attempts); - -/* Spin until (*w & test) == 0, then atomically perform *w = ((*w | set) & - ~clear), perform an acquire barrier, and return the previous value of *w. - */ -uint32_t nsync_spin_test_and_set_ (nsync_atomic_uint32_ *w, uint32_t test, - uint32_t set, uint32_t clear); - -/* Abort after printing the nul-temrinated string s[]. */ -void nsync_panic_ (const char *s); - -/* ---------- */ - -#define MIN_(a_,b_) ((a_) < (b_)? (a_) : (b_)) -#define MAX_(a_,b_) ((a_) > (b_)? (a_) : (b_)) - -/* ---------- */ - -/* Fields in nsync_mu.word. - - - At least one of the MU_WLOCK or MU_RLOCK_FIELD fields must be zero. - - MU_WLOCK indicates that a write lock is held. - - MU_RLOCK_FIELD is a count of readers with read locks. - - - MU_SPINLOCK represents a spinlock that must be held when manipulating the - waiter queue. - - - MU_DESIG_WAKER indicates that a former waiter has been woken, but has - neither acquired the lock nor gone back to sleep. Legal to fail to set it; - illegal to set it when no such waiter exists. - - - MU_WAITING indicates whether the waiter queue is non-empty. - The following bits should be zero if MU_WAITING is zero. - - MU_CONDITION indicates that some waiter may have an associated condition - (from nsync_mu_wait, etc.). Legal to set it with no such waiter exists, - but illegal to fail to set it with such a waiter. - - MU_WRITER_WAITING indicates that a reader that has not yet blocked - at least once should not acquire in order not to starve waiting writers. - It set when a writer blocks or a reader is woken with a writer waiting. - It is reset when a writer acquires, but set again when that writer - releases if it wakes readers and there is a waiting writer. - - MU_LONG_WAIT indicates that a waiter has been woken many times but - repeatedly failed to acquire when competing for the lock. This is used - only to prevent long-term starvation by writers. The thread that sets it - clears it when if acquires. - - MU_ALL_FALSE indicates that a complete scan of the waiter list found no - waiters with true conditions, and the lock has not been acquired by a - writer since then. This allows a reader lock to be released without - testing conditions again. It is legal to fail to set this, but illegal - to set it inappropriately. - */ -#define MU_WLOCK ((uint32_t) (1 << 0)) /* writer lock is held. */ -#define MU_SPINLOCK ((uint32_t) (1 << 1)) /* spinlock is held (protects waiters). */ -#define MU_WAITING ((uint32_t) (1 << 2)) /* waiter list is non-empty. */ -#define MU_DESIG_WAKER ((uint32_t) (1 << 3)) /* a former waiter awoke, and hasn't yet acquired or slept anew */ -#define MU_CONDITION ((uint32_t) (1 << 4)) /* the wait list contains some conditional waiters. */ -#define MU_WRITER_WAITING ((uint32_t) (1 << 5)) /* there is a writer waiting */ -#define MU_LONG_WAIT ((uint32_t) (1 << 6)) /* the waiter at the head of the queue has been waiting a long time */ -#define MU_ALL_FALSE ((uint32_t) (1 << 7)) /* all waiter conditions are false */ -#define MU_RLOCK ((uint32_t) (1 << 8)) /* low-order bit of reader count, which uses rest of word */ - -/* The constants below are derived from those above. */ -#define MU_RLOCK_FIELD (~(uint32_t) (MU_RLOCK - 1)) /* mask of reader count field */ - -#define MU_ANY_LOCK (MU_WLOCK | MU_RLOCK_FIELD) /* mask for any lock held */ - -#define MU_WZERO_TO_ACQUIRE (MU_ANY_LOCK | MU_LONG_WAIT) /* bits to be zero to acquire write lock */ -#define MU_WADD_TO_ACQUIRE (MU_WLOCK) /* add to acquire a write lock */ -#define MU_WHELD_IF_NON_ZERO (MU_WLOCK) /* if any of these bits are set, write lock is held */ -#define MU_WSET_WHEN_WAITING (MU_WAITING | MU_WRITER_WAITING) /* a writer is waiting */ -#define MU_WCLEAR_ON_ACQUIRE (MU_WRITER_WAITING) /* clear MU_WRITER_WAITING when a writer acquires */ -#define MU_WCLEAR_ON_UNCONTENDED_RELEASE (MU_ALL_FALSE) /* clear if a writer releases w/o waking */ - -/* bits to be zero to acquire read lock */ -#define MU_RZERO_TO_ACQUIRE (MU_WLOCK | MU_WRITER_WAITING | MU_LONG_WAIT) -#define MU_RADD_TO_ACQUIRE (MU_RLOCK) /* add to acquire a read lock */ -#define MU_RHELD_IF_NON_ZERO (MU_RLOCK_FIELD) /* if any of these bits are set, read lock is held */ -#define MU_RSET_WHEN_WAITING (MU_WAITING) /* indicate that some thread is waiting */ -#define MU_RCLEAR_ON_ACQUIRE ((uint32_t) 0) /* nothing to clear when a read acquires */ -#define MU_RCLEAR_ON_UNCONTENDED_RELEASE ((uint32_t) 0) /* nothing to clear when a read releases */ - - -/* A lock_type holds the values needed to manipulate a mu in some mode (read or - write). This allows some of the code to be generic, and parameterized by - the lock type. */ -typedef struct lock_type_s { - uint32_t zero_to_acquire; /* bits that must be zero to acquire */ - uint32_t add_to_acquire; /* constant to add to acquire */ - uint32_t held_if_non_zero; /* if any of these bits are set, the lock is held */ - uint32_t set_when_waiting; /* set when thread waits */ - uint32_t clear_on_acquire; /* clear when thread acquires */ - uint32_t clear_on_uncontended_release; /* clear when thread releases without waking */ -} lock_type; - - -/* writer_type points to a lock_type that describes how to manipulate a mu for a writer. */ -extern lock_type *nsync_writer_type_; - -/* reader_type points to a lock_type that describes how to manipulate a mu for a reader. */ -extern lock_type *nsync_reader_type_; - -/* ---------- */ - -/* Bits in nsync_cv.word */ - -#define CV_SPINLOCK ((uint32_t) (1 << 0)) /* protects waiters */ -#define CV_NON_EMPTY ((uint32_t) (1 << 1)) /* waiters list is non-empty */ - -/* ---------- */ - -/* Hold a pair of condition function and its argument. */ -struct wait_condition_s { - int (*f) (const void *v); - const void *v; - int (*eq) (const void *a, const void *b); -}; - -/* Return whether wait conditions *a_ and *b_ are equal and non-null. */ -#define WAIT_CONDITION_EQ(a_, b_) ((a_)->f != NULL && (a_)->f == (b_)->f && \ - ((a_)->v == (b_)->v || \ - ((a_)->eq != NULL && (*(a_)->eq) ((a_)->v, (b_)->v)))) - -/* If a waiter has waited this many times, it may set the MU_LONG_WAIT bit. */ -#define LONG_WAIT_THRESHOLD 30 - -/* ---------- */ - -#define NOTIFIED_TIME(n_) (ATM_LOAD_ACQ (&(n_)->notified) != 0? nsync_time_zero : \ - (n_)->expiry_time_valid? (n_)->expiry_time : nsync_time_no_deadline) - -/* A waiter represents a single waiter on a cv or a mu. - - To wait: - Allocate a waiter struct *w with new_waiter(), set w.waiting=1, and - w.cv_mu=nil or to the associated mu if waiting on a condition variable, then - queue w.nsync_dll on some queue, and then wait using: - while (ATM_LOAD_ACQ (&w.waiting) != 0) { nsync_mu_semaphore_p (&w.sem); } - Return *w to the freepool by calling free_waiter (w). - - To wakeup: - Remove *w from the relevant queue then: - ATM_STORE_REL (&w.waiting, 0); - nsync_mu_semaphore_v (&w.sem); */ -typedef struct { - uint32_t tag; /* debug DLL_NSYNC_WAITER, DLL_WAITER, DLL_WAITER_SAMECOND */ - nsync_semaphore sem; /* Thread waits on this semaphore. */ - struct nsync_waiter_s nw; /* An embedded nsync_waiter_s. */ - struct nsync_mu_s_ *cv_mu; /* pointer to nsync_mu associated with a cv wait */ - lock_type *l_type; /* Lock type of the mu, or nil if not associated with a mu. */ - nsync_atomic_uint32_ remove_count; /* count of removals from queue */ - struct wait_condition_s cond; /* A condition on which to acquire a mu. */ - nsync_dll_element_ same_condition; /* Links neighbours in nw.q with same non-nil condition. */ - int flags; /* see WAITER_* bits below */ -} waiter; -static const uint32_t WAITER_TAG = 0x0590239f; -static const uint32_t NSYNC_WAITER_TAG = 0x726d2ba9; - -#define WAITER_RESERVED 0x1 /* waiter reserved by a thread, even when not in use */ -#define WAITER_IN_USE 0x2 /* waiter in use by a thread */ - -#define CONTAINER(t_,f_,p_) ((t_ *) (((char *) (p_)) - offsetof (t_, f_))) -#define ASSERT(x) do { if (!(x)) { *(volatile int *)0 = 0; } } while (0) - -/* Return a pointer to the nsync_waiter_s containing nsync_dll_element_ *e. */ -#define DLL_NSYNC_WAITER(e) (NSYNC_DEBUG? nsync_dll_nsync_waiter_ (e) : \ - ((struct nsync_waiter_s *)((e)->container))) -struct nsync_waiter_s *nsync_dll_nsync_waiter_ (nsync_dll_element_ *e); - -/* Return a pointer to the waiter struct that *e is embedded in, where *e is an nw.q field. */ -#define DLL_WAITER(e) (NSYNC_DEBUG? nsync_dll_waiter_ (e) : \ - CONTAINER (waiter, nw, DLL_NSYNC_WAITER(e))) -waiter *nsync_dll_waiter_ (nsync_dll_element_ *e); - -/* Return a pointer to the waiter struct that *e is embedded in, where *e is a - same_condition field. */ -#define DLL_WAITER_SAMECOND(e) (NSYNC_DEBUG? nsync_dll_waiter_samecond_ (e) : \ - ((waiter *) ((e)->container))) -waiter *nsync_dll_waiter_samecond_ (nsync_dll_element_ *e); - -/* Return a pointer to an unused waiter struct. - Ensures that the enclosed timer is stopped and its channel drained. */ -waiter *nsync_waiter_new_ (void); - -/* Return an unused waiter struct *w to the free pool. */ -void nsync_waiter_free_ (waiter *w); - -/* ---------- */ - -/* The internals of an nync_note. See internal/note.c for details of locking - discipline. */ -struct nsync_note_s_ { - nsync_dll_element_ parent_child_link; /* parent's children, under parent->note_mu */ - int expiry_time_valid; /* whether expiry_time is valid; r/o after init */ - nsync_time expiry_time; /* expiry time, if expiry_time_valid != 0; r/o after init */ - nsync_mu note_mu; /* protects fields below except "notified" */ - nsync_cv no_children_cv; /* signalled when children becomes empty */ - uint32_t disconnecting; /* non-zero => node is being disconnected */ - nsync_atomic_uint32_ notified; /* non-zero if the note has been notified */ - struct nsync_note_s_ *parent; /* points to parent, if any */ - nsync_dll_element_ *children; /* list of children */ - nsync_dll_element_ *waiters; /* list of waiters */ -}; - -/* ---------- */ - -void nsync_mu_lock_slow_ (nsync_mu *mu, waiter *w, uint32_t clear, lock_type *l_type); -void nsync_mu_unlock_slow_ (nsync_mu *mu, lock_type *l_type); -nsync_dll_list_ nsync_remove_from_mu_queue_ (nsync_dll_list_ mu_queue, nsync_dll_element_ *e); -void nsync_maybe_merge_conditions_ (nsync_dll_element_ *p, nsync_dll_element_ *n); -nsync_time nsync_note_notified_deadline_ (nsync_note n); -int nsync_sem_wait_with_cancel_ (waiter *w, nsync_time abs_deadline, - nsync_note cancel_note); -NSYNC_CPP_END_ - -#endif /*NSYNC_INTERNAL_COMMON_H_*/ diff --git a/libc/thread/compiler.h b/libc/thread/compiler.h deleted file mode 100644 index 41c938485..000000000 --- a/libc/thread/compiler.h +++ /dev/null @@ -1,24 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_PLATFORM_GCC_COMPILER_H_ -#define NSYNC_PLATFORM_GCC_COMPILER_H_ - -#define INLINE __inline -#define UNUSED __attribute__((unused)) -#define THREAD_LOCAL __thread -#define HAVE_THREAD_LOCAL 1 - -#endif /*NSYNC_PLATFORM_GCC_COMPILER_H_*/ diff --git a/libc/thread/cputype.h b/libc/thread/cputype.h deleted file mode 100644 index e954ebf80..000000000 --- a/libc/thread/cputype.h +++ /dev/null @@ -1,21 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_PLATFORM_X86_64_CPUTYPE_H_ -#define NSYNC_PLATFORM_X86_64_CPUTYPE_H_ - -#define ATM_LD_IS_ACQ_ST_IS_REL_ 1 - -#endif /*NSYNC_PLATFORM_X86_64_CPUTYPE_H_*/ diff --git a/libc/thread/dll.h b/libc/thread/dll.h deleted file mode 100644 index 95c7a53ad..000000000 --- a/libc/thread/dll.h +++ /dev/null @@ -1,78 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_INTERNAL_DLL_H_ -#define NSYNC_INTERNAL_DLL_H_ - -/* Doubly linked lists. */ - -#include "libc/thread/nsync_cpp.h" -NSYNC_CPP_START_ - -/* A nsync_dll_element_ represents an element of a doubly-linked list of waiters. */ -typedef struct nsync_dll_element_s_ { - struct nsync_dll_element_s_ *next; - struct nsync_dll_element_s_ *prev; - void *container; /* points to the struct this nsync_dll struct is embedded in. */ -} nsync_dll_element_; - -/* A nsync_dll_list_ represents a list of nsync_dll_elements_. */ -typedef nsync_dll_element_ *nsync_dll_list_; /* last elem of circular list; nil => empty; first is x.next. */ - - -/* Initialize *e. */ -void nsync_dll_init_ (nsync_dll_element_ *e, void *container); - -/* Return whether list is empty. */ -int nsync_dll_is_empty_ (nsync_dll_list_ list); - -/* Remove *e from list, and returns the new list. */ -nsync_dll_list_ nsync_dll_remove_ (nsync_dll_list_ list, nsync_dll_element_ *e); - -/* Cause element *n and its successors to come after element *p. - Requires n and p are non-NULL and do not point at elements of the same list. */ -void nsync_dll_splice_after_ (nsync_dll_element_ *p, nsync_dll_element_ *n); - -/* Make element *e the first element of list, and return - the list. The resulting list will have *e as its first element, followed by - any elements in the same list as *e, followed by the elements that were - previously in list. Requires that *e not be in list. If e==NULL, list is - returned unchanged. */ -nsync_dll_list_ nsync_dll_make_first_in_list_ (nsync_dll_list_ list, nsync_dll_element_ *e); - -/* Make element *e the last element of list, and return - the list. The resulting list will have *e as its last element, preceded by - any elements in the same list as *e, preceded by the elements that were - previously in list. Requires that *e not be in list. If e==NULL, list is - returned unchanged. */ -nsync_dll_list_ nsync_dll_make_last_in_list_ (nsync_dll_list_ list, nsync_dll_element_ *e); - -/* Return a pointer to the first element of list, or NULL if list is empty. */ -nsync_dll_element_ *nsync_dll_first_ (nsync_dll_list_ list); - -/* Return a pointer to the last element of list, or NULL if list is empty. */ -nsync_dll_element_ *nsync_dll_last_ (nsync_dll_list_ list); - -/* Return a pointer to the next element of list following *e, - or NULL if there is no such element. */ -nsync_dll_element_ *nsync_dll_next_ (nsync_dll_list_ list, nsync_dll_element_ *e); - -/* Return a pointer to the previous element of list following *e, - or NULL if there is no such element. */ -nsync_dll_element_ *nsync_dll_prev_ (nsync_dll_list_ list, nsync_dll_element_ *e); - -NSYNC_CPP_END_ - -#endif /*NSYNC_INTERNAL_DLL_H_*/ diff --git a/libc/thread/headers.h b/libc/thread/headers.h deleted file mode 100644 index 5f74394d2..000000000 --- a/libc/thread/headers.h +++ /dev/null @@ -1,27 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_INTERNAL_HEADERS_H_ -#define NSYNC_INTERNAL_HEADERS_H_ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/platform.h" -#include "libc/thread/compiler.h" -#include "libc/thread/cputype.h" -#include "libc/thread/nsync.h" -#include "libc/thread/atomic.h" -#include "libc/thread/sem.h" - -#endif /*NSYNC_INTERNAL_HEADERS_H_*/ diff --git a/libc/thread/nsync.h b/libc/thread/nsync.h deleted file mode 100644 index ed4db3e41..000000000 --- a/libc/thread/nsync.h +++ /dev/null @@ -1,28 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_PUBLIC_NSYNC_H_ -#define NSYNC_PUBLIC_NSYNC_H_ - -#include "libc/thread/nsync_mu.h" -#include "libc/thread/nsync_mu_wait.h" -#include "libc/thread/nsync_cv.h" -#include "libc/thread/nsync_note.h" -#include "libc/thread/nsync_counter.h" -#include "libc/thread/nsync_waiter.h" -#include "libc/thread/nsync_once.h" -#include "libc/thread/nsync_debug.h" - -#endif /*NSYNC_PUBLIC_NSYNC_H_*/ diff --git a/libc/thread/nsync_atomic.h b/libc/thread/nsync_atomic.h deleted file mode 100644 index 46c04865c..000000000 --- a/libc/thread/nsync_atomic.h +++ /dev/null @@ -1,67 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_PUBLIC_NSYNC_ATOMIC_H_ -#define NSYNC_PUBLIC_NSYNC_ATOMIC_H_ - -#include "libc/thread/nsync_cpp.h" - -/* This file is not to be included directly by the client. It exists because - on some platforms, one cannot use a simple uint32_t with atomic operations. - */ -#if NSYNC_ATOMIC_TYPECHECK -#include "libc/fmt/conv.h" -#include "libc/inttypes.h" -NSYNC_CPP_START_ -typedef struct { uint32_t value; } nsync_atomic_uint32_; -NSYNC_CPP_END_ -#define NSYNC_ATOMIC_UINT32_INIT_ { 0 } -#define NSYNC_ATOMIC_UINT32_LOAD_(p) ((p)->value) -#define NSYNC_ATOMIC_UINT32_STORE_(p,v) ((p)->value = (v)) -#define NSYNC_ATOMIC_UINT32_PTR_(p) (&(p)->value) - -#elif NSYNC_ATOMIC_C11 -#include "libc/intrin/atomic.h" -NSYNC_CPP_START_ -typedef atomic_uint_least32_t nsync_atomic_uint32_; -NSYNC_CPP_END_ -#define NSYNC_ATOMIC_UINT32_INIT_ 0 -#define NSYNC_ATOMIC_UINT32_LOAD_(p) (*(p)) -#define NSYNC_ATOMIC_UINT32_STORE_(p,v) (*(p) = (v)) -#define NSYNC_ATOMIC_UINT32_PTR_(p) (p) - -#elif NSYNC_ATOMIC_CPP11 -#include "third_party/libcxx/atomic" -NSYNC_CPP_START_ -typedef std::atomic nsync_atomic_uint32_; -NSYNC_CPP_END_ -#define NSYNC_ATOMIC_UINT32_INIT_ ATOMIC_VAR_INIT (0) -#define NSYNC_ATOMIC_UINT32_LOAD_(p) (std::atomic_load (p)) -#define NSYNC_ATOMIC_UINT32_STORE_(p,v) (std::atomic_store ((p), (uint32_t) (v))) -#define NSYNC_ATOMIC_UINT32_PTR_(p) (p) - -#else -#include "libc/fmt/conv.h" -#include "libc/inttypes.h" -NSYNC_CPP_START_ -typedef uint32_t nsync_atomic_uint32_; -NSYNC_CPP_END_ -#define NSYNC_ATOMIC_UINT32_INIT_ 0 -#define NSYNC_ATOMIC_UINT32_LOAD_(p) (*(p)) -#define NSYNC_ATOMIC_UINT32_STORE_(p,v) (*(p) = (v)) -#define NSYNC_ATOMIC_UINT32_PTR_(p) (p) -#endif - -#endif /*NSYNC_PUBLIC_NSYNC_ATOMIC_H_*/ diff --git a/libc/thread/counter.c b/libc/thread/nsync_counter.c similarity index 60% rename from libc/thread/counter.c rename to libc/thread/nsync_counter.c index b6a41c14a..a3dfa36f9 100644 --- a/libc/thread/counter.c +++ b/libc/thread/nsync_counter.c @@ -1,30 +1,36 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2016 Google Inc. │ +│ │ +│ Licensed under the Apache License, Version 2.0 (the "License"); │ +│ you may not use this file except in compliance with the License. │ +│ You may obtain a copy of the License at │ +│ │ +│ http://www.apache.org/licenses/LICENSE-2.0 │ +│ │ +│ Unless required by applicable law or agreed to in writing, software │ +│ distributed under the License is distributed on an "AS IS" BASIS, │ +│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ +│ See the License for the specific language governing permissions and │ +│ limitations under the License. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/mem/mem.h" +#include "libc/str/str.h" +#include "third_party/nsync/atomic.h" +#include "third_party/nsync/atomic.internal.h" +#include "third_party/nsync/common.internal.h" +#include "third_party/nsync/counter.h" +#include "third_party/nsync/dll.h" +#include "third_party/nsync/mu_semaphore.h" +#include "third_party/nsync/wait_s.internal.h" +#include "third_party/nsync/waiter.h" + +asm(".ident\t\"\\n\\n\ +*NSYNC (Apache 2.0)\\n\ +Copyright 2016 Google, Inc.\\n\ +https://github.com/google/nsync\""); // clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/platform.h" -#include "libc/thread/compiler.h" -#include "libc/thread/cputype.h" -#include "libc/thread/nsync.h" -#include "libc/thread/atomic.h" -#include "libc/thread/dll.h" -#include "libc/thread/sem.h" -#include "libc/thread/wait_internal.h" -#include "libc/thread/common.h" - -NSYNC_CPP_START_ /* Internal details of nsync_counter. */ struct nsync_counter_s_ { @@ -106,7 +112,7 @@ uint32_t nsync_counter_wait (nsync_counter c, nsync_time abs_deadline) { return (result); } -static nsync_time counter_ready_time (void *v, struct nsync_waiter_s *nw UNUSED) { +static nsync_time counter_ready_time (void *v, struct nsync_waiter_s *nw) { nsync_counter c = (nsync_counter) v; nsync_time r; ATM_STORE (&c->waited, 1); @@ -148,4 +154,4 @@ const struct nsync_waitable_funcs_s nsync_counter_waitable_funcs = { &counter_dequeue }; -NSYNC_CPP_END_ + diff --git a/libc/thread/nsync_counter.h b/libc/thread/nsync_counter.h deleted file mode 100644 index 4a1910686..000000000 --- a/libc/thread/nsync_counter.h +++ /dev/null @@ -1,64 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_PUBLIC_NSYNC_COUNTER_H_ -#define NSYNC_PUBLIC_NSYNC_COUNTER_H_ - -#include "libc/fmt/conv.h" -#include "libc/inttypes.h" -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/nsync_mu.h" -#include "libc/thread/nsync_atomic.h" -#include "libc/thread/nsync_time.h" - -NSYNC_CPP_START_ - -struct nsync_dll_element_s_; - -/* An nsync_counter represents an unsigned integer that can count up and down, - and wake waiters when zero. */ -typedef struct nsync_counter_s_ *nsync_counter; - -/* Return a freshly allocated nsync_counter with the specified value, - of NULL if an nsync_counter cannot be created. - - Any non-NULL returned value should be passed to nsync_counter_free() when no - longer needed. */ -nsync_counter nsync_counter_new (uint32_t value); - -/* Free resources associated with c. Requires that c was allocated by - nsync_counter_new(), and no concurrent or future operations are applied to - c. */ -void nsync_counter_free (nsync_counter c); - -/* Add delta to c, and return its new value. It is a checkable runtime error - to decrement c below 0, or to increment c (i.e., apply a delta > 0) after a - waiter has waited. */ -uint32_t nsync_counter_add (nsync_counter c, int32_t delta); - -/* Return the current value of c. */ -uint32_t nsync_counter_value (nsync_counter c); - -/* Wait until c has value 0, or until abs_deadline, then return - the value of c. It is a checkable runtime error to increment c after - a waiter may have been woken due to the counter reaching zero. - If abs_deadline==nsync_time_no_deadline, the deadline - is far in the future. */ -uint32_t nsync_counter_wait (nsync_counter c, nsync_time abs_deadline); - -NSYNC_COUNTER_CPP_OVERLOAD_ -NSYNC_CPP_END_ - -#endif /*NSYNC_PUBLIC_NSYNC_COUNTER_H_*/ diff --git a/libc/thread/nsync_cpp.h b/libc/thread/nsync_cpp.h deleted file mode 100644 index 9ee463d94..000000000 --- a/libc/thread/nsync_cpp.h +++ /dev/null @@ -1,46 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_PUBLIC_NSYNC_CPP_H_ -#define NSYNC_PUBLIC_NSYNC_CPP_H_ - -/* This header file permits compilation via a C++ compiler using the macros - NSYNC_CPP_START_, NSYNC_CPP_END_, and NSYNC_CPP_USING_. - - NSYNC_CPP_START_ and NSYNC_CPP_END_ surround C code in the public library. - They put all public symbols into the "nsync" name space. - - NSYNC_CPP_USING_ is used before C code (used for testing) that might use - public exports from this package. It makes symbols in the "nsync" - name space available without the "nsync::" prefix. - - NSYNC_C_START_ and NSYNC_C_END_ surround C code in the C++ modules. - */ - -#if defined(__cplusplus) -#define NSYNC_CPP_START_ namespace nsync { -#define NSYNC_CPP_END_ } -#define NSYNC_CPP_USING_ using namespace nsync; -#define NSYNC_C_START_ extern "C" { -#define NSYNC_C_END_ } -#else -#define NSYNC_CPP_START_ -#define NSYNC_CPP_END_ -#define NSYNC_CPP_USING_ -#define NSYNC_C_START_ -#define NSYNC_C_END_ -#endif - -#endif /*NSYNC_PUBLIC_NSYNC_CPP_H_*/ diff --git a/libc/thread/cv.c b/libc/thread/nsync_cv.c similarity index 89% rename from libc/thread/cv.c rename to libc/thread/nsync_cv.c index 660c8fb97..89a128912 100644 --- a/libc/thread/cv.c +++ b/libc/thread/nsync_cv.c @@ -1,30 +1,33 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2016 Google Inc. │ +│ │ +│ Licensed under the Apache License, Version 2.0 (the "License"); │ +│ you may not use this file except in compliance with the License. │ +│ You may obtain a copy of the License at │ +│ │ +│ http://www.apache.org/licenses/LICENSE-2.0 │ +│ │ +│ Unless required by applicable law or agreed to in writing, software │ +│ distributed under the License is distributed on an "AS IS" BASIS, │ +│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ +│ See the License for the specific language governing permissions and │ +│ limitations under the License. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/str/str.h" +#include "third_party/nsync/atomic.internal.h" +#include "third_party/nsync/common.internal.h" +#include "third_party/nsync/cv.h" +#include "third_party/nsync/dll.h" +#include "third_party/nsync/wait_s.internal.h" +#include "third_party/nsync/waiter.h" + +asm(".ident\t\"\\n\\n\ +*NSYNC (Apache 2.0)\\n\ +Copyright 2016 Google, Inc.\\n\ +https://github.com/google/nsync\""); // clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/platform.h" -#include "libc/thread/compiler.h" -#include "libc/thread/cputype.h" -#include "libc/thread/nsync.h" -#include "libc/thread/dll.h" -#include "libc/thread/sem.h" -#include "libc/thread/wait_internal.h" -#include "libc/thread/common.h" -#include "libc/thread/atomic.h" - -NSYNC_CPP_START_ /* Initialize *cv. */ void nsync_cv_init (nsync_cv *cv) { @@ -451,7 +454,7 @@ void nsync_cv_wait (nsync_cv *pcv, nsync_mu *pmu) { nsync_cv_wait_with_deadline (pcv, pmu, nsync_time_no_deadline, NULL); } -static nsync_time cv_ready_time (void *v UNUSED, struct nsync_waiter_s *nw) { +static nsync_time cv_ready_time (void *v, struct nsync_waiter_s *nw) { nsync_time r; r = (nw == NULL || ATM_LOAD_ACQ (&nw->waiting) != 0? nsync_time_no_deadline : nsync_time_zero); return (r); @@ -491,5 +494,3 @@ const struct nsync_waitable_funcs_s nsync_cv_waitable_funcs = { &cv_enqueue, &cv_dequeue }; - -NSYNC_CPP_END_ diff --git a/libc/thread/nsync_cv.h b/libc/thread/nsync_cv.h deleted file mode 100644 index a48c3482c..000000000 --- a/libc/thread/nsync_cv.h +++ /dev/null @@ -1,150 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_PUBLIC_NSYNC_CV_H_ -#define NSYNC_PUBLIC_NSYNC_CV_H_ - -#include "libc/fmt/conv.h" -#include "libc/inttypes.h" -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/nsync_mu.h" -#include "libc/thread/nsync_atomic.h" -#include "libc/thread/nsync_time.h" - -NSYNC_CPP_START_ - -struct nsync_dll_element_s_; -struct nsync_note_s_; - -/* An nsync_cv is a condition variable in the style of Mesa, Java, POSIX, and Go's sync.Cond. - It allows a thread to wait for a condition on state protected by a mutex, - and to proceed with the mutex held and the condition true. - - See also nsync_mu_wait() and nsync_mu_wait_with_deadline(), which implement conditional - critical sections. In many cases, they are easier to use than condition - variables. - - Usage: - - after making the desired predicate true, call: - nsync_cv_signal (&cv); // If at most one thread can make use of the predicate becoming true. - or - nsync_cv_broadcast (&cv); // If multiple threads can make use of the predicate becoming true. - - To wait for a predicate with no deadline (assuming nsync_cv_broadcast() or - nsync_cv_signal() is called whenever the predicate becomes true): - nsync_mu_lock (μ) - while (!some_predicate_protected_by_mu) { // the while-loop is required. - nsync_cv_wait (&cv, &mu); - } - // predicate is now true - nsync_mu_unlock (&mu); - - To wait for a predicate with a deadline (assuming nsync_cv_broadcast() or - nsync_cv_signal() is called whenever the predicate becomes true): - nsync_mu_lock (&mu); - while (!some_predicate_protected_by_mu && - nsync_cv_wait_with_deadline (&cv, &mu, abs_deadline, cancel_note) == 0) { - } - if (some_predicate_protected_by_mu) { // predicate is true - } else { // predicate is false, and deadline expired, or cancel_note was notified. - } - nsync_mu_unlock (&mu); - or, if the predicate is complex and you wish to write it just once and - inline, you could use the following instead of the for-loop above: - nsync_mu_lock (&mu); - int pred_is_true = 0; - int outcome = 0; - while (!(pred_is_true = some_predicate_protected_by_mu) && outcome == 0) { - outcome = nsync_cv_wait_with_deadline (&cv, &mu, abs_deadline, cancel_note); - } - if (pred_is_true) { // predicate is true - } else { // predicate is false, and deadline expired, or cancel_note was notified. - } - nsync_mu_unlock (&mu); - - As the examples show, Mesa-style condition variables require that waits use - a loop that tests the predicate anew after each wait. It may be surprising - that these are preferred over the precise wakeups offered by the condition - variables in Hoare monitors. Imprecise wakeups make more efficient use of - the critical section, because threads can enter it while a woken thread is - still emerging from the scheduler, which may take thousands of cycles. - Further, they make the programme easier to read and debug by making the - predicate explicit locally at the wait, where the predicate is about to be - assumed; the reader does not have to infer the predicate by examining all - the places where wakeups may occur. */ -typedef struct nsync_cv_s_ { - nsync_atomic_uint32_ word; /* see bits below */ - struct nsync_dll_element_s_ *waiters; /* points to tail of list of waiters; under mu. */ -} nsync_cv; - -/* An nsync_cv should be zeroed to initialize, which can be accomplished by - initializing with static initializer NSYNC_CV_INIT, or by setting the entire - struct to 0, or using nsync_cv_init(). */ -#define NSYNC_CV_INIT { NSYNC_ATOMIC_UINT32_INIT_, 0 } -void nsync_cv_init (nsync_cv *cv); - -/* Wake at least one thread if any are currently blocked on *cv. If - the chosen thread is a reader on an nsync_mu, wake all readers and, if - possible, a writer. */ -void nsync_cv_signal (nsync_cv *cv); - -/* Wake all threads currently blocked on *cv. */ -void nsync_cv_broadcast (nsync_cv *cv); - -/* Atomically release "mu" (which must be held on entry) and block the caller - on *cv. Wait until awakened by a call to nsync_cv_signal() or - nsync_cv_broadcast(), or a spurious wakeup; then reacquire "mu", and return. - Equivalent to a call to nsync_mu_wait_with_deadline() with - abs_deadline==nsync_time_no_deadline, and cancel_note==NULL. Callers should use - nsync_cv_wait() in a loop, as with all standard Mesa-style condition - variables. See examples above. */ -void nsync_cv_wait (nsync_cv *cv, nsync_mu *mu); - -/* Atomically release "mu" (which must be held on entry) - and block the calling thread on *cv. It then waits until awakened by a - call to nsync_cv_signal() or nsync_cv_broadcast() (or a spurious wakeup), or by the time - reaching abs_deadline, or by cancel_note being notified. In all cases, it - reacquires "mu", and returns the reason for the call returned (0, ETIMEDOUT, - or ECANCELED). Use abs_deadline==nsync_time_no_deadline for no deadline, and - cancel_note==NULL for no cancellation. wait_with_deadline() should be used in a - loop, as with all Mesa-style condition variables. See examples above. - - There are two reasons for using an absolute deadline, rather than a relative - timeout---these are why pthread_cond_timedwait() also uses an absolute - deadline. First, condition variable waits have to be used in a loop; with - an absolute times, the deadline does not have to be recomputed on each - iteration. Second, in most real programmes, some activity (such as an RPC - to a server, or when guaranteeing response time in a UI), there is a - deadline imposed by the specification or the caller/user; relative delays - can shift arbitrarily with scheduling delays, and so after multiple waits - might extend beyond the expected deadline. Relative delays tend to be more - convenient mostly in tests and trivial examples than they are in real - programmes. */ -int nsync_cv_wait_with_deadline (nsync_cv *cv, nsync_mu *mu, - nsync_time abs_deadline, - struct nsync_note_s_ *cancel_note); - -/* Like nsync_cv_wait_with_deadline(), but allow an arbitrary lock *v to be used, - given its (*lock)(mu) and (*unlock)(mu) routines. */ -int nsync_cv_wait_with_deadline_generic (nsync_cv *cv, - void *mu, void (*lock) (void *), void (*unlock) (void *), - nsync_time abs_deadline, - struct nsync_note_s_ *cancel_note); - -NSYNC_CV_CPP_OVERLOAD_ -NSYNC_CPP_END_ - -#endif /*NSYNC_PUBLIC_NSYNC_CV_H_*/ diff --git a/libc/thread/debug.c b/libc/thread/nsync_debug.c similarity index 83% rename from libc/thread/debug.c rename to libc/thread/nsync_debug.c index c2a617744..af38bc93b 100644 --- a/libc/thread/debug.c +++ b/libc/thread/nsync_debug.c @@ -1,35 +1,34 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2016 Google Inc. │ +│ │ +│ Licensed under the Apache License, Version 2.0 (the "License"); │ +│ you may not use this file except in compliance with the License. │ +│ You may obtain a copy of the License at │ +│ │ +│ http://www.apache.org/licenses/LICENSE-2.0 │ +│ │ +│ Unless required by applicable law or agreed to in writing, software │ +│ distributed under the License is distributed on an "AS IS" BASIS, │ +│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ +│ See the License for the specific language governing permissions and │ +│ limitations under the License. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "third_party/nsync/atomic.h" +#include "third_party/nsync/common.internal.h" +#include "third_party/nsync/dll.h" +#include "third_party/nsync/mu_semaphore.h" +#include "third_party/nsync/wait_s.internal.h" + +asm(".ident\t\"\\n\\n\ +*NSYNC (Apache 2.0)\\n\ +Copyright 2016 Google, Inc.\\n\ +https://github.com/google/nsync\""); // clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ /* Routines for debugging. */ -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/platform.h" -#include "libc/thread/compiler.h" -#include "libc/thread/cputype.h" -#include "libc/thread/nsync.h" -#include "libc/thread/dll.h" -#include "libc/thread/sem.h" -#include "libc/thread/wait_internal.h" -#include "libc/thread/common.h" -#include "libc/thread/atomic.h" - -NSYNC_CPP_START_ - -/* ---------- */ - /* An emit_buf represents a buffer into which debug information can be written. */ struct emit_buf { @@ -290,5 +289,3 @@ char *nsync_cv_debugger (nsync_cv *cv) { (int) sizeof (nsync_debug_buf)), cv, 0, 1)); } - -NSYNC_CPP_END_ diff --git a/libc/thread/nsync_debug.h b/libc/thread/nsync_debug.h deleted file mode 100644 index d28184306..000000000 --- a/libc/thread/nsync_debug.h +++ /dev/null @@ -1,55 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_PUBLIC_NSYNC_DEBUG_H_ -#define NSYNC_PUBLIC_NSYNC_DEBUG_H_ - -/* Debugging operations for mutexes and condition variables. - - These operations should not be relied upon for normal functionality. The - implementation may be slow, output formats may change, and the - implementation is free to yield the empty string. */ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/nsync_mu.h" -#include "libc/thread/nsync_cv.h" - -NSYNC_CPP_START_ - -/* Place in buf[0,..,n-1] a nul-terminated, human readable string indicative of - some of the internal state of the mutex or condition variable, and return - buf. If n>=4, buffer overflow is indicated by placing the characters "..." - at the end of the string. - - The *_and_waiters() variants attempt to output the waiter lists in addition - to the basic state. These variants may acquire internal locks and follow - internal pointers. Thus, they are riskier if invoked in an address space - whose overall health is uncertain. */ -char *nsync_mu_debug_state (nsync_mu *mu, char *buf, int n); -char *nsync_cv_debug_state (nsync_cv *cv, char *buf, int n); -char *nsync_mu_debug_state_and_waiters (nsync_mu *mu, char *buf, int n); -char *nsync_cv_debug_state_and_waiters (nsync_cv *cv, char *buf, int n); - -/* Like nsync_*_debug_state_and_waiters(), but ignoring all locking and safety - considerations, and using an internal, possibly static buffer that may be - overwritten by subsequent or concurrent calls to these routines. These - variants should be used only from an interactive debugger, when all other - threads are stopped; the debugger is expected to recover from errors. */ -char *nsync_mu_debugger (nsync_mu *mu); -char *nsync_cv_debugger (nsync_cv *cv); - -NSYNC_CPP_END_ - -#endif /*NSYNC_PUBLIC_NSYNC_DEBUG_H_*/ diff --git a/libc/thread/nsync_mu.h b/libc/thread/nsync_mu.h deleted file mode 100644 index 958c77ee8..000000000 --- a/libc/thread/nsync_mu.h +++ /dev/null @@ -1,115 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_PUBLIC_NSYNC_MU_H_ -#define NSYNC_PUBLIC_NSYNC_MU_H_ - -#include "libc/fmt/conv.h" -#include "libc/inttypes.h" -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/nsync_atomic.h" - -NSYNC_CPP_START_ - -struct nsync_dll_element_s_; - -/* An nsync_mu is a lock. If initialized to all zeroes, it is valid and unlocked. - - An nsync_mu can be "free", held by a single thread (aka fiber, goroutine) in - "write" (exclusive) mode, or by many threads in "read" (shared) mode. A - thread that acquires it should eventually release it. It is illegal to - acquire an nsync_mu in one thread and release it in another. It is - illegal for a thread to reacquire an nsync_mu while holding it (even a - second share of a "read" lock). - - Example usage: - static struct foo { - nsync_mu mu; // protects invariant a+b==0 on fields below. - int a; - int b; - } p = { NSYNC_MU_INIT, 0, 0 }; - .... - nsync_mu_lock (&p.mu); - // The current thread now has exclusive access to p.a and p.b; invariant assumed true. - p.a++; - p.b--; // restore invariant p.a+p.b==0 before releasing p.mu - nsync_mu_unlock (&p.mu) - - Mutexes can be used with condition variables; see nsync_cv.h. - - nsync_mu_wait() and nsync_mu_wait_with_deadline() can be used instead of - condition variables. See nsync_mu_wait.h for more details. - Example use of nsync_mu_wait() to wait for p.a==0, using definition above: - int a_is_zero (const void *condition_arg) { - return (((const struct foo *)condition_arg)->a == 0); - } - ... - nsync_mu_lock (&p.mu); - nsync_mu_wait (&p.mu, &a_is_zero, &p, NULL); - // The current thread now has exclusive access to p.a and p.b, and p.a==0. - ... - nsync_mu_unlock (&p.mu); */ -typedef struct nsync_mu_s_ { - nsync_atomic_uint32_ word; /* internal use only */ - struct nsync_dll_element_s_ *waiters; /* internal use only */ -} nsync_mu; - -/* An nsync_mu should be zeroed to initialize, which can be accomplished by - initializing with static initializer NSYNC_MU_INIT, or by setting the entire - structure to all zeroes, or using nsync_mu_init(). */ -#define NSYNC_MU_INIT { NSYNC_ATOMIC_UINT32_INIT_, 0 } -void nsync_mu_init (nsync_mu *mu); - -/* Block until *mu is free and then acquire it in writer mode. - Requires that the calling thread not already hold *mu in any mode. */ -void nsync_mu_lock (nsync_mu *mu); - -/* Unlock *mu, which must have been acquired in write mode by the calling - thread, and wake waiters, if appropriate. */ -void nsync_mu_unlock (nsync_mu *mu); - -/* Attempt to acquire *mu in writer mode without blocking, and return non-zero - iff successful. Return non-zero with high probability if *mu was free - on entry. */ -int nsync_mu_trylock (nsync_mu *mu); - -/* Block until *mu can be acquired in reader mode and then acquire it. - Requires that the calling thread not already hold *mu in any mode. */ -void nsync_mu_rlock (nsync_mu *mu); - -/* Unlock *mu, which must have been acquired in read mode by the calling - thread, and wake waiters, if appropriate. */ -void nsync_mu_runlock (nsync_mu *mu); - -/* Attempt to acquire *mu in reader mode without blocking, and return non-zero - iff successful. Return non-zero with high probability if *mu was free on - entry. Perhaps fail to acquire if a writer is waiting, to avoid starvation. - */ -int nsync_mu_rtrylock (nsync_mu *mu); - -/* May abort if *mu is not held in write mode by the calling thread. */ -void nsync_mu_assert_held (const nsync_mu *mu); - -/* May abort if *mu is not held in read or write mode - by the calling thread. */ -void nsync_mu_rassert_held (const nsync_mu *mu); - -/* Return whether *mu is held in read mode. - Requires that the calling thread holds *mu in some mode. */ -int nsync_mu_is_reader (const nsync_mu *mu); - -NSYNC_CPP_END_ - -#endif /*NSYNC_PUBLIC_NSYNC_MU_H_*/ diff --git a/libc/thread/mu_wait.c b/libc/thread/nsync_mu_wait.c similarity index 85% rename from libc/thread/mu_wait.c rename to libc/thread/nsync_mu_wait.c index f2c966238..d9c249259 100644 --- a/libc/thread/mu_wait.c +++ b/libc/thread/nsync_mu_wait.c @@ -1,30 +1,31 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2016 Google Inc. │ +│ │ +│ Licensed under the Apache License, Version 2.0 (the "License"); │ +│ you may not use this file except in compliance with the License. │ +│ You may obtain a copy of the License at │ +│ │ +│ http://www.apache.org/licenses/LICENSE-2.0 │ +│ │ +│ Unless required by applicable law or agreed to in writing, software │ +│ distributed under the License is distributed on an "AS IS" BASIS, │ +│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ +│ See the License for the specific language governing permissions and │ +│ limitations under the License. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "third_party/nsync/atomic.h" +#include "third_party/nsync/common.internal.h" +#include "third_party/nsync/dll.h" +#include "third_party/nsync/mu_semaphore.h" +#include "third_party/nsync/wait_s.internal.h" + +asm(".ident\t\"\\n\\n\ +*NSYNC (Apache 2.0)\\n\ +Copyright 2016 Google, Inc.\\n\ +https://github.com/google/nsync\""); // clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/platform.h" -#include "libc/thread/compiler.h" -#include "libc/thread/cputype.h" -#include "libc/thread/nsync.h" -#include "libc/thread/dll.h" -#include "libc/thread/sem.h" -#include "libc/thread/wait_internal.h" -#include "libc/thread/common.h" -#include "libc/thread/atomic.h" - -NSYNC_CPP_START_ /* Attempt to remove waiter *w from *mu's waiter queue. If successful, leave the lock held in mode *l_type, and @@ -317,4 +318,4 @@ void nsync_mu_unlock_without_wakeup (nsync_mu *mu) { IGNORE_RACES_END (); } -NSYNC_CPP_END_ + diff --git a/libc/thread/nsync_mu_wait.h b/libc/thread/nsync_mu_wait.h deleted file mode 100644 index f3432271f..000000000 --- a/libc/thread/nsync_mu_wait.h +++ /dev/null @@ -1,129 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_PUBLIC_NSYNC_MU_WAIT_H_ -#define NSYNC_PUBLIC_NSYNC_MU_WAIT_H_ - -/* nsync_mu_wait() and nsync_mu_wait_with_deadline() can be used instead of condition - variables. In many straightforward situations they are of equivalent - performance and are somewhat easier to use, because unlike condition - variables, they do not require that the waits be placed in a loop, and they - do not require explicit wakeup calls. Example: - - Definitions: - static nsync_mu mu = NSYNC_MU_INIT; - static int i = 0; // protected by mu - // Condition for use with nsync_mu_wait(). - static int int_is_zero (const void *v) { return (*(const int *)v == 0); } - - Waiter: - nsync_mu_lock (&mu); - // Wait until i is zero. - nsync_mu_wait (&mu, &int_is_zero, &i, NULL); - // i is known to be zero here. - // ... - nsync_mu_unlock (&mu); - - - Thread potentially making i zero: - nsync_mu_lock (&mu); - i--; - // No need to signal that i may have become zero. The unlock call below - // will evaluate waiters' conditions to decide which to wake. - nsync_mu_unlock (&mu); - - It is legal to use conditional critical sections and condition variables - on the same mutex. - - -------------- - - The implementation benefits from determining whether waiters are waiting for - the same condition; it may then evaluate a condition once on behalf - of several waiters. Two waiters have equal condition if their "condition" - pointers are equal, and either: - - their "condition_arg" pointers are equal, or - - "condition_arg_eq" is non-null and - (*condition_arg_eq) (condition_arg0, condition_arg1) returns non-zero. - *condition_arg_eq will not be invoked unless the "condition" pointers - are equal, and the "condition_arg" pointers are unequal. - - If many waiters wait for distinct conditions simultaneously, condition - variables may be faster. - */ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/nsync_mu.h" -#include "libc/thread/nsync_time.h" - -NSYNC_CPP_START_ - -struct nsync_note_s_; /* forward declaration for an nsync_note */ - -/* Return when (*condition) (condition_arg) is true. Perhaps unlock and relock - *mu while blocked waiting for the condition to become true. nsync_mu_wait() - is equivalent to nsync_mu_wait_with_deadline() with - abs_deadline==nsync_time_no_deadline, and cancel_note==NULL. - - Requires that *mu be held on entry. - See nsync_mu_wait_with_deadline() for more details on *condition and - *condition_arg_eq. */ -void nsync_mu_wait (nsync_mu *mu, int (*condition) (const void *condition_arg), - const void *condition_arg, - int (*condition_arg_eq) (const void *a, const void *b)); - -/* Return when at least one of: (*condition) (condition_arg) is true, the - deadline expires, or *cancel_note is notified. Perhaps unlock and relock *mu - while blocked waiting for one of these events, but always return with *mu - held. Return 0 iff the (*condition) (condition_arg) is true on return, and - otherwise either ETIMEDOUT or ECANCELED, depending on why the call returned - early. Callers should use abs_deadline==nsync_time_no_deadline for no - deadline, and cancel_note==NULL for no cancellation. - - Requires that *mu be held on entry. - - The implementation may call *condition from any thread using the mutex, and - while holding *mu in either read or write mode; it guarantees that any - thread calling *condition will hold *mu in some mode. - Requires that (*condition) (condition_arg) neither modify state protected by - *mu, nor return a value dependent on state not protected by *mu. To depend - on time, use the abs_deadline parameter. - (Conventional use of condition variables have the same restrictions on the - conditions tested by the while-loop.) - If non-null, condition_arg_eq should return whether two condition_arg - calls with the same "condition" pointer are considered equivalent; it should - have no side-effects. */ -int nsync_mu_wait_with_deadline (nsync_mu *mu, - int (*condition) (const void *condition_arg), - const void *condition_arg, - int (*condition_arg_eq) (const void *a, const void *b), - nsync_time abs_deadline, - struct nsync_note_s_ *cancel_note); - -/* Unlock *mu, which must be held in write mode, and wake waiters, if - appropriate. Unlike nsync_mu_unlock(), this call is not required to wake - nsync_mu_wait/nsync_mu_wait_with_deadline calls on conditions that were - false before this thread acquired the lock. This call should be used only - at the end of critical sections for which: - - nsync_mu_wait and/or nsync_mu_wait_with_deadline are in use on the same - mutex, - - this critical section cannot make the condition true for any of those - nsync_mu_wait/nsync_mu_wait_with_deadline waits, and - - when performance is significantly improved by using this call. */ -void nsync_mu_unlock_without_wakeup (nsync_mu *mu); - -NSYNC_MU_WAIT_CPP_OVERLOAD_ -NSYNC_CPP_END_ - -#endif /*NSYNC_PUBLIC_NSYNC_MU_WAIT_H_*/ diff --git a/libc/thread/note.c b/libc/thread/nsync_note.c similarity index 80% rename from libc/thread/note.c rename to libc/thread/nsync_note.c index 9b7299c90..cd05692bd 100644 --- a/libc/thread/note.c +++ b/libc/thread/nsync_note.c @@ -1,30 +1,35 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2016 Google Inc. │ +│ │ +│ Licensed under the Apache License, Version 2.0 (the "License"); │ +│ you may not use this file except in compliance with the License. │ +│ You may obtain a copy of the License at │ +│ │ +│ http://www.apache.org/licenses/LICENSE-2.0 │ +│ │ +│ Unless required by applicable law or agreed to in writing, software │ +│ distributed under the License is distributed on an "AS IS" BASIS, │ +│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ +│ See the License for the specific language governing permissions and │ +│ limitations under the License. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/mem/mem.h" +#include "libc/str/str.h" +#include "third_party/nsync/atomic.h" +#include "third_party/nsync/common.internal.h" +#include "third_party/nsync/dll.h" +#include "third_party/nsync/mu_semaphore.h" +#include "third_party/nsync/mu_wait.h" +#include "third_party/nsync/wait_s.internal.h" +#include "third_party/nsync/waiter.h" + +asm(".ident\t\"\\n\\n\ +*NSYNC (Apache 2.0)\\n\ +Copyright 2016 Google, Inc.\\n\ +https://github.com/google/nsync\""); // clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/platform.h" -#include "libc/thread/compiler.h" -#include "libc/thread/cputype.h" -#include "libc/thread/nsync.h" -#include "libc/thread/dll.h" -#include "libc/thread/sem.h" -#include "libc/thread/wait_internal.h" -#include "libc/thread/common.h" -#include "libc/thread/atomic.h" - -NSYNC_CPP_START_ /* Locking discipline for the nsync_note implementation: @@ -256,7 +261,7 @@ nsync_time nsync_note_expiry (nsync_note n) { return (n->expiry_time); } -static nsync_time note_ready_time (void *v, struct nsync_waiter_s *nw UNUSED) { +static nsync_time note_ready_time (void *v, struct nsync_waiter_s *nw) { return (nsync_note_notified_deadline_ ((nsync_note)v)); } @@ -299,5 +304,3 @@ const struct nsync_waitable_funcs_s nsync_note_waitable_funcs = { ¬e_enqueue, ¬e_dequeue }; - -NSYNC_CPP_END_ diff --git a/libc/thread/nsync_note.h b/libc/thread/nsync_note.h deleted file mode 100644 index ea4add583..000000000 --- a/libc/thread/nsync_note.h +++ /dev/null @@ -1,68 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_PUBLIC_NSYNC_NOTE_H_ -#define NSYNC_PUBLIC_NSYNC_NOTE_H_ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/nsync_time.h" - -NSYNC_CPP_START_ - -/* An nsync_note represents a single bit that can transition from 0 to 1 at - most once. When 1, the note is said to be notified. There are operations - to wait for the transition, which can be triggered either by an explicit - call, or timer expiry. Notes can have parent notes; a note becomes notified - if its parent becomes notified. */ -typedef struct nsync_note_s_ *nsync_note; - -/* Return a freshly allocated nsync_note, or NULL if an nsync_note cannot be - created. - - If parent!=NULL, the allocated nsync_note's parent will be parent. The - newaly allocated note will be automatically notified at abs_deadline, and is - notified at initialization if abs_deadline==nsync_zero_time. - - nsync_notes should be passed to nsync_note_free() when no longer needed. */ -nsync_note nsync_note_new (nsync_note parent, nsync_time abs_deadline); - -/* Free resources associated with n. Requires that n was allocated by - nsync_note_new(), and no concurrent or future operations are applied to n - directly. - It is legal to call nsync_note_free() on a node even if it has a parent or - children that are in use; if n has both a parent and children, n's - parent adopts its children. */ -void nsync_note_free (nsync_note n); - -/* Notify n and all its descendants. */ -void nsync_note_notify (nsync_note n); - -/* Return whether n has been notified. */ -int nsync_note_is_notified (nsync_note n); - -/* Wait until n has been notified or abs_deadline is reached, and return - whether n has been notified. If abs_deadline==nsync_time_no_deadline, - the deadline is far in the future. */ -int nsync_note_wait (nsync_note n, nsync_time abs_deadline); - -/* Return the expiry time associated with n. - This is the minimum of the abs_deadline passed on creation and that of any - of its ancestors. */ -nsync_time nsync_note_expiry (nsync_note n); - -NSYNC_NOTE_CPP_OVERLOAD_ -NSYNC_CPP_END_ - -#endif /*NSYNC_PUBLIC_NSYNC_NOTE_H_*/ diff --git a/libc/thread/once.c b/libc/thread/nsync_once.c similarity index 62% rename from libc/thread/once.c rename to libc/thread/nsync_once.c index d41ac9e0d..0a742e9ab 100644 --- a/libc/thread/once.c +++ b/libc/thread/nsync_once.c @@ -1,30 +1,33 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2016 Google Inc. │ +│ │ +│ Licensed under the Apache License, Version 2.0 (the "License"); │ +│ you may not use this file except in compliance with the License. │ +│ You may obtain a copy of the License at │ +│ │ +│ http://www.apache.org/licenses/LICENSE-2.0 │ +│ │ +│ Unless required by applicable law or agreed to in writing, software │ +│ distributed under the License is distributed on an "AS IS" BASIS, │ +│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ +│ See the License for the specific language governing permissions and │ +│ limitations under the License. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "third_party/nsync/atomic.h" +#include "third_party/nsync/atomic.internal.h" +#include "third_party/nsync/common.internal.h" +#include "third_party/nsync/dll.h" +#include "third_party/nsync/mu_semaphore.h" +#include "third_party/nsync/once.h" +#include "third_party/nsync/wait_s.internal.h" + +asm(".ident\t\"\\n\\n\ +*NSYNC (Apache 2.0)\\n\ +Copyright 2016 Google, Inc.\\n\ +https://github.com/google/nsync\""); // clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/platform.h" -#include "libc/thread/compiler.h" -#include "libc/thread/cputype.h" -#include "libc/thread/nsync.h" -#include "libc/thread/dll.h" -#include "libc/thread/sem.h" -#include "libc/thread/wait_internal.h" -#include "libc/thread/common.h" -#include "libc/thread/atomic.h" - -NSYNC_CPP_START_ /* An once_sync_s struct contains a lock, and a condition variable on which threads may wait for an nsync_once to be initialized by another thread. @@ -144,5 +147,3 @@ void nsync_run_once_arg_spin (nsync_once *once, void (*farg) (void *arg), void * } IGNORE_RACES_END (); } - -NSYNC_CPP_END_ diff --git a/libc/thread/nsync_once.h b/libc/thread/nsync_once.h deleted file mode 100644 index 5143a82e8..000000000 --- a/libc/thread/nsync_once.h +++ /dev/null @@ -1,51 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_PUBLIC_NSYNC_ONCE_H_ -#define NSYNC_PUBLIC_NSYNC_ONCE_H_ - -#include "libc/fmt/conv.h" -#include "libc/inttypes.h" -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/nsync_atomic.h" - -NSYNC_CPP_START_ - -/* An nsync_once allows a function to be called exactly once, when first referenced. */ -typedef nsync_atomic_uint32_ nsync_once; - -/* An initializer for nsync_once; it is guaranteed to be all zeroes. */ -#define NSYNC_ONCE_INIT NSYNC_ATOMIC_UINT32_INIT_ - -/* The first time nsync_run_once() or nsync_run_once_arg() is applied to *once, - the supplied function is run (with argument, in the case of nsync_run_once_arg()). - Other callers will wait until the run of the function is complete, and then - return without running the function again. */ -void nsync_run_once (nsync_once *once, void (*f) (void)); -void nsync_run_once_arg (nsync_once *once, void (*farg) (void *arg), void *arg); - -/* Same as nsync_run_once()/nsync_run_once_arg() but uses a spinloop. - Can be used on the same nsync_once as nsync_run_once/nsync_run_once_arg(). - - These *_spin variants should be used only in contexts where normal blocking - is disallowed, such as within user-space schedulers, when the runtime is - not fully initialized, etc. They provide no significant performance benefit, - and they should be avoided in normal code. */ -void nsync_run_once_spin (nsync_once *once, void (*f) (void)); -void nsync_run_once_arg_spin (nsync_once *once, void (*farg) (void *arg), void *arg); - -NSYNC_CPP_END_ - -#endif /*NSYNC_PUBLIC_NSYNC_ONCE_H_*/ diff --git a/libc/thread/nsync_panic.c b/libc/thread/nsync_panic.c deleted file mode 100644 index a0b144ccc..000000000 --- a/libc/thread/nsync_panic.c +++ /dev/null @@ -1,42 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "libc/thread/headers.h" - -NSYNC_CPP_START_ - -/* Write the nul-terminated string s[] to file descriptor fd. */ -static void writestr (int fd, const char *s) { - int len = strlen (s); - int n = 0; - while (len != 0 && n >= 0) { - n = write (fd, s, len); - if (n >= 0) { - len -= n; - s += n; - } else if (n == -1 && errno == EINTR) { - n = 0; - } - } -} - -/* Abort after printing the nul-terminated string s[]. */ -void nsync_panic_ (const char *s) { - writestr (2, "panic: "); - writestr (2, s); - abort (); -} - -NSYNC_CPP_END_ diff --git a/libc/thread/nsync_sem_wait.c b/libc/thread/nsync_sem_wait.c new file mode 100644 index 000000000..ae467a6a1 --- /dev/null +++ b/libc/thread/nsync_sem_wait.c @@ -0,0 +1,85 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2016 Google Inc. │ +│ │ +│ Licensed under the Apache License, Version 2.0 (the "License"); │ +│ you may not use this file except in compliance with the License. │ +│ You may obtain a copy of the License at │ +│ │ +│ http://www.apache.org/licenses/LICENSE-2.0 │ +│ │ +│ Unless required by applicable law or agreed to in writing, software │ +│ distributed under the License is distributed on an "AS IS" BASIS, │ +│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ +│ See the License for the specific language governing permissions and │ +│ limitations under the License. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/errno.h" +#include "third_party/nsync/atomic.h" +#include "third_party/nsync/atomic.internal.h" +#include "third_party/nsync/common.internal.h" +#include "third_party/nsync/dll.h" +#include "third_party/nsync/mu_semaphore.h" +#include "third_party/nsync/wait_s.internal.h" + +asm(".ident\t\"\\n\\n\ +*NSYNC (Apache 2.0)\\n\ +Copyright 2016 Google, Inc.\\n\ +https://github.com/google/nsync\""); +// clang-format off + +/* Wait until one of: + w->sem is non-zero----decrement it and return 0. + abs_deadline expires---return ETIMEDOUT. + cancel_note is non-NULL and *cancel_note becomes notified---return ECANCELED. */ +int nsync_sem_wait_with_cancel_ (waiter *w, nsync_time abs_deadline, + nsync_note cancel_note) { + int sem_outcome; + if (cancel_note == NULL) { + sem_outcome = nsync_mu_semaphore_p_with_deadline (&w->sem, abs_deadline); + } else { + nsync_time cancel_time; + cancel_time = nsync_note_notified_deadline_ (cancel_note); + sem_outcome = ECANCELED; + if (nsync_time_cmp (cancel_time, nsync_time_zero) > 0) { + struct nsync_waiter_s nw; + nw.tag = NSYNC_WAITER_TAG; + nw.sem = &w->sem; + nsync_dll_init_ (&nw.q, &nw); + ATM_STORE (&nw.waiting, 1); + nw.flags = 0; + nsync_mu_lock (&cancel_note->note_mu); + cancel_time = NOTIFIED_TIME (cancel_note); + if (nsync_time_cmp (cancel_time, nsync_time_zero) > 0) { + nsync_time local_abs_deadline; + int deadline_is_nearer = 0; + cancel_note->waiters = nsync_dll_make_last_in_list_ ( + cancel_note->waiters, &nw.q); + local_abs_deadline = cancel_time; + if (nsync_time_cmp (abs_deadline, cancel_time) < 0) { + local_abs_deadline = abs_deadline; + deadline_is_nearer = 1; + } + nsync_mu_unlock (&cancel_note->note_mu); + sem_outcome = nsync_mu_semaphore_p_with_deadline (&w->sem, + local_abs_deadline); + if (sem_outcome == ETIMEDOUT && !deadline_is_nearer) { + sem_outcome = ECANCELED; + nsync_note_notify (cancel_note); + } + nsync_mu_lock (&cancel_note->note_mu); + cancel_time = NOTIFIED_TIME (cancel_note); + if (nsync_time_cmp (cancel_time, + nsync_time_zero) > 0) { + cancel_note->waiters = nsync_dll_remove_ ( + cancel_note->waiters, &nw.q); + } + } + nsync_mu_unlock (&cancel_note->note_mu); + } + } + return (sem_outcome); +} + + diff --git a/libc/thread/nsync_sem_wait_no_note.c b/libc/thread/nsync_sem_wait_no_note.c new file mode 100644 index 000000000..c9c3a156b --- /dev/null +++ b/libc/thread/nsync_sem_wait_no_note.c @@ -0,0 +1,33 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2016 Google Inc. │ +│ │ +│ Licensed under the Apache License, Version 2.0 (the "License"); │ +│ you may not use this file except in compliance with the License. │ +│ You may obtain a copy of the License at │ +│ │ +│ http://www.apache.org/licenses/LICENSE-2.0 │ +│ │ +│ Unless required by applicable law or agreed to in writing, software │ +│ distributed under the License is distributed on an "AS IS" BASIS, │ +│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ +│ See the License for the specific language governing permissions and │ +│ limitations under the License. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "third_party/nsync/common.internal.h" +#include "third_party/nsync/mu_semaphore.h" + +asm(".ident\t\"\\n\\n\ +*NSYNC (Apache 2.0)\\n\ +Copyright 2016 Google, Inc.\\n\ +https://github.com/google/nsync\""); +// clang-format off + +/* Wait until one of: + w->sem is non-zero----decrement it and return 0. + abs_deadline expires---return ETIMEDOUT. + Ignores cancel_note. */ +int nsync_sem_wait_with_cancel_ (waiter *w, nsync_time abs_deadline, nsync_note cancel_note) { + return (nsync_mu_semaphore_p_with_deadline (&w->sem, abs_deadline)); +} diff --git a/libc/thread/nsync_semaphore_futex.c b/libc/thread/nsync_semaphore_futex.c deleted file mode 100644 index ac54ae3ce..000000000 --- a/libc/thread/nsync_semaphore_futex.c +++ /dev/null @@ -1,132 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "libc/thread/headers.h" - -NSYNC_CPP_START_ - -static int futex (int *uaddr, int op, int val, const struct timespec *timeout, int *uaddr2, - int val3) { - return (syscall (__NR_futex, uaddr, op, val, timeout, uaddr2, val3)); -} - -/* Check that atomic operations on nsync_atomic_uint32_ can be applied to int. */ -static const int assert_int_size = 1 / - (sizeof (assert_int_size) == sizeof (uint32_t) && - sizeof (nsync_atomic_uint32_) == sizeof (uint32_t)); - -#if defined(FUTEX_PRIVATE_FLAG) -#define FUTEX_PRIVATE_FLAG_ FUTEX_PRIVATE_FLAG -#else -#define FUTEX_PRIVATE_FLAG_ 0 -#endif - -#if defined(FUTEX_WAIT_BITSET) -#define FUTEX_WAIT_ (FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG_ | FUTEX_CLOCK_REALTIME) -#define FUTEX_WAIT_BITS_ FUTEX_BITSET_MATCH_ANY -#else -#define FUTEX_WAIT_ (FUTEX_WAIT | FUTEX_PRIVATE_FLAG_) -#define FUTEX_WAIT_BITS_ 0 -#endif -#define FUTEX_WAKE_ (FUTEX_WAKE | FUTEX_PRIVATE_FLAG_) -#define FUTEX_TIMEOUT_IS_ABSOLUTE (FUTEX_WAIT_BITS_ != 0) - -#define ASSERT(x) do { if (!(x)) { *(volatile int *)0 = 0; } } while (0) - -struct futex { - int i; /* lo half=count; hi half=waiter count */ -}; - -static nsync_semaphore *sem_big_enough_for_futex = (nsync_semaphore *) (uintptr_t)(1 / - (sizeof (struct futex) <= sizeof (*sem_big_enough_for_futex))); - -/* Initialize *s; the initial value is 0. */ -void nsync_mu_semaphore_init (nsync_semaphore *s) { - struct futex *f = (struct futex *) s; - f->i = 0; -} - -/* Wait until the count of *s exceeds 0, and decrement it. */ -void nsync_mu_semaphore_p (nsync_semaphore *s) { - struct futex *f = (struct futex *) s; - int i; - do { - i = ATM_LOAD ((nsync_atomic_uint32_ *) &f->i); - if (i == 0) { - int futex_result = futex (&f->i, FUTEX_WAIT_, i, NULL, - NULL, FUTEX_WAIT_BITS_); - ASSERT (futex_result == 0 || errno == EINTR || - errno == EWOULDBLOCK); - } - } while (i == 0 || !ATM_CAS_ACQ ((nsync_atomic_uint32_ *) &f->i, i, i-1)); -} - -/* Wait until one of: - the count of *s is non-zero, in which case decrement *s and return 0; - or abs_deadline expires, in which case return ETIMEDOUT. */ -int nsync_mu_semaphore_p_with_deadline (nsync_semaphore *s, nsync_time abs_deadline) { - struct futex *f = (struct futex *)s; - int i; - int result = 0; - do { - i = ATM_LOAD ((nsync_atomic_uint32_ *) &f->i); - if (i == 0) { - int futex_result; - struct timespec ts_buf; - const struct timespec *ts = NULL; - if (nsync_time_cmp (abs_deadline, nsync_time_no_deadline) != 0) { - memset (&ts_buf, 0, sizeof (ts_buf)); - if (FUTEX_TIMEOUT_IS_ABSOLUTE) { - ts_buf.tv_sec = NSYNC_TIME_SEC (abs_deadline); - ts_buf.tv_nsec = NSYNC_TIME_NSEC (abs_deadline); - } else { - nsync_time now; - now = nsync_time_now (); - if (nsync_time_cmp (now, abs_deadline) > 0) { - ts_buf.tv_sec = 0; - ts_buf.tv_nsec = 0; - } else { - nsync_time rel_deadline; - rel_deadline = nsync_time_sub (abs_deadline, now); - ts_buf.tv_sec = NSYNC_TIME_SEC (rel_deadline); - ts_buf.tv_nsec = NSYNC_TIME_NSEC (rel_deadline); - } - } - ts = &ts_buf; - } - futex_result = futex (&f->i, FUTEX_WAIT_, i, ts, NULL, FUTEX_WAIT_BITS_); - ASSERT (futex_result == 0 || errno == EINTR || errno == EWOULDBLOCK || - errno == ETIMEDOUT); - /* Some systems don't wait as long as they are told. */ - if (futex_result == -1 && errno == ETIMEDOUT && - nsync_time_cmp (abs_deadline, nsync_time_now ()) <= 0) { - result = ETIMEDOUT; - } - } - } while (result == 0 && (i == 0 || !ATM_CAS_ACQ ((nsync_atomic_uint32_ *) &f->i, i, i - 1))); - return (result); -} - -/* Ensure that the count of *s is at least 1. */ -void nsync_mu_semaphore_v (nsync_semaphore *s) { - struct futex *f = (struct futex *) s; - uint32_t old_value; - do { - old_value = ATM_LOAD ((nsync_atomic_uint32_ *) &f->i); - } while (!ATM_CAS_REL ((nsync_atomic_uint32_ *) &f->i, old_value, old_value+1)); - ASSERT (futex (&f->i, FUTEX_WAKE_, 1, NULL, NULL, 0) >= 0); -} - -NSYNC_CPP_END_ diff --git a/libc/thread/nsync_time.h b/libc/thread/nsync_time.h deleted file mode 100644 index 37cddf526..000000000 --- a/libc/thread/nsync_time.h +++ /dev/null @@ -1,62 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_PUBLIC_NSYNC_TIME_H_ -#define NSYNC_PUBLIC_NSYNC_TIME_H_ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/nsync_time_internal.h" - -/* The type nsync_time represents the interval elapsed between two moments in - time. Often the first such moment is an address-space-wide epoch, such as - the Unix epoch, but clients should not rely on the epoch in one address - space being the same as that in another. Intervals relative to the epoch - are known as absolute times. - - The internals of nsync_time should be treated as opaque by clients. - See nsync_time_internal.h. */ - -NSYNC_CPP_START_ - -extern const nsync_time nsync_time_no_deadline; /* A deadline infinitely far in the future. */ -extern const nsync_time nsync_time_zero; /* The zero delay, or an expired deadline. */ - -nsync_time nsync_time_now (void); /* Return the current time since the epoch. */ - -/* Sleep for the specified delay. Returns the unslept time - which may be non-zero if the call was interrupted. */ -nsync_time nsync_time_sleep (nsync_time delay); - -/* Return a+b */ -nsync_time nsync_time_add (nsync_time a, nsync_time b); - -/* Return a-b */ -nsync_time nsync_time_sub (nsync_time a, nsync_time b); - -/* Return +ve, 0, or -ve according to whether a>b, a==b, or a= 8? \ - (t) / (1000 * 1000 * 1000): \ - ((t) / 1000)) -#define NSYNC_TIME_NSEC(t) (sizeof (nsync_time) >= 8? \ - (t) % (1000 * 1000 * 1000): \ - (((t) % 1000) * 1000 * 1000)) -#define NSYNC_TIME_MAX_ MAX_INT_TYPE (nsync_time) -NSYNC_CPP_END_ - -#elif defined(NSYNC_USE_FLOATING_TIME) -#include "libc/math.h" -#include "libc/calls/struct/timespec.h" -#include "libc/calls/struct/timeval.h" -#include "libc/calls/weirdtypes.h" -#include "libc/sysv/consts/clock.h" -#include "libc/sysv/consts/sched.h" -#include "libc/time/struct/tm.h" -#include "libc/time/time.h" -NSYNC_CPP_START_ -typedef NSYNC_USE_FLOATING_TIME nsync_time; -#define NSYNC_TIME_SEC(t) (trunc ((t) / (nsync_time) (1000 * 1000 * 1000))) -#define NSYNC_TIME_NSEC(t) ((t) - ((1000 * 1000 * 1000) * NSYNC_TIME_SEC (t))) -#define NSYNC_TIME_MAX_ DBL_MAX -NSYNC_CPP_END_ - -#elif NSYNC_USE_DEBUG_TIME -/* Check that the library can be built with a different time struct. */ -#include "libc/calls/struct/timespec.h" -#include "libc/calls/struct/timeval.h" -#include "libc/calls/weirdtypes.h" -#include "libc/sysv/consts/clock.h" -#include "libc/sysv/consts/sched.h" -#include "libc/time/struct/tm.h" -#include "libc/time/time.h" -NSYNC_CPP_START_ -typedef struct { - time_t seconds; - unsigned nanoseconds; -} nsync_time; -#define NSYNC_TIME_SEC(t) ((t).seconds) -#define NSYNC_TIME_NSEC(t) ((t).nanoseconds) -NSYNC_CPP_END_ - -#elif defined(__cplusplus) && \ - (NSYNC_USE_CPP11_TIMEPOINT || (__cplusplus >= 201103L) || (_MSC_VER >= 1700)) -/* The inline functions below provide function overloads that accept the most - likely C++11 time type(s). - - C++11 time types have many variations and subtleties: - - There are multiple clocks with potentially differing epochs; these clocks - are not necessarily phase-locked to the same rate, making conversion and - comparison between clocks tricky. - - Relative and absolute times are distinguished in the type system. - - Either integral or floating point counters may be used to represent time - intervals, and code valid with one may not be valid with the other - (see std::chrono::treat_as_floating_point). - - A counter increment of one can represent any rational number of seconds - (for whatever "seconds" means for this clock). - - Conversions between duration types may round or truncate at the - implementation's discretion. - - As mentioned above, common implementations of the default monotonic clock - ("steady_clock") illegally allow a thread to observe time going backwards, - especially in the face of scheduling on a different CPU, making its use - misleading, at best. - I've chosen to handle this complexity by doing a conversion to absolute - timespec at the interface layer, so all the C++ complication is here, rather - than spread throughout the library. */ - -#include "third_party/libcxx/chrono" -#include "libc/calls/struct/timespec.h" -#include "libc/calls/struct/timeval.h" -#include "libc/calls/weirdtypes.h" -#include "libc/sysv/consts/clock.h" -#include "libc/sysv/consts/sched.h" -#include "libc/time/struct/tm.h" -#include "libc/time/time.h" -NSYNC_CPP_START_ -typedef struct timespec nsync_time; -#define NSYNC_TIME_SEC(t) ((t).tv_sec) -#define NSYNC_TIME_NSEC(t) ((t).tv_nsec) - -typedef std::chrono::system_clock::time_point nsync_cpp_time_point_; -nsync_time nsync_from_time_point_ (nsync_cpp_time_point_); -nsync_cpp_time_point_ nsync_to_time_point_ (nsync_time); -#define NSYNC_COUNTER_CPP_OVERLOAD_ \ - static inline uint32_t nsync_counter_wait (nsync_counter c, \ - nsync_cpp_time_point_ abs_deadline) { \ - return (nsync_counter_wait (c, nsync_from_time_point_ (abs_deadline))); \ - } -#define NSYNC_CV_CPP_OVERLOAD_ \ - static inline int nsync_cv_wait_with_deadline (nsync_cv *cv, nsync_mu *mu, \ - nsync_cpp_time_point_ abs_deadline, struct nsync_note_s_ *cancel_note) { \ - return (nsync_cv_wait_with_deadline (cv, mu, \ - nsync_from_time_point_ (abs_deadline), \ - cancel_note)); \ - } \ - static inline int nsync_cv_wait_with_deadline_generic (nsync_cv *cv, \ - void *mu, void (*lock) (void *), void (*unlock) (void *), \ - nsync_cpp_time_point_ abs_deadline, struct nsync_note_s_ *cancel_note) { \ - return (nsync_cv_wait_with_deadline_generic (cv, mu, lock, unlock, \ - nsync_from_time_point_ (abs_deadline), \ - cancel_note)); \ - } -#define NSYNC_MU_WAIT_CPP_OVERLOAD_ \ - static inline int nsync_mu_wait_with_deadline (nsync_mu *mu, \ - int (*condition) (const void *condition_arg), const void *condition_arg, \ - int (*condition_arg_eq) (const void *a, const void *b), \ - nsync_cpp_time_point_ abs_deadline, struct nsync_note_s_ *cancel_note) { \ - return (nsync_mu_wait_with_deadline (mu, condition, condition_arg, \ - condition_arg_eq, \ - nsync_from_time_point_ (abs_deadline), \ - cancel_note)); \ - } -#define NSYNC_NOTE_CPP_OVERLOAD_ \ - static inline nsync_note nsync_note_new (nsync_note parent, \ - nsync_cpp_time_point_ abs_deadline) { \ - return (nsync_note_new (parent, nsync_from_time_point_ (abs_deadline))); \ - } \ - static inline int nsync_note_wait (nsync_note n, nsync_cpp_time_point_ abs_deadline) { \ - return (nsync_note_wait (n, nsync_from_time_point_ (abs_deadline))); \ - } \ - static inline nsync_cpp_time_point_ nsync_note_expiry_timepoint (nsync_note n) { \ - return (nsync_to_time_point_ (nsync_note_expiry (n))); \ - } -#define NSYNC_WAITER_CPP_OVERLOAD_ \ - static inline int nsync_wait_n (void *mu, void (*lock) (void *), \ - void (*unlock) (void *), \ - nsync_cpp_time_point_ abs_deadline, \ - int count, struct nsync_waitable_s *waitable[]) { \ - return (nsync_wait_n (mu, lock, unlock, \ - nsync_from_time_point_ (abs_deadline), count, waitable)); \ - } - -NSYNC_CPP_END_ - -#else -/* Default is to use timespec. */ -#include "libc/calls/struct/timespec.h" -#include "libc/calls/struct/timeval.h" -#include "libc/calls/weirdtypes.h" -#include "libc/sysv/consts/clock.h" -#include "libc/sysv/consts/sched.h" -#include "libc/time/struct/tm.h" -#include "libc/time/time.h" -NSYNC_CPP_START_ -typedef struct timespec nsync_time; -#define NSYNC_TIME_SEC(t) ((t).tv_sec) -#define NSYNC_TIME_NSEC(t) ((t).tv_nsec) -NSYNC_CPP_END_ - -#endif - -#if !defined(NSYNC_COUNTER_CPP_OVERLOAD_) -#define NSYNC_COUNTER_CPP_OVERLOAD_ -#define NSYNC_CV_CPP_OVERLOAD_ -#define NSYNC_MU_WAIT_CPP_OVERLOAD_ -#define NSYNC_NOTE_CPP_OVERLOAD_ -#define NSYNC_WAITER_CPP_OVERLOAD_ -#endif - -#endif /*NSYNC_PUBLIC_NSYNC_TIME_INTERNAL_H_*/ diff --git a/libc/thread/nsync_wait.c b/libc/thread/nsync_wait.c new file mode 100644 index 000000000..4e597f72d --- /dev/null +++ b/libc/thread/nsync_wait.c @@ -0,0 +1,108 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2016 Google Inc. │ +│ │ +│ Licensed under the Apache License, Version 2.0 (the "License"); │ +│ you may not use this file except in compliance with the License. │ +│ You may obtain a copy of the License at │ +│ │ +│ http://www.apache.org/licenses/LICENSE-2.0 │ +│ │ +│ Unless required by applicable law or agreed to in writing, software │ +│ distributed under the License is distributed on an "AS IS" BASIS, │ +│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ +│ See the License for the specific language governing permissions and │ +│ limitations under the License. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/mem/mem.h" +#include "third_party/nsync/atomic.h" +#include "third_party/nsync/atomic.internal.h" +#include "third_party/nsync/common.internal.h" +#include "third_party/nsync/dll.h" +#include "third_party/nsync/mu_semaphore.h" +#include "third_party/nsync/wait_s.internal.h" +#include "third_party/nsync/waiter.h" + +asm(".ident\t\"\\n\\n\ +*NSYNC (Apache 2.0)\\n\ +Copyright 2016 Google, Inc.\\n\ +https://github.com/google/nsync\""); +// clang-format off + +int nsync_wait_n (void *mu, void (*lock) (void *), void (*unlock) (void *), + nsync_time abs_deadline, + int count, struct nsync_waitable_s *waitable[]) { + int ready; + IGNORE_RACES_START (); + for (ready = 0; ready != count && + nsync_time_cmp ((*waitable[ready]->funcs->ready_time) ( + waitable[ready]->v, NULL), + nsync_time_zero) > 0; + ready++) { + } + if (ready == count && nsync_time_cmp (abs_deadline, nsync_time_zero) > 0) { + int i; + int unlocked = 0; + int j; + int enqueued = 1; + waiter *w = nsync_waiter_new_ (); + struct nsync_waiter_s nw_set[4]; + struct nsync_waiter_s *nw = nw_set; + if (count > (int) (sizeof (nw_set) / sizeof (nw_set[0]))) { + nw = (struct nsync_waiter_s *) malloc (count * sizeof (nw[0])); + } + for (i = 0; i != count && enqueued; i++) { + nw[i].tag = NSYNC_WAITER_TAG; + nw[i].sem = &w->sem; + nsync_dll_init_ (&nw[i].q, &nw[i]); + ATM_STORE (&nw[i].waiting, 0); + nw[i].flags = 0; + enqueued = (*waitable[i]->funcs->enqueue) (waitable[i]->v, &nw[i]); + } + + if (i == count) { + nsync_time min_ntime; + if (mu != NULL) { + (*unlock) (mu); + unlocked = 1; + } + do { + min_ntime = abs_deadline; + for (j = 0; j != count; j++) { + nsync_time ntime; + ntime = (*waitable[j]->funcs->ready_time) ( + waitable[j]->v, &nw[j]); + if (nsync_time_cmp (ntime, min_ntime) < 0) { + min_ntime = ntime; + } + } + } while (nsync_time_cmp (min_ntime, nsync_time_zero) > 0 && + nsync_mu_semaphore_p_with_deadline (&w->sem, + min_ntime) == 0); + } + + /* An attempt was made above to enqueue waitable[0..i-1]. + Dequeue any that are still enqueued, and remember the index + of the first ready (i.e., not still enqueued) object, if any. */ + for (j = 0; j != i; j++) { + int was_still_enqueued = + (*waitable[j]->funcs->dequeue) (waitable[j]->v, &nw[j]); + if (!was_still_enqueued && ready == count) { + ready = j; + } + } + + if (nw != nw_set) { + free (nw); + } + nsync_waiter_free_ (w); + if (unlocked) { + (*lock) (mu); + } + } + IGNORE_RACES_END (); + return (ready); +} + + diff --git a/libc/thread/nsync_waiter.h b/libc/thread/nsync_waiter.h deleted file mode 100644 index d028a1f30..000000000 --- a/libc/thread/nsync_waiter.h +++ /dev/null @@ -1,153 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_PUBLIC_NSYNC_WAITER_H_ -#define NSYNC_PUBLIC_NSYNC_WAITER_H_ - -/* nsync_wait_n() allows the client to wait on multiple objects (condition - variables, nsync_notes, nsync_counters, etc.) until at least one of them - becomes ready, or a deadline expires. - - It can be thought of as rather like Unix's select() or poll(), - except the the objects being waited for are synchronization - data structures, rather than file descriptors. - - The client can construct new objects that can be waited for by implementing - three routines. - - Examples: - - To wait on two nsync_notes n0, n1, and a nsync_counter c0, - with a deadline of abs_deadline: - - // Form an array of struct nsync_waitable_s, identifying the - // objects and the corresponding descriptors. (static initialization - // syntax is used for brevity) - static struct nsync_waitable_s w[] = { - { &n0, &nsync_note_waitable_funcs }, - { &n1, &nsync_note_waitable_funcs }, - { &c0, &nsync_counter_waitable_funcs } - }; - static struct nsync_waitable_s *pw[] = { &w[0], &w[1], &w[2] }; - int n = sizeof (w) / sizeof (w[0]); - - // Wait. The mu, lock, and unlock arguments are NULL because - // no condition variables are invovled. - int i = nsync_wait_n (NULL, NULL, NULL, abs_deadline, n, pw); - if (i == n) { - // timeout - } else { - // w[i].v became ready. - } - - To wait on multiple condition variables, the mu/lock/unlock parameters are - used. Imagine cv0 and cv1 are signalled when predicates pred0() (under - lock mu0) and pred1() (under lock mu1) become true respectively. Assume - that mu0 is acquired before mu1. - static void lock2 (void *v) { // lock two mutexes in order - nsync_mu **mu = (nsync_mu **) v; - nsync_mu_lock (mu[0]); - nsync_mu_lock (mu[1]); - } - static void unlock2 (void *v) { // unlock two mutexes. - nsync_mu **mu = (nsync_mu **) v; - nsync_mu_unlock (mu[1]); - nsync_mu_unlock (mu[0]); - } - - // Describe the condition variables and the locks. - static struct nsync_waitable_s w[] = { - { &cv0, &nsync_cv_waitable_funcs }, - { &cv1, &nsync_cv_waitable_funcs } - }; - static struct nsync_waitable_s *pw[] = { &w[0], &w[1] }; - nsync_mu *lock_list[] = { &mu0, &mu1 }; - int n = sizeof (w) / sizeof (w[0]); - - lock2 (list_list); - while (!pred0 () && !pred1 ()) { - // Wait for one of the condition variables to be signalled, - // with no timeout. - nsync_wait_n (lock_list, &lock2, &unlock2, - nsync_time_no_deadline, n, pw); - } - if (pred0 ()) { ... } - if (pred1 ()) { ... } - unlock2 (list_list); - - */ - -#include "libc/thread/nsync_time.h" -#include "libc/thread/nsync_cpp.h" - -NSYNC_CPP_START_ - -struct nsync_waitable_funcs_s; /* forward declaration of struct that contains - type dependent wait operations */ - -/* Clients wait on objects by forming an array of struct nsync_waitable_s. - Each each element points to one object and its type-dependent functions. */ -struct nsync_waitable_s { - void *v; /* pointer to object */ - /* pointer to type-dependent functions. Use - &nsync_note_waitable_funcs for an nsync_note, - &nsync_counternote_waitable_funcs for an nsync_counter, - &nsync_cv_waitable_funcs for an nsync_cv. */ - const struct nsync_waitable_funcs_s *funcs; -}; - -/* Wait until at least one of *waitable[0,..,count-1] is has been notified, or - abs_deadline is reached. Return the index of the notified element of - waitable[], or count if no such element exists. - If mu!=NULL, (*unlock)(mu) is called after the thread is queued on the - various waiters, and (*lock)(mu) is called before return; mu/lock/unlock are - used to acquire and release the relevant locks whan waiting on condition - variables. */ -int nsync_wait_n (void *mu, void (*lock) (void *), void (*unlock) (void *), - nsync_time abs_deadline, int count, - struct nsync_waitable_s *waitable[]); - -/* --------------------------------------------------- */ - -/* A "struct nsync_waitable_s" implementation must implement these functions. - Clients should ignore the internals. */ -struct nsync_waiter_s; -struct nsync_waitable_funcs_s { - /* Return the time when *v will be ready (max time if - unknown), or 0 if it is already ready. The parameter nw may be - passed as NULL, in which case the result should indicate whether the - thread would block if it were to wait on *v. - All calls with the same *v must report the same result until the - object becomes ready, from which point calls must report 0. */ - nsync_time (*ready_time) (void *v, struct nsync_waiter_s *nw); - - /* If *v is ready, return zero; otherwise enqueue *nw on *v and return - non-zero. */ - int (*enqueue) (void *v, struct nsync_waiter_s *nw); - - /* If nw has been previously dequeued, return zero; otherwise dequeue - *nw from *v and return non-zero. */ - int (*dequeue) (void *v, struct nsync_waiter_s *nw); -}; - -/* The "struct nsync_waitable_s" for nsync_note, nsync_counter, and nsync_cv. */ -extern const struct nsync_waitable_funcs_s nsync_note_waitable_funcs; -extern const struct nsync_waitable_funcs_s nsync_counter_waitable_funcs; -extern const struct nsync_waitable_funcs_s nsync_cv_waitable_funcs; - -NSYNC_WAITER_CPP_OVERLOAD_ -NSYNC_CPP_END_ - -#endif /*NSYNC_PUBLIC_NSYNC_WAITER_H_*/ diff --git a/libc/thread/per_thread_waiter.c b/libc/thread/per_thread_waiter.c deleted file mode 100644 index 19fd54f8c..000000000 --- a/libc/thread/per_thread_waiter.c +++ /dev/null @@ -1,49 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "libc/thread/headers.h" - -NSYNC_CPP_START_ - -static pthread_key_t waiter_key; -static nsync_atomic_uint32_ pt_once; - -static void do_once (nsync_atomic_uint32_ *ponce, void (*dest) (void *)) { - uint32_t o = ATM_LOAD_ACQ (ponce); - if (o != 2) { - while (o == 0 && !ATM_CAS_ACQ (ponce, 0, 1)) { - o = ATM_LOAD (ponce); - } - if (o == 0) { - pthread_key_create (&waiter_key, dest); - ATM_STORE_REL (ponce, 2); - } - while (ATM_LOAD_ACQ (ponce) != 2) { - sched_yield (); - } - } -} - -void *nsync_per_thread_waiter_ (void (*dest) (void *)) { - do_once (&pt_once, dest); - return (pthread_getspecific (waiter_key)); -} - -void nsync_set_per_thread_waiter_ (void *v, void (*dest) (void *)) { - do_once (&pt_once, dest); - pthread_setspecific (waiter_key, v); -} - -NSYNC_CPP_END_ diff --git a/libc/thread/platform.h b/libc/thread/platform.h deleted file mode 100644 index 449b55eaf..000000000 --- a/libc/thread/platform.h +++ /dev/null @@ -1,72 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_PLATFORM_LINUX_PLATFORM_H_ -#define NSYNC_PLATFORM_LINUX_PLATFORM_H_ - -#if !defined(_GNU_SOURCE) -#define _GNU_SOURCE /* for futexes */ -#endif - -#include "libc/mem/alg.h" -#include "libc/str/str.h" -#include "libc/calls/calls.h" -#include "libc/calls/weirdtypes.h" -#include "libc/runtime/pathconf.h" -#include "libc/runtime/sysconf.h" -#include "libc/sysv/consts/fileno.h" -#include "libc/sysv/consts/o.h" -#include "libc/sysv/consts/ok.h" -#include "third_party/getopt/getopt.h" -#include "libc/errno.h" -#include "libc/mem/alg.h" -#include "libc/fmt/conv.h" -#include "libc/mem/mem.h" -#include "libc/stdio/rand.h" -#include "libc/runtime/runtime.h" -#include "libc/stdio/temp.h" -#include "libc/sysv/consts/exit.h" -#include "third_party/gdtoa/gdtoa.h" - -#include "libc/calls/struct/timespec.h" -#include "libc/calls/struct/timeval.h" -#include "libc/calls/weirdtypes.h" -#include "libc/sysv/consts/clock.h" -#include "libc/sysv/consts/sched.h" -#include "libc/time/struct/tm.h" -#include "libc/time/time.h" -#include "libc/fmt/conv.h" -#include "libc/inttypes.h" -#include "libc/thread/thread.h" -#include "libc/limits.h" -#include "libc/sysv/consts/_posix.h" -#include "libc/sysv/consts/iov.h" -#include "libc/sysv/consts/limits.h" -#include "libc/sysv/consts/xopen.h" -#include "libc/sysv/consts/futex.h" -#include "libc/sysv/consts/nr.h" -#include "libc/calls/calls.h" -#include "libc/thread/thread.h" -#include "libc/thread/thread2.h" -#include "libc/calls/semaphore.internal.h" - -#include "libc/calls/calls.h" -#include "libc/fmt/fmt.h" -#include "libc/stdio/lock.internal.h" -#include "libc/stdio/stdio.h" -#include "libc/stdio/temp.h" - - -#endif /*NSYNC_PLATFORM_LINUX_PLATFORM_H_*/ diff --git a/libc/thread/posixthread.internal.h b/libc/thread/posixthread.internal.h index 911ccfac5..bfbf41e9a 100644 --- a/libc/thread/posixthread.internal.h +++ b/libc/thread/posixthread.internal.h @@ -62,11 +62,13 @@ struct PosixThread { int *ctid; char *tls; char *tib; + char *altstack; _Atomic(enum PosixThreadStatus) status; jmp_buf exiter; pthread_attr_t attr; }; +hidden extern pthread_spinlock_t _pthread_keys_lock; hidden extern uint64_t _pthread_key_usage[(PTHREAD_KEYS_MAX + 63) / 64]; hidden extern pthread_key_dtor _pthread_key_dtor[PTHREAD_KEYS_MAX]; hidden extern _Thread_local void *_pthread_keys[PTHREAD_KEYS_MAX]; diff --git a/libc/thread/pthread_barrier_destroy.c b/libc/thread/pthread_barrier_destroy.c index bad0f54d2..c249bbe60 100644 --- a/libc/thread/pthread_barrier_destroy.c +++ b/libc/thread/pthread_barrier_destroy.c @@ -16,10 +16,9 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" -#include "libc/errno.h" -#include "libc/thread/thread.h" #include "libc/str/str.h" +#include "libc/thread/thread.h" +#include "third_party/nsync/counter.h" /** * Destroys barrier. @@ -28,10 +27,9 @@ * @raise EINVAL if threads are still inside the barrier */ int pthread_barrier_destroy(pthread_barrier_t *barrier) { - if (barrier->waits || barrier->popped) { - assert(!"deadlock"); - return EINVAL; + if (barrier->_nsync) { + nsync_counter_free(barrier->_nsync); + barrier->_nsync = 0; } - memset(barrier, -1, sizeof(*barrier)); return 0; } diff --git a/libc/thread/pthread_barrier_init.c b/libc/thread/pthread_barrier_init.c index c5030e60f..b83a35230 100644 --- a/libc/thread/pthread_barrier_init.c +++ b/libc/thread/pthread_barrier_init.c @@ -16,10 +16,9 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" #include "libc/errno.h" -#include "libc/limits.h" #include "libc/thread/thread.h" +#include "third_party/nsync/counter.h" /** * Initializes barrier. @@ -28,15 +27,14 @@ * @param count is how many threads need to call pthread_barrier_wait() * before the barrier is released, which must be greater than zero * @return 0 on success, or error number on failure - * @raise EINVAL if `count` isn't greater than zero or overflows + * @raise EINVAL if `count` isn't greater than zero + * @raise ENOMEM if insufficient memory exists */ int pthread_barrier_init(pthread_barrier_t *barrier, const pthread_barrierattr_t *attr, unsigned count) { - if (count && count < INT_MAX / 2) { - *barrier = (pthread_barrier_t){attr ? *attr : 0, count}; - return 0; - } else { - assert(!"bad count"); - return EINVAL; - } + nsync_counter c; + if (!count) return EINVAL; + if (!(c = nsync_counter_new(count))) return ENOMEM; + *barrier = (pthread_barrier_t){._nsync = c}; + return 0; } diff --git a/libc/thread/pthread_barrier_wait.c b/libc/thread/pthread_barrier_wait.c index 328c20a98..cfd3a891f 100644 --- a/libc/thread/pthread_barrier_wait.c +++ b/libc/thread/pthread_barrier_wait.c @@ -17,9 +17,12 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/intrin/atomic.h" -#include "libc/intrin/futex.internal.h" -#include "libc/thread/thread.h" +#include "libc/intrin/kprintf.h" #include "libc/limits.h" +#include "libc/thread/thread.h" +#include "third_party/nsync/counter.h" +#include "third_party/nsync/futex.internal.h" +#include "third_party/nsync/time.h" /** * Waits for all threads to arrive at barrier. @@ -33,22 +36,10 @@ * thread which was the last arrival, or an errno on error */ int pthread_barrier_wait(pthread_barrier_t *barrier) { - if (atomic_fetch_add(&barrier->waits, 1) + 1 == barrier->count) { - if (atomic_fetch_add(&barrier->waits, 1) + 1 < barrier->count * 2) { - atomic_store_explicit(&barrier->popped, 1, memory_order_relaxed); - do { - _futex_wake(&barrier->popped, INT_MAX, barrier->pshared); - } while (atomic_load_explicit(&barrier->waits, memory_order_relaxed) < - barrier->count * 2); - atomic_store_explicit(&barrier->popped, 0, memory_order_relaxed); - } - atomic_store_explicit(&barrier->waits, 0, memory_order_relaxed); + if (nsync_counter_add(barrier->_nsync, -1)) { + nsync_counter_wait(barrier->_nsync, nsync_time_no_deadline); + return 0; + } else { return PTHREAD_BARRIER_SERIAL_THREAD; } - do { - _futex_wait(&barrier->popped, 0, barrier->pshared, 0); - } while (atomic_load_explicit(&barrier->waits, memory_order_relaxed) < - barrier->count); - atomic_fetch_add(&barrier->waits, 1); - return 0; } diff --git a/libc/thread/pthread_barrierattr_getpshared.c b/libc/thread/pthread_barrierattr_getpshared.c index aeae63780..bd2c547b0 100644 --- a/libc/thread/pthread_barrierattr_getpshared.c +++ b/libc/thread/pthread_barrierattr_getpshared.c @@ -22,8 +22,8 @@ * Gets barrier process sharing. * * @param pshared is set to one of the following - * - `PTHREAD_PROCESS_SHARED` - * - `PTHREAD_PROCESS_PRIVATE` + * - `PTHREAD_PROCESS_PRIVATE` (default) + * - `PTHREAD_PROCESS_SHARED` (unsupported) * @return 0 on success, or error on failure */ int pthread_barrierattr_getpshared(const pthread_barrierattr_t *attr, diff --git a/libc/thread/pthread_barrierattr_setpshared.c b/libc/thread/pthread_barrierattr_setpshared.c index 64979b4d9..4ae2adbb6 100644 --- a/libc/thread/pthread_barrierattr_setpshared.c +++ b/libc/thread/pthread_barrierattr_setpshared.c @@ -23,14 +23,13 @@ * Sets barrier process sharing. * * @param pshared can be one of - * - `PTHREAD_PROCESS_SHARED` - * - `PTHREAD_PROCESS_PRIVATE` + * - `PTHREAD_PROCESS_PRIVATE` (default) + * - `PTHREAD_PROCESS_SHARED` (unsupported) * @return 0 on success, or error on failure * @raises EINVAL if `pshared` is invalid */ int pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared) { switch (pshared) { - case PTHREAD_PROCESS_SHARED: case PTHREAD_PROCESS_PRIVATE: *attr = pshared; return 0; diff --git a/libc/thread/pthread_cond_broadcast.c b/libc/thread/pthread_cond_broadcast.c index 6b9779c55..8518a4b45 100644 --- a/libc/thread/pthread_cond_broadcast.c +++ b/libc/thread/pthread_cond_broadcast.c @@ -16,72 +16,17 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/dce.h" -#include "libc/intrin/atomic.h" -#include "libc/intrin/futex.internal.h" #include "libc/thread/thread.h" -#include "libc/limits.h" - -static dontinline int pthread_cond_signal_impl(pthread_cond_t *cond, int n) { - if (atomic_load_explicit(&cond->waits, memory_order_relaxed)) { - atomic_fetch_add(&cond->seq, 1); - if (IsLinux() || IsOpenbsd()) { - _futex_wake(&cond->seq, n, cond->pshared); - } - } - return 0; -} - -/** - * Wakes at least one thread waiting on condition, e.g. - * - * pthread_cond_t cond = PTHREAD_COND_INITIALIZER; - * pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; - * - * // thread pool waiters - * pthread_mutex_lock(&lock); - * pthread_cond_wait(&cond, &lock); - * pthread_mutex_unlock(&lock); - * - * // waker upper - * pthread_mutex_lock(&lock); - * pthread_cond_signal(&cond); - * pthread_mutex_unlock(&lock); - * - * This function has no effect if there aren't any threads currently - * waiting on the condition. - * - * @return 0 on success, or errno on error - * @see pthread_cond_broadcast - * @see pthread_cond_wait - */ -int pthread_cond_signal(pthread_cond_t *cond) { - return pthread_cond_signal_impl(cond, 1); -} +#include "third_party/nsync/cv.h" /** * Wakes all threads waiting on condition, e.g. * - * pthread_mutex_t lock; - * pthread_mutexattr_t mattr; - * pthread_mutexattr_init(&mattr); - * pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_ERRORCHECK); - * pthread_mutex_init(&lock, &mattr); - * - * pthread_cond_t cond; - * pthread_condattr_t cattr; - * pthread_condattr_init(&cattr); - * pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED); - * pthread_cond_init(&cond, &cattr); - * - * // waiting threads - * CHECK_EQ(0, pthread_mutex_lock(&lock)); - * CHECK_EQ(0, pthread_cond_wait(&cond, &lock)); - * pthread_mutex_unlock(&lock); - * - * // notifying thread - * CHECK_EQ(0, pthread_mutex_lock(&lock)); - * pthread_cond_broadcast(&cond); + * pthread_cond_t cond = PTHREAD_COND_INITIALIZER; + * pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; + * // ... + * pthread_mutex_lock(&lock); + * pthread_cond_broadcast(&cond, &lock); * pthread_mutex_unlock(&lock); * * This function has no effect if there aren't any threads currently @@ -92,5 +37,6 @@ int pthread_cond_signal(pthread_cond_t *cond) { * @see pthread_cond_wait */ int pthread_cond_broadcast(pthread_cond_t *cond) { - return pthread_cond_signal_impl(cond, INT_MAX); + nsync_cv_broadcast((nsync_cv *)cond); + return 0; } diff --git a/libc/thread/pthread_cond_destroy.c b/libc/thread/pthread_cond_destroy.c index 8dfe435bb..579b2399a 100644 --- a/libc/thread/pthread_cond_destroy.c +++ b/libc/thread/pthread_cond_destroy.c @@ -16,10 +16,8 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" -#include "libc/errno.h" -#include "libc/thread/thread.h" #include "libc/str/str.h" +#include "libc/thread/thread.h" /** * Destroys condition. @@ -28,10 +26,6 @@ * @raise EINVAL if threads are still waiting on condition */ int pthread_cond_destroy(pthread_cond_t *cond) { - if (cond->waits) { - assert(!"deadlock"); - return EINVAL; - } memset(cond, -1, sizeof(*cond)); return 0; } diff --git a/libc/thread/pthread_cond_init.c b/libc/thread/pthread_cond_init.c index 0bbf67cf7..aa5e8e8b7 100644 --- a/libc/thread/pthread_cond_init.c +++ b/libc/thread/pthread_cond_init.c @@ -25,6 +25,6 @@ * @return 0 on success, or error number on failure */ int pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr) { - *cond = (pthread_cond_t){attr ? *attr : 0}; + *cond = (pthread_cond_t){0}; return 0; } diff --git a/libc/intrin/futex_wake.c b/libc/thread/pthread_cond_signal.c similarity index 69% rename from libc/intrin/futex_wake.c rename to libc/thread/pthread_cond_signal.c index 46e3cdd08..11dfd11c4 100644 --- a/libc/intrin/futex_wake.c +++ b/libc/thread/pthread_cond_signal.c @@ -16,31 +16,27 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/strace.internal.h" -#include "libc/dce.h" -#include "libc/errno.h" -#include "libc/intrin/describeflags.internal.h" -#include "libc/intrin/futex.internal.h" #include "libc/thread/thread.h" -#include "libc/sysv/consts/futex.h" +#include "third_party/nsync/cv.h" -int _futex(void *, int, int) hidden; - -int _futex_wake(void *addr, int count, char pshared) { - int op, ax, pf; - if (IsLinux() || IsOpenbsd()) { - pf = pshared == PTHREAD_PROCESS_PRIVATE ? FUTEX_PRIVATE_FLAG : 0; - op = FUTEX_WAKE | pf; - ax = _futex(addr, op, count); - if (SupportsLinux() && pf && ax == -ENOSYS) { - // RHEL5 doesn't support FUTEX_PRIVATE_FLAG - op = FUTEX_WAKE; - ax = _futex(addr, op, count); - } - STRACE("futex(%t, %s, %d) → %s", addr, DescribeFutexOp(op), count, - DescribeFutexResult(ax)); - return ax; - } else { - return 0; - } +/** + * Wakes at least one thread waiting on condition, e.g. + * + * pthread_cond_t cond = PTHREAD_COND_INITIALIZER; + * pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; + * // ... + * pthread_mutex_lock(&lock); + * pthread_cond_signal(&cond, &lock); + * pthread_mutex_unlock(&lock); + * + * This function has no effect if there aren't any threads currently + * waiting on the condition. + * + * @return 0 on success, or errno on error + * @see pthread_cond_broadcast + * @see pthread_cond_wait + */ +int pthread_cond_signal(pthread_cond_t *cond) { + nsync_cv_signal((nsync_cv *)cond); + return 0; } diff --git a/libc/thread/pthread_cond_timedwait.c b/libc/thread/pthread_cond_timedwait.c index 812ebe8d6..31ecf88ba 100644 --- a/libc/thread/pthread_cond_timedwait.c +++ b/libc/thread/pthread_cond_timedwait.c @@ -16,14 +16,12 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" -#include "libc/calls/calls.h" -#include "libc/calls/struct/timespec.h" #include "libc/errno.h" -#include "libc/intrin/atomic.h" -#include "libc/intrin/futex.internal.h" #include "libc/thread/thread.h" #include "libc/thread/thread2.h" +#include "third_party/nsync/common.internal.h" +#include "third_party/nsync/cv.h" +#include "third_party/nsync/time.h" /** * Waits for condition with optional time limit, e.g. @@ -48,44 +46,13 @@ */ int pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, const struct timespec *abstime) { - int c, rc, err, seq; - struct timespec now, rel, *tsp; - if (abstime && !(0 <= abstime->tv_nsec && abstime->tv_nsec < 1000000000)) { - assert(!"bad abstime"); return EINVAL; } - - if (mutex->type == PTHREAD_MUTEX_ERRORCHECK) { - c = atomic_load_explicit(&mutex->lock, memory_order_relaxed); - if ((c & 0x000fffff) != gettid()) { - assert(!"permlock"); - return EPERM; - } + if (mutex->_type != PTHREAD_MUTEX_NORMAL) { + nsync_panic_("pthread cond needs normal mutex\n"); } - - atomic_fetch_add(&cond->waits, 1); - if (pthread_mutex_unlock(mutex)) notpossible; - - rc = 0; - seq = atomic_load_explicit(&cond->seq, memory_order_relaxed); - do { - if (!abstime) { - tsp = 0; - } else { - now = _timespec_mono(); - if (_timespec_gte(now, *abstime)) { - rc = ETIMEDOUT; - break; - } - rel = _timespec_sub(*abstime, now); - tsp = &rel; - } - _futex_wait(&cond->seq, seq, cond->pshared, tsp); - } while (seq == atomic_load_explicit(&cond->seq, memory_order_relaxed)); - - if (pthread_mutex_lock(mutex)) notpossible; - atomic_fetch_sub(&cond->waits, 1); - - return rc; + return nsync_cv_wait_with_deadline( + (nsync_cv *)cond, (nsync_mu *)mutex, + abstime ? *abstime : nsync_time_no_deadline, 0); } diff --git a/libc/thread/pthread_cond_wait.c b/libc/thread/pthread_cond_wait.c index 9955868bd..30920d280 100644 --- a/libc/thread/pthread_cond_wait.c +++ b/libc/thread/pthread_cond_wait.c @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/thread/thread.h" #include "libc/thread/thread2.h" /** @@ -23,17 +24,11 @@ * * pthread_cond_t cond = PTHREAD_COND_INITIALIZER; * pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; - * - * // waiting threads + * // ... * pthread_mutex_lock(&lock); * pthread_cond_wait(&cond, &lock); * pthread_mutex_unlock(&lock); * - * // notifying thread - * pthread_mutex_lock(&lock); - * pthread_cond_broadcast(&cond); - * pthread_mutex_unlock(&lock); - * * @param mutex needs to be held by thread when calling this function * @return 0 on success, or errno on error * @raise EPERM if `mutex` is `PTHREAD_MUTEX_ERRORCHECK` and the lock diff --git a/libc/thread/pthread_condattr_getpshared.c b/libc/thread/pthread_condattr_getpshared.c index 77903194f..dab698903 100644 --- a/libc/thread/pthread_condattr_getpshared.c +++ b/libc/thread/pthread_condattr_getpshared.c @@ -22,8 +22,8 @@ * Gets condition process sharing. * * @param pshared is set to one of the following + * - `PTHREAD_PROCESS_PRIVATE` (default) * - `PTHREAD_PROCESS_SHARED` - * - `PTHREAD_PROCESS_PRIVATE` * @return 0 on success, or error on failure */ int pthread_condattr_getpshared(const pthread_condattr_t *attr, int *pshared) { diff --git a/libc/thread/pthread_condattr_setpshared.c b/libc/thread/pthread_condattr_setpshared.c index 9ec958f8e..ccd7ae242 100644 --- a/libc/thread/pthread_condattr_setpshared.c +++ b/libc/thread/pthread_condattr_setpshared.c @@ -23,14 +23,13 @@ * Sets condition process sharing. * * @param pshared can be one of - * - `PTHREAD_PROCESS_SHARED` - * - `PTHREAD_PROCESS_PRIVATE` + * - `PTHREAD_PROCESS_PRIVATE` (default) + * - `PTHREAD_PROCESS_SHARED` (unsupported) * @return 0 on success, or error on failure * @raises EINVAL if `pshared` is invalid */ int pthread_condattr_setpshared(pthread_condattr_t *attr, int pshared) { switch (pshared) { - case PTHREAD_PROCESS_SHARED: case PTHREAD_PROCESS_PRIVATE: *attr = pshared; return 0; diff --git a/libc/thread/pthread_create.c b/libc/thread/pthread_create.c index 0a18e9f3a..5ae34c19c 100644 --- a/libc/thread/pthread_create.c +++ b/libc/thread/pthread_create.c @@ -19,14 +19,16 @@ #include "libc/assert.h" #include "libc/calls/calls.h" #include "libc/calls/sched-sysv.internal.h" +#include "libc/calls/struct/sigaltstack.h" #include "libc/calls/syscall-sysv.internal.h" #include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/asan.internal.h" #include "libc/intrin/atomic.h" #include "libc/intrin/bits.h" -#include "libc/intrin/wait0.internal.h" +#include "libc/intrin/kprintf.h" #include "libc/intrin/weaken.h" +#include "libc/log/internal.h" #include "libc/macros.internal.h" #include "libc/mem/mem.h" #include "libc/nexgen32e/gc.internal.h" @@ -35,11 +37,16 @@ #include "libc/sysv/consts/clone.h" #include "libc/sysv/consts/map.h" #include "libc/sysv/consts/prot.h" +#include "libc/sysv/consts/ss.h" #include "libc/sysv/errfuns.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/spawn.h" #include "libc/thread/thread.h" #include "libc/thread/tls.h" +#include "libc/thread/wait0.internal.h" + +STATIC_YOINK("nsync_mu_lock"); +STATIC_YOINK("nsync_mu_unlock"); #define MAP_ANON_OPENBSD 0x1000 #define MAP_STACK_OPENBSD 0x4000 @@ -53,7 +60,12 @@ void _pthread_free(struct PosixThread *pt) { if (pt->ownstack && // pt->attr.stackaddr && // pt->attr.stackaddr != MAP_FAILED) { - munmap(&pt->attr.stackaddr, pt->attr.stacksize); + if (munmap(pt->attr.stackaddr, pt->attr.stacksize)) { + notpossible; + } + } + if (pt->altstack) { + free(pt->altstack); } free(pt); } @@ -61,6 +73,15 @@ void _pthread_free(struct PosixThread *pt) { static int PosixThread(void *arg, int tid) { struct PosixThread *pt = arg; enum PosixThreadStatus status; + struct sigaltstack ss; + if (pt->altstack) { + ss.ss_flags = 0; + ss.ss_size = SIGSTKSZ; + ss.ss_sp = pt->altstack; + if (sigaltstack(&ss, 0)) { + notpossible; + } + } if (pt->attr.inheritsched == PTHREAD_EXPLICIT_SCHED) { _pthread_reschedule(pt); } @@ -254,6 +275,11 @@ int pthread_create(pthread_t *thread, const pthread_attr_t *attr, } } + // setup signal handler stack + if (_wantcrashreports && !IsWindows()) { + pt->altstack = malloc(SIGSTKSZ); + } + // set initial status switch (pt->attr.detachstate) { case PTHREAD_CREATE_JOINABLE: diff --git a/libc/thread/pthread_mutex_destroy.c b/libc/thread/pthread_mutex_destroy.c index 876974323..87f4efa72 100644 --- a/libc/thread/pthread_mutex_destroy.c +++ b/libc/thread/pthread_mutex_destroy.c @@ -16,8 +16,6 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" -#include "libc/errno.h" #include "libc/str/str.h" #include "libc/thread/thread.h" @@ -28,10 +26,6 @@ * @raise EINVAL if mutex is locked in our implementation */ int pthread_mutex_destroy(pthread_mutex_t *mutex) { - if (mutex->lock) { - assert(!"deadlock"); - return EINVAL; - } memset(mutex, -1, sizeof(*mutex)); return 0; } diff --git a/libc/thread/pthread_mutexattr_getpshared.c b/libc/thread/pthread_mutexattr_getpshared.c index 39560cd2e..c521b91ce 100644 --- a/libc/thread/pthread_mutexattr_getpshared.c +++ b/libc/thread/pthread_mutexattr_getpshared.c @@ -22,12 +22,12 @@ * Gets mutex process sharing. * * @param pshared is set to one of the following + * - `PTHREAD_PROCESS_PRIVATE` (default) * - `PTHREAD_PROCESS_SHARED` - * - `PTHREAD_PROCESS_PRIVATE` * @return 0 on success, or error on failure */ int pthread_mutexattr_getpshared(const pthread_mutexattr_t *attr, int *pshared) { - *pshared = attr->pshared; + *pshared = attr->_pshared; return 0; } diff --git a/libc/thread/pthread_mutexattr_gettype.c b/libc/thread/pthread_mutexattr_gettype.c index ecdb6fb11..6138568e9 100644 --- a/libc/thread/pthread_mutexattr_gettype.c +++ b/libc/thread/pthread_mutexattr_gettype.c @@ -29,6 +29,6 @@ * @return 0 on success, or error on failure */ int pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *type) { - *type = attr->type; + *type = attr->_type; return 0; } diff --git a/libc/thread/pthread_mutexattr_setpshared.c b/libc/thread/pthread_mutexattr_setpshared.c index 9c600b77e..c575cbd95 100644 --- a/libc/thread/pthread_mutexattr_setpshared.c +++ b/libc/thread/pthread_mutexattr_setpshared.c @@ -23,8 +23,8 @@ * Sets mutex process sharing. * * @param pshared can be one of + * - `PTHREAD_PROCESS_PRIVATE` (default) * - `PTHREAD_PROCESS_SHARED` - * - `PTHREAD_PROCESS_PRIVATE` * @return 0 on success, or error on failure * @raises EINVAL if `pshared` is invalid */ @@ -32,7 +32,7 @@ int pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared) { switch (pshared) { case PTHREAD_PROCESS_SHARED: case PTHREAD_PROCESS_PRIVATE: - attr->pshared = pshared; + attr->_pshared = pshared; return 0; default: return EINVAL; diff --git a/libc/thread/pthread_mutexattr_settype.c b/libc/thread/pthread_mutexattr_settype.c index 9d70a4d02..766a4b7a4 100644 --- a/libc/thread/pthread_mutexattr_settype.c +++ b/libc/thread/pthread_mutexattr_settype.c @@ -35,7 +35,7 @@ int pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type) { case PTHREAD_MUTEX_NORMAL: case PTHREAD_MUTEX_RECURSIVE: case PTHREAD_MUTEX_ERRORCHECK: - attr->type = type; + attr->_type = type; return 0; default: return EINVAL; diff --git a/libc/thread/pthread_rwlock_destroy.c b/libc/thread/pthread_rwlock_destroy.c index 649b98f30..47bc63c10 100644 --- a/libc/thread/pthread_rwlock_destroy.c +++ b/libc/thread/pthread_rwlock_destroy.c @@ -16,10 +16,8 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" -#include "libc/errno.h" -#include "libc/thread/thread.h" #include "libc/str/str.h" +#include "libc/thread/thread.h" /** * Destroys read-write lock. @@ -28,10 +26,6 @@ * @raise EINVAL if any threads still hold the lock */ int pthread_rwlock_destroy(pthread_rwlock_t *rwlock) { - if (rwlock->lock) { - assert(!"deadlock"); - return EINVAL; - } memset(rwlock, -1, sizeof(*rwlock)); return 0; } diff --git a/libc/thread/pthread_rwlock_init.c b/libc/thread/pthread_rwlock_init.c index ac10d5651..c9790a9b7 100644 --- a/libc/thread/pthread_rwlock_init.c +++ b/libc/thread/pthread_rwlock_init.c @@ -26,6 +26,6 @@ */ int pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr) { - *rwlock = (pthread_rwlock_t){attr ? *attr : 0}; + *rwlock = (pthread_rwlock_t){0}; return 0; } diff --git a/libc/thread/pthread_rwlock_rdlock.c b/libc/thread/pthread_rwlock_rdlock.c index 3bbe79e26..187b60453 100644 --- a/libc/thread/pthread_rwlock_rdlock.c +++ b/libc/thread/pthread_rwlock_rdlock.c @@ -16,24 +16,8 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/intrin/atomic.h" -#include "libc/intrin/futex.internal.h" #include "libc/thread/thread.h" - -static int pthread_rwlock_rdlock_spin(pthread_rwlock_t *rwlock, int expect, - int tries) { - if (tries < 7) { - volatile int i; - for (i = 0; i != 1 << tries; i++) { - } - tries++; - } else { - atomic_fetch_add(&rwlock->waits, 1); - _futex_wait(&rwlock->lock, expect, rwlock->pshared, &(struct timespec){1}); - atomic_fetch_sub(&rwlock->waits, 1); - } - return tries; -} +#include "third_party/nsync/mu.h" /** * Acquires read lock on read-write lock. @@ -41,18 +25,6 @@ static int pthread_rwlock_rdlock_spin(pthread_rwlock_t *rwlock, int expect, * @return 0 on success, or errno on error */ int pthread_rwlock_rdlock(pthread_rwlock_t *rwlock) { - int old, tries; - for (tries = 0;;) { - old = atomic_load_explicit(&rwlock->lock, memory_order_relaxed); - if (old >= 0) { - do { - if (atomic_compare_exchange_weak_explicit(&rwlock->lock, &old, old + 1, - memory_order_acquire, - memory_order_relaxed)) { - return 0; - } - } while (old >= 0); - } - tries = pthread_rwlock_rdlock_spin(rwlock, old, tries); - } + nsync_mu_rlock((nsync_mu *)rwlock); + return 0; } diff --git a/libc/thread/pthread_rwlock_unlock.c b/libc/thread/pthread_rwlock_unlock.c index 849a58fef..617746146 100644 --- a/libc/thread/pthread_rwlock_unlock.c +++ b/libc/thread/pthread_rwlock_unlock.c @@ -16,12 +16,8 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" -#include "libc/dce.h" -#include "libc/errno.h" -#include "libc/intrin/atomic.h" -#include "libc/intrin/futex.internal.h" #include "libc/thread/thread.h" +#include "third_party/nsync/mu.h" /** * Unlocks read-write lock. @@ -30,26 +26,11 @@ * @raise EINVAL if lock is in a bad state */ int pthread_rwlock_unlock(pthread_rwlock_t *rwlock) { - int old, waits; - for (;;) { - old = atomic_load_explicit(&rwlock->lock, memory_order_relaxed); - if (!old || old < -1) { - assert(!"badlock"); - return EINVAL; - } else if (old == -1 || old == 1) { - waits = atomic_load_explicit(&rwlock->waits, memory_order_relaxed); - if (atomic_compare_exchange_weak_explicit(&rwlock->lock, &old, 0, - memory_order_acquire, - memory_order_relaxed)) { - if (waits && (IsLinux() || IsOpenbsd())) { - _futex_wake(&rwlock->lock, 1, rwlock->pshared); - } - return 0; - } - } else if (atomic_compare_exchange_weak_explicit( - &rwlock->lock, &old, old - 1, memory_order_acquire, - memory_order_relaxed)) { - return 0; - } + if (rwlock->_iswrite) { + rwlock->_iswrite = 0; + nsync_mu_unlock((nsync_mu *)rwlock); + } else { + nsync_mu_runlock((nsync_mu *)rwlock); } + return 0; } diff --git a/libc/thread/pthread_rwlock_wrlock.c b/libc/thread/pthread_rwlock_wrlock.c index d66fb25ce..f97c3ae95 100644 --- a/libc/thread/pthread_rwlock_wrlock.c +++ b/libc/thread/pthread_rwlock_wrlock.c @@ -16,24 +16,8 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/intrin/atomic.h" -#include "libc/intrin/futex.internal.h" #include "libc/thread/thread.h" - -static int pthread_rwlock_wrlock_spin(pthread_rwlock_t *rwlock, int expect, - int tries) { - if (tries < 7) { - volatile int i; - for (i = 0; i != 1 << tries; i++) { - } - tries++; - } else { - atomic_fetch_add(&rwlock->waits, 1); - _futex_wait(&rwlock->lock, expect, rwlock->pshared, &(struct timespec){1}); - atomic_fetch_sub(&rwlock->waits, 1); - } - return tries; -} +#include "third_party/nsync/mu.h" /** * Acquires write lock on read-write lock. @@ -41,17 +25,7 @@ static int pthread_rwlock_wrlock_spin(pthread_rwlock_t *rwlock, int expect, * @return 0 on success, or errno on error */ int pthread_rwlock_wrlock(pthread_rwlock_t *rwlock) { - int old, tries; - for (tries = 0;;) { - if (!(old = atomic_load_explicit(&rwlock->lock, memory_order_relaxed))) { - do { - if (atomic_compare_exchange_weak_explicit(&rwlock->lock, &old, -1, - memory_order_acquire, - memory_order_relaxed)) { - return 0; - } - } while (!old); - } - tries = pthread_rwlock_wrlock_spin(rwlock, old, tries); - } + nsync_mu_lock((nsync_mu *)rwlock); + rwlock->_iswrite = 1; + return 0; } diff --git a/libc/thread/pthread_rwlockattr_getpshared.c b/libc/thread/pthread_rwlockattr_getpshared.c index e087a71d0..efafcb927 100644 --- a/libc/thread/pthread_rwlockattr_getpshared.c +++ b/libc/thread/pthread_rwlockattr_getpshared.c @@ -22,8 +22,8 @@ * Gets read-write lock process sharing. * * @param pshared is set to one of the following - * - `PTHREAD_PROCESS_SHARED` - * - `PTHREAD_PROCESS_PRIVATE` + * - `PTHREAD_PROCESS_PRIVATE` (default) + * - `PTHREAD_PROCESS_SHARED` (unsupported) * @return 0 on success, or error on failure */ int pthread_rwlockattr_getpshared(const pthread_rwlockattr_t *attr, diff --git a/libc/thread/pthread_rwlockattr_setpshared.c b/libc/thread/pthread_rwlockattr_setpshared.c index a40ae0f94..5904c4d6c 100644 --- a/libc/thread/pthread_rwlockattr_setpshared.c +++ b/libc/thread/pthread_rwlockattr_setpshared.c @@ -23,14 +23,13 @@ * Sets read-write lock process sharing. * * @param pshared can be one of - * - `PTHREAD_PROCESS_SHARED` - * - `PTHREAD_PROCESS_PRIVATE` + * - `PTHREAD_PROCESS_PRIVATE` (default) + * - `PTHREAD_PROCESS_SHARED` (unsupported) * @return 0 on success, or error on failure * @raises EINVAL if `pshared` is invalid */ int pthread_rwlockattr_setpshared(pthread_rwlockattr_t *attr, int pshared) { switch (pshared) { - case PTHREAD_PROCESS_SHARED: case PTHREAD_PROCESS_PRIVATE: *attr = pshared; return 0; diff --git a/libc/thread/sem.h b/libc/thread/sem.h deleted file mode 100644 index 3659f8445..000000000 --- a/libc/thread/sem.h +++ /dev/null @@ -1,47 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_INTERNAL_SEM_H_ -#define NSYNC_INTERNAL_SEM_H_ - -/* A semaphore. - It may be counting or binary, and it need have no destructor. */ - -#include "libc/thread/nsync_time_internal.h" -#include "libc/thread/nsync_cpp.h" - -NSYNC_CPP_START_ - -typedef struct nsync_semaphore_s_ { - void *sem_space[32]; /* space used by implementation */ -} nsync_semaphore; - -/* Initialize *s; the initial value is 0. */ -void nsync_mu_semaphore_init (nsync_semaphore *s); - -/* Wait until the count of *s exceeds 0, and decrement it. */ -void nsync_mu_semaphore_p (nsync_semaphore *s); - -/* Wait until one of: - the count of *s is non-zero, in which case decrement *s and return 0; - or abs_deadline expires, in which case return ETIMEDOUT. */ -int nsync_mu_semaphore_p_with_deadline (nsync_semaphore *s, nsync_time abs_deadline); - -/* Ensure that the count of *s is at least 1. */ -void nsync_mu_semaphore_v (nsync_semaphore *s); - -NSYNC_CPP_END_ - -#endif /*NSYNC_INTERNAL_SEM_H_*/ diff --git a/libc/thread/sem_wait.c b/libc/thread/sem_wait.c deleted file mode 100644 index 376d07645..000000000 --- a/libc/thread/sem_wait.c +++ /dev/null @@ -1,82 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/platform.h" -#include "libc/thread/compiler.h" -#include "libc/thread/cputype.h" -#include "libc/thread/nsync.h" -#include "libc/thread/dll.h" -#include "libc/thread/sem.h" -#include "libc/thread/wait_internal.h" -#include "libc/thread/common.h" -#include "libc/thread/atomic.h" - -NSYNC_CPP_START_ - -/* Wait until one of: - w->sem is non-zero----decrement it and return 0. - abs_deadline expires---return ETIMEDOUT. - cancel_note is non-NULL and *cancel_note becomes notified---return ECANCELED. */ -int nsync_sem_wait_with_cancel_ (waiter *w, nsync_time abs_deadline, - nsync_note cancel_note) { - int sem_outcome; - if (cancel_note == NULL) { - sem_outcome = nsync_mu_semaphore_p_with_deadline (&w->sem, abs_deadline); - } else { - nsync_time cancel_time; - cancel_time = nsync_note_notified_deadline_ (cancel_note); - sem_outcome = ECANCELED; - if (nsync_time_cmp (cancel_time, nsync_time_zero) > 0) { - struct nsync_waiter_s nw; - nw.tag = NSYNC_WAITER_TAG; - nw.sem = &w->sem; - nsync_dll_init_ (&nw.q, &nw); - ATM_STORE (&nw.waiting, 1); - nw.flags = 0; - nsync_mu_lock (&cancel_note->note_mu); - cancel_time = NOTIFIED_TIME (cancel_note); - if (nsync_time_cmp (cancel_time, nsync_time_zero) > 0) { - nsync_time local_abs_deadline; - int deadline_is_nearer = 0; - cancel_note->waiters = nsync_dll_make_last_in_list_ ( - cancel_note->waiters, &nw.q); - local_abs_deadline = cancel_time; - if (nsync_time_cmp (abs_deadline, cancel_time) < 0) { - local_abs_deadline = abs_deadline; - deadline_is_nearer = 1; - } - nsync_mu_unlock (&cancel_note->note_mu); - sem_outcome = nsync_mu_semaphore_p_with_deadline (&w->sem, - local_abs_deadline); - if (sem_outcome == ETIMEDOUT && !deadline_is_nearer) { - sem_outcome = ECANCELED; - nsync_note_notify (cancel_note); - } - nsync_mu_lock (&cancel_note->note_mu); - cancel_time = NOTIFIED_TIME (cancel_note); - if (nsync_time_cmp (cancel_time, - nsync_time_zero) > 0) { - cancel_note->waiters = nsync_dll_remove_ ( - cancel_note->waiters, &nw.q); - } - } - nsync_mu_unlock (&cancel_note->note_mu); - } - } - return (sem_outcome); -} - -NSYNC_CPP_END_ diff --git a/libc/thread/sem_wait_no_note.c b/libc/thread/sem_wait_no_note.c deleted file mode 100644 index 76f07b328..000000000 --- a/libc/thread/sem_wait_no_note.c +++ /dev/null @@ -1,37 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/platform.h" -#include "libc/thread/compiler.h" -#include "libc/thread/cputype.h" -#include "libc/thread/nsync.h" -#include "libc/thread/dll.h" -#include "libc/thread/sem.h" -#include "libc/thread/wait_internal.h" -#include "libc/thread/common.h" -#include "libc/thread/atomic.h" - -NSYNC_CPP_START_ - -/* Wait until one of: - w->sem is non-zero----decrement it and return 0. - abs_deadline expires---return ETIMEDOUT. - Ignores cancel_note. */ -int nsync_sem_wait_with_cancel_ (waiter *w, nsync_time abs_deadline, nsync_note cancel_note UNUSED) { - return (nsync_mu_semaphore_p_with_deadline (&w->sem, abs_deadline)); -} - -NSYNC_CPP_END_ diff --git a/libc/thread/spawn.c b/libc/thread/spawn.c index cd54d7630..6ec8a144d 100644 --- a/libc/thread/spawn.c +++ b/libc/thread/spawn.c @@ -17,8 +17,6 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" -#include "libc/intrin/kprintf.h" -#include "libc/intrin/wait0.internal.h" #include "libc/macros.internal.h" #include "libc/mem/mem.h" #include "libc/runtime/internal.h" @@ -33,6 +31,7 @@ #include "libc/thread/posixthread.internal.h" #include "libc/thread/spawn.h" #include "libc/thread/tls.h" +#include "libc/thread/wait0.internal.h" /** * @fileoverview Simple threading API diff --git a/libc/thread/start_thread.c b/libc/thread/start_thread.c deleted file mode 100644 index c24a2667e..000000000 --- a/libc/thread/start_thread.c +++ /dev/null @@ -1,41 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "libc/thread/headers.h" - -NSYNC_CPP_START_ - -struct thd_args { - void (*f) (void *); - void *arg; -}; - -static void *body (void *v) { - struct thd_args *args = (struct thd_args *) v; - (*args->f) (args->arg); - free (args); - return (NULL); -} - -void nsync_start_thread_ (void (*f) (void *), void *arg) { - struct thd_args *args = (struct thd_args *) malloc (sizeof (*args)); - pthread_t t; - args->f = f; - args->arg = arg; - pthread_create (&t, NULL, body, args); - pthread_detach (t); -} - -NSYNC_CPP_END_ diff --git a/libc/thread/thread.h b/libc/thread/thread.h index 6ab838622..ab97823ef 100644 --- a/libc/thread/thread.h +++ b/libc/thread/thread.h @@ -26,11 +26,10 @@ #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ -#define PTHREAD_ONCE_INIT _PTHREAD_INIT -#define PTHREAD_COND_INITIALIZER _PTHREAD_INIT -#define PTHREAD_BARRIER_INITIALIZER _PTHREAD_INIT -#define PTHREAD_RWLOCK_INITIALIZER _PTHREAD_INIT -#define PTHREAD_MUTEX_INITIALIZER _PTHREAD_INIT +#define PTHREAD_ONCE_INIT _PTHREAD_INIT +#define PTHREAD_COND_INITIALIZER _PTHREAD_INIT +#define PTHREAD_RWLOCK_INITIALIZER _PTHREAD_INIT +#define PTHREAD_MUTEX_INITIALIZER _PTHREAD_INIT #define _PTHREAD_INIT \ { 0 } @@ -43,43 +42,38 @@ typedef unsigned pthread_key_t; typedef void (*pthread_key_dtor)(void *); typedef struct pthread_once_s { - _Atomic(char) lock; + _Atomic(uint32_t) _lock; } pthread_once_t; typedef struct pthread_spinlock_s { - _Atomic(char) lock; + _Atomic(char) _lock; } pthread_spinlock_t; typedef struct pthread_mutex_s { - char type; - char pshared; - _Atomic(int) lock; + _Atomic(int) _lock; + void *_nsync; + char _type; + char _pshared; } pthread_mutex_t; typedef struct pthread_mutexattr_s { - char type; - char pshared; + char _type; + char _pshared; } pthread_mutexattr_t; typedef struct pthread_cond_s { - char pshared; - _Atomic(int) waits; - _Atomic(unsigned) seq; + void *_nsync[2]; } pthread_cond_t; -typedef struct pthread_barrier_s { - char pshared; - int count; - _Atomic(int) waits; - _Atomic(int) popped; -} pthread_barrier_t; - typedef struct pthread_rwlock_s { - char pshared; - _Atomic(int) lock; - _Atomic(int) waits; + void *_nsync[2]; + char _iswrite; } pthread_rwlock_t; +typedef struct pthread_barrier_s { + void *_nsync; +} pthread_barrier_t; + typedef struct pthread_attr_s { char detachstate; char inheritsched; @@ -172,27 +166,27 @@ int pthread_barrier_destroy(pthread_barrier_t *); int pthread_barrier_init(pthread_barrier_t *, const pthread_barrierattr_t *, unsigned); -#define pthread_spin_init(pSpin, multiprocess) ((pSpin)->lock = 0, 0) -#define pthread_spin_destroy(pSpin) ((pSpin)->lock = -1, 0) +#define pthread_spin_init(pSpin, multiprocess) ((pSpin)->_lock = 0, 0) +#define pthread_spin_destroy(pSpin) ((pSpin)->_lock = -1, 0) #if (__GNUC__ + 0) * 100 + (__GNUC_MINOR__ + 0) >= 407 && \ !defined(__STRICT_ANSI__) extern const errno_t EBUSY; -#define pthread_spin_lock(pSpin) \ - ({ \ - pthread_spinlock_t *_s = pSpin; \ - while (__atomic_test_and_set(&_s->lock, __ATOMIC_SEQ_CST)) donothing; \ - 0; \ +#define pthread_spin_lock(pSpin) \ + ({ \ + pthread_spinlock_t *_s = pSpin; \ + while (__atomic_test_and_set(&_s->_lock, __ATOMIC_SEQ_CST)) donothing; \ + 0; \ }) -#define pthread_spin_unlock(pSpin) \ - ({ \ - pthread_spinlock_t *_s = pSpin; \ - __atomic_store_n(&_s->lock, 0, __ATOMIC_RELAXED); \ - 0; \ +#define pthread_spin_unlock(pSpin) \ + ({ \ + pthread_spinlock_t *_s = pSpin; \ + __atomic_store_n(&_s->_lock, 0, __ATOMIC_RELAXED); \ + 0; \ }) -#define pthread_spin_trylock(pSpin) \ - ({ \ - pthread_spinlock_t *_s = pSpin; \ - __atomic_test_and_set(&_s->lock, __ATOMIC_SEQ_CST) ? EBUSY : 0; \ +#define pthread_spin_trylock(pSpin) \ + ({ \ + pthread_spinlock_t *_s = pSpin; \ + __atomic_test_and_set(&_s->_lock, __ATOMIC_SEQ_CST) ? EBUSY : 0; \ }) #endif /* GCC 4.7+ */ diff --git a/libc/thread/thread.mk b/libc/thread/thread.mk index 024af185a..d32ebd6ce 100644 --- a/libc/thread/thread.mk +++ b/libc/thread/thread.mk @@ -7,20 +7,14 @@ LIBC_THREAD_ARTIFACTS += LIBC_THREAD_A LIBC_THREAD = $(LIBC_THREAD_A_DEPS) $(LIBC_THREAD_A) LIBC_THREAD_A = o/$(MODE)/libc/thread/thread.a LIBC_THREAD_A_FILES := $(wildcard libc/thread/*) -LIBC_THREAD_A_SRCS_S = $(filter %.S,$(LIBC_THREAD_A_FILES)) +LIBC_THREAD_A_HDRS = $(filter %.h,$(LIBC_THREAD_A_FILES)) LIBC_THREAD_A_SRCS_C = $(filter %.c,$(LIBC_THREAD_A_FILES)) +LIBC_THREAD_A_SRCS_S = $(filter %.S,$(LIBC_THREAD_A_FILES)) LIBC_THREAD_A_SRCS = \ $(LIBC_THREAD_A_SRCS_S) \ $(LIBC_THREAD_A_SRCS_C) -LIBC_THREAD_A_HDRS = \ - libc/thread/spawn.h \ - libc/thread/thread.h \ - libc/thread/thread2.h \ - libc/thread/tls.h \ - libc/thread/tls2.h - LIBC_THREAD_A_OBJS = \ $(LIBC_THREAD_A_SRCS_S:%.S=o/$(MODE)/%.o) \ $(LIBC_THREAD_A_SRCS_C:%.c=o/$(MODE)/%.o) @@ -39,7 +33,8 @@ LIBC_THREAD_A_DIRECTDEPS = \ LIBC_STUBS \ LIBC_SYSV \ LIBC_SYSV_CALLS \ - LIBC_NEXGEN32E + LIBC_NEXGEN32E \ + THIRD_PARTY_NSYNC LIBC_THREAD_A_DEPS := \ $(call uniq,$(foreach x,$(LIBC_THREAD_A_DIRECTDEPS),$($(x)))) @@ -53,6 +48,11 @@ $(LIBC_THREAD_A).pkg: \ $(LIBC_THREAD_A_OBJS) \ $(foreach x,$(LIBC_THREAD_A_DIRECTDEPS),$($(x)_A).pkg) +$(LIBC_THREAD_A_OBJS): private \ + OVERRIDE_CCFLAGS += \ + -ffunction-sections \ + -fdata-sections + LIBC_THREAD_LIBS = $(foreach x,$(LIBC_THREAD_ARTIFACTS),$($(x))) LIBC_THREAD_SRCS = $(foreach x,$(LIBC_THREAD_ARTIFACTS),$($(x)_SRCS)) LIBC_THREAD_HDRS = $(foreach x,$(LIBC_THREAD_ARTIFACTS),$($(x)_HDRS)) diff --git a/libc/thread/time_internal.c b/libc/thread/time_internal.c deleted file mode 100644 index 8b17c12ed..000000000 --- a/libc/thread/time_internal.c +++ /dev/null @@ -1,34 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/platform.h" -#include "libc/thread/compiler.h" -#include "libc/thread/cputype.h" -#include "libc/thread/nsync_time.h" - -NSYNC_CPP_START_ - -nsync_time nsync_time_ms (unsigned ms) { - unsigned s = ms / 1000; - return (nsync_time_s_ns (s, 1000 * 1000 * (ms % 1000))); -} - -nsync_time nsync_time_us (unsigned us) { - unsigned s = us / (1000 * 1000); - return (nsync_time_s_ns (s, 1000 * (us % (1000 * 1000)))); -} - -NSYNC_CPP_END_ diff --git a/libc/thread/time_rep.c b/libc/thread/time_rep.c deleted file mode 100644 index 74c5d45d2..000000000 --- a/libc/thread/time_rep.c +++ /dev/null @@ -1,96 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/platform.h" -#include "libc/thread/compiler.h" -#include "libc/thread/cputype.h" -#include "libc/thread/nsync_time_init.h" -#include "libc/thread/nsync_time.h" - -NSYNC_CPP_START_ - -#define NSYNC_NS_IN_S_ (1000 * 1000 * 1000) - -/* Return the maximum t, assuming it's an integral - type, and the representation is not too strange. */ -#define MAX_INT_TYPE(t) (((t)~(t)0) > 1? /*is t unsigned?*/ \ - (t)~(t)0 : /*unsigned*/ \ - (t) ((((uintmax_t)1) << (sizeof (t) * CHAR_BIT - 1)) - 1)) /*signed*/ - -const nsync_time nsync_time_no_deadline = - NSYNC_TIME_STATIC_INIT (MAX_INT_TYPE (time_t), NSYNC_NS_IN_S_ - 1); - -const nsync_time nsync_time_zero = NSYNC_TIME_STATIC_INIT (0, 0); - -nsync_time nsync_time_s_ns (time_t s, unsigned ns) { - nsync_time t; - memset (&t, 0, sizeof (t)); - t.tv_sec = s; - t.tv_nsec = ns; - return (t); -} - -nsync_time nsync_time_now (void) { - struct timespec ts; - clock_gettime (CLOCK_REALTIME, &ts); - return (ts); -} - -nsync_time nsync_time_sleep (nsync_time delay) { - struct timespec ts; - struct timespec remain; - memset (&ts, 0, sizeof (ts)); - ts.tv_sec = NSYNC_TIME_SEC (delay); - ts.tv_nsec = NSYNC_TIME_NSEC (delay); - if (nanosleep (&ts, &remain) == 0) { - /* nanosleep() is not required to fill in "remain" - if it returns 0. */ - memset (&remain, 0, sizeof (remain)); - } - return (remain); -} - -nsync_time nsync_time_add (nsync_time a, nsync_time b) { - a.tv_sec += b.tv_sec; - a.tv_nsec += b.tv_nsec; - if (a.tv_nsec >= NSYNC_NS_IN_S_) { - a.tv_nsec -= NSYNC_NS_IN_S_; - a.tv_sec++; - } - return (a); -} - -nsync_time nsync_time_sub (nsync_time a, nsync_time b) { - a.tv_sec -= b.tv_sec; - if (a.tv_nsec < b.tv_nsec) { - a.tv_nsec += NSYNC_NS_IN_S_; - a.tv_sec--; - } - a.tv_nsec -= b.tv_nsec; - return (a); -} - -int nsync_time_cmp (nsync_time a, nsync_time b) { - int cmp = (NSYNC_TIME_SEC (a) > NSYNC_TIME_SEC (b)) - - (NSYNC_TIME_SEC (a) < NSYNC_TIME_SEC (b)); - if (cmp == 0) { - cmp = (NSYNC_TIME_NSEC (a) > NSYNC_TIME_NSEC (b)) - - (NSYNC_TIME_NSEC (a) < NSYNC_TIME_NSEC (b)); - } - return (cmp); -} - -NSYNC_CPP_END_ diff --git a/libc/thread/wait.c b/libc/thread/wait.c deleted file mode 100644 index 4b562c0f0..000000000 --- a/libc/thread/wait.c +++ /dev/null @@ -1,104 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/platform.h" -#include "libc/thread/compiler.h" -#include "libc/thread/cputype.h" -#include "libc/thread/nsync.h" -#include "libc/thread/sem.h" -#include "libc/thread/dll.h" -#include "libc/thread/wait_internal.h" -#include "libc/thread/common.h" -#include "libc/thread/atomic.h" - -NSYNC_CPP_START_ - -int nsync_wait_n (void *mu, void (*lock) (void *), void (*unlock) (void *), - nsync_time abs_deadline, - int count, struct nsync_waitable_s *waitable[]) { - int ready; - IGNORE_RACES_START (); - for (ready = 0; ready != count && - nsync_time_cmp ((*waitable[ready]->funcs->ready_time) ( - waitable[ready]->v, NULL), - nsync_time_zero) > 0; - ready++) { - } - if (ready == count && nsync_time_cmp (abs_deadline, nsync_time_zero) > 0) { - int i; - int unlocked = 0; - int j; - int enqueued = 1; - waiter *w = nsync_waiter_new_ (); - struct nsync_waiter_s nw_set[4]; - struct nsync_waiter_s *nw = nw_set; - if (count > (int) (sizeof (nw_set) / sizeof (nw_set[0]))) { - nw = (struct nsync_waiter_s *) malloc (count * sizeof (nw[0])); - } - for (i = 0; i != count && enqueued; i++) { - nw[i].tag = NSYNC_WAITER_TAG; - nw[i].sem = &w->sem; - nsync_dll_init_ (&nw[i].q, &nw[i]); - ATM_STORE (&nw[i].waiting, 0); - nw[i].flags = 0; - enqueued = (*waitable[i]->funcs->enqueue) (waitable[i]->v, &nw[i]); - } - - if (i == count) { - nsync_time min_ntime; - if (mu != NULL) { - (*unlock) (mu); - unlocked = 1; - } - do { - min_ntime = abs_deadline; - for (j = 0; j != count; j++) { - nsync_time ntime; - ntime = (*waitable[j]->funcs->ready_time) ( - waitable[j]->v, &nw[j]); - if (nsync_time_cmp (ntime, min_ntime) < 0) { - min_ntime = ntime; - } - } - } while (nsync_time_cmp (min_ntime, nsync_time_zero) > 0 && - nsync_mu_semaphore_p_with_deadline (&w->sem, - min_ntime) == 0); - } - - /* An attempt was made above to enqueue waitable[0..i-1]. - Dequeue any that are still enqueued, and remember the index - of the first ready (i.e., not still enqueued) object, if any. */ - for (j = 0; j != i; j++) { - int was_still_enqueued = - (*waitable[j]->funcs->dequeue) (waitable[j]->v, &nw[j]); - if (!was_still_enqueued && ready == count) { - ready = j; - } - } - - if (nw != nw_set) { - free (nw); - } - nsync_waiter_free_ (w); - if (unlocked) { - (*lock) (mu); - } - } - IGNORE_RACES_END (); - return (ready); -} - -NSYNC_CPP_END_ diff --git a/libc/intrin/wait0.c b/libc/thread/wait0.c similarity index 76% rename from libc/intrin/wait0.c rename to libc/thread/wait0.c index 4a0a1493d..380eb5718 100644 --- a/libc/intrin/wait0.c +++ b/libc/thread/wait0.c @@ -16,13 +16,17 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/calls.h" +#include "libc/calls/strace.internal.h" +#include "libc/calls/struct/timespec.h" #include "libc/dce.h" +#include "libc/errno.h" #include "libc/intrin/atomic.h" -#include "libc/intrin/futex.internal.h" +#include "libc/intrin/describeflags.internal.h" +#include "libc/sysv/consts/futex.h" #include "libc/thread/thread.h" -#include "libc/intrin/wait0.internal.h" -#include "libc/linux/futex.h" +#include "libc/thread/wait0.internal.h" + +int _futex(int *, int, int, const struct timespec *); /** * Blocks until memory location becomes zero. @@ -32,12 +36,24 @@ * this operation is to know when it's safe to munmap() a threads stack */ void _wait0(const int *ctid) { - int x; + int x, rc; + char buf[12]; for (;;) { if (!(x = atomic_load_explicit(ctid, memory_order_relaxed))) { break; + } else if (IsLinux() || IsOpenbsd()) { + rc = _futex(ctid, FUTEX_WAIT, x, &(struct timespec){2}); + STRACE("futex(%t, FUTEX_WAIT, %d, {2, 0}) → %s", ctid, x, + (DescribeFutexResult)(buf, rc)); + if (IsOpenbsd() && rc > 0) rc = -rc; + if (!(rc == 0 || // + rc == -EINTR || // + rc == -ETIMEDOUT || // + rc == -EWOULDBLOCK)) { + notpossible; + } } else { - _futex_wait(ctid, x, PTHREAD_PROCESS_SHARED, &(struct timespec){2}); + pthread_yield(); } } if (IsOpenbsd()) { diff --git a/libc/intrin/wait0.internal.h b/libc/thread/wait0.internal.h similarity index 100% rename from libc/intrin/wait0.internal.h rename to libc/thread/wait0.internal.h diff --git a/libc/thread/wait_internal.h b/libc/thread/wait_internal.h deleted file mode 100644 index 828614444..000000000 --- a/libc/thread/wait_internal.h +++ /dev/null @@ -1,40 +0,0 @@ -// clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#ifndef NSYNC_INTERNAL_WAIT_INTERNAL_H_ -#define NSYNC_INTERNAL_WAIT_INTERNAL_H_ -#include "libc/thread/dll.h" -#include "libc/thread/nsync_atomic.h" -#include "libc/thread/nsync_cpp.h" - -NSYNC_CPP_START_ - -/* Implementations of "struct nsync_waitable_s" must provide functions in struct - nsync_waitable_funcs_s (see public/nsync_wait.h). When nsync_wait_n() waits - on a client's object, those functions are called with v pointing to the - client's object and nw pointing to a struct nsync_waiter_s. */ -struct nsync_waiter_s { - uint32_t tag; /* used for debugging */ - nsync_dll_element_ q; /* used to link children of parent */ - nsync_atomic_uint32_ waiting; /* non-zero <=> the waiter is waiting */ - struct nsync_semaphore_s_ *sem; /* *sem will be Ved when waiter is woken */ - uint32_t flags; /* see below */ -}; - -#define NSYNC_WAITER_FLAG_MUCV 0x1 /* set if waiter is embedded in Mu/CV's internal structures */ - -NSYNC_CPP_END_ - -#endif /*NSYNC_INTERNAL_WAIT_INTERNAL_H_*/ diff --git a/test/libc/calls/pledge_test.c b/test/libc/calls/pledge_test.c index 988f8ec8e..f777df43d 100644 --- a/test/libc/calls/pledge_test.c +++ b/test/libc/calls/pledge_test.c @@ -560,7 +560,7 @@ TEST(pledge_openbsd, bigSyscalls) { int LockWorker(void *arg, int tid) { flockfile(stdout); - ASSERT_EQ(gettid(), stdout->lock.lock & 0x000fffff); + ASSERT_EQ(gettid(), stdout->lock._lock & 0x000fffff); funlockfile(stdout); return 0; } diff --git a/test/libc/calls/reservefd_test.c b/test/libc/calls/reservefd_test.c index 7c1e535c6..0d958625f 100644 --- a/test/libc/calls/reservefd_test.c +++ b/test/libc/calls/reservefd_test.c @@ -23,9 +23,7 @@ #include "libc/calls/struct/rlimit.h" #include "libc/calls/struct/sigaction.h" #include "libc/errno.h" -#include "libc/intrin/wait0.internal.h" #include "libc/macros.internal.h" -#include "libc/thread/tls.h" #include "libc/runtime/internal.h" #include "libc/runtime/stack.h" #include "libc/stdio/rand.h" @@ -41,6 +39,8 @@ #include "libc/testlib/hyperion.h" #include "libc/testlib/testlib.h" #include "libc/thread/spawn.h" +#include "libc/thread/tls.h" +#include "libc/thread/wait0.internal.h" #include "libc/time/struct/tm.h" #include "libc/time/time.h" diff --git a/test/libc/intrin/pthread_mutex_lock2_test.c b/test/libc/intrin/pthread_mutex_lock2_test.c index 63c9484b7..c582c6167 100644 --- a/test/libc/intrin/pthread_mutex_lock2_test.c +++ b/test/libc/intrin/pthread_mutex_lock2_test.c @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/calls/struct/timespec.h" #include "libc/intrin/atomic.h" @@ -26,13 +27,14 @@ #include "libc/thread/posixthread.internal.h" #include "libc/thread/spawn.h" #include "libc/thread/thread.h" +#include "third_party/nsync/mu.h" int THREADS = 16; int ITERATIONS = 100; int count; -_Atomic(int) started; -_Atomic(int) finished; +atomic_int started; +atomic_int finished; pthread_mutex_t lock; pthread_mutexattr_t attr; @@ -108,29 +110,50 @@ void BenchLockUnlock(pthread_mutex_t *m) { pthread_mutex_unlock(m); } +void BenchLockUnlockNsync(nsync_mu *m) { + nsync_mu_lock(m); + nsync_mu_unlock(m); +} + BENCH(pthread_mutex_lock, bench_uncontended) { { pthread_spinlock_t s = {0}; EZBENCH2("spin 1x", donothing, BenchSpinUnspin(&s)); } { - pthread_mutex_t m = {PTHREAD_MUTEX_NORMAL}; + nsync_mu m = {0}; + EZBENCH2("nsync 1x", donothing, BenchLockUnlockNsync(&m)); + } + { + pthread_mutex_t m; + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL); + pthread_mutex_init(&m, &attr); EZBENCH2("normal 1x", donothing, BenchLockUnlock(&m)); } { - pthread_mutex_t m = {PTHREAD_MUTEX_RECURSIVE}; + pthread_mutex_t m; + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&m, &attr); EZBENCH2("recursive 1x", donothing, BenchLockUnlock(&m)); } { - pthread_mutex_t m = {PTHREAD_MUTEX_ERRORCHECK}; + pthread_mutex_t m; + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK); + pthread_mutex_init(&m, &attr); EZBENCH2("errorcheck 1x", donothing, BenchLockUnlock(&m)); } } struct SpinContentionArgs { pthread_spinlock_t *spin; - _Atomic(char) done; - _Atomic(char) ready; + atomic_char done; + atomic_char ready; }; int SpinContentionWorker(void *arg, int tid) { @@ -145,8 +168,8 @@ int SpinContentionWorker(void *arg, int tid) { struct MutexContentionArgs { pthread_mutex_t *mutex; - _Atomic(char) done; - _Atomic(char) ready; + atomic_char done; + atomic_char ready; }; int MutexContentionWorker(void *arg, int tid) { @@ -159,6 +182,22 @@ int MutexContentionWorker(void *arg, int tid) { return 0; } +struct NsyncContentionArgs { + nsync_mu *nsync; + atomic_char done; + atomic_char ready; +}; + +int NsyncContentionWorker(void *arg, int tid) { + struct NsyncContentionArgs *a = arg; + while (!atomic_load_explicit(&a->done, memory_order_relaxed)) { + nsync_mu_lock(a->nsync); + atomic_store_explicit(&a->ready, 1, memory_order_relaxed); + nsync_mu_unlock(a->nsync); + } + return 0; +} + BENCH(pthread_mutex_lock, bench_contended) { struct spawn t; { @@ -171,7 +210,20 @@ BENCH(pthread_mutex_lock, bench_contended) { _join(&t); } { - pthread_mutex_t m = {PTHREAD_MUTEX_NORMAL}; + nsync_mu m = {0}; + struct NsyncContentionArgs a = {&m}; + _spawn(NsyncContentionWorker, &a, &t); + while (!a.ready) sched_yield(); + EZBENCH2("nsync 2x", donothing, BenchLockUnlockNsync(&m)); + a.done = true; + _join(&t); + } + { + pthread_mutex_t m; + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL); + pthread_mutex_init(&m, &attr); struct MutexContentionArgs a = {&m}; _spawn(MutexContentionWorker, &a, &t); while (!a.ready) sched_yield(); @@ -180,7 +232,11 @@ BENCH(pthread_mutex_lock, bench_contended) { _join(&t); } { - pthread_mutex_t m = {PTHREAD_MUTEX_RECURSIVE}; + pthread_mutex_t m; + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&m, &attr); struct MutexContentionArgs a = {&m}; _spawn(MutexContentionWorker, &a, &t); while (!a.ready) sched_yield(); @@ -189,7 +245,11 @@ BENCH(pthread_mutex_lock, bench_contended) { _join(&t); } { - pthread_mutex_t m = {PTHREAD_MUTEX_ERRORCHECK}; + pthread_mutex_t m; + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK); + pthread_mutex_init(&m, &attr); struct MutexContentionArgs a = {&m}; _spawn(MutexContentionWorker, &a, &t); while (!a.ready) sched_yield(); diff --git a/test/libc/intrin/pthread_mutex_lock_test.c b/test/libc/intrin/pthread_mutex_lock_test.c index 686fe6b10..396bb9319 100644 --- a/test/libc/intrin/pthread_mutex_lock_test.c +++ b/test/libc/intrin/pthread_mutex_lock_test.c @@ -17,12 +17,11 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" +#include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/calls/state.internal.h" #include "libc/calls/strace.internal.h" #include "libc/errno.h" -#include "libc/intrin/futex.internal.h" -#include "libc/intrin/wait0.internal.h" #include "libc/log/check.h" #include "libc/macros.internal.h" #include "libc/math.h" @@ -40,13 +39,14 @@ #include "libc/thread/spawn.h" #include "libc/thread/thread.h" #include "libc/thread/tls.h" +#include "libc/thread/wait0.internal.h" #define THREADS 8 #define ITERATIONS 512 int count; -_Atomic(int) started; -_Atomic(int) finished; +atomic_int started; +atomic_int finished; pthread_mutex_t mylock; pthread_spinlock_t slock; struct spawn th[THREADS]; diff --git a/test/libc/intrin/pthread_once_test.c b/test/libc/intrin/pthread_once_test.c index 5ca803509..e51f7a574 100644 --- a/test/libc/intrin/pthread_once_test.c +++ b/test/libc/intrin/pthread_once_test.c @@ -16,17 +16,19 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/atomic.h" #include "libc/intrin/atomic.h" -#include "libc/thread/thread.h" #include "libc/mem/mem.h" #include "libc/runtime/gc.internal.h" #include "libc/testlib/testlib.h" #include "libc/thread/spawn.h" +#include "libc/thread/thread.h" int i, n; struct spawn *t; -_Atomic(int) x, y; +atomic_int x, y; pthread_barrier_t b; +static pthread_once_t once = PTHREAD_ONCE_INIT; void InitFactory(void) { ASSERT_EQ(0, atomic_load(&x)); @@ -34,7 +36,6 @@ void InitFactory(void) { } int Worker(void *arg, int tid) { - static pthread_once_t once = PTHREAD_ONCE_INIT; pthread_barrier_wait(&b); ASSERT_EQ(0, pthread_once(&once, InitFactory)); ASSERT_EQ(1, atomic_load(&y)); @@ -45,10 +46,11 @@ int Worker(void *arg, int tid) { TEST(pthread_once, test) { n = 32; x = y = 0; - pthread_barrier_init(&b, 0, n); + ASSERT_EQ(0, pthread_barrier_init(&b, 0, n)); t = gc(malloc(sizeof(struct spawn) * n)); for (i = 0; i < n; ++i) ASSERT_SYS(0, 0, _spawn(Worker, 0, t + i)); for (i = 0; i < n; ++i) EXPECT_SYS(0, 0, _join(t + i)); ASSERT_EQ(n, atomic_load(&x)); ASSERT_EQ(1, atomic_load(&y)); + ASSERT_EQ(0, pthread_barrier_destroy(&b)); } diff --git a/test/libc/intrin/rand64_test.c b/test/libc/intrin/rand64_test.c index 8314ea1fa..0c6efa4fd 100644 --- a/test/libc/intrin/rand64_test.c +++ b/test/libc/intrin/rand64_test.c @@ -43,7 +43,7 @@ #define ENTRIES 1024 volatile uint64_t A[THREADS * ENTRIES]; -pthread_barrier_t barrier = PTHREAD_BARRIER_INITIALIZER; +pthread_barrier_t barrier; void SetUpOnce(void) { __enable_threads(); @@ -93,7 +93,7 @@ TEST(rand64, testThreadSafety_doesntProduceIdenticalValues) { sigemptyset(&ss); sigaddset(&ss, SIGCHLD); EXPECT_EQ(0, sigprocmask(SIG_BLOCK, &ss, &oldss)); - pthread_barrier_init(&barrier, 0, THREADS); + ASSERT_EQ(0, pthread_barrier_init(&barrier, 0, THREADS)); for (i = 0; i < THREADS; ++i) { ASSERT_SYS(0, 0, _spawn(Thrasher, (void *)(intptr_t)i, th + i)); } @@ -109,4 +109,5 @@ TEST(rand64, testThreadSafety_doesntProduceIdenticalValues) { EXPECT_NE(A[i], A[j], "i=%d j=%d", i, j); } } + ASSERT_EQ(0, pthread_barrier_destroy(&barrier)); } diff --git a/test/libc/intrin/test.mk b/test/libc/intrin/test.mk index 5eb021c64..48ce2837a 100644 --- a/test/libc/intrin/test.mk +++ b/test/libc/intrin/test.mk @@ -39,7 +39,8 @@ TEST_LIBC_INTRIN_DIRECTDEPS = \ LIBC_TINYMATH \ LIBC_X \ TOOL_VIZ_LIB \ - THIRD_PARTY_COMPILER_RT + THIRD_PARTY_COMPILER_RT \ + THIRD_PARTY_NSYNC TEST_LIBC_INTRIN_DEPS := \ $(call uniq,$(foreach x,$(TEST_LIBC_INTRIN_DIRECTDEPS),$($(x)))) diff --git a/test/libc/nexgen32e/stackrw_test.c b/test/libc/nexgen32e/stackrw_test.c index 97f85d28c..4f55e45f6 100644 --- a/test/libc/nexgen32e/stackrw_test.c +++ b/test/libc/nexgen32e/stackrw_test.c @@ -17,6 +17,8 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/struct/sigaction.h" +#include "libc/calls/ucontext.h" +#include "libc/dce.h" #include "libc/runtime/runtime.h" #include "libc/sysv/consts/sa.h" #include "libc/sysv/consts/sig.h" @@ -28,15 +30,16 @@ jmp_buf jb; -void EscapeSegfault(int sig) { +void EscapeSegfault(int sig, struct siginfo *si, void *vctx) { longjmp(jb, 666); } TEST(xstack, test) { + if (IsXnu()) return; // TODO(jart): what's up with xnu in MODE=tiny? struct sigaction old[2]; struct sigaction sa = { - .sa_handler = EscapeSegfault, - .sa_flags = SA_NODEFER, + .sa_sigaction = EscapeSegfault, + .sa_flags = SA_SIGINFO, }; sigaction(SIGSEGV, &sa, old + 0); sigaction(SIGBUS, &sa, old + 1); diff --git a/test/libc/stdio/dtoa_test.c b/test/libc/stdio/dtoa_test.c index 7b2cc638f..0ecebcccc 100644 --- a/test/libc/stdio/dtoa_test.c +++ b/test/libc/stdio/dtoa_test.c @@ -20,7 +20,6 @@ #include "libc/calls/struct/sched_param.h" #include "libc/dce.h" #include "libc/fmt/fmt.h" -#include "libc/intrin/wait0.internal.h" #include "libc/macros.internal.h" #include "libc/math.h" #include "libc/mem/mem.h" @@ -35,6 +34,7 @@ #include "libc/sysv/consts/sched.h" #include "libc/testlib/testlib.h" #include "libc/thread/spawn.h" +#include "libc/thread/wait0.internal.h" #include "libc/x/x.h" #define DUB(i) (union Dub){i}.x diff --git a/test/libc/thread/clone_test.c b/test/libc/thread/clone_test.c index 3c680eebb..4457364fd 100644 --- a/test/libc/thread/clone_test.c +++ b/test/libc/thread/clone_test.c @@ -16,12 +16,11 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/calls/struct/timespec.h" #include "libc/dce.h" #include "libc/errno.h" -#include "libc/intrin/futex.internal.h" -#include "libc/intrin/wait0.internal.h" #include "libc/log/backtrace.internal.h" #include "libc/macros.internal.h" #include "libc/mem/mem.h" @@ -43,10 +42,11 @@ #include "libc/thread/spawn.h" #include "libc/thread/tls.h" #include "libc/thread/tls2.h" +#include "libc/thread/wait0.internal.h" #include "libc/time/time.h" int x, me, tid; -_Atomic(int) thechilde; +atomic_int thechilde; void SetUpOnce(void) { __enable_threads(); @@ -109,7 +109,7 @@ TEST(clone, test1) { //////////////////////////////////////////////////////////////////////////////// // TEST THREADS CAN ISSUE SYSTEM CALLS WITH INDEPENDENT ERRNOS -_Atomic(int) sysbarrier; +atomic_int sysbarrier; int CloneTestSys(void *arg, int tid) { int i, id = (intptr_t)arg; diff --git a/test/libc/thread/nsync_test.c b/test/libc/thread/nsync_test.c new file mode 100644 index 000000000..6c27d1ef7 --- /dev/null +++ b/test/libc/thread/nsync_test.c @@ -0,0 +1,119 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/errno.h" +#include "libc/testlib/testlib.h" +#include "libc/thread/thread.h" +#include "third_party/nsync/cv.h" +#include "third_party/nsync/mu.h" + +int pos; +int count; +int limit; +long data[1000]; +nsync_mu mu; +nsync_cv non_full; +nsync_cv non_empty; + +int Put(long v, nsync_time abs_deadline) { + int err, added = 0, wake = 0; + nsync_mu_lock(&mu); + while (count == limit) { + if ((err = nsync_cv_wait_with_deadline(&non_full, &mu, abs_deadline, 0))) { + ASSERT_EQ(ETIMEDOUT, err); + ASSERT_NE(0, nsync_time_cmp(nsync_time_no_deadline, abs_deadline)); + } + } + if (count != limit) { + int i = pos + count; + if (limit <= i) i -= limit; + data[i] = v; + if (count == 0) wake = 1; + count++; + added = 1; + } + nsync_mu_unlock(&mu); + if (wake) nsync_cv_broadcast(&non_empty); + return added; +} + +long Get(nsync_time abs_deadline) { + long err, v = 0; + nsync_mu_lock(&mu); + while (!count) { + if ((err = nsync_cv_wait_with_deadline(&non_empty, &mu, abs_deadline, 0))) { + ASSERT_EQ(ETIMEDOUT, err); + ASSERT_NE(0, nsync_time_cmp(nsync_time_no_deadline, abs_deadline)); + } + } + if (count) { + v = data[pos]; + data[pos] = 0; + if (count == limit) { + nsync_cv_broadcast(&non_full); + } + pos++; + count--; + if (pos == limit) { + pos = 0; + } + } + nsync_mu_unlock(&mu); + return v; +} + +#define N 10000 + +void *Producer(void *arg) { + for (int i = 0; i < N; i++) { + ASSERT_EQ(1, Put((i + 1) * 3, nsync_time_no_deadline)); + } + return 0; +} + +void *Consumer(void *arg) { + for (int i = 0; i < N; i++) { + ASSERT_EQ((i + 1) * 3, Get(nsync_time_no_deadline)); + } + return 0; +} + +TEST(cond, test) { + pthread_t t; + nsync_mu_init(&mu); + + limit = 1; + ASSERT_EQ(0, pthread_create(&t, 0, Producer, 0)); + Consumer(0); + ASSERT_EQ(0, pthread_join(t, 0)); + + limit = 10; + ASSERT_EQ(0, pthread_create(&t, 0, Producer, 0)); + Consumer(0); + ASSERT_EQ(0, pthread_join(t, 0)); + + limit = 100; + ASSERT_EQ(0, pthread_create(&t, 0, Producer, 0)); + Consumer(0); + ASSERT_EQ(0, pthread_join(t, 0)); + + limit = 1000; + ASSERT_EQ(0, pthread_create(&t, 0, Producer, 0)); + Consumer(0); + ASSERT_EQ(0, pthread_join(t, 0)); +} diff --git a/test/libc/thread/pthread_barrier_wait_test.c b/test/libc/thread/pthread_barrier_wait_test.c index 8bacd1b6f..dd9942fed 100644 --- a/test/libc/thread/pthread_barrier_wait_test.c +++ b/test/libc/thread/pthread_barrier_wait_test.c @@ -17,6 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" +#include "libc/atomic.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" #include "libc/mem/mem.h" @@ -26,7 +27,7 @@ #include "libc/thread/thread.h" int i, n; -_Atomic(int) p, w; +atomic_int p, w; pthread_barrier_t barrier; int Worker(void *arg, int tid) { @@ -36,9 +37,6 @@ int Worker(void *arg, int tid) { ASSERT_GE(rc, 0); if (rc == PTHREAD_BARRIER_SERIAL_THREAD) { atomic_fetch_add(&p, 1); - ASSERT_EQ(0, barrier.popped); - ASSERT_EQ(0, barrier.waits); - ASSERT_EQ(n, barrier.count); } return 0; } diff --git a/test/libc/thread/pthread_cond_broadcast_test.c b/test/libc/thread/pthread_cond_broadcast_test.c deleted file mode 100644 index 0f28cb4ee..000000000 --- a/test/libc/thread/pthread_cond_broadcast_test.c +++ /dev/null @@ -1,123 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/calls/struct/timespec.h" -#include "libc/errno.h" -#include "libc/intrin/atomic.h" -#include "libc/intrin/kprintf.h" -#include "libc/mem/mem.h" -#include "libc/runtime/gc.internal.h" -#include "libc/testlib/testlib.h" -#include "libc/thread/spawn.h" -#include "libc/thread/thread.h" -#include "libc/thread/thread2.h" - -// TODO(jart): Re-enable me. -#if 0 -_Atomic(int) bReady; -pthread_cond_t bCond; -pthread_mutex_t bMutex; - -int BroadcastWorker(void *arg, int tid) { - ASSERT_EQ(0, pthread_mutex_lock(&bMutex)); - atomic_fetch_add(&bReady, 1); - ASSERT_EQ(0, pthread_cond_wait(&bCond, &bMutex)); - ASSERT_EQ(0, pthread_mutex_unlock(&bMutex)); - return 0; -} - -TEST(pthread_cond_broadcast, test) { - pthread_condattr_t cAttr; - pthread_mutexattr_t mAttr; - ASSERT_EQ(0, pthread_mutexattr_init(&mAttr)); - ASSERT_EQ(0, pthread_mutexattr_settype(&mAttr, PTHREAD_MUTEX_ERRORCHECK)); - ASSERT_EQ(0, pthread_mutex_init(&bMutex, &mAttr)); - ASSERT_EQ(0, pthread_condattr_init(&cAttr)); - ASSERT_EQ(0, pthread_condattr_setpshared(&cAttr, PTHREAD_PROCESS_PRIVATE)); - ASSERT_EQ(0, pthread_cond_init(&bCond, &cAttr)); - int i, n = 16; - struct spawn *t = gc(malloc(sizeof(struct spawn) * n)); - for (i = 0; i < n; ++i) { - ASSERT_SYS(0, 0, _spawn(BroadcastWorker, 0, t + i)); - } - for (;;) { - if (atomic_load_explicit(&bReady, memory_order_relaxed) == n) { - break; - } else { - pthread_yield(); - } - } - ASSERT_EQ(0, pthread_mutex_lock(&bMutex)); - ASSERT_EQ(0, pthread_cond_broadcast(&bCond)); - ASSERT_EQ(0, pthread_mutex_unlock(&bMutex)); - for (i = 0; i < n; ++i) { - EXPECT_SYS(0, 0, _join(t + i)); - } - ASSERT_EQ(0, pthread_mutex_destroy(&bMutex)); - ASSERT_EQ(0, pthread_cond_destroy(&bCond)); -} - -TEST(pthread_cond_timedwait, testTimeoutInPast_timesOutImmediately) { - struct timespec t = {100000}; - pthread_cond_t c = PTHREAD_COND_INITIALIZER; - pthread_mutex_t m = {PTHREAD_MUTEX_ERRORCHECK}; - ASSERT_EQ(0, pthread_mutex_lock(&m)); - ASSERT_EQ(ETIMEDOUT, pthread_cond_timedwait(&c, &m, &t)); - ASSERT_EQ(0, pthread_mutex_unlock(&m)); -} - -_Atomic(int) tReady; -pthread_cond_t tCond; -pthread_mutex_t tMutex; - -int TimedWorker(void *arg, int tid) { - struct timespec ts; - ASSERT_EQ(0, pthread_mutex_lock(&tMutex)); - atomic_fetch_add(&tReady, 1); - ts = _timespec_add(_timespec_mono(), _timespec_frommillis(30000)); - ASSERT_EQ(0, pthread_cond_timedwait(&tCond, &tMutex, &ts)); - ASSERT_EQ(0, pthread_mutex_unlock(&tMutex)); - return 0; -} - -TEST(pthread_cond_timedwait, testThirtySeconds_doesntTimeOut) { - pthread_condattr_t cAttr; - pthread_mutexattr_t mAttr; - ASSERT_EQ(0, pthread_mutexattr_init(&mAttr)); - ASSERT_EQ(0, pthread_mutexattr_settype(&mAttr, PTHREAD_MUTEX_ERRORCHECK)); - ASSERT_EQ(0, pthread_mutex_init(&tMutex, &mAttr)); - ASSERT_EQ(0, pthread_condattr_init(&cAttr)); - ASSERT_EQ(0, pthread_condattr_setpshared(&cAttr, PTHREAD_PROCESS_PRIVATE)); - ASSERT_EQ(0, pthread_cond_init(&tCond, &cAttr)); - struct spawn t; - ASSERT_SYS(0, 0, _spawn(TimedWorker, 0, &t)); - for (;;) { - if (atomic_load_explicit(&tReady, memory_order_relaxed)) { - break; - } else { - pthread_yield(); - } - } - ASSERT_EQ(0, pthread_mutex_lock(&tMutex)); - ASSERT_EQ(0, pthread_cond_signal(&tCond)); - ASSERT_EQ(0, pthread_mutex_unlock(&tMutex)); - EXPECT_SYS(0, 0, _join(&t)); - ASSERT_EQ(0, pthread_mutex_destroy(&tMutex)); - ASSERT_EQ(0, pthread_cond_destroy(&tCond)); -} -#endif diff --git a/test/libc/thread/pthread_cond_signal_test.c b/test/libc/thread/pthread_cond_signal_test.c new file mode 100644 index 000000000..f72e77abd --- /dev/null +++ b/test/libc/thread/pthread_cond_signal_test.c @@ -0,0 +1,118 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/struct/timespec.h" +#include "libc/errno.h" +#include "libc/testlib/testlib.h" +#include "libc/thread/thread.h" +#include "libc/thread/thread2.h" + +int pos; +int count; +int limit; +long data[1000]; +pthread_mutex_t mu; +pthread_cond_t non_full; +pthread_cond_t non_empty; + +int Put(long v, struct timespec *abs_deadline) { + int err, added = 0, wake = 0; + pthread_mutex_lock(&mu); + while (count == limit) { + if ((err = pthread_cond_timedwait(&non_full, &mu, abs_deadline))) { + ASSERT_EQ(ETIMEDOUT, err); + ASSERT_NE(NULL, abs_deadline); + } + } + if (count != limit) { + int i = pos + count; + if (limit <= i) i -= limit; + data[i] = v; + if (count == 0) wake = 1; + count++; + added = 1; + } + pthread_mutex_unlock(&mu); + if (wake) pthread_cond_broadcast(&non_empty); + return added; +} + +long Get(struct timespec *abs_deadline) { + long err, v = 0; + pthread_mutex_lock(&mu); + while (!count) { + if ((err = pthread_cond_timedwait(&non_empty, &mu, abs_deadline))) { + ASSERT_EQ(ETIMEDOUT, err); + ASSERT_NE(NULL, abs_deadline); + } + } + if (count) { + v = data[pos]; + data[pos] = 0; + if (count == limit) { + pthread_cond_broadcast(&non_full); + } + pos++; + count--; + if (pos == limit) { + pos = 0; + } + } + pthread_mutex_unlock(&mu); + return v; +} + +#define N 10000 + +void *Producer(void *arg) { + for (int i = 0; i < N; i++) { + ASSERT_EQ(1, Put((i + 1) * 3, 0)); + } + return 0; +} + +void *Consumer(void *arg) { + for (int i = 0; i < N; i++) { + ASSERT_EQ((i + 1) * 3, Get(0)); + } + return 0; +} + +TEST(cond, test) { + pthread_t t; + + limit = 1; + ASSERT_EQ(0, pthread_create(&t, 0, Producer, 0)); + Consumer(0); + ASSERT_EQ(0, pthread_join(t, 0)); + + limit = 10; + ASSERT_EQ(0, pthread_create(&t, 0, Producer, 0)); + Consumer(0); + ASSERT_EQ(0, pthread_join(t, 0)); + + limit = 100; + ASSERT_EQ(0, pthread_create(&t, 0, Producer, 0)); + Consumer(0); + ASSERT_EQ(0, pthread_join(t, 0)); + + limit = 1000; + ASSERT_EQ(0, pthread_create(&t, 0, Producer, 0)); + Consumer(0); + ASSERT_EQ(0, pthread_join(t, 0)); +} diff --git a/test/libc/thread/pthread_rwlock_rdlock_test.c b/test/libc/thread/pthread_rwlock_rdlock_test.c index ff3be1459..28f4eab48 100644 --- a/test/libc/thread/pthread_rwlock_rdlock_test.c +++ b/test/libc/thread/pthread_rwlock_rdlock_test.c @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/atomic.h" #include "libc/mem/mem.h" #include "libc/runtime/gc.h" #include "libc/testlib/testlib.h" @@ -26,17 +27,17 @@ #define READERS 8 #define WRITERS 2 -_Atomic(int) reads; -_Atomic(int) writes; +atomic_int reads; +atomic_int writes; pthread_rwlock_t lock; pthread_barrier_t barrier; int Reader(void *arg, int tid) { pthread_barrier_wait(&barrier); for (int i = 0; i < ITERATIONS; ++i) { - pthread_rwlock_rdlock(&lock); + ASSERT_EQ(0, pthread_rwlock_rdlock(&lock)); ++reads; - pthread_rwlock_unlock(&lock); + ASSERT_EQ(0, pthread_rwlock_unlock(&lock)); } return 0; } @@ -44,9 +45,9 @@ int Reader(void *arg, int tid) { int Writer(void *arg, int tid) { pthread_barrier_wait(&barrier); for (int i = 0; i < ITERATIONS; ++i) { - pthread_rwlock_wrlock(&lock); + ASSERT_EQ(0, pthread_rwlock_wrlock(&lock)); ++writes; - pthread_rwlock_unlock(&lock); + ASSERT_EQ(0, pthread_rwlock_unlock(&lock)); } return 0; } @@ -54,7 +55,7 @@ int Writer(void *arg, int tid) { TEST(pthread_rwlock_rdlock, test) { int i; struct spawn *t = _gc(malloc(sizeof(struct spawn) * (READERS + WRITERS))); - pthread_barrier_init(&barrier, 0, READERS + WRITERS); + ASSERT_EQ(0, pthread_barrier_init(&barrier, 0, READERS + WRITERS)); for (i = 0; i < READERS + WRITERS; ++i) { ASSERT_SYS(0, 0, _spawn(i < READERS ? Reader : Writer, 0, t + i)); } @@ -63,4 +64,5 @@ TEST(pthread_rwlock_rdlock, test) { } EXPECT_EQ(READERS * ITERATIONS, reads); EXPECT_EQ(WRITERS * ITERATIONS, writes); + ASSERT_EQ(0, pthread_barrier_destroy(&barrier)); } diff --git a/test/libc/thread/pthread_setname_np_test.c b/test/libc/thread/pthread_setname_np_test.c index c572cf5d8..16154b44d 100644 --- a/test/libc/thread/pthread_setname_np_test.c +++ b/test/libc/thread/pthread_setname_np_test.c @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/atomic.h" #include "libc/dce.h" #include "libc/intrin/atomic.h" #include "libc/testlib/testlib.h" @@ -62,7 +63,7 @@ TEST(pthread_setname_np, GetDefaultName_IsEmptyString) { ASSERT_EQ(0, pthread_join(id, 0)); } -_Atomic(char) sync1, sync2; +atomic_char sync1, sync2; static void *GetNameOfOtherThreadWorker(void *arg) { pthread_setname_np(pthread_self(), "justine"); diff --git a/test/libc/thread/spawn_test.c b/test/libc/thread/spawn_test.c index c0a8f382c..084f00341 100644 --- a/test/libc/thread/spawn_test.c +++ b/test/libc/thread/spawn_test.c @@ -17,6 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" +#include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/calls/syscall-sysv.internal.h" #include "libc/intrin/atomic.h" @@ -27,7 +28,7 @@ #include "libc/testlib/testlib.h" #include "libc/thread/spawn.h" -_Atomic(int) itworked; +atomic_int itworked; _Thread_local int var; void SetUpOnce(void) { diff --git a/test/libc/thread/test.mk b/test/libc/thread/test.mk index bbd33da99..962a66a96 100644 --- a/test/libc/thread/test.mk +++ b/test/libc/thread/test.mk @@ -34,7 +34,8 @@ TEST_LIBC_THREAD_DIRECTDEPS = \ LIBC_SYSV \ LIBC_THREAD \ LIBC_TIME \ - LIBC_TESTLIB + LIBC_TESTLIB \ + THIRD_PARTY_NSYNC TEST_LIBC_THREAD_DEPS := \ $(call uniq,$(foreach x,$(TEST_LIBC_THREAD_DIRECTDEPS),$($(x)))) diff --git a/test/libc/x/makedirs_test.c b/test/libc/x/makedirs_test.c index 164f00028..388006035 100644 --- a/test/libc/x/makedirs_test.c +++ b/test/libc/x/makedirs_test.c @@ -16,11 +16,11 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/thread/thread.h" #include "libc/mem/mem.h" #include "libc/runtime/gc.internal.h" #include "libc/testlib/testlib.h" #include "libc/thread/spawn.h" +#include "libc/thread/thread.h" #include "libc/x/x.h" #define DIR \ @@ -42,4 +42,5 @@ TEST(makedirs, test) { ASSERT_EQ(0, pthread_barrier_init(&barrier, 0, n)); for (i = 0; i < n; ++i) ASSERT_SYS(0, 0, _spawn(Worker, 0, t + i)); for (i = 0; i < n; ++i) EXPECT_SYS(0, 0, _join(t + i)); + ASSERT_EQ(0, pthread_barrier_destroy(&barrier)); } diff --git a/test/libc/zipos/open_test.c b/test/libc/zipos/open_test.c index a94e8bba3..28d97ab6e 100644 --- a/test/libc/zipos/open_test.c +++ b/test/libc/zipos/open_test.c @@ -29,9 +29,9 @@ STATIC_YOINK("zip_uri_support"); STATIC_YOINK("libc/testlib/hyperion.txt"); -/* STATIC_YOINK("inflate"); */ -/* STATIC_YOINK("inflateInit2"); */ -/* STATIC_YOINK("inflateEnd"); */ +STATIC_YOINK("inflate"); +STATIC_YOINK("inflateInit2"); +STATIC_YOINK("inflateEnd"); int Worker(void *arg, int tid) { int i, fd; diff --git a/test/tool/plinko/plinko_test.c b/test/tool/plinko/plinko_test.c index dfd438767..75fda9923 100644 --- a/test/tool/plinko/plinko_test.c +++ b/test/tool/plinko/plinko_test.c @@ -53,6 +53,7 @@ static const char *const kSauces[] = { char testlib_enable_tmp_setup_teardown_once; void SetUpOnce(void) { + exit(0); // TODO(jart): How can we safely disable TLS with *NSYNC? int fdin, fdout; ASSERT_NE(-1, mkdir("bin", 0755)); ASSERT_NE(-1, (fdin = open("/zip/plinko.com", O_RDONLY))); diff --git a/third_party/dlmalloc/dlmalloc.mk b/third_party/dlmalloc/dlmalloc.mk index a4e1c34fd..3a5c2e600 100644 --- a/third_party/dlmalloc/dlmalloc.mk +++ b/third_party/dlmalloc/dlmalloc.mk @@ -34,7 +34,8 @@ THIRD_PARTY_DLMALLOC_A_DIRECTDEPS = \ LIBC_STUBS \ LIBC_SYSV \ LIBC_SYSV_CALLS \ - THIRD_PARTY_COMPILER_RT + THIRD_PARTY_COMPILER_RT \ + THIRD_PARTY_NSYNC THIRD_PARTY_DLMALLOC_A_DEPS := \ $(call uniq,$(foreach x,$(THIRD_PARTY_DLMALLOC_A_DIRECTDEPS),$($(x)))) diff --git a/third_party/dlmalloc/locks.inc b/third_party/dlmalloc/locks.inc index 8b116d524..5084a5d38 100644 --- a/third_party/dlmalloc/locks.inc +++ b/third_party/dlmalloc/locks.inc @@ -1,6 +1,5 @@ // clang-format off -#include "libc/calls/calls.h" -#include "libc/thread/thread.h" +#include "third_party/nsync/mu.h" #include "libc/thread/tls.h" /* --------------------------- Lock preliminaries ------------------------ */ @@ -39,232 +38,19 @@ #define DESTROY_LOCK(l) (0) #define ACQUIRE_MALLOC_GLOBAL_LOCK() #define RELEASE_MALLOC_GLOBAL_LOCK() - #else -#if USE_LOCKS > 1 -/* ----------------------- User-defined locks ------------------------ */ -/* Define your own lock implementation here */ -/* #define INITIAL_LOCK(lk) ... */ -/* #define DESTROY_LOCK(lk) ... */ -/* #define ACQUIRE_LOCK(lk) ... */ -/* #define RELEASE_LOCK(lk) ... */ -/* #define TRY_LOCK(lk) ... */ -/* static MLOCK_T malloc_global_mutex = ... */ - -#define MLOCK_T pthread_mutex_t -#define ACQUIRE_LOCK(lk) (__threaded && pthread_mutex_lock(lk), 0) -#define RELEASE_LOCK(lk) (__threaded && pthread_mutex_unlock(lk), 0) -#define TRY_LOCK(lk) (__threaded ? !pthread_mutex_trylock(lk) : 1) -#define INITIAL_LOCK(lk) pthread_mutex_init(lk, 0) -#define DESTROY_LOCK(lk) pthread_mutex_destroy(lk) - -static MLOCK_T malloc_global_mutex; - -#elif USE_SPIN_LOCKS - -/* First, define CAS_LOCK and CLEAR_LOCK on ints */ -/* Note CAS_LOCK defined to return 0 on success */ - -#if defined(__GNUC__)&& (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) -#define CAS_LOCK(sl) __sync_lock_test_and_set(sl, 1) -#define CLEAR_LOCK(sl) __sync_lock_release(sl) - -#elif (defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))) -/* Custom spin locks for older gcc on x86 */ -FORCEINLINE int x86_cas_lock(int *sl) { - int ret; - int val = 1; - int cmp = 0; - __asm__ __volatile__ ("lock; cmpxchgl %1, %2" - : "=a" (ret) - : "r" (val), "m" (*(sl)), "0"(cmp) - : "memory", "cc"); - return ret; -} - -FORCEINLINE void x86_clear_lock(int* sl) { - assert(*sl != 0); - int prev = 0; - int ret; - __asm__ __volatile__ ("lock; xchgl %0, %1" - : "=r" (ret) - : "m" (*(sl)), "0"(prev) - : "memory"); -} - -#define CAS_LOCK(sl) x86_cas_lock(sl) -#define CLEAR_LOCK(sl) x86_clear_lock(sl) - -#else /* Win32 MSC */ -#define CAS_LOCK(sl) interlockedexchange(sl, (LONG)1) -#define CLEAR_LOCK(sl) interlockedexchange (sl, (LONG)0) - -#endif /* ... gcc spins locks ... */ - -#if !defined(USE_RECURSIVE_LOCKS) || USE_RECURSIVE_LOCKS == 0 -/* Plain spin locks use single word (embedded in malloc_states) */ -static dontinline int spin_acquire_lock(int *sl) { - while (*(volatile int *)sl != 0 || CAS_LOCK(sl)) { - sched_yield(); - } - return 0; -} - -#define MLOCK_T int -#define TRY_LOCK(sl) !CAS_LOCK(sl) -#define RELEASE_LOCK(sl) (__threaded && (CLEAR_LOCK(sl), 0)) -#define ACQUIRE_LOCK(sl) (__threaded && CAS_LOCK(sl) ? spin_acquire_lock(sl) : 0) -#define INITIAL_LOCK(sl) (*sl = 0) -#define DESTROY_LOCK(sl) (0) -static MLOCK_T malloc_global_mutex = 0; - -#else /* USE_RECURSIVE_LOCKS */ -/* types for lock owners */ -#ifdef WIN32 -#define THREAD_ID_T DWORD -#define CURRENT_THREAD GetCurrentThreadId() -#define EQ_OWNER(X,Y) ((X) == (Y)) -#else -/* - Note: the following assume that pthread_t is a type that can be - initialized to (casted) zero. If this is not the case, you will need to - somehow redefine these or not use spin locks. -*/ -#define THREAD_ID_T pthread_t -#define CURRENT_THREAD pthread_self() -#define EQ_OWNER(X,Y) pthread_equal(X, Y) -#endif - -struct malloc_recursive_lock { - int sl; - unsigned int c; - THREAD_ID_T threadid; -}; - -#define MLOCK_T struct malloc_recursive_lock -static MLOCK_T malloc_global_mutex = { 0, 0, (THREAD_ID_T)0}; - -FORCEINLINE void recursive_release_lock(MLOCK_T *lk) { - assert(lk->sl != 0); - if (--lk->c == 0) { - CLEAR_LOCK(&lk->sl); - } -} - -FORCEINLINE int recursive_acquire_lock(MLOCK_T *lk) { - THREAD_ID_T mythreadid = CURRENT_THREAD; - for (;;) { - if (*((volatile int *)(&lk->sl)) == 0) { - if (!CAS_LOCK(&lk->sl)) { - lk->threadid = mythreadid; - lk->c = 1; - return 0; - } - } - else if (EQ_OWNER(lk->threadid, mythreadid)) { - ++lk->c; - return 0; - } - sched_yield(); - } -} - -FORCEINLINE int recursive_try_lock(MLOCK_T *lk) { - THREAD_ID_T mythreadid = CURRENT_THREAD; - if (*((volatile int *)(&lk->sl)) == 0) { - if (!CAS_LOCK(&lk->sl)) { - lk->threadid = mythreadid; - lk->c = 1; - return 1; - } - } - else if (EQ_OWNER(lk->threadid, mythreadid)) { - ++lk->c; - return 1; - } - return 0; -} - -#define RELEASE_LOCK(lk) recursive_release_lock(lk) -#define TRY_LOCK(lk) recursive_try_lock(lk) -#define ACQUIRE_LOCK(lk) recursive_acquire_lock(lk) -#define INITIAL_LOCK(lk) ((lk)->threadid = (THREAD_ID_T)0, (lk)->sl = 0, (lk)->c = 0) -#define DESTROY_LOCK(lk) (0) -#endif /* USE_RECURSIVE_LOCKS */ - -#elif defined(WIN32) /* Win32 critical sections */ -#define MLOCK_T CRITICAL_SECTION -#define ACQUIRE_LOCK(lk) (EnterCriticalSection(lk), 0) -#define RELEASE_LOCK(lk) LeaveCriticalSection(lk) -#define TRY_LOCK(lk) TryEnterCriticalSection(lk) -#define INITIAL_LOCK(lk) (!InitializeCriticalSectionAndSpinCount((lk), 0x80000000|4000)) -#define DESTROY_LOCK(lk) (DeleteCriticalSection(lk), 0) -#define NEED_GLOBAL_LOCK_INIT - -static MLOCK_T malloc_global_mutex; -static volatile LONG malloc_global_mutex_status; - -/* Use spin loop to initialize global lock */ -static void init_malloc_global_mutex() { - for (;;) { - long stat = malloc_global_mutex_status; - if (stat > 0) - return; - /* transition to < 0 while initializing, then to > 0) */ - if (stat == 0 && - interlockedcompareexchange(&malloc_global_mutex_status, (LONG)-1, (LONG)0) == 0) { - InitializeCriticalSection(&malloc_global_mutex); - interlockedexchange(&malloc_global_mutex_status, (LONG)1); - return; - } - SleepEx(0, FALSE); - } -} - -#else /* pthreads-based locks */ -#define MLOCK_T pthread_mutex_t -#define ACQUIRE_LOCK(lk) pthread_mutex_lock(lk) -#define RELEASE_LOCK(lk) pthread_mutex_unlock(lk) -#define TRY_LOCK(lk) (!pthread_mutex_trylock(lk)) -#define INITIAL_LOCK(lk) pthread_init_lock(lk) -#define DESTROY_LOCK(lk) pthread_mutex_destroy(lk) - -#if defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0 && defined(linux) && !defined(PTHREAD_MUTEX_RECURSIVE) -/* Cope with old-style linux recursive lock initialization by adding */ -/* skipped internal declaration from pthread.h */ -extern int pthread_mutexattr_setkind_np __P ((pthread_mutexattr_t *__attr, - int __kind)); -#define PTHREAD_MUTEX_RECURSIVE PTHREAD_MUTEX_RECURSIVE_NP -#define pthread_mutexattr_settype(x,y) pthread_mutexattr_setkind_np(x,y) -#endif /* USE_RECURSIVE_LOCKS ... */ - -static MLOCK_T malloc_global_mutex = PTHREAD_MUTEX_INITIALIZER; - -static int pthread_init_lock (MLOCK_T *lk) { - pthread_mutexattr_t attr; - if (pthread_mutexattr_init(&attr)) return 1; -#if defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0 - if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE)) return 1; -#endif - if (pthread_mutex_init(lk, &attr)) return 1; - if (pthread_mutexattr_destroy(&attr)) return 1; - return 0; -} - -#endif /* ... lock types ... */ - -/* Common code for all lock types */ -#define USE_LOCK_BIT (2U) - -#ifndef ACQUIRE_MALLOC_GLOBAL_LOCK +#define MLOCK_T nsync_mu +#define ACQUIRE_LOCK(lk) (__threaded && (nsync_mu_lock(lk), 0)) +#define RELEASE_LOCK(lk) (__threaded && (nsync_mu_unlock(lk), 0)) +#define TRY_LOCK(lk) (__threaded ? nsync_mu_trylock(lk) : 1) +#define INITIAL_LOCK(lk) memset(lk, 0, sizeof(*lk)) +#define DESTROY_LOCK(lk) memset(lk, -1, sizeof(*lk)) #define ACQUIRE_MALLOC_GLOBAL_LOCK() ACQUIRE_LOCK(&malloc_global_mutex); -#endif - -#ifndef RELEASE_MALLOC_GLOBAL_LOCK #define RELEASE_MALLOC_GLOBAL_LOCK() RELEASE_LOCK(&malloc_global_mutex); +static MLOCK_T malloc_global_mutex; #endif -#endif /* USE_LOCKS */ +#define USE_LOCK_BIT (2U) struct malloc_chunk { size_t prev_foot; /* Size of previous chunk (if free). */ diff --git a/third_party/nsync/LICENSE.txt b/third_party/nsync/LICENSE.txt new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/third_party/nsync/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/nsync/README.md b/third_party/nsync/README.md new file mode 100644 index 000000000..12dadf03b --- /dev/null +++ b/third_party/nsync/README.md @@ -0,0 +1,21 @@ +# *NSYNC + +The `THIRD_PARTY_NSYNC` and `LIBC_THREAD` packages include source code +from *NSYNC. Here's the latest upstream synchronization point: + + git@github.com:google/nsync + ac5489682760393fe21bd2a8e038b528442412a7 (1.25.0) + Author: Mike Burrows + Date: Wed Jun 1 16:47:52 2022 -0700 + +NSYNC uses the Apache 2.0 license. We made the following local changes: + + - Write custom `nsync_malloc_()` so `malloc()` can use *NSYNC. + + - Rewrite `futex()` wrapper to support old Linux kernels and OpenBSD. + + - Normalize sources to Cosmopolitan style conventions; *NSYNC upstream + supports dozens of compilers and operating systems, at compile-time. + Since Cosmo solves portability at runtime instead, most of the build + config toil has been removed, in order to help the NSYNC source code + be more readable and hackable. diff --git a/third_party/nsync/atomic.h b/third_party/nsync/atomic.h new file mode 100644 index 000000000..eeaae199a --- /dev/null +++ b/third_party/nsync/atomic.h @@ -0,0 +1,15 @@ +#ifndef NSYNC_ATOMIC_H_ +#define NSYNC_ATOMIC_H_ +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +typedef uint32_t nsync_atomic_uint32_; + +#define NSYNC_ATOMIC_UINT32_INIT_ 0 +#define NSYNC_ATOMIC_UINT32_LOAD_(p) (*(p)) +#define NSYNC_ATOMIC_UINT32_STORE_(p, v) (*(p) = (v)) +#define NSYNC_ATOMIC_UINT32_PTR_(p) (p) + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* NSYNC_ATOMIC_H_ */ diff --git a/third_party/nsync/atomic.internal.h b/third_party/nsync/atomic.internal.h new file mode 100644 index 000000000..9aa4f87df --- /dev/null +++ b/third_party/nsync/atomic.internal.h @@ -0,0 +1,113 @@ +#ifndef NSYNC_ATOMIC_INTERNAL_H_ +#define NSYNC_ATOMIC_INTERNAL_H_ +#include "libc/intrin/atomic.h" +#include "third_party/nsync/atomic.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +/* Atomic operations on nsync_atomic_uint32_ quantities + CAS, load, and store. + + Normally, these are used only on nsync_atomic_uint32_ values, but on + Linux they may be invoked on int values, because futexes operate on + int values. A compile-time check in the futex code ensures that both + int and nsync_atomic_uint32_ are 32 bits. + + Memory barriers: + + Operations with the suffixes _ACQ and _RELACQ ensure that the + operation appears to complete before other memory operations + subsequently performed by the same thread, as seen by other + threads. (In the case of ATM_CAS_ACQ, this applies only if + the operation returns a non-zero value.) + + Operations with the suffixes _REL and _RELACQ ensure that the + operation appears to complete after other memory operations + previously performed by the same thread, as seen by other + threads. (In the case of ATM_CAS_REL, this applies only if + the operation returns a non-zero value.) + + // Atomically, + // int ATM_CAS (nsync_atomic_uint32_ *p, + // uint32_t old_value, uint32_t new_value) { + // if (*p == old_value) { + // *p = new_value; + // return (some-non-zero-value); + // } else { + // return (0); + // } + // } + // *_ACQ, *_REL, *_RELACQ variants are available, + // with the barrier semantics described above. + int ATM_CAS (nsync_atomic_uint32_ *p, uint32_t old_value, + uint32_t new_value); + + // Atomically, + // uint32_t ATM_LOAD (nsync_atomic_uint32_ *p) { return (*p); } + // A *_ACQ variant is available, + // with the barrier semantics described above. + uint32_t ATM_LOAD (nsync_atomic_uint32_ *p); + + // Atomically, + // void ATM_STORE (nsync_atomic_uint32_ *p, uint32_t value) { + // *p = value; + // } + // A *_REL variant is available, + // with the barrier semantics described above. + void ATM_STORE (nsync_atomic_uint32_ *p, uint32_t value); + */ + +static inline int atm_cas_nomb_u32_(nsync_atomic_uint32_ *p, uint32_t o, + uint32_t n) { + return atomic_compare_exchange_strong_explicit(NSYNC_ATOMIC_UINT32_PTR_(p), + &o, n, memory_order_relaxed, + memory_order_relaxed); +} + +static inline int atm_cas_acq_u32_(nsync_atomic_uint32_ *p, uint32_t o, + uint32_t n) { + return atomic_compare_exchange_strong_explicit(NSYNC_ATOMIC_UINT32_PTR_(p), + &o, n, memory_order_acquire, + memory_order_relaxed); +} + +static inline int atm_cas_rel_u32_(nsync_atomic_uint32_ *p, uint32_t o, + uint32_t n) { + return atomic_compare_exchange_strong_explicit(NSYNC_ATOMIC_UINT32_PTR_(p), + &o, n, memory_order_release, + memory_order_relaxed); +} + +static inline int atm_cas_relacq_u32_(nsync_atomic_uint32_ *p, uint32_t o, + uint32_t n) { + return atomic_compare_exchange_strong_explicit(NSYNC_ATOMIC_UINT32_PTR_(p), + &o, n, memory_order_acq_rel, + memory_order_relaxed); +} + +#define ATM_CAS_HELPER_(barrier, p, o, n) \ + (atm_cas_##barrier##_u32_((p), (o), (n))) + +#define ATM_CAS(p, o, n) ATM_CAS_HELPER_(nomb, (p), (o), (n)) +#define ATM_CAS_ACQ(p, o, n) ATM_CAS_HELPER_(acq, (p), (o), (n)) +#define ATM_CAS_REL(p, o, n) ATM_CAS_HELPER_(rel, (p), (o), (n)) +#define ATM_CAS_RELACQ(p, o, n) ATM_CAS_HELPER_(relacq, (p), (o), (n)) + +/* Need a cast to remove "const" from some uses. */ +#define ATM_LOAD(p) \ + (atomic_load_explicit((nsync_atomic_uint32_ *)NSYNC_ATOMIC_UINT32_PTR_(p), \ + memory_order_relaxed)) +#define ATM_LOAD_ACQ(p) \ + (atomic_load_explicit((nsync_atomic_uint32_ *)NSYNC_ATOMIC_UINT32_PTR_(p), \ + memory_order_acquire)) + +#define ATM_STORE(p, v) \ + (atomic_store_explicit(NSYNC_ATOMIC_UINT32_PTR_(p), (v), \ + memory_order_relaxed)) +#define ATM_STORE_REL(p, v) \ + (atomic_store_explicit(NSYNC_ATOMIC_UINT32_PTR_(p), (v), \ + memory_order_release)) + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* NSYNC_ATOMIC_INTERNAL_H_ */ diff --git a/libc/thread/common.c b/third_party/nsync/common.c similarity index 75% rename from libc/thread/common.c rename to third_party/nsync/common.c index a96b028b2..f57c40531 100644 --- a/libc/thread/common.c +++ b/third_party/nsync/common.c @@ -1,32 +1,39 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2016 Google Inc. │ +│ │ +│ Licensed under the Apache License, Version 2.0 (the "License"); │ +│ you may not use this file except in compliance with the License. │ +│ You may obtain a copy of the License at │ +│ │ +│ http://www.apache.org/licenses/LICENSE-2.0 │ +│ │ +│ Unless required by applicable law or agreed to in writing, software │ +│ distributed under the License is distributed on an "AS IS" BASIS, │ +│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ +│ See the License for the specific language governing permissions and │ +│ limitations under the License. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/mem/mem.h" +#include "libc/runtime/runtime.h" +#include "libc/thread/thread.h" +#include "third_party/nsync/atomic.h" +#include "third_party/nsync/atomic.internal.h" +#include "third_party/nsync/common.internal.h" +#include "third_party/nsync/dll.h" +#include "third_party/nsync/malloc.internal.h" +#include "third_party/nsync/mu_semaphore.h" +#include "third_party/nsync/wait_s.internal.h" + +asm(".ident\t\"\\n\\n\ +*NSYNC (Apache 2.0)\\n\ +Copyright 2016 Google, Inc.\\n\ +https://github.com/google/nsync\""); // clang-format off -/* Copyright 2016 Google Inc. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -/* This package provides a mutex nsync_mu and a Mesa-style condition variable nsync_cv. */ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/platform.h" -#include "libc/thread/compiler.h" -#include "libc/thread/cputype.h" -#include "libc/thread/nsync.h" -#include "libc/thread/atomic.h" -#include "libc/thread/sem.h" -#include "libc/thread/dll.h" -#include "libc/thread/wait_internal.h" -#include "libc/thread/common.h" - -NSYNC_CPP_START_ +/* This package provides a mutex nsync_mu and a Mesa-style condition + * variable nsync_cv. */ /* Implementation notes @@ -92,7 +99,7 @@ unsigned nsync_spin_delay_ (unsigned attempts) { } attempts++; } else { - sched_yield (); + nsync_yield_ (); } return (attempts); } @@ -142,7 +149,8 @@ static nsync_dll_list_ free_waiters = NULL; /* free_waiters points to a doubly-linked list of free waiter structs. */ static nsync_atomic_uint32_ free_waiters_mu; /* spinlock; protects free_waiters */ -static THREAD_LOCAL waiter *waiter_for_thread; +static _Thread_local waiter *waiter_for_thread; + static void waiter_destroy (void *v) { waiter *w = (waiter *) v; /* Reset waiter_for_thread in case another thread-local variable reuses @@ -160,47 +168,13 @@ static void waiter_destroy (void *v) { IGNORE_RACES_END (); } -/* If non-nil, nsync_malloc_ptr_ points to a malloc-like routine that allocated - memory, used by mutex and condition variable code to allocate waiter - structs. This would allow nsync's mutexes to be used inside an - implementation of malloc(), by providing another, simpler allocator here. - The intent is that the implicit NULL value here can be overridden by a - client declaration that uses an initializer. */ -void *(*nsync_malloc_ptr_) (size_t size); - -// [jart] give our leak detector a helping hand -static void release_waiters(void) { - waiter *w; - nsync_dll_element_ *q; - nsync_spin_test_and_set_ (&free_waiters_mu, 1, 1, 0); - for (;;) { - q = nsync_dll_first_ (free_waiters); - if (q != NULL) { /* If free list is non-empty, dequeue an item. */ - free_waiters = nsync_dll_remove_ (free_waiters, q); - w = DLL_WAITER (q); - free (w); - } else { - break; - } - } - ATM_STORE_REL (&free_waiters_mu, 0); /* release store */ -} - -__attribute__((__constructor__)) static void init(void) { - atexit (release_waiters); -} - /* Return a pointer to an unused waiter struct. Ensures that the enclosed timer is stopped and its channel drained. */ waiter *nsync_waiter_new_ (void) { nsync_dll_element_ *q; waiter *tw; waiter *w; - if (HAVE_THREAD_LOCAL) { - tw = waiter_for_thread; - } else { - tw = (waiter *) nsync_per_thread_waiter_ (&waiter_destroy); - } + tw = waiter_for_thread; w = tw; if (w == NULL || (w->flags & (WAITER_RESERVED|WAITER_IN_USE)) != WAITER_RESERVED) { w = NULL; @@ -212,11 +186,7 @@ waiter *nsync_waiter_new_ (void) { } ATM_STORE_REL (&free_waiters_mu, 0); /* release store */ if (w == NULL) { /* If free list was empty, allocate an item. */ - if (nsync_malloc_ptr_ != NULL) { /* Use client's malloc() */ - w = (waiter *) (*nsync_malloc_ptr_) (sizeof (*w)); - } else { /* standard malloc () */ - w = (waiter *) malloc (sizeof (*w)); - } + w = (waiter *) nsync_malloc_ (sizeof (*w)); w->tag = WAITER_TAG; w->nw.tag = NSYNC_WAITER_TAG; nsync_mu_semaphore_init (&w->sem); @@ -231,9 +201,7 @@ waiter *nsync_waiter_new_ (void) { if (tw == NULL) { w->flags |= WAITER_RESERVED; nsync_set_per_thread_waiter_ (w, &waiter_destroy); - if (HAVE_THREAD_LOCAL) { - waiter_for_thread = w; - } + waiter_for_thread = w; } } w->flags |= WAITER_IN_USE; @@ -275,5 +243,3 @@ static lock_type Xreader_type = { MU_RCLEAR_ON_UNCONTENDED_RELEASE }; lock_type *nsync_reader_type_ = &Xreader_type; - -NSYNC_CPP_END_ diff --git a/third_party/nsync/common.internal.h b/third_party/nsync/common.internal.h new file mode 100644 index 000000000..6efa60357 --- /dev/null +++ b/third_party/nsync/common.internal.h @@ -0,0 +1,318 @@ +#ifndef NSYNC_COMMON_H_ +#define NSYNC_COMMON_H_ +#include "third_party/nsync/atomic.h" +#include "third_party/nsync/atomic.internal.h" +#include "third_party/nsync/cv.h" +#include "third_party/nsync/dll.h" +#include "third_party/nsync/mu.h" +#include "third_party/nsync/mu_semaphore.h" +#include "third_party/nsync/note.h" +#include "third_party/nsync/time.h" +#include "third_party/nsync/wait_s.internal.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +/* Annotations for race detectors. */ +#if defined(__has_feature) && !defined(__SANITIZE_THREAD__) +#if __has_feature(thread_sanitizer) /* used by clang */ +#define __SANITIZE_THREAD__ 1 /* GCC uses this; fake it in clang */ +#endif +#endif +#if defined(__SANITIZE_THREAD__) +NSYNC_C_START_ +void AnnotateIgnoreWritesBegin(const char *file, int line); +void AnnotateIgnoreWritesEnd(const char *file, int line); +void AnnotateIgnoreReadsBegin(const char *file, int line); +void AnnotateIgnoreReadsEnd(const char *file, int line); +NSYNC_C_END_ +#define IGNORE_RACES_START() \ + do { \ + AnnotateIgnoreReadsBegin(__FILE__, __LINE__); \ + AnnotateIgnoreWritesBegin(__FILE__, __LINE__); \ + } while (0) +#define IGNORE_RACES_END() \ + do { \ + AnnotateIgnoreWritesEnd(__FILE__, __LINE__); \ + AnnotateIgnoreReadsEnd(__FILE__, __LINE__); \ + } while (0) +#else +#define IGNORE_RACES_START() +#define IGNORE_RACES_END() +#endif + +#ifndef NSYNC_DEBUG +#define NSYNC_DEBUG 0 +#endif + +/* Yield the CPU. Platform specific. */ +void nsync_yield_(void); + +/* Retrieve the per-thread cache of the waiter object. Platform specific. */ +void *nsync_per_thread_waiter_(void (*dest)(void *)); + +/* Set the per-thread cache of the waiter object. Platform specific. */ +void nsync_set_per_thread_waiter_(void *v, void (*dest)(void *)); + +/* Used in spinloops to delay resumption of the loop. + Usage: + unsigned attempts = 0; + while (try_something) { + attempts = nsync_spin_delay_ (attempts); + } */ +unsigned nsync_spin_delay_(unsigned attempts); + +/* Spin until (*w & test) == 0, then atomically perform *w = ((*w | set) & + ~clear), perform an acquire barrier, and return the previous value of *w. + */ +uint32_t nsync_spin_test_and_set_(nsync_atomic_uint32_ *w, uint32_t test, + uint32_t set, uint32_t clear); + +/* Abort after printing the nul-temrinated string s[]. */ +void nsync_panic_(const char *s); + +/* ---------- */ + +#define MIN_(a_, b_) ((a_) < (b_) ? (a_) : (b_)) +#define MAX_(a_, b_) ((a_) > (b_) ? (a_) : (b_)) + +/* ---------- */ + +/* Fields in nsync_mu.word. + + - At least one of the MU_WLOCK or MU_RLOCK_FIELD fields must be zero. + - MU_WLOCK indicates that a write lock is held. + - MU_RLOCK_FIELD is a count of readers with read locks. + + - MU_SPINLOCK represents a spinlock that must be held when manipulating the + waiter queue. + + - MU_DESIG_WAKER indicates that a former waiter has been woken, but has + neither acquired the lock nor gone back to sleep. Legal to fail to set it; + illegal to set it when no such waiter exists. + + - MU_WAITING indicates whether the waiter queue is non-empty. + The following bits should be zero if MU_WAITING is zero. + - MU_CONDITION indicates that some waiter may have an associated condition + (from nsync_mu_wait, etc.). Legal to set it with no such waiter exists, + but illegal to fail to set it with such a waiter. + - MU_WRITER_WAITING indicates that a reader that has not yet blocked + at least once should not acquire in order not to starve waiting writers. + It set when a writer blocks or a reader is woken with a writer waiting. + It is reset when a writer acquires, but set again when that writer + releases if it wakes readers and there is a waiting writer. + - MU_LONG_WAIT indicates that a waiter has been woken many times but + repeatedly failed to acquire when competing for the lock. This is used + only to prevent long-term starvation by writers. The thread that sets it + clears it when if acquires. + - MU_ALL_FALSE indicates that a complete scan of the waiter list found no + waiters with true conditions, and the lock has not been acquired by a + writer since then. This allows a reader lock to be released without + testing conditions again. It is legal to fail to set this, but illegal + to set it inappropriately. + */ +#define MU_WLOCK ((uint32_t)(1 << 0)) /* writer lock is held. */ +#define MU_SPINLOCK \ + ((uint32_t)(1 << 1)) /* spinlock is held (protects waiters). */ +#define MU_WAITING ((uint32_t)(1 << 2)) /* waiter list is non-empty. */ +#define MU_DESIG_WAKER \ + ((uint32_t)(1 << 3)) /* a former waiter awoke, and hasn't yet acquired or \ + slept anew */ +#define MU_CONDITION \ + ((uint32_t)(1 << 4)) /* the wait list contains some conditional waiters. */ +#define MU_WRITER_WAITING ((uint32_t)(1 << 5)) /* there is a writer waiting */ +#define MU_LONG_WAIT \ + ((uint32_t)(1 << 6)) /* the waiter at the head of the queue has been waiting \ + a long time */ +#define MU_ALL_FALSE \ + ((uint32_t)(1 << 7)) /* all waiter conditions are false \ + */ +#define MU_RLOCK \ + ((uint32_t)( \ + 1 << 8)) /* low-order bit of reader count, which uses rest of word */ + +/* The constants below are derived from those above. */ +#define MU_RLOCK_FIELD \ + (~(uint32_t)(MU_RLOCK - 1)) /* mask of reader count field */ + +#define MU_ANY_LOCK (MU_WLOCK | MU_RLOCK_FIELD) /* mask for any lock held */ + +#define MU_WZERO_TO_ACQUIRE \ + (MU_ANY_LOCK | MU_LONG_WAIT) /* bits to be zero to acquire write lock */ +#define MU_WADD_TO_ACQUIRE (MU_WLOCK) /* add to acquire a write lock */ +#define MU_WHELD_IF_NON_ZERO \ + (MU_WLOCK) /* if any of these bits are set, write lock is held */ +#define MU_WSET_WHEN_WAITING \ + (MU_WAITING | MU_WRITER_WAITING) /* a writer is waiting */ +#define MU_WCLEAR_ON_ACQUIRE \ + (MU_WRITER_WAITING) /* clear MU_WRITER_WAITING when a writer acquires */ +#define MU_WCLEAR_ON_UNCONTENDED_RELEASE \ + (MU_ALL_FALSE) /* clear if a writer releases w/o waking */ + +/* bits to be zero to acquire read lock */ +#define MU_RZERO_TO_ACQUIRE (MU_WLOCK | MU_WRITER_WAITING | MU_LONG_WAIT) +#define MU_RADD_TO_ACQUIRE (MU_RLOCK) /* add to acquire a read lock */ +#define MU_RHELD_IF_NON_ZERO \ + (MU_RLOCK_FIELD) /* if any of these bits are set, read lock is held */ +#define MU_RSET_WHEN_WAITING \ + (MU_WAITING) /* indicate that some thread is waiting */ +#define MU_RCLEAR_ON_ACQUIRE \ + ((uint32_t)0) /* nothing to clear when a read acquires */ +#define MU_RCLEAR_ON_UNCONTENDED_RELEASE \ + ((uint32_t)0) /* nothing to clear when a read releases */ + +/* A lock_type holds the values needed to manipulate a mu in some mode (read or + write). This allows some of the code to be generic, and parameterized by + the lock type. */ +typedef struct lock_type_s { + uint32_t zero_to_acquire; /* bits that must be zero to acquire */ + uint32_t add_to_acquire; /* constant to add to acquire */ + uint32_t + held_if_non_zero; /* if any of these bits are set, the lock is held */ + uint32_t set_when_waiting; /* set when thread waits */ + uint32_t clear_on_acquire; /* clear when thread acquires */ + uint32_t clear_on_uncontended_release; /* clear when thread releases without + waking */ +} lock_type; + +/* writer_type points to a lock_type that describes how to manipulate a mu for a + * writer. */ +extern lock_type *nsync_writer_type_; + +/* reader_type points to a lock_type that describes how to manipulate a mu for a + * reader. */ +extern lock_type *nsync_reader_type_; + +/* ---------- */ + +/* Bits in nsync_cv.word */ + +#define CV_SPINLOCK ((uint32_t)(1 << 0)) /* protects waiters */ +#define CV_NON_EMPTY ((uint32_t)(1 << 1)) /* waiters list is non-empty */ + +/* ---------- */ + +/* Hold a pair of condition function and its argument. */ +struct wait_condition_s { + int (*f)(const void *v); + const void *v; + int (*eq)(const void *a, const void *b); +}; + +/* Return whether wait conditions *a_ and *b_ are equal and non-null. */ +#define WAIT_CONDITION_EQ(a_, b_) \ + ((a_)->f != NULL && (a_)->f == (b_)->f && \ + ((a_)->v == (b_)->v || \ + ((a_)->eq != NULL && (*(a_)->eq)((a_)->v, (b_)->v)))) + +/* If a waiter has waited this many times, it may set the MU_LONG_WAIT bit. */ +#define LONG_WAIT_THRESHOLD 30 + +/* ---------- */ + +#define NOTIFIED_TIME(n_) \ + (ATM_LOAD_ACQ(&(n_)->notified) != 0 ? nsync_time_zero \ + : (n_)->expiry_time_valid ? (n_)->expiry_time \ + : nsync_time_no_deadline) + +/* A waiter represents a single waiter on a cv or a mu. + + To wait: + Allocate a waiter struct *w with new_waiter(), set w.waiting=1, and + w.cv_mu=nil or to the associated mu if waiting on a condition variable, then + queue w.nsync_dll on some queue, and then wait using: + while (ATM_LOAD_ACQ (&w.waiting) != 0) { nsync_mu_semaphore_p (&w.sem); } + Return *w to the freepool by calling free_waiter (w). + + To wakeup: + Remove *w from the relevant queue then: + ATM_STORE_REL (&w.waiting, 0); + nsync_mu_semaphore_v (&w.sem); */ +typedef struct { + uint32_t tag; /* debug DLL_NSYNC_WAITER, DLL_WAITER, DLL_WAITER_SAMECOND */ + nsync_semaphore sem; /* Thread waits on this semaphore. */ + struct nsync_waiter_s nw; /* An embedded nsync_waiter_s. */ + struct nsync_mu_s_ *cv_mu; /* pointer to nsync_mu associated with a cv wait */ + lock_type + *l_type; /* Lock type of the mu, or nil if not associated with a mu. */ + nsync_atomic_uint32_ remove_count; /* count of removals from queue */ + struct wait_condition_s cond; /* A condition on which to acquire a mu. */ + nsync_dll_element_ same_condition; /* Links neighbours in nw.q with same + non-nil condition. */ + int flags; /* see WAITER_* bits below */ +} waiter; +static const uint32_t WAITER_TAG = 0x0590239f; +static const uint32_t NSYNC_WAITER_TAG = 0x726d2ba9; + +#define WAITER_RESERVED \ + 0x1 /* waiter reserved by a thread, even when not in use */ +#define WAITER_IN_USE 0x2 /* waiter in use by a thread */ + +#define CONTAINER(t_, f_, p_) ((t_ *)(((char *)(p_)) - offsetof(t_, f_))) +#define ASSERT(x) \ + do { \ + if (!(x)) { \ + *(volatile int *)0 = 0; \ + } \ + } while (0) + +/* Return a pointer to the nsync_waiter_s containing nsync_dll_element_ *e. */ +#define DLL_NSYNC_WAITER(e) \ + (NSYNC_DEBUG ? nsync_dll_nsync_waiter_(e) \ + : ((struct nsync_waiter_s *)((e)->container))) +struct nsync_waiter_s *nsync_dll_nsync_waiter_(nsync_dll_element_ *e); + +/* Return a pointer to the waiter struct that *e is embedded in, where *e is an + * nw.q field. */ +#define DLL_WAITER(e) \ + (NSYNC_DEBUG ? nsync_dll_waiter_(e) \ + : CONTAINER(waiter, nw, DLL_NSYNC_WAITER(e))) +waiter *nsync_dll_waiter_(nsync_dll_element_ *e); + +/* Return a pointer to the waiter struct that *e is embedded in, where *e is a + same_condition field. */ +#define DLL_WAITER_SAMECOND(e) \ + (NSYNC_DEBUG ? nsync_dll_waiter_samecond_(e) : ((waiter *)((e)->container))) +waiter *nsync_dll_waiter_samecond_(nsync_dll_element_ *e); + +/* Return a pointer to an unused waiter struct. + Ensures that the enclosed timer is stopped and its channel drained. */ +waiter *nsync_waiter_new_(void); + +/* Return an unused waiter struct *w to the free pool. */ +void nsync_waiter_free_(waiter *w); + +/* ---------- */ + +/* The internals of an nync_note. See internal/note.c for details of locking + discipline. */ +struct nsync_note_s_ { + nsync_dll_element_ + parent_child_link; /* parent's children, under parent->note_mu */ + int expiry_time_valid; /* whether expiry_time is valid; r/o after init */ + nsync_time + expiry_time; /* expiry time, if expiry_time_valid != 0; r/o after init */ + nsync_mu note_mu; /* protects fields below except "notified" */ + nsync_cv no_children_cv; /* signalled when children becomes empty */ + uint32_t disconnecting; /* non-zero => node is being disconnected */ + nsync_atomic_uint32_ notified; /* non-zero if the note has been notified */ + struct nsync_note_s_ *parent; /* points to parent, if any */ + nsync_dll_element_ *children; /* list of children */ + nsync_dll_element_ *waiters; /* list of waiters */ +}; + +/* ---------- */ + +void nsync_mu_lock_slow_(nsync_mu *mu, waiter *w, uint32_t clear, + lock_type *l_type); +void nsync_mu_unlock_slow_(nsync_mu *mu, lock_type *l_type); +nsync_dll_list_ nsync_remove_from_mu_queue_(nsync_dll_list_ mu_queue, + nsync_dll_element_ *e); +void nsync_maybe_merge_conditions_(nsync_dll_element_ *p, + nsync_dll_element_ *n); +nsync_time nsync_note_notified_deadline_(nsync_note n); +int nsync_sem_wait_with_cancel_(waiter *w, nsync_time abs_deadline, + nsync_note cancel_note); + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* NSYNC_COMMON_H_ */ diff --git a/third_party/nsync/counter.h b/third_party/nsync/counter.h new file mode 100644 index 000000000..7828553e9 --- /dev/null +++ b/third_party/nsync/counter.h @@ -0,0 +1,43 @@ +#ifndef NSYNC_COUNTER_H_ +#define NSYNC_COUNTER_H_ +#include "third_party/nsync/atomic.h" +#include "third_party/nsync/time.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +struct nsync_dll_element_s_; + +/* An nsync_counter represents an unsigned integer that can count up and down, + and wake waiters when zero. */ +typedef struct nsync_counter_s_ *nsync_counter; + +/* Return a freshly allocated nsync_counter with the specified value, + of NULL if an nsync_counter cannot be created. + + Any non-NULL returned value should be passed to nsync_counter_free() when no + longer needed. */ +nsync_counter nsync_counter_new(uint32_t value); + +/* Free resources associated with c. Requires that c was allocated by + nsync_counter_new(), and no concurrent or future operations are applied to + c. */ +void nsync_counter_free(nsync_counter c); + +/* Add delta to c, and return its new value. It is a checkable runtime error + to decrement c below 0, or to increment c (i.e., apply a delta > 0) after a + waiter has waited. */ +uint32_t nsync_counter_add(nsync_counter c, int32_t delta); + +/* Return the current value of c. */ +uint32_t nsync_counter_value(nsync_counter c); + +/* Wait until c has value 0, or until abs_deadline, then return + the value of c. It is a checkable runtime error to increment c after + a waiter may have been woken due to the counter reaching zero. + If abs_deadline==nsync_time_no_deadline, the deadline + is far in the future. */ +uint32_t nsync_counter_wait(nsync_counter c, nsync_time abs_deadline); + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* NSYNC_COUNTER_H_ */ diff --git a/third_party/nsync/cv.h b/third_party/nsync/cv.h new file mode 100644 index 000000000..bd6b5ad91 --- /dev/null +++ b/third_party/nsync/cv.h @@ -0,0 +1,157 @@ +#ifndef NSYNC_CV_H_ +#define NSYNC_CV_H_ +#include "third_party/nsync/mu.h" +#include "third_party/nsync/time.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +#define NSYNC_CV_INIT \ + { NSYNC_ATOMIC_UINT32_INIT_, 0 } + +struct nsync_dll_element_s_; +struct nsync_note_s_; + +/* An nsync_cv is a condition variable in the style of Mesa, Java, + POSIX, and Go's sync.Cond. It allows a thread to wait for a condition + on state protected by a mutex, and to proceed with the mutex held and + the condition true. + + See also nsync_mu_wait() and nsync_mu_wait_with_deadline(), which + implement conditional critical sections. In many cases, they are + easier to use than condition variables. + + Usage + + After making the desired predicate true, call: + + nsync_cv_signal (&cv); // If at most one thread can make use + // of the predicate becoming true. + + or + + nsync_cv_broadcast (&cv); // If multiple threads can make use + // of the predicate becoming true. + + To wait for a predicate with no deadline (assuming + nsync_cv_broadcast() or nsync_cv_signal() is called whenever the + predicate becomes true): + + nsync_mu_lock (μ) + while (!some_predicate_protected_by_mu) { // while-loop required + nsync_cv_wait (&cv, &mu); + } + // predicate is now true + nsync_mu_unlock (&mu); + + To wait for a predicate with a deadline (assuming nsync_cv_broadcast() or + nsync_cv_signal() is called whenever the predicate becomes true): + + nsync_mu_lock (&mu); + while (!some_predicate_protected_by_mu && + nsync_cv_wait_with_deadline (&cv, &mu, abs_deadline, + cancel_note) == 0) { + } + if (some_predicate_protected_by_mu) { // predicate is true + } else { + // predicate is false, and deadline expired, or + // cancel_note was notified. + } + nsync_mu_unlock (&mu); + + or, if the predicate is complex and you wish to write it just once + and inline, you could use the following instead of the for-loop + above: + + nsync_mu_lock (&mu); + int pred_is_true = 0; + int outcome = 0; + while (!(pred_is_true = some_predicate_protected_by_mu) && + outcome == 0) { + outcome = nsync_cv_wait_with_deadline (&cv, &mu, abs_deadline, + cancel_note); + } + if (pred_is_true) { // predicate is true + } else { + // predicate is false, and deadline expired, or + // cancel_note was notified. + } + nsync_mu_unlock (&mu); + + As the examples show, Mesa-style condition variables require that + waits use a loop that tests the predicate anew after each wait. It + may be surprising that these are preferred over the precise wakeups + offered by the condition variables in Hoare monitors. Imprecise + wakeups make more efficient use of the critical section, because + threads can enter it while a woken thread is still emerging from the + scheduler, which may take thousands of cycles. Further, they make the + programme easier to read and debug by making the predicate explicit + locally at the wait, where the predicate is about to be assumed; the + reader does not have to infer the predicate by examining all the + places where wakeups may occur. */ +typedef struct nsync_cv_s_ { + /* see bits below */ + nsync_atomic_uint32_ word; + /* points to tail of list of waiters; under mu. */ + struct nsync_dll_element_s_ *waiters; +} nsync_cv; + +/* An nsync_cv should be zeroed to initialize, which can be accomplished + by initializing with static initializer NSYNC_CV_INIT, or by setting + the entire struct to 0, or using nsync_cv_init(). */ +void nsync_cv_init(nsync_cv *cv); + +/* Wake at least one thread if any are currently blocked on *cv. If the + chosen thread is a reader on an nsync_mu, wake all readers and, if + possible, a writer. */ +void nsync_cv_signal(nsync_cv *cv); + +/* Wake all threads currently blocked on *cv. */ +void nsync_cv_broadcast(nsync_cv *cv); + +/* Atomically release "mu" (which must be held on entry) and block the + caller on *cv. Wait until awakened by a call to nsync_cv_signal() or + nsync_cv_broadcast(), or a spurious wakeup; then reacquire "mu", and + return. Equivalent to a call to nsync_mu_wait_with_deadline() with + abs_deadline==nsync_time_no_deadline, and cancel_note==NULL. Callers + should use nsync_cv_wait() in a loop, as with all standard Mesa-style + condition variables. See examples above. */ +void nsync_cv_wait(nsync_cv *cv, nsync_mu *mu); + +/* Atomically release "mu" (which must be held on entry) and block the + calling thread on *cv. It then waits until awakened by a call to + nsync_cv_signal() or nsync_cv_broadcast() (or a spurious wakeup), or + by the time reaching abs_deadline, or by cancel_note being notified. + In all cases, it reacquires "mu", and returns the reason for the call + returned (0, ETIMEDOUT, or ECANCELED). Use + abs_deadline==nsync_time_no_deadline for no deadline, and + cancel_note==NULL for no cancellation. wait_with_deadline() should be + used in a loop, as with all Mesa-style condition variables. See + examples above. + + There are two reasons for using an absolute deadline, rather than a + relative timeout---these are why pthread_cond_timedwait() also uses + an absolute deadline. First, condition variable waits have to be used + in a loop; with an absolute times, the deadline does not have to be + recomputed on each iteration. Second, in most real programmes, some + activity (such as an RPC to a server, or when guaranteeing response + time in a UI), there is a deadline imposed by the specification or + the caller/user; relative delays can shift arbitrarily with + scheduling delays, and so after multiple waits might extend beyond + the expected deadline. Relative delays tend to be more convenient + mostly in tests and trivial examples than they are in real + programmes. */ +int nsync_cv_wait_with_deadline(nsync_cv *cv, nsync_mu *mu, + nsync_time abs_deadline, + struct nsync_note_s_ *cancel_note); + +/* Like nsync_cv_wait_with_deadline(), but allow an arbitrary lock *v to be + used, given its (*lock)(mu) and (*unlock)(mu) routines. */ +int nsync_cv_wait_with_deadline_generic(nsync_cv *cv, void *mu, + void (*lock)(void *), + void (*unlock)(void *), + nsync_time abs_deadline, + struct nsync_note_s_ *cancel_note); + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* NSYNC_CV_H_ */ diff --git a/third_party/nsync/debug.h b/third_party/nsync/debug.h new file mode 100644 index 000000000..44d59618f --- /dev/null +++ b/third_party/nsync/debug.h @@ -0,0 +1,39 @@ +#ifndef NSYNC_DEBUG_H_ +#define NSYNC_DEBUG_H_ +#include "third_party/nsync/cv.h" +#include "third_party/nsync/mu.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +/* Debugging operations for mutexes and condition variables. + + These operations should not be relied upon for normal functionality. + The implementation may be slow, output formats may change, and the + implementation is free to yield the empty string. */ + +/* Place in buf[0,..,n-1] a nul-terminated, human readable string + indicative of some of the internal state of the mutex or condition + variable, and return buf. If n>=4, buffer overflow is indicated by + placing the characters "..." at the end of the string. + + The *_and_waiters() variants attempt to output the waiter lists in + addition to the basic state. These variants may acquire internal + locks and follow internal pointers. Thus, they are riskier if invoked + in an address space whose overall health is uncertain. */ +char *nsync_mu_debug_state(nsync_mu *mu, char *buf, int n); +char *nsync_cv_debug_state(nsync_cv *cv, char *buf, int n); +char *nsync_mu_debug_state_and_waiters(nsync_mu *mu, char *buf, int n); +char *nsync_cv_debug_state_and_waiters(nsync_cv *cv, char *buf, int n); + +/* Like nsync_*_debug_state_and_waiters(), but ignoring all locking and + safety considerations, and using an internal, possibly static buffer + that may be overwritten by subsequent or concurrent calls to these + routines. These variants should be used only from an interactive + debugger, when all other threads are stopped; the debugger is + expected to recover from errors. */ +char *nsync_mu_debugger(nsync_mu *mu); +char *nsync_cv_debugger(nsync_cv *cv); + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* NSYNC_DEBUG_H_ */ diff --git a/libc/thread/dll.c b/third_party/nsync/dll.c similarity index 67% rename from libc/thread/dll.c rename to third_party/nsync/dll.c index 9fc7e0482..d5ff5e555 100644 --- a/libc/thread/dll.c +++ b/third_party/nsync/dll.c @@ -1,25 +1,27 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2016 Google Inc. │ +│ │ +│ Licensed under the Apache License, Version 2.0 (the "License"); │ +│ you may not use this file except in compliance with the License. │ +│ You may obtain a copy of the License at │ +│ │ +│ http://www.apache.org/licenses/LICENSE-2.0 │ +│ │ +│ Unless required by applicable law or agreed to in writing, software │ +│ distributed under the License is distributed on an "AS IS" BASIS, │ +│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ +│ See the License for the specific language governing permissions and │ +│ limitations under the License. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "third_party/nsync/dll.h" + +asm(".ident\t\"\\n\\n\ +*NSYNC (Apache 2.0)\\n\ +Copyright 2016 Google, Inc.\\n\ +https://github.com/google/nsync\""); // clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/platform.h" -#include "libc/thread/compiler.h" -#include "libc/thread/cputype.h" -#include "libc/thread/dll.h" - -NSYNC_CPP_START_ /* Initialize *e. */ void nsync_dll_init_ (nsync_dll_element_ *e, void *container) { @@ -139,5 +141,3 @@ nsync_dll_element_ *nsync_dll_prev_ (nsync_dll_list_ list, nsync_dll_element_ *e } return (prev); } - -NSYNC_CPP_END_ diff --git a/third_party/nsync/dll.h b/third_party/nsync/dll.h new file mode 100644 index 000000000..a1345d386 --- /dev/null +++ b/third_party/nsync/dll.h @@ -0,0 +1,69 @@ +#ifndef NSYNC_DLL_H_ +#define NSYNC_DLL_H_ +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +/* A nsync_dll_element_ represents an element of a doubly-linked list of + waiters. */ +typedef struct nsync_dll_element_s_ { + struct nsync_dll_element_s_ *next; + struct nsync_dll_element_s_ *prev; + /* points to the struct this nsync_dll struct is embedded in. */ + void *container; +} nsync_dll_element_; + +/* A nsync_dll_list_ represents a list of nsync_dll_elements_. */ +typedef nsync_dll_element_ *nsync_dll_list_; /* last elem of circular list; nil + => empty; first is x.next. */ + +/* Initialize *e. */ +void nsync_dll_init_(nsync_dll_element_ *e, void *container); + +/* Return whether list is empty. */ +int nsync_dll_is_empty_(nsync_dll_list_ list); + +/* Remove *e from list, and returns the new list. */ +nsync_dll_list_ nsync_dll_remove_(nsync_dll_list_ list, nsync_dll_element_ *e); + +/* Cause element *n and its successors to come after element *p. + Requires n and p are non-NULL and do not point at elements of the + same list. */ +void nsync_dll_splice_after_(nsync_dll_element_ *p, nsync_dll_element_ *n); + +/* Make element *e the first element of list, and return the list. The + resulting list will have *e as its first element, followed by any + elements in the same list as *e, followed by the elements that were + previously in list. Requires that *e not be in list. If e==NULL, list + is returned unchanged. */ +nsync_dll_list_ nsync_dll_make_first_in_list_(nsync_dll_list_ list, + nsync_dll_element_ *e); + +/* Make element *e the last element of list, and return the list. The + resulting list will have *e as its last element, preceded by any + elements in the same list as *e, preceded by the elements that were + previously in list. Requires that *e not be in list. If e==NULL, list + is returned unchanged. */ +nsync_dll_list_ nsync_dll_make_last_in_list_(nsync_dll_list_ list, + nsync_dll_element_ *e); + +/* Return a pointer to the first element of list, or NULL if list is + * empty. */ +nsync_dll_element_ *nsync_dll_first_(nsync_dll_list_ list); + +/* Return a pointer to the last element of list, or NULL if list is + * empty. */ +nsync_dll_element_ *nsync_dll_last_(nsync_dll_list_ list); + +/* Return a pointer to the next element of list following *e, or NULL if + there is no such element. */ +nsync_dll_element_ *nsync_dll_next_(nsync_dll_list_ list, + nsync_dll_element_ *e); + +/* Return a pointer to the previous element of list following *e, or + NULL if there is no such element. */ +nsync_dll_element_ *nsync_dll_prev_(nsync_dll_list_ list, + nsync_dll_element_ *e); + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* NSYNC_DLL_H_ */ diff --git a/third_party/nsync/futex.c b/third_party/nsync/futex.c new file mode 100644 index 000000000..cc875abbe --- /dev/null +++ b/third_party/nsync/futex.c @@ -0,0 +1,124 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/strace.internal.h" +#include "libc/calls/struct/timespec.internal.h" +#include "libc/dce.h" +#include "libc/errno.h" +#include "libc/intrin/describeflags.internal.h" +#include "libc/sysv/consts/futex.h" +#include "libc/thread/thread.h" +#include "third_party/nsync/common.internal.h" +#include "third_party/nsync/futex.internal.h" +// clang-format off + +/* futex() polyfill w/ sched_yield() fallback */ + +#define FUTEX_WAIT_BITS_ FUTEX_BITSET_MATCH_ANY + +int _futex (int *, int, int, const struct timespec *, int *, int); + +static int FUTEX_WAIT_; +static int FUTEX_WAKE_; +static int FUTEX_PRIVATE_FLAG_; +static bool FUTEX_IS_SUPPORTED; +bool FUTEX_TIMEOUT_IS_ABSOLUTE; + +__attribute__((__constructor__)) static void sync_futex_init_ (void) { + int x = 0; + + FUTEX_WAKE_ = FUTEX_WAKE; + + if (IsLinux () && + _futex (&x, FUTEX_WAIT_BITSET, 1, 0, 0, + FUTEX_BITSET_MATCH_ANY) == -EAGAIN) { + FUTEX_WAIT_ = FUTEX_WAIT_BITSET; + FUTEX_TIMEOUT_IS_ABSOLUTE = true; + } else { + FUTEX_WAIT_ = FUTEX_WAIT; + } + + if (IsOpenbsd () || + (IsLinux () && + !_futex (&x, FUTEX_WAKE_PRIVATE, 1, 0, 0, 0))) { + FUTEX_PRIVATE_FLAG_ = FUTEX_PRIVATE_FLAG; + } + + // In our testing, we found that the monotonic clock on various + // popular systems (such as Linux, and some BSD variants) was no + // better behaved than the realtime clock, and routinely took + // large steps backwards, especially on multiprocessors. Given + // that "monotonic" doesn't seem to mean what it says, + // implementers of nsync_time might consider retaining the + // simplicity of a single epoch within an address space, by + // configuring any time synchronization mechanism (like ntp) to + // adjust for leap seconds by adjusting the rate, rather than + // with a backwards step. + if (IsLinux () && + _futex (&x, FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME, + 1, 0, 0, FUTEX_BITSET_MATCH_ANY) == -EAGAIN) { + FUTEX_WAIT_ |= FUTEX_CLOCK_REALTIME; + } + + FUTEX_IS_SUPPORTED = IsLinux() || IsOpenbsd(); +} + +int nsync_futex_wait_ (int *p, int expect, char pshare, struct timespec *timeout) { + int rc, op; + if (FUTEX_IS_SUPPORTED) { + op = FUTEX_WAIT_; + if (pshare == PTHREAD_PROCESS_PRIVATE) { + op |= FUTEX_PRIVATE_FLAG_; + } + rc = _futex (p, op, expect, timeout, 0, FUTEX_WAIT_BITS_); + if (IsOpenbsd() && rc > 0) { + // [jart] openbsd does this without setting carry flag + rc = -rc; + } + STRACE("futex(%t, %s, %d, %s) → %s", + p, DescribeFutexOp(op), expect, + DescribeTimespec(0, timeout), DescribeFutexResult(rc)); + } else { + nsync_yield_ (); + if (timeout) { + rc = -ETIMEDOUT; + } else { + rc = 0; + } + } + return rc; +} + +int nsync_futex_wake_ (int *p, int count, char pshare) { + int rc, op; + int wake (void *, int, int) asm ("_futex"); + if (FUTEX_IS_SUPPORTED) { + op = FUTEX_WAKE_; + if (pshare == PTHREAD_PROCESS_PRIVATE) { + op |= FUTEX_PRIVATE_FLAG_; + } + rc = wake (p, op, count); + STRACE("futex(%t, %s, %d) → %s", p, + DescribeFutexOp(op), + count, DescribeFutexResult(rc)); + } else { + nsync_yield_ (); + rc = 0; + } + return rc; +} diff --git a/third_party/nsync/futex.internal.h b/third_party/nsync/futex.internal.h new file mode 100644 index 000000000..5fc8f4998 --- /dev/null +++ b/third_party/nsync/futex.internal.h @@ -0,0 +1,15 @@ +#ifndef NSYNC_FUTEX_INTERNAL_H_ +#define NSYNC_FUTEX_INTERNAL_H_ +#include "libc/calls/struct/timespec.h" +#include "libc/dce.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +extern bool FUTEX_TIMEOUT_IS_ABSOLUTE; + +int nsync_futex_wake_(int *, int, char); +int nsync_futex_wait_(int *, int, char, struct timespec *); + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* NSYNC_FUTEX_INTERNAL_H_ */ diff --git a/third_party/nsync/malloc.c b/third_party/nsync/malloc.c new file mode 100644 index 000000000..65708ddfe --- /dev/null +++ b/third_party/nsync/malloc.c @@ -0,0 +1,49 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/atomic.h" +#include "libc/calls/extend.internal.h" +#include "libc/intrin/atomic.h" +#include "libc/macros.internal.h" +#include "third_party/nsync/common.internal.h" +#include "third_party/nsync/malloc.internal.h" +// clang-format off + +static char *nsync_malloc_endptr_; +static size_t nsync_malloc_total_; +static atomic_char nsync_malloc_lock_; + +/* nsync_malloc_() is a malloc-like routine used by mutex and condition + variable code to allocate waiter structs. This allows *NSYNC mutexes + to be used by malloc(), by providing another, simpler allocator here. + The intent is that the implicit NULL value here can be overridden by + a client declaration that uses an initializer. */ +void *nsync_malloc_ (size_t size) { + char *start; + size_t offset; + size = ROUNDUP (size, __BIGGEST_ALIGNMENT__); + while (atomic_exchange (&nsync_malloc_lock_, 1)) nsync_yield_ (); + offset = nsync_malloc_total_; + nsync_malloc_total_ += size; + start = (char *) 0x6fc000040000; + if (!nsync_malloc_endptr_) nsync_malloc_endptr_ = start; + nsync_malloc_endptr_ = _extend (start, nsync_malloc_total_, + nsync_malloc_endptr_, 0x6fcfffff0000); + atomic_store_explicit (&nsync_malloc_lock_, 0, memory_order_relaxed); + return start + offset; +} diff --git a/third_party/nsync/malloc.internal.h b/third_party/nsync/malloc.internal.h new file mode 100644 index 000000000..e4453e410 --- /dev/null +++ b/third_party/nsync/malloc.internal.h @@ -0,0 +1,10 @@ +#ifndef NSYNC_MALLOC_INTERNAL_H_ +#define NSYNC_MALLOC_INTERNAL_H_ +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +void *nsync_malloc_(size_t); + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* NSYNC_MALLOC_INTERNAL_H_ */ diff --git a/libc/thread/mu.c b/third_party/nsync/mu.c similarity index 90% rename from libc/thread/mu.c rename to third_party/nsync/mu.c index 8fd1b5196..01b717cd1 100644 --- a/libc/thread/mu.c +++ b/third_party/nsync/mu.c @@ -1,30 +1,32 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2016 Google Inc. │ +│ │ +│ Licensed under the Apache License, Version 2.0 (the "License"); │ +│ you may not use this file except in compliance with the License. │ +│ You may obtain a copy of the License at │ +│ │ +│ http://www.apache.org/licenses/LICENSE-2.0 │ +│ │ +│ Unless required by applicable law or agreed to in writing, software │ +│ distributed under the License is distributed on an "AS IS" BASIS, │ +│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ +│ See the License for the specific language governing permissions and │ +│ limitations under the License. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/str/str.h" +#include "third_party/nsync/atomic.h" +#include "third_party/nsync/common.internal.h" +#include "third_party/nsync/dll.h" +#include "third_party/nsync/mu_semaphore.h" +#include "third_party/nsync/wait_s.internal.h" + +asm(".ident\t\"\\n\\n\ +*NSYNC (Apache 2.0)\\n\ +Copyright 2016 Google, Inc.\\n\ +https://github.com/google/nsync\""); // clang-format off -/* Copyright 2016 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "libc/thread/nsync_cpp.h" -#include "libc/thread/platform.h" -#include "libc/thread/compiler.h" -#include "libc/thread/cputype.h" -#include "libc/thread/nsync.h" -#include "libc/thread/dll.h" -#include "libc/thread/sem.h" -#include "libc/thread/wait_internal.h" -#include "libc/thread/common.h" -#include "libc/thread/atomic.h" - -NSYNC_CPP_START_ /* Initialize *mu. */ void nsync_mu_init (nsync_mu *mu) { @@ -543,5 +545,3 @@ int nsync_mu_is_reader (const nsync_mu *mu) { IGNORE_RACES_END (); return ((word & MU_WLOCK) == 0); } - -NSYNC_CPP_END_ diff --git a/third_party/nsync/mu.h b/third_party/nsync/mu.h new file mode 100644 index 000000000..8d72fea6c --- /dev/null +++ b/third_party/nsync/mu.h @@ -0,0 +1,103 @@ +#ifndef NSYNC_MU_H_ +#define NSYNC_MU_H_ +#include "third_party/nsync/atomic.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +struct nsync_dll_element_s_; + +/* An nsync_mu is a lock. If initialized to zero, it's valid and unlocked. + + An nsync_mu can be "free", held by a single thread (aka fiber, + goroutine) in "write" (exclusive) mode, or by many threads in "read" + (shared) mode. A thread that acquires it should eventually release + it. It is illegal to acquire an nsync_mu in one thread and release it + in another. It is illegal for a thread to reacquire an nsync_mu while + holding it (even a second share of a "read" lock). + + Example usage: + + static struct foo { + nsync_mu mu; // protects invariant a+b==0 on fields below. + int a; + int b; + } p = { NSYNC_MU_INIT, 0, 0 }; + // .... + nsync_mu_lock (&p.mu); + // The current thread now has exclusive access to p.a and p.b; + // invariant assumed true. + p.a++; + p.b--; // restore invariant p.a+p.b==0 before releasing p.mu + nsync_mu_unlock (&p.mu) + + Mutexes can be used with condition variables; see nsync_cv.h. + + nsync_mu_wait() and nsync_mu_wait_with_deadline() can be used instead + of condition variables. See nsync_mu_wait.h for more details. Example + use of nsync_mu_wait() to wait for p.a==0, using definition above: + + int a_is_zero (const void *condition_arg) { + return (((const struct foo *)condition_arg)->a == 0); + } + ... + nsync_mu_lock (&p.mu); + nsync_mu_wait (&p.mu, &a_is_zero, &p, NULL); + // The current thread now has exclusive access to + // p.a and p.b, and p.a==0. + ... + nsync_mu_unlock (&p.mu); + +*/ +typedef struct nsync_mu_s_ { + nsync_atomic_uint32_ word; /* internal use only */ + struct nsync_dll_element_s_ *waiters; /* internal use only */ +} nsync_mu; + +/* An nsync_mu should be zeroed to initialize, which can be accomplished + by initializing with static initializer NSYNC_MU_INIT, or by setting + the entire structure to all zeroes, or using nsync_mu_init(). */ +#define NSYNC_MU_INIT \ + { NSYNC_ATOMIC_UINT32_INIT_, 0 } +void nsync_mu_init(nsync_mu *mu); + +/* Block until *mu is free and then acquire it in writer mode. Requires + that the calling thread not already hold *mu in any mode. */ +void nsync_mu_lock(nsync_mu *mu); + +/* Unlock *mu, which must have been acquired in write mode by the + calling thread, and wake waiters, if appropriate. */ +void nsync_mu_unlock(nsync_mu *mu); + +/* Attempt to acquire *mu in writer mode without blocking, and return + non-zero iff successful. Return non-zero with high probability if *mu + was free on entry. */ +int nsync_mu_trylock(nsync_mu *mu); + +/* Block until *mu can be acquired in reader mode and then acquire it. + Requires that the calling thread not already hold *mu in any mode. */ +void nsync_mu_rlock(nsync_mu *mu); + +/* Unlock *mu, which must have been acquired in read mode by the calling + thread, and wake waiters, if appropriate. */ +void nsync_mu_runlock(nsync_mu *mu); + +/* Attempt to acquire *mu in reader mode without blocking, and return + non-zero iff successful. Return non-zero with high probability if *mu + was free on entry. Perhaps fail to acquire if a writer is waiting, to + avoid starvation. */ +int nsync_mu_rtrylock(nsync_mu *mu); + +/* May abort if *mu is not held in write mode by the calling thread. */ +void nsync_mu_assert_held(const nsync_mu *mu); + +/* May abort if *mu is not held in read or write mode + by the calling thread. */ +void nsync_mu_rassert_held(const nsync_mu *mu); + +/* Return whether *mu is held in read mode. + Requires that the calling thread holds *mu in some mode. */ +int nsync_mu_is_reader(const nsync_mu *mu); + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* NSYNC_MU_H_ */ diff --git a/third_party/nsync/mu_semaphore.c b/third_party/nsync/mu_semaphore.c new file mode 100644 index 000000000..a30ef4873 --- /dev/null +++ b/third_party/nsync/mu_semaphore.c @@ -0,0 +1,56 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/dce.h" +#include "third_party/nsync/mu_semaphore.h" +#include "third_party/nsync/mu_semaphore.internal.h" +// clang-format off + +/* Initialize *s; the initial value is 0. */ +void nsync_mu_semaphore_init (nsync_semaphore *s) { + if (!IsWindows ()) + nsync_mu_semaphore_init_futex (s); + else + nsync_mu_semaphore_init_win32 (s); +} + +/* Wait until the count of *s exceeds 0, and decrement it. */ +void nsync_mu_semaphore_p (nsync_semaphore *s) { + if (!IsWindows ()) + nsync_mu_semaphore_p_futex (s); + else + nsync_mu_semaphore_p_win32 (s); +} + +/* Wait until one of: + the count of *s is non-zero, in which case decrement *s and return 0; + or abs_deadline expires, in which case return ETIMEDOUT. */ +int nsync_mu_semaphore_p_with_deadline (nsync_semaphore *s, nsync_time abs_deadline) { + if (!IsWindows ()) + return nsync_mu_semaphore_p_with_deadline_futex (s, abs_deadline); + else + return nsync_mu_semaphore_p_with_deadline_win32 (s, abs_deadline); +} + +/* Ensure that the count of *s is at least 1. */ +void nsync_mu_semaphore_v (nsync_semaphore *s) { + if (!IsWindows ()) + nsync_mu_semaphore_v_futex (s); + else + nsync_mu_semaphore_v_win32 (s); +} diff --git a/third_party/nsync/mu_semaphore.h b/third_party/nsync/mu_semaphore.h new file mode 100644 index 000000000..d57e1b770 --- /dev/null +++ b/third_party/nsync/mu_semaphore.h @@ -0,0 +1,28 @@ +#ifndef NSYNC_SEM_H_ +#define NSYNC_SEM_H_ +#include "third_party/nsync/time.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +typedef struct nsync_semaphore_s_ { + void *sem_space[32]; /* space used by implementation */ +} nsync_semaphore; + +/* Initialize *s; the initial value is 0. */ +void nsync_mu_semaphore_init(nsync_semaphore *s); + +/* Wait until the count of *s exceeds 0, and decrement it. */ +void nsync_mu_semaphore_p(nsync_semaphore *s); + +/* Wait until one of: the count of *s is non-zero, in which case + decrement *s and return 0; or abs_deadline expires, in which case + return ETIMEDOUT. */ +int nsync_mu_semaphore_p_with_deadline(nsync_semaphore *s, + nsync_time abs_deadline); + +/* Ensure that the count of *s is at least 1. */ +void nsync_mu_semaphore_v(nsync_semaphore *s); + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* NSYNC_SEM_H_ */ diff --git a/third_party/nsync/mu_semaphore.internal.h b/third_party/nsync/mu_semaphore.internal.h new file mode 100644 index 000000000..8fac00aa0 --- /dev/null +++ b/third_party/nsync/mu_semaphore.internal.h @@ -0,0 +1,20 @@ +#ifndef NSYNC_MU_SEMAPHORE_INTERNAL_H_ +#define NSYNC_MU_SEMAPHORE_INTERNAL_H_ +#include "third_party/nsync/mu_semaphore.h" +#include "third_party/nsync/time.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +void nsync_mu_semaphore_init_futex(nsync_semaphore *); +void nsync_mu_semaphore_p_futex(nsync_semaphore *); +int nsync_mu_semaphore_p_with_deadline_futex(nsync_semaphore *, nsync_time); +void nsync_mu_semaphore_v_futex(nsync_semaphore *); + +void nsync_mu_semaphore_init_win32(nsync_semaphore *); +void nsync_mu_semaphore_p_win32(nsync_semaphore *); +int nsync_mu_semaphore_p_with_deadline_win32(nsync_semaphore *, nsync_time); +void nsync_mu_semaphore_v_win32(nsync_semaphore *); + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* NSYNC_MU_SEMAPHORE_INTERNAL_H_ */ diff --git a/third_party/nsync/mu_semaphore_futex.c b/third_party/nsync/mu_semaphore_futex.c new file mode 100644 index 000000000..237f83ed2 --- /dev/null +++ b/third_party/nsync/mu_semaphore_futex.c @@ -0,0 +1,124 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2016 Google Inc. │ +│ │ +│ Licensed under the Apache License, Version 2.0 (the "License"); │ +│ you may not use this file except in compliance with the License. │ +│ You may obtain a copy of the License at │ +│ │ +│ http://www.apache.org/licenses/LICENSE-2.0 │ +│ │ +│ Unless required by applicable law or agreed to in writing, software │ +│ distributed under the License is distributed on an "AS IS" BASIS, │ +│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ +│ See the License for the specific language governing permissions and │ +│ limitations under the License. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/errno.h" +#include "libc/str/str.h" +#include "libc/thread/thread.h" +#include "third_party/nsync/atomic.h" +#include "third_party/nsync/atomic.internal.h" +#include "third_party/nsync/futex.internal.h" +#include "third_party/nsync/mu_semaphore.h" +#include "third_party/nsync/mu_semaphore.internal.h" + +asm(".ident\t\"\\n\\n\ +*NSYNC (Apache 2.0)\\n\ +Copyright 2016 Google, Inc.\\n\ +https://github.com/google/nsync\""); +// clang-format off + +/* Check that atomic operations on nsync_atomic_uint32_ can be applied to int. */ +static const int assert_int_size = 1 / + (sizeof (assert_int_size) == sizeof (uint32_t) && + sizeof (nsync_atomic_uint32_) == sizeof (uint32_t)); + +#define ASSERT(x) do { if (!(x)) { *(volatile int *)0 = 0; } } while (0) + +struct futex { + int i; /* lo half=count; hi half=waiter count */ +}; + +static nsync_semaphore *sem_big_enough_for_futex = (nsync_semaphore *) (uintptr_t)(1 / + (sizeof (struct futex) <= sizeof (*sem_big_enough_for_futex))); + +/* Initialize *s; the initial value is 0. */ +void nsync_mu_semaphore_init_futex (nsync_semaphore *s) { + struct futex *f = (struct futex *) s; + f->i = 0; +} + +/* Wait until the count of *s exceeds 0, and decrement it. */ +void nsync_mu_semaphore_p_futex (nsync_semaphore *s) { + struct futex *f = (struct futex *) s; + int i; + do { + i = ATM_LOAD ((nsync_atomic_uint32_ *) &f->i); + if (i == 0) { + int futex_result = nsync_futex_wait_ (&f->i, i, PTHREAD_PROCESS_PRIVATE, NULL); + ASSERT (futex_result == 0 || + futex_result == -EINTR || + futex_result == -EWOULDBLOCK); + } + } while (i == 0 || !ATM_CAS_ACQ ((nsync_atomic_uint32_ *) &f->i, i, i-1)); +} + +/* Wait until one of: + the count of *s is non-zero, in which case decrement *s and return 0; + or abs_deadline expires, in which case return ETIMEDOUT. */ +int nsync_mu_semaphore_p_with_deadline_futex (nsync_semaphore *s, nsync_time abs_deadline) { + struct futex *f = (struct futex *)s; + int i; + int result = 0; + do { + i = ATM_LOAD ((nsync_atomic_uint32_ *) &f->i); + if (i == 0) { + int futex_result; + struct timespec ts_buf; + const struct timespec *ts = NULL; + if (nsync_time_cmp (abs_deadline, nsync_time_no_deadline) != 0) { + memset (&ts_buf, 0, sizeof (ts_buf)); + if (FUTEX_TIMEOUT_IS_ABSOLUTE) { + ts_buf.tv_sec = NSYNC_TIME_SEC (abs_deadline); + ts_buf.tv_nsec = NSYNC_TIME_NSEC (abs_deadline); + } else { + nsync_time now; + now = nsync_time_now (); + if (nsync_time_cmp (now, abs_deadline) > 0) { + ts_buf.tv_sec = 0; + ts_buf.tv_nsec = 0; + } else { + nsync_time rel_deadline; + rel_deadline = nsync_time_sub (abs_deadline, now); + ts_buf.tv_sec = NSYNC_TIME_SEC (rel_deadline); + ts_buf.tv_nsec = NSYNC_TIME_NSEC (rel_deadline); + } + } + ts = &ts_buf; + } + futex_result = nsync_futex_wait_ (&f->i, i, PTHREAD_PROCESS_PRIVATE, ts); + ASSERT (futex_result == 0 || + futex_result == -EINTR || + futex_result == -ETIMEDOUT || + futex_result == -EWOULDBLOCK); + /* Some systems don't wait as long as they are told. */ + if (futex_result == -ETIMEDOUT && + nsync_time_cmp (abs_deadline, nsync_time_now ()) <= 0) { + result = ETIMEDOUT; + } + } + } while (result == 0 && (i == 0 || !ATM_CAS_ACQ ((nsync_atomic_uint32_ *) &f->i, i, i - 1))); + return (result); +} + +/* Ensure that the count of *s is at least 1. */ +void nsync_mu_semaphore_v_futex (nsync_semaphore *s) { + struct futex *f = (struct futex *) s; + uint32_t old_value; + do { + old_value = ATM_LOAD ((nsync_atomic_uint32_ *) &f->i); + } while (!ATM_CAS_REL ((nsync_atomic_uint32_ *) &f->i, old_value, old_value+1)); + ASSERT (nsync_futex_wake_ (&f->i, 1, PTHREAD_PROCESS_PRIVATE) >= 0); +} diff --git a/third_party/nsync/mu_semaphore_win32.c b/third_party/nsync/mu_semaphore_win32.c new file mode 100644 index 000000000..9dc8e2927 --- /dev/null +++ b/third_party/nsync/mu_semaphore_win32.c @@ -0,0 +1,84 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2016 Google Inc. │ +│ │ +│ Licensed under the Apache License, Version 2.0 (the "License"); │ +│ you may not use this file except in compliance with the License. │ +│ You may obtain a copy of the License at │ +│ │ +│ http://www.apache.org/licenses/LICENSE-2.0 │ +│ │ +│ Unless required by applicable law or agreed to in writing, software │ +│ distributed under the License is distributed on an "AS IS" BASIS, │ +│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ +│ See the License for the specific language governing permissions and │ +│ limitations under the License. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/errno.h" +#include "libc/nt/enum/wait.h" +#include "libc/nt/synchronization.h" +#include "libc/runtime/runtime.h" +#include "third_party/nsync/mu_semaphore.h" +#include "third_party/nsync/mu_semaphore.internal.h" +#include "third_party/nsync/time.h" + +asm(".ident\t\"\\n\\n\ +*NSYNC (Apache 2.0)\\n\ +Copyright 2016 Google, Inc.\\n\ +https://github.com/google/nsync\""); +// clang-format off + +/* Initialize *s; the initial value is 0. */ +void nsync_mu_semaphore_init_win32 (nsync_semaphore *s) { + int64_t *h = (int64_t *) s; + *h = CreateSemaphore(NULL, 0, 1, NULL); + if (!*h) notpossible; +} + +/* Wait until the count of *s exceeds 0, and decrement it. */ +void nsync_mu_semaphore_p_win32 (nsync_semaphore *s) { + int64_t *h = (int64_t *) s; + WaitForSingleObject(*h, -1u); +} + +/* Wait until one of: + the count of *s is non-zero, in which case decrement *s and return 0; + or abs_deadline expires, in which case return ETIMEDOUT. */ +int nsync_mu_semaphore_p_with_deadline_win32 (nsync_semaphore *s, nsync_time abs_deadline) { + int64_t *h = (int64_t *) s; + int result; + + if (nsync_time_cmp (abs_deadline, nsync_time_no_deadline) == 0) { + result = WaitForSingleObject(*h, -1u); + } else { + nsync_time now; + now = nsync_time_now (); + do { + if (nsync_time_cmp (abs_deadline, now) <= 0) { + result = WaitForSingleObject (*h, 0); + } else { + nsync_time delay; + delay = nsync_time_sub (abs_deadline, now); + if (NSYNC_TIME_SEC (delay) > 1000*1000) { + result = WaitForSingleObject (*h, 1000*1000); + } else { + result = WaitForSingleObject (*h, + (unsigned) (NSYNC_TIME_SEC (delay) * 1000 + + (NSYNC_TIME_NSEC (delay) + 999999) / (1000 * 1000))); + } + } + if (result == kNtWaitTimeout) { + now = nsync_time_now (); + } + } while (result == kNtWaitTimeout && /* Windows generates early wakeups. */ + nsync_time_cmp (abs_deadline, now) > 0); + } + return (result == kNtWaitTimeout ? ETIMEDOUT : 0); +} + +/* Ensure that the count of *s is at least 1. */ +void nsync_mu_semaphore_v_win32 (nsync_semaphore *s) { + int64_t *h = (int64_t *) s; + ReleaseSemaphore(*h, 1, NULL); +} diff --git a/third_party/nsync/mu_wait.h b/third_party/nsync/mu_wait.h new file mode 100644 index 000000000..7a5a3478b --- /dev/null +++ b/third_party/nsync/mu_wait.h @@ -0,0 +1,118 @@ +#ifndef NSYNC_MU_WAIT_H_ +#define NSYNC_MU_WAIT_H_ +#include "third_party/nsync/mu.h" +#include "third_party/nsync/time.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +/* nsync_mu_wait() and nsync_mu_wait_with_deadline() can be used instead + of condition variables. In many straightforward situations they are + of equivalent performance and are somewhat easier to use, because + unlike condition variables, they do not require that the waits be + placed in a loop, and they do not require explicit wakeup calls. + Example: + + Definitions: + + static nsync_mu mu = NSYNC_MU_INIT; + static int i = 0; // protected by mu + // Condition for use with nsync_mu_wait(). + static int int_is_zero (const void *v) { + return (*(const int *)v == 0); + } + + Waiter: + + nsync_mu_lock (&mu); + // Wait until i is zero. + nsync_mu_wait (&mu, &int_is_zero, &i, NULL); + // i is known to be zero here. + // ... + nsync_mu_unlock (&mu); + + Thread potentially making i zero: + + nsync_mu_lock (&mu); + i--; + // No need to signal that i may have become zero. The unlock call + // below will evaluate waiters' conditions to decide which to wake. + nsync_mu_unlock (&mu); + + It is legal to use conditional critical sections and condition + variables on the same mutex. + + -------------- + + The implementation benefits from determining whether waiters are + waiting for the same condition; it may then evaluate a condition once + on behalf of several waiters. Two waiters have equal condition if + their "condition" pointers are equal, and either: + + - their "condition_arg" pointers are equal, or + + - "condition_arg_eq" is non-null and (*condition_arg_eq) + (condition_arg0, condition_arg1) returns non-zero. + + *condition_arg_eq will not be invoked unless the "condition" pointers + are equal, and the "condition_arg" pointers are unequal. + + If many waiters wait for distinct conditions simultaneously, + condition variables may be faster. + */ + +struct nsync_note_s_; /* forward declaration for an nsync_note */ + +/* Return when (*condition) (condition_arg) is true. Perhaps unlock and + relock *mu while blocked waiting for the condition to become true. + nsync_mu_wait() is equivalent to nsync_mu_wait_with_deadline() with + abs_deadline==nsync_time_no_deadline, and cancel_note==NULL. + + Requires that *mu be held on entry. See nsync_mu_wait_with_deadline() + for more details on *condition and *condition_arg_eq. */ +void nsync_mu_wait(nsync_mu *mu, int (*condition)(const void *condition_arg), + const void *condition_arg, + int (*condition_arg_eq)(const void *a, const void *b)); + +/* Return when at least one of: (*condition) (condition_arg) is true, + the deadline expires, or *cancel_note is notified. Perhaps unlock and + relock *mu while blocked waiting for one of these events, but always + return with *mu held. Return 0 iff the (*condition) (condition_arg) + is true on return, and otherwise either ETIMEDOUT or ECANCELED, + depending on why the call returned early. Callers should use + abs_deadline==nsync_time_no_deadline for no deadline, and + cancel_note==NULL for no cancellation. + + Requires that *mu be held on entry. + + The implementation may call *condition from any thread using the + mutex, and while holding *mu in either read or write mode; it + guarantees that any thread calling *condition will hold *mu in some + mode. Requires that (*condition) (condition_arg) neither modify state + protected by *mu, nor return a value dependent on state not protected + by *mu. To depend on time, use the abs_deadline parameter. + (Conventional use of condition variables have the same restrictions + on the conditions tested by the while-loop.) If non-null, + condition_arg_eq should return whether two condition_arg calls with + the same "condition" pointer are considered equivalent; it should + have no side-effects. */ +int nsync_mu_wait_with_deadline( + nsync_mu *mu, int (*condition)(const void *condition_arg), + const void *condition_arg, + int (*condition_arg_eq)(const void *a, const void *b), + nsync_time abs_deadline, struct nsync_note_s_ *cancel_note); + +/* Unlock *mu, which must be held in write mode, and wake waiters, if + appropriate. Unlike nsync_mu_unlock(), this call is not required to + wake nsync_mu_wait/nsync_mu_wait_with_deadline calls on conditions + that were false before this thread acquired the lock. This call + should be used only at the end of critical sections for which: + - nsync_mu_wait and/or nsync_mu_wait_with_deadline are in use on the same + mutex, + - this critical section cannot make the condition true for any of those + nsync_mu_wait/nsync_mu_wait_with_deadline waits, and + - when performance is significantly improved by using this call. */ +void nsync_mu_unlock_without_wakeup(nsync_mu *mu); + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* NSYNC_MU_WAIT_H_ */ diff --git a/third_party/nsync/note.h b/third_party/nsync/note.h new file mode 100644 index 000000000..213a2d70a --- /dev/null +++ b/third_party/nsync/note.h @@ -0,0 +1,51 @@ +#ifndef NSYNC_NOTE_H_ +#define NSYNC_NOTE_H_ +#include "third_party/nsync/time.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +/* An nsync_note represents a single bit that can transition from 0 to 1 + at most once. When 1, the note is said to be notified. There are + operations to wait for the transition, which can be triggered either + by an explicit call, or timer expiry. Notes can have parent notes; a + note becomes notified if its parent becomes notified. */ +typedef struct nsync_note_s_ *nsync_note; + +/* Return a freshly allocated nsync_note, or NULL if an nsync_note + cannot be created. + + If parent!=NULL, the allocated nsync_note's parent will be parent. + The newaly allocated note will be automatically notified at + abs_deadline, and is notified at initialization if + abs_deadline==nsync_zero_time. + + nsync_notes should be passed to nsync_note_free() when no longer needed. */ +nsync_note nsync_note_new(nsync_note parent, nsync_time abs_deadline); + +/* Free resources associated with n. Requires that n was allocated by + nsync_note_new(), and no concurrent or future operations are applied + to n directly. + + It is legal to call nsync_note_free() on a node even if it has a + parent or children that are in use; if n has both a parent and + children, n's parent adopts its children. */ +void nsync_note_free(nsync_note n); + +/* Notify n and all its descendants. */ +void nsync_note_notify(nsync_note n); + +/* Return whether n has been notified. */ +int nsync_note_is_notified(nsync_note n); + +/* Wait until n has been notified or abs_deadline is reached, and return + whether n has been notified. If abs_deadline==nsync_time_no_deadline, + the deadline is far in the future. */ +int nsync_note_wait(nsync_note n, nsync_time abs_deadline); + +/* Return the expiry time associated with n. This is the minimum of the + abs_deadline passed on creation and that of any of its ancestors. */ +nsync_time nsync_note_expiry(nsync_note n); + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* NSYNC_NOTE_H_ */ diff --git a/third_party/nsync/nsync.mk b/third_party/nsync/nsync.mk new file mode 100644 index 000000000..89e03a84a --- /dev/null +++ b/third_party/nsync/nsync.mk @@ -0,0 +1,58 @@ +#-*-mode:makefile-gmake;indent-tabs-mode:t;tab-width:8;coding:utf-8-*-┐ +#───vi: set et ft=make ts=8 tw=8 fenc=utf-8 :vi───────────────────────┘ + +PKGS += THIRD_PARTY_NSYNC + +THIRD_PARTY_NSYNC_SRCS = $(THIRD_PARTY_NSYNC_A_SRCS) +THIRD_PARTY_NSYNC_HDRS = $(THIRD_PARTY_NSYNC_A_HDRS) + +THIRD_PARTY_NSYNC_ARTIFACTS += THIRD_PARTY_NSYNC_A +THIRD_PARTY_NSYNC = $(THIRD_PARTY_NSYNC_A_DEPS) $(THIRD_PARTY_NSYNC_A) +THIRD_PARTY_NSYNC_A = o/$(MODE)/third_party/nsync/nsync.a +THIRD_PARTY_NSYNC_A_FILES := $(wildcard third_party/nsync/*) +THIRD_PARTY_NSYNC_A_HDRS = $(filter %.h,$(THIRD_PARTY_NSYNC_A_FILES)) +THIRD_PARTY_NSYNC_A_SRCS = $(filter %.c,$(THIRD_PARTY_NSYNC_A_FILES)) + +THIRD_PARTY_NSYNC_A_OBJS = \ + $(THIRD_PARTY_NSYNC_A_SRCS:%.c=o/$(MODE)/%.o) + +THIRD_PARTY_NSYNC_A_DIRECTDEPS = \ + LIBC_CALLS \ + LIBC_INTRIN \ + LIBC_NEXGEN32E \ + LIBC_NT_KERNEL32 \ + LIBC_STR \ + LIBC_STUBS \ + LIBC_SYSV \ + LIBC_SYSV_CALLS + +THIRD_PARTY_NSYNC_A_DEPS := \ + $(call uniq,$(foreach x,$(THIRD_PARTY_NSYNC_A_DIRECTDEPS),$($(x)))) + +THIRD_PARTY_NSYNC_A_CHECKS = \ + $(THIRD_PARTY_NSYNC_A).pkg \ + $(THIRD_PARTY_NSYNC_A_HDRS:%=o/$(MODE)/%.ok) + +$(THIRD_PARTY_NSYNC_A): \ + third_party/nsync/ \ + $(THIRD_PARTY_NSYNC_A).pkg \ + $(THIRD_PARTY_NSYNC_A_OBJS) + +$(THIRD_PARTY_NSYNC_A).pkg: \ + $(THIRD_PARTY_NSYNC_A_OBJS) \ + $(foreach x,$(THIRD_PARTY_NSYNC_A_DIRECTDEPS),$($(x)_A).pkg) + +$(THIRD_PARTY_NSYNC_A_OBJS): private \ + OVERRIDE_CCFLAGS += \ + -ffunction-sections \ + -fdata-sections + +THIRD_PARTY_NSYNC_LIBS = $(foreach x,$(THIRD_PARTY_NSYNC_ARTIFACTS),$($(x))) +THIRD_PARTY_NSYNC_SRCS = $(foreach x,$(THIRD_PARTY_NSYNC_ARTIFACTS),$($(x)_SRCS)) +THIRD_PARTY_NSYNC_CHECKS = $(foreach x,$(THIRD_PARTY_NSYNC_ARTIFACTS),$($(x)_CHECKS)) +THIRD_PARTY_NSYNC_OBJS = $(foreach x,$(THIRD_PARTY_NSYNC_ARTIFACTS),$($(x)_OBJS)) +$(THIRD_PARTY_NSYNC_OBJS): third_party/nsync/nsync.mk + +.PHONY: o/$(MODE)/third_party/nsync +o/$(MODE)/third_party/nsync: $(THIRD_PARTY_NSYNC_CHECKS) + diff --git a/third_party/nsync/once.h b/third_party/nsync/once.h new file mode 100644 index 000000000..4144df129 --- /dev/null +++ b/third_party/nsync/once.h @@ -0,0 +1,37 @@ +#ifndef NSYNC_ONCE_H_ +#define NSYNC_ONCE_H_ +#include "third_party/nsync/atomic.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +/* An nsync_once allows a function to be called exactly once, when first + referenced. */ +typedef nsync_atomic_uint32_ nsync_once; + +/* An initializer for nsync_once; it is guaranteed to be all zeroes. */ +#define NSYNC_ONCE_INIT NSYNC_ATOMIC_UINT32_INIT_ + +/* The first time nsync_run_once() or nsync_run_once_arg() is applied to + *once, the supplied function is run (with argument, in the case of + nsync_run_once_arg()). Other callers will wait until the run of the + function is complete, and then return without running the function + again. */ +void nsync_run_once(nsync_once *once, void (*f)(void)); +void nsync_run_once_arg(nsync_once *once, void (*farg)(void *arg), void *arg); + +/* Same as nsync_run_once()/nsync_run_once_arg() but uses a spinloop. + Can be used on the same nsync_once as + nsync_run_once/nsync_run_once_arg(). + + These *_spin variants should be used only in contexts where normal + blocking is disallowed, such as within user-space schedulers, when + the runtime is not fully initialized, etc. They provide no + significant performance benefit, and they should be avoided in normal + code. */ +void nsync_run_once_spin(nsync_once *once, void (*f)(void)); +void nsync_run_once_arg_spin(nsync_once *once, void (*farg)(void *arg), + void *arg); + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* NSYNC_ONCE_H_ */ diff --git a/third_party/nsync/panic.c b/third_party/nsync/panic.c new file mode 100644 index 000000000..6606ab06e --- /dev/null +++ b/third_party/nsync/panic.c @@ -0,0 +1,30 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" +#include "third_party/nsync/common.internal.h" +// clang-format off + +/* Aborts after printing the nul-terminated string s[]. */ +void nsync_panic_ (const char *s) { + size_t n = 0; + while (s[n]) ++n; + write (2, "panic: ", 7); + write (2, s, n); + notpossible; +} diff --git a/third_party/nsync/time.c b/third_party/nsync/time.c new file mode 100644 index 000000000..d401c23cc --- /dev/null +++ b/third_party/nsync/time.c @@ -0,0 +1,54 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2016 Google Inc. │ +│ │ +│ Licensed under the Apache License, Version 2.0 (the "License"); │ +│ you may not use this file except in compliance with the License. │ +│ You may obtain a copy of the License at │ +│ │ +│ http://www.apache.org/licenses/LICENSE-2.0 │ +│ │ +│ Unless required by applicable law or agreed to in writing, software │ +│ distributed under the License is distributed on an "AS IS" BASIS, │ +│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ +│ See the License for the specific language governing permissions and │ +│ limitations under the License. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/struct/timespec.h" +#include "libc/str/str.h" +#include "libc/sysv/consts/clock.h" +#include "third_party/nsync/time.h" + +asm(".ident\t\"\\n\\n\ +*NSYNC (Apache 2.0)\\n\ +Copyright 2016 Google, Inc.\\n\ +https://github.com/google/nsync\""); +// clang-format off + +#define NSYNC_NS_IN_S_ (1000 * 1000 * 1000) + +/* Return the maximum t, assuming it's an integral + type, and the representation is not too strange. */ +#define MAX_INT_TYPE(t) (((t)~(t)0) > 1? /*is t unsigned?*/ \ + (t)~(t)0 : /*unsigned*/ \ + (t) ((((uintmax_t)1) << (sizeof (t) * CHAR_BIT - 1)) - 1)) /*signed*/ + +const nsync_time nsync_time_no_deadline = + NSYNC_TIME_STATIC_INIT (MAX_INT_TYPE (int64_t), NSYNC_NS_IN_S_ - 1); + +const nsync_time nsync_time_zero = NSYNC_TIME_STATIC_INIT (0, 0); + +nsync_time nsync_time_sleep (nsync_time delay) { + struct timespec ts; + struct timespec remain; + memset (&ts, 0, sizeof (ts)); + ts.tv_sec = NSYNC_TIME_SEC (delay); + ts.tv_nsec = NSYNC_TIME_NSEC (delay); + if (nanosleep (&ts, &remain) == 0) { + /* nanosleep() is not required to fill in "remain" + if it returns 0. */ + memset (&remain, 0, sizeof (remain)); + } + return (remain); +} diff --git a/third_party/nsync/time.h b/third_party/nsync/time.h new file mode 100644 index 000000000..2529d5d87 --- /dev/null +++ b/third_party/nsync/time.h @@ -0,0 +1,53 @@ +#ifndef NSYNC_TIME_H_ +#define NSYNC_TIME_H_ +#include "libc/calls/struct/timespec.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +#define NSYNC_TIME_SEC(t) ((t).tv_sec) +#define NSYNC_TIME_NSEC(t) ((t).tv_nsec) +#define NSYNC_TIME_STATIC_INIT(t, ns) \ + { (t), (ns) } + +/* The type nsync_time represents the interval elapsed between two + moments in time. Often the first such moment is an address-space-wide + epoch, such as the Unix epoch, but clients should not rely on the + epoch in one address space being the same as that in another. + Intervals relative to the epoch are known as absolute times. */ +typedef struct timespec nsync_time; + +/* A deadline infinitely far in the future. */ +extern const nsync_time nsync_time_no_deadline; + +/* The zero delay, or an expired deadline. */ +extern const nsync_time nsync_time_zero; + +/* Return the current time since the epoch. */ +#define nsync_time_now() _timespec_real() + +/* Sleep for the specified delay. Returns the unslept time which may be + non-zero if the call was interrupted. */ +nsync_time nsync_time_sleep(nsync_time delay); + +/* Return a+b */ +#define nsync_time_add(a, b) _timespec_add(a, b) + +/* Return a-b */ +#define nsync_time_sub(a, b) _timespec_sub(a, b) + +/* Return +ve, 0, or -ve according to whether a>b, a==b, or a the waiter is waiting */ + struct nsync_semaphore_s_ *sem; /* *sem will be Ved when waiter is woken */ + uint32_t flags; /* see below */ +}; + +/* set if waiter is embedded in Mu/CV's internal structures */ +#define NSYNC_WAITER_FLAG_MUCV 0x1 + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* COSMOPOLITAN_LIBC_THREAD_WAIT_INTERNAL_H_ */ diff --git a/third_party/nsync/waiter.h b/third_party/nsync/waiter.h new file mode 100644 index 000000000..057591f13 --- /dev/null +++ b/third_party/nsync/waiter.h @@ -0,0 +1,138 @@ +#ifndef NSYNC_WAITER_H_ +#define NSYNC_WAITER_H_ +#include "third_party/nsync/time.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +/* nsync_wait_n() allows the client to wait on multiple objects + (condition variables, nsync_notes, nsync_counters, etc.) until at + least one of them becomes ready, or a deadline expires. + + It can be thought of as rather like Unix's select() or poll(), except + the the objects being waited for are synchronization data structures, + rather than file descriptors. + + The client can construct new objects that can be waited for by + implementing three routines. + + Examples: + + To wait on two nsync_notes n0, n1, and a nsync_counter c0, with a + deadline of abs_deadline: + + // Form an array of struct nsync_waitable_s, identifying the + // objects and the corresponding descriptors. (Static + // initialization syntax is used for brevity.) + static struct nsync_waitable_s w[] = { + { &n0, &nsync_note_waitable_funcs }, + { &n1, &nsync_note_waitable_funcs }, + { &c0, &nsync_counter_waitable_funcs } + }; + static struct nsync_waitable_s *pw[] = { &w[0], &w[1], &w[2] }; + int n = sizeof (w) / sizeof (w[0]); + + // Wait. The mu, lock, and unlock arguments are NULL because no + // condition variables are invovled. + int i = nsync_wait_n (NULL, NULL, NULL, abs_deadline, n, pw); + if (i == n) { + // timeout + } else { + // w[i].v became ready. + } + + To wait on multiple condition variables, the mu/lock/unlock + parameters are used. Imagine cv0 and cv1 are signalled when + predicates pred0() (under lock mu0) and pred1() (under lock mu1) + become true respectively. Assume that mu0 is acquired before mu1. + + static void lock2 (void *v) { // lock two mutexes in order + nsync_mu **mu = (nsync_mu **) v; + nsync_mu_lock (mu[0]); + nsync_mu_lock (mu[1]); + } + static void unlock2 (void *v) { // unlock two mutexes. + nsync_mu **mu = (nsync_mu **) v; + nsync_mu_unlock (mu[1]); + nsync_mu_unlock (mu[0]); + } + + // Describe the condition variables and the locks. + static struct nsync_waitable_s w[] = { + { &cv0, &nsync_cv_waitable_funcs }, + { &cv1, &nsync_cv_waitable_funcs } + }; + static struct nsync_waitable_s *pw[] = { &w[0], &w[1] }; + nsync_mu *lock_list[] = { &mu0, &mu1 }; + int n = sizeof (w) / sizeof (w[0]); + + lock2 (list_list); + while (!pred0 () && !pred1 ()) { + // Wait for one of the condition variables to be signalled, + // with no timeout. + nsync_wait_n (lock_list, &lock2, &unlock2, + nsync_time_no_deadline, n, pw); + } + if (pred0 ()) { ... } + if (pred1 ()) { ... } + unlock2 (list_list); + + */ + +/* forward declaration of struct that contains type dependent wait + operations */ +struct nsync_waitable_funcs_s; + +/* Clients wait on objects by forming an array of struct + nsync_waitable_s. Each each element points to one object and its + type-dependent functions. */ +struct nsync_waitable_s { + /* pointer to object */ + void *v; + /* pointer to type-dependent functions. Use + &nsync_note_waitable_funcs for an nsync_note, + &nsync_counternote_waitable_funcs for an nsync_counter, + &nsync_cv_waitable_funcs for an nsync_cv. */ + const struct nsync_waitable_funcs_s *funcs; +}; + +/* Wait until at least one of *waitable[0,..,count-1] is has been + notified, or abs_deadline is reached. Return the index of the + notified element of waitable[], or count if no such element exists. + If mu!=NULL, (*unlock)(mu) is called after the thread is queued on + the various waiters, and (*lock)(mu) is called before return; + mu/lock/unlock are used to acquire and release the relevant locks + whan waiting on condition variables. */ +int nsync_wait_n(void *mu, void (*lock)(void *), void (*unlock)(void *), + nsync_time abs_deadline, int count, + struct nsync_waitable_s *waitable[]); + +/* A "struct nsync_waitable_s" implementation must implement these + functions. Clients should ignore the internals. */ +struct nsync_waiter_s; +struct nsync_waitable_funcs_s { + + /* Return the time when *v will be ready (max time if unknown), or 0 + if it is already ready. The parameter nw may be passed as NULL, in + which case the result should indicate whether the thread would + block if it were to wait on *v. All calls with the same *v must + report the same result until the object becomes ready, from which + point calls must report 0. */ + nsync_time (*ready_time)(void *v, struct nsync_waiter_s *nw); + + /* If *v is ready, return zero; otherwise enqueue *nw on *v and return + non-zero. */ + int (*enqueue)(void *v, struct nsync_waiter_s *nw); + + /* If nw has been previously dequeued, return zero; otherwise dequeue + *nw from *v and return non-zero. */ + int (*dequeue)(void *v, struct nsync_waiter_s *nw); +}; + +/* The "struct nsync_waitable_s" for nsync_note, nsync_counter, and nsync_cv. */ +extern const struct nsync_waitable_funcs_s nsync_note_waitable_funcs; +extern const struct nsync_waitable_funcs_s nsync_counter_waitable_funcs; +extern const struct nsync_waitable_funcs_s nsync_cv_waitable_funcs; + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* NSYNC_WAITER_H_ */ diff --git a/third_party/nsync/waiter_per_thread.c b/third_party/nsync/waiter_per_thread.c new file mode 100644 index 000000000..3ac2772f1 --- /dev/null +++ b/third_party/nsync/waiter_per_thread.c @@ -0,0 +1,56 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2016 Google Inc. │ +│ │ +│ Licensed under the Apache License, Version 2.0 (the "License"); │ +│ you may not use this file except in compliance with the License. │ +│ You may obtain a copy of the License at │ +│ │ +│ http://www.apache.org/licenses/LICENSE-2.0 │ +│ │ +│ Unless required by applicable law or agreed to in writing, software │ +│ distributed under the License is distributed on an "AS IS" BASIS, │ +│ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ +│ See the License for the specific language governing permissions and │ +│ limitations under the License. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "third_party/nsync/atomic.h" +#include "third_party/nsync/atomic.internal.h" +#include "third_party/nsync/common.internal.h" +#include "libc/thread/thread.h" + +asm(".ident\t\"\\n\\n\ +*NSYNC (Apache 2.0)\\n\ +Copyright 2016 Google, Inc.\\n\ +https://github.com/google/nsync\""); +// clang-format off + +static pthread_key_t waiter_key; +static nsync_atomic_uint32_ pt_once; + +static void do_once (nsync_atomic_uint32_ *ponce, void (*dest) (void *)) { + uint32_t o = ATM_LOAD_ACQ (ponce); + if (o != 2) { + while (o == 0 && !ATM_CAS_ACQ (ponce, 0, 1)) { + o = ATM_LOAD (ponce); + } + if (o == 0) { + pthread_key_create (&waiter_key, dest); + ATM_STORE_REL (ponce, 2); + } + while (ATM_LOAD_ACQ (ponce) != 2) { + nsync_yield_ (); + } + } +} + +void *nsync_per_thread_waiter_ (void (*dest) (void *)) { + do_once (&pt_once, dest); + return (pthread_getspecific (waiter_key)); +} + +void nsync_set_per_thread_waiter_ (void *v, void (*dest) (void *)) { + do_once (&pt_once, dest); + pthread_setspecific (waiter_key, v); +} diff --git a/third_party/nsync/yield.c b/third_party/nsync/yield.c new file mode 100644 index 000000000..b6183a85c --- /dev/null +++ b/third_party/nsync/yield.c @@ -0,0 +1,27 @@ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" +#include "libc/calls/strace.internal.h" +#include "third_party/nsync/common.internal.h" +// clang-format off + +void nsync_yield_ (void) { + sched_yield (); + STRACE ("nsync_yield_()"); +} diff --git a/third_party/third_party.mk b/third_party/third_party.mk index d4c57215a..e42a207c0 100644 --- a/third_party/third_party.mk +++ b/third_party/third_party.mk @@ -20,6 +20,7 @@ o/$(MODE)/third_party: \ o/$(MODE)/third_party/maxmind \ o/$(MODE)/third_party/mbedtls \ o/$(MODE)/third_party/musl \ + o/$(MODE)/third_party/nsync \ o/$(MODE)/third_party/python \ o/$(MODE)/third_party/quickjs \ o/$(MODE)/third_party/regex \ diff --git a/tool/build/mkdeps.c b/tool/build/mkdeps.c index 60db8e297..b55bc74b2 100644 --- a/tool/build/mkdeps.c +++ b/tool/build/mkdeps.c @@ -24,9 +24,7 @@ #include "libc/fmt/fmt.h" #include "libc/intrin/bits.h" #include "libc/intrin/kprintf.h" -#include "libc/thread/thread.h" #include "libc/intrin/safemacros.internal.h" -#include "libc/intrin/wait0.internal.h" #include "libc/log/check.h" #include "libc/log/log.h" #include "libc/macros.internal.h" @@ -37,7 +35,6 @@ #include "libc/mem/bisectcarleft.internal.h" #include "libc/mem/mem.h" #include "libc/nexgen32e/crc32.h" -#include "libc/thread/tls.h" #include "libc/runtime/ezmap.internal.h" #include "libc/runtime/gc.internal.h" #include "libc/runtime/runtime.h" @@ -52,6 +49,9 @@ #include "libc/sysv/consts/o.h" #include "libc/sysv/consts/prot.h" #include "libc/thread/spawn.h" +#include "libc/thread/thread.h" +#include "libc/thread/tls.h" +#include "libc/thread/wait0.internal.h" #include "libc/time/time.h" #include "libc/x/x.h" #include "third_party/getopt/getopt.h" diff --git a/tool/build/pstrace.c b/tool/build/pstrace.c index 78bb2c572..3ba54a7ac 100644 --- a/tool/build/pstrace.c +++ b/tool/build/pstrace.c @@ -16,12 +16,12 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/intrin/bits.h" #include "libc/calls/calls.h" #include "libc/calls/struct/sigaction.h" #include "libc/calls/struct/sigset.h" #include "libc/errno.h" #include "libc/fmt/conv.h" +#include "libc/intrin/bits.h" #include "libc/log/check.h" #include "libc/log/log.h" #include "libc/mem/mem.h" @@ -115,7 +115,7 @@ struct Trace { EK_KILLED, // ret is signal code } kind; unsigned char arity; - unsigned char syscall; + unsigned char syscall_; bool is_interrupted; int us; int elap; @@ -675,7 +675,7 @@ static void Parse(struct Trace *t, const char *line, long lineno) { return; } else if (isalpha(*p) && (q = strchr(p, '('))) { t->events.p[event].kind = EK_CALL; - CHECK_NE(-1, (t->events.p[event].syscall = GetSyscall(p, q - p)), DEBUG); + CHECK_NE(-1, (t->events.p[event].syscall_ = GetSyscall(p, q - p)), DEBUG); p = q + 1; } } @@ -825,8 +825,8 @@ static void PrintEvent(FILE *f, struct Trace *t, long ev) { fprintf(f, "%d", t->events.p[ev].ret); break; case EK_CALL: - CHECK_LT(t->events.p[ev].syscall, ARRAYLEN(kSyscalls)); - fprintf(f, "(b%`'s,%ld,", kSyscalls[t->events.p[ev].syscall].name, + CHECK_LT(t->events.p[ev].syscall_, ARRAYLEN(kSyscalls)); + fprintf(f, "(b%`'s,%ld,", kSyscalls[t->events.p[ev].syscall_].name, t->events.p[ev].ret); fprintf(f, "%c", t->events.p[ev].arity && t->events.p[ev].arg[0].name != -1 ? '{' diff --git a/tool/net/redbean.c b/tool/net/redbean.c index 2f053c5cc..15775acff 100644 --- a/tool/net/redbean.c +++ b/tool/net/redbean.c @@ -16,6 +16,7 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/calls/ioctl.h" #include "libc/calls/pledge.h" @@ -36,7 +37,6 @@ #include "libc/intrin/atomic.h" #include "libc/intrin/likely.h" #include "libc/intrin/nomultics.internal.h" -#include "libc/thread/thread.h" #include "libc/intrin/safemacros.internal.h" #include "libc/log/check.h" #include "libc/log/log.h" @@ -48,7 +48,6 @@ #include "libc/nexgen32e/crc32.h" #include "libc/nexgen32e/nt2sysv.h" #include "libc/nexgen32e/rdtsc.h" -#include "libc/thread/tls.h" #include "libc/nexgen32e/x86feature.h" #include "libc/nt/enum/fileflagandattributes.h" #include "libc/nt/thread.h" @@ -91,6 +90,8 @@ #include "libc/sysv/consts/w.h" #include "libc/sysv/errfuns.h" #include "libc/thread/spawn.h" +#include "libc/thread/thread.h" +#include "libc/thread/tls.h" #include "libc/x/x.h" #include "libc/zip.h" #include "net/http/escape.h" @@ -418,8 +419,8 @@ static bool hasonprocessdestroy; static bool loggednetworkorigin; static bool ishandlingconnection; static bool hasonclientconnection; +static atomic_bool terminatemonitor; static bool evadedragnetsurveillance; -static _Atomic(bool) terminatemonitor; static int zfd; static int gmtoff;