From a359de7893c33bfbd99aa82d570f97c55bf2b935 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Mon, 11 Sep 2023 21:34:53 -0700 Subject: [PATCH] Get rid of kmalloc() This changes *NSYNC to allocate waiters on the stack so our locks don't need to depend on dynamic memory. This make our runtiem simpler, and it also fixes bugs with thread cancellation support. --- ape/ape-m1.c | 7 +- ape/ape.h | 2 +- libc/calls/execve.c | 2 +- libc/calls/fcntl-nt.c | 11 +- libc/calls/ntspawn.c | 2 - libc/calls/syscall-sysv.internal.h | 1 + libc/calls/tcsetattr-nt.c | 2 + libc/calls/tkill.c | 35 ++- libc/intrin/asan.c | 4 +- libc/intrin/describeframe.c | 2 - libc/intrin/kmalloc.c | 95 -------- libc/intrin/kmalloc.h | 17 -- libc/intrin/leaky.internal.h | 9 +- libc/intrin/pthread_mutex_trylock.c | 1 + .../pthread_mutexattr_destroy.c | 0 .../pthread_mutexattr_getpshared.c | 0 .../pthread_mutexattr_gettype.c | 0 .../pthread_mutexattr_init.c | 0 .../pthread_mutexattr_setpshared.c | 0 .../pthread_mutexattr_settype.c | 0 libc/intrin/putenv.c | 10 +- libc/intrin/setenv.c | 10 +- libc/log/oncrash_amd64.c | 9 +- libc/runtime/fork-nt.c | 1 - libc/runtime/memtrack.internal.h | 29 +-- libc/runtime/memtrack64.txt | 2 +- libc/runtime/syslib.internal.h | 1 + libc/runtime/zipos-get.c | 3 +- libc/stdio/alloc.c | 45 +--- libc/stdio/internal.h | 31 +-- libc/thread/pthread_atfork.c | 13 +- libc/thread/tls.h | 2 +- libc/x/xload.c | 4 +- test/libc/calls/sched_getaffinity_test.c | 2 + test/libc/stdio/popen_test.c | 1 - test/libc/stdio/posix_spawn_test.c | 2 +- test/libc/thread/pthread_cancel_test.c | 9 +- third_party/finger/util.c | 4 +- third_party/nsync/README.cosmo | 25 +- third_party/nsync/common.c | 91 ++----- third_party/nsync/common.internal.h | 8 +- third_party/nsync/mem/nsync_cv.c | 229 ++++++++++-------- third_party/nsync/mem/nsync_mu_wait.c | 18 +- third_party/nsync/mem/nsync_wait.c | 10 +- third_party/nsync/mu.c | 17 +- third_party/nsync/mu_semaphore.c | 30 ++- third_party/nsync/mu_semaphore.h | 3 + third_party/nsync/mu_semaphore.internal.h | 2 + third_party/nsync/mu_semaphore_gcd.c | 10 + third_party/nsync/mu_semaphore_sem.c | 22 +- third_party/nsync/testing/cv_test.c | 2 +- .../nsync/testing/cv_wait_example_test.c | 2 +- third_party/nsync/testing/dll_test.c | 3 +- .../nsync/testing/mu_wait_example_test.c | 2 +- third_party/nsync/testing/smprintf.c | 6 +- .../python/Modules/cjkcodecs/xloadzd.c | 3 +- tool/build/runitd.c | 26 +- 57 files changed, 405 insertions(+), 472 deletions(-) delete mode 100644 libc/intrin/kmalloc.c delete mode 100644 libc/intrin/kmalloc.h rename libc/{thread => intrin}/pthread_mutexattr_destroy.c (100%) rename libc/{thread => intrin}/pthread_mutexattr_getpshared.c (100%) rename libc/{thread => intrin}/pthread_mutexattr_gettype.c (100%) rename libc/{thread => intrin}/pthread_mutexattr_init.c (100%) rename libc/{thread => intrin}/pthread_mutexattr_setpshared.c (100%) rename libc/{thread => intrin}/pthread_mutexattr_settype.c (100%) diff --git a/ape/ape-m1.c b/ape/ape-m1.c index 3a09c80e2..5488357d9 100644 --- a/ape/ape-m1.c +++ b/ape/ape-m1.c @@ -33,7 +33,7 @@ #define pagesz 16384 #define SYSLIB_MAGIC ('s' | 'l' << 8 | 'i' << 16 | 'b' << 24) -#define SYSLIB_VERSION 1 +#define SYSLIB_VERSION 2 struct Syslib { int magic; @@ -56,6 +56,9 @@ struct Syslib { long (*dispatch_semaphore_signal)(dispatch_semaphore_t); long (*dispatch_semaphore_wait)(dispatch_semaphore_t, dispatch_time_t); dispatch_time_t (*dispatch_walltime)(const struct timespec *, int64_t); + /* v2 (2023-09-10) */ + pthread_t (*pthread_self)(void); + void (*dispatch_release)(dispatch_semaphore_t); }; #define ELFCLASS32 1 @@ -829,6 +832,8 @@ int main(int argc, char **argv, char **envp) { M->lib.dispatch_semaphore_signal = dispatch_semaphore_signal; M->lib.dispatch_semaphore_wait = dispatch_semaphore_wait; M->lib.dispatch_walltime = dispatch_walltime; + M->lib.pthread_self = pthread_self; + M->lib.dispatch_release = dispatch_release; /* getenv("_") is close enough to at_execfn */ execfn = argc > 0 ? argv[0] : 0; diff --git a/ape/ape.h b/ape/ape.h index 70cebe4c4..76935344e 100644 --- a/ape/ape.h +++ b/ape/ape.h @@ -2,7 +2,7 @@ #define COSMOPOLITAN_APE_APE_H_ #define APE_VERSION_MAJOR 1 -#define APE_VERSION_MINOR 7 +#define APE_VERSION_MINOR 8 #define APE_VERSION_STR APE_VERSION_STR_(APE_VERSION_MAJOR, APE_VERSION_MINOR) #define APE_VERSION_NOTE APE_VERSION_NOTE_(APE_VERSION_MAJOR, APE_VERSION_MINOR) diff --git a/libc/calls/execve.c b/libc/calls/execve.c index 3f2a53ae2..2bd7b2a75 100644 --- a/libc/calls/execve.c +++ b/libc/calls/execve.c @@ -71,7 +71,7 @@ int execve(const char *prog, char *const argv[], char *const envp[]) { rc = _weaken(sys_pledge_linux)(__execpromises, __pledge_mode); } if (!rc) { - if (_weaken(__zipos_parseuri) && + if (0 && _weaken(__zipos_parseuri) && (_weaken(__zipos_parseuri)(prog, &uri) != -1)) { rc = _weaken(__zipos_open)(&uri, O_RDONLY | O_CLOEXEC); if (rc != -1) { diff --git a/libc/calls/fcntl-nt.c b/libc/calls/fcntl-nt.c index 42c48815d..e9d928cc4 100644 --- a/libc/calls/fcntl-nt.c +++ b/libc/calls/fcntl-nt.c @@ -25,12 +25,13 @@ #include "libc/calls/syscall_support-nt.internal.h" #include "libc/calls/wincrash.internal.h" #include "libc/errno.h" -#include "libc/intrin/kmalloc.h" #include "libc/intrin/kprintf.h" +#include "libc/intrin/leaky.internal.h" #include "libc/intrin/weaken.h" #include "libc/limits.h" #include "libc/log/backtrace.internal.h" #include "libc/macros.internal.h" +#include "libc/mem/mem.h" #include "libc/nt/createfile.h" #include "libc/nt/enum/fileflagandattributes.h" #include "libc/nt/enum/filelockflags.h" @@ -72,7 +73,7 @@ static textwindows struct FileLock *NewFileLock(void) { fl = g_locks.free; g_locks.free = fl->next; } else { - fl = kmalloc(sizeof(*fl)); + unassert((fl = _weaken(malloc)(sizeof(*fl)))); } bzero(fl, sizeof(*fl)); fl->next = g_locks.list; @@ -80,6 +81,8 @@ static textwindows struct FileLock *NewFileLock(void) { return fl; } +IGNORE_LEAKS(NewFileLock) + static textwindows void FreeFileLock(struct FileLock *fl) { fl->next = g_locks.free; g_locks.free = fl; @@ -129,6 +132,10 @@ static textwindows int sys_fcntl_nt_lock(struct Fd *f, int fd, int cmd, int64_t pos, off, len, end; struct FileLock *fl, *ft, **flp; + if (!_weaken(malloc)) { + return enomem(); + } + l = (struct flock *)arg; len = l->l_len; off = l->l_start; diff --git a/libc/calls/ntspawn.c b/libc/calls/ntspawn.c index 3240d9140..2c41b44f2 100644 --- a/libc/calls/ntspawn.c +++ b/libc/calls/ntspawn.c @@ -87,8 +87,6 @@ textwindows int ntspawn( block = NULL; _init_sigchld(); if (__mkntpath(prog, prog16) == -1) return -1; - // we can't call malloc() because we're higher in the topological order - // we can't call kmalloc() because fork() calls this when kmalloc is locked if ((handle = CreateFileMapping(-1, 0, pushpop(kNtPageReadwrite), 0, sizeof(*block), 0)) && (block = MapViewOfFileEx(handle, kNtFileMapRead | kNtFileMapWrite, 0, 0, diff --git a/libc/calls/syscall-sysv.internal.h b/libc/calls/syscall-sysv.internal.h index 6ec1f3bd6..500886a48 100644 --- a/libc/calls/syscall-sysv.internal.h +++ b/libc/calls/syscall-sysv.internal.h @@ -87,6 +87,7 @@ i32 sys_sem_close(i64); i32 sys_sem_destroy(i64); i32 sys_sem_getvalue(i64, u32 *); i32 sys_sem_init(u32, i64 *); +i32 sys_sem_destroy(i64); i32 sys_sem_open(const char *, int, u32, i64 *); i32 sys_sem_post(i64); i32 sys_sem_trywait(i64); diff --git a/libc/calls/tcsetattr-nt.c b/libc/calls/tcsetattr-nt.c index 8d7acb02c..3227039fc 100644 --- a/libc/calls/tcsetattr-nt.c +++ b/libc/calls/tcsetattr-nt.c @@ -24,6 +24,7 @@ #include "libc/calls/termios.internal.h" #include "libc/calls/ttydefaults.h" #include "libc/dce.h" +#include "libc/errno.h" #include "libc/intrin/describeflags.internal.h" #include "libc/intrin/nomultics.internal.h" #include "libc/intrin/strace.internal.h" @@ -261,6 +262,7 @@ textwindows int tcsetattr_nt(int fd, int opt, const struct termios *tio) { __attribute__((__constructor__)) static void tcsetattr_nt_init(void) { if (!getenv("TERM")) { setenv("TERM", "xterm-256color", true); + errno = 0; // ignore malloc not linked } } diff --git a/libc/calls/tkill.c b/libc/calls/tkill.c index b66a605ab..2d4fa2cfd 100644 --- a/libc/calls/tkill.c +++ b/libc/calls/tkill.c @@ -98,16 +98,43 @@ static dontinline textwindows int __tkill_nt(int tid, int sig, } } -static int __tkill_m1(int tid, int sig, struct CosmoTib *tib) { - struct PosixThread *pt = (struct PosixThread *)__get_tls()->tib_pthread; - return __syslib->pthread_kill(pt->next, sig); +static int __tkill_posix(int tid, int sig, struct CosmoTib *tib) { + + // avoid lock when killing self + int me; + struct PosixThread *pt; + pt = (struct PosixThread *)__get_tls()->tib_pthread; + me = atomic_load_explicit(&__get_tls()->tib_tid, memory_order_relaxed); + if (tid == me && (!tib || tib == __get_tls())) { + return __syslib->pthread_kill(pt->next, sig); + } + + // otherwise look for matching thread + struct Dll *e; + pthread_spin_lock(&_pthread_lock); + for (e = dll_first(_pthread_list); e; e = dll_next(_pthread_list, e)) { + enum PosixThreadStatus status; + struct PosixThread *pt = POSIXTHREAD_CONTAINER(e); + int rhs = atomic_load_explicit(&pt->tib->tib_tid, memory_order_acquire); + if (rhs <= 0 || tid != rhs) continue; + if (tib && tib != pt->tib) continue; + status = atomic_load_explicit(&pt->status, memory_order_acquire); + pthread_spin_unlock(&_pthread_lock); + if (status < kPosixThreadTerminated) { + return __syslib->pthread_kill(pt->next, sig); + } else { + return 0; + } + } + pthread_spin_unlock(&_pthread_lock); + return esrch(); } // OpenBSD has an optional `tib` parameter for extra safety. int __tkill(int tid, int sig, void *tib) { int rc; if (IsXnuSilicon()) { - return __tkill_m1(tid, sig, tib); + return __tkill_posix(tid, sig, tib); } else if (IsLinux() || IsXnu() || IsFreebsd() || IsOpenbsd() || IsNetbsd()) { rc = sys_tkill(tid, sig, tib); } else if (IsWindows()) { diff --git a/libc/intrin/asan.c b/libc/intrin/asan.c index 5b2e6315f..9509c9f71 100644 --- a/libc/intrin/asan.c +++ b/libc/intrin/asan.c @@ -27,7 +27,6 @@ #include "libc/intrin/bits.h" #include "libc/intrin/cmpxchg.h" #include "libc/intrin/directmap.internal.h" -#include "libc/intrin/kmalloc.h" #include "libc/intrin/kprintf.h" #include "libc/intrin/leaky.internal.h" #include "libc/intrin/likely.h" @@ -835,10 +834,9 @@ static __wur __asan_die_f *__asan_report(const void *addr, int size, wint_t c; signed char t; uint64_t x, y, z; - char *p, *q, *buf, *base; struct MemoryIntervals *m; + char buf[8192], *base, *q, *p = buf; ftrace_enabled(-1); - p = buf = kmalloc(1024 * 1024); kprintf("\n\e[J\e[1;31masan error\e[0m: %s %d-byte %s at %p shadow %p\n", __asan_describe_access_poison(kind), size, message, addr, SHADOW(addr)); diff --git a/libc/intrin/describeframe.c b/libc/intrin/describeframe.c index 486b73c21..95228000b 100644 --- a/libc/intrin/describeframe.c +++ b/libc/intrin/describeframe.c @@ -43,8 +43,6 @@ static const char *GetFrameName(int x) { return "g_fds"; } else if (IsZiposFrame(x)) { return "zipos"; - } else if (IsKmallocFrame(x)) { - return "kmalloc"; } else if (IsMemtrackFrame(x)) { return "memtrack"; } else if (IsOldStackFrame(x)) { diff --git a/libc/intrin/kmalloc.c b/libc/intrin/kmalloc.c deleted file mode 100644 index 1f2202bcc..000000000 --- a/libc/intrin/kmalloc.c +++ /dev/null @@ -1,95 +0,0 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ -╞══════════════════════════════════════════════════════════════════════════════╡ -│ Copyright 2022 Justine Alexandra Roberts Tunney │ -│ │ -│ Permission to use, copy, modify, and/or distribute this software for │ -│ any purpose with or without fee is hereby granted, provided that the │ -│ above copyright notice and this permission notice appear in all copies. │ -│ │ -│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ -│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ -│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ -│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ -│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ -│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ -│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ -│ PERFORMANCE OF THIS SOFTWARE. │ -╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/intrin/kmalloc.h" -#include "libc/assert.h" -#include "libc/atomic.h" -#include "libc/calls/syscall-sysv.internal.h" -#include "libc/dce.h" -#include "libc/intrin/asan.internal.h" -#include "libc/intrin/atomic.h" -#include "libc/intrin/describebacktrace.internal.h" -#include "libc/intrin/extend.internal.h" -#include "libc/log/libfatal.internal.h" -#include "libc/macros.internal.h" -#include "libc/runtime/memtrack.internal.h" -#include "libc/sysv/consts/map.h" -#include "libc/thread/thread.h" -#include "libc/thread/tls.h" - -#define KMALLOC_ALIGN sizeof(intptr_t) - -static struct { - char *endptr; - size_t total; - pthread_spinlock_t lock; -} g_kmalloc; - -void __kmalloc_lock(void) { - pthread_spin_lock(&g_kmalloc.lock); -} - -void __kmalloc_unlock(void) { - pthread_spin_unlock(&g_kmalloc.lock); -} - -#ifdef _NOPL0 -#define __kmalloc_lock() _NOPL0("__threadcalls", __kmalloc_lock) -#define __kmalloc_unlock() _NOPL0("__threadcalls", __kmalloc_unlock) -#endif - -/** - * Allocates permanent memory. - * - * The code malloc() depends upon uses this function to allocate memory. - * The returned memory can't be freed, and leak detection is impossible. - * This function panics when memory isn't available. - * - * Memory returned by this function is aligned on the word size, and as - * such, kmalloc() shouldn't be used for vector operations. - * - * @return zero-initialized memory on success, or null w/ errno - * @raise ENOMEM if we require more vespene gas - */ -dontasan void *kmalloc(size_t size) { - char *p, *e; - size_t i, n, t; - n = ROUNDUP(size + (IsAsan() * 8), KMALLOC_ALIGN); - __kmalloc_lock(); - t = g_kmalloc.total; - e = g_kmalloc.endptr; - i = t; - t += n; - p = (char *)kMemtrackKmallocStart; - if (!e) e = p; - if ((e = _extend(p, t, e, MAP_PRIVATE, - kMemtrackKmallocStart + kMemtrackKmallocSize))) { - g_kmalloc.endptr = e; - g_kmalloc.total = t; - } else { - p = 0; - } - __kmalloc_unlock(); - if (p) { - unassert(!((intptr_t)(p + i) & (KMALLOC_ALIGN - 1))); - if (IsAsan()) __asan_poison(p + i + size, n - size, kAsanHeapOverrun); - return p + i; - } else { - return 0; - } -} diff --git a/libc/intrin/kmalloc.h b/libc/intrin/kmalloc.h deleted file mode 100644 index c97d8a47a..000000000 --- a/libc/intrin/kmalloc.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef COSMOPOLITAN_LIBC_INTRIN_KMALLOC_H_ -#define COSMOPOLITAN_LIBC_INTRIN_KMALLOC_H_ -#ifdef _COSMO_SOURCE -#define kmalloc __kmalloc -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -void *kmalloc(size_t) -mallocesque attributeallocsize((1)) returnsaligned((8)); - -void __kmalloc_lock(void); -void __kmalloc_unlock(void); - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* _COSMO_SOURCE */ -#endif /* COSMOPOLITAN_LIBC_INTRIN_KMALLOC_H_ */ diff --git a/libc/intrin/leaky.internal.h b/libc/intrin/leaky.internal.h index da18dbc2d..403eece31 100644 --- a/libc/intrin/leaky.internal.h +++ b/libc/intrin/leaky.internal.h @@ -1,12 +1,17 @@ #ifndef COSMOPOLITAN_LIBC_INTRIN_LEAKY_INTERNAL_H_ #define COSMOPOLITAN_LIBC_INTRIN_LEAKY_INTERNAL_H_ +#include "libc/dce.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ +#if IsAsan() #define IGNORE_LEAKS(FUNC) \ - __static_yoink("_leaky_start"); \ + __static_yoink("_leaky_start"); \ void *_leaky_##FUNC[] _Section(".piro.relo.sort.leaky.2." #FUNC \ - ",\"aw\",@init_array #") = {FUNC} + ",\"aw\",@init_array #") = {FUNC}; +#else +#define IGNORE_LEAKS(FUNC) +#endif extern intptr_t _leaky_end[] __attribute__((__weak__)); extern intptr_t _leaky_start[] __attribute__((__weak__)); diff --git a/libc/intrin/pthread_mutex_trylock.c b/libc/intrin/pthread_mutex_trylock.c index 585735426..93fc8e565 100644 --- a/libc/intrin/pthread_mutex_trylock.c +++ b/libc/intrin/pthread_mutex_trylock.c @@ -19,6 +19,7 @@ #include "libc/calls/calls.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" +#include "libc/intrin/kprintf.h" #include "libc/intrin/weaken.h" #include "libc/thread/thread.h" #include "libc/thread/tls.h" diff --git a/libc/thread/pthread_mutexattr_destroy.c b/libc/intrin/pthread_mutexattr_destroy.c similarity index 100% rename from libc/thread/pthread_mutexattr_destroy.c rename to libc/intrin/pthread_mutexattr_destroy.c diff --git a/libc/thread/pthread_mutexattr_getpshared.c b/libc/intrin/pthread_mutexattr_getpshared.c similarity index 100% rename from libc/thread/pthread_mutexattr_getpshared.c rename to libc/intrin/pthread_mutexattr_getpshared.c diff --git a/libc/thread/pthread_mutexattr_gettype.c b/libc/intrin/pthread_mutexattr_gettype.c similarity index 100% rename from libc/thread/pthread_mutexattr_gettype.c rename to libc/intrin/pthread_mutexattr_gettype.c diff --git a/libc/thread/pthread_mutexattr_init.c b/libc/intrin/pthread_mutexattr_init.c similarity index 100% rename from libc/thread/pthread_mutexattr_init.c rename to libc/intrin/pthread_mutexattr_init.c diff --git a/libc/thread/pthread_mutexattr_setpshared.c b/libc/intrin/pthread_mutexattr_setpshared.c similarity index 100% rename from libc/thread/pthread_mutexattr_setpshared.c rename to libc/intrin/pthread_mutexattr_setpshared.c diff --git a/libc/thread/pthread_mutexattr_settype.c b/libc/intrin/pthread_mutexattr_settype.c similarity index 100% rename from libc/thread/pthread_mutexattr_settype.c rename to libc/intrin/pthread_mutexattr_settype.c diff --git a/libc/intrin/putenv.c b/libc/intrin/putenv.c index 41d5bf4ed..7aa732410 100644 --- a/libc/intrin/putenv.c +++ b/libc/intrin/putenv.c @@ -17,11 +17,14 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/intrin/getenv.internal.h" -#include "libc/intrin/kmalloc.h" +#include "libc/intrin/leaky.internal.h" #include "libc/intrin/strace.internal.h" +#include "libc/intrin/weaken.h" #include "libc/macros.internal.h" #include "libc/mem/internal.h" +#include "libc/mem/mem.h" #include "libc/runtime/runtime.h" +#include "libc/sysv/errfuns.h" #define ToUpper(c) ((c) >= 'a' && (c) <= 'z' ? (c) - 'a' + 'A' : (c)) @@ -40,7 +43,7 @@ static char **GrowEnviron(char **a) { if (!a) a = environ; n = a ? GetEnvironLen(a) : 0; c = MAX(16ul, n) << 1; - if ((b = kmalloc(c * sizeof(char *)))) { + if (_weaken(malloc) && (b = _weaken(malloc)(c * sizeof(char *)))) { if (a) { for (p = b; *a;) { *p++ = *a++; @@ -51,10 +54,13 @@ static char **GrowEnviron(char **a) { capacity = c; return b; } else { + enomem(); return 0; } } +IGNORE_LEAKS(GrowEnviron) + int PutEnvImpl(char *s, bool overwrite) { char **p; struct Env e; diff --git a/libc/intrin/setenv.c b/libc/intrin/setenv.c index bed6840c1..5af9c4a95 100644 --- a/libc/intrin/setenv.c +++ b/libc/intrin/setenv.c @@ -16,9 +16,11 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/intrin/kmalloc.h" +#include "libc/intrin/leaky.internal.h" #include "libc/intrin/strace.internal.h" +#include "libc/intrin/weaken.h" #include "libc/mem/internal.h" +#include "libc/mem/mem.h" #include "libc/runtime/runtime.h" #include "libc/str/str.h" #include "libc/sysv/errfuns.h" @@ -43,9 +45,13 @@ int setenv(const char *name, const char *value, int overwrite) { } n = strlen(name); m = strlen(value); - if (!(s = kmalloc(n + 1 + m + 1))) return -1; + if (!_weaken(malloc) || !(s = _weaken(malloc)(n + 1 + m + 1))) { + return enomem(); + } memcpy(mempcpy(mempcpy(s, name, n), "=", 1), value, m + 1); rc = PutEnvImpl(s, overwrite); STRACE("setenv(%#s, %#s, %d) → %d% m", name, value, overwrite, rc); return rc; } + +IGNORE_LEAKS(setenv) diff --git a/libc/log/oncrash_amd64.c b/libc/log/oncrash_amd64.c index 3cacbf7d5..b80bbe9a1 100644 --- a/libc/log/oncrash_amd64.c +++ b/libc/log/oncrash_amd64.c @@ -32,7 +32,6 @@ #include "libc/intrin/atomic.h" #include "libc/intrin/describebacktrace.internal.h" #include "libc/intrin/describeflags.internal.h" -#include "libc/intrin/kmalloc.h" #include "libc/intrin/kprintf.h" #include "libc/intrin/strace.internal.h" #include "libc/intrin/weaken.h" @@ -207,9 +206,9 @@ void ShowCrashReportHook(int, int, int, struct siginfo *, ucontext_t *); relegated void ShowCrashReport(int err, int sig, struct siginfo *si, ucontext_t *ctx) { int i; - size_t n; + char *p; char host[64]; - char *p, *buf; + char buf[3000]; struct utsname names; if (_weaken(ShowCrashReportHook)) { ShowCrashReportHook(2, err, sig, si, ctx); @@ -223,9 +222,9 @@ relegated void ShowCrashReport(int err, int sig, struct siginfo *si, uname(&names); errno = err; // TODO(jart): Buffer the WHOLE crash report with backtrace for atomic write. - npassert((p = buf = kmalloc((n = 1024 * 1024)))); + p = buf; p += ksnprintf( - p, n, + p, 10000, "\n%serror%s: Uncaught %G (%s) on %s pid %d tid %d\n" " %s\n" " %s\n" diff --git a/libc/runtime/fork-nt.c b/libc/runtime/fork-nt.c index 8ec61223f..845ccbda2 100644 --- a/libc/runtime/fork-nt.c +++ b/libc/runtime/fork-nt.c @@ -68,7 +68,6 @@ extern int64_t __wincrashearly; bool32 __onntconsoleevent(uint32_t); void sys_setitimer_nt_reset(void); -void kmalloc_unlock(void); static textwindows wontreturn void AbortFork(const char *func) { #ifdef SYSDEBUG diff --git a/libc/runtime/memtrack.internal.h b/libc/runtime/memtrack.internal.h index 785a7accf..639aa7a10 100644 --- a/libc/runtime/memtrack.internal.h +++ b/libc/runtime/memtrack.internal.h @@ -12,19 +12,17 @@ #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ -#define kAutomapStart 0x100080040000 -#define kAutomapSize (kMemtrackStart - kAutomapStart) -#define kMemtrackStart 0x1fe7fffc0000 -#define kMemtrackSize (0x1ffffffc0000 - kMemtrackStart) -#define kFixedmapStart 0x300000040000 -#define kFixedmapSize (0x400000040000 - kFixedmapStart) -#define kMemtrackFdsStart 0x6fe000040000 -#define kMemtrackFdsSize (0x6feffffc0000 - kMemtrackFdsStart) -#define kMemtrackZiposStart 0x6fd000040000 -#define kMemtrackZiposSize (0x6fdffffc0000 - kMemtrackZiposStart) -#define kMemtrackKmallocStart 0x6fc000040000 -#define kMemtrackKmallocSize (0x6fcffffc0000 - kMemtrackKmallocStart) -#define kMemtrackGran (!IsAsan() ? FRAMESIZE : FRAMESIZE * 8) +#define kAutomapStart 0x100080040000 +#define kAutomapSize (kMemtrackStart - kAutomapStart) +#define kMemtrackStart 0x1fe7fffc0000 +#define kMemtrackSize (0x1ffffffc0000 - kMemtrackStart) +#define kFixedmapStart 0x300000040000 +#define kFixedmapSize (0x400000040000 - kFixedmapStart) +#define kMemtrackFdsStart 0x6fe000040000 +#define kMemtrackFdsSize (0x6feffffc0000 - kMemtrackFdsStart) +#define kMemtrackZiposStart 0x6fd000040000 +#define kMemtrackZiposSize (0x6fdffffc0000 - kMemtrackZiposStart) +#define kMemtrackGran (!IsAsan() ? FRAMESIZE : FRAMESIZE * 8) struct MemoryInterval { int x; @@ -132,11 +130,6 @@ forceinline pureconst bool IsZiposFrame(int x) { x <= (int)((kMemtrackZiposStart + kMemtrackZiposSize - 1) >> 16); } -forceinline pureconst bool IsKmallocFrame(int x) { - return (int)(kMemtrackKmallocStart >> 16) <= x && - x <= (int)((kMemtrackKmallocStart + kMemtrackKmallocSize - 1) >> 16); -} - forceinline pureconst bool IsShadowFrame(int x) { return 0x7fff <= x && x < 0x10008000; } diff --git a/libc/runtime/memtrack64.txt b/libc/runtime/memtrack64.txt index 44ff03a37..f85c6a520 100644 --- a/libc/runtime/memtrack64.txt +++ b/libc/runtime/memtrack64.txt @@ -1808,7 +1808,7 @@ 6f900000-6f9fffff 64gb free 6fa00000-6fafffff 64gb free 6fb00000-6fbfffff 64gb free -6fc00004-6fcffffb 64gb kmalloc +6fc00004-6fcffffb 64gb free 6fd00004-6fdffffb 64gb zipos 6fe00004-6feffffb 64gb g_fds 6ff00004-70000003 64gb free diff --git a/libc/runtime/syslib.internal.h b/libc/runtime/syslib.internal.h index 27f982385..d58037913 100644 --- a/libc/runtime/syslib.internal.h +++ b/libc/runtime/syslib.internal.h @@ -44,6 +44,7 @@ struct Syslib { dispatch_time_t (*dispatch_walltime)(const struct timespec *, int64_t); /* v2 (2023-09-10) */ long (*pthread_self)(void); + void (*dispatch_release)(dispatch_semaphore_t); }; extern struct Syslib *__syslib; diff --git a/libc/runtime/zipos-get.c b/libc/runtime/zipos-get.c index 3e13f7614..31a3ab5dd 100644 --- a/libc/runtime/zipos-get.c +++ b/libc/runtime/zipos-get.c @@ -23,7 +23,6 @@ #include "libc/cosmo.h" #include "libc/fmt/conv.h" #include "libc/intrin/cmpxchg.h" -#include "libc/intrin/kmalloc.h" #include "libc/intrin/promises.internal.h" #include "libc/intrin/strace.internal.h" #include "libc/macros.internal.h" @@ -91,7 +90,7 @@ static int __zipos_compare_names(const void *a, const void *b, void *c) { static void __zipos_generate_index(struct Zipos *zipos) { size_t c, i; zipos->records = GetZipCdirRecords(zipos->cdir); - zipos->index = kmalloc(zipos->records * sizeof(size_t)); + zipos->index = _mapanon(zipos->records * sizeof(size_t)); for (i = 0, c = GetZipCdirOffset(zipos->cdir); i < zipos->records; ++i, c += ZIP_CFILE_HDRSIZE(zipos->map + c)) { zipos->index[i] = c; diff --git a/libc/stdio/alloc.c b/libc/stdio/alloc.c index bfa1dae35..084881c0f 100644 --- a/libc/stdio/alloc.c +++ b/libc/stdio/alloc.c @@ -16,49 +16,26 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/intrin/atomic.h" -#include "libc/intrin/kmalloc.h" +#include "libc/mem/mem.h" #include "libc/stdio/internal.h" -#include "libc/stdio/stdio.h" -#include "libc/str/str.h" #include "libc/thread/thread.h" -static _Atomic(FILE *) __stdio_freelist; - FILE *__stdio_alloc(void) { FILE *f; - f = atomic_load_explicit(&__stdio_freelist, memory_order_acquire); - while (f) { - if (atomic_compare_exchange_weak_explicit( - &__stdio_freelist, &f, - atomic_load_explicit((_Atomic(struct FILE *) *)&f->next, - memory_order_acquire), - memory_order_release, memory_order_relaxed)) { - atomic_store_explicit((_Atomic(struct FILE *) *)&f->next, 0, - memory_order_release); - break; - } - } - if (!f) { - f = kmalloc(sizeof(FILE)); - } - if (f) { - f->lock._type = PTHREAD_MUTEX_RECURSIVE; + if ((f = calloc(1, sizeof(FILE)))) { + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&f->lock, &attr); + pthread_mutexattr_destroy(&attr); + f->dynamic = 1; } return f; } void __stdio_free(FILE *f) { - FILE *g; - bzero(f, sizeof(*f)); - g = atomic_load_explicit(&__stdio_freelist, memory_order_acquire); - for (;;) { - atomic_store_explicit((_Atomic(struct FILE *) *)&f->next, g, - memory_order_release); - if (atomic_compare_exchange_weak_explicit(&__stdio_freelist, &g, f, - memory_order_release, - memory_order_relaxed)) { - break; - } + pthread_mutex_destroy(&f->lock); + if (f->dynamic) { + free(f); } } diff --git a/libc/stdio/internal.h b/libc/stdio/internal.h index c834e4ab5..7a7b3ee31 100644 --- a/libc/stdio/internal.h +++ b/libc/stdio/internal.h @@ -9,21 +9,22 @@ COSMOPOLITAN_C_START_ struct FILE { - uint8_t bufmode; /* 0x00 _IOFBF, etc. (ignored if fd=-1) */ - char noclose; /* 0x01 for fake dup() todo delete! */ - uint32_t iomode; /* 0x04 O_RDONLY, etc. (ignored if fd=-1) */ - int32_t state; /* 0x08 0=OK, -1=EOF, >0=errno */ - int fd; /* 0x0c ≥0=fd, -1=closed|buffer */ - uint32_t beg; /* 0x10 */ - uint32_t end; /* 0x14 */ - char *buf; /* 0x18 */ - uint32_t size; /* 0x20 */ - uint32_t nofree; /* 0x24 */ - int pid; /* 0x28 */ - char *getln; /* 0x30 */ - pthread_mutex_t lock; /* 0x38 */ - struct FILE *next; /* 0x48 */ - char mem[BUFSIZ]; /* 0x50 */ + uint8_t bufmode; /* _IOFBF, etc. (ignored if fd=-1) */ + char noclose; /* for fake dup() todo delete! */ + char dynamic; /* did malloc() create this object? */ + uint32_t iomode; /* O_RDONLY, etc. (ignored if fd=-1) */ + int32_t state; /* 0=OK, -1=EOF, >0=errno */ + int fd; /* ≥0=fd, -1=closed|buffer */ + uint32_t beg; + uint32_t end; + char *buf; + uint32_t size; + uint32_t nofree; + int pid; + char *getln; + pthread_mutex_t lock; + struct FILE *next; + char mem[BUFSIZ]; }; extern uint64_t g_rando; diff --git a/libc/thread/pthread_atfork.c b/libc/thread/pthread_atfork.c index 2e7bb84d4..ce6674ada 100644 --- a/libc/thread/pthread_atfork.c +++ b/libc/thread/pthread_atfork.c @@ -17,13 +17,17 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" +#include "libc/atomic.h" #include "libc/calls/state.internal.h" +#include "libc/cosmo.h" +#include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" #include "libc/intrin/dll.h" -#include "libc/intrin/kmalloc.h" +#include "libc/intrin/leaky.internal.h" #include "libc/mem/mem.h" #include "libc/runtime/memtrack.internal.h" +#include "libc/runtime/runtime.h" #include "libc/str/str.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/thread.h" @@ -61,11 +65,9 @@ void _pthread_onfork_prepare(void) { pthread_spin_lock(&_pthread_lock); __fds_lock(); __mmi_lock(); - __kmalloc_lock(); } void _pthread_onfork_parent(void) { - __kmalloc_unlock(); __mmi_unlock(); __fds_unlock(); pthread_spin_unlock(&_pthread_lock); @@ -85,7 +87,6 @@ void _pthread_onfork_child(void) { atomic_store_explicit(&pt->cancelled, false, memory_order_relaxed); // wipe core runtime locks - __kmalloc_unlock(); pthread_mutexattr_init(&attr); pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); pthread_mutex_init(&__mmi_lock_obj, &attr); @@ -105,7 +106,7 @@ void _pthread_onfork_child(void) { int _pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) { int rc; struct AtFork *a; - if (!(a = kmalloc(sizeof(struct AtFork)))) return ENOMEM; + if (!(a = malloc(sizeof(struct AtFork)))) return ENOMEM; a->f[0] = prepare; a->f[1] = parent; a->f[2] = child; @@ -118,3 +119,5 @@ int _pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) { rc = 0; return rc; } + +IGNORE_LEAKS(_pthread_atfork) diff --git a/libc/thread/tls.h b/libc/thread/tls.h index 77d31fe77..07a152b8b 100644 --- a/libc/thread/tls.h +++ b/libc/thread/tls.h @@ -27,7 +27,7 @@ struct CosmoTib { _Atomic(int32_t) tib_tid; /* 0x38 transitions -1 → tid → 0 */ int32_t tib_errno; /* 0x3c */ uint64_t tib_flags; /* 0x40 */ - void *tib_nsync; + long __padding; int tib_ftrace; /* inherited */ int tib_strace; /* inherited */ uint64_t tib_sigmask; /* inherited */ diff --git a/libc/x/xload.c b/libc/x/xload.c index 072378a69..89dae7e9c 100644 --- a/libc/x/xload.c +++ b/libc/x/xload.c @@ -17,7 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/intrin/atomic.h" -#include "libc/intrin/kmalloc.h" +#include "libc/mem/mem.h" #include "libc/runtime/internal.h" #include "libc/x/x.h" #include "third_party/zlib/zlib.h" @@ -44,7 +44,7 @@ void *xload(_Atomic(void *) *a, const void *p, size_t n, size_t m) { void *r, *z; if ((r = atomic_load_explicit(a, memory_order_acquire))) return r; - if (!(r = kmalloc(m))) return 0; + if (!(r = malloc(m))) return 0; if (__inflate(r, m, p, n)) return 0; z = 0; if (!atomic_compare_exchange_strong_explicit(a, &z, r, memory_order_release, diff --git a/test/libc/calls/sched_getaffinity_test.c b/test/libc/calls/sched_getaffinity_test.c index cdd43d0c4..1c20849ea 100644 --- a/test/libc/calls/sched_getaffinity_test.c +++ b/test/libc/calls/sched_getaffinity_test.c @@ -89,6 +89,7 @@ __attribute__((__constructor__)) static void init(void) { } } +#ifdef __x86_64__ TEST(sched_setaffinity, isInheritedAcrossExecve) { cpu_set_t x; CPU_ZERO(&x); @@ -103,6 +104,7 @@ TEST(sched_setaffinity, isInheritedAcrossExecve) { EXPECT_TRUE(WIFEXITED(ws)); EXPECT_EQ(42, WEXITSTATUS(ws)); } +#endif /* __x86_64__ */ TEST(sched_getaffinity, getpid) { cpu_set_t x, y; diff --git a/test/libc/stdio/popen_test.c b/test/libc/stdio/popen_test.c index a31a47c02..aeafd2f7c 100644 --- a/test/libc/stdio/popen_test.c +++ b/test/libc/stdio/popen_test.c @@ -28,7 +28,6 @@ #include "libc/mem/gc.h" #include "libc/mem/mem.h" #include "libc/runtime/runtime.h" -#include "libc/stdio/lock.internal.h" #include "libc/stdio/rand.h" #include "libc/stdio/stdio.h" #include "libc/str/str.h" diff --git a/test/libc/stdio/posix_spawn_test.c b/test/libc/stdio/posix_spawn_test.c index 11e730924..ba149861f 100644 --- a/test/libc/stdio/posix_spawn_test.c +++ b/test/libc/stdio/posix_spawn_test.c @@ -251,7 +251,7 @@ void PosixSpawnWait(const char *prog) { ASSERT_EQ(42, WEXITSTATUS(ws)); } -TEST(posix_spawn, bench) { +BENCH(posix_spawn, bench) { long n = 128L * 1000 * 1000; memset(gc(malloc(n)), -1, n); creat("tiny64", 0755); diff --git a/test/libc/thread/pthread_cancel_test.c b/test/libc/thread/pthread_cancel_test.c index 522f34d03..d77e55f5d 100644 --- a/test/libc/thread/pthread_cancel_test.c +++ b/test/libc/thread/pthread_cancel_test.c @@ -169,18 +169,17 @@ void *CondWaitDeferredWorker(void *arg) { pthread_setcancelstate(PTHREAD_CANCEL_DEFERRED, 0); ASSERT_EQ(0, pthread_mutex_lock(&mu)); ASSERT_EQ(ECANCELED, pthread_cond_timedwait(&cv, &mu, 0)); - ASSERT_EQ(0, pthread_mutex_unlock(&mu)); - return 0; + __builtin_trap(); } -TEST(pthread_cancel, condDeferredWait) { +TEST(pthread_cancel, condDeferredWait_reacquiresMutex) { void *rc; pthread_t th; ASSERT_EQ(0, pthread_create(&th, 0, CondWaitDeferredWorker, 0)); pthread_cancel(th); ASSERT_EQ(0, pthread_join(th, &rc)); ASSERT_EQ(PTHREAD_CANCELED, rc); - ASSERT_EQ(0, pthread_mutex_trylock(&mu)); + ASSERT_EQ(EBUSY, pthread_mutex_trylock(&mu)); ASSERT_EQ(0, pthread_mutex_unlock(&mu)); } @@ -192,7 +191,7 @@ TEST(pthread_cancel, condDeferredWaitDelayed) { pthread_cancel(th); ASSERT_EQ(0, pthread_join(th, &rc)); ASSERT_EQ(PTHREAD_CANCELED, rc); - ASSERT_EQ(0, pthread_mutex_trylock(&mu)); + ASSERT_EQ(EBUSY, pthread_mutex_trylock(&mu)); ASSERT_EQ(0, pthread_mutex_unlock(&mu)); } diff --git a/third_party/finger/util.c b/third_party/finger/util.c index 1478ccc4c..b1853a1f0 100644 --- a/third_party/finger/util.c +++ b/third_party/finger/util.c @@ -93,7 +93,7 @@ static void userinfo(PERSON *pn, struct passwd *pw) { pn->shell = strdup(pw->pw_shell); /* make a private copy of gecos to munge */ - strncpy(tbuf, pw->pw_gecos, TBUFLEN); + strlcpy(tbuf, pw->pw_gecos, TBUFLEN); tbuf[TBUFLEN-1] = 0; /* ensure null termination */ bp = tbuf; @@ -176,7 +176,7 @@ match(struct passwd *pw, const char *user) int i, j, ct, rv=0; char *rname; - strncpy(tbuf, pw->pw_gecos, TBUFLEN); + strlcpy(tbuf, pw->pw_gecos, TBUFLEN); tbuf[TBUFLEN-1] = 0; /* guarantee null termination */ p = tbuf; diff --git a/third_party/nsync/README.cosmo b/third_party/nsync/README.cosmo index fc280d7ca..bd79cc03b 100644 --- a/third_party/nsync/README.cosmo +++ b/third_party/nsync/README.cosmo @@ -15,9 +15,22 @@ ORIGIN LOCAL CHANGES - - nsync_malloc_() is implemented as kmalloc() - - nsync_mu_semaphore uses Cosmopolitan Futexes - - block pthread cancellations in nsync_mu_lock_slow_ - - support posix thread cancellations in nsync_cv_wait - - timespec api was so good that it's now part of libc - - linked list api was so good that it's now part of libc + - Time APIs were so good that they're now part of our libc + + - Double linked list API was so good that it's now part of our libc + + - Modified *NSYNC to allocate waiter objects on the stack. We need it + because we use *NSYNC mutexes to implement POSIX mutexes, which are + too low-level to safely depend on malloc, or even mmap in our case. + + - Rewrote most of the semaphore and futex system call support code so + it works well with Cosmopolitan's fat runtime portability. *NSYNC's + unit test suite passes on all supported platforms. However the BSDs + currently appear to be overutilizing CPU time compared with others. + + - Support POSIX thread cancellation. APIs that wait on condition vars + are now cancellation points. In PTHREAD_CANCEL_MASKED mode they may + return ECANCELED. In PTHREAD_CANCEL_DEFERRED mode the POSIX threads + library will unwind the stack to unlock any locks and free waiters. + On the other hand the *NSYNC APIs for mutexes will now safely block + thread cancellation, but you can still use *NSYNC notes to do that. diff --git a/third_party/nsync/common.c b/third_party/nsync/common.c index 88c69db3d..780533814 100644 --- a/third_party/nsync/common.c +++ b/third_party/nsync/common.c @@ -16,10 +16,6 @@ │ limitations under the License. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/intrin/dll.h" -#include "libc/intrin/kmalloc.h" -#include "libc/mem/mem.h" -#include "libc/runtime/runtime.h" -#include "libc/thread/thread.h" #include "libc/thread/tls.h" #include "third_party/nsync/atomic.h" #include "third_party/nsync/atomic.internal.h" @@ -147,78 +143,25 @@ waiter *nsync_dll_waiter_samecond_ (struct Dll *e) { /* -------------------------------- */ -static struct Dll *free_waiters = NULL; - -/* free_waiters points to a doubly-linked list of free waiter structs. */ -static nsync_atomic_uint32_ free_waiters_mu; /* spinlock; protects free_waiters */ - -#define waiter_for_thread __get_tls()->tib_nsync - -static void waiter_destroy (void *v) { - waiter *w = (waiter *) v; - /* Reset waiter_for_thread in case another thread-local variable reuses - the waiter in its destructor while the waiter is taken by the other - thread from free_waiters. This can happen as the destruction order - of thread-local variables can be arbitrary in some platform e.g. - POSIX. */ - waiter_for_thread = NULL; - IGNORE_RACES_START (); - ASSERT ((w->flags & (WAITER_RESERVED|WAITER_IN_USE)) == WAITER_RESERVED); - w->flags &= ~WAITER_RESERVED; - nsync_spin_test_and_set_ (&free_waiters_mu, 1, 1, 0); - dll_make_first (&free_waiters, &w->nw.q); - ATM_STORE_REL (&free_waiters_mu, 0); /* release store */ - IGNORE_RACES_END (); +/* Initializes waiter struct. */ +void nsync_waiter_init_ (waiter *w) { + w->tag = WAITER_TAG; + w->nw.tag = NSYNC_WAITER_TAG; + nsync_mu_semaphore_init (&w->sem); + w->nw.sem = &w->sem; + dll_init (&w->nw.q); + NSYNC_ATOMIC_UINT32_STORE_ (&w->nw.waiting, 0); + w->nw.flags = NSYNC_WAITER_FLAG_MUCV; + ATM_STORE (&w->remove_count, 0); + dll_init (&w->same_condition); + w->flags = WAITER_IN_USE; } -/* Return a pointer to an unused waiter struct. - Ensures that the enclosed timer is stopped and its channel drained. */ -waiter *nsync_waiter_new_ (void) { - struct Dll *q; - waiter *tw; - waiter *w; - tw = waiter_for_thread; - w = tw; - if (w == NULL || (w->flags & (WAITER_RESERVED|WAITER_IN_USE)) != WAITER_RESERVED) { - w = NULL; - nsync_spin_test_and_set_ (&free_waiters_mu, 1, 1, 0); - q = dll_first (free_waiters); - if (q != NULL) { /* If free list is non-empty, dequeue an item. */ - dll_remove (&free_waiters, q); - w = DLL_WAITER (q); - } - ATM_STORE_REL (&free_waiters_mu, 0); /* release store */ - if (w == NULL) { /* If free list was empty, allocate an item. */ - w = (waiter *) kmalloc (sizeof (*w)); - w->tag = WAITER_TAG; - w->nw.tag = NSYNC_WAITER_TAG; - nsync_mu_semaphore_init (&w->sem); - w->nw.sem = &w->sem; - dll_init (&w->nw.q); - NSYNC_ATOMIC_UINT32_STORE_ (&w->nw.waiting, 0); - w->nw.flags = NSYNC_WAITER_FLAG_MUCV; - ATM_STORE (&w->remove_count, 0); - dll_init (&w->same_condition); - w->flags = 0; - } - if (tw == NULL) { - w->flags |= WAITER_RESERVED; - nsync_set_per_thread_waiter_ (w, &waiter_destroy); - waiter_for_thread = w; - } - } - w->flags |= WAITER_IN_USE; - return (w); -} - -/* Return an unused waiter struct *w to the free pool. */ -void nsync_waiter_free_ (waiter *w) { - ASSERT ((w->flags & WAITER_IN_USE) != 0); - w->flags &= ~WAITER_IN_USE; - if ((w->flags & WAITER_RESERVED) == 0) { - nsync_spin_test_and_set_ (&free_waiters_mu, 1, 1, 0); - dll_make_first (&free_waiters, &w->nw.q); - ATM_STORE_REL (&free_waiters_mu, 0); /* release store */ +/* Destroys waiter struct. */ +void nsync_waiter_destroy_ (waiter *w) { + if (w->tag) { + nsync_mu_semaphore_destroy (&w->sem); + w->tag = 0; } } diff --git a/third_party/nsync/common.internal.h b/third_party/nsync/common.internal.h index 7d452c7be..15b04f113 100644 --- a/third_party/nsync/common.internal.h +++ b/third_party/nsync/common.internal.h @@ -244,12 +244,8 @@ waiter *nsync_dll_waiter_(struct Dll *e); : DLL_CONTAINER(struct waiter_s, same_condition, e)) waiter *nsync_dll_waiter_samecond_(struct Dll *e); -/* Return a pointer to an unused waiter struct. - Ensures that the enclosed timer is stopped and its channel drained. */ -waiter *nsync_waiter_new_(void); - -/* Return an unused waiter struct *w to the free pool. */ -void nsync_waiter_free_(waiter *w); +void nsync_waiter_init_(waiter *); +void nsync_waiter_destroy_(waiter *); /* ---------- */ diff --git a/third_party/nsync/mem/nsync_cv.c b/third_party/nsync/mem/nsync_cv.c index 63b7a1c6c..e3f19e9c3 100644 --- a/third_party/nsync/mem/nsync_cv.c +++ b/third_party/nsync/mem/nsync_cv.c @@ -16,6 +16,7 @@ │ limitations under the License. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/cp.internal.h" +#include "libc/errno.h" #include "libc/intrin/dll.h" #include "libc/str/str.h" #include "libc/thread/thread.h" @@ -169,6 +170,95 @@ static void void_mu_unlock (void *mu) { nsync_mu_unlock ((nsync_mu *) mu); } +/* Memory needed for a single nsync_cv_wait_with_deadline_generic() call. */ +struct nsync_cv_wait_with_deadline_s { + nsync_cv *pcv; + int sem_outcome; + int is_reader_mu; + uint32_t old_word; + uint32_t remove_count; + void *pmu; + void (*lock) (void *); + nsync_mu *cv_mu; + nsync_time abs_deadline; + nsync_note cancel_note; + waiter w; +}; + +/* Wait until awoken or timeout, or back out of wait if the thread is being cancelled. */ +static int nsync_cv_wait_with_deadline_impl_ (struct nsync_cv_wait_with_deadline_s *c) { + int outcome = 0; + int attempts = 0; + IGNORE_RACES_START (); + while (ATM_LOAD_ACQ (&c->w.nw.waiting) != 0) { /* acquire load */ + if (c->sem_outcome == 0) { + c->sem_outcome = nsync_sem_wait_with_cancel_ (&c->w, c->abs_deadline, c->cancel_note); + } + if (c->sem_outcome != 0 && ATM_LOAD (&c->w.nw.waiting) != 0) { + /* A timeout or cancellation occurred, and no wakeup. + Acquire *pcv's spinlock, and confirm. */ + c->old_word = nsync_spin_test_and_set_ (&c->pcv->word, CV_SPINLOCK, + CV_SPINLOCK, 0); + /* Check that w wasn't removed from the queue after we + checked above, but before we acquired the spinlock. + The test of remove_count confirms that the waiter *w + is still governed by *pcv's spinlock; otherwise, some + other thread is about to set w.waiting==0. */ + if (ATM_LOAD (&c->w.nw.waiting) != 0) { + if (c->remove_count == ATM_LOAD (&c->w.remove_count)) { + uint32_t old_value; + /* still in cv waiter queue */ + /* Not woken, so remove *w from cv + queue, and declare a + timeout/cancellation. */ + outcome = c->sem_outcome; + dll_remove (&c->pcv->waiters, &c->w.nw.q); + do { + old_value = ATM_LOAD (&c->w.remove_count); + } while (!ATM_CAS (&c->w.remove_count, old_value, old_value+1)); + if (dll_is_empty (c->pcv->waiters)) { + c->old_word &= ~(CV_NON_EMPTY); + } + ATM_STORE_REL (&c->w.nw.waiting, 0); /* release store */ + } + } + /* Release spinlock. */ + ATM_STORE_REL (&c->pcv->word, c->old_word); /* release store */ + } + if (ATM_LOAD (&c->w.nw.waiting) != 0) { + /* The delay here causes this thread ultimately to + yield to another that has dequeued this thread, but + has not yet set the waiting field to zero; a + cancellation or timeout may prevent this thread + from blocking above on the semaphore. */ + attempts = nsync_spin_delay_ (attempts); + } + } + if (c->cv_mu != NULL && c->w.cv_mu == NULL) { /* waiter was moved to *pmu's queue, and woken. */ + /* Requeue on *pmu using existing waiter struct; current thread + is the designated waker. */ + nsync_mu_lock_slow_ (c->cv_mu, &c->w, MU_DESIG_WAKER, c->w.l_type); + } else { + /* Traditional case: We've woken from the cv, and need to reacquire *pmu. */ + if (c->is_reader_mu) { + nsync_mu_rlock (c->cv_mu); + } else { + (*c->lock) (c->pmu); + } + } + nsync_waiter_destroy_ (&c->w); + IGNORE_RACES_END (); + return (outcome); +} + +/* Handle POSIX thread DEFERRED mode cancellation. */ +static void nsync_cv_wait_with_deadline_unwind_ (void *arg) { + struct nsync_cv_wait_with_deadline_s *c; + c = (struct nsync_cv_wait_with_deadline_s *)arg; + c->sem_outcome = ECANCELED; + nsync_cv_wait_with_deadline_impl_ (c); +} + /* Atomically release *pmu (which must be held on entry) and block the calling thread on *pcv. Then wait until awakened by a call to nsync_cv_signal() or nsync_cv_broadcast() (or a spurious wakeup), or by the time @@ -194,33 +284,33 @@ int nsync_cv_wait_with_deadline_generic (nsync_cv *pcv, void *pmu, void (*lock) (void *), void (*unlock) (void *), nsync_time abs_deadline, nsync_note cancel_note) { - nsync_mu *cv_mu = NULL; - int is_reader_mu; - uint32_t old_word; - uint32_t remove_count; - int sem_outcome; - unsigned attempts; - int outcome = 0; - waiter *w; + int outcome; + struct nsync_cv_wait_with_deadline_s c; IGNORE_RACES_START (); - BEGIN_CANCELLATION_POINT; - w = nsync_waiter_new_ (); - pthread_cleanup_push((void *)nsync_waiter_free_, w); - ATM_STORE (&w->nw.waiting, 1); - w->cond.f = NULL; /* Not using a conditional critical section. */ - w->cond.v = NULL; - w->cond.eq = NULL; + + nsync_waiter_init_ (&c.w); + c.abs_deadline = abs_deadline; + c.cancel_note = cancel_note; + c.cv_mu = NULL; + c.lock = lock; + c.pcv = pcv; + c.pmu = pmu; + + ATM_STORE (&c.w.nw.waiting, 1); + c.w.cond.f = NULL; /* Not using a conditional critical section. */ + c.w.cond.v = NULL; + c.w.cond.eq = NULL; if (lock == &void_mu_lock || lock == (void (*) (void *)) &nsync_mu_lock || lock == (void (*) (void *)) &nsync_mu_rlock) { - cv_mu = (nsync_mu *) pmu; + c.cv_mu = (nsync_mu *) pmu; } - w->cv_mu = cv_mu; /* If *pmu is an nsync_mu, record its address, else record NULL. */ - is_reader_mu = 0; /* If true, an nsync_mu in reader mode. */ - if (cv_mu == NULL) { - w->l_type = NULL; + c.w.cv_mu = c.cv_mu; /* If *pmu is an nsync_mu, record its address, else record NULL. */ + c.is_reader_mu = 0; /* If true, an nsync_mu in reader mode. */ + if (c.cv_mu == NULL) { + c.w.l_type = NULL; } else { - uint32_t old_mu_word = ATM_LOAD (&cv_mu->word); + uint32_t old_mu_word = ATM_LOAD (&c.cv_mu->word); int is_writer = (old_mu_word & MU_WHELD_IF_NON_ZERO) != 0; int is_reader = (old_mu_word & MU_RHELD_IF_NON_ZERO) != 0; if (is_writer) { @@ -228,95 +318,34 @@ int nsync_cv_wait_with_deadline_generic (nsync_cv *pcv, void *pmu, nsync_panic_ ("mu held in reader and writer mode simultaneously " "on entry to nsync_cv_wait_with_deadline()\n"); } - w->l_type = nsync_writer_type_; + c.w.l_type = nsync_writer_type_; } else if (is_reader) { - w->l_type = nsync_reader_type_; - is_reader_mu = 1; + c.w.l_type = nsync_reader_type_; + c.is_reader_mu = 1; } else { nsync_panic_ ("mu not held on entry to nsync_cv_wait_with_deadline()\n"); } } /* acquire spinlock, set non-empty */ - old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK|CV_NON_EMPTY, 0); - dll_make_last (&pcv->waiters, &w->nw.q); - remove_count = ATM_LOAD (&w->remove_count); + c.old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK|CV_NON_EMPTY, 0); + dll_make_last (&pcv->waiters, &c.w.nw.q); + c.remove_count = ATM_LOAD (&c.w.remove_count); /* Release the spin lock. */ - ATM_STORE_REL (&pcv->word, old_word|CV_NON_EMPTY); /* release store */ + ATM_STORE_REL (&pcv->word, c.old_word|CV_NON_EMPTY); /* release store */ /* Release *pmu. */ - if (is_reader_mu) { - nsync_mu_runlock (cv_mu); + if (c.is_reader_mu) { + nsync_mu_runlock (c.cv_mu); } else { (*unlock) (pmu); } - /* wait until awoken or a timeout. */ - sem_outcome = 0; - attempts = 0; - while (ATM_LOAD_ACQ (&w->nw.waiting) != 0) { /* acquire load */ - if (sem_outcome == 0) { - sem_outcome = nsync_sem_wait_with_cancel_ (w, abs_deadline, cancel_note); - } - - if (sem_outcome != 0 && ATM_LOAD (&w->nw.waiting) != 0) { - /* A timeout or cancellation occurred, and no wakeup. - Acquire *pcv's spinlock, and confirm. */ - old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, - CV_SPINLOCK, 0); - /* Check that w wasn't removed from the queue after we - checked above, but before we acquired the spinlock. - The test of remove_count confirms that the waiter *w - is still governed by *pcv's spinlock; otherwise, some - other thread is about to set w.waiting==0. */ - if (ATM_LOAD (&w->nw.waiting) != 0) { - if (remove_count == ATM_LOAD (&w->remove_count)) { - uint32_t old_value; - /* still in cv waiter queue */ - /* Not woken, so remove *w from cv - queue, and declare a - timeout/cancellation. */ - outcome = sem_outcome; - dll_remove (&pcv->waiters, &w->nw.q); - do { - old_value = ATM_LOAD (&w->remove_count); - } while (!ATM_CAS (&w->remove_count, old_value, old_value+1)); - if (dll_is_empty (pcv->waiters)) { - old_word &= ~(CV_NON_EMPTY); - } - ATM_STORE_REL (&w->nw.waiting, 0); /* release store */ - } - } - /* Release spinlock. */ - ATM_STORE_REL (&pcv->word, old_word); /* release store */ - } - - if (ATM_LOAD (&w->nw.waiting) != 0) { - /* The delay here causes this thread ultimately to - yield to another that has dequeued this thread, but - has not yet set the waiting field to zero; a - cancellation or timeout may prevent this thread - from blocking above on the semaphore. */ - attempts = nsync_spin_delay_ (attempts); - } - } - - if (cv_mu != NULL && w->cv_mu == NULL) { /* waiter was moved to *pmu's queue, and woken. */ - /* Requeue on *pmu using existing waiter struct; current thread - is the designated waker. */ - nsync_mu_lock_slow_ (cv_mu, w, MU_DESIG_WAKER, w->l_type); - nsync_waiter_free_ (w); - } else { - /* Traditional case: We've woken from the cv, and need to reacquire *pmu. */ - nsync_waiter_free_ (w); - if (is_reader_mu) { - nsync_mu_rlock (cv_mu); - } else { - (*lock) (pmu); - } - } - pthread_cleanup_pop(0); - END_CANCELLATION_POINT; + /* Wait until awoken or a timeout. */ + c.sem_outcome = 0; + pthread_cleanup_push (nsync_cv_wait_with_deadline_unwind_, &c); + outcome = nsync_cv_wait_with_deadline_impl_ (&c); + pthread_cleanup_pop (0); IGNORE_RACES_END (); return (outcome); } @@ -341,7 +370,7 @@ void nsync_cv_signal (nsync_cv *pcv) { first_nw = DLL_NSYNC_WAITER (first); if ((first_nw->flags & NSYNC_WAITER_FLAG_MUCV) != 0) { uint32_t old_value; - do { + do { old_value = ATM_LOAD (&DLL_WAITER (first)->remove_count); } while (!ATM_CAS (&DLL_WAITER (first)->remove_count, old_value, old_value+1)); @@ -353,7 +382,7 @@ void nsync_cv_signal (nsync_cv *pcv) { /* If the first waiter is a reader, wake all readers, and if it's possible, one writer. This allows reader-regions to be added to a monitor without invalidating code in which - a client has optimized broadcast calls by converting them to + a client has optimized broadcast calls by converting them to signal calls. In particular, we wake a writer when waking readers because the readers will not invalidate the condition that motivated the client to call nsync_cv_signal(). But we @@ -382,7 +411,7 @@ void nsync_cv_signal (nsync_cv *pcv) { dll_remove (&pcv->waiters, p); if ((p_nw->flags & NSYNC_WAITER_FLAG_MUCV) != 0) { uint32_t old_value; - do { + do { old_value = ATM_LOAD ( &DLL_WAITER (p)->remove_count); } while (!ATM_CAS (&DLL_WAITER (p)->remove_count, @@ -427,7 +456,7 @@ void nsync_cv_broadcast (nsync_cv *pcv) { dll_remove (&pcv->waiters, p); if ((p_nw->flags & NSYNC_WAITER_FLAG_MUCV) != 0) { uint32_t old_value; - do { + do { old_value = ATM_LOAD (&DLL_WAITER (p)->remove_count); } while (!ATM_CAS (&DLL_WAITER (p)->remove_count, old_value, old_value+1)); diff --git a/third_party/nsync/mem/nsync_mu_wait.c b/third_party/nsync/mem/nsync_mu_wait.c index 4f03138b8..04683bdd9 100644 --- a/third_party/nsync/mem/nsync_mu_wait.c +++ b/third_party/nsync/mem/nsync_mu_wait.c @@ -149,12 +149,12 @@ int nsync_mu_wait_with_deadline (nsync_mu *mu, lock_type *l_type; int first_wait; int condition_is_true; - waiter *w; + waiter w[1]; int outcome; /* Work out in which mode the lock is held. */ uint32_t old_word; IGNORE_RACES_START (); - BLOCK_CANCELLATIONS; /* not supported yet */ + BLOCK_CANCELLATIONS; old_word = ATM_LOAD (&mu->word); if ((old_word & MU_ANY_LOCK) == 0) { nsync_panic_ ("nsync_mu not held in some mode when calling " @@ -165,12 +165,12 @@ int nsync_mu_wait_with_deadline (nsync_mu *mu, l_type = nsync_reader_type_; } + w->tag = 0; /* avoid allocating system resources */ first_wait = 1; /* first time through the loop below. */ condition_is_true = (condition == NULL || (*condition) (condition_arg)); /* Loop until either the condition becomes true, or "outcome" indicates cancellation or timeout. */ - w = NULL; outcome = 0; while (outcome == 0 && !condition_is_true) { uint32_t has_condition; @@ -180,8 +180,10 @@ int nsync_mu_wait_with_deadline (nsync_mu *mu, int sem_outcome; unsigned attempts; int have_lock; - if (w == NULL) { - w = nsync_waiter_new_ (); /* get a waiter struct if we need one. */ + + /* initialize the waiter if we haven't already */ + if (!w->tag) { + nsync_waiter_init_ (w); } /* Prepare to wait. */ @@ -259,9 +261,7 @@ int nsync_mu_wait_with_deadline (nsync_mu *mu, } condition_is_true = (condition == NULL || (*condition) (condition_arg)); } - if (w != NULL) { - nsync_waiter_free_ (w); /* free waiter if we allocated one. */ - } + nsync_waiter_destroy_ (w); if (condition_is_true) { outcome = 0; /* condition is true trumps other outcomes. */ } @@ -319,5 +319,3 @@ void nsync_mu_unlock_without_wakeup (nsync_mu *mu) { } IGNORE_RACES_END (); } - - diff --git a/third_party/nsync/mem/nsync_wait.c b/third_party/nsync/mem/nsync_wait.c index e33fe815d..0277a2fd8 100644 --- a/third_party/nsync/mem/nsync_wait.c +++ b/third_party/nsync/mem/nsync_wait.c @@ -17,6 +17,7 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/blockcancel.internal.h" #include "libc/mem/mem.h" +#include "libc/thread/thread.h" #include "third_party/nsync/atomic.h" #include "third_party/nsync/atomic.internal.h" #include "third_party/nsync/common.internal.h" @@ -36,7 +37,7 @@ int nsync_wait_n (void *mu, void (*lock) (void *), void (*unlock) (void *), int count, struct nsync_waitable_s *waitable[]) { int ready; IGNORE_RACES_START (); - BLOCK_CANCELLATIONS; /* TODO(jart): Does this need pthread cancellations? */ + BLOCK_CANCELLATIONS; for (ready = 0; ready != count && nsync_time_cmp ((*waitable[ready]->funcs->ready_time) ( waitable[ready]->v, NULL), @@ -48,7 +49,8 @@ int nsync_wait_n (void *mu, void (*lock) (void *), void (*unlock) (void *), int unlocked = 0; int j; int enqueued = 1; - waiter *w = nsync_waiter_new_ (); + waiter w[1]; + nsync_waiter_init_ (w); struct nsync_waiter_s nw_set[4]; struct nsync_waiter_s *nw = nw_set; if (count > (int) (sizeof (nw_set) / sizeof (nw_set[0]))) { @@ -95,10 +97,10 @@ int nsync_wait_n (void *mu, void (*lock) (void *), void (*unlock) (void *), } } + nsync_waiter_destroy_ (w); if (nw != nw_set) { free (nw); } - nsync_waiter_free_ (w); if (unlocked) { (*lock) (mu); } @@ -107,5 +109,3 @@ int nsync_wait_n (void *mu, void (*lock) (void *), void (*unlock) (void *), IGNORE_RACES_END (); return (ready); } - - diff --git a/third_party/nsync/mu.c b/third_party/nsync/mu.c index 4b8db1103..1000075fd 100644 --- a/third_party/nsync/mu.c +++ b/third_party/nsync/mu.c @@ -103,7 +103,8 @@ void nsync_mu_lock_slow_ (nsync_mu *mu, waiter *w, uint32_t clear, lock_type *l_ /* wait until awoken. */ while (ATM_LOAD_ACQ (&w->nw.waiting) != 0) { /* acquire load */ - nsync_mu_semaphore_p (&w->sem); + /* This can only return 0 or ECANCELED. */ + ASSERT (nsync_mu_semaphore_p (&w->sem) == 0); } wait_count++; /* If the thread has been woken more than this many @@ -155,9 +156,10 @@ void nsync_mu_lock (nsync_mu *mu) { if ((old_word&MU_WZERO_TO_ACQUIRE) != 0 || !ATM_CAS_ACQ (&mu->word, old_word, (old_word+MU_WADD_TO_ACQUIRE) & ~MU_WCLEAR_ON_ACQUIRE)) { - waiter *w = nsync_waiter_new_ (); - nsync_mu_lock_slow_ (mu, w, 0, nsync_writer_type_); - nsync_waiter_free_ (w); + waiter w; + nsync_waiter_init_ (&w); + nsync_mu_lock_slow_ (mu, &w, 0, nsync_writer_type_); + nsync_waiter_destroy_ (&w); } } IGNORE_RACES_END (); @@ -190,9 +192,10 @@ void nsync_mu_rlock (nsync_mu *mu) { if ((old_word&MU_RZERO_TO_ACQUIRE) != 0 || !ATM_CAS_ACQ (&mu->word, old_word, (old_word+MU_RADD_TO_ACQUIRE) & ~MU_RCLEAR_ON_ACQUIRE)) { - waiter *w = nsync_waiter_new_ (); - nsync_mu_lock_slow_ (mu, w, 0, nsync_reader_type_); - nsync_waiter_free_ (w); + waiter w; + nsync_waiter_init_ (&w); + nsync_mu_lock_slow_ (mu, &w, 0, nsync_reader_type_); + nsync_waiter_destroy_ (&w); } } IGNORE_RACES_END (); diff --git a/third_party/nsync/mu_semaphore.c b/third_party/nsync/mu_semaphore.c index 7ff8f13b0..000ca90cd 100644 --- a/third_party/nsync/mu_semaphore.c +++ b/third_party/nsync/mu_semaphore.c @@ -16,6 +16,7 @@ │ limitations under the License. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "third_party/nsync/mu_semaphore.h" +#include "libc/calls/cp.internal.h" #include "libc/dce.h" #include "third_party/nsync/mu_semaphore.internal.h" @@ -36,28 +37,45 @@ void nsync_mu_semaphore_init (nsync_semaphore *s) { } } +/* Releases system resources associated with *s. */ +void nsync_mu_semaphore_destroy (nsync_semaphore *s) { + if (IsXnuSilicon ()) { + return nsync_mu_semaphore_destroy_gcd (s); + } else if (IsNetbsd ()) { + return nsync_mu_semaphore_destroy_sem (s); + } +} + /* Wait until the count of *s exceeds 0, and decrement it. */ errno_t nsync_mu_semaphore_p (nsync_semaphore *s) { + errno_t err; + BEGIN_CANCELLATION_POINT; if (IsXnuSilicon ()) { - return nsync_mu_semaphore_p_gcd (s); + err = nsync_mu_semaphore_p_gcd (s); } else if (IsNetbsd ()) { - return nsync_mu_semaphore_p_sem (s); + err = nsync_mu_semaphore_p_sem (s); } else { - return nsync_mu_semaphore_p_futex (s); + err = nsync_mu_semaphore_p_futex (s); } + END_CANCELLATION_POINT; + return err; } /* Wait until one of: the count of *s is non-zero, in which case decrement *s and return 0; or abs_deadline expires, in which case return ETIMEDOUT. */ errno_t nsync_mu_semaphore_p_with_deadline (nsync_semaphore *s, nsync_time abs_deadline) { + errno_t err; + BEGIN_CANCELLATION_POINT; if (IsXnuSilicon ()) { - return nsync_mu_semaphore_p_with_deadline_gcd (s, abs_deadline); + err = nsync_mu_semaphore_p_with_deadline_gcd (s, abs_deadline); } else if (IsNetbsd ()) { - return nsync_mu_semaphore_p_with_deadline_sem (s, abs_deadline); + err = nsync_mu_semaphore_p_with_deadline_sem (s, abs_deadline); } else { - return nsync_mu_semaphore_p_with_deadline_futex (s, abs_deadline); + err = nsync_mu_semaphore_p_with_deadline_futex (s, abs_deadline); } + END_CANCELLATION_POINT; + return err; } /* Ensure that the count of *s is at least 1. */ diff --git a/third_party/nsync/mu_semaphore.h b/third_party/nsync/mu_semaphore.h index a2c9c6b45..5632d4a50 100644 --- a/third_party/nsync/mu_semaphore.h +++ b/third_party/nsync/mu_semaphore.h @@ -11,6 +11,9 @@ typedef struct nsync_semaphore_s_ { /* Initialize *s; the initial value is 0. */ void nsync_mu_semaphore_init(nsync_semaphore *s); +/* Releases system resources associated with *s. */ +void nsync_mu_semaphore_destroy(nsync_semaphore *s); + /* Wait until the count of *s exceeds 0, and decrement it. */ errno_t nsync_mu_semaphore_p(nsync_semaphore *s); diff --git a/third_party/nsync/mu_semaphore.internal.h b/third_party/nsync/mu_semaphore.internal.h index 3a0bed3b4..e535acd1f 100755 --- a/third_party/nsync/mu_semaphore.internal.h +++ b/third_party/nsync/mu_semaphore.internal.h @@ -11,11 +11,13 @@ errno_t nsync_mu_semaphore_p_with_deadline_futex(nsync_semaphore *, nsync_time); void nsync_mu_semaphore_v_futex(nsync_semaphore *); void nsync_mu_semaphore_init_sem(nsync_semaphore *); +void nsync_mu_semaphore_destroy_sem(nsync_semaphore *); errno_t nsync_mu_semaphore_p_sem(nsync_semaphore *); errno_t nsync_mu_semaphore_p_with_deadline_sem(nsync_semaphore *, nsync_time); void nsync_mu_semaphore_v_sem(nsync_semaphore *); void nsync_mu_semaphore_init_gcd(nsync_semaphore *); +void nsync_mu_semaphore_destroy_gcd(nsync_semaphore *); errno_t nsync_mu_semaphore_p_gcd(nsync_semaphore *); errno_t nsync_mu_semaphore_p_with_deadline_gcd(nsync_semaphore *, nsync_time); void nsync_mu_semaphore_v_gcd(nsync_semaphore *); diff --git a/third_party/nsync/mu_semaphore_gcd.c b/third_party/nsync/mu_semaphore_gcd.c index 5781a5e10..a2a1e1dba 100644 --- a/third_party/nsync/mu_semaphore_gcd.c +++ b/third_party/nsync/mu_semaphore_gcd.c @@ -41,6 +41,11 @@ static dispatch_semaphore_t dispatch_semaphore_create(long count) { return (ds); } +static void dispatch_release (dispatch_semaphore_t ds) { + __syslib->dispatch_release (ds); + STRACE ("dispatch_release(%#lx)", ds); +} + static long dispatch_semaphore_wait (dispatch_semaphore_t ds, dispatch_time_t dt) { long rc = __syslib->dispatch_semaphore_wait (ds, dt); @@ -65,6 +70,11 @@ void nsync_mu_semaphore_init_gcd (nsync_semaphore *s) { *(dispatch_semaphore_t *)s = dispatch_semaphore_create (0); } +/* Releases system resources associated with *s. */ +void nsync_mu_semaphore_destroy_gcd (nsync_semaphore *s) { + dispatch_release (*(dispatch_semaphore_t *)s); +} + /* Wait until the count of *s exceeds 0, and decrement it. */ errno_t nsync_mu_semaphore_p_gcd (nsync_semaphore *s) { dispatch_semaphore_wait (*(dispatch_semaphore_t *)s, diff --git a/third_party/nsync/mu_semaphore_sem.c b/third_party/nsync/mu_semaphore_sem.c index 8c120085b..cc6027a26 100644 --- a/third_party/nsync/mu_semaphore_sem.c +++ b/third_party/nsync/mu_semaphore_sem.c @@ -1,5 +1,5 @@ -/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ -│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│ +│vi: set et ft=c ts=8 tw=8 fenc=utf-8 :vi│ ╞══════════════════════════════════════════════════════════════════════════════╡ │ Copyright 2022 Justine Alexandra Roberts Tunney │ │ │ @@ -25,6 +25,7 @@ #include "libc/intrin/strace.internal.h" #include "libc/str/str.h" #include "libc/sysv/consts/f.h" +#include "libc/sysv/consts/fd.h" #include "third_party/nsync/mu_semaphore.h" #include "third_party/nsync/time.h" // clang-format off @@ -44,14 +45,19 @@ static nsync_semaphore *sem_big_enough_for_sem = (nsync_semaphore *) (uintptr_t) /* Initialize *s; the initial value is 0. */ void nsync_mu_semaphore_init_sem (nsync_semaphore *s) { - int newfd; struct sem *f = (struct sem *) s; f->id = 0; ASSERT (!sys_sem_init (0, &f->id)); STRACE ("sem_init(0, [%ld]) → 0", f->id); - ASSERT ((newfd = __sys_fcntl (f->id, F_DUPFD_CLOEXEC, 100)) != -1); - ASSERT (!sys_sem_destroy (f->id)); - f->id = newfd; + ASSERT (__sys_fcntl (f->id, F_SETFD, FD_CLOEXEC) == 0); // ouch +} + +/* Releases system resources associated with *s. */ +void nsync_mu_semaphore_destroy_sem (nsync_semaphore *s) { + int rc; + struct sem *f = (struct sem *) s; + ASSERT (!(rc = sys_sem_destroy (f->id))); + STRACE ("sem_destroy(%ld) → %d", rc); } /* Wait until the count of *s exceeds 0, and decrement it. */ @@ -60,9 +66,7 @@ errno_t nsync_mu_semaphore_p_sem (nsync_semaphore *s) { errno_t result; struct sem *f = (struct sem *) s; e = errno; - BEGIN_CANCELLATION_POINT; rc = sys_sem_wait (f->id); - END_CANCELLATION_POINT; STRACE ("sem_wait(%ld) → %d% m", f->id, rc); if (!rc) { result = 0; @@ -82,9 +86,7 @@ errno_t nsync_mu_semaphore_p_with_deadline_sem (nsync_semaphore *s, nsync_time a errno_t result; struct sem *f = (struct sem *) s; e = errno; - BEGIN_CANCELLATION_POINT; rc = sys_sem_timedwait (f->id, &abs_deadline); - END_CANCELLATION_POINT; STRACE ("sem_timedwait(%ld, %s) → %d% m", f->id, DescribeTimespec(0, &abs_deadline), rc); if (!rc) { diff --git a/third_party/nsync/testing/cv_test.c b/third_party/nsync/testing/cv_test.c index 01c042d9a..6c62d4a99 100644 --- a/third_party/nsync/testing/cv_test.c +++ b/third_party/nsync/testing/cv_test.c @@ -17,9 +17,9 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "third_party/nsync/cv.h" #include "libc/errno.h" -#include "libc/fmt/fmt.h" #include "libc/mem/mem.h" #include "libc/runtime/runtime.h" +#include "libc/stdio/stdio.h" #include "libc/str/str.h" #include "third_party/nsync/debug.h" #include "third_party/nsync/mu.h" diff --git a/third_party/nsync/testing/cv_wait_example_test.c b/third_party/nsync/testing/cv_wait_example_test.c index 0682e9fe3..b8537dde7 100644 --- a/third_party/nsync/testing/cv_wait_example_test.c +++ b/third_party/nsync/testing/cv_wait_example_test.c @@ -15,7 +15,7 @@ │ See the License for the specific language governing permissions and │ │ limitations under the License. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/fmt/fmt.h" +#include "libc/stdio/stdio.h" #include "libc/str/str.h" #include "third_party/nsync/array.internal.h" #include "third_party/nsync/cv.h" diff --git a/third_party/nsync/testing/dll_test.c b/third_party/nsync/testing/dll_test.c index 425825a5a..94952a46a 100644 --- a/third_party/nsync/testing/dll_test.c +++ b/third_party/nsync/testing/dll_test.c @@ -16,9 +16,8 @@ │ limitations under the License. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/intrin/dll.h" -#include "libc/fmt/fmt.h" -#include "libc/intrin/dll.h" #include "libc/mem/mem.h" +#include "libc/stdio/stdio.h" #include "libc/str/str.h" #include "third_party/nsync/array.internal.h" #include "third_party/nsync/testing/smprintf.h" diff --git a/third_party/nsync/testing/mu_wait_example_test.c b/third_party/nsync/testing/mu_wait_example_test.c index 8052ca041..f81eeb737 100644 --- a/third_party/nsync/testing/mu_wait_example_test.c +++ b/third_party/nsync/testing/mu_wait_example_test.c @@ -15,7 +15,7 @@ │ See the License for the specific language governing permissions and │ │ limitations under the License. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/fmt/fmt.h" +#include "libc/stdio/stdio.h" #include "libc/str/str.h" #include "third_party/nsync/array.internal.h" #include "third_party/nsync/heap.internal.h" diff --git a/third_party/nsync/testing/smprintf.c b/third_party/nsync/testing/smprintf.c index ca9156acf..c6c29c81e 100644 --- a/third_party/nsync/testing/smprintf.c +++ b/third_party/nsync/testing/smprintf.c @@ -15,10 +15,10 @@ │ See the License for the specific language governing permissions and │ │ limitations under the License. │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/fmt/fmt.h" -#include "libc/mem/mem.h" -#include "libc/str/str.h" #include "third_party/nsync/testing/smprintf.h" +#include "libc/mem/mem.h" +#include "libc/stdio/stdio.h" +#include "libc/str/str.h" // clang-format off char *smprintf (const char *fmt, ...) { diff --git a/third_party/python/Modules/cjkcodecs/xloadzd.c b/third_party/python/Modules/cjkcodecs/xloadzd.c index 718f078cb..a79bdb4cb 100644 --- a/third_party/python/Modules/cjkcodecs/xloadzd.c +++ b/third_party/python/Modules/cjkcodecs/xloadzd.c @@ -19,7 +19,6 @@ #include "libc/assert.h" #include "libc/fmt/leb128.h" #include "libc/intrin/atomic.h" -#include "libc/intrin/kmalloc.h" #include "libc/mem/mem.h" #include "libc/nexgen32e/crc32.h" #include "libc/runtime/internal.h" @@ -53,7 +52,7 @@ void *xloadzd(_Atomic(void *) *a, const void *p, size_t n, size_t m, size_t c, free(q); return 0; } - if (!(r = kmalloc(c * z))) { + if (!(r = malloc(c * z))) { free(q); return 0; } diff --git a/tool/build/runitd.c b/tool/build/runitd.c index 0d064d149..7491e6e7c 100644 --- a/tool/build/runitd.c +++ b/tool/build/runitd.c @@ -264,9 +264,8 @@ void StartTcpServer(void) { INFOF("listening on tcp:%s", DescribeAddress(&g_servaddr)); if (g_sendready) { printf("ready %hu\n", ntohs(g_servaddr.sin_port)); - fflush(stdout); - fclose(stdout); - dup2(g_bogusfd, fileno(stdout)); + close(1); + dup2(g_bogusfd, 1); } } @@ -698,6 +697,7 @@ WaitAgain: } void HandleClient(void) { + struct stat st; struct Client *client; client = calloc(1, sizeof(struct Client)); client->addrsize = sizeof(client->addr); @@ -720,6 +720,9 @@ void HandleClient(void) { WARNF("poll failed %m"); } } + if (fstat(2, &st) != -1 && st.st_size > kLogMaxBytes) { + ftruncate(2, 0); // auto rotate log + } sigset_t mask; pthread_attr_t attr; sigfillset(&mask); @@ -749,17 +752,18 @@ int Serve(void) { } void Daemonize(void) { - VERBF("Daemonize"); - struct stat st; if (fork() > 0) _exit(0); setsid(); if (fork() > 0) _exit(0); - dup2(g_bogusfd, fileno(stdin)); - if (!g_sendready) dup2(g_bogusfd, fileno(stdout)); - freopen(kLogFile, "ae", stderr); - if (fstat(fileno(stderr), &st) != -1 && st.st_size > kLogMaxBytes) { - ftruncate(fileno(stderr), 0); + dup2(g_bogusfd, 0); + if (!g_sendready) dup2(g_bogusfd, 1); + close(2); + open(kLogFile, O_CREAT | O_WRONLY | O_APPEND | O_CLOEXEC, 0644); + extern long __klog_handle; + if (__klog_handle > 0) { + close(__klog_handle); } + __klog_handle = 2; } int main(int argc, char *argv[]) { @@ -779,8 +783,8 @@ int main(int argc, char *argv[]) { } else { g_bogusfd = open("/dev/zero", O_RDONLY | O_CLOEXEC); } - if (g_daemonize) Daemonize(); mkdir("o", 0700); + if (g_daemonize) Daemonize(); Serve(); free(g_psk); #if IsModeDbg()