Improve multithreading

libc/thread/posixthread.internal.h

@@ -74,7 +74,7 @@ struct PosixThread {
   atomic_int pt_canceled;  // 0x04: thread has bad beliefs
   _Atomic(enum PosixThreadStatus) pt_status;
   atomic_int ptid;            // transitions 0 → tid
-  atomic_int pt_refs;         // negative means free
+  atomic_int pt_refs;         // prevents decimation
   void *(*pt_start)(void *);  // creation callback
   void *pt_arg;               // start's parameter
   void *pt_rc;                // start's return value
@@ -103,14 +103,13 @@ int _pthread_setschedparam_freebsd(int, int, const struct sched_param *);
 int _pthread_tid(struct PosixThread *) libcesque;
 intptr_t _pthread_syshand(struct PosixThread *) libcesque;
 long _pthread_cancel_ack(void) libcesque;
-void _pthread_decimate(void) libcesque;
-void _pthread_free(struct PosixThread *, bool) libcesque;
+void _pthread_decimate(bool) libcesque;
+void _pthread_free(struct PosixThread *) libcesque;
 void _pthread_lock(void) libcesque;
 void _pthread_onfork_child(void) libcesque;
 void _pthread_onfork_parent(void) libcesque;
 void _pthread_onfork_prepare(void) libcesque;
 void _pthread_unlock(void) libcesque;
-void _pthread_unref(struct PosixThread *) libcesque;
 void _pthread_zombify(struct PosixThread *) libcesque;
 
 forceinline pureconst struct PosixThread *_pthread_self(void) {
@@ -118,7 +117,11 @@ forceinline pureconst struct PosixThread *_pthread_self(void) {
 }
 
 forceinline void _pthread_ref(struct PosixThread *pt) {
-  atomic_fetch_add_explicit(&pt->pt_refs, 1, memory_order_relaxed);
+  atomic_fetch_add_explicit(&pt->pt_refs, 1, memory_order_acq_rel);
+}
+
+forceinline void _pthread_unref(struct PosixThread *pt) {
+  atomic_fetch_sub_explicit(&pt->pt_refs, 1, memory_order_acq_rel);
 }
 
 COSMOPOLITAN_C_END_

libc/thread/pt.internal.h

@@ -8,5 +8,7 @@
 #define PT_MASKED         16
 #define PT_INCANCEL       32
 #define PT_OPENBSD_KLUDGE 64
+#define PT_EXITING        128
+#define PT_OWNSIGALTSTACK 256
 
 #endif /* COSMOPOLITAN_LIBC_THREAD_PT_H_ */

libc/thread/pthread_attr_getguardsize.c

@@ -19,7 +19,7 @@
 #include "libc/thread/thread.h"
 
 /**
- * Returns size of unmapped pages at bottom of stack.
+ * Returns size of protected region at bottom of thread stack.
  *
  * @param guardsize will be set to guard size in bytes
  * @return 0 on success, or errno on error

libc/thread/pthread_attr_getsigaltstack_np.c

@@ -0,0 +1,34 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/runtime/stack.h"
+#include "libc/thread/thread.h"
+
+/**
+ * Returns configuration for thread signal stack.
+ *
+ * @param stackaddr will be set to signal stack address
+ * @return 0 on success, or errno on error
+ * @see pthread_attr_setsigaltstacksize_np()
+ */
+errno_t pthread_attr_getsigaltstack_np(const pthread_attr_t *attr,
+                                       void **stackaddr, size_t *stacksize) {
+  *stackaddr = attr->__sigaltstackaddr;
+  *stacksize = attr->__sigaltstacksize;
+  return 0;
+}

libc/thread/pthread_attr_getsigaltstacksize_np.c

@@ -0,0 +1,36 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/runtime/stack.h"
+#include "libc/thread/thread.h"
+
+/**
+ * Returns size of thread signal stack.
+ *
+ * This defaults to zero, which means that cosmo won't allocate a
+ * managed signal stack for newly created threads.
+ *
+ * @param x will be set to stack size in bytes
+ * @return 0 on success, or errno on error
+ * @see pthread_attr_setsigaltstacksize_np()
+ */
+errno_t pthread_attr_getsigaltstacksize_np(const pthread_attr_t *a,
+                                           size_t *stacksize) {
+  *stacksize = a->__sigaltstacksize;
+  return 0;
+}

libc/thread/pthread_attr_setguardsize.c

@@ -19,18 +19,16 @@
 #include "libc/thread/thread.h"
 
 /**
- * Sets size of unmapped pages at bottom of stack.
+ * Sets size of protected region at bottom of thread stack.
  *
- * This value will be rounded up to the host microprocessor page size,
- * which is usually 4096 or 16384. It's important to write code in such
- * a way that that code can't skip over the guard area. GCC has warnings
- * like `-Wframe-larger-than=4096 -Walloca-larger-than=4096` which help
- * guarantee your code is safe in this regard. It should be assumed the
- * guard pages exist beneath the stack pointer, rather than the bottom
- * of the stack, since guard pages are also used to grow down commit,
- * which can be poked using CheckLargeStackAllocation().
+ * Cosmopolitan sets this value to `sysconf(_SC_PAGESIZE)` by default.
  *
- * @param guardsize contains guard size in bytes
+ * You may set `guardsize` to disable the stack guard feature and gain a
+ * slight performance advantage by avoiding mprotect() calls. Note that
+ * it could make your code more prone to silent unreported corruption.
+ *
+ * @param guardsize contains guard size in bytes, which is implicitly
+ *     rounded up to `sysconf(_SC_PAGESIZE)`, or zero to disable
  * @return 0 on success, or errno on error
  */
 errno_t pthread_attr_setguardsize(pthread_attr_t *attr, size_t guardsize) {

libc/thread/pthread_attr_setsigaltstack_np.c

@@ -0,0 +1,42 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/dce.h"
+#include "libc/errno.h"
+#include "libc/limits.h"
+#include "libc/runtime/runtime.h"
+#include "libc/thread/thread.h"
+
+/**
+ * Defines user-owned signal stack for thread.
+ */
+errno_t pthread_attr_setsigaltstack_np(pthread_attr_t *attr, void *stackaddr,
+                                       size_t stacksize) {
+  if (!stackaddr) {
+    attr->__sigaltstackaddr = 0;
+    attr->__sigaltstacksize = 0;
+    return 0;
+  }
+  if (stacksize > INT_MAX)
+    return EINVAL;
+  if (stacksize < __get_minsigstksz())
+    return EINVAL;
+  attr->__sigaltstackaddr = stackaddr;
+  attr->__sigaltstacksize = stacksize;
+  return 0;
+}

libc/thread/pthread_attr_setsigaltstacksize_np.c

@@ -0,0 +1,78 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/errno.h"
+#include "libc/limits.h"
+#include "libc/runtime/runtime.h"
+#include "libc/thread/thread.h"
+
+/**
+ * Defines size of cosmo-owned signal stack for thread.
+ *
+ * The sigaltstack() function is useful for writing robust programs that
+ * can recover from the occasional thread having a stack overflow rather
+ * than having the entire process crash. To use it normally, sigaltstack
+ * needs to be called at the start of each thread with a unique piece of
+ * memory. However this is challenging to do *correctly* without support
+ * from the POSIX threads runtime, since canceled or crashed threads may
+ * need to execute on the signal stack during pthread_exit() which would
+ * prevent a thread-local storage key destructor from free()'ing it.
+ *
+ * By default pthread_create() will not install a sigaltstack() on newly
+ * created threads. If this function is called, on the attributes object
+ * that gets passed to pthread_create(), then it'll use malloc() to make
+ * a stack for the thread using the size you specify here. The threading
+ * runtime will also free that memory safely after complete termination.
+ *
+ *     pthread_t id;
+ *     pthread_attr_t attr;
+ *     pthread_attr_init(&attr);
+ *     pthread_attr_setguardsize(&attr, getpagesize());
+ *     pthread_attr_setsigaltstacksize_np(&attr, stacksize);
+ *     pthread_create(&id, &attr, func, 0);
+ *     pthread_attr_destroy(&attr);
+ *     pthread_join(id, 0);
+ *
+ * Try using a size of `sysconf(_SC_SIGSTKSZ)`. If you want the smallest
+ * size possible, then `sysconf(_SC_MINSIGSTKSZ) + 2048` is probably the
+ * smallest value that can reasonably expected to work with pthread_exit
+ *
+ *     struct sigaction sa;
+ *     sigemptyset(&sa.sa_mask);
+ *     sa.sa_flags = SA_SIGINFO | SA_ONSTACK;
+ *     sa.sa_sigaction = on_crash_signal;
+ *     sigaction(SIGSEGV, &sa, 0);
+ *
+ * Please note that in order for this to work, your handlers for signals
+ * such as SIGSEGV and SIGBUS need to use SA_ONSTACK in your sa_flags.
+ *
+ * @param stacksize contains stack size in bytes, or 0 to disable
+ * @return 0 on success, or errno on error
+ * @raise EINVAL if `stacksize` is less than `sysconf(_SC_MINSIGSTKSZ)`
+ */
+errno_t pthread_attr_setsigaltstacksize_np(pthread_attr_t *a,
+                                           size_t stacksize) {
+  if (stacksize) {
+    if (stacksize > INT_MAX)
+      return EINVAL;
+    if (stacksize < __get_minsigstksz())
+      return EINVAL;
+  }
+  a->__sigaltstacksize = stacksize;
+  return 0;
+}

libc/thread/pthread_attr_setstack.c

@@ -18,6 +18,7 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/dce.h"
 #include "libc/errno.h"
+#include "libc/limits.h"
 #include "libc/thread/thread.h"
 
 /**
@@ -71,6 +72,8 @@ errno_t pthread_attr_setstack(pthread_attr_t *attr, void *stackaddr,
     attr->__stacksize = 0;
     return 0;
   }
+  if (stacksize > INT_MAX)
+    return EINVAL;
   if (stacksize < PTHREAD_STACK_MIN)
     return EINVAL;
   attr->__stackaddr = stackaddr;

libc/thread/pthread_attr_setstacksize.c

@@ -17,6 +17,7 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/errno.h"
+#include "libc/limits.h"
 #include "libc/thread/thread.h"
 
 /**
@@ -27,6 +28,8 @@
  * @raise EINVAL if `stacksize` is less than `PTHREAD_STACK_MIN`
  */
 errno_t pthread_attr_setstacksize(pthread_attr_t *a, size_t stacksize) {
+  if (stacksize > INT_MAX)
+    return EINVAL;
   if (stacksize < PTHREAD_STACK_MIN)
     return EINVAL;
   a->__stacksize = stacksize;

libc/thread/pthread_create.c

@@ -18,6 +18,7 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/assert.h"
 #include "libc/calls/calls.h"
+#include "libc/calls/struct/sigaltstack.h"
 #include "libc/calls/struct/sigset.h"
 #include "libc/calls/struct/sigset.internal.h"
 #include "libc/calls/syscall-sysv.internal.h"
@@ -67,54 +68,104 @@ __static_yoink("_pthread_onfork_child");
 #define MAP_ANON_OPENBSD  0x1000
 #define MAP_STACK_OPENBSD 0x4000
 
-void _pthread_free(struct PosixThread *pt, bool isfork) {
+void _pthread_free(struct PosixThread *pt) {
+
+  // thread must be removed from _pthread_list before calling
   unassert(dll_is_alone(&pt->list) && &pt->list != _pthread_list);
+
+  // do nothing for the one and only magical statical posix thread
   if (pt->pt_flags & PT_STATIC)
     return;
+
+  // unmap stack if the cosmo runtime was responsible for mapping it
   if (pt->pt_flags & PT_OWNSTACK)
     unassert(!munmap(pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize));
-  if (!isfork) {
-    uint64_t syshand =
-        atomic_load_explicit(&pt->tib->tib_syshand, memory_order_acquire);
-    if (syshand) {
-      if (IsWindows())
-        unassert(CloseHandle(syshand));
-      else if (IsXnuSilicon())
-        __syslib->__pthread_join(syshand, 0);
-    }
+
+  // free any additional upstream system resources
+  // our fork implementation wipes this handle in child automatically
+  uint64_t syshand =
+      atomic_load_explicit(&pt->tib->tib_syshand, memory_order_acquire);
+  if (syshand) {
+    if (IsWindows())
+      unassert(CloseHandle(syshand));  // non-inheritable
+    else if (IsXnuSilicon())
+      unassert(!__syslib->__pthread_join(syshand, 0));
   }
+
+  // free heap memory associated with thread
+  if (pt->pt_flags & PT_OWNSIGALTSTACK)
+    free(pt->pt_attr.__sigaltstackaddr);
   free(pt->pt_tls);
   free(pt);
 }
 
-void _pthread_decimate(void) {
-  struct Dll *e;
+void _pthread_decimate(bool annihilation_only) {
   struct PosixThread *pt;
+  struct Dll *e, *e2, *list = 0;
   enum PosixThreadStatus status;
-StartOver:
+
+  // acquire posix threads gil
   _pthread_lock();
-  for (e = dll_last(_pthread_list); e; e = dll_prev(_pthread_list, e)) {
+
+  // swiftly remove every single zombie
+  // that isn't being held by a killing thread
+  for (e = dll_last(_pthread_list); e; e = e2) {
+    e2 = dll_prev(_pthread_list, e);
     pt = POSIXTHREAD_CONTAINER(e);
+    if (atomic_load_explicit(&pt->pt_refs, memory_order_acquire) > 0)
+      continue;  // pthread_kill() has a lease on this thread
     status = atomic_load_explicit(&pt->pt_status, memory_order_acquire);
     if (status != kPosixThreadZombie)
-      break;
-    if (!atomic_load_explicit(&pt->tib->tib_tid, memory_order_acquire)) {
-      dll_remove(&_pthread_list, e);
-      _pthread_unlock();
-      _pthread_unref(pt);
-      goto StartOver;
-    }
+      break;  // zombies only exist at the end of the linked list
+    if (atomic_load_explicit(&pt->tib->tib_tid, memory_order_acquire))
+      continue;  // undead thread should that'll stop existing soon
+    dll_remove(&_pthread_list, e);
+    dll_make_first(&list, e);
   }
+
+  // code like pthread_exit() needs to call this in order to know if
+  // it's appropriate to run exit() handlers however we really don't
+  // want to have a thread exiting block on a bunch of __maps locks!
+  // therefore we only take action if we'll destroy all but the self
+  if (annihilation_only)
+    if (!(_pthread_list == _pthread_list->prev &&
+          _pthread_list == _pthread_list->next)) {
+      dll_make_last(&_pthread_list, list);
+      list = 0;
+    }
+
+  // release posix threads gil
   _pthread_unlock();
+
+  // now free our thread local batch of zombies
+  // because death is a release and not a punishment
+  // this is advantaged by not holding locks over munmap
+  while ((e = dll_first(list))) {
+    pt = POSIXTHREAD_CONTAINER(e);
+    dll_remove(&list, e);
+    _pthread_free(pt);
+  }
 }
 
 static int PosixThread(void *arg, int tid) {
   void *rc;
   struct PosixThread *pt = arg;
+
+  // setup scheduling
   if (pt->pt_attr.__inheritsched == PTHREAD_EXPLICIT_SCHED) {
     unassert(_weaken(_pthread_reschedule));
     _weaken(_pthread_reschedule)(pt);  // yoinked by attribute builder
   }
+
+  // setup signal stack
+  if (pt->pt_attr.__sigaltstacksize) {
+    struct sigaltstack ss;
+    ss.ss_sp = pt->pt_attr.__sigaltstackaddr;
+    ss.ss_size = pt->pt_attr.__sigaltstacksize;
+    ss.ss_flags = 0;
+    unassert(!sigaltstack(&ss, 0));
+  }
+
   // set long jump handler so pthread_exit can bring control back here
   if (!setjmp(pt->pt_exiter)) {
     sigdelset(&pt->pt_attr.__sigmask, SIGTHR);
@@ -130,41 +181,45 @@ static int PosixThread(void *arg, int tid) {
     // calling pthread_exit() will either jump back here, or call exit
     pthread_exit(rc);
   }
+
   // avoid signal handler being triggered after we trash our own stack
   __sig_block();
+
   // return to clone polyfill which clears tid, wakes futex, and exits
   return 0;
 }
 
-static int FixupCustomStackOnOpenbsd(pthread_attr_t *attr) {
-  // OpenBSD: Only permits RSP to occupy memory that's been explicitly
-  // defined as stack memory. We need to squeeze the provided interval
-  // in order to successfully call mmap(), which will return EINVAL if
-  // these calculations should overflow.
-  size_t n;
-  uintptr_t x, y;
-  int e, rc, pagesz;
-  pagesz = __pagesize;
-  n = attr->__stacksize;
-  x = (uintptr_t)attr->__stackaddr;
-  y = ROUNDUP(x, pagesz);
-  n -= y - x;
-  n = ROUNDDOWN(n, pagesz);
-  e = errno;
-  if (__sys_mmap((void *)y, n, PROT_READ | PROT_WRITE,
-                 MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD,
-                 -1, 0, 0) == (void *)y) {
-    attr->__stackaddr = (void *)y;
-    attr->__stacksize = n;
-    return 0;
-  } else {
-    rc = errno;
-    errno = e;
-    if (rc == EOVERFLOW) {
-      rc = EINVAL;
-    }
-    return rc;
+static bool TellOpenbsdThisIsStackMemory(void *addr, size_t size) {
+  return __sys_mmap(
+             addr, size, PROT_READ | PROT_WRITE,
+             MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD, -1,
+             0, 0) == addr;
+}
+
+// OpenBSD only permits RSP to occupy memory that's been explicitly
+// defined as stack memory, i.e. `lo <= %rsp < hi` must be the case
+static errno_t FixupCustomStackOnOpenbsd(pthread_attr_t *attr) {
+
+  // get interval
+  uintptr_t lo = (uintptr_t)attr->__stackaddr;
+  uintptr_t hi = lo + attr->__stacksize;
+
+  // squeeze interval
+  lo = (lo + __pagesize - 1) & -__pagesize;
+  hi = hi & -__pagesize;
+
+  // tell os it's stack memory
+  errno_t olderr = errno;
+  if (!TellOpenbsdThisIsStackMemory((void *)lo, hi - lo)) {
+    errno_t err = errno;
+    errno = olderr;
+    return err;
   }
+
+  // update attributes with usable stack address
+  attr->__stackaddr = (void *)lo;
+  attr->__stacksize = hi - lo;
+  return 0;
 }
 
 static errno_t pthread_create_impl(pthread_t *thread,
@@ -204,7 +259,7 @@ static errno_t pthread_create_impl(pthread_t *thread,
     // assume they know what they're doing as much as possible
     if (IsOpenbsd()) {
       if ((rc = FixupCustomStackOnOpenbsd(&pt->pt_attr))) {
-        _pthread_free(pt, false);
+        _pthread_free(pt);
         return rc;
       }
     }
@@ -214,21 +269,17 @@ static errno_t pthread_create_impl(pthread_t *thread,
     pt->pt_attr.__guardsize = ROUNDUP(pt->pt_attr.__guardsize, pagesize);
     pt->pt_attr.__stacksize = pt->pt_attr.__stacksize;
     if (pt->pt_attr.__guardsize + pagesize > pt->pt_attr.__stacksize) {
-      _pthread_free(pt, false);
+      _pthread_free(pt);
       return EINVAL;
     }
     pt->pt_attr.__stackaddr =
         mmap(0, pt->pt_attr.__stacksize, PROT_READ | PROT_WRITE,
              MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
     if (pt->pt_attr.__stackaddr != MAP_FAILED) {
-      if (IsOpenbsd() &&
-          __sys_mmap(
-              pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize,
-              PROT_READ | PROT_WRITE,
-              MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD,
-              -1, 0, 0) != pt->pt_attr.__stackaddr) {
-        notpossible;
-      }
+      if (IsOpenbsd())
+        if (!TellOpenbsdThisIsStackMemory(pt->pt_attr.__stackaddr,
+                                          pt->pt_attr.__stacksize))
+          notpossible;
       if (pt->pt_attr.__guardsize)
         if (mprotect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize,
                      PROT_NONE | PROT_GUARD))
@@ -236,7 +287,7 @@ static errno_t pthread_create_impl(pthread_t *thread,
     }
     if (!pt->pt_attr.__stackaddr || pt->pt_attr.__stackaddr == MAP_FAILED) {
       rc = errno;
-      _pthread_free(pt, false);
+      _pthread_free(pt);
       errno = e;
       if (rc == EINVAL || rc == EOVERFLOW) {
         return EINVAL;
@@ -247,6 +298,18 @@ static errno_t pthread_create_impl(pthread_t *thread,
     pt->pt_flags |= PT_OWNSTACK;
   }
 
+  // setup signal stack
+  if (pt->pt_attr.__sigaltstacksize) {
+    if (!pt->pt_attr.__sigaltstackaddr) {
+      if (!(pt->pt_attr.__sigaltstackaddr =
+                malloc(pt->pt_attr.__sigaltstacksize))) {
+        _pthread_free(pt);
+        return errno;
+      }
+      pt->pt_flags |= PT_OWNSIGALTSTACK;
+    }
+  }
+
   // set initial status
   pt->tib->tib_pthread = (pthread_t)pt;
   atomic_store_explicit(&pt->tib->tib_sigmask, -1, memory_order_relaxed);
@@ -264,7 +327,7 @@ static errno_t pthread_create_impl(pthread_t *thread,
                             memory_order_relaxed);
       break;
     default:
-      _pthread_free(pt, false);
+      _pthread_free(pt);
       return EINVAL;
   }
 
@@ -284,7 +347,7 @@ static errno_t pthread_create_impl(pthread_t *thread,
     _pthread_lock();
     dll_remove(&_pthread_list, &pt->list);
     _pthread_unlock();
-    _pthread_free(pt, false);
+    _pthread_free(pt);
     return rc;
   }
 
@@ -353,7 +416,7 @@ static const char *DescribeHandle(char buf[12], errno_t err, pthread_t *th) {
 errno_t pthread_create(pthread_t *thread, const pthread_attr_t *attr,
                        void *(*start_routine)(void *), void *arg) {
   errno_t err;
-  _pthread_decimate();
+  _pthread_decimate(false);
   BLOCK_SIGNALS;
   err = pthread_create_impl(thread, attr, start_routine, arg, _SigMask);
   ALLOW_SIGNALS;

libc/thread/pthread_decimate_np.c

@@ -32,6 +32,6 @@
  * @return 0 on success, or errno on error
  */
 int pthread_decimate_np(void) {
-  _pthread_decimate();
+  _pthread_decimate(false);
   return 0;
 }

libc/thread/pthread_detach.c

@@ -40,10 +40,8 @@ static errno_t pthread_detach_impl(struct PosixThread *pt) {
     if (atomic_compare_exchange_weak_explicit(&pt->pt_status, &status,
                                               transition, memory_order_release,
                                               memory_order_relaxed)) {
-      if (transition == kPosixThreadZombie) {
+      if (transition == kPosixThreadZombie)
         _pthread_zombify(pt);
-      }
-      _pthread_decimate();
       return 0;
     }
   }

libc/thread/pthread_exit.c

@@ -69,23 +69,33 @@
  * @noreturn
  */
 wontreturn void pthread_exit(void *rc) {
+  int orphan;
   struct CosmoTib *tib;
   struct PosixThread *pt;
   enum PosixThreadStatus status, transition;
 
+  STRACE("pthread_exit(%p)", rc);
+
+  // get posix thread object
   tib = __get_tls();
   pt = (struct PosixThread *)tib->tib_pthread;
-  pt->pt_flags |= PT_NOCANCEL;
-  pt->pt_rc = rc;
 
-  STRACE("pthread_exit(%p)", rc);
+  // "The behavior of pthread_exit() is undefined if called from a
+  //  cancellation cleanup handler or destructor function that was
+  //  invoked as a result of either an implicit or explicit call to
+  //  pthread_exit()." ──Quoth POSIX.1-2017
+  unassert(!(pt->pt_flags & PT_EXITING));
+
+  // set state
+  pt->pt_flags |= PT_NOCANCEL | PT_EXITING;
+  pt->pt_rc = rc;
 
   // free resources
   __cxa_thread_finalize();
-  _pthread_decimate();
 
   // run atexit handlers if orphaned thread
-  if (pthread_orphan_np())
+  _pthread_decimate(true);
+  if ((orphan = pthread_orphan_np()))
     if (_weaken(__cxa_finalize))
       _weaken(__cxa_finalize)(NULL);
 
@@ -113,8 +123,11 @@ wontreturn void pthread_exit(void *rc) {
   if (transition == kPosixThreadZombie)
     _pthread_zombify(pt);
 
-  // check if this is the last survivor
-  if (pthread_orphan_np()) {
+  // "The process shall exit with an exit status of 0 after the last
+  //  thread has been terminated. The behavior shall be as if the
+  //  implementation called exit() with a zero argument at thread
+  //  termination time." ──Quoth POSIX.1-2017
+  if (orphan) {
     for (const uintptr_t *p = __fini_array_end; p > __fini_array_start;)
       ((void (*)(void))(*--p))();
     _Exit(0);

libc/thread/pthread_timedjoin_np.c

@@ -55,29 +55,38 @@ static const char *DescribeReturnValue(char buf[30], int err, void **value) {
  *
  * @return 0 on success, or errno on error
  * @raise ECANCELED if calling thread was cancelled in masked mode
+ * @raise EDEADLK if `ctid` refers calling thread's own ctid futex
  * @raise EBUSY if `abstime` was specified and deadline expired
  * @cancelationpoint
  */
 static errno_t _pthread_wait(atomic_int *ctid, struct timespec *abstime) {
-  int x, e, rc = 0;
-  unassert(ctid != &__get_tls()->tib_tid);
-  // "If the thread calling pthread_join() is canceled, then the target
-  //  thread shall not be detached."  ──Quoth POSIX.1-2017
-  if (!(rc = pthread_testcancel_np())) {
-    BEGIN_CANCELATION_POINT;
-    while ((x = atomic_load_explicit(ctid, memory_order_acquire))) {
-      e = nsync_futex_wait_(ctid, x, !IsWindows() && !IsXnu(), abstime);
-      if (e == -ECANCELED) {
-        rc = ECANCELED;
-        break;
-      } else if (e == -ETIMEDOUT) {
-        rc = EBUSY;
-        break;
+  int x, e;
+  errno_t err = 0;
+  if (ctid == &__get_tls()->tib_tid) {
+    // "If an implementation detects that the value specified by the
+    //  thread argument to pthread_join() refers to the calling thread,
+    //  it is recommended that the function should fail and report an
+    //  [EDEADLK] error." ──Quoth POSIX.1-2017
+    err = EDEADLK;
+  } else {
+    // "If the thread calling pthread_join() is canceled, then the target
+    //  thread shall not be detached."  ──Quoth POSIX.1-2017
+    if (!(err = pthread_testcancel_np())) {
+      BEGIN_CANCELATION_POINT;
+      while ((x = atomic_load_explicit(ctid, memory_order_acquire))) {
+        e = nsync_futex_wait_(ctid, x, !IsWindows() && !IsXnu(), abstime);
+        if (e == -ECANCELED) {
+          err = ECANCELED;
+          break;
+        } else if (e == -ETIMEDOUT) {
+          err = EBUSY;
+          break;
+        }
       }
+      END_CANCELATION_POINT;
     }
-    END_CANCELATION_POINT;
   }
-  return rc;
+  return err;
 }
 
 /**
@@ -97,6 +106,7 @@ static errno_t _pthread_wait(atomic_int *ctid, struct timespec *abstime) {
  *     when we'll stop waiting; if this is null we will wait forever
  * @return 0 on success, or errno on error
  * @raise ECANCELED if calling thread was cancelled in masked mode
+ * @raise EDEADLK if `thread` refers to calling thread
  * @raise EBUSY if `abstime` deadline elapsed
  * @cancelationpoint
  * @returnserrno
@@ -104,26 +114,29 @@ static errno_t _pthread_wait(atomic_int *ctid, struct timespec *abstime) {
 errno_t pthread_timedjoin_np(pthread_t thread, void **value_ptr,
                              struct timespec *abstime) {
   int tid;
-  errno_t err;
+  errno_t err = 0;
   struct PosixThread *pt;
   enum PosixThreadStatus status;
   pt = (struct PosixThread *)thread;
-  tid = _pthread_tid(pt);
-  unassert(_pthread_tid(pt));
-  status = atomic_load_explicit(&pt->pt_status, memory_order_acquire);
+
   // "The behavior is undefined if the value specified by the thread
   //  argument to pthread_join() does not refer to a joinable thread."
   //                                  ──Quoth POSIX.1-2017
+  unassert((tid = _pthread_tid(pt)));
+  status = atomic_load_explicit(&pt->pt_status, memory_order_acquire);
   unassert(status == kPosixThreadJoinable || status == kPosixThreadTerminated);
+
+  // "The results of multiple simultaneous calls to pthread_join()
+  //  specifying the same target thread are undefined."
+  //                                  ──Quoth POSIX.1-2017
   if (!(err = _pthread_wait(&pt->tib->tib_tid, abstime))) {
-    _pthread_lock();
-    dll_remove(&_pthread_list, &pt->list);
-    _pthread_unlock();
-    if (value_ptr) {
+    atomic_store_explicit(&pt->pt_status, kPosixThreadZombie,
+                          memory_order_release);
+    _pthread_zombify(pt);
+    if (value_ptr)
       *value_ptr = pt->pt_rc;
-    }
-    _pthread_unref(pt);
   }
+
   STRACE("pthread_timedjoin_np(%d, %s, %s) → %s", tid,
          DescribeReturnValue(alloca(30), err, value_ptr),
          DescribeTimespec(err ? -1 : 0, abstime), DescribeErrno(err));

libc/thread/thread.h

@@ -94,9 +94,11 @@ typedef struct pthread_attr_s {
   int __schedpolicy;
   int __contentionscope;
   int __guardsize;
-  size_t __stacksize;
+  int __stacksize;
+  int __sigaltstacksize;
   uint64_t __sigmask;
   void *__stackaddr;
+  void *__sigaltstackaddr;
 } pthread_attr_t;
 
 struct _pthread_cleanup_buffer {
@@ -117,6 +119,8 @@ int pthread_attr_getschedpolicy(const pthread_attr_t *, int *) libcesque paramsn
 int pthread_attr_getscope(const pthread_attr_t *, int *) libcesque paramsnonnull();
 int pthread_attr_getstack(const pthread_attr_t *, void **, size_t *) libcesque paramsnonnull();
 int pthread_attr_getstacksize(const pthread_attr_t *, size_t *) libcesque paramsnonnull();
+int pthread_attr_getsigaltstack_np(const pthread_attr_t *, void **, size_t *) libcesque paramsnonnull();
+int pthread_attr_getsigaltstacksize_np(const pthread_attr_t *, size_t *) libcesque paramsnonnull();
 int pthread_attr_init(pthread_attr_t *) libcesque paramsnonnull();
 int pthread_attr_setdetachstate(pthread_attr_t *, int) libcesque paramsnonnull();
 int pthread_attr_setguardsize(pthread_attr_t *, size_t) libcesque paramsnonnull();
@@ -125,6 +129,8 @@ int pthread_attr_setschedpolicy(pthread_attr_t *, int) libcesque paramsnonnull()
 int pthread_attr_setscope(pthread_attr_t *, int) libcesque paramsnonnull();
 int pthread_attr_setstack(pthread_attr_t *, void *, size_t) libcesque paramsnonnull((1));
 int pthread_attr_setstacksize(pthread_attr_t *, size_t) libcesque paramsnonnull();
+int pthread_attr_setsigaltstack_np(pthread_attr_t *, void *, size_t) libcesque paramsnonnull((1));
+int pthread_attr_setsigaltstacksize_np(pthread_attr_t *, size_t);
 int pthread_barrier_destroy(pthread_barrier_t *) libcesque paramsnonnull();
 int pthread_barrier_init(pthread_barrier_t *, const pthread_barrierattr_t *, unsigned) libcesque paramsnonnull((1));
 int pthread_barrier_wait(pthread_barrier_t *) libcesque paramsnonnull();