Make threads faster and more reliable

This change doubles the performance of thread spawning. That's thanks to
our new stack manager, which allows us to avoid zeroing stacks. It gives
us 15µs spawns rather than 30µs spawns on Linux. Also, pthread_exit() is
faster now, since it doesn't need to acquire the pthread GIL. On NetBSD,
that helps us avoid allocating too many semaphores. Even if that happens
we're now able to survive semaphores running out and even memory running
out, when allocating *NSYNC waiter objects. I found a lot more rare bugs
in the POSIX threads runtime that could cause things to crash, if you've
got dozens of threads all spawning and joining dozens of threads. I want
cosmo to be world class production worthy for 2025 so happy holidays all

libc/intrin/count.c

@@ -0,0 +1,26 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2024 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/atomic.h"
+#include "libc/stdalign.h"
+#include "libc/thread/thread.h"
+
+// this counter is important because pthread_exit() needs to know if
+// it's an orphan thread, without needing to acquire _pthread_lock()
+// which causes contention and a file descriptor explosion on netbsd
+alignas(64) atomic_uint _pthread_count = 1;

libc/intrin/itimer.c

@@ -0,0 +1,42 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2024 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/thread/itimer.h"
+#include "libc/str/str.h"
+
+struct IntervalTimer __itimer = {
+    .lock = PTHREAD_MUTEX_INITIALIZER,
+    .cond = PTHREAD_COND_INITIALIZER,
+};
+
+textwindows void __itimer_lock(void) {
+  pthread_mutex_lock(&__itimer.lock);
+}
+
+textwindows void __itimer_unlock(void) {
+  pthread_mutex_unlock(&__itimer.lock);
+}
+
+textwindows void __itimer_wipe_and_reset(void) {
+  // timers aren't inherited by forked subprocesses
+  bzero(&__itimer.it, sizeof(__itimer.it));
+  pthread_mutex_wipe_np(&__itimer.lock);
+  pthread_cond_init(&__itimer.cond, 0);
+  __itimer.thread = 0;
+  __itimer.once = 0;
+}

libc/intrin/kisdangerous.c

@@ -0,0 +1,36 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2024 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/intrin/kprintf.h"
+#include "libc/intrin/maps.h"
+
+privileged optimizesize bool32 kisdangerous(const void *addr) {
+  bool32 res = true;
+  __maps_lock();
+  if (__maps.maps) {
+    struct Map *map;
+    if ((map = __maps_floor(addr)))
+      if ((const char *)addr >= map->addr &&
+          (const char *)addr < map->addr + map->size)
+        res = false;
+  } else {
+    res = false;
+  }
+  __maps_unlock();
+  return res;
+}

libc/intrin/kprintf.greg.c

@@ -160,22 +160,6 @@ __funline bool kischarmisaligned(const char *p, signed char t) {
   return false;
 }
 
-ABI bool32 kisdangerous(const void *addr) {
-  bool32 res = true;
-  __maps_lock();
-  if (__maps.maps) {
-    struct Map *map;
-    if ((map = __maps_floor(addr)))
-      if ((const char *)addr >= map->addr &&
-          (const char *)addr < map->addr + map->size)
-        res = false;
-  } else {
-    res = false;
-  }
-  __maps_unlock();
-  return res;
-}
-
 ABI static void klogclose(long fd) {
 #ifdef __x86_64__
   long ax = __NR_close;

libc/intrin/pthread_cond_init.c

@@ -0,0 +1,37 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/dce.h"
+#include "libc/sysv/consts/clock.h"
+#include "libc/thread/thread.h"
+
+/**
+ * Initializes condition variable.
+ *
+ * @param attr may be null
+ * @return 0 on success, or error number on failure
+ */
+errno_t pthread_cond_init(pthread_cond_t *cond,
+                          const pthread_condattr_t *attr) {
+  *cond = (pthread_cond_t){0};
+  if (attr) {
+    cond->_pshared = attr->_pshared;
+    cond->_clock = attr->_clock;
+  }
+  return 0;
+}

libc/intrin/pthread_orphan_np.c

@@ -16,6 +16,8 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/assert.h"
+#include "libc/intrin/atomic.h"
 #include "libc/thread/posixthread.internal.h"
 #include "libc/thread/thread.h"
 
@@ -28,5 +30,6 @@ int pthread_orphan_np(void) {
   res = _pthread_list == _pthread_list->prev &&
         _pthread_list == _pthread_list->next;
   _pthread_unlock();
+  unassert(!res || atomic_load(&_pthread_count) <= 1);
   return res;
 }

libc/intrin/pthreadlock.c

@@ -16,9 +16,10 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/stdalign.h"
 #include "libc/thread/posixthread.internal.h"
 
-pthread_mutex_t __pthread_lock_obj = PTHREAD_MUTEX_INITIALIZER;
+alignas(64) pthread_mutex_t __pthread_lock_obj = PTHREAD_MUTEX_INITIALIZER;
 
 void _pthread_lock(void) {
   pthread_mutex_lock(&__pthread_lock_obj);

libc/intrin/sig.c

@@ -696,35 +696,40 @@ textwindows dontinstrument static uint32_t __sig_worker(void *arg) {
     }
 
     // unblock stalled asynchronous signals in threads
-    _pthread_lock();
-    for (struct Dll *e = dll_first(_pthread_list); e;
-         e = dll_next(_pthread_list, e)) {
-      struct PosixThread *pt = POSIXTHREAD_CONTAINER(e);
-      if (atomic_load_explicit(&pt->pt_status, memory_order_acquire) >=
-          kPosixThreadTerminated) {
-        break;
+    struct PosixThread *mark;
+    for (;;) {
+      sigset_t pending, mask;
+      mark = 0;
+      _pthread_lock();
+      for (struct Dll *e = dll_first(_pthread_list); e;
+           e = dll_next(_pthread_list, e)) {
+        struct PosixThread *pt = POSIXTHREAD_CONTAINER(e);
+        if (atomic_load_explicit(&pt->pt_status, memory_order_acquire) >=
+            kPosixThreadTerminated)
+          break;
+        pending = atomic_load_explicit(&pt->tib->tib_sigpending,
+                                       memory_order_acquire);
+        mask =
+            atomic_load_explicit(&pt->tib->tib_sigmask, memory_order_acquire);
+        if (pending & ~mask) {
+          _pthread_ref(pt);
+          mark = pt;
+          break;
+        }
       }
-      sigset_t pending =
-          atomic_load_explicit(&pt->tib->tib_sigpending, memory_order_acquire);
-      sigset_t mask =
-          atomic_load_explicit(&pt->tib->tib_sigmask, memory_order_acquire);
-      if (pending & ~mask) {
-        _pthread_ref(pt);
-        _pthread_unlock();
-        while (!atomic_compare_exchange_weak_explicit(
-            &pt->tib->tib_sigpending, &pending, pending & ~mask,
-            memory_order_acq_rel, memory_order_relaxed)) {
-        }
-        while ((pending = pending & ~mask)) {
-          int sig = bsfl(pending) + 1;
-          pending &= ~(1ull << (sig - 1));
-          __sig_killer(pt, sig, SI_KERNEL);
-        }
-        _pthread_lock();
-        _pthread_unref(pt);
+      _pthread_unlock();
+      if (!mark)
+        break;
+      while (!atomic_compare_exchange_weak_explicit(
+          &mark->tib->tib_sigpending, &pending, pending & ~mask,
+          memory_order_acq_rel, memory_order_relaxed)) {
+      }
+      while ((pending = pending & ~mask)) {
+        int sig = bsfl(pending) + 1;
+        pending &= ~(1ull << (sig - 1));
+        __sig_killer(mark, sig, SI_KERNEL);
       }
     }
-    _pthread_unlock();
 
     // wait until next scheduler quantum
     pthread_mutex_unlock(&__sig_worker_lock);

libc/intrin/stack.c

@@ -0,0 +1,354 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2024 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/intrin/stack.h"
+#include "libc/assert.h"
+#include "libc/atomic.h"
+#include "libc/calls/calls.h"
+#include "libc/calls/syscall-sysv.internal.h"
+#include "libc/cosmo.h"
+#include "libc/dce.h"
+#include "libc/errno.h"
+#include "libc/intrin/dll.h"
+#include "libc/runtime/runtime.h"
+#include "libc/sysv/consts/map.h"
+#include "libc/sysv/consts/prot.h"
+#include "libc/thread/thread.h"
+
+/**
+ * @fileoverview cosmo stack memory manager
+ */
+
+#define MAP_ANON_OPENBSD  0x1000
+#define MAP_STACK_OPENBSD 0x4000
+
+#define THREADSTACK_CONTAINER(e) DLL_CONTAINER(struct CosmoStack, elem, e)
+
+struct CosmoStack {
+  struct Dll elem;
+  void *stackaddr;
+  size_t stacksize;
+  size_t guardsize;
+};
+
+struct CosmoStacks {
+  atomic_uint once;
+  pthread_mutex_t lock;
+  struct Dll *stacks;
+  struct Dll *objects;
+  unsigned count;
+};
+
+struct CosmoStacksConfig {
+  unsigned maxstacks;
+};
+
+static struct CosmoStacks cosmo_stacks = {
+    .lock = PTHREAD_MUTEX_INITIALIZER,
+};
+
+static struct CosmoStacksConfig cosmo_stacks_config = {
+    .maxstacks = 16,
+};
+
+void cosmo_stack_lock(void) {
+  pthread_mutex_lock(&cosmo_stacks.lock);
+}
+
+void cosmo_stack_unlock(void) {
+  pthread_mutex_unlock(&cosmo_stacks.lock);
+}
+
+void cosmo_stack_wipe(void) {
+  pthread_mutex_wipe_np(&cosmo_stacks.lock);
+}
+
+static errno_t cosmo_stack_munmap(void *addr, size_t size) {
+  errno_t r = 0;
+  errno_t e = errno;
+  if (!munmap(addr, size)) {
+    r = errno;
+    errno = e;
+  }
+  return r;
+}
+
+static void cosmo_stack_populate(void) {
+  errno_t e = errno;
+  void *map = mmap(0, __pagesize, PROT_READ | PROT_WRITE,
+                   MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  errno = e;
+  if (map != MAP_FAILED) {
+    struct CosmoStack *ts = map;
+    int n = __pagesize / sizeof(struct CosmoStack);
+    for (int i = 0; i < n; ++i) {
+      dll_init(&ts[i].elem);
+      dll_make_first(&cosmo_stacks.objects, &ts[i].elem);
+    }
+  }
+}
+
+static struct Dll *cosmo_stack_decimate(unsigned maxstacks) {
+  struct Dll *surplus = 0;
+  while (cosmo_stacks.count > maxstacks) {
+    struct Dll *e = dll_last(cosmo_stacks.stacks);
+    dll_remove(&cosmo_stacks.stacks, e);
+    dll_make_first(&surplus, e);
+    --cosmo_stacks.count;
+  }
+  return surplus;
+}
+
+static void cosmo_stack_rehabilitate(struct Dll *stacks) {
+  struct Dll *e;
+  for (e = dll_first(stacks); e; e = dll_next(stacks, e))
+    cosmo_stack_munmap(THREADSTACK_CONTAINER(e)->stackaddr,
+                       THREADSTACK_CONTAINER(e)->stacksize);
+  cosmo_stack_lock();
+  dll_make_first(&cosmo_stacks.objects, stacks);
+  cosmo_stack_unlock();
+}
+
+/**
+ * Empties unused stack cache.
+ *
+ * To make POSIX threads as cheap as possible to spawn, we recycle their
+ * stacks without zeroing their memory. On Linux for an 80kb stack size,
+ * that makes launching a thread take 40µs rather than 80µs. However the
+ * stack cache needs to be cleared in certain cases. This is called upon
+ * exit() automatically but anyone can clear this at any other time too.
+ *
+ * @see pthread_decimate_np()
+ */
+void cosmo_stack_clear(void) {
+  cosmo_stack_lock();
+  struct Dll *stacks = cosmo_stacks.stacks;
+  cosmo_stacks.stacks = 0;
+  cosmo_stacks.count = 0;
+  cosmo_stack_unlock();
+  cosmo_stack_rehabilitate(stacks);
+}
+
+/**
+ * Gets maximum number of unused stacks cosmo should cache.
+ * @see cosmo_stack_setmaxstacks()
+ */
+int cosmo_stack_getmaxstacks(void) {
+  return cosmo_stacks_config.maxstacks;
+}
+
+/**
+ * Sets maximum number of unused stacks cosmo should cache.
+ *
+ * This lets you place some limitations on how much stack memory the
+ * cosmo runtime will cache. This number is a count of stacks rather
+ * than the number of bytes they contain. Old stacks are freed in a
+ * least recently used fashion once the cache exceeds this limit.
+ *
+ * If this is set to zero, then the cosmo stack allocator enters a
+ * highly secure hardening mode where cosmo_stack_alloc() zeroes all
+ * stack memory that's returned.
+ *
+ * Setting this to a negative number makes the cache size unlimited.
+ *
+ * Please note this limit only applies to stacks that aren't in use.
+ *
+ * Your default is sixteen stacks may be cached at any given moment.
+ *
+ * If `maxstacks` is less than the current cache size, then surplus
+ * entries will be evicted and freed before this function returns.
+ */
+void cosmo_stack_setmaxstacks(int maxstacks) {
+  cosmo_stack_lock();
+  cosmo_stacks_config.maxstacks = maxstacks;
+  struct Dll *stacks = cosmo_stack_decimate(maxstacks);
+  cosmo_stack_unlock();
+  cosmo_stack_rehabilitate(stacks);
+}
+
+/**
+ * Allocates stack memory.
+ *
+ * This is a caching stack allocator that's used by the POSIX threads
+ * runtime but you may also find it useful for setcontext() coroutines
+ * or sigaltstack(). Normally you can get away with using malloc() for
+ * creating stacks. However some OSes (e.g. OpenBSD) forbid you from
+ * doing that for anything except sigaltstack(). This API serves to
+ * abstract all the gory details of gaining authorized memory, and
+ * additionally implements caching for lightning fast performance.
+ *
+ * The stack size must be nonzero. It is rounded up to the granularity
+ * of the underlying system allocator, which is normally the page size.
+ * Your parameter will be updated with the selected value upon success.
+ *
+ * The guard size specifies how much memory should be protected at the
+ * bottom of your stack. This is helpful for ensuring stack overflows
+ * will result in a segmentation fault, rather than corrupting memory
+ * silently. This may be set to zero, in which case no guard pages will
+ * be protected. This value is rounded up to the system page size. The
+ * corrected value will be returned upon success. Your guard size needs
+ * to be small enough to leave room for at least one memory page in your
+ * stack size i.e. `guardsize + pagesize <= stacksize` must be the case.
+ * Otherwise this function will return an `EINVAL` error.
+ *
+ * When you're done using your stack, pass it to cosmo_stack_free() so
+ * it can be recycled. Stacks are only recycled when the `stacksize` and
+ * `guardsize` parameters are an exact match after correction. Otherwise
+ * they'll likely be freed eventually, in a least-recently used fashion,
+ * based upon the configurable cosmo_stack_setmaxstacks() setting.
+ *
+ * This function returns 0 on success, or an errno on error. See the
+ * documentation of mmap() for a list possible errors that may occur.
+ */
+errno_t cosmo_stack_alloc(size_t *inout_stacksize,  //
+                          size_t *inout_guardsize,  //
+                          void **out_addr) {
+
+  // validate arguments
+  size_t stacksize = *inout_stacksize;
+  size_t guardsize = *inout_guardsize;
+  stacksize = (stacksize + __gransize - 1) & -__gransize;
+  guardsize = (guardsize + __pagesize - 1) & -__pagesize;
+  if (guardsize + __pagesize > stacksize)
+    return EINVAL;
+
+  // recycle stack
+  void *stackaddr = 0;
+  cosmo_stack_lock();
+  for (struct Dll *e = dll_first(cosmo_stacks.stacks); e;
+       e = dll_next(cosmo_stacks.stacks, e)) {
+    struct CosmoStack *ts = THREADSTACK_CONTAINER(e);
+    if (ts->stacksize == stacksize &&  //
+        ts->guardsize == guardsize) {
+      dll_remove(&cosmo_stacks.stacks, e);
+      stackaddr = ts->stackaddr;
+      dll_make_first(&cosmo_stacks.objects, e);
+      --cosmo_stacks.count;
+      break;
+    }
+  }
+  cosmo_stack_unlock();
+
+  // create stack
+  if (!stackaddr) {
+    errno_t e = errno;
+    stackaddr = mmap(0, stacksize, PROT_READ | PROT_WRITE,
+                     MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    if (stackaddr == MAP_FAILED) {
+      errno_t err = errno;
+      errno = e;
+      return err;
+    }
+    if (IsOpenbsd())
+      if (!TellOpenbsdThisIsStackMemory(stackaddr, stacksize))
+        notpossible;
+    if (guardsize)
+      if (mprotect(stackaddr, guardsize, PROT_NONE | PROT_GUARD))
+        notpossible;
+  }
+
+  // return stack
+  *inout_stacksize = stacksize;
+  *inout_guardsize = guardsize;
+  *out_addr = stackaddr;
+  return 0;
+}
+
+static void cosmo_stack_setup(void) {
+  atexit(cosmo_stack_clear);
+}
+
+/**
+ * Frees stack memory.
+ *
+ * While not strictly required, it's assumed these three values would be
+ * those returned by an earlier call to cosmo_stack_alloc().
+ *
+ * This function returns 0 on success, or an errno on error. The `errno`
+ * variable is never clobbered. You can only dependably count on this to
+ * return an error on failure when you say `cosmo_stack_setmaxstacks(0)`
+ */
+errno_t cosmo_stack_free(void *stackaddr, size_t stacksize, size_t guardsize) {
+  stacksize = (stacksize + __gransize - 1) & -__gransize;
+  guardsize = (guardsize + __pagesize - 1) & -__pagesize;
+  if (guardsize + __pagesize > stacksize)
+    return EINVAL;
+  if ((uintptr_t)stackaddr & (__gransize - 1))
+    return EINVAL;
+  cosmo_once(&cosmo_stacks.once, cosmo_stack_setup);
+  cosmo_stack_lock();
+  struct Dll *surplus = 0;
+  if (cosmo_stacks_config.maxstacks) {
+    surplus = cosmo_stack_decimate(cosmo_stacks_config.maxstacks - 1);
+    struct CosmoStack *ts = 0;
+    if (dll_is_empty(cosmo_stacks.objects))
+      cosmo_stack_populate();
+    struct Dll *e;
+    if ((e = dll_first(cosmo_stacks.objects))) {
+      dll_remove(&cosmo_stacks.objects, e);
+      ts = THREADSTACK_CONTAINER(e);
+    }
+    if (ts) {
+      ts->stackaddr = stackaddr;
+      ts->stacksize = stacksize;
+      ts->guardsize = guardsize;
+      dll_make_first(&cosmo_stacks.stacks, &ts->elem);
+      ++cosmo_stacks.count;
+      stackaddr = 0;
+    }
+  }
+  cosmo_stack_unlock();
+  cosmo_stack_rehabilitate(surplus);
+  errno_t err = 0;
+  if (stackaddr)
+    err = cosmo_stack_munmap(stackaddr, stacksize);
+  return err;
+}
+
+relegated bool TellOpenbsdThisIsStackMemory(void *addr, size_t size) {
+  return __sys_mmap(
+             addr, size, PROT_READ | PROT_WRITE,
+             MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD, -1,
+             0, 0) == addr;
+}
+
+// OpenBSD only permits RSP to occupy memory that's been explicitly
+// defined as stack memory, i.e. `lo <= %rsp < hi` must be the case
+relegated errno_t FixupCustomStackOnOpenbsd(pthread_attr_t *attr) {
+
+  // get interval
+  uintptr_t lo = (uintptr_t)attr->__stackaddr;
+  uintptr_t hi = lo + attr->__stacksize;
+
+  // squeeze interval
+  lo = (lo + __pagesize - 1) & -__pagesize;
+  hi = hi & -__pagesize;
+
+  // tell os it's stack memory
+  errno_t olderr = errno;
+  if (!TellOpenbsdThisIsStackMemory((void *)lo, hi - lo)) {
+    errno_t err = errno;
+    errno = olderr;
+    return err;
+  }
+
+  // update attributes with usable stack address
+  attr->__stackaddr = (void *)lo;
+  attr->__stacksize = hi - lo;
+  return 0;
+}

libc/intrin/stack.h

@@ -0,0 +1,14 @@
+#ifndef COSMOPOLITAN_LIBC_STACK_H_
+#define COSMOPOLITAN_LIBC_STACK_H_
+#include "libc/thread/thread.h"
+COSMOPOLITAN_C_START_
+
+void cosmo_stack_lock(void);
+void cosmo_stack_unlock(void);
+void cosmo_stack_wipe(void);
+
+bool TellOpenbsdThisIsStackMemory(void *, size_t);
+errno_t FixupCustomStackOnOpenbsd(pthread_attr_t *);
+
+COSMOPOLITAN_C_END_
+#endif /* COSMOPOLITAN_LIBC_STACK_H_ */

libc/intrin/ulock.c

@@ -17,12 +17,12 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/intrin/ulock.h"
-#include "libc/assert.h"
-#include "libc/calls/calls.h"
 #include "libc/calls/syscall_support-sysv.internal.h"
-#include "libc/dce.h"
+#include "libc/errno.h"
 #include "libc/intrin/describeflags.h"
+#include "libc/intrin/kprintf.h"
 #include "libc/intrin/strace.h"
+#include "libc/intrin/ulock.h"
 
 // XNU futexes
 // https://opensource.apple.com/source/xnu/xnu-7195.50.7.100.1/bsd/sys/ulock.h.auto.html
@@ -32,6 +32,26 @@ int sys_ulock_wait(uint32_t operation, void *addr, uint64_t value,
                    uint32_t timeout_micros) asm("sys_futex_cp");
 
 // returns number of other waiters, or -1 w/ errno
+//
+// - EINTR means a signal handler was called. This is how we support
+//   things like POSIX thread cancelation.
+//
+// - EFAULT if XNU couldn't read `addr`. This is normally considered a
+//   programming error, but with ulock it can actually be a transient
+//   error due to low memory conditions. Apple recommends retrying.
+//
+// - ENOMEM means XNU wasn't able to allocate memory for kernel internal
+//   data structures. Apple doesn't provide any advice on what to do. We
+//   simply turn this into EAGAIN.
+//
+// - EAGAIN if XNU told us EFAULT but cosmo believes the address exists.
+//   This value is also used as a substitute for ENOMEM.
+//
+// - EINVAL could mean operation is invalid, addr is null or misaligned;
+//   it could also mean another thread calling ulock on this address was
+//   configured (via operation) in an inconsistent way.
+//
+// see also os_sync_wait_on_address.h from xcode sdk
 int ulock_wait(uint32_t operation, void *addr, uint64_t value,
                uint32_t timeout_micros) {
   int rc;
@@ -39,12 +59,26 @@ int ulock_wait(uint32_t operation, void *addr, uint64_t value,
   LOCKTRACE("ulock_wait(%#x, %p, %lx, %u) → ...", operation, addr, value,
             timeout_micros);
   rc = sys_ulock_wait(operation, addr, value, timeout_micros);
+  if (rc == -1) {
+    if (errno == ENOMEM)
+      errno = EAGAIN;
+    if (errno == EFAULT)
+      if (!kisdangerous(addr))
+        errno = EAGAIN;
+  }
   LOCKTRACE("ulock_wait(%#x, %p, %lx, %u) → %d% m", operation, addr, value,
             timeout_micros, rc);
   return rc;
 }
 
 // returns -errno
+//
+// - ENOENT means there wasn't anyone to wake
+//
+// - EINVAL could mean operation is invalid, addr is null or misaligned;
+//   it could also mean another thread calling ulock on this address was
+//   configured (via operation) in an inconsistent way.
+//
 int ulock_wake(uint32_t operation, void *addr, uint64_t wake_value) {
   int rc;
   rc = __syscall3i(operation, (long)addr, wake_value, 0x2000000 | 516);