Make detached threads work better

This change adds a double linked list of threads, so that pthread_exit()
will know when it should call exit() from an orphaned child. This change
also improves ftrace and strace logging.

libc/intrin/asan.c

@@ -953,7 +953,7 @@ static void *__asan_morgue_add(void *p) {
       p, memory_order_acq_rel);
 }
 
-static void __asan_morgue_flush(void) {
+__attribute__((__destructor__)) static void __asan_morgue_flush(void) {
   unsigned i;
   for (i = 0; i < ARRAYLEN(__asan_morgue.p); ++i) {
     if (atomic_load_explicit(__asan_morgue.p + i, memory_order_acquire)) {
@@ -1461,8 +1461,8 @@ static textstartup void __asan_shadow_existing_mappings(void) {
   __asan_poison((void *)GetStackAddr(), PAGESIZE, kAsanStackOverflow);
 }
 
-textstartup void __asan_init(int argc, char **argv, char **envp,
+__attribute__((__constructor__)) void __asan_init(int argc, char **argv,
-                             intptr_t *auxv) {
+                                                  char **envp, intptr_t *auxv) {
   static bool once;
   if (!_cmpxchg(&once, false, true)) return;
   if (IsWindows() && NtGetVersion() < kNtVersionWindows10) {
@@ -1497,13 +1497,3 @@ textstartup void __asan_init(int argc, char **argv, char **envp,
   STRACE("/_/   \\_\\____/_/   \\_\\_| \\_|");
   STRACE("cosmopolitan memory safety module initialized");
 }
-
-static textstartup void __asan_ctor(void) {
-  if (_weaken(__cxa_atexit)) {
-    _weaken(__cxa_atexit)(__asan_morgue_flush, NULL, NULL);
-  }
-}
-
-const void *const g_asan_ctor[] initarray = {
-    __asan_ctor,
-};

libc/intrin/exit1.greg.c

@@ -18,8 +18,6 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/dce.h"
 #include "libc/intrin/asmflag.h"
-#include "libc/intrin/promises.internal.h"
-#include "libc/intrin/strace.internal.h"
 #include "libc/nt/thread.h"
 #include "libc/runtime/runtime.h"
 #include "libc/sysv/consts/nr.h"
@@ -47,7 +45,6 @@ __msabi extern typeof(ExitThread) *const __imp_ExitThread;
 privileged wontreturn void _Exit1(int rc) {
   char cf;
   int ax, dx, di, si;
-  STRACE("_Exit1(%d)", rc);
   if (!IsWindows() && !IsMetal()) {
     // exit() on Linux
     // thr_exit() on FreeBSD

libc/intrin/g_fds.c

@@ -18,6 +18,7 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/calls/internal.h"
 #include "libc/calls/state.internal.h"
+#include "libc/intrin/atomic.h"
 #include "libc/intrin/extend.internal.h"
 #include "libc/intrin/pushpop.h"
 #include "libc/intrin/weaken.h"
@@ -38,7 +39,7 @@ static textwindows dontinline void SetupWinStd(struct Fds *fds, int i, int x) {
   if (!h || h == -1) return;
   fds->p[i].kind = pushpop(kFdFile);
   fds->p[i].handle = h;
-  fds->f = i + 1;
+  atomic_store_explicit(&fds->f, i + 1, memory_order_relaxed);
 }
 
 textstartup void InitializeFileDescriptors(void) {
@@ -49,7 +50,7 @@ textstartup void InitializeFileDescriptors(void) {
   fds = VEIL("r", &g_fds);
   fds->p = fds->e = (void *)kMemtrackFdsStart;
   fds->n = 4;
-  fds->f = 3;
+  atomic_store_explicit(&fds->f, 3, memory_order_relaxed);
   fds->e = _extend(fds->p, fds->n * sizeof(*fds->p), fds->e, MAP_PRIVATE,
                    kMemtrackFdsStart + kMemtrackFdsSize);
   if (IsMetal()) {

libc/intrin/pthread_getspecific.c

@@ -16,16 +16,26 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/assert.h"
+#include "libc/intrin/atomic.h"
+#include "libc/thread/posixthread.internal.h"
 #include "libc/thread/thread.h"
 #include "libc/thread/tls.h"
 
 /**
  * Gets value of TLS slot for current thread.
+ *
+ * If `k` wasn't created by pthread_key_create() then the behavior is
+ * undefined. If `k` was unregistered earlier by pthread_key_delete()
+ * then the behavior is undefined.
  */
-void *pthread_getspecific(pthread_key_t key) {
+void *pthread_getspecific(pthread_key_t k) {
-  if (0 <= key && key < PTHREAD_KEYS_MAX) {
+  // "The effect of calling pthread_getspecific() or
-    return __get_tls()->tib_keys[key];
+  //  pthread_setspecific() with a key value not obtained from
-  } else {
+  //  pthread_key_create() or after key has been deleted with
-    return 0;
+  //  pthread_key_delete() is undefined."
-  }
+  //                                  ──Quoth POSIX.1-2017
+  _unassert(0 <= k && k < PTHREAD_KEYS_MAX);
+  _unassert(atomic_load_explicit(_pthread_key_dtor + k, memory_order_acquire));
+  return __get_tls()->tib_keys[k];
 }

libc/intrin/pthread_key_create.c

@@ -18,18 +18,24 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/errno.h"
 #include "libc/intrin/atomic.h"
-#include "libc/runtime/runtime.h"
 #include "libc/thread/posixthread.internal.h"
 #include "libc/thread/thread.h"
 
 /**
  * Allocates TLS slot.
  *
- * If `dtor` is non-null, then it'll be called upon thread exit when the
+ * This function creates a thread-local storage registration, that will
- * key's value is nonzero. The key's value is set to zero before it gets
+ * apply to all threads. The new identifier is written to `key`, and it
- * called. The ordering for multiple destructor calls is unspecified.
+ * can be passed to the pthread_setspecific() and pthread_getspecific()
+ * functions to set and get its associated value. Each thread will have
+ * its key value initialized to zero upon creation. It is also possible
+ * to use pthread_key_delete() to unregister a key.
  *
- * The result should be passed to pthread_key_delete() later.
+ * If `dtor` is non-null, then it'll be called upon pthread_exit() when
+ * the key's value is nonzero. The key's value is set to zero before it
+ * is called. The ordering of multiple destructor calls is unspecified.
+ * The same key can be destroyed `PTHREAD_DESTRUCTOR_ITERATIONS` times,
+ * in cases where it gets set again by a destructor.
  *
  * @param key is set to the allocated key on success
  * @param dtor specifies an optional destructor callback
@@ -42,9 +48,9 @@ int pthread_key_create(pthread_key_t *key, pthread_key_dtor dtor) {
   if (!dtor) dtor = (pthread_key_dtor)-1;
   for (i = 0; i < PTHREAD_KEYS_MAX; ++i) {
     if (!(expect = atomic_load_explicit(_pthread_key_dtor + i,
-                                        memory_order_relaxed)) &&
+                                        memory_order_acquire)) &&
         atomic_compare_exchange_strong_explicit(_pthread_key_dtor + i, &expect,
-                                                dtor, memory_order_relaxed,
+                                                dtor, memory_order_release,
                                                 memory_order_relaxed)) {
       *key = i;
       return 0;
@@ -52,7 +58,3 @@ int pthread_key_create(pthread_key_t *key, pthread_key_dtor dtor) {
   }
   return EAGAIN;
 }
-
-__attribute__((__constructor__)) static textstartup void _pthread_key_init() {
-  atexit(_pthread_key_destruct);
-}

libc/intrin/pthread_key_delete.c

@@ -16,6 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/assert.h"
 #include "libc/errno.h"
 #include "libc/intrin/atomic.h"
 #include "libc/thread/posixthread.internal.h"
@@ -24,16 +25,18 @@
 /**
  * Deletes TLS slot.
  *
+ * This function should only be called if all threads have finished
+ * using the key registration. If a key is used after being deleted
+ * then the behavior is undefined. If `k` was not registered by the
+ * pthread_key_create() function then the behavior is undefined.
+ *
  * @param key was created by pthread_key_create()
  * @return 0 on success, or errno on error
- * @raise EINVAL if `key` isn't valid
  */
-int pthread_key_delete(pthread_key_t key) {
+int pthread_key_delete(pthread_key_t k) {
   uint64_t mask;
-  if (key < PTHREAD_KEYS_MAX) {
+  _unassert(0 <= k && k < PTHREAD_KEYS_MAX);
-    atomic_store_explicit(_pthread_key_dtor + key, 0, memory_order_relaxed);
+  _unassert(atomic_load_explicit(_pthread_key_dtor + k, memory_order_acquire));
-    return 0;
+  atomic_store_explicit(_pthread_key_dtor + k, 0, memory_order_release);
-  } else {
+  return 0;
-    return EINVAL;
-  }
 }

libc/intrin/pthread_setspecific.c

@@ -16,18 +16,27 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/errno.h"
+#include "libc/assert.h"
+#include "libc/intrin/atomic.h"
+#include "libc/thread/posixthread.internal.h"
 #include "libc/thread/thread.h"
 #include "libc/thread/tls.h"
 
 /**
  * Sets value of TLS slot for current thread.
+ *
+ * If `k` wasn't created by pthread_key_create() then the behavior is
+ * undefined. If `k` was unregistered earlier by pthread_key_delete()
+ * then the behavior is undefined.
  */
-int pthread_setspecific(pthread_key_t key, const void *val) {
+int pthread_setspecific(pthread_key_t k, const void *val) {
-  if (0 <= key && key < PTHREAD_KEYS_MAX) {
+  // "The effect of calling pthread_getspecific() or
-    __get_tls()->tib_keys[key] = val;
+  //  pthread_setspecific() with a key value not obtained from
-    return 0;
+  //  pthread_key_create() or after key has been deleted with
-  } else {
+  //  pthread_key_delete() is undefined."
-    return EINVAL;
+  //                                  ──Quoth POSIX.1-2017
-  }
+  _unassert(0 <= k && k < PTHREAD_KEYS_MAX);
+  _unassert(atomic_load_explicit(_pthread_key_dtor + k, memory_order_acquire));
+  __get_tls()->tib_keys[k] = val;
+  return 0;
 }

libc/intrin/pthread_spin_destroy.c

@@ -16,13 +16,19 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/intrin/atomic.h"
 #include "libc/thread/thread.h"
 
+#ifdef pthread_spin_destroy
+#undef pthread_spin_destroy
+#endif
+
 /**
  * Destroys spin lock.
  *
  * @return 0 on success, or errno on error
  */
-errno_t(pthread_spin_destroy)(pthread_spinlock_t *spin) {
+errno_t pthread_spin_destroy(pthread_spinlock_t *spin) {
-  return pthread_spin_destroy(spin);
+  atomic_store_explicit(&spin->_lock, -1, memory_order_relaxed);
+  return 0;
 }

libc/intrin/pthread_spin_init.c

@@ -16,8 +16,13 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/intrin/atomic.h"
 #include "libc/thread/thread.h"
 
+#ifdef pthread_spin_init
+#undef pthread_spin_init
+#endif
+
 /**
  * Initializes spin lock.
  *
@@ -27,6 +32,7 @@
  * @see pthread_spin_destroy
  * @see pthread_spin_lock
  */
-errno_t(pthread_spin_init)(pthread_spinlock_t *spin, int pshared) {
+errno_t pthread_spin_init(pthread_spinlock_t *spin, int pshared) {
-  return pthread_spin_init(spin, pshared);
+  atomic_store_explicit(&spin->_lock, 0, memory_order_relaxed);
+  return 0;
 }

libc/intrin/pthread_spin_lock.c

@@ -16,20 +16,16 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/assert.h"
+#include "libc/intrin/atomic.h"
 #include "libc/thread/thread.h"
 
+#ifdef pthread_spin_lock
+#undef pthread_spin_lock
+#endif
+
 /**
- * Acquires spin lock.
+ * Acquires spin lock, e.g.
- *
- * If the lock is already held, this function will wait for it to become
- * available. No genuine error conditions are currently defined. This is
- * similar to pthread_mutex_lock() except spin locks are much simpler so
- * this API is able to offer a performance advantage in situations where
- * scalable contention handling isn't necessary. Spinlocks are also very
- * small especially in MODE=tiny where a lock needs 16 bytes of code and
- * unlocking needs just 5 bytes. The lock object also only takes 1 byte.
- *
- * The posixly correct way to use this API is as follows:
  *
  *     pthread_spinlock_t lock;
  *     pthread_spin_init(&lock, PTHREAD_PROCESS_PRIVATE);
@@ -38,18 +34,20 @@
  *     pthread_spin_unlock(&lock);
  *     pthread_spin_destroy(&lock);
  *
- * Cosmopolitan permits succinct notation for spin locks:
+ * This function has undefined behavior when `spin` wasn't intialized,
- *
+ * was destroyed, or if the lock's already held by the calling thread.
- *     pthread_spinlock_t lock = {0};
- *     pthread_spin_lock(&lock);
- *     // do work...
- *     pthread_spin_unlock(&lock);
  *
  * @return 0 on success, or errno on error
  * @see pthread_spin_trylock
  * @see pthread_spin_unlock
  * @see pthread_spin_init
  */
-errno_t(pthread_spin_lock)(pthread_spinlock_t *spin) {
+errno_t pthread_spin_lock(pthread_spinlock_t *spin) {
-  return pthread_spin_lock(spin);
+  int x;
+  for (;;) {
+    x = atomic_exchange_explicit(&spin->_lock, 1, memory_order_acquire);
+    if (!x) break;
+    _unassert(x == 1);
+  }
+  return 0;
 }

libc/intrin/pthread_spin_trylock.c

@@ -16,17 +16,28 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/assert.h"
+#include "libc/errno.h"
+#include "libc/intrin/atomic.h"
 #include "libc/thread/thread.h"
 
+#ifdef pthread_spin_trylock
+#undef pthread_spin_trylock
+#endif
+
 /**
  * Acquires spin lock if available.
  *
- * Unlike pthread_spin_lock() this function won't block, and instead
+ * This function has undefined behavior when `spin` wasn't intialized,
- * returns an error immediately if the spinlock couldn't be acquired
+ * was destroyed, or if the lock's already held by the calling thread.
  *
  * @return 0 on success, or errno on error
  * @raise EBUSY if lock is already held
  */
-errno_t(pthread_spin_trylock)(pthread_spinlock_t *spin) {
+errno_t pthread_spin_trylock(pthread_spinlock_t *spin) {
-  return pthread_spin_trylock(spin);
+  int x;
+  x = atomic_exchange_explicit(&spin->_lock, 1, memory_order_acquire);
+  if (!x) return 0;
+  _unassert(x == 1);
+  return EBUSY;
 }

libc/intrin/pthread_spin_unlock.c

@@ -16,14 +16,23 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/intrin/atomic.h"
 #include "libc/thread/thread.h"
 
+#ifdef pthread_spin_unlock
+#undef pthread_spin_unlock
+#endif
+
 /**
  * Releases spin lock.
  *
+ * Calling this function when the lock isn't held by the calling thread
+ * has undefined behavior.
+ *
  * @return 0 on success, or errno on error
  * @see pthread_spin_lock
  */
-errno_t(pthread_spin_unlock)(pthread_spinlock_t *spin) {
+errno_t pthread_spin_unlock(pthread_spinlock_t *spin) {
-  return pthread_spin_unlock(spin);
+  atomic_store_explicit(&spin->_lock, 0, memory_order_release);
+  return 0;
 }

libc/log/backtrace2.c

@@ -16,6 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/blockcancel.internal.h"
 #include "libc/calls/calls.h"
 #include "libc/calls/struct/rusage.internal.h"
 #include "libc/calls/syscall-sysv.internal.h"
@@ -62,10 +63,12 @@ static int PrintBacktraceUsingAddr2line(int fd, const struct StackFrame *bp) {
   }
 
   if (!PLEDGED(STDIO) || !PLEDGED(EXEC) || !PLEDGED(EXEC)) {
+    ShowHint("won't print addr2line backtrace because pledge");
     return -1;
   }
 
   if (!(debugbin = FindDebugBinary())) {
+    ShowHint("won't print addr2line backtrace because no debug binary");
     return -1;
   }
 
@@ -171,11 +174,14 @@ static int PrintBacktrace(int fd, const struct StackFrame *bp) {
       return 0;
     }
   }
+#else
+  ShowHint("won't print addr2line backtrace because no dwarf");
 #endif
   return PrintBacktraceUsingSymbols(fd, bp, GetSymbolTable());
 }
 
 void ShowBacktrace(int fd, const struct StackFrame *bp) {
+  BLOCK_CANCELLATIONS;
 #ifdef __FNO_OMIT_FRAME_POINTER__
   /* asan runtime depends on this function */
   ftrace_enabled(-1);
@@ -189,4 +195,5 @@ void ShowBacktrace(int fd, const struct StackFrame *bp) {
                     "\t-D__FNO_OMIT_FRAME_POINTER__\n"
                     "\t-fno-omit-frame-pointer\n");
 #endif
+  ALLOW_CANCELLATIONS;
 }

libc/runtime/clone.c

@@ -108,7 +108,7 @@ WinThreadEntry(int rdi,                                 // rcx
   *wt->ztid = 0;
   __imp_WakeByAddressAll(wt->ztid);
   // since we didn't indirect this function through NT2SYSV() it's not
-  // safe to simply return, and as such, we just call ExitThread().
+  // safe to simply return, and as such, we need ExitThread().
   __imp_ExitThread(rc);
   notpossible;
 }

libc/runtime/enable_tls.c

@@ -28,15 +28,19 @@
 #include "libc/runtime/internal.h"
 #include "libc/runtime/runtime.h"
 #include "libc/thread/posixthread.internal.h"
+#include "libc/thread/thread.h"
 #include "libc/thread/tls.h"
 
 #define _TLSZ ((intptr_t)_tls_size)
 #define _TLDZ ((intptr_t)_tdata_size)
 #define _TIBZ sizeof(struct CosmoTib)
 
-struct PosixThread _pthread_main;
 extern unsigned char __tls_mov_nt_rax[];
 extern unsigned char __tls_add_nt_rax[];
+
+nsync_dll_list_ _pthread_list;
+pthread_spinlock_t _pthread_lock;
+static struct PosixThread _pthread_main;
 _Alignas(TLS_ALIGNMENT) static char __static_tls[5008];
 
 /**
@@ -79,6 +83,7 @@ void __enable_tls(void) {
   size_t siz;
   struct CosmoTib *tib;
   char *mem, *tls;
+
   siz = ROUNDUP(_TLSZ + _TIBZ, _Alignof(__static_tls));
   if (siz <= sizeof(__static_tls)) {
     // if tls requirement is small then use the static tls block
@@ -95,12 +100,15 @@ void __enable_tls(void) {
     mem = _weaken(_mapanon)(siz);
     _npassert(mem);
   }
+
   if (IsAsan()) {
     // poison the space between .tdata and .tbss
     __asan_poison(mem + (intptr_t)_tdata_size,
                   (intptr_t)_tbss_offset - (intptr_t)_tdata_size,
                   kAsanProtected);
   }
+
+  // initialize main thread tls memory
   tib = (struct CosmoTib *)(mem + siz - _TIBZ);
   tls = mem + siz - _TIBZ - _TLSZ;
   tib->tib_self = tib;
@@ -117,9 +125,16 @@ void __enable_tls(void) {
     tid = sys_gettid();
   }
   atomic_store_explicit(&tib->tib_tid, tid, memory_order_relaxed);
-  _pthread_main.ptid = tid;
+
+  // initialize posix threads
   _pthread_main.tib = tib;
-  _pthread_main.flags = PT_MAINTHREAD;
+  _pthread_main.flags = PT_STATIC;
+  _pthread_main.list.prev = _pthread_main.list.next =  //
+      _pthread_list = VEIL("r", &_pthread_main.list);
+  _pthread_main.list.container = &_pthread_main;
+  atomic_store_explicit(&_pthread_main.ptid, tid, memory_order_relaxed);
+
+  // copy in initialized data section
   __repmovsb(tls, _tdata_start, _TLDZ);
 
   // ask the operating system to change the x86 segment register

libc/runtime/fork-nt.c

@@ -24,6 +24,7 @@
 #include "libc/calls/wincrash.internal.h"
 #include "libc/errno.h"
 #include "libc/fmt/itoa.h"
+#include "libc/intrin/atomic.h"
 #include "libc/intrin/directmap.internal.h"
 #include "libc/intrin/kprintf.h"
 #include "libc/intrin/strace.internal.h"
@@ -237,7 +238,7 @@ textwindows void WinMainForked(void) {
   for (i = 0; i < fds->n; ++i) {
     if (fds->p[i].kind == kFdProcess) {
       fds->p[i].kind = 0;
-      fds->f = MIN(i, fds->f);
+      atomic_store_explicit(&fds->f, MIN(i, fds->f), memory_order_relaxed);
     }
   }
 

libc/testlib/testrunner.c

@@ -218,6 +218,8 @@ void testlib_runtestcases(testfn_t *start, testfn_t *end, testfn_t warmup) {
   }
   if (_weaken(SetUpOnce)) _weaken(SetUpOnce)();
   for (x = 0, fn = start; fn != end; ++fn) {
+    STRACE("");
+    STRACE("# setting up %t", fn);
     if (_weaken(testlib_enable_tmp_setup_teardown)) SetupTmpDir();
     if (_weaken(SetUp)) _weaken(SetUp)();
     errno = 0;
@@ -229,6 +231,7 @@ void testlib_runtestcases(testfn_t *start, testfn_t *end, testfn_t warmup) {
     STRACE("# running test %t", fn);
     (*fn)();
     STRACE("");
+    STRACE("# tearing down %t", fn);
     if (!IsWindows()) sys_getpid();
     if (_weaken(TearDown)) _weaken(TearDown)();
     if (_weaken(testlib_enable_tmp_setup_teardown)) TearDownTmpDir();

libc/thread/posixthread.internal.h

@@ -5,14 +5,16 @@
 #include "libc/runtime/runtime.h"
 #include "libc/thread/thread.h"
 #include "libc/thread/tls.h"
+#include "third_party/nsync/dll.h"
 
 #define PT_OWNSTACK       1
-#define PT_MAINTHREAD     2
+#define PT_STATIC         2
 #define PT_ASYNC          4
 #define PT_NOCANCEL       8
 #define PT_MASKED         16
 #define PT_INCANCEL       32
 #define PT_OPENBSD_KLUDGE 64
+#define PT_EXITING        128
 
 #if !(__ASSEMBLER__ + __LINKER__ + 0)
 COSMOPOLITAN_C_START_
@@ -66,14 +68,15 @@ struct PosixThread {
   int flags;               // 0x00: see PT_* constants
   _Atomic(int) cancelled;  // 0x04: thread has bad beliefs
   _Atomic(enum PosixThreadStatus) status;
-  _Atomic(int) ptid;       // transitions 0 → tid
+  _Atomic(int) ptid;        // transitions 0 → tid
-  void *(*start)(void *);  // creation callback
+  void *(*start)(void *);   // creation callback
-  void *arg;               // start's parameter
+  void *arg;                // start's parameter
-  void *rc;                // start's return value
+  void *rc;                 // start's return value
-  char *altstack;          // thread sigaltstack
+  char *altstack;           // thread sigaltstack
-  char *tls;               // bottom of tls allocation
+  char *tls;                // bottom of tls allocation
-  struct CosmoTib *tib;    // middle of tls allocation
+  struct CosmoTib *tib;     // middle of tls allocation
-  jmp_buf exiter;          // for pthread_exit
+  nsync_dll_element_ list;  // list of threads
+  jmp_buf exiter;           // for pthread_exit
   pthread_attr_t attr;
   sigset_t sigmask;
   struct _pthread_cleanup_buffer *cleanup;
@@ -81,24 +84,19 @@ struct PosixThread {
 
 typedef void (*atfork_f)(void);
 
-extern struct PosixThread _pthread_main;
+extern nsync_dll_list_ _pthread_list;
+extern pthread_spinlock_t _pthread_lock;
 extern _Atomic(pthread_key_dtor) _pthread_key_dtor[PTHREAD_KEYS_MAX] _Hide;
 
 int _pthread_atfork(atfork_f, atfork_f, atfork_f) _Hide;
 int _pthread_reschedule(struct PosixThread *) _Hide;
 int _pthread_setschedparam_freebsd(int, int, const struct sched_param *) _Hide;
-int _pthread_wait(struct PosixThread *) _Hide;
+void _pthread_zombify(struct PosixThread *) _Hide;
 void _pthread_free(struct PosixThread *) _Hide;
-void _pthread_cleanup(struct PosixThread *) _Hide;
-void _pthread_ungarbage(void) _Hide;
-void _pthread_zombies_add(struct PosixThread *) _Hide;
-void _pthread_zombies_purge(void) _Hide;
-void _pthread_zombies_decimate(void) _Hide;
-void _pthread_zombies_harvest(void) _Hide;
-void _pthread_key_destruct(void) _Hide;
 void _pthread_onfork_prepare(void) _Hide;
 void _pthread_onfork_parent(void) _Hide;
 void _pthread_onfork_child(void) _Hide;
+void _pthread_ungarbage(void) _Hide;
 int _pthread_cancel_sys(void) _Hide;
 
 COSMOPOLITAN_C_END_

libc/thread/pthread_atfork.c

@@ -20,6 +20,7 @@
 #include "libc/errno.h"
 #include "libc/intrin/atomic.h"
 #include "libc/intrin/kmalloc.h"
+#include "libc/mem/mem.h"
 #include "libc/runtime/memtrack.internal.h"
 #include "libc/str/str.h"
 #include "libc/thread/posixthread.internal.h"
@@ -34,6 +35,14 @@ static struct AtForks {
   } * list;
 } _atforks;
 
+static void _pthread_purge(void) {
+  nsync_dll_element_ *e;
+  while ((e = nsync_dll_first_(_pthread_list))) {
+    _pthread_list = nsync_dll_remove_(_pthread_list, e);
+    _pthread_free(e->container);
+  }
+}
+
 static void _pthread_onfork(int i) {
   struct AtFork *a;
   struct PosixThread *pt;
@@ -44,17 +53,11 @@ static void _pthread_onfork(int i) {
     _atforks.list = a;
   }
   if (i) pthread_spin_unlock(&_atforks.lock);
-  if (i == 2) {
-    _pthread_zombies_purge();
-    if (__tls_enabled) {
-      pt = (struct PosixThread *)__get_tls()->tib_pthread;
-      pt->flags |= PT_MAINTHREAD;
-    }
-  }
 }
 
 void _pthread_onfork_prepare(void) {
   _pthread_onfork(0);
+  pthread_spin_lock(&_pthread_lock);
   __kmalloc_lock();
   __mmi_lock();
 }
@@ -62,6 +65,7 @@ void _pthread_onfork_prepare(void) {
 void _pthread_onfork_parent(void) {
   __mmi_unlock();
   __kmalloc_unlock();
+  pthread_spin_unlock(&_pthread_lock);
   _pthread_onfork(1);
 }
 
@@ -72,12 +76,26 @@ void _pthread_onfork_child(void) {
   extern pthread_mutex_t __mmi_lock_obj;
   tib = __get_tls();
   pt = (struct PosixThread *)tib->tib_pthread;
+
+  // let's choose to let the new process live.
+  // even though it's unclear what to do with this kind of race.
   atomic_store_explicit(&pt->cancelled, false, memory_order_relaxed);
+
+  // wipe core runtime locks
   pthread_mutexattr_init(&attr);
   pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
   pthread_mutex_init(&__mmi_lock_obj, &attr);
+  pthread_spin_init(&_pthread_lock, 0);
   __kmalloc_unlock();
+
+  // call user-supplied forked child callbacks
   _pthread_onfork(2);
+
+  // delete other threads that existed before forking
+  // this must come after onfork, since it calls free
+  _pthread_list = nsync_dll_remove_(_pthread_list, &pt->list);
+  _pthread_purge();
+  _pthread_list = nsync_dll_make_first_in_list_(_pthread_list, &pt->list);
 }
 
 int _pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) {

libc/thread/pthread_cancel.c

@@ -307,7 +307,7 @@ errno_t pthread_cancel(pthread_t thread) {
 void pthread_testcancel(void) {
   struct PosixThread *pt;
   if (!__tls_enabled) return;
-  pt = (struct PosixThread *)__get_tls()->tib_pthread;
+  if (!(pt = (struct PosixThread *)__get_tls()->tib_pthread)) return;
   if (pt->flags & PT_NOCANCEL) return;
   if ((!(pt->flags & PT_MASKED) || (pt->flags & PT_ASYNC)) &&
       atomic_load_explicit(&pt->cancelled, memory_order_acquire)) {
@@ -335,7 +335,7 @@ errno_t pthread_testcancel_np(void) {
   int rc;
   struct PosixThread *pt;
   if (!__tls_enabled) return 0;
-  pt = (struct PosixThread *)__get_tls()->tib_pthread;
+  if (!(pt = (struct PosixThread *)__get_tls()->tib_pthread)) return 0;
   if (pt->flags & PT_NOCANCEL) return 0;
   if (!atomic_load_explicit(&pt->cancelled, memory_order_acquire)) return 0;
   if (!(pt->flags & PT_MASKED) || (pt->flags & PT_ASYNC)) {

libc/thread/pthread_create.c

@@ -39,7 +39,9 @@
 #include "libc/thread/posixthread.internal.h"
 #include "libc/thread/spawn.h"
 #include "libc/thread/thread.h"
+#include "libc/thread/tls.h"
 #include "libc/thread/wait0.internal.h"
+#include "third_party/nsync/dll.h"
 
 STATIC_YOINK("nsync_mu_lock");
 STATIC_YOINK("nsync_mu_unlock");
@@ -48,19 +50,13 @@ STATIC_YOINK("_pthread_atfork");
 #define MAP_ANON_OPENBSD  0x1000
 #define MAP_STACK_OPENBSD 0x4000
 
-errno_t _pthread_wait(struct PosixThread *pt) {
-  return _wait0(&pt->tib->tib_tid);
-}
-
 void _pthread_free(struct PosixThread *pt) {
-  if (pt->flags & PT_MAINTHREAD) return;
+  if (pt->flags & PT_STATIC) return;
   free(pt->tls);
   if ((pt->flags & PT_OWNSTACK) &&  //
       pt->attr.__stackaddr &&       //
       pt->attr.__stackaddr != MAP_FAILED) {
-    if (munmap(pt->attr.__stackaddr, pt->attr.__stacksize)) {
+    _npassert(!munmap(pt->attr.__stackaddr, pt->attr.__stacksize));
-      notpossible;
-    }
   }
   if (pt->altstack) {
     free(pt->altstack);
@@ -69,9 +65,11 @@ void _pthread_free(struct PosixThread *pt) {
 }
 
 static int PosixThread(void *arg, int tid) {
+  void *rc;
+  struct sigaltstack ss;
   struct PosixThread *pt = arg;
   enum PosixThreadStatus status;
-  struct sigaltstack ss;
+  _unassert(__get_tls()->tib_tid > 0);
   if (pt->altstack) {
     ss.ss_flags = 0;
     ss.ss_size = SIGSTKSZ;
@@ -87,12 +85,12 @@ static int PosixThread(void *arg, int tid) {
   if (!setjmp(pt->exiter)) {
     __get_tls()->tib_pthread = (pthread_t)pt;
     _sigsetmask(pt->sigmask);
-    pt->rc = pt->start(pt->arg);
+    rc = pt->start(pt->arg);
     // ensure pthread_cleanup_pop(), and pthread_exit() popped cleanup
     _npassert(!pt->cleanup);
+    // calling pthread_exit() will either jump back here, or call exit
+    pthread_exit(rc);
   }
-  // run garbage collector, call key destructors, and set change state
-  _pthread_cleanup(pt);
   // return to clone polyfill which clears tid, wakes futex, and exits
   return 0;
 }
@@ -234,13 +232,19 @@ static errno_t pthread_create_impl(pthread_t *thread,
     case PTHREAD_CREATE_DETACHED:
       atomic_store_explicit(&pt->status, kPosixThreadDetached,
                             memory_order_relaxed);
-      _pthread_zombies_add(pt);
       break;
     default:
       _pthread_free(pt);
       return EINVAL;
   }
 
+  // add thread to global list
+  // we add it to the end since zombies go at the beginning
+  nsync_dll_init_(&pt->list, pt);
+  pthread_spin_lock(&_pthread_lock);
+  _pthread_list = nsync_dll_make_last_in_list_(_pthread_list, &pt->list);
+  pthread_spin_unlock(&_pthread_lock);
+
   // launch PosixThread(pt) in new thread
   pt->sigmask = oldsigs;
   if ((rc = clone(PosixThread, pt->attr.__stackaddr,
@@ -249,6 +253,9 @@ static errno_t pthread_create_impl(pthread_t *thread,
                       CLONE_SIGHAND | CLONE_SETTLS | CLONE_PARENT_SETTID |
                       CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID,
                   pt, &pt->ptid, pt->tib, &pt->tib->tib_tid))) {
+    pthread_spin_lock(&_pthread_lock);
+    _pthread_list = nsync_dll_remove_(_pthread_list, &pt->list);
+    pthread_spin_unlock(&_pthread_lock);
     _pthread_free(pt);
     return rc;
   }
@@ -313,7 +320,7 @@ errno_t pthread_create(pthread_t *thread, const pthread_attr_t *attr,
                        void *(*start_routine)(void *), void *arg) {
   errno_t rc;
   __require_tls();
-  _pthread_zombies_decimate();
+  pthread_decimate_np();
   BLOCK_SIGNALS;
   rc = pthread_create_impl(thread, attr, start_routine, arg, _SigMask);
   ALLOW_SIGNALS;

libc/thread/pthread_decimate_np.c

@@ -17,68 +17,33 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/intrin/atomic.h"
-#include "libc/mem/mem.h"
 #include "libc/runtime/runtime.h"
 #include "libc/thread/posixthread.internal.h"
-#include "libc/thread/spawn.h"
 #include "libc/thread/thread.h"
+#include "libc/thread/tls.h"
+#include "third_party/nsync/dll.h"
 
-// TODO(jart): track all threads, not just zombies
+/**
-
+ * Releases memory of detached threads that have terminated.
-static struct Zombie {
+ */
-  struct Zombie *next;
+void pthread_decimate_np(void) {
+  nsync_dll_element_ *e;
   struct PosixThread *pt;
-} * _pthread_zombies;
+  enum PosixThreadStatus status;
-
+StartOver:
-void _pthread_zombies_add(struct PosixThread *pt) {
+  pthread_spin_lock(&_pthread_lock);
-  struct Zombie *z;
+  for (e = nsync_dll_first_(_pthread_list); e;
-  if ((z = malloc(sizeof(struct Zombie)))) {
+       e = nsync_dll_next_(_pthread_list, e)) {
-    z->pt = pt;
+    pt = (struct PosixThread *)e->container;
-    z->next = atomic_load(&_pthread_zombies);
+    if (pt->tib == __get_tls()) continue;
-    for (;;) {
+    status = atomic_load_explicit(&pt->status, memory_order_acquire);
-      if (atomic_compare_exchange_weak(&_pthread_zombies, &z->next, z)) {
+    if (status != kPosixThreadZombie) break;
-        break;
+    if (!atomic_load_explicit(&pt->tib->tib_tid, memory_order_acquire)) {
-      }
+      _pthread_list = nsync_dll_remove_(_pthread_list, e);
+      pthread_spin_unlock(&_pthread_lock);
+      _pthread_free(pt);
+      goto StartOver;
     }
   }
-}
+  pthread_spin_unlock(&_pthread_lock);
-
-static void _pthread_zombies_collect(struct Zombie *z) {
-  // TODO(jart): We need a trywait() op here to avoid cancellation.
-  _pthread_wait(z->pt);
-  _pthread_free(z->pt);
-  free(z);
-}
-
-void _pthread_zombies_decimate(void) {
-  struct Zombie *z;
-  while ((z = atomic_load_explicit(&_pthread_zombies, memory_order_relaxed)) &&
-         atomic_load(&z->pt->status) == kPosixThreadZombie) {
-    if (atomic_compare_exchange_weak(&_pthread_zombies, &z, z->next)) {
-      _pthread_zombies_collect(z);
-    }
-  }
-}
-
-void _pthread_zombies_harvest(void) {
-  struct Zombie *z;
-  while ((z = atomic_load_explicit(&_pthread_zombies, memory_order_relaxed))) {
-    if (atomic_compare_exchange_weak(&_pthread_zombies, &z, z->next)) {
-      _pthread_zombies_collect(z);
-    }
-  }
-}
-
-void _pthread_zombies_purge(void) {
-  struct Zombie *z, *n;
-  while ((z = atomic_load_explicit(&_pthread_zombies, memory_order_relaxed))) {
-    _pthread_free(z->pt);
-    n = z->next;
-    free(z);
-    atomic_store_explicit(&_pthread_zombies, n, memory_order_relaxed);
-  }
-}
-
-__attribute__((__constructor__)) static void _pthread_zombies_init(void) {
-  atexit(_pthread_zombies_harvest);
 }

libc/thread/pthread_detach.c

@@ -16,45 +16,52 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/assert.h"
 #include "libc/errno.h"
 #include "libc/intrin/atomic.h"
+#include "libc/intrin/strace.internal.h"
 #include "libc/macros.internal.h"
 #include "libc/mem/mem.h"
 #include "libc/thread/posixthread.internal.h"
 #include "libc/thread/spawn.h"
 #include "libc/thread/thread.h"
+#include "third_party/nsync/dll.h"
 
 /**
- * Asks POSIX thread to free itself automatically on termination.
+ * Asks POSIX thread to free itself automatically upon termination.
+ *
+ * If this function is used, then it's important to use pthread_exit()
+ * rather than exit() since otherwise your program isn't guaranteed to
+ * gracefully terminate.
+ *
+ * Detaching a non-joinable thread is undefined behavior. For example,
+ * pthread_detach() can't be called twice on the same thread.
  *
  * @return 0 on success, or errno with error
- * @raise EINVAL if thread is null or already detached
  * @returnserrno
  * @threadsafe
  */
 errno_t pthread_detach(pthread_t thread) {
   struct PosixThread *pt;
   enum PosixThreadStatus status, transition;
-  if (!(pt = (struct PosixThread *)thread)) return EINVAL;
+  for (pt = (struct PosixThread *)thread;;) {
-  for (;;) {
     status = atomic_load_explicit(&pt->status, memory_order_acquire);
-    if (status == kPosixThreadDetached || status == kPosixThreadZombie) {
+    if (status == kPosixThreadJoinable) {
-      // these two states indicate the thread was already detached, in
-      // which case it's already listed under _pthread_zombies.
-      return EINVAL;
-    } else if (status == kPosixThreadJoinable) {
       transition = kPosixThreadDetached;
     } else if (status == kPosixThreadTerminated) {
       transition = kPosixThreadZombie;
     } else {
-      notpossible;
+      unreachable;
     }
     if (atomic_compare_exchange_weak_explicit(&pt->status, &status, transition,
                                               memory_order_release,
                                               memory_order_relaxed)) {
-      _pthread_zombies_add(pt);
       break;
     }
   }
+  if (transition == kPosixThreadZombie) {
+    _pthread_zombify(pt);
+  }
+  pthread_decimate_np();
   return 0;
 }

libc/thread/pthread_exit.c

@@ -16,16 +16,52 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/assert.h"
+#include "libc/atomic.h"
 #include "libc/dce.h"
 #include "libc/intrin/atomic.h"
+#include "libc/intrin/kprintf.h"
+#include "libc/intrin/strace.internal.h"
+#include "libc/intrin/weaken.h"
 #include "libc/limits.h"
 #include "libc/mem/gc.h"
 #include "libc/runtime/runtime.h"
 #include "libc/thread/posixthread.internal.h"
 #include "libc/thread/thread.h"
 #include "libc/thread/tls.h"
+#include "third_party/nsync/dll.h"
 #include "third_party/nsync/futex.internal.h"
 
+static void CleanupThread(struct PosixThread *pt) {
+  struct _pthread_cleanup_buffer *cb;
+  while ((cb = pt->cleanup)) {
+    pt->cleanup = cb->__prev;
+    cb->__routine(cb->__arg);
+  }
+}
+
+static void DestroyTlsKeys(struct CosmoTib *tib) {
+  int i, j, gotsome;
+  void *val, **keys;
+  pthread_key_dtor dtor;
+  keys = tib->tib_keys;
+  for (j = 0; j < PTHREAD_DESTRUCTOR_ITERATIONS; ++j) {
+    for (gotsome = i = 0; i < PTHREAD_KEYS_MAX; ++i) {
+      if ((val = keys[i]) &&
+          (dtor = atomic_load_explicit(_pthread_key_dtor + i,
+                                       memory_order_relaxed)) &&
+          dtor != (pthread_key_dtor)-1) {
+        gotsome = 1;
+        keys[i] = 0;
+        dtor(val);
+      }
+    }
+    if (!gotsome) {
+      break;
+    }
+  }
+}
+
 /**
  * Terminates current POSIX thread.
  *
@@ -60,35 +96,59 @@
  * @noreturn
  */
 wontreturn void pthread_exit(void *rc) {
+  struct CosmoTib *tib;
   struct PosixThread *pt;
-  struct _pthread_cleanup_buffer *cb;
+  enum PosixThreadStatus status, transition;
-  pt = (struct PosixThread *)__get_tls()->tib_pthread;
+
+  STRACE("pthread_exit(%p)", rc);
+
+  tib = __get_tls();
+  pt = (struct PosixThread *)tib->tib_pthread;
+  _unassert(~pt->flags & PT_EXITING);
+  pt->flags |= PT_EXITING;
   pt->rc = rc;
-  // the memory of pthread cleanup objects lives on the stack
+
-  // so we need to harvest them before calling longjmp()
+  // free resources
-  while ((cb = pt->cleanup)) {
+  CleanupThread(pt);
-    pt->cleanup = cb->__prev;
+  DestroyTlsKeys(tib);
-    cb->__routine(cb->__arg);
+  _pthread_ungarbage();
+  pthread_decimate_np();
+
+  // transition the thread to a terminated state
+  status = atomic_load_explicit(&pt->status, memory_order_acquire);
+  do {
+    switch (status) {
+      case kPosixThreadJoinable:
+        transition = kPosixThreadTerminated;
+        break;
+      case kPosixThreadDetached:
+        transition = kPosixThreadZombie;
+        break;
+      default:
+        unreachable;
+    }
+  } while (!atomic_compare_exchange_weak_explicit(
+      &pt->status, &status, transition, memory_order_release,
+      memory_order_relaxed));
+
+  // make this thread a zombie if it was detached
+  if (transition == kPosixThreadZombie) {
+    _pthread_zombify(pt);
   }
-  // TODO(jart): An orphaned thread should become the main thread.
+
-  // TODO(jart): This should call __cxa_finalize() for the orphan.
+  // check if this is the main thread or an orphaned thread
-  if (~pt->flags & PT_MAINTHREAD) {
+  if (pthread_orphan_np()) {
-    // this thread was created by pthread_create()
+    exit(0);
-    // garbage collector memory exists on a shadow stack. we don't need
+  }
-    // to use _gclongjmp() since _pthread_ungarbage() will collect them
+
-    // at the setjmp() site.
+  // check if the main thread has died whilst children live
-    longjmp(pt->exiter, 1);
+  // note that the main thread is joinable by child threads
-  } else {
+  if (pt->flags & PT_STATIC) {
-    // this is the main thread
+    atomic_store_explicit(&tib->tib_tid, 0, memory_order_release);
-    // release as much resources and possible and mark it terminated
+    nsync_futex_wake_(&tib->tib_tid, INT_MAX, !IsWindows());
-    _pthread_cleanup(pt);
-    // it's kind of irregular for a child thread to join the main thread
-    // so we don't bother freeing the main thread's stack since it makes
-    // this implementation so much simpler for example we want't to call
-    // set_tid_address() upon every program startup which isn't possible
-    // on non-linux platforms anyway.
-    atomic_store_explicit(&__get_tls()->tib_tid, 0, memory_order_release);
-    nsync_futex_wake_(&__get_tls()->tib_tid, INT_MAX, !IsWindows());
     _Exit1(0);
   }
+
+  // this is a child thread
+  longjmp(pt->exiter, 1);
 }

libc/thread/pthread_join.c

@@ -16,21 +16,29 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/assert.h"
 #include "libc/errno.h"
+#include "libc/intrin/atomic.h"
 #include "libc/thread/posixthread.internal.h"
 #include "libc/thread/thread.h"
 #include "libc/thread/tls.h"
+#include "libc/thread/wait0.internal.h"
 
 /**
  * Waits for thread to terminate.
  *
+ * Multiple threads joining the same thread is undefined behavior. If a
+ * deferred or masked cancellation happens to the calling thread either
+ * before or during the waiting process then the target thread will not
+ * be joined. Calling pthread_join() on a non-joinable thread, e.g. one
+ * that's been detached, is undefined behavior. If a thread attempts to
+ * join itself, then the behavior is undefined.
+ *
  * @param value_ptr if non-null will receive pthread_exit() argument
  *     if the thread called pthread_exit(), or `PTHREAD_CANCELED` if
  *     pthread_cancel() destroyed the thread instead
  * @return 0 on success, or errno on error
  * @raise ECANCELED if calling thread was cancelled in masked mode
- * @raise EDEADLK if `thread` is the current thread
- * @raise EINVAL if `thread` is detached
  * @cancellationpoint
  * @returnserrno
  * @threadsafe
@@ -38,20 +46,22 @@
 errno_t pthread_join(pthread_t thread, void **value_ptr) {
   errno_t rc;
   struct PosixThread *pt;
-  if (thread == __get_tls()->tib_pthread) {
+  enum PosixThreadStatus status;
-    return EDEADLK;
+  pt = (struct PosixThread *)thread;
+  status = atomic_load_explicit(&pt->status, memory_order_acquire);
+  // "The behavior is undefined if the value specified by the thread
+  //  argument to pthread_join() does not refer to a joinable thread."
+  //                                  ──Quoth POSIX.1-2017
+  _unassert(status == kPosixThreadJoinable || status == kPosixThreadTerminated);
+  if (!(rc = _wait0(&pt->tib->tib_tid))) {
+    pthread_spin_lock(&_pthread_lock);
+    _pthread_list = nsync_dll_remove_(_pthread_list, &pt->list);
+    pthread_spin_unlock(&_pthread_lock);
+    if (value_ptr) {
+      *value_ptr = pt->rc;
+    }
+    _pthread_free(pt);
+    pthread_decimate_np();
   }
-  if (!(pt = (struct PosixThread *)thread) ||  //
-      pt->status == kPosixThreadZombie ||      //
-      pt->status == kPosixThreadDetached) {
-    return EINVAL;
-  }
-  if ((rc = _pthread_wait(pt))) {
-    return rc;
-  }
-  if (value_ptr) {
-    *value_ptr = pt->rc;
-  }
-  _pthread_free(pt);
   return 0;
 }

libc/thread/pthread_orphan_np.c

@@ -16,23 +16,17 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/assert.h"
-#include "libc/intrin/atomic.h"
-#include "libc/intrin/weaken.h"
 #include "libc/thread/posixthread.internal.h"
 #include "libc/thread/thread.h"
 
-void _pthread_cleanup(struct PosixThread *pt) {
+/**
-  _pthread_ungarbage();
+ * Returns true if calling thread is the only thread.
-  if (_weaken(_pthread_key_destruct)) {
+ */
-    _weaken(_pthread_key_destruct)();
+bool pthread_orphan_np(void) {
-  }
+  bool res;
-  if (atomic_load_explicit(&pt->status, memory_order_acquire) ==
+  pthread_spin_lock(&_pthread_lock);
-      kPosixThreadDetached) {
+  res = _pthread_list == _pthread_list->prev &&
-    atomic_store_explicit(&pt->status, kPosixThreadZombie,
+        _pthread_list == _pthread_list->next;
-                          memory_order_release);
+  pthread_spin_unlock(&_pthread_lock);
-  } else {
+  return res;
-    atomic_store_explicit(&pt->status, kPosixThreadTerminated,
-                          memory_order_release);
-  }
 }

libc/thread/pthread_print_np.c

@@ -0,0 +1,91 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/blockcancel.internal.h"
+#include "libc/calls/calls.h"
+#include "libc/intrin/kprintf.h"
+#include "libc/runtime/runtime.h"
+#include "libc/str/str.h"
+#include "libc/thread/posixthread.internal.h"
+#include "libc/thread/thread.h"
+#include "third_party/nsync/dll.h"
+
+#define N 2048
+#define M 15
+
+#define append(f, ...) o += f(buf + o, N - o, __VA_ARGS__)
+
+static const char *DescribeStatus(enum PosixThreadStatus status) {
+  switch (status) {
+    case kPosixThreadJoinable:
+      return "JOINAB";
+    case kPosixThreadDetached:
+      return "DETACH";
+    case kPosixThreadTerminated:
+      return "TERMIN";
+    case kPosixThreadZombie:
+      return "ZOMBIE";
+    default:
+      unreachable;
+  }
+}
+
+static const char *DescribeFlags(char buf[M], struct PosixThread *pt) {
+  char *p = buf;
+  if (pt->cancelled) *p++ = '*';
+  if (pt->flags & PT_EXITING) *p++ = 'X';
+  if (pt->flags & PT_STATIC) *p++ = 'S';
+  if (pt->flags & PT_OWNSTACK) *p++ = 'O';
+  if (pt->flags & PT_ASYNC) *p++ = 'A';
+  if (pt->flags & PT_MASKED) *p++ = 'M';
+  if (pt->flags & PT_OPENBSD_KLUDGE) *p++ = 'K';
+  if (pt->flags & PT_INCANCEL) *p++ = '?';
+  if (pt->flags & PT_NOCANCEL) *p++ = '!';
+  *p = 0;
+  return buf;
+}
+
+int pthread_print_np(int fd, const char *fmt, ...) {
+  va_list va;
+  int rc, o = 0;
+  nsync_dll_element_ *e;
+  struct PosixThread *pt;
+  char buf[N], flagbuf[M];
+  pthread_spin_lock(&_pthread_lock);
+  if (fmt) {
+    va_start(va, fmt);
+    append(kvsnprintf, fmt, va);
+    va_end(va);
+    append(ksnprintf, "\n");
+  }
+  append(ksnprintf, "%6s %6s %6s %6s %s\n", "ptid", "tid", "status", "flags",
+         "start");
+  for (e = nsync_dll_first_(_pthread_list); e;
+       e = nsync_dll_next_(_pthread_list, e)) {
+    pt = (struct PosixThread *)e->container;
+    append(ksnprintf, "%-6d %-6d %6s %6s %t\n", pt->ptid, pt->tib->tib_tid,
+           DescribeStatus(pt->status), DescribeFlags(flagbuf, pt), pt->start);
+  }
+  pthread_spin_unlock(&_pthread_lock);
+  BLOCK_CANCELLATIONS;
+  strace_enabled(-1);
+  rc = write(fd, buf, strlen(buf));
+  strace_enabled(+1);
+  ALLOW_CANCELLATIONS;
+  return rc;
+}

libc/thread/pthread_ungarbage.c

@@ -23,9 +23,12 @@
 void _pthread_ungarbage(void) {
   int i;
   struct Garbages *g;
-  if ((g = __get_tls()->tib_garbages)) {
+  struct CosmoTib *tib;
-    for (i = g->i; i--;) {
+  tib = __get_tls();
-      ((void (*)(intptr_t))g->p[i].fn)(g->p[i].arg);
+  while ((g = tib->tib_garbages)) {
+    tib->tib_garbages = 0;
+    while (g->i--) {
+      ((void (*)(intptr_t))g->p[g->i].fn)(g->p[g->i].arg);
     }
     free(g->p);
     free(g);

libc/thread/pthread_zombify.c

@@ -16,30 +16,13 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/intrin/atomic.h"
 #include "libc/thread/posixthread.internal.h"
 #include "libc/thread/thread.h"
-#include "libc/thread/tls.h"
+#include "third_party/nsync/dll.h"
 
-void _pthread_key_destruct(void) {
+void _pthread_zombify(struct PosixThread *pt) {
-  int i, j, gotsome;
+  pthread_spin_lock(&_pthread_lock);
-  void *val, **keys;
+  _pthread_list = nsync_dll_remove_(_pthread_list, &pt->list);
-  pthread_key_dtor dtor;
+  _pthread_list = nsync_dll_make_first_in_list_(_pthread_list, &pt->list);
-  if (!__tls_enabled) return;
+  pthread_spin_unlock(&_pthread_lock);
-  keys = __get_tls()->tib_keys;
-  for (j = 0; j < PTHREAD_DESTRUCTOR_ITERATIONS; ++j) {
-    for (gotsome = i = 0; i < PTHREAD_KEYS_MAX; ++i) {
-      if ((val = keys[i]) &&
-          (dtor = atomic_load_explicit(_pthread_key_dtor + i,
-                                       memory_order_relaxed)) &&
-          dtor != (pthread_key_dtor)-1) {
-        gotsome = 1;
-        keys[i] = 0;
-        dtor(val);
-      }
-    }
-    if (!gotsome) {
-      break;
-    }
-  }
 }

libc/thread/thread.h

@@ -110,10 +110,13 @@ int pthread_create(pthread_t *, const pthread_attr_t *, void *(*)(void *),
                    void *);
 
 int pthread_yield(void);
+bool pthread_orphan_np(void);
 void pthread_testcancel(void);
+void pthread_decimate_np(void);
 int pthread_testcancel_np(void);
 void pthread_exit(void *) wontreturn;
 pthread_t pthread_self(void) pureconst;
+int pthread_print_np(int, const char *, ...);
 pthread_id_np_t pthread_getthreadid_np(void);
 int pthread_getunique_np(pthread_t, pthread_id_np_t *);
 int pthread_setname_np(pthread_t, const char *);
@@ -208,7 +211,7 @@ void _pthread_cleanup_push(struct _pthread_cleanup_buffer *, void (*)(void *),
   }
 
 #if (__GNUC__ + 0) * 100 + (__GNUC_MINOR__ + 0) >= 407 && \
-    !defined(__STRICT_ANSI__)
+    !defined(__STRICT_ANSI__) && !defined(MODE_DBG)
 extern const errno_t EBUSY;
 #define pthread_spin_lock(pSpin)                                           \
   ({                                                                       \

libc/thread/tls.h

@@ -45,18 +45,17 @@ extern unsigned __tls_index;
 void __require_tls(void);
 void __set_tls(struct CosmoTib *);
 
-#if defined(__GNUC__) && defined(__x86_64__) && !defined(__STRICT_ANSI__)
 /**
  * Returns location of thread information block.
  *
  * This can't be used in privileged functions.
  */
-static inline struct CosmoTib *__get_tls(void) {
+#define __get_tls()                                                \
-  struct CosmoTib *_tib;
+  ({                                                               \
-  asm("mov\t%%fs:0,%0" : "=r"(_tib) : /* no inputs */ : "memory");
+    struct CosmoTib *_t;                                           \
-  return _tib;
+    asm("mov\t%%fs:0,%0" : "=r"(_t) : /* no inputs */ : "memory"); \
-}
+    _t;                                                            \
-#endif /* GNU x86-64 */
+  })
 
 COSMOPOLITAN_C_END_
 #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

libc/thread/wait0.c

@@ -16,10 +16,13 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/assert.h"
 #include "libc/calls/cp.internal.h"
 #include "libc/dce.h"
 #include "libc/errno.h"
 #include "libc/intrin/atomic.h"
+#include "libc/thread/thread.h"
+#include "libc/thread/tls.h"
 #include "libc/thread/wait0.internal.h"
 #include "third_party/nsync/futex.internal.h"
 
@@ -38,13 +41,21 @@
  */
 errno_t _wait0(const atomic_int *ctid) {
   int x, rc = 0;
-  BEGIN_CANCELLATION_POINT;
+  // "The behavior is undefined if the value specified by the thread
-  while ((x = atomic_load_explicit(ctid, memory_order_acquire))) {
+  //  argument to pthread_join() refers to the calling thread."
-    if (nsync_futex_wait_(ctid, x, !IsWindows(), 0) == -ECANCELED) {
+  //                                  ──Quoth POSIX.1-2017
-      rc = ECANCELED;
+  _unassert(ctid != &__get_tls()->tib_tid);
-      break;
+  // "If the thread calling pthread_join() is canceled, then the target
+  //  thread shall not be detached."  ──Quoth POSIX.1-2017
+  if (!(rc = pthread_testcancel_np())) {
+    BEGIN_CANCELLATION_POINT;
+    while ((x = atomic_load_explicit(ctid, memory_order_acquire))) {
+      if (nsync_futex_wait_(ctid, x, !IsWindows(), 0) == -ECANCELED) {
+        rc = ECANCELED;
+        break;
+      }
     }
+    END_CANCELLATION_POINT;
   }
-  END_CANCELLATION_POINT;
   return rc;
 }

libc/zipos/open.c

@@ -116,7 +116,9 @@ static int __zipos_mkfd(int minfd) {
 
 static int __zipos_setfd(int fd, struct ZiposHandle *h, unsigned flags,
                          int mode) {
-  _cmpxchg(&g_fds.f, fd, fd + 1);
+  int want = fd;
+  atomic_compare_exchange_strong_explicit(
+      &g_fds.f, &want, fd + 1, memory_order_release, memory_order_relaxed);
   g_fds.p[fd].kind = kFdZip;
   g_fds.p[fd].handle = (intptr_t)h;
   g_fds.p[fd].flags = flags | O_CLOEXEC;

test/libc/thread/pthread_create_test.c

@@ -54,16 +54,11 @@ void TriggerSignal(void) {
 }
 
 static void *Increment(void *arg) {
-  ASSERT_EQ(EDEADLK, pthread_join(pthread_self(), 0));
   ASSERT_EQ(gettid(), pthread_getthreadid_np());
   TriggerSignal();
   return (void *)((uintptr_t)arg + 1);
 }
 
-TEST(pthread_create, joinSelfDeadlocks) {
-  ASSERT_EQ(EDEADLK, pthread_join(pthread_self(), 0));
-}
-
 TEST(pthread_create, testCreateReturnJoin) {
   void *rc;
   pthread_t id;
@@ -279,4 +274,7 @@ BENCH(pthread_create, bench) {
   EZBENCH2("CreateJoin", donothing, CreateJoin());
   EZBENCH2("CreateDetach", donothing, CreateDetach());
   EZBENCH2("CreateDetached", donothing, CreateDetached());
+  while (!pthread_orphan_np()) {
+    pthread_decimate_np();
+  }
 }

test/libc/thread/pthread_detach_test.c

@@ -19,10 +19,13 @@
 #include "libc/calls/calls.h"
 #include "libc/calls/struct/sigaction.h"
 #include "libc/errno.h"
+#include "libc/intrin/strace.internal.h"
 #include "libc/sysv/consts/sa.h"
 #include "libc/sysv/consts/sig.h"
 #include "libc/testlib/testlib.h"
+#include "libc/thread/posixthread.internal.h"
 #include "libc/thread/thread.h"
+#include "third_party/nsync/dll.h"
 
 void OnUsr1(int sig, struct siginfo *si, void *vctx) {
   struct ucontext *ctx = vctx;
@@ -40,7 +43,6 @@ void TriggerSignal(void) {
 }
 
 static void *Increment(void *arg) {
-  ASSERT_EQ(EDEADLK, pthread_join(pthread_self(), 0));
   ASSERT_EQ(gettid(), pthread_getthreadid_np());
   TriggerSignal();
   return (void *)((uintptr_t)arg + 1);
@@ -50,6 +52,9 @@ TEST(pthread_detach, testCreateReturn) {
   pthread_t id;
   ASSERT_EQ(0, pthread_create(&id, 0, Increment, 0));
   ASSERT_EQ(0, pthread_detach(id));
+  while (!pthread_orphan_np()) {
+    pthread_decimate_np();
+  }
 }
 
 TEST(pthread_detach, testDetachUponCreation) {
@@ -58,4 +63,7 @@ TEST(pthread_detach, testDetachUponCreation) {
   ASSERT_EQ(0, pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED));
   ASSERT_EQ(0, pthread_create(0, &attr, Increment, 0));
   ASSERT_EQ(0, pthread_attr_destroy(&attr));
+  while (!pthread_orphan_np()) {
+    pthread_decimate_np();
+  }
 }

test/libc/thread/pthread_exit_test.c

@@ -0,0 +1,66 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/runtime/runtime.h"
+#include "libc/testlib/subprocess.h"
+#include "libc/testlib/testlib.h"
+#include "libc/thread/thread.h"
+
+#define MAIN  150
+#define CHILD 170
+
+pthread_t main_thread;
+_Thread_local static int exit_code;
+
+void OnExit(void) {
+  _Exit(exit_code);
+}
+
+void *Worker(void *arg) {
+  ASSERT_EQ(0, pthread_join(main_thread, 0));
+  exit_code = CHILD;
+  pthread_exit(0);
+}
+
+TEST(pthread_exit, joinableOrphanedChild_runsAtexitHandlers) {
+  pthread_attr_t attr;
+  SPAWN(fork);
+  atexit(OnExit);
+  exit_code = MAIN;
+  main_thread = pthread_self();
+  ASSERT_EQ(0, pthread_attr_init(&attr));
+  ASSERT_EQ(0, pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE));
+  ASSERT_EQ(0, pthread_create(0, &attr, Worker, 0));
+  ASSERT_EQ(0, pthread_attr_destroy(&attr));
+  pthread_exit(0);
+  EXITS(CHILD);
+}
+
+TEST(pthread_exit, detachedOrphanedChild_runsAtexitHandlers) {
+  pthread_attr_t attr;
+  SPAWN(fork);
+  atexit(OnExit);
+  exit_code = MAIN;
+  main_thread = pthread_self();
+  ASSERT_EQ(0, pthread_attr_init(&attr));
+  ASSERT_EQ(0, pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED));
+  ASSERT_EQ(0, pthread_create(0, &attr, Worker, 0));
+  ASSERT_EQ(0, pthread_attr_destroy(&attr));
+  pthread_exit(0);
+  EXITS(CHILD);
+}

test/libc/thread/pthread_key_create_test.c

@@ -16,23 +16,56 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/atomic.h"
+#include "libc/intrin/kprintf.h"
 #include "libc/mem/mem.h"
+#include "libc/nexgen32e/nexgen32e.h"
 #include "libc/testlib/testlib.h"
 #include "libc/thread/thread.h"
 
-TEST(pthread_key_create, testRunsDtors_becauseNoLeakReport) {
+pthread_key_t mykey;
-  void *x;
+int destructor_calls;
-  pthread_key_t key;
+
-  x = malloc(123);
+void KeyDestructor(void *arg) {
-  EXPECT_EQ(0, pthread_key_create(&key, free));
+  EXPECT_EQ(0, pthread_getspecific(mykey));
-  EXPECT_EQ(0, pthread_setspecific(key, x));
+  ASSERT_EQ(31337, (intptr_t)arg);
-  EXPECT_EQ(x, pthread_getspecific(key));
+  ++destructor_calls;
-  x = malloc(123);
+}
-  EXPECT_EQ(0, pthread_key_create(&key, free));
+
-  EXPECT_EQ(0, pthread_setspecific(key, x));
+void *Worker(void *arg) {
-  EXPECT_EQ(x, pthread_getspecific(key));
+  ASSERT_EQ(0, pthread_getspecific(mykey));
-  x = malloc(123);
+  ASSERT_EQ(0, pthread_setspecific(mykey, (void *)31337));
-  EXPECT_EQ(0, pthread_key_create(&key, free));
+  ASSERT_EQ((void *)31337, pthread_getspecific(mykey));
-  EXPECT_EQ(0, pthread_setspecific(key, x));
+  return 0;
-  EXPECT_EQ(x, pthread_getspecific(key));
+}
+
+TEST(pthread_key_create, test) {
+  pthread_t th;
+  destructor_calls = 0;
+  ASSERT_EQ(0, pthread_key_create(&mykey, KeyDestructor));
+  ASSERT_EQ(0, pthread_setspecific(mykey, (void *)666));
+  ASSERT_EQ(0, pthread_create(&th, 0, Worker, 0));
+  ASSERT_EQ(0, pthread_join(th, 0));
+  ASSERT_EQ(1, destructor_calls);
+  ASSERT_EQ((void *)666, pthread_getspecific(mykey));
+  ASSERT_EQ(0, pthread_key_delete(mykey));
+}
+
+void KeyDestructorCraze(void *arg) {
+  EXPECT_EQ(0, pthread_getspecific(mykey));
+  ASSERT_EQ(31337, (intptr_t)arg);
+  EXPECT_EQ(0, pthread_setspecific(mykey, (void *)31337));
+  ++destructor_calls;
+}
+
+TEST(pthread_key_create, destructorKeepsSettingKey_willHalt) {
+  pthread_t th;
+  destructor_calls = 0;
+  ASSERT_EQ(0, pthread_key_create(&mykey, KeyDestructorCraze));
+  ASSERT_EQ(0, pthread_setspecific(mykey, (void *)666));
+  ASSERT_EQ(0, pthread_create(&th, 0, Worker, 0));
+  ASSERT_EQ(0, pthread_join(th, 0));
+  ASSERT_EQ(PTHREAD_DESTRUCTOR_ITERATIONS, destructor_calls);
+  ASSERT_EQ((void *)666, pthread_getspecific(mykey));
+  ASSERT_EQ(0, pthread_key_delete(mykey));
 }