Make more threading improvements

- ASAN memory morgue is now lockless
- Make C11 atomics header more portable
- Rewrote pthread keys support to be lockless
- Simplify Python's unicode table unpacking code
- Make crash report write(2) closer to being atomic
- Make it possible to strace/ftrace a single thread
- ASAN now checks nul-terminated strings fast and properly
- Windows fork() now restores TLS memory of calling thread

libc/thread/mktls.c

@@ -38,7 +38,8 @@ void Bzero(void *, size_t) asm("bzero");  // gcc bug
  */
 char *_mktls(struct CosmoTib **out_tib) {
   char *tls;
-  struct CosmoTib *tib;
+  struct CosmoTib *neu, *old;
+  __require_tls();
 
   // allocate memory for tdata, tbss, and tib
   tls = memalign(TLS_ALIGNMENT, I(_tls_size) + sizeof(struct CosmoTib));
@@ -55,14 +56,17 @@ char *_mktls(struct CosmoTib **out_tib) {
   Bzero(tls + I(_tbss_offset), I(_tbss_size) + sizeof(struct CosmoTib));
 
   // set up thread information block
-  tib = (struct CosmoTib *)(tls + I(_tls_size));
-  tib->tib_self = tib;
-  tib->tib_self2 = tib;
-  tib->tib_sigmask = __get_tls()->tib_sigmask;
-  atomic_store_explicit(&tib->tib_tid, -1, memory_order_relaxed);
+  old = __get_tls();
+  neu = (struct CosmoTib *)(tls + I(_tls_size));
+  neu->tib_self = neu;
+  neu->tib_self2 = neu;
+  neu->tib_ftrace = old->tib_ftrace;
+  neu->tib_strace = old->tib_strace;
+  neu->tib_sigmask = old->tib_sigmask;
+  atomic_store_explicit(&neu->tib_tid, -1, memory_order_relaxed);
 
   if (out_tib) {
-    *out_tib = tib;
+    *out_tib = neu;
   }
   return tls;
 }

libc/thread/posixthread.internal.h

@@ -81,10 +81,7 @@ struct PosixThread {
 typedef void (*atfork_f)(void);
 
 extern struct PosixThread _pthread_main;
-hidden extern pthread_spinlock_t _pthread_keys_lock;
-hidden extern uint64_t _pthread_key_usage[(PTHREAD_KEYS_MAX + 63) / 64];
-hidden extern pthread_key_dtor _pthread_key_dtor[PTHREAD_KEYS_MAX];
-hidden extern _Thread_local void *_pthread_keys[PTHREAD_KEYS_MAX];
+extern _Atomic(pthread_key_dtor) _pthread_key_dtor[PTHREAD_KEYS_MAX] hidden;
 
 int _pthread_atfork(atfork_f, atfork_f, atfork_f) hidden;
 int _pthread_reschedule(struct PosixThread *) hidden;
@@ -97,9 +94,7 @@ void _pthread_zombies_add(struct PosixThread *) hidden;
 void _pthread_zombies_purge(void) hidden;
 void _pthread_zombies_decimate(void) hidden;
 void _pthread_zombies_harvest(void) hidden;
-void _pthread_key_destruct(void *[PTHREAD_KEYS_MAX]) hidden;
-void _pthread_key_lock(void) hidden;
-void _pthread_key_unlock(void) hidden;
+void _pthread_key_destruct(void) hidden;
 void _pthread_onfork_prepare(void) hidden;
 void _pthread_onfork_parent(void) hidden;
 void _pthread_onfork_child(void) hidden;

libc/thread/pthread_atfork.c

@@ -17,13 +17,16 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/assert.h"
+#include "libc/errno.h"
 #include "libc/intrin/kmalloc.h"
+#include "libc/runtime/memtrack.internal.h"
+#include "libc/str/str.h"
 #include "libc/thread/posixthread.internal.h"
 #include "libc/thread/thread.h"
 #include "libc/thread/tls.h"
 
 static struct AtForks {
-  pthread_mutex_t lock;
+  pthread_spinlock_t lock;
   struct AtFork {
     struct AtFork *p[2];
     atfork_f f[3];
@@ -34,12 +37,12 @@ static void _pthread_onfork(int i) {
   struct AtFork *a;
   struct PosixThread *pt;
   _unassert(0 <= i && i <= 2);
-  if (!i) pthread_mutex_lock(&_atforks.lock);
+  if (!i) pthread_spin_lock(&_atforks.lock);
   for (a = _atforks.list; a; a = a->p[!i]) {
     if (a->f[i]) a->f[i]();
     _atforks.list = a;
   }
-  if (i) pthread_mutex_unlock(&_atforks.lock);
+  if (i) pthread_spin_unlock(&_atforks.lock);
   if (i == 2) {
     _pthread_zombies_purge();
     if (__tls_enabled) {
@@ -51,29 +54,36 @@ static void _pthread_onfork(int i) {
 
 void _pthread_onfork_prepare(void) {
   _pthread_onfork(0);
+  __kmalloc_lock();
+  __mmi_lock();
 }
 
 void _pthread_onfork_parent(void) {
+  __mmi_unlock();
+  __kmalloc_unlock();
   _pthread_onfork(1);
 }
 
 void _pthread_onfork_child(void) {
+  extern pthread_mutex_t __mmi_lock_obj;
+  bzero(&__mmi_lock_obj, sizeof(__mmi_lock_obj));
+  __kmalloc_unlock();
   _pthread_onfork(2);
 }
 
 int _pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) {
   int rc;
   struct AtFork *a;
-  a = kmalloc(sizeof(struct AtFork));
+  if (!(a = kmalloc(sizeof(struct AtFork)))) return ENOMEM;
   a->f[0] = prepare;
   a->f[1] = parent;
   a->f[2] = child;
-  pthread_mutex_lock(&_atforks.lock);
+  pthread_spin_lock(&_atforks.lock);
   a->p[0] = 0;
   a->p[1] = _atforks.list;
   if (_atforks.list) _atforks.list->p[0] = a;
   _atforks.list = a;
-  pthread_mutex_unlock(&_atforks.lock);
+  pthread_spin_unlock(&_atforks.lock);
   rc = 0;
   return rc;
 }

libc/thread/pthread_cleanup.c

@@ -25,7 +25,7 @@
 void _pthread_cleanup(struct PosixThread *pt) {
   _pthread_ungarbage();
   if (_weaken(_pthread_key_destruct)) {
-    _weaken(_pthread_key_destruct)(0);
+    _weaken(_pthread_key_destruct)();
   }
   if (atomic_load_explicit(&pt->status, memory_order_acquire) ==
       kPosixThreadDetached) {

libc/thread/pthread_exit.c

@@ -70,6 +70,8 @@ wontreturn void pthread_exit(void *rc) {
     pt->cleanup = cb->__prev;
     cb->__routine(cb->__arg);
   }
+  // TODO(jart): An orphaned thread should become the main thread.
+  // TODO(jart): This should call __cxa_finalize() for the orphan.
   if (~pt->flags & PT_MAINTHREAD) {
     // this thread was created by pthread_create()
     // garbage collector memory exists on a shadow stack. we don't need

libc/thread/pthread_zombies.c

@@ -78,6 +78,6 @@ void _pthread_zombies_purge(void) {
   }
 }
 
-__attribute__((__constructor__)) static void init(void) {
+__attribute__((__constructor__)) static void _pthread_zombies_init(void) {
   atexit(_pthread_zombies_harvest);
 }

libc/thread/tls.h

@@ -18,18 +18,19 @@ struct CosmoFtrace {   /* 16 */
 };
 
 struct CosmoTib {
-  struct CosmoTib *tib_self;     /* 0x00 */
-  struct CosmoFtrace tib_ftrace; /* 0x08 */
-  void *tib_garbages;            /* 0x18 */
-  intptr_t tib_locale;           /* 0x20 */
-  intptr_t tib_pthread;          /* 0x28 */
-  struct CosmoTib *tib_self2;    /* 0x30 */
-  _Atomic(int32_t) tib_tid;      /* 0x38 */
-  int32_t tib_errno;             /* 0x3c */
-  uint64_t tib_flags;            /* 0x40 */
+  struct CosmoTib *tib_self;      /* 0x00 */
+  struct CosmoFtrace tib_ftracer; /* 0x08 */
+  void *tib_garbages;             /* 0x18 */
+  intptr_t tib_locale;            /* 0x20 */
+  intptr_t tib_pthread;           /* 0x28 */
+  struct CosmoTib *tib_self2;     /* 0x30 */
+  _Atomic(int32_t) tib_tid;       /* 0x38 */
+  int32_t tib_errno;              /* 0x3c */
+  uint64_t tib_flags;             /* 0x40 */
   void *tib_nsync;
-  uint64_t tib_sigmask;
-  void *tib_reserved3;
+  int tib_ftrace;       /* inherited */
+  int tib_strace;       /* inherited */
+  uint64_t tib_sigmask; /* inherited */
   void *tib_reserved4;
   void *tib_reserved5;
   void *tib_reserved6;
@@ -42,6 +43,7 @@ extern bool __tls_enabled;
 extern unsigned __tls_index;
 
 void __require_tls(void);
+void __set_tls(struct CosmoTib *);
 
 #if defined(__GNUC__) && defined(__x86_64__) && !defined(__STRICT_ANSI__)
 /**