Make detached threads work better

This change adds a double linked list of threads, so that pthread_exit()
will know when it should call exit() from an orphaned child. This change
also improves ftrace and strace logging.

libc/thread/pthread_exit.c

@@ -16,16 +16,52 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/assert.h"
+#include "libc/atomic.h"
 #include "libc/dce.h"
 #include "libc/intrin/atomic.h"
+#include "libc/intrin/kprintf.h"
+#include "libc/intrin/strace.internal.h"
+#include "libc/intrin/weaken.h"
 #include "libc/limits.h"
 #include "libc/mem/gc.h"
 #include "libc/runtime/runtime.h"
 #include "libc/thread/posixthread.internal.h"
 #include "libc/thread/thread.h"
 #include "libc/thread/tls.h"
+#include "third_party/nsync/dll.h"
 #include "third_party/nsync/futex.internal.h"
 
+static void CleanupThread(struct PosixThread *pt) {
+  struct _pthread_cleanup_buffer *cb;
+  while ((cb = pt->cleanup)) {
+    pt->cleanup = cb->__prev;
+    cb->__routine(cb->__arg);
+  }
+}
+
+static void DestroyTlsKeys(struct CosmoTib *tib) {
+  int i, j, gotsome;
+  void *val, **keys;
+  pthread_key_dtor dtor;
+  keys = tib->tib_keys;
+  for (j = 0; j < PTHREAD_DESTRUCTOR_ITERATIONS; ++j) {
+    for (gotsome = i = 0; i < PTHREAD_KEYS_MAX; ++i) {
+      if ((val = keys[i]) &&
+          (dtor = atomic_load_explicit(_pthread_key_dtor + i,
+                                       memory_order_relaxed)) &&
+          dtor != (pthread_key_dtor)-1) {
+        gotsome = 1;
+        keys[i] = 0;
+        dtor(val);
+      }
+    }
+    if (!gotsome) {
+      break;
+    }
+  }
+}
+
 /**
  * Terminates current POSIX thread.
  *
@@ -60,35 +96,59 @@
  * @noreturn
  */
 wontreturn void pthread_exit(void *rc) {
+  struct CosmoTib *tib;
   struct PosixThread *pt;
-  struct _pthread_cleanup_buffer *cb;
-  pt = (struct PosixThread *)__get_tls()->tib_pthread;
+  enum PosixThreadStatus status, transition;
+
+  STRACE("pthread_exit(%p)", rc);
+
+  tib = __get_tls();
+  pt = (struct PosixThread *)tib->tib_pthread;
+  _unassert(~pt->flags & PT_EXITING);
+  pt->flags |= PT_EXITING;
   pt->rc = rc;
-  // the memory of pthread cleanup objects lives on the stack
-  // so we need to harvest them before calling longjmp()
-  while ((cb = pt->cleanup)) {
-    pt->cleanup = cb->__prev;
-    cb->__routine(cb->__arg);
+
+  // free resources
+  CleanupThread(pt);
+  DestroyTlsKeys(tib);
+  _pthread_ungarbage();
+  pthread_decimate_np();
+
+  // transition the thread to a terminated state
+  status = atomic_load_explicit(&pt->status, memory_order_acquire);
+  do {
+    switch (status) {
+      case kPosixThreadJoinable:
+        transition = kPosixThreadTerminated;
+        break;
+      case kPosixThreadDetached:
+        transition = kPosixThreadZombie;
+        break;
+      default:
+        unreachable;
+    }
+  } while (!atomic_compare_exchange_weak_explicit(
+      &pt->status, &status, transition, memory_order_release,
+      memory_order_relaxed));
+
+  // make this thread a zombie if it was detached
+  if (transition == kPosixThreadZombie) {
+    _pthread_zombify(pt);
   }
-  // TODO(jart): An orphaned thread should become the main thread.
-  // TODO(jart): This should call __cxa_finalize() for the orphan.
-  if (~pt->flags & PT_MAINTHREAD) {
-    // this thread was created by pthread_create()
-    // garbage collector memory exists on a shadow stack. we don't need
-    // to use _gclongjmp() since _pthread_ungarbage() will collect them
-    // at the setjmp() site.
-    longjmp(pt->exiter, 1);
-  } else {
-    // this is the main thread
-    // release as much resources and possible and mark it terminated
-    _pthread_cleanup(pt);
-    // it's kind of irregular for a child thread to join the main thread
-    // so we don't bother freeing the main thread's stack since it makes
-    // this implementation so much simpler for example we want't to call
-    // set_tid_address() upon every program startup which isn't possible
-    // on non-linux platforms anyway.
-    atomic_store_explicit(&__get_tls()->tib_tid, 0, memory_order_release);
-    nsync_futex_wake_(&__get_tls()->tib_tid, INT_MAX, !IsWindows());
+
+  // check if this is the main thread or an orphaned thread
+  if (pthread_orphan_np()) {
+    exit(0);
+  }
+
+  // check if the main thread has died whilst children live
+  // note that the main thread is joinable by child threads
+  if (pt->flags & PT_STATIC) {
+    atomic_store_explicit(&tib->tib_tid, 0, memory_order_release);
+    nsync_futex_wake_(&tib->tib_tid, INT_MAX, !IsWindows());
     _Exit1(0);
   }
+
+  // this is a child thread
+  longjmp(pt->exiter, 1);
 }