Make locks more reliable

This change switches most of the core locks to be re-entrant, in order
to reduce the chance of deadlocking code that does, clever things with
asynchronous signal handlers. This change implements it it in pthreads
so we're one step closer to having a standardized threading primitives

libc/intrin/asan.c

@@ -31,6 +31,7 @@
 #include "libc/intrin/kprintf.h"
 #include "libc/intrin/lockcmpxchg.h"
 #include "libc/intrin/nomultics.internal.h"
+#include "libc/intrin/pthread.h"
 #include "libc/intrin/spinlock.h"
 #include "libc/log/backtrace.internal.h"
 #include "libc/log/internal.h"
@@ -169,7 +170,7 @@ struct ReportOriginHeap {
 };
 
 static int __asan_noreentry;
-_Alignas(64) static int __asan_lock;
+static pthread_mutex_t __asan_lock;
 static struct AsanMorgue __asan_morgue;
 
 #define __asan_unreachable()   \
@@ -852,25 +853,25 @@ dontdiscard __asan_die_f *__asan_report_memory_fault(void *addr, int size,
 void *__asan_morgue_add(void *p) {
   int i;
   void *r;
-  _spinlock_cooperative(&__asan_lock);
+  pthread_mutex_lock(&__asan_lock);
   i = __asan_morgue.i++ & (ARRAYLEN(__asan_morgue.p) - 1);
   r = __asan_morgue.p[i];
   __asan_morgue.p[i] = p;
-  _spunlock(&__asan_lock);
+  pthread_mutex_unlock(&__asan_lock);
   return r;
 }
 
 static void __asan_morgue_flush(void) {
   int i;
   void *p;
-  _spinlock_cooperative(&__asan_lock);
+  pthread_mutex_lock(&__asan_lock);
   for (i = 0; i < ARRAYLEN(__asan_morgue.p); ++i) {
     if (__asan_morgue.p[i] && weaken(dlfree)) {
       weaken(dlfree)(__asan_morgue.p[i]);
     }
     __asan_morgue.p[i] = 0;
   }
-  _spunlock(&__asan_lock);
+  pthread_mutex_unlock(&__asan_lock);
 }
 
 static size_t __asan_user_size(size_t n) {

libc/intrin/cxaatexit.c

@@ -19,6 +19,7 @@
 #include "libc/assert.h"
 #include "libc/bits/weaken.h"
 #include "libc/calls/strace.internal.h"
+#include "libc/intrin/pthread.h"
 #include "libc/intrin/spinlock.h"
 #include "libc/macros.internal.h"
 #include "libc/mem/mem.h"
@@ -29,7 +30,7 @@
 
 STATIC_YOINK("__cxa_finalize");
 
-static int __cxa_lock;
+static pthread_mutex_t __cxa_lock;
 
 /**
  * Adds global destructor.
@@ -50,7 +51,7 @@ noasan int __cxa_atexit(void *fp, void *arg, void *pred) {
   unsigned i;
   struct CxaAtexitBlock *b, *b2;
   _Static_assert(ATEXIT_MAX == CHAR_BIT * sizeof(b->mask), "");
-  _spinlock(&__cxa_lock);
+  pthread_mutex_lock(&__cxa_lock);
   b = __cxa_blocks.p;
   if (!b) b = __cxa_blocks.p = &__cxa_blocks.root;
   if (!~b->mask) {
@@ -59,7 +60,7 @@ noasan int __cxa_atexit(void *fp, void *arg, void *pred) {
       b2->next = b;
       __cxa_blocks.p = b = b2;
     } else {
-      _spunlock(&__cxa_lock);
+      pthread_mutex_unlock(&__cxa_lock);
       return enomem();
     }
   }
@@ -69,6 +70,6 @@ noasan int __cxa_atexit(void *fp, void *arg, void *pred) {
   b->p[i].fp = fp;
   b->p[i].arg = arg;
   b->p[i].pred = pred;
-  _spunlock(&__cxa_lock);
+  pthread_mutex_unlock(&__cxa_lock);
   return 0;
 }

libc/intrin/g_fds.c

@@ -19,6 +19,7 @@
 #include "libc/bits/pushpop.h"
 #include "libc/calls/internal.h"
 #include "libc/calls/strace.internal.h"
+#include "libc/intrin/pthread.h"
 #include "libc/intrin/spinlock.h"
 #include "libc/nt/runtime.h"
 #include "libc/sysv/consts/o.h"
@@ -26,11 +27,7 @@
 STATIC_YOINK("_init_g_fds");
 
 struct Fds g_fds;
-_Alignas(64) int __fds_lock_obj;
-
-void __fds_lock(void) {
-  _spinlock(&__fds_lock_obj);
-}
+pthread_mutex_t __fds_lock_obj;
 
 textstartup void InitializeFileDescriptors(void) {
   struct Fds *fds;

libc/intrin/pthread.c

@@ -0,0 +1,28 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2021 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/errno.h"
+#include "libc/intrin/pthread.h"
+
+int pthread_cancel(pthread_t thread) {
+  return ESRCH;
+}
+
+void *__tls_get_addr(size_t v[2]) {
+  return NULL;
+}

libc/intrin/pthread.h

@@ -0,0 +1,102 @@
+#ifndef COSMOPOLITAN_LIBC_RUNTIME_PTHREAD_H_
+#define COSMOPOLITAN_LIBC_RUNTIME_PTHREAD_H_
+#include "libc/calls/struct/timespec.h"
+#if !(__ASSEMBLER__ + __LINKER__ + 0)
+COSMOPOLITAN_C_START_
+
+#define PTHREAD_ONCE_INIT 0
+
+#define PTHREAD_MUTEX_NORMAL     0
+#define PTHREAD_MUTEX_DEFAULT    0
+#define PTHREAD_MUTEX_RECURSIVE  1
+#define PTHREAD_MUTEX_ERRORCHECK 2
+#define PTHREAD_MUTEX_STALLED    0
+#define PTHREAD_MUTEX_ROBUST     1
+
+/* clang-format off */
+#define PTHREAD_MUTEX_INITIALIZER {0}
+#define PTHREAD_RWLOCK_INITIALIZER {{{0}}}
+#define PTHREAD_COND_INITIALIZER {{{0}}}
+/* clang-format on */
+
+typedef unsigned long *pthread_t;
+typedef int pthread_once_t;
+
+typedef struct {
+  int reent;
+  int owner;
+  int waits;
+} pthread_mutex_t;
+
+typedef struct {
+  unsigned __attr;
+} pthread_mutexattr_t;
+
+typedef struct {
+  unsigned __attr;
+} pthread_condattr_t;
+
+typedef struct {
+  unsigned __attr[2];
+} pthread_rwlockattr_t;
+
+typedef struct {
+  union {
+    int __i[9];
+    volatile int __vi[9];
+    unsigned __s[9];
+  } __u;
+} pthread_attr_t;
+
+typedef struct {
+  union {
+    int __i[12];
+    volatile int __vi[12];
+    void *__p[12];
+  } __u;
+} pthread_cond_t;
+
+typedef struct {
+  union {
+    int __i[8];
+    volatile int __vi[8];
+    void *__p[8];
+  } __u;
+} pthread_rwlock_t;
+
+wontreturn void pthread_exit(void *);
+pureconst pthread_t pthread_self(void);
+int pthread_create(pthread_t *, const pthread_attr_t *, void *(*)(void *),
+                   void *);
+int pthread_detach(pthread_t);
+int pthread_join(pthread_t, void **);
+int pthread_equal(pthread_t, pthread_t);
+int pthread_once(pthread_once_t *, void (*)(void));
+int pthread_mutex_init(pthread_mutex_t *, const pthread_mutexattr_t *);
+int pthread_mutex_lock(pthread_mutex_t *);
+int pthread_mutex_unlock(pthread_mutex_t *);
+int pthread_mutex_trylock(pthread_mutex_t *);
+int pthread_mutex_timedlock(pthread_mutex_t *, const struct timespec *);
+int pthread_mutex_destroy(pthread_mutex_t *);
+int pthread_mutex_consistent(pthread_mutex_t *);
+int pthread_cond_init(pthread_cond_t *, const pthread_condattr_t *);
+int pthread_cond_destroy(pthread_cond_t *);
+int pthread_cond_wait(pthread_cond_t *, pthread_mutex_t *);
+int pthread_cond_timedwait(pthread_cond_t *, pthread_mutex_t *,
+                           const struct timespec *);
+int pthread_cond_broadcast(pthread_cond_t *);
+int pthread_cancel(pthread_t);
+int pthread_cond_signal(pthread_cond_t *);
+int pthread_rwlock_init(pthread_rwlock_t *, const pthread_rwlockattr_t *);
+int pthread_rwlock_destroy(pthread_rwlock_t *);
+int pthread_rwlock_rdlock(pthread_rwlock_t *);
+int pthread_rwlock_tryrdlock(pthread_rwlock_t *);
+int pthread_rwlock_timedrdlock(pthread_rwlock_t *, const struct timespec *);
+int pthread_rwlock_wrlock(pthread_rwlock_t *);
+int pthread_rwlock_trywrlock(pthread_rwlock_t *);
+int pthread_rwlock_timedwrlock(pthread_rwlock_t *, const struct timespec *);
+int pthread_rwlock_unlock(pthread_rwlock_t *);
+
+COSMOPOLITAN_C_END_
+#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
+#endif /* COSMOPOLITAN_LIBC_RUNTIME_PTHREAD_H_ */

libc/intrin/pthread_mutex_lock.c

@@ -0,0 +1,51 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/bits/atomic.h"
+#include "libc/calls/calls.h"
+#include "libc/intrin/lockcmpxchgp.h"
+#include "libc/intrin/pthread.h"
+#include "libc/nexgen32e/threaded.h"
+#include "libc/sysv/consts/futex.h"
+
+/**
+ * Acquires mutex.
+ */
+int pthread_mutex_lock(pthread_mutex_t *mutex) {
+  int me, owner;
+  unsigned tries;
+  if (__threaded) {
+    for (tries = 0, me = gettid();;) {
+      owner = 0;
+      if (_lockcmpxchgp(&mutex->owner, &owner, me) || owner == me) {
+        break;
+      }
+      atomic_fetch_add(&mutex->waits, +1);
+      if (!IsLinux() || futex((void *)&mutex->owner, FUTEX_WAIT, owner, 0, 0)) {
+        if (++tries & 7) {
+          __builtin_ia32_pause();
+        } else {
+          sched_yield();
+        }
+      }
+      atomic_fetch_add(&mutex->waits, -1);
+    }
+  }
+  ++mutex->reent;
+  return 0;
+}

libc/intrin/pthread_mutex_trylock.c

@@ -0,0 +1,43 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/calls.h"
+#include "libc/errno.h"
+#include "libc/intrin/lockcmpxchgp.h"
+#include "libc/intrin/pthread.h"
+#include "libc/nexgen32e/threaded.h"
+
+/**
+ * Tries to acquire mutex.
+ */
+int pthread_mutex_trylock(pthread_mutex_t *mutex) {
+  int rc, me, owner = 0;
+  if (__threaded) {
+    me = gettid();
+    if (!_lockcmpxchgp(&mutex->owner, &owner, me) && owner == me) {
+      rc = 0;
+      ++mutex->reent;
+    } else {
+      rc = EBUSY;
+    }
+  } else {
+    rc = 0;
+    ++mutex->reent;
+  }
+  return rc;
+}

libc/intrin/pthread_mutex_unlock.c

@@ -0,0 +1,46 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/assert.h"
+#include "libc/bits/atomic.h"
+#include "libc/calls/calls.h"
+#include "libc/dce.h"
+#include "libc/intrin/pthread.h"
+#include "libc/nexgen32e/threaded.h"
+#include "libc/sysv/consts/futex.h"
+
+/**
+ * Releases mutex.
+ */
+int pthread_mutex_unlock(pthread_mutex_t *mutex) {
+  int owner;
+  bool shouldunlock;
+  assert(mutex->reent > 0);
+  shouldunlock = --mutex->reent <= 0;
+  if (__threaded) {
+    assert(mutex->owner == gettid());
+    if (shouldunlock) {
+      atomic_store_explicit(&mutex->owner, 0, memory_order_relaxed);
+      if (IsLinux() &&
+          atomic_load_explicit(&mutex->waits, memory_order_acquire)) {
+        futex((void *)&mutex->owner, FUTEX_WAKE, 1, 0, 0);
+      }
+    }
+  }
+  return 0;
+}

libc/intrin/pthread_once.c

@@ -0,0 +1,48 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/bits/atomic.h"
+#include "libc/calls/calls.h"
+#include "libc/intrin/pthread.h"
+
+int pthread_once(pthread_once_t *once, void init(void)) {
+  int x;
+  unsigned tries;
+  switch ((x = atomic_load(once))) {
+    case 0:
+      if (atomic_compare_exchange_strong(once, &x, 1)) {
+        init();
+        atomic_store(once, 2);
+        break;
+      }
+      // fallthrough
+    case 1:
+      tries = 0;
+      do {
+        if (++tries & 7) {
+          __builtin_ia32_pause();
+        } else {
+          sched_yield();
+        }
+      } while (atomic_load(once) == 1);
+      break;
+    default:
+      break;
+  }
+  return 0;
+}