Make pthread mutexes more scalable

pthread_mutex_lock() now uses a better algorithm which goes much faster
in multithreaded environments that have lock contention. This comes at
the cost of adding some fixed-cost overhead to mutex invocations. That
doesn't matter for Cosmopolitan because our core libraries all encode
locking operations as NOP instructions when in single-threaded mode.
Overhead only applies starting the moment you first call clone().

libc/calls/clock_gettime-mono.c

@@ -16,42 +16,37 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/intrin/likely.h"
 #include "libc/calls/clock_gettime.internal.h"
-#include "libc/intrin/spinlock.h"
+#include "libc/calls/struct/timespec.h"
+#include "libc/intrin/pthread.h"
 #include "libc/nexgen32e/rdtsc.h"
-#include "libc/nexgen32e/threaded.h"
 #include "libc/nexgen32e/x86feature.h"
 #include "libc/sysv/consts/clock.h"
 #include "libc/sysv/errfuns.h"
 #include "libc/time/clockstonanos.internal.h"
-#include "libc/time/time.h"
 
 static struct {
-  bool once;
-  char lock;
+  pthread_once_t once;
   uint64_t base;
   struct timespec mono;
 } g_mono;
 
+static void sys_clock_gettime_mono_init(void) {
+  clock_gettime(CLOCK_REALTIME, &g_mono.mono);
+  g_mono.base = rdtsc();
+  g_mono.once = true;
+}
+
 int sys_clock_gettime_mono(struct timespec *ts) {
   // this routine stops being monotonic after 194 years of uptime
   uint64_t nanos;
   struct timespec res;
   if (X86_HAVE(INVTSC)) {
-    if (__threaded) {
-      _spinlock(&g_mono.lock);
-    }
-    if (UNLIKELY(!g_mono.once)) {
-      clock_gettime(CLOCK_REALTIME, &g_mono.mono);
-      g_mono.base = rdtsc();
-      g_mono.once = true;
-    }
+    pthread_once(&g_mono.once, sys_clock_gettime_mono_init);
     nanos = ClocksToNanos(rdtsc(), g_mono.base);
     res = g_mono.mono;
     res.tv_sec += nanos / 1000000000;
     res.tv_nsec += nanos % 1000000000;
-    _spunlock(&g_mono.lock);
     *ts = res;
     return 0;
   } else {

libc/calls/close.c

@@ -24,7 +24,6 @@
 #include "libc/calls/syscall-nt.internal.h"
 #include "libc/calls/syscall-sysv.internal.h"
 #include "libc/dce.h"
-#include "libc/intrin/spinlock.h"
 #include "libc/intrin/weaken.h"
 #include "libc/sock/syscall_fd.internal.h"
 #include "libc/sysv/errfuns.h"

libc/calls/dup-nt.c

@@ -20,7 +20,6 @@
 #include "libc/calls/internal.h"
 #include "libc/calls/state.internal.h"
 #include "libc/calls/syscall_support-nt.internal.h"
-#include "libc/intrin/spinlock.h"
 #include "libc/intrin/weaken.h"
 #include "libc/nt/files.h"
 #include "libc/nt/runtime.h"

libc/calls/fadvise-nt.c

@@ -19,7 +19,6 @@
 #include "libc/calls/internal.h"
 #include "libc/calls/state.internal.h"
 #include "libc/calls/syscall_support-nt.internal.h"
-#include "libc/intrin/spinlock.h"
 #include "libc/nt/createfile.h"
 #include "libc/nt/enum/fileflagandattributes.h"
 #include "libc/nt/files.h"

libc/calls/getloadavg-nt.c

@@ -20,7 +20,7 @@
 #include "libc/calls/syscall_support-nt.internal.h"
 #include "libc/dce.h"
 #include "libc/fmt/conv.h"
-#include "libc/intrin/spinlock.h"
+#include "libc/intrin/pthread.h"
 #include "libc/macros.internal.h"
 #include "libc/nt/accounting.h"
 #include "libc/runtime/sysconf.h"
@@ -29,14 +29,14 @@
 
 static int cpus;
 static double load;
-_Alignas(64) static char lock;
+static pthread_spinlock_t lock;
 static struct NtFileTime idle1, kern1, user1;
 
 textwindows int sys_getloadavg_nt(double *a, int n) {
   int i, rc;
   uint64_t elapsed, used;
   struct NtFileTime idle, kern, user;
-  _spinlock(&lock);
+  pthread_spin_lock(&lock);
   if (GetSystemTimes(&idle, &kern, &user)) {
     elapsed = (FT(kern) - FT(kern1)) + (FT(user) - FT(user1));
     if (elapsed) {
@@ -52,7 +52,7 @@ textwindows int sys_getloadavg_nt(double *a, int n) {
   } else {
     rc = __winerr();
   }
-  _spunlock(&lock);
+  pthread_spin_unlock(&lock);
   return rc;
 }
 

libc/calls/getrusage-nt.c

@@ -22,7 +22,6 @@
 #include "libc/calls/struct/rusage.h"
 #include "libc/calls/syscall_support-nt.internal.h"
 #include "libc/fmt/conv.h"
-#include "libc/intrin/spinlock.h"
 #include "libc/nt/accounting.h"
 #include "libc/nt/process.h"
 #include "libc/nt/runtime.h"

libc/calls/ioctl_tiocgwinsz-nt.c

@@ -25,7 +25,6 @@
 #include "libc/calls/struct/winsize.h"
 #include "libc/calls/syscall_support-nt.internal.h"
 #include "libc/errno.h"
-#include "libc/intrin/spinlock.h"
 #include "libc/intrin/weaken.h"
 #include "libc/log/log.h"
 #include "libc/nt/console.h"

libc/calls/open-nt.c

@@ -22,7 +22,6 @@
 #include "libc/calls/state.internal.h"
 #include "libc/calls/syscall-nt.internal.h"
 #include "libc/calls/syscall_support-nt.internal.h"
-#include "libc/intrin/spinlock.h"
 #include "libc/nt/createfile.h"
 #include "libc/nt/enum/filetype.h"
 #include "libc/nt/files.h"

libc/calls/pipe-nt.c

@@ -19,7 +19,6 @@
 #include "libc/calls/internal.h"
 #include "libc/calls/state.internal.h"
 #include "libc/calls/syscall_support-nt.internal.h"
-#include "libc/intrin/spinlock.h"
 #include "libc/nt/createfile.h"
 #include "libc/nt/enum/accessmask.h"
 #include "libc/nt/enum/creationdisposition.h"

libc/calls/poll-nt.c

@@ -24,7 +24,6 @@
 #include "libc/calls/struct/sigaction.h"
 #include "libc/errno.h"
 #include "libc/intrin/bits.h"
-#include "libc/intrin/spinlock.h"
 #include "libc/intrin/weaken.h"
 #include "libc/macros.internal.h"
 #include "libc/mem/mem.h"

libc/calls/printfds.c

@@ -20,7 +20,6 @@
 #include "libc/calls/state.internal.h"
 #include "libc/calls/struct/fd.internal.h"
 #include "libc/intrin/kprintf.h"
-#include "libc/intrin/spinlock.h"
 
 static const char *__fdkind2str(int x) {
   switch (x) {

libc/calls/reservefd.c

@@ -24,7 +24,6 @@
 #include "libc/calls/struct/sigset.h"
 #include "libc/dce.h"
 #include "libc/intrin/cmpxchg.h"
-#include "libc/intrin/spinlock.h"
 #include "libc/intrin/weaken.h"
 #include "libc/macros.internal.h"
 #include "libc/mem/mem.h"

libc/calls/samplepids.c

@@ -19,7 +19,7 @@
 #include "libc/assert.h"
 #include "libc/calls/internal.h"
 #include "libc/dce.h"
-#include "libc/intrin/spinlock.h"
+#include "libc/intrin/pthread.h"
 #include "libc/nexgen32e/threaded.h"
 #include "libc/stdio/lcg.internal.h"
 
@@ -35,12 +35,12 @@
 textwindows int __sample_pids(int pids[hasatleast 64],
                               int64_t handles[hasatleast 64],
                               bool exploratory) {
-  static char lock;
   static uint64_t rando = 1;
+  static pthread_spinlock_t lock;
   uint32_t i, j, base, count;
-  if (__threaded) _spinlock(&lock);
+  if (__threaded) pthread_spin_lock(&lock);
   base = KnuthLinearCongruentialGenerator(&rando) >> 32;
-  _spunlock(&lock);
+  pthread_spin_unlock(&lock);
   for (count = i = 0; i < g_fds.n; ++i) {
     j = (base + i) % g_fds.n;
     if (g_fds.p[j].kind == kFdProcess && (!exploratory || !g_fds.p[j].zombie)) {

libc/calls/sig.c

@@ -18,7 +18,6 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/calls/sig.internal.h"
 #include "libc/calls/state.internal.h"
-#include "libc/intrin/spinlock.h"
 #include "libc/macros.internal.h"
 #include "libc/sysv/consts/sig.h"
 #include "libc/sysv/errfuns.h"

libc/calls/sig2.c

@@ -23,7 +23,6 @@
 #include "libc/calls/struct/siginfo.h"
 #include "libc/intrin/cmpxchg.h"
 #include "libc/intrin/lockcmpxchg.h"
-#include "libc/intrin/spinlock.h"
 #include "libc/log/libfatal.internal.h"
 #include "libc/macros.internal.h"
 #include "libc/runtime/internal.h"

libc/calls/sigchld-nt.c

@@ -25,7 +25,6 @@
 #include "libc/calls/struct/siginfo.h"
 #include "libc/calls/syscall_support-nt.internal.h"
 #include "libc/dce.h"
-#include "libc/intrin/spinlock.h"
 #include "libc/nt/enum/wait.h"
 #include "libc/nt/runtime.h"
 #include "libc/nt/synchronization.h"

libc/calls/symlinkat-nt.c

@@ -20,7 +20,6 @@
 #include "libc/calls/syscall_support-nt.internal.h"
 #include "libc/errno.h"
 #include "libc/intrin/once.h"
-#include "libc/intrin/spinlock.h"
 #include "libc/nt/enum/accessmask.h"
 #include "libc/nt/enum/fileflagandattributes.h"
 #include "libc/nt/enum/symboliclink.h"

libc/calls/wait4-nt.c

@@ -25,7 +25,6 @@
 #include "libc/calls/struct/rusage.h"
 #include "libc/calls/syscall_support-nt.internal.h"
 #include "libc/fmt/conv.h"
-#include "libc/intrin/spinlock.h"
 #include "libc/macros.internal.h"
 #include "libc/nt/accounting.h"
 #include "libc/nt/enum/accessmask.h"

libc/intrin/futex_wait.c

@@ -23,6 +23,7 @@
 #include "libc/errno.h"
 #include "libc/intrin/describeflags.internal.h"
 #include "libc/intrin/futex.internal.h"
+#include "libc/intrin/pthread.h"
 #include "libc/sysv/consts/futex.h"
 
 int _futex(void *, int, int, struct timespec *) hidden;
@@ -30,17 +31,21 @@ int _futex(void *, int, int, struct timespec *) hidden;
 static dontinline int _futex_wait_impl(void *addr, int expect,
                                        struct timespec *timeout, int private) {
   int op, ax;
-  op = FUTEX_WAIT | private;
-  ax = _futex(addr, op, expect, timeout);
-  if (SupportsLinux() && private && ax == -ENOSYS) {
-    // RHEL5 doesn't support FUTEX_PRIVATE_FLAG
-    op = FUTEX_WAIT;
+  if (IsLinux() || IsOpenbsd()) {
+    op = FUTEX_WAIT | private;
     ax = _futex(addr, op, expect, timeout);
+    if (SupportsLinux() && private && ax == -ENOSYS) {
+      // RHEL5 doesn't support FUTEX_PRIVATE_FLAG
+      op = FUTEX_WAIT;
+      ax = _futex(addr, op, expect, timeout);
+    }
+    if (IsOpenbsd() && ax > 0) ax = -ax;  // yes openbsd does this w/o cf
+    STRACE("futex(%t, %s, %d, %s) → %s", addr, DescribeFutexOp(op), expect,
+           DescribeTimespec(0, timeout), DescribeFutexResult(ax));
+    return ax;
+  } else {
+    return pthread_yield();
   }
-  if (IsOpenbsd() && ax > 0) ax = -ax;  // yes openbsd does this w/o cf
-  STRACE("futex(%t, %s, %d, %s) → %s", addr, DescribeFutexOp(op), expect,
-         DescribeTimespec(0, timeout), DescribeFutexResult(ax));
-  return ax;
 }
 
 int _futex_wait_public(void *addr, int expect, struct timespec *timeout) {

libc/intrin/futex_wake.c

@@ -27,16 +27,20 @@ int _futex(void *, int, int) hidden;
 
 static dontinline int _futex_wake_impl(void *addr, int count, int private) {
   int op, ax;
-  op = FUTEX_WAKE | private;
-  ax = _futex(addr, op, count);
-  if (SupportsLinux() && private && ax == -ENOSYS) {
-    // RHEL5 doesn't support FUTEX_PRIVATE_FLAG
-    op = FUTEX_WAKE;
+  if (IsLinux() || IsOpenbsd()) {
+    op = FUTEX_WAKE | private;
     ax = _futex(addr, op, count);
+    if (SupportsLinux() && private && ax == -ENOSYS) {
+      // RHEL5 doesn't support FUTEX_PRIVATE_FLAG
+      op = FUTEX_WAKE;
+      ax = _futex(addr, op, count);
+    }
+    STRACE("futex(%t, %s, %d) → %s", addr, DescribeFutexOp(op), count,
+           DescribeFutexResult(ax));
+    return ax;
+  } else {
+    return 0;
   }
-  STRACE("futex(%t, %s, %d) → %s", addr, DescribeFutexOp(op), count,
-         DescribeFutexResult(ax));
-  return ax;
 }
 
 int _futex_wake_public(void *addr, int count) {

libc/intrin/pthread.h

@@ -111,6 +111,8 @@ int pthread_mutexattr_init(pthread_mutexattr_t *);
 int pthread_mutexattr_destroy(pthread_mutexattr_t *);
 int pthread_mutexattr_gettype(const pthread_mutexattr_t *, int *);
 int pthread_mutexattr_settype(pthread_mutexattr_t *, int);
+int pthread_mutexattr_setpshared(pthread_mutexattr_t *, int);
+int pthread_mutexattr_getpshared(const pthread_mutexattr_t *, int *);
 int pthread_mutex_init(pthread_mutex_t *, const pthread_mutexattr_t *);
 int pthread_mutex_lock(pthread_mutex_t *);
 int pthread_mutex_unlock(pthread_mutex_t *);
@@ -208,22 +210,6 @@ extern const errno_t EBUSY;
   })
 #endif
 
-#if (__GNUC__ + 0) * 100 + (__GNUC_MINOR__ + 0) >= 407
-#define pthread_mutex_lock(mutex)                              \
-  (((mutex)->attr == PTHREAD_MUTEX_NORMAL &&                   \
-    !__atomic_load_n(&(mutex)->lock, __ATOMIC_RELAXED) &&      \
-    !__atomic_exchange_n(&(mutex)->lock, 1, __ATOMIC_SEQ_CST)) \
-       ? 0                                                     \
-       : pthread_mutex_lock(mutex))
-#define pthread_mutex_unlock(mutex)                              \
-  ((mutex)->attr == PTHREAD_MUTEX_NORMAL                         \
-       ? (__atomic_store_n(&(mutex)->lock, 0, __ATOMIC_RELAXED), \
-          (__atomic_load_n(&(mutex)->waits, __ATOMIC_RELAXED) && \
-           _pthread_mutex_wake(mutex)),                          \
-          0)                                                     \
-       : pthread_mutex_unlock(mutex))
-#endif
-
 #define pthread_condattr_init(pAttr)                 (*(pAttr) = PTHREAD_PROCESS_DEFAULT, 0)
 #define pthread_condattr_destroy(pAttr)              (*(pAttr) = 0)
 #define pthread_condattr_getpshared(pAttr, pPshared) (*(pPshared) = *(pAttr), 0)

libc/intrin/pthread_mutex_lock.c

@@ -50,11 +50,6 @@ static int pthread_mutex_lock_spin(pthread_mutex_t *mutex, int expect,
 /**
  * Locks mutex.
  *
- *     spin                l:   181,570c    58,646ns
- *     mutex normal        l:   297,965c    96,241ns
- *     mutex recursive     l: 1,112,166c   359,223ns
- *     mutex errorcheck    l: 1,449,723c   468,252ns
- *
  * Here's an example of using a normal mutex:
  *
  *     pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
@@ -90,19 +85,40 @@ static int pthread_mutex_lock_spin(pthread_mutex_t *mutex, int expect,
  *     // do work...
  *     pthread_mutex_unlock(&lock);
  *
+ * Microbenchmarks for single-threaded lock + unlock:
+ *
+ *     pthread_spinlock_t          :    12c (    4ns)
+ *     PTHREAD_MUTEX_NORMAL        :    37c (   12ns)
+ *     PTHREAD_MUTEX_RECURSIVE     :    22c (    7ns)
+ *     PTHREAD_MUTEX_ERRORCHECK    :    27c (    9ns)
+ *
+ * Microbenchmarks for multi-threaded lock + unlock:
+ *
+ *     pthread_spinlock_t          : 6,162c (1,990ns)
+ *     PTHREAD_MUTEX_NORMAL        :   780c (  252ns)
+ *     PTHREAD_MUTEX_RECURSIVE     : 1,047c (  338ns)
+ *     PTHREAD_MUTEX_ERRORCHECK    : 1,044c (  337ns)
+ *
  * @return 0 on success, or error number on failure
  * @see pthread_spin_lock
  */
-int(pthread_mutex_lock)(pthread_mutex_t *mutex) {
-  int me, owner, tries;
+int pthread_mutex_lock(pthread_mutex_t *mutex) {
+  int c, me, owner, tries;
   switch (mutex->attr) {
     case PTHREAD_MUTEX_NORMAL:
-      for (tries = 0;;) {
-        if (!atomic_load_explicit(&mutex->lock, memory_order_relaxed) &&
-            !atomic_exchange_explicit(&mutex->lock, 1, memory_order_acquire)) {
-          break;
+      // From Futexes Are Tricky Version 1.1 § Mutex, Take 3;
+      // Ulrich Drepper, Red Hat Incorporated, June 27, 2004.
+      c = 0;
+      if (!atomic_compare_exchange_strong_explicit(&mutex->lock, &c, 1,
+                                                   memory_order_acquire,
+                                                   memory_order_relaxed)) {
+        if (c != 2) {
+          c = atomic_exchange_explicit(&mutex->lock, 2, memory_order_acquire);
+        }
+        while (c) {
+          _futex_wait_private(&mutex->lock, 2, 0);
+          c = atomic_exchange_explicit(&mutex->lock, 2, memory_order_acquire);
         }
-        tries = pthread_mutex_lock_spin(mutex, 1, tries);
       }
       return 0;
     case PTHREAD_MUTEX_RECURSIVE:

libc/intrin/pthread_mutex_trylock.c

@@ -16,25 +16,53 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/intrin/atomic.h"
+#include "libc/assert.h"
 #include "libc/calls/calls.h"
 #include "libc/errno.h"
+#include "libc/intrin/atomic.h"
 #include "libc/intrin/pthread.h"
-#include "libc/nexgen32e/threaded.h"
 
 /**
- * Tries to acquire mutex.
+ * Locks mutex if it isn't locked already.
+ *
+ * Unlike pthread_mutex_lock() this function won't block and instead
+ * returns an error immediately if the lock couldn't be acquired.
+ *
+ * @return 0 on success, or errno on error
+ * @raise EBUSY if lock is already held
+ * @raise ENOTRECOVERABLE if `mutex` is corrupted
  */
 int pthread_mutex_trylock(pthread_mutex_t *mutex) {
-  int rc, me, owner;
-  me = gettid();
-  owner = 0;
-  if (!atomic_compare_exchange_strong(&mutex->lock, &owner, me) &&
-      owner == me) {
-    rc = 0;
-    ++mutex->reent;
-  } else {
-    rc = EBUSY;
+  int c, me, owner;
+  switch (mutex->attr) {
+    case PTHREAD_MUTEX_NORMAL:
+      c = 0;
+      if (atomic_compare_exchange_strong_explicit(&mutex->lock, &c, 1,
+                                                  memory_order_acquire,
+                                                  memory_order_relaxed)) {
+        return 0;
+      } else {
+        return EBUSY;
+      }
+    case PTHREAD_MUTEX_RECURSIVE:
+    case PTHREAD_MUTEX_ERRORCHECK:
+      owner = 0;
+      me = gettid();
+      if (!atomic_compare_exchange_strong_explicit(&mutex->lock, &owner, me,
+                                                   memory_order_acquire,
+                                                   memory_order_relaxed)) {
+        if (owner == me) {
+          if (mutex->attr == PTHREAD_MUTEX_ERRORCHECK) {
+            return EBUSY;
+          }
+        } else {
+          return EBUSY;
+        }
+      }
+      ++mutex->reent;
+      return 0;
+    default:
+      assert(!"badlock");
+      return ENOTRECOVERABLE;
   }
-  return rc;
 }

libc/intrin/pthread_mutex_unlock.c

@@ -20,6 +20,7 @@
 #include "libc/calls/calls.h"
 #include "libc/errno.h"
 #include "libc/intrin/atomic.h"
+#include "libc/intrin/futex.internal.h"
 #include "libc/intrin/pthread.h"
 
 /**
@@ -28,9 +29,18 @@
  * @return 0 on success or error number on failure
  * @raises EPERM if in error check mode and not owned by caller
  */
-int(pthread_mutex_unlock)(pthread_mutex_t *mutex) {
-  int me, owner;
+int pthread_mutex_unlock(pthread_mutex_t *mutex) {
+  int c, me, owner;
   switch (mutex->attr) {
+    case PTHREAD_MUTEX_NORMAL:
+      // From Futexes Are Tricky Version 1.1 § Mutex, Take 3;
+      // Ulrich Drepper, Red Hat Incorporated, June 27, 2004.
+      if ((c = atomic_fetch_sub_explicit(&mutex->lock, 1,
+                                         memory_order_release)) != 1) {
+        atomic_store_explicit(&mutex->lock, 0, memory_order_release);
+        _futex_wake_private(&mutex->lock, 1);
+      }
+      return 0;
     case PTHREAD_MUTEX_ERRORCHECK:
       me = gettid();
       owner = atomic_load_explicit(&mutex->lock, memory_order_relaxed);
@@ -41,8 +51,6 @@ int(pthread_mutex_unlock)(pthread_mutex_t *mutex) {
       // fallthrough
     case PTHREAD_MUTEX_RECURSIVE:
       if (--mutex->reent) return 0;
-      // fallthrough
-    case PTHREAD_MUTEX_NORMAL:
       atomic_store_explicit(&mutex->lock, 0, memory_order_relaxed);
       if (atomic_load_explicit(&mutex->waits, memory_order_relaxed) > 0) {
         _pthread_mutex_wake(mutex);

libc/intrin/pthread_mutexattr_getpshared.c

@@ -0,0 +1,32 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/intrin/pthread.h"
+
+/**
+ * Gets mutex process sharing.
+ *
+ * @param pshared is set to one of the following
+ *     - `PTHREAD_PROCESS_SHARED`
+ *     - `PTHREAD_PROCESS_PRIVATE`
+ * @return 0 on success, or error on failure
+ */
+int(pthread_mutexattr_getpshared)(const pthread_mutexattr_t *attr,
+                                  int *pshared) {
+  return pthread_mutexattr_getpshared(attr, pshared);
+}

libc/intrin/pthread_mutexattr_setpshared.c

@@ -0,0 +1,40 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/errno.h"
+#include "libc/intrin/pthread.h"
+
+/**
+ * Sets mutex process sharing.
+ *
+ * @param pshared can be one of
+ *     - `PTHREAD_PROCESS_SHARED`
+ *     - `PTHREAD_PROCESS_PRIVATE`
+ * @return 0 on success, or error on failure
+ * @raises EINVAL if `pshared` is invalid
+ */
+int pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared) {
+  switch (pshared) {
+    case PTHREAD_PROCESS_SHARED:
+    case PTHREAD_PROCESS_PRIVATE:
+      *attr = pshared;
+      return 0;
+    default:
+      return EINVAL;
+  }
+}

libc/intrin/pthread_spin_trylock.c

@@ -23,7 +23,6 @@
  *
  * Unlike pthread_spin_lock() this function won't block, and instead
  * returns an error immediately if the spinlock couldn't be acquired
- * furthermore this function doesn't define any error conditions now
  *
  * @return 0 on success, or errno on error
  * @raise EBUSY if lock is already held

test/libc/intrin/pthread_mutex_lock2_test.c

@@ -0,0 +1,199 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/calls.h"
+#include "libc/calls/struct/timespec.h"
+#include "libc/intrin/atomic.h"
+#include "libc/intrin/pthread.h"
+#include "libc/mem/mem.h"
+#include "libc/runtime/gc.internal.h"
+#include "libc/testlib/ezbench.h"
+#include "libc/testlib/testlib.h"
+#include "libc/thread/posixthread.internal.h"
+
+int THREADS = 16;
+int ITERATIONS = 100;
+
+int count;
+_Atomic(int) started;
+_Atomic(int) finished;
+pthread_mutex_t lock;
+pthread_mutexattr_t attr;
+
+FIXTURE(pthread_mutex_lock, normal) {
+  ASSERT_EQ(0, pthread_mutexattr_init(&attr));
+  ASSERT_EQ(0, pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL));
+  ASSERT_EQ(0, pthread_mutex_init(&lock, &attr));
+  ASSERT_EQ(0, pthread_mutexattr_destroy(&attr));
+}
+
+FIXTURE(pthread_mutex_lock, recursive) {
+  ASSERT_EQ(0, pthread_mutexattr_init(&attr));
+  ASSERT_EQ(0, pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE));
+  ASSERT_EQ(0, pthread_mutex_init(&lock, &attr));
+  ASSERT_EQ(0, pthread_mutexattr_destroy(&attr));
+}
+
+FIXTURE(pthread_mutex_lock, errorcheck) {
+  ASSERT_EQ(0, pthread_mutexattr_init(&attr));
+  ASSERT_EQ(0, pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK));
+  ASSERT_EQ(0, pthread_mutex_init(&lock, &attr));
+  ASSERT_EQ(0, pthread_mutexattr_destroy(&attr));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// TESTS
+
+int MutexWorker(void *p, int tid) {
+  int i;
+  ++started;
+  for (i = 0; i < ITERATIONS; ++i) {
+    pthread_mutex_lock(&lock);
+    ++count;
+    sched_yield();
+    pthread_mutex_unlock(&lock);
+  }
+  ++finished;
+  return 0;
+}
+
+TEST(pthread_mutex_lock, contention) {
+  int i;
+  struct spawn *th = gc(malloc(sizeof(struct spawn) * THREADS));
+  pthread_mutexattr_init(&attr);
+  pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL);
+  pthread_mutex_init(&lock, &attr);
+  pthread_mutexattr_destroy(&attr);
+  count = 0;
+  started = 0;
+  finished = 0;
+  for (i = 0; i < THREADS; ++i) {
+    ASSERT_SYS(0, 0, _spawn(MutexWorker, (void *)(intptr_t)i, th + i));
+  }
+  for (i = 0; i < THREADS; ++i) {
+    ASSERT_SYS(0, 0, _join(th + i));
+  }
+  EXPECT_EQ(THREADS, started);
+  EXPECT_EQ(THREADS, finished);
+  EXPECT_EQ(THREADS * ITERATIONS, count);
+  EXPECT_EQ(0, pthread_mutex_destroy(&lock));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// BENCHMARKS
+
+void BenchSpinUnspin(pthread_spinlock_t *s) {
+  pthread_spin_lock(s);
+  pthread_spin_unlock(s);
+}
+
+void BenchLockUnlock(pthread_mutex_t *m) {
+  pthread_mutex_lock(m);
+  pthread_mutex_unlock(m);
+}
+
+BENCH(pthread_mutex_lock, bench_uncontended) {
+  {
+    pthread_spinlock_t s = 0;
+    EZBENCH2("spin 1x", donothing, BenchSpinUnspin(&s));
+  }
+  {
+    pthread_mutex_t m = {PTHREAD_MUTEX_NORMAL};
+    EZBENCH2("normal 1x", donothing, BenchLockUnlock(&m));
+  }
+  {
+    pthread_mutex_t m = {PTHREAD_MUTEX_RECURSIVE};
+    EZBENCH2("recursive 1x", donothing, BenchLockUnlock(&m));
+  }
+  {
+    pthread_mutex_t m = {PTHREAD_MUTEX_ERRORCHECK};
+    EZBENCH2("errorcheck 1x", donothing, BenchLockUnlock(&m));
+  }
+}
+
+struct SpinContentionArgs {
+  pthread_spinlock_t *spin;
+  _Atomic(char) done;
+  _Atomic(char) ready;
+};
+
+int SpinContentionWorker(void *arg, int tid) {
+  struct SpinContentionArgs *a = arg;
+  while (!atomic_load_explicit(&a->done, memory_order_relaxed)) {
+    pthread_spin_lock(a->spin);
+    atomic_store_explicit(&a->ready, 1, memory_order_relaxed);
+    pthread_spin_unlock(a->spin);
+  }
+  return 0;
+}
+
+struct MutexContentionArgs {
+  pthread_mutex_t *mutex;
+  _Atomic(char) done;
+  _Atomic(char) ready;
+};
+
+int MutexContentionWorker(void *arg, int tid) {
+  struct MutexContentionArgs *a = arg;
+  while (!atomic_load_explicit(&a->done, memory_order_relaxed)) {
+    pthread_mutex_lock(a->mutex);
+    atomic_store_explicit(&a->ready, 1, memory_order_relaxed);
+    pthread_mutex_unlock(a->mutex);
+  }
+  return 0;
+}
+
+BENCH(pthread_mutex_lock, bench_contended) {
+  struct spawn t;
+  {
+    pthread_spinlock_t s = 0;
+    struct SpinContentionArgs a = {&s};
+    _spawn(SpinContentionWorker, &a, &t);
+    while (!a.ready) sched_yield();
+    EZBENCH2("spin 2x", donothing, BenchSpinUnspin(&s));
+    a.done = true;
+    _join(&t);
+  }
+  {
+    pthread_mutex_t m = {PTHREAD_MUTEX_NORMAL};
+    struct MutexContentionArgs a = {&m};
+    _spawn(MutexContentionWorker, &a, &t);
+    while (!a.ready) sched_yield();
+    EZBENCH2("normal 2x", donothing, BenchLockUnlock(&m));
+    a.done = true;
+    _join(&t);
+  }
+  {
+    pthread_mutex_t m = {PTHREAD_MUTEX_RECURSIVE};
+    struct MutexContentionArgs a = {&m};
+    _spawn(MutexContentionWorker, &a, &t);
+    while (!a.ready) sched_yield();
+    EZBENCH2("recursive 2x", donothing, BenchLockUnlock(&m));
+    a.done = true;
+    _join(&t);
+  }
+  {
+    pthread_mutex_t m = {PTHREAD_MUTEX_ERRORCHECK};
+    struct MutexContentionArgs a = {&m};
+    _spawn(MutexContentionWorker, &a, &t);
+    while (!a.ready) sched_yield();
+    EZBENCH2("errorcheck 2x", donothing, BenchLockUnlock(&m));
+    a.done = true;
+    _join(&t);
+  }
+}

test/libc/intrin/pthread_mutex_lock_test.c

@@ -220,12 +220,3 @@ TEST(pthread_spin_lock, test) {
   EXPECT_EQ(THREADS * ITERATIONS, count);
   EXPECT_EQ(0, pthread_spin_destroy(&slock));
 }
-
-BENCH(pthread_mutex_lock, bench) {
-  char schar = 0;
-  pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
-  EZBENCH2("spin", donothing, pthread_spin_lock_test());
-  EZBENCH2("normal", donothing, pthread_mutex_lock_contention());
-  EZBENCH2("recursive", donothing, pthread_mutex_lock_rcontention());
-  EZBENCH2("errorcheck", donothing, pthread_mutex_lock_econtention());
-}

third_party/dlmalloc/dlmalloc.c

@@ -25,8 +25,7 @@
 #define HAVE_MMAP 1
 #define HAVE_MREMAP 0
 #define HAVE_MORECORE 0
-#define USE_LOCKS 1
-#define USE_SPIN_LOCKS 1
+#define USE_LOCKS 2
 #define MORECORE_CONTIGUOUS 0
 #define MALLOC_INSPECT_ALL 1
 

third_party/dlmalloc/locks.inc

@@ -1,5 +1,6 @@
 // clang-format off
 #include "libc/calls/calls.h"
+#include "libc/intrin/pthread.h"
 #include "libc/nexgen32e/threaded.h"
 
 /* --------------------------- Lock preliminaries ------------------------ */
@@ -50,6 +51,15 @@
 /* #define TRY_LOCK(lk) ... */
 /* static MLOCK_T malloc_global_mutex = ... */
 
+#define MLOCK_T               pthread_mutex_t
+#define ACQUIRE_LOCK(lk)      (__threaded && pthread_mutex_lock(lk), 0)
+#define RELEASE_LOCK(lk)      (__threaded && pthread_mutex_unlock(lk), 0)
+#define TRY_LOCK(lk)          (__threaded ? !pthread_mutex_trylock(lk) : 1)
+#define INITIAL_LOCK(lk)      pthread_mutex_init(lk, 0)
+#define DESTROY_LOCK(lk)      pthread_mutex_destroy(lk)
+
+static MLOCK_T malloc_global_mutex;
+
 #elif USE_SPIN_LOCKS
 
 /* First, define CAS_LOCK and CLEAR_LOCK on ints */

third_party/libcxx/thread.cc

@@ -7,6 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "libc/runtime/sysconf.h"
 #include "third_party/libcxx/__config"
 #ifndef _LIBCPP_HAS_NO_THREADS