Eliminate cyclic locks in runtime

This change introduces a new deadlock detector for Cosmo's POSIX threads
implementation. Error check mutexes will now track a DAG of nested locks
and report EDEADLK when a deadlock is theoretically possible. These will
occur rarely, but it's important for production hardening your code. You
don't even need to change your mutexes to use the POSIX error check mode
because `cosmocc -mdbg` will enable error checking on mutexes by default
globally. When cycles are found, an error message showing your demangled
symbols describing the strongly connected component are printed and then
the SIGTRAP is raised, which means you'll also get a backtrace if you're
using ShowCrashReports() too. This new error checker is so low-level and
so pure that it's able to verify the relationships of every libc runtime
lock, including those locks upon which the mutex implementation depends.

third_party/dlmalloc/README.cosmo

@@ -9,6 +9,7 @@ LICENSE
 
 LOCAL CHANGES
 
+  - Fix MT-safety bugs in DEBUG mode
   - Fix bug in dlmalloc_inspect_all()
   - Define dlmalloc_requires_more_vespene_gas()
   - Make dlmalloc scalable using sched_getcpu()

third_party/dlmalloc/dlmalloc.c

@@ -31,13 +31,14 @@
 #define FOOTERS 1
 #define MSPACES 1
 #define ONLY_MSPACES 1 // enables scalable multi-threaded malloc
-#define USE_SPIN_LOCKS 0 // only profitable using sched_getcpu()
+#define USE_SPIN_LOCKS 0 // set to 0 to use scalable nsync locks
 #else
 #define INSECURE 1
 #define PROCEED_ON_ERROR 1
 #define FOOTERS 0
 #define MSPACES 0
 #define ONLY_MSPACES 0
+#define USE_SPIN_LOCKS 1
 #endif
 
 #define HAVE_MMAP 1
@@ -1263,12 +1264,15 @@ void* dlrealloc_single(void* oldmem, size_t bytes) {
 #endif /* FOOTERS */
     if (!PREACTION(m)) {
       mchunkptr newp = try_realloc_chunk(m, oldp, nb, MREMAP_MAYMOVE);
-      POSTACTION(m);
       if (newp != 0) {
+        /* [jart] fix realloc MT bug in DEBUG mode
+                  https://github.com/intel/linux-sgx/issues/534 */
         check_inuse_chunk(m, newp);
+        POSTACTION(m);
         mem = chunk2mem(newp);
       }
       else {
+        POSTACTION(m);
         mem = internal_malloc(m, bytes);
         if (mem != 0) {
           size_t oc = chunksize(oldp) - overhead_for(oldp);
@@ -1301,11 +1305,13 @@ void* dlrealloc_in_place(void* oldmem, size_t bytes) {
 #endif /* FOOTERS */
       if (!PREACTION(m)) {
         mchunkptr newp = try_realloc_chunk(m, oldp, nb, 0);
-        POSTACTION(m);
         if (newp == oldp) {
+          /* [jart] fix realloc MT bug in DEBUG mode
+                    https://github.com/intel/linux-sgx/issues/534 */
           check_inuse_chunk(m, newp);
           mem = oldmem;
         }
+        POSTACTION(m);
       }
     }
   }
@@ -1319,13 +1325,6 @@ void* dlmemalign_single(size_t alignment, size_t bytes) {
   return internal_memalign(gm, alignment, bytes);
 }
 
-#if USE_LOCKS
-void dlmalloc_atfork(void) {
-  bzero(&gm->mutex, sizeof(gm->mutex));
-  bzero(&malloc_global_mutex, sizeof(malloc_global_mutex));
-}
-#endif
-
 void** dlindependent_calloc(size_t n_elements, size_t elem_size,
                             void* chunks[]) {
   size_t sz = elem_size; /* serves as 1-element array */

third_party/dlmalloc/dlmalloc.h

@@ -9,7 +9,6 @@
 #define dlmallinfo                   __dlmallinfo
 #define dlmalloc                     __dlmalloc
 #define dlmalloc_abort               __dlmalloc_abort
-#define dlmalloc_atfork              __dlmalloc_atfork
 #define dlmalloc_footprint           __dlmalloc_footprint
 #define dlmalloc_footprint_limit     __dlmalloc_footprint_limit
 #define dlmalloc_inspect_all         __dlmalloc_inspect_all
@@ -527,7 +526,10 @@ void mspace_inspect_all(mspace msp,
                         void (*handler)(void*, void*, size_t, void*),
                         void* arg);
 
-void dlmalloc_atfork(void);
+void dlmalloc_pre_fork(void) libcesque;
+void dlmalloc_post_fork_parent(void) libcesque;
+void dlmalloc_post_fork_child(void) libcesque;
+
 void dlmalloc_abort(void) relegated wontreturn;
 
 COSMOPOLITAN_C_END_

third_party/dlmalloc/init.inc

@@ -7,31 +7,34 @@
 #if LOCK_AT_FORK
 #if ONLY_MSPACES
 
-static void dlmalloc_pre_fork(void) {
+void dlmalloc_pre_fork(void) {
   mstate h;
-  for (unsigned i = 0; i < ARRAYLEN(g_heaps); ++i)
+  ACQUIRE_MALLOC_GLOBAL_LOCK();
+  for (unsigned i = ARRAYLEN(g_heaps); i--;)
     if ((h = atomic_load_explicit(&g_heaps[i], memory_order_acquire)))
       ACQUIRE_LOCK(&h->mutex);
 }
 
-static void dlmalloc_post_fork_parent(void) {
+void dlmalloc_post_fork_parent(void) {
   mstate h;
   for (unsigned i = 0; i < ARRAYLEN(g_heaps); ++i)
     if ((h = atomic_load_explicit(&g_heaps[i], memory_order_acquire)))
       RELEASE_LOCK(&h->mutex);
+  RELEASE_MALLOC_GLOBAL_LOCK();
 }
 
-static void dlmalloc_post_fork_child(void) {
+void dlmalloc_post_fork_child(void) {
   mstate h;
   for (unsigned i = 0; i < ARRAYLEN(g_heaps); ++i)
     if ((h = atomic_load_explicit(&g_heaps[i], memory_order_acquire)))
-      (void)INITIAL_LOCK(&h->mutex);
+      (void)REFRESH_LOCK(&h->mutex);
+  (void)REFRESH_MALLOC_GLOBAL_LOCK();
 }
 
 #else
-static void dlmalloc_pre_fork(void)         { ACQUIRE_LOCK(&(gm)->mutex); }
-static void dlmalloc_post_fork_parent(void) { RELEASE_LOCK(&(gm)->mutex); }
-static void dlmalloc_post_fork_child(void)  { (void)INITIAL_LOCK(&(gm)->mutex); }
+void dlmalloc_pre_fork(void)         { ACQUIRE_LOCK(&(gm)->mutex); }
+void dlmalloc_post_fork_parent(void) { RELEASE_LOCK(&(gm)->mutex); }
+void dlmalloc_post_fork_child(void)  { (void)REFRESH_LOCK(&(gm)->mutex); }
 #endif /* ONLY_MSPACES */
 #endif /* LOCK_AT_FORK */
 
@@ -95,12 +98,6 @@ __attribute__((__constructor__(49))) int init_mparams(void) {
     (void)INITIAL_LOCK(&gm->mutex);
 #endif
 
-#if LOCK_AT_FORK
-    pthread_atfork(&dlmalloc_pre_fork,
-                   &dlmalloc_post_fork_parent,
-                   &dlmalloc_post_fork_child);
-#endif
-
     {
 #if USE_DEV_RANDOM
       int fd;

third_party/dlmalloc/locks.inc

@@ -1,3 +1,7 @@
+#include "libc/cosmo.h"
+#include "libc/intrin/kprintf.h"
+#include "libc/intrin/maps.h"
+#include "libc/thread/thread.h"
 
 /* --------------------------- Lock preliminaries ------------------------ */
 
@@ -33,11 +37,20 @@
 
 #define MLOCK_T atomic_uint
 
+static int malloc_inlk(MLOCK_T *lk) {
+  atomic_store_explicit(lk, 0, memory_order_relaxed);
+  return 0;
+}
+
 static int malloc_wipe(MLOCK_T *lk) {
   atomic_store_explicit(lk, 0, memory_order_relaxed);
   return 0;
 }
 
+static int malloc_kilk(MLOCK_T *lk) {
+  return 0;
+}
+
 static int malloc_lock(MLOCK_T *lk) {
   for (;;) {
     if (!atomic_exchange_explicit(lk, 1, memory_order_acquire))
@@ -49,36 +62,71 @@ static int malloc_lock(MLOCK_T *lk) {
   return 0;
 }
 
-static int malloc_unlock(MLOCK_T *lk) {
+static int malloc_unlk(MLOCK_T *lk) {
   atomic_store_explicit(lk, 0, memory_order_release);
   return 0;
 }
 
 #else
 
-#define MLOCK_T nsync_mu
+#define MLOCK_T struct MallocLock
 
-static int malloc_wipe(MLOCK_T *lk) {
+struct MallocLock {
+#if DEBUG
+  void *edges;
+#endif
+  nsync_mu mu;
+};
+
+static int malloc_inlk(MLOCK_T *lk) {
   bzero(lk, sizeof(*lk));
   return 0;
 }
 
-static int malloc_lock(MLOCK_T *lk) {
-  nsync_mu_lock(lk);
+static int malloc_wipe(MLOCK_T *lk) {
+  bzero(&lk->mu, sizeof(lk->mu));
   return 0;
 }
 
-static int malloc_unlock(MLOCK_T *lk) {
-  nsync_mu_unlock(lk);
+static int malloc_kilk(MLOCK_T *lk) {
+  return 0;
+}
+
+static int malloc_lock(MLOCK_T *lk) {
+#if DEBUG
+  __deadlock_check(lk, 0);
+#endif
+  nsync_mu_lock(&lk->mu);
+#if DEBUG
+  __deadlock_record(lk, 0);
+  __deadlock_track(lk, 0);
+#endif
+  return 0;
+}
+
+static int malloc_unlk(MLOCK_T *lk) {
+#if DEBUG
+  if (__deadlock_tracked(lk) == 0) {
+    kprintf("error: unlock malloc mutex not owned by caller: %t\n", lk);
+    DebugBreak();
+  }
+#endif
+  nsync_mu_unlock(&lk->mu);
+#if DEBUG
+  __deadlock_untrack(lk);
+#endif
   return 0;
 }
 
 #endif
 
 #define ACQUIRE_LOCK(lk) malloc_lock(lk)
-#define RELEASE_LOCK(lk) malloc_unlock(lk)
-#define INITIAL_LOCK(lk) malloc_wipe(lk)
-#define DESTROY_LOCK(lk) malloc_wipe(lk)
+#define RELEASE_LOCK(lk) malloc_unlk(lk)
+#define INITIAL_LOCK(lk) malloc_inlk(lk)
+#define REFRESH_LOCK(lk) malloc_wipe(lk)
+#define DESTROY_LOCK(lk) malloc_kilk(lk)
+#define INITIAL_MALLOC_GLOBAL_LOCK() INITIAL_LOCK(&malloc_global_mutex);
+#define REFRESH_MALLOC_GLOBAL_LOCK() REFRESH_LOCK(&malloc_global_mutex);
 #define ACQUIRE_MALLOC_GLOBAL_LOCK() ACQUIRE_LOCK(&malloc_global_mutex);
 #define RELEASE_MALLOC_GLOBAL_LOCK() RELEASE_LOCK(&malloc_global_mutex);
 

third_party/dlmalloc/mspaces.inc

@@ -368,12 +368,15 @@ void* mspace_realloc(mspace msp, void* oldmem, size_t bytes) {
 #endif /* FOOTERS */
     if (!PREACTION(m)) {
       mchunkptr newp = try_realloc_chunk(m, oldp, nb, 1);
-      POSTACTION(m);
       if (newp != 0) {
+        /* [jart] fix realloc MT bug in DEBUG mode
+                  https://github.com/intel/linux-sgx/issues/534 */
         check_inuse_chunk(m, newp);
+        POSTACTION(m);
         mem = chunk2mem(newp);
       }
       else {
+        POSTACTION(m);
         mem = mspace_malloc(m, bytes);
         if (mem != 0) {
           size_t oc = chunksize(oldp) - overhead_for(oldp);
@@ -407,11 +410,13 @@ void* mspace_realloc_in_place(mspace msp, void* oldmem, size_t bytes) {
 #endif /* FOOTERS */
       if (!PREACTION(m)) {
         mchunkptr newp = try_realloc_chunk(m, oldp, nb, 0);
-        POSTACTION(m);
         if (newp == oldp) {
+          /* [jart] fix realloc_in_place MT bug in DEBUG mode
+                    https://github.com/intel/linux-sgx/issues/534 */
           check_inuse_chunk(m, newp);
           mem = oldmem;
         }
+        POSTACTION(m);
       }
     }
   }