Eliminate cyclic locks in runtime

This change introduces a new deadlock detector for Cosmo's POSIX threads
implementation. Error check mutexes will now track a DAG of nested locks
and report EDEADLK when a deadlock is theoretically possible. These will
occur rarely, but it's important for production hardening your code. You
don't even need to change your mutexes to use the POSIX error check mode
because `cosmocc -mdbg` will enable error checking on mutexes by default
globally. When cycles are found, an error message showing your demangled
symbols describing the strongly connected component are printed and then
the SIGTRAP is raised, which means you'll also get a backtrace if you're
using ShowCrashReports() too. This new error checker is so low-level and
so pure that it's able to verify the relationships of every libc runtime
lock, including those locks upon which the mutex implementation depends.
This commit is contained in:
Justine Tunney 2024-12-16 20:51:27 -08:00
parent 26c051c297
commit af7bd80430
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
141 changed files with 2094 additions and 1601 deletions

View file

@ -9,6 +9,7 @@ LICENSE
LOCAL CHANGES
- Fix MT-safety bugs in DEBUG mode
- Fix bug in dlmalloc_inspect_all()
- Define dlmalloc_requires_more_vespene_gas()
- Make dlmalloc scalable using sched_getcpu()

View file

@ -31,13 +31,14 @@
#define FOOTERS 1
#define MSPACES 1
#define ONLY_MSPACES 1 // enables scalable multi-threaded malloc
#define USE_SPIN_LOCKS 0 // only profitable using sched_getcpu()
#define USE_SPIN_LOCKS 0 // set to 0 to use scalable nsync locks
#else
#define INSECURE 1
#define PROCEED_ON_ERROR 1
#define FOOTERS 0
#define MSPACES 0
#define ONLY_MSPACES 0
#define USE_SPIN_LOCKS 1
#endif
#define HAVE_MMAP 1
@ -1263,12 +1264,15 @@ void* dlrealloc_single(void* oldmem, size_t bytes) {
#endif /* FOOTERS */
if (!PREACTION(m)) {
mchunkptr newp = try_realloc_chunk(m, oldp, nb, MREMAP_MAYMOVE);
POSTACTION(m);
if (newp != 0) {
/* [jart] fix realloc MT bug in DEBUG mode
https://github.com/intel/linux-sgx/issues/534 */
check_inuse_chunk(m, newp);
POSTACTION(m);
mem = chunk2mem(newp);
}
else {
POSTACTION(m);
mem = internal_malloc(m, bytes);
if (mem != 0) {
size_t oc = chunksize(oldp) - overhead_for(oldp);
@ -1301,11 +1305,13 @@ void* dlrealloc_in_place(void* oldmem, size_t bytes) {
#endif /* FOOTERS */
if (!PREACTION(m)) {
mchunkptr newp = try_realloc_chunk(m, oldp, nb, 0);
POSTACTION(m);
if (newp == oldp) {
/* [jart] fix realloc MT bug in DEBUG mode
https://github.com/intel/linux-sgx/issues/534 */
check_inuse_chunk(m, newp);
mem = oldmem;
}
POSTACTION(m);
}
}
}
@ -1319,13 +1325,6 @@ void* dlmemalign_single(size_t alignment, size_t bytes) {
return internal_memalign(gm, alignment, bytes);
}
#if USE_LOCKS
void dlmalloc_atfork(void) {
bzero(&gm->mutex, sizeof(gm->mutex));
bzero(&malloc_global_mutex, sizeof(malloc_global_mutex));
}
#endif
void** dlindependent_calloc(size_t n_elements, size_t elem_size,
void* chunks[]) {
size_t sz = elem_size; /* serves as 1-element array */

View file

@ -9,7 +9,6 @@
#define dlmallinfo __dlmallinfo
#define dlmalloc __dlmalloc
#define dlmalloc_abort __dlmalloc_abort
#define dlmalloc_atfork __dlmalloc_atfork
#define dlmalloc_footprint __dlmalloc_footprint
#define dlmalloc_footprint_limit __dlmalloc_footprint_limit
#define dlmalloc_inspect_all __dlmalloc_inspect_all
@ -527,7 +526,10 @@ void mspace_inspect_all(mspace msp,
void (*handler)(void*, void*, size_t, void*),
void* arg);
void dlmalloc_atfork(void);
void dlmalloc_pre_fork(void) libcesque;
void dlmalloc_post_fork_parent(void) libcesque;
void dlmalloc_post_fork_child(void) libcesque;
void dlmalloc_abort(void) relegated wontreturn;
COSMOPOLITAN_C_END_

View file

@ -7,31 +7,34 @@
#if LOCK_AT_FORK
#if ONLY_MSPACES
static void dlmalloc_pre_fork(void) {
void dlmalloc_pre_fork(void) {
mstate h;
for (unsigned i = 0; i < ARRAYLEN(g_heaps); ++i)
ACQUIRE_MALLOC_GLOBAL_LOCK();
for (unsigned i = ARRAYLEN(g_heaps); i--;)
if ((h = atomic_load_explicit(&g_heaps[i], memory_order_acquire)))
ACQUIRE_LOCK(&h->mutex);
}
static void dlmalloc_post_fork_parent(void) {
void dlmalloc_post_fork_parent(void) {
mstate h;
for (unsigned i = 0; i < ARRAYLEN(g_heaps); ++i)
if ((h = atomic_load_explicit(&g_heaps[i], memory_order_acquire)))
RELEASE_LOCK(&h->mutex);
RELEASE_MALLOC_GLOBAL_LOCK();
}
static void dlmalloc_post_fork_child(void) {
void dlmalloc_post_fork_child(void) {
mstate h;
for (unsigned i = 0; i < ARRAYLEN(g_heaps); ++i)
if ((h = atomic_load_explicit(&g_heaps[i], memory_order_acquire)))
(void)INITIAL_LOCK(&h->mutex);
(void)REFRESH_LOCK(&h->mutex);
(void)REFRESH_MALLOC_GLOBAL_LOCK();
}
#else
static void dlmalloc_pre_fork(void) { ACQUIRE_LOCK(&(gm)->mutex); }
static void dlmalloc_post_fork_parent(void) { RELEASE_LOCK(&(gm)->mutex); }
static void dlmalloc_post_fork_child(void) { (void)INITIAL_LOCK(&(gm)->mutex); }
void dlmalloc_pre_fork(void) { ACQUIRE_LOCK(&(gm)->mutex); }
void dlmalloc_post_fork_parent(void) { RELEASE_LOCK(&(gm)->mutex); }
void dlmalloc_post_fork_child(void) { (void)REFRESH_LOCK(&(gm)->mutex); }
#endif /* ONLY_MSPACES */
#endif /* LOCK_AT_FORK */
@ -95,12 +98,6 @@ __attribute__((__constructor__(49))) int init_mparams(void) {
(void)INITIAL_LOCK(&gm->mutex);
#endif
#if LOCK_AT_FORK
pthread_atfork(&dlmalloc_pre_fork,
&dlmalloc_post_fork_parent,
&dlmalloc_post_fork_child);
#endif
{
#if USE_DEV_RANDOM
int fd;

View file

@ -1,3 +1,7 @@
#include "libc/cosmo.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/maps.h"
#include "libc/thread/thread.h"
/* --------------------------- Lock preliminaries ------------------------ */
@ -33,11 +37,20 @@
#define MLOCK_T atomic_uint
static int malloc_inlk(MLOCK_T *lk) {
atomic_store_explicit(lk, 0, memory_order_relaxed);
return 0;
}
static int malloc_wipe(MLOCK_T *lk) {
atomic_store_explicit(lk, 0, memory_order_relaxed);
return 0;
}
static int malloc_kilk(MLOCK_T *lk) {
return 0;
}
static int malloc_lock(MLOCK_T *lk) {
for (;;) {
if (!atomic_exchange_explicit(lk, 1, memory_order_acquire))
@ -49,36 +62,71 @@ static int malloc_lock(MLOCK_T *lk) {
return 0;
}
static int malloc_unlock(MLOCK_T *lk) {
static int malloc_unlk(MLOCK_T *lk) {
atomic_store_explicit(lk, 0, memory_order_release);
return 0;
}
#else
#define MLOCK_T nsync_mu
#define MLOCK_T struct MallocLock
static int malloc_wipe(MLOCK_T *lk) {
struct MallocLock {
#if DEBUG
void *edges;
#endif
nsync_mu mu;
};
static int malloc_inlk(MLOCK_T *lk) {
bzero(lk, sizeof(*lk));
return 0;
}
static int malloc_lock(MLOCK_T *lk) {
nsync_mu_lock(lk);
static int malloc_wipe(MLOCK_T *lk) {
bzero(&lk->mu, sizeof(lk->mu));
return 0;
}
static int malloc_unlock(MLOCK_T *lk) {
nsync_mu_unlock(lk);
static int malloc_kilk(MLOCK_T *lk) {
return 0;
}
static int malloc_lock(MLOCK_T *lk) {
#if DEBUG
__deadlock_check(lk, 0);
#endif
nsync_mu_lock(&lk->mu);
#if DEBUG
__deadlock_record(lk, 0);
__deadlock_track(lk, 0);
#endif
return 0;
}
static int malloc_unlk(MLOCK_T *lk) {
#if DEBUG
if (__deadlock_tracked(lk) == 0) {
kprintf("error: unlock malloc mutex not owned by caller: %t\n", lk);
DebugBreak();
}
#endif
nsync_mu_unlock(&lk->mu);
#if DEBUG
__deadlock_untrack(lk);
#endif
return 0;
}
#endif
#define ACQUIRE_LOCK(lk) malloc_lock(lk)
#define RELEASE_LOCK(lk) malloc_unlock(lk)
#define INITIAL_LOCK(lk) malloc_wipe(lk)
#define DESTROY_LOCK(lk) malloc_wipe(lk)
#define RELEASE_LOCK(lk) malloc_unlk(lk)
#define INITIAL_LOCK(lk) malloc_inlk(lk)
#define REFRESH_LOCK(lk) malloc_wipe(lk)
#define DESTROY_LOCK(lk) malloc_kilk(lk)
#define INITIAL_MALLOC_GLOBAL_LOCK() INITIAL_LOCK(&malloc_global_mutex);
#define REFRESH_MALLOC_GLOBAL_LOCK() REFRESH_LOCK(&malloc_global_mutex);
#define ACQUIRE_MALLOC_GLOBAL_LOCK() ACQUIRE_LOCK(&malloc_global_mutex);
#define RELEASE_MALLOC_GLOBAL_LOCK() RELEASE_LOCK(&malloc_global_mutex);

View file

@ -368,12 +368,15 @@ void* mspace_realloc(mspace msp, void* oldmem, size_t bytes) {
#endif /* FOOTERS */
if (!PREACTION(m)) {
mchunkptr newp = try_realloc_chunk(m, oldp, nb, 1);
POSTACTION(m);
if (newp != 0) {
/* [jart] fix realloc MT bug in DEBUG mode
https://github.com/intel/linux-sgx/issues/534 */
check_inuse_chunk(m, newp);
POSTACTION(m);
mem = chunk2mem(newp);
}
else {
POSTACTION(m);
mem = mspace_malloc(m, bytes);
if (mem != 0) {
size_t oc = chunksize(oldp) - overhead_for(oldp);
@ -407,11 +410,13 @@ void* mspace_realloc_in_place(mspace msp, void* oldmem, size_t bytes) {
#endif /* FOOTERS */
if (!PREACTION(m)) {
mchunkptr newp = try_realloc_chunk(m, oldp, nb, 0);
POSTACTION(m);
if (newp == oldp) {
/* [jart] fix realloc_in_place MT bug in DEBUG mode
https://github.com/intel/linux-sgx/issues/534 */
check_inuse_chunk(m, newp);
mem = oldmem;
}
POSTACTION(m);
}
}
}

59
third_party/gdtoa/lock.c vendored Normal file
View file

@ -0,0 +1,59 @@
/*-*- mode:c;indent-tabs-mode:t;c-basic-offset:8;tab-width:8;coding:utf-8 -*-│
vi: set noet ft=c ts=8 sw=8 fenc=utf-8 :vi
The author of this software is David M. Gay.
Please send bug reports to David M. Gay <dmg@acm.org>
or Justine Tunney <jtunney@gmail.com>
Copyright (C) 1998, 1999 by Lucent Technologies
All Rights Reserved
Permission to use, copy, modify, and distribute this software and
its documentation for any purpose and without fee is hereby
granted, provided that the above copyright notice appear in all
copies and that both that the copyright notice and this
permission notice and warranty disclaimer appear in supporting
documentation, and that the name of Lucent or any of its entities
not be used in advertising or publicity pertaining to
distribution of the software without specific, written prior
permission.
LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.
IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY
SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
THIS SOFTWARE.
*/
#include "third_party/gdtoa/lock.h"
pthread_mutex_t __gdtoa_lock_obj = PTHREAD_MUTEX_INITIALIZER;
pthread_mutex_t __gdtoa_lock1_obj = PTHREAD_MUTEX_INITIALIZER;
void
__gdtoa_lock(void)
{
pthread_mutex_lock(&__gdtoa_lock_obj);
}
void
__gdtoa_unlock(void)
{
pthread_mutex_unlock(&__gdtoa_lock_obj);
}
void
__gdtoa_lock1(void)
{
pthread_mutex_lock(&__gdtoa_lock1_obj);
}
void
__gdtoa_unlock1(void)
{
pthread_mutex_unlock(&__gdtoa_lock1_obj);
}

15
third_party/gdtoa/lock.h vendored Normal file
View file

@ -0,0 +1,15 @@
#ifndef COSMOPOLITAN_THIRD_PARTY_GDTOA_LOCK_H_
#define COSMOPOLITAN_THIRD_PARTY_GDTOA_LOCK_H_
#include "libc/thread/thread.h"
COSMOPOLITAN_C_START_
extern pthread_mutex_t __gdtoa_lock_obj;
extern pthread_mutex_t __gdtoa_lock1_obj;
void __gdtoa_lock(void);
void __gdtoa_unlock(void);
void __gdtoa_lock1(void);
void __gdtoa_unlock1(void);
COSMOPOLITAN_C_END_
#endif /* COSMOPOLITAN_THIRD_PARTY_GDTOA_LOCK_H_ */

View file

@ -35,46 +35,9 @@
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
#include "third_party/gdtoa/gdtoa.internal.h"
#include "third_party/gdtoa/lock.h"
static ThInfo TI0;
static pthread_mutex_t __gdtoa_lock_obj;
static pthread_mutex_t __gdtoa_lock1_obj;
static void
__gdtoa_lock(void)
{
pthread_mutex_lock(&__gdtoa_lock_obj);
}
static void
__gdtoa_unlock(void)
{
pthread_mutex_unlock(&__gdtoa_lock_obj);
}
static void
__gdtoa_initlock(void)
{
pthread_mutex_init(&__gdtoa_lock_obj, 0);
}
static void
__gdtoa_lock1(void)
{
pthread_mutex_lock(&__gdtoa_lock1_obj);
}
static void
__gdtoa_unlock1(void)
{
pthread_mutex_unlock(&__gdtoa_lock1_obj);
}
static void
__gdtoa_initlock1(void)
{
pthread_mutex_init(&__gdtoa_lock1_obj, 0);
}
static void
__gdtoa_Brelease(Bigint *rv)
@ -88,24 +51,20 @@ static void
__gdtoa_Bclear(void)
{
int i;
__gdtoa_lock();
__gdtoa_lock1();
for (i = 0; i < ARRAYLEN(TI0.Freelist); ++i)
__gdtoa_Brelease(TI0.Freelist[i]);
__gdtoa_lock1();
__gdtoa_lock();
__gdtoa_Brelease(TI0.P5s);
__gdtoa_unlock1();
bzero(&TI0, sizeof(TI0));
__gdtoa_unlock();
bzero(&TI0, sizeof(TI0));
__gdtoa_unlock1();
}
__attribute__((__constructor__(60))) static void
__gdtoa_Binit(void)
{
__gdtoa_initlock();
__gdtoa_initlock1();
atexit(__gdtoa_Bclear);
pthread_atfork(__gdtoa_lock1, __gdtoa_unlock1, __gdtoa_initlock1);
pthread_atfork(__gdtoa_lock, __gdtoa_unlock, __gdtoa_initlock);
}
static ThInfo *

View file

@ -19,12 +19,16 @@
#include "libc/thread/thread.h"
#include "third_party/lua/lrepl.h"
static pthread_mutex_t lua_repl_lock_obj;
static pthread_mutex_t lua_repl_lock_obj = PTHREAD_MUTEX_INITIALIZER;
void(lua_repl_lock)(void) {
void lua_repl_wock(void) {
lua_repl_lock_obj = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER;
}
void lua_repl_lock(void) {
pthread_mutex_lock(&lua_repl_lock_obj);
}
void(lua_repl_unlock)(void) {
void lua_repl_unlock(void) {
pthread_mutex_unlock(&lua_repl_lock_obj);
}

View file

@ -11,6 +11,7 @@ extern struct linenoiseState *lua_repl_linenoise;
extern linenoiseCompletionCallback *lua_repl_completions_callback;
void lua_freerepl(void);
void lua_repl_wock(void);
void lua_repl_lock(void);
void lua_repl_unlock(void);
int lua_loadline(lua_State *);

View file

@ -2959,7 +2959,7 @@ static int LuaUnixMapshared(lua_State *L) {
m->mapsize = c;
m->lock = (pthread_mutex_t *)p;
pthread_mutexattr_init(&mattr);
pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_NORMAL);
pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_DEFAULT);
pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED);
pthread_mutex_init(m->lock, &mattr);
pthread_mutexattr_destroy(&mattr);

View file

@ -39,6 +39,7 @@
#include "third_party/nsync/common.internal.h"
#include "third_party/nsync/mu_semaphore.h"
#include "third_party/nsync/mu_semaphore.internal.h"
#include "libc/intrin/kprintf.h"
#include "third_party/nsync/wait_s.internal.h"
__static_yoink("nsync_notice");

View file

@ -78,7 +78,7 @@ static bool nsync_mu_semaphore_sem_create (struct sem *f) {
return true;
}
static void nsync_mu_semaphore_sem_fork_child (void) {
void nsync_mu_semaphore_sem_fork_child (void) {
struct sem *f;
for (f = atomic_load_explicit (&g_sems, memory_order_relaxed); f; f = f->next) {
int rc = sys_close (f->id);
@ -87,17 +87,11 @@ static void nsync_mu_semaphore_sem_fork_child (void) {
}
}
static void nsync_mu_semaphore_sem_init (void) {
pthread_atfork (0, 0, nsync_mu_semaphore_sem_fork_child);
}
/* Initialize *s; the initial value is 0. */
bool nsync_mu_semaphore_init_sem (nsync_semaphore *s) {
static atomic_uint once;
struct sem *f = (struct sem *) s;
if (!nsync_mu_semaphore_sem_create (f))
return false;
cosmo_once (&once, nsync_mu_semaphore_sem_init);
sems_push(f);
return true;
}

View file

@ -28,5 +28,5 @@ void nsync_panic_ (const char *s) {
"cosmoaddr2line ", program_invocation_name, " ",
DescribeBacktrace (__builtin_frame_address (0)), "\n",
NULL);
_Exit (44);
__builtin_trap ();
}

View file

@ -2,6 +2,10 @@
vi: set noet ft=c ts=8 sw=8 fenc=utf-8 :vi
*/
#define LOCALTIME_IMPLEMENTATION
#include "lock.h"
#include "tzdir.h"
#include "tzfile.h"
#include "private.h"
#include "libc/calls/blockcancel.internal.h"
#include "libc/calls/calls.h"
#include "libc/cxxabi.h"
@ -10,20 +14,15 @@
#include "libc/serialize.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/o.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
#include "libc/time.h"
#include "libc/inttypes.h"
#include "libc/sysv/consts/ok.h"
#include "libc/runtime/runtime.h"
#include "libc/stdckdint.h"
#include "libc/time.h"
#include "tzdir.h"
#include "tzfile.h"
#include "libc/nt/struct/timezoneinformation.h"
#include "libc/nt/time.h"
#include "libc/dce.h"
#include "private.h"
/* Convert timestamp from time_t to struct tm. */
@ -624,34 +623,10 @@ localtime_windows_init(void)
setenv("TZ", buf, true);
}
static pthread_mutex_t locallock = PTHREAD_MUTEX_INITIALIZER;
static dontinline void
localtime_wipe(void)
{
pthread_mutex_init(&locallock, 0);
}
static dontinline void
localtime_lock(void)
{
pthread_mutex_lock(&locallock);
}
static dontinline void
localtime_unlock(void)
{
pthread_mutex_unlock(&locallock);
}
__attribute__((__constructor__(80)))
textstartup static void
localtime_init(void)
{
localtime_wipe();
pthread_atfork(localtime_lock,
localtime_unlock,
localtime_wipe);
if (IsWindows())
localtime_windows_init();
}
@ -2052,9 +2027,9 @@ localtime_tzset_unlocked(void)
void
tzset(void)
{
localtime_lock();
__localtime_lock();
localtime_tzset_unlocked();
localtime_unlock();
__localtime_unlock();
}
static void
@ -2067,7 +2042,7 @@ static void
localtime_gmtcheck(void)
{
static bool gmt_is_set;
localtime_lock();
__localtime_lock();
if (! gmt_is_set) {
#ifdef ALL_STATE
gmtptr = malloc(sizeof *gmtptr);
@ -2077,7 +2052,7 @@ localtime_gmtcheck(void)
localtime_gmtload(gmtptr);
gmt_is_set = true;
}
localtime_unlock();
__localtime_unlock();
}
/*
@ -2193,11 +2168,11 @@ localsub(struct state const *sp, time_t const *timep, int_fast32_t setname,
static struct tm *
localtime_tzset(time_t const *timep, struct tm *tmp, bool setname)
{
localtime_lock();
__localtime_lock();
if (setname || !lcl_is_set)
localtime_tzset_unlocked();
tmp = localsub(lclptr, timep, setname, tmp);
localtime_unlock();
__localtime_unlock();
return tmp;
}
@ -2834,10 +2809,10 @@ time_t
mktime(struct tm *tmp)
{
time_t t;
localtime_lock();
__localtime_lock();
localtime_tzset_unlocked();
t = mktime_tzname(lclptr, tmp, true);
localtime_unlock();
__localtime_unlock();
return t;
}

12
third_party/tz/lock.h vendored Normal file
View file

@ -0,0 +1,12 @@
#ifndef COSMOPOLITAN_THIRD_PARTY_TZ_LOCK_H_
#define COSMOPOLITAN_THIRD_PARTY_TZ_LOCK_H_
#include "libc/thread/thread.h"
COSMOPOLITAN_C_START_
extern pthread_mutex_t __localtime_lock_obj;
void __localtime_lock(void);
void __localtime_unlock(void);
COSMOPOLITAN_C_END_
#endif /* COSMOPOLITAN_THIRD_PARTY_TZ_LOCK_H_ */