Make recursive mutexes faster

Recursive mutexes now go as fast as normal mutexes. The tradeoff is they
are no longer safe to use in signal handlers. However you can still have
signal safe mutexes if you set your mutex to both recursive and pshared.
You can also make functions that use recursive mutexes signal safe using
sigprocmask to ensure recursion doesn't happen due to any signal handler

The impact of this change is that, on Windows, many functions which edit
the file descriptor table rely on recursive mutexes, e.g. open(). If you
develop your app so it uses pread() and pwrite() then your app should go
very fast when performing a heavily multithreaded and contended workload

For example, when scaling to 40+ cores, *NSYNC mutexes can go as much as
1000x faster (in CPU time) than the naive recursive lock implementation.
Now recursive will use *NSYNC under the hood when it's possible to do so
This commit is contained in:
Justine Tunney 2024-09-09 22:07:03 -07:00
parent 58d252f3db
commit 2f48a02b44
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
37 changed files with 2684 additions and 2209 deletions

View file

@ -111,6 +111,37 @@ static errno_t pthread_mutex_lock_recursive(pthread_mutex_t *mutex,
}
}
#if PTHREAD_USE_NSYNC
static errno_t pthread_mutex_lock_recursive_nsync(pthread_mutex_t *mutex,
uint64_t word) {
int me = gettid();
for (;;) {
if (MUTEX_OWNER(word) == me) {
if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) {
if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) {
if (atomic_compare_exchange_weak_explicit(
&mutex->_word, &word, MUTEX_INC_DEPTH(word),
memory_order_relaxed, memory_order_relaxed))
return 0;
continue;
} else {
return EAGAIN;
}
} else {
return EDEADLK;
}
}
_weaken(nsync_mu_lock)((nsync_mu *)mutex->_nsyncx);
word = MUTEX_UNLOCK(word);
word = MUTEX_LOCK(word);
word = MUTEX_SET_OWNER(word, me);
mutex->_word = word;
mutex->_pid = __pid;
return 0;
}
}
#endif
static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex) {
uint64_t word;
@ -141,8 +172,17 @@ static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex) {
return 0;
}
// handle recursive and error checking mutexes
// handle recursive and error checking mutexes
#if PTHREAD_USE_NSYNC
if (_weaken(nsync_mu_lock) &&
MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE) {
return pthread_mutex_lock_recursive_nsync(mutex, word);
} else {
return pthread_mutex_lock_recursive(mutex, word);
}
#else
return pthread_mutex_lock_recursive(mutex, word);
#endif
}
/**