Rewrite recursive mutex code

This commit is contained in:
Justine Tunney 2022-09-10 09:14:40 -07:00
parent bae7367774
commit cfcf5918bc
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
27 changed files with 164 additions and 257 deletions

View file

@ -20,11 +20,14 @@
#include "libc/macros.internal.h"
.privileged
_futex: mov %rcx,%r10
_futex: push %rbp
mov %rsp,%rbp
mov %rcx,%r10
mov __NR_futex,%eax
clc
syscall
jnc 1f
neg %eax
1: ret
1: pop %rbp
ret
.endfn _futex,globl,hidden

View file

@ -16,11 +16,9 @@
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/calls/state.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/thread/tls.h"
/**

View file

@ -181,19 +181,22 @@ privileged static void klog(const char *b, size_t n) {
: "=a"(rax), "=D"(rdi), "=S"(rsi), "=d"(rdx)
: "0"(__NR_write), "1"(2), "2"(b), "3"(n)
: "rcx", "r8", "r9", "r10", "r11", "memory", "cc");
if (rax < 0) {
notpossible;
}
}
}
privileged static size_t kformat(char *b, size_t n, const char *fmt,
va_list va) {
int si;
int si, y;
wint_t t, u;
const char *abet;
signed char type;
const char *s, *f;
unsigned long long x;
unsigned i, j, m, rem, sign, hash, cols, prec;
char c, *p, *e, pdot, zero, flip, dang, base, quot, uppr, z[128];
char c, *p, *e, pdot, zero, flip, dang, base, quot, uppr, ansi, z[128];
if (kistextpointer(b) || kisdangerous(b)) n = 0;
if (!kistextpointer(fmt)) fmt = "!!WONTFMT";
p = b;
@ -314,6 +317,16 @@ privileged static size_t kformat(char *b, size_t n, const char *fmt,
} else {
x = __get_tls_privileged()->tib_tid;
}
if (!__nocolor && p + 7 <= e) {
*p++ = '\e';
*p++ = '[';
*p++ = '1';
*p++ = ';';
*p++ = '3';
*p++ = '0' + x % 8;
*p++ = 'm';
ansi = true;
}
} else {
x = 666;
}
@ -690,6 +703,15 @@ privileged static size_t kformat(char *b, size_t n, const char *fmt,
}
break;
}
if (ansi) {
if (p + 4 <= e) {
*p++ = '\e';
*p++ = '[';
*p++ = '0';
*p++ = 'm';
}
ansi = false;
}
break;
}
}

View file

@ -1,44 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi
Copyright 2021 Justine Alexandra Roberts Tunney
Permission to use, copy, modify, and/or distribute this software for
any purpose with or without fee is hereby granted, provided that the
above copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/intrin/lockxadd.h"
#include "libc/runtime/runtime.h"
/**
* Compares and exchanges w/ lock prefix.
*
* @param ifthing is uint𝑘_t[hasatleast 1] where 𝑘 {8,16,32,64}
* @param size is automatically supplied by macro wrapper
* @return value at location `*ifthing` *before* addition
* @see InterlockedAdd() for a very similar API
* @see xadd() if only written by one thread
*/
intptr_t(_lockxadd)(void *ifthing, intptr_t replaceitwithme, size_t size) {
switch (size) {
case 1:
return _lockxadd((int8_t *)ifthing, (int8_t)replaceitwithme);
case 2:
return _lockxadd((int16_t *)ifthing, (int16_t)replaceitwithme);
case 4:
return _lockxadd((int32_t *)ifthing, (int32_t)replaceitwithme);
case 8:
return _lockxadd((int64_t *)ifthing, (int64_t)replaceitwithme);
default:
abort();
}
}

View file

@ -1,42 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi
Copyright 2021 Justine Alexandra Roberts Tunney
Permission to use, copy, modify, and/or distribute this software for
any purpose with or without fee is hereby granted, provided that the
above copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/intrin/lockxchg.h"
/**
* Compares and exchanges w/ lock prefix.
*
* @param memory is uint𝑘_t[hasatleast 1] where 𝑘 {8,16,32,64}
* @param size is automatically supplied by macro wrapper
* @return true if value was exchanged, otherwise false
* @see xchg()
*/
intptr_t(lockxchg)(void *memory, void *localvar, size_t size) {
switch (size) {
case 1:
return lockxchg((int8_t *)memory, (int8_t *)localvar);
case 2:
return lockxchg((int16_t *)memory, (int16_t *)localvar);
case 4:
return lockxchg((int32_t *)memory, (int32_t *)localvar);
case 8:
return lockxchg((int64_t *)memory, (int64_t *)localvar);
default:
return false;
}
}

View file

@ -19,29 +19,9 @@
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/errno.h"
#include "libc/intrin/asmflag.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/futex.internal.h"
#include "libc/thread/thread.h"
#include "libc/linux/futex.h"
#include "libc/thread/tls.h"
#include "libc/sysv/consts/futex.h"
#include "libc/sysv/consts/nr.h"
static int pthread_mutex_lock_spin(pthread_mutex_t *mutex, int expect,
int tries) {
if (tries < 7) {
volatile int i;
for (i = 0; i != 1 << tries; i++) {
}
tries++;
} else {
atomic_fetch_add(&mutex->waits, 1);
_futex_wait(&mutex->lock, expect, mutex->pshared, &(struct timespec){1});
atomic_fetch_sub(&mutex->waits, 1);
}
return tries;
}
/**
* Locks mutex.
@ -74,26 +54,18 @@ static int pthread_mutex_lock_spin(pthread_mutex_t *mutex, int expect,
* pthread_mutex_unlock(&lock);
* pthread_mutex_destroy(&lock);
*
* Alternatively, Cosmopolitan lets you do the folllowing instead:
*
* pthread_mutex_t lock = {PTHREAD_MUTEX_RECURSIVE};
* pthread_mutex_lock(&lock);
* // do work...
* pthread_mutex_unlock(&lock);
*
* @return 0 on success, or error number on failure
* @see pthread_spin_lock()
*/
int pthread_mutex_lock(pthread_mutex_t *mutex) {
int c, me, owner, tries;
switch (mutex->type) {
case PTHREAD_MUTEX_NORMAL:
int c, d, t;
if (mutex->type == PTHREAD_MUTEX_NORMAL) {
// From Futexes Are Tricky Version 1.1 § Mutex, Take 3;
// Ulrich Drepper, Red Hat Incorporated, June 27, 2004.
c = 0;
if (!atomic_compare_exchange_strong_explicit(&mutex->lock, &c, 1,
memory_order_acquire,
memory_order_relaxed)) {
if (!atomic_compare_exchange_strong_explicit(
&mutex->lock, &c, 1, memory_order_acquire, memory_order_relaxed)) {
if (c != 2) {
c = atomic_exchange_explicit(&mutex->lock, 2, memory_order_acquire);
}
@ -103,28 +75,47 @@ int pthread_mutex_lock(pthread_mutex_t *mutex) {
}
}
return 0;
case PTHREAD_MUTEX_RECURSIVE:
case PTHREAD_MUTEX_ERRORCHECK:
for (tries = 0, me = gettid();;) {
owner = atomic_load_explicit(&mutex->lock, memory_order_relaxed);
if (!owner && atomic_compare_exchange_weak_explicit(
&mutex->lock, &owner, me, memory_order_acquire,
memory_order_relaxed)) {
break;
} else if (owner == me) {
if (mutex->type != PTHREAD_MUTEX_ERRORCHECK) {
break;
} else {
}
t = gettid();
if (mutex->type == PTHREAD_MUTEX_ERRORCHECK) {
c = atomic_load_explicit(&mutex->lock, memory_order_relaxed);
if ((c & 0x000fffff) == t) {
assert(!"deadlock");
return EDEADLK;
}
}
tries = pthread_mutex_lock_spin(mutex, owner, tries);
for (;;) {
c = 0;
d = 0x10100000 | t;
if (atomic_compare_exchange_weak_explicit(
&mutex->lock, &c, d, memory_order_acquire, memory_order_relaxed)) {
break;
} else {
if ((c & 0x000fffff) == t) {
if ((c & 0x0ff00000) < 0x0ff00000) {
c = atomic_fetch_add_explicit(&mutex->lock, 0x00100000,
memory_order_relaxed);
break;
} else {
assert(!"recurse");
return EAGAIN;
}
++mutex->reent;
}
if ((c & 0xf0000000) == 0x10000000) {
d = 0x20000000 | c;
if (atomic_compare_exchange_weak_explicit(&mutex->lock, &c, d,
memory_order_acquire,
memory_order_relaxed)) {
c = d;
}
}
if ((c & 0xf0000000) == 0x20000000) {
_futex_wait(&mutex->lock, c, mutex->pshared, 0);
}
}
}
return 0;
default:
assert(!"badlock");
return EINVAL;
}
}

View file

@ -33,36 +33,31 @@
* @raise ENOTRECOVERABLE if `mutex` is corrupted
*/
int pthread_mutex_trylock(pthread_mutex_t *mutex) {
int c, me, owner;
switch (mutex->type) {
case PTHREAD_MUTEX_NORMAL:
int c, d, t;
if (mutex->type == PTHREAD_MUTEX_NORMAL) {
c = 0;
if (atomic_compare_exchange_strong_explicit(&mutex->lock, &c, 1,
memory_order_acquire,
memory_order_relaxed)) {
if (atomic_compare_exchange_strong_explicit(
&mutex->lock, &c, 1, memory_order_acquire, memory_order_relaxed)) {
return 0;
} else {
return EBUSY;
}
case PTHREAD_MUTEX_RECURSIVE:
case PTHREAD_MUTEX_ERRORCHECK:
owner = 0;
me = gettid();
if (!atomic_compare_exchange_strong_explicit(&mutex->lock, &owner, me,
memory_order_acquire,
memory_order_relaxed)) {
if (owner == me) {
if (mutex->type == PTHREAD_MUTEX_ERRORCHECK) {
}
c = 0;
t = gettid();
d = 0x10100000 | t;
if (!atomic_compare_exchange_strong_explicit(
&mutex->lock, &c, d, memory_order_acquire, memory_order_relaxed)) {
if ((c & 0x000fffff) != t || mutex->type == PTHREAD_MUTEX_ERRORCHECK) {
return EBUSY;
}
} else {
return EBUSY;
if ((c & 0x0ff00000) == 0x0ff00000) {
return EAGAIN;
}
atomic_fetch_add_explicit(&mutex->lock, 0x00100000, memory_order_relaxed);
}
++mutex->reent;
return 0;
default:
assert(!"badlock");
return ENOTRECOVERABLE;
}
}

View file

@ -18,7 +18,6 @@
*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/futex.internal.h"
@ -31,35 +30,34 @@
* @raises EPERM if in error check mode and not owned by caller
*/
int pthread_mutex_unlock(pthread_mutex_t *mutex) {
int c, me, owner;
switch (mutex->type) {
case PTHREAD_MUTEX_NORMAL:
int c, t;
if (mutex->type == PTHREAD_MUTEX_NORMAL) {
// From Futexes Are Tricky Version 1.1 § Mutex, Take 3;
// Ulrich Drepper, Red Hat Incorporated, June 27, 2004.
if ((c = atomic_fetch_sub_explicit(&mutex->lock, 1,
memory_order_release)) != 1) {
if ((c = atomic_fetch_sub(&mutex->lock, 1)) != 1) {
atomic_store_explicit(&mutex->lock, 0, memory_order_release);
_futex_wake(&mutex->lock, 1, mutex->pshared);
}
return 0;
case PTHREAD_MUTEX_ERRORCHECK:
me = gettid();
owner = atomic_load_explicit(&mutex->lock, memory_order_relaxed);
if (owner != me) {
}
t = gettid();
if (mutex->type == PTHREAD_MUTEX_ERRORCHECK) {
c = atomic_load_explicit(&mutex->lock, memory_order_relaxed);
if ((c & 0x000fffff) != t) {
assert(!"permlock");
return EPERM;
}
// fallthrough
case PTHREAD_MUTEX_RECURSIVE:
if (--mutex->reent) return 0;
atomic_store_explicit(&mutex->lock, 0, memory_order_relaxed);
if ((IsLinux() || IsOpenbsd()) &&
atomic_load_explicit(&mutex->waits, memory_order_relaxed) > 0) {
return _futex_wake(&mutex->lock, 1, mutex->pshared);
}
c = atomic_fetch_sub(&mutex->lock, 0x00100000);
if ((c & 0x0ff00000) == 0x00100000) {
atomic_store_explicit(&mutex->lock, 0, memory_order_release);
if ((c & 0xf0000000) == 0x20000000) {
_futex_wake(&mutex->lock, 1, mutex->pshared);
}
}
return 0;
default:
assert(!"badlock");
return EINVAL;
}
}

View file

@ -16,12 +16,8 @@
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/intrin/weaken.h"
#include "libc/calls/strace.internal.h"
#include "libc/dce.h"
#include "libc/nt/console.h"
#include "libc/nt/process.h"
#include "libc/nt/runtime.h"
#include "libc/intrin/weaken.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/stdio.h"

View file

@ -1,15 +0,0 @@
#ifndef COSMOPOLITAN_LIBC_THREAD_INTERNAL_H_
#define COSMOPOLITAN_LIBC_THREAD_INTERNAL_H_
#include "libc/thread/thread.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
hidden extern uint64_t _pthread_key_usage[(PTHREAD_KEYS_MAX + 63) / 64];
hidden extern pthread_key_dtor _pthread_key_dtor[PTHREAD_KEYS_MAX];
hidden extern _Thread_local void *_pthread_keys[PTHREAD_KEYS_MAX];
void _pthread_key_destruct(void *[PTHREAD_KEYS_MAX]);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_THREAD_INTERNAL_H_ */

View file

@ -67,6 +67,10 @@ struct PosixThread {
pthread_attr_t attr;
};
hidden extern uint64_t _pthread_key_usage[(PTHREAD_KEYS_MAX + 63) / 64];
hidden extern pthread_key_dtor _pthread_key_dtor[PTHREAD_KEYS_MAX];
hidden extern _Thread_local void *_pthread_keys[PTHREAD_KEYS_MAX];
int _pthread_reschedule(struct PosixThread *) hidden;
int _pthread_setschedparam_freebsd(int, int, const struct sched_param *) hidden;
void _pthread_free(struct PosixThread *) hidden;
@ -75,6 +79,7 @@ void _pthread_wait(struct PosixThread *) hidden;
void _pthread_zombies_add(struct PosixThread *) hidden;
void _pthread_zombies_decimate(void) hidden;
void _pthread_zombies_harvest(void) hidden;
void _pthread_key_destruct(void *[PTHREAD_KEYS_MAX]);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

View file

@ -36,7 +36,6 @@
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
#include "libc/thread/internal.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/spawn.h"
#include "libc/thread/thread.h"

View file

@ -17,7 +17,8 @@
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/errno.h"
#include "libc/thread/internal.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
/**

View file

@ -19,7 +19,8 @@
#include "libc/errno.h"
#include "libc/nexgen32e/bsr.h"
#include "libc/runtime/runtime.h"
#include "libc/thread/internal.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
/**

View file

@ -17,7 +17,8 @@
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/errno.h"
#include "libc/thread/internal.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
/**
* Deletes TLS slot.

View file

@ -17,7 +17,7 @@
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/nexgen32e/bsr.h"
#include "libc/thread/internal.h"
#include "libc/thread/posixthread.internal.h"
void _pthread_key_destruct(void *key[PTHREAD_KEYS_MAX]) {
int i, j;

View file

@ -16,7 +16,7 @@
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/thread/internal.h"
#include "libc/thread/posixthread.internal.h"
// tls value slots for pthread keys api
_Thread_local void *_pthread_keys[PTHREAD_KEYS_MAX];

View file

@ -18,8 +18,8 @@
*/
#include "libc/assert.h"
#include "libc/errno.h"
#include "libc/thread/thread.h"
#include "libc/str/str.h"
#include "libc/thread/thread.h"
/**
* Destroys mutex.
@ -28,7 +28,7 @@
* @raise EINVAL if mutex is locked in our implementation
*/
int pthread_mutex_destroy(pthread_mutex_t *mutex) {
if (mutex->lock || mutex->waits) {
if (mutex->lock) {
assert(!"deadlock");
return EINVAL;
}

View file

@ -17,8 +17,8 @@
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/errno.h"
#include "libc/thread/internal.h"
#include "libc/thread/tls.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
/**
* Sets value of TLS slot for current thread.

View file

@ -17,11 +17,11 @@
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/intrin/atomic.h"
#include "libc/thread/thread.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/spawn.h"
#include "libc/thread/thread.h"
/**
* @fileoverview Memory collector for detached threads.

View file

@ -53,9 +53,7 @@ typedef struct pthread_spinlock_s {
typedef struct pthread_mutex_s {
char type;
char pshared;
int reent;
_Atomic(int) lock;
_Atomic(int) waits;
} pthread_mutex_t;
typedef struct pthread_mutexattr_s {

View file

@ -560,7 +560,7 @@ TEST(pledge_openbsd, bigSyscalls) {
int LockWorker(void *arg, int tid) {
flockfile(stdout);
ASSERT_EQ(gettid(), stdout->lock.lock);
ASSERT_EQ(gettid(), stdout->lock.lock & 0x000fffff);
funlockfile(stdout);
return 0;
}

View file

@ -19,13 +19,13 @@
#include "libc/calls/calls.h"
#include "libc/calls/struct/timespec.h"
#include "libc/intrin/atomic.h"
#include "libc/thread/thread.h"
#include "libc/mem/mem.h"
#include "libc/runtime/gc.internal.h"
#include "libc/testlib/ezbench.h"
#include "libc/testlib/testlib.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/spawn.h"
#include "libc/thread/thread.h"
int THREADS = 16;
int ITERATIONS = 100;

View file

@ -27,7 +27,6 @@
#include "libc/macros.internal.h"
#include "libc/math.h"
#include "libc/mem/mem.h"
#include "libc/thread/tls.h"
#include "libc/runtime/gc.internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
@ -38,8 +37,9 @@
#include "libc/sysv/consts/rlimit.h"
#include "libc/testlib/ezbench.h"
#include "libc/testlib/testlib.h"
#include "libc/thread/thread.h"
#include "libc/thread/spawn.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
#define THREADS 8
#define ITERATIONS 512