mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-06-26 22:38:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Rewrite recursive mutex code

Browse source
This commit is contained in:
Justine Tunney 2022-09-10 09:14:40 -07:00
parent bae7367774
commit cfcf5918bc
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
27 changed files with 164 additions and 257 deletions
Download patch file Download diff file Expand all files Collapse all files

7
libc/intrin/futex.S
View file

@ -20,11 +20,14 @@
#include "libc/macros.internal.h"
.privileged
_futex: mov %rcx,%r10
_futex: push %rbp
mov %rsp,%rbp
mov %rcx,%r10
mov __NR_futex,%eax
clc
syscall
jnc 1f
neg %eax
1: ret
1: pop %rbp
ret
.endfn _futex,globl,hidden

2
libc/intrin/gettid.c
View file

@ -16,11 +16,9 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/calls/state.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/thread/tls.h"
/**

26
libc/intrin/kprintf.greg.c
View file

@ -181,19 +181,22 @@ privileged static void klog(const char *b, size_t n) {
: "=a"(rax), "=D"(rdi), "=S"(rsi), "=d"(rdx)
: "0"(__NR_write), "1"(2), "2"(b), "3"(n)
: "rcx", "r8", "r9", "r10", "r11", "memory", "cc");
if (rax < 0) {
notpossible;
}
}
}
privileged static size_t kformat(char *b, size_t n, const char *fmt,
va_list va) {
int si;
int si, y;
wint_t t, u;
const char *abet;
signed char type;
const char *s, *f;
unsigned long long x;
unsigned i, j, m, rem, sign, hash, cols, prec;
char c, *p, *e, pdot, zero, flip, dang, base, quot, uppr, z[128];
char c, *p, *e, pdot, zero, flip, dang, base, quot, uppr, ansi, z[128];
if (kistextpointer(b) || kisdangerous(b)) n = 0;
if (!kistextpointer(fmt)) fmt = "!!WONTFMT";
p = b;
@ -314,6 +317,16 @@ privileged static size_t kformat(char *b, size_t n, const char *fmt,
} else {
x = __get_tls_privileged()->tib_tid;
}
if (!__nocolor && p + 7 <= e) {
*p++ = '\e';
*p++ = '[';
*p++ = '1';
*p++ = ';';
*p++ = '3';
*p++ = '0' + x % 8;
*p++ = 'm';
ansi = true;
}
} else {
x = 666;
}
@ -690,6 +703,15 @@ privileged static size_t kformat(char *b, size_t n, const char *fmt,
}
break;
}
if (ansi) {
if (p + 4 <= e) {
*p++ = '\e';
*p++ = '[';
*p++ = '0';
*p++ = 'm';
}
ansi = false;
}
break;
}
}

44
libc/intrin/lockxadd.c
View file

@ -1,44 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/lockxadd.h"
#include "libc/runtime/runtime.h"
/**
* Compares and exchanges w/ lock prefix.
*
* @param ifthing is uint𝑘_t[hasatleast 1] where 𝑘 ∈ {8,16,32,64}
* @param size is automatically supplied by macro wrapper
* @return value at location `*ifthing` *before* addition
* @see InterlockedAdd() for a very similar API
* @see xadd() if only written by one thread
*/
intptr_t(_lockxadd)(void *ifthing, intptr_t replaceitwithme, size_t size) {
switch (size) {
case 1:
return _lockxadd((int8_t *)ifthing, (int8_t)replaceitwithme);
case 2:
return _lockxadd((int16_t *)ifthing, (int16_t)replaceitwithme);
case 4:
return _lockxadd((int32_t *)ifthing, (int32_t)replaceitwithme);
case 8:
return _lockxadd((int64_t *)ifthing, (int64_t)replaceitwithme);
default:
abort();
}
}

42
libc/intrin/lockxchg.c
View file

@ -1,42 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/lockxchg.h"
/**
* Compares and exchanges w/ lock prefix.
*
* @param memory is uint𝑘_t[hasatleast 1] where 𝑘 ∈ {8,16,32,64}
* @param size is automatically supplied by macro wrapper
* @return true if value was exchanged, otherwise false
* @see xchg()
*/
intptr_t(lockxchg)(void *memory, void *localvar, size_t size) {
switch (size) {
case 1:
return lockxchg((int8_t *)memory, (int8_t *)localvar);
case 2:
return lockxchg((int16_t *)memory, (int16_t *)localvar);
case 4:
return lockxchg((int32_t *)memory, (int32_t *)localvar);
case 8:
return lockxchg((int64_t *)memory, (int64_t *)localvar);
default:
return false;
}
}

47
libc/intrin/phtread_attr_setschedpolicy.c
View file

@ -1,47 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/thread/thread.h"
/**
* Sets thread scheduler policy attribute, e.g.
*
* pthread_t id;
* pthread_attr_t attr;
* pthread_attr_init(&attr);
* struct sched_param pri = {sched_get_priority_min(SCHED_OTHER)};
* pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED);
* pthread_attr_setschedpolicy(&attr, SCHED_OTHER);
* pthread_attr_setschedparam(&attr, &pri);
* pthread_create(&id, &attr, func, 0);
* pthread_attr_destroy(&attr);
* pthread_join(id, 0);
*
* @param policy may be one of:
* - `SCHED_OTHER` the default policy
* - `SCHED_FIFO` for real-time scheduling (usually needs root)
* - `SCHED_RR` for round-robin scheduling (usually needs root)
* - `SCHED_IDLE` for lowest effort (Linux and FreeBSD only)
* - `SCHED_BATCH` for "batch" style execution of processes if
* supported (Linux), otherwise it's treated as `SCHED_OTHER`
* @see sched_setscheduler()
*/
int pthread_attr_setschedpolicy(pthread_attr_t *attr, int policy) {
attr->schedpolicy = policy;
return 0;
}

123
libc/intrin/pthread_mutex_lock.c
View file

@ -19,29 +19,9 @@
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/errno.h"
#include "libc/intrin/asmflag.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/futex.internal.h"
#include "libc/thread/thread.h"
#include "libc/linux/futex.h"
#include "libc/thread/tls.h"
#include "libc/sysv/consts/futex.h"
#include "libc/sysv/consts/nr.h"
static int pthread_mutex_lock_spin(pthread_mutex_t *mutex, int expect,
int tries) {
if (tries < 7) {
volatile int i;
for (i = 0; i != 1 << tries; i++) {
}
tries++;
} else {
atomic_fetch_add(&mutex->waits, 1);
_futex_wait(&mutex->lock, expect, mutex->pshared, &(struct timespec){1});
atomic_fetch_sub(&mutex->waits, 1);
}
return tries;
}
/**
* Locks mutex.
@ -74,57 +54,68 @@ static int pthread_mutex_lock_spin(pthread_mutex_t *mutex, int expect,
* pthread_mutex_unlock(&lock);
* pthread_mutex_destroy(&lock);
*
* Alternatively, Cosmopolitan lets you do the folllowing instead:
*
* pthread_mutex_t lock = {PTHREAD_MUTEX_RECURSIVE};
* pthread_mutex_lock(&lock);
* // do work...
* pthread_mutex_unlock(&lock);
*
* @return 0 on success, or error number on failure
* @see pthread_spin_lock()
*/
int pthread_mutex_lock(pthread_mutex_t *mutex) {
int c, me, owner, tries;
switch (mutex->type) {
case PTHREAD_MUTEX_NORMAL:
// From Futexes Are Tricky Version 1.1 § Mutex, Take 3;
// Ulrich Drepper, Red Hat Incorporated, June 27, 2004.
c = 0;
if (!atomic_compare_exchange_strong_explicit(&mutex->lock, &c, 1,
memory_order_acquire,
memory_order_relaxed)) {
if (c != 2) {
c = atomic_exchange_explicit(&mutex->lock, 2, memory_order_acquire);
}
while (c) {
_futex_wait(&mutex->lock, 2, mutex->pshared, 0);
c = atomic_exchange_explicit(&mutex->lock, 2, memory_order_acquire);
}
int c, d, t;
if (mutex->type == PTHREAD_MUTEX_NORMAL) {
// From Futexes Are Tricky Version 1.1 § Mutex, Take 3;
// Ulrich Drepper, Red Hat Incorporated, June 27, 2004.
c = 0;
if (!atomic_compare_exchange_strong_explicit(
&mutex->lock, &c, 1, memory_order_acquire, memory_order_relaxed)) {
if (c != 2) {
c = atomic_exchange_explicit(&mutex->lock, 2, memory_order_acquire);
}
return 0;
case PTHREAD_MUTEX_RECURSIVE:
case PTHREAD_MUTEX_ERRORCHECK:
for (tries = 0, me = gettid();;) {
owner = atomic_load_explicit(&mutex->lock, memory_order_relaxed);
if (!owner && atomic_compare_exchange_weak_explicit(
&mutex->lock, &owner, me, memory_order_acquire,
memory_order_relaxed)) {
break;
} else if (owner == me) {
if (mutex->type != PTHREAD_MUTEX_ERRORCHECK) {
break;
} else {
assert(!"deadlock");
return EDEADLK;
}
}
tries = pthread_mutex_lock_spin(mutex, owner, tries);
while (c) {
_futex_wait(&mutex->lock, 2, mutex->pshared, 0);
c = atomic_exchange_explicit(&mutex->lock, 2, memory_order_acquire);
}
++mutex->reent;
return 0;
default:
assert(!"badlock");
return EINVAL;
}
return 0;
}
t = gettid();
if (mutex->type == PTHREAD_MUTEX_ERRORCHECK) {
c = atomic_load_explicit(&mutex->lock, memory_order_relaxed);
if ((c & 0x000fffff) == t) {
assert(!"deadlock");
return EDEADLK;
}
}
for (;;) {
c = 0;
d = 0x10100000 | t;
if (atomic_compare_exchange_weak_explicit(
&mutex->lock, &c, d, memory_order_acquire, memory_order_relaxed)) {
break;
} else {
if ((c & 0x000fffff) == t) {
if ((c & 0x0ff00000) < 0x0ff00000) {
c = atomic_fetch_add_explicit(&mutex->lock, 0x00100000,
memory_order_relaxed);
break;
} else {
assert(!"recurse");
return EAGAIN;
}
}
if ((c & 0xf0000000) == 0x10000000) {
d = 0x20000000 | c;
if (atomic_compare_exchange_weak_explicit(&mutex->lock, &c, d,
memory_order_acquire,
memory_order_relaxed)) {
c = d;
}
}
if ((c & 0xf0000000) == 0x20000000) {
_futex_wait(&mutex->lock, c, mutex->pshared, 0);
}
}
}
return 0;
}

55
libc/intrin/pthread_mutex_trylock.c
View file

@ -33,36 +33,31 @@
* @raise ENOTRECOVERABLE if `mutex` is corrupted
*/
int pthread_mutex_trylock(pthread_mutex_t *mutex) {
int c, me, owner;
switch (mutex->type) {
case PTHREAD_MUTEX_NORMAL:
c = 0;
if (atomic_compare_exchange_strong_explicit(&mutex->lock, &c, 1,
memory_order_acquire,
memory_order_relaxed)) {
return 0;
} else {
return EBUSY;
}
case PTHREAD_MUTEX_RECURSIVE:
case PTHREAD_MUTEX_ERRORCHECK:
owner = 0;
me = gettid();
if (!atomic_compare_exchange_strong_explicit(&mutex->lock, &owner, me,
memory_order_acquire,
memory_order_relaxed)) {
if (owner == me) {
if (mutex->type == PTHREAD_MUTEX_ERRORCHECK) {
return EBUSY;
}
} else {
return EBUSY;
}
}
++mutex->reent;
int c, d, t;
if (mutex->type == PTHREAD_MUTEX_NORMAL) {
c = 0;
if (atomic_compare_exchange_strong_explicit(
&mutex->lock, &c, 1, memory_order_acquire, memory_order_relaxed)) {
return 0;
default:
assert(!"badlock");
return ENOTRECOVERABLE;
} else {
return EBUSY;
}
}
c = 0;
t = gettid();
d = 0x10100000 | t;
if (!atomic_compare_exchange_strong_explicit(
&mutex->lock, &c, d, memory_order_acquire, memory_order_relaxed)) {
if ((c & 0x000fffff) != t || mutex->type == PTHREAD_MUTEX_ERRORCHECK) {
return EBUSY;
}
if ((c & 0x0ff00000) == 0x0ff00000) {
return EAGAIN;
}
atomic_fetch_add_explicit(&mutex->lock, 0x00100000, memory_order_relaxed);
}
return 0;
}

60
libc/intrin/pthread_mutex_unlock.c
View file

@ -18,7 +18,6 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/futex.internal.h"
@ -31,35 +30,34 @@
* @raises EPERM if in error check mode and not owned by caller
*/
int pthread_mutex_unlock(pthread_mutex_t *mutex) {
int c, me, owner;
switch (mutex->type) {
case PTHREAD_MUTEX_NORMAL:
// From Futexes Are Tricky Version 1.1 § Mutex, Take 3;
// Ulrich Drepper, Red Hat Incorporated, June 27, 2004.
if ((c = atomic_fetch_sub_explicit(&mutex->lock, 1,
memory_order_release)) != 1) {
atomic_store_explicit(&mutex->lock, 0, memory_order_release);
_futex_wake(&mutex->lock, 1, mutex->pshared);
}
return 0;
case PTHREAD_MUTEX_ERRORCHECK:
me = gettid();
owner = atomic_load_explicit(&mutex->lock, memory_order_relaxed);
if (owner != me) {
assert(!"permlock");
return EPERM;
}
// fallthrough
case PTHREAD_MUTEX_RECURSIVE:
if (--mutex->reent) return 0;
atomic_store_explicit(&mutex->lock, 0, memory_order_relaxed);
if ((IsLinux() || IsOpenbsd()) &&
atomic_load_explicit(&mutex->waits, memory_order_relaxed) > 0) {
return _futex_wake(&mutex->lock, 1, mutex->pshared);
}
return 0;
default:
assert(!"badlock");
return EINVAL;
int c, t;
if (mutex->type == PTHREAD_MUTEX_NORMAL) {
// From Futexes Are Tricky Version 1.1 § Mutex, Take 3;
// Ulrich Drepper, Red Hat Incorporated, June 27, 2004.
if ((c = atomic_fetch_sub(&mutex->lock, 1)) != 1) {
atomic_store_explicit(&mutex->lock, 0, memory_order_release);
_futex_wake(&mutex->lock, 1, mutex->pshared);
}
return 0;
}
t = gettid();
if (mutex->type == PTHREAD_MUTEX_ERRORCHECK) {
c = atomic_load_explicit(&mutex->lock, memory_order_relaxed);
if ((c & 0x000fffff) != t) {
assert(!"permlock");
return EPERM;
}
}
c = atomic_fetch_sub(&mutex->lock, 0x00100000);
if ((c & 0x0ff00000) == 0x00100000) {
atomic_store_explicit(&mutex->lock, 0, memory_order_release);
if ((c & 0xf0000000) == 0x20000000) {
_futex_wake(&mutex->lock, 1, mutex->pshared);
}
}
return 0;
}

6
libc/intrin/quick_exit.c
View file

@ -16,12 +16,8 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/weaken.h"
#include "libc/calls/strace.internal.h"
#include "libc/dce.h"
#include "libc/nt/console.h"
#include "libc/nt/process.h"
#include "libc/nt/runtime.h"
#include "libc/intrin/weaken.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/stdio.h"