mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-07-07 03:38:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Revert "Make spin locks go faster"

Browse source 
This reverts commit c8e25d811c.
This commit is contained in:
Justine Tunney 2024-07-25 22:24:32 -07:00
parent 0679cfeb41
commit 02e1cbcd00
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
16 changed files with 122 additions and 149 deletions
Download patch file Download diff file Expand all files Collapse all files

1
libc/intrin/cxalock.c
View file

@ -35,5 +35,4 @@ void __cxa_unlock(void) {
__attribute__((__constructor__(60))) static textstartup void __cxa_init() {
pthread_atfork(__cxa_lock, __cxa_unlock, __cxa_wipe);
__cxa_wipe();
}

20
libc/intrin/maps.c
View file

@ -91,10 +91,22 @@ privileged bool __maps_lock(void) {
tib = __get_tls_privileged();
if (atomic_fetch_add_explicit(&tib->tib_relock_maps, 1, memory_order_relaxed))
return true;
while (atomic_exchange_explicit(&__maps.lock, 1, memory_order_acquire))
for (;;)
if (!atomic_load_explicit(&__maps.lock, memory_order_relaxed))
break;
int backoff = 0;
while (atomic_exchange_explicit(&__maps.lock, 1, memory_order_acquire)) {
if (backoff < 7) {
volatile int i;
for (i = 0; i != 1 << backoff; i++) {
}
backoff++;
} else {
// STRACE("pthread_delay_np(__maps)");
#if defined(__GNUC__) && defined(__aarch64__)
__asm__ volatile("yield");
#elif defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__))
__asm__ volatile("pause");
#endif
}
}
return false;
}

66
libc/intrin/pthread_atfork_actual.c
View file

@ -16,11 +16,21 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/atomic.h"
#include "libc/calls/state.internal.h"
#include "libc/cosmo.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/strace.h"
#include "libc/macros.internal.h"
#include "libc/proc/proc.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
struct AtFork {
struct AtFork *p[2];
@ -28,16 +38,16 @@ struct AtFork {
};
static struct AtForks {
pthread_mutex_t lock;
pthread_spinlock_t lock;
struct AtFork *list;
struct AtFork pool[256];
int allocated;
} _atforks = {
PTHREAD_MUTEX_INITIALIZER,
};
struct AtFork pool[64];
atomic_int allocated;
} _atforks;
static void _pthread_onfork(int i, const char *op) {
struct AtFork *a;
if (!i)
pthread_spin_lock(&_atforks.lock);
for (a = _atforks.list; a; a = a->p[!i]) {
if (a->f[i]) {
STRACE("pthread_atfork(%s, %t)", op, a->f[i]);
@ -45,41 +55,47 @@ static void _pthread_onfork(int i, const char *op) {
}
_atforks.list = a;
}
if (i)
pthread_spin_unlock(&_atforks.lock);
}
void _pthread_onfork_prepare(void) {
pthread_mutex_lock(&_atforks.lock);
_pthread_onfork(0, "prepare");
}
void _pthread_onfork_parent(void) {
_pthread_onfork(1, "parent");
pthread_mutex_unlock(&_atforks.lock);
}
void _pthread_onfork_child(void) {
pthread_mutex_init(&_atforks.lock, 0);
_pthread_onfork(2, "child");
}
static struct AtFork *_pthread_atfork_alloc(void) {
int i, n = ARRAYLEN(_atforks.pool);
if (atomic_load_explicit(&_atforks.allocated, memory_order_relaxed) < n &&
(i = atomic_fetch_add(&_atforks.allocated, 1)) < n) {
return _atforks.pool + i;
} else {
return 0;
}
}
int _pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) {
int rc;
struct AtFork *a;
pthread_mutex_lock(&_atforks.lock);
if (_atforks.allocated < ARRAYLEN(_atforks.pool)) {
a = &_atforks.pool[_atforks.allocated++];
a->f[0] = prepare;
a->f[1] = parent;
a->f[2] = child;
a->p[0] = 0;
a->p[1] = _atforks.list;
if (_atforks.list)
_atforks.list->p[0] = a;
_atforks.list = a;
rc = 0;
} else {
rc = ENOMEM;
}
pthread_mutex_unlock(&_atforks.lock);
if (!(a = _pthread_atfork_alloc()))
return ENOMEM;
a->f[0] = prepare;
a->f[1] = parent;
a->f[2] = child;
pthread_spin_lock(&_atforks.lock);
a->p[0] = 0;
a->p[1] = _atforks.list;
if (_atforks.list)
_atforks.list->p[0] = a;
_atforks.list = a;
pthread_spin_unlock(&_atforks.lock);
rc = 0;
return rc;
}

26
libc/intrin/pthread_mutex_lock.c
View file

@ -31,16 +31,17 @@
#include "third_party/nsync/futex.internal.h"
#include "third_party/nsync/mu.h"
static void pthread_mutex_lock_spin(atomic_int *word) {
static void pthread_mutex_lock_naive(pthread_mutex_t *mutex, uint64_t word) {
int backoff = 0;
uint64_t lock;
for (;;) {
if (!atomic_exchange_explicit(word, 1, memory_order_acquire))
break;
for (;;) {
if (!atomic_load_explicit(word, memory_order_relaxed))
break;
backoff = pthread_delay_np(word, backoff);
}
word = MUTEX_UNLOCK(word);
lock = MUTEX_LOCK(word);
if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
memory_order_acquire,
memory_order_relaxed))
return;
backoff = pthread_delay_np(mutex, backoff);
}
}
@ -95,12 +96,7 @@ static errno_t pthread_mutex_lock_recursive(pthread_mutex_t *mutex,
mutex->_pid = __pid;
return 0;
}
for (;;) {
word = atomic_load_explicit(&mutex->_word, memory_order_relaxed);
if (!MUTEX_LOCKED(word))
break;
backoff = pthread_delay_np(mutex, backoff);
}
backoff = pthread_delay_np(mutex, backoff);
}
}
@ -125,7 +121,7 @@ static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex) {
if (_weaken(nsync_futex_wait_)) {
pthread_mutex_lock_drepper(&mutex->_futex, MUTEX_PSHARED(word));
} else {
pthread_mutex_lock_spin(&mutex->_futex);
pthread_mutex_lock_naive(mutex, word);
}
return 0;
}

12
libc/intrin/pthread_mutex_trylock.c
View file

@ -27,8 +27,14 @@
#include "third_party/nsync/futex.internal.h"
#include "third_party/nsync/mu.h"
static errno_t pthread_mutex_trylock_spin(atomic_int *word) {
if (!atomic_exchange_explicit(word, 1, memory_order_acquire))
static errno_t pthread_mutex_trylock_naive(pthread_mutex_t *mutex,
uint64_t word) {
uint64_t lock;
word = MUTEX_UNLOCK(word);
lock = MUTEX_LOCK(word);
if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
memory_order_acquire,
memory_order_relaxed))
return 0;
return EBUSY;
}
@ -110,7 +116,7 @@ errno_t pthread_mutex_trylock(pthread_mutex_t *mutex) {
if (_weaken(nsync_futex_wait_)) {
return pthread_mutex_trylock_drepper(&mutex->_futex);
} else {
return pthread_mutex_trylock_spin(&mutex->_futex);
return pthread_mutex_trylock_naive(mutex, word);
}
}

7
libc/intrin/pthread_mutex_unlock.c
View file

@ -28,8 +28,9 @@
#include "third_party/nsync/futex.internal.h"
#include "third_party/nsync/mu.h"
static void pthread_mutex_unlock_spin(atomic_int *word) {
atomic_store_explicit(word, 0, memory_order_release);
static void pthread_mutex_unlock_naive(pthread_mutex_t *mutex, uint64_t word) {
uint64_t lock = MUTEX_UNLOCK(word);
atomic_store_explicit(&mutex->_word, lock, memory_order_release);
}
// see "take 3" algorithm in "futexes are tricky" by ulrich drepper
@ -101,7 +102,7 @@ errno_t pthread_mutex_unlock(pthread_mutex_t *mutex) {
if (_weaken(nsync_futex_wake_)) {
pthread_mutex_unlock_drepper(&mutex->_futex, MUTEX_PSHARED(word));
} else {
pthread_mutex_unlock_spin(&mutex->_futex);
pthread_mutex_unlock_naive(mutex, word);
}
return 0;
}

30
libc/intrin/pthread_spin_destroy.c Normal file
View file

@ -0,0 +1,30 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/atomic.h"
#include "libc/thread/thread.h"
/**
* Destroys spin lock.
*
* @return 0 on success, or errno on error
*/
errno_t pthread_spin_destroy(pthread_spinlock_t *spin) {
atomic_store_explicit(&spin->_lock, -1, memory_order_relaxed);
return 0;
}

34
libc/intrin/pthread_spin_init.c Normal file
View file

@ -0,0 +1,34 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/atomic.h"
#include "libc/thread/thread.h"
/**
* Initializes spin lock.
*
* @param pshared is ignored, since this implementation always permits
* multiple processes to operate on the same spin locks
* @return 0 on success, or errno on error
* @see pthread_spin_destroy
* @see pthread_spin_lock
*/
errno_t pthread_spin_init(pthread_spinlock_t *spin, int pshared) {
atomic_store_explicit(&spin->_lock, 0, memory_order_relaxed);
return 0;
}

45
libc/intrin/pthread_spin_lock.c Normal file
View file

@ -0,0 +1,45 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/atomic.h"
#include "libc/thread/thread.h"
/**
* Acquires spin lock, e.g.
*
* pthread_spinlock_t lock;
* pthread_spin_init(&lock, PTHREAD_PROCESS_PRIVATE);
* pthread_spin_lock(&lock);
* // do work...
* pthread_spin_unlock(&lock);
* pthread_spin_destroy(&lock);
*
* This function has undefined behavior when `spin` wasn't intialized,
* was destroyed, or if the lock's already held by the calling thread.
*
* @return 0 on success, or errno on error
* @see pthread_spin_trylock
* @see pthread_spin_unlock
* @see pthread_spin_init
*/
errno_t pthread_spin_lock(pthread_spinlock_t *spin) {
while (atomic_exchange_explicit(&spin->_lock, 1, memory_order_acquire)) {
pthread_pause_np();
}
return 0;
}

38
libc/intrin/pthread_spin_trylock.c Normal file
View file

@ -0,0 +1,38 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/thread/thread.h"
/**
* Acquires spin lock if available.
*
* This function has undefined behavior when `spin` wasn't intialized,
* was destroyed, or if the lock's already held by the calling thread.
*
* @return 0 on success, or errno on error
* @raise EBUSY if lock is already held
*/
errno_t pthread_spin_trylock(pthread_spinlock_t *spin) {
if (!atomic_exchange_explicit(&spin->_lock, 1, memory_order_acquire)) {
return 0;
} else {
return EBUSY;
}
}

34
libc/intrin/pthread_spin_unlock.c Normal file
View file

@ -0,0 +1,34 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/atomic.h"
#include "libc/thread/thread.h"
/**
* Releases spin lock.
*
* Calling this function when the lock isn't held by the calling thread
* has undefined behavior.
*
* @return 0 on success, or errno on error
* @see pthread_spin_lock
*/
errno_t pthread_spin_unlock(pthread_spinlock_t *spin) {
atomic_store_explicit(&spin->_lock, 0, memory_order_release);
return 0;
}