cosmopolitan/libc/thread/pthread_exit.c
Justine Tunney 3609f65de3
Make malloc() go 200x faster
If pthread_create() is linked into the binary, then the cosmo runtime
will create an independent dlmalloc arena for each core. Whenever the
malloc() function is used it will index `g_heaps[sched_getcpu() / 2]`
to find the arena with the greatest hyperthread / numa locality. This
may be configured via an environment variable. For example if you say
`export COSMOPOLITAN_HEAP_COUNT=1` then you can restore the old ways.
Your process may be configured to have anywhere between 1 - 128 heaps

We need this revision because it makes multithreaded C++ applications
faster. For example, an HTTP server I'm working on that makes extreme
use of the STL went from 16k to 2000k requests per second, after this
change was made. To understand why, try out the malloc_test benchmark
which calls malloc() + realloc() in a loop across many threads, which
sees a a 250x improvement in process clock time and 200x on wall time

The tradeoff is this adds ~25ns of latency to individual malloc calls
compared to MODE=tiny, once the cosmo runtime has transitioned into a
fully multi-threaded state. If you don't need malloc() to be scalable
then cosmo provides many options for you. For starters the heap count
variable above can be set to put the process back in single heap mode
plus you can go even faster still, if you include tinymalloc.inc like
many of the programs in tool/build/.. are already doing since that'll
shave tens of kb off your binary footprint too. Theres also MODE=tiny
which is configured to use just 1 plain old dlmalloc arena by default

Another tradeoff is we need more memory now (except in MODE=tiny), to
track the provenance of memory allocation. This is so allocations can
be freely shared across threads, and because OSes can reschedule code
to different CPUs at any time.
2024-06-05 02:02:14 -07:00

174 lines
6.3 KiB
C

/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/atomic.h"
#include "libc/cxxabi.h"
#include "libc/dce.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/cxaatexit.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/limits.h"
#include "libc/mem/gc.h"
#include "libc/mem/mem.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
#include "third_party/nsync/futex.internal.h"
#include "third_party/nsync/wait_s.internal.h"
void _pthread_unwind(struct PosixThread *pt) {
struct _pthread_cleanup_buffer *cb;
while ((cb = pt->pt_cleanup)) {
pt->pt_cleanup = cb->__prev;
cb->__routine(cb->__arg);
}
}
void _pthread_unkey(struct CosmoTib *tib) {
void *val;
int i, j, gotsome;
pthread_key_dtor dtor;
for (j = 0; j < PTHREAD_DESTRUCTOR_ITERATIONS; ++j) {
for (gotsome = i = 0; i < PTHREAD_KEYS_MAX; ++i) {
if ((val = tib->tib_keys[i]) &&
(dtor = atomic_load_explicit(_pthread_key_dtor + i,
memory_order_relaxed)) &&
dtor != (pthread_key_dtor)-1) {
gotsome = 1;
tib->tib_keys[i] = 0;
dtor(val);
}
}
if (!gotsome) {
break;
}
}
}
/**
* Terminates current POSIX thread.
*
* For example, a thread could terminate early as follows:
*
* pthread_exit((void *)123);
*
* The result value could then be obtained when joining the thread:
*
* void *rc;
* pthread_join(id, &rc);
* assert((intptr_t)rc == 123);
*
* Under normal circumstances a thread can exit by simply returning from
* the callback function that was supplied to pthread_create(). This may
* be used if the thread wishes to exit at any other point in the thread
* lifecycle, in which case this function is responsible for ensuring we
* invoke gc(), _defer(), and pthread_cleanup_push() callbacks, and also
* pthread_key_create() destructors.
*
* If the current thread is an orphaned thread, or is the main thread
* when no other threads were created, then this will terminated your
* process with an exit code of zero. It's not possible to supply a
* non-zero exit status to wait4() via this function.
*
* Once a thread has exited, access to its stack memory is undefined.
* The behavior of calling pthread_exit() from cleanup handlers and key
* destructors is also undefined.
*
* @param rc is reported later to pthread_join()
* @noreturn
*/
wontreturn void pthread_exit(void *rc) {
struct CosmoTib *tib;
struct PosixThread *pt;
enum PosixThreadStatus status, transition;
tib = __get_tls();
pt = (struct PosixThread *)tib->tib_pthread;
pt->pt_flags |= PT_NOCANCEL;
pt->pt_rc = rc;
STRACE("pthread_exit(%p)", rc);
// free resources
_pthread_unwind(pt);
if (_weaken(__cxa_thread_finalize)) {
_weaken(__cxa_thread_finalize)();
}
_pthread_unkey(tib);
if (tib->tib_nsync) {
nsync_waiter_destroy(tib->tib_nsync);
}
_pthread_ungarbage();
_pthread_decimate();
// run atexit handlers if orphaned thread
if (pthread_orphan_np()) {
if (_weaken(__cxa_finalize)) {
_weaken(__cxa_finalize)(NULL);
}
}
// transition the thread to a terminated state
status = atomic_load_explicit(&pt->pt_status, memory_order_acquire);
do {
switch (status) {
case kPosixThreadJoinable:
transition = kPosixThreadTerminated;
break;
case kPosixThreadDetached:
transition = kPosixThreadZombie;
break;
default:
__builtin_unreachable();
}
} while (!atomic_compare_exchange_weak_explicit(
&pt->pt_status, &status, transition, memory_order_release,
memory_order_relaxed));
// make this thread a zombie if it was detached
if (transition == kPosixThreadZombie) {
_pthread_zombify(pt);
}
// check if this is the last survivor
if (pthread_orphan_np()) {
for (const uintptr_t *p = __fini_array_end; p > __fini_array_start;) {
((void (*)(void))(*--p))();
}
_Exit(0);
}
// check if the main thread has died whilst children live
// note that the main thread is joinable by child threads
if (pt->pt_flags & PT_STATIC) {
atomic_store_explicit(&tib->tib_tid, 0, memory_order_release);
nsync_futex_wake_((atomic_int *)&tib->tib_tid, INT_MAX,
!IsWindows() && !IsXnu());
_Exit1(0);
}
// this is a child thread
longjmp(pt->pt_exiter, 1);
}
__weak_reference(pthread_exit, thr_exit);