mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-01-30 19:17:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Make threads faster and more reliable

Browse source 
This change doubles the performance of thread spawning. That's thanks to
our new stack manager, which allows us to avoid zeroing stacks. It gives
us 15µs spawns rather than 30µs spawns on Linux. Also, pthread_exit() is
faster now, since it doesn't need to acquire the pthread GIL. On NetBSD,
that helps us avoid allocating too many semaphores. Even if that happens
we're now able to survive semaphores running out and even memory running
out, when allocating *NSYNC waiter objects. I found a lot more rare bugs
in the POSIX threads runtime that could cause things to crash, if you've
got dozens of threads all spawning and joining dozens of threads. I want
cosmo to be world class production worthy for 2025 so happy holidays all
This commit is contained in:
Justine Tunney 2024-12-18 04:59:02 -08:00
parent 906bd06a5a
commit 624573207e
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
51 changed files with 1006 additions and 321 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
libc/calls/clock_nanosleep-openbsd.c
View file

@ -23,9 +23,9 @@
#include "libc/sysv/consts/clock.h"
#include "libc/sysv/errfuns.h"
int sys_clock_nanosleep_openbsd(int clock, int flags,
const struct timespec *req,
struct timespec *rem) {
relegated int sys_clock_nanosleep_openbsd(int clock, int flags,
const struct timespec *req,
struct timespec *rem) {
int res;
struct timespec start, relative, remainder;
if (!flags) {

23
libc/calls/time.c
View file

@ -16,10 +16,9 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/time.h"
#include "libc/calls/struct/timeval.h"
#include "libc/dce.h"
#include "libc/sysv/errfuns.h"
#include "libc/calls/calls.h"
#include "libc/calls/struct/timespec.h"
#include "libc/sysv/consts/clock.h"
/**
* Returns time as seconds from UNIX epoch.
@ -29,15 +28,11 @@
* @asyncsignalsafe
*/
int64_t time(int64_t *opt_out_ret) {
int64_t secs;
struct timeval tv;
if (gettimeofday(&tv, 0) != -1) {
secs = tv.tv_sec;
if (opt_out_ret) {
*opt_out_ret = secs;
}
} else {
secs = -1;
}
int64_t secs = -1;
struct timespec ts;
if (!clock_gettime(CLOCK_REALTIME, &ts))
secs = ts.tv_sec;
if (opt_out_ret)
*opt_out_ret = secs;
return secs;
}

13
libc/cosmo.h
View file

@ -13,15 +13,24 @@ errno_t cosmo_once(_COSMO_ATOMIC(unsigned) *, void (*)(void)) libcesque;
int systemvpe(const char *, char *const[], char *const[]) libcesque;
char *GetProgramExecutableName(void) libcesque;
void unleaf(void) libcesque;
bool32 IsLinuxModern(void) libcesque;
int __demangle(char *, const char *, size_t) libcesque;
int __is_mangled(const char *) libcesque;
bool32 IsLinuxModern(void) libcesque;
int LoadZipArgs(int *, char ***) libcesque;
int cosmo_args(const char *, char ***) libcesque;
int LoadZipArgs(int *, char ***) libcesque;
int cosmo_futex_wake(_COSMO_ATOMIC(int) *, int, char);
int cosmo_futex_wait(_COSMO_ATOMIC(int) *, int, char, int,
const struct timespec *);
errno_t cosmo_stack_alloc(size_t *, size_t *, void **) libcesque;
errno_t cosmo_stack_free(void *, size_t, size_t) libcesque;
void cosmo_stack_clear(void) libcesque;
void cosmo_stack_setmaxstacks(int) libcesque;
int cosmo_stack_getmaxstacks(void) libcesque;
int __deadlock_check(void *, int) libcesque;
int __deadlock_tracked(void *) libcesque;
void __deadlock_record(void *, int) libcesque;

26
libc/intrin/count.c Normal file
View file

@ -0,0 +1,26 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/atomic.h"
#include "libc/stdalign.h"
#include "libc/thread/thread.h"
// this counter is important because pthread_exit() needs to know if
// it's an orphan thread, without needing to acquire _pthread_lock()
// which causes contention and a file descriptor explosion on netbsd
alignas(64) atomic_uint _pthread_count = 1;

42
libc/intrin/itimer.c Normal file
View file

@ -0,0 +1,42 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/thread/itimer.h"
#include "libc/str/str.h"
struct IntervalTimer __itimer = {
.lock = PTHREAD_MUTEX_INITIALIZER,
.cond = PTHREAD_COND_INITIALIZER,
};
textwindows void __itimer_lock(void) {
pthread_mutex_lock(&__itimer.lock);
}
textwindows void __itimer_unlock(void) {
pthread_mutex_unlock(&__itimer.lock);
}
textwindows void __itimer_wipe_and_reset(void) {
// timers aren't inherited by forked subprocesses
bzero(&__itimer.it, sizeof(__itimer.it));
pthread_mutex_wipe_np(&__itimer.lock);
pthread_cond_init(&__itimer.cond, 0);
__itimer.thread = 0;
__itimer.once = 0;
}

36
libc/intrin/kisdangerous.c Normal file
View file

@ -0,0 +1,36 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/kprintf.h"
#include "libc/intrin/maps.h"
privileged optimizesize bool32 kisdangerous(const void *addr) {
bool32 res = true;
__maps_lock();
if (__maps.maps) {
struct Map *map;
if ((map = __maps_floor(addr)))
if ((const char *)addr >= map->addr &&
(const char *)addr < map->addr + map->size)
res = false;
} else {
res = false;
}
__maps_unlock();
return res;
}

16
libc/intrin/kprintf.greg.c
View file

@ -160,22 +160,6 @@ __funline bool kischarmisaligned(const char *p, signed char t) {
return false;
}
ABI bool32 kisdangerous(const void *addr) {
bool32 res = true;
__maps_lock();
if (__maps.maps) {
struct Map *map;
if ((map = __maps_floor(addr)))
if ((const char *)addr >= map->addr &&
(const char *)addr < map->addr + map->size)
res = false;
} else {
res = false;
}
__maps_unlock();
return res;
}
ABI static void klogclose(long fd) {
#ifdef __x86_64__
long ax = __NR_close;

0
libc/thread/pthread_cond_init.c → libc/intrin/pthread_cond_init.c
View file

3
libc/intrin/pthread_orphan_np.c
View file

@ -16,6 +16,8 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/intrin/atomic.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
@ -28,5 +30,6 @@ int pthread_orphan_np(void) {
res = _pthread_list == _pthread_list->prev &&
_pthread_list == _pthread_list->next;
_pthread_unlock();
unassert(!res || atomic_load(&_pthread_count) <= 1);
return res;
}

3
libc/intrin/pthreadlock.c
View file

@ -16,9 +16,10 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/stdalign.h"
#include "libc/thread/posixthread.internal.h"
pthread_mutex_t __pthread_lock_obj = PTHREAD_MUTEX_INITIALIZER;
alignas(64) pthread_mutex_t __pthread_lock_obj = PTHREAD_MUTEX_INITIALIZER;
void _pthread_lock(void) {
pthread_mutex_lock(&__pthread_lock_obj);

57
libc/intrin/sig.c
View file

@ -696,35 +696,40 @@ textwindows dontinstrument static uint32_t __sig_worker(void *arg) {
}
// unblock stalled asynchronous signals in threads
_pthread_lock();
for (struct Dll *e = dll_first(_pthread_list); e;
e = dll_next(_pthread_list, e)) {
struct PosixThread *pt = POSIXTHREAD_CONTAINER(e);
if (atomic_load_explicit(&pt->pt_status, memory_order_acquire) >=
kPosixThreadTerminated) {
break;
struct PosixThread *mark;
for (;;) {
sigset_t pending, mask;
mark = 0;
_pthread_lock();
for (struct Dll *e = dll_first(_pthread_list); e;
e = dll_next(_pthread_list, e)) {
struct PosixThread *pt = POSIXTHREAD_CONTAINER(e);
if (atomic_load_explicit(&pt->pt_status, memory_order_acquire) >=
kPosixThreadTerminated)
break;
pending = atomic_load_explicit(&pt->tib->tib_sigpending,
memory_order_acquire);
mask =
atomic_load_explicit(&pt->tib->tib_sigmask, memory_order_acquire);
if (pending & ~mask) {
_pthread_ref(pt);
mark = pt;
break;
}
}
sigset_t pending =
atomic_load_explicit(&pt->tib->tib_sigpending, memory_order_acquire);
sigset_t mask =
atomic_load_explicit(&pt->tib->tib_sigmask, memory_order_acquire);
if (pending & ~mask) {
_pthread_ref(pt);
_pthread_unlock();
while (!atomic_compare_exchange_weak_explicit(
&pt->tib->tib_sigpending, &pending, pending & ~mask,
memory_order_acq_rel, memory_order_relaxed)) {
}
while ((pending = pending & ~mask)) {
int sig = bsfl(pending) + 1;
pending &= ~(1ull << (sig - 1));
__sig_killer(pt, sig, SI_KERNEL);
}
_pthread_lock();
_pthread_unref(pt);
_pthread_unlock();
if (!mark)
break;
while (!atomic_compare_exchange_weak_explicit(
&mark->tib->tib_sigpending, &pending, pending & ~mask,
memory_order_acq_rel, memory_order_relaxed)) {
}
while ((pending = pending & ~mask)) {
int sig = bsfl(pending) + 1;
pending &= ~(1ull << (sig - 1));
__sig_killer(mark, sig, SI_KERNEL);
}
}
_pthread_unlock();
// wait until next scheduler quantum
pthread_mutex_unlock(&__sig_worker_lock);

354
libc/intrin/stack.c Normal file
View file

@ -0,0 +1,354 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/stack.h"
#include "libc/assert.h"
#include "libc/atomic.h"
#include "libc/calls/calls.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/cosmo.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/dll.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
#include "libc/thread/thread.h"
/**
* @fileoverview cosmo stack memory manager
*/
#define MAP_ANON_OPENBSD 0x1000
#define MAP_STACK_OPENBSD 0x4000
#define THREADSTACK_CONTAINER(e) DLL_CONTAINER(struct CosmoStack, elem, e)
struct CosmoStack {
struct Dll elem;
void *stackaddr;
size_t stacksize;
size_t guardsize;
};
struct CosmoStacks {
atomic_uint once;
pthread_mutex_t lock;
struct Dll *stacks;
struct Dll *objects;
unsigned count;
};
struct CosmoStacksConfig {
unsigned maxstacks;
};
static struct CosmoStacks cosmo_stacks = {
.lock = PTHREAD_MUTEX_INITIALIZER,
};
static struct CosmoStacksConfig cosmo_stacks_config = {
.maxstacks = 16,
};
void cosmo_stack_lock(void) {
pthread_mutex_lock(&cosmo_stacks.lock);
}
void cosmo_stack_unlock(void) {
pthread_mutex_unlock(&cosmo_stacks.lock);
}
void cosmo_stack_wipe(void) {
pthread_mutex_wipe_np(&cosmo_stacks.lock);
}
static errno_t cosmo_stack_munmap(void *addr, size_t size) {
errno_t r = 0;
errno_t e = errno;
if (!munmap(addr, size)) {
r = errno;
errno = e;
}
return r;
}
static void cosmo_stack_populate(void) {
errno_t e = errno;
void *map = mmap(0, __pagesize, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
errno = e;
if (map != MAP_FAILED) {
struct CosmoStack *ts = map;
int n = __pagesize / sizeof(struct CosmoStack);
for (int i = 0; i < n; ++i) {
dll_init(&ts[i].elem);
dll_make_first(&cosmo_stacks.objects, &ts[i].elem);
}
}
}
static struct Dll *cosmo_stack_decimate(unsigned maxstacks) {
struct Dll *surplus = 0;
while (cosmo_stacks.count > maxstacks) {
struct Dll *e = dll_last(cosmo_stacks.stacks);
dll_remove(&cosmo_stacks.stacks, e);
dll_make_first(&surplus, e);
--cosmo_stacks.count;
}
return surplus;
}
static void cosmo_stack_rehabilitate(struct Dll *stacks) {
struct Dll *e;
for (e = dll_first(stacks); e; e = dll_next(stacks, e))
cosmo_stack_munmap(THREADSTACK_CONTAINER(e)->stackaddr,
THREADSTACK_CONTAINER(e)->stacksize);
cosmo_stack_lock();
dll_make_first(&cosmo_stacks.objects, stacks);
cosmo_stack_unlock();
}
/**
* Empties unused stack cache.
*
* To make POSIX threads as cheap as possible to spawn, we recycle their
* stacks without zeroing their memory. On Linux for an 80kb stack size,
* that makes launching a thread take 40µs rather than 80µs. However the
* stack cache needs to be cleared in certain cases. This is called upon
* exit() automatically but anyone can clear this at any other time too.
*
* @see pthread_decimate_np()
*/
void cosmo_stack_clear(void) {
cosmo_stack_lock();
struct Dll *stacks = cosmo_stacks.stacks;
cosmo_stacks.stacks = 0;
cosmo_stacks.count = 0;
cosmo_stack_unlock();
cosmo_stack_rehabilitate(stacks);
}
/**
* Gets maximum number of unused stacks cosmo should cache.
* @see cosmo_stack_setmaxstacks()
*/
int cosmo_stack_getmaxstacks(void) {
return cosmo_stacks_config.maxstacks;
}
/**
* Sets maximum number of unused stacks cosmo should cache.
*
* This lets you place some limitations on how much stack memory the
* cosmo runtime will cache. This number is a count of stacks rather
* than the number of bytes they contain. Old stacks are freed in a
* least recently used fashion once the cache exceeds this limit.
*
* If this is set to zero, then the cosmo stack allocator enters a
* highly secure hardening mode where cosmo_stack_alloc() zeroes all
* stack memory that's returned.
*
* Setting this to a negative number makes the cache size unlimited.
*
* Please note this limit only applies to stacks that aren't in use.
*
* Your default is sixteen stacks may be cached at any given moment.
*
* If `maxstacks` is less than the current cache size, then surplus
* entries will be evicted and freed before this function returns.
*/
void cosmo_stack_setmaxstacks(int maxstacks) {
cosmo_stack_lock();
cosmo_stacks_config.maxstacks = maxstacks;
struct Dll *stacks = cosmo_stack_decimate(maxstacks);
cosmo_stack_unlock();
cosmo_stack_rehabilitate(stacks);
}
/**
* Allocates stack memory.
*
* This is a caching stack allocator that's used by the POSIX threads
* runtime but you may also find it useful for setcontext() coroutines
* or sigaltstack(). Normally you can get away with using malloc() for
* creating stacks. However some OSes (e.g. OpenBSD) forbid you from
* doing that for anything except sigaltstack(). This API serves to
* abstract all the gory details of gaining authorized memory, and
* additionally implements caching for lightning fast performance.
*
* The stack size must be nonzero. It is rounded up to the granularity
* of the underlying system allocator, which is normally the page size.
* Your parameter will be updated with the selected value upon success.
*
* The guard size specifies how much memory should be protected at the
* bottom of your stack. This is helpful for ensuring stack overflows
* will result in a segmentation fault, rather than corrupting memory
* silently. This may be set to zero, in which case no guard pages will
* be protected. This value is rounded up to the system page size. The
* corrected value will be returned upon success. Your guard size needs
* to be small enough to leave room for at least one memory page in your
* stack size i.e. `guardsize + pagesize <= stacksize` must be the case.
* Otherwise this function will return an `EINVAL` error.
*
* When you're done using your stack, pass it to cosmo_stack_free() so
* it can be recycled. Stacks are only recycled when the `stacksize` and
* `guardsize` parameters are an exact match after correction. Otherwise
* they'll likely be freed eventually, in a least-recently used fashion,
* based upon the configurable cosmo_stack_setmaxstacks() setting.
*
* This function returns 0 on success, or an errno on error. See the
* documentation of mmap() for a list possible errors that may occur.
*/
errno_t cosmo_stack_alloc(size_t *inout_stacksize, //
size_t *inout_guardsize, //
void **out_addr) {
// validate arguments
size_t stacksize = *inout_stacksize;
size_t guardsize = *inout_guardsize;
stacksize = (stacksize + __gransize - 1) & -__gransize;
guardsize = (guardsize + __pagesize - 1) & -__pagesize;
if (guardsize + __pagesize > stacksize)
return EINVAL;
// recycle stack
void *stackaddr = 0;
cosmo_stack_lock();
for (struct Dll *e = dll_first(cosmo_stacks.stacks); e;
e = dll_next(cosmo_stacks.stacks, e)) {
struct CosmoStack *ts = THREADSTACK_CONTAINER(e);
if (ts->stacksize == stacksize && //
ts->guardsize == guardsize) {
dll_remove(&cosmo_stacks.stacks, e);
stackaddr = ts->stackaddr;
dll_make_first(&cosmo_stacks.objects, e);
--cosmo_stacks.count;
break;
}
}
cosmo_stack_unlock();
// create stack
if (!stackaddr) {
errno_t e = errno;
stackaddr = mmap(0, stacksize, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (stackaddr == MAP_FAILED) {
errno_t err = errno;
errno = e;
return err;
}
if (IsOpenbsd())
if (!TellOpenbsdThisIsStackMemory(stackaddr, stacksize))
notpossible;
if (guardsize)
if (mprotect(stackaddr, guardsize, PROT_NONE | PROT_GUARD))
notpossible;
}
// return stack
*inout_stacksize = stacksize;
*inout_guardsize = guardsize;
*out_addr = stackaddr;
return 0;
}
static void cosmo_stack_setup(void) {
atexit(cosmo_stack_clear);
}
/**
* Frees stack memory.
*
* While not strictly required, it's assumed these three values would be
* those returned by an earlier call to cosmo_stack_alloc().
*
* This function returns 0 on success, or an errno on error. The `errno`
* variable is never clobbered. You can only dependably count on this to
* return an error on failure when you say `cosmo_stack_setmaxstacks(0)`
*/
errno_t cosmo_stack_free(void *stackaddr, size_t stacksize, size_t guardsize) {
stacksize = (stacksize + __gransize - 1) & -__gransize;
guardsize = (guardsize + __pagesize - 1) & -__pagesize;
if (guardsize + __pagesize > stacksize)
return EINVAL;
if ((uintptr_t)stackaddr & (__gransize - 1))
return EINVAL;
cosmo_once(&cosmo_stacks.once, cosmo_stack_setup);
cosmo_stack_lock();
struct Dll *surplus = 0;
if (cosmo_stacks_config.maxstacks) {
surplus = cosmo_stack_decimate(cosmo_stacks_config.maxstacks - 1);
struct CosmoStack *ts = 0;
if (dll_is_empty(cosmo_stacks.objects))
cosmo_stack_populate();
struct Dll *e;
if ((e = dll_first(cosmo_stacks.objects))) {
dll_remove(&cosmo_stacks.objects, e);
ts = THREADSTACK_CONTAINER(e);
}
if (ts) {
ts->stackaddr = stackaddr;
ts->stacksize = stacksize;
ts->guardsize = guardsize;
dll_make_first(&cosmo_stacks.stacks, &ts->elem);
++cosmo_stacks.count;
stackaddr = 0;
}
}
cosmo_stack_unlock();
cosmo_stack_rehabilitate(surplus);
errno_t err = 0;
if (stackaddr)
err = cosmo_stack_munmap(stackaddr, stacksize);
return err;
}
relegated bool TellOpenbsdThisIsStackMemory(void *addr, size_t size) {
return __sys_mmap(
addr, size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD, -1,
0, 0) == addr;
}
// OpenBSD only permits RSP to occupy memory that's been explicitly
// defined as stack memory, i.e. `lo <= %rsp < hi` must be the case
relegated errno_t FixupCustomStackOnOpenbsd(pthread_attr_t *attr) {
// get interval
uintptr_t lo = (uintptr_t)attr->__stackaddr;
uintptr_t hi = lo + attr->__stacksize;
// squeeze interval
lo = (lo + __pagesize - 1) & -__pagesize;
hi = hi & -__pagesize;
// tell os it's stack memory
errno_t olderr = errno;
if (!TellOpenbsdThisIsStackMemory((void *)lo, hi - lo)) {
errno_t err = errno;
errno = olderr;
return err;
}
// update attributes with usable stack address
attr->__stackaddr = (void *)lo;
attr->__stacksize = hi - lo;
return 0;
}

14
libc/intrin/stack.h Normal file
View file

@ -0,0 +1,14 @@
#ifndef COSMOPOLITAN_LIBC_STACK_H_
#define COSMOPOLITAN_LIBC_STACK_H_
#include "libc/thread/thread.h"
COSMOPOLITAN_C_START_
void cosmo_stack_lock(void);
void cosmo_stack_unlock(void);
void cosmo_stack_wipe(void);
bool TellOpenbsdThisIsStackMemory(void *, size_t);
errno_t FixupCustomStackOnOpenbsd(pthread_attr_t *);
COSMOPOLITAN_C_END_
#endif /* COSMOPOLITAN_LIBC_STACK_H_ */

40
libc/intrin/ulock.c
View file

@ -17,12 +17,12 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/ulock.h"
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/calls/syscall_support-sysv.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/describeflags.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/strace.h"
#include "libc/intrin/ulock.h"
// XNU futexes
// https://opensource.apple.com/source/xnu/xnu-7195.50.7.100.1/bsd/sys/ulock.h.auto.html
@ -32,6 +32,26 @@ int sys_ulock_wait(uint32_t operation, void *addr, uint64_t value,
uint32_t timeout_micros) asm("sys_futex_cp");
// returns number of other waiters, or -1 w/ errno
//
// - EINTR means a signal handler was called. This is how we support
// things like POSIX thread cancelation.
//
// - EFAULT if XNU couldn't read `addr`. This is normally considered a
// programming error, but with ulock it can actually be a transient
// error due to low memory conditions. Apple recommends retrying.
//
// - ENOMEM means XNU wasn't able to allocate memory for kernel internal
// data structures. Apple doesn't provide any advice on what to do. We
// simply turn this into EAGAIN.
//
// - EAGAIN if XNU told us EFAULT but cosmo believes the address exists.
// This value is also used as a substitute for ENOMEM.
//
// - EINVAL could mean operation is invalid, addr is null or misaligned;
// it could also mean another thread calling ulock on this address was
// configured (via operation) in an inconsistent way.
//
// see also os_sync_wait_on_address.h from xcode sdk
int ulock_wait(uint32_t operation, void *addr, uint64_t value,
uint32_t timeout_micros) {
int rc;
@ -39,12 +59,26 @@ int ulock_wait(uint32_t operation, void *addr, uint64_t value,
LOCKTRACE("ulock_wait(%#x, %p, %lx, %u) → ...", operation, addr, value,
timeout_micros);
rc = sys_ulock_wait(operation, addr, value, timeout_micros);
if (rc == -1) {
if (errno == ENOMEM)
errno = EAGAIN;
if (errno == EFAULT)
if (!kisdangerous(addr))
errno = EAGAIN;
}
LOCKTRACE("ulock_wait(%#x, %p, %lx, %u) → %d% m", operation, addr, value,
timeout_micros, rc);
return rc;
}
// returns -errno
//
// - ENOENT means there wasn't anyone to wake
//
// - EINVAL could mean operation is invalid, addr is null or misaligned;
// it could also mean another thread calling ulock on this address was
// configured (via operation) in an inconsistent way.
//
int ulock_wake(uint32_t operation, void *addr, uint64_t wake_value) {
int rc;
rc = __syscall3i(operation, (long)addr, wake_value, 0x2000000 | 516);

2
libc/mem/leaks.c
View file

@ -79,7 +79,7 @@ void CheckForMemoryLeaks(void) {
// validate usage of this api
if (_weaken(_pthread_decimate))
_weaken(_pthread_decimate)(false);
_weaken(_pthread_decimate)();
if (!pthread_orphan_np())
kprintf("warning: called CheckForMemoryLeaks() from non-orphaned thread\n");

15
libc/proc/fork-nt.c
View file

@ -62,7 +62,7 @@
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/consts/sig.h"
#include "libc/sysv/errfuns.h"
#include "libc/thread/itimer.internal.h"
#include "libc/thread/itimer.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/tls.h"
#ifdef __x86_64__
@ -189,7 +189,7 @@ static textwindows void *Malloc(size_t size) {
}
textwindows void WinMainForked(void) {
jmp_buf jb;
intptr_t jb[5];
int64_t reader;
int64_t savetsc;
uint32_t varlen;
@ -305,14 +305,14 @@ textwindows void WinMainForked(void) {
#endif
// jump back into function below
longjmp(jb, 1);
__builtin_longjmp(jb, 1);
}
textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
char ok;
jmp_buf jb;
char **args;
int rc = -1;
intptr_t jb[5];
struct Proc *proc;
struct CosmoTib *tib;
char16_t pipename[64];
@ -325,7 +325,7 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
return -1;
ftrace_enabled(-1);
strace_enabled(-1);
if (!setjmp(jb)) {
if (!__builtin_setjmp(jb)) {
reader = CreateNamedPipe(__create_pipe_name(pipename), kNtPipeAccessInbound,
kNtPipeTypeByte | kNtPipeReadmodeByte, 1, PIPE_BUF,
PIPE_BUF, 0, &kNtIsInheritable);
@ -467,12 +467,7 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
if (ftrace_stackdigs)
_weaken(__hook)(_weaken(ftrace_hook), _weaken(GetSymbolTable)());
// reset core runtime services
__proc_wipe();
WipeKeystrokes();
if (_weaken(__sig_init))
_weaken(__sig_init)();
if (_weaken(__itimer_wipe))
_weaken(__itimer_wipe)();
// notify pthread join
atomic_store_explicit(&_pthread_static.ptid, GetCurrentThreadId(),
memory_order_release);

25
libc/proc/fork.c
View file

@ -17,6 +17,7 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/calls/sig.internal.h"
#include "libc/calls/state.internal.h"
#include "libc/calls/struct/sigset.internal.h"
#include "libc/calls/struct/timespec.h"
@ -27,6 +28,7 @@
#include "libc/intrin/cxaatexit.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/stack.h"
#include "libc/intrin/strace.h"
#include "libc/intrin/weaken.h"
#include "libc/nt/files.h"
@ -39,6 +41,7 @@
#include "libc/runtime/syslib.internal.h"
#include "libc/stdio/internal.h"
#include "libc/str/str.h"
#include "libc/thread/itimer.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
#include "third_party/dlmalloc/dlmalloc.h"
@ -104,10 +107,6 @@ static void fork_prepare(void) {
pthread_mutex_lock(&supreme_lock);
if (_weaken(_pthread_onfork_prepare))
_weaken(_pthread_onfork_prepare)();
if (IsWindows()) {
pthread_mutex_lock(&__sig_worker_lock);
__proc_lock();
}
fork_prepare_stdio();
__localtime_lock();
__cxa_lock();
@ -117,12 +116,16 @@ static void fork_prepare(void) {
dlmalloc_pre_fork();
__fds_lock();
pthread_mutex_lock(&__rand64_lock_obj);
if (_weaken(cosmo_stack_lock))
_weaken(cosmo_stack_lock)();
__maps_lock();
LOCKTRACE("READY TO LOCK AND ROLL");
}
static void fork_parent(void) {
__maps_unlock();
if (_weaken(cosmo_stack_unlock))
_weaken(cosmo_stack_unlock)();
pthread_mutex_unlock(&__rand64_lock_obj);
__fds_unlock();
dlmalloc_post_fork_parent();
@ -132,10 +135,6 @@ static void fork_parent(void) {
__cxa_unlock();
__localtime_unlock();
fork_parent_stdio();
if (IsWindows()) {
__proc_unlock();
pthread_mutex_unlock(&__sig_worker_lock);
}
if (_weaken(_pthread_onfork_parent))
_weaken(_pthread_onfork_parent)();
pthread_mutex_unlock(&supreme_lock);
@ -143,6 +142,8 @@ static void fork_parent(void) {
static void fork_child(void) {
nsync_mu_semaphore_sem_fork_child();
if (_weaken(cosmo_stack_wipe))
_weaken(cosmo_stack_wipe)();
pthread_mutex_wipe_np(&__rand64_lock_obj);
pthread_mutex_wipe_np(&__fds_lock_obj);
dlmalloc_post_fork_child();
@ -153,8 +154,13 @@ static void fork_child(void) {
pthread_mutex_wipe_np(&__cxa_lock_obj);
pthread_mutex_wipe_np(&__localtime_lock_obj);
if (IsWindows()) {
__proc_wipe();
// we don't bother locking the proc/itimer/sig locks above since
// their state is reset in the forked child. nothing to protect.
__proc_wipe_and_reset();
__itimer_wipe_and_reset();
pthread_mutex_wipe_np(&__sig_worker_lock);
if (_weaken(__sig_init))
_weaken(__sig_init)();
}
if (_weaken(_pthread_onfork_child))
_weaken(_pthread_onfork_child)();
@ -211,6 +217,7 @@ int _fork(uint32_t dwCreationFlags) {
memory_order_relaxed);
}
dll_make_first(&_pthread_list, &pt->list);
atomic_store_explicit(&_pthread_count, 1, memory_order_relaxed);
// get new system thread handle
intptr_t syshand = 0;

3
libc/proc/proc.c
View file

@ -268,7 +268,8 @@ textwindows void __proc_unlock(void) {
/**
* Resets process tracker from forked child.
*/
textwindows void __proc_wipe(void) {
textwindows void __proc_wipe_and_reset(void) {
// TODO(jart): Should we preserve this state in forked children?
pthread_mutex_t lock = __proc.lock;
bzero(&__proc, sizeof(__proc));
__proc.lock = lock;

2
libc/proc/proc.internal.h
View file

@ -41,7 +41,6 @@ struct Procs {
extern struct Procs __proc;
void __proc_wipe(void) libcesque;
void __proc_lock(void) libcesque;
void __proc_unlock(void) libcesque;
int64_t __proc_handle(int) libcesque;
@ -49,6 +48,7 @@ int64_t __proc_search(int) libcesque;
struct Proc *__proc_new(void) libcesque;
void __proc_add(struct Proc *) libcesque;
void __proc_free(struct Proc *) libcesque;
void __proc_wipe_and_reset(void) libcesque;
int __proc_harvest(struct Proc *, bool) libcesque;
int sys_wait4_nt(int, int *, int, struct rusage *) libcesque;

87
libc/runtime/clone.c
View file

@ -29,6 +29,7 @@
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/describeflags.h"
#include "libc/intrin/strace.h"
#include "libc/intrin/ulock.h"
#include "libc/intrin/weaken.h"
#include "libc/limits.h"
@ -56,6 +57,7 @@
#include "libc/sysv/errfuns.h"
#include "libc/thread/freebsd.internal.h"
#include "libc/thread/openbsd.internal.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
#include "libc/thread/xnu.internal.h"
@ -188,6 +190,7 @@ XnuThreadMain(void *pthread, // rdi
struct CloneArgs *wt, // r8
unsigned xnuflags) { // r9
int ax;
wt->tid = tid;
*wt->ctid = tid;
*wt->ptid = tid;
@ -259,7 +262,7 @@ static errno_t CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags,
// we can't use address sanitizer because:
// 1. __asan_handle_no_return wipes stack [todo?]
static wontreturn void OpenbsdThreadMain(void *p) {
relegated static wontreturn void OpenbsdThreadMain(void *p) {
struct CloneArgs *wt = p;
*wt->ctid = wt->tid;
wt->func(wt->arg, wt->tid);
@ -276,9 +279,9 @@ static wontreturn void OpenbsdThreadMain(void *p) {
__builtin_unreachable();
}
static errno_t CloneOpenbsd(int (*func)(void *, int), char *stk, size_t stksz,
int flags, void *arg, void *tls, atomic_int *ptid,
atomic_int *ctid) {
relegated errno_t CloneOpenbsd(int (*func)(void *, int), char *stk,
size_t stksz, int flags, void *arg, void *tls,
atomic_int *ptid, atomic_int *ctid) {
int rc;
intptr_t sp;
struct __tfork *tf;
@ -299,10 +302,8 @@ static errno_t CloneOpenbsd(int (*func)(void *, int), char *stk, size_t stksz,
tf->tf_tcb = flags & CLONE_SETTLS ? tls : 0;
tf->tf_tid = &wt->tid;
if ((rc = __tfork_thread(tf, sizeof(*tf), OpenbsdThreadMain, wt)) >= 0) {
npassert(rc);
if (flags & CLONE_PARENT_SETTID) {
if (flags & CLONE_PARENT_SETTID)
*ptid = rc;
}
return 0;
} else {
return -rc;
@ -314,18 +315,20 @@ static errno_t CloneOpenbsd(int (*func)(void *, int), char *stk, size_t stksz,
static wontreturn void NetbsdThreadMain(void *arg, // rdi
int (*func)(void *, int), // rsi
int *tid, // rdx
atomic_int *ctid, // rcx
int *ztid) { // r9
int flags, // rdx
atomic_int *ctid) { // rcx
int ax, dx;
// TODO(jart): Why are we seeing flakes where *tid is zero?
// ax = *tid;
static atomic_int clobber;
atomic_int *ztid = &clobber;
ax = sys_gettid();
*ctid = ax;
if (flags & CLONE_CHILD_SETTID)
atomic_store_explicit(ctid, ax, memory_order_release);
if (flags & CLONE_CHILD_CLEARTID)
ztid = ctid;
func(arg, ax);
// we no longer use the stack after this point
// %eax = int __lwp_exit(void);
asm volatile("movl\t$0,%2\n\t" // *wt->ztid = 0
asm volatile("movl\t$0,%2\n\t" // *ztid = 0
"syscall" // __lwp_exit()
: "=a"(ax), "=d"(dx), "=m"(*ztid)
: "0"(310)
@ -340,7 +343,6 @@ static int CloneNetbsd(int (*func)(void *, int), char *stk, size_t stksz,
// second-class API, intended to help Linux folks migrate to this.
int ax;
bool failed;
atomic_int *tid;
intptr_t dx, sp;
static bool once;
struct ucontext_netbsd *ctx;
@ -357,12 +359,6 @@ static int CloneNetbsd(int (*func)(void *, int), char *stk, size_t stksz,
}
sp = (intptr_t)stk + stksz;
// allocate memory for tid
sp -= sizeof(atomic_int);
sp = sp & -alignof(atomic_int);
tid = (atomic_int *)sp;
*tid = 0;
// align the stack
sp = AlignStack(sp, stk, stksz, 16);
@ -383,9 +379,8 @@ static int CloneNetbsd(int (*func)(void *, int), char *stk, size_t stksz,
ctx->uc_mcontext.rip = (intptr_t)NetbsdThreadMain;
ctx->uc_mcontext.rdi = (intptr_t)arg;
ctx->uc_mcontext.rsi = (intptr_t)func;
ctx->uc_mcontext.rdx = (intptr_t)tid;
ctx->uc_mcontext.rcx = (intptr_t)(flags & CLONE_CHILD_SETTID ? ctid : tid);
ctx->uc_mcontext.r8 = (intptr_t)(flags & CLONE_CHILD_CLEARTID ? ctid : tid);
ctx->uc_mcontext.rdx = flags;
ctx->uc_mcontext.rcx = (intptr_t)ctid;
ctx->uc_flags |= _UC_STACK;
ctx->uc_stack.ss_sp = stk;
ctx->uc_stack.ss_size = stksz;
@ -396,15 +391,15 @@ static int CloneNetbsd(int (*func)(void *, int), char *stk, size_t stksz,
}
// perform the system call
int tid = 0;
asm volatile(CFLAG_ASM("syscall")
: CFLAG_CONSTRAINT(failed), "=a"(ax), "=d"(dx)
: "1"(__NR__lwp_create), "D"(ctx), "S"(LWP_DETACHED), "2"(tid)
: "1"(__NR__lwp_create), "D"(ctx), "S"(LWP_DETACHED), "2"(&tid)
: "rcx", "r8", "r9", "r10", "r11", "memory");
if (!failed) {
npassert(*tid);
if (flags & CLONE_PARENT_SETTID) {
*ptid = *tid;
}
unassert(tid);
if (flags & CLONE_PARENT_SETTID)
*ptid = tid;
return 0;
} else {
return ax;
@ -744,43 +739,47 @@ static int CloneLinux(int (*func)(void *arg, int rc), char *stk, size_t stksz,
*/
errno_t clone(void *func, void *stk, size_t stksz, int flags, void *arg,
void *ptid, void *tls, void *ctid) {
int rc;
errno_t err;
atomic_fetch_add(&_pthread_count, 1);
if (!func) {
rc = EINVAL;
err = EINVAL;
} else if (IsLinux()) {
rc = CloneLinux(func, stk, stksz, flags, arg, tls, ptid, ctid);
err = CloneLinux(func, stk, stksz, flags, arg, tls, ptid, ctid);
} else if (!IsTiny() &&
(flags & ~(CLONE_SETTLS | CLONE_PARENT_SETTID |
CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID)) !=
(CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_FILES |
CLONE_SIGHAND | CLONE_SYSVSEM)) {
rc = EINVAL;
err = EINVAL;
} else if (IsXnu()) {
#ifdef __x86_64__
rc = CloneXnu(func, stk, stksz, flags, arg, tls, ptid, ctid);
err = CloneXnu(func, stk, stksz, flags, arg, tls, ptid, ctid);
#elif defined(__aarch64__)
rc = CloneSilicon(func, stk, stksz, flags, arg, tls, ptid, ctid);
err = CloneSilicon(func, stk, stksz, flags, arg, tls, ptid, ctid);
#else
#error "unsupported architecture"
#endif
} else if (IsFreebsd()) {
rc = CloneFreebsd(func, stk, stksz, flags, arg, tls, ptid, ctid);
err = CloneFreebsd(func, stk, stksz, flags, arg, tls, ptid, ctid);
#ifdef __x86_64__
} else if (IsNetbsd()) {
rc = CloneNetbsd(func, stk, stksz, flags, arg, tls, ptid, ctid);
err = CloneNetbsd(func, stk, stksz, flags, arg, tls, ptid, ctid);
} else if (IsOpenbsd()) {
rc = CloneOpenbsd(func, stk, stksz, flags, arg, tls, ptid, ctid);
err = CloneOpenbsd(func, stk, stksz, flags, arg, tls, ptid, ctid);
} else if (IsWindows()) {
rc = CloneWindows(func, stk, stksz, flags, arg, tls, ptid, ctid);
err = CloneWindows(func, stk, stksz, flags, arg, tls, ptid, ctid);
#endif /* __x86_64__ */
} else {
rc = ENOSYS;
err = ENOSYS;
}
if (SupportsBsd() && rc == EPROCLIM) {
rc = EAGAIN;
}
if (SupportsBsd() && err == EPROCLIM)
err = EAGAIN;
return rc;
if (err)
unassert(atomic_fetch_sub(&_pthread_count, 1) > 1);
return err;
}

2
libc/system/popen.c
View file

@ -87,7 +87,6 @@ FILE *popen(const char *cmdline, const char *mode) {
// "The popen() function shall ensure that any streams from
// previous popen() calls that remain open in the parent
// process are closed in the new child process." -POSIX
__stdio_lock();
for (struct Dll *e = dll_first(__stdio.files); e;
e = dll_next(__stdio.files, e)) {
FILE *f2 = FILE_CONTAINER(e);
@ -96,7 +95,6 @@ FILE *popen(const char *cmdline, const char *mode) {
f2->fd = -1;
}
}
__stdio_unlock();
_Exit(_cocmd(3,
(char *[]){

3
libc/testlib/BUILD.mk
View file

@ -27,6 +27,7 @@ LIBC_TESTLIB_A_HDRS = \
libc/testlib/ezbench.h \
libc/testlib/fastrandomstring.h \
libc/testlib/hyperion.h \
libc/testlib/manystack.h \
libc/testlib/moby.h \
libc/testlib/subprocess.h \
libc/testlib/testlib.h \
@ -70,6 +71,7 @@ LIBC_TESTLIB_A_SRCS_C = \
libc/testlib/globals.c \
libc/testlib/hexequals.c \
libc/testlib/incrementfailed.c \
libc/testlib/manystack.c \
libc/testlib/memoryexists.c \
libc/testlib/seterrno.c \
libc/testlib/shoulddebugbreak.c \
@ -110,6 +112,7 @@ LIBC_TESTLIB_A_DIRECTDEPS = \
LIBC_STR \
LIBC_SYSV \
LIBC_SYSV_CALLS \
LIBC_THREAD \
LIBC_TINYMATH \
LIBC_X \
THIRD_PARTY_COMPILER_RT \

69
libc/testlib/manystack.c Normal file
View file

@ -0,0 +1,69 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/testlib/manystack.h"
#include "libc/atomic.h"
#include "libc/calls/struct/sigaction.h"
#include "libc/calls/struct/sigset.h"
#include "libc/intrin/dll.h"
#include "libc/sysv/consts/sig.h"
#include "libc/thread/posixthread.internal.h"
static atomic_int manystack_gotsig;
static atomic_bool manystack_shutdown;
static void manystack_signal(int sig) {
manystack_gotsig = sig;
}
static void *manystack_thread(void *arg) {
sigset_t ss;
sigfillset(&ss);
sigdelset(&ss, SIGUSR2);
while (!manystack_shutdown) {
sigsuspend(&ss);
if (!manystack_shutdown) {
_pthread_lock();
for (struct Dll *e = dll_first(_pthread_list); e;
e = dll_next(_pthread_list, e)) {
pthread_t th = (pthread_t)POSIXTHREAD_CONTAINER(e);
if (!pthread_equal(th, pthread_self()))
pthread_kill(th, SIGQUIT);
}
_pthread_unlock();
}
}
return 0;
}
pthread_t manystack_start(void) {
sigset_t ss;
pthread_t msh;
sigemptyset(&ss);
sigaddset(&ss, SIGUSR2);
sigprocmask(SIG_BLOCK, &ss, 0);
signal(SIGUSR2, manystack_signal);
pthread_create(&msh, 0, manystack_thread, 0);
return msh;
}
void manystack_stop(pthread_t msh) {
manystack_shutdown = true;
pthread_kill(msh, SIGUSR2);
pthread_join(msh, 0);
}

10
libc/testlib/manystack.h Normal file
View file

@ -0,0 +1,10 @@
#ifndef COSMOPOLITAN_LIBC_TESTLIB_MANYSTACK_H_
#define COSMOPOLITAN_LIBC_TESTLIB_MANYSTACK_H_
#include "libc/thread/thread.h"
COSMOPOLITAN_C_START_
pthread_t manystack_start(void);
void manystack_stop(pthread_t);
COSMOPOLITAN_C_END_
#endif /* COSMOPOLITAN_LIBC_TESTLIB_MANYSTACK_H_ */

2
libc/testlib/testmain.c
View file

@ -156,7 +156,7 @@ int main(int argc, char *argv[]) {
// make sure threads are in a good state
if (_weaken(_pthread_decimate))
_weaken(_pthread_decimate)(false);
_weaken(_pthread_decimate)();
if (_weaken(pthread_orphan_np) && !_weaken(pthread_orphan_np)()) {
tinyprint(2, "error: tests ended with threads still active\n", NULL);
_Exit(1);

34
libc/thread/itimer.c
View file

@ -33,18 +33,13 @@
#include "libc/sysv/consts/sicode.h"
#include "libc/sysv/consts/sig.h"
#include "libc/sysv/errfuns.h"
#include "libc/thread/itimer.internal.h"
#include "libc/thread/itimer.h"
#include "libc/thread/thread2.h"
#include "libc/thread/tls.h"
#ifdef __x86_64__
#define STACK_SIZE 65536
struct IntervalTimer __itimer = {
.lock = PTHREAD_MUTEX_INITIALIZER,
.cond = PTHREAD_COND_INITIALIZER,
};
static textwindows dontinstrument uint32_t __itimer_worker(void *arg) {
struct CosmoTib tls;
char *sp = __builtin_frame_address(0);
@ -55,7 +50,7 @@ static textwindows dontinstrument uint32_t __itimer_worker(void *arg) {
for (;;) {
bool dosignal = false;
struct timeval now, waituntil;
pthread_mutex_lock(&__itimer.lock);
__itimer_lock();
now = timeval_real();
if (timeval_iszero(__itimer.it.it_value)) {
waituntil = timeval_max;
@ -76,13 +71,13 @@ static textwindows dontinstrument uint32_t __itimer_worker(void *arg) {
dosignal = true;
}
}
pthread_mutex_unlock(&__itimer.lock);
__itimer_unlock();
if (dosignal)
__sig_generate(SIGALRM, SI_TIMER);
pthread_mutex_lock(&__itimer.lock);
__itimer_lock();
struct timespec deadline = timeval_totimespec(waituntil);
pthread_cond_timedwait(&__itimer.cond, &__itimer.lock, &deadline);
pthread_mutex_unlock(&__itimer.lock);
__itimer_unlock();
}
return 0;
}
@ -92,39 +87,30 @@ static textwindows void __itimer_setup(void) {
kNtStackSizeParamIsAReservation, 0);
}
textwindows void __itimer_wipe(void) {
// this function is called by fork(), because
// timers aren't inherited by forked subprocesses
bzero(&__itimer, sizeof(__itimer));
}
textwindows int sys_setitimer_nt(int which, const struct itimerval *neu,
struct itimerval *old) {
struct itimerval config;
cosmo_once(&__itimer.once, __itimer_setup);
if (which != ITIMER_REAL || (neu && (!timeval_isvalid(neu->it_value) ||
!timeval_isvalid(neu->it_interval)))) {
!timeval_isvalid(neu->it_interval))))
return einval();
}
if (neu) {
if (neu)
// POSIX defines setitimer() with the restrict keyword but let's
// accommodate the usage setitimer(ITIMER_REAL, &it, &it) anyway
config = *neu;
}
BLOCK_SIGNALS;
pthread_mutex_lock(&__itimer.lock);
__itimer_lock();
if (old) {
old->it_interval = __itimer.it.it_interval;
old->it_value = timeval_subz(__itimer.it.it_value, timeval_real());
}
if (neu) {
if (!timeval_iszero(config.it_value)) {
if (!timeval_iszero(config.it_value))
config.it_value = timeval_add(config.it_value, timeval_real());
}
__itimer.it = config;
pthread_cond_signal(&__itimer.cond);
}
pthread_mutex_unlock(&__itimer.lock);
__itimer_unlock();
ALLOW_SIGNALS;
return 0;
}

4
libc/thread/itimer.internal.h → libc/thread/itimer.h
View file

@ -15,7 +15,9 @@ struct IntervalTimer {
extern struct IntervalTimer __itimer;
void __itimer_wipe(void);
void __itimer_lock(void);
void __itimer_unlock(void);
void __itimer_wipe_and_reset(void);
COSMOPOLITAN_C_END_
#endif /* COSMOPOLITAN_LIBC_ITIMER_H_ */

44
libc/runtime/mapstack.c → libc/thread/mapstack.c
View file

@ -16,18 +16,9 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/cosmo.h"
#include "libc/errno.h"
#include "libc/runtime/stack.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
#define MAP_ANON_OPENBSD 0x1000
#define MAP_STACK_OPENBSD 0x4000
/**
* Allocates stack.
@ -43,28 +34,23 @@
* @return stack bottom address on success, or null w/ errno
*/
void *NewCosmoStack(void) {
char *p;
size_t n = GetStackSize();
if ((p = mmap(0, n, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1,
0)) != MAP_FAILED) {
if (IsOpenbsd() && __sys_mmap(p, n, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD |
MAP_STACK_OPENBSD,
-1, 0, 0) != p)
notpossible;
if (mprotect(p, GetGuardSize(), PROT_NONE | PROT_GUARD))
notpossible;
return p;
} else {
return 0;
}
void *stackaddr;
size_t stacksize = GetStackSize();
size_t guardsize = GetGuardSize();
errno_t err = cosmo_stack_alloc(&stacksize, &guardsize, &stackaddr);
if (!err)
return stackaddr;
errno = err;
return 0;
}
/**
* Frees stack.
*
* @param stk was allocated by NewCosmoStack()
* @param stackaddr was allocated by NewCosmoStack()
* @return 0 on success, or -1 w/ errno
*/
int FreeCosmoStack(void *stk) {
return munmap(stk, GetStackSize());
int FreeCosmoStack(void *stackaddr) {
cosmo_stack_free(stackaddr, GetStackSize(), GetGuardSize());
return 0;
}

7
libc/thread/posixthread.internal.h
View file

@ -74,6 +74,7 @@ struct PosixThread {
int pt_flags; // 0x00: see PT_* constants
atomic_int pt_canceled; // 0x04: thread has bad beliefs
_Atomic(enum PosixThreadStatus) pt_status;
_Atomic(atomic_int *) pt_blocker;
atomic_int ptid; // transitions 0 → tid
atomic_int pt_refs; // prevents decimation
void *(*pt_start)(void *); // creation callback
@ -83,11 +84,10 @@ struct PosixThread {
struct CosmoTib *tib; // middle of tls allocation
struct Dll list; // list of threads
struct _pthread_cleanup_buffer *pt_cleanup;
_Atomic(atomic_int *) pt_blocker;
uint64_t pt_blkmask;
int64_t pt_event;
locale_t pt_locale;
jmp_buf pt_exiter;
intptr_t pt_exiter[5];
pthread_attr_t pt_attr;
atomic_bool pt_intoff;
};
@ -95,6 +95,7 @@ struct PosixThread {
typedef void (*atfork_f)(void);
extern struct Dll *_pthread_list;
extern _Atomic(unsigned) _pthread_count;
extern struct PosixThread _pthread_static;
extern _Atomic(pthread_key_dtor) _pthread_key_dtor[PTHREAD_KEYS_MAX];
@ -103,7 +104,7 @@ int _pthread_setschedparam_freebsd(int, int, const struct sched_param *);
int _pthread_tid(struct PosixThread *) libcesque;
intptr_t _pthread_syshand(struct PosixThread *) libcesque;
long _pthread_cancel_ack(void) libcesque;
void _pthread_decimate(bool) libcesque;
void _pthread_decimate(void) libcesque;
void _pthread_free(struct PosixThread *) libcesque;
void _pthread_lock(void) libcesque;
void _pthread_onfork_child(void) libcesque;

7
libc/thread/pthread_atfork.c
View file

@ -16,6 +16,8 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/atomic.h"
#include "libc/cosmo.h"
#include "libc/errno.h"
#include "libc/intrin/strace.h"
#include "libc/mem/mem.h"
@ -28,13 +30,12 @@ struct AtFork {
};
struct AtForks {
pthread_once_t once;
atomic_uint once;
pthread_mutex_t lock;
struct AtFork *list;
};
static struct AtForks _atforks = {
.once = PTHREAD_ONCE_INIT,
.lock = PTHREAD_MUTEX_INITIALIZER,
};
@ -161,7 +162,7 @@ void _pthread_onfork_child(void) {
* @raise ENOMEM if we require more vespene gas
*/
int pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) {
pthread_once(&_atforks.once, pthread_atfork_init);
cosmo_once(&_atforks.once, pthread_atfork_init);
struct AtFork *a;
if (!(a = calloc(1, sizeof(struct AtFork))))
return ENOMEM;

4
libc/thread/pthread_attr_setdetachstate.c
View file

@ -28,6 +28,10 @@
* pthread_create(0, &attr, func, 0);
* pthread_attr_destroy(&attr);
*
* If you use this, please be warned that your thread might run and exit
* before pthread_create() even returns. You really should assume it can
* not be used with any pthread APIs from the calling thread.
*
* @param detachstate can be one of
* - `PTHREAD_CREATE_JOINABLE` (default)
* - `PTHREAD_CREATE_DETACHED`

1
libc/thread/pthread_attr_setsigmask_np.c
View file

@ -17,6 +17,7 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/str/str.h"
#include "libc/sysv/consts/sig.h"
#include "libc/thread/thread2.h"
/**

106
libc/thread/pthread_create.c
View file

@ -18,10 +18,12 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/calls/sig.internal.h"
#include "libc/calls/struct/sigaltstack.h"
#include "libc/calls/struct/sigset.h"
#include "libc/calls/struct/sigset.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/cosmo.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/fmt/itoa.h"
@ -29,6 +31,7 @@
#include "libc/intrin/describeflags.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/stack.h"
#include "libc/intrin/strace.h"
#include "libc/intrin/weaken.h"
#include "libc/log/internal.h"
@ -48,7 +51,6 @@
#include "libc/str/str.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/clone.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/consts/sig.h"
#include "libc/sysv/consts/ss.h"
@ -65,9 +67,6 @@ __static_yoink("_pthread_onfork_prepare");
__static_yoink("_pthread_onfork_parent");
__static_yoink("_pthread_onfork_child");
#define MAP_ANON_OPENBSD 0x1000
#define MAP_STACK_OPENBSD 0x4000
void _pthread_free(struct PosixThread *pt) {
// thread must be removed from _pthread_list before calling
@ -79,7 +78,8 @@ void _pthread_free(struct PosixThread *pt) {
// unmap stack if the cosmo runtime was responsible for mapping it
if (pt->pt_flags & PT_OWNSTACK)
unassert(!munmap(pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize));
cosmo_stack_free(pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize,
pt->pt_attr.__guardsize);
// free any additional upstream system resources
// our fork implementation wipes this handle in child automatically
@ -99,7 +99,7 @@ void _pthread_free(struct PosixThread *pt) {
free(pt);
}
void _pthread_decimate(bool annihilation_only) {
void _pthread_decimate(void) {
struct PosixThread *pt;
struct Dll *e, *e2, *list = 0;
enum PosixThreadStatus status;
@ -123,17 +123,6 @@ void _pthread_decimate(bool annihilation_only) {
dll_make_first(&list, e);
}
// code like pthread_exit() needs to call this in order to know if
// it's appropriate to run exit() handlers however we really don't
// want to have a thread exiting block on a bunch of __maps locks!
// therefore we only take action if we'll destroy all but the self
if (annihilation_only)
if (!(_pthread_list == _pthread_list->prev &&
_pthread_list == _pthread_list->next)) {
dll_make_last(&_pthread_list, list);
list = 0;
}
// release posix threads gil
_pthread_unlock();
@ -167,11 +156,14 @@ static int PosixThread(void *arg, int tid) {
}
// set long jump handler so pthread_exit can bring control back here
if (!setjmp(pt->pt_exiter)) {
sigdelset(&pt->pt_attr.__sigmask, SIGTHR);
if (!__builtin_setjmp(pt->pt_exiter)) {
// setup signals for new thread
pt->pt_attr.__sigmask &= ~(1ull << (SIGTHR - 1));
if (IsWindows() || IsMetal()) {
atomic_store_explicit(&__get_tls()->tib_sigmask, pt->pt_attr.__sigmask,
memory_order_release);
if (_weaken(__sig_check))
_weaken(__sig_check)();
} else {
sys_sigprocmask(SIG_SETMASK, &pt->pt_attr.__sigmask, 0);
}
@ -189,39 +181,6 @@ static int PosixThread(void *arg, int tid) {
return 0;
}
static bool TellOpenbsdThisIsStackMemory(void *addr, size_t size) {
return __sys_mmap(
addr, size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD, -1,
0, 0) == addr;
}
// OpenBSD only permits RSP to occupy memory that's been explicitly
// defined as stack memory, i.e. `lo <= %rsp < hi` must be the case
static errno_t FixupCustomStackOnOpenbsd(pthread_attr_t *attr) {
// get interval
uintptr_t lo = (uintptr_t)attr->__stackaddr;
uintptr_t hi = lo + attr->__stacksize;
// squeeze interval
lo = (lo + __pagesize - 1) & -__pagesize;
hi = hi & -__pagesize;
// tell os it's stack memory
errno_t olderr = errno;
if (!TellOpenbsdThisIsStackMemory((void *)lo, hi - lo)) {
errno_t err = errno;
errno = olderr;
return err;
}
// update attributes with usable stack address
attr->__stackaddr = (void *)lo;
attr->__stacksize = hi - lo;
return 0;
}
static errno_t pthread_create_impl(pthread_t *thread,
const pthread_attr_t *attr,
void *(*start_routine)(void *), void *arg,
@ -266,37 +225,18 @@ static errno_t pthread_create_impl(pthread_t *thread,
}
} else {
// cosmo is managing the stack
int pagesize = __pagesize;
pt->pt_attr.__guardsize = ROUNDUP(pt->pt_attr.__guardsize, pagesize);
pt->pt_attr.__stacksize = pt->pt_attr.__stacksize;
if (pt->pt_attr.__guardsize + pagesize > pt->pt_attr.__stacksize) {
pt->pt_flags |= PT_OWNSTACK;
errno_t err =
cosmo_stack_alloc(&pt->pt_attr.__stacksize, &pt->pt_attr.__guardsize,
&pt->pt_attr.__stackaddr);
if (err) {
_pthread_free(pt);
return EINVAL;
}
pt->pt_attr.__stackaddr =
mmap(0, pt->pt_attr.__stacksize, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (pt->pt_attr.__stackaddr != MAP_FAILED) {
if (IsOpenbsd())
if (!TellOpenbsdThisIsStackMemory(pt->pt_attr.__stackaddr,
pt->pt_attr.__stacksize))
notpossible;
if (pt->pt_attr.__guardsize)
if (mprotect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize,
PROT_NONE | PROT_GUARD))
notpossible;
}
if (!pt->pt_attr.__stackaddr || pt->pt_attr.__stackaddr == MAP_FAILED) {
rc = errno;
_pthread_free(pt);
errno = e;
if (rc == EINVAL || rc == EOVERFLOW) {
if (err == EINVAL || err == EOVERFLOW) {
return EINVAL;
} else {
return EAGAIN;
}
}
pt->pt_flags |= PT_OWNSTACK;
}
// setup signal stack
@ -338,6 +278,10 @@ static errno_t pthread_create_impl(pthread_t *thread,
dll_make_first(&_pthread_list, &pt->list);
_pthread_unlock();
// if pthread_attr_setdetachstate() was used then it's possible for
// the `pt` object to be freed before this clone call has returned!
_pthread_ref(pt);
// launch PosixThread(pt) in new thread
if ((rc = clone(PosixThread, pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize,
CLONE_VM | CLONE_THREAD | CLONE_FS | CLONE_FILES |
@ -400,8 +344,8 @@ static const char *DescribeHandle(char buf[12], errno_t err, pthread_t *th) {
* │ _lwp_create │
* └──────────────┘
*
* @param thread if non-null is used to output the thread id
* upon successful completion
* @param thread is used to output the thread id upon success, which
* must be non-null
* @param attr points to launch configuration, or may be null
* to use sensible defaults; it must be initialized using
* pthread_attr_init()
@ -417,12 +361,14 @@ static const char *DescribeHandle(char buf[12], errno_t err, pthread_t *th) {
errno_t pthread_create(pthread_t *thread, const pthread_attr_t *attr,
void *(*start_routine)(void *), void *arg) {
errno_t err;
_pthread_decimate(false);
_pthread_decimate();
BLOCK_SIGNALS;
err = pthread_create_impl(thread, attr, start_routine, arg, _SigMask);
ALLOW_SIGNALS;
STRACE("pthread_create([%s], %p, %t, %p) → %s",
DescribeHandle(alloca(12), err, thread), attr, start_routine, arg,
DescribeErrno(err));
if (!err)
_pthread_unref(*(struct PosixThread **)thread);
return err;
}

16
libc/thread/pthread_decimate_np.c
View file

@ -16,22 +16,32 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/cosmo.h"
#include "libc/intrin/stack.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
/**
* Garbage collects POSIX threads runtime.
*
* Let's say you want to run a memory leak detector. You can say:
* This function frees unreferenced zombie threads and empties cache
* memory associated with the Cosmopolitan POSIX threads runtime.
*
* Here's an example use case for this function. Let's say you want to
* create a malloc() memory leak detector. If your program was running
* threads earlier, then there might still be allocations lingering
* around, that'll give you false positives. To fix this, what you would
* do is call the following, right before running your leak detector:
*
* while (!pthread_orphan_np())
* pthread_decimate_np();
*
* To wait until all threads have exited.
* Which will wait until all threads have exited and their memory freed.
*
* @return 0 on success, or errno on error
*/
int pthread_decimate_np(void) {
_pthread_decimate(false);
_pthread_decimate();
cosmo_stack_clear();
return 0;
}

25
libc/thread/pthread_exit.c
View file

@ -69,7 +69,7 @@
* @noreturn
*/
wontreturn void pthread_exit(void *rc) {
int orphan;
unsigned population;
struct CosmoTib *tib;
struct PosixThread *pt;
enum PosixThreadStatus status, transition;
@ -94,10 +94,21 @@ wontreturn void pthread_exit(void *rc) {
__cxa_thread_finalize();
// run atexit handlers if orphaned thread
_pthread_decimate(true);
if ((orphan = pthread_orphan_np()))
if (_weaken(__cxa_finalize))
_weaken(__cxa_finalize)(NULL);
// notice how we avoid acquiring the pthread gil
if (!(population = atomic_fetch_sub(&_pthread_count, 1) - 1)) {
// we know for certain we're an orphan. any other threads that
// exist, will terminate and clear their tid very soon. but...
// some goofball could spawn more threads from atexit handlers
for (;;) {
_pthread_decimate();
if (pthread_orphan_np()) {
if (_weaken(__cxa_finalize))
_weaken(__cxa_finalize)(NULL);
population = atomic_load(&_pthread_count);
break;
}
}
}
// transition the thread to a terminated state
status = atomic_load_explicit(&pt->pt_status, memory_order_acquire);
@ -127,7 +138,7 @@ wontreturn void pthread_exit(void *rc) {
// thread has been terminated. The behavior shall be as if the
// implementation called exit() with a zero argument at thread
// termination time." ──Quoth POSIX.1-2017
if (orphan) {
if (!population) {
for (int i = __fini_array_end - __fini_array_start; i--;)
((void (*)(void))__fini_array_start[i])();
_Exit(0);
@ -143,7 +154,7 @@ wontreturn void pthread_exit(void *rc) {
}
// this is a child thread
longjmp(pt->pt_exiter, 1);
__builtin_longjmp(pt->pt_exiter, 1);
}
__weak_reference(pthread_exit, thr_exit);

4
libc/thread/pthread_kill.c
View file

@ -43,6 +43,8 @@ errno_t pthread_kill(pthread_t thread, int sig) {
int err = 0;
struct PosixThread *pt;
pt = (struct PosixThread *)thread;
if (pt)
_pthread_ref(pt);
if (!thread) {
err = EFAULT;
} else if (!(1 <= sig && sig <= 64)) {
@ -69,5 +71,7 @@ errno_t pthread_kill(pthread_t thread, int sig) {
}
STRACE("pthread_kill(%d, %G) → %s", _pthread_tid(pt), sig,
DescribeErrno(err));
if (pt)
_pthread_unref(pt);
return err;
}

10
libc/thread/sem_open.c
View file

@ -17,10 +17,12 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/atomic.h"
#include "libc/calls/blockcancel.internal.h"
#include "libc/calls/calls.h"
#include "libc/calls/struct/stat.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/cosmo.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
@ -40,7 +42,7 @@
#include "libc/thread/tls.h"
static struct Semaphores {
pthread_once_t once;
atomic_uint once;
pthread_mutex_t lock;
struct Semaphore {
struct Semaphore *next;
@ -49,7 +51,9 @@ static struct Semaphores {
bool dead;
int refs;
} *list;
} g_semaphores;
} g_semaphores = {
.lock = PTHREAD_MUTEX_INITIALIZER,
};
static void sem_open_lock(void) {
pthread_mutex_lock(&g_semaphores.lock);
@ -69,7 +73,7 @@ static void sem_open_setup(void) {
}
static void sem_open_init(void) {
pthread_once(&g_semaphores.once, sem_open_setup);
cosmo_once(&g_semaphores.once, sem_open_setup);
}
static sem_t *sem_open_impl(const char *path, int oflag, unsigned mode,

4
libc/thread/thread.h
View file

@ -128,10 +128,10 @@ typedef struct pthread_attr_s {
int __schedparam;
int __schedpolicy;
int __contentionscope;
int __guardsize;
int __stacksize;
int __sigaltstacksize;
uint64_t __sigmask;
size_t __guardsize;
size_t __stacksize;
void *__stackaddr;
void *__sigaltstackaddr;
} pthread_attr_t;

2
test/libc/system/BUILD.mk
View file

@ -33,6 +33,7 @@ TEST_LIBC_SYSTEM_DIRECTDEPS = \
LIBC_RUNTIME \
LIBC_STDIO \
LIBC_STDIO \
LIBC_STR \
LIBC_SYSTEM \
LIBC_SYSV \
LIBC_TESTLIB \
@ -40,6 +41,7 @@ TEST_LIBC_SYSTEM_DIRECTDEPS = \
LIBC_X \
THIRD_PARTY_MUSL \
THIRD_PARTY_TR \
THIRD_PARTY_TZ \
TEST_LIBC_SYSTEM_DEPS := \
$(call uniq,$(foreach x,$(TEST_LIBC_SYSTEM_DIRECTDEPS),$($(x))))

26
test/libc/system/popen_test.c
View file

@ -17,6 +17,7 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/calls/struct/itimerval.h"
#include "libc/calls/struct/sigaction.h"
#include "libc/dce.h"
#include "libc/errno.h"
@ -31,15 +32,40 @@
#include "libc/stdio/stdio.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/f.h"
#include "libc/sysv/consts/itimer.h"
#include "libc/sysv/consts/sig.h"
#include "libc/testlib/testlib.h"
#include "libc/thread/thread.h"
#include "libc/time.h"
FILE *f;
char buf[32];
void OnAlarm(int sig) {
}
void *LolThread(void *arg) {
return 0;
}
void SetUpOnce(void) {
testlib_enable_tmp_setup_teardown();
// give deadlock detector more information
int64_t t = 0x5cd04d0e;
localtime(&t);
pthread_t th;
pthread_create(&th, 0, LolThread, 0);
pthread_join(th, 0);
char buf[32];
sprintf(buf, "%g", 3.14);
atexit((void *)LolThread);
FILE *f = fopen("/zip/.cosmo", "r");
fgetc(f);
fclose(f);
signal(SIGALRM, OnAlarm);
struct itimerval it = {{0, 1000}, {0, 1}};
setitimer(ITIMER_REAL, &it, 0);
}
void CheckForFdLeaks(void) {

64
test/libc/thread/pthread_create_test.c
View file

@ -22,6 +22,8 @@
#include "libc/calls/struct/sched_param.h"
#include "libc/calls/struct/sigaction.h"
#include "libc/calls/struct/siginfo.h"
#include "libc/calls/struct/sigset.h"
#include "libc/cosmo.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/kprintf.h"
@ -40,7 +42,9 @@
#include "libc/sysv/consts/sched.h"
#include "libc/sysv/consts/sig.h"
#include "libc/sysv/consts/ss.h"
#include "libc/testlib/benchmark.h"
#include "libc/testlib/ezbench.h"
#include "libc/testlib/manystack.h"
#include "libc/testlib/subprocess.h"
#include "libc/testlib/testlib.h"
#include "libc/thread/posixthread.internal.h"
@ -50,6 +54,10 @@
void OnUsr1(int sig, siginfo_t *si, void *vctx) {
}
void SetUpOnce(void) {
cosmo_stack_setmaxstacks((_rand64() & 7) - 1);
}
void SetUp(void) {
struct sigaction sig = {.sa_sigaction = OnUsr1, .sa_flags = SA_SIGINFO};
sigaction(SIGUSR1, &sig, 0);
@ -280,10 +288,60 @@ static void CreateDetached(void) {
ASSERT_EQ(0, pthread_attr_destroy(&attr));
}
#define LAUNCHES 10
#define LAUNCHERS 10
errno_t pthread_create2(pthread_t *thread, const pthread_attr_t *attr,
void *(*start_routine)(void *), void *arg) {
for (int i = 1;; i <<= 1) {
errno_t err = pthread_create(thread, attr, start_routine, arg);
if (err != EAGAIN)
return err;
usleep(i);
}
}
static void *CreateDetachedParallelThreads(void *arg) {
for (int i = 0; i < LAUNCHES; ++i)
CreateDetached();
return 0;
}
static void CreateDetachedParallel(void) {
pthread_t th[LAUNCHERS];
for (int i = 0; i < LAUNCHERS; ++i)
ASSERT_EQ(0, pthread_create2(&th[i], 0, CreateDetachedParallelThreads, 0));
for (int i = 0; i < LAUNCHERS; ++i)
ASSERT_EQ(0, pthread_join(th[i], 0));
}
static void *CreateJoinParallelThreads(void *arg) {
for (int i = 0; i < LAUNCHES; ++i)
CreateJoin();
return 0;
}
static void CreateJoinParallel(void) {
pthread_t th[LAUNCHERS];
for (int i = 0; i < LAUNCHERS; ++i)
ASSERT_EQ(0, pthread_create2(&th[i], 0, CreateJoinParallelThreads, 0));
for (int i = 0; i < LAUNCHERS; ++i)
ASSERT_EQ(0, pthread_join(th[i], 0));
}
TEST(pthread_create, bench) {
EZBENCH2("CreateJoin", donothing, CreateJoin());
EZBENCH2("CreateDetach", donothing, CreateDetach());
EZBENCH2("CreateDetached", donothing, CreateDetached());
kprintf("cosmo_stack_getmaxstacks() = %d\n", cosmo_stack_getmaxstacks());
pthread_t msh = manystack_start();
BENCHMARK(100, 1, CreateJoin());
BENCHMARK(100, 1, CreateDetach());
usleep(10000);
pthread_decimate_np();
BENCHMARK(100, 1, CreateDetached());
usleep(10000);
pthread_decimate_np();
BENCHMARK(1, LAUNCHERS + LAUNCHERS * LAUNCHES, CreateJoinParallel());
BENCHMARK(1, LAUNCHERS + LAUNCHERS * LAUNCHES, CreateDetachedParallel());
manystack_stop(msh);
while (!pthread_orphan_np())
pthread_decimate_np();
}

6
test/posix/file_offset_exec_test.c
View file

@ -13,6 +13,7 @@
// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
#include <cosmo.h>
#include <signal.h>
#include <stdatomic.h>
#include <stdio.h>
@ -36,6 +37,11 @@ void on_unexpected_death(int sig) {
}
int main() {
// TODO(jart): fix flakes
if (IsWindows())
return 0;
signal(SIGCHLD, on_unexpected_death);
// extract test program

29
test/posix/fork_bench_test.c Normal file
View file

@ -0,0 +1,29 @@
// Copyright 2024 Justine Alexandra Roberts Tunney
//
// Permission to use, copy, modify, and/or distribute this software for
// any purpose with or without fee is hereby granted, provided that the
// above copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
// WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
// AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
// DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
// PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
#include <stdlib.h>
#include <unistd.h>
#include "libc/testlib/benchmark.h"
void TestFork(void) {
int pid;
if (!(pid = fork()))
_Exit(0);
wait(0);
}
int main(int argc, char *argv[]) {
BENCHMARK(100, 1, TestFork());
}

9
test/posix/mutex_async_signal_safety_test.c
View file

@ -25,7 +25,7 @@
//
// glibc fails this test
// musl passes this test
// cosmo only guarantees this in process shared mode
// cosmo only guarantees this in process-shared non-debug mode
atomic_bool done;
atomic_bool ready;
@ -51,7 +51,14 @@ void* work(void* arg) {
int main() {
if (IsQemuUser()) {
// qemu is believed to be the one at fault
kprintf("mutex_async_signal_safety_test flakes on qemu\n");
return 0;
}
if (IsModeDbg()) {
// the deadlock detector gets in the way of our glorious spin lock
kprintf("mutex_async_signal_safety_test not feasible in debug mode\n");
return 0;
}

5
test/posix/signal_latency_async_test.c
View file

@ -13,6 +13,7 @@
// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
#include <cosmo.h>
#include <pthread.h>
#include <signal.h>
#include <stdatomic.h>
@ -107,6 +108,10 @@ int compare(const void *a, const void *b) {
int main() {
// TODO(jart): fix flakes
if (IsWindows())
return 0;
// Install signal handlers
struct sigaction sa;
sa.sa_handler = receiver_signal_handler;

1
third_party/dlmalloc/dlmalloc.c vendored
View file

@ -45,7 +45,6 @@
#define USE_LOCKS 2
#define MALLOC_INSPECT_ALL 1
#define ABORT_ON_ASSERT_FAILURE 0
#define LOCK_AT_FORK 1
#define NO_MALLOC_STATS 1
#if IsModeDbg()

26
third_party/dlmalloc/init.inc vendored
View file

@ -3,38 +3,38 @@
#include "libc/nexgen32e/rdtsc.h"
#include "libc/runtime/runtime.h"
/* ---------------------------- setting mparams -------------------------- */
#if LOCK_AT_FORK
#if ONLY_MSPACES
void dlmalloc_pre_fork(void) {
#if ONLY_MSPACES
mstate h;
for (unsigned i = ARRAYLEN(g_heaps); i--;)
if ((h = atomic_load_explicit(&g_heaps[i], memory_order_acquire)))
ACQUIRE_LOCK(&h->mutex);
#else
ACQUIRE_LOCK(&(gm)->mutex);
#endif
}
void dlmalloc_post_fork_parent(void) {
#if ONLY_MSPACES
mstate h;
for (unsigned i = 0; i < ARRAYLEN(g_heaps); ++i)
if ((h = atomic_load_explicit(&g_heaps[i], memory_order_acquire)))
RELEASE_LOCK(&h->mutex);
#else
RELEASE_LOCK(&(gm)->mutex);
#endif
}
void dlmalloc_post_fork_child(void) {
#if ONLY_MSPACES
mstate h;
for (unsigned i = 0; i < ARRAYLEN(g_heaps); ++i)
if ((h = atomic_load_explicit(&g_heaps[i], memory_order_acquire)))
(void)REFRESH_LOCK(&h->mutex);
}
REFRESH_LOCK(&h->mutex);
#else
void dlmalloc_pre_fork(void) { ACQUIRE_LOCK(&(gm)->mutex); }
void dlmalloc_post_fork_parent(void) { RELEASE_LOCK(&(gm)->mutex); }
void dlmalloc_post_fork_child(void) { (void)REFRESH_LOCK(&(gm)->mutex); }
#endif /* ONLY_MSPACES */
#endif /* LOCK_AT_FORK */
REFRESH_LOCK(&(gm)->mutex);
#endif
}
/* Initialize mparams */
__attribute__((__constructor__(49))) int init_mparams(void) {

4
third_party/dlmalloc/platform.inc vendored
View file

@ -151,10 +151,6 @@
========================================================================
*/
#ifndef LOCK_AT_FORK
#define LOCK_AT_FORK 0
#endif
/* ------------------- size_t and alignment properties -------------------- */
/* The byte and bit size of a size_t */

26
third_party/nsync/common.c vendored
View file

@ -40,6 +40,7 @@
#include "third_party/nsync/mu_semaphore.h"
#include "third_party/nsync/mu_semaphore.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/strace.h"
#include "third_party/nsync/wait_s.internal.h"
__static_yoink("nsync_notice");
@ -179,10 +180,10 @@ static waiter *free_waiters_pop (void) {
return w;
}
static void free_waiters_populate (void) {
static bool free_waiters_populate (void) {
int n;
if (IsNetbsd ()) {
// netbsd needs a real file descriptor per semaphore
// netbsd semaphores are file descriptors
n = 1;
} else {
n = __pagesize / sizeof(waiter);
@ -192,14 +193,17 @@ static void free_waiters_populate (void) {
MAP_PRIVATE | MAP_ANONYMOUS,
-1, 0);
if (waiters == MAP_FAILED)
nsync_panic_ ("out of memory\n");
return false;
for (size_t i = 0; i < n; ++i) {
waiter *w = &waiters[i];
w->tag = WAITER_TAG;
w->nw.tag = NSYNC_WAITER_TAG;
if (!nsync_mu_semaphore_init (&w->sem)) {
if (!i)
nsync_panic_ ("out of semaphores\n");
if (!i) {
// netbsd can run out of semaphores
munmap (waiters, n * sizeof (waiter));
return false;
}
break;
}
w->nw.sem = &w->sem;
@ -208,6 +212,7 @@ static void free_waiters_populate (void) {
dll_init (&w->same_condition);
free_waiters_push (w);
}
return true;
}
/* -------------------------------- */
@ -232,11 +237,18 @@ void nsync_waiter_destroy (void *v) {
waiter *nsync_waiter_new_ (void) {
waiter *w;
waiter *tw;
unsigned attempts = 0;
bool out_of_semaphores = false;
tw = waiter_for_thread;
w = tw;
if (w == NULL || (w->flags & (WAITER_RESERVED|WAITER_IN_USE)) != WAITER_RESERVED) {
while (!(w = free_waiters_pop ()))
free_waiters_populate ();
while (!(w = free_waiters_pop ())) {
if (!out_of_semaphores)
if (!free_waiters_populate ())
out_of_semaphores = true;
if (out_of_semaphores)
attempts = pthread_delay_np (&free_waiters, attempts);
}
if (tw == NULL) {
w->flags |= WAITER_RESERVED;
waiter_for_thread = w;

6
third_party/nsync/mu_semaphore_sem.c vendored
View file

@ -33,7 +33,6 @@
#include "third_party/nsync/time.h"
#include "third_party/nsync/mu_semaphore.h"
#include "libc/intrin/atomic.h"
#include "libc/atomic.h"
#include "third_party/nsync/time.h"
/**
@ -83,8 +82,9 @@ void nsync_mu_semaphore_sem_fork_child (void) {
for (f = atomic_load_explicit (&g_sems, memory_order_relaxed); f; f = f->next) {
int rc = sys_close (f->id);
STRACE ("close(%ld) → %d", f->id, rc);
ASSERT (nsync_mu_semaphore_sem_create (f));
}
for (f = atomic_load_explicit (&g_sems, memory_order_relaxed); f; f = f->next)
ASSERT (nsync_mu_semaphore_sem_create (f));
}
/* Initialize *s; the initial value is 0. */
@ -92,7 +92,7 @@ bool nsync_mu_semaphore_init_sem (nsync_semaphore *s) {
struct sem *f = (struct sem *) s;
if (!nsync_mu_semaphore_sem_create (f))
return false;
sems_push(f);
sems_push (f);
return true;
}

4
tool/build/runit.c
View file

@ -384,6 +384,10 @@ int RunOnHost(char *spec) {
handshake_latency = timespec_tomicros(timespec_sub(timespec_mono(), start));
if (!err)
break;
if (err == MBEDTLS_ERR_NET_CONN_RESET) {
close(g_sock);
continue;
}
WARNF("handshake with %s:%d failed -0x%04x (%s)", //
g_hostname, g_runitdport, err, GetTlsError(err));
close(g_sock);