/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ │ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ ╞══════════════════════════════════════════════════════════════════════════════╡ │ Copyright 2024 Justine Alexandra Roberts Tunney │ │ │ │ Permission to use, copy, modify, and/or distribute this software for │ │ any purpose with or without fee is hereby granted, provided that the │ │ above copyright notice and this permission notice appear in all copies. │ │ │ │ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ │ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ │ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ │ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ │ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ │ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/intrin/stack.h" #include "libc/assert.h" #include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/calls/syscall-sysv.internal.h" #include "libc/cosmo.h" #include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/dll.h" #include "libc/runtime/runtime.h" #include "libc/sysv/consts/map.h" #include "libc/sysv/consts/prot.h" #include "libc/thread/posixthread.internal.h" #include "libc/thread/thread.h" /** * @fileoverview cosmo stack memory manager */ #define MAP_ANON_OPENBSD 0x1000 #define MAP_STACK_OPENBSD 0x4000 #define THREADSTACK_CONTAINER(e) DLL_CONTAINER(struct CosmoStack, elem, e) struct CosmoStack { struct Dll elem; void *stackaddr; unsigned stacksize; unsigned guardsize; }; struct CosmoStacks { atomic_uint once; pthread_mutex_t lock; struct Dll *stacks; struct Dll *objects; unsigned count; }; struct CosmoStacksConfig { unsigned maxstacks; }; static struct CosmoStacks cosmo_stacks = { .lock = PTHREAD_MUTEX_INITIALIZER, }; static struct CosmoStacksConfig cosmo_stacks_config = { .maxstacks = 16, }; void cosmo_stack_lock(void) { _pthread_mutex_lock(&cosmo_stacks.lock); } void cosmo_stack_unlock(void) { _pthread_mutex_unlock(&cosmo_stacks.lock); } void cosmo_stack_wipe(void) { _pthread_mutex_wipe_np(&cosmo_stacks.lock); } static errno_t cosmo_stack_munmap(void *addr, size_t size) { errno_t r = 0; errno_t e = errno; if (!munmap(addr, size)) { r = errno; errno = e; } return r; } static void cosmo_stack_populate(void) { errno_t e = errno; void *map = mmap(0, __pagesize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); errno = e; if (map != MAP_FAILED) { struct CosmoStack *ts = map; int n = __pagesize / sizeof(struct CosmoStack); for (int i = 0; i < n; ++i) { dll_init(&ts[i].elem); dll_make_first(&cosmo_stacks.objects, &ts[i].elem); } } } static struct Dll *cosmo_stack_decimate(unsigned maxstacks) { struct Dll *surplus = 0; while (cosmo_stacks.count > maxstacks) { struct Dll *e = dll_last(cosmo_stacks.stacks); dll_remove(&cosmo_stacks.stacks, e); dll_make_first(&surplus, e); --cosmo_stacks.count; } return surplus; } static void cosmo_stack_rehabilitate(struct Dll *stacks) { struct Dll *e; for (e = dll_first(stacks); e; e = dll_next(stacks, e)) cosmo_stack_munmap(THREADSTACK_CONTAINER(e)->stackaddr, THREADSTACK_CONTAINER(e)->stacksize); cosmo_stack_lock(); dll_make_first(&cosmo_stacks.objects, stacks); cosmo_stack_unlock(); } /** * Empties unused stack cache. * * To make POSIX threads as cheap as possible to spawn, we recycle their * stacks without zeroing their memory. On Linux for an 80kb stack size, * that makes launching a thread take 40µs rather than 80µs. However the * stack cache needs to be cleared in certain cases. This is called upon * exit() automatically but anyone can clear this at any other time too. * * @see pthread_decimate_np() */ void cosmo_stack_clear(void) { cosmo_stack_lock(); struct Dll *stacks = cosmo_stacks.stacks; cosmo_stacks.stacks = 0; cosmo_stacks.count = 0; cosmo_stack_unlock(); cosmo_stack_rehabilitate(stacks); } /** * Gets maximum number of unused stacks cosmo should cache. * @see cosmo_stack_setmaxstacks() */ int cosmo_stack_getmaxstacks(void) { return cosmo_stacks_config.maxstacks; } /** * Sets maximum number of unused stacks cosmo should cache. * * This lets you place some limitations on how much stack memory the * cosmo runtime will cache. This number is a count of stacks rather * than the number of bytes they contain. Old stacks are freed in a * least recently used fashion once the cache exceeds this limit. * * If this is set to zero, then the cosmo stack allocator enters a * highly secure hardening mode where cosmo_stack_alloc() zeroes all * stack memory that's returned. * * Setting this to a negative number makes the cache size unlimited. * * Please note this limit only applies to stacks that aren't in use. * * Your default is sixteen stacks may be cached at any given moment. * * If `maxstacks` is less than the current cache size, then surplus * entries will be evicted and freed before this function returns. */ void cosmo_stack_setmaxstacks(int maxstacks) { cosmo_stack_lock(); cosmo_stacks_config.maxstacks = maxstacks; struct Dll *stacks = cosmo_stack_decimate(maxstacks); cosmo_stack_unlock(); cosmo_stack_rehabilitate(stacks); } /** * Allocates stack memory. * * This is a caching stack allocator that's used by the POSIX threads * runtime but you may also find it useful for setcontext() coroutines * or sigaltstack(). Normally you can get away with using malloc() for * creating stacks. However some OSes (e.g. OpenBSD) forbid you from * doing that for anything except sigaltstack(). This API serves to * abstract all the gory details of gaining authorized memory, and * additionally implements caching for lightning fast performance. * * The stack size must be nonzero. It is rounded up to the granularity * of the underlying system allocator, which is normally the page size. * Your parameter will be updated with the selected value upon success. * * The guard size specifies how much memory should be protected at the * bottom of your stack. This is helpful for ensuring stack overflows * will result in a segmentation fault, rather than corrupting memory * silently. This may be set to zero, in which case no guard pages will * be protected. This value is rounded up to the system page size. The * corrected value will be returned upon success. Your guard size needs * to be small enough to leave room for at least one memory page in your * stack size i.e. `guardsize + pagesize <= stacksize` must be the case. * Otherwise this function will return an `EINVAL` error. * * When you're done using your stack, pass it to cosmo_stack_free() so * it can be recycled. Stacks are only recycled when the `stacksize` and * `guardsize` parameters are an exact match after correction. Otherwise * they'll likely be freed eventually, in a least-recently used fashion, * based upon the configurable cosmo_stack_setmaxstacks() setting. * * This function returns 0 on success, or an errno on error. See the * documentation of mmap() for a list possible errors that may occur. */ errno_t cosmo_stack_alloc(unsigned *inout_stacksize, // unsigned *inout_guardsize, // void **out_addr) { // validate arguments unsigned stacksize = *inout_stacksize; unsigned guardsize = *inout_guardsize; stacksize = (stacksize + __gransize - 1) & -__gransize; guardsize = (guardsize + __pagesize - 1) & -__pagesize; if (guardsize + __pagesize > stacksize) return EINVAL; // recycle stack void *stackaddr = 0; cosmo_stack_lock(); for (struct Dll *e = dll_first(cosmo_stacks.stacks); e; e = dll_next(cosmo_stacks.stacks, e)) { struct CosmoStack *ts = THREADSTACK_CONTAINER(e); if (ts->stacksize == stacksize && // ts->guardsize == guardsize) { dll_remove(&cosmo_stacks.stacks, e); stackaddr = ts->stackaddr; dll_make_first(&cosmo_stacks.objects, e); --cosmo_stacks.count; break; } } cosmo_stack_unlock(); // create stack if (!stackaddr) { errno_t e = errno; stackaddr = mmap(0, stacksize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (stackaddr == MAP_FAILED) { errno_t err = errno; errno = e; return err; } if (IsOpenbsd()) if (!TellOpenbsdThisIsStackMemory(stackaddr, stacksize)) notpossible; if (guardsize) if (mprotect(stackaddr, guardsize, PROT_NONE | PROT_GUARD)) notpossible; } // return stack *inout_stacksize = stacksize; *inout_guardsize = guardsize; *out_addr = stackaddr; return 0; } static void cosmo_stack_setup(void) { atexit(cosmo_stack_clear); } /** * Frees stack memory. * * While not strictly required, it's assumed these three values would be * those returned by an earlier call to cosmo_stack_alloc(). * * This function returns 0 on success, or an errno on error. The `errno` * variable is never clobbered. You can only dependably count on this to * return an error on failure when you say `cosmo_stack_setmaxstacks(0)` */ errno_t cosmo_stack_free(void *stackaddr, unsigned stacksize, unsigned guardsize) { stacksize = (stacksize + __gransize - 1) & -__gransize; guardsize = (guardsize + __pagesize - 1) & -__pagesize; if (guardsize + __pagesize > stacksize) return EINVAL; if ((uintptr_t)stackaddr & (__gransize - 1)) return EINVAL; cosmo_once(&cosmo_stacks.once, cosmo_stack_setup); cosmo_stack_lock(); struct Dll *surplus = 0; if (cosmo_stacks_config.maxstacks) { surplus = cosmo_stack_decimate(cosmo_stacks_config.maxstacks - 1); struct CosmoStack *ts = 0; if (dll_is_empty(cosmo_stacks.objects)) cosmo_stack_populate(); struct Dll *e; if ((e = dll_first(cosmo_stacks.objects))) { dll_remove(&cosmo_stacks.objects, e); ts = THREADSTACK_CONTAINER(e); } if (ts) { ts->stackaddr = stackaddr; ts->stacksize = stacksize; ts->guardsize = guardsize; dll_make_first(&cosmo_stacks.stacks, &ts->elem); ++cosmo_stacks.count; stackaddr = 0; } } cosmo_stack_unlock(); cosmo_stack_rehabilitate(surplus); errno_t err = 0; if (stackaddr) err = cosmo_stack_munmap(stackaddr, stacksize); return err; } relegated bool TellOpenbsdThisIsStackMemory(void *addr, size_t size) { return __sys_mmap( addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD, -1, 0, 0) == addr; } // OpenBSD only permits RSP to occupy memory that's been explicitly // defined as stack memory, i.e. `lo <= %rsp < hi` must be the case relegated errno_t FixupCustomStackOnOpenbsd(pthread_attr_t *attr) { // get interval uintptr_t lo = (uintptr_t)attr->__stackaddr; uintptr_t hi = lo + attr->__stacksize; // squeeze interval lo = (lo + __pagesize - 1) & -__pagesize; hi = hi & -__pagesize; // tell os it's stack memory errno_t olderr = errno; if (!TellOpenbsdThisIsStackMemory((void *)lo, hi - lo)) { errno_t err = errno; errno = olderr; return err; } // update attributes with usable stack address attr->__stackaddr = (void *)lo; attr->__stacksize = hi - lo; return 0; }