From d461c6f47d69635e7bc13ef45debb28193797dd5 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Mon, 24 Jun 2024 06:53:49 -0700 Subject: [PATCH] Do more quality assurance work --- ape/aarch64.lds | 5 -- ape/ape.S | 2 +- ape/ape.lds | 5 -- libc/calls/BUILD.mk | 2 - libc/crt/crt.S | 6 +- libc/intrin/BUILD.mk | 2 + libc/intrin/maps.h | 4 +- libc/intrin/mmap.c | 123 ++++++++++++++++++++++----- libc/intrin/mprotect.c | 10 ++- libc/intrin/msync-nt.c | 2 + libc/intrin/printmaps.c | 18 ++-- libc/{calls => intrin}/sched_yield.S | 3 +- libc/proc/fork-nt.c | 6 ++ libc/proc/fork.c | 7 ++ libc/runtime/cxa_thread_atexit.c | 22 +++-- libc/runtime/mapstack.c | 2 +- libc/runtime/stack.h | 5 -- libc/runtime/winmain.greg.c | 2 + libc/thread/posixthread.internal.h | 1 - libc/thread/pthread_create.c | 17 ++-- libc/thread/pthread_exit.c | 9 -- test/libc/calls/setrlimit_test.c | 2 + third_party/lua/lua.main.c | 3 - third_party/nsync/common.c | 25 ++++-- third_party/python/freeze.c | 3 - third_party/python/launch.c | 3 - third_party/python/pyobj.c | 2 - third_party/python/python3.c | 3 - third_party/python/pythontester.c | 3 - third_party/python/repl.c | 3 - tool/net/redbean.c | 2 - 31 files changed, 194 insertions(+), 108 deletions(-) rename libc/{calls => intrin}/sched_yield.S (99%) diff --git a/ape/aarch64.lds b/ape/aarch64.lds index f25d232cb..c63ce4295 100644 --- a/ape/aarch64.lds +++ b/ape/aarch64.lds @@ -286,12 +286,7 @@ SECTIONS { ape_stack_vaddr = DEFINED(ape_stack_vaddr) ? ape_stack_vaddr : 0x700000000000; ape_stack_memsz = DEFINED(ape_stack_memsz) ? ape_stack_memsz : 8 * 1024 * 1024; -ape_stack_align = DEFINED(ape_stack_align) ? MAX(ape_stack_align, 16) : 16; ape_stack_prot = PROT_READ | PROT_WRITE; -ape_stack_round = -ape_stack_align; - -ASSERT(ape_stack_align < 1048576, "stack align too big"); -ASSERT(!(ape_stack_align & (ape_stack_align - 1)), "stack align must be two power"); _tls_size = _tbss_end - _tdata_start; _tdata_size = _tdata_end - _tdata_start; diff --git a/ape/ape.S b/ape/ape.S index 79594ddd4..2ec05e963 100644 --- a/ape/ape.S +++ b/ape/ape.S @@ -761,7 +761,7 @@ ape_phdrs: .quad 0 .quad 0 .quad 0 - .stub ape_stack_align,quad + .quad 16 #if SupportsOpenbsd() || SupportsNetbsd() .long PT_NOTE diff --git a/ape/ape.lds b/ape/ape.lds index ef8dfbb3a..4e6db724a 100644 --- a/ape/ape.lds +++ b/ape/ape.lds @@ -588,11 +588,6 @@ ape_stack_vaddr = DEFINED(ape_stack_vaddr) ? ape_stack_vaddr : 0x700000000000; ape_stack_paddr = ape_ram_paddr + ape_ram_filesz; ape_stack_filesz = 0; ape_stack_memsz = DEFINED(ape_stack_memsz) ? ape_stack_memsz : 8 * 1024 * 1024; -ape_stack_align = DEFINED(ape_stack_align) ? MAX(ape_stack_align, 16) : 16; -ape_stack_round = -ape_stack_align; - -ASSERT(ape_stack_align < 1048576, "stack align too big"); -ASSERT(!(ape_stack_align & (ape_stack_align - 1)), "stack align must be two power"); ape_note_offset = ape_cod_offset + (ape_note - ape_cod_vaddr); ape_note_filesz = ape_note_end - ape_note; diff --git a/libc/calls/BUILD.mk b/libc/calls/BUILD.mk index b2b633a7a..442dab18e 100644 --- a/libc/calls/BUILD.mk +++ b/libc/calls/BUILD.mk @@ -163,8 +163,6 @@ o/$(MODE)/libc/calls/tailcontext.o: libc/calls/tailcontext.S @$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $< o/$(MODE)/libc/calls/stackjump.o: libc/calls/stackjump.S @$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $< -o/$(MODE)/libc/calls/sched_yield.o: libc/calls/sched_yield.S - @$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $< o/$(MODE)/libc/calls/uname.o: libc/integral/normalize.inc diff --git a/libc/crt/crt.S b/libc/crt/crt.S index e2917b2ec..b5ba61a59 100644 --- a/libc/crt/crt.S +++ b/libc/crt/crt.S @@ -79,7 +79,7 @@ _start: // setup stack xor %ebp,%ebp - and $ape_stack_round,%rsp + and $-16,%rsp #if SupportsWindows() && !IsTiny() // make win32 imps crash @@ -136,9 +136,7 @@ _start: mov x28,#0 mov x29,#0 mov x30,#0 - ldr x1,=ape_stack_round - and x1,x0,x1 - mov sp,x1 + and sp,x0,#-16 // second arg shall be struct Syslib passed by ape-m1.c // used to talk to apple's authoritarian libraries diff --git a/libc/intrin/BUILD.mk b/libc/intrin/BUILD.mk index f9742c42b..3cc80c6b6 100644 --- a/libc/intrin/BUILD.mk +++ b/libc/intrin/BUILD.mk @@ -135,6 +135,8 @@ o/$(MODE)/libc/intrin/kweekdayname.o: libc/intrin/kweekdayname.S @$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $< o/$(MODE)/libc/intrin/kweekdaynameshort.o: libc/intrin/kweekdaynameshort.S @$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $< +o/$(MODE)/libc/intrin/sched_yield.o: libc/intrin/sched_yield.S + @$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -c $< LIBC_INTRIN_LIBS = $(foreach x,$(LIBC_INTRIN_ARTIFACTS),$($(x))) LIBC_INTRIN_HDRS = $(foreach x,$(LIBC_INTRIN_ARTIFACTS),$($(x)_HDRS)) diff --git a/libc/intrin/maps.h b/libc/intrin/maps.h index 090bb060b..bd03e2266 100644 --- a/libc/intrin/maps.h +++ b/libc/intrin/maps.h @@ -26,6 +26,8 @@ struct Maps { struct Dll *free; struct Map stack; struct Dll *used; + size_t count; + size_t pages; }; struct AddrSize { @@ -35,9 +37,9 @@ struct AddrSize { extern struct Maps __maps; -int maps_check(void); void __maps_init(void); void __maps_lock(void); +void __maps_check(void); void __maps_unlock(void); struct Map *__maps_alloc(void); void __maps_free(struct Map *); diff --git a/libc/intrin/mmap.c b/libc/intrin/mmap.c index 95d34515c..8011618a0 100644 --- a/libc/intrin/mmap.c +++ b/libc/intrin/mmap.c @@ -25,9 +25,11 @@ #include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" +#include "libc/intrin/describebacktrace.internal.h" #include "libc/intrin/describeflags.internal.h" #include "libc/intrin/directmap.internal.h" #include "libc/intrin/dll.h" +#include "libc/intrin/kprintf.h" #include "libc/intrin/maps.h" #include "libc/intrin/strace.internal.h" #include "libc/intrin/weaken.h" @@ -44,6 +46,7 @@ #include "libc/sysv/errfuns.h" #include "libc/thread/thread.h" +#define MMDEBUG 0 // this code is too slow for openbsd/windows #define WINBASE 0x100080040000 // TODO: Can we support Windows Vista again? #define WINMAXX 0x200080000000 @@ -51,21 +54,80 @@ #define PGUP(x) (((x) + granularity - 1) & -granularity) +#if MMDEBUG +#define ASSERT(x) (void)0 +#else +#define ASSERT(x) \ + do { \ + if (!(x)) { \ + char bt[160]; \ + struct StackFrame *bp = __builtin_frame_address(0); \ + kprintf("%!s:%d: assertion failed: %!s\n", __FILE__, __LINE__, #x); \ + kprintf("bt %!s\n", (DescribeBacktrace)(bt, bp)); \ + __print_maps(); \ + _Exit(99); \ + } \ + } while (0) +#endif + static atomic_ulong rollo; +static bool overlaps_existing_map(const char *addr, size_t size) { + int granularity = __granularity(); + for (struct Map *map = __maps.maps; map; map = map->next) + if (MAX(addr, map->addr) < + MIN(addr + PGUP(size), map->addr + PGUP(map->size))) + return true; + return false; +} + +void __maps_check(void) { +#if MMDEBUG + size_t maps = 0; + size_t pages = 0; + int granularity = getauxval(AT_PAGESZ); + for (struct Map *map = __maps.maps; map; map = map->next) { + ASSERT(map->addr != MAP_FAILED); + ASSERT(map->size); + pages += PGUP(map->size) / granularity; + maps += 1; + } + ASSERT(maps = __maps.count); + ASSERT(pages == __maps.pages); + for (struct Map *m1 = __maps.maps; m1; m1 = m1->next) + for (struct Map *m2 = m1->next; m2; m2 = m2->next) + ASSERT(MAX(m1->addr, m2->addr) >= + MIN(m1->addr + PGUP(m1->size), m2->addr + PGUP(m2->size))); +#endif +} + void __maps_free(struct Map *map) { + map->next = 0; + map->size = 0; + map->addr = MAP_FAILED; + ASSERT(dll_is_alone(&map->elem)); dll_make_last(&__maps.free, &map->elem); } void __maps_insert(struct Map *map) { struct Map *last = __maps.maps; - if (last && // + int granularity = getauxval(AT_PAGESZ); + __maps.pages += PGUP(map->size) / granularity; + if (last && !IsWindows() && // map->addr == last->addr + last->size && // + (map->flags & MAP_ANONYMOUS) && // map->flags == last->flags && // - map->prot == last->prot && // - map->off == last->off && // - map->h == last->h && // - map->off == -1) { + map->prot == last->prot) { + last->size += map->size; + dll_remove(&__maps.used, &last->elem); + dll_make_first(&__maps.used, &last->elem); + __maps_free(map); + } else if (last && !IsWindows() && // + map->addr + map->size == last->addr && // + (map->flags & MAP_ANONYMOUS) && // + map->flags == last->flags && // + map->prot == last->prot) { + last->addr -= map->size; last->size += map->size; dll_remove(&__maps.used, &last->elem); dll_make_first(&__maps.used, &last->elem); @@ -74,7 +136,9 @@ void __maps_insert(struct Map *map) { dll_make_first(&__maps.used, &map->elem); map->next = __maps.maps; __maps.maps = map; + ++__maps.count; } + __maps_check(); } struct Map *__maps_alloc(void) { @@ -104,14 +168,6 @@ struct Map *__maps_alloc(void) { return map; } -static bool __overlaps_existing_map(const char *addr, size_t size) { - for (struct Map *map = __maps.maps; map; map = map->next) { - if (MAX(addr, map->addr) < MIN(addr + size, map->addr + map->size)) - return true; - } - return false; -} - static int __munmap_chunk(void *addr, size_t size) { return sys_munmap(addr, size); } @@ -119,6 +175,7 @@ static int __munmap_chunk(void *addr, size_t size) { static int __munmap(char *addr, size_t size, bool untrack_only) { // validate arguments + int pagesz = getauxval(AT_PAGESZ); int granularity = __granularity(); if (((uintptr_t)addr & (granularity - 1)) || // !size || (uintptr_t)addr + size < size) @@ -127,7 +184,6 @@ static int __munmap(char *addr, size_t size, bool untrack_only) { // untrack and delete mapping int rc = 0; __maps_lock(); - // we can't call strace, kprintf, or nothing StartOver:; struct Map *map = __maps.maps; _Atomic(struct Map *) *prev = &__maps.maps; @@ -141,12 +197,16 @@ StartOver:; // remove mapping completely dll_remove(&__maps.used, &map->elem); *prev = next; - map->size = 0; - map->addr = MAP_FAILED; + __maps.pages -= (map_size + pagesz - 1) / pagesz; + __maps.count -= 1; if (untrack_only) { __maps_free(map); + __maps_check(); } else { + __maps_unlock(); if (!IsWindows()) { + ASSERT(addr <= map_addr); + ASSERT(map_addr + PGUP(map_size) <= addr + PGUP(size)); if (__munmap_chunk(map_addr, map_size)) rc = -1; } else { @@ -155,7 +215,9 @@ StartOver:; if (!CloseHandle(map->h)) rc = -1; } + __maps_lock(); __maps_free(map); + __maps_check(); goto StartOver; } map = next; @@ -167,15 +229,24 @@ StartOver:; rc = einval(); } else if (addr <= map_addr) { // shave off lefthand side of mapping - size_t left = addr + size - map_addr; - size_t right = map_addr + map_size - (addr + size); + ASSERT(addr + size < map_addr + PGUP(map_size)); + size_t left = PGUP(addr + size - map_addr); + size_t right = map_size - left; + ASSERT(right > 0); + ASSERT(left > 0); map->addr += left; map->size = right; if (map->off != -1) map->off += left; + __maps.pages -= (left + pagesz - 1) / pagesz; + __maps_check(); if (!untrack_only) { + __maps_unlock(); + ASSERT(addr <= map_addr); + ASSERT(map_addr + PGUP(left) <= addr + PGUP(size)); if (__munmap_chunk(map_addr, left) == -1) rc = -1; + __maps_lock(); goto StartOver; } } else if (addr + PGUP(size) >= map_addr + PGUP(map_size)) { @@ -183,9 +254,14 @@ StartOver:; size_t left = addr - map_addr; size_t right = map_addr + map_size - addr; map->size = left; + __maps.pages -= (right + pagesz - 1) / pagesz; + __maps_check(); if (!untrack_only) { + __maps_unlock(); + ASSERT(PGUP(right) <= PGUP(size)); if (__munmap_chunk(addr, right) == -1) rc = -1; + __maps_lock(); goto StartOver; } } else { @@ -207,9 +283,14 @@ StartOver:; map->off += left + middle; dll_make_first(&__maps.used, &leftmap->elem); *prev = leftmap; + __maps.pages -= (middle + pagesz - 1) / pagesz; + __maps.count += 1; + __maps_check(); if (!untrack_only) { + __maps_unlock(); if (__munmap_chunk(addr, size) == -1) rc = -1; + __maps_lock(); goto StartOver; } } else { @@ -241,7 +322,7 @@ static void *__mmap_chunk(void *addr, size_t size, int prot, int flags, int fd, sysflags |= MAP_FIXED_NOREPLACE_linux; } else if (IsFreebsd() || IsNetbsd()) { sysflags |= MAP_FIXED; - if (__overlaps_existing_map(addr, size)) + if (overlaps_existing_map(addr, size)) return (void *)eexist(); } else { noreplace = true; @@ -296,7 +377,7 @@ TryAgain: // untrack mapping we blew away if (should_untrack) - __munmap(addr, size, true); + __munmap(res.addr, size, true); // track Map object map->addr = res.addr; @@ -346,7 +427,7 @@ static void *__mmap_impl(char *addr, size_t size, int prot, int flags, int fd, // so we create an separate map for each granule in the mapping if (!(flags & MAP_FIXED)) { - while (__overlaps_existing_map(addr, size)) { + while (overlaps_existing_map(addr, size)) { if (flags & MAP_FIXED_NOREPLACE) return (void *)eexist(); addr += granularity; diff --git a/libc/intrin/mprotect.c b/libc/intrin/mprotect.c index 7cf095356..b1af88552 100644 --- a/libc/intrin/mprotect.c +++ b/libc/intrin/mprotect.c @@ -91,8 +91,8 @@ int __mprotect(char *addr, size_t size, int prot) { rc = -1; } else if (addr <= map_addr) { // cleave lefthand side of mapping - size_t left = addr + size - map_addr; - size_t right = map_addr + map_size - (addr + size); + size_t left = PGUP(addr + size - map_addr); + size_t right = map_size - left; struct Map *leftmap; if ((leftmap = __maps_alloc())) { if (!__mprotect_chunk(map_addr, left, prot, false)) { @@ -108,6 +108,8 @@ int __mprotect(char *addr, size_t size, int prot) { map->off += left; dll_make_first(&__maps.used, &leftmap->elem); *prev = leftmap; + __maps.count += 1; + __maps_check(); } else { __maps_free(leftmap); rc = -1; @@ -135,6 +137,8 @@ int __mprotect(char *addr, size_t size, int prot) { map->off += left; dll_make_first(&__maps.used, &leftmap->elem); *prev = leftmap; + __maps.count += 1; + __maps_check(); } else { __maps_free(leftmap); rc = -1; @@ -171,6 +175,8 @@ int __mprotect(char *addr, size_t size, int prot) { dll_make_first(&__maps.used, &leftmap->elem); dll_make_first(&__maps.used, &midlmap->elem); *prev = leftmap; + __maps.count += 2; + __maps_check(); } else { __maps_free(midlmap); __maps_free(leftmap); diff --git a/libc/intrin/msync-nt.c b/libc/intrin/msync-nt.c index 3b240c893..78cbd5444 100644 --- a/libc/intrin/msync-nt.c +++ b/libc/intrin/msync-nt.c @@ -34,6 +34,7 @@ textwindows int sys_msync_nt(char *addr, size_t size, int flags) { return einval(); int rc = 0; + __maps_lock(); for (struct Map *map = __maps.maps; map; map = map->next) { char *beg = MAX(addr, map->addr); char *end = MIN(addr + size, map->addr + map->size); @@ -42,6 +43,7 @@ textwindows int sys_msync_nt(char *addr, size_t size, int flags) { rc = -1; // TODO(jart): FlushFileBuffers too on g_fds handle if MS_SYNC? } + __maps_unlock(); return rc; } diff --git a/libc/intrin/printmaps.c b/libc/intrin/printmaps.c index 1c217ba9f..468c0f928 100644 --- a/libc/intrin/printmaps.c +++ b/libc/intrin/printmaps.c @@ -24,15 +24,20 @@ #include "libc/macros.internal.h" #include "libc/runtime/memtrack.internal.h" #include "libc/runtime/runtime.h" +#include "libc/sysv/consts/auxv.h" /** * Prints memory mappings. */ void __print_maps(void) { - int limit = 10; + int limit = 13; long maptally = 0; char mappingbuf[8], sb[16]; - for (struct Map *map = __maps.maps; map; map = map->next) { + __maps_lock(); + struct Dll *e, *e2; + for (e = dll_first(__maps.used); e; e = e2) { + e2 = dll_next(__maps.used, e); + struct Map *map = MAP_CONTAINER(e); maptally += map->size; kprintf("%012lx-%012lx %!s", map->addr, map->addr + map->size, (DescribeMapping)(mappingbuf, map->prot, map->flags)); @@ -45,11 +50,10 @@ void __print_maps(void) { if (map->readonlyfile) kprintf(" readonlyfile"); kprintf("\n"); - if (!--limit) { - kprintf("...\n"); + if (!--limit) break; - } } - sizefmt(sb, maptally, 1024); - kprintf("# %!sb mapped memory\n", sb); + kprintf("# %'zu bytes in %'zu mappings\n", + __maps.pages * getauxval(AT_PAGESZ), __maps.count); + __maps_unlock(); } diff --git a/libc/calls/sched_yield.S b/libc/intrin/sched_yield.S similarity index 99% rename from libc/calls/sched_yield.S rename to libc/intrin/sched_yield.S index 6357c5fe8..42433fa6d 100644 --- a/libc/calls/sched_yield.S +++ b/libc/intrin/sched_yield.S @@ -19,13 +19,12 @@ #include "libc/dce.h" #include "libc/sysv/consts/nr.h" #include "libc/macros.internal.h" +.privileged // Relinquishes scheduled quantum. // // @return 0 on success, or -1 w/ errno - .ftrace1 sched_yield: - .ftrace2 #ifdef __x86_64__ push %rbp mov %rsp,%rbp diff --git a/libc/proc/fork-nt.c b/libc/proc/fork-nt.c index cab228285..7e8f89cac 100644 --- a/libc/proc/fork-nt.c +++ b/libc/proc/fork-nt.c @@ -18,6 +18,7 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "ape/sections.internal.h" #include "libc/assert.h" +#include "libc/atomic.h" #include "libc/calls/internal.h" #include "libc/calls/sig.internal.h" #include "libc/calls/state.internal.h" @@ -66,6 +67,7 @@ #ifdef __x86_64__ extern long __klog_handle; +extern atomic_uint free_waiters_mu; void WipeKeystrokes(void); __msabi extern typeof(GetCurrentProcessId) *const __imp_GetCurrentProcessId; @@ -262,9 +264,13 @@ textwindows void WinMainForked(void) { __maps.free = 0; __maps.used = 0; __maps.maps = maps; + __maps.count = 0; + __maps.pages = 0; dll_init(&__maps.stack.elem); dll_make_first(&__maps.used, &__maps.stack.elem); for (struct Map *map = maps; map; map = map->next) { + __maps.count += 1; + __maps.pages += (map->size + 4095) / 4096; dll_make_last(&__maps.used, &map->elem); if (!VirtualProtect(map->addr, map->size, __prot2nt(map->prot, map->iscow), &oldprot)) { diff --git a/libc/proc/fork.c b/libc/proc/fork.c index af54d3295..d814902c9 100644 --- a/libc/proc/fork.c +++ b/libc/proc/fork.c @@ -17,6 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/assert.h" +#include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/calls/state.internal.h" #include "libc/calls/struct/sigset.h" @@ -44,15 +45,20 @@ #include "libc/thread/posixthread.internal.h" #include "libc/thread/tls.h" +extern atomic_uint free_waiters_mu; + static void _onfork_prepare(void) { if (_weaken(_pthread_onfork_prepare)) _weaken(_pthread_onfork_prepare)(); _pthread_lock(); __maps_lock(); __fds_lock(); + while (atomic_exchange_explicit(&free_waiters_mu, 1, memory_order_acquire)) { + } } static void _onfork_parent(void) { + atomic_store_explicit(&free_waiters_mu, 0, memory_order_release); __fds_unlock(); __maps_unlock(); _pthread_unlock(); @@ -65,6 +71,7 @@ static void _onfork_child(void) { pthread_mutexattr_init(&attr); pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); pthread_mutex_init(&__fds_lock_obj, &attr); + atomic_store_explicit(&free_waiters_mu, 0, memory_order_relaxed); pthread_mutexattr_destroy(&attr); _pthread_init(); __maps_unlock(); diff --git a/libc/runtime/cxa_thread_atexit.c b/libc/runtime/cxa_thread_atexit.c index ced061833..9d8eaa3f1 100644 --- a/libc/runtime/cxa_thread_atexit.c +++ b/libc/runtime/cxa_thread_atexit.c @@ -32,6 +32,16 @@ struct Dtor { static _Thread_local struct Dtor *__cxa_thread_atexit_list; +static void _pthread_unwind(struct CosmoTib *tib) { + struct PosixThread *pt; + struct _pthread_cleanup_buffer *cb; + pt = (struct PosixThread *)tib->tib_pthread; + while ((cb = pt->pt_cleanup)) { + pt->pt_cleanup = cb->__prev; + cb->__routine(cb->__arg); + } +} + static void __cxa_thread_unkey(struct CosmoTib *tib) { void *val; int i, j, gotsome; @@ -67,16 +77,18 @@ static void _pthread_ungarbage(struct CosmoTib *tib) { void __cxa_thread_finalize(void) { struct Dtor *dtor; + struct CosmoTib *tib; + tib = __get_tls(); + _pthread_unwind(tib); + if (tib->tib_nsync) + _weaken(nsync_waiter_destroy)(tib->tib_nsync); + __cxa_thread_unkey(tib); + _pthread_ungarbage(tib); while ((dtor = __cxa_thread_atexit_list)) { __cxa_thread_atexit_list = dtor->next; ((void (*)(void *))dtor->fun)(dtor->arg); _weaken(free)(dtor); } - struct CosmoTib *tib = __get_tls(); - __cxa_thread_unkey(tib); - if (tib->tib_nsync) - _weaken(nsync_waiter_destroy)(tib->tib_nsync); - _pthread_ungarbage(tib); } int __cxa_thread_atexit_impl(void *fun, void *arg, void *dso_symbol) { diff --git a/libc/runtime/mapstack.c b/libc/runtime/mapstack.c index 6e9a508ae..3bec5d639 100644 --- a/libc/runtime/mapstack.c +++ b/libc/runtime/mapstack.c @@ -44,7 +44,7 @@ */ void *NewCosmoStack(void) { char *p; - size_t n = GetStackSize() + (uintptr_t)ape_stack_align; + size_t n = GetStackSize(); if ((p = mmap(0, n, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)) != MAP_FAILED) { if (IsOpenbsd() && __sys_mmap(p, n, PROT_READ | PROT_WRITE, diff --git a/libc/runtime/stack.h b/libc/runtime/stack.h index 590ce3255..a09fe61d8 100644 --- a/libc/runtime/stack.h +++ b/libc/runtime/stack.h @@ -14,9 +14,6 @@ */ #define GetGuardSize() 16384 -#define STATIC_STACK_ALIGN(BYTES) \ - _STACK_SYMBOL("ape_stack_align", _STACK_STRINGIFY(BYTES) _STACK_EXTRA) - /** * Makes program stack executable if declared, e.g. * @@ -50,8 +47,6 @@ COSMOPOLITAN_C_START_ extern char ape_stack_prot[] __attribute__((__weak__)); extern char ape_stack_memsz[] __attribute__((__weak__)); -extern char ape_stack_align[] __attribute__((__weak__)); -extern char ape_stack_round[] __attribute__((__weak__)); uintptr_t GetStackBottom(void) pureconst; diff --git a/libc/runtime/winmain.greg.c b/libc/runtime/winmain.greg.c index 0c0e96fc4..1d92f976a 100644 --- a/libc/runtime/winmain.greg.c +++ b/libc/runtime/winmain.greg.c @@ -212,6 +212,8 @@ static abi wontreturn void WinInit(const char16_t *cmdline) { __maps.stack.size = stacksize; __maps.stack.prot = prot; __maps.maps = &__maps.stack; + __maps.pages = (stacksize + 4095) / 4096; + __maps.count = 1; dll_init(&__maps.stack.elem); dll_make_first(&__maps.used, &__maps.stack.elem); struct WinArgs *wa = diff --git a/libc/thread/posixthread.internal.h b/libc/thread/posixthread.internal.h index 319646e0e..cc9389d8c 100644 --- a/libc/thread/posixthread.internal.h +++ b/libc/thread/posixthread.internal.h @@ -112,7 +112,6 @@ void _pthread_onfork_parent(void) libcesque; void _pthread_onfork_prepare(void) libcesque; void _pthread_unlock(void) libcesque; void _pthread_unref(struct PosixThread *) libcesque; -void _pthread_unwind(struct PosixThread *) libcesque; void _pthread_zombify(struct PosixThread *) libcesque; forceinline pureconst struct PosixThread *_pthread_self(void) { diff --git a/libc/thread/pthread_create.c b/libc/thread/pthread_create.c index 15624b7ec..4312c114d 100644 --- a/libc/thread/pthread_create.c +++ b/libc/thread/pthread_create.c @@ -71,10 +71,8 @@ void _pthread_free(struct PosixThread *pt, bool isfork) { unassert(dll_is_alone(&pt->list) && &pt->list != _pthread_list); if (pt->pt_flags & PT_STATIC) return; - if (pt->pt_flags & PT_OWNSTACK) { - unassert(!munmap(pt->pt_attr.__stackaddr, - pt->pt_attr.__stacksize + (uintptr_t)ape_stack_align)); - } + if (pt->pt_flags & PT_OWNSTACK) + unassert(!munmap(pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize)); if (!isfork) { uint64_t syshand = atomic_load_explicit(&pt->tib->tib_syshand, memory_order_acquire); @@ -146,7 +144,7 @@ static int FixupCustomStackOnOpenbsd(pthread_attr_t *attr) { uintptr_t x, y; int e, rc, pagesz; pagesz = getauxval(AT_PAGESZ); - n = attr->__stacksize + (uintptr_t)ape_stack_align; + n = attr->__stacksize; x = (uintptr_t)attr->__stackaddr; y = ROUNDUP(x, pagesz); n -= y - x; @@ -156,7 +154,7 @@ static int FixupCustomStackOnOpenbsd(pthread_attr_t *attr) { MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD, -1, 0, 0) == (void *)y) { attr->__stackaddr = (void *)y; - attr->__stacksize = n - (uintptr_t)ape_stack_align; + attr->__stacksize = n; return 0; } else { rc = errno; @@ -219,13 +217,12 @@ static errno_t pthread_create_impl(pthread_t *thread, return EINVAL; } pt->pt_attr.__stackaddr = - mmap(0, pt->pt_attr.__stacksize + (uintptr_t)ape_stack_align, - PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + mmap(0, pt->pt_attr.__stacksize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (pt->pt_attr.__stackaddr != MAP_FAILED) { if (IsOpenbsd() && __sys_mmap( - pt->pt_attr.__stackaddr, - pt->pt_attr.__stacksize + (uintptr_t)ape_stack_align, + pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD, -1, 0, 0) != pt->pt_attr.__stackaddr) { diff --git a/libc/thread/pthread_exit.c b/libc/thread/pthread_exit.c index f6558670d..60524f5e7 100644 --- a/libc/thread/pthread_exit.c +++ b/libc/thread/pthread_exit.c @@ -36,14 +36,6 @@ #include "third_party/nsync/futex.internal.h" #include "third_party/nsync/wait_s.internal.h" -void _pthread_unwind(struct PosixThread *pt) { - struct _pthread_cleanup_buffer *cb; - while ((cb = pt->pt_cleanup)) { - pt->pt_cleanup = cb->__prev; - cb->__routine(cb->__arg); - } -} - /** * Terminates current POSIX thread. * @@ -89,7 +81,6 @@ wontreturn void pthread_exit(void *rc) { STRACE("pthread_exit(%p)", rc); // free resources - _pthread_unwind(pt); __cxa_thread_finalize(); _pthread_decimate(); diff --git a/test/libc/calls/setrlimit_test.c b/test/libc/calls/setrlimit_test.c index b9f57bcfc..2b7629ee6 100644 --- a/test/libc/calls/setrlimit_test.c +++ b/test/libc/calls/setrlimit_test.c @@ -131,6 +131,8 @@ TEST(setrlimit, testMemoryLimit) { int i, wstatus; if (IsXnu()) return; + if (IsOpenbsd()) + return; // simply too slow until mmap() becomes O(logn) ASSERT_NE(-1, (wstatus = xspawn(0))); if (wstatus == -2) { ASSERT_EQ(0, SetKernelEnforcedMemoryLimit(MEM)); diff --git a/third_party/lua/lua.main.c b/third_party/lua/lua.main.c index 18c450639..469c5638f 100644 --- a/third_party/lua/lua.main.c +++ b/third_party/lua/lua.main.c @@ -54,9 +54,6 @@ #include "tool/args/args.h" __static_yoink("lua_notice"); - -STATIC_STACK_ALIGN(GetStackSize()); - #if !defined(LUA_PROGNAME) #define LUA_PROGNAME "lua" #endif diff --git a/third_party/nsync/common.c b/third_party/nsync/common.c index 7250d8015..479ade601 100644 --- a/third_party/nsync/common.c +++ b/third_party/nsync/common.c @@ -36,6 +36,10 @@ #include "libc/runtime/runtime.h" #include "libc/runtime/runtime.h" #include "libc/sysv/consts/map.h" +#include "libc/nt/runtime.h" +#include "libc/intrin/directmap.internal.h" +#include "libc/thread/thread.h" +#include "libc/dce.h" #include "third_party/nsync/wait_s.internal.h" __static_yoink("nsync_notice"); @@ -154,10 +158,19 @@ waiter *nsync_dll_waiter_samecond_ (struct Dll *e) { static void *nsync_malloc (size_t size) { void *res; - res = mmap (0, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, 0); + if (!IsWindows ()) { + // too much of a performance hit to track + res = __sys_mmap ((void *)0x7110000000, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0, 0); + } else { + // must be tracked for fork() resurrection + res = mmap (0, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + } if (res == MAP_FAILED) nsync_panic_ ("out of memory\n"); return res; @@ -168,7 +181,7 @@ static void *nsync_malloc (size_t size) { static struct Dll *free_waiters = NULL; /* free_waiters points to a doubly-linked list of free waiter structs. */ -static nsync_atomic_uint32_ free_waiters_mu; /* spinlock; protects free_waiters */ +nsync_atomic_uint32_ free_waiters_mu; /* spinlock; protects free_waiters */ #define waiter_for_thread __get_tls()->tib_nsync @@ -236,6 +249,8 @@ void nsync_waiter_free_ (waiter *w) { nsync_spin_test_and_set_ (&free_waiters_mu, 1, 1, 0); dll_make_first (&free_waiters, &w->nw.q); ATM_STORE_REL (&free_waiters_mu, 0); /* release store */ + if (w == waiter_for_thread) + waiter_for_thread = 0; } } diff --git a/third_party/python/freeze.c b/third_party/python/freeze.c index d5f103ec6..48aacf823 100644 --- a/third_party/python/freeze.c +++ b/third_party/python/freeze.c @@ -7,7 +7,6 @@ #include "libc/assert.h" #include "libc/log/log.h" #include "libc/mem/mem.h" -#include "libc/runtime/stack.h" #include "third_party/python/Include/bytesobject.h" #include "third_party/python/Include/compile.h" #include "third_party/python/Include/fileutils.h" @@ -18,8 +17,6 @@ #include "third_party/python/Include/pymacro.h" #include "third_party/python/Include/pythonrun.h" -STATIC_STACK_ALIGN(GetStackSize()); - #define HEADER "\ /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-│\n\ │ vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi │\n\ diff --git a/third_party/python/launch.c b/third_party/python/launch.c index 75ddc987f..f74dce921 100644 --- a/third_party/python/launch.c +++ b/third_party/python/launch.c @@ -13,7 +13,6 @@ #include "libc/log/log.h" #include "libc/mem/mem.h" #include "libc/runtime/runtime.h" -#include "libc/runtime/stack.h" #include "libc/testlib/testlib.h" #include "libc/x/x.h" #include "third_party/python/Include/abstract.h" @@ -39,8 +38,6 @@ #include "third_party/python/Include/warnings.h" #include "third_party/python/Include/yoink.h" -STATIC_STACK_ALIGN(GetStackSize()); - #define USE_COSMO_CRASH MODE_DBG + 0 __static_yoink("zipos"); diff --git a/third_party/python/pyobj.c b/third_party/python/pyobj.c index cb7cd7d28..4f9def529 100644 --- a/third_party/python/pyobj.c +++ b/third_party/python/pyobj.c @@ -64,8 +64,6 @@ #include "libc/serialize.h" #include "tool/build/lib/stripcomponents.h" -STATIC_STACK_ALIGN(GetStackSize()); - __static_yoink("_PyUnicode_GetCode"); #define MANUAL "\ diff --git a/third_party/python/python3.c b/third_party/python/python3.c index 7bd3671aa..907bd32a7 100644 --- a/third_party/python/python3.c +++ b/third_party/python/python3.c @@ -4,13 +4,10 @@ │ Python 3 │ │ https://docs.python.org/3/license.html │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/runtime/stack.h" #include "third_party/python/Include/yoink.h" #include "third_party/python/runpythonmodule.h" #include "tool/args/args.h" -STATIC_STACK_ALIGN(GetStackSize()); - PYTHON_YOINK("xed"); PYTHON_YOINK("xterm"); diff --git a/third_party/python/pythontester.c b/third_party/python/pythontester.c index 7455efc3e..07549673b 100644 --- a/third_party/python/pythontester.c +++ b/third_party/python/pythontester.c @@ -6,13 +6,10 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/log/log.h" #include "libc/runtime/runtime.h" -#include "libc/runtime/stack.h" #include "third_party/python/Include/yoink.h" #include "third_party/python/runpythonmodule.h" #include "tool/args/args.h" -STATIC_STACK_ALIGN(GetStackSize()); - int main(int argc, char **argv) { diff --git a/third_party/python/repl.c b/third_party/python/repl.c index 2064e2c3b..9528e4e83 100644 --- a/third_party/python/repl.c +++ b/third_party/python/repl.c @@ -4,13 +4,10 @@ │ Python 3 │ │ https://docs.python.org/3/license.html │ ╚─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/runtime/stack.h" #include "third_party/python/Include/yoink.h" #include "third_party/python/runpythonmodule.h" #include "tool/args/args.h" -STATIC_STACK_ALIGN(GetStackSize()); - int main(int argc, char **argv) { diff --git a/tool/net/redbean.c b/tool/net/redbean.c index 2ef31064b..8696121e9 100644 --- a/tool/net/redbean.c +++ b/tool/net/redbean.c @@ -139,8 +139,6 @@ #pragma GCC diagnostic ignored "-Wunused-variable" -STATIC_STACK_ALIGN(GetStackSize()); - __static_yoink("zipos"); #ifdef USE_BLINK