From 8c645fa1ee6fe95e52ae76360d061fe9b5c3ce81 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Fri, 5 Jul 2024 23:13:20 -0700 Subject: [PATCH] Make mmap() scalable It's now possible to create thousands of thousands of sparse independent memory mappings, without any slowdown. The memory manager is better with tracking memory protection now, particularly on Windows in a precise way that can be restored during fork(). You now have the highest quality mem manager possible. It's even better than some OSes like XNU, where mmap() is implemented as an O(n) operation which means sadly things aren't much improved over there. With this change the llamafile HTTP server endpoint at /tokenize with a prompt of 50 tokens is now able to handle 2.6m r/sec --- libc/calls/setrlimit.c | 2 +- libc/intrin/BUILD.mk | 4 + libc/intrin/describemapping.c | 4 +- libc/intrin/describeprotflags.c | 12 +- libc/intrin/describerlimit.c | 16 +- libc/intrin/directmap-nt.c | 3 +- libc/intrin/dll.c | 8 +- libc/intrin/kprintf.greg.c | 30 +- libc/intrin/maps.c | 4 +- libc/intrin/maps.h | 54 ++- libc/intrin/mmap.c | 260 +++++++-------- libc/intrin/mprotect.c | 189 ++++++----- libc/intrin/msync-nt.c | 21 +- libc/intrin/printmaps.c | 11 +- libc/intrin/prot2nt.greg.c | 2 + libc/intrin/pthread_delay_np.c | 40 +++ libc/intrin/pthread_mutex_lock.c | 125 +++---- libc/intrin/pthread_mutex_trylock.c | 1 + libc/intrin/pthread_mutex_unlock.c | 1 + libc/intrin/pthread_syshand.c | 2 +- libc/intrin/pthread_tid.c | 2 +- libc/{calls => intrin}/pthread_yield_np.c | 0 libc/intrin/rbtree.c | 312 ------------------ libc/intrin/rbtree.h | 57 ---- libc/intrin/tree.c | 271 +++++++++++++++ libc/intrin/tree.h | 91 +++++ libc/log/oncrash_amd64.c | 10 +- libc/proc/fork-nt.c | 142 ++++---- libc/proc/fork.c | 17 +- libc/proc/posix_spawn.c | 57 ++-- libc/runtime/mapstack.c | 5 +- libc/runtime/runtime.h | 2 +- libc/stdio/printargs.c | 2 +- libc/sysv/consts.sh | 1 + libc/sysv/consts/PROT_GUARD.S | 2 + libc/sysv/consts/prot.h | 5 +- libc/thread/pthread_create.c | 18 +- libc/thread/thread.h | 1 + test/ctl/set_test.cc | 16 +- test/libc/calls/raise_test.c | 1 - test/libc/calls/setrlimit_test.c | 24 +- test/libc/intrin/mmap_scalability_test.c | 76 +++++ test/libc/intrin/mmap_test.c | 16 + .../intrin/{rbtree_test.c => tree_test.c} | 76 +++-- test/libc/thread/pthread_atfork_test.c | 17 +- third_party/nsync/common.c | 146 ++++---- third_party/nsync/common.internal.h | 11 +- third_party/nsync/mem/nsync_cv.c | 14 +- third_party/nsync/mem/nsync_debug.c | 4 +- third_party/nsync/mem/nsync_mu_wait.c | 7 +- third_party/nsync/mem/nsync_once.c | 3 +- third_party/nsync/mu.c | 7 +- third_party/nsync/mu_semaphore.c | 2 +- third_party/nsync/mu_semaphore.h | 2 +- third_party/nsync/mu_semaphore.internal.h | 6 +- third_party/nsync/mu_semaphore_futex.c | 3 +- third_party/nsync/mu_semaphore_gcd.c | 4 +- third_party/nsync/mu_semaphore_sem.c | 31 +- tool/build/runitd.c | 55 ++- 59 files changed, 1238 insertions(+), 1067 deletions(-) create mode 100644 libc/intrin/pthread_delay_np.c rename libc/{calls => intrin}/pthread_yield_np.c (100%) delete mode 100644 libc/intrin/rbtree.c delete mode 100644 libc/intrin/rbtree.h create mode 100644 libc/intrin/tree.c create mode 100644 libc/intrin/tree.h create mode 100644 libc/sysv/consts/PROT_GUARD.S create mode 100644 test/libc/intrin/mmap_scalability_test.c rename test/libc/intrin/{rbtree_test.c => tree_test.c} (55%) diff --git a/libc/calls/setrlimit.c b/libc/calls/setrlimit.c index 45a8c532c..de7b5adc9 100644 --- a/libc/calls/setrlimit.c +++ b/libc/calls/setrlimit.c @@ -85,7 +85,7 @@ int setrlimit(int resource, const struct rlimit *rlim) { rc = efault(); } else if (IsXnuSilicon()) { rc = _sysret(__syslib->__setrlimit(resource, rlim)); - } else if (!IsWindows()) { + } else if (!IsWindows() && !(IsNetbsd() && resource == RLIMIT_AS)) { rc = sys_setrlimit(resource, rlim); } else if (resource == RLIMIT_STACK) { rc = enotsup(); diff --git a/libc/intrin/BUILD.mk b/libc/intrin/BUILD.mk index a8052956e..5f05cfc5e 100644 --- a/libc/intrin/BUILD.mk +++ b/libc/intrin/BUILD.mk @@ -62,6 +62,10 @@ o/$(MODE)/libc/intrin/kprintf.o: private \ -Wframe-larger-than=128 \ -Walloca-larger-than=128 +o/$(MODE)/libc/intrin/tree.o: private \ + CFLAGS += \ + -ffunction-sections + o//libc/intrin/memmove.o: private \ CFLAGS += \ -fno-toplevel-reorder diff --git a/libc/intrin/describemapping.c b/libc/intrin/describemapping.c index 258e9a4ab..d4c54ef49 100644 --- a/libc/intrin/describemapping.c +++ b/libc/intrin/describemapping.c @@ -24,7 +24,7 @@ static char DescribeMapType(int flags) { switch (flags & MAP_TYPE) { case MAP_FILE: - return 'f'; + return '-'; case MAP_PRIVATE: return 'p'; case MAP_SHARED: @@ -47,7 +47,7 @@ const char *(DescribeMapping)(char p[8], int prot, int flags) { DescribeProt(p, prot); p[3] = DescribeMapType(flags); p[4] = (flags & MAP_ANONYMOUS) ? 'a' : '-'; - p[5] = (flags & MAP_FIXED) ? 'F' : '-'; + p[5] = (flags & MAP_FIXED) ? 'f' : '-'; p[6] = 0; return p; } diff --git a/libc/intrin/describeprotflags.c b/libc/intrin/describeprotflags.c index f281dffc2..5761ac0fe 100644 --- a/libc/intrin/describeprotflags.c +++ b/libc/intrin/describeprotflags.c @@ -20,12 +20,12 @@ #include "libc/macros.internal.h" #include "libc/sysv/consts/prot.h" -static const struct DescribeFlags kProtFlags[] = { - {PROT_READ, "READ"}, // - {PROT_WRITE, "WRITE"}, // - {PROT_EXEC, "EXEC"}, // -}; - const char *(DescribeProtFlags)(char buf[48], int x) { + const struct DescribeFlags kProtFlags[] = { + {PROT_READ, "READ"}, // + {PROT_WRITE, "WRITE"}, // + {PROT_EXEC, "EXEC"}, // + {PROT_GUARD, "GUARD"}, // + }; return DescribeFlags(buf, 48, kProtFlags, ARRAYLEN(kProtFlags), "PROT_", x); } diff --git a/libc/intrin/describerlimit.c b/libc/intrin/describerlimit.c index 79d9a7d46..9d211a6c4 100644 --- a/libc/intrin/describerlimit.c +++ b/libc/intrin/describerlimit.c @@ -18,8 +18,11 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/struct/rlimit.h" #include "libc/dce.h" +#include "libc/fmt/itoa.h" #include "libc/intrin/kprintf.h" #include "libc/intrin/strace.internal.h" +#include "libc/str/str.h" +#include "libc/sysv/consts/rlim.h" const char *DescribeRlimit(char buf[64], int rc, const struct rlimit *rlim) { if (rc == -1) @@ -29,7 +32,18 @@ const char *DescribeRlimit(char buf[64], int rc, const struct rlimit *rlim) { if (kisdangerous(rlim)) { ksnprintf(buf, 64, "%p", rlim); } else { - ksnprintf(buf, 64, "{%'ld, %'ld}", rlim->rlim_cur, rlim->rlim_max); + char str[2][21]; + if (rlim->rlim_cur == RLIM_INFINITY) { + strcpy(str[0], "RLIM_INFINITY"); + } else { + FormatInt64(str[0], rlim->rlim_cur); + } + if (rlim->rlim_max == RLIM_INFINITY) { + strcpy(str[1], "RLIM_INFINITY"); + } else { + FormatInt64(str[1], rlim->rlim_max); + } + ksnprintf(buf, 64, "{%s, %s}", str[0], str[1]); } return buf; } diff --git a/libc/intrin/directmap-nt.c b/libc/intrin/directmap-nt.c index a6fd1d2f1..00b93463d 100644 --- a/libc/intrin/directmap-nt.c +++ b/libc/intrin/directmap-nt.c @@ -86,9 +86,8 @@ TryAgain: if ((dm.addr = MapViewOfFileEx(dm.maphandle, fl.flags2, off >> 32, off, size, addr))) { uint32_t oldprot; - if (VirtualProtect(dm.addr, size, __prot2nt(prot, iscow), &oldprot)) { + if (VirtualProtect(dm.addr, size, __prot2nt(prot, iscow), &oldprot)) return dm; - } UnmapViewOfFile(dm.addr); } CloseHandle(dm.maphandle); diff --git a/libc/intrin/dll.c b/libc/intrin/dll.c index b94cfd7b2..b6337273d 100644 --- a/libc/intrin/dll.c +++ b/libc/intrin/dll.c @@ -28,7 +28,7 @@ * * It's required that `elem` and `succ` aren't part of the same list. */ -privileged void dll_splice_after(struct Dll *elem, struct Dll *succ) { +void dll_splice_after(struct Dll *elem, struct Dll *succ) { struct Dll *tmp1, *tmp2; tmp1 = elem->next; tmp2 = succ->prev; @@ -43,7 +43,7 @@ privileged void dll_splice_after(struct Dll *elem, struct Dll *succ) { * * @param list is a doubly-linked list, where `!*list` means empty */ -privileged void dll_remove(struct Dll **list, struct Dll *elem) { +void dll_remove(struct Dll **list, struct Dll *elem) { if (*list == elem) { if ((*list)->prev == *list) { *list = 0; @@ -66,7 +66,7 @@ privileged void dll_remove(struct Dll **list, struct Dll *elem) { * @param list is a doubly-linked list, where `!*list` means empty * @param elem must not be a member of `list`, or null for no-op */ -privileged void dll_make_first(struct Dll **list, struct Dll *elem) { +void dll_make_first(struct Dll **list, struct Dll *elem) { if (elem) { if (!*list) { *list = elem->prev; @@ -85,7 +85,7 @@ privileged void dll_make_first(struct Dll **list, struct Dll *elem) { * @param list is a doubly-linked list, where `!*list` means empty * @param elem must not be a member of `list`, or null for no-op */ -privileged void dll_make_last(struct Dll **list, struct Dll *elem) { +void dll_make_last(struct Dll **list, struct Dll *elem) { if (elem) { dll_make_first(list, elem->next); *list = elem; diff --git a/libc/intrin/kprintf.greg.c b/libc/intrin/kprintf.greg.c index 2bd649244..d32c52c64 100644 --- a/libc/intrin/kprintf.greg.c +++ b/libc/intrin/kprintf.greg.c @@ -25,7 +25,6 @@ #include "libc/fmt/magnumstrs.internal.h" #include "libc/intrin/asmflag.h" #include "libc/intrin/atomic.h" -#include "libc/intrin/dll.h" #include "libc/intrin/getenv.internal.h" #include "libc/intrin/kprintf.h" #include "libc/intrin/likely.h" @@ -154,27 +153,18 @@ __funline bool kischarmisaligned(const char *p, signed char t) { return false; } -privileged static bool32 kisdangerous_unlocked(const char *addr) { - struct Dll *e; - if ((e = dll_first(__maps.used))) { - do { - struct Map *map = MAP_CONTAINER(e); - if (map->addr <= addr && addr < map->addr + map->size) { - dll_remove(&__maps.used, e); - dll_make_first(&__maps.used, e); - return !(map->prot & PROT_READ); - } - } while ((e = dll_next(__maps.used, e))); - return true; - } else { - return false; - } -} - privileged bool32 kisdangerous(const void *addr) { - bool32 res; + bool32 res = true; __maps_lock(); - res = kisdangerous_unlocked(addr); + if (__maps.maps) { + struct Map *map; + if ((map = __maps_floor(addr))) + if ((const char *)addr >= map->addr && + (const char *)addr < map->addr + map->size) + res = false; + } else { + res = false; + } __maps_unlock(); return res; } diff --git a/libc/intrin/maps.c b/libc/intrin/maps.c index e2786ee75..e845679e5 100644 --- a/libc/intrin/maps.c +++ b/libc/intrin/maps.c @@ -20,6 +20,7 @@ #include "ape/sections.internal.h" #include "libc/dce.h" #include "libc/intrin/dll.h" +#include "libc/intrin/maps.h" #include "libc/runtime/runtime.h" #include "libc/runtime/stack.h" #include "libc/sysv/consts/auxv.h" @@ -32,8 +33,7 @@ __static_yoink("_init_maps"); struct Maps __maps; void __maps_add(struct Map *map) { - dll_init(&map->elem); - dll_make_first(&__maps.used, &map->elem); + tree_insert(&__maps.maps, &map->tree, __maps_compare); ++__maps.count; } diff --git a/libc/intrin/maps.h b/libc/intrin/maps.h index 5b5aba76a..387975b41 100644 --- a/libc/intrin/maps.h +++ b/libc/intrin/maps.h @@ -1,36 +1,41 @@ -#ifndef COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_ -#define COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_ +#ifndef COSMOPOLITAN_MAPS_H_ +#define COSMOPOLITAN_MAPS_H_ #include "libc/intrin/atomic.h" #include "libc/intrin/dll.h" +#include "libc/intrin/tree.h" +#include "libc/runtime/runtime.h" #include "libc/thread/tls2.internal.h" COSMOPOLITAN_C_START_ -#define MAP_CONTAINER(e) DLL_CONTAINER(struct Map, elem, e) +#define MAP_TREE_CONTAINER(e) TREE_CONTAINER(struct Map, tree, e) +#define MAP_FREE_CONTAINER(e) DLL_CONTAINER(struct Map, free, e) struct Map { char *addr; /* granule aligned */ size_t size; /* must be nonzero */ - struct Dll elem; /* for __maps.free */ - int64_t off; /* -1 if anonymous */ + int64_t off; /* ignore for anon */ int prot; /* memory protects */ int flags; /* memory map flag */ bool iscow; /* windows nt only */ bool readonlyfile; /* windows nt only */ unsigned visited; /* used for checks */ + unsigned oldprot; /* in windows fork */ intptr_t hand; /* windows nt only */ + union { + struct Tree tree; + struct Dll free; + }; }; struct Maps { - unsigned mono; atomic_int lock; + struct Tree *maps; struct Dll *free; - struct Dll *used; size_t count; size_t pages; + atomic_ulong rollo; struct Map stack; struct Map guard; - bool once; - atomic_ulong rollo; }; struct AddrSize { @@ -45,10 +50,37 @@ bool __maps_lock(void); void __maps_check(void); void __maps_unlock(void); void __maps_add(struct Map *); -struct Map *__maps_alloc(void); void __maps_free(struct Map *); +struct Map *__maps_alloc(void); +struct Map *__maps_floor(const char *); void __maps_stack(char *, int, int, size_t, int, intptr_t); +int __maps_compare(const struct Tree *, const struct Tree *); struct AddrSize __get_main_stack(void); +forceinline optimizespeed int __maps_search(const void *key, + const struct Tree *node) { + const char *addr = (const char *)key; + const struct Map *map = (const struct Map *)MAP_TREE_CONTAINER(node); + if (addr < map->addr) + return +1; + if (addr >= map->addr + map->size) + return -1; + return 0; +} + +static struct Map *__maps_next(struct Map *map) { + struct Tree *node; + if ((node = tree_next(&map->tree))) + return MAP_TREE_CONTAINER(node); + return 0; +} + +static struct Map *__maps_first(void) { + struct Tree *node; + if ((node = tree_first(__maps.maps))) + return MAP_TREE_CONTAINER(node); + return 0; +} + COSMOPOLITAN_C_END_ -#endif /* COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_ */ +#endif /* COSMOPOLITAN_MAPS_H_ */ diff --git a/libc/intrin/mmap.c b/libc/intrin/mmap.c index c4229924e..02ef9614e 100644 --- a/libc/intrin/mmap.c +++ b/libc/intrin/mmap.c @@ -67,20 +67,31 @@ struct StackFrame *bp = __builtin_frame_address(0); \ kprintf("%!s:%d: assertion failed: %!s\n", __FILE__, __LINE__, #x); \ kprintf("bt %!s\n", (DescribeBacktrace)(bt, bp)); \ - __print_maps(); \ - _Exit(99); \ + __print_maps(0); \ + __builtin_trap(); \ } \ } while (0) #endif +int __maps_compare(const struct Tree *ra, const struct Tree *rb) { + const struct Map *a = (const struct Map *)MAP_TREE_CONTAINER(ra); + const struct Map *b = (const struct Map *)MAP_TREE_CONTAINER(rb); + return (a->addr > b->addr) - (a->addr < b->addr); +} + +privileged optimizespeed struct Map *__maps_floor(const char *addr) { + struct Tree *node; + if ((node = tree_floor(__maps.maps, addr, __maps_search))) + return MAP_TREE_CONTAINER(node); + return 0; +} + static bool overlaps_existing_map(const char *addr, size_t size, int pagesz) { - for (struct Dll *e = dll_first(__maps.used); e; - e = dll_next(__maps.used, e)) { - struct Map *map = MAP_CONTAINER(e); + struct Map *map; + if ((map = __maps_floor(addr))) if (MAX(addr, map->addr) < MIN(addr + PGUP(size), map->addr + PGUP(map->size))) return true; - } return false; } @@ -89,66 +100,51 @@ void __maps_check(void) { size_t maps = 0; size_t pages = 0; int pagesz = getpagesize(); - unsigned id = ++__maps.mono; - for (struct Dll *e = dll_first(__maps.used); e; - e = dll_next(__maps.used, e)) { - struct Map *map = MAP_CONTAINER(e); + static unsigned mono; + unsigned id = ++mono; + for (struct Map *map = __maps_first(); map; map = __maps_next(map)) { ASSERT(map->addr != MAP_FAILED); ASSERT(map->visited != id); ASSERT(map->size); map->visited = id; pages += (map->size + getpagesize() - 1) / getpagesize(); maps += 1; + struct Map *next; + if ((next = __maps_next(map))) { + ASSERT(map->addr < next->addr); + ASSERT( + !(MAX(map->addr, next->addr) < + MIN(map->addr + PGUP(map->size), next->addr + PGUP(next->size)))); + } } ASSERT(maps = __maps.count); ASSERT(pages == __maps.pages); - for (struct Dll *e = dll_first(__maps.used); e; - e = dll_next(__maps.used, e)) { - struct Map *m1 = MAP_CONTAINER(e); - for (struct Dll *f = dll_next(__maps.used, e); f; - f = dll_next(__maps.used, f)) { - struct Map *m2 = MAP_CONTAINER(f); - ASSERT(MAX(m1->addr, m2->addr) >= - MIN(m1->addr + PGUP(m1->size), m2->addr + PGUP(m2->size))); - } - } #endif } void __maps_free(struct Map *map) { map->size = 0; map->addr = MAP_FAILED; - ASSERT(dll_is_alone(&map->elem)); - dll_make_last(&__maps.free, &map->elem); + dll_init(&map->free); + dll_make_first(&__maps.free, &map->free); } static void __maps_insert(struct Map *map) { - struct Dll *e = dll_first(__maps.used); - struct Map *last = e ? MAP_CONTAINER(e) : 0; __maps.pages += (map->size + getpagesize() - 1) / getpagesize(); - if (last && !IsWindows() && // - map->addr == last->addr + last->size && // + struct Map *floor = __maps_floor(map->addr); + if (floor && !IsWindows() && // + map->addr + map->size == floor->addr && // (map->flags & MAP_ANONYMOUS) && // - map->flags == last->flags && // - map->prot == last->prot) { - last->size += map->size; - dll_remove(&__maps.used, &last->elem); - dll_make_first(&__maps.used, &last->elem); - __maps_free(map); - } else if (last && !IsWindows() && // - map->addr + map->size == last->addr && // - (map->flags & MAP_ANONYMOUS) && // - map->flags == last->flags && // - map->prot == last->prot) { - last->addr -= map->size; - last->size += map->size; - dll_remove(&__maps.used, &last->elem); - dll_make_first(&__maps.used, &last->elem); + map->flags == floor->flags && // + map->prot == floor->prot) { + floor->addr -= map->size; + floor->size += map->size; __maps_free(map); + __maps_check(); } else { __maps_add(map); + __maps_check(); } - __maps_check(); } struct Map *__maps_alloc(void) { @@ -156,7 +152,7 @@ struct Map *__maps_alloc(void) { struct Map *map; if ((e = dll_first(__maps.free))) { dll_remove(&__maps.free, e); - map = MAP_CONTAINER(e); + map = MAP_FREE_CONTAINER(e); return map; } int granularity = __granularity(); @@ -168,11 +164,8 @@ struct Map *__maps_alloc(void) { CloseHandle(sys.maphandle); map = sys.addr; map->addr = MAP_FAILED; - dll_init(&map->elem); - for (int i = 1; i < granularity / sizeof(struct Map); ++i) { - dll_init(&map[i].elem); + for (int i = 1; i < granularity / sizeof(struct Map); ++i) __maps_free(map + i); - } return map; } @@ -190,106 +183,106 @@ static int __munmap(char *addr, size_t size, bool untrack_only) { // untrack mappings int rc = 0; - struct Dll *cur; - struct Dll *next; - struct Dll *delete = 0; + struct Map *map; + struct Map *next; + struct Dll *deleted = 0; if (__maps_lock()) { __maps_unlock(); return edeadlk(); } - for (cur = dll_first(__maps.used); cur; cur = next) { - next = dll_next(__maps.used, cur); - struct Map *map = MAP_CONTAINER(cur); + for (map = __maps_floor(addr); map; map = next) { + next = __maps_next(map); char *map_addr = map->addr; size_t map_size = map->size; - if (MAX(addr, map_addr) < MIN(addr + size, map_addr + PGUP(map_size))) { - if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) { - // remove mapping completely - dll_remove(&__maps.used, cur); - dll_make_first(&delete, cur); - __maps.pages -= (map_size + pagesz - 1) / pagesz; - __maps.count -= 1; - __maps_check(); - } else if (IsWindows()) { - // you can't carve up memory maps on windows. our mmap() makes - // this not a problem (for non-enormous memory maps) by making - // independent mappings for each 64 kb granule, under the hood - rc = einval(); - } else if (addr <= map_addr) { - // shave off lefthand side of mapping - ASSERT(addr + size < map_addr + PGUP(map_size)); - size_t left = PGUP(addr + size - map_addr); - size_t right = map_size - left; - ASSERT(right > 0); - ASSERT(left > 0); - struct Map *leftmap; - if ((leftmap = __maps_alloc())) { - map->addr += left; - map->size = right; - if (!(map->flags & MAP_ANONYMOUS)) - map->off += left; - __maps.pages -= (left + pagesz - 1) / pagesz; + if (!(MAX(addr, map_addr) < MIN(addr + size, map_addr + PGUP(map_size)))) + break; + if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) { + // remove mapping completely + tree_remove(&__maps.maps, &map->tree); + dll_init(&map->free); + dll_make_first(&deleted, &map->free); + __maps.pages -= (map_size + pagesz - 1) / pagesz; + __maps.count -= 1; + } else if (IsWindows()) { + // you can't carve up memory maps on windows. our mmap() makes + // this not a problem (for non-enormous memory maps) by making + // independent mappings for each 64 kb granule, under the hood + rc = einval(); + } else if (addr <= map_addr) { + // shave off lefthand side of mapping + ASSERT(addr + size < map_addr + PGUP(map_size)); + size_t left = PGUP(addr + size - map_addr); + size_t right = map_size - left; + ASSERT(right > 0); + ASSERT(left > 0); + struct Map *leftmap; + if ((leftmap = __maps_alloc())) { + map->addr += left; + map->size = right; + if (!(map->flags & MAP_ANONYMOUS)) + map->off += left; + __maps.pages -= (left + pagesz - 1) / pagesz; + leftmap->addr = map_addr; + leftmap->size = left; + dll_init(&leftmap->free); + dll_make_first(&deleted, &leftmap->free); + } else { + rc = -1; + } + } else if (addr + size >= map_addr + PGUP(map_size)) { + // shave off righthand side of mapping + size_t left = addr - map_addr; + size_t right = map_addr + map_size - addr; + struct Map *rightmap; + if ((rightmap = __maps_alloc())) { + map->size = left; + __maps.pages -= (right + pagesz - 1) / pagesz; + rightmap->addr = addr; + rightmap->size = right; + dll_init(&rightmap->free); + dll_make_first(&deleted, &rightmap->free); + } else { + rc = -1; + } + } else { + // punch hole in mapping + size_t left = addr - map_addr; + size_t middle = size; + size_t right = map_size - middle - left; + struct Map *leftmap; + if ((leftmap = __maps_alloc())) { + struct Map *middlemap; + if ((middlemap = __maps_alloc())) { leftmap->addr = map_addr; leftmap->size = left; - dll_make_first(&delete, &leftmap->elem); - __maps_check(); - } else { - rc = -1; - } - } else if (addr + size >= map_addr + PGUP(map_size)) { - // shave off righthand side of mapping - size_t left = addr - map_addr; - size_t right = map_addr + map_size - addr; - struct Map *rightmap; - if ((rightmap = __maps_alloc())) { - map->size = left; - __maps.pages -= (right + pagesz - 1) / pagesz; - rightmap->addr = addr; - rightmap->size = right; - dll_make_first(&delete, &rightmap->elem); - __maps_check(); + leftmap->off = map->off; + leftmap->prot = map->prot; + leftmap->flags = map->flags; + map->addr += left + middle; + map->size = right; + if (!(map->flags & MAP_ANONYMOUS)) + map->off += left + middle; + tree_insert(&__maps.maps, &leftmap->tree, __maps_compare); + __maps.pages -= (middle + pagesz - 1) / pagesz; + __maps.count += 1; + middlemap->addr = addr; + middlemap->size = size; + dll_init(&middlemap->free); + dll_make_first(&deleted, &middlemap->free); } else { rc = -1; } } else { - // punch hole in mapping - size_t left = addr - map_addr; - size_t middle = size; - size_t right = map_size - middle - left; - struct Map *leftmap; - if ((leftmap = __maps_alloc())) { - struct Map *middlemap; - if ((middlemap = __maps_alloc())) { - leftmap->addr = map_addr; - leftmap->size = left; - leftmap->off = map->off; - leftmap->prot = map->prot; - leftmap->flags = map->flags; - map->addr += left + middle; - map->size = right; - if (!(map->flags & MAP_ANONYMOUS)) - map->off += left + middle; - dll_make_first(&__maps.used, &leftmap->elem); - __maps.pages -= (middle + pagesz - 1) / pagesz; - __maps.count += 1; - middlemap->addr = addr; - middlemap->size = size; - dll_make_first(&delete, &middlemap->elem); - __maps_check(); - } else { - rc = -1; - } - } else { - rc = -1; - } + rc = -1; } } + __maps_check(); } __maps_unlock(); // delete mappings - for (struct Dll *e = dll_first(delete); e; e = dll_next(delete, e)) { - struct Map *map = MAP_CONTAINER(e); + for (struct Dll *e = dll_first(deleted); e; e = dll_next(deleted, e)) { + struct Map *map = MAP_FREE_CONTAINER(e); if (!untrack_only) { if (!IsWindows()) { if (sys_munmap(map->addr, map->size)) @@ -305,12 +298,12 @@ static int __munmap(char *addr, size_t size, bool untrack_only) { } // free mappings - if (!dll_is_empty(delete)) { + if (!dll_is_empty(deleted)) { __maps_lock(); struct Dll *e; - while ((e = dll_first(delete))) { - dll_remove(&delete, e); - __maps_free(MAP_CONTAINER(e)); + while ((e = dll_first(deleted))) { + dll_remove(&deleted, e); + __maps_free(MAP_FREE_CONTAINER(e)); } __maps_check(); __maps_unlock(); @@ -350,6 +343,7 @@ static void *__mmap_chunk(void *addr, size_t size, int prot, int flags, int fd, __maps_unlock(); return (void *)edeadlk(); } + __maps_check(); map = __maps_alloc(); __maps_unlock(); if (!map) diff --git a/libc/intrin/mprotect.c b/libc/intrin/mprotect.c index 83dc39926..76a40175e 100644 --- a/libc/intrin/mprotect.c +++ b/libc/intrin/mprotect.c @@ -25,6 +25,7 @@ #include "libc/intrin/kprintf.h" #include "libc/intrin/maps.h" #include "libc/intrin/strace.internal.h" +#include "libc/intrin/tree.h" #include "libc/nt/memory.h" #include "libc/runtime/internal.h" #include "libc/runtime/runtime.h" @@ -51,7 +52,7 @@ static int __mprotect_chunk(char *addr, size_t size, int prot, bool iscow) { int __mprotect(char *addr, size_t size, int prot) { // unix checks prot before checking size - if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC)) + if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC | PROT_GUARD)) return einval(); // make new technology consistent with unix @@ -68,64 +69,97 @@ int __mprotect(char *addr, size_t size, int prot) { // change mappings int rc = 0; - struct Dll *cur; bool found = false; if (__maps_lock()) { __maps_unlock(); return edeadlk(); } - for (cur = dll_first(__maps.used); cur; cur = dll_next(__maps.used, cur)) { - struct Map *map = MAP_CONTAINER(cur); + for (struct Map *map = __maps_floor(addr); map; map = __maps_next(map)) { char *map_addr = map->addr; size_t map_size = map->size; char *beg = MAX(addr, map_addr); char *end = MIN(addr + size, map_addr + PGUP(map_size)); - if (beg < end) { - found = true; - if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) { - // change protection of entire mapping - if (!__mprotect_chunk(map_addr, map_size, prot, map->iscow)) { + if (beg >= end) + break; + found = true; + if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) { + // change protection of entire mapping + if (!__mprotect_chunk(map_addr, map_size, prot, map->iscow)) { + map->prot = prot; + } else { + rc = -1; + } + } else if (addr <= map_addr) { + // change lefthand side of mapping + size_t left = PGUP(addr + size - map_addr); + size_t right = map_size - left; + struct Map *leftmap; + if ((leftmap = __maps_alloc())) { + if (!__mprotect_chunk(map_addr, left, prot, false)) { + leftmap->addr = map_addr; + leftmap->size = left; + leftmap->prot = prot; + leftmap->off = map->off; + leftmap->flags = map->flags; + leftmap->iscow = map->iscow; + leftmap->readonlyfile = map->readonlyfile; + leftmap->hand = map->hand; + map->addr += left; + map->size = right; + map->hand = -1; + if (!(map->flags & MAP_ANONYMOUS)) + map->off += left; + tree_insert(&__maps.maps, &leftmap->tree, __maps_compare); + __maps.count += 1; + __maps_check(); + } else { + __maps_free(leftmap); + rc = -1; + } + } else { + rc = -1; + } + } else if (addr + size >= map_addr + PGUP(map_size)) { + // change righthand side of mapping + size_t left = addr - map_addr; + size_t right = map_addr + map_size - addr; + struct Map *leftmap; + if ((leftmap = __maps_alloc())) { + if (!__mprotect_chunk(map_addr + left, right, prot, false)) { + leftmap->addr = map_addr; + leftmap->size = left; + leftmap->off = map->off; + leftmap->prot = map->prot; + leftmap->flags = map->flags; + leftmap->iscow = map->iscow; + leftmap->readonlyfile = map->readonlyfile; + leftmap->hand = map->hand; + map->addr += left; + map->size = right; map->prot = prot; + map->hand = -1; + if (!(map->flags & MAP_ANONYMOUS)) + map->off += left; + tree_insert(&__maps.maps, &leftmap->tree, __maps_compare); + __maps.count += 1; + __maps_check(); } else { + __maps_free(leftmap); rc = -1; } - } else if (addr <= map_addr) { - // change lefthand side of mapping - size_t left = PGUP(addr + size - map_addr); - size_t right = map_size - left; - struct Map *leftmap; - if ((leftmap = __maps_alloc())) { - if (!__mprotect_chunk(map_addr, left, prot, false)) { - leftmap->addr = map_addr; - leftmap->size = left; - leftmap->prot = prot; - leftmap->off = map->off; - leftmap->flags = map->flags; - leftmap->iscow = map->iscow; - leftmap->readonlyfile = map->readonlyfile; - leftmap->hand = map->hand; - map->addr += left; - map->size = right; - map->hand = -1; - if (!(map->flags & MAP_ANONYMOUS)) - map->off += left; - dll_make_first(&__maps.used, &leftmap->elem); - __maps.count += 1; - __maps_check(); - } else { - __maps_free(leftmap); - rc = -1; - } - } else { - rc = -1; - } - } else if (addr + size >= map_addr + PGUP(map_size)) { - // change righthand side of mapping - size_t left = addr - map_addr; - size_t right = map_addr + map_size - addr; - struct Map *leftmap; - if ((leftmap = __maps_alloc())) { - if (!__mprotect_chunk(map_addr + left, right, prot, false)) { + } else { + rc = -1; + } + } else { + // change middle of mapping + size_t left = addr - map_addr; + size_t middle = size; + size_t right = map_size - middle - left; + struct Map *leftmap; + if ((leftmap = __maps_alloc())) { + struct Map *midlmap; + if ((midlmap = __maps_alloc())) { + if (!__mprotect_chunk(map_addr + left, middle, prot, false)) { leftmap->addr = map_addr; leftmap->size = left; leftmap->off = map->off; @@ -134,67 +168,32 @@ int __mprotect(char *addr, size_t size, int prot) { leftmap->iscow = map->iscow; leftmap->readonlyfile = map->readonlyfile; leftmap->hand = map->hand; - map->addr += left; + midlmap->addr = map_addr + left; + midlmap->size = middle; + midlmap->off = (map->flags & MAP_ANONYMOUS) ? 0 : map->off + left; + midlmap->prot = prot; + midlmap->flags = map->flags; + midlmap->hand = -1; + map->addr += left + middle; map->size = right; - map->prot = prot; map->hand = -1; if (!(map->flags & MAP_ANONYMOUS)) - map->off += left; - dll_make_first(&__maps.used, &leftmap->elem); - __maps.count += 1; + map->off += left + middle; + tree_insert(&__maps.maps, &leftmap->tree, __maps_compare); + tree_insert(&__maps.maps, &midlmap->tree, __maps_compare); + __maps.count += 2; __maps_check(); } else { + __maps_free(midlmap); __maps_free(leftmap); rc = -1; } } else { + __maps_free(leftmap); rc = -1; } } else { - // change middle of mapping - size_t left = addr - map_addr; - size_t middle = size; - size_t right = map_size - middle - left; - struct Map *leftmap; - if ((leftmap = __maps_alloc())) { - struct Map *midlmap; - if ((midlmap = __maps_alloc())) { - if (!__mprotect_chunk(map_addr + left, middle, prot, false)) { - leftmap->addr = map_addr; - leftmap->size = left; - leftmap->off = map->off; - leftmap->prot = map->prot; - leftmap->flags = map->flags; - leftmap->iscow = map->iscow; - leftmap->readonlyfile = map->readonlyfile; - leftmap->hand = map->hand; - midlmap->addr = map_addr + left; - midlmap->size = middle; - midlmap->off = (map->flags & MAP_ANONYMOUS) ? 0 : map->off + left; - midlmap->prot = prot; - midlmap->flags = map->flags; - midlmap->hand = -1; - map->addr += left + middle; - map->size = right; - map->hand = -1; - if (!(map->flags & MAP_ANONYMOUS)) - map->off += left + middle; - dll_make_first(&__maps.used, &midlmap->elem); - dll_make_first(&__maps.used, &leftmap->elem); - __maps.count += 2; - __maps_check(); - } else { - __maps_free(midlmap); - __maps_free(leftmap); - rc = -1; - } - } else { - __maps_free(leftmap); - rc = -1; - } - } else { - rc = -1; - } + rc = -1; } } } diff --git a/libc/intrin/msync-nt.c b/libc/intrin/msync-nt.c index 3ff6524ed..883a11c06 100644 --- a/libc/intrin/msync-nt.c +++ b/libc/intrin/msync-nt.c @@ -34,16 +34,17 @@ textwindows int sys_msync_nt(char *addr, size_t size, int flags) { return einval(); int rc = 0; - __maps_lock(); - for (struct Dll *e = dll_first(__maps.used); e; - e = dll_next(__maps.used, e)) { - struct Map *map = MAP_CONTAINER(e); - char *beg = MAX(addr, map->addr); - char *end = MIN(addr + size, map->addr + map->size); - if (beg < end) - if (!FlushViewOfFile(beg, end - beg)) - rc = -1; - // TODO(jart): FlushFileBuffers too on g_fds handle if MS_SYNC? + if (__maps_lock()) { + rc = edeadlk(); + } else { + for (struct Map *map = __maps_floor(addr); map; map = __maps_next(map)) { + char *beg = MAX(addr, map->addr); + char *end = MIN(addr + size, map->addr + map->size); + if (beg < end) + if (!FlushViewOfFile(beg, end - beg)) + rc = -1; + // TODO(jart): FlushFileBuffers too on g_fds handle if MS_SYNC? + } } __maps_unlock(); diff --git a/libc/intrin/printmaps.c b/libc/intrin/printmaps.c index b84a1cc25..18c02812e 100644 --- a/libc/intrin/printmaps.c +++ b/libc/intrin/printmaps.c @@ -29,16 +29,11 @@ /** * Prints memory mappings. */ -void __print_maps(void) { - int limit = 15; - long maptally = 0; +void __print_maps(size_t limit) { char mappingbuf[8], sb[16]; __maps_lock(); - struct Dll *e, *e2; - for (e = dll_first(__maps.used); e; e = e2) { - e2 = dll_next(__maps.used, e); - struct Map *map = MAP_CONTAINER(e); - maptally += map->size; + for (struct Tree *e = tree_first(__maps.maps); e; e = tree_next(e)) { + struct Map *map = MAP_TREE_CONTAINER(e); kprintf("%012lx-%012lx %!s", map->addr, map->addr + map->size, (DescribeMapping)(mappingbuf, map->prot, map->flags)); sizefmt(sb, map->size, 1024); diff --git a/libc/intrin/prot2nt.greg.c b/libc/intrin/prot2nt.greg.c index 67a59430b..59dd01491 100644 --- a/libc/intrin/prot2nt.greg.c +++ b/libc/intrin/prot2nt.greg.c @@ -42,6 +42,8 @@ privileged int __prot2nt(int prot, int iscow) { return kNtPageExecuteReadwrite; } default: + if (prot & PROT_GUARD) + return kNtPageReadwrite | kNtPageGuard; return kNtPageNoaccess; } } diff --git a/libc/intrin/pthread_delay_np.c b/libc/intrin/pthread_delay_np.c new file mode 100644 index 000000000..3a7c98024 --- /dev/null +++ b/libc/intrin/pthread_delay_np.c @@ -0,0 +1,40 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/strace.internal.h" +#include "libc/thread/thread.h" + +/** + * Delays execution for brief moment. + * + * @param symbol may be used to strace names of static locks + * @param backoff should start at zero and be feed back in + * @return new value for backoff + */ +int pthread_delay_np(const void *symbol, int backoff) { + if (backoff < 7) { + volatile int i; + for (i = 0; i != 1 << backoff; i++) { + } + backoff++; + } else { + STRACE("pthread_delay_np(%t)", symbol); + pthread_yield_np(); + } + return backoff; +} diff --git a/libc/intrin/pthread_mutex_lock.c b/libc/intrin/pthread_mutex_lock.c index 6fd2c6bfb..509af6e04 100644 --- a/libc/intrin/pthread_mutex_lock.c +++ b/libc/intrin/pthread_mutex_lock.c @@ -18,8 +18,10 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" #include "libc/calls/state.internal.h" +#include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" +#include "libc/intrin/describeflags.internal.h" #include "libc/intrin/strace.internal.h" #include "libc/intrin/weaken.h" #include "libc/runtime/internal.h" @@ -27,6 +29,66 @@ #include "libc/thread/thread.h" #include "third_party/nsync/mu.h" +static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex) { + int me; + int backoff = 0; + uint64_t word, lock; + + // get current state of lock + word = atomic_load_explicit(&mutex->_word, memory_order_relaxed); + + // use fancy nsync mutex if possible + if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL && // + MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && // + _weaken(nsync_mu_lock)) { + _weaken(nsync_mu_lock)((nsync_mu *)mutex); + return 0; + } + + // implement barebones normal mutexes + if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) { + for (;;) { + word = MUTEX_UNLOCK(word); + lock = MUTEX_LOCK(word); + if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock, + memory_order_acquire, + memory_order_relaxed)) + return 0; + backoff = pthread_delay_np(mutex, backoff); + } + } + + // implement recursive mutexes + me = gettid(); + for (;;) { + if (MUTEX_OWNER(word) == me) { + if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) { + if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) { + if (atomic_compare_exchange_weak_explicit( + &mutex->_word, &word, MUTEX_INC_DEPTH(word), + memory_order_relaxed, memory_order_relaxed)) + return 0; + continue; + } else { + return EAGAIN; + } + } else { + return EDEADLK; + } + } + word = MUTEX_UNLOCK(word); + lock = MUTEX_LOCK(word); + lock = MUTEX_SET_OWNER(lock, me); + if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock, + memory_order_acquire, + memory_order_relaxed)) { + mutex->_pid = __pid; + return 0; + } + backoff = pthread_delay_np(mutex, backoff); + } +} + /** * Locks mutex. * @@ -65,65 +127,10 @@ * @vforksafe */ errno_t pthread_mutex_lock(pthread_mutex_t *mutex) { - int me; - uint64_t word, lock; - - LOCKTRACE("pthread_mutex_lock(%t)", mutex); - if (__vforked) return 0; - - // get current state of lock - word = atomic_load_explicit(&mutex->_word, memory_order_relaxed); - - // use fancy nsync mutex if possible - if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL && // - MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && // - _weaken(nsync_mu_lock)) { - _weaken(nsync_mu_lock)((nsync_mu *)mutex); - return 0; - } - - // implement barebones normal mutexes - if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) { - for (;;) { - word = MUTEX_UNLOCK(word); - lock = MUTEX_LOCK(word); - if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock, - memory_order_acquire, - memory_order_relaxed)) - return 0; - pthread_pause_np(); - } - } - - // implement recursive mutexes - me = gettid(); - for (;;) { - if (MUTEX_OWNER(word) == me) { - if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) { - if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) { - if (atomic_compare_exchange_weak_explicit( - &mutex->_word, &word, MUTEX_INC_DEPTH(word), - memory_order_relaxed, memory_order_relaxed)) - return 0; - continue; - } else { - return EAGAIN; - } - } else { - return EDEADLK; - } - } - word = MUTEX_UNLOCK(word); - lock = MUTEX_LOCK(word); - lock = MUTEX_SET_OWNER(lock, me); - if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock, - memory_order_acquire, - memory_order_relaxed)) { - mutex->_pid = __pid; - return 0; - } - pthread_pause_np(); - } + LOCKTRACE("acquiring %t...", mutex); + errno_t err = pthread_mutex_lock_impl(mutex); + LOCKTRACE("pthread_mutex_lock(%t) → %s", mutex, DescribeErrno(err)); + return err; } diff --git a/libc/intrin/pthread_mutex_trylock.c b/libc/intrin/pthread_mutex_trylock.c index f4e0dde6a..21513656a 100644 --- a/libc/intrin/pthread_mutex_trylock.c +++ b/libc/intrin/pthread_mutex_trylock.c @@ -17,6 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" +#include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" #include "libc/intrin/weaken.h" diff --git a/libc/intrin/pthread_mutex_unlock.c b/libc/intrin/pthread_mutex_unlock.c index c6fc274b1..dc8d0fb1f 100644 --- a/libc/intrin/pthread_mutex_unlock.c +++ b/libc/intrin/pthread_mutex_unlock.c @@ -17,6 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" +#include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" #include "libc/intrin/strace.internal.h" diff --git a/libc/intrin/pthread_syshand.c b/libc/intrin/pthread_syshand.c index 856ce3323..b15460580 100644 --- a/libc/intrin/pthread_syshand.c +++ b/libc/intrin/pthread_syshand.c @@ -29,6 +29,6 @@ intptr_t _pthread_syshand(struct PosixThread *pt) { syshand = atomic_load_explicit(&pt->tib->tib_syshand, memory_order_acquire); if (syshand) return syshand; - pthread_pause_np(); + pthread_yield_np(); } } diff --git a/libc/intrin/pthread_tid.c b/libc/intrin/pthread_tid.c index 27e3ee5a1..4f7553e9a 100644 --- a/libc/intrin/pthread_tid.c +++ b/libc/intrin/pthread_tid.c @@ -24,6 +24,6 @@ int _pthread_tid(struct PosixThread *pt) { int tid = 0; while (pt && !(tid = atomic_load_explicit(&pt->ptid, memory_order_acquire))) - pthread_pause_np(); + pthread_yield_np(); return tid; } diff --git a/libc/calls/pthread_yield_np.c b/libc/intrin/pthread_yield_np.c similarity index 100% rename from libc/calls/pthread_yield_np.c rename to libc/intrin/pthread_yield_np.c diff --git a/libc/intrin/rbtree.c b/libc/intrin/rbtree.c deleted file mode 100644 index 3fc3ae5cc..000000000 --- a/libc/intrin/rbtree.c +++ /dev/null @@ -1,312 +0,0 @@ -// Copyright 2024 Justine Alexandra Roberts Tunney -// -// Permission to use, copy, modify, and/or distribute this software for -// any purpose with or without fee is hereby granted, provided that the -// above copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL -// WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE -// AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL -// DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR -// PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -// PERFORMANCE OF THIS SOFTWARE. - -#include "rbtree.h" -#include "libc/dce.h" -#include "libc/str/str.h" - -#define RBTREE_DEBUG - -struct rbtree *rbtree_next(struct rbtree *node) { - if (!node) - return 0; - if (node->right) - return rbtree_first(node->right); - struct rbtree *parent = node->parent; - while (parent && node == parent->right) { - node = parent; - parent = parent->parent; - } - return parent; -} - -struct rbtree *rbtree_prev(struct rbtree *node) { - if (!node) - return 0; - if (rbtree_get_left(node)) - return rbtree_last(rbtree_get_left(node)); - struct rbtree *parent = node->parent; - while (parent && node == rbtree_get_left(parent)) { - node = parent; - parent = parent->parent; - } - return parent; -} - -struct rbtree *rbtree_first(struct rbtree *node) { - while (node && rbtree_get_left(node)) - node = rbtree_get_left(node); - return node; -} - -struct rbtree *rbtree_last(struct rbtree *node) { - while (node && node->right) - node = node->right; - return node; -} - -static void rbtree_rotate_left(struct rbtree **root, struct rbtree *x) { - struct rbtree *y = x->right; - x->right = rbtree_get_left(y); - if (rbtree_get_left(y)) - rbtree_get_left(y)->parent = x; - y->parent = x->parent; - if (!x->parent) { - *root = y; - } else if (x == rbtree_get_left(x->parent)) { - rbtree_set_left(x->parent, y); - } else { - x->parent->right = y; - } - rbtree_set_left(y, x); - x->parent = y; -} - -static void rbtree_rotate_right(struct rbtree **root, struct rbtree *y) { - struct rbtree *x = rbtree_get_left(y); - rbtree_set_left(y, x->right); - if (x->right) - x->right->parent = y; - x->parent = y->parent; - if (!y->parent) { - *root = x; - } else if (y == y->parent->right) { - y->parent->right = x; - } else { - rbtree_set_left(y->parent, x); - } - x->right = y; - y->parent = x; -} - -static void rbtree_insert_fixup(struct rbtree **root, struct rbtree *node) { - rbtree_set_red(node, 1); - while (node != *root && rbtree_get_red(node->parent)) { - if (node->parent == rbtree_get_left(node->parent->parent)) { - struct rbtree *uncle = node->parent->parent->right; - if (uncle && rbtree_get_red(uncle)) { - rbtree_set_red(node->parent, 0); - rbtree_set_red(uncle, 0); - rbtree_set_red(node->parent->parent, 1); - node = node->parent->parent; - } else { - if (node == node->parent->right) { - node = node->parent; - rbtree_rotate_left(root, node); - } - rbtree_set_red(node->parent, 0); - rbtree_set_red(node->parent->parent, 1); - rbtree_rotate_right(root, node->parent->parent); - } - } else { - struct rbtree *uncle = rbtree_get_left(node->parent->parent); - if (uncle && rbtree_get_red(uncle)) { - rbtree_set_red(node->parent, 0); - rbtree_set_red(uncle, 0); - rbtree_set_red(node->parent->parent, 1); - node = node->parent->parent; - } else { - if (node == rbtree_get_left(node->parent)) { - node = node->parent; - rbtree_rotate_right(root, node); - } - rbtree_set_red(node->parent, 0); - rbtree_set_red(node->parent->parent, 1); - rbtree_rotate_left(root, node->parent->parent); - } - } - } - rbtree_set_red(*root, 0); -} - -void rbtree_insert(struct rbtree **root, struct rbtree *node, - rbtree_cmp_f *cmp) { - bzero(node, sizeof(*node)); - if (!*root) { - *root = node; - } else { - struct rbtree *search = *root; - struct rbtree *parent = 0; - do { - parent = search; - if (cmp(node, search) < 0) { - search = rbtree_get_left(search); - } else { - search = search->right; - } - } while (search); - if (cmp(node, parent) < 0) { - rbtree_set_left(parent, node); - } else { - parent->right = node; - } - node->parent = parent; - rbtree_insert_fixup(root, node); - } -} - -static void rbtree_transplant(struct rbtree **root, struct rbtree *u, - struct rbtree *v) { - if (!u->parent) { - *root = v; - } else if (u == rbtree_get_left(u->parent)) { - rbtree_set_left(u->parent, v); - } else { - u->parent->right = v; - } - if (v) - v->parent = u->parent; -} - -static void rbtree_remove_fixup(struct rbtree **root, struct rbtree *node, - struct rbtree *parent) { - while (node != *root && (!node || !rbtree_get_red(node))) { - if (node == rbtree_get_left(parent)) { - struct rbtree *sibling = parent->right; - if (rbtree_get_red(sibling)) { - rbtree_set_red(sibling, 0); - rbtree_set_red(parent, 1); - rbtree_rotate_left(root, parent); - sibling = parent->right; - } - if ((!rbtree_get_left(sibling) || - !rbtree_get_red(rbtree_get_left(sibling))) && - (!sibling->right || !rbtree_get_red(sibling->right))) { - rbtree_set_red(sibling, 1); - node = parent; - parent = node->parent; - } else { - if (!sibling->right || !rbtree_get_red(sibling->right)) { - rbtree_set_red(rbtree_get_left(sibling), 0); - rbtree_set_red(sibling, 1); - rbtree_rotate_right(root, sibling); - sibling = parent->right; - } - rbtree_set_red(sibling, rbtree_get_red(parent)); - rbtree_set_red(parent, 0); - rbtree_set_red(sibling->right, 0); - rbtree_rotate_left(root, parent); - node = *root; - break; - } - } else { - struct rbtree *sibling = rbtree_get_left(parent); - if (rbtree_get_red(sibling)) { - rbtree_set_red(sibling, 0); - rbtree_set_red(parent, 1); - rbtree_rotate_right(root, parent); - sibling = rbtree_get_left(parent); - } - if ((!sibling->right || !rbtree_get_red(sibling->right)) && - (!rbtree_get_left(sibling) || - !rbtree_get_red(rbtree_get_left(sibling)))) { - rbtree_set_red(sibling, 1); - node = parent; - parent = node->parent; - } else { - if (!rbtree_get_left(sibling) || - !rbtree_get_red(rbtree_get_left(sibling))) { - rbtree_set_red(sibling->right, 0); - rbtree_set_red(sibling, 1); - rbtree_rotate_left(root, sibling); - sibling = rbtree_get_left(parent); - } - rbtree_set_red(sibling, rbtree_get_red(parent)); - rbtree_set_red(parent, 0); - rbtree_set_red(rbtree_get_left(sibling), 0); - rbtree_rotate_right(root, parent); - node = *root; - break; - } - } - } - if (node) - rbtree_set_red(node, 0); -} - -void rbtree_remove(struct rbtree **root, struct rbtree *node) { - struct rbtree *y = node; - struct rbtree *x = 0; - struct rbtree *x_parent = 0; - int y_original_color = rbtree_get_red(y); - if (!rbtree_get_left(node)) { - x = node->right; - rbtree_transplant(root, node, node->right); - x_parent = node->parent; - } else if (!node->right) { - x = rbtree_get_left(node); - rbtree_transplant(root, node, rbtree_get_left(node)); - x_parent = node->parent; - } else { - y = rbtree_first(node->right); - y_original_color = rbtree_get_red(y); - x = y->right; - if (y->parent == node) { - if (x) - x->parent = y; - x_parent = y; - } else { - rbtree_transplant(root, y, y->right); - y->right = node->right; - y->right->parent = y; - x_parent = y->parent; - } - rbtree_transplant(root, node, y); - rbtree_set_left(y, rbtree_get_left(node)); - rbtree_get_left(y)->parent = y; - rbtree_set_red(y, rbtree_get_red(node)); - } - if (!y_original_color) - rbtree_remove_fixup(root, x, x_parent); -} - -struct rbtree *rbtree_get(const struct rbtree *node, const struct rbtree *key, - rbtree_cmp_f *cmp) { - while (node) { - int c = cmp(key, node); - if (c < 0) { - node = rbtree_get_left(node); - } else if (c > 0) { - node = node->right; - } else { - return (struct rbtree *)node; - } - } - return 0; -} - -struct rbtree *rbtree_floor(const struct rbtree *node, const struct rbtree *key, - rbtree_cmp_f *cmp) { - while (node) { - if (cmp(key, node) < 0) { - node = rbtree_get_left(node); - } else { - node = node->right; - } - } - return (struct rbtree *)node; -} - -struct rbtree *rbtree_ceil(const struct rbtree *node, const struct rbtree *key, - rbtree_cmp_f *cmp) { - while (node) { - if (cmp(node, key) < 0) { - node = rbtree_get_left(node); - } else { - node = node->right; - } - } - return (struct rbtree *)node; -} diff --git a/libc/intrin/rbtree.h b/libc/intrin/rbtree.h deleted file mode 100644 index 0c63f67dd..000000000 --- a/libc/intrin/rbtree.h +++ /dev/null @@ -1,57 +0,0 @@ -#ifdef _COSMO_SOURCE -#ifndef COSMOPOLITAN_RBTREE_H_ -#define COSMOPOLITAN_RBTREE_H_ -#define rbtree_ceil __rbtree_ceil -#define rbtree_first __rbtree_first -#define rbtree_floor __rbtree_floor -#define rbtree_get __rbtree_get -#define rbtree_insert __rbtree_insert -#define rbtree_last __rbtree_last -#define rbtree_next __rbtree_next -#define rbtree_prev __rbtree_prev -#define rbtree_remove __rbtree_remove -COSMOPOLITAN_C_START_ - -#define RBTREE_CONTAINER(t, f, p) ((t *)(((char *)(p)) - offsetof(t, f))) - -struct rbtree { - uintptr_t word; - struct rbtree *right; - struct rbtree *parent; -}; - -typedef int rbtree_cmp_f(const struct rbtree *, const struct rbtree *); - -static inline struct rbtree *rbtree_get_left(const struct rbtree *node) { - return (struct rbtree *)(node->word & -2); -} - -static inline void rbtree_set_left(struct rbtree *node, struct rbtree *left) { - node->word = (uintptr_t)left | (node->word & 1); -} - -static inline int rbtree_get_red(const struct rbtree *node) { - return node->word & 1; -} - -static inline void rbtree_set_red(struct rbtree *node, int red) { - node->word &= -2; - node->word |= red; -} - -struct rbtree *rbtree_next(struct rbtree *) libcesque; -struct rbtree *rbtree_prev(struct rbtree *) libcesque; -struct rbtree *rbtree_first(struct rbtree *) libcesque; -struct rbtree *rbtree_last(struct rbtree *) libcesque; -void rbtree_remove(struct rbtree **, struct rbtree *) libcesque; -void rbtree_insert(struct rbtree **, struct rbtree *, rbtree_cmp_f *) libcesque; -struct rbtree *rbtree_get(const struct rbtree *, const struct rbtree *, - rbtree_cmp_f *) libcesque; -struct rbtree *rbtree_ceil(const struct rbtree *, const struct rbtree *, - rbtree_cmp_f *) libcesque; -struct rbtree *rbtree_floor(const struct rbtree *, const struct rbtree *, - rbtree_cmp_f *) libcesque; - -COSMOPOLITAN_C_END_ -#endif /* COSMOPOLITAN_RBTREE_H_ */ -#endif /* _COSMO_SOURCE */ diff --git a/libc/intrin/tree.c b/libc/intrin/tree.c new file mode 100644 index 000000000..23e25f7f5 --- /dev/null +++ b/libc/intrin/tree.c @@ -0,0 +1,271 @@ +// Copyright 2024 Justine Alexandra Roberts Tunney +// +// Permission to use, copy, modify, and/or distribute this software for +// any purpose with or without fee is hereby granted, provided that the +// above copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL +// WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE +// AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL +// DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR +// PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +#include "tree.h" + +struct Tree *tree_last(struct Tree *node) { + while (node && node->right) + node = node->right; + return node; +} + +struct Tree *tree_first(struct Tree *node) { + while (node && tree_get_left(node)) + node = tree_get_left(node); + return node; +} + +struct Tree *tree_next(struct Tree *node) { + if (!node) + return 0; + if (node->right) + return tree_first(node->right); + struct Tree *parent = node->parent; + while (parent && node == parent->right) { + node = parent; + parent = parent->parent; + } + return parent; +} + +struct Tree *tree_prev(struct Tree *node) { + struct Tree *parent; + if (!node) + return 0; + if (tree_get_left(node)) + return tree_last(tree_get_left(node)); + parent = node->parent; + while (parent && node == tree_get_left(parent)) { + node = parent; + parent = parent->parent; + } + return parent; +} + +static void tree_rotate_left(struct Tree **root, struct Tree *x) { + struct Tree *y = x->right; + x->right = tree_get_left(y); + if (tree_get_left(y)) + tree_get_left(y)->parent = x; + y->parent = x->parent; + if (!x->parent) { + *root = y; + } else if (x == tree_get_left(x->parent)) { + tree_set_left(x->parent, y); + } else { + x->parent->right = y; + } + tree_set_left(y, x); + x->parent = y; +} + +static void tree_rotate_right(struct Tree **root, struct Tree *y) { + struct Tree *x = tree_get_left(y); + tree_set_left(y, x->right); + if (x->right) + x->right->parent = y; + x->parent = y->parent; + if (!y->parent) { + *root = x; + } else if (y == y->parent->right) { + y->parent->right = x; + } else { + tree_set_left(y->parent, x); + } + y->parent = x; + x->right = y; +} + +static void tree_rebalance_insert(struct Tree **root, struct Tree *node) { + struct Tree *uncle; + tree_set_red(node, 1); + while (node != *root && tree_get_red(node->parent)) { + if (node->parent == tree_get_left(node->parent->parent)) { + uncle = node->parent->parent->right; + if (uncle && tree_get_red(uncle)) { + tree_set_red(node->parent, 0); + tree_set_red(uncle, 0); + tree_set_red(node->parent->parent, 1); + node = node->parent->parent; + } else { + if (node == node->parent->right) { + node = node->parent; + tree_rotate_left(root, node); + } + tree_set_red(node->parent, 0); + tree_set_red(node->parent->parent, 1); + tree_rotate_right(root, node->parent->parent); + } + } else { + uncle = tree_get_left(node->parent->parent); + if (uncle && tree_get_red(uncle)) { + tree_set_red(node->parent, 0); + tree_set_red(uncle, 0); + tree_set_red(node->parent->parent, 1); + node = node->parent->parent; + } else { + if (node == tree_get_left(node->parent)) { + node = node->parent; + tree_rotate_right(root, node); + } + tree_set_red(node->parent, 0); + tree_set_red(node->parent->parent, 1); + tree_rotate_left(root, node->parent->parent); + } + } + } + tree_set_red(*root, 0); +} + +void tree_insert(struct Tree **root, struct Tree *node, tree_cmp_f *cmp) { + struct Tree *search, *parent; + node->word = 0; + node->right = 0; + node->parent = 0; + if (!*root) { + *root = node; + } else { + search = *root; + parent = 0; + do { + parent = search; + if (cmp(node, search) < 0) { + search = tree_get_left(search); + } else { + search = search->right; + } + } while (search); + if (cmp(node, parent) < 0) { + tree_set_left(parent, node); + } else { + parent->right = node; + } + node->parent = parent; + tree_rebalance_insert(root, node); + } +} + +static void tree_transplant(struct Tree **root, struct Tree *u, + struct Tree *v) { + if (!u->parent) { + *root = v; + } else if (u == tree_get_left(u->parent)) { + tree_set_left(u->parent, v); + } else { + u->parent->right = v; + } + if (v) + v->parent = u->parent; +} + +static void tree_rebalance_remove(struct Tree **root, struct Tree *node, + struct Tree *parent) { + struct Tree *sibling; + while (node != *root && (!node || !tree_get_red(node))) { + if (node == tree_get_left(parent)) { + sibling = parent->right; + if (tree_get_red(sibling)) { + tree_set_red(sibling, 0); + tree_set_red(parent, 1); + tree_rotate_left(root, parent); + sibling = parent->right; + } + if ((!tree_get_left(sibling) || !tree_get_red(tree_get_left(sibling))) && + (!sibling->right || !tree_get_red(sibling->right))) { + tree_set_red(sibling, 1); + node = parent; + parent = node->parent; + } else { + if (!sibling->right || !tree_get_red(sibling->right)) { + tree_set_red(tree_get_left(sibling), 0); + tree_set_red(sibling, 1); + tree_rotate_right(root, sibling); + sibling = parent->right; + } + tree_set_red(sibling, tree_get_red(parent)); + tree_set_red(parent, 0); + tree_set_red(sibling->right, 0); + tree_rotate_left(root, parent); + node = *root; + break; + } + } else { + sibling = tree_get_left(parent); + if (tree_get_red(sibling)) { + tree_set_red(sibling, 0); + tree_set_red(parent, 1); + tree_rotate_right(root, parent); + sibling = tree_get_left(parent); + } + if ((!sibling->right || !tree_get_red(sibling->right)) && + (!tree_get_left(sibling) || !tree_get_red(tree_get_left(sibling)))) { + tree_set_red(sibling, 1); + node = parent; + parent = node->parent; + } else { + if (!tree_get_left(sibling) || !tree_get_red(tree_get_left(sibling))) { + tree_set_red(sibling->right, 0); + tree_set_red(sibling, 1); + tree_rotate_left(root, sibling); + sibling = tree_get_left(parent); + } + tree_set_red(sibling, tree_get_red(parent)); + tree_set_red(parent, 0); + tree_set_red(tree_get_left(sibling), 0); + tree_rotate_right(root, parent); + node = *root; + break; + } + } + } + if (node) + tree_set_red(node, 0); +} + +void tree_remove(struct Tree **root, struct Tree *node) { + struct Tree *x = 0; + struct Tree *y = node; + struct Tree *x_parent = 0; + int y_original_color = tree_get_red(y); + if (!tree_get_left(node)) { + x = node->right; + tree_transplant(root, node, node->right); + x_parent = node->parent; + } else if (!node->right) { + x = tree_get_left(node); + tree_transplant(root, node, tree_get_left(node)); + x_parent = node->parent; + } else { + y = tree_first(node->right); + y_original_color = tree_get_red(y); + x = y->right; + if (y->parent == node) { + if (x) + x->parent = y; + x_parent = y; + } else { + tree_transplant(root, y, y->right); + y->right = node->right; + y->right->parent = y; + x_parent = y->parent; + } + tree_transplant(root, node, y); + tree_set_left(y, tree_get_left(node)); + tree_get_left(y)->parent = y; + tree_set_red(y, tree_get_red(node)); + } + if (!y_original_color) + tree_rebalance_remove(root, x, x_parent); +} diff --git a/libc/intrin/tree.h b/libc/intrin/tree.h new file mode 100644 index 000000000..fae2e08cb --- /dev/null +++ b/libc/intrin/tree.h @@ -0,0 +1,91 @@ +#ifndef COSMOPOLITAN_TREE_H_ +#define COSMOPOLITAN_TREE_H_ +#define tree_first __tree_first +#define tree_insert __tree_insert +#define tree_last __tree_last +#define tree_next __tree_next +#define tree_prev __tree_prev +#define tree_remove __tree_remove +COSMOPOLITAN_C_START_ + +#define TREE_CONTAINER(t, f, p) ((t *)(((char *)(p)) - offsetof(t, f))) + +struct Tree { + uintptr_t word; + struct Tree *right; + struct Tree *parent; +}; + +typedef int tree_search_f(const void *, const struct Tree *); +typedef int tree_cmp_f(const struct Tree *, const struct Tree *); + +forceinline struct Tree *tree_get_left(const struct Tree *node) { + return (struct Tree *)(node->word & -2); +} + +static inline void tree_set_left(struct Tree *node, struct Tree *left) { + node->word = (uintptr_t)left | (node->word & 1); +} + +static inline int tree_get_red(const struct Tree *node) { + return node->word & 1; +} + +static inline void tree_set_red(struct Tree *node, int red) { + node->word &= -2; + node->word |= red; +} + +forceinline optimizespeed struct Tree *tree_floor(const struct Tree *node, + const void *key, + tree_search_f *cmp) { + struct Tree *left = 0; + while (node) { + if (cmp(key, node) >= 0) { + left = (struct Tree *)node; + node = tree_get_left(node); + } else { + node = node->right; + } + } + return left; +} + +static inline struct Tree *tree_ceil(const struct Tree *node, const void *key, + tree_search_f *cmp) { + struct Tree *right = 0; + while (node) { + if (cmp(key, node) < 0) { + right = (struct Tree *)node; + node = tree_get_left(node); + } else { + node = node->right; + } + } + return right; +} + +static inline struct Tree *tree_get(const struct Tree *node, const void *key, + tree_search_f *cmp) { + while (node) { + int c = cmp(key, node); + if (c < 0) { + node = tree_get_left(node); + } else if (c > 0) { + node = node->right; + } else { + return (struct Tree *)node; + } + } + return 0; +} + +struct Tree *tree_next(struct Tree *) libcesque; +struct Tree *tree_prev(struct Tree *) libcesque; +struct Tree *tree_first(struct Tree *) libcesque; +struct Tree *tree_last(struct Tree *) libcesque; +void tree_remove(struct Tree **, struct Tree *) libcesque; +void tree_insert(struct Tree **, struct Tree *, tree_cmp_f *) libcesque; + +COSMOPOLITAN_C_END_ +#endif /* COSMOPOLITAN_TREE_H_ */ diff --git a/libc/log/oncrash_amd64.c b/libc/log/oncrash_amd64.c index 914e2aa44..f9df5f99b 100644 --- a/libc/log/oncrash_amd64.c +++ b/libc/log/oncrash_amd64.c @@ -241,14 +241,10 @@ static relegated void ShowCrashReport(int err, int sig, siginfo_t *si, klog(buf, p - buf); } kprintf("\n"); - if (!IsWindows()) { - __print_maps(); - } - if (__argv) { - for (i = 0; i < __argc; ++i) { + __print_maps(15); + if (__argv) + for (i = 0; i < __argc; ++i) kprintf("%s ", __argv[i]); - } - } kprintf("\n"); } diff --git a/libc/proc/fork-nt.c b/libc/proc/fork-nt.c index 6dc06bf3f..7e58445b6 100644 --- a/libc/proc/fork-nt.c +++ b/libc/proc/fork-nt.c @@ -30,6 +30,7 @@ #include "libc/intrin/kprintf.h" #include "libc/intrin/maps.h" #include "libc/intrin/strace.internal.h" +#include "libc/intrin/tree.h" #include "libc/intrin/weaken.h" #include "libc/macros.internal.h" #include "libc/nt/createfile.h" @@ -95,7 +96,7 @@ static inline textwindows ssize_t ForkIo(int64_t h, char *p, size_t n, size_t i; uint32_t x; for (i = 0; i < n; i += x) { - if (!f(h, p + i, n - i, &x, NULL)) + if (!f(h, p + i, n - i, &x, 0)) return __winerr(); if (!x) break; @@ -109,10 +110,11 @@ static dontinline textwindows ssize_t ForkIo2( const char *sf, bool ischild) { ssize_t rc = ForkIo(h, buf, n, fn); if (ischild) { - __tls_enabled_set(false); // prevent tls crash in kprintf + // prevent crashes + __tls_enabled_set(false); __pid = __imp_GetCurrentProcessId(); __klog_handle = 0; - __maps.used = 0; + __maps.maps = 0; } NTTRACE("%s(%ld, %p, %'zu) → %'zd% m", sf, h, buf, n, rc); return rc; @@ -121,9 +123,11 @@ static dontinline textwindows ssize_t ForkIo2( static dontinline textwindows bool WriteAll(int64_t h, void *buf, size_t n) { bool ok; ok = ForkIo2(h, buf, n, (void *)WriteFile, "WriteFile", false) != -1; - if (!ok) - AbortFork("WriteAll"); - // Sleep(10); + if (!ok) { + STRACE("fork() failed in parent due to WriteAll(%ld, %p, %'zu) → %u", h, + buf, n, GetLastError()); + __print_maps(0); + } return ok; } @@ -185,30 +189,6 @@ static textwindows void *Malloc(size_t size) { return HeapAlloc(GetProcessHeap(), 0, size); } -static textwindows void Free(void *addr) { - HeapFree(GetProcessHeap(), 0, addr); -} - -static int CountMaps(struct Dll *maps) { - int count = 0; - for (struct Dll *e = dll_first(maps); e; e = dll_next(maps, e)) - ++count; - return count; -} - -static struct Map **SortMaps(struct Dll *maps, int count) { - int j, i = 0; - struct Map **sorted = Malloc(count * sizeof(struct Map *)); - for (struct Dll *e = dll_first(maps); e; e = dll_next(maps, e)) { - struct Map *map = MAP_CONTAINER(e); - for (j = i; j > 0 && sorted[j - 1]->addr > map->addr; --j) - sorted[j] = sorted[j - 1]; - sorted[j] = map; - ++i; - } - return sorted; -} - textwindows void WinMainForked(void) { jmp_buf jb; int64_t reader; @@ -233,35 +213,30 @@ textwindows void WinMainForked(void) { ReadOrDie(reader, jb, sizeof(jb)); // read memory mappings from parent process - int n = 0; - struct Dll *maps = 0; + struct Tree *maps = 0; for (;;) { struct Map *map = Malloc(sizeof(struct Map)); ReadOrDie(reader, map, sizeof(struct Map)); - if (map->addr == MAP_FAILED) { - Free(map); + if (map->addr == MAP_FAILED) break; - } - dll_init(&map->elem); - dll_make_first(&maps, &map->elem); - ++n; + tree_insert(&maps, &map->tree, __maps_compare); } - // created sorted array of maps - struct Map **sorted = SortMaps(maps, n); - // map memory into process int granularity = __granularity(); - for (int i = 0; i < n; ++i) { - struct Map *map = sorted[i]; + for (struct Tree *e = tree_first(maps); e; e = tree_next(e)) { + struct Map *map = MAP_TREE_CONTAINER(e); if ((uintptr_t)map->addr & (granularity - 1)) continue; - size_t size = map->size; // get true length in case mprotect() chopped up actual win32 map - for (int j = i + 1; - j < n && sorted[j]->hand == -1 && map->addr + size == sorted[j]->addr; - ++j) { - size += sorted[j]->size; + size_t size = map->size; + for (struct Tree *e2 = tree_next(e); e2; e2 = tree_next(e2)) { + struct Map *map2 = MAP_TREE_CONTAINER(e2); + if (map2->hand == -1 && map->addr + size == map2->addr) { + size += map2->size; + } else { + break; + } } // obtain the most permissive access possible unsigned prot, access; @@ -295,11 +270,11 @@ textwindows void WinMainForked(void) { // fixup memory manager __maps.free = 0; - __maps.used = 0; + __maps.maps = 0; __maps.count = 0; __maps.pages = 0; - for (int i = 0; i < n; ++i) { - struct Map *map = sorted[i]; + for (struct Tree *e = tree_first(maps); e; e = tree_next(e)) { + struct Map *map = MAP_TREE_CONTAINER(e); __maps.count += 1; __maps.pages += (map->size + getpagesize() - 1) / getpagesize(); unsigned old_protect; @@ -307,8 +282,7 @@ textwindows void WinMainForked(void) { &old_protect)) AbortFork("VirtualProtect"); } - Free(sorted); - __maps.used = maps; + __maps.maps = maps; __maps_init(); // mitosis complete @@ -393,19 +367,13 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) { if (spawnrc != -1) { CloseHandle(procinfo.hThread); ok = WriteAll(writer, jb, sizeof(jb)); - int count = 0; // this list will be populated with the maps we're transferring - struct Dll *e2, *maps = 0; - for (struct Dll *e = dll_first(__maps.used); ok && e; e = e2) { - e2 = dll_next(__maps.used, e); - struct Map *map = MAP_CONTAINER(e); + for (struct Map *map = __maps_first(); ok && map; + map = __maps_next(map)) { if (MAX((char *)__executable_start, map->addr) < MIN((char *)_end, map->addr + map->size)) continue; // executable image is loaded by windows - dll_remove(&__maps.used, e); - dll_make_last(&maps, e); ok = WriteAll(writer, map, sizeof(*map)); - ++count; } // send a terminating Map struct to child if (ok) { @@ -415,40 +383,44 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) { } // now write content of each map to child int granularity = __granularity(); - struct Map **sorted = SortMaps(maps, count); - uint32_t *old_protect = Malloc(count * 4); - for (int i = 0; ok && i < count; ++i) { - struct Map *map = sorted[i]; + for (struct Map *map = __maps_first(); ok && map; + map = __maps_next(map)) { // we only need to worry about the base mapping if ((uintptr_t)map->addr & (granularity - 1)) continue; + if (MAX((char *)__executable_start, map->addr) < + MIN((char *)_end, map->addr + map->size)) + continue; // executable image is loaded by windows // shared mappings don't need to be copied if ((map->flags & MAP_TYPE) == MAP_SHARED) continue; // get true length in case mprotect() chopped up actual win32 map - int j; size_t size = map->size; - for (j = i + 1; j < count && sorted[j]->hand == -1 && - map->addr + size == sorted[j]->addr; - ++j) { - size += sorted[j]->size; + for (struct Map *map2 = __maps_next(map); map2; + map2 = __maps_next(map2)) { + if (map2->hand == -1 && map->addr + size == map2->addr) { + size += map2->size; + } else { + break; + } + } + for (struct Map *map2 = map; ok && map2; map2 = __maps_next(map2)) { + if (!(map2->prot & PROT_READ)) + if (map->addr >= map2->addr && map->addr < map->addr + size) + ok = VirtualProtect( + map2->addr, map2->size, + __prot2nt(map2->prot | PROT_READ, map2->iscow), + &map2->oldprot); } - for (int k = i; ok && k < j; ++k) - if (!(sorted[k]->prot & PROT_READ)) - ok = VirtualProtect( - sorted[k]->addr, sorted[k]->size, - __prot2nt(sorted[k]->prot | PROT_READ, map->iscow), - &old_protect[k]); if (ok) ok = WriteAll(writer, map->addr, size); - for (int k = i; ok && k < j; ++k) - if (!(sorted[k]->prot & PROT_READ)) - ok = VirtualProtect(sorted[k]->addr, sorted[k]->size, - old_protect[k], &old_protect[k]); + for (struct Map *map2 = map; ok && map2; map2 = __maps_next(map2)) { + if (!(map2->prot & PROT_READ)) + if (map->addr >= map2->addr && map->addr < map->addr + size) + ok = VirtualProtect(map2->addr, map2->size, map2->oldprot, + &map2->oldprot); + } } - Free(old_protect); - Free(sorted); - dll_make_first(&__maps.used, maps); if (ok) ok = WriteAll(writer, __data_start, __data_end - __data_start); if (ok) @@ -466,6 +438,7 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) { } else { TerminateProcess(procinfo.hProcess, SIGKILL); CloseHandle(procinfo.hProcess); + rc = -1; } } } @@ -473,9 +446,8 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) { CloseHandle(reader); if (writer != -1) CloseHandle(writer); - if (rc == -1 && errno != ENOMEM) { + if (rc == -1 && errno != ENOMEM) eagain(); // posix fork() only specifies two errors - } } else { rc = 0; // re-apply code morphing for thread-local storage diff --git a/libc/proc/fork.c b/libc/proc/fork.c index 05431bcb5..3a48db798 100644 --- a/libc/proc/fork.c +++ b/libc/proc/fork.c @@ -22,6 +22,7 @@ #include "libc/calls/state.internal.h" #include "libc/calls/struct/sigset.h" #include "libc/calls/struct/sigset.internal.h" +#include "libc/calls/struct/timespec.h" #include "libc/calls/syscall-nt.internal.h" #include "libc/calls/syscall-sysv.internal.h" #include "libc/dce.h" @@ -45,7 +46,6 @@ #include "libc/thread/posixthread.internal.h" #include "libc/thread/tls.h" -extern pthread_mutex_t nsync_waiters_mu; extern pthread_mutex_t _pthread_lock_obj; static void _onfork_prepare(void) { @@ -54,11 +54,10 @@ static void _onfork_prepare(void) { _pthread_lock(); __maps_lock(); __fds_lock(); - pthread_mutex_lock(&nsync_waiters_mu); + LOCKTRACE("READY TO ROCK AND ROLL"); } static void _onfork_parent(void) { - pthread_mutex_unlock(&nsync_waiters_mu); __fds_unlock(); __maps_unlock(); _pthread_unlock(); @@ -68,7 +67,6 @@ static void _onfork_parent(void) { static void _onfork_child(void) { __fds_lock_obj = (pthread_mutex_t)PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; - nsync_waiters_mu = (pthread_mutex_t)PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; _pthread_lock_obj = (pthread_mutex_t)PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; atomic_store_explicit(&__maps.lock, 0, memory_order_relaxed); atomic_store_explicit(&__get_tls()->tib_relock_maps, 0, memory_order_relaxed); @@ -77,7 +75,9 @@ static void _onfork_child(void) { } int _fork(uint32_t dwCreationFlags) { + long micros; struct Dll *e; + struct timespec started; int ax, dx, tid, parent; parent = __pid; BLOCK_SIGNALS; @@ -85,11 +85,13 @@ int _fork(uint32_t dwCreationFlags) { __proc_lock(); if (__threaded) _onfork_prepare(); + started = timespec_real(); if (!IsWindows()) { ax = sys_fork(); } else { ax = sys_fork_nt(dwCreationFlags); } + micros = timespec_tomicros(timespec_sub(timespec_real(), started)); if (!ax) { // get new process id @@ -136,15 +138,14 @@ int _fork(uint32_t dwCreationFlags) { // run user fork callbacks if (__threaded) _onfork_child(); - STRACE("fork() → 0 (child of %d)", parent); + STRACE("fork() → 0 (child of %d; took %ld us)", parent, micros); } else { // this is the parent process - if (__threaded) { + if (__threaded) _onfork_parent(); - } if (IsWindows()) __proc_unlock(); - STRACE("fork() → %d% m", ax); + STRACE("fork() → %d% m (took %ld us)", ax, micros); } ALLOW_SIGNALS; return ax; diff --git a/libc/proc/posix_spawn.c b/libc/proc/posix_spawn.c index 4e85e49b4..967392ad8 100644 --- a/libc/proc/posix_spawn.c +++ b/libc/proc/posix_spawn.c @@ -482,9 +482,8 @@ errno_t posix_spawn(int *pid, const char *path, const posix_spawn_file_actions_t *file_actions, const posix_spawnattr_t *attrp, char *const argv[], char *const envp[]) { - if (IsWindows()) { + if (IsWindows()) return posix_spawn_nt(pid, path, file_actions, attrp, argv, envp); - } int pfds[2]; bool use_pipe; volatile int status = 0; @@ -516,66 +515,55 @@ errno_t posix_spawn(int *pid, const char *path, sigaction(sig, &dfl, 0); } } - if (flags & POSIX_SPAWN_SETSID) { + if (flags & POSIX_SPAWN_SETSID) setsid(); - } - if ((flags & POSIX_SPAWN_SETPGROUP) && setpgid(0, (*attrp)->pgroup)) { + if ((flags & POSIX_SPAWN_SETPGROUP) && setpgid(0, (*attrp)->pgroup)) goto ChildFailed; - } - if ((flags & POSIX_SPAWN_RESETIDS) && setgid(getgid())) { + if ((flags & POSIX_SPAWN_RESETIDS) && setgid(getgid())) goto ChildFailed; - } - if ((flags & POSIX_SPAWN_RESETIDS) && setuid(getuid())) { + if ((flags & POSIX_SPAWN_RESETIDS) && setuid(getuid())) goto ChildFailed; - } if (file_actions) { struct _posix_faction *a; for (a = *file_actions; a; a = a->next) { if (use_pipe && pfds[1] == a->fildes) { int p2; - if ((p2 = dup(pfds[1])) == -1) { + if ((p2 = dup(pfds[1])) == -1) goto ChildFailed; - } lost_cloexec = true; close(pfds[1]); pfds[1] = p2; } switch (a->action) { case _POSIX_SPAWN_CLOSE: - if (close(a->fildes)) { + if (close(a->fildes)) goto ChildFailed; - } break; case _POSIX_SPAWN_DUP2: - if (dup2(a->fildes, a->newfildes) == -1) { + if (dup2(a->fildes, a->newfildes) == -1) goto ChildFailed; - } break; case _POSIX_SPAWN_OPEN: { int t; - if ((t = openat(AT_FDCWD, a->path, a->oflag, a->mode)) == -1) { + if ((t = openat(AT_FDCWD, a->path, a->oflag, a->mode)) == -1) goto ChildFailed; - } if (t != a->fildes) { if (dup2(t, a->fildes) == -1) { close(t); goto ChildFailed; } - if (close(t)) { + if (close(t)) goto ChildFailed; - } } break; } case _POSIX_SPAWN_CHDIR: - if (chdir(a->path) == -1) { + if (chdir(a->path) == -1) goto ChildFailed; - } break; case _POSIX_SPAWN_FCHDIR: - if (fchdir(a->fildes) == -1) { + if (fchdir(a->fildes) == -1) goto ChildFailed; - } break; default: __builtin_unreachable(); @@ -583,17 +571,13 @@ errno_t posix_spawn(int *pid, const char *path, } } if (IsLinux() || IsFreebsd() || IsNetbsd()) { - if (flags & POSIX_SPAWN_SETSCHEDULER) { + if (flags & POSIX_SPAWN_SETSCHEDULER) if (sched_setscheduler(0, (*attrp)->schedpolicy, - &(*attrp)->schedparam) == -1) { + &(*attrp)->schedparam) == -1) goto ChildFailed; - } - } - if (flags & POSIX_SPAWN_SETSCHEDPARAM) { - if (sched_setparam(0, &(*attrp)->schedparam)) { + if (flags & POSIX_SPAWN_SETSCHEDPARAM) + if (sched_setparam(0, &(*attrp)->schedparam)) goto ChildFailed; - } - } } if (flags & POSIX_SPAWN_SETRLIMIT) { int rlimset = (*attrp)->rlimset; @@ -608,9 +592,8 @@ errno_t posix_spawn(int *pid, const char *path, } } } - if (lost_cloexec) { + if (lost_cloexec) fcntl(pfds[1], F_SETFD, FD_CLOEXEC); - } if (flags & POSIX_SPAWN_SETSIGMASK) { childmask = (*attrp)->sigmask; } else { @@ -636,9 +619,8 @@ errno_t posix_spawn(int *pid, const char *path, if (!use_pipe) { res = status; } else { - if (can_clobber) { + if (can_clobber) atomic_store_explicit(&has_vfork, true, memory_order_release); - } res = 0; read(pfds[0], &res, sizeof(res)); } @@ -651,9 +633,8 @@ errno_t posix_spawn(int *pid, const char *path, } else { res = errno; } - if (use_pipe) { + if (use_pipe) close(pfds[0]); - } ParentFailed: sigprocmask(SIG_SETMASK, &oldmask, 0); pthread_setcancelstate(cs, 0); diff --git a/libc/runtime/mapstack.c b/libc/runtime/mapstack.c index 3bec5d639..eccd5cefc 100644 --- a/libc/runtime/mapstack.c +++ b/libc/runtime/mapstack.c @@ -50,9 +50,10 @@ void *NewCosmoStack(void) { if (IsOpenbsd() && __sys_mmap(p, n, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD, - -1, 0, 0) != p) { + -1, 0, 0) != p) + notpossible; + if (mprotect(p, GetGuardSize(), PROT_NONE | PROT_GUARD)) notpossible; - } return p; } else { return 0; diff --git a/libc/runtime/runtime.h b/libc/runtime/runtime.h index 236a06f2d..295f0fe02 100644 --- a/libc/runtime/runtime.h +++ b/libc/runtime/runtime.h @@ -91,7 +91,7 @@ void ShowCrashReports(void) libcesque; int ftrace_install(void) libcesque; int ftrace_enabled(int) libcesque; int strace_enabled(int) libcesque; -void __print_maps(void) libcesque; +void __print_maps(size_t) libcesque; void __printargs(const char *) libcesque; /* builtin sh-like system/popen dsl */ int _cocmd(int, char **, char **) libcesque; diff --git a/libc/stdio/printargs.c b/libc/stdio/printargs.c index 2b549cbdf..5d2a61c2f 100644 --- a/libc/stdio/printargs.c +++ b/libc/stdio/printargs.c @@ -468,7 +468,7 @@ textstartup void __printargs(const char *prologue) { PRINT(""); PRINT("MEMTRACK"); - __print_maps(); + __print_maps(0); PRINT(""); PRINT("TERMIOS"); diff --git a/libc/sysv/consts.sh b/libc/sysv/consts.sh index 233c11edc..22cd4c998 100755 --- a/libc/sysv/consts.sh +++ b/libc/sysv/consts.sh @@ -292,6 +292,7 @@ syscon mprot PROT_NONE 0 0 0 0 0 0 0 0 # mmap, mprotect, unix syscon mprot PROT_READ 1 1 1 1 1 1 1 1 # mmap, mprotect, unix consensus syscon mprot PROT_WRITE 2 2 2 2 2 2 2 2 # mmap, mprotect, unix consensus syscon mprot PROT_EXEC 4 4 4 4 4 4 4 4 # mmap, mprotect, unix consensus +syscon mprot PROT_GUARD 0 0 0 0 0 0 0 0x100 # mmap, mprotect, unix consensus # mremap() flags # the revolutionary praxis of realloc() diff --git a/libc/sysv/consts/PROT_GUARD.S b/libc/sysv/consts/PROT_GUARD.S new file mode 100644 index 000000000..e04cb73ee --- /dev/null +++ b/libc/sysv/consts/PROT_GUARD.S @@ -0,0 +1,2 @@ +#include "libc/sysv/consts/syscon.internal.h" +.syscon mprot,PROT_GUARD,0,0,0,0,0,0,0,0x100 diff --git a/libc/sysv/consts/prot.h b/libc/sysv/consts/prot.h index 032a6b1a7..ea026f869 100644 --- a/libc/sysv/consts/prot.h +++ b/libc/sysv/consts/prot.h @@ -3,10 +3,7 @@ #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ -extern const int PROT_NONE; -extern const int PROT_READ; -extern const int PROT_WRITE; -extern const int PROT_EXEC; +extern const int PROT_GUARD; COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ diff --git a/libc/thread/pthread_create.c b/libc/thread/pthread_create.c index b9515272e..55db00ef4 100644 --- a/libc/thread/pthread_create.c +++ b/libc/thread/pthread_create.c @@ -229,20 +229,10 @@ static errno_t pthread_create_impl(pthread_t *thread, -1, 0, 0) != pt->pt_attr.__stackaddr) { notpossible; } - if (pt->pt_attr.__guardsize) { - if (!IsWindows()) { - if (mprotect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize, - PROT_NONE)) { - notpossible; - } - } else { - uint32_t oldattr; - if (!VirtualProtect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize, - kNtPageReadwrite | kNtPageGuard, &oldattr)) { - notpossible; - } - } - } + if (pt->pt_attr.__guardsize) + if (mprotect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize, + PROT_NONE | PROT_GUARD)) + notpossible; } if (!pt->pt_attr.__stackaddr || pt->pt_attr.__stackaddr == MAP_FAILED) { rc = errno; diff --git a/libc/thread/thread.h b/libc/thread/thread.h index 7fd38f42c..872fdcc37 100644 --- a/libc/thread/thread.h +++ b/libc/thread/thread.h @@ -190,6 +190,7 @@ int pthread_spin_trylock(pthread_spinlock_t *) libcesque paramsnonnull(); int pthread_spin_unlock(pthread_spinlock_t *) libcesque paramsnonnull(); int pthread_testcancel_np(void) libcesque; int pthread_tryjoin_np(pthread_t, void **) libcesque; +int pthread_delay_np(const void *, int) libcesque; int pthread_yield_np(void) libcesque; int pthread_yield(void) libcesque; pthread_id_np_t pthread_getthreadid_np(void) libcesque; diff --git a/test/ctl/set_test.cc b/test/ctl/set_test.cc index 5eecca5e2..c435533db 100644 --- a/test/ctl/set_test.cc +++ b/test/ctl/set_test.cc @@ -19,9 +19,9 @@ #include "ctl/set.h" #include "libc/mem/leaks.h" -// #include -// #define ctl std -// #define check() size() +#include +#define ctl std +#define check() size() int rand32(void) @@ -148,6 +148,16 @@ main() s.check(); } + { + // Test lower_bound and upper_bound + ctl::set s{ 1, 3, 4, 5, 7, 9 }; + auto lower = s.lower_bound(4); + auto upper = s.upper_bound(4); + if (*lower != 4 || *upper != 5) + return 18; + s.check(); + } + { // Test emplace ctl::set> s; diff --git a/test/libc/calls/raise_test.c b/test/libc/calls/raise_test.c index 5f764fa38..5ebb8189a 100644 --- a/test/libc/calls/raise_test.c +++ b/test/libc/calls/raise_test.c @@ -20,7 +20,6 @@ #include "libc/calls/struct/sigaction.h" #include "libc/calls/struct/siginfo.h" #include "libc/dce.h" -#include "libc/intrin/kprintf.h" #include "libc/runtime/runtime.h" #include "libc/sysv/consts/sa.h" #include "libc/sysv/consts/sicode.h" diff --git a/test/libc/calls/setrlimit_test.c b/test/libc/calls/setrlimit_test.c index bc40f8542..59ad24782 100644 --- a/test/libc/calls/setrlimit_test.c +++ b/test/libc/calls/setrlimit_test.c @@ -118,10 +118,11 @@ TEST(setrlimit, testFileSizeLimit) { EXPECT_EQ(0, WTERMSIG(wstatus)); } -int SetKernelEnforcedMemoryLimit(size_t n) { - struct rlimit rlim; +int SetMemoryLimit(size_t n) { + struct rlimit rlim = {0}; getrlimit(RLIMIT_AS, &rlim); rlim.rlim_cur = n; + rlim.rlim_max = n; return setrlimit(RLIMIT_AS, &rlim); } @@ -129,27 +130,20 @@ TEST(setrlimit, testMemoryLimit) { char *p; bool gotsome; int i, wstatus; - if (IsXnu()) - return; - if (IsOpenbsd()) - return; // simply too slow until mmap() becomes O(logn) ASSERT_NE(-1, (wstatus = xspawn(0))); if (wstatus == -2) { - ASSERT_EQ(0, SetKernelEnforcedMemoryLimit(MEM)); - for (gotsome = false, i = 0; i < (MEM * 2) / __granularity(); ++i) { - p = mmap(0, __granularity(), PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE | MAP_POPULATE, -1, 0); + ASSERT_EQ(0, SetMemoryLimit(MEM)); + for (gotsome = false, i = 0; i < (MEM * 2) / getpagesize(); ++i) { + p = mmap(0, getpagesize(), PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (p != MAP_FAILED) { gotsome = true; } else { - if (!IsNetbsd()) { - // TODO(jart): what's going on with NetBSD? - ASSERT_TRUE(gotsome); - } + ASSERT_TRUE(gotsome); ASSERT_EQ(ENOMEM, errno); _Exit(0); } - rngset(p, __granularity(), _rand64, -1); + rngset(p, getpagesize(), _rand64, -1); } _Exit(1); } diff --git a/test/libc/intrin/mmap_scalability_test.c b/test/libc/intrin/mmap_scalability_test.c new file mode 100644 index 000000000..26982fbbe --- /dev/null +++ b/test/libc/intrin/mmap_scalability_test.c @@ -0,0 +1,76 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" +#include "libc/calls/struct/timespec.h" +#include "libc/dce.h" +#include "libc/intrin/kprintf.h" +#include "libc/runtime/runtime.h" +#include "libc/sysv/consts/map.h" +#include "libc/sysv/consts/prot.h" + +#define BENCH(ITERATIONS, WORK_PER_RUN, CODE) \ + do { \ + struct timespec start = timespec_real(); \ + for (int __i = 0; __i < ITERATIONS; ++__i) { \ + asm volatile("" ::: "memory"); \ + CODE; \ + } \ + long long work = (WORK_PER_RUN) * (ITERATIONS); \ + double nanos = \ + (timespec_tonanos(timespec_sub(timespec_real(), start)) + work - 1) / \ + (double)work; \ + kprintf("%'20ld ns %2dx %s\n", (long)nanos, (ITERATIONS), #CODE); \ + } while (0) + +void *randaddr(void) { + static unsigned long lcg = 1; + lcg *= 6364136223846793005; + lcg += 1442695040888963407; + return (void *)(lcg >> 48 << 28); +} + +void map_unmap_one_page(void) { + void *p; + if ((p = mmap(randaddr(), 1, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)) == MAP_FAILED) + __builtin_trap(); + if (munmap(p, 1)) + __builtin_trap(); +} + +int main() { + kprintf("\n"); + BENCH(1000, 1, map_unmap_one_page()); + + // macos doesn't scale + if (IsXnu()) + return 0; + + // create lots of sparse mappings to put + // weight on __maps.maps. red-black tree + int n = 10000; + kprintf("%20s creating %d sparse maps...\n", "", n); + for (int i = 0; i < n; ++i) { + if (mmap(randaddr(), 1, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0) == MAP_FAILED) + __builtin_trap(); + } + + BENCH(1000, 1, map_unmap_one_page()); +} diff --git a/test/libc/intrin/mmap_test.c b/test/libc/intrin/mmap_test.c index 44e9216ca..bf9eee74a 100644 --- a/test/libc/intrin/mmap_test.c +++ b/test/libc/intrin/mmap_test.c @@ -461,7 +461,23 @@ void BenchUnmap(void) { __builtin_trap(); } +void BenchBigMmap(void) { + void *p; + p = mmap(0, 101 * 1024 * 1024, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (p == MAP_FAILED) + __builtin_trap(); + ptrs[count++] = p; +} + +void BenchBigMunmap(void) { + if (munmap(ptrs[--count], 101 * 1024 * 1024)) + __builtin_trap(); +} + BENCH(mmap, bench) { EZBENCH2("mmap", donothing, BenchMmapPrivate()); EZBENCH2("munmap", donothing, BenchUnmap()); + // EZBENCH2("big mmap", donothing, BenchBigMmap()); + // EZBENCH2("big munmap", donothing, BenchBigMunmap()); } diff --git a/test/libc/intrin/rbtree_test.c b/test/libc/intrin/tree_test.c similarity index 55% rename from test/libc/intrin/rbtree_test.c rename to test/libc/intrin/tree_test.c index 5bcea721c..09a8c807e 100644 --- a/test/libc/intrin/rbtree_test.c +++ b/test/libc/intrin/tree_test.c @@ -13,16 +13,18 @@ // TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR // PERFORMANCE OF THIS SOFTWARE. -#include "libc/intrin/rbtree.h" +#include "libc/intrin/tree.h" #include "libc/intrin/kprintf.h" +#include "libc/intrin/maps.h" #include "libc/macros.internal.h" +#include "libc/mem/mem.h" #include "libc/runtime/runtime.h" #include "libc/stdio/rand.h" -#define NUMBER_CONTAINER(e) RBTREE_CONTAINER(struct number, elem, e) +#define NUMBER_CONTAINER(e) TREE_CONTAINER(struct number, elem, e) -void rbtree_checker(const struct rbtree *node, const struct rbtree *parent, - int black_count, int *black_height, rbtree_cmp_f *cmp) { +void tree_checker(const struct Tree *node, const struct Tree *parent, + int black_count, int *black_height, tree_cmp_f *cmp) { if (!node) { // Leaf nodes are considered black if (*black_height == -1) { @@ -37,89 +39,101 @@ void rbtree_checker(const struct rbtree *node, const struct rbtree *parent, // ILLEGAL TREE: Parent link is incorrect __builtin_trap(); if (parent) { - if (rbtree_get_left(parent) == node && cmp(parent, node) < 0) + if (tree_get_left(parent) == node && cmp(parent, node) < 0) // ILLEGAL TREE: Binary search property violated on left child __builtin_trap(); if (parent->right == node && cmp(node, parent) < 0) // ILLEGAL TREE: Binary search property violated on right child __builtin_trap(); } - if (!rbtree_get_red(node)) { + if (!tree_get_red(node)) { black_count++; - } else if (parent && rbtree_get_red(parent)) { + } else if (parent && tree_get_red(parent)) { // ILLEGAL TREE: Red node has red child __builtin_trap(); } - rbtree_checker(rbtree_get_left(node), node, black_count, black_height, cmp); - rbtree_checker(node->right, node, black_count, black_height, cmp); + tree_checker(tree_get_left(node), node, black_count, black_height, cmp); + tree_checker(node->right, node, black_count, black_height, cmp); } -void rbtree_check(struct rbtree *root, rbtree_cmp_f *cmp) { +void tree_check(struct Tree *root, tree_cmp_f *cmp) { if (root) { - if (rbtree_get_red(root)) + if (tree_get_red(root)) // ILLEGAL TREE: root node must be black __builtin_trap(); int black_height = -1; - rbtree_checker(root, 0, 0, &black_height, cmp); + tree_checker(root, 0, 0, &black_height, cmp); } } struct number { - int number; - struct rbtree elem; + long number; + struct Tree elem; }; -int number_compare(const struct rbtree *ra, const struct rbtree *rb) { +int number_search(const void *ra, const struct Tree *rb) { + long a = (long)ra; + const struct number *b = NUMBER_CONTAINER(rb); + return (a > b->number) - (a < b->number); +} + +int number_compare(const struct Tree *ra, const struct Tree *rb) { const struct number *a = NUMBER_CONTAINER(ra); const struct number *b = NUMBER_CONTAINER(rb); return (a->number > b->number) - (a->number < b->number); } struct number *number_new(int number) { - static int used; - static struct number heap[8192]; - if (used == ARRAYLEN(heap)) - return 0; - struct number *res = &heap[used++]; - res->number = number; + struct number *res; + if ((res = malloc(sizeof(struct number)))) + res->number = number; return res; } -struct rbtree *tree = 0; +struct Tree *tree = 0; void print(void) { - for (struct rbtree *e = rbtree_first(tree); e; e = rbtree_next(e)) + for (struct Tree *e = tree_first(tree); e; e = tree_next(e)) kprintf("%3d", NUMBER_CONTAINER(e)->number); kprintf("\n"); } void print_reversed(void) { - for (struct rbtree *e = rbtree_last(tree); e; e = rbtree_prev(e)) + for (struct Tree *e = tree_last(tree); e; e = tree_prev(e)) kprintf("%3d", NUMBER_CONTAINER(e)->number); kprintf("\n"); } void simple_test(void) { - static const int kNumba[] = {74, 53, 96, 70, 34, 95, 30, 2, 89, 46, - 23, 2, 52, 0, 34, 12, 90, 95, 32, 65}; + static const long kNumba[] = {74, 53, 96, 70, 34, 95, 30, 2, 96, 46, + 23, 2, 52, 0, 34, 94, 90, 95, 32, 65}; for (int i = 0; i < 20; ++i) { int number = kNumba[i]; kprintf("%3d", number); - rbtree_insert(&tree, &number_new(number)->elem, number_compare); - rbtree_check(tree, number_compare); + tree_insert(&tree, &number_new(number)->elem, number_compare); + tree_check(tree, number_compare); } kprintf("\n"); print(); print_reversed(); for (int i = 0; i < 20; ++i) { - rbtree_remove(&tree, rbtree_get(tree, &(&(struct number){kNumba[i]})->elem, - number_compare)); - rbtree_check(tree, number_compare); + tree_remove(&tree, tree_get(tree, (void *)kNumba[i], number_search)); + tree_check(tree, number_compare); print(); } } int main() { ShowCrashReports(); + tree_check(__maps.maps, __maps_compare); + kprintf("\n"); + __print_maps(15); + kprintf("\n"); simple_test(); + tree_check(__maps.maps, __maps_compare); + for (int i = 0; i < 100000; ++i) + tree_insert(&tree, &number_new(rand())->elem, number_compare); + tree_check(tree, number_compare); + tree_check(__maps.maps, __maps_compare); + __print_maps(15); } diff --git a/test/libc/thread/pthread_atfork_test.c b/test/libc/thread/pthread_atfork_test.c index dc977b9aa..f1e44139b 100644 --- a/test/libc/thread/pthread_atfork_test.c +++ b/test/libc/thread/pthread_atfork_test.c @@ -16,8 +16,11 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/dce.h" +#include "libc/intrin/atomic.h" +#include "libc/intrin/kprintf.h" #include "libc/mem/gc.h" #include "libc/mem/mem.h" #include "libc/runtime/internal.h" @@ -66,7 +69,7 @@ TEST(pthread_atfork, test) { EXITS(0); } -pthread_mutex_t mu; +pthread_mutex_t mu = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; void mu_lock(void) { pthread_mutex_lock(&mu); @@ -80,8 +83,12 @@ void mu_wipe(void) { pthread_mutex_init(&mu, 0); } +static atomic_bool once; + void *Worker(void *arg) { for (int i = 0; i < 20; ++i) { + if (!atomic_exchange(&once, true)) + __print_maps(0); mu_lock(); usleep(20); mu_unlock(); @@ -99,14 +106,14 @@ void *Worker(void *arg) { } TEST(pthread_atfork, fork_exit_torture) { + if (!IsFreebsd()) + return; mu_wipe(); pthread_atfork(mu_lock, mu_unlock, mu_wipe); int i, n = 4; pthread_t *t = gc(malloc(sizeof(pthread_t) * n)); - for (i = 0; i < n; ++i) { + for (i = 0; i < n; ++i) ASSERT_EQ(0, pthread_create(t + i, 0, Worker, 0)); - } - for (i = 0; i < n; ++i) { + for (i = 0; i < n; ++i) ASSERT_EQ(0, pthread_join(t[i], 0)); - } } diff --git a/third_party/nsync/common.c b/third_party/nsync/common.c index 4c77e7d4e..3a20bcdbb 100644 --- a/third_party/nsync/common.c +++ b/third_party/nsync/common.c @@ -18,6 +18,7 @@ #include "libc/calls/calls.h" #include "libc/calls/syscall-sysv.internal.h" #include "libc/dce.h" +#include "libc/intrin/directmap.internal.h" #include "libc/intrin/dll.h" #include "libc/intrin/extend.internal.h" #include "libc/nt/enum/filemapflags.h" @@ -25,21 +26,17 @@ #include "libc/nt/memory.h" #include "libc/nt/runtime.h" #include "libc/runtime/memtrack.internal.h" +#include "libc/runtime/runtime.h" +#include "libc/stdalign.internal.h" +#include "libc/stdalign.internal.h" #include "libc/sysv/consts/map.h" #include "libc/sysv/consts/prot.h" +#include "libc/thread/thread.h" #include "libc/thread/tls.h" #include "third_party/nsync/atomic.h" #include "third_party/nsync/atomic.internal.h" #include "third_party/nsync/common.internal.h" #include "third_party/nsync/mu_semaphore.h" -#include "third_party/nsync/races.internal.h" -#include "libc/runtime/runtime.h" -#include "libc/runtime/runtime.h" -#include "libc/sysv/consts/map.h" -#include "libc/nt/runtime.h" -#include "libc/intrin/directmap.internal.h" -#include "libc/thread/thread.h" -#include "libc/dce.h" #include "third_party/nsync/wait_s.internal.h" __static_yoink("nsync_notice"); @@ -97,33 +94,15 @@ __static_yoink("nsync_notice"); distinct wakeup conditions were high. So clients are advised to resort to condition variables if they have many distinct wakeup conditions. */ -/* Used in spinloops to delay resumption of the loop. - Usage: - unsigned attempts = 0; - while (try_something) { - attempts = nsync_spin_delay_ (attempts); - } */ -unsigned nsync_spin_delay_ (unsigned attempts) { - if (attempts < 7) { - volatile int i; - for (i = 0; i != 1 << attempts; i++) { - } - attempts++; - } else { - nsync_yield_ (); - } - return (attempts); -} - /* Spin until (*w & test) == 0, then atomically perform *w = ((*w | set) & ~clear), perform an acquire barrier, and return the previous value of *w. */ uint32_t nsync_spin_test_and_set_ (nsync_atomic_uint32_ *w, uint32_t test, - uint32_t set, uint32_t clear) { + uint32_t set, uint32_t clear, void *symbol) { unsigned attempts = 0; /* CV_SPINLOCK retry count */ uint32_t old = ATM_LOAD (w); while ((old & test) != 0 || !ATM_CAS_ACQ (w, old, (old | set) & ~clear)) { - attempts = nsync_spin_delay_ (attempts); + attempts = pthread_delay_np (symbol, attempts); old = ATM_LOAD (w); } return (old); @@ -156,33 +135,69 @@ waiter *nsync_dll_waiter_samecond_ (struct Dll *e) { /* -------------------------------- */ -static void *nsync_malloc (size_t size) { - void *res; - if (!IsWindows ()) { - // too much of a performance hit to track - res = __sys_mmap ((void *)0x7110000000, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, 0, 0); +static _Atomic(waiter *) free_waiters; + +static void free_waiters_push (waiter *w) { + int backoff = 0; + w->next_free = atomic_load_explicit (&free_waiters, memory_order_relaxed); + while (!atomic_compare_exchange_weak_explicit (&free_waiters, &w->next_free, w, + memory_order_acq_rel, memory_order_relaxed)) + backoff = pthread_delay_np(free_waiters, backoff); +} + +static void free_waiters_populate (void) { + int n; + if (IsNetbsd () || IsXnuSilicon ()) { + // netbsd needs one file descriptor per semaphore (!!) + // tim cook wants us to use his grand central dispatch + n = 1; } else { - // must be tracked for fork() resurrection - res = mmap (0, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, 0); + n = getpagesize() / sizeof(waiter); } - if (res == MAP_FAILED) + waiter *waiters = mmap (0, n * sizeof(waiter), + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + if (waiters == MAP_FAILED) nsync_panic_ ("out of memory\n"); - return res; + for (size_t i = 0; i < n; ++i) { + waiter *w = &waiters[i]; + w->tag = WAITER_TAG; + w->nw.tag = NSYNC_WAITER_TAG; + if (!nsync_mu_semaphore_init (&w->sem)) { + if (!i) + nsync_panic_ ("out of semaphores\n"); + break; + } + w->nw.sem = &w->sem; + dll_init (&w->nw.q); + NSYNC_ATOMIC_UINT32_STORE_ (&w->nw.waiting, 0); + w->nw.flags = NSYNC_WAITER_FLAG_MUCV; + ATM_STORE (&w->remove_count, 0); + dll_init (&w->same_condition); + w->flags = 0; + free_waiters_push (w); + } +} + +static waiter *free_waiters_pop (void) { + waiter *w; + int backoff = 0; + for (;;) { + if ((w = atomic_load_explicit (&free_waiters, memory_order_relaxed))) { + if (atomic_compare_exchange_weak_explicit (&free_waiters, &w, w->next_free, + memory_order_acq_rel, memory_order_relaxed)) + return w; + backoff = pthread_delay_np(free_waiters, backoff); + } else { + free_waiters_populate (); + } + } + return w; } /* -------------------------------- */ -static struct Dll *free_waiters = NULL; - -/* free_waiters points to a doubly-linked list of free waiter structs. */ -pthread_mutex_t nsync_waiters_mu = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; - #define waiter_for_thread __get_tls()->tib_nsync void nsync_waiter_destroy (void *v) { @@ -193,45 +208,20 @@ void nsync_waiter_destroy (void *v) { of thread-local variables can be arbitrary in some platform e.g. POSIX. */ waiter_for_thread = NULL; - IGNORE_RACES_START (); ASSERT ((w->flags & (WAITER_RESERVED|WAITER_IN_USE)) == WAITER_RESERVED); w->flags &= ~WAITER_RESERVED; - pthread_mutex_lock (&nsync_waiters_mu); - dll_make_first (&free_waiters, &w->nw.q); - pthread_mutex_unlock (&nsync_waiters_mu); - IGNORE_RACES_END (); + free_waiters_push (w); } /* Return a pointer to an unused waiter struct. Ensures that the enclosed timer is stopped and its channel drained. */ waiter *nsync_waiter_new_ (void) { - struct Dll *q; - waiter *tw; waiter *w; + waiter *tw; tw = waiter_for_thread; w = tw; if (w == NULL || (w->flags & (WAITER_RESERVED|WAITER_IN_USE)) != WAITER_RESERVED) { - w = NULL; - pthread_mutex_lock (&nsync_waiters_mu); - q = dll_first (free_waiters); - if (q != NULL) { /* If free list is non-empty, dequeue an item. */ - dll_remove (&free_waiters, q); - w = DLL_WAITER (q); - } - pthread_mutex_unlock (&nsync_waiters_mu); - if (w == NULL) { /* If free list was empty, allocate an item. */ - w = (waiter *) nsync_malloc (sizeof (*w)); - w->tag = WAITER_TAG; - w->nw.tag = NSYNC_WAITER_TAG; - nsync_mu_semaphore_init (&w->sem); - w->nw.sem = &w->sem; - dll_init (&w->nw.q); - NSYNC_ATOMIC_UINT32_STORE_ (&w->nw.waiting, 0); - w->nw.flags = NSYNC_WAITER_FLAG_MUCV; - ATM_STORE (&w->remove_count, 0); - dll_init (&w->same_condition); - w->flags = 0; - } + w = free_waiters_pop (); if (tw == NULL) { w->flags |= WAITER_RESERVED; waiter_for_thread = w; @@ -246,9 +236,7 @@ void nsync_waiter_free_ (waiter *w) { ASSERT ((w->flags & WAITER_IN_USE) != 0); w->flags &= ~WAITER_IN_USE; if ((w->flags & WAITER_RESERVED) == 0) { - pthread_mutex_lock (&nsync_waiters_mu); - dll_make_first (&free_waiters, &w->nw.q); - pthread_mutex_unlock (&nsync_waiters_mu); + free_waiters_push (w); if (w == waiter_for_thread) waiter_for_thread = 0; } diff --git a/third_party/nsync/common.internal.h b/third_party/nsync/common.internal.h index 90205e102..a0b2c0ffe 100644 --- a/third_party/nsync/common.internal.h +++ b/third_party/nsync/common.internal.h @@ -24,19 +24,11 @@ void nsync_yield_(void); /* Retrieve the per-thread cache of the waiter object. Platform specific. */ void *nsync_per_thread_waiter_(void (*dest)(void *)); -/* Used in spinloops to delay resumption of the loop. - Usage: - unsigned attempts = 0; - while (try_something) { - attempts = nsync_spin_delay_ (attempts); - } */ -unsigned nsync_spin_delay_(unsigned attempts); - /* Spin until (*w & test) == 0, then atomically perform *w = ((*w | set) & ~clear), perform an acquire barrier, and return the previous value of *w. */ uint32_t nsync_spin_test_and_set_(nsync_atomic_uint32_ *w, uint32_t test, - uint32_t set, uint32_t clear); + uint32_t set, uint32_t clear, void *symbol); /* Abort after printing the nul-temrinated string s[]. */ void nsync_panic_(const char *s) wontreturn; @@ -210,6 +202,7 @@ typedef struct waiter_s { struct wait_condition_s cond; /* A condition on which to acquire a mu. */ struct Dll same_condition; /* Links neighbours in nw.q with same non-nil condition. */ + struct waiter_s * next_free; } waiter; static const uint32_t WAITER_TAG = 0x0590239f; static const uint32_t NSYNC_WAITER_TAG = 0x726d2ba9; diff --git a/third_party/nsync/mem/nsync_cv.c b/third_party/nsync/mem/nsync_cv.c index 926379340..8e363f77c 100644 --- a/third_party/nsync/mem/nsync_cv.c +++ b/third_party/nsync/mem/nsync_cv.c @@ -193,7 +193,7 @@ static int nsync_cv_wait_with_deadline_impl_ (struct nsync_cv_wait_with_deadline /* A timeout or cancellation occurred, and no wakeup. Acquire *pcv's spinlock, and confirm. */ c->old_word = nsync_spin_test_and_set_ (&c->pcv->word, CV_SPINLOCK, - CV_SPINLOCK, 0); + CV_SPINLOCK, 0, c->cv_mu); /* Check that w wasn't removed from the queue after we checked above, but before we acquired the spinlock. The test of remove_count confirms that the waiter *w @@ -226,7 +226,7 @@ static int nsync_cv_wait_with_deadline_impl_ (struct nsync_cv_wait_with_deadline has not yet set the waiting field to zero; a cancellation or timeout may prevent this thread from blocking above on the semaphore. */ - attempts = nsync_spin_delay_ (attempts); + attempts = pthread_delay_np (c->cv_mu, attempts); } } if (c->cv_mu != NULL && c->w->cv_mu == NULL) { /* waiter was moved to *pmu's queue, and woken. */ @@ -323,7 +323,7 @@ int nsync_cv_wait_with_deadline_generic (nsync_cv *pcv, void *pmu, } /* acquire spinlock, set non-empty */ - c.old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK|CV_NON_EMPTY, 0); + c.old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK|CV_NON_EMPTY, 0, pmu); dll_make_last (&pcv->waiters, &c.w->nw.q); c.remove_count = ATM_LOAD (&c.w->remove_count); /* Release the spin lock. */ @@ -355,7 +355,7 @@ void nsync_cv_signal (nsync_cv *pcv) { int all_readers = 0; /* acquire spinlock */ uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, - CV_SPINLOCK, 0); + CV_SPINLOCK, 0, pcv); if (!dll_is_empty (pcv->waiters)) { /* Point to first waiter that enqueued itself, and detach it from all others. */ @@ -438,7 +438,7 @@ void nsync_cv_broadcast (nsync_cv *pcv) { int all_readers; struct Dll *to_wake_list = NULL; /* waiters that we will wake */ /* acquire spinlock */ - nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0); + nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0, pcv); p = NULL; next = NULL; all_readers = 1; @@ -497,7 +497,7 @@ static nsync_time cv_ready_time (void *v, struct nsync_waiter_s *nw) { static int cv_enqueue (void *v, struct nsync_waiter_s *nw) { nsync_cv *pcv = (nsync_cv *) v; /* acquire spinlock */ - uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0); + uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0, pcv); dll_make_last (&pcv->waiters, &nw->q); ATM_STORE (&nw->waiting, 1); /* Release spinlock. */ @@ -509,7 +509,7 @@ static int cv_dequeue (void *v, struct nsync_waiter_s *nw) { nsync_cv *pcv = (nsync_cv *) v; int was_queued = 0; /* acquire spinlock */ - uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0); + uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0, pcv); if (ATM_LOAD_ACQ (&nw->waiting) != 0) { dll_remove (&pcv->waiters, &nw->q); ATM_STORE (&nw->waiting, 0); diff --git a/third_party/nsync/mem/nsync_debug.c b/third_party/nsync/mem/nsync_debug.c index 1b72c1175..a3d847286 100644 --- a/third_party/nsync/mem/nsync_debug.c +++ b/third_party/nsync/mem/nsync_debug.c @@ -197,7 +197,7 @@ static char *emit_mu_state (struct emit_buf *b, nsync_mu *mu, word = ATM_LOAD (&mu->word); if ((word & MU_WAITING) != 0 && print_waiters && /* can benefit from lock */ (blocking || (word & MU_SPINLOCK) == 0)) { /* willing, or no need to wait */ - word = nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK, MU_SPINLOCK, 0); + word = nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK, MU_SPINLOCK, 0, mu); acquired = 1; } readers = word / MU_RLOCK; @@ -234,7 +234,7 @@ static char *emit_cv_state (struct emit_buf *b, nsync_cv *cv, word = ATM_LOAD (&cv->word); if ((word & CV_NON_EMPTY) != 0 && print_waiters && /* can benefit from lock */ (blocking || (word & CV_SPINLOCK) == 0)) { /* willing, or no need to wait */ - word = nsync_spin_test_and_set_ (&cv->word, CV_SPINLOCK, CV_SPINLOCK, 0); + word = nsync_spin_test_and_set_ (&cv->word, CV_SPINLOCK, CV_SPINLOCK, 0, cv); acquired = 1; } emit_print (b, "cv 0x%i -> 0x%i = {", (uintptr_t) cv, word); diff --git a/third_party/nsync/mem/nsync_mu_wait.c b/third_party/nsync/mem/nsync_mu_wait.c index 3c48ea599..e46492aa9 100644 --- a/third_party/nsync/mem/nsync_mu_wait.c +++ b/third_party/nsync/mem/nsync_mu_wait.c @@ -21,6 +21,7 @@ #include "third_party/nsync/common.internal.h" #include "third_party/nsync/mu_semaphore.h" #include "third_party/nsync/races.internal.h" +#include "libc/thread/thread.h" #include "third_party/nsync/wait_s.internal.h" __static_yoink("nsync_notice"); @@ -84,7 +85,7 @@ static int mu_try_acquire_after_timeout_or_cancel (nsync_mu *mu, lock_type *l_ty ATM_CAS_RELACQ (&mu->word, old_word, old_word|MU_WRITER_WAITING); } - spin_attempts = nsync_spin_delay_ (spin_attempts); + spin_attempts = pthread_delay_np (mu, spin_attempts); old_word = ATM_LOAD (&mu->word); } /* Check that w wasn't removed from the queue after our caller checked, @@ -194,7 +195,7 @@ int nsync_mu_wait_with_deadline (nsync_mu *mu, /* Acquire spinlock. */ old_word = nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK, - MU_SPINLOCK|MU_WAITING|has_condition, MU_ALL_FALSE); + MU_SPINLOCK|MU_WAITING|has_condition, MU_ALL_FALSE, mu); had_waiters = ((old_word & (MU_DESIG_WAKER | MU_WAITING)) == MU_WAITING); /* Queue the waiter. */ if (first_wait) { @@ -244,7 +245,7 @@ int nsync_mu_wait_with_deadline (nsync_mu *mu, } if (ATM_LOAD (&w->nw.waiting) != 0) { - attempts = nsync_spin_delay_ (attempts); /* will ultimately yield */ + attempts = pthread_delay_np (mu, attempts); /* will ultimately yield */ } } diff --git a/third_party/nsync/mem/nsync_once.c b/third_party/nsync/mem/nsync_once.c index 163923359..873766b99 100644 --- a/third_party/nsync/mem/nsync_once.c +++ b/third_party/nsync/mem/nsync_once.c @@ -21,6 +21,7 @@ #include "third_party/nsync/mu_semaphore.h" #include "third_party/nsync/once.h" #include "third_party/nsync/races.internal.h" +#include "libc/thread/thread.h" #include "third_party/nsync/wait_s.internal.h" __static_yoink("nsync_notice"); @@ -92,7 +93,7 @@ static void nsync_run_once_impl (nsync_once *once, struct once_sync_s *s, deadline = nsync_time_add (nsync_time_now (), nsync_time_ms (attempts)); nsync_cv_wait_with_deadline (&s->once_cv, &s->once_mu, deadline, NULL); } else { - attempts = nsync_spin_delay_ (attempts); + attempts = pthread_delay_np (once, attempts); } } if (s != NULL) { diff --git a/third_party/nsync/mu.c b/third_party/nsync/mu.c index 4cae68328..20eac1e68 100644 --- a/third_party/nsync/mu.c +++ b/third_party/nsync/mu.c @@ -22,6 +22,7 @@ #include "third_party/nsync/common.internal.h" #include "third_party/nsync/mu_semaphore.h" #include "third_party/nsync/races.internal.h" +#include "libc/thread/thread.h" #include "third_party/nsync/wait_s.internal.h" __static_yoink("nsync_notice"); @@ -120,7 +121,7 @@ void nsync_mu_lock_slow_ (nsync_mu *mu, waiter *w, uint32_t clear, lock_type *l_ about waiting writers or long waiters. */ zero_to_acquire &= ~(MU_WRITER_WAITING | MU_LONG_WAIT); } - attempts = nsync_spin_delay_ (attempts); + attempts = pthread_delay_np (mu, attempts); } ALLOW_CANCELATION; } @@ -393,7 +394,7 @@ void nsync_mu_unlock_slow_ (nsync_mu *mu, lock_type *l_type) { released above. */ if (testing_conditions) { nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK, - MU_SPINLOCK, 0); + MU_SPINLOCK, 0, mu); } /* add the new_waiters to the last of the waiters. */ @@ -444,7 +445,7 @@ void nsync_mu_unlock_slow_ (nsync_mu *mu, lock_type *l_type) { } return; } - attempts = nsync_spin_delay_ (attempts); + attempts = pthread_delay_np (mu, attempts); } } diff --git a/third_party/nsync/mu_semaphore.c b/third_party/nsync/mu_semaphore.c index 493efa2c4..fd9d855ce 100644 --- a/third_party/nsync/mu_semaphore.c +++ b/third_party/nsync/mu_semaphore.c @@ -27,7 +27,7 @@ __static_yoink("nsync_notice"); #define PREFER_GCD_OVER_ULOCK 1 /* Initialize *s; the initial value is 0. */ -void nsync_mu_semaphore_init (nsync_semaphore *s) { +bool nsync_mu_semaphore_init (nsync_semaphore *s) { if (PREFER_GCD_OVER_ULOCK && IsXnuSilicon ()) { return nsync_mu_semaphore_init_gcd (s); } else if (IsNetbsd ()) { diff --git a/third_party/nsync/mu_semaphore.h b/third_party/nsync/mu_semaphore.h index 64c7d563d..992d4849f 100644 --- a/third_party/nsync/mu_semaphore.h +++ b/third_party/nsync/mu_semaphore.h @@ -8,7 +8,7 @@ typedef struct nsync_semaphore_s_ { } nsync_semaphore; /* Initialize *s; the initial value is 0. */ -void nsync_mu_semaphore_init(nsync_semaphore *s); +bool nsync_mu_semaphore_init(nsync_semaphore *s); /* Wait until the count of *s exceeds 0, and decrement it. */ errno_t nsync_mu_semaphore_p(nsync_semaphore *s); diff --git a/third_party/nsync/mu_semaphore.internal.h b/third_party/nsync/mu_semaphore.internal.h index ce3181fc3..31b51975d 100755 --- a/third_party/nsync/mu_semaphore.internal.h +++ b/third_party/nsync/mu_semaphore.internal.h @@ -4,17 +4,17 @@ #include "third_party/nsync/time.h" COSMOPOLITAN_C_START_ -void nsync_mu_semaphore_init_futex(nsync_semaphore *); +bool nsync_mu_semaphore_init_futex(nsync_semaphore *); errno_t nsync_mu_semaphore_p_futex(nsync_semaphore *); errno_t nsync_mu_semaphore_p_with_deadline_futex(nsync_semaphore *, nsync_time); void nsync_mu_semaphore_v_futex(nsync_semaphore *); -void nsync_mu_semaphore_init_sem(nsync_semaphore *); +bool nsync_mu_semaphore_init_sem(nsync_semaphore *); errno_t nsync_mu_semaphore_p_sem(nsync_semaphore *); errno_t nsync_mu_semaphore_p_with_deadline_sem(nsync_semaphore *, nsync_time); void nsync_mu_semaphore_v_sem(nsync_semaphore *); -void nsync_mu_semaphore_init_gcd(nsync_semaphore *); +bool nsync_mu_semaphore_init_gcd(nsync_semaphore *); errno_t nsync_mu_semaphore_p_gcd(nsync_semaphore *); errno_t nsync_mu_semaphore_p_with_deadline_gcd(nsync_semaphore *, nsync_time); void nsync_mu_semaphore_v_gcd(nsync_semaphore *); diff --git a/third_party/nsync/mu_semaphore_futex.c b/third_party/nsync/mu_semaphore_futex.c index 14d0ec20e..12964e7c8 100644 --- a/third_party/nsync/mu_semaphore_futex.c +++ b/third_party/nsync/mu_semaphore_futex.c @@ -43,9 +43,10 @@ static nsync_semaphore *sem_big_enough_for_futex = (nsync_semaphore *) (uintptr_ (sizeof (struct futex) <= sizeof (*sem_big_enough_for_futex))); /* Initialize *s; the initial value is 0. */ -void nsync_mu_semaphore_init_futex (nsync_semaphore *s) { +bool nsync_mu_semaphore_init_futex (nsync_semaphore *s) { struct futex *f = (struct futex *) s; f->i = 0; + return true; } /* Wait until the count of *s exceeds 0, and decrement it. If POSIX cancellations diff --git a/third_party/nsync/mu_semaphore_gcd.c b/third_party/nsync/mu_semaphore_gcd.c index fd36135de..1f88d5e0d 100644 --- a/third_party/nsync/mu_semaphore_gcd.c +++ b/third_party/nsync/mu_semaphore_gcd.c @@ -85,8 +85,8 @@ static errno_t nsync_dispatch_semaphore_wait (nsync_semaphore *s, } /* Initialize *s; the initial value is 0. */ -void nsync_mu_semaphore_init_gcd (nsync_semaphore *s) { - *(dispatch_semaphore_t *)s = dispatch_semaphore_create (0); +bool nsync_mu_semaphore_init_gcd (nsync_semaphore *s) { + return !!(*(dispatch_semaphore_t *)s = dispatch_semaphore_create (0)); } /* Wait until the count of *s exceeds 0, and decrement it. If POSIX cancellations diff --git a/third_party/nsync/mu_semaphore_sem.c b/third_party/nsync/mu_semaphore_sem.c index d2c6377ac..3eb8a22f6 100644 --- a/third_party/nsync/mu_semaphore_sem.c +++ b/third_party/nsync/mu_semaphore_sem.c @@ -54,14 +54,22 @@ static struct { static nsync_semaphore *sem_big_enough_for_sem = (nsync_semaphore *) (uintptr_t)(1 / (sizeof (struct sem) <= sizeof (*sem_big_enough_for_sem))); -static void nsync_mu_semaphore_sem_create (struct sem *f) { +static bool nsync_mu_semaphore_sem_create (struct sem *f) { + int rc; int lol; f->id = 0; - ASSERT (!sys_sem_init (0, &f->id)); - if ((lol = __sys_fcntl (f->id, F_DUPFD_CLOEXEC, 50)) >= 50) { - sys_close (f->id); + rc = sys_sem_init (0, &f->id); + STRACE ("sem_init(0, [%ld]) → %d", f->id, rc); + if (rc != 0) + return false; + lol = __sys_fcntl (f->id, F_DUPFD_CLOEXEC, 50); + STRACE ("fcntl(%ld, F_DUPFD_CLOEXEC, 50) → %d", f->id, lol); + if (lol >= 50) { + rc = sys_close (f->id); + STRACE ("close(%ld) → %d", f->id, rc); f->id = lol; } + return true; } static void nsync_mu_semaphore_sem_fork_child (void) { @@ -69,8 +77,12 @@ static void nsync_mu_semaphore_sem_fork_child (void) { struct sem *f; for (e = dll_first (g_sems.list); e; e = dll_next (g_sems.list, e)) { f = SEM_CONTAINER (e); - sys_close (f->id); - nsync_mu_semaphore_sem_create (f); + int rc = sys_close (f->id); + STRACE ("close(%ld) → %d", f->id, rc); + } + for (e = dll_first (g_sems.list); e; e = dll_next (g_sems.list, e)) { + f = SEM_CONTAINER (e); + ASSERT (nsync_mu_semaphore_sem_create (f)); } (void) pthread_spin_init (&g_sems.lock, 0); } @@ -80,15 +92,16 @@ static void nsync_mu_semaphore_sem_init (void) { } /* Initialize *s; the initial value is 0. */ -void nsync_mu_semaphore_init_sem (nsync_semaphore *s) { +bool nsync_mu_semaphore_init_sem (nsync_semaphore *s) { struct sem *f = (struct sem *) s; - nsync_mu_semaphore_sem_create (f); + if (!nsync_mu_semaphore_sem_create (f)) + return false; cosmo_once (&g_sems.once, nsync_mu_semaphore_sem_init); pthread_spin_lock (&g_sems.lock); dll_init (&f->list); dll_make_first (&g_sems.list, &f->list); pthread_spin_unlock (&g_sems.lock); - STRACE ("sem_init(0, [%ld]) → 0", f->id); + return true; } /* Wait until the count of *s exceeds 0, and decrement it. If POSIX cancellations diff --git a/tool/build/runitd.c b/tool/build/runitd.c index d1b0f375f..577876772 100644 --- a/tool/build/runitd.c +++ b/tool/build/runitd.c @@ -134,12 +134,12 @@ #define INFOF(FMT, ...) LOGF(INFO, FMT, ##__VA_ARGS__) #define WARNF(FMT, ...) LOGF(WARN, FMT, ##__VA_ARGS__) -#define LOGF(LVL, FMT, ...) \ - do { \ - if (g_log_level >= LOG_LEVEL_##LVL) { \ - kprintf("%r" #LVL " %6P %'18T %s:%d " FMT "\n", __FILE__, __LINE__, \ - ##__VA_ARGS__); \ - } \ +#define LOGF(LVL, FMT, ...) \ + do { \ + if (g_log_level >= LOG_LEVEL_##LVL) { \ + kprintf("%r" #LVL " %6P %6H %'18T %s:%d " FMT "\n", __FILE__, __LINE__, \ + ##__VA_ARGS__); \ + } \ } while (0) struct Client { @@ -443,12 +443,18 @@ void FreeClient(struct Client *client) { void *ClientWorker(void *arg) { uint32_t crc; sigset_t sigmask; + struct timespec ts0; + struct timespec ts1; + struct timespec ts2; int events, wstatus; struct Client *client = arg; uint32_t namesize, filesize; char *addrstr, *origname; unsigned char msg[4 + 1 + 4 + 4 + 4]; + ts0 = timespec_real(); + ts1 = timespec_real(); + SetupPresharedKeySsl(MBEDTLS_SSL_IS_SERVER, g_psk); defer(FreeClient, client); @@ -458,9 +464,15 @@ void *ClientWorker(void *arg) { EzHandshake(); addrstr = DescribeAddress(&client->addr); DEBUF("%s %s %s", DescribeAddress(&g_servaddr), "accepted", addrstr); + DEBUF("it took %'zu us to handshake client", + timespec_tomicros(timespec_sub(timespec_real(), ts1))); // get the executable + ts1 = timespec_real(); + ts2 = timespec_real(); Recv(client, msg, sizeof(msg)); + DEBUF("it took %'zu us to receive #1", + timespec_tomicros(timespec_sub(timespec_real(), ts2))); if (READ32BE(msg) != RUNITD_MAGIC) { WARNF("%s magic mismatch!", addrstr); pthread_exit(0); @@ -473,11 +485,19 @@ void *ClientWorker(void *arg) { filesize = READ32BE(msg + 9); crc = READ32BE(msg + 13); origname = gc(calloc(1, namesize + 1)); + ts2 = timespec_real(); Recv(client, origname, namesize); + DEBUF("it took %'zu us to receive #2", + timespec_tomicros(timespec_sub(timespec_real(), ts2))); VERBF("%s sent %#s (%'u bytes @ %#s)", addrstr, origname, filesize, client->tmpexepath); char *exedata = gc(malloc(filesize)); + ts2 = timespec_real(); Recv(client, exedata, filesize); + DEBUF("it took %'zu us to receive #3", + timespec_tomicros(timespec_sub(timespec_real(), ts2))); + DEBUF("it took %'zu us to receive executable from network", + timespec_tomicros(timespec_sub(timespec_real(), ts1))); if (crc32_z(0, exedata, filesize) != crc) { WARNF("%s crc mismatch! %#s", addrstr, origname); pthread_exit(0); @@ -488,6 +508,7 @@ void *ClientWorker(void *arg) { // condition can happen, where etxtbsy is raised by our execve // we're using o_cloexec so it's guaranteed to fix itself fast // thus we use an optimistic approach to avoid expensive locks + ts1 = timespec_real(); sprintf(client->tmpexepath, "o/%s.XXXXXX", basename(stripext(gc(strdup(origname))))); int exefd = openatemp(AT_FDCWD, client->tmpexepath, 0, O_CLOEXEC, 0700); @@ -510,6 +531,8 @@ void *ClientWorker(void *arg) { WARNF("%s failed to close %#s due to %m", addrstr, origname); pthread_exit(0); } + DEBUF("it took %'zu us to write executable to disk", + timespec_tomicros(timespec_sub(timespec_real(), ts1))); // do the args int i = 0; @@ -560,8 +583,11 @@ RetryOnEtxtbsyRaceCondition: posix_spawn_file_actions_adddup2(&spawnfila, g_bogusfd, 0); posix_spawn_file_actions_adddup2(&spawnfila, client->pipe[1], 1); posix_spawn_file_actions_adddup2(&spawnfila, client->pipe[1], 2); + ts1 = timespec_real(); err = posix_spawn(&client->pid, client->tmpexepath, &spawnfila, &spawnattr, args, environ); + DEBUF("it took %'zu us to call posix_spawn", + timespec_tomicros(timespec_sub(timespec_real(), ts1))); if (err) { if (err == ETXTBSY) { goto RetryOnEtxtbsyRaceCondition; @@ -599,8 +625,11 @@ RetryOnEtxtbsyRaceCondition: fds[0].events = POLLIN; fds[1].fd = client->pipe[0]; fds[1].events = POLLIN; + ts1 = timespec_real(); events = poll(fds, ARRAYLEN(fds), timespec_tomillis(timespec_sub(deadline, now))); + DEBUF("it took %'zu us to call poll", + timespec_tomicros(timespec_sub(timespec_real(), ts1))); if (events == -1) { if (errno == EINTR) { INFOF("poll interrupted"); @@ -615,7 +644,10 @@ RetryOnEtxtbsyRaceCondition: if (fds[0].revents) { int received; char buf[512]; + ts1 = timespec_real(); received = mbedtls_ssl_read(&ezssl, buf, sizeof(buf)); + DEBUF("it took %'zu us to call mbedtls_ssl_read", + timespec_tomicros(timespec_sub(timespec_real(), ts1))); if (!received) { WARNF("%s client disconnected so killing worker %d", origname, client->pid); @@ -640,7 +672,10 @@ RetryOnEtxtbsyRaceCondition: } if (fds[1].revents) { char buf[512]; + ts1 = timespec_real(); ssize_t got = read(client->pipe[0], buf, sizeof(buf)); + DEBUF("it took %'zu us to call read", + timespec_tomicros(timespec_sub(timespec_real(), ts1))); if (got == -1) { WARNF("got %s reading %s output", strerror(errno), origname); goto HangupClientAndTerminateJob; @@ -658,7 +693,10 @@ RetryOnEtxtbsyRaceCondition: WaitAgain: DEBUF("waitpid"); struct rusage rusage; + ts1 = timespec_real(); int wrc = wait4(client->pid, &wstatus, 0, &rusage); + DEBUF("it took %'zu us to call wait4", + timespec_tomicros(timespec_sub(timespec_real(), ts1))); if (wrc == -1) { if (errno == EINTR) { WARNF("waitpid interrupted; killing %s pid %d", origname, client->pid); @@ -711,13 +749,18 @@ WaitAgain: AppendResourceReport(&client->output, &rusage, "\n"); PrintProgramOutput(client); } + ts1 = timespec_real(); SendProgramOutput(client); SendExitMessage(exitcode); mbedtls_ssl_close_notify(&ezssl); + DEBUF("it took %'zu us to send result to client", + timespec_tomicros(timespec_sub(timespec_real(), ts1))); if (etxtbsy_tries > 1) { WARNF("encountered %d ETXTBSY race conditions spawning %s", etxtbsy_tries - 1, origname); } + DEBUF("it took %'zu us TO DO EVERYTHING", + timespec_tomicros(timespec_sub(timespec_real(), ts0))); pthread_exit(0); }