diff --git a/libc/calls/setrlimit.c b/libc/calls/setrlimit.c index 45a8c532c..de7b5adc9 100644 --- a/libc/calls/setrlimit.c +++ b/libc/calls/setrlimit.c @@ -85,7 +85,7 @@ int setrlimit(int resource, const struct rlimit *rlim) { rc = efault(); } else if (IsXnuSilicon()) { rc = _sysret(__syslib->__setrlimit(resource, rlim)); - } else if (!IsWindows()) { + } else if (!IsWindows() && !(IsNetbsd() && resource == RLIMIT_AS)) { rc = sys_setrlimit(resource, rlim); } else if (resource == RLIMIT_STACK) { rc = enotsup(); diff --git a/libc/intrin/BUILD.mk b/libc/intrin/BUILD.mk index a8052956e..5f05cfc5e 100644 --- a/libc/intrin/BUILD.mk +++ b/libc/intrin/BUILD.mk @@ -62,6 +62,10 @@ o/$(MODE)/libc/intrin/kprintf.o: private \ -Wframe-larger-than=128 \ -Walloca-larger-than=128 +o/$(MODE)/libc/intrin/tree.o: private \ + CFLAGS += \ + -ffunction-sections + o//libc/intrin/memmove.o: private \ CFLAGS += \ -fno-toplevel-reorder diff --git a/libc/intrin/describemapping.c b/libc/intrin/describemapping.c index 258e9a4ab..d4c54ef49 100644 --- a/libc/intrin/describemapping.c +++ b/libc/intrin/describemapping.c @@ -24,7 +24,7 @@ static char DescribeMapType(int flags) { switch (flags & MAP_TYPE) { case MAP_FILE: - return 'f'; + return '-'; case MAP_PRIVATE: return 'p'; case MAP_SHARED: @@ -47,7 +47,7 @@ const char *(DescribeMapping)(char p[8], int prot, int flags) { DescribeProt(p, prot); p[3] = DescribeMapType(flags); p[4] = (flags & MAP_ANONYMOUS) ? 'a' : '-'; - p[5] = (flags & MAP_FIXED) ? 'F' : '-'; + p[5] = (flags & MAP_FIXED) ? 'f' : '-'; p[6] = 0; return p; } diff --git a/libc/intrin/describeprotflags.c b/libc/intrin/describeprotflags.c index f281dffc2..5761ac0fe 100644 --- a/libc/intrin/describeprotflags.c +++ b/libc/intrin/describeprotflags.c @@ -20,12 +20,12 @@ #include "libc/macros.internal.h" #include "libc/sysv/consts/prot.h" -static const struct DescribeFlags kProtFlags[] = { - {PROT_READ, "READ"}, // - {PROT_WRITE, "WRITE"}, // - {PROT_EXEC, "EXEC"}, // -}; - const char *(DescribeProtFlags)(char buf[48], int x) { + const struct DescribeFlags kProtFlags[] = { + {PROT_READ, "READ"}, // + {PROT_WRITE, "WRITE"}, // + {PROT_EXEC, "EXEC"}, // + {PROT_GUARD, "GUARD"}, // + }; return DescribeFlags(buf, 48, kProtFlags, ARRAYLEN(kProtFlags), "PROT_", x); } diff --git a/libc/intrin/describerlimit.c b/libc/intrin/describerlimit.c index 79d9a7d46..9d211a6c4 100644 --- a/libc/intrin/describerlimit.c +++ b/libc/intrin/describerlimit.c @@ -18,8 +18,11 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/struct/rlimit.h" #include "libc/dce.h" +#include "libc/fmt/itoa.h" #include "libc/intrin/kprintf.h" #include "libc/intrin/strace.internal.h" +#include "libc/str/str.h" +#include "libc/sysv/consts/rlim.h" const char *DescribeRlimit(char buf[64], int rc, const struct rlimit *rlim) { if (rc == -1) @@ -29,7 +32,18 @@ const char *DescribeRlimit(char buf[64], int rc, const struct rlimit *rlim) { if (kisdangerous(rlim)) { ksnprintf(buf, 64, "%p", rlim); } else { - ksnprintf(buf, 64, "{%'ld, %'ld}", rlim->rlim_cur, rlim->rlim_max); + char str[2][21]; + if (rlim->rlim_cur == RLIM_INFINITY) { + strcpy(str[0], "RLIM_INFINITY"); + } else { + FormatInt64(str[0], rlim->rlim_cur); + } + if (rlim->rlim_max == RLIM_INFINITY) { + strcpy(str[1], "RLIM_INFINITY"); + } else { + FormatInt64(str[1], rlim->rlim_max); + } + ksnprintf(buf, 64, "{%s, %s}", str[0], str[1]); } return buf; } diff --git a/libc/intrin/directmap-nt.c b/libc/intrin/directmap-nt.c index a6fd1d2f1..00b93463d 100644 --- a/libc/intrin/directmap-nt.c +++ b/libc/intrin/directmap-nt.c @@ -86,9 +86,8 @@ TryAgain: if ((dm.addr = MapViewOfFileEx(dm.maphandle, fl.flags2, off >> 32, off, size, addr))) { uint32_t oldprot; - if (VirtualProtect(dm.addr, size, __prot2nt(prot, iscow), &oldprot)) { + if (VirtualProtect(dm.addr, size, __prot2nt(prot, iscow), &oldprot)) return dm; - } UnmapViewOfFile(dm.addr); } CloseHandle(dm.maphandle); diff --git a/libc/intrin/dll.c b/libc/intrin/dll.c index b94cfd7b2..b6337273d 100644 --- a/libc/intrin/dll.c +++ b/libc/intrin/dll.c @@ -28,7 +28,7 @@ * * It's required that `elem` and `succ` aren't part of the same list. */ -privileged void dll_splice_after(struct Dll *elem, struct Dll *succ) { +void dll_splice_after(struct Dll *elem, struct Dll *succ) { struct Dll *tmp1, *tmp2; tmp1 = elem->next; tmp2 = succ->prev; @@ -43,7 +43,7 @@ privileged void dll_splice_after(struct Dll *elem, struct Dll *succ) { * * @param list is a doubly-linked list, where `!*list` means empty */ -privileged void dll_remove(struct Dll **list, struct Dll *elem) { +void dll_remove(struct Dll **list, struct Dll *elem) { if (*list == elem) { if ((*list)->prev == *list) { *list = 0; @@ -66,7 +66,7 @@ privileged void dll_remove(struct Dll **list, struct Dll *elem) { * @param list is a doubly-linked list, where `!*list` means empty * @param elem must not be a member of `list`, or null for no-op */ -privileged void dll_make_first(struct Dll **list, struct Dll *elem) { +void dll_make_first(struct Dll **list, struct Dll *elem) { if (elem) { if (!*list) { *list = elem->prev; @@ -85,7 +85,7 @@ privileged void dll_make_first(struct Dll **list, struct Dll *elem) { * @param list is a doubly-linked list, where `!*list` means empty * @param elem must not be a member of `list`, or null for no-op */ -privileged void dll_make_last(struct Dll **list, struct Dll *elem) { +void dll_make_last(struct Dll **list, struct Dll *elem) { if (elem) { dll_make_first(list, elem->next); *list = elem; diff --git a/libc/intrin/kprintf.greg.c b/libc/intrin/kprintf.greg.c index 2bd649244..d32c52c64 100644 --- a/libc/intrin/kprintf.greg.c +++ b/libc/intrin/kprintf.greg.c @@ -25,7 +25,6 @@ #include "libc/fmt/magnumstrs.internal.h" #include "libc/intrin/asmflag.h" #include "libc/intrin/atomic.h" -#include "libc/intrin/dll.h" #include "libc/intrin/getenv.internal.h" #include "libc/intrin/kprintf.h" #include "libc/intrin/likely.h" @@ -154,27 +153,18 @@ __funline bool kischarmisaligned(const char *p, signed char t) { return false; } -privileged static bool32 kisdangerous_unlocked(const char *addr) { - struct Dll *e; - if ((e = dll_first(__maps.used))) { - do { - struct Map *map = MAP_CONTAINER(e); - if (map->addr <= addr && addr < map->addr + map->size) { - dll_remove(&__maps.used, e); - dll_make_first(&__maps.used, e); - return !(map->prot & PROT_READ); - } - } while ((e = dll_next(__maps.used, e))); - return true; - } else { - return false; - } -} - privileged bool32 kisdangerous(const void *addr) { - bool32 res; + bool32 res = true; __maps_lock(); - res = kisdangerous_unlocked(addr); + if (__maps.maps) { + struct Map *map; + if ((map = __maps_floor(addr))) + if ((const char *)addr >= map->addr && + (const char *)addr < map->addr + map->size) + res = false; + } else { + res = false; + } __maps_unlock(); return res; } diff --git a/libc/intrin/maps.c b/libc/intrin/maps.c index e2786ee75..e845679e5 100644 --- a/libc/intrin/maps.c +++ b/libc/intrin/maps.c @@ -20,6 +20,7 @@ #include "ape/sections.internal.h" #include "libc/dce.h" #include "libc/intrin/dll.h" +#include "libc/intrin/maps.h" #include "libc/runtime/runtime.h" #include "libc/runtime/stack.h" #include "libc/sysv/consts/auxv.h" @@ -32,8 +33,7 @@ __static_yoink("_init_maps"); struct Maps __maps; void __maps_add(struct Map *map) { - dll_init(&map->elem); - dll_make_first(&__maps.used, &map->elem); + tree_insert(&__maps.maps, &map->tree, __maps_compare); ++__maps.count; } diff --git a/libc/intrin/maps.h b/libc/intrin/maps.h index 5b5aba76a..387975b41 100644 --- a/libc/intrin/maps.h +++ b/libc/intrin/maps.h @@ -1,36 +1,41 @@ -#ifndef COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_ -#define COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_ +#ifndef COSMOPOLITAN_MAPS_H_ +#define COSMOPOLITAN_MAPS_H_ #include "libc/intrin/atomic.h" #include "libc/intrin/dll.h" +#include "libc/intrin/tree.h" +#include "libc/runtime/runtime.h" #include "libc/thread/tls2.internal.h" COSMOPOLITAN_C_START_ -#define MAP_CONTAINER(e) DLL_CONTAINER(struct Map, elem, e) +#define MAP_TREE_CONTAINER(e) TREE_CONTAINER(struct Map, tree, e) +#define MAP_FREE_CONTAINER(e) DLL_CONTAINER(struct Map, free, e) struct Map { char *addr; /* granule aligned */ size_t size; /* must be nonzero */ - struct Dll elem; /* for __maps.free */ - int64_t off; /* -1 if anonymous */ + int64_t off; /* ignore for anon */ int prot; /* memory protects */ int flags; /* memory map flag */ bool iscow; /* windows nt only */ bool readonlyfile; /* windows nt only */ unsigned visited; /* used for checks */ + unsigned oldprot; /* in windows fork */ intptr_t hand; /* windows nt only */ + union { + struct Tree tree; + struct Dll free; + }; }; struct Maps { - unsigned mono; atomic_int lock; + struct Tree *maps; struct Dll *free; - struct Dll *used; size_t count; size_t pages; + atomic_ulong rollo; struct Map stack; struct Map guard; - bool once; - atomic_ulong rollo; }; struct AddrSize { @@ -45,10 +50,37 @@ bool __maps_lock(void); void __maps_check(void); void __maps_unlock(void); void __maps_add(struct Map *); -struct Map *__maps_alloc(void); void __maps_free(struct Map *); +struct Map *__maps_alloc(void); +struct Map *__maps_floor(const char *); void __maps_stack(char *, int, int, size_t, int, intptr_t); +int __maps_compare(const struct Tree *, const struct Tree *); struct AddrSize __get_main_stack(void); +forceinline optimizespeed int __maps_search(const void *key, + const struct Tree *node) { + const char *addr = (const char *)key; + const struct Map *map = (const struct Map *)MAP_TREE_CONTAINER(node); + if (addr < map->addr) + return +1; + if (addr >= map->addr + map->size) + return -1; + return 0; +} + +static struct Map *__maps_next(struct Map *map) { + struct Tree *node; + if ((node = tree_next(&map->tree))) + return MAP_TREE_CONTAINER(node); + return 0; +} + +static struct Map *__maps_first(void) { + struct Tree *node; + if ((node = tree_first(__maps.maps))) + return MAP_TREE_CONTAINER(node); + return 0; +} + COSMOPOLITAN_C_END_ -#endif /* COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_ */ +#endif /* COSMOPOLITAN_MAPS_H_ */ diff --git a/libc/intrin/mmap.c b/libc/intrin/mmap.c index c4229924e..02ef9614e 100644 --- a/libc/intrin/mmap.c +++ b/libc/intrin/mmap.c @@ -67,20 +67,31 @@ struct StackFrame *bp = __builtin_frame_address(0); \ kprintf("%!s:%d: assertion failed: %!s\n", __FILE__, __LINE__, #x); \ kprintf("bt %!s\n", (DescribeBacktrace)(bt, bp)); \ - __print_maps(); \ - _Exit(99); \ + __print_maps(0); \ + __builtin_trap(); \ } \ } while (0) #endif +int __maps_compare(const struct Tree *ra, const struct Tree *rb) { + const struct Map *a = (const struct Map *)MAP_TREE_CONTAINER(ra); + const struct Map *b = (const struct Map *)MAP_TREE_CONTAINER(rb); + return (a->addr > b->addr) - (a->addr < b->addr); +} + +privileged optimizespeed struct Map *__maps_floor(const char *addr) { + struct Tree *node; + if ((node = tree_floor(__maps.maps, addr, __maps_search))) + return MAP_TREE_CONTAINER(node); + return 0; +} + static bool overlaps_existing_map(const char *addr, size_t size, int pagesz) { - for (struct Dll *e = dll_first(__maps.used); e; - e = dll_next(__maps.used, e)) { - struct Map *map = MAP_CONTAINER(e); + struct Map *map; + if ((map = __maps_floor(addr))) if (MAX(addr, map->addr) < MIN(addr + PGUP(size), map->addr + PGUP(map->size))) return true; - } return false; } @@ -89,66 +100,51 @@ void __maps_check(void) { size_t maps = 0; size_t pages = 0; int pagesz = getpagesize(); - unsigned id = ++__maps.mono; - for (struct Dll *e = dll_first(__maps.used); e; - e = dll_next(__maps.used, e)) { - struct Map *map = MAP_CONTAINER(e); + static unsigned mono; + unsigned id = ++mono; + for (struct Map *map = __maps_first(); map; map = __maps_next(map)) { ASSERT(map->addr != MAP_FAILED); ASSERT(map->visited != id); ASSERT(map->size); map->visited = id; pages += (map->size + getpagesize() - 1) / getpagesize(); maps += 1; + struct Map *next; + if ((next = __maps_next(map))) { + ASSERT(map->addr < next->addr); + ASSERT( + !(MAX(map->addr, next->addr) < + MIN(map->addr + PGUP(map->size), next->addr + PGUP(next->size)))); + } } ASSERT(maps = __maps.count); ASSERT(pages == __maps.pages); - for (struct Dll *e = dll_first(__maps.used); e; - e = dll_next(__maps.used, e)) { - struct Map *m1 = MAP_CONTAINER(e); - for (struct Dll *f = dll_next(__maps.used, e); f; - f = dll_next(__maps.used, f)) { - struct Map *m2 = MAP_CONTAINER(f); - ASSERT(MAX(m1->addr, m2->addr) >= - MIN(m1->addr + PGUP(m1->size), m2->addr + PGUP(m2->size))); - } - } #endif } void __maps_free(struct Map *map) { map->size = 0; map->addr = MAP_FAILED; - ASSERT(dll_is_alone(&map->elem)); - dll_make_last(&__maps.free, &map->elem); + dll_init(&map->free); + dll_make_first(&__maps.free, &map->free); } static void __maps_insert(struct Map *map) { - struct Dll *e = dll_first(__maps.used); - struct Map *last = e ? MAP_CONTAINER(e) : 0; __maps.pages += (map->size + getpagesize() - 1) / getpagesize(); - if (last && !IsWindows() && // - map->addr == last->addr + last->size && // + struct Map *floor = __maps_floor(map->addr); + if (floor && !IsWindows() && // + map->addr + map->size == floor->addr && // (map->flags & MAP_ANONYMOUS) && // - map->flags == last->flags && // - map->prot == last->prot) { - last->size += map->size; - dll_remove(&__maps.used, &last->elem); - dll_make_first(&__maps.used, &last->elem); - __maps_free(map); - } else if (last && !IsWindows() && // - map->addr + map->size == last->addr && // - (map->flags & MAP_ANONYMOUS) && // - map->flags == last->flags && // - map->prot == last->prot) { - last->addr -= map->size; - last->size += map->size; - dll_remove(&__maps.used, &last->elem); - dll_make_first(&__maps.used, &last->elem); + map->flags == floor->flags && // + map->prot == floor->prot) { + floor->addr -= map->size; + floor->size += map->size; __maps_free(map); + __maps_check(); } else { __maps_add(map); + __maps_check(); } - __maps_check(); } struct Map *__maps_alloc(void) { @@ -156,7 +152,7 @@ struct Map *__maps_alloc(void) { struct Map *map; if ((e = dll_first(__maps.free))) { dll_remove(&__maps.free, e); - map = MAP_CONTAINER(e); + map = MAP_FREE_CONTAINER(e); return map; } int granularity = __granularity(); @@ -168,11 +164,8 @@ struct Map *__maps_alloc(void) { CloseHandle(sys.maphandle); map = sys.addr; map->addr = MAP_FAILED; - dll_init(&map->elem); - for (int i = 1; i < granularity / sizeof(struct Map); ++i) { - dll_init(&map[i].elem); + for (int i = 1; i < granularity / sizeof(struct Map); ++i) __maps_free(map + i); - } return map; } @@ -190,106 +183,106 @@ static int __munmap(char *addr, size_t size, bool untrack_only) { // untrack mappings int rc = 0; - struct Dll *cur; - struct Dll *next; - struct Dll *delete = 0; + struct Map *map; + struct Map *next; + struct Dll *deleted = 0; if (__maps_lock()) { __maps_unlock(); return edeadlk(); } - for (cur = dll_first(__maps.used); cur; cur = next) { - next = dll_next(__maps.used, cur); - struct Map *map = MAP_CONTAINER(cur); + for (map = __maps_floor(addr); map; map = next) { + next = __maps_next(map); char *map_addr = map->addr; size_t map_size = map->size; - if (MAX(addr, map_addr) < MIN(addr + size, map_addr + PGUP(map_size))) { - if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) { - // remove mapping completely - dll_remove(&__maps.used, cur); - dll_make_first(&delete, cur); - __maps.pages -= (map_size + pagesz - 1) / pagesz; - __maps.count -= 1; - __maps_check(); - } else if (IsWindows()) { - // you can't carve up memory maps on windows. our mmap() makes - // this not a problem (for non-enormous memory maps) by making - // independent mappings for each 64 kb granule, under the hood - rc = einval(); - } else if (addr <= map_addr) { - // shave off lefthand side of mapping - ASSERT(addr + size < map_addr + PGUP(map_size)); - size_t left = PGUP(addr + size - map_addr); - size_t right = map_size - left; - ASSERT(right > 0); - ASSERT(left > 0); - struct Map *leftmap; - if ((leftmap = __maps_alloc())) { - map->addr += left; - map->size = right; - if (!(map->flags & MAP_ANONYMOUS)) - map->off += left; - __maps.pages -= (left + pagesz - 1) / pagesz; + if (!(MAX(addr, map_addr) < MIN(addr + size, map_addr + PGUP(map_size)))) + break; + if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) { + // remove mapping completely + tree_remove(&__maps.maps, &map->tree); + dll_init(&map->free); + dll_make_first(&deleted, &map->free); + __maps.pages -= (map_size + pagesz - 1) / pagesz; + __maps.count -= 1; + } else if (IsWindows()) { + // you can't carve up memory maps on windows. our mmap() makes + // this not a problem (for non-enormous memory maps) by making + // independent mappings for each 64 kb granule, under the hood + rc = einval(); + } else if (addr <= map_addr) { + // shave off lefthand side of mapping + ASSERT(addr + size < map_addr + PGUP(map_size)); + size_t left = PGUP(addr + size - map_addr); + size_t right = map_size - left; + ASSERT(right > 0); + ASSERT(left > 0); + struct Map *leftmap; + if ((leftmap = __maps_alloc())) { + map->addr += left; + map->size = right; + if (!(map->flags & MAP_ANONYMOUS)) + map->off += left; + __maps.pages -= (left + pagesz - 1) / pagesz; + leftmap->addr = map_addr; + leftmap->size = left; + dll_init(&leftmap->free); + dll_make_first(&deleted, &leftmap->free); + } else { + rc = -1; + } + } else if (addr + size >= map_addr + PGUP(map_size)) { + // shave off righthand side of mapping + size_t left = addr - map_addr; + size_t right = map_addr + map_size - addr; + struct Map *rightmap; + if ((rightmap = __maps_alloc())) { + map->size = left; + __maps.pages -= (right + pagesz - 1) / pagesz; + rightmap->addr = addr; + rightmap->size = right; + dll_init(&rightmap->free); + dll_make_first(&deleted, &rightmap->free); + } else { + rc = -1; + } + } else { + // punch hole in mapping + size_t left = addr - map_addr; + size_t middle = size; + size_t right = map_size - middle - left; + struct Map *leftmap; + if ((leftmap = __maps_alloc())) { + struct Map *middlemap; + if ((middlemap = __maps_alloc())) { leftmap->addr = map_addr; leftmap->size = left; - dll_make_first(&delete, &leftmap->elem); - __maps_check(); - } else { - rc = -1; - } - } else if (addr + size >= map_addr + PGUP(map_size)) { - // shave off righthand side of mapping - size_t left = addr - map_addr; - size_t right = map_addr + map_size - addr; - struct Map *rightmap; - if ((rightmap = __maps_alloc())) { - map->size = left; - __maps.pages -= (right + pagesz - 1) / pagesz; - rightmap->addr = addr; - rightmap->size = right; - dll_make_first(&delete, &rightmap->elem); - __maps_check(); + leftmap->off = map->off; + leftmap->prot = map->prot; + leftmap->flags = map->flags; + map->addr += left + middle; + map->size = right; + if (!(map->flags & MAP_ANONYMOUS)) + map->off += left + middle; + tree_insert(&__maps.maps, &leftmap->tree, __maps_compare); + __maps.pages -= (middle + pagesz - 1) / pagesz; + __maps.count += 1; + middlemap->addr = addr; + middlemap->size = size; + dll_init(&middlemap->free); + dll_make_first(&deleted, &middlemap->free); } else { rc = -1; } } else { - // punch hole in mapping - size_t left = addr - map_addr; - size_t middle = size; - size_t right = map_size - middle - left; - struct Map *leftmap; - if ((leftmap = __maps_alloc())) { - struct Map *middlemap; - if ((middlemap = __maps_alloc())) { - leftmap->addr = map_addr; - leftmap->size = left; - leftmap->off = map->off; - leftmap->prot = map->prot; - leftmap->flags = map->flags; - map->addr += left + middle; - map->size = right; - if (!(map->flags & MAP_ANONYMOUS)) - map->off += left + middle; - dll_make_first(&__maps.used, &leftmap->elem); - __maps.pages -= (middle + pagesz - 1) / pagesz; - __maps.count += 1; - middlemap->addr = addr; - middlemap->size = size; - dll_make_first(&delete, &middlemap->elem); - __maps_check(); - } else { - rc = -1; - } - } else { - rc = -1; - } + rc = -1; } } + __maps_check(); } __maps_unlock(); // delete mappings - for (struct Dll *e = dll_first(delete); e; e = dll_next(delete, e)) { - struct Map *map = MAP_CONTAINER(e); + for (struct Dll *e = dll_first(deleted); e; e = dll_next(deleted, e)) { + struct Map *map = MAP_FREE_CONTAINER(e); if (!untrack_only) { if (!IsWindows()) { if (sys_munmap(map->addr, map->size)) @@ -305,12 +298,12 @@ static int __munmap(char *addr, size_t size, bool untrack_only) { } // free mappings - if (!dll_is_empty(delete)) { + if (!dll_is_empty(deleted)) { __maps_lock(); struct Dll *e; - while ((e = dll_first(delete))) { - dll_remove(&delete, e); - __maps_free(MAP_CONTAINER(e)); + while ((e = dll_first(deleted))) { + dll_remove(&deleted, e); + __maps_free(MAP_FREE_CONTAINER(e)); } __maps_check(); __maps_unlock(); @@ -350,6 +343,7 @@ static void *__mmap_chunk(void *addr, size_t size, int prot, int flags, int fd, __maps_unlock(); return (void *)edeadlk(); } + __maps_check(); map = __maps_alloc(); __maps_unlock(); if (!map) diff --git a/libc/intrin/mprotect.c b/libc/intrin/mprotect.c index 83dc39926..76a40175e 100644 --- a/libc/intrin/mprotect.c +++ b/libc/intrin/mprotect.c @@ -25,6 +25,7 @@ #include "libc/intrin/kprintf.h" #include "libc/intrin/maps.h" #include "libc/intrin/strace.internal.h" +#include "libc/intrin/tree.h" #include "libc/nt/memory.h" #include "libc/runtime/internal.h" #include "libc/runtime/runtime.h" @@ -51,7 +52,7 @@ static int __mprotect_chunk(char *addr, size_t size, int prot, bool iscow) { int __mprotect(char *addr, size_t size, int prot) { // unix checks prot before checking size - if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC)) + if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC | PROT_GUARD)) return einval(); // make new technology consistent with unix @@ -68,64 +69,97 @@ int __mprotect(char *addr, size_t size, int prot) { // change mappings int rc = 0; - struct Dll *cur; bool found = false; if (__maps_lock()) { __maps_unlock(); return edeadlk(); } - for (cur = dll_first(__maps.used); cur; cur = dll_next(__maps.used, cur)) { - struct Map *map = MAP_CONTAINER(cur); + for (struct Map *map = __maps_floor(addr); map; map = __maps_next(map)) { char *map_addr = map->addr; size_t map_size = map->size; char *beg = MAX(addr, map_addr); char *end = MIN(addr + size, map_addr + PGUP(map_size)); - if (beg < end) { - found = true; - if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) { - // change protection of entire mapping - if (!__mprotect_chunk(map_addr, map_size, prot, map->iscow)) { + if (beg >= end) + break; + found = true; + if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) { + // change protection of entire mapping + if (!__mprotect_chunk(map_addr, map_size, prot, map->iscow)) { + map->prot = prot; + } else { + rc = -1; + } + } else if (addr <= map_addr) { + // change lefthand side of mapping + size_t left = PGUP(addr + size - map_addr); + size_t right = map_size - left; + struct Map *leftmap; + if ((leftmap = __maps_alloc())) { + if (!__mprotect_chunk(map_addr, left, prot, false)) { + leftmap->addr = map_addr; + leftmap->size = left; + leftmap->prot = prot; + leftmap->off = map->off; + leftmap->flags = map->flags; + leftmap->iscow = map->iscow; + leftmap->readonlyfile = map->readonlyfile; + leftmap->hand = map->hand; + map->addr += left; + map->size = right; + map->hand = -1; + if (!(map->flags & MAP_ANONYMOUS)) + map->off += left; + tree_insert(&__maps.maps, &leftmap->tree, __maps_compare); + __maps.count += 1; + __maps_check(); + } else { + __maps_free(leftmap); + rc = -1; + } + } else { + rc = -1; + } + } else if (addr + size >= map_addr + PGUP(map_size)) { + // change righthand side of mapping + size_t left = addr - map_addr; + size_t right = map_addr + map_size - addr; + struct Map *leftmap; + if ((leftmap = __maps_alloc())) { + if (!__mprotect_chunk(map_addr + left, right, prot, false)) { + leftmap->addr = map_addr; + leftmap->size = left; + leftmap->off = map->off; + leftmap->prot = map->prot; + leftmap->flags = map->flags; + leftmap->iscow = map->iscow; + leftmap->readonlyfile = map->readonlyfile; + leftmap->hand = map->hand; + map->addr += left; + map->size = right; map->prot = prot; + map->hand = -1; + if (!(map->flags & MAP_ANONYMOUS)) + map->off += left; + tree_insert(&__maps.maps, &leftmap->tree, __maps_compare); + __maps.count += 1; + __maps_check(); } else { + __maps_free(leftmap); rc = -1; } - } else if (addr <= map_addr) { - // change lefthand side of mapping - size_t left = PGUP(addr + size - map_addr); - size_t right = map_size - left; - struct Map *leftmap; - if ((leftmap = __maps_alloc())) { - if (!__mprotect_chunk(map_addr, left, prot, false)) { - leftmap->addr = map_addr; - leftmap->size = left; - leftmap->prot = prot; - leftmap->off = map->off; - leftmap->flags = map->flags; - leftmap->iscow = map->iscow; - leftmap->readonlyfile = map->readonlyfile; - leftmap->hand = map->hand; - map->addr += left; - map->size = right; - map->hand = -1; - if (!(map->flags & MAP_ANONYMOUS)) - map->off += left; - dll_make_first(&__maps.used, &leftmap->elem); - __maps.count += 1; - __maps_check(); - } else { - __maps_free(leftmap); - rc = -1; - } - } else { - rc = -1; - } - } else if (addr + size >= map_addr + PGUP(map_size)) { - // change righthand side of mapping - size_t left = addr - map_addr; - size_t right = map_addr + map_size - addr; - struct Map *leftmap; - if ((leftmap = __maps_alloc())) { - if (!__mprotect_chunk(map_addr + left, right, prot, false)) { + } else { + rc = -1; + } + } else { + // change middle of mapping + size_t left = addr - map_addr; + size_t middle = size; + size_t right = map_size - middle - left; + struct Map *leftmap; + if ((leftmap = __maps_alloc())) { + struct Map *midlmap; + if ((midlmap = __maps_alloc())) { + if (!__mprotect_chunk(map_addr + left, middle, prot, false)) { leftmap->addr = map_addr; leftmap->size = left; leftmap->off = map->off; @@ -134,67 +168,32 @@ int __mprotect(char *addr, size_t size, int prot) { leftmap->iscow = map->iscow; leftmap->readonlyfile = map->readonlyfile; leftmap->hand = map->hand; - map->addr += left; + midlmap->addr = map_addr + left; + midlmap->size = middle; + midlmap->off = (map->flags & MAP_ANONYMOUS) ? 0 : map->off + left; + midlmap->prot = prot; + midlmap->flags = map->flags; + midlmap->hand = -1; + map->addr += left + middle; map->size = right; - map->prot = prot; map->hand = -1; if (!(map->flags & MAP_ANONYMOUS)) - map->off += left; - dll_make_first(&__maps.used, &leftmap->elem); - __maps.count += 1; + map->off += left + middle; + tree_insert(&__maps.maps, &leftmap->tree, __maps_compare); + tree_insert(&__maps.maps, &midlmap->tree, __maps_compare); + __maps.count += 2; __maps_check(); } else { + __maps_free(midlmap); __maps_free(leftmap); rc = -1; } } else { + __maps_free(leftmap); rc = -1; } } else { - // change middle of mapping - size_t left = addr - map_addr; - size_t middle = size; - size_t right = map_size - middle - left; - struct Map *leftmap; - if ((leftmap = __maps_alloc())) { - struct Map *midlmap; - if ((midlmap = __maps_alloc())) { - if (!__mprotect_chunk(map_addr + left, middle, prot, false)) { - leftmap->addr = map_addr; - leftmap->size = left; - leftmap->off = map->off; - leftmap->prot = map->prot; - leftmap->flags = map->flags; - leftmap->iscow = map->iscow; - leftmap->readonlyfile = map->readonlyfile; - leftmap->hand = map->hand; - midlmap->addr = map_addr + left; - midlmap->size = middle; - midlmap->off = (map->flags & MAP_ANONYMOUS) ? 0 : map->off + left; - midlmap->prot = prot; - midlmap->flags = map->flags; - midlmap->hand = -1; - map->addr += left + middle; - map->size = right; - map->hand = -1; - if (!(map->flags & MAP_ANONYMOUS)) - map->off += left + middle; - dll_make_first(&__maps.used, &midlmap->elem); - dll_make_first(&__maps.used, &leftmap->elem); - __maps.count += 2; - __maps_check(); - } else { - __maps_free(midlmap); - __maps_free(leftmap); - rc = -1; - } - } else { - __maps_free(leftmap); - rc = -1; - } - } else { - rc = -1; - } + rc = -1; } } } diff --git a/libc/intrin/msync-nt.c b/libc/intrin/msync-nt.c index 3ff6524ed..883a11c06 100644 --- a/libc/intrin/msync-nt.c +++ b/libc/intrin/msync-nt.c @@ -34,16 +34,17 @@ textwindows int sys_msync_nt(char *addr, size_t size, int flags) { return einval(); int rc = 0; - __maps_lock(); - for (struct Dll *e = dll_first(__maps.used); e; - e = dll_next(__maps.used, e)) { - struct Map *map = MAP_CONTAINER(e); - char *beg = MAX(addr, map->addr); - char *end = MIN(addr + size, map->addr + map->size); - if (beg < end) - if (!FlushViewOfFile(beg, end - beg)) - rc = -1; - // TODO(jart): FlushFileBuffers too on g_fds handle if MS_SYNC? + if (__maps_lock()) { + rc = edeadlk(); + } else { + for (struct Map *map = __maps_floor(addr); map; map = __maps_next(map)) { + char *beg = MAX(addr, map->addr); + char *end = MIN(addr + size, map->addr + map->size); + if (beg < end) + if (!FlushViewOfFile(beg, end - beg)) + rc = -1; + // TODO(jart): FlushFileBuffers too on g_fds handle if MS_SYNC? + } } __maps_unlock(); diff --git a/libc/intrin/printmaps.c b/libc/intrin/printmaps.c index b84a1cc25..18c02812e 100644 --- a/libc/intrin/printmaps.c +++ b/libc/intrin/printmaps.c @@ -29,16 +29,11 @@ /** * Prints memory mappings. */ -void __print_maps(void) { - int limit = 15; - long maptally = 0; +void __print_maps(size_t limit) { char mappingbuf[8], sb[16]; __maps_lock(); - struct Dll *e, *e2; - for (e = dll_first(__maps.used); e; e = e2) { - e2 = dll_next(__maps.used, e); - struct Map *map = MAP_CONTAINER(e); - maptally += map->size; + for (struct Tree *e = tree_first(__maps.maps); e; e = tree_next(e)) { + struct Map *map = MAP_TREE_CONTAINER(e); kprintf("%012lx-%012lx %!s", map->addr, map->addr + map->size, (DescribeMapping)(mappingbuf, map->prot, map->flags)); sizefmt(sb, map->size, 1024); diff --git a/libc/intrin/prot2nt.greg.c b/libc/intrin/prot2nt.greg.c index 67a59430b..59dd01491 100644 --- a/libc/intrin/prot2nt.greg.c +++ b/libc/intrin/prot2nt.greg.c @@ -42,6 +42,8 @@ privileged int __prot2nt(int prot, int iscow) { return kNtPageExecuteReadwrite; } default: + if (prot & PROT_GUARD) + return kNtPageReadwrite | kNtPageGuard; return kNtPageNoaccess; } } diff --git a/libc/intrin/pthread_delay_np.c b/libc/intrin/pthread_delay_np.c new file mode 100644 index 000000000..3a7c98024 --- /dev/null +++ b/libc/intrin/pthread_delay_np.c @@ -0,0 +1,40 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/intrin/strace.internal.h" +#include "libc/thread/thread.h" + +/** + * Delays execution for brief moment. + * + * @param symbol may be used to strace names of static locks + * @param backoff should start at zero and be feed back in + * @return new value for backoff + */ +int pthread_delay_np(const void *symbol, int backoff) { + if (backoff < 7) { + volatile int i; + for (i = 0; i != 1 << backoff; i++) { + } + backoff++; + } else { + STRACE("pthread_delay_np(%t)", symbol); + pthread_yield_np(); + } + return backoff; +} diff --git a/libc/intrin/pthread_mutex_lock.c b/libc/intrin/pthread_mutex_lock.c index 6fd2c6bfb..509af6e04 100644 --- a/libc/intrin/pthread_mutex_lock.c +++ b/libc/intrin/pthread_mutex_lock.c @@ -18,8 +18,10 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" #include "libc/calls/state.internal.h" +#include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" +#include "libc/intrin/describeflags.internal.h" #include "libc/intrin/strace.internal.h" #include "libc/intrin/weaken.h" #include "libc/runtime/internal.h" @@ -27,6 +29,66 @@ #include "libc/thread/thread.h" #include "third_party/nsync/mu.h" +static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex) { + int me; + int backoff = 0; + uint64_t word, lock; + + // get current state of lock + word = atomic_load_explicit(&mutex->_word, memory_order_relaxed); + + // use fancy nsync mutex if possible + if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL && // + MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && // + _weaken(nsync_mu_lock)) { + _weaken(nsync_mu_lock)((nsync_mu *)mutex); + return 0; + } + + // implement barebones normal mutexes + if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) { + for (;;) { + word = MUTEX_UNLOCK(word); + lock = MUTEX_LOCK(word); + if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock, + memory_order_acquire, + memory_order_relaxed)) + return 0; + backoff = pthread_delay_np(mutex, backoff); + } + } + + // implement recursive mutexes + me = gettid(); + for (;;) { + if (MUTEX_OWNER(word) == me) { + if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) { + if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) { + if (atomic_compare_exchange_weak_explicit( + &mutex->_word, &word, MUTEX_INC_DEPTH(word), + memory_order_relaxed, memory_order_relaxed)) + return 0; + continue; + } else { + return EAGAIN; + } + } else { + return EDEADLK; + } + } + word = MUTEX_UNLOCK(word); + lock = MUTEX_LOCK(word); + lock = MUTEX_SET_OWNER(lock, me); + if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock, + memory_order_acquire, + memory_order_relaxed)) { + mutex->_pid = __pid; + return 0; + } + backoff = pthread_delay_np(mutex, backoff); + } +} + /** * Locks mutex. * @@ -65,65 +127,10 @@ * @vforksafe */ errno_t pthread_mutex_lock(pthread_mutex_t *mutex) { - int me; - uint64_t word, lock; - - LOCKTRACE("pthread_mutex_lock(%t)", mutex); - if (__vforked) return 0; - - // get current state of lock - word = atomic_load_explicit(&mutex->_word, memory_order_relaxed); - - // use fancy nsync mutex if possible - if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL && // - MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && // - _weaken(nsync_mu_lock)) { - _weaken(nsync_mu_lock)((nsync_mu *)mutex); - return 0; - } - - // implement barebones normal mutexes - if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) { - for (;;) { - word = MUTEX_UNLOCK(word); - lock = MUTEX_LOCK(word); - if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock, - memory_order_acquire, - memory_order_relaxed)) - return 0; - pthread_pause_np(); - } - } - - // implement recursive mutexes - me = gettid(); - for (;;) { - if (MUTEX_OWNER(word) == me) { - if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) { - if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) { - if (atomic_compare_exchange_weak_explicit( - &mutex->_word, &word, MUTEX_INC_DEPTH(word), - memory_order_relaxed, memory_order_relaxed)) - return 0; - continue; - } else { - return EAGAIN; - } - } else { - return EDEADLK; - } - } - word = MUTEX_UNLOCK(word); - lock = MUTEX_LOCK(word); - lock = MUTEX_SET_OWNER(lock, me); - if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock, - memory_order_acquire, - memory_order_relaxed)) { - mutex->_pid = __pid; - return 0; - } - pthread_pause_np(); - } + LOCKTRACE("acquiring %t...", mutex); + errno_t err = pthread_mutex_lock_impl(mutex); + LOCKTRACE("pthread_mutex_lock(%t) → %s", mutex, DescribeErrno(err)); + return err; } diff --git a/libc/intrin/pthread_mutex_trylock.c b/libc/intrin/pthread_mutex_trylock.c index f4e0dde6a..21513656a 100644 --- a/libc/intrin/pthread_mutex_trylock.c +++ b/libc/intrin/pthread_mutex_trylock.c @@ -17,6 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" +#include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" #include "libc/intrin/weaken.h" diff --git a/libc/intrin/pthread_mutex_unlock.c b/libc/intrin/pthread_mutex_unlock.c index c6fc274b1..dc8d0fb1f 100644 --- a/libc/intrin/pthread_mutex_unlock.c +++ b/libc/intrin/pthread_mutex_unlock.c @@ -17,6 +17,7 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" +#include "libc/dce.h" #include "libc/errno.h" #include "libc/intrin/atomic.h" #include "libc/intrin/strace.internal.h" diff --git a/libc/intrin/pthread_syshand.c b/libc/intrin/pthread_syshand.c index 856ce3323..b15460580 100644 --- a/libc/intrin/pthread_syshand.c +++ b/libc/intrin/pthread_syshand.c @@ -29,6 +29,6 @@ intptr_t _pthread_syshand(struct PosixThread *pt) { syshand = atomic_load_explicit(&pt->tib->tib_syshand, memory_order_acquire); if (syshand) return syshand; - pthread_pause_np(); + pthread_yield_np(); } } diff --git a/libc/intrin/pthread_tid.c b/libc/intrin/pthread_tid.c index 27e3ee5a1..4f7553e9a 100644 --- a/libc/intrin/pthread_tid.c +++ b/libc/intrin/pthread_tid.c @@ -24,6 +24,6 @@ int _pthread_tid(struct PosixThread *pt) { int tid = 0; while (pt && !(tid = atomic_load_explicit(&pt->ptid, memory_order_acquire))) - pthread_pause_np(); + pthread_yield_np(); return tid; } diff --git a/libc/calls/pthread_yield_np.c b/libc/intrin/pthread_yield_np.c similarity index 100% rename from libc/calls/pthread_yield_np.c rename to libc/intrin/pthread_yield_np.c diff --git a/libc/intrin/rbtree.c b/libc/intrin/rbtree.c deleted file mode 100644 index 3fc3ae5cc..000000000 --- a/libc/intrin/rbtree.c +++ /dev/null @@ -1,312 +0,0 @@ -// Copyright 2024 Justine Alexandra Roberts Tunney -// -// Permission to use, copy, modify, and/or distribute this software for -// any purpose with or without fee is hereby granted, provided that the -// above copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL -// WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE -// AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL -// DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR -// PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -// PERFORMANCE OF THIS SOFTWARE. - -#include "rbtree.h" -#include "libc/dce.h" -#include "libc/str/str.h" - -#define RBTREE_DEBUG - -struct rbtree *rbtree_next(struct rbtree *node) { - if (!node) - return 0; - if (node->right) - return rbtree_first(node->right); - struct rbtree *parent = node->parent; - while (parent && node == parent->right) { - node = parent; - parent = parent->parent; - } - return parent; -} - -struct rbtree *rbtree_prev(struct rbtree *node) { - if (!node) - return 0; - if (rbtree_get_left(node)) - return rbtree_last(rbtree_get_left(node)); - struct rbtree *parent = node->parent; - while (parent && node == rbtree_get_left(parent)) { - node = parent; - parent = parent->parent; - } - return parent; -} - -struct rbtree *rbtree_first(struct rbtree *node) { - while (node && rbtree_get_left(node)) - node = rbtree_get_left(node); - return node; -} - -struct rbtree *rbtree_last(struct rbtree *node) { - while (node && node->right) - node = node->right; - return node; -} - -static void rbtree_rotate_left(struct rbtree **root, struct rbtree *x) { - struct rbtree *y = x->right; - x->right = rbtree_get_left(y); - if (rbtree_get_left(y)) - rbtree_get_left(y)->parent = x; - y->parent = x->parent; - if (!x->parent) { - *root = y; - } else if (x == rbtree_get_left(x->parent)) { - rbtree_set_left(x->parent, y); - } else { - x->parent->right = y; - } - rbtree_set_left(y, x); - x->parent = y; -} - -static void rbtree_rotate_right(struct rbtree **root, struct rbtree *y) { - struct rbtree *x = rbtree_get_left(y); - rbtree_set_left(y, x->right); - if (x->right) - x->right->parent = y; - x->parent = y->parent; - if (!y->parent) { - *root = x; - } else if (y == y->parent->right) { - y->parent->right = x; - } else { - rbtree_set_left(y->parent, x); - } - x->right = y; - y->parent = x; -} - -static void rbtree_insert_fixup(struct rbtree **root, struct rbtree *node) { - rbtree_set_red(node, 1); - while (node != *root && rbtree_get_red(node->parent)) { - if (node->parent == rbtree_get_left(node->parent->parent)) { - struct rbtree *uncle = node->parent->parent->right; - if (uncle && rbtree_get_red(uncle)) { - rbtree_set_red(node->parent, 0); - rbtree_set_red(uncle, 0); - rbtree_set_red(node->parent->parent, 1); - node = node->parent->parent; - } else { - if (node == node->parent->right) { - node = node->parent; - rbtree_rotate_left(root, node); - } - rbtree_set_red(node->parent, 0); - rbtree_set_red(node->parent->parent, 1); - rbtree_rotate_right(root, node->parent->parent); - } - } else { - struct rbtree *uncle = rbtree_get_left(node->parent->parent); - if (uncle && rbtree_get_red(uncle)) { - rbtree_set_red(node->parent, 0); - rbtree_set_red(uncle, 0); - rbtree_set_red(node->parent->parent, 1); - node = node->parent->parent; - } else { - if (node == rbtree_get_left(node->parent)) { - node = node->parent; - rbtree_rotate_right(root, node); - } - rbtree_set_red(node->parent, 0); - rbtree_set_red(node->parent->parent, 1); - rbtree_rotate_left(root, node->parent->parent); - } - } - } - rbtree_set_red(*root, 0); -} - -void rbtree_insert(struct rbtree **root, struct rbtree *node, - rbtree_cmp_f *cmp) { - bzero(node, sizeof(*node)); - if (!*root) { - *root = node; - } else { - struct rbtree *search = *root; - struct rbtree *parent = 0; - do { - parent = search; - if (cmp(node, search) < 0) { - search = rbtree_get_left(search); - } else { - search = search->right; - } - } while (search); - if (cmp(node, parent) < 0) { - rbtree_set_left(parent, node); - } else { - parent->right = node; - } - node->parent = parent; - rbtree_insert_fixup(root, node); - } -} - -static void rbtree_transplant(struct rbtree **root, struct rbtree *u, - struct rbtree *v) { - if (!u->parent) { - *root = v; - } else if (u == rbtree_get_left(u->parent)) { - rbtree_set_left(u->parent, v); - } else { - u->parent->right = v; - } - if (v) - v->parent = u->parent; -} - -static void rbtree_remove_fixup(struct rbtree **root, struct rbtree *node, - struct rbtree *parent) { - while (node != *root && (!node || !rbtree_get_red(node))) { - if (node == rbtree_get_left(parent)) { - struct rbtree *sibling = parent->right; - if (rbtree_get_red(sibling)) { - rbtree_set_red(sibling, 0); - rbtree_set_red(parent, 1); - rbtree_rotate_left(root, parent); - sibling = parent->right; - } - if ((!rbtree_get_left(sibling) || - !rbtree_get_red(rbtree_get_left(sibling))) && - (!sibling->right || !rbtree_get_red(sibling->right))) { - rbtree_set_red(sibling, 1); - node = parent; - parent = node->parent; - } else { - if (!sibling->right || !rbtree_get_red(sibling->right)) { - rbtree_set_red(rbtree_get_left(sibling), 0); - rbtree_set_red(sibling, 1); - rbtree_rotate_right(root, sibling); - sibling = parent->right; - } - rbtree_set_red(sibling, rbtree_get_red(parent)); - rbtree_set_red(parent, 0); - rbtree_set_red(sibling->right, 0); - rbtree_rotate_left(root, parent); - node = *root; - break; - } - } else { - struct rbtree *sibling = rbtree_get_left(parent); - if (rbtree_get_red(sibling)) { - rbtree_set_red(sibling, 0); - rbtree_set_red(parent, 1); - rbtree_rotate_right(root, parent); - sibling = rbtree_get_left(parent); - } - if ((!sibling->right || !rbtree_get_red(sibling->right)) && - (!rbtree_get_left(sibling) || - !rbtree_get_red(rbtree_get_left(sibling)))) { - rbtree_set_red(sibling, 1); - node = parent; - parent = node->parent; - } else { - if (!rbtree_get_left(sibling) || - !rbtree_get_red(rbtree_get_left(sibling))) { - rbtree_set_red(sibling->right, 0); - rbtree_set_red(sibling, 1); - rbtree_rotate_left(root, sibling); - sibling = rbtree_get_left(parent); - } - rbtree_set_red(sibling, rbtree_get_red(parent)); - rbtree_set_red(parent, 0); - rbtree_set_red(rbtree_get_left(sibling), 0); - rbtree_rotate_right(root, parent); - node = *root; - break; - } - } - } - if (node) - rbtree_set_red(node, 0); -} - -void rbtree_remove(struct rbtree **root, struct rbtree *node) { - struct rbtree *y = node; - struct rbtree *x = 0; - struct rbtree *x_parent = 0; - int y_original_color = rbtree_get_red(y); - if (!rbtree_get_left(node)) { - x = node->right; - rbtree_transplant(root, node, node->right); - x_parent = node->parent; - } else if (!node->right) { - x = rbtree_get_left(node); - rbtree_transplant(root, node, rbtree_get_left(node)); - x_parent = node->parent; - } else { - y = rbtree_first(node->right); - y_original_color = rbtree_get_red(y); - x = y->right; - if (y->parent == node) { - if (x) - x->parent = y; - x_parent = y; - } else { - rbtree_transplant(root, y, y->right); - y->right = node->right; - y->right->parent = y; - x_parent = y->parent; - } - rbtree_transplant(root, node, y); - rbtree_set_left(y, rbtree_get_left(node)); - rbtree_get_left(y)->parent = y; - rbtree_set_red(y, rbtree_get_red(node)); - } - if (!y_original_color) - rbtree_remove_fixup(root, x, x_parent); -} - -struct rbtree *rbtree_get(const struct rbtree *node, const struct rbtree *key, - rbtree_cmp_f *cmp) { - while (node) { - int c = cmp(key, node); - if (c < 0) { - node = rbtree_get_left(node); - } else if (c > 0) { - node = node->right; - } else { - return (struct rbtree *)node; - } - } - return 0; -} - -struct rbtree *rbtree_floor(const struct rbtree *node, const struct rbtree *key, - rbtree_cmp_f *cmp) { - while (node) { - if (cmp(key, node) < 0) { - node = rbtree_get_left(node); - } else { - node = node->right; - } - } - return (struct rbtree *)node; -} - -struct rbtree *rbtree_ceil(const struct rbtree *node, const struct rbtree *key, - rbtree_cmp_f *cmp) { - while (node) { - if (cmp(node, key) < 0) { - node = rbtree_get_left(node); - } else { - node = node->right; - } - } - return (struct rbtree *)node; -} diff --git a/libc/intrin/rbtree.h b/libc/intrin/rbtree.h deleted file mode 100644 index 0c63f67dd..000000000 --- a/libc/intrin/rbtree.h +++ /dev/null @@ -1,57 +0,0 @@ -#ifdef _COSMO_SOURCE -#ifndef COSMOPOLITAN_RBTREE_H_ -#define COSMOPOLITAN_RBTREE_H_ -#define rbtree_ceil __rbtree_ceil -#define rbtree_first __rbtree_first -#define rbtree_floor __rbtree_floor -#define rbtree_get __rbtree_get -#define rbtree_insert __rbtree_insert -#define rbtree_last __rbtree_last -#define rbtree_next __rbtree_next -#define rbtree_prev __rbtree_prev -#define rbtree_remove __rbtree_remove -COSMOPOLITAN_C_START_ - -#define RBTREE_CONTAINER(t, f, p) ((t *)(((char *)(p)) - offsetof(t, f))) - -struct rbtree { - uintptr_t word; - struct rbtree *right; - struct rbtree *parent; -}; - -typedef int rbtree_cmp_f(const struct rbtree *, const struct rbtree *); - -static inline struct rbtree *rbtree_get_left(const struct rbtree *node) { - return (struct rbtree *)(node->word & -2); -} - -static inline void rbtree_set_left(struct rbtree *node, struct rbtree *left) { - node->word = (uintptr_t)left | (node->word & 1); -} - -static inline int rbtree_get_red(const struct rbtree *node) { - return node->word & 1; -} - -static inline void rbtree_set_red(struct rbtree *node, int red) { - node->word &= -2; - node->word |= red; -} - -struct rbtree *rbtree_next(struct rbtree *) libcesque; -struct rbtree *rbtree_prev(struct rbtree *) libcesque; -struct rbtree *rbtree_first(struct rbtree *) libcesque; -struct rbtree *rbtree_last(struct rbtree *) libcesque; -void rbtree_remove(struct rbtree **, struct rbtree *) libcesque; -void rbtree_insert(struct rbtree **, struct rbtree *, rbtree_cmp_f *) libcesque; -struct rbtree *rbtree_get(const struct rbtree *, const struct rbtree *, - rbtree_cmp_f *) libcesque; -struct rbtree *rbtree_ceil(const struct rbtree *, const struct rbtree *, - rbtree_cmp_f *) libcesque; -struct rbtree *rbtree_floor(const struct rbtree *, const struct rbtree *, - rbtree_cmp_f *) libcesque; - -COSMOPOLITAN_C_END_ -#endif /* COSMOPOLITAN_RBTREE_H_ */ -#endif /* _COSMO_SOURCE */ diff --git a/libc/intrin/tree.c b/libc/intrin/tree.c new file mode 100644 index 000000000..23e25f7f5 --- /dev/null +++ b/libc/intrin/tree.c @@ -0,0 +1,271 @@ +// Copyright 2024 Justine Alexandra Roberts Tunney +// +// Permission to use, copy, modify, and/or distribute this software for +// any purpose with or without fee is hereby granted, provided that the +// above copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL +// WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE +// AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL +// DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR +// PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +#include "tree.h" + +struct Tree *tree_last(struct Tree *node) { + while (node && node->right) + node = node->right; + return node; +} + +struct Tree *tree_first(struct Tree *node) { + while (node && tree_get_left(node)) + node = tree_get_left(node); + return node; +} + +struct Tree *tree_next(struct Tree *node) { + if (!node) + return 0; + if (node->right) + return tree_first(node->right); + struct Tree *parent = node->parent; + while (parent && node == parent->right) { + node = parent; + parent = parent->parent; + } + return parent; +} + +struct Tree *tree_prev(struct Tree *node) { + struct Tree *parent; + if (!node) + return 0; + if (tree_get_left(node)) + return tree_last(tree_get_left(node)); + parent = node->parent; + while (parent && node == tree_get_left(parent)) { + node = parent; + parent = parent->parent; + } + return parent; +} + +static void tree_rotate_left(struct Tree **root, struct Tree *x) { + struct Tree *y = x->right; + x->right = tree_get_left(y); + if (tree_get_left(y)) + tree_get_left(y)->parent = x; + y->parent = x->parent; + if (!x->parent) { + *root = y; + } else if (x == tree_get_left(x->parent)) { + tree_set_left(x->parent, y); + } else { + x->parent->right = y; + } + tree_set_left(y, x); + x->parent = y; +} + +static void tree_rotate_right(struct Tree **root, struct Tree *y) { + struct Tree *x = tree_get_left(y); + tree_set_left(y, x->right); + if (x->right) + x->right->parent = y; + x->parent = y->parent; + if (!y->parent) { + *root = x; + } else if (y == y->parent->right) { + y->parent->right = x; + } else { + tree_set_left(y->parent, x); + } + y->parent = x; + x->right = y; +} + +static void tree_rebalance_insert(struct Tree **root, struct Tree *node) { + struct Tree *uncle; + tree_set_red(node, 1); + while (node != *root && tree_get_red(node->parent)) { + if (node->parent == tree_get_left(node->parent->parent)) { + uncle = node->parent->parent->right; + if (uncle && tree_get_red(uncle)) { + tree_set_red(node->parent, 0); + tree_set_red(uncle, 0); + tree_set_red(node->parent->parent, 1); + node = node->parent->parent; + } else { + if (node == node->parent->right) { + node = node->parent; + tree_rotate_left(root, node); + } + tree_set_red(node->parent, 0); + tree_set_red(node->parent->parent, 1); + tree_rotate_right(root, node->parent->parent); + } + } else { + uncle = tree_get_left(node->parent->parent); + if (uncle && tree_get_red(uncle)) { + tree_set_red(node->parent, 0); + tree_set_red(uncle, 0); + tree_set_red(node->parent->parent, 1); + node = node->parent->parent; + } else { + if (node == tree_get_left(node->parent)) { + node = node->parent; + tree_rotate_right(root, node); + } + tree_set_red(node->parent, 0); + tree_set_red(node->parent->parent, 1); + tree_rotate_left(root, node->parent->parent); + } + } + } + tree_set_red(*root, 0); +} + +void tree_insert(struct Tree **root, struct Tree *node, tree_cmp_f *cmp) { + struct Tree *search, *parent; + node->word = 0; + node->right = 0; + node->parent = 0; + if (!*root) { + *root = node; + } else { + search = *root; + parent = 0; + do { + parent = search; + if (cmp(node, search) < 0) { + search = tree_get_left(search); + } else { + search = search->right; + } + } while (search); + if (cmp(node, parent) < 0) { + tree_set_left(parent, node); + } else { + parent->right = node; + } + node->parent = parent; + tree_rebalance_insert(root, node); + } +} + +static void tree_transplant(struct Tree **root, struct Tree *u, + struct Tree *v) { + if (!u->parent) { + *root = v; + } else if (u == tree_get_left(u->parent)) { + tree_set_left(u->parent, v); + } else { + u->parent->right = v; + } + if (v) + v->parent = u->parent; +} + +static void tree_rebalance_remove(struct Tree **root, struct Tree *node, + struct Tree *parent) { + struct Tree *sibling; + while (node != *root && (!node || !tree_get_red(node))) { + if (node == tree_get_left(parent)) { + sibling = parent->right; + if (tree_get_red(sibling)) { + tree_set_red(sibling, 0); + tree_set_red(parent, 1); + tree_rotate_left(root, parent); + sibling = parent->right; + } + if ((!tree_get_left(sibling) || !tree_get_red(tree_get_left(sibling))) && + (!sibling->right || !tree_get_red(sibling->right))) { + tree_set_red(sibling, 1); + node = parent; + parent = node->parent; + } else { + if (!sibling->right || !tree_get_red(sibling->right)) { + tree_set_red(tree_get_left(sibling), 0); + tree_set_red(sibling, 1); + tree_rotate_right(root, sibling); + sibling = parent->right; + } + tree_set_red(sibling, tree_get_red(parent)); + tree_set_red(parent, 0); + tree_set_red(sibling->right, 0); + tree_rotate_left(root, parent); + node = *root; + break; + } + } else { + sibling = tree_get_left(parent); + if (tree_get_red(sibling)) { + tree_set_red(sibling, 0); + tree_set_red(parent, 1); + tree_rotate_right(root, parent); + sibling = tree_get_left(parent); + } + if ((!sibling->right || !tree_get_red(sibling->right)) && + (!tree_get_left(sibling) || !tree_get_red(tree_get_left(sibling)))) { + tree_set_red(sibling, 1); + node = parent; + parent = node->parent; + } else { + if (!tree_get_left(sibling) || !tree_get_red(tree_get_left(sibling))) { + tree_set_red(sibling->right, 0); + tree_set_red(sibling, 1); + tree_rotate_left(root, sibling); + sibling = tree_get_left(parent); + } + tree_set_red(sibling, tree_get_red(parent)); + tree_set_red(parent, 0); + tree_set_red(tree_get_left(sibling), 0); + tree_rotate_right(root, parent); + node = *root; + break; + } + } + } + if (node) + tree_set_red(node, 0); +} + +void tree_remove(struct Tree **root, struct Tree *node) { + struct Tree *x = 0; + struct Tree *y = node; + struct Tree *x_parent = 0; + int y_original_color = tree_get_red(y); + if (!tree_get_left(node)) { + x = node->right; + tree_transplant(root, node, node->right); + x_parent = node->parent; + } else if (!node->right) { + x = tree_get_left(node); + tree_transplant(root, node, tree_get_left(node)); + x_parent = node->parent; + } else { + y = tree_first(node->right); + y_original_color = tree_get_red(y); + x = y->right; + if (y->parent == node) { + if (x) + x->parent = y; + x_parent = y; + } else { + tree_transplant(root, y, y->right); + y->right = node->right; + y->right->parent = y; + x_parent = y->parent; + } + tree_transplant(root, node, y); + tree_set_left(y, tree_get_left(node)); + tree_get_left(y)->parent = y; + tree_set_red(y, tree_get_red(node)); + } + if (!y_original_color) + tree_rebalance_remove(root, x, x_parent); +} diff --git a/libc/intrin/tree.h b/libc/intrin/tree.h new file mode 100644 index 000000000..fae2e08cb --- /dev/null +++ b/libc/intrin/tree.h @@ -0,0 +1,91 @@ +#ifndef COSMOPOLITAN_TREE_H_ +#define COSMOPOLITAN_TREE_H_ +#define tree_first __tree_first +#define tree_insert __tree_insert +#define tree_last __tree_last +#define tree_next __tree_next +#define tree_prev __tree_prev +#define tree_remove __tree_remove +COSMOPOLITAN_C_START_ + +#define TREE_CONTAINER(t, f, p) ((t *)(((char *)(p)) - offsetof(t, f))) + +struct Tree { + uintptr_t word; + struct Tree *right; + struct Tree *parent; +}; + +typedef int tree_search_f(const void *, const struct Tree *); +typedef int tree_cmp_f(const struct Tree *, const struct Tree *); + +forceinline struct Tree *tree_get_left(const struct Tree *node) { + return (struct Tree *)(node->word & -2); +} + +static inline void tree_set_left(struct Tree *node, struct Tree *left) { + node->word = (uintptr_t)left | (node->word & 1); +} + +static inline int tree_get_red(const struct Tree *node) { + return node->word & 1; +} + +static inline void tree_set_red(struct Tree *node, int red) { + node->word &= -2; + node->word |= red; +} + +forceinline optimizespeed struct Tree *tree_floor(const struct Tree *node, + const void *key, + tree_search_f *cmp) { + struct Tree *left = 0; + while (node) { + if (cmp(key, node) >= 0) { + left = (struct Tree *)node; + node = tree_get_left(node); + } else { + node = node->right; + } + } + return left; +} + +static inline struct Tree *tree_ceil(const struct Tree *node, const void *key, + tree_search_f *cmp) { + struct Tree *right = 0; + while (node) { + if (cmp(key, node) < 0) { + right = (struct Tree *)node; + node = tree_get_left(node); + } else { + node = node->right; + } + } + return right; +} + +static inline struct Tree *tree_get(const struct Tree *node, const void *key, + tree_search_f *cmp) { + while (node) { + int c = cmp(key, node); + if (c < 0) { + node = tree_get_left(node); + } else if (c > 0) { + node = node->right; + } else { + return (struct Tree *)node; + } + } + return 0; +} + +struct Tree *tree_next(struct Tree *) libcesque; +struct Tree *tree_prev(struct Tree *) libcesque; +struct Tree *tree_first(struct Tree *) libcesque; +struct Tree *tree_last(struct Tree *) libcesque; +void tree_remove(struct Tree **, struct Tree *) libcesque; +void tree_insert(struct Tree **, struct Tree *, tree_cmp_f *) libcesque; + +COSMOPOLITAN_C_END_ +#endif /* COSMOPOLITAN_TREE_H_ */ diff --git a/libc/log/oncrash_amd64.c b/libc/log/oncrash_amd64.c index 914e2aa44..f9df5f99b 100644 --- a/libc/log/oncrash_amd64.c +++ b/libc/log/oncrash_amd64.c @@ -241,14 +241,10 @@ static relegated void ShowCrashReport(int err, int sig, siginfo_t *si, klog(buf, p - buf); } kprintf("\n"); - if (!IsWindows()) { - __print_maps(); - } - if (__argv) { - for (i = 0; i < __argc; ++i) { + __print_maps(15); + if (__argv) + for (i = 0; i < __argc; ++i) kprintf("%s ", __argv[i]); - } - } kprintf("\n"); } diff --git a/libc/proc/fork-nt.c b/libc/proc/fork-nt.c index 6dc06bf3f..7e58445b6 100644 --- a/libc/proc/fork-nt.c +++ b/libc/proc/fork-nt.c @@ -30,6 +30,7 @@ #include "libc/intrin/kprintf.h" #include "libc/intrin/maps.h" #include "libc/intrin/strace.internal.h" +#include "libc/intrin/tree.h" #include "libc/intrin/weaken.h" #include "libc/macros.internal.h" #include "libc/nt/createfile.h" @@ -95,7 +96,7 @@ static inline textwindows ssize_t ForkIo(int64_t h, char *p, size_t n, size_t i; uint32_t x; for (i = 0; i < n; i += x) { - if (!f(h, p + i, n - i, &x, NULL)) + if (!f(h, p + i, n - i, &x, 0)) return __winerr(); if (!x) break; @@ -109,10 +110,11 @@ static dontinline textwindows ssize_t ForkIo2( const char *sf, bool ischild) { ssize_t rc = ForkIo(h, buf, n, fn); if (ischild) { - __tls_enabled_set(false); // prevent tls crash in kprintf + // prevent crashes + __tls_enabled_set(false); __pid = __imp_GetCurrentProcessId(); __klog_handle = 0; - __maps.used = 0; + __maps.maps = 0; } NTTRACE("%s(%ld, %p, %'zu) → %'zd% m", sf, h, buf, n, rc); return rc; @@ -121,9 +123,11 @@ static dontinline textwindows ssize_t ForkIo2( static dontinline textwindows bool WriteAll(int64_t h, void *buf, size_t n) { bool ok; ok = ForkIo2(h, buf, n, (void *)WriteFile, "WriteFile", false) != -1; - if (!ok) - AbortFork("WriteAll"); - // Sleep(10); + if (!ok) { + STRACE("fork() failed in parent due to WriteAll(%ld, %p, %'zu) → %u", h, + buf, n, GetLastError()); + __print_maps(0); + } return ok; } @@ -185,30 +189,6 @@ static textwindows void *Malloc(size_t size) { return HeapAlloc(GetProcessHeap(), 0, size); } -static textwindows void Free(void *addr) { - HeapFree(GetProcessHeap(), 0, addr); -} - -static int CountMaps(struct Dll *maps) { - int count = 0; - for (struct Dll *e = dll_first(maps); e; e = dll_next(maps, e)) - ++count; - return count; -} - -static struct Map **SortMaps(struct Dll *maps, int count) { - int j, i = 0; - struct Map **sorted = Malloc(count * sizeof(struct Map *)); - for (struct Dll *e = dll_first(maps); e; e = dll_next(maps, e)) { - struct Map *map = MAP_CONTAINER(e); - for (j = i; j > 0 && sorted[j - 1]->addr > map->addr; --j) - sorted[j] = sorted[j - 1]; - sorted[j] = map; - ++i; - } - return sorted; -} - textwindows void WinMainForked(void) { jmp_buf jb; int64_t reader; @@ -233,35 +213,30 @@ textwindows void WinMainForked(void) { ReadOrDie(reader, jb, sizeof(jb)); // read memory mappings from parent process - int n = 0; - struct Dll *maps = 0; + struct Tree *maps = 0; for (;;) { struct Map *map = Malloc(sizeof(struct Map)); ReadOrDie(reader, map, sizeof(struct Map)); - if (map->addr == MAP_FAILED) { - Free(map); + if (map->addr == MAP_FAILED) break; - } - dll_init(&map->elem); - dll_make_first(&maps, &map->elem); - ++n; + tree_insert(&maps, &map->tree, __maps_compare); } - // created sorted array of maps - struct Map **sorted = SortMaps(maps, n); - // map memory into process int granularity = __granularity(); - for (int i = 0; i < n; ++i) { - struct Map *map = sorted[i]; + for (struct Tree *e = tree_first(maps); e; e = tree_next(e)) { + struct Map *map = MAP_TREE_CONTAINER(e); if ((uintptr_t)map->addr & (granularity - 1)) continue; - size_t size = map->size; // get true length in case mprotect() chopped up actual win32 map - for (int j = i + 1; - j < n && sorted[j]->hand == -1 && map->addr + size == sorted[j]->addr; - ++j) { - size += sorted[j]->size; + size_t size = map->size; + for (struct Tree *e2 = tree_next(e); e2; e2 = tree_next(e2)) { + struct Map *map2 = MAP_TREE_CONTAINER(e2); + if (map2->hand == -1 && map->addr + size == map2->addr) { + size += map2->size; + } else { + break; + } } // obtain the most permissive access possible unsigned prot, access; @@ -295,11 +270,11 @@ textwindows void WinMainForked(void) { // fixup memory manager __maps.free = 0; - __maps.used = 0; + __maps.maps = 0; __maps.count = 0; __maps.pages = 0; - for (int i = 0; i < n; ++i) { - struct Map *map = sorted[i]; + for (struct Tree *e = tree_first(maps); e; e = tree_next(e)) { + struct Map *map = MAP_TREE_CONTAINER(e); __maps.count += 1; __maps.pages += (map->size + getpagesize() - 1) / getpagesize(); unsigned old_protect; @@ -307,8 +282,7 @@ textwindows void WinMainForked(void) { &old_protect)) AbortFork("VirtualProtect"); } - Free(sorted); - __maps.used = maps; + __maps.maps = maps; __maps_init(); // mitosis complete @@ -393,19 +367,13 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) { if (spawnrc != -1) { CloseHandle(procinfo.hThread); ok = WriteAll(writer, jb, sizeof(jb)); - int count = 0; // this list will be populated with the maps we're transferring - struct Dll *e2, *maps = 0; - for (struct Dll *e = dll_first(__maps.used); ok && e; e = e2) { - e2 = dll_next(__maps.used, e); - struct Map *map = MAP_CONTAINER(e); + for (struct Map *map = __maps_first(); ok && map; + map = __maps_next(map)) { if (MAX((char *)__executable_start, map->addr) < MIN((char *)_end, map->addr + map->size)) continue; // executable image is loaded by windows - dll_remove(&__maps.used, e); - dll_make_last(&maps, e); ok = WriteAll(writer, map, sizeof(*map)); - ++count; } // send a terminating Map struct to child if (ok) { @@ -415,40 +383,44 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) { } // now write content of each map to child int granularity = __granularity(); - struct Map **sorted = SortMaps(maps, count); - uint32_t *old_protect = Malloc(count * 4); - for (int i = 0; ok && i < count; ++i) { - struct Map *map = sorted[i]; + for (struct Map *map = __maps_first(); ok && map; + map = __maps_next(map)) { // we only need to worry about the base mapping if ((uintptr_t)map->addr & (granularity - 1)) continue; + if (MAX((char *)__executable_start, map->addr) < + MIN((char *)_end, map->addr + map->size)) + continue; // executable image is loaded by windows // shared mappings don't need to be copied if ((map->flags & MAP_TYPE) == MAP_SHARED) continue; // get true length in case mprotect() chopped up actual win32 map - int j; size_t size = map->size; - for (j = i + 1; j < count && sorted[j]->hand == -1 && - map->addr + size == sorted[j]->addr; - ++j) { - size += sorted[j]->size; + for (struct Map *map2 = __maps_next(map); map2; + map2 = __maps_next(map2)) { + if (map2->hand == -1 && map->addr + size == map2->addr) { + size += map2->size; + } else { + break; + } + } + for (struct Map *map2 = map; ok && map2; map2 = __maps_next(map2)) { + if (!(map2->prot & PROT_READ)) + if (map->addr >= map2->addr && map->addr < map->addr + size) + ok = VirtualProtect( + map2->addr, map2->size, + __prot2nt(map2->prot | PROT_READ, map2->iscow), + &map2->oldprot); } - for (int k = i; ok && k < j; ++k) - if (!(sorted[k]->prot & PROT_READ)) - ok = VirtualProtect( - sorted[k]->addr, sorted[k]->size, - __prot2nt(sorted[k]->prot | PROT_READ, map->iscow), - &old_protect[k]); if (ok) ok = WriteAll(writer, map->addr, size); - for (int k = i; ok && k < j; ++k) - if (!(sorted[k]->prot & PROT_READ)) - ok = VirtualProtect(sorted[k]->addr, sorted[k]->size, - old_protect[k], &old_protect[k]); + for (struct Map *map2 = map; ok && map2; map2 = __maps_next(map2)) { + if (!(map2->prot & PROT_READ)) + if (map->addr >= map2->addr && map->addr < map->addr + size) + ok = VirtualProtect(map2->addr, map2->size, map2->oldprot, + &map2->oldprot); + } } - Free(old_protect); - Free(sorted); - dll_make_first(&__maps.used, maps); if (ok) ok = WriteAll(writer, __data_start, __data_end - __data_start); if (ok) @@ -466,6 +438,7 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) { } else { TerminateProcess(procinfo.hProcess, SIGKILL); CloseHandle(procinfo.hProcess); + rc = -1; } } } @@ -473,9 +446,8 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) { CloseHandle(reader); if (writer != -1) CloseHandle(writer); - if (rc == -1 && errno != ENOMEM) { + if (rc == -1 && errno != ENOMEM) eagain(); // posix fork() only specifies two errors - } } else { rc = 0; // re-apply code morphing for thread-local storage diff --git a/libc/proc/fork.c b/libc/proc/fork.c index 05431bcb5..3a48db798 100644 --- a/libc/proc/fork.c +++ b/libc/proc/fork.c @@ -22,6 +22,7 @@ #include "libc/calls/state.internal.h" #include "libc/calls/struct/sigset.h" #include "libc/calls/struct/sigset.internal.h" +#include "libc/calls/struct/timespec.h" #include "libc/calls/syscall-nt.internal.h" #include "libc/calls/syscall-sysv.internal.h" #include "libc/dce.h" @@ -45,7 +46,6 @@ #include "libc/thread/posixthread.internal.h" #include "libc/thread/tls.h" -extern pthread_mutex_t nsync_waiters_mu; extern pthread_mutex_t _pthread_lock_obj; static void _onfork_prepare(void) { @@ -54,11 +54,10 @@ static void _onfork_prepare(void) { _pthread_lock(); __maps_lock(); __fds_lock(); - pthread_mutex_lock(&nsync_waiters_mu); + LOCKTRACE("READY TO ROCK AND ROLL"); } static void _onfork_parent(void) { - pthread_mutex_unlock(&nsync_waiters_mu); __fds_unlock(); __maps_unlock(); _pthread_unlock(); @@ -68,7 +67,6 @@ static void _onfork_parent(void) { static void _onfork_child(void) { __fds_lock_obj = (pthread_mutex_t)PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; - nsync_waiters_mu = (pthread_mutex_t)PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; _pthread_lock_obj = (pthread_mutex_t)PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; atomic_store_explicit(&__maps.lock, 0, memory_order_relaxed); atomic_store_explicit(&__get_tls()->tib_relock_maps, 0, memory_order_relaxed); @@ -77,7 +75,9 @@ static void _onfork_child(void) { } int _fork(uint32_t dwCreationFlags) { + long micros; struct Dll *e; + struct timespec started; int ax, dx, tid, parent; parent = __pid; BLOCK_SIGNALS; @@ -85,11 +85,13 @@ int _fork(uint32_t dwCreationFlags) { __proc_lock(); if (__threaded) _onfork_prepare(); + started = timespec_real(); if (!IsWindows()) { ax = sys_fork(); } else { ax = sys_fork_nt(dwCreationFlags); } + micros = timespec_tomicros(timespec_sub(timespec_real(), started)); if (!ax) { // get new process id @@ -136,15 +138,14 @@ int _fork(uint32_t dwCreationFlags) { // run user fork callbacks if (__threaded) _onfork_child(); - STRACE("fork() → 0 (child of %d)", parent); + STRACE("fork() → 0 (child of %d; took %ld us)", parent, micros); } else { // this is the parent process - if (__threaded) { + if (__threaded) _onfork_parent(); - } if (IsWindows()) __proc_unlock(); - STRACE("fork() → %d% m", ax); + STRACE("fork() → %d% m (took %ld us)", ax, micros); } ALLOW_SIGNALS; return ax; diff --git a/libc/proc/posix_spawn.c b/libc/proc/posix_spawn.c index 4e85e49b4..967392ad8 100644 --- a/libc/proc/posix_spawn.c +++ b/libc/proc/posix_spawn.c @@ -482,9 +482,8 @@ errno_t posix_spawn(int *pid, const char *path, const posix_spawn_file_actions_t *file_actions, const posix_spawnattr_t *attrp, char *const argv[], char *const envp[]) { - if (IsWindows()) { + if (IsWindows()) return posix_spawn_nt(pid, path, file_actions, attrp, argv, envp); - } int pfds[2]; bool use_pipe; volatile int status = 0; @@ -516,66 +515,55 @@ errno_t posix_spawn(int *pid, const char *path, sigaction(sig, &dfl, 0); } } - if (flags & POSIX_SPAWN_SETSID) { + if (flags & POSIX_SPAWN_SETSID) setsid(); - } - if ((flags & POSIX_SPAWN_SETPGROUP) && setpgid(0, (*attrp)->pgroup)) { + if ((flags & POSIX_SPAWN_SETPGROUP) && setpgid(0, (*attrp)->pgroup)) goto ChildFailed; - } - if ((flags & POSIX_SPAWN_RESETIDS) && setgid(getgid())) { + if ((flags & POSIX_SPAWN_RESETIDS) && setgid(getgid())) goto ChildFailed; - } - if ((flags & POSIX_SPAWN_RESETIDS) && setuid(getuid())) { + if ((flags & POSIX_SPAWN_RESETIDS) && setuid(getuid())) goto ChildFailed; - } if (file_actions) { struct _posix_faction *a; for (a = *file_actions; a; a = a->next) { if (use_pipe && pfds[1] == a->fildes) { int p2; - if ((p2 = dup(pfds[1])) == -1) { + if ((p2 = dup(pfds[1])) == -1) goto ChildFailed; - } lost_cloexec = true; close(pfds[1]); pfds[1] = p2; } switch (a->action) { case _POSIX_SPAWN_CLOSE: - if (close(a->fildes)) { + if (close(a->fildes)) goto ChildFailed; - } break; case _POSIX_SPAWN_DUP2: - if (dup2(a->fildes, a->newfildes) == -1) { + if (dup2(a->fildes, a->newfildes) == -1) goto ChildFailed; - } break; case _POSIX_SPAWN_OPEN: { int t; - if ((t = openat(AT_FDCWD, a->path, a->oflag, a->mode)) == -1) { + if ((t = openat(AT_FDCWD, a->path, a->oflag, a->mode)) == -1) goto ChildFailed; - } if (t != a->fildes) { if (dup2(t, a->fildes) == -1) { close(t); goto ChildFailed; } - if (close(t)) { + if (close(t)) goto ChildFailed; - } } break; } case _POSIX_SPAWN_CHDIR: - if (chdir(a->path) == -1) { + if (chdir(a->path) == -1) goto ChildFailed; - } break; case _POSIX_SPAWN_FCHDIR: - if (fchdir(a->fildes) == -1) { + if (fchdir(a->fildes) == -1) goto ChildFailed; - } break; default: __builtin_unreachable(); @@ -583,17 +571,13 @@ errno_t posix_spawn(int *pid, const char *path, } } if (IsLinux() || IsFreebsd() || IsNetbsd()) { - if (flags & POSIX_SPAWN_SETSCHEDULER) { + if (flags & POSIX_SPAWN_SETSCHEDULER) if (sched_setscheduler(0, (*attrp)->schedpolicy, - &(*attrp)->schedparam) == -1) { + &(*attrp)->schedparam) == -1) goto ChildFailed; - } - } - if (flags & POSIX_SPAWN_SETSCHEDPARAM) { - if (sched_setparam(0, &(*attrp)->schedparam)) { + if (flags & POSIX_SPAWN_SETSCHEDPARAM) + if (sched_setparam(0, &(*attrp)->schedparam)) goto ChildFailed; - } - } } if (flags & POSIX_SPAWN_SETRLIMIT) { int rlimset = (*attrp)->rlimset; @@ -608,9 +592,8 @@ errno_t posix_spawn(int *pid, const char *path, } } } - if (lost_cloexec) { + if (lost_cloexec) fcntl(pfds[1], F_SETFD, FD_CLOEXEC); - } if (flags & POSIX_SPAWN_SETSIGMASK) { childmask = (*attrp)->sigmask; } else { @@ -636,9 +619,8 @@ errno_t posix_spawn(int *pid, const char *path, if (!use_pipe) { res = status; } else { - if (can_clobber) { + if (can_clobber) atomic_store_explicit(&has_vfork, true, memory_order_release); - } res = 0; read(pfds[0], &res, sizeof(res)); } @@ -651,9 +633,8 @@ errno_t posix_spawn(int *pid, const char *path, } else { res = errno; } - if (use_pipe) { + if (use_pipe) close(pfds[0]); - } ParentFailed: sigprocmask(SIG_SETMASK, &oldmask, 0); pthread_setcancelstate(cs, 0); diff --git a/libc/runtime/mapstack.c b/libc/runtime/mapstack.c index 3bec5d639..eccd5cefc 100644 --- a/libc/runtime/mapstack.c +++ b/libc/runtime/mapstack.c @@ -50,9 +50,10 @@ void *NewCosmoStack(void) { if (IsOpenbsd() && __sys_mmap(p, n, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD, - -1, 0, 0) != p) { + -1, 0, 0) != p) + notpossible; + if (mprotect(p, GetGuardSize(), PROT_NONE | PROT_GUARD)) notpossible; - } return p; } else { return 0; diff --git a/libc/runtime/runtime.h b/libc/runtime/runtime.h index 236a06f2d..295f0fe02 100644 --- a/libc/runtime/runtime.h +++ b/libc/runtime/runtime.h @@ -91,7 +91,7 @@ void ShowCrashReports(void) libcesque; int ftrace_install(void) libcesque; int ftrace_enabled(int) libcesque; int strace_enabled(int) libcesque; -void __print_maps(void) libcesque; +void __print_maps(size_t) libcesque; void __printargs(const char *) libcesque; /* builtin sh-like system/popen dsl */ int _cocmd(int, char **, char **) libcesque; diff --git a/libc/stdio/printargs.c b/libc/stdio/printargs.c index 2b549cbdf..5d2a61c2f 100644 --- a/libc/stdio/printargs.c +++ b/libc/stdio/printargs.c @@ -468,7 +468,7 @@ textstartup void __printargs(const char *prologue) { PRINT(""); PRINT("MEMTRACK"); - __print_maps(); + __print_maps(0); PRINT(""); PRINT("TERMIOS"); diff --git a/libc/sysv/consts.sh b/libc/sysv/consts.sh index 233c11edc..22cd4c998 100755 --- a/libc/sysv/consts.sh +++ b/libc/sysv/consts.sh @@ -292,6 +292,7 @@ syscon mprot PROT_NONE 0 0 0 0 0 0 0 0 # mmap, mprotect, unix syscon mprot PROT_READ 1 1 1 1 1 1 1 1 # mmap, mprotect, unix consensus syscon mprot PROT_WRITE 2 2 2 2 2 2 2 2 # mmap, mprotect, unix consensus syscon mprot PROT_EXEC 4 4 4 4 4 4 4 4 # mmap, mprotect, unix consensus +syscon mprot PROT_GUARD 0 0 0 0 0 0 0 0x100 # mmap, mprotect, unix consensus # mremap() flags # the revolutionary praxis of realloc() diff --git a/libc/sysv/consts/PROT_GUARD.S b/libc/sysv/consts/PROT_GUARD.S new file mode 100644 index 000000000..e04cb73ee --- /dev/null +++ b/libc/sysv/consts/PROT_GUARD.S @@ -0,0 +1,2 @@ +#include "libc/sysv/consts/syscon.internal.h" +.syscon mprot,PROT_GUARD,0,0,0,0,0,0,0,0x100 diff --git a/libc/sysv/consts/prot.h b/libc/sysv/consts/prot.h index 032a6b1a7..ea026f869 100644 --- a/libc/sysv/consts/prot.h +++ b/libc/sysv/consts/prot.h @@ -3,10 +3,7 @@ #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ -extern const int PROT_NONE; -extern const int PROT_READ; -extern const int PROT_WRITE; -extern const int PROT_EXEC; +extern const int PROT_GUARD; COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ diff --git a/libc/thread/pthread_create.c b/libc/thread/pthread_create.c index b9515272e..55db00ef4 100644 --- a/libc/thread/pthread_create.c +++ b/libc/thread/pthread_create.c @@ -229,20 +229,10 @@ static errno_t pthread_create_impl(pthread_t *thread, -1, 0, 0) != pt->pt_attr.__stackaddr) { notpossible; } - if (pt->pt_attr.__guardsize) { - if (!IsWindows()) { - if (mprotect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize, - PROT_NONE)) { - notpossible; - } - } else { - uint32_t oldattr; - if (!VirtualProtect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize, - kNtPageReadwrite | kNtPageGuard, &oldattr)) { - notpossible; - } - } - } + if (pt->pt_attr.__guardsize) + if (mprotect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize, + PROT_NONE | PROT_GUARD)) + notpossible; } if (!pt->pt_attr.__stackaddr || pt->pt_attr.__stackaddr == MAP_FAILED) { rc = errno; diff --git a/libc/thread/thread.h b/libc/thread/thread.h index 7fd38f42c..872fdcc37 100644 --- a/libc/thread/thread.h +++ b/libc/thread/thread.h @@ -190,6 +190,7 @@ int pthread_spin_trylock(pthread_spinlock_t *) libcesque paramsnonnull(); int pthread_spin_unlock(pthread_spinlock_t *) libcesque paramsnonnull(); int pthread_testcancel_np(void) libcesque; int pthread_tryjoin_np(pthread_t, void **) libcesque; +int pthread_delay_np(const void *, int) libcesque; int pthread_yield_np(void) libcesque; int pthread_yield(void) libcesque; pthread_id_np_t pthread_getthreadid_np(void) libcesque; diff --git a/test/ctl/set_test.cc b/test/ctl/set_test.cc index 5eecca5e2..c435533db 100644 --- a/test/ctl/set_test.cc +++ b/test/ctl/set_test.cc @@ -19,9 +19,9 @@ #include "ctl/set.h" #include "libc/mem/leaks.h" -// #include -// #define ctl std -// #define check() size() +#include +#define ctl std +#define check() size() int rand32(void) @@ -148,6 +148,16 @@ main() s.check(); } + { + // Test lower_bound and upper_bound + ctl::set s{ 1, 3, 4, 5, 7, 9 }; + auto lower = s.lower_bound(4); + auto upper = s.upper_bound(4); + if (*lower != 4 || *upper != 5) + return 18; + s.check(); + } + { // Test emplace ctl::set> s; diff --git a/test/libc/calls/raise_test.c b/test/libc/calls/raise_test.c index 5f764fa38..5ebb8189a 100644 --- a/test/libc/calls/raise_test.c +++ b/test/libc/calls/raise_test.c @@ -20,7 +20,6 @@ #include "libc/calls/struct/sigaction.h" #include "libc/calls/struct/siginfo.h" #include "libc/dce.h" -#include "libc/intrin/kprintf.h" #include "libc/runtime/runtime.h" #include "libc/sysv/consts/sa.h" #include "libc/sysv/consts/sicode.h" diff --git a/test/libc/calls/setrlimit_test.c b/test/libc/calls/setrlimit_test.c index bc40f8542..59ad24782 100644 --- a/test/libc/calls/setrlimit_test.c +++ b/test/libc/calls/setrlimit_test.c @@ -118,10 +118,11 @@ TEST(setrlimit, testFileSizeLimit) { EXPECT_EQ(0, WTERMSIG(wstatus)); } -int SetKernelEnforcedMemoryLimit(size_t n) { - struct rlimit rlim; +int SetMemoryLimit(size_t n) { + struct rlimit rlim = {0}; getrlimit(RLIMIT_AS, &rlim); rlim.rlim_cur = n; + rlim.rlim_max = n; return setrlimit(RLIMIT_AS, &rlim); } @@ -129,27 +130,20 @@ TEST(setrlimit, testMemoryLimit) { char *p; bool gotsome; int i, wstatus; - if (IsXnu()) - return; - if (IsOpenbsd()) - return; // simply too slow until mmap() becomes O(logn) ASSERT_NE(-1, (wstatus = xspawn(0))); if (wstatus == -2) { - ASSERT_EQ(0, SetKernelEnforcedMemoryLimit(MEM)); - for (gotsome = false, i = 0; i < (MEM * 2) / __granularity(); ++i) { - p = mmap(0, __granularity(), PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE | MAP_POPULATE, -1, 0); + ASSERT_EQ(0, SetMemoryLimit(MEM)); + for (gotsome = false, i = 0; i < (MEM * 2) / getpagesize(); ++i) { + p = mmap(0, getpagesize(), PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (p != MAP_FAILED) { gotsome = true; } else { - if (!IsNetbsd()) { - // TODO(jart): what's going on with NetBSD? - ASSERT_TRUE(gotsome); - } + ASSERT_TRUE(gotsome); ASSERT_EQ(ENOMEM, errno); _Exit(0); } - rngset(p, __granularity(), _rand64, -1); + rngset(p, getpagesize(), _rand64, -1); } _Exit(1); } diff --git a/test/libc/intrin/mmap_scalability_test.c b/test/libc/intrin/mmap_scalability_test.c new file mode 100644 index 000000000..26982fbbe --- /dev/null +++ b/test/libc/intrin/mmap_scalability_test.c @@ -0,0 +1,76 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2024 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" +#include "libc/calls/struct/timespec.h" +#include "libc/dce.h" +#include "libc/intrin/kprintf.h" +#include "libc/runtime/runtime.h" +#include "libc/sysv/consts/map.h" +#include "libc/sysv/consts/prot.h" + +#define BENCH(ITERATIONS, WORK_PER_RUN, CODE) \ + do { \ + struct timespec start = timespec_real(); \ + for (int __i = 0; __i < ITERATIONS; ++__i) { \ + asm volatile("" ::: "memory"); \ + CODE; \ + } \ + long long work = (WORK_PER_RUN) * (ITERATIONS); \ + double nanos = \ + (timespec_tonanos(timespec_sub(timespec_real(), start)) + work - 1) / \ + (double)work; \ + kprintf("%'20ld ns %2dx %s\n", (long)nanos, (ITERATIONS), #CODE); \ + } while (0) + +void *randaddr(void) { + static unsigned long lcg = 1; + lcg *= 6364136223846793005; + lcg += 1442695040888963407; + return (void *)(lcg >> 48 << 28); +} + +void map_unmap_one_page(void) { + void *p; + if ((p = mmap(randaddr(), 1, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)) == MAP_FAILED) + __builtin_trap(); + if (munmap(p, 1)) + __builtin_trap(); +} + +int main() { + kprintf("\n"); + BENCH(1000, 1, map_unmap_one_page()); + + // macos doesn't scale + if (IsXnu()) + return 0; + + // create lots of sparse mappings to put + // weight on __maps.maps. red-black tree + int n = 10000; + kprintf("%20s creating %d sparse maps...\n", "", n); + for (int i = 0; i < n; ++i) { + if (mmap(randaddr(), 1, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0) == MAP_FAILED) + __builtin_trap(); + } + + BENCH(1000, 1, map_unmap_one_page()); +} diff --git a/test/libc/intrin/mmap_test.c b/test/libc/intrin/mmap_test.c index 44e9216ca..bf9eee74a 100644 --- a/test/libc/intrin/mmap_test.c +++ b/test/libc/intrin/mmap_test.c @@ -461,7 +461,23 @@ void BenchUnmap(void) { __builtin_trap(); } +void BenchBigMmap(void) { + void *p; + p = mmap(0, 101 * 1024 * 1024, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (p == MAP_FAILED) + __builtin_trap(); + ptrs[count++] = p; +} + +void BenchBigMunmap(void) { + if (munmap(ptrs[--count], 101 * 1024 * 1024)) + __builtin_trap(); +} + BENCH(mmap, bench) { EZBENCH2("mmap", donothing, BenchMmapPrivate()); EZBENCH2("munmap", donothing, BenchUnmap()); + // EZBENCH2("big mmap", donothing, BenchBigMmap()); + // EZBENCH2("big munmap", donothing, BenchBigMunmap()); } diff --git a/test/libc/intrin/rbtree_test.c b/test/libc/intrin/tree_test.c similarity index 55% rename from test/libc/intrin/rbtree_test.c rename to test/libc/intrin/tree_test.c index 5bcea721c..09a8c807e 100644 --- a/test/libc/intrin/rbtree_test.c +++ b/test/libc/intrin/tree_test.c @@ -13,16 +13,18 @@ // TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR // PERFORMANCE OF THIS SOFTWARE. -#include "libc/intrin/rbtree.h" +#include "libc/intrin/tree.h" #include "libc/intrin/kprintf.h" +#include "libc/intrin/maps.h" #include "libc/macros.internal.h" +#include "libc/mem/mem.h" #include "libc/runtime/runtime.h" #include "libc/stdio/rand.h" -#define NUMBER_CONTAINER(e) RBTREE_CONTAINER(struct number, elem, e) +#define NUMBER_CONTAINER(e) TREE_CONTAINER(struct number, elem, e) -void rbtree_checker(const struct rbtree *node, const struct rbtree *parent, - int black_count, int *black_height, rbtree_cmp_f *cmp) { +void tree_checker(const struct Tree *node, const struct Tree *parent, + int black_count, int *black_height, tree_cmp_f *cmp) { if (!node) { // Leaf nodes are considered black if (*black_height == -1) { @@ -37,89 +39,101 @@ void rbtree_checker(const struct rbtree *node, const struct rbtree *parent, // ILLEGAL TREE: Parent link is incorrect __builtin_trap(); if (parent) { - if (rbtree_get_left(parent) == node && cmp(parent, node) < 0) + if (tree_get_left(parent) == node && cmp(parent, node) < 0) // ILLEGAL TREE: Binary search property violated on left child __builtin_trap(); if (parent->right == node && cmp(node, parent) < 0) // ILLEGAL TREE: Binary search property violated on right child __builtin_trap(); } - if (!rbtree_get_red(node)) { + if (!tree_get_red(node)) { black_count++; - } else if (parent && rbtree_get_red(parent)) { + } else if (parent && tree_get_red(parent)) { // ILLEGAL TREE: Red node has red child __builtin_trap(); } - rbtree_checker(rbtree_get_left(node), node, black_count, black_height, cmp); - rbtree_checker(node->right, node, black_count, black_height, cmp); + tree_checker(tree_get_left(node), node, black_count, black_height, cmp); + tree_checker(node->right, node, black_count, black_height, cmp); } -void rbtree_check(struct rbtree *root, rbtree_cmp_f *cmp) { +void tree_check(struct Tree *root, tree_cmp_f *cmp) { if (root) { - if (rbtree_get_red(root)) + if (tree_get_red(root)) // ILLEGAL TREE: root node must be black __builtin_trap(); int black_height = -1; - rbtree_checker(root, 0, 0, &black_height, cmp); + tree_checker(root, 0, 0, &black_height, cmp); } } struct number { - int number; - struct rbtree elem; + long number; + struct Tree elem; }; -int number_compare(const struct rbtree *ra, const struct rbtree *rb) { +int number_search(const void *ra, const struct Tree *rb) { + long a = (long)ra; + const struct number *b = NUMBER_CONTAINER(rb); + return (a > b->number) - (a < b->number); +} + +int number_compare(const struct Tree *ra, const struct Tree *rb) { const struct number *a = NUMBER_CONTAINER(ra); const struct number *b = NUMBER_CONTAINER(rb); return (a->number > b->number) - (a->number < b->number); } struct number *number_new(int number) { - static int used; - static struct number heap[8192]; - if (used == ARRAYLEN(heap)) - return 0; - struct number *res = &heap[used++]; - res->number = number; + struct number *res; + if ((res = malloc(sizeof(struct number)))) + res->number = number; return res; } -struct rbtree *tree = 0; +struct Tree *tree = 0; void print(void) { - for (struct rbtree *e = rbtree_first(tree); e; e = rbtree_next(e)) + for (struct Tree *e = tree_first(tree); e; e = tree_next(e)) kprintf("%3d", NUMBER_CONTAINER(e)->number); kprintf("\n"); } void print_reversed(void) { - for (struct rbtree *e = rbtree_last(tree); e; e = rbtree_prev(e)) + for (struct Tree *e = tree_last(tree); e; e = tree_prev(e)) kprintf("%3d", NUMBER_CONTAINER(e)->number); kprintf("\n"); } void simple_test(void) { - static const int kNumba[] = {74, 53, 96, 70, 34, 95, 30, 2, 89, 46, - 23, 2, 52, 0, 34, 12, 90, 95, 32, 65}; + static const long kNumba[] = {74, 53, 96, 70, 34, 95, 30, 2, 96, 46, + 23, 2, 52, 0, 34, 94, 90, 95, 32, 65}; for (int i = 0; i < 20; ++i) { int number = kNumba[i]; kprintf("%3d", number); - rbtree_insert(&tree, &number_new(number)->elem, number_compare); - rbtree_check(tree, number_compare); + tree_insert(&tree, &number_new(number)->elem, number_compare); + tree_check(tree, number_compare); } kprintf("\n"); print(); print_reversed(); for (int i = 0; i < 20; ++i) { - rbtree_remove(&tree, rbtree_get(tree, &(&(struct number){kNumba[i]})->elem, - number_compare)); - rbtree_check(tree, number_compare); + tree_remove(&tree, tree_get(tree, (void *)kNumba[i], number_search)); + tree_check(tree, number_compare); print(); } } int main() { ShowCrashReports(); + tree_check(__maps.maps, __maps_compare); + kprintf("\n"); + __print_maps(15); + kprintf("\n"); simple_test(); + tree_check(__maps.maps, __maps_compare); + for (int i = 0; i < 100000; ++i) + tree_insert(&tree, &number_new(rand())->elem, number_compare); + tree_check(tree, number_compare); + tree_check(__maps.maps, __maps_compare); + __print_maps(15); } diff --git a/test/libc/thread/pthread_atfork_test.c b/test/libc/thread/pthread_atfork_test.c index dc977b9aa..f1e44139b 100644 --- a/test/libc/thread/pthread_atfork_test.c +++ b/test/libc/thread/pthread_atfork_test.c @@ -16,8 +16,11 @@ │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/atomic.h" #include "libc/calls/calls.h" #include "libc/dce.h" +#include "libc/intrin/atomic.h" +#include "libc/intrin/kprintf.h" #include "libc/mem/gc.h" #include "libc/mem/mem.h" #include "libc/runtime/internal.h" @@ -66,7 +69,7 @@ TEST(pthread_atfork, test) { EXITS(0); } -pthread_mutex_t mu; +pthread_mutex_t mu = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; void mu_lock(void) { pthread_mutex_lock(&mu); @@ -80,8 +83,12 @@ void mu_wipe(void) { pthread_mutex_init(&mu, 0); } +static atomic_bool once; + void *Worker(void *arg) { for (int i = 0; i < 20; ++i) { + if (!atomic_exchange(&once, true)) + __print_maps(0); mu_lock(); usleep(20); mu_unlock(); @@ -99,14 +106,14 @@ void *Worker(void *arg) { } TEST(pthread_atfork, fork_exit_torture) { + if (!IsFreebsd()) + return; mu_wipe(); pthread_atfork(mu_lock, mu_unlock, mu_wipe); int i, n = 4; pthread_t *t = gc(malloc(sizeof(pthread_t) * n)); - for (i = 0; i < n; ++i) { + for (i = 0; i < n; ++i) ASSERT_EQ(0, pthread_create(t + i, 0, Worker, 0)); - } - for (i = 0; i < n; ++i) { + for (i = 0; i < n; ++i) ASSERT_EQ(0, pthread_join(t[i], 0)); - } } diff --git a/third_party/nsync/common.c b/third_party/nsync/common.c index 4c77e7d4e..3a20bcdbb 100644 --- a/third_party/nsync/common.c +++ b/third_party/nsync/common.c @@ -18,6 +18,7 @@ #include "libc/calls/calls.h" #include "libc/calls/syscall-sysv.internal.h" #include "libc/dce.h" +#include "libc/intrin/directmap.internal.h" #include "libc/intrin/dll.h" #include "libc/intrin/extend.internal.h" #include "libc/nt/enum/filemapflags.h" @@ -25,21 +26,17 @@ #include "libc/nt/memory.h" #include "libc/nt/runtime.h" #include "libc/runtime/memtrack.internal.h" +#include "libc/runtime/runtime.h" +#include "libc/stdalign.internal.h" +#include "libc/stdalign.internal.h" #include "libc/sysv/consts/map.h" #include "libc/sysv/consts/prot.h" +#include "libc/thread/thread.h" #include "libc/thread/tls.h" #include "third_party/nsync/atomic.h" #include "third_party/nsync/atomic.internal.h" #include "third_party/nsync/common.internal.h" #include "third_party/nsync/mu_semaphore.h" -#include "third_party/nsync/races.internal.h" -#include "libc/runtime/runtime.h" -#include "libc/runtime/runtime.h" -#include "libc/sysv/consts/map.h" -#include "libc/nt/runtime.h" -#include "libc/intrin/directmap.internal.h" -#include "libc/thread/thread.h" -#include "libc/dce.h" #include "third_party/nsync/wait_s.internal.h" __static_yoink("nsync_notice"); @@ -97,33 +94,15 @@ __static_yoink("nsync_notice"); distinct wakeup conditions were high. So clients are advised to resort to condition variables if they have many distinct wakeup conditions. */ -/* Used in spinloops to delay resumption of the loop. - Usage: - unsigned attempts = 0; - while (try_something) { - attempts = nsync_spin_delay_ (attempts); - } */ -unsigned nsync_spin_delay_ (unsigned attempts) { - if (attempts < 7) { - volatile int i; - for (i = 0; i != 1 << attempts; i++) { - } - attempts++; - } else { - nsync_yield_ (); - } - return (attempts); -} - /* Spin until (*w & test) == 0, then atomically perform *w = ((*w | set) & ~clear), perform an acquire barrier, and return the previous value of *w. */ uint32_t nsync_spin_test_and_set_ (nsync_atomic_uint32_ *w, uint32_t test, - uint32_t set, uint32_t clear) { + uint32_t set, uint32_t clear, void *symbol) { unsigned attempts = 0; /* CV_SPINLOCK retry count */ uint32_t old = ATM_LOAD (w); while ((old & test) != 0 || !ATM_CAS_ACQ (w, old, (old | set) & ~clear)) { - attempts = nsync_spin_delay_ (attempts); + attempts = pthread_delay_np (symbol, attempts); old = ATM_LOAD (w); } return (old); @@ -156,33 +135,69 @@ waiter *nsync_dll_waiter_samecond_ (struct Dll *e) { /* -------------------------------- */ -static void *nsync_malloc (size_t size) { - void *res; - if (!IsWindows ()) { - // too much of a performance hit to track - res = __sys_mmap ((void *)0x7110000000, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, 0, 0); +static _Atomic(waiter *) free_waiters; + +static void free_waiters_push (waiter *w) { + int backoff = 0; + w->next_free = atomic_load_explicit (&free_waiters, memory_order_relaxed); + while (!atomic_compare_exchange_weak_explicit (&free_waiters, &w->next_free, w, + memory_order_acq_rel, memory_order_relaxed)) + backoff = pthread_delay_np(free_waiters, backoff); +} + +static void free_waiters_populate (void) { + int n; + if (IsNetbsd () || IsXnuSilicon ()) { + // netbsd needs one file descriptor per semaphore (!!) + // tim cook wants us to use his grand central dispatch + n = 1; } else { - // must be tracked for fork() resurrection - res = mmap (0, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, 0); + n = getpagesize() / sizeof(waiter); } - if (res == MAP_FAILED) + waiter *waiters = mmap (0, n * sizeof(waiter), + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + if (waiters == MAP_FAILED) nsync_panic_ ("out of memory\n"); - return res; + for (size_t i = 0; i < n; ++i) { + waiter *w = &waiters[i]; + w->tag = WAITER_TAG; + w->nw.tag = NSYNC_WAITER_TAG; + if (!nsync_mu_semaphore_init (&w->sem)) { + if (!i) + nsync_panic_ ("out of semaphores\n"); + break; + } + w->nw.sem = &w->sem; + dll_init (&w->nw.q); + NSYNC_ATOMIC_UINT32_STORE_ (&w->nw.waiting, 0); + w->nw.flags = NSYNC_WAITER_FLAG_MUCV; + ATM_STORE (&w->remove_count, 0); + dll_init (&w->same_condition); + w->flags = 0; + free_waiters_push (w); + } +} + +static waiter *free_waiters_pop (void) { + waiter *w; + int backoff = 0; + for (;;) { + if ((w = atomic_load_explicit (&free_waiters, memory_order_relaxed))) { + if (atomic_compare_exchange_weak_explicit (&free_waiters, &w, w->next_free, + memory_order_acq_rel, memory_order_relaxed)) + return w; + backoff = pthread_delay_np(free_waiters, backoff); + } else { + free_waiters_populate (); + } + } + return w; } /* -------------------------------- */ -static struct Dll *free_waiters = NULL; - -/* free_waiters points to a doubly-linked list of free waiter structs. */ -pthread_mutex_t nsync_waiters_mu = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; - #define waiter_for_thread __get_tls()->tib_nsync void nsync_waiter_destroy (void *v) { @@ -193,45 +208,20 @@ void nsync_waiter_destroy (void *v) { of thread-local variables can be arbitrary in some platform e.g. POSIX. */ waiter_for_thread = NULL; - IGNORE_RACES_START (); ASSERT ((w->flags & (WAITER_RESERVED|WAITER_IN_USE)) == WAITER_RESERVED); w->flags &= ~WAITER_RESERVED; - pthread_mutex_lock (&nsync_waiters_mu); - dll_make_first (&free_waiters, &w->nw.q); - pthread_mutex_unlock (&nsync_waiters_mu); - IGNORE_RACES_END (); + free_waiters_push (w); } /* Return a pointer to an unused waiter struct. Ensures that the enclosed timer is stopped and its channel drained. */ waiter *nsync_waiter_new_ (void) { - struct Dll *q; - waiter *tw; waiter *w; + waiter *tw; tw = waiter_for_thread; w = tw; if (w == NULL || (w->flags & (WAITER_RESERVED|WAITER_IN_USE)) != WAITER_RESERVED) { - w = NULL; - pthread_mutex_lock (&nsync_waiters_mu); - q = dll_first (free_waiters); - if (q != NULL) { /* If free list is non-empty, dequeue an item. */ - dll_remove (&free_waiters, q); - w = DLL_WAITER (q); - } - pthread_mutex_unlock (&nsync_waiters_mu); - if (w == NULL) { /* If free list was empty, allocate an item. */ - w = (waiter *) nsync_malloc (sizeof (*w)); - w->tag = WAITER_TAG; - w->nw.tag = NSYNC_WAITER_TAG; - nsync_mu_semaphore_init (&w->sem); - w->nw.sem = &w->sem; - dll_init (&w->nw.q); - NSYNC_ATOMIC_UINT32_STORE_ (&w->nw.waiting, 0); - w->nw.flags = NSYNC_WAITER_FLAG_MUCV; - ATM_STORE (&w->remove_count, 0); - dll_init (&w->same_condition); - w->flags = 0; - } + w = free_waiters_pop (); if (tw == NULL) { w->flags |= WAITER_RESERVED; waiter_for_thread = w; @@ -246,9 +236,7 @@ void nsync_waiter_free_ (waiter *w) { ASSERT ((w->flags & WAITER_IN_USE) != 0); w->flags &= ~WAITER_IN_USE; if ((w->flags & WAITER_RESERVED) == 0) { - pthread_mutex_lock (&nsync_waiters_mu); - dll_make_first (&free_waiters, &w->nw.q); - pthread_mutex_unlock (&nsync_waiters_mu); + free_waiters_push (w); if (w == waiter_for_thread) waiter_for_thread = 0; } diff --git a/third_party/nsync/common.internal.h b/third_party/nsync/common.internal.h index 90205e102..a0b2c0ffe 100644 --- a/third_party/nsync/common.internal.h +++ b/third_party/nsync/common.internal.h @@ -24,19 +24,11 @@ void nsync_yield_(void); /* Retrieve the per-thread cache of the waiter object. Platform specific. */ void *nsync_per_thread_waiter_(void (*dest)(void *)); -/* Used in spinloops to delay resumption of the loop. - Usage: - unsigned attempts = 0; - while (try_something) { - attempts = nsync_spin_delay_ (attempts); - } */ -unsigned nsync_spin_delay_(unsigned attempts); - /* Spin until (*w & test) == 0, then atomically perform *w = ((*w | set) & ~clear), perform an acquire barrier, and return the previous value of *w. */ uint32_t nsync_spin_test_and_set_(nsync_atomic_uint32_ *w, uint32_t test, - uint32_t set, uint32_t clear); + uint32_t set, uint32_t clear, void *symbol); /* Abort after printing the nul-temrinated string s[]. */ void nsync_panic_(const char *s) wontreturn; @@ -210,6 +202,7 @@ typedef struct waiter_s { struct wait_condition_s cond; /* A condition on which to acquire a mu. */ struct Dll same_condition; /* Links neighbours in nw.q with same non-nil condition. */ + struct waiter_s * next_free; } waiter; static const uint32_t WAITER_TAG = 0x0590239f; static const uint32_t NSYNC_WAITER_TAG = 0x726d2ba9; diff --git a/third_party/nsync/mem/nsync_cv.c b/third_party/nsync/mem/nsync_cv.c index 926379340..8e363f77c 100644 --- a/third_party/nsync/mem/nsync_cv.c +++ b/third_party/nsync/mem/nsync_cv.c @@ -193,7 +193,7 @@ static int nsync_cv_wait_with_deadline_impl_ (struct nsync_cv_wait_with_deadline /* A timeout or cancellation occurred, and no wakeup. Acquire *pcv's spinlock, and confirm. */ c->old_word = nsync_spin_test_and_set_ (&c->pcv->word, CV_SPINLOCK, - CV_SPINLOCK, 0); + CV_SPINLOCK, 0, c->cv_mu); /* Check that w wasn't removed from the queue after we checked above, but before we acquired the spinlock. The test of remove_count confirms that the waiter *w @@ -226,7 +226,7 @@ static int nsync_cv_wait_with_deadline_impl_ (struct nsync_cv_wait_with_deadline has not yet set the waiting field to zero; a cancellation or timeout may prevent this thread from blocking above on the semaphore. */ - attempts = nsync_spin_delay_ (attempts); + attempts = pthread_delay_np (c->cv_mu, attempts); } } if (c->cv_mu != NULL && c->w->cv_mu == NULL) { /* waiter was moved to *pmu's queue, and woken. */ @@ -323,7 +323,7 @@ int nsync_cv_wait_with_deadline_generic (nsync_cv *pcv, void *pmu, } /* acquire spinlock, set non-empty */ - c.old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK|CV_NON_EMPTY, 0); + c.old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK|CV_NON_EMPTY, 0, pmu); dll_make_last (&pcv->waiters, &c.w->nw.q); c.remove_count = ATM_LOAD (&c.w->remove_count); /* Release the spin lock. */ @@ -355,7 +355,7 @@ void nsync_cv_signal (nsync_cv *pcv) { int all_readers = 0; /* acquire spinlock */ uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, - CV_SPINLOCK, 0); + CV_SPINLOCK, 0, pcv); if (!dll_is_empty (pcv->waiters)) { /* Point to first waiter that enqueued itself, and detach it from all others. */ @@ -438,7 +438,7 @@ void nsync_cv_broadcast (nsync_cv *pcv) { int all_readers; struct Dll *to_wake_list = NULL; /* waiters that we will wake */ /* acquire spinlock */ - nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0); + nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0, pcv); p = NULL; next = NULL; all_readers = 1; @@ -497,7 +497,7 @@ static nsync_time cv_ready_time (void *v, struct nsync_waiter_s *nw) { static int cv_enqueue (void *v, struct nsync_waiter_s *nw) { nsync_cv *pcv = (nsync_cv *) v; /* acquire spinlock */ - uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0); + uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0, pcv); dll_make_last (&pcv->waiters, &nw->q); ATM_STORE (&nw->waiting, 1); /* Release spinlock. */ @@ -509,7 +509,7 @@ static int cv_dequeue (void *v, struct nsync_waiter_s *nw) { nsync_cv *pcv = (nsync_cv *) v; int was_queued = 0; /* acquire spinlock */ - uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0); + uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0, pcv); if (ATM_LOAD_ACQ (&nw->waiting) != 0) { dll_remove (&pcv->waiters, &nw->q); ATM_STORE (&nw->waiting, 0); diff --git a/third_party/nsync/mem/nsync_debug.c b/third_party/nsync/mem/nsync_debug.c index 1b72c1175..a3d847286 100644 --- a/third_party/nsync/mem/nsync_debug.c +++ b/third_party/nsync/mem/nsync_debug.c @@ -197,7 +197,7 @@ static char *emit_mu_state (struct emit_buf *b, nsync_mu *mu, word = ATM_LOAD (&mu->word); if ((word & MU_WAITING) != 0 && print_waiters && /* can benefit from lock */ (blocking || (word & MU_SPINLOCK) == 0)) { /* willing, or no need to wait */ - word = nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK, MU_SPINLOCK, 0); + word = nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK, MU_SPINLOCK, 0, mu); acquired = 1; } readers = word / MU_RLOCK; @@ -234,7 +234,7 @@ static char *emit_cv_state (struct emit_buf *b, nsync_cv *cv, word = ATM_LOAD (&cv->word); if ((word & CV_NON_EMPTY) != 0 && print_waiters && /* can benefit from lock */ (blocking || (word & CV_SPINLOCK) == 0)) { /* willing, or no need to wait */ - word = nsync_spin_test_and_set_ (&cv->word, CV_SPINLOCK, CV_SPINLOCK, 0); + word = nsync_spin_test_and_set_ (&cv->word, CV_SPINLOCK, CV_SPINLOCK, 0, cv); acquired = 1; } emit_print (b, "cv 0x%i -> 0x%i = {", (uintptr_t) cv, word); diff --git a/third_party/nsync/mem/nsync_mu_wait.c b/third_party/nsync/mem/nsync_mu_wait.c index 3c48ea599..e46492aa9 100644 --- a/third_party/nsync/mem/nsync_mu_wait.c +++ b/third_party/nsync/mem/nsync_mu_wait.c @@ -21,6 +21,7 @@ #include "third_party/nsync/common.internal.h" #include "third_party/nsync/mu_semaphore.h" #include "third_party/nsync/races.internal.h" +#include "libc/thread/thread.h" #include "third_party/nsync/wait_s.internal.h" __static_yoink("nsync_notice"); @@ -84,7 +85,7 @@ static int mu_try_acquire_after_timeout_or_cancel (nsync_mu *mu, lock_type *l_ty ATM_CAS_RELACQ (&mu->word, old_word, old_word|MU_WRITER_WAITING); } - spin_attempts = nsync_spin_delay_ (spin_attempts); + spin_attempts = pthread_delay_np (mu, spin_attempts); old_word = ATM_LOAD (&mu->word); } /* Check that w wasn't removed from the queue after our caller checked, @@ -194,7 +195,7 @@ int nsync_mu_wait_with_deadline (nsync_mu *mu, /* Acquire spinlock. */ old_word = nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK, - MU_SPINLOCK|MU_WAITING|has_condition, MU_ALL_FALSE); + MU_SPINLOCK|MU_WAITING|has_condition, MU_ALL_FALSE, mu); had_waiters = ((old_word & (MU_DESIG_WAKER | MU_WAITING)) == MU_WAITING); /* Queue the waiter. */ if (first_wait) { @@ -244,7 +245,7 @@ int nsync_mu_wait_with_deadline (nsync_mu *mu, } if (ATM_LOAD (&w->nw.waiting) != 0) { - attempts = nsync_spin_delay_ (attempts); /* will ultimately yield */ + attempts = pthread_delay_np (mu, attempts); /* will ultimately yield */ } } diff --git a/third_party/nsync/mem/nsync_once.c b/third_party/nsync/mem/nsync_once.c index 163923359..873766b99 100644 --- a/third_party/nsync/mem/nsync_once.c +++ b/third_party/nsync/mem/nsync_once.c @@ -21,6 +21,7 @@ #include "third_party/nsync/mu_semaphore.h" #include "third_party/nsync/once.h" #include "third_party/nsync/races.internal.h" +#include "libc/thread/thread.h" #include "third_party/nsync/wait_s.internal.h" __static_yoink("nsync_notice"); @@ -92,7 +93,7 @@ static void nsync_run_once_impl (nsync_once *once, struct once_sync_s *s, deadline = nsync_time_add (nsync_time_now (), nsync_time_ms (attempts)); nsync_cv_wait_with_deadline (&s->once_cv, &s->once_mu, deadline, NULL); } else { - attempts = nsync_spin_delay_ (attempts); + attempts = pthread_delay_np (once, attempts); } } if (s != NULL) { diff --git a/third_party/nsync/mu.c b/third_party/nsync/mu.c index 4cae68328..20eac1e68 100644 --- a/third_party/nsync/mu.c +++ b/third_party/nsync/mu.c @@ -22,6 +22,7 @@ #include "third_party/nsync/common.internal.h" #include "third_party/nsync/mu_semaphore.h" #include "third_party/nsync/races.internal.h" +#include "libc/thread/thread.h" #include "third_party/nsync/wait_s.internal.h" __static_yoink("nsync_notice"); @@ -120,7 +121,7 @@ void nsync_mu_lock_slow_ (nsync_mu *mu, waiter *w, uint32_t clear, lock_type *l_ about waiting writers or long waiters. */ zero_to_acquire &= ~(MU_WRITER_WAITING | MU_LONG_WAIT); } - attempts = nsync_spin_delay_ (attempts); + attempts = pthread_delay_np (mu, attempts); } ALLOW_CANCELATION; } @@ -393,7 +394,7 @@ void nsync_mu_unlock_slow_ (nsync_mu *mu, lock_type *l_type) { released above. */ if (testing_conditions) { nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK, - MU_SPINLOCK, 0); + MU_SPINLOCK, 0, mu); } /* add the new_waiters to the last of the waiters. */ @@ -444,7 +445,7 @@ void nsync_mu_unlock_slow_ (nsync_mu *mu, lock_type *l_type) { } return; } - attempts = nsync_spin_delay_ (attempts); + attempts = pthread_delay_np (mu, attempts); } } diff --git a/third_party/nsync/mu_semaphore.c b/third_party/nsync/mu_semaphore.c index 493efa2c4..fd9d855ce 100644 --- a/third_party/nsync/mu_semaphore.c +++ b/third_party/nsync/mu_semaphore.c @@ -27,7 +27,7 @@ __static_yoink("nsync_notice"); #define PREFER_GCD_OVER_ULOCK 1 /* Initialize *s; the initial value is 0. */ -void nsync_mu_semaphore_init (nsync_semaphore *s) { +bool nsync_mu_semaphore_init (nsync_semaphore *s) { if (PREFER_GCD_OVER_ULOCK && IsXnuSilicon ()) { return nsync_mu_semaphore_init_gcd (s); } else if (IsNetbsd ()) { diff --git a/third_party/nsync/mu_semaphore.h b/third_party/nsync/mu_semaphore.h index 64c7d563d..992d4849f 100644 --- a/third_party/nsync/mu_semaphore.h +++ b/third_party/nsync/mu_semaphore.h @@ -8,7 +8,7 @@ typedef struct nsync_semaphore_s_ { } nsync_semaphore; /* Initialize *s; the initial value is 0. */ -void nsync_mu_semaphore_init(nsync_semaphore *s); +bool nsync_mu_semaphore_init(nsync_semaphore *s); /* Wait until the count of *s exceeds 0, and decrement it. */ errno_t nsync_mu_semaphore_p(nsync_semaphore *s); diff --git a/third_party/nsync/mu_semaphore.internal.h b/third_party/nsync/mu_semaphore.internal.h index ce3181fc3..31b51975d 100755 --- a/third_party/nsync/mu_semaphore.internal.h +++ b/third_party/nsync/mu_semaphore.internal.h @@ -4,17 +4,17 @@ #include "third_party/nsync/time.h" COSMOPOLITAN_C_START_ -void nsync_mu_semaphore_init_futex(nsync_semaphore *); +bool nsync_mu_semaphore_init_futex(nsync_semaphore *); errno_t nsync_mu_semaphore_p_futex(nsync_semaphore *); errno_t nsync_mu_semaphore_p_with_deadline_futex(nsync_semaphore *, nsync_time); void nsync_mu_semaphore_v_futex(nsync_semaphore *); -void nsync_mu_semaphore_init_sem(nsync_semaphore *); +bool nsync_mu_semaphore_init_sem(nsync_semaphore *); errno_t nsync_mu_semaphore_p_sem(nsync_semaphore *); errno_t nsync_mu_semaphore_p_with_deadline_sem(nsync_semaphore *, nsync_time); void nsync_mu_semaphore_v_sem(nsync_semaphore *); -void nsync_mu_semaphore_init_gcd(nsync_semaphore *); +bool nsync_mu_semaphore_init_gcd(nsync_semaphore *); errno_t nsync_mu_semaphore_p_gcd(nsync_semaphore *); errno_t nsync_mu_semaphore_p_with_deadline_gcd(nsync_semaphore *, nsync_time); void nsync_mu_semaphore_v_gcd(nsync_semaphore *); diff --git a/third_party/nsync/mu_semaphore_futex.c b/third_party/nsync/mu_semaphore_futex.c index 14d0ec20e..12964e7c8 100644 --- a/third_party/nsync/mu_semaphore_futex.c +++ b/third_party/nsync/mu_semaphore_futex.c @@ -43,9 +43,10 @@ static nsync_semaphore *sem_big_enough_for_futex = (nsync_semaphore *) (uintptr_ (sizeof (struct futex) <= sizeof (*sem_big_enough_for_futex))); /* Initialize *s; the initial value is 0. */ -void nsync_mu_semaphore_init_futex (nsync_semaphore *s) { +bool nsync_mu_semaphore_init_futex (nsync_semaphore *s) { struct futex *f = (struct futex *) s; f->i = 0; + return true; } /* Wait until the count of *s exceeds 0, and decrement it. If POSIX cancellations diff --git a/third_party/nsync/mu_semaphore_gcd.c b/third_party/nsync/mu_semaphore_gcd.c index fd36135de..1f88d5e0d 100644 --- a/third_party/nsync/mu_semaphore_gcd.c +++ b/third_party/nsync/mu_semaphore_gcd.c @@ -85,8 +85,8 @@ static errno_t nsync_dispatch_semaphore_wait (nsync_semaphore *s, } /* Initialize *s; the initial value is 0. */ -void nsync_mu_semaphore_init_gcd (nsync_semaphore *s) { - *(dispatch_semaphore_t *)s = dispatch_semaphore_create (0); +bool nsync_mu_semaphore_init_gcd (nsync_semaphore *s) { + return !!(*(dispatch_semaphore_t *)s = dispatch_semaphore_create (0)); } /* Wait until the count of *s exceeds 0, and decrement it. If POSIX cancellations diff --git a/third_party/nsync/mu_semaphore_sem.c b/third_party/nsync/mu_semaphore_sem.c index d2c6377ac..3eb8a22f6 100644 --- a/third_party/nsync/mu_semaphore_sem.c +++ b/third_party/nsync/mu_semaphore_sem.c @@ -54,14 +54,22 @@ static struct { static nsync_semaphore *sem_big_enough_for_sem = (nsync_semaphore *) (uintptr_t)(1 / (sizeof (struct sem) <= sizeof (*sem_big_enough_for_sem))); -static void nsync_mu_semaphore_sem_create (struct sem *f) { +static bool nsync_mu_semaphore_sem_create (struct sem *f) { + int rc; int lol; f->id = 0; - ASSERT (!sys_sem_init (0, &f->id)); - if ((lol = __sys_fcntl (f->id, F_DUPFD_CLOEXEC, 50)) >= 50) { - sys_close (f->id); + rc = sys_sem_init (0, &f->id); + STRACE ("sem_init(0, [%ld]) → %d", f->id, rc); + if (rc != 0) + return false; + lol = __sys_fcntl (f->id, F_DUPFD_CLOEXEC, 50); + STRACE ("fcntl(%ld, F_DUPFD_CLOEXEC, 50) → %d", f->id, lol); + if (lol >= 50) { + rc = sys_close (f->id); + STRACE ("close(%ld) → %d", f->id, rc); f->id = lol; } + return true; } static void nsync_mu_semaphore_sem_fork_child (void) { @@ -69,8 +77,12 @@ static void nsync_mu_semaphore_sem_fork_child (void) { struct sem *f; for (e = dll_first (g_sems.list); e; e = dll_next (g_sems.list, e)) { f = SEM_CONTAINER (e); - sys_close (f->id); - nsync_mu_semaphore_sem_create (f); + int rc = sys_close (f->id); + STRACE ("close(%ld) → %d", f->id, rc); + } + for (e = dll_first (g_sems.list); e; e = dll_next (g_sems.list, e)) { + f = SEM_CONTAINER (e); + ASSERT (nsync_mu_semaphore_sem_create (f)); } (void) pthread_spin_init (&g_sems.lock, 0); } @@ -80,15 +92,16 @@ static void nsync_mu_semaphore_sem_init (void) { } /* Initialize *s; the initial value is 0. */ -void nsync_mu_semaphore_init_sem (nsync_semaphore *s) { +bool nsync_mu_semaphore_init_sem (nsync_semaphore *s) { struct sem *f = (struct sem *) s; - nsync_mu_semaphore_sem_create (f); + if (!nsync_mu_semaphore_sem_create (f)) + return false; cosmo_once (&g_sems.once, nsync_mu_semaphore_sem_init); pthread_spin_lock (&g_sems.lock); dll_init (&f->list); dll_make_first (&g_sems.list, &f->list); pthread_spin_unlock (&g_sems.lock); - STRACE ("sem_init(0, [%ld]) → 0", f->id); + return true; } /* Wait until the count of *s exceeds 0, and decrement it. If POSIX cancellations diff --git a/tool/build/runitd.c b/tool/build/runitd.c index d1b0f375f..577876772 100644 --- a/tool/build/runitd.c +++ b/tool/build/runitd.c @@ -134,12 +134,12 @@ #define INFOF(FMT, ...) LOGF(INFO, FMT, ##__VA_ARGS__) #define WARNF(FMT, ...) LOGF(WARN, FMT, ##__VA_ARGS__) -#define LOGF(LVL, FMT, ...) \ - do { \ - if (g_log_level >= LOG_LEVEL_##LVL) { \ - kprintf("%r" #LVL " %6P %'18T %s:%d " FMT "\n", __FILE__, __LINE__, \ - ##__VA_ARGS__); \ - } \ +#define LOGF(LVL, FMT, ...) \ + do { \ + if (g_log_level >= LOG_LEVEL_##LVL) { \ + kprintf("%r" #LVL " %6P %6H %'18T %s:%d " FMT "\n", __FILE__, __LINE__, \ + ##__VA_ARGS__); \ + } \ } while (0) struct Client { @@ -443,12 +443,18 @@ void FreeClient(struct Client *client) { void *ClientWorker(void *arg) { uint32_t crc; sigset_t sigmask; + struct timespec ts0; + struct timespec ts1; + struct timespec ts2; int events, wstatus; struct Client *client = arg; uint32_t namesize, filesize; char *addrstr, *origname; unsigned char msg[4 + 1 + 4 + 4 + 4]; + ts0 = timespec_real(); + ts1 = timespec_real(); + SetupPresharedKeySsl(MBEDTLS_SSL_IS_SERVER, g_psk); defer(FreeClient, client); @@ -458,9 +464,15 @@ void *ClientWorker(void *arg) { EzHandshake(); addrstr = DescribeAddress(&client->addr); DEBUF("%s %s %s", DescribeAddress(&g_servaddr), "accepted", addrstr); + DEBUF("it took %'zu us to handshake client", + timespec_tomicros(timespec_sub(timespec_real(), ts1))); // get the executable + ts1 = timespec_real(); + ts2 = timespec_real(); Recv(client, msg, sizeof(msg)); + DEBUF("it took %'zu us to receive #1", + timespec_tomicros(timespec_sub(timespec_real(), ts2))); if (READ32BE(msg) != RUNITD_MAGIC) { WARNF("%s magic mismatch!", addrstr); pthread_exit(0); @@ -473,11 +485,19 @@ void *ClientWorker(void *arg) { filesize = READ32BE(msg + 9); crc = READ32BE(msg + 13); origname = gc(calloc(1, namesize + 1)); + ts2 = timespec_real(); Recv(client, origname, namesize); + DEBUF("it took %'zu us to receive #2", + timespec_tomicros(timespec_sub(timespec_real(), ts2))); VERBF("%s sent %#s (%'u bytes @ %#s)", addrstr, origname, filesize, client->tmpexepath); char *exedata = gc(malloc(filesize)); + ts2 = timespec_real(); Recv(client, exedata, filesize); + DEBUF("it took %'zu us to receive #3", + timespec_tomicros(timespec_sub(timespec_real(), ts2))); + DEBUF("it took %'zu us to receive executable from network", + timespec_tomicros(timespec_sub(timespec_real(), ts1))); if (crc32_z(0, exedata, filesize) != crc) { WARNF("%s crc mismatch! %#s", addrstr, origname); pthread_exit(0); @@ -488,6 +508,7 @@ void *ClientWorker(void *arg) { // condition can happen, where etxtbsy is raised by our execve // we're using o_cloexec so it's guaranteed to fix itself fast // thus we use an optimistic approach to avoid expensive locks + ts1 = timespec_real(); sprintf(client->tmpexepath, "o/%s.XXXXXX", basename(stripext(gc(strdup(origname))))); int exefd = openatemp(AT_FDCWD, client->tmpexepath, 0, O_CLOEXEC, 0700); @@ -510,6 +531,8 @@ void *ClientWorker(void *arg) { WARNF("%s failed to close %#s due to %m", addrstr, origname); pthread_exit(0); } + DEBUF("it took %'zu us to write executable to disk", + timespec_tomicros(timespec_sub(timespec_real(), ts1))); // do the args int i = 0; @@ -560,8 +583,11 @@ RetryOnEtxtbsyRaceCondition: posix_spawn_file_actions_adddup2(&spawnfila, g_bogusfd, 0); posix_spawn_file_actions_adddup2(&spawnfila, client->pipe[1], 1); posix_spawn_file_actions_adddup2(&spawnfila, client->pipe[1], 2); + ts1 = timespec_real(); err = posix_spawn(&client->pid, client->tmpexepath, &spawnfila, &spawnattr, args, environ); + DEBUF("it took %'zu us to call posix_spawn", + timespec_tomicros(timespec_sub(timespec_real(), ts1))); if (err) { if (err == ETXTBSY) { goto RetryOnEtxtbsyRaceCondition; @@ -599,8 +625,11 @@ RetryOnEtxtbsyRaceCondition: fds[0].events = POLLIN; fds[1].fd = client->pipe[0]; fds[1].events = POLLIN; + ts1 = timespec_real(); events = poll(fds, ARRAYLEN(fds), timespec_tomillis(timespec_sub(deadline, now))); + DEBUF("it took %'zu us to call poll", + timespec_tomicros(timespec_sub(timespec_real(), ts1))); if (events == -1) { if (errno == EINTR) { INFOF("poll interrupted"); @@ -615,7 +644,10 @@ RetryOnEtxtbsyRaceCondition: if (fds[0].revents) { int received; char buf[512]; + ts1 = timespec_real(); received = mbedtls_ssl_read(&ezssl, buf, sizeof(buf)); + DEBUF("it took %'zu us to call mbedtls_ssl_read", + timespec_tomicros(timespec_sub(timespec_real(), ts1))); if (!received) { WARNF("%s client disconnected so killing worker %d", origname, client->pid); @@ -640,7 +672,10 @@ RetryOnEtxtbsyRaceCondition: } if (fds[1].revents) { char buf[512]; + ts1 = timespec_real(); ssize_t got = read(client->pipe[0], buf, sizeof(buf)); + DEBUF("it took %'zu us to call read", + timespec_tomicros(timespec_sub(timespec_real(), ts1))); if (got == -1) { WARNF("got %s reading %s output", strerror(errno), origname); goto HangupClientAndTerminateJob; @@ -658,7 +693,10 @@ RetryOnEtxtbsyRaceCondition: WaitAgain: DEBUF("waitpid"); struct rusage rusage; + ts1 = timespec_real(); int wrc = wait4(client->pid, &wstatus, 0, &rusage); + DEBUF("it took %'zu us to call wait4", + timespec_tomicros(timespec_sub(timespec_real(), ts1))); if (wrc == -1) { if (errno == EINTR) { WARNF("waitpid interrupted; killing %s pid %d", origname, client->pid); @@ -711,13 +749,18 @@ WaitAgain: AppendResourceReport(&client->output, &rusage, "\n"); PrintProgramOutput(client); } + ts1 = timespec_real(); SendProgramOutput(client); SendExitMessage(exitcode); mbedtls_ssl_close_notify(&ezssl); + DEBUF("it took %'zu us to send result to client", + timespec_tomicros(timespec_sub(timespec_real(), ts1))); if (etxtbsy_tries > 1) { WARNF("encountered %d ETXTBSY race conditions spawning %s", etxtbsy_tries - 1, origname); } + DEBUF("it took %'zu us TO DO EVERYTHING", + timespec_tomicros(timespec_sub(timespec_real(), ts0))); pthread_exit(0); }