mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-01-31 03:27:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Make mmap() scalable

Browse source 
It's now possible to create thousands of thousands of sparse independent
memory mappings, without any slowdown. The memory manager is better with
tracking memory protection now, particularly on Windows in a precise way
that can be restored during fork(). You now have the highest quality mem
manager possible. It's even better than some OSes like XNU, where mmap()
is implemented as an O(n) operation which means sadly things aren't much
improved over there. With this change the llamafile HTTP server endpoint
at /tokenize with a prompt of 50 tokens is now able to handle 2.6m r/sec
This commit is contained in:
Justine Tunney 2024-07-05 23:13:20 -07:00
parent 3756870635
commit 8c645fa1ee
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
59 changed files with 1238 additions and 1067 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
libc/calls/setrlimit.c
View file

@ -85,7 +85,7 @@ int setrlimit(int resource, const struct rlimit *rlim) {
rc = efault();
} else if (IsXnuSilicon()) {
rc = _sysret(__syslib->__setrlimit(resource, rlim));
} else if (!IsWindows()) {
} else if (!IsWindows() && !(IsNetbsd() && resource == RLIMIT_AS)) {
rc = sys_setrlimit(resource, rlim);
} else if (resource == RLIMIT_STACK) {
rc = enotsup();

4
libc/intrin/BUILD.mk
View file

@ -62,6 +62,10 @@ o/$(MODE)/libc/intrin/kprintf.o: private \
-Wframe-larger-than=128 \
-Walloca-larger-than=128
o/$(MODE)/libc/intrin/tree.o: private \
CFLAGS += \
-ffunction-sections
o//libc/intrin/memmove.o: private \
CFLAGS += \
-fno-toplevel-reorder

4
libc/intrin/describemapping.c
View file

@ -24,7 +24,7 @@
static char DescribeMapType(int flags) {
switch (flags & MAP_TYPE) {
case MAP_FILE:
return 'f';
return '-';
case MAP_PRIVATE:
return 'p';
case MAP_SHARED:
@ -47,7 +47,7 @@ const char *(DescribeMapping)(char p[8], int prot, int flags) {
DescribeProt(p, prot);
p[3] = DescribeMapType(flags);
p[4] = (flags & MAP_ANONYMOUS) ? 'a' : '-';
p[5] = (flags & MAP_FIXED) ? 'F' : '-';
p[5] = (flags & MAP_FIXED) ? 'f' : '-';
p[6] = 0;
return p;
}

12
libc/intrin/describeprotflags.c
View file

@ -20,12 +20,12 @@
#include "libc/macros.internal.h"
#include "libc/sysv/consts/prot.h"
static const struct DescribeFlags kProtFlags[] = {
{PROT_READ, "READ"}, //
{PROT_WRITE, "WRITE"}, //
{PROT_EXEC, "EXEC"}, //
};
const char *(DescribeProtFlags)(char buf[48], int x) {
const struct DescribeFlags kProtFlags[] = {
{PROT_READ, "READ"}, //
{PROT_WRITE, "WRITE"}, //
{PROT_EXEC, "EXEC"}, //
{PROT_GUARD, "GUARD"}, //
};
return DescribeFlags(buf, 48, kProtFlags, ARRAYLEN(kProtFlags), "PROT_", x);
}

16
libc/intrin/describerlimit.c
View file

@ -18,8 +18,11 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/struct/rlimit.h"
#include "libc/dce.h"
#include "libc/fmt/itoa.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/strace.internal.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/rlim.h"
const char *DescribeRlimit(char buf[64], int rc, const struct rlimit *rlim) {
if (rc == -1)
@ -29,7 +32,18 @@ const char *DescribeRlimit(char buf[64], int rc, const struct rlimit *rlim) {
if (kisdangerous(rlim)) {
ksnprintf(buf, 64, "%p", rlim);
} else {
ksnprintf(buf, 64, "{%'ld, %'ld}", rlim->rlim_cur, rlim->rlim_max);
char str[2][21];
if (rlim->rlim_cur == RLIM_INFINITY) {
strcpy(str[0], "RLIM_INFINITY");
} else {
FormatInt64(str[0], rlim->rlim_cur);
}
if (rlim->rlim_max == RLIM_INFINITY) {
strcpy(str[1], "RLIM_INFINITY");
} else {
FormatInt64(str[1], rlim->rlim_max);
}
ksnprintf(buf, 64, "{%s, %s}", str[0], str[1]);
}
return buf;
}

3
libc/intrin/directmap-nt.c
View file

@ -86,9 +86,8 @@ TryAgain:
if ((dm.addr = MapViewOfFileEx(dm.maphandle, fl.flags2, off >> 32, off,
size, addr))) {
uint32_t oldprot;
if (VirtualProtect(dm.addr, size, __prot2nt(prot, iscow), &oldprot)) {
if (VirtualProtect(dm.addr, size, __prot2nt(prot, iscow), &oldprot))
return dm;
}
UnmapViewOfFile(dm.addr);
}
CloseHandle(dm.maphandle);

8
libc/intrin/dll.c
View file

@ -28,7 +28,7 @@
*
* It's required that `elem` and `succ` aren't part of the same list.
*/
privileged void dll_splice_after(struct Dll *elem, struct Dll *succ) {
void dll_splice_after(struct Dll *elem, struct Dll *succ) {
struct Dll *tmp1, *tmp2;
tmp1 = elem->next;
tmp2 = succ->prev;
@ -43,7 +43,7 @@ privileged void dll_splice_after(struct Dll *elem, struct Dll *succ) {
*
* @param list is a doubly-linked list, where `!*list` means empty
*/
privileged void dll_remove(struct Dll **list, struct Dll *elem) {
void dll_remove(struct Dll **list, struct Dll *elem) {
if (*list == elem) {
if ((*list)->prev == *list) {
*list = 0;
@ -66,7 +66,7 @@ privileged void dll_remove(struct Dll **list, struct Dll *elem) {
* @param list is a doubly-linked list, where `!*list` means empty
* @param elem must not be a member of `list`, or null for no-op
*/
privileged void dll_make_first(struct Dll **list, struct Dll *elem) {
void dll_make_first(struct Dll **list, struct Dll *elem) {
if (elem) {
if (!*list) {
*list = elem->prev;
@ -85,7 +85,7 @@ privileged void dll_make_first(struct Dll **list, struct Dll *elem) {
* @param list is a doubly-linked list, where `!*list` means empty
* @param elem must not be a member of `list`, or null for no-op
*/
privileged void dll_make_last(struct Dll **list, struct Dll *elem) {
void dll_make_last(struct Dll **list, struct Dll *elem) {
if (elem) {
dll_make_first(list, elem->next);
*list = elem;

30
libc/intrin/kprintf.greg.c
View file

@ -25,7 +25,6 @@
#include "libc/fmt/magnumstrs.internal.h"
#include "libc/intrin/asmflag.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/getenv.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/likely.h"
@ -154,27 +153,18 @@ __funline bool kischarmisaligned(const char *p, signed char t) {
return false;
}
privileged static bool32 kisdangerous_unlocked(const char *addr) {
struct Dll *e;
if ((e = dll_first(__maps.used))) {
do {
struct Map *map = MAP_CONTAINER(e);
if (map->addr <= addr && addr < map->addr + map->size) {
dll_remove(&__maps.used, e);
dll_make_first(&__maps.used, e);
return !(map->prot & PROT_READ);
}
} while ((e = dll_next(__maps.used, e)));
return true;
} else {
return false;
}
}
privileged bool32 kisdangerous(const void *addr) {
bool32 res;
bool32 res = true;
__maps_lock();
res = kisdangerous_unlocked(addr);
if (__maps.maps) {
struct Map *map;
if ((map = __maps_floor(addr)))
if ((const char *)addr >= map->addr &&
(const char *)addr < map->addr + map->size)
res = false;
} else {
res = false;
}
__maps_unlock();
return res;
}

4
libc/intrin/maps.c
View file

@ -20,6 +20,7 @@
#include "ape/sections.internal.h"
#include "libc/dce.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/maps.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/sysv/consts/auxv.h"
@ -32,8 +33,7 @@ __static_yoink("_init_maps");
struct Maps __maps;
void __maps_add(struct Map *map) {
dll_init(&map->elem);
dll_make_first(&__maps.used, &map->elem);
tree_insert(&__maps.maps, &map->tree, __maps_compare);
++__maps.count;
}

54
libc/intrin/maps.h
View file

@ -1,36 +1,41 @@
#ifndef COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_
#define COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_
#ifndef COSMOPOLITAN_MAPS_H_
#define COSMOPOLITAN_MAPS_H_
#include "libc/intrin/atomic.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/tree.h"
#include "libc/runtime/runtime.h"
#include "libc/thread/tls2.internal.h"
COSMOPOLITAN_C_START_
#define MAP_CONTAINER(e) DLL_CONTAINER(struct Map, elem, e)
#define MAP_TREE_CONTAINER(e) TREE_CONTAINER(struct Map, tree, e)
#define MAP_FREE_CONTAINER(e) DLL_CONTAINER(struct Map, free, e)
struct Map {
char *addr; /* granule aligned */
size_t size; /* must be nonzero */
struct Dll elem; /* for __maps.free */
int64_t off; /* -1 if anonymous */
int64_t off; /* ignore for anon */
int prot; /* memory protects */
int flags; /* memory map flag */
bool iscow; /* windows nt only */
bool readonlyfile; /* windows nt only */
unsigned visited; /* used for checks */
unsigned oldprot; /* in windows fork */
intptr_t hand; /* windows nt only */
union {
struct Tree tree;
struct Dll free;
};
};
struct Maps {
unsigned mono;
atomic_int lock;
struct Tree *maps;
struct Dll *free;
struct Dll *used;
size_t count;
size_t pages;
atomic_ulong rollo;
struct Map stack;
struct Map guard;
bool once;
atomic_ulong rollo;
};
struct AddrSize {
@ -45,10 +50,37 @@ bool __maps_lock(void);
void __maps_check(void);
void __maps_unlock(void);
void __maps_add(struct Map *);
struct Map *__maps_alloc(void);
void __maps_free(struct Map *);
struct Map *__maps_alloc(void);
struct Map *__maps_floor(const char *);
void __maps_stack(char *, int, int, size_t, int, intptr_t);
int __maps_compare(const struct Tree *, const struct Tree *);
struct AddrSize __get_main_stack(void);
forceinline optimizespeed int __maps_search(const void *key,
const struct Tree *node) {
const char *addr = (const char *)key;
const struct Map *map = (const struct Map *)MAP_TREE_CONTAINER(node);
if (addr < map->addr)
return +1;
if (addr >= map->addr + map->size)
return -1;
return 0;
}
static struct Map *__maps_next(struct Map *map) {
struct Tree *node;
if ((node = tree_next(&map->tree)))
return MAP_TREE_CONTAINER(node);
return 0;
}
static struct Map *__maps_first(void) {
struct Tree *node;
if ((node = tree_first(__maps.maps)))
return MAP_TREE_CONTAINER(node);
return 0;
}
COSMOPOLITAN_C_END_
#endif /* COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_ */
#endif /* COSMOPOLITAN_MAPS_H_ */

260
libc/intrin/mmap.c
View file

@ -67,20 +67,31 @@
struct StackFrame *bp = __builtin_frame_address(0); \
kprintf("%!s:%d: assertion failed: %!s\n", __FILE__, __LINE__, #x); \
kprintf("bt %!s\n", (DescribeBacktrace)(bt, bp)); \
__print_maps(); \
_Exit(99); \
__print_maps(0); \
__builtin_trap(); \
} \
} while (0)
#endif
int __maps_compare(const struct Tree *ra, const struct Tree *rb) {
const struct Map *a = (const struct Map *)MAP_TREE_CONTAINER(ra);
const struct Map *b = (const struct Map *)MAP_TREE_CONTAINER(rb);
return (a->addr > b->addr) - (a->addr < b->addr);
}
privileged optimizespeed struct Map *__maps_floor(const char *addr) {
struct Tree *node;
if ((node = tree_floor(__maps.maps, addr, __maps_search)))
return MAP_TREE_CONTAINER(node);
return 0;
}
static bool overlaps_existing_map(const char *addr, size_t size, int pagesz) {
for (struct Dll *e = dll_first(__maps.used); e;
e = dll_next(__maps.used, e)) {
struct Map *map = MAP_CONTAINER(e);
struct Map *map;
if ((map = __maps_floor(addr)))
if (MAX(addr, map->addr) <
MIN(addr + PGUP(size), map->addr + PGUP(map->size)))
return true;
}
return false;
}
@ -89,66 +100,51 @@ void __maps_check(void) {
size_t maps = 0;
size_t pages = 0;
int pagesz = getpagesize();
unsigned id = ++__maps.mono;
for (struct Dll *e = dll_first(__maps.used); e;
e = dll_next(__maps.used, e)) {
struct Map *map = MAP_CONTAINER(e);
static unsigned mono;
unsigned id = ++mono;
for (struct Map *map = __maps_first(); map; map = __maps_next(map)) {
ASSERT(map->addr != MAP_FAILED);
ASSERT(map->visited != id);
ASSERT(map->size);
map->visited = id;
pages += (map->size + getpagesize() - 1) / getpagesize();
maps += 1;
struct Map *next;
if ((next = __maps_next(map))) {
ASSERT(map->addr < next->addr);
ASSERT(
!(MAX(map->addr, next->addr) <
MIN(map->addr + PGUP(map->size), next->addr + PGUP(next->size))));
}
}
ASSERT(maps = __maps.count);
ASSERT(pages == __maps.pages);
for (struct Dll *e = dll_first(__maps.used); e;
e = dll_next(__maps.used, e)) {
struct Map *m1 = MAP_CONTAINER(e);
for (struct Dll *f = dll_next(__maps.used, e); f;
f = dll_next(__maps.used, f)) {
struct Map *m2 = MAP_CONTAINER(f);
ASSERT(MAX(m1->addr, m2->addr) >=
MIN(m1->addr + PGUP(m1->size), m2->addr + PGUP(m2->size)));
}
}
#endif
}
void __maps_free(struct Map *map) {
map->size = 0;
map->addr = MAP_FAILED;
ASSERT(dll_is_alone(&map->elem));
dll_make_last(&__maps.free, &map->elem);
dll_init(&map->free);
dll_make_first(&__maps.free, &map->free);
}
static void __maps_insert(struct Map *map) {
struct Dll *e = dll_first(__maps.used);
struct Map *last = e ? MAP_CONTAINER(e) : 0;
__maps.pages += (map->size + getpagesize() - 1) / getpagesize();
if (last && !IsWindows() && //
map->addr == last->addr + last->size && //
struct Map *floor = __maps_floor(map->addr);
if (floor && !IsWindows() && //
map->addr + map->size == floor->addr && //
(map->flags & MAP_ANONYMOUS) && //
map->flags == last->flags && //
map->prot == last->prot) {
last->size += map->size;
dll_remove(&__maps.used, &last->elem);
dll_make_first(&__maps.used, &last->elem);
__maps_free(map);
} else if (last && !IsWindows() && //
map->addr + map->size == last->addr && //
(map->flags & MAP_ANONYMOUS) && //
map->flags == last->flags && //
map->prot == last->prot) {
last->addr -= map->size;
last->size += map->size;
dll_remove(&__maps.used, &last->elem);
dll_make_first(&__maps.used, &last->elem);
map->flags == floor->flags && //
map->prot == floor->prot) {
floor->addr -= map->size;
floor->size += map->size;
__maps_free(map);
__maps_check();
} else {
__maps_add(map);
__maps_check();
}
__maps_check();
}
struct Map *__maps_alloc(void) {
@ -156,7 +152,7 @@ struct Map *__maps_alloc(void) {
struct Map *map;
if ((e = dll_first(__maps.free))) {
dll_remove(&__maps.free, e);
map = MAP_CONTAINER(e);
map = MAP_FREE_CONTAINER(e);
return map;
}
int granularity = __granularity();
@ -168,11 +164,8 @@ struct Map *__maps_alloc(void) {
CloseHandle(sys.maphandle);
map = sys.addr;
map->addr = MAP_FAILED;
dll_init(&map->elem);
for (int i = 1; i < granularity / sizeof(struct Map); ++i) {
dll_init(&map[i].elem);
for (int i = 1; i < granularity / sizeof(struct Map); ++i)
__maps_free(map + i);
}
return map;
}
@ -190,106 +183,106 @@ static int __munmap(char *addr, size_t size, bool untrack_only) {
// untrack mappings
int rc = 0;
struct Dll *cur;
struct Dll *next;
struct Dll *delete = 0;
struct Map *map;
struct Map *next;
struct Dll *deleted = 0;
if (__maps_lock()) {
__maps_unlock();
return edeadlk();
}
for (cur = dll_first(__maps.used); cur; cur = next) {
next = dll_next(__maps.used, cur);
struct Map *map = MAP_CONTAINER(cur);
for (map = __maps_floor(addr); map; map = next) {
next = __maps_next(map);
char *map_addr = map->addr;
size_t map_size = map->size;
if (MAX(addr, map_addr) < MIN(addr + size, map_addr + PGUP(map_size))) {
if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) {
// remove mapping completely
dll_remove(&__maps.used, cur);
dll_make_first(&delete, cur);
__maps.pages -= (map_size + pagesz - 1) / pagesz;
__maps.count -= 1;
__maps_check();
} else if (IsWindows()) {
// you can't carve up memory maps on windows. our mmap() makes
// this not a problem (for non-enormous memory maps) by making
// independent mappings for each 64 kb granule, under the hood
rc = einval();
} else if (addr <= map_addr) {
// shave off lefthand side of mapping
ASSERT(addr + size < map_addr + PGUP(map_size));
size_t left = PGUP(addr + size - map_addr);
size_t right = map_size - left;
ASSERT(right > 0);
ASSERT(left > 0);
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
map->addr += left;
map->size = right;
if (!(map->flags & MAP_ANONYMOUS))
map->off += left;
__maps.pages -= (left + pagesz - 1) / pagesz;
if (!(MAX(addr, map_addr) < MIN(addr + size, map_addr + PGUP(map_size))))
break;
if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) {
// remove mapping completely
tree_remove(&__maps.maps, &map->tree);
dll_init(&map->free);
dll_make_first(&deleted, &map->free);
__maps.pages -= (map_size + pagesz - 1) / pagesz;
__maps.count -= 1;
} else if (IsWindows()) {
// you can't carve up memory maps on windows. our mmap() makes
// this not a problem (for non-enormous memory maps) by making
// independent mappings for each 64 kb granule, under the hood
rc = einval();
} else if (addr <= map_addr) {
// shave off lefthand side of mapping
ASSERT(addr + size < map_addr + PGUP(map_size));
size_t left = PGUP(addr + size - map_addr);
size_t right = map_size - left;
ASSERT(right > 0);
ASSERT(left > 0);
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
map->addr += left;
map->size = right;
if (!(map->flags & MAP_ANONYMOUS))
map->off += left;
__maps.pages -= (left + pagesz - 1) / pagesz;
leftmap->addr = map_addr;
leftmap->size = left;
dll_init(&leftmap->free);
dll_make_first(&deleted, &leftmap->free);
} else {
rc = -1;
}
} else if (addr + size >= map_addr + PGUP(map_size)) {
// shave off righthand side of mapping
size_t left = addr - map_addr;
size_t right = map_addr + map_size - addr;
struct Map *rightmap;
if ((rightmap = __maps_alloc())) {
map->size = left;
__maps.pages -= (right + pagesz - 1) / pagesz;
rightmap->addr = addr;
rightmap->size = right;
dll_init(&rightmap->free);
dll_make_first(&deleted, &rightmap->free);
} else {
rc = -1;
}
} else {
// punch hole in mapping
size_t left = addr - map_addr;
size_t middle = size;
size_t right = map_size - middle - left;
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
struct Map *middlemap;
if ((middlemap = __maps_alloc())) {
leftmap->addr = map_addr;
leftmap->size = left;
dll_make_first(&delete, &leftmap->elem);
__maps_check();
} else {
rc = -1;
}
} else if (addr + size >= map_addr + PGUP(map_size)) {
// shave off righthand side of mapping
size_t left = addr - map_addr;
size_t right = map_addr + map_size - addr;
struct Map *rightmap;
if ((rightmap = __maps_alloc())) {
map->size = left;
__maps.pages -= (right + pagesz - 1) / pagesz;
rightmap->addr = addr;
rightmap->size = right;
dll_make_first(&delete, &rightmap->elem);
__maps_check();
leftmap->off = map->off;
leftmap->prot = map->prot;
leftmap->flags = map->flags;
map->addr += left + middle;
map->size = right;
if (!(map->flags & MAP_ANONYMOUS))
map->off += left + middle;
tree_insert(&__maps.maps, &leftmap->tree, __maps_compare);
__maps.pages -= (middle + pagesz - 1) / pagesz;
__maps.count += 1;
middlemap->addr = addr;
middlemap->size = size;
dll_init(&middlemap->free);
dll_make_first(&deleted, &middlemap->free);
} else {
rc = -1;
}
} else {
// punch hole in mapping
size_t left = addr - map_addr;
size_t middle = size;
size_t right = map_size - middle - left;
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
struct Map *middlemap;
if ((middlemap = __maps_alloc())) {
leftmap->addr = map_addr;
leftmap->size = left;
leftmap->off = map->off;
leftmap->prot = map->prot;
leftmap->flags = map->flags;
map->addr += left + middle;
map->size = right;
if (!(map->flags & MAP_ANONYMOUS))
map->off += left + middle;
dll_make_first(&__maps.used, &leftmap->elem);
__maps.pages -= (middle + pagesz - 1) / pagesz;
__maps.count += 1;
middlemap->addr = addr;
middlemap->size = size;
dll_make_first(&delete, &middlemap->elem);
__maps_check();
} else {
rc = -1;
}
} else {
rc = -1;
}
rc = -1;
}
}
__maps_check();
}
__maps_unlock();
// delete mappings
for (struct Dll *e = dll_first(delete); e; e = dll_next(delete, e)) {
struct Map *map = MAP_CONTAINER(e);
for (struct Dll *e = dll_first(deleted); e; e = dll_next(deleted, e)) {
struct Map *map = MAP_FREE_CONTAINER(e);
if (!untrack_only) {
if (!IsWindows()) {
if (sys_munmap(map->addr, map->size))
@ -305,12 +298,12 @@ static int __munmap(char *addr, size_t size, bool untrack_only) {
}
// free mappings
if (!dll_is_empty(delete)) {
if (!dll_is_empty(deleted)) {
__maps_lock();
struct Dll *e;
while ((e = dll_first(delete))) {
dll_remove(&delete, e);
__maps_free(MAP_CONTAINER(e));
while ((e = dll_first(deleted))) {
dll_remove(&deleted, e);
__maps_free(MAP_FREE_CONTAINER(e));
}
__maps_check();
__maps_unlock();
@ -350,6 +343,7 @@ static void *__mmap_chunk(void *addr, size_t size, int prot, int flags, int fd,
__maps_unlock();
return (void *)edeadlk();
}
__maps_check();
map = __maps_alloc();
__maps_unlock();
if (!map)

189
libc/intrin/mprotect.c
View file

@ -25,6 +25,7 @@
#include "libc/intrin/kprintf.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/tree.h"
#include "libc/nt/memory.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
@ -51,7 +52,7 @@ static int __mprotect_chunk(char *addr, size_t size, int prot, bool iscow) {
int __mprotect(char *addr, size_t size, int prot) {
// unix checks prot before checking size
if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC))
if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC | PROT_GUARD))
return einval();
// make new technology consistent with unix
@ -68,64 +69,97 @@ int __mprotect(char *addr, size_t size, int prot) {
// change mappings
int rc = 0;
struct Dll *cur;
bool found = false;
if (__maps_lock()) {
__maps_unlock();
return edeadlk();
}
for (cur = dll_first(__maps.used); cur; cur = dll_next(__maps.used, cur)) {
struct Map *map = MAP_CONTAINER(cur);
for (struct Map *map = __maps_floor(addr); map; map = __maps_next(map)) {
char *map_addr = map->addr;
size_t map_size = map->size;
char *beg = MAX(addr, map_addr);
char *end = MIN(addr + size, map_addr + PGUP(map_size));
if (beg < end) {
found = true;
if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) {
// change protection of entire mapping
if (!__mprotect_chunk(map_addr, map_size, prot, map->iscow)) {
if (beg >= end)
break;
found = true;
if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) {
// change protection of entire mapping
if (!__mprotect_chunk(map_addr, map_size, prot, map->iscow)) {
map->prot = prot;
} else {
rc = -1;
}
} else if (addr <= map_addr) {
// change lefthand side of mapping
size_t left = PGUP(addr + size - map_addr);
size_t right = map_size - left;
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
if (!__mprotect_chunk(map_addr, left, prot, false)) {
leftmap->addr = map_addr;
leftmap->size = left;
leftmap->prot = prot;
leftmap->off = map->off;
leftmap->flags = map->flags;
leftmap->iscow = map->iscow;
leftmap->readonlyfile = map->readonlyfile;
leftmap->hand = map->hand;
map->addr += left;
map->size = right;
map->hand = -1;
if (!(map->flags & MAP_ANONYMOUS))
map->off += left;
tree_insert(&__maps.maps, &leftmap->tree, __maps_compare);
__maps.count += 1;
__maps_check();
} else {
__maps_free(leftmap);
rc = -1;
}
} else {
rc = -1;
}
} else if (addr + size >= map_addr + PGUP(map_size)) {
// change righthand side of mapping
size_t left = addr - map_addr;
size_t right = map_addr + map_size - addr;
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
if (!__mprotect_chunk(map_addr + left, right, prot, false)) {
leftmap->addr = map_addr;
leftmap->size = left;
leftmap->off = map->off;
leftmap->prot = map->prot;
leftmap->flags = map->flags;
leftmap->iscow = map->iscow;
leftmap->readonlyfile = map->readonlyfile;
leftmap->hand = map->hand;
map->addr += left;
map->size = right;
map->prot = prot;
map->hand = -1;
if (!(map->flags & MAP_ANONYMOUS))
map->off += left;
tree_insert(&__maps.maps, &leftmap->tree, __maps_compare);
__maps.count += 1;
__maps_check();
} else {
__maps_free(leftmap);
rc = -1;
}
} else if (addr <= map_addr) {
// change lefthand side of mapping
size_t left = PGUP(addr + size - map_addr);
size_t right = map_size - left;
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
if (!__mprotect_chunk(map_addr, left, prot, false)) {
leftmap->addr = map_addr;
leftmap->size = left;
leftmap->prot = prot;
leftmap->off = map->off;
leftmap->flags = map->flags;
leftmap->iscow = map->iscow;
leftmap->readonlyfile = map->readonlyfile;
leftmap->hand = map->hand;
map->addr += left;
map->size = right;
map->hand = -1;
if (!(map->flags & MAP_ANONYMOUS))
map->off += left;
dll_make_first(&__maps.used, &leftmap->elem);
__maps.count += 1;
__maps_check();
} else {
__maps_free(leftmap);
rc = -1;
}
} else {
rc = -1;
}
} else if (addr + size >= map_addr + PGUP(map_size)) {
// change righthand side of mapping
size_t left = addr - map_addr;
size_t right = map_addr + map_size - addr;
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
if (!__mprotect_chunk(map_addr + left, right, prot, false)) {
} else {
rc = -1;
}
} else {
// change middle of mapping
size_t left = addr - map_addr;
size_t middle = size;
size_t right = map_size - middle - left;
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
struct Map *midlmap;
if ((midlmap = __maps_alloc())) {
if (!__mprotect_chunk(map_addr + left, middle, prot, false)) {
leftmap->addr = map_addr;
leftmap->size = left;
leftmap->off = map->off;
@ -134,67 +168,32 @@ int __mprotect(char *addr, size_t size, int prot) {
leftmap->iscow = map->iscow;
leftmap->readonlyfile = map->readonlyfile;
leftmap->hand = map->hand;
map->addr += left;
midlmap->addr = map_addr + left;
midlmap->size = middle;
midlmap->off = (map->flags & MAP_ANONYMOUS) ? 0 : map->off + left;
midlmap->prot = prot;
midlmap->flags = map->flags;
midlmap->hand = -1;
map->addr += left + middle;
map->size = right;
map->prot = prot;
map->hand = -1;
if (!(map->flags & MAP_ANONYMOUS))
map->off += left;
dll_make_first(&__maps.used, &leftmap->elem);
__maps.count += 1;
map->off += left + middle;
tree_insert(&__maps.maps, &leftmap->tree, __maps_compare);
tree_insert(&__maps.maps, &midlmap->tree, __maps_compare);
__maps.count += 2;
__maps_check();
} else {
__maps_free(midlmap);
__maps_free(leftmap);
rc = -1;
}
} else {
__maps_free(leftmap);
rc = -1;
}
} else {
// change middle of mapping
size_t left = addr - map_addr;
size_t middle = size;
size_t right = map_size - middle - left;
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
struct Map *midlmap;
if ((midlmap = __maps_alloc())) {
if (!__mprotect_chunk(map_addr + left, middle, prot, false)) {
leftmap->addr = map_addr;
leftmap->size = left;
leftmap->off = map->off;
leftmap->prot = map->prot;
leftmap->flags = map->flags;
leftmap->iscow = map->iscow;
leftmap->readonlyfile = map->readonlyfile;
leftmap->hand = map->hand;
midlmap->addr = map_addr + left;
midlmap->size = middle;
midlmap->off = (map->flags & MAP_ANONYMOUS) ? 0 : map->off + left;
midlmap->prot = prot;
midlmap->flags = map->flags;
midlmap->hand = -1;
map->addr += left + middle;
map->size = right;
map->hand = -1;
if (!(map->flags & MAP_ANONYMOUS))
map->off += left + middle;
dll_make_first(&__maps.used, &midlmap->elem);
dll_make_first(&__maps.used, &leftmap->elem);
__maps.count += 2;
__maps_check();
} else {
__maps_free(midlmap);
__maps_free(leftmap);
rc = -1;
}
} else {
__maps_free(leftmap);
rc = -1;
}
} else {
rc = -1;
}
rc = -1;
}
}
}

21
libc/intrin/msync-nt.c
View file

@ -34,16 +34,17 @@ textwindows int sys_msync_nt(char *addr, size_t size, int flags) {
return einval();
int rc = 0;
__maps_lock();
for (struct Dll *e = dll_first(__maps.used); e;
e = dll_next(__maps.used, e)) {
struct Map *map = MAP_CONTAINER(e);
char *beg = MAX(addr, map->addr);
char *end = MIN(addr + size, map->addr + map->size);
if (beg < end)
if (!FlushViewOfFile(beg, end - beg))
rc = -1;
// TODO(jart): FlushFileBuffers too on g_fds handle if MS_SYNC?
if (__maps_lock()) {
rc = edeadlk();
} else {
for (struct Map *map = __maps_floor(addr); map; map = __maps_next(map)) {
char *beg = MAX(addr, map->addr);
char *end = MIN(addr + size, map->addr + map->size);
if (beg < end)
if (!FlushViewOfFile(beg, end - beg))
rc = -1;
// TODO(jart): FlushFileBuffers too on g_fds handle if MS_SYNC?
}
}
__maps_unlock();

11
libc/intrin/printmaps.c
View file

@ -29,16 +29,11 @@
/**
* Prints memory mappings.
*/
void __print_maps(void) {
int limit = 15;
long maptally = 0;
void __print_maps(size_t limit) {
char mappingbuf[8], sb[16];
__maps_lock();
struct Dll *e, *e2;
for (e = dll_first(__maps.used); e; e = e2) {
e2 = dll_next(__maps.used, e);
struct Map *map = MAP_CONTAINER(e);
maptally += map->size;
for (struct Tree *e = tree_first(__maps.maps); e; e = tree_next(e)) {
struct Map *map = MAP_TREE_CONTAINER(e);
kprintf("%012lx-%012lx %!s", map->addr, map->addr + map->size,
(DescribeMapping)(mappingbuf, map->prot, map->flags));
sizefmt(sb, map->size, 1024);

2
libc/intrin/prot2nt.greg.c
View file

@ -42,6 +42,8 @@ privileged int __prot2nt(int prot, int iscow) {
return kNtPageExecuteReadwrite;
}
default:
if (prot & PROT_GUARD)
return kNtPageReadwrite | kNtPageGuard;
return kNtPageNoaccess;
}
}

40
libc/intrin/pthread_delay_np.c Normal file
View file

@ -0,0 +1,40 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/strace.internal.h"
#include "libc/thread/thread.h"
/**
* Delays execution for brief moment.
*
* @param symbol may be used to strace names of static locks
* @param backoff should start at zero and be feed back in
* @return new value for backoff
*/
int pthread_delay_np(const void *symbol, int backoff) {
if (backoff < 7) {
volatile int i;
for (i = 0; i != 1 << backoff; i++) {
}
backoff++;
} else {
STRACE("pthread_delay_np(%t)", symbol);
pthread_yield_np();
}
return backoff;
}

125
libc/intrin/pthread_mutex_lock.c
View file

@ -18,8 +18,10 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/calls/state.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/runtime/internal.h"
@ -27,6 +29,66 @@
#include "libc/thread/thread.h"
#include "third_party/nsync/mu.h"
static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex) {
int me;
int backoff = 0;
uint64_t word, lock;
// get current state of lock
word = atomic_load_explicit(&mutex->_word, memory_order_relaxed);
// use fancy nsync mutex if possible
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL && //
MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && //
_weaken(nsync_mu_lock)) {
_weaken(nsync_mu_lock)((nsync_mu *)mutex);
return 0;
}
// implement barebones normal mutexes
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) {
for (;;) {
word = MUTEX_UNLOCK(word);
lock = MUTEX_LOCK(word);
if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
memory_order_acquire,
memory_order_relaxed))
return 0;
backoff = pthread_delay_np(mutex, backoff);
}
}
// implement recursive mutexes
me = gettid();
for (;;) {
if (MUTEX_OWNER(word) == me) {
if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) {
if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) {
if (atomic_compare_exchange_weak_explicit(
&mutex->_word, &word, MUTEX_INC_DEPTH(word),
memory_order_relaxed, memory_order_relaxed))
return 0;
continue;
} else {
return EAGAIN;
}
} else {
return EDEADLK;
}
}
word = MUTEX_UNLOCK(word);
lock = MUTEX_LOCK(word);
lock = MUTEX_SET_OWNER(lock, me);
if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
memory_order_acquire,
memory_order_relaxed)) {
mutex->_pid = __pid;
return 0;
}
backoff = pthread_delay_np(mutex, backoff);
}
}
/**
* Locks mutex.
*
@ -65,65 +127,10 @@
* @vforksafe
*/
errno_t pthread_mutex_lock(pthread_mutex_t *mutex) {
int me;
uint64_t word, lock;
LOCKTRACE("pthread_mutex_lock(%t)", mutex);
if (__vforked)
return 0;
// get current state of lock
word = atomic_load_explicit(&mutex->_word, memory_order_relaxed);
// use fancy nsync mutex if possible
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL && //
MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && //
_weaken(nsync_mu_lock)) {
_weaken(nsync_mu_lock)((nsync_mu *)mutex);
return 0;
}
// implement barebones normal mutexes
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) {
for (;;) {
word = MUTEX_UNLOCK(word);
lock = MUTEX_LOCK(word);
if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
memory_order_acquire,
memory_order_relaxed))
return 0;
pthread_pause_np();
}
}
// implement recursive mutexes
me = gettid();
for (;;) {
if (MUTEX_OWNER(word) == me) {
if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) {
if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) {
if (atomic_compare_exchange_weak_explicit(
&mutex->_word, &word, MUTEX_INC_DEPTH(word),
memory_order_relaxed, memory_order_relaxed))
return 0;
continue;
} else {
return EAGAIN;
}
} else {
return EDEADLK;
}
}
word = MUTEX_UNLOCK(word);
lock = MUTEX_LOCK(word);
lock = MUTEX_SET_OWNER(lock, me);
if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
memory_order_acquire,
memory_order_relaxed)) {
mutex->_pid = __pid;
return 0;
}
pthread_pause_np();
}
LOCKTRACE("acquiring %t...", mutex);
errno_t err = pthread_mutex_lock_impl(mutex);
LOCKTRACE("pthread_mutex_lock(%t) → %s", mutex, DescribeErrno(err));
return err;
}

1
libc/intrin/pthread_mutex_trylock.c
View file

@ -17,6 +17,7 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/weaken.h"

1
libc/intrin/pthread_mutex_unlock.c
View file

@ -17,6 +17,7 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/strace.internal.h"

2
libc/intrin/pthread_syshand.c
View file

@ -29,6 +29,6 @@ intptr_t _pthread_syshand(struct PosixThread *pt) {
syshand = atomic_load_explicit(&pt->tib->tib_syshand, memory_order_acquire);
if (syshand)
return syshand;
pthread_pause_np();
pthread_yield_np();
}
}

2
libc/intrin/pthread_tid.c
View file

@ -24,6 +24,6 @@
int _pthread_tid(struct PosixThread *pt) {
int tid = 0;
while (pt && !(tid = atomic_load_explicit(&pt->ptid, memory_order_acquire)))
pthread_pause_np();
pthread_yield_np();
return tid;
}

0
libc/calls/pthread_yield_np.c → libc/intrin/pthread_yield_np.c
View file

312
libc/intrin/rbtree.c
View file

@ -1,312 +0,0 @@
// Copyright 2024 Justine Alexandra Roberts Tunney
//
// Permission to use, copy, modify, and/or distribute this software for
// any purpose with or without fee is hereby granted, provided that the
// above copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
// WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
// AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
// DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
// PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
#include "rbtree.h"
#include "libc/dce.h"
#include "libc/str/str.h"
#define RBTREE_DEBUG
struct rbtree *rbtree_next(struct rbtree *node) {
if (!node)
return 0;
if (node->right)
return rbtree_first(node->right);
struct rbtree *parent = node->parent;
while (parent && node == parent->right) {
node = parent;
parent = parent->parent;
}
return parent;
}
struct rbtree *rbtree_prev(struct rbtree *node) {
if (!node)
return 0;
if (rbtree_get_left(node))
return rbtree_last(rbtree_get_left(node));
struct rbtree *parent = node->parent;
while (parent && node == rbtree_get_left(parent)) {
node = parent;
parent = parent->parent;
}
return parent;
}
struct rbtree *rbtree_first(struct rbtree *node) {
while (node && rbtree_get_left(node))
node = rbtree_get_left(node);
return node;
}
struct rbtree *rbtree_last(struct rbtree *node) {
while (node && node->right)
node = node->right;
return node;
}
static void rbtree_rotate_left(struct rbtree **root, struct rbtree *x) {
struct rbtree *y = x->right;
x->right = rbtree_get_left(y);
if (rbtree_get_left(y))
rbtree_get_left(y)->parent = x;
y->parent = x->parent;
if (!x->parent) {
*root = y;
} else if (x == rbtree_get_left(x->parent)) {
rbtree_set_left(x->parent, y);
} else {
x->parent->right = y;
}
rbtree_set_left(y, x);
x->parent = y;
}
static void rbtree_rotate_right(struct rbtree **root, struct rbtree *y) {
struct rbtree *x = rbtree_get_left(y);
rbtree_set_left(y, x->right);
if (x->right)
x->right->parent = y;
x->parent = y->parent;
if (!y->parent) {
*root = x;
} else if (y == y->parent->right) {
y->parent->right = x;
} else {
rbtree_set_left(y->parent, x);
}
x->right = y;
y->parent = x;
}
static void rbtree_insert_fixup(struct rbtree **root, struct rbtree *node) {
rbtree_set_red(node, 1);
while (node != *root && rbtree_get_red(node->parent)) {
if (node->parent == rbtree_get_left(node->parent->parent)) {
struct rbtree *uncle = node->parent->parent->right;
if (uncle && rbtree_get_red(uncle)) {
rbtree_set_red(node->parent, 0);
rbtree_set_red(uncle, 0);
rbtree_set_red(node->parent->parent, 1);
node = node->parent->parent;
} else {
if (node == node->parent->right) {
node = node->parent;
rbtree_rotate_left(root, node);
}
rbtree_set_red(node->parent, 0);
rbtree_set_red(node->parent->parent, 1);
rbtree_rotate_right(root, node->parent->parent);
}
} else {
struct rbtree *uncle = rbtree_get_left(node->parent->parent);
if (uncle && rbtree_get_red(uncle)) {
rbtree_set_red(node->parent, 0);
rbtree_set_red(uncle, 0);
rbtree_set_red(node->parent->parent, 1);
node = node->parent->parent;
} else {
if (node == rbtree_get_left(node->parent)) {
node = node->parent;
rbtree_rotate_right(root, node);
}
rbtree_set_red(node->parent, 0);
rbtree_set_red(node->parent->parent, 1);
rbtree_rotate_left(root, node->parent->parent);
}
}
}
rbtree_set_red(*root, 0);
}
void rbtree_insert(struct rbtree **root, struct rbtree *node,
rbtree_cmp_f *cmp) {
bzero(node, sizeof(*node));
if (!*root) {
*root = node;
} else {
struct rbtree *search = *root;
struct rbtree *parent = 0;
do {
parent = search;
if (cmp(node, search) < 0) {
search = rbtree_get_left(search);
} else {
search = search->right;
}
} while (search);
if (cmp(node, parent) < 0) {
rbtree_set_left(parent, node);
} else {
parent->right = node;
}
node->parent = parent;
rbtree_insert_fixup(root, node);
}
}
static void rbtree_transplant(struct rbtree **root, struct rbtree *u,
struct rbtree *v) {
if (!u->parent) {
*root = v;
} else if (u == rbtree_get_left(u->parent)) {
rbtree_set_left(u->parent, v);
} else {
u->parent->right = v;
}
if (v)
v->parent = u->parent;
}
static void rbtree_remove_fixup(struct rbtree **root, struct rbtree *node,
struct rbtree *parent) {
while (node != *root && (!node || !rbtree_get_red(node))) {
if (node == rbtree_get_left(parent)) {
struct rbtree *sibling = parent->right;
if (rbtree_get_red(sibling)) {
rbtree_set_red(sibling, 0);
rbtree_set_red(parent, 1);
rbtree_rotate_left(root, parent);
sibling = parent->right;
}
if ((!rbtree_get_left(sibling) ||
!rbtree_get_red(rbtree_get_left(sibling))) &&
(!sibling->right || !rbtree_get_red(sibling->right))) {
rbtree_set_red(sibling, 1);
node = parent;
parent = node->parent;
} else {
if (!sibling->right || !rbtree_get_red(sibling->right)) {
rbtree_set_red(rbtree_get_left(sibling), 0);
rbtree_set_red(sibling, 1);
rbtree_rotate_right(root, sibling);
sibling = parent->right;
}
rbtree_set_red(sibling, rbtree_get_red(parent));
rbtree_set_red(parent, 0);
rbtree_set_red(sibling->right, 0);
rbtree_rotate_left(root, parent);
node = *root;
break;
}
} else {
struct rbtree *sibling = rbtree_get_left(parent);
if (rbtree_get_red(sibling)) {
rbtree_set_red(sibling, 0);
rbtree_set_red(parent, 1);
rbtree_rotate_right(root, parent);
sibling = rbtree_get_left(parent);
}
if ((!sibling->right || !rbtree_get_red(sibling->right)) &&
(!rbtree_get_left(sibling) ||
!rbtree_get_red(rbtree_get_left(sibling)))) {
rbtree_set_red(sibling, 1);
node = parent;
parent = node->parent;
} else {
if (!rbtree_get_left(sibling) ||
!rbtree_get_red(rbtree_get_left(sibling))) {
rbtree_set_red(sibling->right, 0);
rbtree_set_red(sibling, 1);
rbtree_rotate_left(root, sibling);
sibling = rbtree_get_left(parent);
}
rbtree_set_red(sibling, rbtree_get_red(parent));
rbtree_set_red(parent, 0);
rbtree_set_red(rbtree_get_left(sibling), 0);
rbtree_rotate_right(root, parent);
node = *root;
break;
}
}
}
if (node)
rbtree_set_red(node, 0);
}
void rbtree_remove(struct rbtree **root, struct rbtree *node) {
struct rbtree *y = node;
struct rbtree *x = 0;
struct rbtree *x_parent = 0;
int y_original_color = rbtree_get_red(y);
if (!rbtree_get_left(node)) {
x = node->right;
rbtree_transplant(root, node, node->right);
x_parent = node->parent;
} else if (!node->right) {
x = rbtree_get_left(node);
rbtree_transplant(root, node, rbtree_get_left(node));
x_parent = node->parent;
} else {
y = rbtree_first(node->right);
y_original_color = rbtree_get_red(y);
x = y->right;
if (y->parent == node) {
if (x)
x->parent = y;
x_parent = y;
} else {
rbtree_transplant(root, y, y->right);
y->right = node->right;
y->right->parent = y;
x_parent = y->parent;
}
rbtree_transplant(root, node, y);
rbtree_set_left(y, rbtree_get_left(node));
rbtree_get_left(y)->parent = y;
rbtree_set_red(y, rbtree_get_red(node));
}
if (!y_original_color)
rbtree_remove_fixup(root, x, x_parent);
}
struct rbtree *rbtree_get(const struct rbtree *node, const struct rbtree *key,
rbtree_cmp_f *cmp) {
while (node) {
int c = cmp(key, node);
if (c < 0) {
node = rbtree_get_left(node);
} else if (c > 0) {
node = node->right;
} else {
return (struct rbtree *)node;
}
}
return 0;
}
struct rbtree *rbtree_floor(const struct rbtree *node, const struct rbtree *key,
rbtree_cmp_f *cmp) {
while (node) {
if (cmp(key, node) < 0) {
node = rbtree_get_left(node);
} else {
node = node->right;
}
}
return (struct rbtree *)node;
}
struct rbtree *rbtree_ceil(const struct rbtree *node, const struct rbtree *key,
rbtree_cmp_f *cmp) {
while (node) {
if (cmp(node, key) < 0) {
node = rbtree_get_left(node);
} else {
node = node->right;
}
}
return (struct rbtree *)node;
}

57
libc/intrin/rbtree.h
View file

@ -1,57 +0,0 @@
#ifdef _COSMO_SOURCE
#ifndef COSMOPOLITAN_RBTREE_H_
#define COSMOPOLITAN_RBTREE_H_
#define rbtree_ceil __rbtree_ceil
#define rbtree_first __rbtree_first
#define rbtree_floor __rbtree_floor
#define rbtree_get __rbtree_get
#define rbtree_insert __rbtree_insert
#define rbtree_last __rbtree_last
#define rbtree_next __rbtree_next
#define rbtree_prev __rbtree_prev
#define rbtree_remove __rbtree_remove
COSMOPOLITAN_C_START_
#define RBTREE_CONTAINER(t, f, p) ((t *)(((char *)(p)) - offsetof(t, f)))
struct rbtree {
uintptr_t word;
struct rbtree *right;
struct rbtree *parent;
};
typedef int rbtree_cmp_f(const struct rbtree *, const struct rbtree *);
static inline struct rbtree *rbtree_get_left(const struct rbtree *node) {
return (struct rbtree *)(node->word & -2);
}
static inline void rbtree_set_left(struct rbtree *node, struct rbtree *left) {
node->word = (uintptr_t)left | (node->word & 1);
}
static inline int rbtree_get_red(const struct rbtree *node) {
return node->word & 1;
}
static inline void rbtree_set_red(struct rbtree *node, int red) {
node->word &= -2;
node->word |= red;
}
struct rbtree *rbtree_next(struct rbtree *) libcesque;
struct rbtree *rbtree_prev(struct rbtree *) libcesque;
struct rbtree *rbtree_first(struct rbtree *) libcesque;
struct rbtree *rbtree_last(struct rbtree *) libcesque;
void rbtree_remove(struct rbtree **, struct rbtree *) libcesque;
void rbtree_insert(struct rbtree **, struct rbtree *, rbtree_cmp_f *) libcesque;
struct rbtree *rbtree_get(const struct rbtree *, const struct rbtree *,
rbtree_cmp_f *) libcesque;
struct rbtree *rbtree_ceil(const struct rbtree *, const struct rbtree *,
rbtree_cmp_f *) libcesque;
struct rbtree *rbtree_floor(const struct rbtree *, const struct rbtree *,
rbtree_cmp_f *) libcesque;
COSMOPOLITAN_C_END_
#endif /* COSMOPOLITAN_RBTREE_H_ */
#endif /* _COSMO_SOURCE */

271
libc/intrin/tree.c Normal file
View file

@ -0,0 +1,271 @@
// Copyright 2024 Justine Alexandra Roberts Tunney
//
// Permission to use, copy, modify, and/or distribute this software for
// any purpose with or without fee is hereby granted, provided that the
// above copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
// WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
// AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
// DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
// PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
#include "tree.h"
struct Tree *tree_last(struct Tree *node) {
while (node && node->right)
node = node->right;
return node;
}
struct Tree *tree_first(struct Tree *node) {
while (node && tree_get_left(node))
node = tree_get_left(node);
return node;
}
struct Tree *tree_next(struct Tree *node) {
if (!node)
return 0;
if (node->right)
return tree_first(node->right);
struct Tree *parent = node->parent;
while (parent && node == parent->right) {
node = parent;
parent = parent->parent;
}
return parent;
}
struct Tree *tree_prev(struct Tree *node) {
struct Tree *parent;
if (!node)
return 0;
if (tree_get_left(node))
return tree_last(tree_get_left(node));
parent = node->parent;
while (parent && node == tree_get_left(parent)) {
node = parent;
parent = parent->parent;
}
return parent;
}
static void tree_rotate_left(struct Tree **root, struct Tree *x) {
struct Tree *y = x->right;
x->right = tree_get_left(y);
if (tree_get_left(y))
tree_get_left(y)->parent = x;
y->parent = x->parent;
if (!x->parent) {
*root = y;
} else if (x == tree_get_left(x->parent)) {
tree_set_left(x->parent, y);
} else {
x->parent->right = y;
}
tree_set_left(y, x);
x->parent = y;
}
static void tree_rotate_right(struct Tree **root, struct Tree *y) {
struct Tree *x = tree_get_left(y);
tree_set_left(y, x->right);
if (x->right)
x->right->parent = y;
x->parent = y->parent;
if (!y->parent) {
*root = x;
} else if (y == y->parent->right) {
y->parent->right = x;
} else {
tree_set_left(y->parent, x);
}
y->parent = x;
x->right = y;
}
static void tree_rebalance_insert(struct Tree **root, struct Tree *node) {
struct Tree *uncle;
tree_set_red(node, 1);
while (node != *root && tree_get_red(node->parent)) {
if (node->parent == tree_get_left(node->parent->parent)) {
uncle = node->parent->parent->right;
if (uncle && tree_get_red(uncle)) {
tree_set_red(node->parent, 0);
tree_set_red(uncle, 0);
tree_set_red(node->parent->parent, 1);
node = node->parent->parent;
} else {
if (node == node->parent->right) {
node = node->parent;
tree_rotate_left(root, node);
}
tree_set_red(node->parent, 0);
tree_set_red(node->parent->parent, 1);
tree_rotate_right(root, node->parent->parent);
}
} else {
uncle = tree_get_left(node->parent->parent);
if (uncle && tree_get_red(uncle)) {
tree_set_red(node->parent, 0);
tree_set_red(uncle, 0);
tree_set_red(node->parent->parent, 1);
node = node->parent->parent;
} else {
if (node == tree_get_left(node->parent)) {
node = node->parent;
tree_rotate_right(root, node);
}
tree_set_red(node->parent, 0);
tree_set_red(node->parent->parent, 1);
tree_rotate_left(root, node->parent->parent);
}
}
}
tree_set_red(*root, 0);
}
void tree_insert(struct Tree **root, struct Tree *node, tree_cmp_f *cmp) {
struct Tree *search, *parent;
node->word = 0;
node->right = 0;
node->parent = 0;
if (!*root) {
*root = node;
} else {
search = *root;
parent = 0;
do {
parent = search;
if (cmp(node, search) < 0) {
search = tree_get_left(search);
} else {
search = search->right;
}
} while (search);
if (cmp(node, parent) < 0) {
tree_set_left(parent, node);
} else {
parent->right = node;
}
node->parent = parent;
tree_rebalance_insert(root, node);
}
}
static void tree_transplant(struct Tree **root, struct Tree *u,
struct Tree *v) {
if (!u->parent) {
*root = v;
} else if (u == tree_get_left(u->parent)) {
tree_set_left(u->parent, v);
} else {
u->parent->right = v;
}
if (v)
v->parent = u->parent;
}
static void tree_rebalance_remove(struct Tree **root, struct Tree *node,
struct Tree *parent) {
struct Tree *sibling;
while (node != *root && (!node || !tree_get_red(node))) {
if (node == tree_get_left(parent)) {
sibling = parent->right;
if (tree_get_red(sibling)) {
tree_set_red(sibling, 0);
tree_set_red(parent, 1);
tree_rotate_left(root, parent);
sibling = parent->right;
}
if ((!tree_get_left(sibling) || !tree_get_red(tree_get_left(sibling))) &&
(!sibling->right || !tree_get_red(sibling->right))) {
tree_set_red(sibling, 1);
node = parent;
parent = node->parent;
} else {
if (!sibling->right || !tree_get_red(sibling->right)) {
tree_set_red(tree_get_left(sibling), 0);
tree_set_red(sibling, 1);
tree_rotate_right(root, sibling);
sibling = parent->right;
}
tree_set_red(sibling, tree_get_red(parent));
tree_set_red(parent, 0);
tree_set_red(sibling->right, 0);
tree_rotate_left(root, parent);
node = *root;
break;
}
} else {
sibling = tree_get_left(parent);
if (tree_get_red(sibling)) {
tree_set_red(sibling, 0);
tree_set_red(parent, 1);
tree_rotate_right(root, parent);
sibling = tree_get_left(parent);
}
if ((!sibling->right || !tree_get_red(sibling->right)) &&
(!tree_get_left(sibling) || !tree_get_red(tree_get_left(sibling)))) {
tree_set_red(sibling, 1);
node = parent;
parent = node->parent;
} else {
if (!tree_get_left(sibling) || !tree_get_red(tree_get_left(sibling))) {
tree_set_red(sibling->right, 0);
tree_set_red(sibling, 1);
tree_rotate_left(root, sibling);
sibling = tree_get_left(parent);
}
tree_set_red(sibling, tree_get_red(parent));
tree_set_red(parent, 0);
tree_set_red(tree_get_left(sibling), 0);
tree_rotate_right(root, parent);
node = *root;
break;
}
}
}
if (node)
tree_set_red(node, 0);
}
void tree_remove(struct Tree **root, struct Tree *node) {
struct Tree *x = 0;
struct Tree *y = node;
struct Tree *x_parent = 0;
int y_original_color = tree_get_red(y);
if (!tree_get_left(node)) {
x = node->right;
tree_transplant(root, node, node->right);
x_parent = node->parent;
} else if (!node->right) {
x = tree_get_left(node);
tree_transplant(root, node, tree_get_left(node));
x_parent = node->parent;
} else {
y = tree_first(node->right);
y_original_color = tree_get_red(y);
x = y->right;
if (y->parent == node) {
if (x)
x->parent = y;
x_parent = y;
} else {
tree_transplant(root, y, y->right);
y->right = node->right;
y->right->parent = y;
x_parent = y->parent;
}
tree_transplant(root, node, y);
tree_set_left(y, tree_get_left(node));
tree_get_left(y)->parent = y;
tree_set_red(y, tree_get_red(node));
}
if (!y_original_color)
tree_rebalance_remove(root, x, x_parent);
}

91
libc/intrin/tree.h Normal file
View file

@ -0,0 +1,91 @@
#ifndef COSMOPOLITAN_TREE_H_
#define COSMOPOLITAN_TREE_H_
#define tree_first __tree_first
#define tree_insert __tree_insert
#define tree_last __tree_last
#define tree_next __tree_next
#define tree_prev __tree_prev
#define tree_remove __tree_remove
COSMOPOLITAN_C_START_
#define TREE_CONTAINER(t, f, p) ((t *)(((char *)(p)) - offsetof(t, f)))
struct Tree {
uintptr_t word;
struct Tree *right;
struct Tree *parent;
};
typedef int tree_search_f(const void *, const struct Tree *);
typedef int tree_cmp_f(const struct Tree *, const struct Tree *);
forceinline struct Tree *tree_get_left(const struct Tree *node) {
return (struct Tree *)(node->word & -2);
}
static inline void tree_set_left(struct Tree *node, struct Tree *left) {
node->word = (uintptr_t)left | (node->word & 1);
}
static inline int tree_get_red(const struct Tree *node) {
return node->word & 1;
}
static inline void tree_set_red(struct Tree *node, int red) {
node->word &= -2;
node->word |= red;
}
forceinline optimizespeed struct Tree *tree_floor(const struct Tree *node,
const void *key,
tree_search_f *cmp) {
struct Tree *left = 0;
while (node) {
if (cmp(key, node) >= 0) {
left = (struct Tree *)node;
node = tree_get_left(node);
} else {
node = node->right;
}
}
return left;
}
static inline struct Tree *tree_ceil(const struct Tree *node, const void *key,
tree_search_f *cmp) {
struct Tree *right = 0;
while (node) {
if (cmp(key, node) < 0) {
right = (struct Tree *)node;
node = tree_get_left(node);
} else {
node = node->right;
}
}
return right;
}
static inline struct Tree *tree_get(const struct Tree *node, const void *key,
tree_search_f *cmp) {
while (node) {
int c = cmp(key, node);
if (c < 0) {
node = tree_get_left(node);
} else if (c > 0) {
node = node->right;
} else {
return (struct Tree *)node;
}
}
return 0;
}
struct Tree *tree_next(struct Tree *) libcesque;
struct Tree *tree_prev(struct Tree *) libcesque;
struct Tree *tree_first(struct Tree *) libcesque;
struct Tree *tree_last(struct Tree *) libcesque;
void tree_remove(struct Tree **, struct Tree *) libcesque;
void tree_insert(struct Tree **, struct Tree *, tree_cmp_f *) libcesque;
COSMOPOLITAN_C_END_
#endif /* COSMOPOLITAN_TREE_H_ */

10
libc/log/oncrash_amd64.c
View file

@ -241,14 +241,10 @@ static relegated void ShowCrashReport(int err, int sig, siginfo_t *si,
klog(buf, p - buf);
}
kprintf("\n");
if (!IsWindows()) {
__print_maps();
}
if (__argv) {
for (i = 0; i < __argc; ++i) {
__print_maps(15);
if (__argv)
for (i = 0; i < __argc; ++i)
kprintf("%s ", __argv[i]);
}
}
kprintf("\n");
}

142
libc/proc/fork-nt.c
View file

@ -30,6 +30,7 @@
#include "libc/intrin/kprintf.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/tree.h"
#include "libc/intrin/weaken.h"
#include "libc/macros.internal.h"
#include "libc/nt/createfile.h"
@ -95,7 +96,7 @@ static inline textwindows ssize_t ForkIo(int64_t h, char *p, size_t n,
size_t i;
uint32_t x;
for (i = 0; i < n; i += x) {
if (!f(h, p + i, n - i, &x, NULL))
if (!f(h, p + i, n - i, &x, 0))
return __winerr();
if (!x)
break;
@ -109,10 +110,11 @@ static dontinline textwindows ssize_t ForkIo2(
const char *sf, bool ischild) {
ssize_t rc = ForkIo(h, buf, n, fn);
if (ischild) {
__tls_enabled_set(false); // prevent tls crash in kprintf
// prevent crashes
__tls_enabled_set(false);
__pid = __imp_GetCurrentProcessId();
__klog_handle = 0;
__maps.used = 0;
__maps.maps = 0;
}
NTTRACE("%s(%ld, %p, %'zu) → %'zd% m", sf, h, buf, n, rc);
return rc;
@ -121,9 +123,11 @@ static dontinline textwindows ssize_t ForkIo2(
static dontinline textwindows bool WriteAll(int64_t h, void *buf, size_t n) {
bool ok;
ok = ForkIo2(h, buf, n, (void *)WriteFile, "WriteFile", false) != -1;
if (!ok)
AbortFork("WriteAll");
// Sleep(10);
if (!ok) {
STRACE("fork() failed in parent due to WriteAll(%ld, %p, %'zu) → %u", h,
buf, n, GetLastError());
__print_maps(0);
}
return ok;
}
@ -185,30 +189,6 @@ static textwindows void *Malloc(size_t size) {
return HeapAlloc(GetProcessHeap(), 0, size);
}
static textwindows void Free(void *addr) {
HeapFree(GetProcessHeap(), 0, addr);
}
static int CountMaps(struct Dll *maps) {
int count = 0;
for (struct Dll *e = dll_first(maps); e; e = dll_next(maps, e))
++count;
return count;
}
static struct Map **SortMaps(struct Dll *maps, int count) {
int j, i = 0;
struct Map **sorted = Malloc(count * sizeof(struct Map *));
for (struct Dll *e = dll_first(maps); e; e = dll_next(maps, e)) {
struct Map *map = MAP_CONTAINER(e);
for (j = i; j > 0 && sorted[j - 1]->addr > map->addr; --j)
sorted[j] = sorted[j - 1];
sorted[j] = map;
++i;
}
return sorted;
}
textwindows void WinMainForked(void) {
jmp_buf jb;
int64_t reader;
@ -233,35 +213,30 @@ textwindows void WinMainForked(void) {
ReadOrDie(reader, jb, sizeof(jb));
// read memory mappings from parent process
int n = 0;
struct Dll *maps = 0;
struct Tree *maps = 0;
for (;;) {
struct Map *map = Malloc(sizeof(struct Map));
ReadOrDie(reader, map, sizeof(struct Map));
if (map->addr == MAP_FAILED) {
Free(map);
if (map->addr == MAP_FAILED)
break;
}
dll_init(&map->elem);
dll_make_first(&maps, &map->elem);
++n;
tree_insert(&maps, &map->tree, __maps_compare);
}
// created sorted array of maps
struct Map **sorted = SortMaps(maps, n);
// map memory into process
int granularity = __granularity();
for (int i = 0; i < n; ++i) {
struct Map *map = sorted[i];
for (struct Tree *e = tree_first(maps); e; e = tree_next(e)) {
struct Map *map = MAP_TREE_CONTAINER(e);
if ((uintptr_t)map->addr & (granularity - 1))
continue;
size_t size = map->size;
// get true length in case mprotect() chopped up actual win32 map
for (int j = i + 1;
j < n && sorted[j]->hand == -1 && map->addr + size == sorted[j]->addr;
++j) {
size += sorted[j]->size;
size_t size = map->size;
for (struct Tree *e2 = tree_next(e); e2; e2 = tree_next(e2)) {
struct Map *map2 = MAP_TREE_CONTAINER(e2);
if (map2->hand == -1 && map->addr + size == map2->addr) {
size += map2->size;
} else {
break;
}
}
// obtain the most permissive access possible
unsigned prot, access;
@ -295,11 +270,11 @@ textwindows void WinMainForked(void) {
// fixup memory manager
__maps.free = 0;
__maps.used = 0;
__maps.maps = 0;
__maps.count = 0;
__maps.pages = 0;
for (int i = 0; i < n; ++i) {
struct Map *map = sorted[i];
for (struct Tree *e = tree_first(maps); e; e = tree_next(e)) {
struct Map *map = MAP_TREE_CONTAINER(e);
__maps.count += 1;
__maps.pages += (map->size + getpagesize() - 1) / getpagesize();
unsigned old_protect;
@ -307,8 +282,7 @@ textwindows void WinMainForked(void) {
&old_protect))
AbortFork("VirtualProtect");
}
Free(sorted);
__maps.used = maps;
__maps.maps = maps;
__maps_init();
// mitosis complete
@ -393,19 +367,13 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
if (spawnrc != -1) {
CloseHandle(procinfo.hThread);
ok = WriteAll(writer, jb, sizeof(jb));
int count = 0;
// this list will be populated with the maps we're transferring
struct Dll *e2, *maps = 0;
for (struct Dll *e = dll_first(__maps.used); ok && e; e = e2) {
e2 = dll_next(__maps.used, e);
struct Map *map = MAP_CONTAINER(e);
for (struct Map *map = __maps_first(); ok && map;
map = __maps_next(map)) {
if (MAX((char *)__executable_start, map->addr) <
MIN((char *)_end, map->addr + map->size))
continue; // executable image is loaded by windows
dll_remove(&__maps.used, e);
dll_make_last(&maps, e);
ok = WriteAll(writer, map, sizeof(*map));
++count;
}
// send a terminating Map struct to child
if (ok) {
@ -415,40 +383,44 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
}
// now write content of each map to child
int granularity = __granularity();
struct Map **sorted = SortMaps(maps, count);
uint32_t *old_protect = Malloc(count * 4);
for (int i = 0; ok && i < count; ++i) {
struct Map *map = sorted[i];
for (struct Map *map = __maps_first(); ok && map;
map = __maps_next(map)) {
// we only need to worry about the base mapping
if ((uintptr_t)map->addr & (granularity - 1))
continue;
if (MAX((char *)__executable_start, map->addr) <
MIN((char *)_end, map->addr + map->size))
continue; // executable image is loaded by windows
// shared mappings don't need to be copied
if ((map->flags & MAP_TYPE) == MAP_SHARED)
continue;
// get true length in case mprotect() chopped up actual win32 map
int j;
size_t size = map->size;
for (j = i + 1; j < count && sorted[j]->hand == -1 &&
map->addr + size == sorted[j]->addr;
++j) {
size += sorted[j]->size;
for (struct Map *map2 = __maps_next(map); map2;
map2 = __maps_next(map2)) {
if (map2->hand == -1 && map->addr + size == map2->addr) {
size += map2->size;
} else {
break;
}
}
for (struct Map *map2 = map; ok && map2; map2 = __maps_next(map2)) {
if (!(map2->prot & PROT_READ))
if (map->addr >= map2->addr && map->addr < map->addr + size)
ok = VirtualProtect(
map2->addr, map2->size,
__prot2nt(map2->prot | PROT_READ, map2->iscow),
&map2->oldprot);
}
for (int k = i; ok && k < j; ++k)
if (!(sorted[k]->prot & PROT_READ))
ok = VirtualProtect(
sorted[k]->addr, sorted[k]->size,
__prot2nt(sorted[k]->prot | PROT_READ, map->iscow),
&old_protect[k]);
if (ok)
ok = WriteAll(writer, map->addr, size);
for (int k = i; ok && k < j; ++k)
if (!(sorted[k]->prot & PROT_READ))
ok = VirtualProtect(sorted[k]->addr, sorted[k]->size,
old_protect[k], &old_protect[k]);
for (struct Map *map2 = map; ok && map2; map2 = __maps_next(map2)) {
if (!(map2->prot & PROT_READ))
if (map->addr >= map2->addr && map->addr < map->addr + size)
ok = VirtualProtect(map2->addr, map2->size, map2->oldprot,
&map2->oldprot);
}
}
Free(old_protect);
Free(sorted);
dll_make_first(&__maps.used, maps);
if (ok)
ok = WriteAll(writer, __data_start, __data_end - __data_start);
if (ok)
@ -466,6 +438,7 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
} else {
TerminateProcess(procinfo.hProcess, SIGKILL);
CloseHandle(procinfo.hProcess);
rc = -1;
}
}
}
@ -473,9 +446,8 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
CloseHandle(reader);
if (writer != -1)
CloseHandle(writer);
if (rc == -1 && errno != ENOMEM) {
if (rc == -1 && errno != ENOMEM)
eagain(); // posix fork() only specifies two errors
}
} else {
rc = 0;
// re-apply code morphing for thread-local storage

17
libc/proc/fork.c
View file

@ -22,6 +22,7 @@
#include "libc/calls/state.internal.h"
#include "libc/calls/struct/sigset.h"
#include "libc/calls/struct/sigset.internal.h"
#include "libc/calls/struct/timespec.h"
#include "libc/calls/syscall-nt.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
@ -45,7 +46,6 @@
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/tls.h"
extern pthread_mutex_t nsync_waiters_mu;
extern pthread_mutex_t _pthread_lock_obj;
static void _onfork_prepare(void) {
@ -54,11 +54,10 @@ static void _onfork_prepare(void) {
_pthread_lock();
__maps_lock();
__fds_lock();
pthread_mutex_lock(&nsync_waiters_mu);
LOCKTRACE("READY TO ROCK AND ROLL");
}
static void _onfork_parent(void) {
pthread_mutex_unlock(&nsync_waiters_mu);
__fds_unlock();
__maps_unlock();
_pthread_unlock();
@ -68,7 +67,6 @@ static void _onfork_parent(void) {
static void _onfork_child(void) {
__fds_lock_obj = (pthread_mutex_t)PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
nsync_waiters_mu = (pthread_mutex_t)PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
_pthread_lock_obj = (pthread_mutex_t)PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
atomic_store_explicit(&__maps.lock, 0, memory_order_relaxed);
atomic_store_explicit(&__get_tls()->tib_relock_maps, 0, memory_order_relaxed);
@ -77,7 +75,9 @@ static void _onfork_child(void) {
}
int _fork(uint32_t dwCreationFlags) {
long micros;
struct Dll *e;
struct timespec started;
int ax, dx, tid, parent;
parent = __pid;
BLOCK_SIGNALS;
@ -85,11 +85,13 @@ int _fork(uint32_t dwCreationFlags) {
__proc_lock();
if (__threaded)
_onfork_prepare();
started = timespec_real();
if (!IsWindows()) {
ax = sys_fork();
} else {
ax = sys_fork_nt(dwCreationFlags);
}
micros = timespec_tomicros(timespec_sub(timespec_real(), started));
if (!ax) {
// get new process id
@ -136,15 +138,14 @@ int _fork(uint32_t dwCreationFlags) {
// run user fork callbacks
if (__threaded)
_onfork_child();
STRACE("fork() → 0 (child of %d)", parent);
STRACE("fork() → 0 (child of %d; took %ld us)", parent, micros);
} else {
// this is the parent process
if (__threaded) {
if (__threaded)
_onfork_parent();
}
if (IsWindows())
__proc_unlock();
STRACE("fork() → %d% m", ax);
STRACE("fork() → %d% m (took %ld us)", ax, micros);
}
ALLOW_SIGNALS;
return ax;

57
libc/proc/posix_spawn.c
View file

@ -482,9 +482,8 @@ errno_t posix_spawn(int *pid, const char *path,
const posix_spawn_file_actions_t *file_actions,
const posix_spawnattr_t *attrp, char *const argv[],
char *const envp[]) {
if (IsWindows()) {
if (IsWindows())
return posix_spawn_nt(pid, path, file_actions, attrp, argv, envp);
}
int pfds[2];
bool use_pipe;
volatile int status = 0;
@ -516,66 +515,55 @@ errno_t posix_spawn(int *pid, const char *path,
sigaction(sig, &dfl, 0);
}
}
if (flags & POSIX_SPAWN_SETSID) {
if (flags & POSIX_SPAWN_SETSID)
setsid();
}
if ((flags & POSIX_SPAWN_SETPGROUP) && setpgid(0, (*attrp)->pgroup)) {
if ((flags & POSIX_SPAWN_SETPGROUP) && setpgid(0, (*attrp)->pgroup))
goto ChildFailed;
}
if ((flags & POSIX_SPAWN_RESETIDS) && setgid(getgid())) {
if ((flags & POSIX_SPAWN_RESETIDS) && setgid(getgid()))
goto ChildFailed;
}
if ((flags & POSIX_SPAWN_RESETIDS) && setuid(getuid())) {
if ((flags & POSIX_SPAWN_RESETIDS) && setuid(getuid()))
goto ChildFailed;
}
if (file_actions) {
struct _posix_faction *a;
for (a = *file_actions; a; a = a->next) {
if (use_pipe && pfds[1] == a->fildes) {
int p2;
if ((p2 = dup(pfds[1])) == -1) {
if ((p2 = dup(pfds[1])) == -1)
goto ChildFailed;
}
lost_cloexec = true;
close(pfds[1]);
pfds[1] = p2;
}
switch (a->action) {
case _POSIX_SPAWN_CLOSE:
if (close(a->fildes)) {
if (close(a->fildes))
goto ChildFailed;
}
break;
case _POSIX_SPAWN_DUP2:
if (dup2(a->fildes, a->newfildes) == -1) {
if (dup2(a->fildes, a->newfildes) == -1)
goto ChildFailed;
}
break;
case _POSIX_SPAWN_OPEN: {
int t;
if ((t = openat(AT_FDCWD, a->path, a->oflag, a->mode)) == -1) {
if ((t = openat(AT_FDCWD, a->path, a->oflag, a->mode)) == -1)
goto ChildFailed;
}
if (t != a->fildes) {
if (dup2(t, a->fildes) == -1) {
close(t);
goto ChildFailed;
}
if (close(t)) {
if (close(t))
goto ChildFailed;
}
}
break;
}
case _POSIX_SPAWN_CHDIR:
if (chdir(a->path) == -1) {
if (chdir(a->path) == -1)
goto ChildFailed;
}
break;
case _POSIX_SPAWN_FCHDIR:
if (fchdir(a->fildes) == -1) {
if (fchdir(a->fildes) == -1)
goto ChildFailed;
}
break;
default:
__builtin_unreachable();
@ -583,17 +571,13 @@ errno_t posix_spawn(int *pid, const char *path,
}
}
if (IsLinux() || IsFreebsd() || IsNetbsd()) {
if (flags & POSIX_SPAWN_SETSCHEDULER) {
if (flags & POSIX_SPAWN_SETSCHEDULER)
if (sched_setscheduler(0, (*attrp)->schedpolicy,
&(*attrp)->schedparam) == -1) {
&(*attrp)->schedparam) == -1)
goto ChildFailed;
}
}
if (flags & POSIX_SPAWN_SETSCHEDPARAM) {
if (sched_setparam(0, &(*attrp)->schedparam)) {
if (flags & POSIX_SPAWN_SETSCHEDPARAM)
if (sched_setparam(0, &(*attrp)->schedparam))
goto ChildFailed;
}
}
}
if (flags & POSIX_SPAWN_SETRLIMIT) {
int rlimset = (*attrp)->rlimset;
@ -608,9 +592,8 @@ errno_t posix_spawn(int *pid, const char *path,
}
}
}
if (lost_cloexec) {
if (lost_cloexec)
fcntl(pfds[1], F_SETFD, FD_CLOEXEC);
}
if (flags & POSIX_SPAWN_SETSIGMASK) {
childmask = (*attrp)->sigmask;
} else {
@ -636,9 +619,8 @@ errno_t posix_spawn(int *pid, const char *path,
if (!use_pipe) {
res = status;
} else {
if (can_clobber) {
if (can_clobber)
atomic_store_explicit(&has_vfork, true, memory_order_release);
}
res = 0;
read(pfds[0], &res, sizeof(res));
}
@ -651,9 +633,8 @@ errno_t posix_spawn(int *pid, const char *path,
} else {
res = errno;
}
if (use_pipe) {
if (use_pipe)
close(pfds[0]);
}
ParentFailed:
sigprocmask(SIG_SETMASK, &oldmask, 0);
pthread_setcancelstate(cs, 0);

5
libc/runtime/mapstack.c
View file

@ -50,9 +50,10 @@ void *NewCosmoStack(void) {
if (IsOpenbsd() && __sys_mmap(p, n, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD |
MAP_STACK_OPENBSD,
-1, 0, 0) != p) {
-1, 0, 0) != p)
notpossible;
if (mprotect(p, GetGuardSize(), PROT_NONE | PROT_GUARD))
notpossible;
}
return p;
} else {
return 0;

2
libc/runtime/runtime.h
View file

@ -91,7 +91,7 @@ void ShowCrashReports(void) libcesque;
int ftrace_install(void) libcesque;
int ftrace_enabled(int) libcesque;
int strace_enabled(int) libcesque;
void __print_maps(void) libcesque;
void __print_maps(size_t) libcesque;
void __printargs(const char *) libcesque;
/* builtin sh-like system/popen dsl */
int _cocmd(int, char **, char **) libcesque;

2
libc/stdio/printargs.c
View file

@ -468,7 +468,7 @@ textstartup void __printargs(const char *prologue) {
PRINT("");
PRINT("MEMTRACK");
__print_maps();
__print_maps(0);
PRINT("");
PRINT("TERMIOS");

1
libc/sysv/consts.sh
View file

@ -292,6 +292,7 @@ syscon mprot PROT_NONE 0 0 0 0 0 0 0 0 # mmap, mprotect, unix
syscon mprot PROT_READ 1 1 1 1 1 1 1 1 # mmap, mprotect, unix consensus
syscon mprot PROT_WRITE 2 2 2 2 2 2 2 2 # mmap, mprotect, unix consensus
syscon mprot PROT_EXEC 4 4 4 4 4 4 4 4 # mmap, mprotect, unix consensus
syscon mprot PROT_GUARD 0 0 0 0 0 0 0 0x100 # mmap, mprotect, unix consensus
# mremap() flags
# the revolutionary praxis of realloc()

2
libc/sysv/consts/PROT_GUARD.S Normal file
View file

@ -0,0 +1,2 @@
#include "libc/sysv/consts/syscon.internal.h"
.syscon mprot,PROT_GUARD,0,0,0,0,0,0,0,0x100

5
libc/sysv/consts/prot.h
View file

@ -3,10 +3,7 @@
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
extern const int PROT_NONE;
extern const int PROT_READ;
extern const int PROT_WRITE;
extern const int PROT_EXEC;
extern const int PROT_GUARD;
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

18
libc/thread/pthread_create.c
View file

@ -229,20 +229,10 @@ static errno_t pthread_create_impl(pthread_t *thread,
-1, 0, 0) != pt->pt_attr.__stackaddr) {
notpossible;
}
if (pt->pt_attr.__guardsize) {
if (!IsWindows()) {
if (mprotect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize,
PROT_NONE)) {
notpossible;
}
} else {
uint32_t oldattr;
if (!VirtualProtect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize,
kNtPageReadwrite | kNtPageGuard, &oldattr)) {
notpossible;
}
}
}
if (pt->pt_attr.__guardsize)
if (mprotect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize,
PROT_NONE | PROT_GUARD))
notpossible;
}
if (!pt->pt_attr.__stackaddr || pt->pt_attr.__stackaddr == MAP_FAILED) {
rc = errno;

1
libc/thread/thread.h
View file

@ -190,6 +190,7 @@ int pthread_spin_trylock(pthread_spinlock_t *) libcesque paramsnonnull();
int pthread_spin_unlock(pthread_spinlock_t *) libcesque paramsnonnull();
int pthread_testcancel_np(void) libcesque;
int pthread_tryjoin_np(pthread_t, void **) libcesque;
int pthread_delay_np(const void *, int) libcesque;
int pthread_yield_np(void) libcesque;
int pthread_yield(void) libcesque;
pthread_id_np_t pthread_getthreadid_np(void) libcesque;

16
test/ctl/set_test.cc
View file

@ -19,9 +19,9 @@
#include "ctl/set.h"
#include "libc/mem/leaks.h"
// #include <set>
// #define ctl std
// #define check() size()
#include <set>
#define ctl std
#define check() size()
int
rand32(void)
@ -148,6 +148,16 @@ main()
s.check();
}
{
// Test lower_bound and upper_bound
ctl::set<int> s{ 1, 3, 4, 5, 7, 9 };
auto lower = s.lower_bound(4);
auto upper = s.upper_bound(4);
if (*lower != 4 || *upper != 5)
return 18;
s.check();
}
{
// Test emplace
ctl::set<ctl::pair<int, int>> s;

1
test/libc/calls/raise_test.c
View file

@ -20,7 +20,6 @@
#include "libc/calls/struct/sigaction.h"
#include "libc/calls/struct/siginfo.h"
#include "libc/dce.h"
#include "libc/intrin/kprintf.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/sa.h"
#include "libc/sysv/consts/sicode.h"

24
test/libc/calls/setrlimit_test.c
View file

@ -118,10 +118,11 @@ TEST(setrlimit, testFileSizeLimit) {
EXPECT_EQ(0, WTERMSIG(wstatus));
}
int SetKernelEnforcedMemoryLimit(size_t n) {
struct rlimit rlim;
int SetMemoryLimit(size_t n) {
struct rlimit rlim = {0};
getrlimit(RLIMIT_AS, &rlim);
rlim.rlim_cur = n;
rlim.rlim_max = n;
return setrlimit(RLIMIT_AS, &rlim);
}
@ -129,27 +130,20 @@ TEST(setrlimit, testMemoryLimit) {
char *p;
bool gotsome;
int i, wstatus;
if (IsXnu())
return;
if (IsOpenbsd())
return; // simply too slow until mmap() becomes O(logn)
ASSERT_NE(-1, (wstatus = xspawn(0)));
if (wstatus == -2) {
ASSERT_EQ(0, SetKernelEnforcedMemoryLimit(MEM));
for (gotsome = false, i = 0; i < (MEM * 2) / __granularity(); ++i) {
p = mmap(0, __granularity(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE | MAP_POPULATE, -1, 0);
ASSERT_EQ(0, SetMemoryLimit(MEM));
for (gotsome = false, i = 0; i < (MEM * 2) / getpagesize(); ++i) {
p = mmap(0, getpagesize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
if (p != MAP_FAILED) {
gotsome = true;
} else {
if (!IsNetbsd()) {
// TODO(jart): what's going on with NetBSD?
ASSERT_TRUE(gotsome);
}
ASSERT_TRUE(gotsome);
ASSERT_EQ(ENOMEM, errno);
_Exit(0);
}
rngset(p, __granularity(), _rand64, -1);
rngset(p, getpagesize(), _rand64, -1);
}
_Exit(1);
}

76
test/libc/intrin/mmap_scalability_test.c Normal file
View file

@ -0,0 +1,76 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/calls/struct/timespec.h"
#include "libc/dce.h"
#include "libc/intrin/kprintf.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
#define BENCH(ITERATIONS, WORK_PER_RUN, CODE) \
do { \
struct timespec start = timespec_real(); \
for (int __i = 0; __i < ITERATIONS; ++__i) { \
asm volatile("" ::: "memory"); \
CODE; \
} \
long long work = (WORK_PER_RUN) * (ITERATIONS); \
double nanos = \
(timespec_tonanos(timespec_sub(timespec_real(), start)) + work - 1) / \
(double)work; \
kprintf("%'20ld ns %2dx %s\n", (long)nanos, (ITERATIONS), #CODE); \
} while (0)
void *randaddr(void) {
static unsigned long lcg = 1;
lcg *= 6364136223846793005;
lcg += 1442695040888963407;
return (void *)(lcg >> 48 << 28);
}
void map_unmap_one_page(void) {
void *p;
if ((p = mmap(randaddr(), 1, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)) == MAP_FAILED)
__builtin_trap();
if (munmap(p, 1))
__builtin_trap();
}
int main() {
kprintf("\n");
BENCH(1000, 1, map_unmap_one_page());
// macos doesn't scale
if (IsXnu())
return 0;
// create lots of sparse mappings to put
// weight on __maps.maps. red-black tree
int n = 10000;
kprintf("%20s creating %d sparse maps...\n", "", n);
for (int i = 0; i < n; ++i) {
if (mmap(randaddr(), 1, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS,
-1, 0) == MAP_FAILED)
__builtin_trap();
}
BENCH(1000, 1, map_unmap_one_page());
}

16
test/libc/intrin/mmap_test.c
View file

@ -461,7 +461,23 @@ void BenchUnmap(void) {
__builtin_trap();
}
void BenchBigMmap(void) {
void *p;
p = mmap(0, 101 * 1024 * 1024, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
if (p == MAP_FAILED)
__builtin_trap();
ptrs[count++] = p;
}
void BenchBigMunmap(void) {
if (munmap(ptrs[--count], 101 * 1024 * 1024))
__builtin_trap();
}
BENCH(mmap, bench) {
EZBENCH2("mmap", donothing, BenchMmapPrivate());
EZBENCH2("munmap", donothing, BenchUnmap());
// EZBENCH2("big mmap", donothing, BenchBigMmap());
// EZBENCH2("big munmap", donothing, BenchBigMunmap());
}

76
test/libc/intrin/rbtree_test.c → test/libc/intrin/tree_test.c
View file

@ -13,16 +13,18 @@
// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
#include "libc/intrin/rbtree.h"
#include "libc/intrin/tree.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/maps.h"
#include "libc/macros.internal.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/rand.h"
#define NUMBER_CONTAINER(e) RBTREE_CONTAINER(struct number, elem, e)
#define NUMBER_CONTAINER(e) TREE_CONTAINER(struct number, elem, e)
void rbtree_checker(const struct rbtree *node, const struct rbtree *parent,
int black_count, int *black_height, rbtree_cmp_f *cmp) {
void tree_checker(const struct Tree *node, const struct Tree *parent,
int black_count, int *black_height, tree_cmp_f *cmp) {
if (!node) {
// Leaf nodes are considered black
if (*black_height == -1) {
@ -37,89 +39,101 @@ void rbtree_checker(const struct rbtree *node, const struct rbtree *parent,
// ILLEGAL TREE: Parent link is incorrect
__builtin_trap();
if (parent) {
if (rbtree_get_left(parent) == node && cmp(parent, node) < 0)
if (tree_get_left(parent) == node && cmp(parent, node) < 0)
// ILLEGAL TREE: Binary search property violated on left child
__builtin_trap();
if (parent->right == node && cmp(node, parent) < 0)
// ILLEGAL TREE: Binary search property violated on right child
__builtin_trap();
}
if (!rbtree_get_red(node)) {
if (!tree_get_red(node)) {
black_count++;
} else if (parent && rbtree_get_red(parent)) {
} else if (parent && tree_get_red(parent)) {
// ILLEGAL TREE: Red node has red child
__builtin_trap();
}
rbtree_checker(rbtree_get_left(node), node, black_count, black_height, cmp);
rbtree_checker(node->right, node, black_count, black_height, cmp);
tree_checker(tree_get_left(node), node, black_count, black_height, cmp);
tree_checker(node->right, node, black_count, black_height, cmp);
}
void rbtree_check(struct rbtree *root, rbtree_cmp_f *cmp) {
void tree_check(struct Tree *root, tree_cmp_f *cmp) {
if (root) {
if (rbtree_get_red(root))
if (tree_get_red(root))
// ILLEGAL TREE: root node must be black
__builtin_trap();
int black_height = -1;
rbtree_checker(root, 0, 0, &black_height, cmp);
tree_checker(root, 0, 0, &black_height, cmp);
}
}
struct number {
int number;
struct rbtree elem;
long number;
struct Tree elem;
};
int number_compare(const struct rbtree *ra, const struct rbtree *rb) {
int number_search(const void *ra, const struct Tree *rb) {
long a = (long)ra;
const struct number *b = NUMBER_CONTAINER(rb);
return (a > b->number) - (a < b->number);
}
int number_compare(const struct Tree *ra, const struct Tree *rb) {
const struct number *a = NUMBER_CONTAINER(ra);
const struct number *b = NUMBER_CONTAINER(rb);
return (a->number > b->number) - (a->number < b->number);
}
struct number *number_new(int number) {
static int used;
static struct number heap[8192];
if (used == ARRAYLEN(heap))
return 0;
struct number *res = &heap[used++];
res->number = number;
struct number *res;
if ((res = malloc(sizeof(struct number))))
res->number = number;
return res;
}
struct rbtree *tree = 0;
struct Tree *tree = 0;
void print(void) {
for (struct rbtree *e = rbtree_first(tree); e; e = rbtree_next(e))
for (struct Tree *e = tree_first(tree); e; e = tree_next(e))
kprintf("%3d", NUMBER_CONTAINER(e)->number);
kprintf("\n");
}
void print_reversed(void) {
for (struct rbtree *e = rbtree_last(tree); e; e = rbtree_prev(e))
for (struct Tree *e = tree_last(tree); e; e = tree_prev(e))
kprintf("%3d", NUMBER_CONTAINER(e)->number);
kprintf("\n");
}
void simple_test(void) {
static const int kNumba[] = {74, 53, 96, 70, 34, 95, 30, 2, 89, 46,
23, 2, 52, 0, 34, 12, 90, 95, 32, 65};
static const long kNumba[] = {74, 53, 96, 70, 34, 95, 30, 2, 96, 46,
23, 2, 52, 0, 34, 94, 90, 95, 32, 65};
for (int i = 0; i < 20; ++i) {
int number = kNumba[i];
kprintf("%3d", number);
rbtree_insert(&tree, &number_new(number)->elem, number_compare);
rbtree_check(tree, number_compare);
tree_insert(&tree, &number_new(number)->elem, number_compare);
tree_check(tree, number_compare);
}
kprintf("\n");
print();
print_reversed();
for (int i = 0; i < 20; ++i) {
rbtree_remove(&tree, rbtree_get(tree, &(&(struct number){kNumba[i]})->elem,
number_compare));
rbtree_check(tree, number_compare);
tree_remove(&tree, tree_get(tree, (void *)kNumba[i], number_search));
tree_check(tree, number_compare);
print();
}
}
int main() {
ShowCrashReports();
tree_check(__maps.maps, __maps_compare);
kprintf("\n");
__print_maps(15);
kprintf("\n");
simple_test();
tree_check(__maps.maps, __maps_compare);
for (int i = 0; i < 100000; ++i)
tree_insert(&tree, &number_new(rand())->elem, number_compare);
tree_check(tree, number_compare);
tree_check(__maps.maps, __maps_compare);
__print_maps(15);
}

17
test/libc/thread/pthread_atfork_test.c
View file

@ -16,8 +16,11 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/atomic.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/kprintf.h"
#include "libc/mem/gc.h"
#include "libc/mem/mem.h"
#include "libc/runtime/internal.h"
@ -66,7 +69,7 @@ TEST(pthread_atfork, test) {
EXITS(0);
}
pthread_mutex_t mu;
pthread_mutex_t mu = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
void mu_lock(void) {
pthread_mutex_lock(&mu);
@ -80,8 +83,12 @@ void mu_wipe(void) {
pthread_mutex_init(&mu, 0);
}
static atomic_bool once;
void *Worker(void *arg) {
for (int i = 0; i < 20; ++i) {
if (!atomic_exchange(&once, true))
__print_maps(0);
mu_lock();
usleep(20);
mu_unlock();
@ -99,14 +106,14 @@ void *Worker(void *arg) {
}
TEST(pthread_atfork, fork_exit_torture) {
if (!IsFreebsd())
return;
mu_wipe();
pthread_atfork(mu_lock, mu_unlock, mu_wipe);
int i, n = 4;
pthread_t *t = gc(malloc(sizeof(pthread_t) * n));
for (i = 0; i < n; ++i) {
for (i = 0; i < n; ++i)
ASSERT_EQ(0, pthread_create(t + i, 0, Worker, 0));
}
for (i = 0; i < n; ++i) {
for (i = 0; i < n; ++i)
ASSERT_EQ(0, pthread_join(t[i], 0));
}
}

146
third_party/nsync/common.c vendored
View file

@ -18,6 +18,7 @@
#include "libc/calls/calls.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/extend.internal.h"
#include "libc/nt/enum/filemapflags.h"
@ -25,21 +26,17 @@
#include "libc/nt/memory.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/stdalign.internal.h"
#include "libc/stdalign.internal.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
#include "third_party/nsync/atomic.h"
#include "third_party/nsync/atomic.internal.h"
#include "third_party/nsync/common.internal.h"
#include "third_party/nsync/mu_semaphore.h"
#include "third_party/nsync/races.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/map.h"
#include "libc/nt/runtime.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/thread/thread.h"
#include "libc/dce.h"
#include "third_party/nsync/wait_s.internal.h"
__static_yoink("nsync_notice");
@ -97,33 +94,15 @@ __static_yoink("nsync_notice");
distinct wakeup conditions were high. So clients are advised to resort to
condition variables if they have many distinct wakeup conditions. */
/* Used in spinloops to delay resumption of the loop.
Usage:
unsigned attempts = 0;
while (try_something) {
attempts = nsync_spin_delay_ (attempts);
} */
unsigned nsync_spin_delay_ (unsigned attempts) {
if (attempts < 7) {
volatile int i;
for (i = 0; i != 1 << attempts; i++) {
}
attempts++;
} else {
nsync_yield_ ();
}
return (attempts);
}
/* Spin until (*w & test) == 0, then atomically perform *w = ((*w | set) &
~clear), perform an acquire barrier, and return the previous value of *w.
*/
uint32_t nsync_spin_test_and_set_ (nsync_atomic_uint32_ *w, uint32_t test,
uint32_t set, uint32_t clear) {
uint32_t set, uint32_t clear, void *symbol) {
unsigned attempts = 0; /* CV_SPINLOCK retry count */
uint32_t old = ATM_LOAD (w);
while ((old & test) != 0 || !ATM_CAS_ACQ (w, old, (old | set) & ~clear)) {
attempts = nsync_spin_delay_ (attempts);
attempts = pthread_delay_np (symbol, attempts);
old = ATM_LOAD (w);
}
return (old);
@ -156,33 +135,69 @@ waiter *nsync_dll_waiter_samecond_ (struct Dll *e) {
/* -------------------------------- */
static void *nsync_malloc (size_t size) {
void *res;
if (!IsWindows ()) {
// too much of a performance hit to track
res = __sys_mmap ((void *)0x7110000000, size,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS,
-1, 0, 0);
static _Atomic(waiter *) free_waiters;
static void free_waiters_push (waiter *w) {
int backoff = 0;
w->next_free = atomic_load_explicit (&free_waiters, memory_order_relaxed);
while (!atomic_compare_exchange_weak_explicit (&free_waiters, &w->next_free, w,
memory_order_acq_rel, memory_order_relaxed))
backoff = pthread_delay_np(free_waiters, backoff);
}
static void free_waiters_populate (void) {
int n;
if (IsNetbsd () || IsXnuSilicon ()) {
// netbsd needs one file descriptor per semaphore (!!)
// tim cook wants us to use his grand central dispatch
n = 1;
} else {
// must be tracked for fork() resurrection
res = mmap (0, size,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS,
-1, 0);
n = getpagesize() / sizeof(waiter);
}
if (res == MAP_FAILED)
waiter *waiters = mmap (0, n * sizeof(waiter),
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS,
-1, 0);
if (waiters == MAP_FAILED)
nsync_panic_ ("out of memory\n");
return res;
for (size_t i = 0; i < n; ++i) {
waiter *w = &waiters[i];
w->tag = WAITER_TAG;
w->nw.tag = NSYNC_WAITER_TAG;
if (!nsync_mu_semaphore_init (&w->sem)) {
if (!i)
nsync_panic_ ("out of semaphores\n");
break;
}
w->nw.sem = &w->sem;
dll_init (&w->nw.q);
NSYNC_ATOMIC_UINT32_STORE_ (&w->nw.waiting, 0);
w->nw.flags = NSYNC_WAITER_FLAG_MUCV;
ATM_STORE (&w->remove_count, 0);
dll_init (&w->same_condition);
w->flags = 0;
free_waiters_push (w);
}
}
static waiter *free_waiters_pop (void) {
waiter *w;
int backoff = 0;
for (;;) {
if ((w = atomic_load_explicit (&free_waiters, memory_order_relaxed))) {
if (atomic_compare_exchange_weak_explicit (&free_waiters, &w, w->next_free,
memory_order_acq_rel, memory_order_relaxed))
return w;
backoff = pthread_delay_np(free_waiters, backoff);
} else {
free_waiters_populate ();
}
}
return w;
}
/* -------------------------------- */
static struct Dll *free_waiters = NULL;
/* free_waiters points to a doubly-linked list of free waiter structs. */
pthread_mutex_t nsync_waiters_mu = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
#define waiter_for_thread __get_tls()->tib_nsync
void nsync_waiter_destroy (void *v) {
@ -193,45 +208,20 @@ void nsync_waiter_destroy (void *v) {
of thread-local variables can be arbitrary in some platform e.g.
POSIX. */
waiter_for_thread = NULL;
IGNORE_RACES_START ();
ASSERT ((w->flags & (WAITER_RESERVED|WAITER_IN_USE)) == WAITER_RESERVED);
w->flags &= ~WAITER_RESERVED;
pthread_mutex_lock (&nsync_waiters_mu);
dll_make_first (&free_waiters, &w->nw.q);
pthread_mutex_unlock (&nsync_waiters_mu);
IGNORE_RACES_END ();
free_waiters_push (w);
}
/* Return a pointer to an unused waiter struct.
Ensures that the enclosed timer is stopped and its channel drained. */
waiter *nsync_waiter_new_ (void) {
struct Dll *q;
waiter *tw;
waiter *w;
waiter *tw;
tw = waiter_for_thread;
w = tw;
if (w == NULL || (w->flags & (WAITER_RESERVED|WAITER_IN_USE)) != WAITER_RESERVED) {
w = NULL;
pthread_mutex_lock (&nsync_waiters_mu);
q = dll_first (free_waiters);
if (q != NULL) { /* If free list is non-empty, dequeue an item. */
dll_remove (&free_waiters, q);
w = DLL_WAITER (q);
}
pthread_mutex_unlock (&nsync_waiters_mu);
if (w == NULL) { /* If free list was empty, allocate an item. */
w = (waiter *) nsync_malloc (sizeof (*w));
w->tag = WAITER_TAG;
w->nw.tag = NSYNC_WAITER_TAG;
nsync_mu_semaphore_init (&w->sem);
w->nw.sem = &w->sem;
dll_init (&w->nw.q);
NSYNC_ATOMIC_UINT32_STORE_ (&w->nw.waiting, 0);
w->nw.flags = NSYNC_WAITER_FLAG_MUCV;
ATM_STORE (&w->remove_count, 0);
dll_init (&w->same_condition);
w->flags = 0;
}
w = free_waiters_pop ();
if (tw == NULL) {
w->flags |= WAITER_RESERVED;
waiter_for_thread = w;
@ -246,9 +236,7 @@ void nsync_waiter_free_ (waiter *w) {
ASSERT ((w->flags & WAITER_IN_USE) != 0);
w->flags &= ~WAITER_IN_USE;
if ((w->flags & WAITER_RESERVED) == 0) {
pthread_mutex_lock (&nsync_waiters_mu);
dll_make_first (&free_waiters, &w->nw.q);
pthread_mutex_unlock (&nsync_waiters_mu);
free_waiters_push (w);
if (w == waiter_for_thread)
waiter_for_thread = 0;
}

11
third_party/nsync/common.internal.h vendored
View file

@ -24,19 +24,11 @@ void nsync_yield_(void);
/* Retrieve the per-thread cache of the waiter object. Platform specific. */
void *nsync_per_thread_waiter_(void (*dest)(void *));
/* Used in spinloops to delay resumption of the loop.
Usage:
unsigned attempts = 0;
while (try_something) {
attempts = nsync_spin_delay_ (attempts);
} */
unsigned nsync_spin_delay_(unsigned attempts);
/* Spin until (*w & test) == 0, then atomically perform *w = ((*w | set) &
~clear), perform an acquire barrier, and return the previous value of *w.
*/
uint32_t nsync_spin_test_and_set_(nsync_atomic_uint32_ *w, uint32_t test,
uint32_t set, uint32_t clear);
uint32_t set, uint32_t clear, void *symbol);
/* Abort after printing the nul-temrinated string s[]. */
void nsync_panic_(const char *s) wontreturn;
@ -210,6 +202,7 @@ typedef struct waiter_s {
struct wait_condition_s cond; /* A condition on which to acquire a mu. */
struct Dll same_condition; /* Links neighbours in nw.q with same
non-nil condition. */
struct waiter_s * next_free;
} waiter;
static const uint32_t WAITER_TAG = 0x0590239f;
static const uint32_t NSYNC_WAITER_TAG = 0x726d2ba9;

14
third_party/nsync/mem/nsync_cv.c vendored
View file

@ -193,7 +193,7 @@ static int nsync_cv_wait_with_deadline_impl_ (struct nsync_cv_wait_with_deadline
/* A timeout or cancellation occurred, and no wakeup.
Acquire *pcv's spinlock, and confirm. */
c->old_word = nsync_spin_test_and_set_ (&c->pcv->word, CV_SPINLOCK,
CV_SPINLOCK, 0);
CV_SPINLOCK, 0, c->cv_mu);
/* Check that w wasn't removed from the queue after we
checked above, but before we acquired the spinlock.
The test of remove_count confirms that the waiter *w
@ -226,7 +226,7 @@ static int nsync_cv_wait_with_deadline_impl_ (struct nsync_cv_wait_with_deadline
has not yet set the waiting field to zero; a
cancellation or timeout may prevent this thread
from blocking above on the semaphore. */
attempts = nsync_spin_delay_ (attempts);
attempts = pthread_delay_np (c->cv_mu, attempts);
}
}
if (c->cv_mu != NULL && c->w->cv_mu == NULL) { /* waiter was moved to *pmu's queue, and woken. */
@ -323,7 +323,7 @@ int nsync_cv_wait_with_deadline_generic (nsync_cv *pcv, void *pmu,
}
/* acquire spinlock, set non-empty */
c.old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK|CV_NON_EMPTY, 0);
c.old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK|CV_NON_EMPTY, 0, pmu);
dll_make_last (&pcv->waiters, &c.w->nw.q);
c.remove_count = ATM_LOAD (&c.w->remove_count);
/* Release the spin lock. */
@ -355,7 +355,7 @@ void nsync_cv_signal (nsync_cv *pcv) {
int all_readers = 0;
/* acquire spinlock */
uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK,
CV_SPINLOCK, 0);
CV_SPINLOCK, 0, pcv);
if (!dll_is_empty (pcv->waiters)) {
/* Point to first waiter that enqueued itself, and
detach it from all others. */
@ -438,7 +438,7 @@ void nsync_cv_broadcast (nsync_cv *pcv) {
int all_readers;
struct Dll *to_wake_list = NULL; /* waiters that we will wake */
/* acquire spinlock */
nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0);
nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0, pcv);
p = NULL;
next = NULL;
all_readers = 1;
@ -497,7 +497,7 @@ static nsync_time cv_ready_time (void *v, struct nsync_waiter_s *nw) {
static int cv_enqueue (void *v, struct nsync_waiter_s *nw) {
nsync_cv *pcv = (nsync_cv *) v;
/* acquire spinlock */
uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0);
uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0, pcv);
dll_make_last (&pcv->waiters, &nw->q);
ATM_STORE (&nw->waiting, 1);
/* Release spinlock. */
@ -509,7 +509,7 @@ static int cv_dequeue (void *v, struct nsync_waiter_s *nw) {
nsync_cv *pcv = (nsync_cv *) v;
int was_queued = 0;
/* acquire spinlock */
uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0);
uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0, pcv);
if (ATM_LOAD_ACQ (&nw->waiting) != 0) {
dll_remove (&pcv->waiters, &nw->q);
ATM_STORE (&nw->waiting, 0);

4
third_party/nsync/mem/nsync_debug.c vendored
View file

@ -197,7 +197,7 @@ static char *emit_mu_state (struct emit_buf *b, nsync_mu *mu,
word = ATM_LOAD (&mu->word);
if ((word & MU_WAITING) != 0 && print_waiters && /* can benefit from lock */
(blocking || (word & MU_SPINLOCK) == 0)) { /* willing, or no need to wait */
word = nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK, MU_SPINLOCK, 0);
word = nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK, MU_SPINLOCK, 0, mu);
acquired = 1;
}
readers = word / MU_RLOCK;
@ -234,7 +234,7 @@ static char *emit_cv_state (struct emit_buf *b, nsync_cv *cv,
word = ATM_LOAD (&cv->word);
if ((word & CV_NON_EMPTY) != 0 && print_waiters && /* can benefit from lock */
(blocking || (word & CV_SPINLOCK) == 0)) { /* willing, or no need to wait */
word = nsync_spin_test_and_set_ (&cv->word, CV_SPINLOCK, CV_SPINLOCK, 0);
word = nsync_spin_test_and_set_ (&cv->word, CV_SPINLOCK, CV_SPINLOCK, 0, cv);
acquired = 1;
}
emit_print (b, "cv 0x%i -> 0x%i = {", (uintptr_t) cv, word);

7
third_party/nsync/mem/nsync_mu_wait.c vendored
View file

@ -21,6 +21,7 @@
#include "third_party/nsync/common.internal.h"
#include "third_party/nsync/mu_semaphore.h"
#include "third_party/nsync/races.internal.h"
#include "libc/thread/thread.h"
#include "third_party/nsync/wait_s.internal.h"
__static_yoink("nsync_notice");
@ -84,7 +85,7 @@ static int mu_try_acquire_after_timeout_or_cancel (nsync_mu *mu, lock_type *l_ty
ATM_CAS_RELACQ (&mu->word, old_word,
old_word|MU_WRITER_WAITING);
}
spin_attempts = nsync_spin_delay_ (spin_attempts);
spin_attempts = pthread_delay_np (mu, spin_attempts);
old_word = ATM_LOAD (&mu->word);
}
/* Check that w wasn't removed from the queue after our caller checked,
@ -194,7 +195,7 @@ int nsync_mu_wait_with_deadline (nsync_mu *mu,
/* Acquire spinlock. */
old_word = nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK,
MU_SPINLOCK|MU_WAITING|has_condition, MU_ALL_FALSE);
MU_SPINLOCK|MU_WAITING|has_condition, MU_ALL_FALSE, mu);
had_waiters = ((old_word & (MU_DESIG_WAKER | MU_WAITING)) == MU_WAITING);
/* Queue the waiter. */
if (first_wait) {
@ -244,7 +245,7 @@ int nsync_mu_wait_with_deadline (nsync_mu *mu,
}
if (ATM_LOAD (&w->nw.waiting) != 0) {
attempts = nsync_spin_delay_ (attempts); /* will ultimately yield */
attempts = pthread_delay_np (mu, attempts); /* will ultimately yield */
}
}

3
third_party/nsync/mem/nsync_once.c vendored
View file

@ -21,6 +21,7 @@
#include "third_party/nsync/mu_semaphore.h"
#include "third_party/nsync/once.h"
#include "third_party/nsync/races.internal.h"
#include "libc/thread/thread.h"
#include "third_party/nsync/wait_s.internal.h"
__static_yoink("nsync_notice");
@ -92,7 +93,7 @@ static void nsync_run_once_impl (nsync_once *once, struct once_sync_s *s,
deadline = nsync_time_add (nsync_time_now (), nsync_time_ms (attempts));
nsync_cv_wait_with_deadline (&s->once_cv, &s->once_mu, deadline, NULL);
} else {
attempts = nsync_spin_delay_ (attempts);
attempts = pthread_delay_np (once, attempts);
}
}
if (s != NULL) {

7
third_party/nsync/mu.c vendored
View file

@ -22,6 +22,7 @@
#include "third_party/nsync/common.internal.h"
#include "third_party/nsync/mu_semaphore.h"
#include "third_party/nsync/races.internal.h"
#include "libc/thread/thread.h"
#include "third_party/nsync/wait_s.internal.h"
__static_yoink("nsync_notice");
@ -120,7 +121,7 @@ void nsync_mu_lock_slow_ (nsync_mu *mu, waiter *w, uint32_t clear, lock_type *l_
about waiting writers or long waiters. */
zero_to_acquire &= ~(MU_WRITER_WAITING | MU_LONG_WAIT);
}
attempts = nsync_spin_delay_ (attempts);
attempts = pthread_delay_np (mu, attempts);
}
ALLOW_CANCELATION;
}
@ -393,7 +394,7 @@ void nsync_mu_unlock_slow_ (nsync_mu *mu, lock_type *l_type) {
released above. */
if (testing_conditions) {
nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK,
MU_SPINLOCK, 0);
MU_SPINLOCK, 0, mu);
}
/* add the new_waiters to the last of the waiters. */
@ -444,7 +445,7 @@ void nsync_mu_unlock_slow_ (nsync_mu *mu, lock_type *l_type) {
}
return;
}
attempts = nsync_spin_delay_ (attempts);
attempts = pthread_delay_np (mu, attempts);
}
}

2
third_party/nsync/mu_semaphore.c vendored
View file

@ -27,7 +27,7 @@ __static_yoink("nsync_notice");
#define PREFER_GCD_OVER_ULOCK 1
/* Initialize *s; the initial value is 0. */
void nsync_mu_semaphore_init (nsync_semaphore *s) {
bool nsync_mu_semaphore_init (nsync_semaphore *s) {
if (PREFER_GCD_OVER_ULOCK && IsXnuSilicon ()) {
return nsync_mu_semaphore_init_gcd (s);
} else if (IsNetbsd ()) {

2
third_party/nsync/mu_semaphore.h vendored
View file

@ -8,7 +8,7 @@ typedef struct nsync_semaphore_s_ {
} nsync_semaphore;
/* Initialize *s; the initial value is 0. */
void nsync_mu_semaphore_init(nsync_semaphore *s);
bool nsync_mu_semaphore_init(nsync_semaphore *s);
/* Wait until the count of *s exceeds 0, and decrement it. */
errno_t nsync_mu_semaphore_p(nsync_semaphore *s);

6
third_party/nsync/mu_semaphore.internal.h vendored
View file

@ -4,17 +4,17 @@
#include "third_party/nsync/time.h"
COSMOPOLITAN_C_START_
void nsync_mu_semaphore_init_futex(nsync_semaphore *);
bool nsync_mu_semaphore_init_futex(nsync_semaphore *);
errno_t nsync_mu_semaphore_p_futex(nsync_semaphore *);
errno_t nsync_mu_semaphore_p_with_deadline_futex(nsync_semaphore *, nsync_time);
void nsync_mu_semaphore_v_futex(nsync_semaphore *);
void nsync_mu_semaphore_init_sem(nsync_semaphore *);
bool nsync_mu_semaphore_init_sem(nsync_semaphore *);
errno_t nsync_mu_semaphore_p_sem(nsync_semaphore *);
errno_t nsync_mu_semaphore_p_with_deadline_sem(nsync_semaphore *, nsync_time);
void nsync_mu_semaphore_v_sem(nsync_semaphore *);
void nsync_mu_semaphore_init_gcd(nsync_semaphore *);
bool nsync_mu_semaphore_init_gcd(nsync_semaphore *);
errno_t nsync_mu_semaphore_p_gcd(nsync_semaphore *);
errno_t nsync_mu_semaphore_p_with_deadline_gcd(nsync_semaphore *, nsync_time);
void nsync_mu_semaphore_v_gcd(nsync_semaphore *);

3
third_party/nsync/mu_semaphore_futex.c vendored
View file

@ -43,9 +43,10 @@ static nsync_semaphore *sem_big_enough_for_futex = (nsync_semaphore *) (uintptr_
(sizeof (struct futex) <= sizeof (*sem_big_enough_for_futex)));
/* Initialize *s; the initial value is 0. */
void nsync_mu_semaphore_init_futex (nsync_semaphore *s) {
bool nsync_mu_semaphore_init_futex (nsync_semaphore *s) {
struct futex *f = (struct futex *) s;
f->i = 0;
return true;
}
/* Wait until the count of *s exceeds 0, and decrement it. If POSIX cancellations

4
third_party/nsync/mu_semaphore_gcd.c vendored
View file

@ -85,8 +85,8 @@ static errno_t nsync_dispatch_semaphore_wait (nsync_semaphore *s,
}
/* Initialize *s; the initial value is 0. */
void nsync_mu_semaphore_init_gcd (nsync_semaphore *s) {
*(dispatch_semaphore_t *)s = dispatch_semaphore_create (0);
bool nsync_mu_semaphore_init_gcd (nsync_semaphore *s) {
return !!(*(dispatch_semaphore_t *)s = dispatch_semaphore_create (0));
}
/* Wait until the count of *s exceeds 0, and decrement it. If POSIX cancellations

31
third_party/nsync/mu_semaphore_sem.c vendored
View file

@ -54,14 +54,22 @@ static struct {
static nsync_semaphore *sem_big_enough_for_sem = (nsync_semaphore *) (uintptr_t)(1 /
(sizeof (struct sem) <= sizeof (*sem_big_enough_for_sem)));
static void nsync_mu_semaphore_sem_create (struct sem *f) {
static bool nsync_mu_semaphore_sem_create (struct sem *f) {
int rc;
int lol;
f->id = 0;
ASSERT (!sys_sem_init (0, &f->id));
if ((lol = __sys_fcntl (f->id, F_DUPFD_CLOEXEC, 50)) >= 50) {
sys_close (f->id);
rc = sys_sem_init (0, &f->id);
STRACE ("sem_init(0, [%ld]) → %d", f->id, rc);
if (rc != 0)
return false;
lol = __sys_fcntl (f->id, F_DUPFD_CLOEXEC, 50);
STRACE ("fcntl(%ld, F_DUPFD_CLOEXEC, 50) → %d", f->id, lol);
if (lol >= 50) {
rc = sys_close (f->id);
STRACE ("close(%ld) → %d", f->id, rc);
f->id = lol;
}
return true;
}
static void nsync_mu_semaphore_sem_fork_child (void) {
@ -69,8 +77,12 @@ static void nsync_mu_semaphore_sem_fork_child (void) {
struct sem *f;
for (e = dll_first (g_sems.list); e; e = dll_next (g_sems.list, e)) {
f = SEM_CONTAINER (e);
sys_close (f->id);
nsync_mu_semaphore_sem_create (f);
int rc = sys_close (f->id);
STRACE ("close(%ld) → %d", f->id, rc);
}
for (e = dll_first (g_sems.list); e; e = dll_next (g_sems.list, e)) {
f = SEM_CONTAINER (e);
ASSERT (nsync_mu_semaphore_sem_create (f));
}
(void) pthread_spin_init (&g_sems.lock, 0);
}
@ -80,15 +92,16 @@ static void nsync_mu_semaphore_sem_init (void) {
}
/* Initialize *s; the initial value is 0. */
void nsync_mu_semaphore_init_sem (nsync_semaphore *s) {
bool nsync_mu_semaphore_init_sem (nsync_semaphore *s) {
struct sem *f = (struct sem *) s;
nsync_mu_semaphore_sem_create (f);
if (!nsync_mu_semaphore_sem_create (f))
return false;
cosmo_once (&g_sems.once, nsync_mu_semaphore_sem_init);
pthread_spin_lock (&g_sems.lock);
dll_init (&f->list);
dll_make_first (&g_sems.list, &f->list);
pthread_spin_unlock (&g_sems.lock);
STRACE ("sem_init(0, [%ld]) → 0", f->id);
return true;
}
/* Wait until the count of *s exceeds 0, and decrement it. If POSIX cancellations

55
tool/build/runitd.c
View file

@ -134,12 +134,12 @@
#define INFOF(FMT, ...) LOGF(INFO, FMT, ##__VA_ARGS__)
#define WARNF(FMT, ...) LOGF(WARN, FMT, ##__VA_ARGS__)
#define LOGF(LVL, FMT, ...) \
do { \
if (g_log_level >= LOG_LEVEL_##LVL) { \
kprintf("%r" #LVL " %6P %'18T %s:%d " FMT "\n", __FILE__, __LINE__, \
##__VA_ARGS__); \
} \
#define LOGF(LVL, FMT, ...) \
do { \
if (g_log_level >= LOG_LEVEL_##LVL) { \
kprintf("%r" #LVL " %6P %6H %'18T %s:%d " FMT "\n", __FILE__, __LINE__, \
##__VA_ARGS__); \
} \
} while (0)
struct Client {
@ -443,12 +443,18 @@ void FreeClient(struct Client *client) {
void *ClientWorker(void *arg) {
uint32_t crc;
sigset_t sigmask;
struct timespec ts0;
struct timespec ts1;
struct timespec ts2;
int events, wstatus;
struct Client *client = arg;
uint32_t namesize, filesize;
char *addrstr, *origname;
unsigned char msg[4 + 1 + 4 + 4 + 4];
ts0 = timespec_real();
ts1 = timespec_real();
SetupPresharedKeySsl(MBEDTLS_SSL_IS_SERVER, g_psk);
defer(FreeClient, client);
@ -458,9 +464,15 @@ void *ClientWorker(void *arg) {
EzHandshake();
addrstr = DescribeAddress(&client->addr);
DEBUF("%s %s %s", DescribeAddress(&g_servaddr), "accepted", addrstr);
DEBUF("it took %'zu us to handshake client",
timespec_tomicros(timespec_sub(timespec_real(), ts1)));
// get the executable
ts1 = timespec_real();
ts2 = timespec_real();
Recv(client, msg, sizeof(msg));
DEBUF("it took %'zu us to receive #1",
timespec_tomicros(timespec_sub(timespec_real(), ts2)));
if (READ32BE(msg) != RUNITD_MAGIC) {
WARNF("%s magic mismatch!", addrstr);
pthread_exit(0);
@ -473,11 +485,19 @@ void *ClientWorker(void *arg) {
filesize = READ32BE(msg + 9);
crc = READ32BE(msg + 13);
origname = gc(calloc(1, namesize + 1));
ts2 = timespec_real();
Recv(client, origname, namesize);
DEBUF("it took %'zu us to receive #2",
timespec_tomicros(timespec_sub(timespec_real(), ts2)));
VERBF("%s sent %#s (%'u bytes @ %#s)", addrstr, origname, filesize,
client->tmpexepath);
char *exedata = gc(malloc(filesize));
ts2 = timespec_real();
Recv(client, exedata, filesize);
DEBUF("it took %'zu us to receive #3",
timespec_tomicros(timespec_sub(timespec_real(), ts2)));
DEBUF("it took %'zu us to receive executable from network",
timespec_tomicros(timespec_sub(timespec_real(), ts1)));
if (crc32_z(0, exedata, filesize) != crc) {
WARNF("%s crc mismatch! %#s", addrstr, origname);
pthread_exit(0);
@ -488,6 +508,7 @@ void *ClientWorker(void *arg) {
// condition can happen, where etxtbsy is raised by our execve
// we're using o_cloexec so it's guaranteed to fix itself fast
// thus we use an optimistic approach to avoid expensive locks
ts1 = timespec_real();
sprintf(client->tmpexepath, "o/%s.XXXXXX",
basename(stripext(gc(strdup(origname)))));
int exefd = openatemp(AT_FDCWD, client->tmpexepath, 0, O_CLOEXEC, 0700);
@ -510,6 +531,8 @@ void *ClientWorker(void *arg) {
WARNF("%s failed to close %#s due to %m", addrstr, origname);
pthread_exit(0);
}
DEBUF("it took %'zu us to write executable to disk",
timespec_tomicros(timespec_sub(timespec_real(), ts1)));
// do the args
int i = 0;
@ -560,8 +583,11 @@ RetryOnEtxtbsyRaceCondition:
posix_spawn_file_actions_adddup2(&spawnfila, g_bogusfd, 0);
posix_spawn_file_actions_adddup2(&spawnfila, client->pipe[1], 1);
posix_spawn_file_actions_adddup2(&spawnfila, client->pipe[1], 2);
ts1 = timespec_real();
err = posix_spawn(&client->pid, client->tmpexepath, &spawnfila, &spawnattr,
args, environ);
DEBUF("it took %'zu us to call posix_spawn",
timespec_tomicros(timespec_sub(timespec_real(), ts1)));
if (err) {
if (err == ETXTBSY) {
goto RetryOnEtxtbsyRaceCondition;
@ -599,8 +625,11 @@ RetryOnEtxtbsyRaceCondition:
fds[0].events = POLLIN;
fds[1].fd = client->pipe[0];
fds[1].events = POLLIN;
ts1 = timespec_real();
events = poll(fds, ARRAYLEN(fds),
timespec_tomillis(timespec_sub(deadline, now)));
DEBUF("it took %'zu us to call poll",
timespec_tomicros(timespec_sub(timespec_real(), ts1)));
if (events == -1) {
if (errno == EINTR) {
INFOF("poll interrupted");
@ -615,7 +644,10 @@ RetryOnEtxtbsyRaceCondition:
if (fds[0].revents) {
int received;
char buf[512];
ts1 = timespec_real();
received = mbedtls_ssl_read(&ezssl, buf, sizeof(buf));
DEBUF("it took %'zu us to call mbedtls_ssl_read",
timespec_tomicros(timespec_sub(timespec_real(), ts1)));
if (!received) {
WARNF("%s client disconnected so killing worker %d", origname,
client->pid);
@ -640,7 +672,10 @@ RetryOnEtxtbsyRaceCondition:
}
if (fds[1].revents) {
char buf[512];
ts1 = timespec_real();
ssize_t got = read(client->pipe[0], buf, sizeof(buf));
DEBUF("it took %'zu us to call read",
timespec_tomicros(timespec_sub(timespec_real(), ts1)));
if (got == -1) {
WARNF("got %s reading %s output", strerror(errno), origname);
goto HangupClientAndTerminateJob;
@ -658,7 +693,10 @@ RetryOnEtxtbsyRaceCondition:
WaitAgain:
DEBUF("waitpid");
struct rusage rusage;
ts1 = timespec_real();
int wrc = wait4(client->pid, &wstatus, 0, &rusage);
DEBUF("it took %'zu us to call wait4",
timespec_tomicros(timespec_sub(timespec_real(), ts1)));
if (wrc == -1) {
if (errno == EINTR) {
WARNF("waitpid interrupted; killing %s pid %d", origname, client->pid);
@ -711,13 +749,18 @@ WaitAgain:
AppendResourceReport(&client->output, &rusage, "\n");
PrintProgramOutput(client);
}
ts1 = timespec_real();
SendProgramOutput(client);
SendExitMessage(exitcode);
mbedtls_ssl_close_notify(&ezssl);
DEBUF("it took %'zu us to send result to client",
timespec_tomicros(timespec_sub(timespec_real(), ts1)));
if (etxtbsy_tries > 1) {
WARNF("encountered %d ETXTBSY race conditions spawning %s",
etxtbsy_tries - 1, origname);
}
DEBUF("it took %'zu us TO DO EVERYTHING",
timespec_tomicros(timespec_sub(timespec_real(), ts0)));
pthread_exit(0);
}