mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-06-27 06:48:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Simplify memory manager code

Browse source
This commit is contained in:
Justine Tunney 2024-12-28 17:08:18 -08:00
parent 379cd77078
commit aca4214ff6
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
11 changed files with 442 additions and 325 deletions
Download patch file Download diff file Expand all files Collapse all files

84
libc/calls/read-nt.c
View file

@ -48,6 +48,7 @@
#include "libc/nt/enum/wait.h" #include "libc/nt/enum/wait.h"
#include "libc/nt/errors.h" #include "libc/nt/errors.h"
#include "libc/nt/events.h" #include "libc/nt/events.h"
#include "libc/nt/memory.h"
#include "libc/nt/runtime.h" #include "libc/nt/runtime.h"
#include "libc/nt/struct/inputrecord.h" #include "libc/nt/struct/inputrecord.h"
#include "libc/nt/synchronization.h" #include "libc/nt/synchronization.h"
@ -127,38 +128,46 @@ struct Keystrokes {
bool ohno_decckm; bool ohno_decckm;
bool bypass_mode; bool bypass_mode;
uint16_t utf16hs; uint16_t utf16hs;
int16_t freekeys; size_t free_keys;
int64_t cin, cot; int64_t cin, cot;
struct Dll *list; struct Dll *list;
struct Dll *line; struct Dll *line;
struct Dll *free; struct Dll *free;
pthread_mutex_t lock;
struct Keystroke pool[512];
}; };
static struct Keystrokes __keystroke = { static struct Keystrokes __keystroke;
.lock = PTHREAD_MUTEX_INITIALIZER, static pthread_mutex_t __keystroke_lock = PTHREAD_MUTEX_INITIALIZER;
};
textwindows void sys_read_nt_wipe_keystrokes(void) { textwindows void sys_read_nt_wipe_keystrokes(void) {
pthread_mutex_t lock = __keystroke.lock;
bzero(&__keystroke, sizeof(__keystroke)); bzero(&__keystroke, sizeof(__keystroke));
__keystroke.lock = lock; _pthread_mutex_wipe_np(&__keystroke_lock);
_pthread_mutex_wipe_np(&__keystroke.lock);
} }
textwindows static void FreeKeystrokeImpl(struct Dll *key) { textwindows static void FreeKeystrokeImpl(struct Dll *key) {
dll_make_first(&__keystroke.free, key); dll_make_first(&__keystroke.free, key);
++__keystroke.freekeys; ++__keystroke.free_keys;
}
textwindows static struct Keystroke *AllocKeystroke(void) {
struct Keystroke *k;
if (!(k = HeapAlloc(GetProcessHeap(), 0, sizeof(struct Keystroke))))
return 0;
dll_init(&k->elem);
return k;
} }
textwindows static struct Keystroke *NewKeystroke(void) { textwindows static struct Keystroke *NewKeystroke(void) {
struct Dll *e = dll_first(__keystroke.free); struct Dll *e;
if (!e) // See MIN(freekeys) before ReadConsoleInput() struct Keystroke *k;
__builtin_trap(); if ((e = dll_first(__keystroke.free))) {
struct Keystroke *k = KEYSTROKE_CONTAINER(e); dll_remove(&__keystroke.free, e);
dll_remove(&__keystroke.free, &k->elem); k = KEYSTROKE_CONTAINER(e);
--__keystroke.freekeys; --__keystroke.free_keys;
} else {
// PopulateKeystrokes() should make this branch impossible
if (!(k = AllocKeystroke()))
return 0;
}
k->buflen = 0; k->buflen = 0;
return k; return k;
} }
@ -174,15 +183,22 @@ textwindows static void FreeKeystrokes(struct Dll **list) {
FreeKeystroke(list, key); FreeKeystroke(list, key);
} }
textwindows static void PopulateKeystrokes(size_t want) {
struct Keystroke *k;
while (__keystroke.free_keys < want) {
if ((k = AllocKeystroke())) {
FreeKeystrokeImpl(&k->elem);
} else {
break;
}
}
}
textwindows static void OpenConsole(void) { textwindows static void OpenConsole(void) {
__keystroke.cin = CreateFile(u"CONIN$", kNtGenericRead | kNtGenericWrite, __keystroke.cin = CreateFile(u"CONIN$", kNtGenericRead | kNtGenericWrite,
kNtFileShareRead, 0, kNtOpenExisting, 0, 0); kNtFileShareRead, 0, kNtOpenExisting, 0, 0);
__keystroke.cot = CreateFile(u"CONOUT$", kNtGenericRead | kNtGenericWrite, __keystroke.cot = CreateFile(u"CONOUT$", kNtGenericRead | kNtGenericWrite,
kNtFileShareWrite, 0, kNtOpenExisting, 0, 0); kNtFileShareWrite, 0, kNtOpenExisting, 0, 0);
for (int i = 0; i < ARRAYLEN(__keystroke.pool); ++i) {
dll_init(&__keystroke.pool[i].elem);
FreeKeystrokeImpl(&__keystroke.pool[i].elem);
}
} }
textwindows static int AddSignal(int sig) { textwindows static int AddSignal(int sig) {
@ -196,11 +212,11 @@ textwindows static void InitConsole(void) {
} }
textwindows static void LockKeystrokes(void) { textwindows static void LockKeystrokes(void) {
_pthread_mutex_lock(&__keystroke.lock); _pthread_mutex_lock(&__keystroke_lock);
} }
textwindows static void UnlockKeystrokes(void) { textwindows static void UnlockKeystrokes(void) {
_pthread_mutex_unlock(&__keystroke.lock); _pthread_mutex_unlock(&__keystroke_lock);
} }
textwindows int64_t GetConsoleInputHandle(void) { textwindows int64_t GetConsoleInputHandle(void) {
@ -523,14 +539,12 @@ textwindows static void IngestConsoleInputRecord(struct NtInputRecord *r) {
!(__ttyconf.magic & kTtyNoIexten)) { // IEXTEN !(__ttyconf.magic & kTtyNoIexten)) { // IEXTEN
if (__keystroke.bypass_mode) { if (__keystroke.bypass_mode) {
struct Keystroke *k = NewKeystroke(); struct Keystroke *k = NewKeystroke();
if (!k)
return;
memcpy(k->buf, buf, sizeof(k->buf)); memcpy(k->buf, buf, sizeof(k->buf));
k->buflen = len; k->buflen = len;
dll_make_last(&__keystroke.line, &k->elem); dll_make_last(&__keystroke.line, &k->elem);
EchoConsoleNt(buf, len, true); EchoConsoleNt(buf, len, true);
if (!__keystroke.freekeys) {
dll_make_last(&__keystroke.list, __keystroke.line);
__keystroke.line = 0;
}
__keystroke.bypass_mode = false; __keystroke.bypass_mode = false;
return; return;
} else if (len == 1 && buf[0] && // } else if (len == 1 && buf[0] && //
@ -620,6 +634,8 @@ textwindows static void IngestConsoleInputRecord(struct NtInputRecord *r) {
// allocate object to hold keystroke // allocate object to hold keystroke
struct Keystroke *k = NewKeystroke(); struct Keystroke *k = NewKeystroke();
if (!k)
return;
memcpy(k->buf, buf, sizeof(k->buf)); memcpy(k->buf, buf, sizeof(k->buf));
k->buflen = len; k->buflen = len;
@ -633,12 +649,12 @@ textwindows static void IngestConsoleInputRecord(struct NtInputRecord *r) {
} else { } else {
dll_make_last(&__keystroke.line, &k->elem); dll_make_last(&__keystroke.line, &k->elem);
// flush canonical mode line if oom or enter // flush canonical mode line on enter
if (!__keystroke.freekeys || (len == 1 && buf[0] && if (len == 1 && buf[0] &&
((buf[0] & 255) == '\n' || // ((buf[0] & 255) == '\n' || //
(buf[0] & 255) == __ttyconf.veol || // (buf[0] & 255) == __ttyconf.veol || //
((buf[0] & 255) == __ttyconf.veol2 && ((buf[0] & 255) == __ttyconf.veol2 &&
!(__ttyconf.magic & kTtyNoIexten))))) { !(__ttyconf.magic & kTtyNoIexten)))) {
dll_make_last(&__keystroke.list, __keystroke.line); dll_make_last(&__keystroke.list, __keystroke.line);
__keystroke.line = 0; __keystroke.line = 0;
} }
@ -649,15 +665,17 @@ textwindows static void IngestConsoleInput(void) {
uint32_t i, n; uint32_t i, n;
struct NtInputRecord records[16]; struct NtInputRecord records[16];
for (;;) { for (;;) {
if (!__keystroke.freekeys)
return;
if (__keystroke.end_of_file) if (__keystroke.end_of_file)
return; return;
if (!GetNumberOfConsoleInputEvents(__keystroke.cin, &n)) if (!GetNumberOfConsoleInputEvents(__keystroke.cin, &n))
goto UnexpectedEof; goto UnexpectedEof;
if (!n || !__keystroke.freekeys) if (n > ARRAYLEN(records))
n = ARRAYLEN(records);
PopulateKeystrokes(n + 1);
if (n > __keystroke.free_keys)
n = __keystroke.free_keys;
if (!n)
return; return;
n = MIN(__keystroke.freekeys, MIN(ARRAYLEN(records), n));
if (!ReadConsoleInput(__keystroke.cin, records, n, &n)) if (!ReadConsoleInput(__keystroke.cin, records, n, &n))
goto UnexpectedEof; goto UnexpectedEof;
for (i = 0; i < n && !__keystroke.end_of_file; ++i) for (i = 0; i < n && !__keystroke.end_of_file; ++i)

24
libc/intrin/maps.c
View file

@ -32,6 +32,7 @@
#include "libc/runtime/stack.h" #include "libc/runtime/stack.h"
#include "libc/sysv/consts/prot.h" #include "libc/sysv/consts/prot.h"
#include "libc/thread/lock.h" #include "libc/thread/lock.h"
#include "libc/thread/tls.h"
#ifdef __x86_64__ #ifdef __x86_64__
__static_yoink("_init_maps"); __static_yoink("_init_maps");
@ -124,26 +125,33 @@ privileged static void __maps_panic(const char *msg) {
} }
#endif #endif
ABI bool __maps_lock(void) { bool __maps_held(void) {
return __tls_enabled && !(__get_tls()->tib_flags & TIB_FLAG_VFORKED) &&
MUTEX_OWNER(
atomic_load_explicit(&__maps.lock.word, memory_order_relaxed)) ==
atomic_load_explicit(&__get_tls()->tib_tid, memory_order_relaxed);
}
ABI void __maps_lock(void) {
int me; int me;
uint64_t word, lock; uint64_t word, lock;
struct CosmoTib *tib; struct CosmoTib *tib;
if (!__tls_enabled) if (!__tls_enabled)
return false; return;
if (!(tib = __get_tls_privileged())) if (!(tib = __get_tls_privileged()))
return false; return;
if (tib->tib_flags & TIB_FLAG_VFORKED) if (tib->tib_flags & TIB_FLAG_VFORKED)
return false; return;
me = atomic_load_explicit(&tib->tib_tid, memory_order_acquire); me = atomic_load_explicit(&tib->tib_tid, memory_order_relaxed);
if (me <= 0) if (me <= 0)
return false; return;
word = atomic_load_explicit(&__maps.lock.word, memory_order_relaxed); word = atomic_load_explicit(&__maps.lock.word, memory_order_relaxed);
for (;;) { for (;;) {
if (MUTEX_OWNER(word) == me) { if (MUTEX_OWNER(word) == me) {
if (atomic_compare_exchange_weak_explicit( if (atomic_compare_exchange_weak_explicit(
&__maps.lock.word, &word, MUTEX_INC_DEPTH(word), &__maps.lock.word, &word, MUTEX_INC_DEPTH(word),
memory_order_relaxed, memory_order_relaxed)) memory_order_relaxed, memory_order_relaxed))
return true; return;
continue; continue;
} }
#if DEBUG_MAPS_LOCK #if DEBUG_MAPS_LOCK
@ -162,7 +170,7 @@ ABI bool __maps_lock(void) {
__deadlock_track(&__maps.lock, 0); __deadlock_track(&__maps.lock, 0);
__deadlock_record(&__maps.lock, 0); __deadlock_record(&__maps.lock, 0);
#endif #endif
return false; return;
} }
for (;;) { for (;;) {
word = atomic_load_explicit(&__maps.lock.word, memory_order_relaxed); word = atomic_load_explicit(&__maps.lock.word, memory_order_relaxed);

14
libc/intrin/maps.h
View file

@ -5,17 +5,16 @@
#include "libc/runtime/runtime.h" #include "libc/runtime/runtime.h"
COSMOPOLITAN_C_START_ COSMOPOLITAN_C_START_
#define MAPS_RETRY ((void *)-1)
#define MAP_TREE_CONTAINER(e) TREE_CONTAINER(struct Map, tree, e) #define MAP_TREE_CONTAINER(e) TREE_CONTAINER(struct Map, tree, e)
struct Map { struct Map {
char *addr; /* granule aligned */ char *addr; /* granule aligned */
size_t size; /* must be nonzero */ size_t size; /* must be nonzero */
int64_t off; /* ignore for anon */ int64_t off; /* ignore for anon */
int prot; /* memory protects */
int flags; /* memory map flag */ int flags; /* memory map flag */
char prot; /* memory protects */
bool iscow; /* windows nt only */ bool iscow; /* windows nt only */
bool precious; /* windows nt only */
bool readonlyfile; /* windows nt only */ bool readonlyfile; /* windows nt only */
unsigned visited; /* checks and fork */ unsigned visited; /* checks and fork */
intptr_t hand; /* windows nt only */ intptr_t hand; /* windows nt only */
@ -39,7 +38,11 @@ struct Maps {
size_t pages; size_t pages;
struct Map stack; struct Map stack;
struct Map guard; struct Map guard;
struct Map spool[13]; #ifdef MODE_DBG
struct Map spool[1];
#else
struct Map spool[20];
#endif
}; };
struct AddrSize { struct AddrSize {
@ -49,8 +52,9 @@ struct AddrSize {
extern struct Maps __maps; extern struct Maps __maps;
bool __maps_held(void);
void __maps_init(void); void __maps_init(void);
bool __maps_lock(void); void __maps_lock(void);
void __maps_check(void); void __maps_check(void);
void __maps_unlock(void); void __maps_unlock(void);
void *__maps_randaddr(void); void *__maps_randaddr(void);

369
libc/intrin/mmap.c
View file

@ -43,13 +43,16 @@
#include "libc/sysv/consts/o.h" #include "libc/sysv/consts/o.h"
#include "libc/sysv/consts/prot.h" #include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h" #include "libc/sysv/errfuns.h"
#include "libc/thread/lock.h"
#include "libc/thread/tls.h"
#define MMDEBUG 0 #define MMDEBUG 0
#define MAX_SIZE 0x0ff800000000ul #define MAX_SIZE 0x0ff800000000ul
#define MAP_FIXED_NOREPLACE_linux 0x100000 #define MAP_FIXED_NOREPLACE_linux 0x100000
#define PGUP(x) (((x) + pagesz - 1) & -pagesz) #define PGUP(x) (((x) + __pagesize - 1) & -__pagesize)
#define GRUP(x) (((x) + __gransize - 1) & -__gransize)
#define MASQUE 0x00fffffffffffff8 #define MASQUE 0x00fffffffffffff8
#define PTR(x) ((uintptr_t)(x) & MASQUE) #define PTR(x) ((uintptr_t)(x) & MASQUE)
@ -88,7 +91,6 @@ privileged optimizespeed struct Map *__maps_floor(const char *addr) {
} }
static bool __maps_overlaps(const char *addr, size_t size) { static bool __maps_overlaps(const char *addr, size_t size) {
int pagesz = __pagesize;
struct Map *map, *floor = __maps_floor(addr); struct Map *map, *floor = __maps_floor(addr);
for (map = floor; map && map->addr <= addr + size; map = __maps_next(map)) for (map = floor; map && map->addr <= addr + size; map = __maps_next(map))
if (MAX(addr, map->addr) < if (MAX(addr, map->addr) <
@ -101,7 +103,6 @@ void __maps_check(void) {
#if MMDEBUG #if MMDEBUG
size_t maps = 0; size_t maps = 0;
size_t pages = 0; size_t pages = 0;
int pagesz = __pagesize;
static unsigned mono; static unsigned mono;
unsigned id = ++mono; unsigned id = ++mono;
for (struct Map *map = __maps_first(); map; map = __maps_next(map)) { for (struct Map *map = __maps_first(); map; map = __maps_next(map)) {
@ -109,7 +110,7 @@ void __maps_check(void) {
ASSERT(map->visited != id); ASSERT(map->visited != id);
ASSERT(map->size); ASSERT(map->size);
map->visited = id; map->visited = id;
pages += (map->size + pagesz - 1) / pagesz; pages += (map->size + __pagesize - 1) / __pagesize;
maps += 1; maps += 1;
struct Map *next; struct Map *next;
if ((next = __maps_next(map))) { if ((next = __maps_next(map))) {
@ -123,89 +124,77 @@ void __maps_check(void) {
#endif #endif
} }
static int __muntrack(char *addr, size_t size, int pagesz, static int __muntrack(char *addr, size_t size, struct Map **deleted,
struct Map **deleted) { struct Map **untracked, struct Map temp[2]) {
int rc = 0; int rc = 0;
size_t ti = 0;
struct Map *map; struct Map *map;
struct Map *next; struct Map *next;
struct Map *floor; struct Map *floor;
StartOver: size = PGUP(size);
floor = __maps_floor(addr); floor = __maps_floor(addr);
for (map = floor; map && map->addr <= addr + size; map = next) { for (map = floor; map && map->addr <= addr + size; map = next) {
next = __maps_next(map); next = __maps_next(map);
char *map_addr = map->addr; char *map_addr = map->addr;
size_t map_size = map->size; size_t map_size = map->size;
if (!(MAX(addr, map_addr) < if (!(MAX(addr, map_addr) < MIN(addr + size, map_addr + PGUP(map_size))))
MIN(addr + PGUP(size), map_addr + PGUP(map_size)))) continue;
if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) {
if (map->precious)
continue; continue;
if (addr <= map_addr && addr + PGUP(size) >= map_addr + PGUP(map_size)) {
// remove mapping completely // remove mapping completely
tree_remove(&__maps.maps, &map->tree); tree_remove(&__maps.maps, &map->tree);
map->freed = *deleted; map->freed = *deleted;
*deleted = map; *deleted = map;
__maps.pages -= (map_size + pagesz - 1) / pagesz; __maps.pages -= (map_size + __pagesize - 1) / __pagesize;
__maps.count -= 1; __maps.count -= 1;
__maps_check(); __maps_check();
} else if (IsWindows()) { } else if (IsWindows()) {
STRACE("you can't carve up memory maps on windows ;_;"); STRACE("you can't carve up memory maps on windows ;_;");
rc = einval(); rc = enotsup();
} else if (addr <= map_addr) { } else if (addr <= map_addr) {
// shave off lefthand side of mapping // shave off lefthand side of mapping
ASSERT(addr + PGUP(size) < map_addr + PGUP(map_size)); ASSERT(addr + size < map_addr + PGUP(map_size));
size_t left = addr + PGUP(size) - map_addr; size_t left = addr + size - map_addr;
size_t right = map_size - left; size_t right = map_size - left;
ASSERT(right > 0); ASSERT(right > 0);
ASSERT(left > 0); ASSERT(left > 0);
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
if (leftmap == MAPS_RETRY)
goto StartOver;
map->addr += left; map->addr += left;
map->size = right; map->size = right;
if (!(map->flags & MAP_ANONYMOUS)) if (!(map->flags & MAP_ANONYMOUS))
map->off += left; map->off += left;
__maps.pages -= (left + pagesz - 1) / pagesz; __maps.pages -= (left + __pagesize - 1) / __pagesize;
leftmap->addr = map_addr; if (untracked) {
leftmap->size = left; ASSERT(ti < 2);
leftmap->freed = *deleted; temp[ti].addr = map_addr;
*deleted = leftmap; temp[ti].size = left;
__maps_check(); temp[ti].freed = *untracked;
} else { *untracked = temp;
rc = -1; ++ti;
} }
} else if (addr + PGUP(size) >= map_addr + PGUP(map_size)) { __maps_check();
} else if (addr + size >= map_addr + PGUP(map_size)) {
// shave off righthand side of mapping // shave off righthand side of mapping
size_t left = addr - map_addr; size_t left = addr - map_addr;
size_t right = map_addr + map_size - addr; size_t right = map_addr + map_size - addr;
struct Map *rightmap;
if ((rightmap = __maps_alloc())) {
if (rightmap == MAPS_RETRY)
goto StartOver;
map->size = left; map->size = left;
__maps.pages -= (right + pagesz - 1) / pagesz; __maps.pages -= (right + __pagesize - 1) / __pagesize;
rightmap->addr = addr; if (untracked) {
rightmap->size = right; ASSERT(ti < 2);
rightmap->freed = *deleted; temp[ti].addr = addr;
*deleted = rightmap; temp[ti].size = right;
__maps_check(); temp[ti].freed = *untracked;
} else { *untracked = temp;
rc = -1; ++ti;
} }
__maps_check();
} else { } else {
// punch hole in mapping // punch hole in mapping
size_t left = addr - map_addr; size_t left = addr - map_addr;
size_t middle = PGUP(size); size_t middle = size;
size_t right = map_size - middle - left; size_t right = map_size - middle - left;
struct Map *leftmap; struct Map *leftmap;
if ((leftmap = __maps_alloc())) { if ((leftmap = __maps_alloc())) {
if (leftmap == MAPS_RETRY)
goto StartOver;
struct Map *middlemap;
if ((middlemap = __maps_alloc())) {
if (middlemap == MAPS_RETRY) {
__maps_free(leftmap);
goto StartOver;
}
leftmap->addr = map_addr; leftmap->addr = map_addr;
leftmap->size = left; leftmap->size = left;
leftmap->off = map->off; leftmap->off = map->off;
@ -216,17 +205,17 @@ StartOver:
if (!(map->flags & MAP_ANONYMOUS)) if (!(map->flags & MAP_ANONYMOUS))
map->off += left + middle; map->off += left + middle;
tree_insert(&__maps.maps, &leftmap->tree, __maps_compare); tree_insert(&__maps.maps, &leftmap->tree, __maps_compare);
__maps.pages -= (middle + pagesz - 1) / pagesz; __maps.pages -= (middle + __pagesize - 1) / __pagesize;
__maps.count += 1; __maps.count += 1;
middlemap->addr = addr; if (untracked) {
middlemap->size = size; ASSERT(ti < 2);
middlemap->freed = *deleted; temp[ti].addr = addr;
*deleted = middlemap; temp[ti].size = size;
__maps_check(); temp[ti].freed = *untracked;
} else { *untracked = temp;
__maps_free(leftmap); ++ti;
rc = -1;
} }
__maps_check();
} else { } else {
rc = -1; rc = -1;
} }
@ -258,13 +247,33 @@ static void __maps_free_all(struct Map *list) {
} }
} }
static int __maps_funge_prot(int prot) { static void __maps_insert_all(struct Map *list) {
prot &= ~MAP_FIXED; struct Map *next;
prot &= ~MAP_FIXED_NOREPLACE; for (struct Map *map = list; map; map = next) {
return prot; next = map->freed;
__maps_insert(map);
}
}
static int __maps_destroy_all(struct Map *list) {
int rc = 0;
for (struct Map *map = list; map; map = map->freed) {
if (!IsWindows()) {
if (sys_munmap(map->addr, map->size))
rc = -1;
} else if (map->hand != -1) {
if (!UnmapViewOfFile(map->addr))
rc = -1;
if (!CloseHandle(map->hand))
rc = -1;
}
}
return rc;
} }
static int __maps_funge_flags(int flags) { static int __maps_funge_flags(int flags) {
flags &= ~MAP_FIXED;
flags &= ~MAP_FIXED_NOREPLACE;
if ((flags & MAP_TYPE) == MAP_SHARED_VALIDATE) { if ((flags & MAP_TYPE) == MAP_SHARED_VALIDATE) {
flags &= ~MAP_TYPE; flags &= ~MAP_TYPE;
flags |= MAP_SHARED; flags |= MAP_SHARED;
@ -280,20 +289,20 @@ static bool __maps_fungible(const struct Map *map) {
} }
static bool __maps_adjacent(const struct Map *x, const struct Map *y) { static bool __maps_adjacent(const struct Map *x, const struct Map *y) {
char *a = x->addr + ((x->size + __pagesize - 1) & -__pagesize); char *a = x->addr + PGUP(x->size);
char *b = y->addr; char *b = y->addr;
ASSERT(a <= b); ASSERT(a <= b);
return a == b; return a == b;
} }
static bool __maps_mergeable(const struct Map *x, const struct Map *y) { static bool __maps_mergeable(const struct Map *x, const struct Map *y) {
if (!__maps_adjacent(x, y))
return false;
if (!__maps_fungible(x)) if (!__maps_fungible(x))
return false; return false;
if (!__maps_fungible(y)) if (!__maps_fungible(y))
return false; return false;
if (!__maps_adjacent(x, y)) if (x->prot != y->prot)
return false;
if (__maps_funge_prot(x->prot) != __maps_funge_prot(y->prot))
return false; return false;
if (__maps_funge_flags(x->flags) != __maps_funge_flags(y->flags)) if (__maps_funge_flags(x->flags) != __maps_funge_flags(y->flags))
return false; return false;
@ -304,7 +313,6 @@ void __maps_insert(struct Map *map) {
struct Map *left, *right; struct Map *left, *right;
ASSERT(map->size); ASSERT(map->size);
ASSERT(!__maps_overlaps(map->addr, map->size)); ASSERT(!__maps_overlaps(map->addr, map->size));
map->flags &= MAP_TYPE | MAP_ANONYMOUS | MAP_NOFORK;
__maps.pages += (map->size + __pagesize - 1) / __pagesize; __maps.pages += (map->size + __pagesize - 1) / __pagesize;
// find adjacent mappings // find adjacent mappings
@ -317,8 +325,7 @@ void __maps_insert(struct Map *map) {
// avoid insert by making mapping on left bigger // avoid insert by making mapping on left bigger
if (left) if (left)
if (__maps_mergeable(left, map)) { if (__maps_mergeable(left, map)) {
left->size += __pagesize - 1; left->size = PGUP(left->size);
left->size &= -__pagesize;
left->size += map->size; left->size += map->size;
__maps_free(map); __maps_free(map);
map = 0; map = 0;
@ -327,8 +334,7 @@ void __maps_insert(struct Map *map) {
// avoid insert by making mapping on right bigger // avoid insert by making mapping on right bigger
if (map && right) if (map && right)
if (__maps_mergeable(map, right)) { if (__maps_mergeable(map, right)) {
map->size += __pagesize - 1; map->size = PGUP(map->size);
map->size &= -__pagesize;
right->addr -= map->size; right->addr -= map->size;
right->size += map->size; right->size += map->size;
__maps_free(map); __maps_free(map);
@ -338,14 +344,12 @@ void __maps_insert(struct Map *map) {
// check if we filled a hole // check if we filled a hole
if (!map && left && right) if (!map && left && right)
if (__maps_mergeable(left, right)) { if (__maps_mergeable(left, right)) {
left->size += __pagesize - 1; left->size = PGUP(left->size);
left->size &= -__pagesize;
right->addr -= left->size; right->addr -= left->size;
right->size += left->size; right->size += left->size;
tree_remove(&__maps.maps, &left->tree); tree_remove(&__maps.maps, &left->tree);
__maps.count -= 1;
__maps_free(left); __maps_free(left);
map = 0; __maps.count -= 1;
} }
// otherwise just insert // otherwise just insert
@ -356,26 +360,19 @@ void __maps_insert(struct Map *map) {
__maps_check(); __maps_check();
} }
static void __maps_track_insert(struct Map *map, char *addr, size_t size, // adds interval to rbtree (no sys_mmap)
uintptr_t map_handle, int prot, int flags) { bool __maps_track(char *addr, size_t size, int prot, int flags) {
struct Map *map;
if (!(map = __maps_alloc()))
return false;
map->addr = addr; map->addr = addr;
map->size = size; map->size = size;
map->prot = prot; map->prot = prot;
map->flags = flags; map->flags = flags;
map->hand = map_handle; map->hand = -1;
__maps_lock(); __maps_lock();
__maps_insert(map); __maps_insert(map);
__maps_unlock(); __maps_unlock();
}
// adds interval to rbtree (no sys_mmap)
bool __maps_track(char *addr, size_t size, int prot, int flags) {
struct Map *map;
do {
if (!(map = __maps_alloc()))
return false;
} while (map == MAPS_RETRY);
__maps_track_insert(map, addr, size, -1, prot, flags);
return true; return true;
} }
@ -383,7 +380,7 @@ bool __maps_track(char *addr, size_t size, int prot, int flags) {
int __maps_untrack(char *addr, size_t size) { int __maps_untrack(char *addr, size_t size) {
struct Map *deleted = 0; struct Map *deleted = 0;
__maps_lock(); __maps_lock();
int rc = __muntrack(addr, size, __pagesize, &deleted); int rc = __muntrack(addr, size, &deleted, 0, 0);
__maps_unlock(); __maps_unlock();
__maps_free_all(deleted); __maps_free_all(deleted);
return rc; return rc;
@ -399,29 +396,22 @@ struct Map *__maps_alloc(void) {
return map; return map;
pthread_pause_np(); pthread_pause_np();
} }
void *mark;
int size = 65536; int size = 65536;
__maps_lock();
do {
// we're creating sudden surprise memory. the user might be in the // we're creating sudden surprise memory. the user might be in the
// middle of carefully planning a fixed memory structure. we don't // middle of carefully planning a fixed memory structure. we don't
// want the system allocator to put our surprise memory inside it. // want the system allocator to put our surprise memory inside it,
mark = __maps_randaddr(); // and we also want to avoid the chances of accidentally unmapping
} while (__maps_overlaps(mark, size)); struct DirectMap sys =
struct DirectMap sys = sys_mmap(mark, size, PROT_READ | PROT_WRITE, sys_mmap(__maps_randaddr(), size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (sys.addr == MAP_FAILED) { if (sys.addr == MAP_FAILED)
__maps_unlock();
return 0; return 0;
}
map = sys.addr; map = sys.addr;
__maps_track_insert(map, sys.addr, size, sys.maphandle, if (IsWindows())
PROT_READ | PROT_WRITE, CloseHandle(sys.maphandle);
MAP_PRIVATE | MAP_ANONYMOUS | MAP_NOFORK);
__maps_unlock();
for (int i = 1; i < size / sizeof(struct Map); ++i) for (int i = 1; i < size / sizeof(struct Map); ++i)
__maps_free(map + i); __maps_free(map + i);
return MAPS_RETRY; return map;
} }
static int __munmap(char *addr, size_t size) { static int __munmap(char *addr, size_t size) {
@ -431,41 +421,33 @@ static int __munmap(char *addr, size_t size) {
!size || (uintptr_t)addr + size < size) !size || (uintptr_t)addr + size < size)
return einval(); return einval();
// test for signal handler tragedy
if (__maps_held())
return edeadlk();
// lock the memory manager // lock the memory manager
__maps_lock(); __maps_lock();
__maps_check(); __maps_check();
// normalize size // normalize size
// abort if size doesn't include all pages in granule // abort if size doesn't include all pages in granule
size_t pgup_size = (size + __pagesize - 1) & -__pagesize; if (GRUP(size) > PGUP(size))
size_t grup_size = (size + __gransize - 1) & -__gransize; if (__maps_overlaps(addr + PGUP(size), GRUP(size) - PGUP(size))) {
if (grup_size > pgup_size)
if (__maps_overlaps(addr + pgup_size, grup_size - pgup_size)) {
__maps_unlock(); __maps_unlock();
return einval(); return einval();
} }
// untrack mappings // untrack mappings
int rc; int rc;
struct Map temp[2];
struct Map *deleted = 0; struct Map *deleted = 0;
rc = __muntrack(addr, pgup_size, __pagesize, &deleted); struct Map *untracked = 0;
rc = __muntrack(addr, size, &deleted, &untracked, temp);
__maps_unlock(); __maps_unlock();
// delete mappings // ask operating system to remove mappings
for (struct Map *map = deleted; map; map = map->freed) { rc |= __maps_destroy_all(untracked);
if (!IsWindows()) { rc |= __maps_destroy_all(deleted);
if (sys_munmap(map->addr, map->size))
rc = -1;
} else if (map->hand != -1) {
ASSERT(!((uintptr_t)map->addr & (__gransize - 1)));
if (!UnmapViewOfFile(map->addr))
rc = -1;
if (!CloseHandle(map->hand))
rc = -1;
}
}
// freed mappings
__maps_free_all(deleted); __maps_free_all(deleted);
return rc; return rc;
@ -485,14 +467,13 @@ void *__maps_randaddr(void) {
static void *__maps_pickaddr(size_t size) { static void *__maps_pickaddr(size_t size) {
char *addr = 0; char *addr = 0;
struct Map *map, *prev; struct Map *map, *prev;
size += __gransize - 1; size = GRUP(size);
size &= -__gransize;
if ((map = __maps_last())) { if ((map = __maps_last())) {
// choose address beneath higher mapping // choose address beneath higher mapping
for (; map; map = prev) { for (; map; map = prev) {
char *min = (char *)(intptr_t)__gransize; char *min = (char *)(intptr_t)__gransize;
if ((prev = __maps_prev(map))) if ((prev = __maps_prev(map)))
min = prev->addr + ((prev->size + __gransize - 1) & -__gransize); min = prev->addr + GRUP(prev->size);
if (map->addr > min && // if (map->addr > min && //
map->addr - min >= size) { map->addr - min >= size) {
addr = map->addr - size; addr = map->addr - size;
@ -502,7 +483,7 @@ static void *__maps_pickaddr(size_t size) {
// append if existing maps are too dense // append if existing maps are too dense
if (!addr) { if (!addr) {
map = __maps_last(); map = __maps_last();
addr = map->addr + ((map->size + __gransize - 1) & -__gransize); addr = map->addr + GRUP(map->size);
intptr_t end = (intptr_t)addr; intptr_t end = (intptr_t)addr;
if (ckd_add(&end, end, size)) if (ckd_add(&end, end, size))
return 0; return 0;
@ -518,7 +499,12 @@ static void *__mmap_impl(char *addr, size_t size, int prot, int flags, int fd,
int64_t off) { int64_t off) {
// validate file map args // validate file map args
if (!(flags & MAP_ANONYMOUS)) { if (flags & MAP_ANONYMOUS) {
// some operating systems will complain unless we do this
fd = -1;
off = 0;
} else {
// validate arguments for file mapping
if (off & (__gransize - 1)) if (off & (__gransize - 1))
return (void *)einval(); return (void *)einval();
if (IsWindows()) { if (IsWindows()) {
@ -531,10 +517,8 @@ static void *__mmap_impl(char *addr, size_t size, int prot, int flags, int fd,
// allocate Map object // allocate Map object
struct Map *map; struct Map *map;
do {
if (!(map = __maps_alloc())) if (!(map = __maps_alloc()))
return MAP_FAILED; return MAP_FAILED;
} while (map == MAPS_RETRY);
// polyfill nuances of fixed mappings // polyfill nuances of fixed mappings
int sysflags = flags; int sysflags = flags;
@ -588,18 +572,29 @@ static void *__mmap_impl(char *addr, size_t size, int prot, int flags, int fd,
} }
} else { } else {
// remove existing mappings and their tracking objects // remove existing mappings and their tracking objects
if (__munmap(addr, size)) { struct Map *deleted = 0;
if (__muntrack(addr, size, &deleted, 0, 0)) {
__maps_insert_all(deleted);
__maps_unlock(); __maps_unlock();
__maps_free(map); __maps_free(map);
return (void *)enomem(); return MAP_FAILED;
}
int rc = __maps_destroy_all(deleted);
__maps_free_all(deleted);
if (rc) {
__maps_unlock();
__maps_free(map);
return (void *)eperm();
} }
} }
// claims intended interval while still holding the lock // claims intended interval while still holding the lock
if (!__maps_track(addr, size, 0, 0)) { map->addr = addr;
__maps_unlock(); map->size = size;
__maps_free(map); map->prot = 0;
return (void *)enomem(); map->flags = 0;
} map->hand = -1;
map->precious = true;
__maps_insert(map);
__maps_unlock(); __maps_unlock();
} }
@ -611,15 +606,19 @@ static void *__mmap_impl(char *addr, size_t size, int prot, int flags, int fd,
// handle failure // handle failure
if (IsWindows()) { if (IsWindows()) {
// untrack reservation
__maps_lock();
tree_remove(&__maps.maps, &map->tree);
__maps.pages -= (map->size + __pagesize - 1) / __pagesize;
map->precious = false;
__maps_unlock();
if (errno == EADDRNOTAVAIL) { if (errno == EADDRNOTAVAIL) {
// we've encountered mystery memory // we've encountered mystery memory
if (fixedmode) { if (fixedmode) {
// TODO(jart): Use VirtualQuery() to destroy mystery memory. // TODO(jart): Use VirtualQuery() to destroy mystery memory.
__maps_untrack(addr, size);
errno = ENOMEM; errno = ENOMEM;
} else if (noreplace) { } else if (noreplace) {
// we can't try again with a different address in this case // we can't try again with a different address in this case
__maps_untrack(addr, size);
errno = EEXIST; errno = EEXIST;
} else { } else {
// we shall leak the tracking object since it should at least // we shall leak the tracking object since it should at least
@ -629,8 +628,6 @@ static void *__mmap_impl(char *addr, size_t size, int prot, int flags, int fd,
addr = 0; addr = 0;
continue; continue;
} }
} else {
__maps_untrack(addr, size);
} }
} }
__maps_free(map); __maps_free(map);
@ -652,6 +649,7 @@ static void *__mmap_impl(char *addr, size_t size, int prot, int flags, int fd,
map->prot = prot; map->prot = prot;
map->flags = flags; map->flags = flags;
map->hand = res.maphandle; map->hand = res.maphandle;
map->precious = false;
if (IsWindows()) { if (IsWindows()) {
map->iscow = (flags & MAP_TYPE) != MAP_SHARED && fd != -1; map->iscow = (flags & MAP_TYPE) != MAP_SHARED && fd != -1;
map->readonlyfile = (flags & MAP_TYPE) == MAP_SHARED && fd != -1 && map->readonlyfile = (flags & MAP_TYPE) == MAP_SHARED && fd != -1 &&
@ -659,11 +657,18 @@ static void *__mmap_impl(char *addr, size_t size, int prot, int flags, int fd,
} }
// track map object // track map object
if (!IsWindows()) {
struct Map *deleted = 0;
__maps_lock(); __maps_lock();
if (IsWindows() || fixedmode) if (IsWindows() || fixedmode)
__maps_untrack(res.addr, size); if (__muntrack(res.addr, size, &deleted, 0, 0))
STRACE("memtrack compromised by hole punch oom");
__maps_insert(map); __maps_insert(map);
__maps_unlock(); __maps_unlock();
__maps_free_all(deleted);
} else {
atomic_thread_fence(memory_order_release);
}
return res.addr; return res.addr;
} }
@ -686,6 +691,10 @@ static void *__mmap(char *addr, size_t size, int prot, int flags, int fd,
if (__maps.count * __pagesize + size > __virtualmax) if (__maps.count * __pagesize + size > __virtualmax)
return (void *)enomem(); return (void *)enomem();
// test for signal handler reentry
if (__maps_held())
return (void *)edeadlk();
// create memory mappping // create memory mappping
if (!__isfdkind(fd, kFdZip)) { if (!__isfdkind(fd, kFdZip)) {
res = __mmap_impl(addr, size, prot, flags, fd, off); res = __mmap_impl(addr, size, prot, flags, fd, off);
@ -699,40 +708,32 @@ static void *__mmap(char *addr, size_t size, int prot, int flags, int fd,
} }
static void *__mremap_impl(char *old_addr, size_t old_size, size_t new_size, static void *__mremap_impl(char *old_addr, size_t old_size, size_t new_size,
int flags, char *new_addr, int pagesz, int gransz) { int flags, char *new_addr) {
// normalize and validate old size // normalize and validate old size
// abort if size doesn't include all pages in granule // abort if size doesn't include all pages in granule
size_t pgup_old_size = (old_size + pagesz - 1) & -pagesz; if (GRUP(old_size) > PGUP(old_size))
size_t grup_old_size = (old_size + gransz - 1) & -gransz; if (__maps_overlaps(old_addr + PGUP(old_size),
if (grup_old_size > pgup_old_size) GRUP(old_size) - PGUP(old_size)))
if (__maps_overlaps(old_addr + pgup_old_size,
grup_old_size - pgup_old_size))
return (void *)einval(); return (void *)einval();
old_size = pgup_old_size;
// validate new size // validate new size
// abort if size doesn't include all pages in granule // abort if size doesn't include all pages in granule
if (flags & MREMAP_FIXED) { if (flags & MREMAP_FIXED)
size_t pgup_new_size = (new_size + pagesz - 1) & -pagesz; if (GRUP(new_size) > PGUP(new_size))
size_t grup_new_size = (new_size + gransz - 1) & -gransz; if (__maps_overlaps(new_addr + PGUP(new_size),
if (grup_new_size > pgup_new_size) GRUP(new_size) - PGUP(new_size)))
if (__maps_overlaps(new_addr + pgup_new_size,
grup_new_size - pgup_new_size))
return (void *)einval(); return (void *)einval();
}
// allocate object for tracking new mapping // allocate object for tracking new mapping
struct Map *map; struct Map *map;
do {
if (!(map = __maps_alloc())) if (!(map = __maps_alloc()))
return (void *)enomem(); return (void *)enomem();
} while (map == MAPS_RETRY);
// check old interval is fully contained within one mapping // check old interval is fully contained within one mapping
struct Map *old_map; struct Map *old_map;
if (!(old_map = __maps_floor(old_addr)) || if (!(old_map = __maps_floor(old_addr)) ||
old_addr + old_size > old_map->addr + PGUP(old_map->size) || old_addr + PGUP(old_size) > old_map->addr + PGUP(old_map->size) ||
old_addr < old_map->addr) { old_addr < old_map->addr) {
__maps_free(map); __maps_free(map);
return (void *)efault(); return (void *)efault();
@ -777,7 +778,7 @@ static void *__mremap_impl(char *old_addr, size_t old_size, size_t new_size,
// untrack old mapping // untrack old mapping
struct Map *deleted = 0; struct Map *deleted = 0;
__muntrack(old_addr, old_size, pagesz, &deleted); __muntrack(old_addr, old_size, &deleted, 0, 0);
__maps_free_all(deleted); __maps_free_all(deleted);
// track map object // track map object
@ -794,9 +795,6 @@ static void *__mremap_impl(char *old_addr, size_t old_size, size_t new_size,
static void *__mremap(char *old_addr, size_t old_size, size_t new_size, static void *__mremap(char *old_addr, size_t old_size, size_t new_size,
int flags, char *new_addr) { int flags, char *new_addr) {
int pagesz = __pagesize;
int gransz = __gransize;
// kernel support // kernel support
if (!IsLinux() && !IsNetbsd()) if (!IsLinux() && !IsNetbsd())
return (void *)enosys(); return (void *)enosys();
@ -810,17 +808,16 @@ static void *__mremap(char *old_addr, size_t old_size, size_t new_size,
// we support these flags // we support these flags
if (flags & ~(MREMAP_MAYMOVE | MREMAP_FIXED)) if (flags & ~(MREMAP_MAYMOVE | MREMAP_FIXED))
return (void *)einval(); return (void *)einval();
if (IsNetbsd() && !(flags & MREMAP_MAYMOVE) && if (IsNetbsd() && !(flags & MREMAP_MAYMOVE) && PGUP(new_size) > old_size)
((new_size + pagesz - 1) & -pagesz) > old_size)
return (void *)enotsup(); return (void *)enotsup();
if ((flags & MREMAP_FIXED) && !(flags & MREMAP_MAYMOVE)) if ((flags & MREMAP_FIXED) && !(flags & MREMAP_MAYMOVE))
return (void *)einval(); return (void *)einval();
// addresses must be granularity aligned // addresses must be granularity aligned
if ((uintptr_t)old_addr & (gransz - 1)) if ((uintptr_t)old_addr & (__gransize - 1))
return (void *)einval(); return (void *)einval();
if (flags & MREMAP_FIXED) if (flags & MREMAP_FIXED)
if ((uintptr_t)new_addr & (gransz - 1)) if ((uintptr_t)new_addr & (__gransize - 1))
return (void *)einval(); return (void *)einval();
// sizes must not be zero // sizes must not be zero
@ -850,20 +847,19 @@ static void *__mremap(char *old_addr, size_t old_size, size_t new_size,
// memory increase must not exceed RLIMIT_AS // memory increase must not exceed RLIMIT_AS
if (PGUP(new_size) > old_size) if (PGUP(new_size) > old_size)
if (__maps.count * pagesz - old_size + PGUP(new_size) > __virtualmax) if (__maps.count * __pagesize - old_size + PGUP(new_size) > __virtualmax)
return (void *)enomem(); return (void *)enomem();
// lock the memory manager // test for signal handler reentry
// abort on reentry due to signal handler if (__maps_held())
if (__maps_lock()) {
__maps_unlock();
return (void *)edeadlk(); return (void *)edeadlk();
}
// lock the memory manager
__maps_lock();
__maps_check(); __maps_check();
// perform operation // perform operation
char *res = __mremap_impl(old_addr, old_size, new_size, flags, new_addr, char *res = __mremap_impl(old_addr, old_size, new_size, flags, new_addr);
pagesz, gransz);
// return result // return result
__maps_unlock(); __maps_unlock();
@ -940,6 +936,24 @@ static void *__mremap(char *old_addr, size_t old_size, size_t new_size,
* The `MAP_CONCEAL` flag may be passed to prevent a memory mapping from * The `MAP_CONCEAL` flag may be passed to prevent a memory mapping from
* appearing in core dumps. This is currently supported on BSD OSes, and * appearing in core dumps. This is currently supported on BSD OSes, and
* is ignored on everything else. * is ignored on everything else.
*
* POSIX does not require mmap() to be asynchronous signal safe. But you
* should be able to call this from a signal handler safely, if you know
* that your signal will never interrupt the cosmopolitan memory manager
* and the only way you can ensure that, is by blocking signals whenever
* you call mmap(), munmap(), mprotect(), etc.
*
* @raise ENOMEM if `RUSAGE_AS` or similar limits are exceeded
* @raise EEXIST if `flags` has `MAP_FIXED_NOREPLACE` and `addr` is used
* @raise EPERM if `addr` is null and `flags` has `MAP_FIXED`
* @raise ENOTSUP if memory map is cleaved on windows with `MAP_FIXED`
* @raise EINVAL if `addr` isn't granularity aligned with `MAP_FIXED`
* @raise EINVAL if `size` is zero
* @raise EINVAL if `flags` or `prot` hold invalid values
* @raise EACCESS if `fd` isn't a regular file
* @raise EACCESS if `fd` was opened in write-only mode
* @raise EACCESS if `off` isn't getgransize() aligned
* @raise EDEADLK if called from signal handler interrupting mmap()
*/ */
void *mmap(void *addr, size_t size, int prot, int flags, int fd, int64_t off) { void *mmap(void *addr, size_t size, int prot, int flags, int fd, int64_t off) {
void *res = __mmap(addr, size, prot, flags, fd, off); void *res = __mmap(addr, size, prot, flags, fd, off);
@ -985,6 +999,11 @@ void *mremap(void *old_addr, size_t old_size, size_t new_size, int flags, ...) {
* The `size` parameter is implicitly rounded up to the page size. * The `size` parameter is implicitly rounded up to the page size.
* *
* @return 0 on success, or -1 w/ errno. * @return 0 on success, or -1 w/ errno.
* @raise ENOMEM if OOM happened when punching hole in existing mapping
* @raise ENOTSUP if memory map is cleaved on windows with `MAP_FIXED`
* @raise EDEADLK if called from signal handler interrupting mmap()
* @raise EINVAL if `addr` isn't granularity aligned
* @raise EINVAL if `size` didn't include all pages in granule
*/ */
int munmap(void *addr, size_t size) { int munmap(void *addr, size_t size) {
int rc = __munmap(addr, size); int rc = __munmap(addr, size);

35
libc/intrin/mprotect.c
View file

@ -66,15 +66,15 @@ int __mprotect(char *addr, size_t size, int prot) {
// normalize size // normalize size
size = (size + pagesz - 1) & -pagesz; size = (size + pagesz - 1) & -pagesz;
// test for signal handler reentry
if (__maps_held())
return edeadlk();
// change mappings // change mappings
int rc = 0; int rc = 0;
bool found = false; bool found = false;
if (__maps_lock()) { __maps_lock();
__maps_unlock();
return edeadlk();
}
struct Map *map, *floor; struct Map *map, *floor;
StartOver:
floor = __maps_floor(addr); floor = __maps_floor(addr);
for (map = floor; map && map->addr <= addr + size; map = __maps_next(map)) { for (map = floor; map && map->addr <= addr + size; map = __maps_next(map)) {
char *map_addr = map->addr; char *map_addr = map->addr;
@ -97,8 +97,6 @@ StartOver:
size_t right = map_size - left; size_t right = map_size - left;
struct Map *leftmap; struct Map *leftmap;
if ((leftmap = __maps_alloc())) { if ((leftmap = __maps_alloc())) {
if (leftmap == MAPS_RETRY)
goto StartOver;
if (!__mprotect_chunk(map_addr, left, prot, false)) { if (!__mprotect_chunk(map_addr, left, prot, false)) {
leftmap->addr = map_addr; leftmap->addr = map_addr;
leftmap->size = left; leftmap->size = left;
@ -129,8 +127,6 @@ StartOver:
size_t right = map_addr + map_size - addr; size_t right = map_addr + map_size - addr;
struct Map *leftmap; struct Map *leftmap;
if ((leftmap = __maps_alloc())) { if ((leftmap = __maps_alloc())) {
if (leftmap == MAPS_RETRY)
goto StartOver;
if (!__mprotect_chunk(map_addr + left, right, prot, false)) { if (!__mprotect_chunk(map_addr + left, right, prot, false)) {
leftmap->addr = map_addr; leftmap->addr = map_addr;
leftmap->size = left; leftmap->size = left;
@ -163,14 +159,8 @@ StartOver:
size_t right = map_size - middle - left; size_t right = map_size - middle - left;
struct Map *leftmap; struct Map *leftmap;
if ((leftmap = __maps_alloc())) { if ((leftmap = __maps_alloc())) {
if (leftmap == MAPS_RETRY)
goto StartOver;
struct Map *midlmap; struct Map *midlmap;
if ((midlmap = __maps_alloc())) { if ((midlmap = __maps_alloc())) {
if (midlmap == MAPS_RETRY) {
__maps_free(leftmap);
goto StartOver;
}
if (!__mprotect_chunk(map_addr + left, middle, prot, false)) { if (!__mprotect_chunk(map_addr + left, middle, prot, false)) {
leftmap->addr = map_addr; leftmap->addr = map_addr;
leftmap->size = left; leftmap->size = left;
@ -221,11 +211,20 @@ StartOver:
/** /**
* Modifies restrictions on virtual memory address range. * Modifies restrictions on virtual memory address range.
* *
* @param addr needs to be 4kb aligned * POSIX doesn't require mprotect() to be async signal safe. However you
* @param prot can have PROT_{NONE,READ,WRITE,EXEC} * should be able to call this from a signal handler safely, if you know
* that your signal will never interrupt the cosmopolitan memory manager
* and the only way you can ensure that, is by blocking signals whenever
* you call mmap(), munmap(), mprotect(), etc.
*
* @param addr needs to be page size aligned
* @param size is rounded up to the page size
* @param prot can be PROT_NONE or a combination of PROT_READ,
* PROT_WRITE, and PROT_EXEC
* @return 0 on success, or -1 w/ errno * @return 0 on success, or -1 w/ errno
* @raise EINVAL if `size` is zero
* @raise ENOMEM on tracking memory oom * @raise ENOMEM on tracking memory oom
* @see mmap() * @raise EDEADLK if called from signal handler interrupting mmap()
*/ */
int mprotect(void *addr, size_t size, int prot) { int mprotect(void *addr, size_t size, int prot) {
int rc; int rc;

13
libc/intrin/msync-nt.c
View file

@ -26,17 +26,15 @@
#include "libc/sysv/errfuns.h" #include "libc/sysv/errfuns.h"
textwindows int sys_msync_nt(char *addr, size_t size, int flags) { textwindows int sys_msync_nt(char *addr, size_t size, int flags) {
size = (size + __pagesize - 1) & -__pagesize;
int pagesz = __pagesize; if ((uintptr_t)addr & (__pagesize - 1))
size = (size + pagesz - 1) & -pagesz;
if ((uintptr_t)addr & (pagesz - 1))
return einval(); return einval();
if (__maps_held())
return edeadlk();
int rc = 0; int rc = 0;
if (__maps_lock()) { __maps_lock();
rc = edeadlk();
} else {
struct Map *map, *floor; struct Map *map, *floor;
floor = __maps_floor(addr); floor = __maps_floor(addr);
for (map = floor; map && map->addr <= addr + size; map = __maps_next(map)) { for (map = floor; map && map->addr <= addr + size; map = __maps_next(map)) {
@ -47,7 +45,6 @@ textwindows int sys_msync_nt(char *addr, size_t size, int flags) {
rc = -1; rc = -1;
// TODO(jart): FlushFileBuffers too on g_fds handle if MS_SYNC? // TODO(jart): FlushFileBuffers too on g_fds handle if MS_SYNC?
} }
}
__maps_unlock(); __maps_unlock();
return rc; return rc;

1
libc/intrin/msync.c
View file

@ -38,6 +38,7 @@
* @param flags needs MS_ASYNC or MS_SYNC and can have MS_INVALIDATE * @param flags needs MS_ASYNC or MS_SYNC and can have MS_INVALIDATE
* @return 0 on success or -1 w/ errno * @return 0 on success or -1 w/ errno
* @raise ECANCELED if thread was cancelled in masked mode * @raise ECANCELED if thread was cancelled in masked mode
* @raise EDEADLK if called from signal handler interrupting mmap()
* @raise EINTR if we needed to block and a signal was delivered instead * @raise EINTR if we needed to block and a signal was delivered instead
* @raise EINVAL if `MS_SYNC` and `MS_ASYNC` were both specified * @raise EINVAL if `MS_SYNC` and `MS_ASYNC` were both specified
* @raise EINVAL if unknown `flags` were passed * @raise EINVAL if unknown `flags` were passed

34
libc/intrin/sig.c
View file

@ -588,6 +588,22 @@ textwindows static void __sig_unmaskable(struct SignalFrame *sf) {
DescribeBacktrace( DescribeBacktrace(
(struct StackFrame *)sf->ctx.uc_mcontext.gregs[REG_RBP])); (struct StackFrame *)sf->ctx.uc_mcontext.gregs[REG_RBP]));
// kills process if the user did not specify a handler for this signal
// we also don't allow unmaskable signals to be ignored by the program
if (sf->rva == (intptr_t)SIG_DFL || //
sf->rva == (intptr_t)SIG_IGN)
__sig_death(sf->si.si_signo, "uncaught ");
// we kill the process if this thread's signal mask blocks this signal
// then we block some extra signals while executing the signal handler
struct CosmoTib *tib = __get_tls();
sigset_t blocksigs = __sighandmask[sf->si.si_signo];
if (!(sf->flags & SA_NODEFER))
blocksigs |= 1ull << (sf->si.si_signo - 1);
sf->ctx.uc_sigmask = atomic_fetch_or(&tib->tib_sigmask, blocksigs);
if (sf->ctx.uc_sigmask & (1ull << (sf->si.si_signo - 1)))
__sig_death(sf->si.si_signo, "masked ");
// this will restore the guard page if the user is using a sigaltstack // this will restore the guard page if the user is using a sigaltstack
if (sf->si.si_errno == kNtStatusGuardPageViolation) if (sf->si.si_errno == kNtStatusGuardPageViolation)
__sig_reguard(sf->si.si_addr); __sig_reguard(sf->si.si_addr);
@ -620,22 +636,6 @@ __msabi HAIRY static unsigned __sig_crash(struct NtExceptionPointers *ep) {
if (flags & SA_RESETHAND) if (flags & SA_RESETHAND)
__sighandrvas[sig] = (int32_t)(intptr_t)SIG_DFL; __sighandrvas[sig] = (int32_t)(intptr_t)SIG_DFL;
// kills process if the user did not specify a handler for this signal
// we also don't allow unmaskable signals to be ignored by the program
if (rva == (intptr_t)SIG_DFL || //
rva == (intptr_t)SIG_IGN)
__sig_death(sig, "uncaught ");
// we kill the process if this thread's signal mask blocks this signal
// then we block some extra signals while executing the signal handler
struct CosmoTib *tib = __get_tls();
sigset_t blocksigs = __sighandmask[sig];
if (!(flags & SA_NODEFER))
blocksigs |= 1ull << (sig - 1);
sigset_t oldsigmask = atomic_fetch_or(&tib->tib_sigmask, blocksigs);
if (oldsigmask & (1ull << (sig - 1)))
__sig_death(sig, "masked ");
// we don't know if it is safe for signal handlers to longjmp() out of // we don't know if it is safe for signal handlers to longjmp() out of
// win32 vectored exception handlers so let's copy the machine context // win32 vectored exception handlers so let's copy the machine context
// and tell win32 to restore control to __sig_unmaskable() which shall // and tell win32 to restore control to __sig_unmaskable() which shall
@ -643,6 +643,7 @@ __msabi HAIRY static unsigned __sig_crash(struct NtExceptionPointers *ep) {
// was caused by stack overflow, then we're literally executing inside // was caused by stack overflow, then we're literally executing inside
// the guard page so this code can't use more than 4096 bytes of stack // the guard page so this code can't use more than 4096 bytes of stack
uintptr_t sp; uintptr_t sp;
struct CosmoTib *tib = __get_tls();
if (__sig_should_use_altstack(flags, tib)) { if (__sig_should_use_altstack(flags, tib)) {
sp = (uintptr_t)tib->tib_sigstack_addr + tib->tib_sigstack_size; sp = (uintptr_t)tib->tib_sigstack_addr + tib->tib_sigstack_size;
} else { } else {
@ -654,7 +655,6 @@ __msabi HAIRY static unsigned __sig_crash(struct NtExceptionPointers *ep) {
struct SignalFrame *sf = (struct SignalFrame *)sp; struct SignalFrame *sf = (struct SignalFrame *)sp;
__repstosb(sf, 0, sizeof(*sf)); __repstosb(sf, 0, sizeof(*sf));
__sig_translate(&sf->ctx, ep->ContextRecord); __sig_translate(&sf->ctx, ep->ContextRecord);
sf->ctx.uc_sigmask = oldsigmask;
sf->rva = rva; sf->rva = rva;
sf->flags = flags; sf->flags = flags;
sf->si.si_code = sic; sf->si.si_code = sic;

13
libc/proc/proc.c
View file

@ -33,11 +33,13 @@
#include "libc/intrin/weaken.h" #include "libc/intrin/weaken.h"
#include "libc/mem/leaks.h" #include "libc/mem/leaks.h"
#include "libc/nt/accounting.h" #include "libc/nt/accounting.h"
#include "libc/nt/enum/heap.h"
#include "libc/nt/enum/processaccess.h" #include "libc/nt/enum/processaccess.h"
#include "libc/nt/enum/processcreationflags.h" #include "libc/nt/enum/processcreationflags.h"
#include "libc/nt/enum/status.h" #include "libc/nt/enum/status.h"
#include "libc/nt/enum/wait.h" #include "libc/nt/enum/wait.h"
#include "libc/nt/events.h" #include "libc/nt/events.h"
#include "libc/nt/memory.h"
#include "libc/nt/process.h" #include "libc/nt/process.h"
#include "libc/nt/runtime.h" #include "libc/nt/runtime.h"
#include "libc/nt/struct/filetime.h" #include "libc/nt/struct/filetime.h"
@ -292,16 +294,9 @@ textwindows struct Proc *__proc_new(void) {
proc = PROC_CONTAINER(e); proc = PROC_CONTAINER(e);
dll_remove(&__proc.free, &proc->elem); dll_remove(&__proc.free, &proc->elem);
} }
if (proc) { if (!proc && !(proc = HeapAlloc(GetProcessHeap(), 0, sizeof(struct Proc))))
bzero(proc, sizeof(*proc));
} else {
proc = mmap(0, sizeof(struct Proc), PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (proc == MAP_FAILED) {
enomem();
return 0; return 0;
} bzero(proc, sizeof(*proc));
}
dll_init(&proc->elem); dll_init(&proc->elem);
return proc; return proc;
} }

16
test/libc/intrin/mmap_test.c
View file

@ -534,35 +534,31 @@ void BenchMmapPrivate(void) {
void *p; void *p;
p = mmap(0, (sizes[count] = rand() % (pagesz * 500)), PROT_READ | PROT_WRITE, p = mmap(0, (sizes[count] = rand() % (pagesz * 500)), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
if (p == MAP_FAILED) ASSERT_NE(MAP_FAILED, p);
__builtin_trap();
ptrs[count] = p; ptrs[count] = p;
++count; ++count;
} }
void BenchUnmap(void) { void BenchUnmap(void) {
--count; --count;
if (munmap(ptrs[count], sizes[count])) ASSERT_SYS(0, 0, munmap(ptrs[count], sizes[count]));
__builtin_trap();
} }
void BenchBigMmap(void) { void BenchBigMmap(void) {
void *p; void *p;
p = mmap(0, 101 * 1024 * 1024, PROT_READ | PROT_WRITE, p = mmap(0, 101 * 1024 * 1024, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
if (p == MAP_FAILED) ASSERT_NE(MAP_FAILED, p);
__builtin_trap();
ptrs[count++] = p; ptrs[count++] = p;
} }
void BenchBigMunmap(void) { void BenchBigMunmap(void) {
if (munmap(ptrs[--count], 101 * 1024 * 1024)) ASSERT_SYS(0, 0, munmap(ptrs[--count], 101 * 1024 * 1024));
__builtin_trap();
} }
TEST(mmap, bench) { TEST(mmap, bench) {
BENCHMARK(N, 1, BenchMmapPrivate()); BENCHMARK(N, 1, BenchMmapPrivate());
BENCHMARK(N, 1, BenchUnmap()); BENCHMARK(N, 1, BenchUnmap());
// BENCHMARK(N, 1, BenchBigMmap()); /* BENCHMARK(N, 1, BenchBigMmap()); */
// BENCHMARK(N, 1, BenchBigMunmap()); /* BENCHMARK(N, 1, BenchBigMunmap()); */
} }

84
test/libc/intrin/munmap_test.c
View file

@ -53,26 +53,106 @@ TEST(munmap, test) {
EXPECT_FALSE(testlib_memoryexists(p)); EXPECT_FALSE(testlib_memoryexists(p));
} }
TEST(munmap, carveMemory) {
if (IsWindows())
return; // needs carving
char *p;
int count = __maps.count;
ASSERT_NE(MAP_FAILED,
(p = mmap(__maps_randaddr(), gransz * 3, PROT_READ | PROT_WRITE,
MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)));
EXPECT_EQ(count + 1, __maps.count);
count = __maps.count;
EXPECT_TRUE(testlib_memoryexists(p + gransz * 0));
EXPECT_TRUE(testlib_memoryexists(p + gransz * 1));
EXPECT_TRUE(testlib_memoryexists(p + gransz * 2));
EXPECT_SYS(0, 0, munmap(p + gransz * 0, gransz));
EXPECT_EQ(count + 0, __maps.count);
count = __maps.count;
EXPECT_FALSE(testlib_memoryexists(p + gransz * 0));
EXPECT_TRUE(testlib_memoryexists(p + gransz * 1));
EXPECT_TRUE(testlib_memoryexists(p + gransz * 2));
EXPECT_SYS(0, 0, munmap(p + gransz * 2, gransz));
EXPECT_EQ(count + 0, __maps.count);
count = __maps.count;
EXPECT_FALSE(testlib_memoryexists(p + gransz * 0));
EXPECT_TRUE(testlib_memoryexists(p + gransz * 1));
EXPECT_FALSE(testlib_memoryexists(p + gransz * 2));
EXPECT_SYS(0, 0, munmap(p + gransz * 1, gransz));
EXPECT_EQ(count - 1, __maps.count);
count = __maps.count;
EXPECT_FALSE(testlib_memoryexists(p + gransz * 0));
EXPECT_FALSE(testlib_memoryexists(p + gransz * 1));
EXPECT_FALSE(testlib_memoryexists(p + gransz * 2));
}
TEST(munmap, punchHoleInMemory) { TEST(munmap, punchHoleInMemory) {
if (IsWindows()) if (IsWindows())
return; // needs carving return; // needs carving
char *p; char *p;
ASSERT_NE(MAP_FAILED, (p = mmap(0, gransz * 3, PROT_READ | PROT_WRITE, int count = __maps.count;
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0))); ASSERT_NE(MAP_FAILED,
(p = mmap(__maps_randaddr(), gransz * 3, PROT_READ | PROT_WRITE,
MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)));
EXPECT_EQ(count + 1, __maps.count);
count = __maps.count;
EXPECT_TRUE(testlib_memoryexists(p + gransz * 0)); EXPECT_TRUE(testlib_memoryexists(p + gransz * 0));
EXPECT_TRUE(testlib_memoryexists(p + gransz * 1)); EXPECT_TRUE(testlib_memoryexists(p + gransz * 1));
EXPECT_TRUE(testlib_memoryexists(p + gransz * 2)); EXPECT_TRUE(testlib_memoryexists(p + gransz * 2));
EXPECT_SYS(0, 0, munmap(p + gransz, gransz)); EXPECT_SYS(0, 0, munmap(p + gransz, gransz));
EXPECT_EQ(count + 1, __maps.count);
count = __maps.count;
EXPECT_TRUE(testlib_memoryexists(p + gransz * 0)); EXPECT_TRUE(testlib_memoryexists(p + gransz * 0));
EXPECT_FALSE(testlib_memoryexists(p + gransz * 1)); EXPECT_FALSE(testlib_memoryexists(p + gransz * 1));
EXPECT_TRUE(testlib_memoryexists(p + gransz * 2)); EXPECT_TRUE(testlib_memoryexists(p + gransz * 2));
EXPECT_SYS(0, 0, munmap(p, gransz)); EXPECT_SYS(0, 0, munmap(p, gransz));
EXPECT_EQ(count - 1, __maps.count);
count = __maps.count;
EXPECT_SYS(0, 0, munmap(p + gransz * 2, gransz)); EXPECT_SYS(0, 0, munmap(p + gransz * 2, gransz));
EXPECT_EQ(count - 1, __maps.count);
count = __maps.count;
EXPECT_FALSE(testlib_memoryexists(p + gransz * 0)); EXPECT_FALSE(testlib_memoryexists(p + gransz * 0));
EXPECT_FALSE(testlib_memoryexists(p + gransz * 1)); EXPECT_FALSE(testlib_memoryexists(p + gransz * 1));
EXPECT_FALSE(testlib_memoryexists(p + gransz * 2)); EXPECT_FALSE(testlib_memoryexists(p + gransz * 2));
} }
TEST(munmap, fillHoleInMemory) {
if (IsWindows())
return; // needs fungible memory
int count = __maps.count;
char *base = __maps_randaddr();
EXPECT_EQ(base + gransz * 0,
mmap(base + gransz * 0, gransz, PROT_READ | PROT_WRITE,
MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0));
EXPECT_EQ(count + 1, __maps.count);
count = __maps.count;
EXPECT_TRUE(testlib_memoryexists(base + gransz * 0));
EXPECT_FALSE(testlib_memoryexists(base + gransz * 1));
EXPECT_FALSE(testlib_memoryexists(base + gransz * 2));
EXPECT_EQ(base + gransz * 2,
mmap(base + gransz * 2, gransz, PROT_READ | PROT_WRITE,
MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0));
EXPECT_EQ(count + 1, __maps.count);
count = __maps.count;
EXPECT_TRUE(testlib_memoryexists(base + gransz * 0));
EXPECT_FALSE(testlib_memoryexists(base + gransz * 1));
EXPECT_TRUE(testlib_memoryexists(base + gransz * 2));
EXPECT_EQ(base + gransz * 1,
mmap(base + gransz * 1, gransz, PROT_READ | PROT_WRITE,
MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0));
EXPECT_EQ(count - 1, __maps.count);
count = __maps.count;
EXPECT_TRUE(testlib_memoryexists(base + gransz * 0));
EXPECT_TRUE(testlib_memoryexists(base + gransz * 1));
EXPECT_TRUE(testlib_memoryexists(base + gransz * 2));
EXPECT_SYS(0, 0, munmap(base, gransz * 3));
EXPECT_EQ(count - 1, __maps.count);
count = __maps.count;
EXPECT_FALSE(testlib_memoryexists(base + gransz * 0));
EXPECT_FALSE(testlib_memoryexists(base + gransz * 1));
EXPECT_FALSE(testlib_memoryexists(base + gransz * 2));
}
TEST(munmap, memoryHasHole) { TEST(munmap, memoryHasHole) {
if (IsWindows()) if (IsWindows())
return; // needs carving return; // needs carving