mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-07-01 00:38:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Fix minor suboptimalities in memory manager

Browse source
This commit is contained in:
Justine Tunney 2024-07-18 19:19:51 -07:00
parent 76cea6c687
commit 23dfb79d33
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
3 changed files with 49 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

5
libc/intrin/maps.c
View file

@ -89,7 +89,7 @@ privileged bool __maps_lock(void) {
if (!__tls_enabled) if (!__tls_enabled)
return false; return false;
tib = __get_tls_privileged(); tib = __get_tls_privileged();
if (tib->tib_relock_maps++) if (atomic_fetch_add_explicit(&tib->tib_relock_maps, 1, memory_order_relaxed))
return true; return true;
while (atomic_exchange_explicit(&__maps.lock, 1, memory_order_acquire)) { while (atomic_exchange_explicit(&__maps.lock, 1, memory_order_acquire)) {
#if defined(__GNUC__) && defined(__aarch64__) #if defined(__GNUC__) && defined(__aarch64__)
@ -106,6 +106,7 @@ privileged void __maps_unlock(void) {
if (!__tls_enabled) if (!__tls_enabled)
return; return;
tib = __get_tls_privileged(); tib = __get_tls_privileged();
if (!--tib->tib_relock_maps) if (atomic_fetch_sub_explicit(&tib->tib_relock_maps, 1,
memory_order_relaxed) == 1)
atomic_store_explicit(&__maps.lock, 0, memory_order_release); atomic_store_explicit(&__maps.lock, 0, memory_order_release);
} }

10
libc/intrin/maps.h
View file

@ -1,14 +1,12 @@
#ifndef COSMOPOLITAN_MAPS_H_ #ifndef COSMOPOLITAN_MAPS_H_
#define COSMOPOLITAN_MAPS_H_ #define COSMOPOLITAN_MAPS_H_
#include "libc/intrin/atomic.h" #include "libc/intrin/atomic.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/tree.h" #include "libc/intrin/tree.h"
#include "libc/runtime/runtime.h" #include "libc/runtime/runtime.h"
#include "libc/thread/tls2.internal.h" #include "libc/thread/tls2.internal.h"
COSMOPOLITAN_C_START_ COSMOPOLITAN_C_START_
#define MAP_TREE_CONTAINER(e) TREE_CONTAINER(struct Map, tree, e) #define MAP_TREE_CONTAINER(e) TREE_CONTAINER(struct Map, tree, e)
#define MAP_FREE_CONTAINER(e) DLL_CONTAINER(struct Map, free, e)
struct Map { struct Map {
char *addr; /* granule aligned */ char *addr; /* granule aligned */
@ -22,14 +20,14 @@ struct Map {
intptr_t hand; /* windows nt only */ intptr_t hand; /* windows nt only */
union { union {
struct Tree tree; struct Tree tree;
struct Dll free; struct Map *free;
}; };
}; };
struct Maps { struct Maps {
atomic_int lock; atomic_int lock;
struct Tree *maps; struct Tree *maps;
struct Dll *free; _Atomic(struct Map *) free;
size_t count; size_t count;
size_t pages; size_t pages;
atomic_size_t rollo; atomic_size_t rollo;
@ -64,14 +62,14 @@ forceinline optimizespeed int __maps_search(const void *key,
return (addr > map->addr) - (addr < map->addr); return (addr > map->addr) - (addr < map->addr);
} }
static struct Map *__maps_next(struct Map *map) { static inline struct Map *__maps_next(struct Map *map) {
struct Tree *node; struct Tree *node;
if ((node = tree_next(&map->tree))) if ((node = tree_next(&map->tree)))
return MAP_TREE_CONTAINER(node); return MAP_TREE_CONTAINER(node);
return 0; return 0;
} }
static struct Map *__maps_first(void) { static inline struct Map *__maps_first(void) {
struct Tree *node; struct Tree *node;
if ((node = tree_first(__maps.maps))) if ((node = tree_first(__maps.maps)))
return MAP_TREE_CONTAINER(node); return MAP_TREE_CONTAINER(node);

94
libc/intrin/mmap.c
View file

@ -31,7 +31,6 @@
#include "libc/intrin/describebacktrace.internal.h" #include "libc/intrin/describebacktrace.internal.h"
#include "libc/intrin/describeflags.internal.h" #include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/directmap.internal.h" #include "libc/intrin/directmap.internal.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/kprintf.h" #include "libc/intrin/kprintf.h"
#include "libc/intrin/maps.h" #include "libc/intrin/maps.h"
#include "libc/intrin/strace.internal.h" #include "libc/intrin/strace.internal.h"
@ -124,7 +123,7 @@ void __maps_check(void) {
} }
static int __muntrack(char *addr, size_t size, int pagesz, static int __muntrack(char *addr, size_t size, int pagesz,
struct Dll **deleted) { struct Map **deleted) {
int rc = 0; int rc = 0;
struct Map *map; struct Map *map;
struct Map *next; struct Map *next;
@ -140,8 +139,8 @@ static int __muntrack(char *addr, size_t size, int pagesz,
if (addr <= map_addr && addr + PGUP(size) >= map_addr + PGUP(map_size)) { if (addr <= map_addr && addr + PGUP(size) >= map_addr + PGUP(map_size)) {
// remove mapping completely // remove mapping completely
tree_remove(&__maps.maps, &map->tree); tree_remove(&__maps.maps, &map->tree);
dll_init(&map->free); map->free = *deleted;
dll_make_first(deleted, &map->free); *deleted = map;
__maps.pages -= (map_size + pagesz - 1) / pagesz; __maps.pages -= (map_size + pagesz - 1) / pagesz;
__maps.count -= 1; __maps.count -= 1;
__maps_check(); __maps_check();
@ -166,8 +165,8 @@ static int __muntrack(char *addr, size_t size, int pagesz,
__maps.pages -= (left + pagesz - 1) / pagesz; __maps.pages -= (left + pagesz - 1) / pagesz;
leftmap->addr = map_addr; leftmap->addr = map_addr;
leftmap->size = left; leftmap->size = left;
dll_init(&leftmap->free); leftmap->free = *deleted;
dll_make_first(deleted, &leftmap->free); *deleted = leftmap;
__maps_check(); __maps_check();
} else { } else {
rc = -1; rc = -1;
@ -182,8 +181,8 @@ static int __muntrack(char *addr, size_t size, int pagesz,
__maps.pages -= (right + pagesz - 1) / pagesz; __maps.pages -= (right + pagesz - 1) / pagesz;
rightmap->addr = addr; rightmap->addr = addr;
rightmap->size = right; rightmap->size = right;
dll_init(&rightmap->free); rightmap->free = *deleted;
dll_make_first(deleted, &rightmap->free); *deleted = rightmap;
__maps_check(); __maps_check();
} else { } else {
rc = -1; rc = -1;
@ -211,8 +210,8 @@ static int __muntrack(char *addr, size_t size, int pagesz,
__maps.count += 1; __maps.count += 1;
middlemap->addr = addr; middlemap->addr = addr;
middlemap->size = size; middlemap->size = size;
dll_init(&middlemap->free); middlemap->free = *deleted;
dll_make_first(deleted, &middlemap->free); *deleted = middlemap;
__maps_check(); __maps_check();
} else { } else {
rc = -1; rc = -1;
@ -228,8 +227,21 @@ static int __muntrack(char *addr, size_t size, int pagesz,
void __maps_free(struct Map *map) { void __maps_free(struct Map *map) {
map->size = 0; map->size = 0;
map->addr = MAP_FAILED; map->addr = MAP_FAILED;
dll_init(&map->free); map->free = atomic_load_explicit(&__maps.free, memory_order_relaxed);
dll_make_first(&__maps.free, &map->free); for (;;) {
if (atomic_compare_exchange_weak_explicit(&__maps.free, &map->free, map,
memory_order_release,
memory_order_relaxed))
break;
}
}
static void __maps_free_all(struct Map *list) {
struct Map *next;
for (struct Map *map = list; map; map = next) {
next = map->free;
__maps_free(map);
}
} }
static void __maps_insert(struct Map *map) { static void __maps_insert(struct Map *map) {
@ -282,12 +294,13 @@ static void __maps_insert(struct Map *map) {
} }
struct Map *__maps_alloc(void) { struct Map *__maps_alloc(void) {
struct Dll *e;
struct Map *map; struct Map *map;
if ((e = dll_first(__maps.free))) { map = atomic_load_explicit(&__maps.free, memory_order_relaxed);
dll_remove(&__maps.free, e); while (map) {
map = MAP_FREE_CONTAINER(e); if (atomic_compare_exchange_weak_explicit(&__maps.free, &map, map->free,
return map; memory_order_acquire,
memory_order_relaxed))
return map;
} }
int gransz = getgransize(); int gransz = getgransize();
struct DirectMap sys = sys_mmap(0, gransz, PROT_READ | PROT_WRITE, struct DirectMap sys = sys_mmap(0, gransz, PROT_READ | PROT_WRITE,
@ -335,14 +348,13 @@ static int __munmap(char *addr, size_t size) {
} }
// untrack mappings // untrack mappings
struct Dll *deleted = 0; struct Map *deleted = 0;
__muntrack(addr, pgup_size, pagesz, &deleted); __muntrack(addr, pgup_size, pagesz, &deleted);
__maps_unlock(); __maps_unlock();
// delete mappings // delete mappings
int rc = 0; int rc = 0;
for (struct Dll *e = dll_first(deleted); e; e = dll_next(deleted, e)) { for (struct Map *map = deleted; map; map = map->free) {
struct Map *map = MAP_FREE_CONTAINER(e);
if (!IsWindows()) { if (!IsWindows()) {
if (sys_munmap(map->addr, map->size)) if (sys_munmap(map->addr, map->size))
rc = -1; rc = -1;
@ -356,11 +368,7 @@ static int __munmap(char *addr, size_t size) {
} }
// free mappings // free mappings
if (!dll_is_empty(deleted)) { __maps_free_all(deleted);
__maps_lock();
dll_make_first(&__maps.free, deleted);
__maps_unlock();
}
return rc; return rc;
} }
@ -392,20 +400,12 @@ static void *__mmap_chunk(void *addr, size_t size, int prot, int flags, int fd,
// allocate Map object // allocate Map object
struct Map *map; struct Map *map;
if (__maps_lock()) { if (!(map = __maps_alloc()))
__maps_unlock();
return (void *)edeadlk();
}
__maps_check();
map = __maps_alloc();
__maps_unlock();
if (!map)
return MAP_FAILED; return MAP_FAILED;
// remove mapping we blew away // remove mapping we blew away
if (IsWindows() && should_untrack) if (IsWindows() && should_untrack)
if (__munmap(addr, size)) __munmap(addr, size);
return MAP_FAILED;
// obtain mapping from operating system // obtain mapping from operating system
int olderr = errno; int olderr = errno;
@ -424,9 +424,7 @@ TryAgain:
goto TryAgain; goto TryAgain;
} }
} }
__maps_lock();
__maps_free(map); __maps_free(map);
__maps_unlock();
return MAP_FAILED; return MAP_FAILED;
} }
@ -440,21 +438,15 @@ TryAgain:
UnmapViewOfFile(res.addr); UnmapViewOfFile(res.addr);
CloseHandle(res.maphandle); CloseHandle(res.maphandle);
} }
__maps_lock();
__maps_free(map); __maps_free(map);
__maps_unlock();
return (void *)eexist(); return (void *)eexist();
} }
// untrack mapping we blew away // untrack mapping we blew away
if (!IsWindows() && should_untrack) { if (!IsWindows() && should_untrack) {
struct Dll *deleted = 0; struct Map *deleted = 0;
__muntrack(res.addr, size, pagesz, &deleted); __muntrack(res.addr, size, pagesz, &deleted);
if (!dll_is_empty(deleted)) { __maps_free_all(deleted);
__maps_lock();
dll_make_first(&__maps.free, deleted);
__maps_unlock();
}
} }
// track map object // track map object
@ -632,25 +624,23 @@ static void *__mremap_impl(char *old_addr, size_t old_size, size_t new_size,
} else { } else {
res = sys_mremap(old_addr, old_size, new_size, flags, (uintptr_t)new_addr); res = sys_mremap(old_addr, old_size, new_size, flags, (uintptr_t)new_addr);
} }
if (res == MAP_FAILED)
__maps_free(map);
// re-acquire lock if needed // re-acquire lock if needed
if (!flags) if (!flags)
__maps_lock(); __maps_lock();
// check result if (res == MAP_FAILED)
if (res == MAP_FAILED) {
__maps_free(map);
return MAP_FAILED; return MAP_FAILED;
}
if (!(flags & MREMAP_MAYMOVE)) if (!(flags & MREMAP_MAYMOVE))
ASSERT(res == old_addr); ASSERT(res == old_addr);
// untrack old mapping // untrack old mapping
struct Dll *deleted = 0; struct Map *deleted = 0;
__muntrack(old_addr, old_size, pagesz, &deleted); __muntrack(old_addr, old_size, pagesz, &deleted);
dll_make_first(&__maps.free, deleted); __maps_free_all(deleted);
deleted = 0;
// track map object // track map object
map->addr = res; map->addr = res;