mirror of
https://github.com/jart/cosmopolitan.git
synced 2025-07-04 02:08:30 +00:00
Make mmap() scalable
It's now possible to create thousands of thousands of sparse independent memory mappings, without any slowdown. The memory manager is better with tracking memory protection now, particularly on Windows in a precise way that can be restored during fork(). You now have the highest quality mem manager possible. It's even better than some OSes like XNU, where mmap() is implemented as an O(n) operation which means sadly things aren't much improved over there. With this change the llamafile HTTP server endpoint at /tokenize with a prompt of 50 tokens is now able to handle 2.6m r/sec
This commit is contained in:
Justine Tunney
parent
3756870635
commit
8c645fa1ee
59 changed files with 1238 additions and 1067 deletions
|
|
@ -25,6 +25,7 @@
|
|||
#include "libc/intrin/kprintf.h"
|
||||
#include "libc/intrin/maps.h"
|
||||
#include "libc/intrin/strace.internal.h"
|
||||
#include "libc/intrin/tree.h"
|
||||
#include "libc/nt/memory.h"
|
||||
#include "libc/runtime/internal.h"
|
||||
#include "libc/runtime/runtime.h"
|
||||
|
|
@ -51,7 +52,7 @@ static int __mprotect_chunk(char *addr, size_t size, int prot, bool iscow) {
|
|||
int __mprotect(char *addr, size_t size, int prot) {
|
||||
|
||||
// unix checks prot before checking size
|
||||
if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC))
|
||||
if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC | PROT_GUARD))
|
||||
return einval();
|
||||
|
||||
// make new technology consistent with unix
|
||||
|
|
@ -68,64 +69,97 @@ int __mprotect(char *addr, size_t size, int prot) {
|
|||
|
||||
// change mappings
|
||||
int rc = 0;
|
||||
struct Dll *cur;
|
||||
bool found = false;
|
||||
if (__maps_lock()) {
|
||||
__maps_unlock();
|
||||
return edeadlk();
|
||||
}
|
||||
for (cur = dll_first(__maps.used); cur; cur = dll_next(__maps.used, cur)) {
|
||||
struct Map *map = MAP_CONTAINER(cur);
|
||||
for (struct Map *map = __maps_floor(addr); map; map = __maps_next(map)) {
|
||||
char *map_addr = map->addr;
|
||||
size_t map_size = map->size;
|
||||
char *beg = MAX(addr, map_addr);
|
||||
char *end = MIN(addr + size, map_addr + PGUP(map_size));
|
||||
if (beg < end) {
|
||||
found = true;
|
||||
if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) {
|
||||
// change protection of entire mapping
|
||||
if (!__mprotect_chunk(map_addr, map_size, prot, map->iscow)) {
|
||||
if (beg >= end)
|
||||
break;
|
||||
found = true;
|
||||
if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) {
|
||||
// change protection of entire mapping
|
||||
if (!__mprotect_chunk(map_addr, map_size, prot, map->iscow)) {
|
||||
map->prot = prot;
|
||||
} else {
|
||||
rc = -1;
|
||||
}
|
||||
} else if (addr <= map_addr) {
|
||||
// change lefthand side of mapping
|
||||
size_t left = PGUP(addr + size - map_addr);
|
||||
size_t right = map_size - left;
|
||||
struct Map *leftmap;
|
||||
if ((leftmap = __maps_alloc())) {
|
||||
if (!__mprotect_chunk(map_addr, left, prot, false)) {
|
||||
leftmap->addr = map_addr;
|
||||
leftmap->size = left;
|
||||
leftmap->prot = prot;
|
||||
leftmap->off = map->off;
|
||||
leftmap->flags = map->flags;
|
||||
leftmap->iscow = map->iscow;
|
||||
leftmap->readonlyfile = map->readonlyfile;
|
||||
leftmap->hand = map->hand;
|
||||
map->addr += left;
|
||||
map->size = right;
|
||||
map->hand = -1;
|
||||
if (!(map->flags & MAP_ANONYMOUS))
|
||||
map->off += left;
|
||||
tree_insert(&__maps.maps, &leftmap->tree, __maps_compare);
|
||||
__maps.count += 1;
|
||||
__maps_check();
|
||||
} else {
|
||||
__maps_free(leftmap);
|
||||
rc = -1;
|
||||
}
|
||||
} else {
|
||||
rc = -1;
|
||||
}
|
||||
} else if (addr + size >= map_addr + PGUP(map_size)) {
|
||||
// change righthand side of mapping
|
||||
size_t left = addr - map_addr;
|
||||
size_t right = map_addr + map_size - addr;
|
||||
struct Map *leftmap;
|
||||
if ((leftmap = __maps_alloc())) {
|
||||
if (!__mprotect_chunk(map_addr + left, right, prot, false)) {
|
||||
leftmap->addr = map_addr;
|
||||
leftmap->size = left;
|
||||
leftmap->off = map->off;
|
||||
leftmap->prot = map->prot;
|
||||
leftmap->flags = map->flags;
|
||||
leftmap->iscow = map->iscow;
|
||||
leftmap->readonlyfile = map->readonlyfile;
|
||||
leftmap->hand = map->hand;
|
||||
map->addr += left;
|
||||
map->size = right;
|
||||
map->prot = prot;
|
||||
map->hand = -1;
|
||||
if (!(map->flags & MAP_ANONYMOUS))
|
||||
map->off += left;
|
||||
tree_insert(&__maps.maps, &leftmap->tree, __maps_compare);
|
||||
__maps.count += 1;
|
||||
__maps_check();
|
||||
} else {
|
||||
__maps_free(leftmap);
|
||||
rc = -1;
|
||||
}
|
||||
} else if (addr <= map_addr) {
|
||||
// change lefthand side of mapping
|
||||
size_t left = PGUP(addr + size - map_addr);
|
||||
size_t right = map_size - left;
|
||||
struct Map *leftmap;
|
||||
if ((leftmap = __maps_alloc())) {
|
||||
if (!__mprotect_chunk(map_addr, left, prot, false)) {
|
||||
leftmap->addr = map_addr;
|
||||
leftmap->size = left;
|
||||
leftmap->prot = prot;
|
||||
leftmap->off = map->off;
|
||||
leftmap->flags = map->flags;
|
||||
leftmap->iscow = map->iscow;
|
||||
leftmap->readonlyfile = map->readonlyfile;
|
||||
leftmap->hand = map->hand;
|
||||
map->addr += left;
|
||||
map->size = right;
|
||||
map->hand = -1;
|
||||
if (!(map->flags & MAP_ANONYMOUS))
|
||||
map->off += left;
|
||||
dll_make_first(&__maps.used, &leftmap->elem);
|
||||
__maps.count += 1;
|
||||
__maps_check();
|
||||
} else {
|
||||
__maps_free(leftmap);
|
||||
rc = -1;
|
||||
}
|
||||
} else {
|
||||
rc = -1;
|
||||
}
|
||||
} else if (addr + size >= map_addr + PGUP(map_size)) {
|
||||
// change righthand side of mapping
|
||||
size_t left = addr - map_addr;
|
||||
size_t right = map_addr + map_size - addr;
|
||||
struct Map *leftmap;
|
||||
if ((leftmap = __maps_alloc())) {
|
||||
if (!__mprotect_chunk(map_addr + left, right, prot, false)) {
|
||||
} else {
|
||||
rc = -1;
|
||||
}
|
||||
} else {
|
||||
// change middle of mapping
|
||||
size_t left = addr - map_addr;
|
||||
size_t middle = size;
|
||||
size_t right = map_size - middle - left;
|
||||
struct Map *leftmap;
|
||||
if ((leftmap = __maps_alloc())) {
|
||||
struct Map *midlmap;
|
||||
if ((midlmap = __maps_alloc())) {
|
||||
if (!__mprotect_chunk(map_addr + left, middle, prot, false)) {
|
||||
leftmap->addr = map_addr;
|
||||
leftmap->size = left;
|
||||
leftmap->off = map->off;
|
||||
|
|
@ -134,67 +168,32 @@ int __mprotect(char *addr, size_t size, int prot) {
|
|||
leftmap->iscow = map->iscow;
|
||||
leftmap->readonlyfile = map->readonlyfile;
|
||||
leftmap->hand = map->hand;
|
||||
map->addr += left;
|
||||
midlmap->addr = map_addr + left;
|
||||
midlmap->size = middle;
|
||||
midlmap->off = (map->flags & MAP_ANONYMOUS) ? 0 : map->off + left;
|
||||
midlmap->prot = prot;
|
||||
midlmap->flags = map->flags;
|
||||
midlmap->hand = -1;
|
||||
map->addr += left + middle;
|
||||
map->size = right;
|
||||
map->prot = prot;
|
||||
map->hand = -1;
|
||||
if (!(map->flags & MAP_ANONYMOUS))
|
||||
map->off += left;
|
||||
dll_make_first(&__maps.used, &leftmap->elem);
|
||||
__maps.count += 1;
|
||||
map->off += left + middle;
|
||||
tree_insert(&__maps.maps, &leftmap->tree, __maps_compare);
|
||||
tree_insert(&__maps.maps, &midlmap->tree, __maps_compare);
|
||||
__maps.count += 2;
|
||||
__maps_check();
|
||||
} else {
|
||||
__maps_free(midlmap);
|
||||
__maps_free(leftmap);
|
||||
rc = -1;
|
||||
}
|
||||
} else {
|
||||
__maps_free(leftmap);
|
||||
rc = -1;
|
||||
}
|
||||
} else {
|
||||
// change middle of mapping
|
||||
size_t left = addr - map_addr;
|
||||
size_t middle = size;
|
||||
size_t right = map_size - middle - left;
|
||||
struct Map *leftmap;
|
||||
if ((leftmap = __maps_alloc())) {
|
||||
struct Map *midlmap;
|
||||
if ((midlmap = __maps_alloc())) {
|
||||
if (!__mprotect_chunk(map_addr + left, middle, prot, false)) {
|
||||
leftmap->addr = map_addr;
|
||||
leftmap->size = left;
|
||||
leftmap->off = map->off;
|
||||
leftmap->prot = map->prot;
|
||||
leftmap->flags = map->flags;
|
||||
leftmap->iscow = map->iscow;
|
||||
leftmap->readonlyfile = map->readonlyfile;
|
||||
leftmap->hand = map->hand;
|
||||
midlmap->addr = map_addr + left;
|
||||
midlmap->size = middle;
|
||||
midlmap->off = (map->flags & MAP_ANONYMOUS) ? 0 : map->off + left;
|
||||
midlmap->prot = prot;
|
||||
midlmap->flags = map->flags;
|
||||
midlmap->hand = -1;
|
||||
map->addr += left + middle;
|
||||
map->size = right;
|
||||
map->hand = -1;
|
||||
if (!(map->flags & MAP_ANONYMOUS))
|
||||
map->off += left + middle;
|
||||
dll_make_first(&__maps.used, &midlmap->elem);
|
||||
dll_make_first(&__maps.used, &leftmap->elem);
|
||||
__maps.count += 2;
|
||||
__maps_check();
|
||||
} else {
|
||||
__maps_free(midlmap);
|
||||
__maps_free(leftmap);
|
||||
rc = -1;
|
||||
}
|
||||
} else {
|
||||
__maps_free(leftmap);
|
||||
rc = -1;
|
||||
}
|
||||
} else {
|
||||
rc = -1;
|
||||
}
|
||||
rc = -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue