mirror of
https://github.com/jart/cosmopolitan.git
synced 2025-07-19 00:50:30 +00:00
Make realloc() go 100x faster on Linux/NetBSD
Cosmopolitan now supports mremap(), which is only supported on Linux and NetBSD. First, it allows memory mappings to be relocated without copying them; this can dramatically speed up data structures like std::vector if the array size grows larger than 256kb. The mremap() system call is also 10x faster than munmap() when shrinking large memory mappings. There's now two functions, getpagesize() and getgransize() which help to write portable code that uses mmap(MAP_FIXED). Alternative sysconf() may be called with our new _SC_GRANSIZE. The madvise() system call now has a better wrapper with improved documentation.
This commit is contained in:
Justine Tunney
parent
196942084b
commit
f7780de24b
71 changed files with 1301 additions and 640 deletions
|
|
@ -62,6 +62,7 @@ o/$(MODE)/libc/intrin/kprintf.o: private \
|
|||
-Wframe-larger-than=128 \
|
||||
-Walloca-larger-than=128
|
||||
|
||||
o/$(MODE)/libc/intrin/mmap.o \
|
||||
o/$(MODE)/libc/intrin/tree.o: private \
|
||||
CFLAGS += \
|
||||
-ffunction-sections
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ const char *DescribeInOutInt64(char[23], ssize_t, int64_t *) libcesque;
|
|||
const char *DescribeItimer(char[12], int) libcesque;
|
||||
const char *DescribeMapFlags(char[64], int) libcesque;
|
||||
const char *DescribeMapping(char[8], int, int) libcesque;
|
||||
const char *DescribeMremapFlags(char[30], int) libcesque;
|
||||
const char *DescribeNtConsoleInFlags(char[256], uint32_t) libcesque;
|
||||
const char *DescribeNtConsoleOutFlags(char[128], uint32_t) libcesque;
|
||||
const char *DescribeNtCreationDisposition(uint32_t) libcesque;
|
||||
|
|
@ -49,7 +50,6 @@ const char *DescribePollFlags(char[64], int) libcesque;
|
|||
const char *DescribeProtFlags(char[48], int) libcesque;
|
||||
const char *DescribePtrace(char[12], int) libcesque;
|
||||
const char *DescribePtraceEvent(char[32], int) libcesque;
|
||||
const char *DescribeRemapFlags(char[48], int) libcesque;
|
||||
const char *DescribeRlimitName(char[20], int) libcesque;
|
||||
const char *DescribeSchedPolicy(char[48], int) libcesque;
|
||||
const char *DescribeSeccompOperation(int) libcesque;
|
||||
|
|
@ -81,6 +81,7 @@ const char *DescribeWhichPrio(char[12], int) libcesque;
|
|||
#define DescribeItimer(x) DescribeItimer(alloca(12), x)
|
||||
#define DescribeMapFlags(x) DescribeMapFlags(alloca(64), x)
|
||||
#define DescribeMapping(x, y) DescribeMapping(alloca(8), x, y)
|
||||
#define DescribeMremapFlags(x) DescribeMremapFlags(alloca(30), x)
|
||||
#define DescribeNtConsoleInFlags(x) DescribeNtConsoleInFlags(alloca(256), x)
|
||||
#define DescribeNtConsoleOutFlags(x) DescribeNtConsoleOutFlags(alloca(128), x)
|
||||
#define DescribeNtFileAccessFlags(x) DescribeNtFileAccessFlags(alloca(512), x)
|
||||
|
|
@ -103,7 +104,6 @@ const char *DescribeWhichPrio(char[12], int) libcesque;
|
|||
#define DescribeProtFlags(x) DescribeProtFlags(alloca(48), x)
|
||||
#define DescribePtrace(i) DescribePtrace(alloca(12), i)
|
||||
#define DescribePtraceEvent(x) DescribePtraceEvent(alloca(32), x)
|
||||
#define DescribeRemapFlags(x) DescribeRemapFlags(alloca(48), x)
|
||||
#define DescribeRlimitName(rl) DescribeRlimitName(alloca(20), rl)
|
||||
#define DescribeSchedPolicy(x) DescribeSchedPolicy(alloca(48), x)
|
||||
#define DescribeSiCode(x, y) DescribeSiCode(alloca(20), x, y)
|
||||
|
|
|
|||
|
|
@ -26,7 +26,10 @@ static char DescribeMapType(int flags) {
|
|||
case MAP_FILE:
|
||||
return '-';
|
||||
case MAP_PRIVATE:
|
||||
return 'p';
|
||||
if (flags & MAP_NOFORK)
|
||||
return 'P';
|
||||
else
|
||||
return 'p';
|
||||
case MAP_SHARED:
|
||||
return 's';
|
||||
default:
|
||||
|
|
@ -47,7 +50,6 @@ const char *(DescribeMapping)(char p[8], int prot, int flags) {
|
|||
DescribeProt(p, prot);
|
||||
p[3] = DescribeMapType(flags);
|
||||
p[4] = (flags & MAP_ANONYMOUS) ? 'a' : '-';
|
||||
p[5] = (flags & MAP_FIXED) ? 'f' : '-';
|
||||
p[6] = 0;
|
||||
p[5] = 0;
|
||||
return p;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
|
||||
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
|
||||
╞══════════════════════════════════════════════════════════════════════════════╡
|
||||
│ Copyright 2022 Justine Alexandra Roberts Tunney │
|
||||
│ Copyright 2024 Justine Alexandra Roberts Tunney │
|
||||
│ │
|
||||
│ Permission to use, copy, modify, and/or distribute this software for │
|
||||
│ any purpose with or without fee is hereby granted, provided that the │
|
||||
|
|
@ -20,12 +20,12 @@
|
|||
#include "libc/macros.internal.h"
|
||||
#include "libc/sysv/consts/mremap.h"
|
||||
|
||||
static const struct DescribeFlags kRemapFlags[] = {
|
||||
static const struct DescribeFlags kMremapFlags[] = {
|
||||
{MREMAP_MAYMOVE, "MAYMOVE"}, //
|
||||
{MREMAP_FIXED, "FIXED"}, //
|
||||
};
|
||||
|
||||
const char *(DescribeRemapFlags)(char buf[48], int x) {
|
||||
return DescribeFlags(buf, 48, kRemapFlags, ARRAYLEN(kRemapFlags), "MREMAP_",
|
||||
const char *(DescribeMremapFlags)(char buf[30], int x) {
|
||||
return DescribeFlags(buf, 30, kMremapFlags, ARRAYLEN(kMremapFlags), "MREMAP_",
|
||||
x);
|
||||
}
|
||||
|
|
@ -22,7 +22,7 @@
|
|||
#include "libc/sysv/consts/map.h"
|
||||
#include "libc/sysv/consts/prot.h"
|
||||
|
||||
#define G __granularity()
|
||||
#define G getgransize()
|
||||
|
||||
/**
|
||||
* Extends static allocation.
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@
|
|||
#include "libc/runtime/runtime.h"
|
||||
#include "libc/sysv/consts/auxv.h"
|
||||
|
||||
int __granularity(void) {
|
||||
int getgransize(void) {
|
||||
static int res;
|
||||
if (!res) {
|
||||
if (!IsWindows()) {
|
||||
|
|
@ -86,8 +86,6 @@ void __maps_init(void) {
|
|||
|
||||
privileged bool __maps_lock(void) {
|
||||
struct CosmoTib *tib;
|
||||
if (!__threaded)
|
||||
return false;
|
||||
if (!__tls_enabled)
|
||||
return false;
|
||||
tib = __get_tls_privileged();
|
||||
|
|
@ -105,8 +103,6 @@ privileged bool __maps_lock(void) {
|
|||
|
||||
privileged void __maps_unlock(void) {
|
||||
struct CosmoTib *tib;
|
||||
if (!__threaded)
|
||||
return;
|
||||
if (!__tls_enabled)
|
||||
return;
|
||||
tib = __get_tls_privileged();
|
||||
|
|
|
|||
|
|
@ -18,8 +18,7 @@ struct Map {
|
|||
int flags; /* memory map flag */
|
||||
bool iscow; /* windows nt only */
|
||||
bool readonlyfile; /* windows nt only */
|
||||
unsigned visited; /* used for checks */
|
||||
unsigned oldprot; /* in windows fork */
|
||||
unsigned visited; /* checks and fork */
|
||||
intptr_t hand; /* windows nt only */
|
||||
union {
|
||||
struct Tree tree;
|
||||
|
|
@ -33,7 +32,7 @@ struct Maps {
|
|||
struct Dll *free;
|
||||
size_t count;
|
||||
size_t pages;
|
||||
atomic_ulong rollo;
|
||||
atomic_size_t rollo;
|
||||
struct Map stack;
|
||||
struct Map guard;
|
||||
};
|
||||
|
|
@ -45,6 +44,7 @@ struct AddrSize {
|
|||
|
||||
extern struct Maps __maps;
|
||||
|
||||
void *randaddr(void);
|
||||
void __maps_init(void);
|
||||
bool __maps_lock(void);
|
||||
void __maps_check(void);
|
||||
|
|
@ -52,6 +52,7 @@ void __maps_unlock(void);
|
|||
void __maps_add(struct Map *);
|
||||
void __maps_free(struct Map *);
|
||||
struct Map *__maps_alloc(void);
|
||||
struct Map *__maps_ceil(const char *);
|
||||
struct Map *__maps_floor(const char *);
|
||||
void __maps_stack(char *, int, int, size_t, int, intptr_t);
|
||||
int __maps_compare(const struct Tree *, const struct Tree *);
|
||||
|
|
@ -61,11 +62,7 @@ forceinline optimizespeed int __maps_search(const void *key,
|
|||
const struct Tree *node) {
|
||||
const char *addr = (const char *)key;
|
||||
const struct Map *map = (const struct Map *)MAP_TREE_CONTAINER(node);
|
||||
if (addr < map->addr)
|
||||
return +1;
|
||||
if (addr >= map->addr + map->size)
|
||||
return -1;
|
||||
return 0;
|
||||
return (addr > map->addr) - (addr < map->addr);
|
||||
}
|
||||
|
||||
static struct Map *__maps_next(struct Map *map) {
|
||||
|
|
|
|||
|
|
@ -44,14 +44,16 @@
|
|||
#include "libc/stdio/sysparam.h"
|
||||
#include "libc/sysv/consts/auxv.h"
|
||||
#include "libc/sysv/consts/map.h"
|
||||
#include "libc/sysv/consts/mremap.h"
|
||||
#include "libc/sysv/consts/o.h"
|
||||
#include "libc/sysv/consts/prot.h"
|
||||
#include "libc/sysv/errfuns.h"
|
||||
#include "libc/thread/thread.h"
|
||||
#include "libc/thread/tls.h"
|
||||
|
||||
#define MMDEBUG 0 // this code is too slow for openbsd/windows
|
||||
#define WINBASE 0x100080040000 // TODO: Can we support Windows Vista again?
|
||||
#define WINMAXX 0x200080000000
|
||||
#define MMDEBUG IsModeDbg()
|
||||
#define WINBASE (1ul << 35) // 34 gb
|
||||
#define WINMAXX ((1ul << 44) - WINBASE) // 17 tb
|
||||
|
||||
#define MAP_FIXED_NOREPLACE_linux 0x100000
|
||||
|
||||
|
|
@ -86,9 +88,19 @@ privileged optimizespeed struct Map *__maps_floor(const char *addr) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
static bool overlaps_existing_map(const char *addr, size_t size, int pagesz) {
|
||||
struct Map *map;
|
||||
if ((map = __maps_floor(addr)))
|
||||
struct Map *__maps_ceil(const char *addr) {
|
||||
struct Tree *node;
|
||||
if ((node = tree_ceil(__maps.maps, addr, __maps_search)))
|
||||
return MAP_TREE_CONTAINER(node);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool __maps_overlaps(const char *addr, size_t size, int pagesz) {
|
||||
ASSERT(!((uintptr_t)addr & (getgransize() - 1)) && size);
|
||||
struct Map *map, *ceil, *floor;
|
||||
floor = __maps_floor(addr);
|
||||
ceil = __maps_ceil(addr + size);
|
||||
for (map = floor; map && map != ceil; map = __maps_next(map))
|
||||
if (MAX(addr, map->addr) <
|
||||
MIN(addr + PGUP(size), map->addr + PGUP(map->size)))
|
||||
return true;
|
||||
|
|
@ -107,14 +119,13 @@ void __maps_check(void) {
|
|||
ASSERT(map->visited != id);
|
||||
ASSERT(map->size);
|
||||
map->visited = id;
|
||||
pages += (map->size + getpagesize() - 1) / getpagesize();
|
||||
pages += (map->size + pagesz - 1) / pagesz;
|
||||
maps += 1;
|
||||
struct Map *next;
|
||||
if ((next = __maps_next(map))) {
|
||||
ASSERT(map->addr < next->addr);
|
||||
ASSERT(
|
||||
!(MAX(map->addr, next->addr) <
|
||||
MIN(map->addr + PGUP(map->size), next->addr + PGUP(next->size))));
|
||||
ASSERT(MAX(map->addr, next->addr) >=
|
||||
MIN(map->addr + PGUP(map->size), next->addr + PGUP(next->size)));
|
||||
}
|
||||
}
|
||||
ASSERT(maps = __maps.count);
|
||||
|
|
@ -122,87 +133,30 @@ void __maps_check(void) {
|
|||
#endif
|
||||
}
|
||||
|
||||
void __maps_free(struct Map *map) {
|
||||
map->size = 0;
|
||||
map->addr = MAP_FAILED;
|
||||
dll_init(&map->free);
|
||||
dll_make_first(&__maps.free, &map->free);
|
||||
}
|
||||
|
||||
static void __maps_insert(struct Map *map) {
|
||||
__maps.pages += (map->size + getpagesize() - 1) / getpagesize();
|
||||
struct Map *floor = __maps_floor(map->addr);
|
||||
if (floor && !IsWindows() && //
|
||||
map->addr + map->size == floor->addr && //
|
||||
(map->flags & MAP_ANONYMOUS) && //
|
||||
map->flags == floor->flags && //
|
||||
map->prot == floor->prot) {
|
||||
floor->addr -= map->size;
|
||||
floor->size += map->size;
|
||||
__maps_free(map);
|
||||
__maps_check();
|
||||
} else {
|
||||
__maps_add(map);
|
||||
__maps_check();
|
||||
}
|
||||
}
|
||||
|
||||
struct Map *__maps_alloc(void) {
|
||||
struct Dll *e;
|
||||
struct Map *map;
|
||||
if ((e = dll_first(__maps.free))) {
|
||||
dll_remove(&__maps.free, e);
|
||||
map = MAP_FREE_CONTAINER(e);
|
||||
return map;
|
||||
}
|
||||
int granularity = __granularity();
|
||||
struct DirectMap sys = sys_mmap(0, granularity, PROT_READ | PROT_WRITE,
|
||||
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||
if (sys.addr == MAP_FAILED)
|
||||
return 0;
|
||||
if (IsWindows())
|
||||
CloseHandle(sys.maphandle);
|
||||
map = sys.addr;
|
||||
map->addr = MAP_FAILED;
|
||||
for (int i = 1; i < granularity / sizeof(struct Map); ++i)
|
||||
__maps_free(map + i);
|
||||
return map;
|
||||
}
|
||||
|
||||
static int __munmap(char *addr, size_t size, bool untrack_only) {
|
||||
|
||||
// validate arguments
|
||||
int pagesz = getpagesize();
|
||||
int granularity = __granularity();
|
||||
if (((uintptr_t)addr & (granularity - 1)) || //
|
||||
!size || (uintptr_t)addr + size < size)
|
||||
return einval();
|
||||
|
||||
// normalize size
|
||||
size = (size + granularity - 1) & -granularity;
|
||||
|
||||
// untrack mappings
|
||||
static int __muntrack(char *addr, size_t size, int pagesz,
|
||||
struct Dll **deleted) {
|
||||
int rc = 0;
|
||||
struct Map *map;
|
||||
struct Map *next;
|
||||
struct Dll *deleted = 0;
|
||||
if (__maps_lock()) {
|
||||
__maps_unlock();
|
||||
return edeadlk();
|
||||
}
|
||||
for (map = __maps_floor(addr); map; map = next) {
|
||||
struct Map *ceil;
|
||||
struct Map *floor;
|
||||
floor = __maps_floor(addr);
|
||||
ceil = __maps_ceil(addr + size);
|
||||
for (map = floor; map && map != ceil; map = next) {
|
||||
next = __maps_next(map);
|
||||
char *map_addr = map->addr;
|
||||
size_t map_size = map->size;
|
||||
if (!(MAX(addr, map_addr) < MIN(addr + size, map_addr + PGUP(map_size))))
|
||||
break;
|
||||
if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) {
|
||||
if (!(MAX(addr, map_addr) <
|
||||
MIN(addr + PGUP(size), map_addr + PGUP(map_size))))
|
||||
continue;
|
||||
if (addr <= map_addr && addr + PGUP(size) >= map_addr + PGUP(map_size)) {
|
||||
// remove mapping completely
|
||||
tree_remove(&__maps.maps, &map->tree);
|
||||
dll_init(&map->free);
|
||||
dll_make_first(&deleted, &map->free);
|
||||
dll_make_first(deleted, &map->free);
|
||||
__maps.pages -= (map_size + pagesz - 1) / pagesz;
|
||||
__maps.count -= 1;
|
||||
__maps_check();
|
||||
} else if (IsWindows()) {
|
||||
// you can't carve up memory maps on windows. our mmap() makes
|
||||
// this not a problem (for non-enormous memory maps) by making
|
||||
|
|
@ -210,8 +164,8 @@ static int __munmap(char *addr, size_t size, bool untrack_only) {
|
|||
rc = einval();
|
||||
} else if (addr <= map_addr) {
|
||||
// shave off lefthand side of mapping
|
||||
ASSERT(addr + size < map_addr + PGUP(map_size));
|
||||
size_t left = PGUP(addr + size - map_addr);
|
||||
ASSERT(addr + PGUP(size) < map_addr + PGUP(map_size));
|
||||
size_t left = addr + PGUP(size) - map_addr;
|
||||
size_t right = map_size - left;
|
||||
ASSERT(right > 0);
|
||||
ASSERT(left > 0);
|
||||
|
|
@ -225,11 +179,12 @@ static int __munmap(char *addr, size_t size, bool untrack_only) {
|
|||
leftmap->addr = map_addr;
|
||||
leftmap->size = left;
|
||||
dll_init(&leftmap->free);
|
||||
dll_make_first(&deleted, &leftmap->free);
|
||||
dll_make_first(deleted, &leftmap->free);
|
||||
__maps_check();
|
||||
} else {
|
||||
rc = -1;
|
||||
}
|
||||
} else if (addr + size >= map_addr + PGUP(map_size)) {
|
||||
} else if (addr + PGUP(size) >= map_addr + PGUP(map_size)) {
|
||||
// shave off righthand side of mapping
|
||||
size_t left = addr - map_addr;
|
||||
size_t right = map_addr + map_size - addr;
|
||||
|
|
@ -240,14 +195,15 @@ static int __munmap(char *addr, size_t size, bool untrack_only) {
|
|||
rightmap->addr = addr;
|
||||
rightmap->size = right;
|
||||
dll_init(&rightmap->free);
|
||||
dll_make_first(&deleted, &rightmap->free);
|
||||
dll_make_first(deleted, &rightmap->free);
|
||||
__maps_check();
|
||||
} else {
|
||||
rc = -1;
|
||||
}
|
||||
} else {
|
||||
// punch hole in mapping
|
||||
size_t left = addr - map_addr;
|
||||
size_t middle = size;
|
||||
size_t middle = PGUP(size);
|
||||
size_t right = map_size - middle - left;
|
||||
struct Map *leftmap;
|
||||
if ((leftmap = __maps_alloc())) {
|
||||
|
|
@ -268,7 +224,8 @@ static int __munmap(char *addr, size_t size, bool untrack_only) {
|
|||
middlemap->addr = addr;
|
||||
middlemap->size = size;
|
||||
dll_init(&middlemap->free);
|
||||
dll_make_first(&deleted, &middlemap->free);
|
||||
dll_make_first(deleted, &middlemap->free);
|
||||
__maps_check();
|
||||
} else {
|
||||
rc = -1;
|
||||
}
|
||||
|
|
@ -276,36 +233,146 @@ static int __munmap(char *addr, size_t size, bool untrack_only) {
|
|||
rc = -1;
|
||||
}
|
||||
}
|
||||
__maps_check();
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
|
||||
void __maps_free(struct Map *map) {
|
||||
map->size = 0;
|
||||
map->addr = MAP_FAILED;
|
||||
dll_init(&map->free);
|
||||
dll_make_first(&__maps.free, &map->free);
|
||||
}
|
||||
|
||||
static void __maps_insert(struct Map *map) {
|
||||
map->flags &= MAP_TYPE | MAP_ANONYMOUS | MAP_NOFORK;
|
||||
|
||||
// coalesce adjacent mappings
|
||||
if (!IsWindows() && (map->flags & MAP_ANONYMOUS)) {
|
||||
int prot = map->prot & ~(MAP_FIXED | MAP_FIXED_NOREPLACE);
|
||||
int flags = map->flags;
|
||||
bool coalesced = false;
|
||||
struct Map *floor, *ceil, *other, *last = 0;
|
||||
floor = __maps_floor(map->addr);
|
||||
ceil = __maps_ceil(map->addr + map->size);
|
||||
for (other = floor; other; last = other, other = __maps_next(other)) {
|
||||
if (prot == other->prot && flags == other->flags) {
|
||||
if (!coalesced) {
|
||||
if (map->addr == other->addr + other->size) {
|
||||
__maps.pages += (map->size + getpagesize() - 1) / getpagesize();
|
||||
other->size += map->size;
|
||||
__maps_free(map);
|
||||
__maps_check();
|
||||
coalesced = true;
|
||||
} else if (map->addr + map->size == other->addr) {
|
||||
__maps.pages += (map->size + getpagesize() - 1) / getpagesize();
|
||||
other->addr -= map->size;
|
||||
other->size += map->size;
|
||||
__maps_free(map);
|
||||
__maps_check();
|
||||
coalesced = true;
|
||||
}
|
||||
}
|
||||
if (last && other->addr == last->addr + last->size) {
|
||||
other->addr -= last->size;
|
||||
other->size += last->size;
|
||||
tree_remove(&__maps.maps, &last->tree);
|
||||
__maps.count -= 1;
|
||||
__maps_free(last);
|
||||
__maps_check();
|
||||
}
|
||||
}
|
||||
if (other == ceil)
|
||||
break;
|
||||
}
|
||||
if (coalesced)
|
||||
return;
|
||||
}
|
||||
|
||||
// otherwise insert new mapping
|
||||
__maps.pages += (map->size + getpagesize() - 1) / getpagesize();
|
||||
__maps_add(map);
|
||||
__maps_check();
|
||||
}
|
||||
|
||||
struct Map *__maps_alloc(void) {
|
||||
struct Dll *e;
|
||||
struct Map *map;
|
||||
if ((e = dll_first(__maps.free))) {
|
||||
dll_remove(&__maps.free, e);
|
||||
map = MAP_FREE_CONTAINER(e);
|
||||
return map;
|
||||
}
|
||||
int gransz = getgransize();
|
||||
struct DirectMap sys = sys_mmap(0, gransz, PROT_READ | PROT_WRITE,
|
||||
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||
if (sys.addr == MAP_FAILED)
|
||||
return 0;
|
||||
map = sys.addr;
|
||||
map->addr = sys.addr;
|
||||
map->size = gransz;
|
||||
map->prot = PROT_READ | PROT_WRITE;
|
||||
map->flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NOFORK;
|
||||
map->hand = sys.maphandle;
|
||||
__maps_insert(map++);
|
||||
map->addr = MAP_FAILED;
|
||||
for (int i = 1; i < gransz / sizeof(struct Map) - 1; ++i)
|
||||
__maps_free(map + i);
|
||||
return map;
|
||||
}
|
||||
|
||||
static int __munmap(char *addr, size_t size) {
|
||||
|
||||
// validate arguments
|
||||
int pagesz = getpagesize();
|
||||
int gransz = getgransize();
|
||||
if (((uintptr_t)addr & (gransz - 1)) || //
|
||||
!size || (uintptr_t)addr + size < size)
|
||||
return einval();
|
||||
|
||||
// lock the memory manager
|
||||
// abort on reentry due to signal handler
|
||||
if (__maps_lock()) {
|
||||
__maps_unlock();
|
||||
return edeadlk();
|
||||
}
|
||||
__maps_check();
|
||||
|
||||
// normalize size
|
||||
// abort if size doesn't include all pages in granule
|
||||
size_t pgup_size = (size + pagesz - 1) & -pagesz;
|
||||
size_t grup_size = (size + gransz - 1) & -gransz;
|
||||
if (grup_size > pgup_size)
|
||||
if (__maps_overlaps(addr + pgup_size, grup_size - pgup_size, pagesz)) {
|
||||
__maps_unlock();
|
||||
return einval();
|
||||
}
|
||||
|
||||
// untrack mappings
|
||||
struct Dll *deleted = 0;
|
||||
__muntrack(addr, pgup_size, pagesz, &deleted);
|
||||
__maps_unlock();
|
||||
|
||||
// delete mappings
|
||||
int rc = 0;
|
||||
for (struct Dll *e = dll_first(deleted); e; e = dll_next(deleted, e)) {
|
||||
struct Map *map = MAP_FREE_CONTAINER(e);
|
||||
if (!untrack_only) {
|
||||
if (!IsWindows()) {
|
||||
if (sys_munmap(map->addr, map->size))
|
||||
rc = -1;
|
||||
} else if (map->hand != -1) {
|
||||
ASSERT(!((uintptr_t)map->addr & (granularity - 1)));
|
||||
if (!UnmapViewOfFile(map->addr))
|
||||
rc = -1;
|
||||
if (!CloseHandle(map->hand))
|
||||
rc = -1;
|
||||
}
|
||||
if (!IsWindows()) {
|
||||
if (sys_munmap(map->addr, map->size))
|
||||
rc = -1;
|
||||
} else if (map->hand != -1) {
|
||||
ASSERT(!((uintptr_t)map->addr & (gransz - 1)));
|
||||
if (!UnmapViewOfFile(map->addr))
|
||||
rc = -1;
|
||||
if (!CloseHandle(map->hand))
|
||||
rc = -1;
|
||||
}
|
||||
}
|
||||
|
||||
// free mappings
|
||||
if (!dll_is_empty(deleted)) {
|
||||
__maps_lock();
|
||||
struct Dll *e;
|
||||
while ((e = dll_first(deleted))) {
|
||||
dll_remove(&deleted, e);
|
||||
__maps_free(MAP_FREE_CONTAINER(e));
|
||||
}
|
||||
__maps_check();
|
||||
dll_make_first(&__maps.free, deleted);
|
||||
__maps_unlock();
|
||||
}
|
||||
|
||||
|
|
@ -313,7 +380,7 @@ static int __munmap(char *addr, size_t size, bool untrack_only) {
|
|||
}
|
||||
|
||||
static void *__mmap_chunk(void *addr, size_t size, int prot, int flags, int fd,
|
||||
int64_t off, int pagesz, int granularity) {
|
||||
int64_t off, int pagesz, int gransz) {
|
||||
|
||||
// polyfill nuances of fixed mappings
|
||||
int sysflags = flags;
|
||||
|
|
@ -328,7 +395,7 @@ static void *__mmap_chunk(void *addr, size_t size, int prot, int flags, int fd,
|
|||
sysflags |= MAP_FIXED_NOREPLACE_linux;
|
||||
} else if (IsFreebsd() || IsNetbsd()) {
|
||||
sysflags |= MAP_FIXED;
|
||||
if (overlaps_existing_map(addr, size, pagesz))
|
||||
if (__maps_overlaps(addr, size, pagesz))
|
||||
return (void *)eexist();
|
||||
} else {
|
||||
noreplace = true;
|
||||
|
|
@ -351,7 +418,7 @@ static void *__mmap_chunk(void *addr, size_t size, int prot, int flags, int fd,
|
|||
|
||||
// remove mapping we blew away
|
||||
if (IsWindows() && should_untrack)
|
||||
if (__munmap(addr, size, false))
|
||||
if (__munmap(addr, size))
|
||||
return MAP_FAILED;
|
||||
|
||||
// obtain mapping from operating system
|
||||
|
|
@ -366,7 +433,7 @@ TryAgain:
|
|||
} else if (should_untrack) {
|
||||
errno = ENOMEM;
|
||||
} else {
|
||||
addr += granularity;
|
||||
addr += gransz;
|
||||
errno = olderr;
|
||||
goto TryAgain;
|
||||
}
|
||||
|
|
@ -394,10 +461,17 @@ TryAgain:
|
|||
}
|
||||
|
||||
// untrack mapping we blew away
|
||||
if (!IsWindows() && should_untrack)
|
||||
__munmap(res.addr, size, true);
|
||||
if (!IsWindows() && should_untrack) {
|
||||
struct Dll *deleted = 0;
|
||||
__muntrack(res.addr, size, pagesz, &deleted);
|
||||
if (!dll_is_empty(deleted)) {
|
||||
__maps_lock();
|
||||
dll_make_first(&__maps.free, deleted);
|
||||
__maps_unlock();
|
||||
}
|
||||
}
|
||||
|
||||
// track Map object
|
||||
// track map object
|
||||
map->addr = res.addr;
|
||||
map->size = size;
|
||||
map->off = off;
|
||||
|
|
@ -417,11 +491,11 @@ TryAgain:
|
|||
}
|
||||
|
||||
static void *__mmap_impl(char *addr, size_t size, int prot, int flags, int fd,
|
||||
int64_t off, int pagesz, int granularity) {
|
||||
int64_t off, int pagesz, int gransz) {
|
||||
|
||||
// validate file map args
|
||||
if (!(flags & MAP_ANONYMOUS)) {
|
||||
if (off & (granularity - 1))
|
||||
if (off & (gransz - 1))
|
||||
return (void *)einval();
|
||||
if (IsWindows()) {
|
||||
if (!__isfdkind(fd, kFdFile))
|
||||
|
|
@ -433,36 +507,48 @@ static void *__mmap_impl(char *addr, size_t size, int prot, int flags, int fd,
|
|||
|
||||
// mmap works fine on unix
|
||||
if (!IsWindows())
|
||||
return __mmap_chunk(addr, size, prot, flags, fd, off, pagesz, granularity);
|
||||
return __mmap_chunk(addr, size, prot, flags, fd, off, pagesz, gransz);
|
||||
|
||||
// if the concept of pagesz wasn't exciting enough
|
||||
if (!addr && !(flags & (MAP_FIXED | MAP_FIXED_NOREPLACE))) {
|
||||
size_t slab = (size + granularity - 1) & -granularity;
|
||||
addr = (char *)(WINBASE + atomic_fetch_add(&__maps.rollo, slab) % WINMAXX);
|
||||
size_t rollo, rollo2, slab = (size + gransz - 1) & -gransz;
|
||||
rollo = atomic_load_explicit(&__maps.rollo, memory_order_relaxed);
|
||||
for (;;) {
|
||||
if ((rollo2 = rollo + slab) > WINMAXX) {
|
||||
rollo = 0;
|
||||
rollo2 = slab;
|
||||
}
|
||||
if (atomic_compare_exchange_weak_explicit(&__maps.rollo, &rollo, rollo2,
|
||||
memory_order_acq_rel,
|
||||
memory_order_relaxed)) {
|
||||
addr = (char *)WINBASE + rollo;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// windows forbids unmapping a subset of a map once it's made
|
||||
if (size <= granularity || size > 100 * 1024 * 1024)
|
||||
return __mmap_chunk(addr, size, prot, flags, fd, off, pagesz, granularity);
|
||||
if (size <= gransz || size > 100 * 1024 * 1024)
|
||||
return __mmap_chunk(addr, size, prot, flags, fd, off, pagesz, gransz);
|
||||
|
||||
// so we create a separate map for each granule in the mapping
|
||||
if (!(flags & MAP_FIXED)) {
|
||||
while (overlaps_existing_map(addr, size, pagesz)) {
|
||||
while (__maps_overlaps(addr, size, pagesz)) {
|
||||
if (flags & MAP_FIXED_NOREPLACE)
|
||||
return (void *)eexist();
|
||||
addr += granularity;
|
||||
addr += gransz;
|
||||
}
|
||||
}
|
||||
char *res = addr;
|
||||
while (size) {
|
||||
char *got;
|
||||
size_t amt = MIN(size, granularity);
|
||||
got = __mmap_chunk(addr, amt, prot, flags, fd, off, pagesz, granularity);
|
||||
size_t amt = MIN(size, gransz);
|
||||
got = __mmap_chunk(addr, amt, prot, flags, fd, off, pagesz, gransz);
|
||||
if (got != addr) {
|
||||
if (got != MAP_FAILED)
|
||||
__munmap(got, amt, false);
|
||||
__munmap(got, amt);
|
||||
if (addr > res)
|
||||
__munmap(res, addr - res, false);
|
||||
__munmap(res, addr - res);
|
||||
errno = EAGAIN;
|
||||
return MAP_FAILED;
|
||||
}
|
||||
|
|
@ -477,20 +563,20 @@ static void *__mmap(char *addr, size_t size, int prot, int flags, int fd,
|
|||
int64_t off) {
|
||||
char *res;
|
||||
int pagesz = getpagesize();
|
||||
int granularity = __granularity();
|
||||
int gransz = getgransize();
|
||||
|
||||
// validate arguments
|
||||
if (((uintptr_t)addr & (granularity - 1)) || //
|
||||
if (((uintptr_t)addr & (gransz - 1)) || //
|
||||
!size || (uintptr_t)addr + size < size)
|
||||
return (void *)einval();
|
||||
if (size > 0x100000000000)
|
||||
if (size > WINMAXX)
|
||||
return (void *)enomem();
|
||||
if (__maps.count * pagesz + size > __virtualmax)
|
||||
return (void *)enomem();
|
||||
|
||||
// create memory mappping
|
||||
if (!__isfdkind(fd, kFdZip)) {
|
||||
res = __mmap_impl(addr, size, prot, flags, fd, off, pagesz, granularity);
|
||||
res = __mmap_impl(addr, size, prot, flags, fd, off, pagesz, gransz);
|
||||
} else {
|
||||
res = _weaken(__zipos_mmap)(
|
||||
addr, size, prot, flags,
|
||||
|
|
@ -500,6 +586,170 @@ static void *__mmap(char *addr, size_t size, int prot, int flags, int fd,
|
|||
return res;
|
||||
}
|
||||
|
||||
static void *__mremap_impl(char *old_addr, size_t old_size, size_t new_size,
|
||||
int flags, char *new_addr, int pagesz, int gransz) {
|
||||
|
||||
// normalize and validate old size
|
||||
// abort if size doesn't include all pages in granule
|
||||
size_t pgup_old_size = (old_size + pagesz - 1) & -pagesz;
|
||||
size_t grup_old_size = (old_size + gransz - 1) & -gransz;
|
||||
if (grup_old_size > pgup_old_size)
|
||||
if (__maps_overlaps(old_addr + pgup_old_size, grup_old_size - pgup_old_size,
|
||||
pagesz))
|
||||
return (void *)einval();
|
||||
old_size = pgup_old_size;
|
||||
|
||||
// validate new size
|
||||
// abort if size doesn't include all pages in granule
|
||||
if (flags & MREMAP_FIXED) {
|
||||
size_t pgup_new_size = (new_size + pagesz - 1) & -pagesz;
|
||||
size_t grup_new_size = (new_size + gransz - 1) & -gransz;
|
||||
if (grup_new_size > pgup_new_size)
|
||||
if (__maps_overlaps(new_addr + pgup_new_size,
|
||||
grup_new_size - pgup_new_size, pagesz))
|
||||
return (void *)einval();
|
||||
}
|
||||
|
||||
// check old interval is fully contained within one mapping
|
||||
struct Map *old_map;
|
||||
if (!(old_map = __maps_floor(old_addr)) ||
|
||||
old_addr + old_size > old_map->addr + PGUP(old_map->size) ||
|
||||
old_addr < old_map->addr)
|
||||
return (void *)efault();
|
||||
|
||||
// save old properties
|
||||
int old_off = old_map->off;
|
||||
int old_prot = old_map->prot;
|
||||
int old_flags = old_map->flags;
|
||||
|
||||
// allocate object for tracking new mapping
|
||||
struct Map *map;
|
||||
if (!(map = __maps_alloc()))
|
||||
return (void *)enomem();
|
||||
|
||||
// netbsd mremap fixed returns enoent rather than unmapping old pages
|
||||
if (IsNetbsd() && (flags & MREMAP_FIXED))
|
||||
if (__munmap(new_addr, new_size)) {
|
||||
__maps_free(map);
|
||||
return MAP_FAILED;
|
||||
}
|
||||
|
||||
// release lock before system call if possible
|
||||
if (!flags)
|
||||
__maps_unlock();
|
||||
|
||||
// the time has come
|
||||
char *res;
|
||||
if (IsNetbsd()) {
|
||||
int sysfl = (flags & MREMAP_FIXED) ? MAP_FIXED : 0;
|
||||
res = sys_mremap(old_addr, old_size, (uintptr_t)new_addr, new_size, sysfl);
|
||||
} else {
|
||||
res = sys_mremap(old_addr, old_size, new_size, flags, (uintptr_t)new_addr);
|
||||
}
|
||||
|
||||
// re-acquire lock if needed
|
||||
if (!flags)
|
||||
__maps_lock();
|
||||
|
||||
// check result
|
||||
if (res == MAP_FAILED) {
|
||||
__maps_free(map);
|
||||
return MAP_FAILED;
|
||||
}
|
||||
|
||||
if (!(flags & MREMAP_MAYMOVE))
|
||||
ASSERT(res == old_addr);
|
||||
|
||||
// untrack old mapping
|
||||
struct Dll *deleted = 0;
|
||||
__muntrack(old_addr, old_size, pagesz, &deleted);
|
||||
dll_make_first(&__maps.free, deleted);
|
||||
deleted = 0;
|
||||
|
||||
// track map object
|
||||
map->addr = res;
|
||||
map->size = new_size;
|
||||
map->off = old_off;
|
||||
map->prot = old_prot;
|
||||
map->flags = old_flags;
|
||||
__maps_insert(map);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
static void *__mremap(char *old_addr, size_t old_size, size_t new_size,
|
||||
int flags, char *new_addr) {
|
||||
|
||||
int pagesz = getpagesize();
|
||||
int gransz = getgransize();
|
||||
|
||||
// demand kernel support
|
||||
if (!IsLinux() && !IsNetbsd())
|
||||
return (void *)enosys();
|
||||
|
||||
// we support these flags
|
||||
if (flags & ~(MREMAP_MAYMOVE | MREMAP_FIXED))
|
||||
return (void *)einval();
|
||||
if (IsNetbsd() && !(flags & MREMAP_MAYMOVE) &&
|
||||
((new_size + pagesz - 1) & -pagesz) > old_size)
|
||||
return (void *)enotsup();
|
||||
if ((flags & MREMAP_FIXED) && !(flags & MREMAP_MAYMOVE))
|
||||
return (void *)einval();
|
||||
|
||||
// addresses must be granularity aligned
|
||||
if ((uintptr_t)old_addr & (gransz - 1))
|
||||
return (void *)einval();
|
||||
if (flags & MREMAP_FIXED)
|
||||
if ((uintptr_t)new_addr & (gransz - 1))
|
||||
return (void *)einval();
|
||||
|
||||
// sizes must not be zero
|
||||
if (!old_size)
|
||||
return (void *)einval();
|
||||
if (!new_size)
|
||||
return (void *)einval();
|
||||
|
||||
// check for big size
|
||||
if (old_size > WINMAXX)
|
||||
return (void *)enomem();
|
||||
if (new_size > WINMAXX)
|
||||
return (void *)enomem();
|
||||
|
||||
// check for overflow
|
||||
if ((uintptr_t)old_addr + old_size < old_size)
|
||||
return (void *)enomem();
|
||||
if (flags & MREMAP_FIXED)
|
||||
if ((uintptr_t)new_addr + new_size < new_size)
|
||||
return (void *)enomem();
|
||||
|
||||
// old and new intervals must not overlap
|
||||
if (flags & MREMAP_FIXED)
|
||||
if (MAX(old_addr, new_addr) <
|
||||
MIN(old_addr + old_size, new_addr + PGUP(new_size)))
|
||||
return (void *)einval();
|
||||
|
||||
// memory increase must not exceed RLIMIT_AS
|
||||
if (PGUP(new_size) > old_size)
|
||||
if (__maps.count * pagesz - old_size + PGUP(new_size) > __virtualmax)
|
||||
return (void *)enomem();
|
||||
|
||||
// lock the memory manager
|
||||
// abort on reentry due to signal handler
|
||||
if (__maps_lock()) {
|
||||
__maps_unlock();
|
||||
return (void *)edeadlk();
|
||||
}
|
||||
__maps_check();
|
||||
|
||||
// perform operation
|
||||
char *res = __mremap_impl(old_addr, old_size, new_size, flags, new_addr,
|
||||
pagesz, gransz);
|
||||
|
||||
// return result
|
||||
__maps_unlock();
|
||||
return res;
|
||||
}
|
||||
|
||||
void *mmap(void *addr, size_t size, int prot, int flags, int fd, int64_t off) {
|
||||
void *res = __mmap(addr, size, prot, flags, fd, off);
|
||||
STRACE("mmap(%p, %'zu, %s, %s, %d, %'ld) → %p% m", addr, size,
|
||||
|
|
@ -507,8 +757,22 @@ void *mmap(void *addr, size_t size, int prot, int flags, int fd, int64_t off) {
|
|||
return res;
|
||||
}
|
||||
|
||||
void *mremap(void *old_addr, size_t old_size, size_t new_size, int flags, ...) {
|
||||
va_list ap;
|
||||
void *new_addr = 0;
|
||||
if (flags & MREMAP_FIXED) {
|
||||
va_start(ap, flags);
|
||||
new_addr = va_arg(ap, void *);
|
||||
va_end(ap);
|
||||
}
|
||||
void *res = __mremap(old_addr, old_size, new_size, flags, new_addr);
|
||||
STRACE("mremap(%p, %'zu, %'zu, %s, %p) → %p% m", old_addr, old_size, new_size,
|
||||
DescribeMremapFlags(flags), new_addr, res);
|
||||
return res;
|
||||
}
|
||||
|
||||
int munmap(void *addr, size_t size) {
|
||||
int rc = __munmap(addr, size, false);
|
||||
int rc = __munmap(addr, size);
|
||||
STRACE("munmap(%p, %'zu) → %d% m", addr, size, rc);
|
||||
return rc;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -74,13 +74,16 @@ int __mprotect(char *addr, size_t size, int prot) {
|
|||
__maps_unlock();
|
||||
return edeadlk();
|
||||
}
|
||||
for (struct Map *map = __maps_floor(addr); map; map = __maps_next(map)) {
|
||||
struct Map *map, *ceil, *floor;
|
||||
floor = __maps_floor(addr);
|
||||
ceil = __maps_ceil(addr + size);
|
||||
for (map = floor; map && map != ceil; map = __maps_next(map)) {
|
||||
char *map_addr = map->addr;
|
||||
size_t map_size = map->size;
|
||||
char *beg = MAX(addr, map_addr);
|
||||
char *end = MIN(addr + size, map_addr + PGUP(map_size));
|
||||
if (beg >= end)
|
||||
break;
|
||||
continue;
|
||||
found = true;
|
||||
if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) {
|
||||
// change protection of entire mapping
|
||||
|
|
|
|||
|
|
@ -37,7 +37,10 @@ textwindows int sys_msync_nt(char *addr, size_t size, int flags) {
|
|||
if (__maps_lock()) {
|
||||
rc = edeadlk();
|
||||
} else {
|
||||
for (struct Map *map = __maps_floor(addr); map; map = __maps_next(map)) {
|
||||
struct Map *map, *ceil, *floor;
|
||||
floor = __maps_floor(addr);
|
||||
ceil = __maps_ceil(addr + size);
|
||||
for (map = floor; map && map != ceil; map = __maps_next(map)) {
|
||||
char *beg = MAX(addr, map->addr);
|
||||
char *end = MIN(addr + size, map->addr + map->size);
|
||||
if (beg < end)
|
||||
|
|
|
|||
26
libc/intrin/randaddr.c
Normal file
26
libc/intrin/randaddr.c
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
|
||||
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
|
||||
╞══════════════════════════════════════════════════════════════════════════════╡
|
||||
│ Copyright 2024 Justine Alexandra Roberts Tunney │
|
||||
│ │
|
||||
│ Permission to use, copy, modify, and/or distribute this software for │
|
||||
│ any purpose with or without fee is hereby granted, provided that the │
|
||||
│ above copyright notice and this permission notice appear in all copies. │
|
||||
│ │
|
||||
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
|
||||
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
|
||||
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
|
||||
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
|
||||
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
|
||||
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
|
||||
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
|
||||
│ PERFORMANCE OF THIS SOFTWARE. │
|
||||
╚─────────────────────────────────────────────────────────────────────────────*/
|
||||
#include "libc/intrin/maps.h"
|
||||
|
||||
void *randaddr(void) {
|
||||
static unsigned long lcg = 1;
|
||||
lcg *= 6364136223846793005;
|
||||
lcg += 1442695040888963407;
|
||||
return (void *)(lcg >> 48 << 28);
|
||||
}
|
||||
|
|
@ -36,35 +36,12 @@ static inline void tree_set_red(struct Tree *node, int red) {
|
|||
node->word |= red;
|
||||
}
|
||||
|
||||
forceinline optimizespeed struct Tree *tree_floor(const struct Tree *node,
|
||||
const void *key,
|
||||
tree_search_f *cmp) {
|
||||
struct Tree *left = 0;
|
||||
while (node) {
|
||||
if (cmp(key, node) >= 0) {
|
||||
left = (struct Tree *)node;
|
||||
node = tree_get_left(node);
|
||||
} else {
|
||||
node = node->right;
|
||||
}
|
||||
}
|
||||
return left;
|
||||
}
|
||||
|
||||
static inline struct Tree *tree_ceil(const struct Tree *node, const void *key,
|
||||
tree_search_f *cmp) {
|
||||
struct Tree *right = 0;
|
||||
while (node) {
|
||||
if (cmp(key, node) < 0) {
|
||||
right = (struct Tree *)node;
|
||||
node = tree_get_left(node);
|
||||
} else {
|
||||
node = node->right;
|
||||
}
|
||||
}
|
||||
return right;
|
||||
}
|
||||
|
||||
// Returns node equal to given key.
|
||||
//
|
||||
// [1 3 5 7] [1 3 5 7] [1 3 5 7]
|
||||
// NULL ↑ NULL
|
||||
// 4 3 8
|
||||
//
|
||||
static inline struct Tree *tree_get(const struct Tree *node, const void *key,
|
||||
tree_search_f *cmp) {
|
||||
while (node) {
|
||||
|
|
@ -80,6 +57,72 @@ static inline struct Tree *tree_get(const struct Tree *node, const void *key,
|
|||
return 0;
|
||||
}
|
||||
|
||||
// Returns last node less than or equal to given key.
|
||||
//
|
||||
// [1 3 5 7] [1 3 5 7] [1 3 5 7]
|
||||
// ↑ ↑ ↑
|
||||
// 4 3 8
|
||||
//
|
||||
forceinline optimizespeed struct Tree *tree_floor(const struct Tree *node,
|
||||
const void *key,
|
||||
tree_search_f *cmp) {
|
||||
struct Tree *left = 0;
|
||||
while (node) {
|
||||
int c = cmp(key, node);
|
||||
if (c < 0) {
|
||||
node = tree_get_left(node);
|
||||
} else if (c > 0) {
|
||||
left = (struct Tree *)node;
|
||||
node = node->right;
|
||||
} else {
|
||||
return (struct Tree *)node;
|
||||
}
|
||||
}
|
||||
return left;
|
||||
}
|
||||
|
||||
// Returns first node not less than given key.
|
||||
//
|
||||
// [1 3 5 7] [1 3 5 7] [1 3 5 7]
|
||||
// ↑ ↑ NULL
|
||||
// 4 3 8
|
||||
//
|
||||
static inline struct Tree *tree_lower(const struct Tree *node, const void *key,
|
||||
tree_search_f *cmp) {
|
||||
struct Tree *left = 0;
|
||||
while (node) {
|
||||
int c = cmp(key, node);
|
||||
if (c <= 0) {
|
||||
left = (struct Tree *)node;
|
||||
node = tree_get_left(node);
|
||||
} else {
|
||||
node = node->right;
|
||||
}
|
||||
}
|
||||
return left;
|
||||
}
|
||||
|
||||
// Returns first node greater than than given key.
|
||||
//
|
||||
// [1 3 5 7] [1 3 5 7] [1 3 5 7]
|
||||
// ↑ ↑ NULL
|
||||
// 4 3 8
|
||||
//
|
||||
static inline struct Tree *tree_ceil(const struct Tree *node, const void *key,
|
||||
tree_search_f *cmp) {
|
||||
struct Tree *left = 0;
|
||||
while (node) {
|
||||
int c = cmp(key, node);
|
||||
if (c < 0) {
|
||||
left = (struct Tree *)node;
|
||||
node = tree_get_left(node);
|
||||
} else {
|
||||
node = node->right;
|
||||
}
|
||||
}
|
||||
return left;
|
||||
}
|
||||
|
||||
struct Tree *tree_next(struct Tree *) libcesque;
|
||||
struct Tree *tree_prev(struct Tree *) libcesque;
|
||||
struct Tree *tree_first(struct Tree *) libcesque;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue