mirror of
https://github.com/jart/cosmopolitan.git
synced 2025-07-08 20:28:30 +00:00
Make realloc() go 100x faster on Linux/NetBSD
Cosmopolitan now supports mremap(), which is only supported on Linux and NetBSD. First, it allows memory mappings to be relocated without copying them; this can dramatically speed up data structures like std::vector if the array size grows larger than 256kb. The mremap() system call is also 10x faster than munmap() when shrinking large memory mappings. There's now two functions, getpagesize() and getgransize() which help to write portable code that uses mmap(MAP_FIXED). Alternative sysconf() may be called with our new _SC_GRANSIZE. The madvise() system call now has a better wrapper with improved documentation.
This commit is contained in:
Justine Tunney
parent
196942084b
commit
f7780de24b
71 changed files with 1301 additions and 640 deletions
|
|
@ -16,61 +16,31 @@
|
|||
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
|
||||
│ PERFORMANCE OF THIS SOFTWARE. │
|
||||
╚─────────────────────────────────────────────────────────────────────────────*/
|
||||
#include "libc/calls/syscall-nt.internal.h"
|
||||
#include "libc/calls/syscall_support-nt.internal.h"
|
||||
#include "libc/nt/enum/offerpriority.h"
|
||||
#include "libc/nt/memory.h"
|
||||
#include "libc/nt/runtime.h"
|
||||
#include "libc/nt/struct/memoryrangeentry.h"
|
||||
#include "libc/sysv/consts/madv.h"
|
||||
#include "libc/sysv/errfuns.h"
|
||||
|
||||
typedef bool32 (*__msabi PrefetchVirtualMemoryPtr)(
|
||||
int64_t hProcess, uintptr_t NumberOfEntries,
|
||||
struct NtMemoryRangeEntry *VirtualAddresses, uint32_t reserved_Flags);
|
||||
|
||||
textwindows static PrefetchVirtualMemoryPtr GetPrefetchVirtualMemory(void) {
|
||||
static PrefetchVirtualMemoryPtr PrefetchVirtualMemory_;
|
||||
if (!PrefetchVirtualMemory_) {
|
||||
PrefetchVirtualMemory_ = /* win8.1+ */
|
||||
GetProcAddressModule("Kernel32.dll", "PrefetchVirtualMemory");
|
||||
}
|
||||
return PrefetchVirtualMemory_;
|
||||
}
|
||||
|
||||
typedef bool32 (*__msabi OfferVirtualMemoryPtr)(void *inout_VirtualAddress,
|
||||
size_t Size, int Priority);
|
||||
|
||||
textwindows static OfferVirtualMemoryPtr GetOfferVirtualMemory(void) {
|
||||
static OfferVirtualMemoryPtr OfferVirtualMemory_;
|
||||
if (!OfferVirtualMemory_) {
|
||||
OfferVirtualMemory_ = /* win8.1+ */
|
||||
GetProcAddressModule("Kernel32.dll", "OfferVirtualMemory");
|
||||
}
|
||||
return OfferVirtualMemory_;
|
||||
}
|
||||
|
||||
textwindows int sys_madvise_nt(void *addr, size_t length, int advice) {
|
||||
if (advice == MADV_WILLNEED || advice == MADV_SEQUENTIAL) {
|
||||
PrefetchVirtualMemoryPtr fn = GetPrefetchVirtualMemory();
|
||||
if (fn) {
|
||||
if (fn(GetCurrentProcess(), 1, &(struct NtMemoryRangeEntry){addr, length},
|
||||
0)) {
|
||||
return 0;
|
||||
} else {
|
||||
return __winerr();
|
||||
}
|
||||
if (!length)
|
||||
return 0;
|
||||
if (PrefetchVirtualMemory(GetCurrentProcess(), 1,
|
||||
&(struct NtMemoryRangeEntry){addr, length}, 0)) {
|
||||
return 0;
|
||||
} else {
|
||||
return enosys();
|
||||
return __winerr();
|
||||
}
|
||||
} else if (advice == MADV_FREE) {
|
||||
OfferVirtualMemoryPtr fn = GetOfferVirtualMemory();
|
||||
if (fn) {
|
||||
if (fn(addr, length, kNtVmOfferPriorityNormal)) {
|
||||
return 0;
|
||||
} else {
|
||||
return __winerr();
|
||||
}
|
||||
if (!length)
|
||||
return 0;
|
||||
if (OfferVirtualMemory(addr, length, kNtVmOfferPriorityNormal)) {
|
||||
return 0;
|
||||
} else {
|
||||
return enosys();
|
||||
return __winerr();
|
||||
}
|
||||
} else {
|
||||
return einval();
|
||||
|
|
|
|||
|
|
@ -21,27 +21,49 @@
|
|||
#include "libc/calls/syscall-sysv.internal.h"
|
||||
#include "libc/dce.h"
|
||||
#include "libc/intrin/strace.internal.h"
|
||||
#include "libc/runtime/runtime.h"
|
||||
#include "libc/sysv/errfuns.h"
|
||||
|
||||
static int __madvise(void *addr, size_t length, int advice) {
|
||||
|
||||
// simulate linux behavior of validating alignment
|
||||
if ((uintptr_t)addr & (getpagesize() - 1))
|
||||
return einval();
|
||||
|
||||
// simulate linux behavior of checking for negative length
|
||||
if ((ssize_t)length < 0)
|
||||
return einval();
|
||||
|
||||
// madvise(0, 0, advice) may be used to validate advice
|
||||
if (!length && (IsFreebsd() || IsNetbsd()))
|
||||
addr = (void *)65536l;
|
||||
|
||||
if (!IsWindows())
|
||||
return sys_madvise(addr, length, advice);
|
||||
return sys_madvise_nt(addr, length, advice);
|
||||
}
|
||||
|
||||
/**
|
||||
* Drops hints to O/S about intended access patterns of mmap()'d memory.
|
||||
* Declares intent to OS on how memory region will be used.
|
||||
*
|
||||
* `madvise(0, 0, advice)` is recommended for validating `advise` and it
|
||||
* will always be the case that a `length` of zero is a no-op otherwise.
|
||||
*
|
||||
* Having the interval overlap unmapped pages has undefined behavior. On
|
||||
* Linux, this can be counted upon to raise ENOMEM. Other OSes vary much
|
||||
* in behavior here; they'll might ignore unmapped regions or they might
|
||||
* raise EINVAL, EFAULT, or ENOMEM.
|
||||
*
|
||||
* @param advice can be MADV_WILLNEED, MADV_SEQUENTIAL, MADV_FREE, etc.
|
||||
* @return 0 on success, or -1 w/ errno
|
||||
* @raise EINVAL if `advice` isn't valid or supported by system
|
||||
* @raise EINVAL on Linux if addr/length isn't page size aligned with
|
||||
* respect to `getpagesize()`
|
||||
* @raise ENOMEM on Linux if addr/length overlaps unmapped regions
|
||||
* @raise EINVAL if `addr` isn't getpagesize() aligned
|
||||
* @raise EINVAL if `length` is negative
|
||||
* @see libc/sysv/consts.sh
|
||||
* @see fadvise()
|
||||
*/
|
||||
int madvise(void *addr, size_t length, int advice) {
|
||||
int rc;
|
||||
if (!IsWindows()) {
|
||||
rc = sys_madvise(addr, length, advice);
|
||||
} else {
|
||||
rc = sys_madvise_nt(addr, length, advice);
|
||||
}
|
||||
int rc = __madvise(addr, length, advice);
|
||||
STRACE("madvise(%p, %'zu, %d) → %d% m", addr, length, advice, rc);
|
||||
return rc;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,77 +0,0 @@
|
|||
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
|
||||
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
|
||||
╞══════════════════════════════════════════════════════════════════════════════╡
|
||||
│ Copyright 2021 Justine Alexandra Roberts Tunney │
|
||||
│ │
|
||||
│ Permission to use, copy, modify, and/or distribute this software for │
|
||||
│ any purpose with or without fee is hereby granted, provided that the │
|
||||
│ above copyright notice and this permission notice appear in all copies. │
|
||||
│ │
|
||||
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
|
||||
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
|
||||
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
|
||||
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
|
||||
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
|
||||
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
|
||||
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
|
||||
│ PERFORMANCE OF THIS SOFTWARE. │
|
||||
╚─────────────────────────────────────────────────────────────────────────────*/
|
||||
#include "libc/calls/calls.h"
|
||||
#include "libc/calls/syscall-sysv.internal.h"
|
||||
#include "libc/dce.h"
|
||||
#include "libc/errno.h"
|
||||
#include "libc/intrin/asmflag.h"
|
||||
#include "libc/intrin/strace.internal.h"
|
||||
#include "libc/sysv/consts/map.h"
|
||||
#include "libc/sysv/consts/mremap.h"
|
||||
#include "libc/sysv/errfuns.h"
|
||||
|
||||
/**
|
||||
* Relocates memory.
|
||||
*
|
||||
* This function lets you move to to different addresses witohut copying
|
||||
* it. This system call is currently supported on Linux and NetBSD. Your
|
||||
* C library runtime won't have any awareness of this memory, so certain
|
||||
* features like ASAN memory safety and kprintf() won't work as well.
|
||||
*/
|
||||
void *sys_mremap(void *p, size_t n, size_t m, int f, void *q) {
|
||||
#ifdef __x86_64__
|
||||
bool cf;
|
||||
uintptr_t res, rdx;
|
||||
register uintptr_t r8 asm("r8");
|
||||
register uintptr_t r10 asm("r10");
|
||||
if (IsLinux()) {
|
||||
r10 = f;
|
||||
r8 = (uintptr_t)q;
|
||||
asm("syscall"
|
||||
: "=a"(res)
|
||||
: "0"(0x019), "D"(p), "S"(n), "d"(m), "r"(r10), "r"(r8)
|
||||
: "rcx", "r11", "memory", "cc");
|
||||
if (res > -4096ul)
|
||||
errno = -res, res = -1;
|
||||
} else if (IsNetbsd()) {
|
||||
if (f & MREMAP_MAYMOVE) {
|
||||
res = 0x19B;
|
||||
r10 = m;
|
||||
r8 = (f & MREMAP_FIXED) ? MAP_FIXED : 0;
|
||||
asm(CFLAG_ASM("syscall")
|
||||
: CFLAG_CONSTRAINT(cf), "+a"(res), "=d"(rdx)
|
||||
: "D"(p), "S"(n), "2"(q), "r"(r10), "r"(r8)
|
||||
: "rcx", "r9", "r11", "memory", "cc");
|
||||
if (cf)
|
||||
errno = res, res = -1;
|
||||
} else {
|
||||
res = einval();
|
||||
}
|
||||
} else {
|
||||
res = enosys();
|
||||
}
|
||||
#elif defined(__aarch64__)
|
||||
void *res;
|
||||
res = __sys_mremap(p, n, m, f, q);
|
||||
#else
|
||||
#error "arch unsupported"
|
||||
#endif
|
||||
KERNTRACE("sys_mremap(%p, %'zu, %'zu, %#b, %p) → %p% m", p, n, m, f, q, res);
|
||||
return (void *)res;
|
||||
}
|
||||
|
|
@ -136,8 +136,7 @@ u32 sys_getuid(void);
|
|||
u32 sys_umask(u32);
|
||||
unsigned long _sysret(unsigned long);
|
||||
void *__sys_mmap(void *, u64, u32, u32, i64, i64, i64);
|
||||
void *__sys_mremap(void *, u64, u64, i32, void *);
|
||||
void *sys_mremap(void *, u64, u64, i32, void *);
|
||||
void *sys_mremap(void *, u64, u64, u64, u64);
|
||||
void sys_exit(i32);
|
||||
|
||||
#undef i32
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue