mirror of
https://github.com/jart/cosmopolitan.git
synced 2025-06-01 18:22:27 +00:00
Make realloc() go 100x faster on Linux/NetBSD
Cosmopolitan now supports mremap(), which is only supported on Linux and NetBSD. First, it allows memory mappings to be relocated without copying them; this can dramatically speed up data structures like std::vector if the array size grows larger than 256kb. The mremap() system call is also 10x faster than munmap() when shrinking large memory mappings. There's now two functions, getpagesize() and getgransize() which help to write portable code that uses mmap(MAP_FIXED). Alternative sysconf() may be called with our new _SC_GRANSIZE. The madvise() system call now has a better wrapper with improved documentation.
This commit is contained in:
Justine Tunney
parent
196942084b
commit
f7780de24b
71 changed files with 1301 additions and 640 deletions
|
|
@ -19,6 +19,7 @@
|
|||
#include "libc/calls/calls.h"
|
||||
#include "libc/dce.h"
|
||||
#include "libc/errno.h"
|
||||
#include "libc/intrin/maps.h"
|
||||
#include "libc/log/log.h"
|
||||
#include "libc/runtime/runtime.h"
|
||||
#include "libc/sysv/consts/map.h"
|
||||
|
|
@ -26,116 +27,116 @@
|
|||
#include "libc/sysv/consts/prot.h"
|
||||
#include "libc/testlib/testlib.h"
|
||||
|
||||
int granularity;
|
||||
int gransz;
|
||||
|
||||
void SetUpOnce(void) {
|
||||
granularity = __granularity();
|
||||
gransz = getgransize();
|
||||
testlib_enable_tmp_setup_teardown();
|
||||
}
|
||||
|
||||
TEST(munmap, doesntExist_doesntCare) {
|
||||
EXPECT_SYS(0, 0, munmap(0, granularity * 8));
|
||||
EXPECT_SYS(0, 0, munmap(0, gransz * 8));
|
||||
}
|
||||
|
||||
TEST(munmap, invalidParams) {
|
||||
EXPECT_SYS(EINVAL, -1, munmap(0, 0));
|
||||
EXPECT_SYS(EINVAL, -1, munmap((void *)0x100080000000, 0));
|
||||
EXPECT_SYS(EINVAL, -1, munmap((void *)0x100080000001, granularity));
|
||||
EXPECT_SYS(EINVAL, -1, munmap((void *)0x100080000001, gransz));
|
||||
}
|
||||
|
||||
TEST(munmap, test) {
|
||||
char *p;
|
||||
ASSERT_NE(MAP_FAILED, (p = mmap(0, granularity, PROT_READ | PROT_WRITE,
|
||||
ASSERT_NE(MAP_FAILED, (p = mmap(0, gransz, PROT_READ | PROT_WRITE,
|
||||
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)));
|
||||
EXPECT_TRUE(testlib_memoryexists(p));
|
||||
EXPECT_SYS(0, 0, munmap(p, granularity));
|
||||
EXPECT_SYS(0, 0, munmap(p, gransz));
|
||||
EXPECT_FALSE(testlib_memoryexists(p));
|
||||
}
|
||||
|
||||
TEST(munmap, punchHoleInMemory) {
|
||||
char *p;
|
||||
ASSERT_NE(MAP_FAILED, (p = mmap(0, granularity * 3, PROT_READ | PROT_WRITE,
|
||||
ASSERT_NE(MAP_FAILED, (p = mmap(0, gransz * 3, PROT_READ | PROT_WRITE,
|
||||
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + granularity * 0));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + granularity * 1));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + granularity * 2));
|
||||
EXPECT_SYS(0, 0, munmap(p + granularity, granularity));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + granularity * 0));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 1));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + granularity * 2));
|
||||
EXPECT_SYS(0, 0, munmap(p, granularity));
|
||||
EXPECT_SYS(0, 0, munmap(p + granularity * 2, granularity));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 0));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 1));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 2));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + gransz * 0));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + gransz * 1));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + gransz * 2));
|
||||
EXPECT_SYS(0, 0, munmap(p + gransz, gransz));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + gransz * 0));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 1));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + gransz * 2));
|
||||
EXPECT_SYS(0, 0, munmap(p, gransz));
|
||||
EXPECT_SYS(0, 0, munmap(p + gransz * 2, gransz));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 0));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 1));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 2));
|
||||
}
|
||||
|
||||
TEST(munmap, memoryHasHole) {
|
||||
char *p;
|
||||
ASSERT_NE(MAP_FAILED, (p = mmap(0, granularity * 3, PROT_READ | PROT_WRITE,
|
||||
ASSERT_NE(MAP_FAILED, (p = mmap(0, gransz * 3, PROT_READ | PROT_WRITE,
|
||||
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)));
|
||||
EXPECT_SYS(0, 0, munmap(p + granularity, granularity));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + granularity * 0));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 1));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + granularity * 2));
|
||||
EXPECT_SYS(0, 0, munmap(p, granularity * 3));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 0));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 1));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 2));
|
||||
EXPECT_SYS(0, 0, munmap(p + gransz, gransz));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + gransz * 0));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 1));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + gransz * 2));
|
||||
EXPECT_SYS(0, 0, munmap(p, gransz * 3));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 0));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 1));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 2));
|
||||
}
|
||||
|
||||
TEST(munmap, blanketFree) {
|
||||
char *p;
|
||||
ASSERT_NE(MAP_FAILED, (p = mmap(0, granularity * 3, PROT_READ | PROT_WRITE,
|
||||
ASSERT_NE(MAP_FAILED, (p = mmap(0, gransz * 3, PROT_READ | PROT_WRITE,
|
||||
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + granularity * 0));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + granularity * 1));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + granularity * 2));
|
||||
EXPECT_SYS(0, 0, munmap(p + granularity * 0, granularity));
|
||||
EXPECT_SYS(0, 0, munmap(p + granularity * 2, granularity));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 0));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + granularity * 1));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 2));
|
||||
EXPECT_SYS(0, 0, munmap(p, granularity * 3));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 0));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 1));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 2));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + gransz * 0));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + gransz * 1));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + gransz * 2));
|
||||
EXPECT_SYS(0, 0, munmap(p + gransz * 0, gransz));
|
||||
EXPECT_SYS(0, 0, munmap(p + gransz * 2, gransz));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 0));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + gransz * 1));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 2));
|
||||
EXPECT_SYS(0, 0, munmap(p, gransz * 3));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 0));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 1));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 2));
|
||||
}
|
||||
|
||||
TEST(munmap, trimLeft) {
|
||||
char *p;
|
||||
ASSERT_NE(MAP_FAILED, (p = mmap(0, granularity * 2, PROT_READ | PROT_WRITE,
|
||||
ASSERT_NE(MAP_FAILED, (p = mmap(0, gransz * 2, PROT_READ | PROT_WRITE,
|
||||
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + granularity * 0));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + granularity * 1));
|
||||
EXPECT_SYS(0, 0, munmap(p, granularity));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 0));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + granularity * 1));
|
||||
EXPECT_SYS(0, 0, munmap(p, granularity * 2));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 0));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 1));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + gransz * 0));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + gransz * 1));
|
||||
EXPECT_SYS(0, 0, munmap(p, gransz));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 0));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + gransz * 1));
|
||||
EXPECT_SYS(0, 0, munmap(p, gransz * 2));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 0));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 1));
|
||||
}
|
||||
|
||||
TEST(munmap, trimRight) {
|
||||
char *p;
|
||||
ASSERT_NE(MAP_FAILED, (p = mmap(0, granularity * 2, PROT_READ | PROT_WRITE,
|
||||
ASSERT_NE(MAP_FAILED, (p = mmap(0, gransz * 2, PROT_READ | PROT_WRITE,
|
||||
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + granularity * 0));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + granularity * 1));
|
||||
EXPECT_SYS(0, 0, munmap(p + granularity, granularity));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + granularity * 0));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 1));
|
||||
EXPECT_SYS(0, 0, munmap(p, granularity * 2));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 0));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + granularity * 1));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + gransz * 0));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + gransz * 1));
|
||||
EXPECT_SYS(0, 0, munmap(p + gransz, gransz));
|
||||
EXPECT_TRUE(testlib_memoryexists(p + gransz * 0));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 1));
|
||||
EXPECT_SYS(0, 0, munmap(p, gransz * 2));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 0));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + gransz * 1));
|
||||
}
|
||||
|
||||
TEST(munmap, memoryGone) {
|
||||
char *p;
|
||||
ASSERT_NE(MAP_FAILED, (p = mmap(0, granularity, PROT_READ | PROT_WRITE,
|
||||
ASSERT_NE(MAP_FAILED, (p = mmap(0, gransz, PROT_READ | PROT_WRITE,
|
||||
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)));
|
||||
EXPECT_SYS(0, 0, munmap(p, granularity));
|
||||
EXPECT_SYS(0, 0, munmap(p, granularity));
|
||||
EXPECT_SYS(0, 0, munmap(p, gransz));
|
||||
EXPECT_SYS(0, 0, munmap(p, gransz));
|
||||
}
|
||||
|
||||
TEST(munmap, tinyFile_roundupUnmapSize) {
|
||||
|
|
@ -147,7 +148,7 @@ TEST(munmap, tinyFile_roundupUnmapSize) {
|
|||
ASSERT_NE(MAP_FAILED, (p = mmap(0, 5, PROT_READ, MAP_PRIVATE, 3, 0)));
|
||||
ASSERT_SYS(0, 0, close(3));
|
||||
EXPECT_TRUE(testlib_memoryexists(p));
|
||||
EXPECT_SYS(0, 0, munmap(p, granularity));
|
||||
EXPECT_SYS(0, 0, munmap(p, gransz));
|
||||
EXPECT_FALSE(testlib_memoryexists(p));
|
||||
EXPECT_FALSE(testlib_memoryexists(p + 5));
|
||||
}
|
||||
|
|
@ -171,22 +172,22 @@ TEST(munmap, tinyFile_preciseUnmapSize) {
|
|||
}
|
||||
|
||||
// clang-format off
|
||||
/* TEST(munmap, tinyFile_mapThriceUnmapOnce) { */
|
||||
/* char *p = (char *)0x000063d646e20000; */
|
||||
/* ASSERT_SYS(0, 3, open("doge", O_RDWR | O_CREAT | O_TRUNC, 0644)); */
|
||||
/* ASSERT_SYS (0, 5, write(3, "hello", 5)); */
|
||||
/* ASSERT_EQ(p+granularity*0, mmap(p+granularity*0, granularity, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0)); */
|
||||
/* ASSERT_EQ(p+granularity*1, mmap(p+granularity*1, 5, PROT_READ, MAP_PRIVATE|MAP_FIXED, 3, 0)); */
|
||||
/* ASSERT_EQ(p+granularity*3, mmap(p+granularity*3, 5, PROT_READ, MAP_PRIVATE|MAP_FIXED, 3, 0)); */
|
||||
/* ASSERT_SYS(0, 0, close(3)); */
|
||||
/* EXPECT_TRUE(testlib_memoryexists(p+granularity*0)); */
|
||||
/* EXPECT_TRUE(testlib_memoryexists(p+granularity*1)); */
|
||||
/* EXPECT_FALSE(testlib_memoryexists(p+granularity*2)); */
|
||||
/* EXPECT_TRUE(testlib_memoryexists(p+granularity*3)); */
|
||||
/* EXPECT_SYS(0, 0, munmap(p, granularity*5)); */
|
||||
/* EXPECT_FALSE(testlib_memoryexists(p+granularity*0)); */
|
||||
/* EXPECT_FALSE(testlib_memoryexists(p+granularity*1)); */
|
||||
/* EXPECT_FALSE(testlib_memoryexists(p+granularity*2)); */
|
||||
/* EXPECT_FALSE(testlib_memoryexists(p+granularity*3)); */
|
||||
/* } */
|
||||
TEST(munmap, tinyFile_mapThriceUnmapOnce) {
|
||||
char *p = randaddr();
|
||||
ASSERT_SYS(0, 3, open("doge", O_RDWR | O_CREAT | O_TRUNC, 0644));
|
||||
ASSERT_SYS (0, 5, write(3, "hello", 5));
|
||||
ASSERT_EQ(p+gransz*0, mmap(p+gransz*0, gransz, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0));
|
||||
ASSERT_EQ(p+gransz*1, mmap(p+gransz*1, 5, PROT_READ, MAP_PRIVATE|MAP_FIXED, 3, 0));
|
||||
ASSERT_EQ(p+gransz*3, mmap(p+gransz*3, 5, PROT_READ, MAP_PRIVATE|MAP_FIXED, 3, 0));
|
||||
ASSERT_SYS(0, 0, close(3));
|
||||
EXPECT_TRUE(testlib_memoryexists(p+gransz*0));
|
||||
EXPECT_TRUE(testlib_memoryexists(p+gransz*1));
|
||||
EXPECT_FALSE(testlib_memoryexists(p+gransz*2));
|
||||
EXPECT_TRUE(testlib_memoryexists(p+gransz*3));
|
||||
EXPECT_SYS(0, 0, munmap(p, gransz*5));
|
||||
EXPECT_FALSE(testlib_memoryexists(p+gransz*0));
|
||||
EXPECT_FALSE(testlib_memoryexists(p+gransz*1));
|
||||
EXPECT_FALSE(testlib_memoryexists(p+gransz*2));
|
||||
EXPECT_FALSE(testlib_memoryexists(p+gransz*3));
|
||||
}
|
||||
// clang-format on
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue