mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-07-01 00:38:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Make realloc() go 100x faster on Linux/NetBSD

Browse source 
Cosmopolitan now supports mremap(), which is only supported on Linux and
NetBSD. First, it allows memory mappings to be relocated without copying
them; this can dramatically speed up data structures like std::vector if
the array size grows larger than 256kb. The mremap() system call is also
10x faster than munmap() when shrinking large memory mappings.

There's now two functions, getpagesize() and getgransize() which help to
write portable code that uses mmap(MAP_FIXED). Alternative sysconf() may
be called with our new _SC_GRANSIZE. The madvise() system call now has a
better wrapper with improved documentation.
This commit is contained in:
Justine Tunney 2024-07-07 12:24:25 -07:00
parent 196942084b
commit f7780de24b
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
71 changed files with 1301 additions and 640 deletions
Download patch file Download diff file Expand all files Collapse all files

89
test/libc/calls/madvise_test.c
View file

@ -33,63 +33,100 @@ void SetUpOnce(void) {
TEST(madvise, anon) {
char *p;
ASSERT_NE(MAP_FAILED, (p = mmap(0, __granularity(), PROT_READ | PROT_WRITE,
ASSERT_NE(MAP_FAILED, (p = mmap(0, getpagesize(), PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)));
ASSERT_SYS(0, 0, madvise(p, __granularity(), MADV_WILLNEED));
ASSERT_SYS(0, 0, munmap(p, __granularity()));
ASSERT_SYS(0, 0, madvise(p, getpagesize(), MADV_WILLNEED));
ASSERT_SYS(0, 0, munmap(p, getpagesize()));
}
TEST(madvise, file) {
char *p;
ASSERT_SYS(0, 3, creat("foo.dat", 0644));
ASSERT_SYS(0, 0, ftruncate(3, __granularity()));
ASSERT_SYS(0, 0, ftruncate(3, getpagesize()));
ASSERT_SYS(0, 0, close(3));
ASSERT_SYS(0, 3, open("foo.dat", O_RDWR));
ASSERT_NE(MAP_FAILED, (p = mmap(0, __granularity(), PROT_READ | PROT_WRITE,
ASSERT_NE(MAP_FAILED, (p = mmap(0, getpagesize(), PROT_READ | PROT_WRITE,
MAP_PRIVATE, 3, 0)));
ASSERT_SYS(0, 0, madvise(p, __granularity(), MADV_WILLNEED));
ASSERT_SYS(0, 0, munmap(p, __granularity()));
ASSERT_SYS(0, 0, madvise(p, getpagesize(), MADV_WILLNEED));
ASSERT_SYS(0, 0, munmap(p, getpagesize()));
ASSERT_SYS(0, 0, close(3));
}
TEST(madvise, short) {
char *p;
ASSERT_NE(MAP_FAILED, (p = mmap(0, __granularity(), PROT_READ | PROT_WRITE,
ASSERT_NE(MAP_FAILED, (p = mmap(0, getpagesize(), PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)));
ASSERT_SYS(0, 0, madvise(p, __granularity() - 1, MADV_WILLNEED));
ASSERT_SYS(0, 0, munmap(p, __granularity()));
ASSERT_SYS(0, 0, madvise(p, getpagesize() - 1, MADV_WILLNEED));
ASSERT_SYS(0, 0, munmap(p, getpagesize()));
}
TEST(madvise, zero) {
char *p;
ASSERT_NE(MAP_FAILED, (p = mmap(0, getpagesize(), PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)));
ASSERT_SYS(0, 0, madvise(p, 0, MADV_WILLNEED));
ASSERT_SYS(0, 0, madvise(0, 0, MADV_WILLNEED));
ASSERT_SYS(EINVAL, -1, madvise(p, 0, 666));
ASSERT_SYS(EINVAL, -1, madvise(0, 0, 666));
ASSERT_SYS(0, 0, munmap(p, getpagesize()));
}
TEST(madvise, subPages) {
char *p;
ASSERT_NE(MAP_FAILED, (p = mmap(0, __granularity(), PROT_READ | PROT_WRITE,
ASSERT_NE(MAP_FAILED, (p = mmap(0, getpagesize() * 2, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)));
ASSERT_SYS(0, 0,
madvise(p + getpagesize(), __granularity() - getpagesize(),
MADV_WILLNEED));
ASSERT_SYS(0, 0, munmap(p, __granularity()));
ASSERT_SYS(0, 0, madvise(p + getpagesize(), getpagesize(), MADV_WILLNEED));
ASSERT_SYS(0, 0, munmap(p, getpagesize() * 2));
}
TEST(madvise, madvWillNeed_unmappedRegion) {
char *p;
ASSERT_NE(MAP_FAILED, (p = mmap(0, getgransize() * 3, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)));
ASSERT_SYS(0, 0, munmap(p + getgransize(), getgransize()));
if (IsXnu())
ASSERT_SYS(EINVAL, -1, madvise(p, getgransize() * 3, MADV_WILLNEED));
else if (IsBsd() || IsWindows())
ASSERT_SYS(0, 0, madvise(p, getgransize() * 3, MADV_WILLNEED));
else
ASSERT_SYS(ENOMEM, -1, madvise(p, getgransize() * 3, MADV_WILLNEED));
ASSERT_SYS(0, 0, munmap(p, getgransize() * 3));
}
TEST(madvise, madvFree_unmappedRegion) {
char *p;
ASSERT_NE(MAP_FAILED, (p = mmap(0, getgransize() * 3, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)));
ASSERT_SYS(0, 0, munmap(p + getgransize(), getgransize()));
if (IsXnu())
ASSERT_SYS(EINVAL, -1, madvise(p, getgransize() * 3, MADV_FREE));
else if (IsNetbsd() || IsOpenbsd())
ASSERT_SYS(EFAULT, -1, madvise(p, getgransize() * 3, MADV_FREE));
else if (IsFreebsd() || IsWindows())
ASSERT_SYS(0, 0, madvise(p, getgransize() * 3, MADV_FREE));
else
ASSERT_SYS(ENOMEM, -1, madvise(p, getgransize() * 3, MADV_FREE));
ASSERT_SYS(0, 0, munmap(p, getgransize() * 3));
}
TEST(madvise, misalign) {
char *p;
if (!IsLinux())
return; // most platforms don't care
if (IsQemuUser())
return; // qemu claims to be linux but doesn't care
ASSERT_NE(MAP_FAILED, (p = mmap(0, __granularity(), PROT_READ | PROT_WRITE,
ASSERT_NE(MAP_FAILED, (p = mmap(0, getpagesize(), PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)));
ASSERT_SYS(EINVAL, -1, madvise(p + 1, __granularity() - 1, MADV_WILLNEED));
ASSERT_SYS(0, 0, munmap(p, __granularity()));
ASSERT_SYS(EINVAL, -1, madvise(p + 1, getpagesize(), MADV_WILLNEED));
ASSERT_SYS(0, 0, madvise(p, 1, MADV_WILLNEED));
ASSERT_SYS(EINVAL, -1, madvise(p, -1, MADV_WILLNEED));
ASSERT_SYS(0, 0, munmap(p, getpagesize()));
}
TEST(madvise, badAdvice) {
char *p;
if (IsAarch64() && IsQemuUser())
return; // qemu doesn't validate advice
ASSERT_NE(MAP_FAILED, (p = mmap(0, __granularity(), PROT_READ | PROT_WRITE,
ASSERT_NE(MAP_FAILED, (p = mmap(0, getpagesize(), PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)));
ASSERT_SYS(EINVAL, -1, madvise(p, __granularity(), 127));
ASSERT_SYS(0, 0, munmap(p, __granularity()));
ASSERT_SYS(EINVAL, -1, madvise(p, getpagesize(), 127));
ASSERT_SYS(0, 0, munmap(p, getpagesize()));
}
TEST(madvise, missingMemory) {
@ -98,5 +135,5 @@ TEST(madvise, missingMemory) {
if (IsQemuUser())
return; // qemu claims to be linux but doesn't care
ASSERT_SYS(ENOMEM, -1,
madvise((char *)0x83483838000, __granularity(), MADV_WILLNEED));
madvise((char *)0x83483838000, getpagesize(), MADV_WILLNEED));
}

24
test/libc/calls/pledge_test.c
View file

@ -86,7 +86,7 @@ TEST(pledge, default_allowsExit) {
int *job;
int ws, pid;
// create small shared memory region
ASSERT_NE(-1, (job = mmap(0, __granularity(), PROT_READ | PROT_WRITE,
ASSERT_NE(-1, (job = mmap(0, getpagesize(), PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0)));
job[0] = 2; // create workload
job[1] = 2;
@ -100,7 +100,7 @@ TEST(pledge, default_allowsExit) {
EXPECT_TRUE(WIFEXITED(ws));
EXPECT_EQ(0, WEXITSTATUS(ws));
EXPECT_EQ(4, job[0]); // check result
EXPECT_SYS(0, 0, munmap(job, __granularity()));
EXPECT_SYS(0, 0, munmap(job, getpagesize()));
}
TEST(pledge, execpromises_notok) {
@ -298,7 +298,7 @@ TEST(pledge, stdioTty_sendtoRestricted_requiresNullAddr) {
errno = 0;
// set lower 32-bit word of pointer to zero lool
struct sockaddr_in *sin =
mmap((void *)0x300000000000, __granularity(), PROT_READ | PROT_WRITE,
mmap((void *)0x300000000000, getpagesize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
sin->sin_family = AF_INET;
ASSERT_SYS(
@ -334,7 +334,7 @@ TEST(pledge, unix_forbidsInetSockets) {
TEST(pledge, wpath_doesNotImplyRpath) {
int ws, pid;
bool *gotsome;
ASSERT_NE(-1, (gotsome = _mapshared(__granularity())));
ASSERT_NE(-1, (gotsome = _mapshared(getpagesize())));
ASSERT_SYS(0, 0, touch("foo", 0644));
ASSERT_NE(-1, (pid = fork()));
if (!pid) {
@ -412,13 +412,13 @@ TEST(pledge, mmap) {
ASSERT_NE(-1, (pid = fork()));
if (!pid) {
ASSERT_SYS(0, 0, pledge("stdio", 0));
ASSERT_NE(MAP_FAILED, (p = mmap(0, __granularity(), PROT_READ | PROT_WRITE,
ASSERT_NE(MAP_FAILED, (p = mmap(0, getpagesize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)));
ASSERT_SYS(0, 0, mprotect(p, __granularity(), PROT_READ));
ASSERT_SYS(0, 0, mprotect(p, getpagesize(), PROT_READ));
ASSERT_SYS(EPERM, MAP_FAILED,
mprotect(p, __granularity(), PROT_READ | PROT_EXEC));
mprotect(p, getpagesize(), PROT_READ | PROT_EXEC));
ASSERT_SYS(EPERM, MAP_FAILED,
mmap(0, __granularity(), PROT_EXEC | PROT_READ,
mmap(0, getpagesize(), PROT_EXEC | PROT_READ,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0));
_Exit(0);
}
@ -434,11 +434,11 @@ TEST(pledge, mmapProtExec) {
ASSERT_NE(-1, (pid = fork()));
if (!pid) {
ASSERT_SYS(0, 0, pledge("stdio prot_exec", 0));
ASSERT_NE(MAP_FAILED, (p = mmap(0, __granularity(), PROT_READ | PROT_WRITE,
ASSERT_NE(MAP_FAILED, (p = mmap(0, getpagesize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)));
ASSERT_SYS(0, 0, mprotect(p, __granularity(), PROT_READ));
ASSERT_SYS(0, 0, mprotect(p, __granularity(), PROT_READ | PROT_EXEC));
ASSERT_NE(MAP_FAILED, mmap(0, __granularity(), PROT_EXEC | PROT_READ,
ASSERT_SYS(0, 0, mprotect(p, getpagesize(), PROT_READ));
ASSERT_SYS(0, 0, mprotect(p, getpagesize(), PROT_READ | PROT_EXEC));
ASSERT_NE(MAP_FAILED, mmap(0, getpagesize(), PROT_EXEC | PROT_READ,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0));
_Exit(0);
}

12
test/libc/calls/setrlimit_test.c
View file

@ -159,15 +159,15 @@ TEST(setrlimit, testVirtualMemoryLimit) {
ASSERT_NE(-1, (wstatus = xspawn(0)));
if (wstatus == -2) {
ASSERT_EQ(0, setrlimit(RLIMIT_AS, &(struct rlimit){MEM, MEM}));
for (i = 0; i < (MEM * 2) / __granularity(); ++i) {
p = sys_mmap(0, __granularity(), PROT_READ | PROT_WRITE,
for (i = 0; i < (MEM * 2) / getpagesize(); ++i) {
p = sys_mmap(0, getpagesize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE | MAP_POPULATE, -1, 0)
.addr;
if (p == MAP_FAILED) {
ASSERT_EQ(ENOMEM, errno);
_Exit(0);
}
rngset(p, __granularity(), _rand64, -1);
rngset(p, getpagesize(), _rand64, -1);
}
_Exit(1);
}
@ -193,15 +193,15 @@ TEST(setrlimit, testDataMemoryLimit) {
ASSERT_NE(-1, (wstatus = xspawn(0)));
if (wstatus == -2) {
ASSERT_EQ(0, setrlimit(RLIMIT_DATA, &(struct rlimit){MEM, MEM}));
for (i = 0; i < (MEM * 2) / __granularity(); ++i) {
p = sys_mmap(0, __granularity(), PROT_READ | PROT_WRITE,
for (i = 0; i < (MEM * 2) / getpagesize(); ++i) {
p = sys_mmap(0, getpagesize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE | MAP_POPULATE, -1, 0)
.addr;
if (p == MAP_FAILED) {
ASSERT_EQ(ENOMEM, errno);
_Exit(0);
}
rngset(p, __granularity(), _rand64, -1);
rngset(p, getpagesize(), _rand64, -1);
}
_Exit(1);
}

4
test/libc/calls/unveil_test.c
View file

@ -335,7 +335,7 @@ TEST(unveil, isThreadSpecificOnLinux_isProcessWideOnOpenbsd) {
TEST(unveil, usedTwice_forbidden_worksWithPledge) {
int ws, pid;
bool *gotsome;
ASSERT_NE(-1, (gotsome = _mapshared(__granularity())));
ASSERT_NE(-1, (gotsome = _mapshared(getpagesize())));
ASSERT_NE(-1, (pid = fork()));
if (!pid) {
__pledge_mode = PLEDGE_PENALTY_KILL_PROCESS;
@ -359,7 +359,7 @@ TEST(unveil, usedTwice_forbidden_worksWithPledge) {
ASSERT_TRUE(*gotsome);
ASSERT_TRUE(WIFSIGNALED(ws));
ASSERT_EQ(IsOpenbsd() ? SIGABRT : SIGSYS, WTERMSIG(ws));
EXPECT_SYS(0, 0, munmap(gotsome, __granularity()));
EXPECT_SYS(0, 0, munmap(gotsome, getpagesize()));
}
TEST(unveil, lotsOfPaths) {