mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-07-06 11:18:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Rewrite memory manager

Browse source 
Actually Portable Executable now supports Android. Cosmo's old mmap code
required a 47 bit address space. The new implementation is very agnostic
and supports both smaller address spaces (e.g. embedded) and even modern
56-bit PML5T paging for x86 which finally came true on Zen4 Threadripper

Cosmopolitan no longer requires UNIX systems to observe the Windows 64kb
granularity; i.e. sysconf(_SC_PAGE_SIZE) will now report the host native
page size. This fixes a longstanding POSIX conformance issue, concerning
file mappings that overlap the end of file. Other aspects of conformance
have been improved too, such as the subtleties of address assignment and
and the various subtleties surrounding MAP_FIXED and MAP_FIXED_NOREPLACE

On Windows, mappings larger than 100 megabytes won't be broken down into
thousands of independent 64kb mappings. Support for MAP_STACK is removed
by this change; please use NewCosmoStack() instead.

Stack overflow avoidance is now being implemented using the POSIX thread
APIs. Please use GetStackBottom() and GetStackAddr(), instead of the old
error-prone GetStackAddr() and HaveStackMemory() APIs which are removed.
This commit is contained in:
Justine Tunney 2024-06-20 20:46:42 -07:00
parent 7f6d0b8709
commit 6ffed14b9c
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
150 changed files with 1893 additions and 5634 deletions
Download patch file Download diff file Expand all files Collapse all files

32
libc/thread/getstackbottom.c Normal file
View file

@ -0,0 +1,32 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/thread/thread.h"
uintptr_t GetStackBottom(void) {
char *bottom;
void *stackaddr;
pthread_attr_t tattr;
size_t stacksize, guardsize;
pthread_getattr_np(pthread_self(), &tattr);
pthread_attr_getstack(&tattr, &stackaddr, &stacksize);
pthread_attr_getguardsize(&tattr, &guardsize);
bottom = stackaddr;
bottom += guardsize;
return (uintptr_t)bottom;
}

2
libc/thread/openbsd.internal.h
View file

@ -4,7 +4,7 @@ COSMOPOLITAN_C_START_
struct __tfork {
void *tf_tcb;
int32_t *tf_tid;
_Atomic(int) *tf_tid;
void *tf_stack;
};

20
libc/thread/pthread_atfork.c
View file

@ -16,7 +16,6 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/atomic.h"
#include "libc/calls/state.internal.h"
#include "libc/cosmo.h"
@ -25,8 +24,8 @@
#include "libc/intrin/atomic.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/leaky.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/macros.internal.h"
#include "libc/mem/mem.h"
#include "libc/proc/proc.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
@ -42,18 +41,19 @@ struct AtFork {
static struct AtForks {
pthread_spinlock_t lock;
struct AtFork *list;
struct AtFork pool[8];
struct AtFork pool[64];
atomic_int allocated;
} _atforks;
static void _pthread_onfork(int i) {
static void _pthread_onfork(int i, const char *op) {
struct AtFork *a;
unassert(0 <= i && i <= 2);
if (!i)
pthread_spin_lock(&_atforks.lock);
for (a = _atforks.list; a; a = a->p[!i]) {
if (a->f[i])
if (a->f[i]) {
STRACE("pthread_atfork(%s, %t)", op, a->f[i]);
a->f[i]();
}
_atforks.list = a;
}
if (i)
@ -61,15 +61,15 @@ static void _pthread_onfork(int i) {
}
void _pthread_onfork_prepare(void) {
_pthread_onfork(0);
_pthread_onfork(0, "prepare");
}
void _pthread_onfork_parent(void) {
_pthread_onfork(1);
_pthread_onfork(1, "parent");
}
void _pthread_onfork_child(void) {
_pthread_onfork(2);
_pthread_onfork(2, "child");
}
static struct AtFork *_pthread_atfork_alloc(void) {
@ -78,7 +78,7 @@ static struct AtFork *_pthread_atfork_alloc(void) {
(i = atomic_fetch_add(&_atforks.allocated, 1)) < n) {
return _atforks.pool + i;
} else {
return malloc(sizeof(struct AtFork));
return 0;
}
}

76
libc/thread/pthread_create.c
View file

@ -28,6 +28,7 @@
#include "libc/intrin/bsr.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/log/internal.h"
@ -64,10 +65,6 @@ __static_yoink("_pthread_onfork_prepare");
__static_yoink("_pthread_onfork_parent");
__static_yoink("_pthread_onfork_child");
/* #ifndef MODE_DBG */
/* __static_yoink("threaded_dlmalloc"); */
/* #endif */
#define MAP_ANON_OPENBSD 0x1000
#define MAP_STACK_OPENBSD 0x4000
@ -76,7 +73,8 @@ void _pthread_free(struct PosixThread *pt, bool isfork) {
if (pt->pt_flags & PT_STATIC)
return;
if (pt->pt_flags & PT_OWNSTACK) {
unassert(!munmap(pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize));
unassert(!munmap(pt->pt_attr.__stackaddr,
pt->pt_attr.__stacksize + (uintptr_t)ape_stack_align));
}
if (!isfork) {
uint64_t syshand =
@ -149,7 +147,7 @@ static int FixupCustomStackOnOpenbsd(pthread_attr_t *attr) {
uintptr_t x, y;
int e, rc, pagesz;
pagesz = getauxval(AT_PAGESZ);
n = attr->__stacksize;
n = attr->__stacksize + (uintptr_t)ape_stack_align;
x = (uintptr_t)attr->__stackaddr;
y = ROUNDUP(x, pagesz);
n -= y - x;
@ -159,7 +157,7 @@ static int FixupCustomStackOnOpenbsd(pthread_attr_t *attr) {
MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD,
-1, 0, 0) == (void *)y) {
attr->__stackaddr = (void *)y;
attr->__stacksize = n;
attr->__stacksize = n - (uintptr_t)ape_stack_align;
return 0;
} else {
rc = errno;
@ -214,48 +212,37 @@ static errno_t pthread_create_impl(pthread_t *thread,
}
} else {
// cosmo is managing the stack
// 1. in mono repo optimize for tiniest stack possible
// 2. in public world optimize to *work* regardless of memory
int granularity = FRAMESIZE;
int pagesize = getauxval(AT_PAGESZ);
pt->pt_attr.__guardsize = ROUNDUP(pt->pt_attr.__guardsize, pagesize);
pt->pt_attr.__stacksize = ROUNDUP(pt->pt_attr.__stacksize, granularity);
pt->pt_attr.__stacksize = pt->pt_attr.__stacksize;
if (pt->pt_attr.__guardsize + pagesize > pt->pt_attr.__stacksize) {
_pthread_free(pt, false);
return EINVAL;
}
if (pt->pt_attr.__guardsize == pagesize &&
!(IsAarch64() && IsLinux() && IsQemuUser())) {
// MAP_GROWSDOWN doesn't work very well on qemu-aarch64
pt->pt_attr.__stackaddr =
mmap(0, pt->pt_attr.__stacksize, PROT_READ | PROT_WRITE,
MAP_STACK | MAP_ANONYMOUS, -1, 0);
} else {
pt->pt_attr.__stackaddr =
mmap(0, pt->pt_attr.__stacksize, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (pt->pt_attr.__stackaddr != MAP_FAILED) {
if (IsOpenbsd() &&
__sys_mmap(
pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD,
-1, 0, 0) != pt->pt_attr.__stackaddr) {
notpossible;
}
if (pt->pt_attr.__guardsize) {
if (!IsWindows()) {
if (mprotect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize,
PROT_NONE)) {
notpossible;
}
} else {
uint32_t oldattr;
if (!VirtualProtect(pt->pt_attr.__stackaddr,
pt->pt_attr.__guardsize,
kNtPageReadwrite | kNtPageGuard, &oldattr)) {
notpossible;
}
pt->pt_attr.__stackaddr =
mmap(0, pt->pt_attr.__stacksize + (uintptr_t)ape_stack_align,
PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (pt->pt_attr.__stackaddr != MAP_FAILED) {
if (IsOpenbsd() &&
__sys_mmap(
pt->pt_attr.__stackaddr,
pt->pt_attr.__stacksize + (uintptr_t)ape_stack_align,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD,
-1, 0, 0) != pt->pt_attr.__stackaddr) {
notpossible;
}
if (pt->pt_attr.__guardsize) {
if (!IsWindows()) {
if (mprotect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize,
PROT_NONE)) {
notpossible;
}
} else {
uint32_t oldattr;
if (!VirtualProtect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize,
kNtPageReadwrite | kNtPageGuard, &oldattr)) {
notpossible;
}
}
}
@ -305,8 +292,7 @@ static errno_t pthread_create_impl(pthread_t *thread,
_pthread_unlock();
// launch PosixThread(pt) in new thread
if ((rc = clone(PosixThread, pt->pt_attr.__stackaddr,
pt->pt_attr.__stacksize - (IsOpenbsd() ? 16 : 0),
if ((rc = clone(PosixThread, pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize,
CLONE_VM | CLONE_THREAD | CLONE_FS | CLONE_FILES |
CLONE_SIGHAND | CLONE_SYSVSEM | CLONE_SETTLS |
CLONE_PARENT_SETTID | CLONE_CHILD_SETTID |

7
libc/thread/pthread_getattr_np.c
View file

@ -20,10 +20,10 @@
#include "libc/calls/struct/rlimit.h"
#include "libc/dce.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/maps.h"
#include "libc/limits.h"
#include "libc/macros.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/rlim.h"
@ -73,8 +73,9 @@ errno_t pthread_getattr_np(pthread_t thread, pthread_attr_t *attr) {
__builtin_unreachable();
}
if (!attr->__stacksize && (pt->pt_flags & PT_STATIC)) {
__get_main_stack(&attr->__stackaddr, &attr->__stacksize,
&attr->__guardsize);
attr->__stackaddr = __maps.stack.addr;
attr->__stacksize = __maps.stack.size;
attr->__guardsize = 0;
}
return 0;
}

1
libc/thread/tls.h
View file

@ -4,6 +4,7 @@
#define TLS_ALIGNMENT 64
#define TIB_FLAG_VFORKED 1
#define TIB_FLAG_MAPLOCK 2
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_