mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-07-04 18:28:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Rewrite memory manager

Browse source 
Actually Portable Executable now supports Android. Cosmo's old mmap code
required a 47 bit address space. The new implementation is very agnostic
and supports both smaller address spaces (e.g. embedded) and even modern
56-bit PML5T paging for x86 which finally came true on Zen4 Threadripper

Cosmopolitan no longer requires UNIX systems to observe the Windows 64kb
granularity; i.e. sysconf(_SC_PAGE_SIZE) will now report the host native
page size. This fixes a longstanding POSIX conformance issue, concerning
file mappings that overlap the end of file. Other aspects of conformance
have been improved too, such as the subtleties of address assignment and
and the various subtleties surrounding MAP_FIXED and MAP_FIXED_NOREPLACE

On Windows, mappings larger than 100 megabytes won't be broken down into
thousands of independent 64kb mappings. Support for MAP_STACK is removed
by this change; please use NewCosmoStack() instead.

Stack overflow avoidance is now being implemented using the POSIX thread
APIs. Please use GetStackBottom() and GetStackAddr(), instead of the old
error-prone GetStackAddr() and HaveStackMemory() APIs which are removed.
This commit is contained in:
Justine Tunney 2024-06-20 20:46:42 -07:00
parent 7f6d0b8709
commit 6ffed14b9c
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
150 changed files with 1893 additions and 5634 deletions
Download patch file Download diff file Expand all files Collapse all files

140
libc/runtime/clone.c
View file

@ -47,6 +47,7 @@
#include "libc/runtime/syslib.internal.h"
#include "libc/sock/internal.h"
#include "libc/stdalign.internal.h"
#include "libc/stdio/sysparam.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/arch.h"
#include "libc/sysv/consts/clone.h"
@ -80,27 +81,25 @@
struct CloneArgs {
alignas(16) union {
struct {
int tid;
atomic_int tid;
int this;
};
uint32_t utid;
int64_t tid64;
};
int *ptid;
int *ctid;
int *ztid;
atomic_int *ptid;
atomic_int *ctid;
atomic_int *ztid;
char *tls;
int (*func)(void *, int);
void *arg;
long sp;
};
int sys_set_tls(uintptr_t, void *);
int __stack_call(void *, int, long, long, int (*)(void *, int), void *);
int __stack_call(void *, int, long, long, int (*)(void *, int), long);
static struct CloneArgs *AllocateCloneArgs(char *stk, size_t stksz) {
return (struct CloneArgs *)(((uintptr_t)(stk + stksz) -
sizeof(struct CloneArgs)) &
-16);
static long AlignStack(long sp, char *stk, long stksz, int mal) {
return sp & -mal;
}
#ifdef __x86_64__
@ -120,8 +119,8 @@ WinThreadEntry(int rdi, // rcx
int rc;
if (wt->tls)
__set_tls_win32(wt->tls);
*wt->ctid = wt->tid;
rc = __stack_call(wt->arg, wt->tid, 0, 0, wt->func, wt);
*wt->ctid = GetCurrentThreadId();
rc = __stack_call(wt->arg, wt->tid, 0, 0, wt->func, wt->sp);
// we can now clear ctid directly since we're no longer using our own
// stack memory, which can now be safely free'd by the parent thread.
*wt->ztid = 0;
@ -134,24 +133,31 @@ WinThreadEntry(int rdi, // rcx
static textwindows errno_t CloneWindows(int (*func)(void *, int), char *stk,
size_t stksz, int flags, void *arg,
void *tls, int *ptid, int *ctid) {
void *tls, atomic_int *ptid,
atomic_int *ctid) {
long sp;
int64_t h;
uint32_t utid;
struct CloneArgs *wt;
wt = AllocateCloneArgs(stk, stksz);
sp = (intptr_t)stk + stksz;
sp = AlignStack(sp, stk, stksz, 16);
sp -= sizeof(struct CloneArgs);
sp &= -alignof(struct CloneArgs);
wt = (struct CloneArgs *)sp;
wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid;
wt->func = func;
wt->arg = arg;
wt->tls = flags & CLONE_SETTLS ? tls : 0;
wt->sp = sp;
if ((h = CreateThread(&kNtIsInheritable, 65536, (void *)WinThreadEntry, wt,
kNtStackSizeParamIsAReservation, &wt->utid))) {
kNtStackSizeParamIsAReservation, &utid))) {
if (flags & CLONE_PARENT_SETTID)
*ptid = utid;
if (flags & CLONE_SETTLS) {
struct CosmoTib *tib = tls;
atomic_store_explicit(&tib->tib_syshand, h, memory_order_release);
}
if (flags & CLONE_PARENT_SETTID) {
*ptid = wt->tid;
}
return 0;
} else {
return __dos2errno(GetLastError());
@ -222,14 +228,26 @@ XnuThreadMain(void *pthread, // rdi
}
static errno_t CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags,
void *arg, void *tls, int *ptid, int *ctid) {
void *arg, void *tls, atomic_int *ptid,
atomic_int *ctid) {
// perform this weird mandatory system call once
static bool once;
struct CloneArgs *wt;
if (!once) {
npassert(sys_bsdthread_register(XnuThreadThunk, 0, 0, 0, 0, 0, 0) != -1);
once = true;
}
wt = AllocateCloneArgs(stk, stksz);
// setup stack for thread
long sp;
struct CloneArgs *wt;
sp = (intptr_t)stk + stksz;
sp = AlignStack(sp, stk, stksz, 16);
sp -= sizeof(struct CloneArgs);
sp &= -alignof(struct CloneArgs);
wt = (struct CloneArgs *)sp;
// pass parameters to new thread via xnu
wt->ptid = flags & CLONE_PARENT_SETTID ? ptid : &wt->tid;
wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid;
@ -260,8 +278,8 @@ static wontreturn void OpenbsdThreadMain(void *p) {
}
static errno_t CloneOpenbsd(int (*func)(void *, int), char *stk, size_t stksz,
int flags, void *arg, void *tls, int *ptid,
int *ctid) {
int flags, void *arg, void *tls, atomic_int *ptid,
atomic_int *ctid) {
int rc;
intptr_t sp;
struct __tfork *tf;
@ -273,11 +291,12 @@ static errno_t CloneOpenbsd(int (*func)(void *, int), char *stk, size_t stksz,
sp -= sizeof(struct CloneArgs);
sp &= -alignof(struct CloneArgs);
wt = (struct CloneArgs *)sp;
sp = AlignStack(sp, stk, stksz, 16);
wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid;
wt->arg = arg;
wt->func = func;
tf->tf_stack = (char *)wt - 8;
tf->tf_stack = (char *)sp - 8;
tf->tf_tcb = flags & CLONE_SETTLS ? tls : 0;
tf->tf_tid = &wt->tid;
if ((rc = __tfork_thread(tf, sizeof(*tf), OpenbsdThreadMain, wt)) >= 0) {
@ -297,7 +316,7 @@ static errno_t CloneOpenbsd(int (*func)(void *, int), char *stk, size_t stksz,
static wontreturn void NetbsdThreadMain(void *arg, // rdi
int (*func)(void *, int), // rsi
int *tid, // rdx
int *ctid, // rcx
atomic_int *ctid, // rcx
int *ztid) { // r9
int ax, dx;
// TODO(jart): Why are we seeing flakes where *tid is zero?
@ -316,11 +335,13 @@ static wontreturn void NetbsdThreadMain(void *arg, // rdi
}
static int CloneNetbsd(int (*func)(void *, int), char *stk, size_t stksz,
int flags, void *arg, void *tls, int *ptid, int *ctid) {
int flags, void *arg, void *tls, atomic_int *ptid,
atomic_int *ctid) {
// NetBSD has its own clone() and it works, but it's technically a
// second-class API, intended to help Linux folks migrate to this.
int ax;
bool failed;
int ax, *tid;
atomic_int *tid;
intptr_t dx, sp;
static bool once;
struct ucontext_netbsd *ctx;
@ -335,16 +356,16 @@ static int CloneNetbsd(int (*func)(void *, int), char *stk, size_t stksz,
npassert(!failed);
once = true;
}
sp = (intptr_t)(stk + stksz);
sp = (intptr_t)stk + stksz;
// allocate memory for tid
sp -= sizeof(int);
sp = sp & -alignof(int);
tid = (int *)sp;
sp -= sizeof(atomic_int);
sp = sp & -alignof(atomic_int);
tid = (atomic_int *)sp;
*tid = 0;
// align the stack
sp = sp & -16;
sp = AlignStack(sp, stk, stksz, 16);
// simulate call to misalign stack and ensure backtrace looks good
sp -= 8;
@ -439,11 +460,16 @@ static wontreturn void FreebsdThreadMain(void *p) {
}
static errno_t CloneFreebsd(int (*func)(void *, int), char *stk, size_t stksz,
int flags, void *arg, void *tls, int *ptid,
int *ctid) {
int flags, void *arg, void *tls, atomic_int *ptid,
atomic_int *ctid) {
long sp;
int64_t tid;
struct CloneArgs *wt;
wt = AllocateCloneArgs(stk, stksz);
sp = (intptr_t)stk + stksz;
sp -= sizeof(struct CloneArgs);
sp &= -alignof(struct CloneArgs);
wt = (struct CloneArgs *)sp;
sp = AlignStack(sp, stk, stksz, 16);
wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid;
wt->tls = tls;
@ -453,7 +479,7 @@ static errno_t CloneFreebsd(int (*func)(void *, int), char *stk, size_t stksz,
.start_func = FreebsdThreadMain,
.arg = wt,
.stack_base = stk,
.stack_size = (uintptr_t)wt - (uintptr_t)stk,
.stack_size = sp - (long)stk,
.tls_base = flags & CLONE_SETTLS ? tls : 0,
.tls_size = 64,
.child_tid = &wt->tid64,
@ -492,15 +518,16 @@ static void *SiliconThreadMain(void *arg) {
struct CloneArgs *wt = arg;
asm volatile("mov\tx28,%0" : /* no outputs */ : "r"(wt->tls));
*wt->ctid = wt->this;
__stack_call(wt->arg, wt->this, 0, 0, wt->func, wt);
__stack_call(wt->arg, wt->this, 0, 0, wt->func, wt->sp);
*wt->ztid = 0;
ulock_wake(UL_COMPARE_AND_WAIT | ULF_WAKE_ALL, wt->ztid, 0);
return 0;
}
static errno_t CloneSilicon(int (*fn)(void *, int), char *stk, size_t stksz,
int flags, void *arg, void *tls, int *ptid,
int *ctid) {
int flags, void *arg, void *tls, atomic_int *ptid,
atomic_int *ctid) {
long sp;
void *attr;
errno_t res;
unsigned tid;
@ -508,7 +535,11 @@ static errno_t CloneSilicon(int (*fn)(void *, int), char *stk, size_t stksz,
size_t babystack;
struct CloneArgs *wt;
static atomic_uint tids;
wt = AllocateCloneArgs(stk, stksz);
sp = (intptr_t)stk + stksz;
sp -= sizeof(struct CloneArgs);
sp &= -alignof(struct CloneArgs);
wt = (struct CloneArgs *)sp;
sp = AlignStack(sp, stk, stksz, 16);
tid = atomic_fetch_add_explicit(&tids, 1, memory_order_acq_rel);
wt->this = tid = (tid & (kMaxThreadIds - 1)) + kMinThreadId;
wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
@ -516,6 +547,7 @@ static errno_t CloneSilicon(int (*fn)(void *, int), char *stk, size_t stksz,
wt->tls = flags & CLONE_SETTLS ? tls : 0;
wt->func = fn;
wt->arg = arg;
wt->sp = sp;
babystack = __syslib->__pthread_stack_min;
#pragma GCC push_options
#pragma GCC diagnostic ignored "-Walloca-larger-than="
@ -545,16 +577,16 @@ struct LinuxCloneArgs {
int (*func)(void *, int);
void *arg;
char *tls;
int ctid;
atomic_int ctid;
};
int sys_clone_linux(int flags, // rdi
long sp, // rsi
int *ptid, // rdx
int *ctid, // rcx
void *tls, // r8
void *func, // r9
void *arg); // 8(rsp)
int sys_clone_linux(int flags, // rdi
long sp, // rsi
atomic_int *ptid, // rdx
atomic_int *ctid, // rcx
void *tls, // r8
void *func, // r9
void *arg); // 8(rsp)
static int LinuxThreadEntry(void *arg, int tid) {
struct LinuxCloneArgs *wt = arg;
@ -563,19 +595,21 @@ static int LinuxThreadEntry(void *arg, int tid) {
}
static int CloneLinux(int (*func)(void *arg, int rc), char *stk, size_t stksz,
int flags, void *arg, void *tls, int *ptid, int *ctid) {
int flags, void *arg, void *tls, atomic_int *ptid,
atomic_int *ctid) {
int rc;
long sp;
struct LinuxCloneArgs *wt;
sp = (intptr_t)(stk + stksz);
sp = (intptr_t)stk + stksz;
sp -= sizeof(struct LinuxCloneArgs);
sp &= -alignof(struct LinuxCloneArgs);
wt = (struct LinuxCloneArgs *)sp;
// align the stack
#ifdef __aarch64__
sp = sp & -128; // for kernel 4.6 and earlier
sp = AlignStack(sp, stk, stksz, 128); // for kernel <=4.6
#else
sp = sp & -16;
sp = AlignStack(sp, stk, stksz, 16);
#endif
wt = (struct LinuxCloneArgs *)sp;
#ifdef __x86_64__
if (flags & CLONE_SETTLS) {
flags &= ~CLONE_SETTLS;

8
libc/runtime/cosmo2.c
View file

@ -21,6 +21,7 @@
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/strace.internal.h"
#include "libc/limits.h"
#include "libc/macros.internal.h"
@ -162,13 +163,8 @@ wontreturn textstartup void cosmo(long *sp, struct Syslib *m1, char *exename,
// needed by kisdangerous()
__pid = sys_getpid().ax;
// initialize memory manager
_mmi.i = 0;
_mmi.p = _mmi.s;
_mmi.n = ARRAYLEN(_mmi.s);
__virtualmax = -1;
// initialize file system
__maps_init();
__init_fds(argc, argv, envp);
// prepend cwd to executable path

3
libc/runtime/enable_tls.c
View file

@ -25,6 +25,7 @@
#include "libc/intrin/atomic.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/getenv.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/weaken.h"
#include "libc/macros.internal.h"
#include "libc/nt/files.h"
@ -251,7 +252,7 @@ textstartup void __enable_tls(void) {
_pthread_static.pt_flags = PT_STATIC;
dll_init(&_pthread_static.list);
_pthread_list = &_pthread_static.list;
atomic_store_explicit(&_pthread_static.ptid, tid, memory_order_relaxed);
atomic_store_explicit(&_pthread_static.ptid, tid, memory_order_release);
// ask the operating system to change the x86 segment register
__set_tls(tib);

4
libc/runtime/ftrace_install.c
View file

@ -16,8 +16,8 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/fmt/itoa.h"
#include "libc/intrin/kprintf.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
@ -28,7 +28,7 @@ textstartup int ftrace_install(void) {
ftrace_stackdigs = LengthInt64Thousands(GetStackSize());
return __hook(ftrace_hook, GetSymbolTable());
} else {
kprintf("error: --ftrace failed to open symbol table\n");
tinyprint(2, "error: --ftrace failed to open symbol table\n", NULL);
return -1;
}
}

2
libc/runtime/getavphyspages.c
View file

@ -23,5 +23,5 @@ long __get_avphys_pages(void) {
struct sysinfo si;
if (sysinfo(&si) == -1)
return -1;
return (((int64_t)si.freeram + si.bufferram) * si.mem_unit) / FRAMESIZE;
return (((int64_t)si.freeram + si.bufferram) * si.mem_unit) / __granularity();
}

27
libc/runtime/getmemtracksize.c
View file

@ -1,27 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/memtrack.internal.h"
size_t __get_memtrack_size(struct MemoryIntervals *mm) {
size_t i, n;
for (n = i = 0; i < mm->i; ++i) {
n += ((size_t)(mm->p[i].y - mm->p[i].x) + 1) << 16;
}
return n;
}

2
libc/runtime/getpagesize.c
View file

@ -23,5 +23,5 @@
* @see sysconf(_SC_PAGE_SIZE) which is portable
*/
int getpagesize(void) {
return FRAMESIZE;
return __granularity();
}

2
libc/runtime/getphyspages.c
View file

@ -23,5 +23,5 @@ long __get_phys_pages(void) {
struct sysinfo si;
if (sysinfo(&si) == -1)
return -1;
return ((int64_t)si.totalram * si.mem_unit) / FRAMESIZE;
return ((int64_t)si.totalram * si.mem_unit) / __granularity();
}

2
libc/runtime/getsymboltable.c
View file

@ -63,7 +63,7 @@ static struct SymbolTable *GetSymbolTableFromZip(struct Zipos *zipos) {
(cf = GetZipFile(zipos, ".symtab")) != -1) {
lf = GetZipCfileOffset(zipos->map + cf);
size = GetZipLfileUncompressedSize(zipos->map + lf);
size2 = ROUNDUP(size, FRAMESIZE);
size2 = ROUNDUP(size, __granularity());
if ((res = _mapanon(size2))) {
switch (ZIP_LFILE_COMPRESSIONMETHOD(zipos->map + lf)) {
case kZipCompressionNone:

12
libc/runtime/grow.c
View file

@ -16,6 +16,7 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "ape/sections.internal.h"
#include "libc/assert.h"
#include "libc/fmt/conv.h"
#include "libc/intrin/weaken.h"
@ -31,6 +32,13 @@
#define GUARANTEE_TERMINATOR 1
#define INITIAL_CAPACITY (32 - GUARANTEE_TERMINATOR)
static bool isheap(const void *p) {
if (__executable_start <= (const unsigned char *)p &&
(const unsigned char *)p < _end)
return false;
return true;
}
bool __grow(void *pp, size_t *capacity, size_t itemsize, size_t extra) {
void **p, *p1, *p2;
size_t n1, n2;
@ -39,8 +47,8 @@ bool __grow(void *pp, size_t *capacity, size_t itemsize, size_t extra) {
p = (void **)pp;
unassert(itemsize);
unassert((*p && *capacity) || (!*p && !*capacity));
unassert(!_isheap(*p) || ((intptr_t)*p & 15) == 0);
p1 = _isheap(*p) ? *p : NULL;
unassert(!isheap(*p) || ((intptr_t)*p & 15) == 0);
p1 = isheap(*p) ? *p : NULL;
p2 = NULL;
n1 = *capacity;
n2 = (*p ? n1 + (n1 >> 1) : MAX(4, INITIAL_CAPACITY / itemsize)) + extra;

3
libc/runtime/internal.h
View file

@ -6,7 +6,7 @@
#include "libc/runtime/runtime.h"
#define STACK_CEIL 0x700000000000ul
#define STACK_SIZE FRAMESIZE
#define STACK_SIZE 65536
#define RUNLEVEL_MALLOC 1
@ -41,7 +41,6 @@ long _setstack(void *, void *, ...);
int GetDosArgv(const char16_t *, char *, size_t, char **, size_t);
int GetDosEnviron(const char16_t *, char *, size_t, char **, size_t);
bool __intercept_flag(int *, char *[], const char *);
int sys_mprotect_nt(void *, size_t, int);
int __inflate(void *, size_t, const void *, size_t);
void *__mmap_unlocked(void *, size_t, int, int, int, int64_t);
int __munmap_unlocked(char *, size_t);

37
libc/runtime/isheap.c
View file

@ -1,37 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
// TODO(jart): DELETE
/**
* Returns true if address isn't stack and was malloc'd or mmap'd.
*
* @assume stack addresses are always greater than heap addresses
* @assume stack memory isn't stored beneath %rsp (-mno-red-zone)
* @deprecated
*/
optimizesize bool32 _isheap(const void *p) {
intptr_t x, y;
x = kAutomapStart;
y = x + kAutomapSize;
return x <= (intptr_t)p && (intptr_t)p < y;
}

43
libc/runtime/ismemtracked.greg.c
View file

@ -1,43 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/memtrack.internal.h"
static inline bool IsMemtrackedImpl(int x, int y) {
unsigned i;
i = __find_memory(&_mmi, x);
if (i == _mmi.i)
return false;
if (x < _mmi.p[i].x)
return false;
for (;;) {
if (y <= _mmi.p[i].y)
return true;
if (++i == _mmi.i)
return false;
if (_mmi.p[i].x != _mmi.p[i - 1].y + 1)
return false;
}
}
bool IsMemtracked(int x, int y) {
/* assumes __mmi_lock() is held */
bool res;
res = IsMemtrackedImpl(x, y);
return res;
}

7
libc/runtime/mapanon.c
View file

@ -22,7 +22,6 @@
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
/**
* Helper function for allocating anonymous mapping.
@ -59,11 +58,9 @@
void *_mapanon(size_t size) {
void *m;
m = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (m != MAP_FAILED) {
if (m != MAP_FAILED)
return m;
}
if (errno == ENOMEM && _weaken(__oom_hook)) {
if (errno == ENOMEM && _weaken(__oom_hook))
_weaken(__oom_hook)(size);
}
return 0;
}

20
libc/runtime/mapstack.c
View file

@ -17,6 +17,7 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/asancodes.h"
@ -27,6 +28,9 @@
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
#define MAP_ANON_OPENBSD 0x1000
#define MAP_STACK_OPENBSD 0x4000
/**
* Allocates stack.
*
@ -42,13 +46,17 @@
*/
void *NewCosmoStack(void) {
char *p;
if ((p = mmap(0, GetStackSize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS |
(IsAarch64() && IsLinux() && IsQemuUser() ? MAP_PRIVATE
: MAP_STACK),
-1, 0)) != MAP_FAILED) {
size_t n = GetStackSize() + (uintptr_t)ape_stack_align;
if ((p = mmap(0, n, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1,
0)) != MAP_FAILED) {
if (IsOpenbsd() && __sys_mmap(p, n, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD |
MAP_STACK_OPENBSD,
-1, 0, 0) != p) {
notpossible;
}
if (IsAsan()) {
__asan_poison(p + GetStackSize() - 16, 16, kAsanStackOverflow);
__asan_poison(p + n - 16, 16, kAsanStackOverflow);
__asan_poison(p, getauxval(AT_PAGESZ), kAsanStackOverflow);
}
return p;

183
libc/runtime/memtrack.internal.h
View file

@ -1,189 +1,22 @@
#ifndef COSMOPOLITAN_LIBC_RUNTIME_MEMTRACK_H_
#define COSMOPOLITAN_LIBC_RUNTIME_MEMTRACK_H_
#include "ape/sections.internal.h"
#include "libc/dce.h"
#include "libc/macros.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/sysv/consts/ss.h"
#include "libc/thread/tls.h"
COSMOPOLITAN_C_START_
#define kAutomapStart 0x100080040000
#define kAutomapSize (kMemtrackStart - kAutomapStart)
#define kMemtrackStart 0x1fe7fffc0000
#define kMemtrackSize (0x1ffffffc0000 - kMemtrackStart)
#ifndef __SANITIZE_ADDRESS__
#define kFixedmapStart 0x300000000
#define kFixedmapSize (0x400000000 - kFixedmapStart)
#define kMemtrackFdsStart 0x6fe000000
#define kMemtrackFdsSize (0x6ff000000 - kMemtrackFdsStart)
#define kMemtrackZiposStart 0x6fd000000
#define kMemtrackZiposSize (0xafe000000 - kMemtrackZiposStart)
#else
#define kFixedmapStart 0x300000040000
#define kFixedmapSize (0x400000040000 - kFixedmapStart)
#define kMemtrackNsyncStart 0x6fc000040000
#define kMemtrackNsyncSize (0x6fcffffc0000 - kMemtrackNsyncStart)
#define kMemtrackFdsStart 0x6fe000040000
#define kMemtrackFdsSize (0x6feffffc0000 - kMemtrackFdsStart)
#define kMemtrackZiposStart 0x6fd000040000
#define kMemtrackZiposSize (0x6fdffffc0000 - kMemtrackZiposStart)
#define kMemtrackGran (!IsAsan() ? FRAMESIZE : FRAMESIZE * 8)
struct MemoryInterval {
int x;
int y;
long h;
long size;
long offset;
int flags;
char prot;
bool iscow;
bool readonlyfile;
};
struct MemoryIntervals {
size_t i, n;
struct MemoryInterval *p;
struct MemoryInterval s[16];
};
extern struct MemoryIntervals _mmi;
void __mmi_lock(void);
void __mmi_unlock(void);
bool IsMemtracked(int, int);
void PrintSystemMappings(int);
unsigned __find_memory(const struct MemoryIntervals *, int) nosideeffect;
bool __check_memtrack(const struct MemoryIntervals *) nosideeffect;
void PrintMemoryIntervals(int, const struct MemoryIntervals *);
int __track_memory(struct MemoryIntervals *, int, int, long, int, int, bool,
bool, long, long);
int __untrack_memory(struct MemoryIntervals *, int, int,
void (*)(struct MemoryIntervals *, int, int));
void __release_memory_nt(struct MemoryIntervals *, int, int);
int __untrack_memories(void *, size_t);
size_t __get_memtrack_size(struct MemoryIntervals *) nosideeffect;
#ifdef __x86_64__
/*
* AMD64 has 48-bit signed pointers (PML4T)
* AMD64 is trying to go bigger, i.e. 57-bit (PML5T)
* LINUX forbids userspace from leveraging negative pointers
* Q-EMU may impose smaller vaspaces emulating AMD on non-AMD
*
* Having "signed pointers" means these top sixteen bits
*
* 0x0000000000000000
* ^^^^
*
* must be
*
* - 0000 for positive pointers
* - FFFF for negative pointers
*
* otherwise the instruction using the faulty pointer will fault.
*/
#define IsLegalPointer(p) \
(-0x800000000000 <= (intptr_t)(p) && (intptr_t)(p) <= 0x7fffffffffff)
#define ADDR_32_TO_48(x) (intptr_t)((uint64_t)(int)(x) << 16)
#elif defined(__aarch64__)
/*
* ARM64 has 48-bit unsigned pointers (Armv8.0-A)
* ARM64 can possibly go bigger, i.e. 52-bit (Armv8.2-A)
* ARM64 can impose arbitrarily smaller vaspaces, e.g. 40/44-bit
* APPLE in their limitless authoritarianism forbids 32-bit pointers
*/
#define IsLegalPointer(p) ((uintptr_t)(p) <= 0xffffffffffff)
#define ADDR_32_TO_48(x) (uintptr_t)((uint64_t)(uint32_t)(x) << 16)
#else
/* RISC-V Sipeed Nezha has 39-bit vaspace */
#error "unsupported architecture"
#endif
forceinline pureconst bool IsLegalSize(uint64_t n) {
/* subtract frame size so roundup is safe */
return n <= 0x800000000000 - FRAMESIZE;
}
forceinline pureconst bool IsAutoFrame(int x) {
return (int)(kAutomapStart >> 16) <= x &&
x <= (int)((kAutomapStart + kAutomapSize - 1) >> 16);
}
forceinline pureconst bool IsMemtrackFrame(int x) {
return (int)(kAutomapStart >> 16) <= x &&
x <= (int)((kAutomapStart + kAutomapSize - 1) >> 16);
}
forceinline pureconst bool IsGfdsFrame(int x) {
return (int)(kMemtrackFdsStart >> 16) <= x &&
x <= (int)((kMemtrackFdsStart + kMemtrackFdsSize - 1) >> 16);
}
forceinline pureconst bool IsNsyncFrame(int x) {
return (int)(kMemtrackNsyncStart >> 16) <= x &&
x <= (int)((kMemtrackNsyncStart + kMemtrackNsyncSize - 1) >> 16);
}
forceinline pureconst bool IsZiposFrame(int x) {
return (int)(kMemtrackZiposStart >> 16) <= x &&
x <= (int)((kMemtrackZiposStart + kMemtrackZiposSize - 1) >> 16);
}
forceinline pureconst bool IsShadowFrame(int x) {
return 0x7fff <= x && x < 0x10008000;
}
forceinline pureconst bool IsStaticStackFrame(int x) {
intptr_t stack = GetStaticStackAddr(0);
return (int)(stack >> 16) <= x &&
x <= (int)((stack + (GetStackSize() - FRAMESIZE)) >> 16);
}
forceinline pureconst bool IsStackFrame(int x) {
intptr_t stack = GetStackAddr();
return (int)(stack >> 16) <= x &&
x <= (int)((stack + (GetStackSize() - FRAMESIZE)) >> 16);
}
forceinline pureconst bool IsOldStack(const void *x) {
size_t foss_stack_size = 8ul * 1024 * 1024;
uintptr_t top = __oldstack + foss_stack_size;
uintptr_t bot = __oldstack - foss_stack_size;
return bot <= (uintptr_t)x && (uintptr_t)x < top;
}
forceinline pureconst bool IsOldStackFrame(int x) {
size_t foss_stack_size = 8ul * 1024 * 1024;
uintptr_t top = __oldstack + foss_stack_size;
uintptr_t bot = __oldstack - foss_stack_size;
return (int)(bot >> 16) <= x && x <= (int)((top >> 16) - 1);
}
forceinline pureconst bool IsFixedFrame(int x) {
return (kFixedmapStart >> 16) <= x &&
x <= ((kFixedmapStart + (kFixedmapSize - 1)) >> 16);
}
forceinline pureconst bool OverlapsImageSpace(const void *p, size_t n) {
const unsigned char *BegA, *EndA, *BegB, *EndB;
if (n) {
BegA = p;
EndA = BegA + n;
BegB = __executable_start;
EndB = _end;
return MAX(BegA, BegB) < MIN(EndA, EndB);
} else {
return 0;
}
}
forceinline pureconst bool OverlapsShadowSpace(const void *p, size_t n) {
intptr_t BegA, EndA, BegB, EndB;
if (n) {
BegA = (intptr_t)p;
EndA = BegA + n;
BegB = 0x7fff0000;
EndB = 0x100080000000;
return MAX(BegA, BegB) < MIN(EndA, EndB);
} else {
return 0;
}
}
COSMOPOLITAN_C_END_
#endif /* COSMOPOLITAN_LIBC_RUNTIME_MEMTRACK_H_ */

2073
libc/runtime/memtrack64.txt
View file

File diff suppressed because it is too large Load diff

39
libc/runtime/memtracknt.c
View file

@ -1,39 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/intrin/strace.internal.h"
#include "libc/nt/memory.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
static inline void *GetFrameAddr(int f) {
intptr_t a;
a = f;
a *= FRAMESIZE;
return (void *)a;
}
void __release_memory_nt(struct MemoryIntervals *mm, int l, int r) {
int i;
for (i = l; i <= r; ++i) {
UnmapViewOfFile(GetFrameAddr(mm->p[i].x));
CloseHandle(mm->p[i].h);
}
}

503
libc/runtime/mmap.c
View file

@ -1,503 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
#include "libc/calls/struct/sigset.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/asancodes.h"
#include "libc/intrin/bsr.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/safemacros.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/limits.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/libfatal.internal.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
#include "libc/nt/enum/memflags.h"
#include "libc/nt/enum/pageflags.h"
#include "libc/nt/memory.h"
#include "libc/nt/process.h"
#include "libc/nt/runtime.h"
#include "libc/nt/struct/processmemorycounters.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/zipos.internal.h"
#include "libc/stdckdint.h"
#include "libc/stdio/rand.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/o.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/consts/sig.h"
#include "libc/sysv/consts/ss.h"
#include "libc/sysv/errfuns.h"
#include "libc/thread/thread.h"
#define MAP_ANONYMOUS_linux 0x00000020
#define MAP_ANONYMOUS_openbsd 0x00001000
#define MAP_GROWSDOWN_linux 0x00000100
#define MAP_STACK_freebsd 0x00000400
#define MAP_STACK_openbsd 0x00004000
#define IP(X) (intptr_t)(X)
#define VIP(X) (void *)IP(X)
#define ALIGNED(p) (!(IP(p) & (FRAMESIZE - 1)))
#define SHADE(x) (((intptr_t)(x) >> 3) + 0x7fff8000)
#define FRAME(x) ((int)((intptr_t)(x) >> 16))
static inline pureconst unsigned long __rounddown2pow(unsigned long x) {
return x ? 1ul << bsrl(x) : 0;
}
static wontreturn void __mmap_die(const char *s) {
if (_weaken(__die))
_weaken(__die)();
STRACE("%s %m", s);
_Exit(199);
}
static inline bool __overlaps_existing_mapping(char *p, size_t n) {
int a, b, i;
unassert(n > 0);
a = FRAME(p);
b = FRAME(p + (n - 1));
i = __find_memory(&_mmi, a);
if (i < _mmi.i) {
if (a <= _mmi.p[i].x && _mmi.p[i].x <= b)
return true;
if (a <= _mmi.p[i].y && _mmi.p[i].y <= b)
return true;
if (_mmi.p[i].x <= a && b <= _mmi.p[i].y)
return true;
}
return false;
}
static bool __choose_memory(int x, int n, int align, int *res) {
// TODO: improve performance
int i, start, end;
unassert(align > 0);
if (_mmi.i) {
// find the start of the automap memory region
i = __find_memory(&_mmi, x);
if (i < _mmi.i) {
// check to see if there's space available before the first entry
if (!ckd_add(&start, x, align - 1)) {
start &= -align;
if (!ckd_add(&end, start, n - 1)) {
if (end < _mmi.p[i].x) {
*res = start;
return true;
}
}
}
// check to see if there's space available between two entries
while (++i < _mmi.i) {
if (!ckd_add(&start, _mmi.p[i - 1].y, 1) &&
!ckd_add(&start, start, align - 1)) {
start &= -align;
if (!ckd_add(&end, start, n - 1)) {
if (end < _mmi.p[i].x) {
*res = start;
return true;
}
}
}
}
}
// otherwise append after the last entry if space is available
if (!ckd_add(&start, _mmi.p[i - 1].y, 1) &&
!ckd_add(&start, start, align - 1)) {
start &= -align;
if (!ckd_add(&end, start, n - 1)) {
*res = start;
return true;
}
}
} else {
// if memtrack is empty, then just assign the requested address
// assuming it doesn't overflow
if (!ckd_add(&start, x, align - 1)) {
start &= -align;
if (!ckd_add(&end, start, n - 1)) {
*res = start;
return true;
}
}
}
return false;
}
static bool __auto_map(int count, int align, int *res) {
return __choose_memory(FRAME(kAutomapStart), count, align, res) &&
*res + count <= FRAME(kAutomapStart + (kAutomapSize - 1));
}
static void *__finish_memory(void *addr, size_t size, int prot, int flags,
int fd, int64_t off, int f, int x, int n,
struct DirectMap dm) {
if (!IsWindows() && (flags & MAP_FIXED)) {
if (__untrack_memories(addr, size)) {
__mmap_die("FIXED UNTRACK FAILED");
}
}
if (__track_memory(&_mmi, x, x + (n - 1), dm.maphandle, prot, flags, false,
false, off, size)) {
if (sys_munmap(addr, n) == -1) {
__mmap_die("TRACK MUNMAP FAILED");
}
return MAP_FAILED;
}
if (_weaken(__asan_map_shadow) && !OverlapsShadowSpace(addr, size)) {
_weaken(__asan_map_shadow)((intptr_t)addr, size);
}
return addr;
}
static void *__map_memory(void *addr, size_t size, int prot, int flags, int fd,
int64_t off, int f, int x, int n) {
struct DirectMap dm;
dm = sys_mmap(addr, size, prot, f, fd, off);
if (VERY_UNLIKELY(dm.addr == MAP_FAILED)) {
if (IsWindows() && (flags & MAP_FIXED)) {
__mmap_die("can't recover from MAP_FIXED errors on Windows");
}
return MAP_FAILED;
}
if (VERY_UNLIKELY(dm.addr != addr)) {
__mmap_die("KERNEL DIDN'T RESPECT MAP_FIXED");
}
return __finish_memory(addr, size, prot, flags, fd, off, f, x, n, dm);
}
/**
* Maps memory from system, one frame at a time.
*
* This is useful on Windows since it allows us to partially unmap or
* punch holes into existing mappings.
*/
static textwindows dontinline void *__map_memories(char *addr, size_t size,
int prot, int flags, int fd,
int64_t off, int f, int x,
int n) {
size_t i, m;
int64_t oi, sz;
struct DirectMap dm;
bool iscow, readonlyfile;
m = (size_t)(n - 1) << 16;
unassert(m < size);
unassert(m + FRAMESIZE >= size);
oi = fd == -1 ? 0 : off + m;
sz = size - m;
dm = sys_mmap(addr + m, sz, prot, f, fd, oi);
if (dm.addr == MAP_FAILED)
return MAP_FAILED;
iscow = (flags & MAP_TYPE) != MAP_SHARED && fd != -1;
readonlyfile = (flags & MAP_TYPE) == MAP_SHARED && fd != -1 &&
(g_fds.p[fd].flags & O_ACCMODE) == O_RDONLY;
if (__track_memory(&_mmi, x + (n - 1), x + (n - 1), dm.maphandle, prot, flags,
readonlyfile, iscow, oi, sz) == -1) {
__mmap_die("__map_memories unrecoverable #1");
}
for (i = 0; i < m; i += FRAMESIZE) {
oi = fd == -1 ? 0 : off + i;
sz = FRAMESIZE;
dm = sys_mmap(addr + i, sz, prot, f, fd, oi);
if (dm.addr == MAP_FAILED ||
__track_memory(&_mmi, x + i / FRAMESIZE, x + i / FRAMESIZE,
dm.maphandle, prot, flags, readonlyfile, iscow, oi,
sz) == -1) {
__mmap_die("__map_memories unrecoverable #2");
}
}
if (_weaken(__asan_map_shadow) && !OverlapsShadowSpace(addr, size)) {
_weaken(__asan_map_shadow)((intptr_t)addr, size);
}
return addr;
}
inline void *__mmap_unlocked(void *addr, size_t size, int prot, int flags,
int fd, int64_t off) {
char *p = addr;
struct DirectMap dm;
size_t requested_size;
bool needguard, clashes;
int a, f, n, x, pagesize;
size_t virtualused, virtualneed;
if (VERY_UNLIKELY(!size)) {
STRACE("can't mmap zero bytes");
return VIP(einval());
}
if (VERY_UNLIKELY(!ALIGNED(p))) {
STRACE("cosmo mmap is 64kb aligned");
return VIP(einval());
}
if (VERY_UNLIKELY(!IsLegalSize(size))) {
STRACE("mmap size isn't legal");
return VIP(einval());
}
if (VERY_UNLIKELY(!IsLegalPointer(p))) {
STRACE("mmap addr isn't 48-bit");
return VIP(einval());
}
if ((flags & (MAP_SHARED | MAP_PRIVATE)) == (MAP_SHARED | MAP_PRIVATE)) {
flags = MAP_SHARED; // cf. MAP_SHARED_VALIDATE
}
requested_size = size;
pagesize = getauxval(AT_PAGESZ);
if (flags & MAP_ANONYMOUS) {
fd = -1;
off = 0;
size = ROUNDUP(size, FRAMESIZE);
if ((flags & MAP_TYPE) == MAP_FILE) {
STRACE("need MAP_PRIVATE or MAP_SHARED");
return VIP(einval());
}
} else if (__isfdkind(fd, kFdZip)) {
STRACE("mmap fd is zipos handle");
return VIP(einval());
} else if (VERY_UNLIKELY(off < 0)) {
STRACE("mmap negative offset");
return VIP(einval());
} else if (off & ((IsWindows() ? FRAMESIZE : pagesize) - 1)) {
STRACE("mmap offset isn't properly aligned");
return VIP(einval());
} else if (VERY_UNLIKELY(INT64_MAX - size < off)) {
STRACE("mmap too large");
return VIP(einval());
}
if (__virtualmax < LONG_MAX &&
(ckd_add(&virtualneed, (virtualused = __get_memtrack_size(&_mmi)),
size) ||
virtualneed > __virtualmax)) {
STRACE("mmap %'zu size + %'zu inuse exceeds virtual memory limit %'zu",
size, virtualused, __virtualmax);
return VIP(enomem());
}
clashes = OverlapsImageSpace(p, size) || __overlaps_existing_mapping(p, size);
if ((flags & MAP_FIXED_NOREPLACE) == MAP_FIXED_NOREPLACE && clashes) {
STRACE("mmap noreplace overlaps existing");
return VIP(eexist());
}
if (ckd_add(&n, (int)(size >> 16), (int)!!(size & (FRAMESIZE - 1)))) {
STRACE("mmap range overflows");
return VIP(einval());
}
a = MAX(1, __rounddown2pow(size) >> 16);
f = (flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED;
if (flags & MAP_FIXED) {
x = FRAME(p);
if (IsWindows()) {
if (__untrack_memories(p, size)) {
__mmap_die("FIXED UNTRACK FAILED");
}
}
} else if (p && !clashes && !OverlapsShadowSpace(p, size)) {
x = FRAME(p);
} else if (!__auto_map(n, a, &x)) {
STRACE("automap has no room for %d frames with %d alignment", n, a);
return VIP(enomem());
}
needguard = false;
p = (char *)ADDR_32_TO_48(x);
if ((f & MAP_TYPE) == MAP_STACK) {
if (~f & MAP_ANONYMOUS) {
STRACE("MAP_STACK must be anonymous");
return VIP(einval());
}
f &= ~MAP_TYPE;
f |= MAP_PRIVATE;
if (IsOpenbsd()) { // openbsd:dubstack
// on openbsd this is less about scalability of threads, and more
// about defining the legal intervals for the RSP register. sadly
// openbsd doesn't let us create a new fixed stack mapping. but..
// openbsd does allow us to overwrite existing fixed mappings, to
// authorize its usage as a stack.
if (sys_mmap(p, size, prot, f, fd, off).addr == MAP_FAILED) {
return MAP_FAILED;
}
f |= MAP_STACK_openbsd;
needguard = true;
} else if (IsLinux()) {
// make sure there's no existing stuff existing between our stack
// starting page and the bottom guard page, since that would stop
// our stack page from growing down.
npassert(!sys_munmap(p, size));
int guardsize = pagesize, e = errno;
if ((dm = sys_mmap(p + size - guardsize, guardsize, prot,
f | MAP_GROWSDOWN_linux, fd, off))
.addr != MAP_FAILED) {
npassert(sys_mmap(p, pagesize, PROT_NONE,
MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)
.addr == p);
dm.addr = p;
p = __finish_memory(p, size, prot, flags, fd, off, f, x, n, dm);
if (IsAsan() && p != MAP_FAILED) {
__asan_poison(p, pagesize, kAsanStackOverflow);
}
return p;
} else if (errno == ENOTSUP) {
// WSL and Blink don't support MAP_GROWSDOWN
needguard = true;
errno = e;
}
} else {
if (IsFreebsd()) {
f |= MAP_STACK_freebsd;
}
needguard = true;
}
}
if (!IsWindows()) {
p = __map_memory(p, size, prot, flags, fd, off, f, x, n);
} else {
p = __map_memories(p, size, prot, flags, fd, off, f, x, n);
}
if (p != MAP_FAILED) {
if (IsAsan()) {
__asan_poison(p + requested_size, size - requested_size,
kAsanMmapSizeOverrun);
}
if (needguard) {
if (!IsWindows()) {
unassert(!mprotect(p, pagesize, PROT_NONE));
if (IsAsan()) {
__asan_poison(p, pagesize, kAsanStackOverflow);
}
} else {
uint32_t oldattr;
unassert(VirtualProtect(p, pagesize, kNtPageReadwrite | kNtPageGuard,
&oldattr));
}
}
}
return p;
}
/**
* Creates virtual memory, e.g.
*
* char *m;
* m = mmap(NULL, FRAMESIZE, PROT_READ | PROT_WRITE,
* MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
* munmap(m, FRAMESIZE);
*
* @param addr should be 0 to let your memory manager choose address;
* unless MAP_FIXED or MAP_FIXED_NOREPLACE are specified in flags
* in which case this function will do precicely as you ask, even
* if p=0 (in which you need -fno-delete-null-pointer-checks); it
* needs to be 64kb aligned because it's a wise choice that sadly
* needs to be made mandatory because of Windows although you can
* use __sys_mmap() to circumvent it on System Five in which case
* runtime support services, e.g. asan memory safety, could break
* @param size must be >0 otherwise EINVAL is raised
* @param prot can have PROT_READ/PROT_WRITE/PROT_EXEC/PROT_NONE/etc.
* @param flags should have one of the following masked by `MAP_TYPE`
* - `MAP_FILE` in which case `MAP_ANONYMOUS` shouldn't be used
* - `MAP_PRIVATE` for copy-on-write behavior of writeable pages
* - `MAP_SHARED` to create shared memory between processes
* - `MAP_STACK` to create a grows-down alloc, where a guard page
* is automatically protected at the bottom, sized as AT_PAGESZ
* Your `flags` may optionally bitwise or any of the following:
* - `MAP_ANONYMOUS` in which case `fd` and `off` are ignored
* - `MAP_FIXED` in which case `addr` becomes more than a hint
* - `MAP_FIXED_NOREPLACE` to protect existing mappings; this is
* always polyfilled by mmap() which tracks its own memory and
* removed before passing to the kernel, in order to support
* old versions; if you believe mappings exist which only the
* kernel knows, then this flag may be passed to sys_mmap() on
* Linux 4.17+ and FreeBSD (where it has multiple bits)
* - `MAP_CONCEAL` is FreeBSD/NetBSD/OpenBSD-only
* - `MAP_NORESERVE` is Linux/XNU/NetBSD-only
* - `MAP_POPULATE` is Linux/FreeBSD-only
* - `MAP_NONBLOCK` is Linux-only
* - `MAP_NOSYNC` is FreeBSD-only
* - `MAP_INHERIT` is NetBSD-only
* - `MAP_LOCKED` is Linux-only
* @param fd is an open()'d file descriptor, whose contents shall be
* made available w/ automatic reading at the chosen address
* @param off specifies absolute byte index of fd's file for mapping,
* should be zero if MAP_ANONYMOUS is specified, which SHOULD be
* aligned to FRAMESIZE
* @return virtual base address of new mapping, or MAP_FAILED w/ errno
*/
void *mmap(void *addr, size_t size, int prot, int flags, int fd, int64_t off) {
void *res;
#if SYSDEBUG
size_t toto = 0;
#if _KERNTRACE || _NTTRACE
if (IsWindows()) {
STRACE("mmap(%p, %'zu, %s, %s, %d, %'ld) → ...", addr, size,
DescribeProtFlags(prot), DescribeMapFlags(flags), fd, off);
}
#endif
#endif
__mmi_lock();
if (!__isfdkind(fd, kFdZip)) {
res = __mmap_unlocked(addr, size, prot, flags, fd, off);
} else {
res = _weaken(__zipos_mmap)(
addr, size, prot, flags,
(struct ZiposHandle *)(intptr_t)g_fds.p[fd].handle, off);
}
#if SYSDEBUG
toto = __strace > 0 ? __get_memtrack_size(&_mmi) : 0;
#endif
__mmi_unlock();
#if SYSDEBUG
STRACE("mmap(%p, %'zu, %s, %s, %d, %'ld) → %p% m (%'zu bytes total)", addr,
size, DescribeProtFlags(prot), DescribeMapFlags(flags), fd, off, res,
toto);
#endif
return res;
}
__strong_reference(mmap, mmap64);

69
libc/runtime/mprotect-nt.greg.c
View file

@ -1,69 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/struct/sigset.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/nt/memory.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/memtrack.internal.h"
textwindows int sys_mprotect_nt(void *addr, size_t size, int prot) {
int rc = 0;
unsigned i;
uint32_t op;
char *a, *b, *x, *y, *p;
BLOCK_SIGNALS;
__mmi_lock();
size = (size + 4095) & -4096;
p = addr;
i = __find_memory(&_mmi, (intptr_t)p >> 16);
if (i == _mmi.i || (!i && p + size <= (char *)ADDR_32_TO_48(_mmi.p[0].x))) {
// memory isn't in memtrack
// let's just trust the user then
// it's probably part of the executable
if (!VirtualProtect(addr, size, __prot2nt(prot, false), &op)) {
rc = -1;
}
} else {
// memory is in memtrack, so use memtrack, to do dimensioning
// we unfortunately must do something similar to this for cow
for (; i < _mmi.i; ++i) {
x = (char *)ADDR_32_TO_48(_mmi.p[i].x);
y = (char *)ADDR_32_TO_48(_mmi.p[i].y) + 65536;
if ((x <= p && p < y) || (x < p + size && p + size <= y) ||
(p < x && y < p + size)) {
if (p <= x && p + size >= y) {
_mmi.p[i].prot = prot;
} else {
_mmi.p[i].prot |= prot;
}
a = MIN(MAX(p, x), y);
b = MAX(MIN(p + size, y), x);
if (!VirtualProtect(a, b - a, __prot2nt(prot, _mmi.p[i].iscow), &op)) {
rc = -1;
break;
}
} else {
break;
}
}
}
__mmi_unlock();
ALLOW_SIGNALS;
return rc;
}

56
libc/runtime/mprotect.c
View file

@ -1,56 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/strace.internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
/**
* Modifies restrictions on virtual memory address range.
*
* @param addr needs to be 4kb aligned
* @param prot can have PROT_{NONE,READ,WRITE,EXEC,GROWSDOWN,GROWSUP}
* @return 0 on success, or -1 w/ errno
* @see mmap()
*/
int mprotect(void *addr, size_t size, int prot) {
int64_t rc;
if (prot &
~(PROT_READ | PROT_WRITE | PROT_EXEC | PROT_GROWSDOWN | PROT_GROWSUP)) {
rc = einval(); // unix checks prot before checking size
} else if (!size) {
return 0; // make new technology consistent with unix
} else if (UNLIKELY((intptr_t)addr & (getauxval(AT_PAGESZ) - 1))) {
rc = einval(); // unix checks prot before checking size
} else if (!IsWindows()) {
rc = sys_mprotect(addr, size, prot);
} else {
rc = sys_mprotect_nt(addr, size, prot);
}
STRACE("mprotect(%p, %'zu, %s) → %d% m", addr, size, DescribeProtFlags(prot),
rc);
return rc;
}

50
libc/runtime/msync-nt.c
View file

@ -1,50 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macros.internal.h"
#include "libc/nt/files.h"
#include "libc/nt/memory.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/msync.h"
#include "libc/sysv/consts/o.h"
#include "libc/sysv/consts/prot.h"
textwindows int sys_msync_nt(char *addr, size_t size, int flags) {
int i, rc = 0;
char *a, *b, *x, *y;
__mmi_lock();
for (i = __find_memory(&_mmi, (intptr_t)addr >> 16); i < _mmi.i; ++i) {
x = (char *)ADDR_32_TO_48(_mmi.p[i].x);
y = x + _mmi.p[i].size;
if ((x <= addr && addr < y) || (x < addr + size && addr + size <= y) ||
(addr < x && y < addr + size)) {
a = MIN(MAX(addr, x), y);
b = MAX(MIN(addr + size, y), x);
if (!FlushViewOfFile(a, b - a)) {
rc = -1;
break;
}
// TODO(jart): FlushFileBuffers too on g_fds handle if MS_SYNC?
} else {
break;
}
}
__mmi_unlock();
return rc;
}

98
libc/runtime/msync.c
View file

@ -1,98 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/sysv/consts/msync.h"
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/calls/cp.internal.h"
#include "libc/calls/syscall-nt.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/intrin/strace.internal.h"
#include "libc/macros.internal.h"
#include "libc/sysv/errfuns.h"
/**
* Synchronize memory mapping changes to disk.
*
* Without this, there's no guarantee memory is written back to disk.
* Particularly on RHEL5, OpenBSD, and Windows NT.
*
* @param addr needs to be 4096-byte page aligned
* @param flags needs MS_ASYNC or MS_SYNC and can have MS_INVALIDATE
* @return 0 on success or -1 w/ errno
* @raise ECANCELED if thread was cancelled in masked mode
* @raise EINTR if we needed to block and a signal was delivered instead
* @raise EINVAL if `MS_SYNC` and `MS_ASYNC` were both specified
* @raise EINVAL if unknown `flags` were passed
* @cancelationpoint
*/
int msync(void *addr, size_t size, int flags) {
int rc;
if ((flags & ~(MS_SYNC | MS_ASYNC | MS_INVALIDATE)) ||
(flags & (MS_SYNC | MS_ASYNC)) == (MS_SYNC | MS_ASYNC)) {
rc = einval();
goto Finished;
}
// According to POSIX, either MS_SYNC or MS_ASYNC must be specified
// in flags, and indeed failure to include one of these flags will
// cause msync() to fail on some systems. However, Linux permits a
// call to msync() that specifies neither of these flags, with
// semantics that are (currently) equivalent to specifying MS_ASYNC.
// ──Quoth msync(2) of Linux Programmer's Manual
int sysflags = flags;
sysflags = flags;
if (flags & MS_ASYNC) {
sysflags = MS_ASYNC;
} else if (flags & MS_SYNC) {
sysflags = MS_SYNC;
} else {
sysflags = MS_ASYNC;
}
if (flags & MS_INVALIDATE) {
sysflags |= MS_INVALIDATE;
}
// FreeBSD's manual says "The flags argument was both MS_ASYNC and
// MS_INVALIDATE. Only one of these flags is allowed." which makes
// following the POSIX recommendation somewhat difficult.
if (IsFreebsd()) {
if (sysflags == (MS_ASYNC | MS_INVALIDATE)) {
sysflags = MS_INVALIDATE;
}
}
// FreeBSD specifies MS_SYNC as 0 so we shift the Cosmo constants
if (IsFreebsd()) {
sysflags >>= 1;
}
BEGIN_CANCELATION_POINT;
if (!IsWindows()) {
rc = sys_msync(addr, size, sysflags);
} else {
rc = sys_msync_nt(addr, size, sysflags);
}
END_CANCELATION_POINT;
Finished:
STRACE("msync(%p, %'zu, %#x) → %d% m", addr, size, flags, rc);
return rc;
}

160
libc/runtime/munmap.c
View file

@ -1,160 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/state.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/strace.internal.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/libfatal.internal.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
#include "libc/sysv/errfuns.h"
#define IP(X) (intptr_t)(X)
#define ALIGNED(p) (!(IP(p) & (FRAMESIZE - 1)))
#define FRAME(x) ((int)((intptr_t)(x) >> 16))
static void __munmap_shadow(char *p, size_t n) {
intptr_t a, b, x, y;
KERNTRACE("__munmap_shadow(%p, %'zu)", p, n);
a = ((intptr_t)p >> 3) + 0x7fff8000;
b = a + (n >> 3);
if (IsMemtracked(FRAME(a), FRAME(b - 1))) {
x = ROUNDUP(a, FRAMESIZE);
y = ROUNDDOWN(b, FRAMESIZE);
if (0 && x < y) {
// delete shadowspace if unmapping ≥512kb. in practice it has
// to be >1mb since we can only unmap it if it's aligned, and
// as such we poison the edges if there are any.
__repstosb((void *)a, kAsanUnmapped, x - a);
__munmap_unlocked((void *)x, y - x);
__repstosb((void *)y, kAsanUnmapped, b - y);
} else {
// otherwise just poison and assume reuse
__repstosb((void *)a, kAsanUnmapped, b - a);
}
} else {
STRACE("unshadow(%.12p, %p) EFAULT", a, b - a);
}
}
// our api supports doing things like munmap(0, 0x7fffffffffff) but some
// platforms (e.g. openbsd) require that we know the specific intervals
// or else it returns EINVAL. so we munmap a piecewise.
static void __munmap_impl(char *p, size_t n) {
char *q;
size_t m;
intptr_t a, b, c;
int i, l, r, beg, end;
KERNTRACE("__munmap_impl(%p, %'zu)", p, n);
l = FRAME(p);
r = FRAME(p + n - 1);
i = __find_memory(&_mmi, l);
for (; i < _mmi.i && r >= _mmi.p[i].x; ++i) {
if (l >= _mmi.p[i].x && r <= _mmi.p[i].y) {
// it's contained within the entry
beg = l;
end = r;
} else if (l <= _mmi.p[i].x && r >= _mmi.p[i].x) {
// it overlaps with the lefthand side of the entry
beg = _mmi.p[i].x;
end = MIN(r, _mmi.p[i].y);
} else if (l <= _mmi.p[i].y && r >= _mmi.p[i].y) {
// it overlaps with the righthand side of the entry
beg = MAX(_mmi.p[i].x, l);
end = _mmi.p[i].y;
} else {
__builtin_unreachable();
}
// openbsd even requires that if we mapped, for instance a 5 byte
// file, that we be sure to call munmap(file, 5). let's abstract!
a = ADDR_32_TO_48(beg);
b = ADDR_32_TO_48(end) + FRAMESIZE;
c = ADDR_32_TO_48(_mmi.p[i].x) + _mmi.p[i].size;
q = (char *)a;
m = MIN(b, c) - a;
if (!IsWindows()) {
npassert(!sys_munmap(q, m));
} else {
// Handled by __untrack_memories() on Windows
}
if (IsAsan() && !OverlapsShadowSpace(p, n)) {
__munmap_shadow(q, m);
}
}
}
int __munmap_unlocked(char *p, size_t n) {
unassert(!__vforked);
if (UNLIKELY(!n)) {
STRACE("munmap n is 0");
return einval();
}
if (UNLIKELY(!IsLegalSize(n))) {
STRACE("munmap n isn't 48-bit");
return einval();
}
if (UNLIKELY(!IsLegalPointer(p))) {
STRACE("munmap p isn't 48-bit");
return einval();
}
if (UNLIKELY(!IsLegalPointer(p + (n - 1)))) {
STRACE("munmap p+(n-1) isn't 48-bit");
return einval();
}
if (UNLIKELY(!ALIGNED(p))) {
STRACE("munmap(%p) isn't 64kb aligned", p);
return einval();
}
__munmap_impl(p, n);
return __untrack_memories(p, n);
}
/**
* Releases memory pages.
*
* @param p is the beginning of the memory region to unmap
* @param n is the number of bytes to be unmapped
* @return 0 on success, or -1 w/ errno
* @raises EINVAL if `n == 0`
* @raises EINVAL if `n` isn't 48-bit
* @raises EINVAL if `p+(n-1)` isn't 48-bit
* @raises EINVAL if `p` isn't 65536-byte aligned
*/
int munmap(void *p, size_t n) {
int rc;
__mmi_lock();
rc = __munmap_unlocked(p, n);
size_t toto = __strace > 0 ? __get_memtrack_size(&_mmi) : 0;
__mmi_unlock();
STRACE("munmap(%.12p, %'zu) → %d% m (%'zu bytes total)", p, n, rc, toto);
return rc;
}

8
libc/runtime/opensymboltable.greg.c
View file

@ -31,6 +31,7 @@
#include "libc/runtime/runtime.h"
#include "libc/runtime/symbols.internal.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/o.h"
#include "libc/sysv/consts/prot.h"
@ -48,6 +49,7 @@ static struct SymbolTable *OpenSymbolTableImpl(const char *filename) {
size_t n, m, tsz, size;
const Elf64_Sym *symtab, *sym;
ptrdiff_t names_offset, name_base_offset, stp_offset;
long pagesz = getauxval(AT_PAGESZ);
map = MAP_FAILED;
if ((fd = open(filename, O_RDONLY | O_CLOEXEC)) == -1)
return 0;
@ -73,11 +75,11 @@ static struct SymbolTable *OpenSymbolTableImpl(const char *filename) {
tsz += sizeof(unsigned) * n;
name_base_offset = tsz;
tsz += m;
tsz = ROUNDUP(tsz, FRAMESIZE);
tsz = ROUNDUP(tsz, pagesz);
stp_offset = tsz;
size = tsz;
tsz += sizeof(const Elf64_Sym *) * n;
tsz = ROUNDUP(tsz, FRAMESIZE);
tsz = ROUNDUP(tsz, pagesz);
t = mmap(0, tsz, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
if (t == MAP_FAILED)
goto SystemError;
@ -128,7 +130,7 @@ static struct SymbolTable *OpenSymbolTableImpl(const char *filename) {
++j;
}
t->count = j;
munmap(stp, ROUNDUP(sizeof(const Elf64_Sym *) * n, FRAMESIZE));
munmap(stp, sizeof(const Elf64_Sym *) * n);
munmap(map, filesize);
close(fd);
return t;

29
libc/runtime/printmaps.c
View file

@ -1,29 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
/**
* Prints memory mappings to stderr.
*/
void __print_maps(void) {
__mmi_lock();
PrintMemoryIntervals(2, &_mmi);
__mmi_unlock();
}

142
libc/runtime/runtime.h
View file

@ -19,9 +19,9 @@ typedef unsigned long jmp_buf[18];
typedef unsigned long jmp_buf[26];
#endif
void mcount(void);
int daemon(int, int);
unsigned long getauxval(unsigned long);
void mcount(void) libcesque;
int daemon(int, int) libcesque;
unsigned long getauxval(unsigned long) libcesque;
int setjmp(jmp_buf)
libcesque returnstwice paramsnonnull();
void longjmp(jmp_buf, int) libcesque wontreturn paramsnonnull();
@ -37,28 +37,28 @@ void quick_exit(int) wontreturn;
void abort(void) wontreturn;
int atexit(void (*)(void)) paramsnonnull() libcesque;
char *getenv(const char *) paramsnonnull() __wur nosideeffect libcesque;
int putenv(char *);
int setenv(const char *, const char *, int);
int unsetenv(const char *);
int clearenv(void);
void fpreset(void);
void *mmap(void *, uint64_t, int32_t, int32_t, int32_t, int64_t);
int munmap(void *, uint64_t);
int mprotect(void *, uint64_t, int);
int msync(void *, size_t, int);
int mlock(const void *, size_t);
int munlock(const void *, size_t);
long gethostid(void);
int sethostid(long);
char *getlogin(void);
int getlogin_r(char *, size_t);
int login_tty(int);
int getpagesize(void);
int syncfs(int) dontthrow;
int vhangup(void);
int getdtablesize(void);
int sethostname(const char *, size_t);
int acct(const char *);
int putenv(char *) libcesque;
int setenv(const char *, const char *, int) libcesque;
int unsetenv(const char *) libcesque;
int clearenv(void) libcesque;
void fpreset(void) libcesque;
void *mmap(void *, uint64_t, int32_t, int32_t, int32_t, int64_t) libcesque;
int munmap(void *, uint64_t) libcesque;
int mprotect(void *, uint64_t, int) libcesque;
int msync(void *, size_t, int) libcesque;
int mlock(const void *, size_t) libcesque;
int munlock(const void *, size_t) libcesque;
long gethostid(void) libcesque;
int sethostid(long) libcesque;
char *getlogin(void) libcesque;
int getlogin_r(char *, size_t) libcesque;
int login_tty(int) libcesque;
int getpagesize(void) libcesque;
int syncfs(int) dontthrow libcesque;
int vhangup(void) libcesque;
int getdtablesize(void) libcesque;
int sethostname(const char *, size_t) libcesque;
int acct(const char *) libcesque;
#if defined(_GNU_SOURCE) || defined(_COSMO_SOURCE)
extern char **environ;
@ -80,62 +80,62 @@ extern uint64_t kStartTsc;
extern const char kNtSystemDirectory[];
extern const char kNtWindowsDirectory[];
extern size_t __virtualmax;
extern size_t __virtualsize;
extern size_t __stackmax;
extern bool32 __isworker;
/* utilities */
void _intsort(int *, size_t);
void _longsort(long *, size_t);
void _intsort(int *, size_t) libcesque;
void _longsort(long *, size_t) libcesque;
/* diagnostics */
void ShowCrashReports(void);
int ftrace_install(void);
int ftrace_enabled(int);
int strace_enabled(int);
void __print_maps(void);
void __printargs(const char *);
void ShowCrashReports(void) libcesque;
int ftrace_install(void) libcesque;
int ftrace_enabled(int) libcesque;
int strace_enabled(int) libcesque;
void __print_maps(void) libcesque;
void __printargs(const char *) libcesque;
/* builtin sh-like system/popen dsl */
int _cocmd(int, char **, char **);
int _cocmd(int, char **, char **) libcesque;
/* executable program */
char *GetProgramExecutableName(void);
char *GetInterpreterExecutableName(char *, size_t);
int __open_executable(void);
char *GetProgramExecutableName(void) libcesque;
char *GetInterpreterExecutableName(char *, size_t) libcesque;
int __open_executable(void) libcesque;
/* execution control */
int verynice(void);
void __warn_if_powersave(void);
void _Exit1(int) libcesque wontreturn;
void __paginate(int, const char *);
void __paginate_file(int, const char *);
int verynice(void) libcesque;
void __warn_if_powersave(void) libcesque;
void _Exit1(int) libcesque wontreturn libcesque;
void __paginate(int, const char *) libcesque;
void __paginate_file(int, const char *) libcesque;
/* memory management */
void _weakfree(void *);
void *_mapanon(size_t) attributeallocsize((1)) mallocesque;
void *_mapshared(size_t) attributeallocsize((1)) mallocesque;
void CheckForMemoryLeaks(void);
void CheckForFileLeaks(void);
bool32 _isheap(const void *);
void __enable_threads(void);
void __oom_hook(size_t);
void _weakfree(void *) libcesque;
void *_mapanon(size_t) attributeallocsize((1)) mallocesque libcesque;
void *_mapshared(size_t) attributeallocsize((1)) mallocesque libcesque;
void CheckForMemoryLeaks(void) libcesque;
void CheckForFileLeaks(void) libcesque;
void __enable_threads(void) libcesque;
void __oom_hook(size_t) libcesque;
/* code morphing */
void __morph_begin(void);
void __morph_end(void);
void __jit_begin(void);
void __jit_end(void);
void __clear_cache(void *, void *);
void __morph_begin(void) libcesque;
void __morph_end(void) libcesque;
void __jit_begin(void) libcesque;
void __jit_end(void) libcesque;
void __clear_cache(void *, void *) libcesque;
/* portability */
bool32 IsGenuineBlink(void);
bool32 IsCygwin(void);
const char *GetCpuidOs(void);
const char *GetCpuidEmulator(void);
void GetCpuidBrand(char[13], uint32_t);
long __get_rlimit(int);
const char *__describe_os(void);
long __get_sysctl(int, int);
int __get_arg_max(void) pureconst;
int __get_cpu_count(void) pureconst;
long __get_avphys_pages(void) pureconst;
long __get_phys_pages(void) pureconst;
long __get_minsigstksz(void) pureconst;
void __get_main_stack(void **, size_t *, int *);
long __get_safe_size(long, long);
char *__get_tmpdir(void);
bool32 IsGenuineBlink(void) libcesque;
bool32 IsCygwin(void) libcesque;
const char *GetCpuidOs(void) libcesque;
const char *GetCpuidEmulator(void) libcesque;
void GetCpuidBrand(char[13], uint32_t) libcesque;
long __get_rlimit(int) libcesque;
const char *__describe_os(void) libcesque;
long __get_sysctl(int, int) libcesque;
int __granularity(void) pureconst libcesque;
int __get_arg_max(void) pureconst libcesque;
int __get_cpu_count(void) pureconst libcesque;
long __get_avphys_pages(void) pureconst libcesque;
long __get_phys_pages(void) pureconst libcesque;
long __get_minsigstksz(void) pureconst libcesque;
long __get_safe_size(long, long) libcesque;
char *__get_tmpdir(void) libcesque;
forceinline int __trace_disabled(int x) {
return 0;
}

34
libc/runtime/stack.h
View file

@ -14,17 +14,6 @@
*/
#define GetGuardSize() 16384
/**
* Align APE main thread stack at startup.
*
* You need this in your main program module:
*
* STATIC_STACK_ALIGN(GetStackSize());
*
* If you want to use GetStackAddr() and HaveStackMemory() safely on
* your main thread in your process. It causes crt.S to waste a tiny
* amount of memory to ensure those macros go extremely fast.
*/
#define STATIC_STACK_ALIGN(BYTES) \
_STACK_SYMBOL("ape_stack_align", _STACK_STRINGIFY(BYTES) _STACK_EXTRA)
@ -62,29 +51,12 @@ COSMOPOLITAN_C_START_
extern char ape_stack_prot[] __attribute__((__weak__));
extern char ape_stack_memsz[] __attribute__((__weak__));
extern char ape_stack_align[] __attribute__((__weak__));
extern char ape_stack_round[] __attribute__((__weak__));
/**
* Returns address of bottom of current stack.
*
* This always works on threads. If you want it to work on the main
* process too, then you'll need STATIC_STACK_ALIGN(GetStackSize())
* which will burn O(256kb) of memory to ensure thread invariants.
*/
#define GetStackAddr() ((GetStackPointer() - 1) & -GetStackSize())
uintptr_t GetStackBottom(void) pureconst;
#define GetStaticStackSize() ((uintptr_t)ape_stack_memsz)
/**
* Returns true if at least `n` bytes of stack are available.
*
* This always works on threads. If you want it to work on the main
* process too, then you'll need STATIC_STACK_ALIGN(GetStackSize())
* which will burn O(256kb) of memory to ensure thread invariants,
* which make this check exceedingly fast.
*/
#define HaveStackMemory(n) \
(GetStackPointer() >= GetStackAddr() + GetGuardSize() + (n))
/**
* Extends stack memory by poking large allocations.
*
@ -137,8 +109,6 @@ int FreeCosmoStack(void *) libcesque;
: "i"(ADDEND)); \
vAddr; \
})
#else
#define GetStaticStackAddr(ADDEND) (GetStackAddr() + ADDEND)
#endif
#define GetStackPointer() \

4
libc/runtime/sysconf.c
View file

@ -25,9 +25,11 @@
#include "libc/limits.h"
#include "libc/macros.internal.h"
#include "libc/runtime/clktck.h"
#include "libc/intrin/maps.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/sysconf.h"
#include "libc/sysv/consts/_posix.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/limits.h"
#include "libc/sysv/consts/rlimit.h"
#include "libc/sysv/consts/ss.h"
@ -58,7 +60,7 @@ long sysconf(int name) {
case _SC_CLK_TCK:
return CLK_TCK;
case _SC_PAGESIZE:
return FRAMESIZE;
return __granularity();
case _SC_ARG_MAX:
return __get_arg_max();
case _SC_SIGSTKSZ:

31
libc/runtime/untrackmemoryintervals.c
View file

@ -1,31 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/dce.h"
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
int __untrack_memories(void *addr, size_t size) {
int a, b;
unassert(size > 0);
a = ROUNDDOWN((intptr_t)addr, FRAMESIZE) >> 16;
b = ROUNDDOWN((intptr_t)addr + size - 1, FRAMESIZE) >> 16;
return __untrack_memory(&_mmi, a, b,
SupportsWindows() ? __release_memory_nt : 0);
}

28
libc/runtime/winmain.greg.c
View file

@ -20,6 +20,8 @@
#include "libc/calls/internal.h"
#include "libc/calls/sig.internal.h"
#include "libc/calls/syscall_support-nt.internal.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/nomultics.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/limits.h"
@ -192,28 +194,26 @@ static abi wontreturn void WinInit(const char16_t *cmdline) {
__imp_AddVectoredExceptionHandler(true, (void *)OnWinCrash);
// allocate memory for stack and argument block
_mmi.p = _mmi.s;
_mmi.n = ARRAYLEN(_mmi.s);
uintptr_t stackaddr = GetStaticStackAddr(0);
char *stackaddr = (char *)GetStaticStackAddr(0);
size_t stacksize = GetStaticStackSize();
__imp_MapViewOfFileEx(
(_mmi.p[0].h = __imp_CreateFileMappingW(
(__maps.stack.h = __imp_CreateFileMappingW(
-1, 0, kNtPageExecuteReadwrite, stacksize >> 32, stacksize, NULL)),
kNtFileMapWrite | kNtFileMapExecute, 0, 0, stacksize, (void *)stackaddr);
kNtFileMapWrite | kNtFileMapExecute, 0, 0, stacksize, stackaddr);
int prot = (intptr_t)ape_stack_prot;
if (~prot & PROT_EXEC) {
uint32_t old;
__imp_VirtualProtect((void *)stackaddr, stacksize, kNtPageReadwrite, &old);
__imp_VirtualProtect(stackaddr, stacksize, kNtPageReadwrite, &old);
}
uint32_t oldattr;
__imp_VirtualProtect((void *)stackaddr, GetGuardSize(),
__imp_VirtualProtect(stackaddr, GetGuardSize(),
kNtPageReadwrite | kNtPageGuard, &oldattr);
_mmi.p[0].x = stackaddr >> 16;
_mmi.p[0].y = (stackaddr >> 16) + ((stacksize - 1) >> 16);
_mmi.p[0].prot = prot;
_mmi.p[0].flags = 0x00000026; // stack+anonymous
_mmi.p[0].size = stacksize;
_mmi.i = 1;
__maps.stack.addr = stackaddr;
__maps.stack.size = stacksize;
__maps.stack.prot = prot;
__maps.maps = &__maps.stack;
dll_init(&__maps.stack.elem);
dll_make_first(&__maps.used, &__maps.stack.elem);
struct WinArgs *wa =
(struct WinArgs *)(stackaddr + (stacksize - sizeof(struct WinArgs)));
@ -300,7 +300,7 @@ static abi wontreturn void WinInit(const char16_t *cmdline) {
// handover control to cosmopolitan runtime
__stack_call(count, wa->argv, wa->envp, wa->auxv, cosmo,
stackaddr + (stacksize - sizeof(struct WinArgs)));
(uintptr_t)(stackaddr + (stacksize - sizeof(struct WinArgs))));
}
abi int64_t WinMain(int64_t hInstance, int64_t hPrevInstance,

2
libc/runtime/zipos-get.c
View file

@ -62,7 +62,7 @@ static void __zipos_dismiss(uint8_t *map, const uint8_t *cdir, long pg) {
}
// unmap the executable portion beneath the local files
mo = ROUNDDOWN(lo, FRAMESIZE);
mo = ROUNDDOWN(lo, __granularity());
if (mo)
munmap(map, mo);

7
libc/runtime/zipos-mmap.c
View file

@ -20,6 +20,7 @@
#include "libc/calls/struct/iovec.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/strace.internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
@ -37,7 +38,7 @@
* Map zipos file into memory. See mmap.
*
* @param addr should be 0 or a compatible address
* @param size must be >0 and will be rounded up to FRAMESIZE
* @param size must be >0 and will be rounded up to granularity
* automatically.
* @param prot can have PROT_READ/PROT_WRITE/PROT_EXEC/PROT_NONE/etc.
* @param flags cannot have `MAP_SHARED` or `MAP_ANONYMOUS`, there is
@ -76,7 +77,7 @@ void *__zipos_mmap(void *addr, size_t size, int prot, int flags,
flags |= MAP_PRIVATE | MAP_ANONYMOUS;
const int tempProt = !IsXnu() ? prot | PROT_WRITE : PROT_WRITE;
void *outAddr = __mmap_unlocked(addr, size, tempProt, flags, -1, 0);
void *outAddr = __mmap(addr, size, tempProt, flags, -1, 0);
if (outAddr == MAP_FAILED) {
return MAP_FAILED;
}
@ -96,7 +97,7 @@ void *__zipos_mmap(void *addr, size_t size, int prot, int flags,
} while (0);
const int e = errno;
__munmap_unlocked(outAddr, size);
munmap(outAddr, size);
errno = e;
strace_enabled(+1);
return MAP_FAILED;

3
libc/runtime/zipos-stat-impl.c
View file

@ -20,6 +20,7 @@
#include "libc/calls/struct/stat.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/safemacros.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/zipos.internal.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/s.h"
@ -30,7 +31,7 @@ int __zipos_stat_impl(struct Zipos *zipos, size_t cf, struct stat *st) {
bzero(st, sizeof(*st));
st->st_nlink = 1;
st->st_dev = zipos->dev;
st->st_blksize = FRAMESIZE;
st->st_blksize = __granularity();
if (cf == ZIPOS_SYNTHETIC_DIRECTORY) {
st->st_mode = S_IFDIR | (0555 & ~atomic_load_explicit(
&__umask, memory_order_acquire));