mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-03-03 07:29:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Rewrite memory manager

Browse source 
Actually Portable Executable now supports Android. Cosmo's old mmap code
required a 47 bit address space. The new implementation is very agnostic
and supports both smaller address spaces (e.g. embedded) and even modern
56-bit PML5T paging for x86 which finally came true on Zen4 Threadripper

Cosmopolitan no longer requires UNIX systems to observe the Windows 64kb
granularity; i.e. sysconf(_SC_PAGE_SIZE) will now report the host native
page size. This fixes a longstanding POSIX conformance issue, concerning
file mappings that overlap the end of file. Other aspects of conformance
have been improved too, such as the subtleties of address assignment and
and the various subtleties surrounding MAP_FIXED and MAP_FIXED_NOREPLACE

On Windows, mappings larger than 100 megabytes won't be broken down into
thousands of independent 64kb mappings. Support for MAP_STACK is removed
by this change; please use NewCosmoStack() instead.

Stack overflow avoidance is now being implemented using the POSIX thread
APIs. Please use GetStackBottom() and GetStackAddr(), instead of the old
error-prone GetStackAddr() and HaveStackMemory() APIs which are removed.
This commit is contained in:
Justine Tunney 2024-06-20 20:46:42 -07:00
parent 7f6d0b8709
commit 6ffed14b9c
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
150 changed files with 1893 additions and 5634 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
ape/aarch64.lds
View file

@ -1,6 +1,7 @@
/*-*- mode: ld-script; indent-tabs-mode: nil; tab-width: 2; coding: utf-8 -*-│
│ vi: set et sts=2 sw=2 fenc=utf-8 :vi │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/sysv/consts/prot.h"
#include "libc/thread/tls.h"
ENTRY(_start)
@ -285,9 +286,13 @@ SECTIONS {
ape_stack_vaddr = DEFINED(ape_stack_vaddr) ? ape_stack_vaddr : 0x700000000000;
ape_stack_memsz = DEFINED(ape_stack_memsz) ? ape_stack_memsz : 8 * 1024 * 1024;
ape_stack_align = DEFINED(ape_stack_align) ? ape_stack_align : 16;
ape_stack_align = DEFINED(ape_stack_align) ? MAX(ape_stack_align, 16) : 16;
ape_stack_prot = PROT_READ | PROT_WRITE;
ape_stack_round = -ape_stack_align;
ASSERT(ape_stack_align < 1048576, "stack align too big");
ASSERT(!(ape_stack_align & (ape_stack_align - 1)), "stack align must be two power");
_tls_size = _tbss_end - _tdata_start;
_tdata_size = _tdata_end - _tdata_start;
_tbss_size = _tbss_end - _tbss_start;

5
ape/ape.lds
View file

@ -588,9 +588,12 @@ ape_stack_vaddr = DEFINED(ape_stack_vaddr) ? ape_stack_vaddr : 0x700000000000;
ape_stack_paddr = ape_ram_paddr + ape_ram_filesz;
ape_stack_filesz = 0;
ape_stack_memsz = DEFINED(ape_stack_memsz) ? ape_stack_memsz : 8 * 1024 * 1024;
ape_stack_align = DEFINED(ape_stack_align) ? ape_stack_align : 16;
ape_stack_align = DEFINED(ape_stack_align) ? MAX(ape_stack_align, 16) : 16;
ape_stack_round = -ape_stack_align;
ASSERT(ape_stack_align < 1048576, "stack align too big");
ASSERT(!(ape_stack_align & (ape_stack_align - 1)), "stack align must be two power");
ape_note_offset = ape_cod_offset + (ape_note - ape_cod_vaddr);
ape_note_filesz = ape_note_end - ape_note;
ape_note_memsz = ape_note_filesz;

2
ctl/README.md
View file

@ -12,7 +12,7 @@ This package provides an STL alternative that's designed to:
This code is experimental and mostly for Justine to save herself from
the STL trainwreck. That doesn't mean cosmocc users aren't welcome to
use tihs too. You shouldn't expect the design to not change later on.
use this too. You shouldn't expect the design to not change later on.
## Rationale

2
examples/nesemu1.cc
View file

@ -137,7 +137,7 @@ struct Action {
struct Audio {
size_t i;
int16_t p[FRAMESIZE];
int16_t p[65536];
};
struct Status {

1
libc/calls/finddebugbinary.c
View file

@ -27,7 +27,6 @@
#include "libc/elf/tinyelf.internal.h"
#include "libc/errno.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/nt/memory.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/runtime.h"

37
libc/calls/virtualmax.S
View file

@ -1,37 +0,0 @@
/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│
│ vi: set noet ft=asm ts=8 sw=8 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macros.internal.h"
.initbss 201,_init___virtualmax
// Maximum amount of virtual memory in bytes.
//
// mmap() will return ENOMEM once this is reached.
//
// By default no limit is imposed.
__virtualmax:
.quad 0
.endobj __virtualmax,globl
.previous
.init.start 201,_init___virtualmax
push $-1
pop %rax
stosq
.init.end 201,_init___virtualmax

1
libc/integral/normalize.inc
View file

@ -84,7 +84,6 @@
#endif
#ifdef _COSMO_SOURCE
#define FRAMESIZE 65536
#define _PAGESIZE 4096
#endif

3
libc/intrin/BUILD.mk
View file

@ -66,8 +66,7 @@ o/$(MODE)/libc/intrin/asan.o: private \
CFLAGS += \
-O2 \
-finline \
-finline-functions \
-fpatchable-function-entry=0,0
-finline-functions
o//libc/intrin/memmove.o: private \
CFLAGS += \

261
libc/intrin/asan.c
View file

@ -16,6 +16,7 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "ape/sections.internal.h"
#include "libc/calls/state.internal.h"
#include "libc/calls/struct/rlimit.h"
#include "libc/calls/struct/rlimit.internal.h"
@ -30,6 +31,7 @@
#include "libc/intrin/kprintf.h"
#include "libc/intrin/leaky.internal.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/log/libfatal.internal.h"
@ -94,13 +96,6 @@ __static_yoink("_init_asan");
#define ASAN_LOG(...) (void)0 // kprintf(__VA_ARGS__)
#define REQUIRE(FUNC) \
do { \
if (!_weaken(FUNC)) { \
__asan_require_failed(#FUNC); \
} \
} while (0)
struct AsanSourceLocation {
char *filename;
int line;
@ -161,7 +156,7 @@ static char *__asan_stpcpy(char *d, const char *s) {
}
}
void __asan_memset(void *p, char c, size_t n) {
dontinstrument void __asan_memset(void *p, char c, size_t n) {
char *b;
size_t i;
uint64_t x;
@ -313,20 +308,16 @@ static wontreturn void __asan_require_failed(const char *func) {
__asan_unreachable();
}
void __asan_poison(void *p, long n, signed char t) {
signed char k, *s;
s = (signed char *)(((intptr_t)p >> 3) + 0x7fff8000);
if ((k = (intptr_t)p & 7)) {
if ((!*s && n >= 8 - k) || *s > k)
*s = k;
n -= MIN(8 - k, n);
s += 1;
}
__asan_memset(s, t, n >> 3);
if ((k = n & 7)) {
s += n >> 3;
if (*s < 0 || (*s > 0 && *s <= k))
*s = t;
static bool __asan_overlaps_shadow_space(const void *p, size_t n) {
intptr_t BegA, EndA, BegB, EndB;
if (n) {
BegA = (intptr_t)p;
EndA = BegA + n;
BegB = 0x7fff0000;
EndB = 0x100080000000;
return MAX(BegA, BegB) < MIN(EndA, EndB);
} else {
return 0;
}
}
@ -353,17 +344,92 @@ void __asan_unpoison(void *p, long n) {
}
}
bool __asan_is_mapped(int x) {
// xxx: we can't lock because no reentrant locks yet
int i;
bool res;
__mmi_lock();
i = __find_memory(&_mmi, x);
res = i < _mmi.i && x >= _mmi.p[i].x;
__mmi_unlock();
dontinstrument void __asan_poison(void *p, long n, signed char t) {
signed char k, *s;
s = (signed char *)(((intptr_t)p >> 3) + 0x7fff8000);
if ((k = (intptr_t)p & 7)) {
if ((!*s && n >= 8 - k) || *s > k)
*s = k;
n -= MIN(8 - k, n);
s += 1;
}
__asan_memset(s, t, n >> 3);
if ((k = n & 7)) {
s += n >> 3;
if (*s < 0 || (*s > 0 && *s <= k))
*s = t;
}
}
static void __asan_poison_safe(char *p, long n, signed char t) {
int granularity = __granularity();
int chunk = granularity << 3;
while (n > 0) {
char *block = (char *)((uintptr_t)p & -chunk);
signed char *shadow = (signed char *)(((uintptr_t)block >> 3) + 0x7fff8000);
if (__asan_is_mapped(shadow)) {
char *start = MAX(p, block);
size_t bytes = MIN(n, chunk);
__asan_poison(start, bytes, t);
}
p += chunk;
n -= chunk;
}
}
privileged static bool __asan_is_mapped_unlocked(const char *addr) {
struct Dll *e, *e2;
for (e = dll_first(__maps.used); e; e = e2) {
e2 = dll_next(__maps.used, e);
struct Map *map = MAP_CONTAINER(e);
if (map->addr <= addr && addr < map->addr + map->size) {
dll_remove(&__maps.used, e);
dll_make_first(&__maps.used, e);
return true;
}
}
return false;
}
privileged bool __asan_is_mapped(const void *addr) {
bool32 res;
__maps_lock();
res = __asan_is_mapped_unlocked(addr);
__maps_unlock();
return res;
}
static void __asan_ensure_shadow_is_mapped(void *addr, size_t size) {
int m = 0x7fff8000;
int g = __granularity();
size_t n = (size + 7) & -8;
char *p = (char *)((((uintptr_t)addr >> 3) + m) & -g);
char *pe = (char *)(((((uintptr_t)(addr + n) >> 3) + m) + g - 1) & -g);
for (char *pm = p; p < pe; p = pm) {
pm += g;
if (!__asan_is_mapped(p)) {
while (pm < pe && !__asan_is_mapped(pm))
pm += g;
if (__mmap(p, pm - p, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) != MAP_FAILED)
__asan_memset(p, kAsanUnmapped, pm - p);
}
}
}
void __asan_shadow(void *addr, size_t size) {
if (!__asan_overlaps_shadow_space(addr, size)) {
__asan_ensure_shadow_is_mapped(addr, size);
__asan_unpoison(addr, size);
}
}
void __asan_unshadow(void *addr, size_t size) {
if (!__asan_overlaps_shadow_space(addr, size)) {
__asan_poison_safe(addr, size, kAsanUnmapped);
}
}
static bool __asan_is_image(const unsigned char *p) {
return __executable_start <= p && p < _end;
}
@ -387,13 +453,15 @@ static struct AsanFault __asan_fault(const signed char *s, signed char dflt) {
static struct AsanFault __asan_checka(const signed char *s, long ndiv8) {
uint64_t w;
int g = __granularity();
const signed char *o = s;
const signed char *e = s + ndiv8;
for (; ((intptr_t)s & 7) && s < e; ++s) {
if (*s)
return __asan_fault(s - 1, kAsanHeapOverrun);
}
for (; s + 8 <= e; s += 8) {
if (UNLIKELY(!((intptr_t)s & (FRAMESIZE - 1))) && kisdangerous(s)) {
if (UNLIKELY(!((intptr_t)s & (g - 1))) && kisdangerous(s)) {
return (struct AsanFault){kAsanUnmapped, s};
}
if ((w = READ64LE(s))) {
@ -403,7 +471,7 @@ static struct AsanFault __asan_checka(const signed char *s, long ndiv8) {
}
for (; s < e; ++s) {
if (*s)
return __asan_fault(s - 1, kAsanHeapOverrun);
return __asan_fault(s > o ? s - 1 : s, kAsanHeapOverrun);
}
return (struct AsanFault){0};
}
@ -429,7 +497,7 @@ struct AsanFault __asan_check(const void *p, long n) {
if (n > 0) {
k = (intptr_t)p & 7;
s = SHADOW(p);
if (OverlapsShadowSpace(p, n)) {
if (__asan_overlaps_shadow_space(p, n)) {
return (struct AsanFault){kAsanProtected, s};
} else if (kisdangerous(s)) {
return (struct AsanFault){kAsanUnmapped, s};
@ -475,8 +543,9 @@ struct AsanFault __asan_check(const void *p, long n) {
struct AsanFault __asan_check_str(const char *p) {
uint64_t w;
signed char c, k, *s;
int g = __granularity();
s = SHADOW(p);
if (OverlapsShadowSpace(p, 1)) {
if (__asan_overlaps_shadow_space(p, 1)) {
return (struct AsanFault){kAsanProtected, s};
}
if (kisdangerous(s)) {
@ -494,7 +563,7 @@ struct AsanFault __asan_check_str(const char *p) {
++s;
}
for (;; ++s, p += 8) {
if (UNLIKELY(!((intptr_t)s & (FRAMESIZE - 1))) && kisdangerous(s)) {
if (UNLIKELY(!((intptr_t)s & (g - 1))) && kisdangerous(s)) {
return (struct AsanFault){kAsanUnmapped, s};
}
if ((c = *s) < 0) {
@ -749,9 +818,9 @@ void __asan_report_memory_origin(const unsigned char *addr, int size,
}
if (__executable_start <= addr && addr < _end) {
__asan_report_memory_origin_image((intptr_t)addr, size);
} else if (IsAutoFrame((intptr_t)addr >> 16)) {
if (_weaken(__asan_report_memory_origin_heap))
_weaken(__asan_report_memory_origin_heap)(addr, size);
/* } else if (IsAutoFrame((intptr_t)addr >> 16)) { */
/* if (_weaken(__asan_report_memory_origin_heap)) */
/* _weaken(__asan_report_memory_origin_heap)(addr, size); */
}
}
@ -766,9 +835,8 @@ static __wur __asan_die_f *__asan_report(const void *addr, int size,
int i;
wint_t c;
signed char t;
uint64_t x, y, z;
uint64_t z;
char *base, *q, *p = buf;
struct MemoryIntervals *m;
ftrace_enabled(-1);
kprintf("\n\e[J\e[1;31masan error\e[0m: %s %d-byte %s at %p shadow %p\n",
__asan_describe_access_poison(kind), size, message, addr,
@ -832,20 +900,20 @@ static __wur __asan_die_f *__asan_report(const void *addr, int size,
p = __asan_format_section(p, __executable_start, _etext, ".text", addr);
p = __asan_format_section(p, _etext, _edata, ".data", addr);
p = __asan_format_section(p, _end, _edata, ".bss", addr);
__mmi_lock();
for (m = &_mmi, i = 0; i < m->i; ++i) {
x = m->p[i].x;
y = m->p[i].y;
p = __asan_format_interval(p, x << 16, (y << 16) + (FRAMESIZE - 1));
z = (intptr_t)addr >> 16;
if (x <= z && z <= y)
__maps_lock();
struct Dll *e, *e2;
for (e = dll_first(__maps.used); e; e = e2) {
e2 = dll_next(__maps.used, e);
struct Map *map = MAP_CONTAINER(e);
p = __asan_format_interval(p, (uintptr_t)map->addr,
(uintptr_t)map->addr + map->size);
if (!__asan_overlaps_shadow_space(map->addr, map->size))
p = __asan_stpcpy(p, " ←address");
z = (((intptr_t)addr >> 3) + 0x7fff8000) >> 16;
if (x <= z && z <= y)
else
p = __asan_stpcpy(p, " ←shadow");
*p++ = '\n';
}
__mmi_unlock();
__maps_unlock();
*p = 0;
kprintf("%s", buf);
__asan_report_memory_origin(addr, size, kind);
@ -1033,95 +1101,16 @@ void __asan_after_dynamic_init(void) {
ASAN_LOG("__asan_after_dynamic_init()\n");
}
void __asan_map_shadow(uintptr_t p, size_t n) {
// assume _mmi.lock is held
void *addr;
int i, a, b;
size_t size;
int prot, flag;
struct DirectMap sm;
struct MemoryIntervals *m;
if (OverlapsShadowSpace((void *)p, n)) {
kprintf("error: %p size %'zu overlaps shadow space: %s\n", p, n,
DescribeBacktrace(__builtin_frame_address(0)));
_Exit(1);
}
m = &_mmi;
a = (0x7fff8000 + (p >> 3)) >> 16;
b = (0x7fff8000 + (p >> 3) + (n >> 3) + 0xffff) >> 16;
for (; a <= b; a += i) {
i = 1;
if (__asan_is_mapped(a)) {
continue;
}
for (; a + i <= b; ++i) {
if (__asan_is_mapped(a + i)) {
break;
}
}
size = (size_t)i << 16;
addr = (void *)ADDR_32_TO_48(a);
prot = PROT_READ | PROT_WRITE;
flag = MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS;
sm = sys_mmap(addr, size, prot, flag, -1, 0);
if (sm.addr == MAP_FAILED ||
__track_memory(m, a, a + i - 1, sm.maphandle, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, false, false, 0,
size) == -1) {
kprintf("error: could not map asan shadow memory\n");
__asan_die()();
__asan_unreachable();
}
__repstosb(addr, kAsanUnmapped, size);
}
__asan_unpoison((char *)p, n);
}
static textstartup void __asan_shadow_mapping(struct MemoryIntervals *m,
size_t i) {
uintptr_t x, y;
if (i < m->i) {
x = m->p[i].x;
y = m->p[i].y;
__asan_shadow_mapping(m, i + 1);
__asan_map_shadow(x << 16, (y - x + 1) << 16);
}
}
static textstartup void __asan_shadow_existing_mappings(void) {
__asan_shadow_mapping(&_mmi, 0);
if (!IsWindows()) {
int guard;
void *addr;
size_t size;
__get_main_stack(&addr, &size, &guard);
__asan_map_shadow((uintptr_t)addr, size);
__asan_poison(addr, guard, kAsanStackOverflow);
}
}
static size_t __asan_strlen(const char *s) {
size_t i = 0;
while (s[i])
++i;
return i;
}
forceinline ssize_t __write_str(const char *s) {
return sys_write(2, s, __asan_strlen(s));
}
void __asan_init(int argc, char **argv, char **envp, unsigned long *auxv) {
void __asan_init(void) {
static bool once;
if (!_cmpxchg(&once, false, true))
if (once)
return;
__asan_shadow_existing_mappings();
__asan_map_shadow((uintptr_t)__executable_start, _end - __executable_start);
__asan_map_shadow(0, 4096);
once = true;
__asan_shadow(0, 4096);
__asan_shadow(__maps.stack.addr, __maps.stack.size);
__asan_shadow(__executable_start, _end - __executable_start);
__asan_poison((void *)__veil("r", 0L), getauxval(AT_PAGESZ), kAsanNullPage);
if (!IsWindows()) {
sys_mprotect((void *)0x7fff8000, 0x10000, PROT_READ);
}
STRACE(" _ ____ _ _ _ ");
STRACE(" / \\ / ___| / \\ | \\ | |");
STRACE(" / _ \\ \\___ \\ / _ \\ | \\| |");

5
libc/intrin/asan.internal.h
View file

@ -46,7 +46,8 @@ void __asan_unpoison(void *, long);
void __asan_poison(void *, long, signed char);
void __asan_verify(const void *, size_t);
void __asan_verify_str(const char *);
void __asan_map_shadow(uintptr_t, size_t);
void __asan_shadow(void *, size_t);
void __asan_unshadow(void *, size_t);
bool __asan_is_valid(const void *, long) nosideeffect;
bool __asan_is_valid_str(const char *) nosideeffect;
bool __asan_is_valid_strlist(char *const *) nosideeffect;
@ -54,9 +55,9 @@ bool __asan_is_valid_iov(const struct iovec *, int) nosideeffect;
struct AsanFault __asan_check(const void *, long) nosideeffect;
struct AsanFault __asan_check_str(const char *) nosideeffect;
bool __asan_is_mapped(int);
int __asan_is_leaky(void *);
int __asan_print_trace(void *);
bool __asan_is_mapped(const void *);
__asan_die_f *__asan_die(void) __wur;
void __asan_memset(void *, char, size_t);
size_t __asan_get_heap_size(const void *);

9
libc/intrin/asaninit.S
View file

@ -19,14 +19,13 @@
#include "libc/macros.internal.h"
.init.start 303,_init_asan
#ifdef __x86_64__
#ifdef __SANITIZE_ADDRESS__
push %rdi
push %rsi
mov %r12,%rdi
mov %r13,%rsi
mov %r14,%rdx
mov %r15,%rcx
.weak __asan_init
call __asan_init
pop %rsi
pop %rdi
#endif /* __SANITIZE_ADDRESS__ */
#endif /* __x86_64__ */
.init.end 303,_init_asan

2
libc/intrin/describeflags.internal.h
View file

@ -21,7 +21,6 @@ const char *DescribeDnotifyFlags(char[80], int) libcesque;
const char *DescribeErrno(char[30], int) libcesque;
const char *DescribeFcntlCmd(char[20], int) libcesque;
const char *DescribeFlockType(char[12], int) libcesque;
const char *DescribeFrame(char[32], int) libcesque;
const char *DescribeFutexOp(char[64], int) libcesque;
const char *DescribeHow(char[12], int) libcesque;
const char *DescribeInOutInt64(char[23], ssize_t, int64_t *) libcesque;
@ -80,7 +79,6 @@ const char *DescribeWhichPrio(char[12], int) libcesque;
#define DescribeErrno(x) DescribeErrno(alloca(30), x)
#define DescribeFcntlCmd(x) DescribeFcntlCmd(alloca(20), x)
#define DescribeFlockType(x) DescribeFlockType(alloca(12), x)
#define DescribeFrame(x) DescribeFrame(alloca(32), x)
#define DescribeFutexOp(x) DescribeFutexOp(alloca(64), x)
#define DescribeHow(x) DescribeHow(alloca(12), x)
#define DescribeInOutInt64(rc, x) DescribeInOutInt64(alloca(23), rc, x)

72
libc/intrin/describeframe.c
View file

@ -1,72 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/dce.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/weaken.h"
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/winargs.internal.h"
#define UNSHADOW(x) ((int64_t)(MAX(0, (x) - 0x7fff8000)) << 3)
#define FRAME(x) ((int)((x) >> 16))
static const char *GetFrameName(int x) {
if (!x) {
return "null";
} else if (IsShadowFrame(x)) {
return "shadow";
} else if (IsAutoFrame(x)) {
return "automap";
} else if (IsFixedFrame(x)) {
return "fixed";
} else if (IsStaticStackFrame(x)) {
return "stack";
} else if (IsGfdsFrame(x)) {
return "g_fds";
} else if (IsNsyncFrame(x)) {
return "nsync";
} else if (IsZiposFrame(x)) {
return "zipos";
} else if (IsMemtrackFrame(x)) {
return "memtrack";
} else if (IsOldStackFrame(x)) {
return "system stack";
} else if (((GetStaticStackAddr(0) + GetStackSize()) >> 16) <= x &&
x <= ((GetStaticStackAddr(0) + GetStackSize() - 1) >> 16)) {
return "static stack";
} else if ((int)((intptr_t)__executable_start >> 16) <= x &&
x <= (int)(((intptr_t)_end - 1) >> 16)) {
return "image";
} else {
return "unknown";
}
}
const char *(DescribeFrame)(char buf[32], int x) {
if (IsShadowFrame(x)) {
ksnprintf(buf, 32, "%s %s %.8x", GetFrameName(x),
GetFrameName(FRAME(UNSHADOW(ADDR_32_TO_48(x)))),
FRAME(UNSHADOW(ADDR_32_TO_48(x))));
return buf;
} else {
return GetFrameName(x);
}
}

1
libc/intrin/describemapflags.c
View file

@ -24,7 +24,6 @@
const char *(DescribeMapFlags)(char buf[64], int x) {
const struct DescribeFlags kMapFlags[] = {
{MAP_STACK, "STACK"}, // order matters
{MAP_PRIVATE, "PRIVATE"}, //
{MAP_ANONYMOUS, "ANONYMOUS"}, //
{MAP_SHARED, "SHARED"}, //

2
libc/intrin/describemapping.c
View file

@ -29,8 +29,6 @@ static char DescribeMapType(int flags) {
return 'p';
case MAP_SHARED:
return 's';
case MAP_STACK:
return 'S';
default:
return '?';
}

2
libc/intrin/directmap-nt.c
View file

@ -86,7 +86,7 @@ TryAgain:
if ((dm.addr = MapViewOfFileEx(dm.maphandle, fl.flags2, off >> 32, off,
size, addr))) {
uint32_t oldprot;
if (VirtualProtect(addr, size, __prot2nt(prot, iscow), &oldprot)) {
if (VirtualProtect(dm.addr, size, __prot2nt(prot, iscow), &oldprot)) {
return dm;
}
UnmapViewOfFile(dm.addr);

17
libc/intrin/directmap.c
View file

@ -16,6 +16,7 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
@ -24,7 +25,9 @@
#include "libc/intrin/strace.internal.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/syslib.internal.h"
#include "libc/sysv/errfuns.h"
/**
* Obtains memory mapping directly from system.
@ -39,7 +42,10 @@
struct DirectMap sys_mmap(void *addr, size_t size, int prot, int flags, int fd,
int64_t off) {
struct DirectMap d;
if (IsXnuSilicon()) {
if ((__virtualsize += size) >= __virtualmax) {
d.maphandle = kNtInvalidHandleValue;
d.addr = (void *)enomem();
} else if (IsXnuSilicon()) {
long p = _sysret(__syslib->__mmap(addr, size, prot, flags, fd, off));
d.maphandle = kNtInvalidHandleValue;
d.addr = (void *)p;
@ -51,9 +57,10 @@ struct DirectMap sys_mmap(void *addr, size_t size, int prot, int flags, int fd,
} else {
d = sys_mmap_nt(addr, size, prot, flags, fd, off);
}
KERNTRACE("sys_mmap(%.12p /* %s */, %'zu, %s, %s, %d, %'ld) → {%.12p, %p}% m",
addr, DescribeFrame((intptr_t)addr >> 16), size,
DescribeProtFlags(prot), DescribeMapFlags(flags), fd, off, d.addr,
d.maphandle);
if (d.addr == MAP_FAILED)
__virtualsize -= size;
KERNTRACE("sys_mmap(%.12p, %'zu, %s, %s, %d, %'ld) → {%.12p, %p}% m", addr,
size, DescribeProtFlags(prot), DescribeMapFlags(flags), fd, off,
d.addr, d.maphandle);
return d;
}

8
libc/intrin/dll.c
View file

@ -28,7 +28,7 @@
*
* It's required that `elem` and `succ` aren't part of the same list.
*/
void dll_splice_after(struct Dll *elem, struct Dll *succ) {
privileged void dll_splice_after(struct Dll *elem, struct Dll *succ) {
struct Dll *tmp1, *tmp2;
tmp1 = elem->next;
tmp2 = succ->prev;
@ -43,7 +43,7 @@ void dll_splice_after(struct Dll *elem, struct Dll *succ) {
*
* @param list is a doubly-linked list, where `!*list` means empty
*/
void dll_remove(struct Dll **list, struct Dll *elem) {
privileged void dll_remove(struct Dll **list, struct Dll *elem) {
if (*list == elem) {
if ((*list)->prev == *list) {
*list = 0;
@ -66,7 +66,7 @@ void dll_remove(struct Dll **list, struct Dll *elem) {
* @param list is a doubly-linked list, where `!*list` means empty
* @param elem must not be a member of `list`, or null for no-op
*/
void dll_make_first(struct Dll **list, struct Dll *elem) {
privileged void dll_make_first(struct Dll **list, struct Dll *elem) {
if (elem) {
if (!*list) {
*list = elem->prev;
@ -85,7 +85,7 @@ void dll_make_first(struct Dll **list, struct Dll *elem) {
* @param list is a doubly-linked list, where `!*list` means empty
* @param elem must not be a member of `list`, or null for no-op
*/
void dll_make_last(struct Dll **list, struct Dll *elem) {
privileged void dll_make_last(struct Dll **list, struct Dll *elem) {
if (elem) {
dll_make_first(list, elem->next);
*list = elem;

14
libc/intrin/dll.h
View file

@ -14,38 +14,38 @@ struct Dll {
struct Dll *prev;
};
static inline void dll_init(struct Dll *e) {
forceinline void dll_init(struct Dll *e) {
e->next = e;
e->prev = e;
}
static inline int dll_is_alone(struct Dll *e) {
forceinline int dll_is_alone(struct Dll *e) {
return e->next == e && e->prev == e;
}
static inline int dll_is_empty(struct Dll *list) {
forceinline int dll_is_empty(struct Dll *list) {
return !list;
}
static inline struct Dll *dll_last(struct Dll *list) {
forceinline struct Dll *dll_last(struct Dll *list) {
return list;
}
static inline struct Dll *dll_first(struct Dll *list) {
forceinline struct Dll *dll_first(struct Dll *list) {
struct Dll *first = 0;
if (list)
first = list->next;
return first;
}
static inline struct Dll *dll_next(struct Dll *list, struct Dll *e) {
forceinline struct Dll *dll_next(struct Dll *list, struct Dll *e) {
struct Dll *next = 0;
if (e != list)
next = e->next;
return next;
}
static inline struct Dll *dll_prev(struct Dll *list, struct Dll *e) {
forceinline struct Dll *dll_prev(struct Dll *list, struct Dll *e) {
struct Dll *prev = 0;
if (e != list->next)
prev = e->prev;

68
libc/intrin/extend.c
View file

@ -16,51 +16,14 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/calls/struct/sigset.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/asancodes.h"
#include "libc/intrin/describebacktrace.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/log/libfatal.internal.h"
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
#define G FRAMESIZE
static void *_mapframe_impl(void *p, int f) {
int rc, prot, flags;
struct DirectMap dm;
prot = PROT_READ | PROT_WRITE;
flags = f | MAP_ANONYMOUS | MAP_FIXED;
if ((dm = sys_mmap(p, G, prot, flags, -1, 0)).addr == p) {
rc = __track_memory(&_mmi, (uintptr_t)p >> 16, (uintptr_t)p >> 16,
dm.maphandle, prot, flags, false, false, 0, G);
if (!rc) {
return p;
} else {
return 0;
}
} else {
return 0;
}
}
static void *_mapframe(void *p, int f) {
void *res;
// mmap isn't required to be @asyncsignalsafe but this is
BLOCK_SIGNALS;
__mmi_lock();
res = _mapframe_impl(p, f);
__mmi_unlock();
ALLOW_SIGNALS;
return res;
}
#define G __granularity()
/**
* Extends static allocation.
@ -84,32 +47,23 @@ static void *_mapframe(void *p, int f) {
*/
void *_extend(void *p, size_t n, void *e, int f, intptr_t h) {
char *q;
#ifndef NDEBUG
#ifdef __SANITIZE_ADDRESS__
if ((uintptr_t)SHADOW(p) & (G - 1))
notpossible;
__builtin_trap();
if ((uintptr_t)p + (G << kAsanScale) > h)
notpossible;
__builtin_trap();
#endif
// TODO(jart): Make this spin less in non-ASAN mode.
for (q = e; q < ((char *)p + n); q += 8) {
if (!((uintptr_t)q & (G - 1))) {
#ifndef NDEBUG
if (q + G > (char *)h)
notpossible;
#endif
if (!_mapframe(q, f))
__builtin_trap();
if (mmap(q, G, PROT_READ | PROT_WRITE, f | MAP_ANONYMOUS | MAP_FIXED, -1,
0) == MAP_FAILED)
return 0;
if (IsAsan()) {
if (!((uintptr_t)SHADOW(q) & (G - 1))) {
if (!_mapframe(SHADOW(q), f))
return 0;
__asan_poison(q, G << kAsanScale, kAsanProtected);
}
}
}
if (IsAsan()) {
*SHADOW(q) = 0;
}
}
return q;
}

34
libc/intrin/findmemoryinterval.c
View file

@ -1,34 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/memtrack.internal.h"
unsigned __find_memory(const struct MemoryIntervals *mm, int x) {
unsigned l, m, r;
l = 0;
r = mm->i;
while (l < r) {
m = (l & r) + ((l ^ r) >> 1); // floor((a+b)/2)
if (mm->p[m].y < x) {
l = m + 1;
} else {
r = m;
}
}
return l;
}

26
libc/intrin/getmainstack.c
View file

@ -16,14 +16,14 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/struct/rlimit.h"
#include "libc/calls/struct/rlimit.internal.h"
#include "libc/dce.h"
#include "libc/intrin/getauxval.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/maps.h"
#include "libc/macros.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/rlim.h"
#include "libc/sysv/consts/rlimit.h"
@ -101,30 +101,22 @@ static size_t __get_stack_size(int pagesz, uintptr_t start, uintptr_t top) {
/**
* Returns approximate boundaries of main thread stack.
*
* This function works on every OS except Windows.
*/
void __get_main_stack(void **out_addr, size_t *out_size, int *out_guardsize) {
if (IsWindows()) {
*out_addr = (void *)GetStaticStackAddr(0);
*out_size = GetStaticStackSize();
*out_guardsize = getauxval(AT_PAGESZ);
return;
}
struct AddrSize __get_main_stack(void) {
int pagesz = getauxval(AT_PAGESZ);
uintptr_t start = (uintptr_t)__argv;
uintptr_t top = __get_main_top(pagesz);
uintptr_t bot = top - __get_stack_size(pagesz, start, top);
uintptr_t vdso = getauxval(AT_SYSINFO_EHDR);
if (vdso) {
struct AuxiliaryValue avdso = __getauxval(AT_SYSINFO_EHDR);
if (avdso.isfound) {
uintptr_t vdso = avdso.value;
if (vdso > start && vdso < top) {
top = vdso;
} else if (vdso < start && vdso >= bot) {
bot += vdso + pagesz * 2;
}
}
unassert(bot < top);
unassert(bot < start);
unassert(top > start);
*out_addr = (void *)bot;
*out_size = top - bot;
*out_guardsize = pagesz;
return (struct AddrSize){(char *)bot, top - bot};
}

5
libc/intrin/getsafesize.greg.c
View file

@ -37,11 +37,6 @@ privileged long __get_safe_size(long want, long extraspace) {
return want;
struct PosixThread *pt;
struct CosmoTib *tib = __get_tls_privileged();
if (!IsAutoFrame((uintptr_t)tib >> 16) &&
!(__executable_start <= (const unsigned char *)tib &&
(const unsigned char *)tib < _end)) {
return want;
}
long bottom, sp = GetStackPointer();
if ((char *)sp >= tib->tib_sigstack_addr &&
(char *)sp <= tib->tib_sigstack_addr + tib->tib_sigstack_size) {

19
libc/runtime/printmaps.c → libc/intrin/granularity.c
View file

@ -1,7 +1,7 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
@ -16,14 +16,15 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/memtrack.internal.h"
#include "libc/dce.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/auxv.h"
/**
* Prints memory mappings to stderr.
*/
void __print_maps(void) {
__mmi_lock();
PrintMemoryIntervals(2, &_mmi);
__mmi_unlock();
int __granularity(void) {
if (IsWindows())
return 65536;
static int res;
if (!res)
res = getauxval(AT_PAGESZ);
return res;
}

11
libc/runtime/getmemtracksize.c → libc/intrin/klog.c
View file

@ -1,7 +1,7 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
@ -16,12 +16,5 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/memtrack.internal.h"
size_t __get_memtrack_size(struct MemoryIntervals *mm) {
size_t i, n;
for (n = i = 0; i < mm->i; ++i) {
n += ((size_t)(mm->p[i].y - mm->p[i].x) + 1) << 16;
}
return n;
}
long __klog_handle;

86
libc/intrin/kprintf.greg.c
View file

@ -25,8 +25,11 @@
#include "libc/fmt/magnumstrs.internal.h"
#include "libc/intrin/asmflag.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/getenv.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/nomultics.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/log/internal.h"
@ -45,9 +48,11 @@
#include "libc/runtime/internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/runtime/symbols.internal.h"
#include "libc/serialize.h"
#include "libc/stdckdint.h"
#include "libc/stdio/sysparam.h"
#include "libc/str/str.h"
#include "libc/str/tab.internal.h"
#include "libc/str/utf16.h"
@ -117,7 +122,7 @@ __msabi extern typeof(SetLastError) *const __imp_SetLastError;
__msabi extern typeof(WriteFile) *const __imp_WriteFile;
// clang-format on
long __klog_handle;
extern long __klog_handle;
extern struct SymbolTable *__symtab;
__funline char *kadvance(char *p, char *e, long n) {
@ -141,20 +146,6 @@ __funline char *kemitquote(char *p, char *e, signed char t, unsigned c) {
return p;
}
__funline bool kiskernelpointer(const void *p) {
return 0x7f0000000000 <= (intptr_t)p && (intptr_t)p < 0x800000000000;
}
__funline bool kistextpointer(const void *p) {
return __executable_start <= (const unsigned char *)p &&
(const unsigned char *)p < _etext;
}
__funline bool kisimagepointer(const void *p) {
return __executable_start <= (const unsigned char *)p &&
(const unsigned char *)p < _end;
}
__funline bool kischarmisaligned(const char *p, signed char t) {
if (t == -1)
return (intptr_t)p & 1;
@ -163,55 +154,26 @@ __funline bool kischarmisaligned(const char *p, signed char t) {
return false;
}
__funline bool kismemtrackhosed(void) {
return !((_weaken(_mmi)->i <= _weaken(_mmi)->n) &&
(_weaken(_mmi)->p == _weaken(_mmi)->s ||
_weaken(_mmi)->p == (struct MemoryInterval *)kMemtrackStart));
}
privileged static bool kismapped(int x) {
// xxx: we can't lock because no reentrant locks yet
size_t m, r, l = 0;
if (!_weaken(_mmi))
return true;
if (kismemtrackhosed())
return false;
r = _weaken(_mmi)->i;
while (l < r) {
m = (l & r) + ((l ^ r) >> 1); // floor((a+b)/2)
if (_weaken(_mmi)->p[m].y < x) {
l = m + 1;
} else {
r = m;
privileged static bool32 kisdangerous_unlocked(const char *addr) {
struct Dll *e, *e2;
for (e = dll_first(__maps.used); e; e = e2) {
e2 = dll_next(__maps.used, e);
struct Map *map = MAP_CONTAINER(e);
if (map->addr <= addr && addr < map->addr + map->size) {
dll_remove(&__maps.used, e);
dll_make_first(&__maps.used, e);
return !(map->prot & PROT_READ);
}
}
if (l < _weaken(_mmi)->i && x >= _weaken(_mmi)->p[l].x) {
return !!(_weaken(_mmi)->p[l].prot & PROT_READ);
} else {
return false;
}
return true;
}
privileged bool32 kisdangerous(const void *p) {
int frame;
if (kisimagepointer(p))
return false;
if (kiskernelpointer(p))
return false;
if (IsOldStack(p))
return false;
if (IsLegalPointer(p)) {
frame = (uintptr_t)p >> 16;
if (IsStackFrame(frame))
return false;
if (kismapped(frame))
return false;
}
if (GetStackAddr() + GetGuardSize() <= (uintptr_t)p &&
(uintptr_t)p < GetStackAddr() + GetStackSize()) {
return false;
}
return true;
privileged bool32 kisdangerous(const void *addr) {
bool32 res;
__maps_lock();
res = kisdangerous_unlocked(addr);
__maps_unlock();
return res;
}
privileged static void klogclose(long fd) {
@ -463,10 +425,6 @@ privileged static size_t kformat(char *b, size_t n, const char *fmt,
unsigned long long x;
unsigned i, j, m, rem, sign, hash, cols, prec;
char c, *p, *e, pdot, zero, flip, dang, base, quot, uppr, ansi, z[128];
if (kistextpointer(b) || kisdangerous(b))
n = 0;
if (!kistextpointer(fmt))
fmt = "!!WONTFMT";
p = b;
f = fmt;
e = p + n; // assume if n was negative e < p will be the case

95
libc/intrin/maps.c Normal file
View file

@ -0,0 +1,95 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/maps.h"
#include "ape/sections.internal.h"
#include "libc/dce.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/strace.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/prot.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
#ifdef __x86_64__
__static_yoink("_init_maps");
#endif
struct Maps __maps;
void __maps_init(void) {
// record _start() stack mapping
if (!IsWindows()) {
struct AddrSize stack;
stack = __get_main_stack();
dll_init(&__maps.stack.elem);
__maps.stack.addr = stack.addr;
__maps.stack.size = stack.size;
__maps.stack.prot = (uintptr_t)ape_stack_prot;
__maps_insert(&__maps.stack);
}
// record .text and .data mappings
static struct Map text, data;
dll_init(&text.elem);
text.addr = (char *)__executable_start;
text.size = _etext - __executable_start;
text.prot = PROT_READ | PROT_EXEC;
int pagesz = getauxval(AT_PAGESZ);
uintptr_t ds = ((uintptr_t)_etext + pagesz - 1) & -pagesz;
if (ds < (uintptr_t)_end) {
dll_init(&data.elem);
data.addr = (char *)ds;
data.size = (uintptr_t)_end - ds;
data.prot = PROT_READ | PROT_WRITE;
__maps_insert(&data);
}
__maps_insert(&text);
}
privileged void __maps_lock(void) {
struct CosmoTib *tib;
if (!__threaded)
return;
if (!__tls_enabled)
return;
tib = __get_tls_privileged();
if (tib->tib_flags & TIB_FLAG_MAPLOCK)
return;
while (atomic_exchange_explicit(&__maps.lock, 1, memory_order_acquire)) {
#if defined(__GNUC__) && defined(__aarch64__)
__asm__ volatile("yield");
#elif defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__))
__asm__ volatile("pause");
#endif
}
tib->tib_flags |= TIB_FLAG_MAPLOCK;
}
privileged void __maps_unlock(void) {
struct CosmoTib *tib;
atomic_store_explicit(&__maps.lock, 0, memory_order_release);
if (__tls_enabled) {
tib = __get_tls_privileged();
tib->tib_flags &= ~TIB_FLAG_MAPLOCK;
}
}

49
libc/intrin/maps.h Normal file
View file

@ -0,0 +1,49 @@
#ifndef COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_
#define COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_
#include "libc/intrin/atomic.h"
#include "libc/intrin/dll.h"
#include "libc/thread/tls2.internal.h"
COSMOPOLITAN_C_START_
#define MAP_CONTAINER(e) DLL_CONTAINER(struct Map, elem, e)
struct Map {
_Atomic(struct Map *) next; /* for __maps.maps */
char *addr; /* granule aligned */
size_t size; /* must be nonzero */
struct Dll elem; /* for __maps.free */
int64_t off; /* -1 if anonymous */
int prot; /* memory protects */
int flags; /* memory map flag */
bool iscow; /* windows nt only */
bool readonlyfile; /* windows nt only */
intptr_t h; /* windows nt only */
};
struct Maps {
atomic_int lock;
_Atomic(struct Map *) maps;
struct Dll *free;
struct Map stack;
struct Dll *used;
};
struct AddrSize {
char *addr;
size_t size;
};
extern struct Maps __maps;
int maps_check(void);
void __maps_init(void);
void __maps_lock(void);
void __maps_unlock(void);
struct Map *__maps_alloc(void);
void __maps_free(struct Map *);
void __maps_insert(struct Map *);
void *__mmap(char *, size_t, int, int, int, int64_t);
struct AddrSize __get_main_stack(void);
COSMOPOLITAN_C_END_
#endif /* COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_ */

14
libc/intrin/mmi.init.S → libc/intrin/maps_init.S
View file

@ -1,7 +1,7 @@
/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│
│ vi: set noet ft=asm ts=8 sw=8 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
@ -16,10 +16,12 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/thread/thread.h"
#include "libc/macros.internal.h"
.init.start 200,_init__mmi
movb $16,_mmi+8
movl $_mmi+24,_mmi+16
.init.end 200,_init__mmi
.init.start 301,_init_maps
push %rdi
push %rsi
call __maps_init
pop %rsi
pop %rdi
.init.end 301,_init_maps

243
libc/intrin/memtrack.greg.c
View file

@ -1,243 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/log/libfatal.internal.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
#include "libc/mem/mem.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
static void *__shove_memory(struct MemoryInterval *d,
const struct MemoryInterval *s, int n) {
int i;
unassert(n >= 0);
if (d > s) {
for (i = n; i--;) {
d[i] = s[i];
}
} else {
for (i = 0; i < n; ++i) {
d[i] = s[i];
}
}
return d;
}
static void __remove_memory(struct MemoryIntervals *mm, int i, int n) {
unassert(i >= 0);
unassert(i + n <= mm->i);
__shove_memory(mm->p + i, mm->p + i + n, mm->i - (i + n));
mm->i -= n;
}
static bool __extend_memory(struct MemoryIntervals *mm) {
int prot, flags;
char *base, *shad;
size_t gran, size;
struct DirectMap dm;
gran = kMemtrackGran;
base = (char *)kMemtrackStart;
prot = PROT_READ | PROT_WRITE;
flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED;
if (mm->p == mm->s) {
// TODO(jart): How can we detect ASAN mode under GREG?
if (1 || IsAsan()) {
shad = (char *)(((intptr_t)base >> 3) + 0x7fff8000);
dm = sys_mmap(shad, gran >> 3, prot, flags, -1, 0);
if (!dm.addr)
return false;
}
dm = sys_mmap(base, gran, prot, flags, -1, 0);
if (!dm.addr)
return false;
__shove_memory(dm.addr, mm->p, mm->i);
mm->p = dm.addr;
mm->n = gran / sizeof(*mm->p);
} else {
size = ROUNDUP(mm->n * sizeof(*mm->p), gran);
base += size;
if (IsAsan()) {
shad = (char *)(((intptr_t)base >> 3) + 0x7fff8000);
dm = sys_mmap(shad, gran >> 3, prot, flags, -1, 0);
if (!dm.addr)
return false;
}
dm = sys_mmap(base, gran, prot, flags, -1, 0);
if (!dm.addr)
return false;
mm->n = (size + gran) / sizeof(*mm->p);
}
return true;
}
static int __mint_memory(struct MemoryIntervals *mm, int i) {
unassert(i >= 0);
unassert(i <= mm->i);
unassert(mm->n >= 0);
if (mm->i == mm->n && !__extend_memory(mm))
return enomem();
__shove_memory(mm->p + i + 1, mm->p + i, mm->i++ - i);
return 0;
}
static int __punch_memory(struct MemoryIntervals *mm, int x, int y, int i) {
if (__mint_memory(mm, i) == -1)
return -1;
mm->p[i + 0].size -= (size_t)(mm->p[i + 0].y - (x - 1)) * FRAMESIZE;
mm->p[i + 0].y = x - 1;
mm->p[i + 1].size -= (size_t)((y + 1) - mm->p[i + 1].x) * FRAMESIZE;
mm->p[i + 1].x = y + 1;
return 0;
}
int __untrack_memory(struct MemoryIntervals *mm, int x, int y,
void wf(struct MemoryIntervals *, int, int)) {
unsigned l, r;
unassert(y >= x);
if (!mm->i)
return 0;
// binary search for the lefthand side
l = __find_memory(mm, x);
if (l == mm->i)
return 0;
if (y < mm->p[l].x)
return 0;
// binary search for the righthand side
r = __find_memory(mm, y);
if (r == mm->i || (r > l && y < mm->p[r].x))
--r;
unassert(r >= l);
unassert(x <= mm->p[r].y);
// remove the middle of an existing map
//
// ----|mmmmmmmmmmmmmmmm|--------- before
// xxxxx
// ----|mmmm|-----|mmmmm|--------- after
//
// this isn't possible on windows because we track each
// 64kb segment on that platform using a separate entry
if (l == r && x > mm->p[l].x && y < mm->p[l].y) {
return __punch_memory(mm, x, y, l);
}
// trim the right side of the lefthand map
//
// ----|mmmmmmm|-------------- before
// xxxxx
// ----|mmmm|----------------- after
//
if (x > mm->p[l].x && x <= mm->p[l].y) {
unassert(y >= mm->p[l].y);
if (IsWindows())
return einval();
mm->p[l].size -= (size_t)(mm->p[l].y - (x - 1)) * FRAMESIZE;
mm->p[l].y = x - 1;
unassert(mm->p[l].x <= mm->p[l].y);
++l;
}
// trim the left side of the righthand map
//
// ------------|mmmmm|-------- before
// xxxxx
// ---------------|mm|-------- after
//
if (y >= mm->p[r].x && y < mm->p[r].y) {
unassert(x <= mm->p[r].x);
if (IsWindows())
return einval();
mm->p[r].size -= (size_t)((y + 1) - mm->p[r].x) * FRAMESIZE;
mm->p[r].x = y + 1;
unassert(mm->p[r].x <= mm->p[r].y);
--r;
}
if (l <= r) {
if (IsWindows() && wf) {
wf(mm, l, r);
}
__remove_memory(mm, l, r - l + 1);
}
return 0;
}
int __track_memory(struct MemoryIntervals *mm, int x, int y, long h, int prot,
int flags, bool readonlyfile, bool iscow, long offset,
long size) {
unsigned i;
unassert(y >= x);
i = __find_memory(mm, x);
// try to extend the righthand side of the lefthand entry
// we can't do that if we're tracking independent handles
// we can't do that if it's a file map with a small size!
if (i && x == mm->p[i - 1].y + 1 && h == mm->p[i - 1].h &&
prot == mm->p[i - 1].prot && flags == mm->p[i - 1].flags &&
mm->p[i - 1].size ==
(size_t)(mm->p[i - 1].y - mm->p[i - 1].x) * FRAMESIZE + FRAMESIZE) {
mm->p[i - 1].size += (size_t)(y - mm->p[i - 1].y) * FRAMESIZE;
mm->p[i - 1].y = y;
// if we filled the hole then merge the two mappings
if (i < mm->i && y + 1 == mm->p[i].x && h == mm->p[i].h &&
prot == mm->p[i].prot && flags == mm->p[i].flags) {
mm->p[i - 1].y = mm->p[i].y;
mm->p[i - 1].size += mm->p[i].size;
__remove_memory(mm, i, 1);
}
}
// try to extend the lefthand side of the righthand entry
// we can't do that if we're creating a smaller file map!
else if (i < mm->i && y + 1 == mm->p[i].x && h == mm->p[i].h &&
prot == mm->p[i].prot && flags == mm->p[i].flags &&
size == (size_t)(y - x) * FRAMESIZE + FRAMESIZE) {
mm->p[i].size += (size_t)(mm->p[i].x - x) * FRAMESIZE;
mm->p[i].x = x;
}
// otherwise, create a new entry and memmove the items
else {
if (__mint_memory(mm, i) == -1)
return -1;
mm->p[i].x = x;
mm->p[i].y = y;
mm->p[i].h = h;
mm->p[i].prot = prot;
mm->p[i].flags = flags;
mm->p[i].offset = offset;
mm->p[i].size = size;
mm->p[i].iscow = iscow;
mm->p[i].readonlyfile = readonlyfile;
}
return 0;
}

431
libc/intrin/mmap.c Normal file
View file

@ -0,0 +1,431 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "ape/sections.internal.h"
#include "libc/atomic.h"
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
#include "libc/calls/struct/sigset.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/nt/memory.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/runtime/zipos.internal.h"
#include "libc/stdio/sysparam.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/o.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
#include "libc/thread/thread.h"
#define WINBASE 0x100080040000 // TODO: Can we support Windows Vista again?
#define WINMAXX 0x200080000000
#define MAP_FIXED_NOREPLACE_linux 0x100000
#define PGUP(x) (((x) + granularity - 1) & -granularity)
static atomic_ulong rollo;
void __maps_free(struct Map *map) {
dll_make_last(&__maps.free, &map->elem);
}
void __maps_insert(struct Map *map) {
struct Map *last = __maps.maps;
if (last && //
map->addr == last->addr + last->size && //
map->flags == last->flags && //
map->prot == last->prot && //
map->off == last->off && //
map->h == last->h && //
map->off == -1) {
last->size += map->size;
dll_remove(&__maps.used, &last->elem);
dll_make_first(&__maps.used, &last->elem);
__maps_free(map);
} else {
dll_make_first(&__maps.used, &map->elem);
map->next = __maps.maps;
__maps.maps = map;
}
}
struct Map *__maps_alloc(void) {
struct Dll *e;
struct Map *map;
if ((e = dll_first(__maps.free))) {
dll_remove(&__maps.free, e);
map = MAP_CONTAINER(e);
map->next = 0;
return map;
}
int granularity = __granularity();
struct DirectMap sys = sys_mmap(0, granularity, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (sys.addr == MAP_FAILED)
return 0;
if (IsWindows())
CloseHandle(sys.maphandle);
map = sys.addr;
map->addr = MAP_FAILED;
dll_init(&map->elem);
for (int i = 1; i < granularity / sizeof(struct Map); ++i) {
dll_init(&map[i].elem);
__maps_free(map + i);
}
map->next = 0;
return map;
}
static bool __overlaps_existing_map(const char *addr, size_t size) {
for (struct Map *map = __maps.maps; map; map = map->next) {
if (MAX(addr, map->addr) < MIN(addr + size, map->addr + map->size))
return true;
}
return false;
}
static int __munmap_chunk(void *addr, size_t size) {
int rc = sys_munmap(addr, size);
if (IsAsan() && !rc)
__asan_unshadow(addr, size);
return rc;
}
static int __munmap(char *addr, size_t size, bool untrack_only) {
// validate arguments
int granularity = __granularity();
if (((uintptr_t)addr & (granularity - 1)) || //
!size || (uintptr_t)addr + size < size)
return einval();
// untrack and delete mapping
int rc = 0;
__maps_lock();
// we can't call strace, kprintf, or nothing
StartOver:;
struct Map *map = __maps.maps;
_Atomic(struct Map *) *prev = &__maps.maps;
while (map) {
char *map_addr = map->addr;
size_t map_size = map->size;
struct Map *next = map->next;
if (MAX(addr, map_addr) <
MIN(addr + PGUP(size), map_addr + PGUP(map_size))) {
if (addr <= map_addr && addr + PGUP(size) >= map_addr + PGUP(map_size)) {
// remove mapping completely
dll_remove(&__maps.used, &map->elem);
*prev = next;
map->size = 0;
map->addr = MAP_FAILED;
if (untrack_only) {
__maps_free(map);
} else {
if (!IsWindows()) {
if (__munmap_chunk(map_addr, map_size))
rc = -1;
} else {
if (!UnmapViewOfFile(map_addr))
rc = -1;
if (!CloseHandle(map->h))
rc = -1;
}
__maps_free(map);
goto StartOver;
}
map = next;
continue;
} else if (IsWindows()) {
// you can't carve up memory maps on windows. our mmap() makes
// this not a problem (for non-enormous memory maps) by making
// independent mappings for each 64 kb granule, under the hood
rc = einval();
} else if (addr <= map_addr) {
// shave off lefthand side of mapping
size_t left = addr + size - map_addr;
size_t right = map_addr + map_size - (addr + size);
map->addr += left;
map->size = right;
if (map->off != -1)
map->off += left;
if (!untrack_only) {
if (__munmap_chunk(map_addr, left) == -1)
rc = -1;
goto StartOver;
}
} else if (addr + PGUP(size) >= map_addr + PGUP(map_size)) {
// shave off righthand side of mapping
size_t left = addr - map_addr;
size_t right = map_addr + map_size - addr;
map->size = left;
if (!untrack_only) {
if (__munmap_chunk(addr, right) == -1)
rc = -1;
goto StartOver;
}
} else {
// punch hole in mapping
size_t left = addr - map_addr;
size_t middle = PGUP(size);
size_t right = map_size - middle - left;
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
leftmap->next = map;
leftmap->addr = map_addr;
leftmap->size = left;
leftmap->off = map->off;
leftmap->prot = map->prot;
leftmap->flags = map->flags;
map->addr += left + middle;
map->size = right;
if (map->off != -1)
map->off += left + middle;
dll_make_first(&__maps.used, &leftmap->elem);
*prev = leftmap;
if (!untrack_only) {
if (__munmap_chunk(addr, size) == -1)
rc = -1;
goto StartOver;
}
} else {
rc = -1;
}
}
}
prev = &map->next;
map = next;
}
__maps_unlock();
return rc;
}
static void *__mmap_chunk(void *addr, size_t size, int prot, int flags, int fd,
int64_t off, int granularity) {
// polyfill nuances of fixed mappings
int sysflags = flags;
bool noreplace = false;
bool should_untrack = false;
if (flags & MAP_FIXED_NOREPLACE) {
if (flags & MAP_FIXED)
return (void *)einval();
sysflags &= ~MAP_FIXED_NOREPLACE;
if (IsLinux()) {
noreplace = true;
sysflags |= MAP_FIXED_NOREPLACE_linux;
} else if (IsFreebsd() || IsNetbsd()) {
sysflags |= MAP_FIXED;
if (__overlaps_existing_map(addr, size))
return (void *)eexist();
} else {
noreplace = true;
}
} else if (flags & MAP_FIXED) {
should_untrack = true;
}
// allocate Map object
struct Map *map;
__maps_lock();
map = __maps_alloc();
__maps_unlock();
if (!map)
return MAP_FAILED;
// obtain mapping from operating system
int olderr = errno;
struct DirectMap res;
TryAgain:
res = sys_mmap(addr, size, prot, sysflags, fd, off);
if (res.addr == MAP_FAILED) {
if (IsWindows() && errno == EADDRNOTAVAIL) {
if (noreplace) {
errno = EEXIST;
} else if (should_untrack) {
sys_munmap(res.addr, size);
errno = olderr;
goto TryAgain;
} else {
addr += granularity;
errno = olderr;
goto TryAgain;
}
}
__maps_lock();
__maps_free(map);
__maps_unlock();
return MAP_FAILED;
}
// polyfill map fixed noreplace
// we assume non-linux gives us addr if it's free
// that's what linux (e.g. rhel7) did before noreplace
if (noreplace && res.addr != addr) {
sys_munmap(res.addr, size);
__maps_lock();
__maps_free(map);
__maps_unlock();
return (void *)eexist();
}
// untrack mapping we blew away
if (should_untrack)
__munmap(addr, size, true);
// track Map object
map->addr = res.addr;
map->size = size;
map->off = off;
map->prot = prot;
map->flags = flags;
map->h = res.maphandle;
if (IsWindows()) {
map->iscow = (flags & MAP_TYPE) != MAP_SHARED && fd != -1;
map->readonlyfile = (flags & MAP_TYPE) == MAP_SHARED && fd != -1 &&
(g_fds.p[fd].flags & O_ACCMODE) == O_RDONLY;
}
__maps_lock();
__maps_insert(map);
__maps_unlock();
return res.addr;
}
static void *__mmap_impl(char *addr, size_t size, int prot, int flags, int fd,
int64_t off, int granularity) {
// validate file map args
if (fd != -1) {
if (off & (granularity - 1))
return (void *)einval();
if (IsWindows()) {
if (!__isfdkind(fd, kFdFile))
return (void *)eacces();
if ((g_fds.p[fd].flags & O_ACCMODE) == O_WRONLY)
return (void *)eacces();
}
}
// mmap works fine on unix
if (!IsWindows())
return __mmap_chunk(addr, size, prot, flags, fd, off, granularity);
// if the concept of granularity wasn't exciting enough
if (!addr && !(flags & (MAP_FIXED | MAP_FIXED_NOREPLACE)))
addr = (char *)(WINBASE + atomic_fetch_add(&rollo, PGUP(size)) % WINMAXX);
// windows forbids unmapping a subset of a map once it's made
if (size <= granularity || size > 100 * 1024 * 1024)
return __mmap_chunk(addr, size, prot, flags, fd, off, granularity);
// so we create an separate map for each granule in the mapping
if (!(flags & MAP_FIXED)) {
while (__overlaps_existing_map(addr, size)) {
if (flags & MAP_FIXED_NOREPLACE)
return (void *)eexist();
addr += granularity;
}
}
char *res = addr;
while (size) {
char *got;
size_t amt = MIN(size, granularity);
got = __mmap_chunk(addr, amt, prot, flags, fd, off, granularity);
if (got != addr) {
if (got != MAP_FAILED)
__munmap(got, amt, false);
if (addr > res)
__munmap(res, addr - res, false);
errno = EAGAIN;
return MAP_FAILED;
}
size -= amt;
addr += amt;
off += amt;
}
return res;
}
void *__mmap(char *addr, size_t size, int prot, int flags, int fd,
int64_t off) {
char *res;
int granularity = __granularity();
// validate arguments
if (((uintptr_t)addr & (granularity - 1)) || //
!size || (uintptr_t)addr + size < size)
return (void *)einval();
if (size > 0x100000000000)
return (void *)enomem();
// normalize arguments
if (flags & MAP_ANONYMOUS) {
fd = -1;
off = 0;
size = PGUP(size);
}
// create memory mappping
if (!__isfdkind(fd, kFdZip)) {
res = __mmap_impl(addr, size, prot, flags, fd, off, granularity);
} else {
res = _weaken(__zipos_mmap)(
addr, size, prot, flags,
(struct ZiposHandle *)(uintptr_t)g_fds.p[fd].handle, off);
}
return res;
}
void *mmap(void *addr, size_t size, int prot, int flags, int fd, int64_t off) {
void *res;
res = __mmap(addr, size, prot, flags, fd, off);
if (IsAsan() && res != MAP_FAILED) {
__asan_shadow(res, size);
int granularity = __granularity();
if (size != PGUP(size))
__asan_poison(res + size, PGUP(size) - size, kAsanMmapSizeOverrun);
}
STRACE("mmap(%p, %'zu, %s, %s, %d, %'ld) → %p% m", addr, size,
DescribeProtFlags(prot), DescribeMapFlags(flags), fd, off, res);
return res;
}
int munmap(void *addr, size_t size) {
int rc;
rc = __munmap(addr, size, false);
STRACE("munmap(%p, %'zu) → %d% m", addr, size, rc);
return rc;
}
__weak_reference(mmap, mmap64);

28
libc/intrin/mmi.c
View file

@ -1,28 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/thread/thread.h"
#ifdef __x86_64__
__static_yoink("_init__mmi");
#endif
struct MemoryIntervals _mmi;
pthread_mutex_t __mmi_lock_obj = {._type = PTHREAD_MUTEX_RECURSIVE};

215
libc/intrin/mprotect.c Normal file
View file

@ -0,0 +1,215 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/struct/sigset.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/strace.internal.h"
#include "libc/nt/memory.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/sysparam.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
#define PGUP(x) (((x) + pagesz - 1) & -pagesz)
static int __mprotect_chunk(char *addr, size_t size, int prot, bool iscow) {
if (!IsWindows())
return sys_mprotect(addr, size, prot);
uint32_t op;
if (!VirtualProtect(addr, size, __prot2nt(prot, iscow), &op))
return -1;
return 0;
}
int __mprotect(char *addr, size_t size, int prot) {
// unix checks prot before checking size
if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC))
return einval();
// make new technology consistent with unix
if (!size)
return 0;
// unix checks prot before checking size
int pagesz = getauxval(AT_PAGESZ);
if ((intptr_t)addr & (pagesz - 1))
return einval();
// change mappings
int rc = 0;
__maps_lock();
bool found = false;
struct Map *map = __maps.maps;
_Atomic(struct Map *) *prev = &__maps.maps;
while (map) {
char *map_addr = map->addr;
size_t map_size = map->size;
struct Map *next = map->next;
char *beg = MAX(addr, map_addr);
char *end = MIN(addr + PGUP(size), map_addr + PGUP(map_size));
if (beg < end) {
found = true;
if (addr <= map_addr && addr + PGUP(size) >= map_addr + PGUP(map_size)) {
// change protection of entire mapping
if (!__mprotect_chunk(map_addr, map_size, prot, map->iscow)) {
map->prot = prot;
} else {
rc = -1;
}
} else if (IsWindows()) {
// windows does allow changing protection at 4096 byte chunks
// however we currently don't have data structures that track
// this within the 64 kb map granules that can't be broken up
if (__mprotect_chunk(beg, end - beg, prot, map->iscow) == -1)
rc = -1;
} else if (addr <= map_addr) {
// cleave lefthand side of mapping
size_t left = addr + size - map_addr;
size_t right = map_addr + map_size - (addr + size);
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
if (!__mprotect_chunk(map_addr, left, prot, false)) {
leftmap->next = map;
leftmap->addr = map_addr;
leftmap->size = left;
leftmap->prot = prot;
leftmap->off = map->off;
leftmap->flags = map->flags;
map->addr += left;
map->size = right;
if (map->off != -1)
map->off += left;
dll_make_first(&__maps.used, &leftmap->elem);
*prev = leftmap;
} else {
__maps_free(leftmap);
rc = -1;
}
} else {
rc = -1;
}
} else if (addr + PGUP(size) >= map_addr + PGUP(map_size)) {
// cleave righthand side of mapping
size_t left = addr - map_addr;
size_t right = map_addr + map_size - addr;
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
if (!__mprotect_chunk(map_addr + left, right, prot, false)) {
leftmap->next = map;
leftmap->addr = map_addr;
leftmap->size = left;
leftmap->off = map->off;
leftmap->prot = map->prot;
leftmap->flags = map->flags;
map->addr += left;
map->size = right;
map->prot = prot;
if (map->off != -1)
map->off += left;
dll_make_first(&__maps.used, &leftmap->elem);
*prev = leftmap;
} else {
__maps_free(leftmap);
rc = -1;
}
} else {
rc = -1;
}
} else {
// punch hole in mapping
size_t left = addr - map_addr;
size_t middle = PGUP(size);
size_t right = map_size - middle - left;
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
struct Map *midlmap;
if ((midlmap = __maps_alloc())) {
if (!__mprotect_chunk(map_addr + left, middle, prot, false)) {
leftmap->next = midlmap;
leftmap->addr = map_addr;
leftmap->size = left;
leftmap->off = map->off;
leftmap->prot = map->prot;
leftmap->flags = map->flags;
midlmap->next = map;
midlmap->addr = map_addr + left;
midlmap->size = middle;
midlmap->off = map->off == -1 ? -1 : map->off + left;
midlmap->prot = prot;
midlmap->flags = map->flags;
map->addr += left + middle;
map->size = right;
if (map->off != -1)
map->off += left + middle;
dll_make_first(&__maps.used, &leftmap->elem);
dll_make_first(&__maps.used, &midlmap->elem);
*prev = leftmap;
} else {
__maps_free(midlmap);
__maps_free(leftmap);
rc = -1;
}
} else {
__maps_free(leftmap);
rc = -1;
}
} else {
rc = -1;
}
}
}
prev = &map->next;
map = next;
}
// allow user to change mappings unknown to cosmo runtime
if (!found)
rc = __mprotect_chunk(addr, size, prot, false);
__maps_unlock();
return rc;
}
/**
* Modifies restrictions on virtual memory address range.
*
* @param addr needs to be 4kb aligned
* @param prot can have PROT_{NONE,READ,WRITE,EXEC}
* @return 0 on success, or -1 w/ errno
* @raise ENOMEM on tracking memory oom
* @see mmap()
*/
int mprotect(void *addr, size_t size, int prot) {
int rc;
rc = __mprotect(addr, size, prot);
STRACE("mprotect(%p, %'zu, %s) → %d% m", addr, size, DescribeProtFlags(prot),
rc);
return rc;
}

36
libc/runtime/memtracknt.c → libc/intrin/msync-nt.c
View file

@ -16,24 +16,32 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/intrin/strace.internal.h"
#include "libc/calls/syscall-nt.internal.h"
#include "libc/intrin/maps.h"
#include "libc/nt/memory.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/sysparam.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/errfuns.h"
static inline void *GetFrameAddr(int f) {
intptr_t a;
a = f;
a *= FRAMESIZE;
return (void *)a;
}
textwindows int sys_msync_nt(char *addr, size_t size, int flags) {
void __release_memory_nt(struct MemoryIntervals *mm, int l, int r) {
int i;
for (i = l; i <= r; ++i) {
UnmapViewOfFile(GetFrameAddr(mm->p[i].x));
CloseHandle(mm->p[i].h);
int pagesz = getauxval(AT_PAGESZ);
size = (size + pagesz - 1) & -pagesz;
if ((uintptr_t)addr & (pagesz - 1))
return einval();
int rc = 0;
for (struct Map *map = __maps.maps; map; map = map->next) {
char *beg = MAX(addr, map->addr);
char *end = MIN(addr + size, map->addr + map->size);
if (beg < end)
if (!FlushViewOfFile(beg, end - beg))
rc = -1;
// TODO(jart): FlushFileBuffers too on g_fds handle if MS_SYNC?
}
return rc;
}

0
libc/runtime/msync.c → libc/intrin/msync.c
View file

0
libc/calls/munmap-metal.c → libc/intrin/munmap-metal.c
View file

6
libc/calls/munmap-sysv.c → libc/intrin/munmap-sysv.c
View file

@ -21,6 +21,7 @@
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/syslib.internal.h"
/**
@ -40,7 +41,8 @@ int sys_munmap(void *p, size_t n) {
} else {
rc = __sys_munmap(p, n);
}
KERNTRACE("sys_munmap(%p /* %s */, %'zu) → %d", p,
DescribeFrame((intptr_t)p >> 16), n, rc);
if (!rc)
__virtualsize -= n;
KERNTRACE("sys_munmap(%p, %'zu) → %d", p, n, rc);
return rc;
}

34
libc/intrin/nocolor.c
View file

@ -16,39 +16,19 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/dce.h"
#include "libc/log/internal.h"
#include "libc/runtime/runtime.h"
#define IsDumb(s) \
(s[0] == 'd' && s[1] == 'u' && s[2] == 'm' && s[3] == 'b' && !s[4])
/**
* Indicates if ANSI terminal colors are inappropriate.
*
* Normally this variable should be false. We only set it to true if
* we're running on an old version of Windows or the environment
* variable `TERM` is set to `dumb`.
*
* We think colors should be the norm, since most software is usually
* too conservative about removing them. Rather than using `isatty`
* consider using sed for instances where color must be removed:
*
* sed 's/\x1b\[[;[:digit:]]*m//g' <color.txt >uncolor.txt
*
* For some reason, important software is configured by default in many
* operating systems, to not only disable colors, but utf-8 too! Here's
* an example of how a wrapper script can fix that for `less`.
*
* #!/bin/sh
* LESSCHARSET=UTF-8 exec /usr/bin/less -RS "$@"
*
* Thank you for using colors!
*/
bool __nocolor;
__attribute__((__constructor__(20))) optimizesize textstartup void
__nocolor_init(int argc, char **argv, char **envp, intptr_t *auxv) {
char *s;
__nocolor = IsWindows() || ((s = getenv("TERM")) && IsDumb(s));
if ((s = getenv("TERM")))
if (s[0] == 'd' && //
s[1] == 'u' && //
s[2] == 'm' && //
s[3] == 'b' && //
s[4] == '\0')
__nocolor = true;
}

42
libc/runtime/isheap.c → libc/intrin/printmaps.c
View file

@ -16,22 +16,40 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/fmt/conv.h"
#include "libc/fmt/itoa.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/maps.h"
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
// TODO(jart): DELETE
/**
* Returns true if address isn't stack and was malloc'd or mmap'd.
*
* @assume stack addresses are always greater than heap addresses
* @assume stack memory isn't stored beneath %rsp (-mno-red-zone)
* @deprecated
* Prints memory mappings.
*/
optimizesize bool32 _isheap(const void *p) {
intptr_t x, y;
x = kAutomapStart;
y = x + kAutomapSize;
return x <= (intptr_t)p && (intptr_t)p < y;
void __print_maps(void) {
int limit = 10;
long maptally = 0;
char mappingbuf[8], sb[16];
for (struct Map *map = __maps.maps; map; map = map->next) {
maptally += map->size;
kprintf("%012lx-%012lx %!s", map->addr, map->addr + map->size,
(DescribeMapping)(mappingbuf, map->prot, map->flags));
sizefmt(sb, map->size, 1024);
kprintf(" %!sb", sb);
if (map->h && map->h != -1)
kprintf(" h=%ld", map->h);
if (map->iscow)
kprintf(" cow");
if (map->readonlyfile)
kprintf(" readonlyfile");
kprintf("\n");
if (!--limit) {
kprintf("...\n");
break;
}
}
sizefmt(sb, maptally, 1024);
kprintf("# %!sb mapped memory\n", sb);
}

66
libc/intrin/printmemoryintervals.c
View file

@ -1,66 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/fmt/conv.h"
#include "libc/fmt/itoa.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
static bool IsNoteworthyHole(unsigned i, const struct MemoryIntervals *mm) {
// gaps between shadow frames aren't interesting
// the chasm from heap to stack ruins statistics
return !(
(IsShadowFrame(mm->p[i].y) || IsShadowFrame(mm->p[i + 1].x)) ||
(!IsStaticStackFrame(mm->p[i].y) && IsStaticStackFrame(mm->p[i + 1].x)));
}
void PrintMemoryIntervals(int fd, const struct MemoryIntervals *mm) {
long i, w, frames, maptally = 0;
char mappingbuf[8], framebuf[64], sb[16];
for (w = i = 0; i < mm->i; ++i) {
w = MAX(w, LengthInt64Thousands(mm->p[i].y + 1 - mm->p[i].x));
}
for (i = 0; i < mm->i; ++i) {
frames = mm->p[i].y + 1 - mm->p[i].x;
maptally += frames;
kprintf("%08x-%08x %s %'*ldx %s", mm->p[i].x, mm->p[i].y,
(DescribeMapping)(mappingbuf, mm->p[i].prot, mm->p[i].flags), w,
frames, (DescribeFrame)(framebuf, mm->p[i].x));
if (mm->p[i].iscow)
kprintf(" cow");
if (mm->p[i].readonlyfile)
kprintf(" readonlyfile");
sizefmt(sb, mm->p[i].size, 1024);
kprintf(" %sB", sb);
if (i + 1 < mm->i) {
frames = mm->p[i + 1].x - mm->p[i].y - 1;
if (frames && IsNoteworthyHole(i, mm)) {
sizefmt(sb, frames * FRAMESIZE, 1024);
kprintf(" w/ %sB hole", sb);
}
}
if (mm->p[i].h != -1) {
kprintf(" h=%ld", mm->p[i].h);
}
kprintf("\n");
}
sizefmt(sb, maptally * FRAMESIZE, 1024);
kprintf("# %sB total mapped memory\n", sb);
}

38
libc/intrin/printsystemmappings.greg.c
View file

@ -1,38 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/sysv/consts/at.h"
#include "libc/sysv/consts/o.h"
void PrintSystemMappings(int outfd) {
int infd;
ssize_t rc;
char buf[64];
if (!IsWindows()) {
if ((infd = __sys_openat(AT_FDCWD, "/proc/self/maps", O_RDONLY, 0)) >= 0) {
sys_write(outfd, "\n", 1);
while ((rc = sys_read(infd, buf, sizeof(buf))) > 0) {
sys_write(outfd, buf, rc);
}
}
sys_close(infd);
}
}

3
libc/intrin/pthread_tid.c
View file

@ -16,11 +16,14 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/dce.h"
#include "libc/intrin/atomic.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
int _pthread_tid(struct PosixThread *pt) {
if (IsWindows()) // xxx: fixme
return pt->ptid;
int tid = 0;
while (pt && !(tid = atomic_load_explicit(&pt->ptid, memory_order_acquire))) {
pthread_pause_np();

6
libc/calls/virtualmax2.c → libc/intrin/virtualmax.c
View file

@ -17,8 +17,6 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/runtime.h"
#ifndef __x86_64__
size_t __virtualmax;
#endif /* __x86_64__ */
size_t __virtualmax = -1;
size_t __virtualsize = 0;

10
libc/log/backtrace.internal.h
View file

@ -1,19 +1,9 @@
#ifndef COSMOPOLITAN_LIBC_LOG_BACKTRACE_H_
#define COSMOPOLITAN_LIBC_LOG_BACKTRACE_H_
#include "libc/nexgen32e/stackframe.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/symbols.internal.h"
COSMOPOLITAN_C_START_
forceinline pureconst bool IsValidStackFramePointer(struct StackFrame *x) {
/* assumes __mmi_lock() is held */
return IsLegalPointer(x) && !((uintptr_t)x & 15) &&
(IsStaticStackFrame((uintptr_t)x >> 16) ||
IsOldStackFrame((uintptr_t)x >> 16) ||
/* lua coroutines need this */
IsMemtracked((uintptr_t)x >> 16, (uintptr_t)x >> 16));
}
void ShowBacktrace(int, const struct StackFrame *);
int PrintBacktraceUsingSymbols(int, const struct StackFrame *,
struct SymbolTable *);

1
libc/log/leaks.c
View file

@ -126,7 +126,6 @@ dontasan void CheckForMemoryLeaks(void) {
malloc_inspect_all(OnMemory, 0);
kprintf("\n");
/* __print_maps(); */
/* PrintSystemMappings(2); */
/* PrintGarbage(); */
_Exit(78);
}

5
libc/log/oncrash_amd64.c
View file

@ -245,7 +245,6 @@ static relegated void ShowCrashReport(int err, int sig, siginfo_t *si,
if (!IsWindows()) {
__print_maps();
}
/* PrintSystemMappings(2); */
if (__argv) {
for (i = 0; i < __argc; ++i) {
kprintf("%s ", __argv[i]);
@ -269,6 +268,8 @@ static inline void SpinUnlock(atomic_uint *lock) {
relegated void __oncrash(int sig, siginfo_t *si, void *arg) {
static atomic_uint lock;
ftrace_enabled(-1);
strace_enabled(-1);
BLOCK_CANCELATION;
SpinLock(&lock);
int err = errno;
@ -291,6 +292,8 @@ relegated void __oncrash(int sig, siginfo_t *si, void *arg) {
SpinUnlock(&lock);
ALLOW_CANCELATION;
strace_enabled(+1);
ftrace_enabled(+1);
}
#endif /* __x86_64__ */

7
libc/log/oncrash_arm64.c
View file

@ -33,6 +33,7 @@
#include "libc/calls/ucontext.h"
#include "libc/cosmo.h"
#include "libc/cxxabi.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/describebacktrace.internal.h"
@ -211,8 +212,6 @@ static relegated void __oncrash_impl(int sig, siginfo_t *si, ucontext_t *ctx) {
struct utsname names = {0};
struct Buffer b[1] = {{buf, size}};
b->p[b->i++] = '\n';
ftrace_enabled(-1);
strace_enabled(-1);
__restore_tty();
uname(&names);
gethostname(host, sizeof(host));
@ -390,6 +389,8 @@ static inline void SpinUnlock(atomic_uint *lock) {
relegated void __oncrash(int sig, siginfo_t *si, void *arg) {
static atomic_uint lock;
ftrace_enabled(-1);
strace_enabled(-1);
BLOCK_CANCELATION;
SpinLock(&lock);
__oncrash_impl(sig, si, arg);
@ -416,6 +417,8 @@ relegated void __oncrash(int sig, siginfo_t *si, void *arg) {
SpinUnlock(&lock);
ALLOW_CANCELATION;
strace_enabled(+1);
ftrace_enabled(+1);
}
#endif /* __aarch64__ */

3
libc/mem/BUILD.mk
View file

@ -54,8 +54,7 @@ o/$(MODE)/libc/mem/asan.o: private \
-fno-sanitize=all \
-fno-stack-protector \
-Wframe-larger-than=4096 \
-Walloca-larger-than=4096 \
-fpatchable-function-entry=0,0
-Walloca-larger-than=4096
# make asan stack traces shorter
o/$(MODE)/libc/mem/asanthunk.o: private \

24
libc/mem/asan.c
View file

@ -123,21 +123,13 @@ static void *__asan_allocate_heap(size_t a, size_t n, struct AsanTrace *bt) {
}
static struct AsanExtra *__asan_get_extra(const void *p, size_t *c) {
int f;
long x, n;
struct AsanExtra *e;
f = (intptr_t)p >> 16;
if (!kisdangerous(p) && (n = dlmalloc_usable_size((void *)p)) > sizeof(*e) &&
!ckd_add(&x, (intptr_t)p, n) && x <= 0x800000000000 &&
(LIKELY(f == (int)((x - 1) >> 16)) || !kisdangerous((void *)(x - 1))) &&
(LIKELY(f == (int)((x = x - sizeof(*e)) >> 16)) ||
__asan_is_mapped(x >> 16)) &&
!(x & (_Alignof(struct AsanExtra) - 1))) {
*c = n;
return (struct AsanExtra *)x;
} else {
long n;
if (kisdangerous(p))
return 0;
}
if ((n = dlmalloc_usable_size((void *)p)) < sizeof(struct AsanExtra))
return 0;
*c = n;
return (struct AsanExtra *)(p + n - sizeof(struct AsanExtra));
}
// Returns true if `p` was allocated by an IGNORE_LEAKS(function).
@ -153,7 +145,7 @@ int __asan_is_leaky(void *p) {
return 0;
if (!__asan_read48(e->size, &n))
return 0;
if (!__asan_is_mapped((((intptr_t)p >> 3) + 0x7fff8000) >> 16))
if (!__asan_is_mapped((char *)(((intptr_t)p >> 3) + 0x7fff8000)))
return 0;
if (!(st = GetSymbolTable()))
return 0;
@ -182,7 +174,7 @@ int __asan_print_trace(void *p) {
return -1;
}
kprintf("\n%p %,lu bytes [asan]", (char *)p, n);
if (!__asan_is_mapped((((intptr_t)p >> 3) + 0x7fff8000) >> 16))
if (!__asan_is_mapped((char *)(((intptr_t)p >> 3) + 0x7fff8000)))
kprintf(" (shadow not mapped?!)");
for (i = 0; i < ARRAYLEN(e->bt.p) && e->bt.p[i]; ++i)
kprintf("\n%*lx %t", 12, e->bt.p[i], e->bt.p[i]);

5
libc/mem/pvalloc.c
View file

@ -18,6 +18,7 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/errno.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/stdckdint.h"
/**
@ -31,9 +32,9 @@
* @see valloc()
*/
void *pvalloc(size_t n) {
if (ckd_add(&n, n, FRAMESIZE - 1)) {
if (ckd_add(&n, n, __granularity() - 1)) {
errno = ENOMEM;
return 0;
}
return memalign(FRAMESIZE, n & -FRAMESIZE);
return memalign(__granularity(), n & -__granularity());
}

5
libc/mem/tinymalloc.inc
View file

@ -14,6 +14,7 @@
// PERFORMANCE OF THIS SOFTWARE.
#include "libc/assert.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/mem/mem.h"
#include "libc/stdalign.internal.h"
@ -28,7 +29,7 @@
#define TINYMALLOC_MAX_ALIGN 4096
#endif
#ifndef MODE_DBG /* don't interfere with asan malloc */
#if !IsAsan()
alignas(TINYMALLOC_MAX_ALIGN) static struct {
char memory[TINYMALLOC_MAX_BYTES];
@ -167,4 +168,4 @@ OutOfMemory:
return 0;
}
#endif /* MODE_DBG */
#endif /* IsAsan() */

3
libc/mem/valloc.c
View file

@ -17,6 +17,7 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
/**
* Allocates granular aligned memory, i.e.
@ -28,5 +29,5 @@
* @see pvalloc()
*/
void *valloc(size_t n) {
return memalign(FRAMESIZE, n);
return memalign(__granularity(), n);
}

129
libc/proc/fork-nt.c
View file

@ -17,14 +17,17 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "ape/sections.internal.h"
#include "libc/assert.h"
#include "libc/calls/internal.h"
#include "libc/calls/sig.internal.h"
#include "libc/calls/state.internal.h"
#include "libc/calls/syscall_support-nt.internal.h"
#include "libc/errno.h"
#include "libc/fmt/itoa.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/macros.internal.h"
@ -41,11 +44,14 @@
#include "libc/nt/runtime.h"
#include "libc/nt/signals.h"
#include "libc/nt/struct/ntexceptionpointers.h"
#include "libc/nt/thread.h"
#include "libc/nt/thunk/msabi.h"
#include "libc/proc/ntspawn.h"
#include "libc/proc/proc.internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/runtime/symbols.internal.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/at.h"
@ -55,10 +61,13 @@
#include "libc/sysv/consts/sig.h"
#include "libc/sysv/errfuns.h"
#include "libc/thread/itimer.internal.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/tls.h"
#ifdef __x86_64__
extern long __klog_handle;
void WipeKeystrokes(void);
__msabi extern typeof(GetCurrentProcessId) *const __imp_GetCurrentProcessId;
static textwindows wontreturn void AbortFork(const char *func) {
#if SYSDEBUG
@ -98,8 +107,11 @@ static dontinline textwindows bool ForkIo2(int64_t h, void *buf, size_t n,
struct NtOverlapped *),
const char *sf, bool ischild) {
ssize_t rc = ForkIo(h, buf, n, fn);
if (ischild)
if (ischild) {
__tls_enabled_set(false); // prevent tls crash in kprintf
__pid = __imp_GetCurrentProcessId();
__klog_handle = 0;
}
NTTRACE("%s(%ld, %p, %'zu) → %'zd% m", sf, h, buf, n, rc);
return rc != -1;
}
@ -183,16 +195,17 @@ static __msabi textwindows int OnForkCrash(struct NtExceptionPointers *ep) {
TerminateThisProcess(SIGSTKFLT);
}
static textwindows void *Malloc(size_t size) {
return HeapAlloc(GetProcessHeap(), 0, size);
}
textwindows void WinMainForked(void) {
jmp_buf jb;
int64_t reader;
int64_t savetsc;
char *addr, *shad;
uint64_t size, upsize;
struct MemoryInterval *maps;
struct Map *map;
uint32_t varlen, oldprot;
char16_t fvar[21 + 1 + 21 + 1];
uint32_t i, varlen, oldprot, savepid;
long mapcount, mapcapacity, specialz;
struct Fds *fds = __veil("r", &g_fds);
// check to see if the process was actually forked
@ -200,7 +213,7 @@ textwindows void WinMainForked(void) {
varlen = GetEnvironmentVariable(u"_FORK", fvar, ARRAYLEN(fvar));
if (!varlen || varlen >= ARRAYLEN(fvar))
return;
NTTRACE("WinMainForked()");
/* STRACE("WinMainForked()"); */
SetEnvironmentVariable(u"_FORK", NULL);
#if SYSDEBUG
int64_t oncrash = AddVectoredExceptionHandler(1, (void *)OnForkCrash);
@ -208,64 +221,57 @@ textwindows void WinMainForked(void) {
ParseInt(fvar, &reader);
// read the cpu state from the parent process & plus
// read the list of mappings from the parent process
// this is stored in a special secretive memory map!
// read ExtendMemoryIntervals for further details :|
maps = (void *)kMemtrackStart;
ReadOrDie(reader, jb, sizeof(jb));
ReadOrDie(reader, &mapcount, sizeof(_mmi.i));
ReadOrDie(reader, &mapcapacity, sizeof(_mmi.n));
specialz = ROUNDUP(mapcapacity * sizeof(_mmi.p[0]), kMemtrackGran);
ViewOrDie(MapOrDie(kNtPageReadwrite, specialz), kNtFileMapWrite, 0, specialz,
maps);
ReadOrDie(reader, maps, mapcount * sizeof(_mmi.p[0]));
if (IsAsan()) {
shad = (char *)(((intptr_t)maps >> 3) + 0x7fff8000);
size = ROUNDUP(specialz >> 3, FRAMESIZE);
ViewOrDie(MapOrDie(kNtPageReadwrite, size), kNtFileMapWrite, 0, size, shad);
ReadOrDie(reader, shad, (mapcount * sizeof(_mmi.p[0])) >> 3);
}
// read the heap mappings from the parent process
for (i = 0; i < mapcount; ++i) {
addr = (char *)((uint64_t)maps[i].x << 16);
size = maps[i].size;
if ((maps[i].flags & MAP_TYPE) != MAP_SHARED) {
upsize = ROUNDUP(size, FRAMESIZE);
// read memory mappings from parent process
struct Map *maps = __maps.maps;
for (;;) {
map = Malloc(sizeof(*map));
ReadOrDie(reader, map, sizeof(*map));
if ((map->flags & MAP_TYPE) != MAP_SHARED) {
// we don't need to close the map handle because sys_mmap_nt
// doesn't mark it inheritable across fork() for MAP_PRIVATE
ViewOrDie((maps[i].h = MapOrDie(kNtPageExecuteReadwrite, upsize)),
kNtFileMapWrite | kNtFileMapExecute, 0, upsize, addr);
ReadOrDie(reader, addr, size);
ViewOrDie((map->h = MapOrDie(kNtPageExecuteReadwrite, map->size)),
kNtFileMapWrite | kNtFileMapExecute, 0, map->size, map->addr);
ReadOrDie(reader, map->addr, map->size);
} else {
// we can however safely inherit MAP_SHARED with zero copy
ViewOrDie(maps[i].h,
maps[i].readonlyfile ? kNtFileMapRead | kNtFileMapExecute
: kNtFileMapWrite | kNtFileMapExecute,
maps[i].offset, size, addr);
ViewOrDie(map->h,
map->readonlyfile ? kNtFileMapRead | kNtFileMapExecute
: kNtFileMapWrite | kNtFileMapExecute,
map->off, map->size, map->addr);
}
dll_init(&map->elem);
bool isdone = !map->next;
map->next = maps;
maps = map;
if (isdone)
break;
}
// read the .data and .bss program image sections
savepid = __pid;
savetsc = kStartTsc;
ReadOrDie(reader, __data_start, __data_end - __data_start);
ReadOrDie(reader, __bss_start, __bss_end - __bss_start);
__pid = savepid;
kStartTsc = savetsc;
__threaded = false;
__tls_index = 0;
__tls_enabled_set(false);
// apply fixups and reapply memory protections
_mmi.p = maps;
_mmi.n = specialz / sizeof(_mmi.p[0]);
for (i = 0; i < mapcount; ++i) {
if (!VirtualProtect((void *)((uint64_t)maps[i].x << 16), maps[i].size,
__prot2nt(maps[i].prot, maps[i].iscow), &oldprot)) {
// fixup memory manager
__maps.free = 0;
__maps.used = 0;
__maps.maps = maps;
dll_init(&__maps.stack.elem);
dll_make_first(&__maps.used, &__maps.stack.elem);
for (struct Map *map = maps; map; map = map->next) {
dll_make_last(&__maps.used, &map->elem);
if (!VirtualProtect(map->addr, map->size, __prot2nt(map->prot, map->iscow),
&oldprot)) {
AbortFork("VirtualProtect");
}
}
__maps_init();
// mitosis complete
if (!CloseHandle(reader)) {
@ -295,7 +301,7 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
jmp_buf jb;
uint32_t op;
char **args;
int i, rc = -1;
int rc = -1;
struct Proc *proc;
struct CosmoTib *tib;
char16_t pipename[64];
@ -304,10 +310,10 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
struct NtProcessInformation procinfo;
char *p, forkvar[6 + 21 + 1 + 21 + 1];
tib = __get_tls();
ftrace_enabled(-1);
strace_enabled(-1);
if (!(proc = __proc_new()))
return -1;
ftrace_enabled(-1);
strace_enabled(-1);
if (!setjmp(jb)) {
reader = CreateNamedPipe(__create_pipe_name(pipename), kNtPipeAccessInbound,
kNtPipeTypeByte | kNtPipeReadmodeByte, 1, PIPE_BUF,
@ -324,6 +330,7 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
startinfo.hStdError = g_fds.p[2].handle;
args = __argv;
#if SYSDEBUG
int i;
// If --strace was passed to this program, then propagate it the
// forked process since the flag was removed by __intercept_flag
if (strace_enabled(0) > 0) {
@ -350,22 +357,17 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
&procinfo});
if (spawnrc != -1) {
CloseHandle(procinfo.hThread);
ok = WriteAll(writer, jb, sizeof(jb)) &&
WriteAll(writer, &_mmi.i, sizeof(_mmi.i)) &&
WriteAll(writer, &_mmi.n, sizeof(_mmi.n)) &&
WriteAll(writer, _mmi.p, _mmi.i * sizeof(_mmi.p[0]));
if (IsAsan() && ok) {
ok = WriteAll(writer, (char *)(((intptr_t)_mmi.p >> 3) + 0x7fff8000),
(_mmi.i * sizeof(_mmi.p[0])) >> 3);
}
for (i = 0; i < _mmi.i && ok; ++i) {
if ((_mmi.p[i].flags & MAP_TYPE) != MAP_SHARED) {
char *p = (char *)((uint64_t)_mmi.p[i].x << 16);
ok = WriteAll(writer, jb, sizeof(jb));
for (struct Map *map = __maps.maps; ok && map; map = map->next) {
if (MAX((char *)__executable_start, map->addr) <
MIN((char *)_end, map->addr + map->size))
continue; // executable image is loaded by windows
ok = WriteAll(writer, map, sizeof(*map));
if (ok && (map->flags & MAP_TYPE) != MAP_SHARED) {
// XXX: forking destroys thread guard pages currently
VirtualProtect(
p, _mmi.p[i].size,
__prot2nt(_mmi.p[i].prot | PROT_READ, _mmi.p[i].iscow), &op);
ok = WriteAll(writer, p, _mmi.p[i].size);
VirtualProtect(map->addr, map->size,
__prot2nt(map->prot | PROT_READ, map->iscow), &op);
ok = WriteAll(writer, map->addr, map->size);
}
}
if (ok)
@ -416,6 +418,9 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
if (_weaken(__itimer_wipe)) {
_weaken(__itimer_wipe)();
}
// notify pthread join
atomic_store_explicit(&_pthread_static.ptid, GetCurrentThreadId(),
memory_order_release);
}
if (rc == -1) {
dll_make_first(&__proc.free, &proc->elem);

18
libc/proc/fork.c
View file

@ -26,6 +26,8 @@
#include "libc/dce.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/nt/files.h"
@ -43,32 +45,29 @@
#include "libc/thread/tls.h"
static void _onfork_prepare(void) {
if (_weaken(_pthread_onfork_prepare)) {
if (_weaken(_pthread_onfork_prepare))
_weaken(_pthread_onfork_prepare)();
}
_pthread_lock();
__maps_lock();
__fds_lock();
__mmi_lock();
}
static void _onfork_parent(void) {
__mmi_unlock();
__fds_unlock();
__maps_unlock();
_pthread_unlock();
if (_weaken(_pthread_onfork_parent)) {
if (_weaken(_pthread_onfork_parent))
_weaken(_pthread_onfork_parent)();
}
}
static void _onfork_child(void) {
pthread_mutexattr_t attr;
pthread_mutexattr_init(&attr);
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
extern pthread_mutex_t __mmi_lock_obj;
pthread_mutex_init(&__mmi_lock_obj, &attr);
pthread_mutex_init(&__fds_lock_obj, &attr);
pthread_mutexattr_destroy(&attr);
_pthread_init();
__maps_unlock();
if (_weaken(_pthread_onfork_child)) {
_weaken(_pthread_onfork_child)();
}
@ -81,9 +80,8 @@ int _fork(uint32_t dwCreationFlags) {
BLOCK_SIGNALS;
if (IsWindows())
__proc_lock();
if (__threaded) {
if (__threaded)
_onfork_prepare();
}
if (!IsWindows()) {
ax = sys_fork();
} else {

140
libc/runtime/clone.c
View file

@ -47,6 +47,7 @@
#include "libc/runtime/syslib.internal.h"
#include "libc/sock/internal.h"
#include "libc/stdalign.internal.h"
#include "libc/stdio/sysparam.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/arch.h"
#include "libc/sysv/consts/clone.h"
@ -80,27 +81,25 @@
struct CloneArgs {
alignas(16) union {
struct {
int tid;
atomic_int tid;
int this;
};
uint32_t utid;
int64_t tid64;
};
int *ptid;
int *ctid;
int *ztid;
atomic_int *ptid;
atomic_int *ctid;
atomic_int *ztid;
char *tls;
int (*func)(void *, int);
void *arg;
long sp;
};
int sys_set_tls(uintptr_t, void *);
int __stack_call(void *, int, long, long, int (*)(void *, int), void *);
int __stack_call(void *, int, long, long, int (*)(void *, int), long);
static struct CloneArgs *AllocateCloneArgs(char *stk, size_t stksz) {
return (struct CloneArgs *)(((uintptr_t)(stk + stksz) -
sizeof(struct CloneArgs)) &
-16);
static long AlignStack(long sp, char *stk, long stksz, int mal) {
return sp & -mal;
}
#ifdef __x86_64__
@ -120,8 +119,8 @@ WinThreadEntry(int rdi, // rcx
int rc;
if (wt->tls)
__set_tls_win32(wt->tls);
*wt->ctid = wt->tid;
rc = __stack_call(wt->arg, wt->tid, 0, 0, wt->func, wt);
*wt->ctid = GetCurrentThreadId();
rc = __stack_call(wt->arg, wt->tid, 0, 0, wt->func, wt->sp);
// we can now clear ctid directly since we're no longer using our own
// stack memory, which can now be safely free'd by the parent thread.
*wt->ztid = 0;
@ -134,24 +133,31 @@ WinThreadEntry(int rdi, // rcx
static textwindows errno_t CloneWindows(int (*func)(void *, int), char *stk,
size_t stksz, int flags, void *arg,
void *tls, int *ptid, int *ctid) {
void *tls, atomic_int *ptid,
atomic_int *ctid) {
long sp;
int64_t h;
uint32_t utid;
struct CloneArgs *wt;
wt = AllocateCloneArgs(stk, stksz);
sp = (intptr_t)stk + stksz;
sp = AlignStack(sp, stk, stksz, 16);
sp -= sizeof(struct CloneArgs);
sp &= -alignof(struct CloneArgs);
wt = (struct CloneArgs *)sp;
wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid;
wt->func = func;
wt->arg = arg;
wt->tls = flags & CLONE_SETTLS ? tls : 0;
wt->sp = sp;
if ((h = CreateThread(&kNtIsInheritable, 65536, (void *)WinThreadEntry, wt,
kNtStackSizeParamIsAReservation, &wt->utid))) {
kNtStackSizeParamIsAReservation, &utid))) {
if (flags & CLONE_PARENT_SETTID)
*ptid = utid;
if (flags & CLONE_SETTLS) {
struct CosmoTib *tib = tls;
atomic_store_explicit(&tib->tib_syshand, h, memory_order_release);
}
if (flags & CLONE_PARENT_SETTID) {
*ptid = wt->tid;
}
return 0;
} else {
return __dos2errno(GetLastError());
@ -222,14 +228,26 @@ XnuThreadMain(void *pthread, // rdi
}
static errno_t CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags,
void *arg, void *tls, int *ptid, int *ctid) {
void *arg, void *tls, atomic_int *ptid,
atomic_int *ctid) {
// perform this weird mandatory system call once
static bool once;
struct CloneArgs *wt;
if (!once) {
npassert(sys_bsdthread_register(XnuThreadThunk, 0, 0, 0, 0, 0, 0) != -1);
once = true;
}
wt = AllocateCloneArgs(stk, stksz);
// setup stack for thread
long sp;
struct CloneArgs *wt;
sp = (intptr_t)stk + stksz;
sp = AlignStack(sp, stk, stksz, 16);
sp -= sizeof(struct CloneArgs);
sp &= -alignof(struct CloneArgs);
wt = (struct CloneArgs *)sp;
// pass parameters to new thread via xnu
wt->ptid = flags & CLONE_PARENT_SETTID ? ptid : &wt->tid;
wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid;
@ -260,8 +278,8 @@ static wontreturn void OpenbsdThreadMain(void *p) {
}
static errno_t CloneOpenbsd(int (*func)(void *, int), char *stk, size_t stksz,
int flags, void *arg, void *tls, int *ptid,
int *ctid) {
int flags, void *arg, void *tls, atomic_int *ptid,
atomic_int *ctid) {
int rc;
intptr_t sp;
struct __tfork *tf;
@ -273,11 +291,12 @@ static errno_t CloneOpenbsd(int (*func)(void *, int), char *stk, size_t stksz,
sp -= sizeof(struct CloneArgs);
sp &= -alignof(struct CloneArgs);
wt = (struct CloneArgs *)sp;
sp = AlignStack(sp, stk, stksz, 16);
wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid;
wt->arg = arg;
wt->func = func;
tf->tf_stack = (char *)wt - 8;
tf->tf_stack = (char *)sp - 8;
tf->tf_tcb = flags & CLONE_SETTLS ? tls : 0;
tf->tf_tid = &wt->tid;
if ((rc = __tfork_thread(tf, sizeof(*tf), OpenbsdThreadMain, wt)) >= 0) {
@ -297,7 +316,7 @@ static errno_t CloneOpenbsd(int (*func)(void *, int), char *stk, size_t stksz,
static wontreturn void NetbsdThreadMain(void *arg, // rdi
int (*func)(void *, int), // rsi
int *tid, // rdx
int *ctid, // rcx
atomic_int *ctid, // rcx
int *ztid) { // r9
int ax, dx;
// TODO(jart): Why are we seeing flakes where *tid is zero?
@ -316,11 +335,13 @@ static wontreturn void NetbsdThreadMain(void *arg, // rdi
}
static int CloneNetbsd(int (*func)(void *, int), char *stk, size_t stksz,
int flags, void *arg, void *tls, int *ptid, int *ctid) {
int flags, void *arg, void *tls, atomic_int *ptid,
atomic_int *ctid) {
// NetBSD has its own clone() and it works, but it's technically a
// second-class API, intended to help Linux folks migrate to this.
int ax;
bool failed;
int ax, *tid;
atomic_int *tid;
intptr_t dx, sp;
static bool once;
struct ucontext_netbsd *ctx;
@ -335,16 +356,16 @@ static int CloneNetbsd(int (*func)(void *, int), char *stk, size_t stksz,
npassert(!failed);
once = true;
}
sp = (intptr_t)(stk + stksz);
sp = (intptr_t)stk + stksz;
// allocate memory for tid
sp -= sizeof(int);
sp = sp & -alignof(int);
tid = (int *)sp;
sp -= sizeof(atomic_int);
sp = sp & -alignof(atomic_int);
tid = (atomic_int *)sp;
*tid = 0;
// align the stack
sp = sp & -16;
sp = AlignStack(sp, stk, stksz, 16);
// simulate call to misalign stack and ensure backtrace looks good
sp -= 8;
@ -439,11 +460,16 @@ static wontreturn void FreebsdThreadMain(void *p) {
}
static errno_t CloneFreebsd(int (*func)(void *, int), char *stk, size_t stksz,
int flags, void *arg, void *tls, int *ptid,
int *ctid) {
int flags, void *arg, void *tls, atomic_int *ptid,
atomic_int *ctid) {
long sp;
int64_t tid;
struct CloneArgs *wt;
wt = AllocateCloneArgs(stk, stksz);
sp = (intptr_t)stk + stksz;
sp -= sizeof(struct CloneArgs);
sp &= -alignof(struct CloneArgs);
wt = (struct CloneArgs *)sp;
sp = AlignStack(sp, stk, stksz, 16);
wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid;
wt->tls = tls;
@ -453,7 +479,7 @@ static errno_t CloneFreebsd(int (*func)(void *, int), char *stk, size_t stksz,
.start_func = FreebsdThreadMain,
.arg = wt,
.stack_base = stk,
.stack_size = (uintptr_t)wt - (uintptr_t)stk,
.stack_size = sp - (long)stk,
.tls_base = flags & CLONE_SETTLS ? tls : 0,
.tls_size = 64,
.child_tid = &wt->tid64,
@ -492,15 +518,16 @@ static void *SiliconThreadMain(void *arg) {
struct CloneArgs *wt = arg;
asm volatile("mov\tx28,%0" : /* no outputs */ : "r"(wt->tls));
*wt->ctid = wt->this;
__stack_call(wt->arg, wt->this, 0, 0, wt->func, wt);
__stack_call(wt->arg, wt->this, 0, 0, wt->func, wt->sp);
*wt->ztid = 0;
ulock_wake(UL_COMPARE_AND_WAIT | ULF_WAKE_ALL, wt->ztid, 0);
return 0;
}
static errno_t CloneSilicon(int (*fn)(void *, int), char *stk, size_t stksz,
int flags, void *arg, void *tls, int *ptid,
int *ctid) {
int flags, void *arg, void *tls, atomic_int *ptid,
atomic_int *ctid) {
long sp;
void *attr;
errno_t res;
unsigned tid;
@ -508,7 +535,11 @@ static errno_t CloneSilicon(int (*fn)(void *, int), char *stk, size_t stksz,
size_t babystack;
struct CloneArgs *wt;
static atomic_uint tids;
wt = AllocateCloneArgs(stk, stksz);
sp = (intptr_t)stk + stksz;
sp -= sizeof(struct CloneArgs);
sp &= -alignof(struct CloneArgs);
wt = (struct CloneArgs *)sp;
sp = AlignStack(sp, stk, stksz, 16);
tid = atomic_fetch_add_explicit(&tids, 1, memory_order_acq_rel);
wt->this = tid = (tid & (kMaxThreadIds - 1)) + kMinThreadId;
wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
@ -516,6 +547,7 @@ static errno_t CloneSilicon(int (*fn)(void *, int), char *stk, size_t stksz,
wt->tls = flags & CLONE_SETTLS ? tls : 0;
wt->func = fn;
wt->arg = arg;
wt->sp = sp;
babystack = __syslib->__pthread_stack_min;
#pragma GCC push_options
#pragma GCC diagnostic ignored "-Walloca-larger-than="
@ -545,16 +577,16 @@ struct LinuxCloneArgs {
int (*func)(void *, int);
void *arg;
char *tls;
int ctid;
atomic_int ctid;
};
int sys_clone_linux(int flags, // rdi
long sp, // rsi
int *ptid, // rdx
int *ctid, // rcx
void *tls, // r8
void *func, // r9
void *arg); // 8(rsp)
int sys_clone_linux(int flags, // rdi
long sp, // rsi
atomic_int *ptid, // rdx
atomic_int *ctid, // rcx
void *tls, // r8
void *func, // r9
void *arg); // 8(rsp)
static int LinuxThreadEntry(void *arg, int tid) {
struct LinuxCloneArgs *wt = arg;
@ -563,19 +595,21 @@ static int LinuxThreadEntry(void *arg, int tid) {
}
static int CloneLinux(int (*func)(void *arg, int rc), char *stk, size_t stksz,
int flags, void *arg, void *tls, int *ptid, int *ctid) {
int flags, void *arg, void *tls, atomic_int *ptid,
atomic_int *ctid) {
int rc;
long sp;
struct LinuxCloneArgs *wt;
sp = (intptr_t)(stk + stksz);
sp = (intptr_t)stk + stksz;
sp -= sizeof(struct LinuxCloneArgs);
sp &= -alignof(struct LinuxCloneArgs);
wt = (struct LinuxCloneArgs *)sp;
// align the stack
#ifdef __aarch64__
sp = sp & -128; // for kernel 4.6 and earlier
sp = AlignStack(sp, stk, stksz, 128); // for kernel <=4.6
#else
sp = sp & -16;
sp = AlignStack(sp, stk, stksz, 16);
#endif
wt = (struct LinuxCloneArgs *)sp;
#ifdef __x86_64__
if (flags & CLONE_SETTLS) {
flags &= ~CLONE_SETTLS;

8
libc/runtime/cosmo2.c
View file

@ -21,6 +21,7 @@
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/strace.internal.h"
#include "libc/limits.h"
#include "libc/macros.internal.h"
@ -162,13 +163,8 @@ wontreturn textstartup void cosmo(long *sp, struct Syslib *m1, char *exename,
// needed by kisdangerous()
__pid = sys_getpid().ax;
// initialize memory manager
_mmi.i = 0;
_mmi.p = _mmi.s;
_mmi.n = ARRAYLEN(_mmi.s);
__virtualmax = -1;
// initialize file system
__maps_init();
__init_fds(argc, argv, envp);
// prepend cwd to executable path

3
libc/runtime/enable_tls.c
View file

@ -25,6 +25,7 @@
#include "libc/intrin/atomic.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/getenv.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/weaken.h"
#include "libc/macros.internal.h"
#include "libc/nt/files.h"
@ -251,7 +252,7 @@ textstartup void __enable_tls(void) {
_pthread_static.pt_flags = PT_STATIC;
dll_init(&_pthread_static.list);
_pthread_list = &_pthread_static.list;
atomic_store_explicit(&_pthread_static.ptid, tid, memory_order_relaxed);
atomic_store_explicit(&_pthread_static.ptid, tid, memory_order_release);
// ask the operating system to change the x86 segment register
__set_tls(tib);

4
libc/runtime/ftrace_install.c
View file

@ -16,8 +16,8 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/fmt/itoa.h"
#include "libc/intrin/kprintf.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
@ -28,7 +28,7 @@ textstartup int ftrace_install(void) {
ftrace_stackdigs = LengthInt64Thousands(GetStackSize());
return __hook(ftrace_hook, GetSymbolTable());
} else {
kprintf("error: --ftrace failed to open symbol table\n");
tinyprint(2, "error: --ftrace failed to open symbol table\n", NULL);
return -1;
}
}

2
libc/runtime/getavphyspages.c
View file

@ -23,5 +23,5 @@ long __get_avphys_pages(void) {
struct sysinfo si;
if (sysinfo(&si) == -1)
return -1;
return (((int64_t)si.freeram + si.bufferram) * si.mem_unit) / FRAMESIZE;
return (((int64_t)si.freeram + si.bufferram) * si.mem_unit) / __granularity();
}

2
libc/runtime/getpagesize.c
View file

@ -23,5 +23,5 @@
* @see sysconf(_SC_PAGE_SIZE) which is portable
*/
int getpagesize(void) {
return FRAMESIZE;
return __granularity();
}

2
libc/runtime/getphyspages.c
View file

@ -23,5 +23,5 @@ long __get_phys_pages(void) {
struct sysinfo si;
if (sysinfo(&si) == -1)
return -1;
return ((int64_t)si.totalram * si.mem_unit) / FRAMESIZE;
return ((int64_t)si.totalram * si.mem_unit) / __granularity();
}

2
libc/runtime/getsymboltable.c
View file

@ -63,7 +63,7 @@ static struct SymbolTable *GetSymbolTableFromZip(struct Zipos *zipos) {
(cf = GetZipFile(zipos, ".symtab")) != -1) {
lf = GetZipCfileOffset(zipos->map + cf);
size = GetZipLfileUncompressedSize(zipos->map + lf);
size2 = ROUNDUP(size, FRAMESIZE);
size2 = ROUNDUP(size, __granularity());
if ((res = _mapanon(size2))) {
switch (ZIP_LFILE_COMPRESSIONMETHOD(zipos->map + lf)) {
case kZipCompressionNone:

12
libc/runtime/grow.c
View file

@ -16,6 +16,7 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "ape/sections.internal.h"
#include "libc/assert.h"
#include "libc/fmt/conv.h"
#include "libc/intrin/weaken.h"
@ -31,6 +32,13 @@
#define GUARANTEE_TERMINATOR 1
#define INITIAL_CAPACITY (32 - GUARANTEE_TERMINATOR)
static bool isheap(const void *p) {
if (__executable_start <= (const unsigned char *)p &&
(const unsigned char *)p < _end)
return false;
return true;
}
bool __grow(void *pp, size_t *capacity, size_t itemsize, size_t extra) {
void **p, *p1, *p2;
size_t n1, n2;
@ -39,8 +47,8 @@ bool __grow(void *pp, size_t *capacity, size_t itemsize, size_t extra) {
p = (void **)pp;
unassert(itemsize);
unassert((*p && *capacity) || (!*p && !*capacity));
unassert(!_isheap(*p) || ((intptr_t)*p & 15) == 0);
p1 = _isheap(*p) ? *p : NULL;
unassert(!isheap(*p) || ((intptr_t)*p & 15) == 0);
p1 = isheap(*p) ? *p : NULL;
p2 = NULL;
n1 = *capacity;
n2 = (*p ? n1 + (n1 >> 1) : MAX(4, INITIAL_CAPACITY / itemsize)) + extra;

3
libc/runtime/internal.h
View file

@ -6,7 +6,7 @@
#include "libc/runtime/runtime.h"
#define STACK_CEIL 0x700000000000ul
#define STACK_SIZE FRAMESIZE
#define STACK_SIZE 65536
#define RUNLEVEL_MALLOC 1
@ -41,7 +41,6 @@ long _setstack(void *, void *, ...);
int GetDosArgv(const char16_t *, char *, size_t, char **, size_t);
int GetDosEnviron(const char16_t *, char *, size_t, char **, size_t);
bool __intercept_flag(int *, char *[], const char *);
int sys_mprotect_nt(void *, size_t, int);
int __inflate(void *, size_t, const void *, size_t);
void *__mmap_unlocked(void *, size_t, int, int, int, int64_t);
int __munmap_unlocked(char *, size_t);

43
libc/runtime/ismemtracked.greg.c
View file

@ -1,43 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/memtrack.internal.h"
static inline bool IsMemtrackedImpl(int x, int y) {
unsigned i;
i = __find_memory(&_mmi, x);
if (i == _mmi.i)
return false;
if (x < _mmi.p[i].x)
return false;
for (;;) {
if (y <= _mmi.p[i].y)
return true;
if (++i == _mmi.i)
return false;
if (_mmi.p[i].x != _mmi.p[i - 1].y + 1)
return false;
}
}
bool IsMemtracked(int x, int y) {
/* assumes __mmi_lock() is held */
bool res;
res = IsMemtrackedImpl(x, y);
return res;
}

7
libc/runtime/mapanon.c
View file

@ -22,7 +22,6 @@
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
/**
* Helper function for allocating anonymous mapping.
@ -59,11 +58,9 @@
void *_mapanon(size_t size) {
void *m;
m = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (m != MAP_FAILED) {
if (m != MAP_FAILED)
return m;
}
if (errno == ENOMEM && _weaken(__oom_hook)) {
if (errno == ENOMEM && _weaken(__oom_hook))
_weaken(__oom_hook)(size);
}
return 0;
}

20
libc/runtime/mapstack.c
View file

@ -17,6 +17,7 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/asancodes.h"
@ -27,6 +28,9 @@
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
#define MAP_ANON_OPENBSD 0x1000
#define MAP_STACK_OPENBSD 0x4000
/**
* Allocates stack.
*
@ -42,13 +46,17 @@
*/
void *NewCosmoStack(void) {
char *p;
if ((p = mmap(0, GetStackSize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS |
(IsAarch64() && IsLinux() && IsQemuUser() ? MAP_PRIVATE
: MAP_STACK),
-1, 0)) != MAP_FAILED) {
size_t n = GetStackSize() + (uintptr_t)ape_stack_align;
if ((p = mmap(0, n, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1,
0)) != MAP_FAILED) {
if (IsOpenbsd() && __sys_mmap(p, n, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD |
MAP_STACK_OPENBSD,
-1, 0, 0) != p) {
notpossible;
}
if (IsAsan()) {
__asan_poison(p + GetStackSize() - 16, 16, kAsanStackOverflow);
__asan_poison(p + n - 16, 16, kAsanStackOverflow);
__asan_poison(p, getauxval(AT_PAGESZ), kAsanStackOverflow);
}
return p;

183
libc/runtime/memtrack.internal.h
View file

@ -1,189 +1,22 @@
#ifndef COSMOPOLITAN_LIBC_RUNTIME_MEMTRACK_H_
#define COSMOPOLITAN_LIBC_RUNTIME_MEMTRACK_H_
#include "ape/sections.internal.h"
#include "libc/dce.h"
#include "libc/macros.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/sysv/consts/ss.h"
#include "libc/thread/tls.h"
COSMOPOLITAN_C_START_
#define kAutomapStart 0x100080040000
#define kAutomapSize (kMemtrackStart - kAutomapStart)
#define kMemtrackStart 0x1fe7fffc0000
#define kMemtrackSize (0x1ffffffc0000 - kMemtrackStart)
#ifndef __SANITIZE_ADDRESS__
#define kFixedmapStart 0x300000000
#define kFixedmapSize (0x400000000 - kFixedmapStart)
#define kMemtrackFdsStart 0x6fe000000
#define kMemtrackFdsSize (0x6ff000000 - kMemtrackFdsStart)
#define kMemtrackZiposStart 0x6fd000000
#define kMemtrackZiposSize (0xafe000000 - kMemtrackZiposStart)
#else
#define kFixedmapStart 0x300000040000
#define kFixedmapSize (0x400000040000 - kFixedmapStart)
#define kMemtrackNsyncStart 0x6fc000040000
#define kMemtrackNsyncSize (0x6fcffffc0000 - kMemtrackNsyncStart)
#define kMemtrackFdsStart 0x6fe000040000
#define kMemtrackFdsSize (0x6feffffc0000 - kMemtrackFdsStart)
#define kMemtrackZiposStart 0x6fd000040000
#define kMemtrackZiposSize (0x6fdffffc0000 - kMemtrackZiposStart)
#define kMemtrackGran (!IsAsan() ? FRAMESIZE : FRAMESIZE * 8)
struct MemoryInterval {
int x;
int y;
long h;
long size;
long offset;
int flags;
char prot;
bool iscow;
bool readonlyfile;
};
struct MemoryIntervals {
size_t i, n;
struct MemoryInterval *p;
struct MemoryInterval s[16];
};
extern struct MemoryIntervals _mmi;
void __mmi_lock(void);
void __mmi_unlock(void);
bool IsMemtracked(int, int);
void PrintSystemMappings(int);
unsigned __find_memory(const struct MemoryIntervals *, int) nosideeffect;
bool __check_memtrack(const struct MemoryIntervals *) nosideeffect;
void PrintMemoryIntervals(int, const struct MemoryIntervals *);
int __track_memory(struct MemoryIntervals *, int, int, long, int, int, bool,
bool, long, long);
int __untrack_memory(struct MemoryIntervals *, int, int,
void (*)(struct MemoryIntervals *, int, int));
void __release_memory_nt(struct MemoryIntervals *, int, int);
int __untrack_memories(void *, size_t);
size_t __get_memtrack_size(struct MemoryIntervals *) nosideeffect;
#ifdef __x86_64__
/*
* AMD64 has 48-bit signed pointers (PML4T)
* AMD64 is trying to go bigger, i.e. 57-bit (PML5T)
* LINUX forbids userspace from leveraging negative pointers
* Q-EMU may impose smaller vaspaces emulating AMD on non-AMD
*
* Having "signed pointers" means these top sixteen bits
*
* 0x0000000000000000
* ^^^^
*
* must be
*
* - 0000 for positive pointers
* - FFFF for negative pointers
*
* otherwise the instruction using the faulty pointer will fault.
*/
#define IsLegalPointer(p) \
(-0x800000000000 <= (intptr_t)(p) && (intptr_t)(p) <= 0x7fffffffffff)
#define ADDR_32_TO_48(x) (intptr_t)((uint64_t)(int)(x) << 16)
#elif defined(__aarch64__)
/*
* ARM64 has 48-bit unsigned pointers (Armv8.0-A)
* ARM64 can possibly go bigger, i.e. 52-bit (Armv8.2-A)
* ARM64 can impose arbitrarily smaller vaspaces, e.g. 40/44-bit
* APPLE in their limitless authoritarianism forbids 32-bit pointers
*/
#define IsLegalPointer(p) ((uintptr_t)(p) <= 0xffffffffffff)
#define ADDR_32_TO_48(x) (uintptr_t)((uint64_t)(uint32_t)(x) << 16)
#else
/* RISC-V Sipeed Nezha has 39-bit vaspace */
#error "unsupported architecture"
#endif
forceinline pureconst bool IsLegalSize(uint64_t n) {
/* subtract frame size so roundup is safe */
return n <= 0x800000000000 - FRAMESIZE;
}
forceinline pureconst bool IsAutoFrame(int x) {
return (int)(kAutomapStart >> 16) <= x &&
x <= (int)((kAutomapStart + kAutomapSize - 1) >> 16);
}
forceinline pureconst bool IsMemtrackFrame(int x) {
return (int)(kAutomapStart >> 16) <= x &&
x <= (int)((kAutomapStart + kAutomapSize - 1) >> 16);
}
forceinline pureconst bool IsGfdsFrame(int x) {
return (int)(kMemtrackFdsStart >> 16) <= x &&
x <= (int)((kMemtrackFdsStart + kMemtrackFdsSize - 1) >> 16);
}
forceinline pureconst bool IsNsyncFrame(int x) {
return (int)(kMemtrackNsyncStart >> 16) <= x &&
x <= (int)((kMemtrackNsyncStart + kMemtrackNsyncSize - 1) >> 16);
}
forceinline pureconst bool IsZiposFrame(int x) {
return (int)(kMemtrackZiposStart >> 16) <= x &&
x <= (int)((kMemtrackZiposStart + kMemtrackZiposSize - 1) >> 16);
}
forceinline pureconst bool IsShadowFrame(int x) {
return 0x7fff <= x && x < 0x10008000;
}
forceinline pureconst bool IsStaticStackFrame(int x) {
intptr_t stack = GetStaticStackAddr(0);
return (int)(stack >> 16) <= x &&
x <= (int)((stack + (GetStackSize() - FRAMESIZE)) >> 16);
}
forceinline pureconst bool IsStackFrame(int x) {
intptr_t stack = GetStackAddr();
return (int)(stack >> 16) <= x &&
x <= (int)((stack + (GetStackSize() - FRAMESIZE)) >> 16);
}
forceinline pureconst bool IsOldStack(const void *x) {
size_t foss_stack_size = 8ul * 1024 * 1024;
uintptr_t top = __oldstack + foss_stack_size;
uintptr_t bot = __oldstack - foss_stack_size;
return bot <= (uintptr_t)x && (uintptr_t)x < top;
}
forceinline pureconst bool IsOldStackFrame(int x) {
size_t foss_stack_size = 8ul * 1024 * 1024;
uintptr_t top = __oldstack + foss_stack_size;
uintptr_t bot = __oldstack - foss_stack_size;
return (int)(bot >> 16) <= x && x <= (int)((top >> 16) - 1);
}
forceinline pureconst bool IsFixedFrame(int x) {
return (kFixedmapStart >> 16) <= x &&
x <= ((kFixedmapStart + (kFixedmapSize - 1)) >> 16);
}
forceinline pureconst bool OverlapsImageSpace(const void *p, size_t n) {
const unsigned char *BegA, *EndA, *BegB, *EndB;
if (n) {
BegA = p;
EndA = BegA + n;
BegB = __executable_start;
EndB = _end;
return MAX(BegA, BegB) < MIN(EndA, EndB);
} else {
return 0;
}
}
forceinline pureconst bool OverlapsShadowSpace(const void *p, size_t n) {
intptr_t BegA, EndA, BegB, EndB;
if (n) {
BegA = (intptr_t)p;
EndA = BegA + n;
BegB = 0x7fff0000;
EndB = 0x100080000000;
return MAX(BegA, BegB) < MIN(EndA, EndB);
} else {
return 0;
}
}
COSMOPOLITAN_C_END_
#endif /* COSMOPOLITAN_LIBC_RUNTIME_MEMTRACK_H_ */

2073
libc/runtime/memtrack64.txt
View file

File diff suppressed because it is too large Load diff

503
libc/runtime/mmap.c
View file

@ -1,503 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
#include "libc/calls/struct/sigset.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/asancodes.h"
#include "libc/intrin/bsr.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/safemacros.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/limits.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/libfatal.internal.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
#include "libc/nt/enum/memflags.h"
#include "libc/nt/enum/pageflags.h"
#include "libc/nt/memory.h"
#include "libc/nt/process.h"
#include "libc/nt/runtime.h"
#include "libc/nt/struct/processmemorycounters.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/zipos.internal.h"
#include "libc/stdckdint.h"
#include "libc/stdio/rand.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/o.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/consts/sig.h"
#include "libc/sysv/consts/ss.h"
#include "libc/sysv/errfuns.h"
#include "libc/thread/thread.h"
#define MAP_ANONYMOUS_linux 0x00000020
#define MAP_ANONYMOUS_openbsd 0x00001000
#define MAP_GROWSDOWN_linux 0x00000100
#define MAP_STACK_freebsd 0x00000400
#define MAP_STACK_openbsd 0x00004000
#define IP(X) (intptr_t)(X)
#define VIP(X) (void *)IP(X)
#define ALIGNED(p) (!(IP(p) & (FRAMESIZE - 1)))
#define SHADE(x) (((intptr_t)(x) >> 3) + 0x7fff8000)
#define FRAME(x) ((int)((intptr_t)(x) >> 16))
static inline pureconst unsigned long __rounddown2pow(unsigned long x) {
return x ? 1ul << bsrl(x) : 0;
}
static wontreturn void __mmap_die(const char *s) {
if (_weaken(__die))
_weaken(__die)();
STRACE("%s %m", s);
_Exit(199);
}
static inline bool __overlaps_existing_mapping(char *p, size_t n) {
int a, b, i;
unassert(n > 0);
a = FRAME(p);
b = FRAME(p + (n - 1));
i = __find_memory(&_mmi, a);
if (i < _mmi.i) {
if (a <= _mmi.p[i].x && _mmi.p[i].x <= b)
return true;
if (a <= _mmi.p[i].y && _mmi.p[i].y <= b)
return true;
if (_mmi.p[i].x <= a && b <= _mmi.p[i].y)
return true;
}
return false;
}
static bool __choose_memory(int x, int n, int align, int *res) {
// TODO: improve performance
int i, start, end;
unassert(align > 0);
if (_mmi.i) {
// find the start of the automap memory region
i = __find_memory(&_mmi, x);
if (i < _mmi.i) {
// check to see if there's space available before the first entry
if (!ckd_add(&start, x, align - 1)) {
start &= -align;
if (!ckd_add(&end, start, n - 1)) {
if (end < _mmi.p[i].x) {
*res = start;
return true;
}
}
}
// check to see if there's space available between two entries
while (++i < _mmi.i) {
if (!ckd_add(&start, _mmi.p[i - 1].y, 1) &&
!ckd_add(&start, start, align - 1)) {
start &= -align;
if (!ckd_add(&end, start, n - 1)) {
if (end < _mmi.p[i].x) {
*res = start;
return true;
}
}
}
}
}
// otherwise append after the last entry if space is available
if (!ckd_add(&start, _mmi.p[i - 1].y, 1) &&
!ckd_add(&start, start, align - 1)) {
start &= -align;
if (!ckd_add(&end, start, n - 1)) {
*res = start;
return true;
}
}
} else {
// if memtrack is empty, then just assign the requested address
// assuming it doesn't overflow
if (!ckd_add(&start, x, align - 1)) {
start &= -align;
if (!ckd_add(&end, start, n - 1)) {
*res = start;
return true;
}
}
}
return false;
}
static bool __auto_map(int count, int align, int *res) {
return __choose_memory(FRAME(kAutomapStart), count, align, res) &&
*res + count <= FRAME(kAutomapStart + (kAutomapSize - 1));
}
static void *__finish_memory(void *addr, size_t size, int prot, int flags,
int fd, int64_t off, int f, int x, int n,
struct DirectMap dm) {
if (!IsWindows() && (flags & MAP_FIXED)) {
if (__untrack_memories(addr, size)) {
__mmap_die("FIXED UNTRACK FAILED");
}
}
if (__track_memory(&_mmi, x, x + (n - 1), dm.maphandle, prot, flags, false,
false, off, size)) {
if (sys_munmap(addr, n) == -1) {
__mmap_die("TRACK MUNMAP FAILED");
}
return MAP_FAILED;
}
if (_weaken(__asan_map_shadow) && !OverlapsShadowSpace(addr, size)) {
_weaken(__asan_map_shadow)((intptr_t)addr, size);
}
return addr;
}
static void *__map_memory(void *addr, size_t size, int prot, int flags, int fd,
int64_t off, int f, int x, int n) {
struct DirectMap dm;
dm = sys_mmap(addr, size, prot, f, fd, off);
if (VERY_UNLIKELY(dm.addr == MAP_FAILED)) {
if (IsWindows() && (flags & MAP_FIXED)) {
__mmap_die("can't recover from MAP_FIXED errors on Windows");
}
return MAP_FAILED;
}
if (VERY_UNLIKELY(dm.addr != addr)) {
__mmap_die("KERNEL DIDN'T RESPECT MAP_FIXED");
}
return __finish_memory(addr, size, prot, flags, fd, off, f, x, n, dm);
}
/**
* Maps memory from system, one frame at a time.
*
* This is useful on Windows since it allows us to partially unmap or
* punch holes into existing mappings.
*/
static textwindows dontinline void *__map_memories(char *addr, size_t size,
int prot, int flags, int fd,
int64_t off, int f, int x,
int n) {
size_t i, m;
int64_t oi, sz;
struct DirectMap dm;
bool iscow, readonlyfile;
m = (size_t)(n - 1) << 16;
unassert(m < size);
unassert(m + FRAMESIZE >= size);
oi = fd == -1 ? 0 : off + m;
sz = size - m;
dm = sys_mmap(addr + m, sz, prot, f, fd, oi);
if (dm.addr == MAP_FAILED)
return MAP_FAILED;
iscow = (flags & MAP_TYPE) != MAP_SHARED && fd != -1;
readonlyfile = (flags & MAP_TYPE) == MAP_SHARED && fd != -1 &&
(g_fds.p[fd].flags & O_ACCMODE) == O_RDONLY;
if (__track_memory(&_mmi, x + (n - 1), x + (n - 1), dm.maphandle, prot, flags,
readonlyfile, iscow, oi, sz) == -1) {
__mmap_die("__map_memories unrecoverable #1");
}
for (i = 0; i < m; i += FRAMESIZE) {
oi = fd == -1 ? 0 : off + i;
sz = FRAMESIZE;
dm = sys_mmap(addr + i, sz, prot, f, fd, oi);
if (dm.addr == MAP_FAILED ||
__track_memory(&_mmi, x + i / FRAMESIZE, x + i / FRAMESIZE,
dm.maphandle, prot, flags, readonlyfile, iscow, oi,
sz) == -1) {
__mmap_die("__map_memories unrecoverable #2");
}
}
if (_weaken(__asan_map_shadow) && !OverlapsShadowSpace(addr, size)) {
_weaken(__asan_map_shadow)((intptr_t)addr, size);
}
return addr;
}
inline void *__mmap_unlocked(void *addr, size_t size, int prot, int flags,
int fd, int64_t off) {
char *p = addr;
struct DirectMap dm;
size_t requested_size;
bool needguard, clashes;
int a, f, n, x, pagesize;
size_t virtualused, virtualneed;
if (VERY_UNLIKELY(!size)) {
STRACE("can't mmap zero bytes");
return VIP(einval());
}
if (VERY_UNLIKELY(!ALIGNED(p))) {
STRACE("cosmo mmap is 64kb aligned");
return VIP(einval());
}
if (VERY_UNLIKELY(!IsLegalSize(size))) {
STRACE("mmap size isn't legal");
return VIP(einval());
}
if (VERY_UNLIKELY(!IsLegalPointer(p))) {
STRACE("mmap addr isn't 48-bit");
return VIP(einval());
}
if ((flags & (MAP_SHARED | MAP_PRIVATE)) == (MAP_SHARED | MAP_PRIVATE)) {
flags = MAP_SHARED; // cf. MAP_SHARED_VALIDATE
}
requested_size = size;
pagesize = getauxval(AT_PAGESZ);
if (flags & MAP_ANONYMOUS) {
fd = -1;
off = 0;
size = ROUNDUP(size, FRAMESIZE);
if ((flags & MAP_TYPE) == MAP_FILE) {
STRACE("need MAP_PRIVATE or MAP_SHARED");
return VIP(einval());
}
} else if (__isfdkind(fd, kFdZip)) {
STRACE("mmap fd is zipos handle");
return VIP(einval());
} else if (VERY_UNLIKELY(off < 0)) {
STRACE("mmap negative offset");
return VIP(einval());
} else if (off & ((IsWindows() ? FRAMESIZE : pagesize) - 1)) {
STRACE("mmap offset isn't properly aligned");
return VIP(einval());
} else if (VERY_UNLIKELY(INT64_MAX - size < off)) {
STRACE("mmap too large");
return VIP(einval());
}
if (__virtualmax < LONG_MAX &&
(ckd_add(&virtualneed, (virtualused = __get_memtrack_size(&_mmi)),
size) ||
virtualneed > __virtualmax)) {
STRACE("mmap %'zu size + %'zu inuse exceeds virtual memory limit %'zu",
size, virtualused, __virtualmax);
return VIP(enomem());
}
clashes = OverlapsImageSpace(p, size) || __overlaps_existing_mapping(p, size);
if ((flags & MAP_FIXED_NOREPLACE) == MAP_FIXED_NOREPLACE && clashes) {
STRACE("mmap noreplace overlaps existing");
return VIP(eexist());
}
if (ckd_add(&n, (int)(size >> 16), (int)!!(size & (FRAMESIZE - 1)))) {
STRACE("mmap range overflows");
return VIP(einval());
}
a = MAX(1, __rounddown2pow(size) >> 16);
f = (flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED;
if (flags & MAP_FIXED) {
x = FRAME(p);
if (IsWindows()) {
if (__untrack_memories(p, size)) {
__mmap_die("FIXED UNTRACK FAILED");
}
}
} else if (p && !clashes && !OverlapsShadowSpace(p, size)) {
x = FRAME(p);
} else if (!__auto_map(n, a, &x)) {
STRACE("automap has no room for %d frames with %d alignment", n, a);
return VIP(enomem());
}
needguard = false;
p = (char *)ADDR_32_TO_48(x);
if ((f & MAP_TYPE) == MAP_STACK) {
if (~f & MAP_ANONYMOUS) {
STRACE("MAP_STACK must be anonymous");
return VIP(einval());
}
f &= ~MAP_TYPE;
f |= MAP_PRIVATE;
if (IsOpenbsd()) { // openbsd:dubstack
// on openbsd this is less about scalability of threads, and more
// about defining the legal intervals for the RSP register. sadly
// openbsd doesn't let us create a new fixed stack mapping. but..
// openbsd does allow us to overwrite existing fixed mappings, to
// authorize its usage as a stack.
if (sys_mmap(p, size, prot, f, fd, off).addr == MAP_FAILED) {
return MAP_FAILED;
}
f |= MAP_STACK_openbsd;
needguard = true;
} else if (IsLinux()) {
// make sure there's no existing stuff existing between our stack
// starting page and the bottom guard page, since that would stop
// our stack page from growing down.
npassert(!sys_munmap(p, size));
int guardsize = pagesize, e = errno;
if ((dm = sys_mmap(p + size - guardsize, guardsize, prot,
f | MAP_GROWSDOWN_linux, fd, off))
.addr != MAP_FAILED) {
npassert(sys_mmap(p, pagesize, PROT_NONE,
MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)
.addr == p);
dm.addr = p;
p = __finish_memory(p, size, prot, flags, fd, off, f, x, n, dm);
if (IsAsan() && p != MAP_FAILED) {
__asan_poison(p, pagesize, kAsanStackOverflow);
}
return p;
} else if (errno == ENOTSUP) {
// WSL and Blink don't support MAP_GROWSDOWN
needguard = true;
errno = e;
}
} else {
if (IsFreebsd()) {
f |= MAP_STACK_freebsd;
}
needguard = true;
}
}
if (!IsWindows()) {
p = __map_memory(p, size, prot, flags, fd, off, f, x, n);
} else {
p = __map_memories(p, size, prot, flags, fd, off, f, x, n);
}
if (p != MAP_FAILED) {
if (IsAsan()) {
__asan_poison(p + requested_size, size - requested_size,
kAsanMmapSizeOverrun);
}
if (needguard) {
if (!IsWindows()) {
unassert(!mprotect(p, pagesize, PROT_NONE));
if (IsAsan()) {
__asan_poison(p, pagesize, kAsanStackOverflow);
}
} else {
uint32_t oldattr;
unassert(VirtualProtect(p, pagesize, kNtPageReadwrite | kNtPageGuard,
&oldattr));
}
}
}
return p;
}
/**
* Creates virtual memory, e.g.
*
* char *m;
* m = mmap(NULL, FRAMESIZE, PROT_READ | PROT_WRITE,
* MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
* munmap(m, FRAMESIZE);
*
* @param addr should be 0 to let your memory manager choose address;
* unless MAP_FIXED or MAP_FIXED_NOREPLACE are specified in flags
* in which case this function will do precicely as you ask, even
* if p=0 (in which you need -fno-delete-null-pointer-checks); it
* needs to be 64kb aligned because it's a wise choice that sadly
* needs to be made mandatory because of Windows although you can
* use __sys_mmap() to circumvent it on System Five in which case
* runtime support services, e.g. asan memory safety, could break
* @param size must be >0 otherwise EINVAL is raised
* @param prot can have PROT_READ/PROT_WRITE/PROT_EXEC/PROT_NONE/etc.
* @param flags should have one of the following masked by `MAP_TYPE`
* - `MAP_FILE` in which case `MAP_ANONYMOUS` shouldn't be used
* - `MAP_PRIVATE` for copy-on-write behavior of writeable pages
* - `MAP_SHARED` to create shared memory between processes
* - `MAP_STACK` to create a grows-down alloc, where a guard page
* is automatically protected at the bottom, sized as AT_PAGESZ
* Your `flags` may optionally bitwise or any of the following:
* - `MAP_ANONYMOUS` in which case `fd` and `off` are ignored
* - `MAP_FIXED` in which case `addr` becomes more than a hint
* - `MAP_FIXED_NOREPLACE` to protect existing mappings; this is
* always polyfilled by mmap() which tracks its own memory and
* removed before passing to the kernel, in order to support
* old versions; if you believe mappings exist which only the
* kernel knows, then this flag may be passed to sys_mmap() on
* Linux 4.17+ and FreeBSD (where it has multiple bits)
* - `MAP_CONCEAL` is FreeBSD/NetBSD/OpenBSD-only
* - `MAP_NORESERVE` is Linux/XNU/NetBSD-only
* - `MAP_POPULATE` is Linux/FreeBSD-only
* - `MAP_NONBLOCK` is Linux-only
* - `MAP_NOSYNC` is FreeBSD-only
* - `MAP_INHERIT` is NetBSD-only
* - `MAP_LOCKED` is Linux-only
* @param fd is an open()'d file descriptor, whose contents shall be
* made available w/ automatic reading at the chosen address
* @param off specifies absolute byte index of fd's file for mapping,
* should be zero if MAP_ANONYMOUS is specified, which SHOULD be
* aligned to FRAMESIZE
* @return virtual base address of new mapping, or MAP_FAILED w/ errno
*/
void *mmap(void *addr, size_t size, int prot, int flags, int fd, int64_t off) {
void *res;
#if SYSDEBUG
size_t toto = 0;
#if _KERNTRACE || _NTTRACE
if (IsWindows()) {
STRACE("mmap(%p, %'zu, %s, %s, %d, %'ld) → ...", addr, size,
DescribeProtFlags(prot), DescribeMapFlags(flags), fd, off);
}
#endif
#endif
__mmi_lock();
if (!__isfdkind(fd, kFdZip)) {
res = __mmap_unlocked(addr, size, prot, flags, fd, off);
} else {
res = _weaken(__zipos_mmap)(
addr, size, prot, flags,
(struct ZiposHandle *)(intptr_t)g_fds.p[fd].handle, off);
}
#if SYSDEBUG
toto = __strace > 0 ? __get_memtrack_size(&_mmi) : 0;
#endif
__mmi_unlock();
#if SYSDEBUG
STRACE("mmap(%p, %'zu, %s, %s, %d, %'ld) → %p% m (%'zu bytes total)", addr,
size, DescribeProtFlags(prot), DescribeMapFlags(flags), fd, off, res,
toto);
#endif
return res;
}
__strong_reference(mmap, mmap64);

69
libc/runtime/mprotect-nt.greg.c
View file

@ -1,69 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/struct/sigset.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/nt/memory.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/memtrack.internal.h"
textwindows int sys_mprotect_nt(void *addr, size_t size, int prot) {
int rc = 0;
unsigned i;
uint32_t op;
char *a, *b, *x, *y, *p;
BLOCK_SIGNALS;
__mmi_lock();
size = (size + 4095) & -4096;
p = addr;
i = __find_memory(&_mmi, (intptr_t)p >> 16);
if (i == _mmi.i || (!i && p + size <= (char *)ADDR_32_TO_48(_mmi.p[0].x))) {
// memory isn't in memtrack
// let's just trust the user then
// it's probably part of the executable
if (!VirtualProtect(addr, size, __prot2nt(prot, false), &op)) {
rc = -1;
}
} else {
// memory is in memtrack, so use memtrack, to do dimensioning
// we unfortunately must do something similar to this for cow
for (; i < _mmi.i; ++i) {
x = (char *)ADDR_32_TO_48(_mmi.p[i].x);
y = (char *)ADDR_32_TO_48(_mmi.p[i].y) + 65536;
if ((x <= p && p < y) || (x < p + size && p + size <= y) ||
(p < x && y < p + size)) {
if (p <= x && p + size >= y) {
_mmi.p[i].prot = prot;
} else {
_mmi.p[i].prot |= prot;
}
a = MIN(MAX(p, x), y);
b = MAX(MIN(p + size, y), x);
if (!VirtualProtect(a, b - a, __prot2nt(prot, _mmi.p[i].iscow), &op)) {
rc = -1;
break;
}
} else {
break;
}
}
}
__mmi_unlock();
ALLOW_SIGNALS;
return rc;
}

56
libc/runtime/mprotect.c
View file

@ -1,56 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/strace.internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
/**
* Modifies restrictions on virtual memory address range.
*
* @param addr needs to be 4kb aligned
* @param prot can have PROT_{NONE,READ,WRITE,EXEC,GROWSDOWN,GROWSUP}
* @return 0 on success, or -1 w/ errno
* @see mmap()
*/
int mprotect(void *addr, size_t size, int prot) {
int64_t rc;
if (prot &
~(PROT_READ | PROT_WRITE | PROT_EXEC | PROT_GROWSDOWN | PROT_GROWSUP)) {
rc = einval(); // unix checks prot before checking size
} else if (!size) {
return 0; // make new technology consistent with unix
} else if (UNLIKELY((intptr_t)addr & (getauxval(AT_PAGESZ) - 1))) {
rc = einval(); // unix checks prot before checking size
} else if (!IsWindows()) {
rc = sys_mprotect(addr, size, prot);
} else {
rc = sys_mprotect_nt(addr, size, prot);
}
STRACE("mprotect(%p, %'zu, %s) → %d% m", addr, size, DescribeProtFlags(prot),
rc);
return rc;
}

50
libc/runtime/msync-nt.c
View file

@ -1,50 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macros.internal.h"
#include "libc/nt/files.h"
#include "libc/nt/memory.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/msync.h"
#include "libc/sysv/consts/o.h"
#include "libc/sysv/consts/prot.h"
textwindows int sys_msync_nt(char *addr, size_t size, int flags) {
int i, rc = 0;
char *a, *b, *x, *y;
__mmi_lock();
for (i = __find_memory(&_mmi, (intptr_t)addr >> 16); i < _mmi.i; ++i) {
x = (char *)ADDR_32_TO_48(_mmi.p[i].x);
y = x + _mmi.p[i].size;
if ((x <= addr && addr < y) || (x < addr + size && addr + size <= y) ||
(addr < x && y < addr + size)) {
a = MIN(MAX(addr, x), y);
b = MAX(MIN(addr + size, y), x);
if (!FlushViewOfFile(a, b - a)) {
rc = -1;
break;
}
// TODO(jart): FlushFileBuffers too on g_fds handle if MS_SYNC?
} else {
break;
}
}
__mmi_unlock();
return rc;
}

160
libc/runtime/munmap.c
View file

@ -1,160 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/state.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/strace.internal.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/libfatal.internal.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
#include "libc/sysv/errfuns.h"
#define IP(X) (intptr_t)(X)
#define ALIGNED(p) (!(IP(p) & (FRAMESIZE - 1)))
#define FRAME(x) ((int)((intptr_t)(x) >> 16))
static void __munmap_shadow(char *p, size_t n) {
intptr_t a, b, x, y;
KERNTRACE("__munmap_shadow(%p, %'zu)", p, n);
a = ((intptr_t)p >> 3) + 0x7fff8000;
b = a + (n >> 3);
if (IsMemtracked(FRAME(a), FRAME(b - 1))) {
x = ROUNDUP(a, FRAMESIZE);
y = ROUNDDOWN(b, FRAMESIZE);
if (0 && x < y) {
// delete shadowspace if unmapping ≥512kb. in practice it has
// to be >1mb since we can only unmap it if it's aligned, and
// as such we poison the edges if there are any.
__repstosb((void *)a, kAsanUnmapped, x - a);
__munmap_unlocked((void *)x, y - x);
__repstosb((void *)y, kAsanUnmapped, b - y);
} else {
// otherwise just poison and assume reuse
__repstosb((void *)a, kAsanUnmapped, b - a);
}
} else {
STRACE("unshadow(%.12p, %p) EFAULT", a, b - a);
}
}
// our api supports doing things like munmap(0, 0x7fffffffffff) but some
// platforms (e.g. openbsd) require that we know the specific intervals
// or else it returns EINVAL. so we munmap a piecewise.
static void __munmap_impl(char *p, size_t n) {
char *q;
size_t m;
intptr_t a, b, c;
int i, l, r, beg, end;
KERNTRACE("__munmap_impl(%p, %'zu)", p, n);
l = FRAME(p);
r = FRAME(p + n - 1);
i = __find_memory(&_mmi, l);
for (; i < _mmi.i && r >= _mmi.p[i].x; ++i) {
if (l >= _mmi.p[i].x && r <= _mmi.p[i].y) {
// it's contained within the entry
beg = l;
end = r;
} else if (l <= _mmi.p[i].x && r >= _mmi.p[i].x) {
// it overlaps with the lefthand side of the entry
beg = _mmi.p[i].x;
end = MIN(r, _mmi.p[i].y);
} else if (l <= _mmi.p[i].y && r >= _mmi.p[i].y) {
// it overlaps with the righthand side of the entry
beg = MAX(_mmi.p[i].x, l);
end = _mmi.p[i].y;
} else {
__builtin_unreachable();
}
// openbsd even requires that if we mapped, for instance a 5 byte
// file, that we be sure to call munmap(file, 5). let's abstract!
a = ADDR_32_TO_48(beg);
b = ADDR_32_TO_48(end) + FRAMESIZE;
c = ADDR_32_TO_48(_mmi.p[i].x) + _mmi.p[i].size;
q = (char *)a;
m = MIN(b, c) - a;
if (!IsWindows()) {
npassert(!sys_munmap(q, m));
} else {
// Handled by __untrack_memories() on Windows
}
if (IsAsan() && !OverlapsShadowSpace(p, n)) {
__munmap_shadow(q, m);
}
}
}
int __munmap_unlocked(char *p, size_t n) {
unassert(!__vforked);
if (UNLIKELY(!n)) {
STRACE("munmap n is 0");
return einval();
}
if (UNLIKELY(!IsLegalSize(n))) {
STRACE("munmap n isn't 48-bit");
return einval();
}
if (UNLIKELY(!IsLegalPointer(p))) {
STRACE("munmap p isn't 48-bit");
return einval();
}
if (UNLIKELY(!IsLegalPointer(p + (n - 1)))) {
STRACE("munmap p+(n-1) isn't 48-bit");
return einval();
}
if (UNLIKELY(!ALIGNED(p))) {
STRACE("munmap(%p) isn't 64kb aligned", p);
return einval();
}
__munmap_impl(p, n);
return __untrack_memories(p, n);
}
/**
* Releases memory pages.
*
* @param p is the beginning of the memory region to unmap
* @param n is the number of bytes to be unmapped
* @return 0 on success, or -1 w/ errno
* @raises EINVAL if `n == 0`
* @raises EINVAL if `n` isn't 48-bit
* @raises EINVAL if `p+(n-1)` isn't 48-bit
* @raises EINVAL if `p` isn't 65536-byte aligned
*/
int munmap(void *p, size_t n) {
int rc;
__mmi_lock();
rc = __munmap_unlocked(p, n);
size_t toto = __strace > 0 ? __get_memtrack_size(&_mmi) : 0;
__mmi_unlock();
STRACE("munmap(%.12p, %'zu) → %d% m (%'zu bytes total)", p, n, rc, toto);
return rc;
}

8
libc/runtime/opensymboltable.greg.c
View file

@ -31,6 +31,7 @@
#include "libc/runtime/runtime.h"
#include "libc/runtime/symbols.internal.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/o.h"
#include "libc/sysv/consts/prot.h"
@ -48,6 +49,7 @@ static struct SymbolTable *OpenSymbolTableImpl(const char *filename) {
size_t n, m, tsz, size;
const Elf64_Sym *symtab, *sym;
ptrdiff_t names_offset, name_base_offset, stp_offset;
long pagesz = getauxval(AT_PAGESZ);
map = MAP_FAILED;
if ((fd = open(filename, O_RDONLY | O_CLOEXEC)) == -1)
return 0;
@ -73,11 +75,11 @@ static struct SymbolTable *OpenSymbolTableImpl(const char *filename) {
tsz += sizeof(unsigned) * n;
name_base_offset = tsz;
tsz += m;
tsz = ROUNDUP(tsz, FRAMESIZE);
tsz = ROUNDUP(tsz, pagesz);
stp_offset = tsz;
size = tsz;
tsz += sizeof(const Elf64_Sym *) * n;
tsz = ROUNDUP(tsz, FRAMESIZE);
tsz = ROUNDUP(tsz, pagesz);
t = mmap(0, tsz, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
if (t == MAP_FAILED)
goto SystemError;
@ -128,7 +130,7 @@ static struct SymbolTable *OpenSymbolTableImpl(const char *filename) {
++j;
}
t->count = j;
munmap(stp, ROUNDUP(sizeof(const Elf64_Sym *) * n, FRAMESIZE));
munmap(stp, sizeof(const Elf64_Sym *) * n);
munmap(map, filesize);
close(fd);
return t;

142
libc/runtime/runtime.h
View file

@ -19,9 +19,9 @@ typedef unsigned long jmp_buf[18];
typedef unsigned long jmp_buf[26];
#endif
void mcount(void);
int daemon(int, int);
unsigned long getauxval(unsigned long);
void mcount(void) libcesque;
int daemon(int, int) libcesque;
unsigned long getauxval(unsigned long) libcesque;
int setjmp(jmp_buf)
libcesque returnstwice paramsnonnull();
void longjmp(jmp_buf, int) libcesque wontreturn paramsnonnull();
@ -37,28 +37,28 @@ void quick_exit(int) wontreturn;
void abort(void) wontreturn;
int atexit(void (*)(void)) paramsnonnull() libcesque;
char *getenv(const char *) paramsnonnull() __wur nosideeffect libcesque;
int putenv(char *);
int setenv(const char *, const char *, int);
int unsetenv(const char *);
int clearenv(void);
void fpreset(void);
void *mmap(void *, uint64_t, int32_t, int32_t, int32_t, int64_t);
int munmap(void *, uint64_t);
int mprotect(void *, uint64_t, int);
int msync(void *, size_t, int);
int mlock(const void *, size_t);
int munlock(const void *, size_t);
long gethostid(void);
int sethostid(long);
char *getlogin(void);
int getlogin_r(char *, size_t);
int login_tty(int);
int getpagesize(void);
int syncfs(int) dontthrow;
int vhangup(void);
int getdtablesize(void);
int sethostname(const char *, size_t);
int acct(const char *);
int putenv(char *) libcesque;
int setenv(const char *, const char *, int) libcesque;
int unsetenv(const char *) libcesque;
int clearenv(void) libcesque;
void fpreset(void) libcesque;
void *mmap(void *, uint64_t, int32_t, int32_t, int32_t, int64_t) libcesque;
int munmap(void *, uint64_t) libcesque;
int mprotect(void *, uint64_t, int) libcesque;
int msync(void *, size_t, int) libcesque;
int mlock(const void *, size_t) libcesque;
int munlock(const void *, size_t) libcesque;
long gethostid(void) libcesque;
int sethostid(long) libcesque;
char *getlogin(void) libcesque;
int getlogin_r(char *, size_t) libcesque;
int login_tty(int) libcesque;
int getpagesize(void) libcesque;
int syncfs(int) dontthrow libcesque;
int vhangup(void) libcesque;
int getdtablesize(void) libcesque;
int sethostname(const char *, size_t) libcesque;
int acct(const char *) libcesque;
#if defined(_GNU_SOURCE) || defined(_COSMO_SOURCE)
extern char **environ;
@ -80,62 +80,62 @@ extern uint64_t kStartTsc;
extern const char kNtSystemDirectory[];
extern const char kNtWindowsDirectory[];
extern size_t __virtualmax;
extern size_t __virtualsize;
extern size_t __stackmax;
extern bool32 __isworker;
/* utilities */
void _intsort(int *, size_t);
void _longsort(long *, size_t);
void _intsort(int *, size_t) libcesque;
void _longsort(long *, size_t) libcesque;
/* diagnostics */
void ShowCrashReports(void);
int ftrace_install(void);
int ftrace_enabled(int);
int strace_enabled(int);
void __print_maps(void);
void __printargs(const char *);
void ShowCrashReports(void) libcesque;
int ftrace_install(void) libcesque;
int ftrace_enabled(int) libcesque;
int strace_enabled(int) libcesque;
void __print_maps(void) libcesque;
void __printargs(const char *) libcesque;
/* builtin sh-like system/popen dsl */
int _cocmd(int, char **, char **);
int _cocmd(int, char **, char **) libcesque;
/* executable program */
char *GetProgramExecutableName(void);
char *GetInterpreterExecutableName(char *, size_t);
int __open_executable(void);
char *GetProgramExecutableName(void) libcesque;
char *GetInterpreterExecutableName(char *, size_t) libcesque;
int __open_executable(void) libcesque;
/* execution control */
int verynice(void);
void __warn_if_powersave(void);
void _Exit1(int) libcesque wontreturn;
void __paginate(int, const char *);
void __paginate_file(int, const char *);
int verynice(void) libcesque;
void __warn_if_powersave(void) libcesque;
void _Exit1(int) libcesque wontreturn libcesque;
void __paginate(int, const char *) libcesque;
void __paginate_file(int, const char *) libcesque;
/* memory management */
void _weakfree(void *);
void *_mapanon(size_t) attributeallocsize((1)) mallocesque;
void *_mapshared(size_t) attributeallocsize((1)) mallocesque;
void CheckForMemoryLeaks(void);
void CheckForFileLeaks(void);
bool32 _isheap(const void *);
void __enable_threads(void);
void __oom_hook(size_t);
void _weakfree(void *) libcesque;
void *_mapanon(size_t) attributeallocsize((1)) mallocesque libcesque;
void *_mapshared(size_t) attributeallocsize((1)) mallocesque libcesque;
void CheckForMemoryLeaks(void) libcesque;
void CheckForFileLeaks(void) libcesque;
void __enable_threads(void) libcesque;
void __oom_hook(size_t) libcesque;
/* code morphing */
void __morph_begin(void);
void __morph_end(void);
void __jit_begin(void);
void __jit_end(void);
void __clear_cache(void *, void *);
void __morph_begin(void) libcesque;
void __morph_end(void) libcesque;
void __jit_begin(void) libcesque;
void __jit_end(void) libcesque;
void __clear_cache(void *, void *) libcesque;
/* portability */
bool32 IsGenuineBlink(void);
bool32 IsCygwin(void);
const char *GetCpuidOs(void);
const char *GetCpuidEmulator(void);
void GetCpuidBrand(char[13], uint32_t);
long __get_rlimit(int);
const char *__describe_os(void);
long __get_sysctl(int, int);
int __get_arg_max(void) pureconst;
int __get_cpu_count(void) pureconst;
long __get_avphys_pages(void) pureconst;
long __get_phys_pages(void) pureconst;
long __get_minsigstksz(void) pureconst;
void __get_main_stack(void **, size_t *, int *);
long __get_safe_size(long, long);
char *__get_tmpdir(void);
bool32 IsGenuineBlink(void) libcesque;
bool32 IsCygwin(void) libcesque;
const char *GetCpuidOs(void) libcesque;
const char *GetCpuidEmulator(void) libcesque;
void GetCpuidBrand(char[13], uint32_t) libcesque;
long __get_rlimit(int) libcesque;
const char *__describe_os(void) libcesque;
long __get_sysctl(int, int) libcesque;
int __granularity(void) pureconst libcesque;
int __get_arg_max(void) pureconst libcesque;
int __get_cpu_count(void) pureconst libcesque;
long __get_avphys_pages(void) pureconst libcesque;
long __get_phys_pages(void) pureconst libcesque;
long __get_minsigstksz(void) pureconst libcesque;
long __get_safe_size(long, long) libcesque;
char *__get_tmpdir(void) libcesque;
forceinline int __trace_disabled(int x) {
return 0;
}

34
libc/runtime/stack.h
View file

@ -14,17 +14,6 @@
*/
#define GetGuardSize() 16384
/**
* Align APE main thread stack at startup.
*
* You need this in your main program module:
*
* STATIC_STACK_ALIGN(GetStackSize());
*
* If you want to use GetStackAddr() and HaveStackMemory() safely on
* your main thread in your process. It causes crt.S to waste a tiny
* amount of memory to ensure those macros go extremely fast.
*/
#define STATIC_STACK_ALIGN(BYTES) \
_STACK_SYMBOL("ape_stack_align", _STACK_STRINGIFY(BYTES) _STACK_EXTRA)
@ -62,29 +51,12 @@ COSMOPOLITAN_C_START_
extern char ape_stack_prot[] __attribute__((__weak__));
extern char ape_stack_memsz[] __attribute__((__weak__));
extern char ape_stack_align[] __attribute__((__weak__));
extern char ape_stack_round[] __attribute__((__weak__));
/**
* Returns address of bottom of current stack.
*
* This always works on threads. If you want it to work on the main
* process too, then you'll need STATIC_STACK_ALIGN(GetStackSize())
* which will burn O(256kb) of memory to ensure thread invariants.
*/
#define GetStackAddr() ((GetStackPointer() - 1) & -GetStackSize())
uintptr_t GetStackBottom(void) pureconst;
#define GetStaticStackSize() ((uintptr_t)ape_stack_memsz)
/**
* Returns true if at least `n` bytes of stack are available.
*
* This always works on threads. If you want it to work on the main
* process too, then you'll need STATIC_STACK_ALIGN(GetStackSize())
* which will burn O(256kb) of memory to ensure thread invariants,
* which make this check exceedingly fast.
*/
#define HaveStackMemory(n) \
(GetStackPointer() >= GetStackAddr() + GetGuardSize() + (n))
/**
* Extends stack memory by poking large allocations.
*
@ -137,8 +109,6 @@ int FreeCosmoStack(void *) libcesque;
: "i"(ADDEND)); \
vAddr; \
})
#else
#define GetStaticStackAddr(ADDEND) (GetStackAddr() + ADDEND)
#endif
#define GetStackPointer() \

4
libc/runtime/sysconf.c
View file

@ -25,9 +25,11 @@
#include "libc/limits.h"
#include "libc/macros.internal.h"
#include "libc/runtime/clktck.h"
#include "libc/intrin/maps.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/sysconf.h"
#include "libc/sysv/consts/_posix.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/limits.h"
#include "libc/sysv/consts/rlimit.h"
#include "libc/sysv/consts/ss.h"
@ -58,7 +60,7 @@ long sysconf(int name) {
case _SC_CLK_TCK:
return CLK_TCK;
case _SC_PAGESIZE:
return FRAMESIZE;
return __granularity();
case _SC_ARG_MAX:
return __get_arg_max();
case _SC_SIGSTKSZ:

31
libc/runtime/untrackmemoryintervals.c
View file

@ -1,31 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/dce.h"
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
int __untrack_memories(void *addr, size_t size) {
int a, b;
unassert(size > 0);
a = ROUNDDOWN((intptr_t)addr, FRAMESIZE) >> 16;
b = ROUNDDOWN((intptr_t)addr + size - 1, FRAMESIZE) >> 16;
return __untrack_memory(&_mmi, a, b,
SupportsWindows() ? __release_memory_nt : 0);
}

28
libc/runtime/winmain.greg.c
View file

@ -20,6 +20,8 @@
#include "libc/calls/internal.h"
#include "libc/calls/sig.internal.h"
#include "libc/calls/syscall_support-nt.internal.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/nomultics.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/limits.h"
@ -192,28 +194,26 @@ static abi wontreturn void WinInit(const char16_t *cmdline) {
__imp_AddVectoredExceptionHandler(true, (void *)OnWinCrash);
// allocate memory for stack and argument block
_mmi.p = _mmi.s;
_mmi.n = ARRAYLEN(_mmi.s);
uintptr_t stackaddr = GetStaticStackAddr(0);
char *stackaddr = (char *)GetStaticStackAddr(0);
size_t stacksize = GetStaticStackSize();
__imp_MapViewOfFileEx(
(_mmi.p[0].h = __imp_CreateFileMappingW(
(__maps.stack.h = __imp_CreateFileMappingW(
-1, 0, kNtPageExecuteReadwrite, stacksize >> 32, stacksize, NULL)),
kNtFileMapWrite | kNtFileMapExecute, 0, 0, stacksize, (void *)stackaddr);
kNtFileMapWrite | kNtFileMapExecute, 0, 0, stacksize, stackaddr);
int prot = (intptr_t)ape_stack_prot;
if (~prot & PROT_EXEC) {
uint32_t old;
__imp_VirtualProtect((void *)stackaddr, stacksize, kNtPageReadwrite, &old);
__imp_VirtualProtect(stackaddr, stacksize, kNtPageReadwrite, &old);
}
uint32_t oldattr;
__imp_VirtualProtect((void *)stackaddr, GetGuardSize(),
__imp_VirtualProtect(stackaddr, GetGuardSize(),
kNtPageReadwrite | kNtPageGuard, &oldattr);
_mmi.p[0].x = stackaddr >> 16;
_mmi.p[0].y = (stackaddr >> 16) + ((stacksize - 1) >> 16);
_mmi.p[0].prot = prot;
_mmi.p[0].flags = 0x00000026; // stack+anonymous
_mmi.p[0].size = stacksize;
_mmi.i = 1;
__maps.stack.addr = stackaddr;
__maps.stack.size = stacksize;
__maps.stack.prot = prot;
__maps.maps = &__maps.stack;
dll_init(&__maps.stack.elem);
dll_make_first(&__maps.used, &__maps.stack.elem);
struct WinArgs *wa =
(struct WinArgs *)(stackaddr + (stacksize - sizeof(struct WinArgs)));
@ -300,7 +300,7 @@ static abi wontreturn void WinInit(const char16_t *cmdline) {
// handover control to cosmopolitan runtime
__stack_call(count, wa->argv, wa->envp, wa->auxv, cosmo,
stackaddr + (stacksize - sizeof(struct WinArgs)));
(uintptr_t)(stackaddr + (stacksize - sizeof(struct WinArgs))));
}
abi int64_t WinMain(int64_t hInstance, int64_t hPrevInstance,

2
libc/runtime/zipos-get.c
View file

@ -62,7 +62,7 @@ static void __zipos_dismiss(uint8_t *map, const uint8_t *cdir, long pg) {
}
// unmap the executable portion beneath the local files
mo = ROUNDDOWN(lo, FRAMESIZE);
mo = ROUNDDOWN(lo, __granularity());
if (mo)
munmap(map, mo);

7
libc/runtime/zipos-mmap.c
View file

@ -20,6 +20,7 @@
#include "libc/calls/struct/iovec.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/strace.internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
@ -37,7 +38,7 @@
* Map zipos file into memory. See mmap.
*
* @param addr should be 0 or a compatible address
* @param size must be >0 and will be rounded up to FRAMESIZE
* @param size must be >0 and will be rounded up to granularity
* automatically.
* @param prot can have PROT_READ/PROT_WRITE/PROT_EXEC/PROT_NONE/etc.
* @param flags cannot have `MAP_SHARED` or `MAP_ANONYMOUS`, there is
@ -76,7 +77,7 @@ void *__zipos_mmap(void *addr, size_t size, int prot, int flags,
flags |= MAP_PRIVATE | MAP_ANONYMOUS;
const int tempProt = !IsXnu() ? prot | PROT_WRITE : PROT_WRITE;
void *outAddr = __mmap_unlocked(addr, size, tempProt, flags, -1, 0);
void *outAddr = __mmap(addr, size, tempProt, flags, -1, 0);
if (outAddr == MAP_FAILED) {
return MAP_FAILED;
}
@ -96,7 +97,7 @@ void *__zipos_mmap(void *addr, size_t size, int prot, int flags,
} while (0);
const int e = errno;
__munmap_unlocked(outAddr, size);
munmap(outAddr, size);
errno = e;
strace_enabled(+1);
return MAP_FAILED;

3
libc/runtime/zipos-stat-impl.c
View file

@ -20,6 +20,7 @@
#include "libc/calls/struct/stat.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/safemacros.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/zipos.internal.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/s.h"
@ -30,7 +31,7 @@ int __zipos_stat_impl(struct Zipos *zipos, size_t cf, struct stat *st) {
bzero(st, sizeof(*st));
st->st_nlink = 1;
st->st_dev = zipos->dev;
st->st_blksize = FRAMESIZE;
st->st_blksize = __granularity();
if (cf == ZIPOS_SYNTHETIC_DIRECTORY) {
st->st_mode = S_IFDIR | (0555 & ~atomic_load_explicit(
&__umask, memory_order_acquire));

5
libc/stdio/printargs.c
View file

@ -464,11 +464,6 @@ textstartup void __printargs(const char *prologue) {
PRINT(" ☼ %s = %#s", "GetProgramExecutableName", GetProgramExecutableName());
PRINT(" ☼ %s = %#s", "GetInterpreterExecutableName",
GetInterpreterExecutableName(u.path, sizeof(u.path)));
PRINT(" ☼ %s = %p", "GetStackSize()", GetStackSize());
PRINT(" ☼ %s = %p", "GetGuardSize()", GetGuardSize());
PRINT(" ☼ %s = %p", "GetStackAddr()", GetStackAddr());
PRINT(" ☼ %s = %p", "GetStaticStackSize()", GetStaticStackSize());
PRINT(" ☼ %s = %p", "GetStaticStackAddr(0)", GetStaticStackAddr(0));
PRINT(" ☼ %s = %p", "__builtin_frame_address(0)", __builtin_frame_address(0));
PRINT("");

5
libc/sysv/consts.sh
View file

@ -219,10 +219,9 @@ syscon mmap MAP_FILE 0 0 0 0 0 0 0 0 # consensus
syscon mmap MAP_SHARED 1 1 1 1 1 1 1 1 # forced consensus & faked nt
syscon mmap MAP_SHARED_VALIDATE 3 3 1 1 1 1 1 1 # weird linux thing
syscon mmap MAP_PRIVATE 2 2 2 2 2 2 2 2 # forced consensus & faked nt
syscon mmap MAP_STACK 6 6 6 6 6 6 6 6 # our definition
syscon mmap MAP_TYPE 15 15 15 15 15 15 15 15 # mask for type of mapping
syscon mmap MAP_FIXED 0x00000010 0x00000010 0x00000010 0x00000010 0x00000010 0x00000010 0x00000010 0x00000010 # unix consensus; openbsd appears to forbid; faked nt
syscon mmap MAP_FIXED_NOREPLACE 0x08000000 0x08000000 0x00004010 0x00004010 0x08000000 0x08000000 0x08000000 0x08000000 # handled and defined by cosmo runtime; 0x100000 on linux 4.7+; MAP_FIXED|MAP_EXCL on FreeBSD
syscon mmap MAP_FIXED_NOREPLACE 0x08000000 0x08000000 0x00004000 0x00004000 0x08000000 0x08000000 0x08000000 0x08000000 # handled and defined by cosmo runtime; 0x100000 on linux 4.7+; MAP_FIXED|MAP_EXCL on FreeBSD
syscon mmap MAP_ANONYMOUS 0x00000020 0x00000020 0x00001000 0x00001000 0x00001000 0x00001000 0x00001000 0x00000020 # bsd consensus; faked nt
syscon mmap MAP_GROWSDOWN 0x00000100 0x00000100 0 0 0 0 0 0 # use MAP_STACK; abstracted by MAP_STACK; may be passed to __sys_mmap() for low-level Linux fiddling
syscon mmap MAP_LOCKED 0x00002000 0x00002000 0 0 0 0 0 0
@ -293,8 +292,6 @@ syscon mprot PROT_NONE 0 0 0 0 0 0 0 0 # mmap, mprotect, unix
syscon mprot PROT_READ 1 1 1 1 1 1 1 1 # mmap, mprotect, unix consensus
syscon mprot PROT_WRITE 2 2 2 2 2 2 2 2 # mmap, mprotect, unix consensus
syscon mprot PROT_EXEC 4 4 4 4 4 4 4 4 # mmap, mprotect, unix consensus
syscon mprot PROT_GROWSDOWN 0x01000000 0x01000000 0 0 0 0 0 0 # intended for mprotect; see MAP_GROWSDOWN for mmap() (todo: what was 0x01000000 on nt)
syscon mprot PROT_GROWSUP 0x02000000 0x02000000 0 0 0 0 0 0 # intended for mprotect; see MAP_GROWSDOWN for mmap()
# mremap() flags
# the revolutionary praxis of realloc()

2
libc/sysv/consts/MAP_FIXED_NOREPLACE.S
View file

@ -1,2 +1,2 @@
#include "libc/sysv/consts/syscon.internal.h"
.syscon mmap,MAP_FIXED_NOREPLACE,0x08000000,0x08000000,0x00004010,0x00004010,0x08000000,0x08000000,0x08000000,0x08000000
.syscon mmap,MAP_FIXED_NOREPLACE,0x08000000,0x08000000,0x00004000,0x00004000,0x08000000,0x08000000,0x08000000,0x08000000

2
libc/sysv/consts/MAP_STACK.S
View file

@ -1,2 +0,0 @@
#include "libc/sysv/consts/syscon.internal.h"
.syscon mmap,MAP_STACK,6,6,6,6,6,6,6,6

2
libc/sysv/consts/PROT_GROWSDOWN.S
View file

@ -1,2 +0,0 @@
#include "libc/sysv/consts/syscon.internal.h"
.syscon mprot,PROT_GROWSDOWN,0x01000000,0x01000000,0,0,0,0,0,0

2
libc/sysv/consts/PROT_GROWSUP.S
View file

@ -1,2 +0,0 @@
#include "libc/sysv/consts/syscon.internal.h"
.syscon mprot,PROT_GROWSUP,0x02000000,0x02000000,0,0,0,0,0,0

1
libc/sysv/consts/map.h
View file

@ -33,7 +33,6 @@ COSMOPOLITAN_C_END_
#define MAP_FILE 0
#define MAP_SHARED 1
#define MAP_PRIVATE 2
#define MAP_STACK 6
#define MAP_TYPE 15
#define MAP_FIXED 16

2
libc/sysv/consts/prot.h
View file

@ -7,8 +7,6 @@ extern const int PROT_NONE;
extern const int PROT_READ;
extern const int PROT_WRITE;
extern const int PROT_EXEC;
extern const int PROT_GROWSDOWN;
extern const int PROT_GROWSUP;
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

31
libc/sysv/systemfive.S
View file

@ -16,7 +16,6 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/strace.internal.h"
#include "libc/dce.h"
#include "libc/macros.internal.h"
#include "libc/thread/pt.internal.h"
@ -378,30 +377,8 @@ _init_systemfive_magnums:
jnz 3b
xchg %rbx,%rax
stosq
#if SYSDEBUG
inc %r8d
#endif
jmp 2b
5: nop
#if SYSDEBUG
push %rdi
push %rsi
push %r8
call __describe_os
mov %rax,%rdx
pop %r8
.weak __stracef
ezlea __stracef,ax
test %rax,%rax
jz 5f
ezlea .Llog,di
mov %r8d,%esi
push %rax
call *%rax
pop %rax
5: pop %rsi
pop %rdi
#endif /* DEBUGSYS */
pop %rdi
pop %rsi
pop %rbx
@ -447,11 +424,3 @@ _init_systemfive_sigsys:
_init_systemfive_done:
nop
.init.end 300,_init_systemfive,globl,hidden
#if SYSDEBUG
.rodata.str1.1
.Llog: .ascii STRACE_PROLOGUE
.ascii "bell system five system call support"
.asciz " %'u magnums loaded on %s\n"
.previous
#endif /* DEBUGSYS */

24
libc/intrin/mmi_lock.c → libc/thread/getstackbottom.c
View file

@ -1,7 +1,7 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
@ -16,17 +16,17 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/memtrack.internal.h"
#include "libc/thread/thread.h"
// nsync depends on this non-nsync lock
extern pthread_mutex_t __mmi_lock_obj;
void __mmi_lock(void) {
pthread_mutex_lock(&__mmi_lock_obj);
}
void __mmi_unlock(void) {
pthread_mutex_unlock(&__mmi_lock_obj);
uintptr_t GetStackBottom(void) {
char *bottom;
void *stackaddr;
pthread_attr_t tattr;
size_t stacksize, guardsize;
pthread_getattr_np(pthread_self(), &tattr);
pthread_attr_getstack(&tattr, &stackaddr, &stacksize);
pthread_attr_getguardsize(&tattr, &guardsize);
bottom = stackaddr;
bottom += guardsize;
return (uintptr_t)bottom;
}

2
libc/thread/openbsd.internal.h
View file

@ -4,7 +4,7 @@ COSMOPOLITAN_C_START_
struct __tfork {
void *tf_tcb;
int32_t *tf_tid;
_Atomic(int) *tf_tid;
void *tf_stack;
};

20
libc/thread/pthread_atfork.c
View file

@ -16,7 +16,6 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/atomic.h"
#include "libc/calls/state.internal.h"
#include "libc/cosmo.h"
@ -25,8 +24,8 @@
#include "libc/intrin/atomic.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/leaky.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/macros.internal.h"
#include "libc/mem/mem.h"
#include "libc/proc/proc.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
@ -42,18 +41,19 @@ struct AtFork {
static struct AtForks {
pthread_spinlock_t lock;
struct AtFork *list;
struct AtFork pool[8];
struct AtFork pool[64];
atomic_int allocated;
} _atforks;
static void _pthread_onfork(int i) {
static void _pthread_onfork(int i, const char *op) {
struct AtFork *a;
unassert(0 <= i && i <= 2);
if (!i)
pthread_spin_lock(&_atforks.lock);
for (a = _atforks.list; a; a = a->p[!i]) {
if (a->f[i])
if (a->f[i]) {
STRACE("pthread_atfork(%s, %t)", op, a->f[i]);
a->f[i]();
}
_atforks.list = a;
}
if (i)
@ -61,15 +61,15 @@ static void _pthread_onfork(int i) {
}
void _pthread_onfork_prepare(void) {
_pthread_onfork(0);
_pthread_onfork(0, "prepare");
}
void _pthread_onfork_parent(void) {
_pthread_onfork(1);
_pthread_onfork(1, "parent");
}
void _pthread_onfork_child(void) {
_pthread_onfork(2);
_pthread_onfork(2, "child");
}
static struct AtFork *_pthread_atfork_alloc(void) {
@ -78,7 +78,7 @@ static struct AtFork *_pthread_atfork_alloc(void) {
(i = atomic_fetch_add(&_atforks.allocated, 1)) < n) {
return _atforks.pool + i;
} else {
return malloc(sizeof(struct AtFork));
return 0;
}
}

76
libc/thread/pthread_create.c
View file

@ -28,6 +28,7 @@
#include "libc/intrin/bsr.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/log/internal.h"
@ -64,10 +65,6 @@ __static_yoink("_pthread_onfork_prepare");
__static_yoink("_pthread_onfork_parent");
__static_yoink("_pthread_onfork_child");
/* #ifndef MODE_DBG */
/* __static_yoink("threaded_dlmalloc"); */
/* #endif */
#define MAP_ANON_OPENBSD 0x1000
#define MAP_STACK_OPENBSD 0x4000
@ -76,7 +73,8 @@ void _pthread_free(struct PosixThread *pt, bool isfork) {
if (pt->pt_flags & PT_STATIC)
return;
if (pt->pt_flags & PT_OWNSTACK) {
unassert(!munmap(pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize));
unassert(!munmap(pt->pt_attr.__stackaddr,
pt->pt_attr.__stacksize + (uintptr_t)ape_stack_align));
}
if (!isfork) {
uint64_t syshand =
@ -149,7 +147,7 @@ static int FixupCustomStackOnOpenbsd(pthread_attr_t *attr) {
uintptr_t x, y;
int e, rc, pagesz;
pagesz = getauxval(AT_PAGESZ);
n = attr->__stacksize;
n = attr->__stacksize + (uintptr_t)ape_stack_align;
x = (uintptr_t)attr->__stackaddr;
y = ROUNDUP(x, pagesz);
n -= y - x;
@ -159,7 +157,7 @@ static int FixupCustomStackOnOpenbsd(pthread_attr_t *attr) {
MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD,
-1, 0, 0) == (void *)y) {
attr->__stackaddr = (void *)y;
attr->__stacksize = n;
attr->__stacksize = n - (uintptr_t)ape_stack_align;
return 0;
} else {
rc = errno;
@ -214,48 +212,37 @@ static errno_t pthread_create_impl(pthread_t *thread,
}
} else {
// cosmo is managing the stack
// 1. in mono repo optimize for tiniest stack possible
// 2. in public world optimize to *work* regardless of memory
int granularity = FRAMESIZE;
int pagesize = getauxval(AT_PAGESZ);
pt->pt_attr.__guardsize = ROUNDUP(pt->pt_attr.__guardsize, pagesize);
pt->pt_attr.__stacksize = ROUNDUP(pt->pt_attr.__stacksize, granularity);
pt->pt_attr.__stacksize = pt->pt_attr.__stacksize;
if (pt->pt_attr.__guardsize + pagesize > pt->pt_attr.__stacksize) {
_pthread_free(pt, false);
return EINVAL;
}
if (pt->pt_attr.__guardsize == pagesize &&
!(IsAarch64() && IsLinux() && IsQemuUser())) {
// MAP_GROWSDOWN doesn't work very well on qemu-aarch64
pt->pt_attr.__stackaddr =
mmap(0, pt->pt_attr.__stacksize, PROT_READ | PROT_WRITE,
MAP_STACK | MAP_ANONYMOUS, -1, 0);
} else {
pt->pt_attr.__stackaddr =
mmap(0, pt->pt_attr.__stacksize, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (pt->pt_attr.__stackaddr != MAP_FAILED) {
if (IsOpenbsd() &&
__sys_mmap(
pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD,
-1, 0, 0) != pt->pt_attr.__stackaddr) {
notpossible;
}
if (pt->pt_attr.__guardsize) {
if (!IsWindows()) {
if (mprotect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize,
PROT_NONE)) {
notpossible;
}
} else {
uint32_t oldattr;
if (!VirtualProtect(pt->pt_attr.__stackaddr,
pt->pt_attr.__guardsize,
kNtPageReadwrite | kNtPageGuard, &oldattr)) {
notpossible;
}
pt->pt_attr.__stackaddr =
mmap(0, pt->pt_attr.__stacksize + (uintptr_t)ape_stack_align,
PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (pt->pt_attr.__stackaddr != MAP_FAILED) {
if (IsOpenbsd() &&
__sys_mmap(
pt->pt_attr.__stackaddr,
pt->pt_attr.__stacksize + (uintptr_t)ape_stack_align,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD,
-1, 0, 0) != pt->pt_attr.__stackaddr) {
notpossible;
}
if (pt->pt_attr.__guardsize) {
if (!IsWindows()) {
if (mprotect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize,
PROT_NONE)) {
notpossible;
}
} else {
uint32_t oldattr;
if (!VirtualProtect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize,
kNtPageReadwrite | kNtPageGuard, &oldattr)) {
notpossible;
}
}
}
@ -305,8 +292,7 @@ static errno_t pthread_create_impl(pthread_t *thread,
_pthread_unlock();
// launch PosixThread(pt) in new thread
if ((rc = clone(PosixThread, pt->pt_attr.__stackaddr,
pt->pt_attr.__stacksize - (IsOpenbsd() ? 16 : 0),
if ((rc = clone(PosixThread, pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize,
CLONE_VM | CLONE_THREAD | CLONE_FS | CLONE_FILES |
CLONE_SIGHAND | CLONE_SYSVSEM | CLONE_SETTLS |
CLONE_PARENT_SETTID | CLONE_CHILD_SETTID |

7
libc/thread/pthread_getattr_np.c
View file

@ -20,10 +20,10 @@
#include "libc/calls/struct/rlimit.h"
#include "libc/dce.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/maps.h"
#include "libc/limits.h"
#include "libc/macros.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/rlim.h"
@ -73,8 +73,9 @@ errno_t pthread_getattr_np(pthread_t thread, pthread_attr_t *attr) {
__builtin_unreachable();
}
if (!attr->__stacksize && (pt->pt_flags & PT_STATIC)) {
__get_main_stack(&attr->__stackaddr, &attr->__stacksize,
&attr->__guardsize);
attr->__stackaddr = __maps.stack.addr;
attr->__stacksize = __maps.stack.size;
attr->__guardsize = 0;
}
return 0;
}

1
libc/thread/tls.h
View file

@ -4,6 +4,7 @@
#define TLS_ALIGNMENT 64
#define TIB_FLAG_VFORKED 1
#define TIB_FLAG_MAPLOCK 2
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_

3
net/https/finishcertificate.c
View file

@ -18,6 +18,7 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/log/log.h"
#include "libc/mem/gc.h"
#include "libc/runtime/runtime.h"
#include "libc/x/xasprintf.h"
#include "net/https/https.h"
@ -26,7 +27,7 @@ struct Cert FinishCertificate(struct Cert *ca, mbedtls_x509write_cert *wcert,
int i, n, rc;
unsigned char *p;
mbedtls_x509_crt *cert;
p = malloc((n = FRAMESIZE));
p = malloc((n = __granularity()));
i = mbedtls_x509write_crt_der(wcert, p, n, GenerateHardRandom, 0);
if (i < 0)
FATALF("write key (grep -0x%04x)", -i);

2
net/turfwar/turfwar.c
View file

@ -122,7 +122,7 @@
#define SOCK_MAX 100 // max length of socket queue
#define MSG_BUF 512 // small response lookaside
#define INBUF_SIZE FRAMESIZE
#define INBUF_SIZE 65536
#define OUTBUF_SIZE 8192
#define TB_BYTES (1u << TB_CIDR)

Some files were not shown because too many files have changed in this diff Show more