mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-07-24 11:30:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Rewrite memory manager

Browse source 
Actually Portable Executable now supports Android. Cosmo's old mmap code
required a 47 bit address space. The new implementation is very agnostic
and supports both smaller address spaces (e.g. embedded) and even modern
56-bit PML5T paging for x86 which finally came true on Zen4 Threadripper

Cosmopolitan no longer requires UNIX systems to observe the Windows 64kb
granularity; i.e. sysconf(_SC_PAGE_SIZE) will now report the host native
page size. This fixes a longstanding POSIX conformance issue, concerning
file mappings that overlap the end of file. Other aspects of conformance
have been improved too, such as the subtleties of address assignment and
and the various subtleties surrounding MAP_FIXED and MAP_FIXED_NOREPLACE

On Windows, mappings larger than 100 megabytes won't be broken down into
thousands of independent 64kb mappings. Support for MAP_STACK is removed
by this change; please use NewCosmoStack() instead.

Stack overflow avoidance is now being implemented using the POSIX thread
APIs. Please use GetStackBottom() and GetStackAddr(), instead of the old
error-prone GetStackAddr() and HaveStackMemory() APIs which are removed.
This commit is contained in:
Justine Tunney 2024-06-20 20:46:42 -07:00
parent 7f6d0b8709
commit 6ffed14b9c
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
150 changed files with 1893 additions and 5634 deletions
Download patch file Download diff file Expand all files Collapse all files

3
libc/intrin/BUILD.mk
View file

@ -66,8 +66,7 @@ o/$(MODE)/libc/intrin/asan.o: private \
CFLAGS += \
-O2 \
-finline \
-finline-functions \
-fpatchable-function-entry=0,0
-finline-functions
o//libc/intrin/memmove.o: private \
CFLAGS += \

261
libc/intrin/asan.c
View file

@ -16,6 +16,7 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "ape/sections.internal.h"
#include "libc/calls/state.internal.h"
#include "libc/calls/struct/rlimit.h"
#include "libc/calls/struct/rlimit.internal.h"
@ -30,6 +31,7 @@
#include "libc/intrin/kprintf.h"
#include "libc/intrin/leaky.internal.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/log/libfatal.internal.h"
@ -94,13 +96,6 @@ __static_yoink("_init_asan");
#define ASAN_LOG(...) (void)0 // kprintf(__VA_ARGS__)
#define REQUIRE(FUNC) \
do { \
if (!_weaken(FUNC)) { \
__asan_require_failed(#FUNC); \
} \
} while (0)
struct AsanSourceLocation {
char *filename;
int line;
@ -161,7 +156,7 @@ static char *__asan_stpcpy(char *d, const char *s) {
}
}
void __asan_memset(void *p, char c, size_t n) {
dontinstrument void __asan_memset(void *p, char c, size_t n) {
char *b;
size_t i;
uint64_t x;
@ -313,20 +308,16 @@ static wontreturn void __asan_require_failed(const char *func) {
__asan_unreachable();
}
void __asan_poison(void *p, long n, signed char t) {
signed char k, *s;
s = (signed char *)(((intptr_t)p >> 3) + 0x7fff8000);
if ((k = (intptr_t)p & 7)) {
if ((!*s && n >= 8 - k) || *s > k)
*s = k;
n -= MIN(8 - k, n);
s += 1;
}
__asan_memset(s, t, n >> 3);
if ((k = n & 7)) {
s += n >> 3;
if (*s < 0 || (*s > 0 && *s <= k))
*s = t;
static bool __asan_overlaps_shadow_space(const void *p, size_t n) {
intptr_t BegA, EndA, BegB, EndB;
if (n) {
BegA = (intptr_t)p;
EndA = BegA + n;
BegB = 0x7fff0000;
EndB = 0x100080000000;
return MAX(BegA, BegB) < MIN(EndA, EndB);
} else {
return 0;
}
}
@ -353,17 +344,92 @@ void __asan_unpoison(void *p, long n) {
}
}
bool __asan_is_mapped(int x) {
// xxx: we can't lock because no reentrant locks yet
int i;
bool res;
__mmi_lock();
i = __find_memory(&_mmi, x);
res = i < _mmi.i && x >= _mmi.p[i].x;
__mmi_unlock();
dontinstrument void __asan_poison(void *p, long n, signed char t) {
signed char k, *s;
s = (signed char *)(((intptr_t)p >> 3) + 0x7fff8000);
if ((k = (intptr_t)p & 7)) {
if ((!*s && n >= 8 - k) || *s > k)
*s = k;
n -= MIN(8 - k, n);
s += 1;
}
__asan_memset(s, t, n >> 3);
if ((k = n & 7)) {
s += n >> 3;
if (*s < 0 || (*s > 0 && *s <= k))
*s = t;
}
}
static void __asan_poison_safe(char *p, long n, signed char t) {
int granularity = __granularity();
int chunk = granularity << 3;
while (n > 0) {
char *block = (char *)((uintptr_t)p & -chunk);
signed char *shadow = (signed char *)(((uintptr_t)block >> 3) + 0x7fff8000);
if (__asan_is_mapped(shadow)) {
char *start = MAX(p, block);
size_t bytes = MIN(n, chunk);
__asan_poison(start, bytes, t);
}
p += chunk;
n -= chunk;
}
}
privileged static bool __asan_is_mapped_unlocked(const char *addr) {
struct Dll *e, *e2;
for (e = dll_first(__maps.used); e; e = e2) {
e2 = dll_next(__maps.used, e);
struct Map *map = MAP_CONTAINER(e);
if (map->addr <= addr && addr < map->addr + map->size) {
dll_remove(&__maps.used, e);
dll_make_first(&__maps.used, e);
return true;
}
}
return false;
}
privileged bool __asan_is_mapped(const void *addr) {
bool32 res;
__maps_lock();
res = __asan_is_mapped_unlocked(addr);
__maps_unlock();
return res;
}
static void __asan_ensure_shadow_is_mapped(void *addr, size_t size) {
int m = 0x7fff8000;
int g = __granularity();
size_t n = (size + 7) & -8;
char *p = (char *)((((uintptr_t)addr >> 3) + m) & -g);
char *pe = (char *)(((((uintptr_t)(addr + n) >> 3) + m) + g - 1) & -g);
for (char *pm = p; p < pe; p = pm) {
pm += g;
if (!__asan_is_mapped(p)) {
while (pm < pe && !__asan_is_mapped(pm))
pm += g;
if (__mmap(p, pm - p, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) != MAP_FAILED)
__asan_memset(p, kAsanUnmapped, pm - p);
}
}
}
void __asan_shadow(void *addr, size_t size) {
if (!__asan_overlaps_shadow_space(addr, size)) {
__asan_ensure_shadow_is_mapped(addr, size);
__asan_unpoison(addr, size);
}
}
void __asan_unshadow(void *addr, size_t size) {
if (!__asan_overlaps_shadow_space(addr, size)) {
__asan_poison_safe(addr, size, kAsanUnmapped);
}
}
static bool __asan_is_image(const unsigned char *p) {
return __executable_start <= p && p < _end;
}
@ -387,13 +453,15 @@ static struct AsanFault __asan_fault(const signed char *s, signed char dflt) {
static struct AsanFault __asan_checka(const signed char *s, long ndiv8) {
uint64_t w;
int g = __granularity();
const signed char *o = s;
const signed char *e = s + ndiv8;
for (; ((intptr_t)s & 7) && s < e; ++s) {
if (*s)
return __asan_fault(s - 1, kAsanHeapOverrun);
}
for (; s + 8 <= e; s += 8) {
if (UNLIKELY(!((intptr_t)s & (FRAMESIZE - 1))) && kisdangerous(s)) {
if (UNLIKELY(!((intptr_t)s & (g - 1))) && kisdangerous(s)) {
return (struct AsanFault){kAsanUnmapped, s};
}
if ((w = READ64LE(s))) {
@ -403,7 +471,7 @@ static struct AsanFault __asan_checka(const signed char *s, long ndiv8) {
}
for (; s < e; ++s) {
if (*s)
return __asan_fault(s - 1, kAsanHeapOverrun);
return __asan_fault(s > o ? s - 1 : s, kAsanHeapOverrun);
}
return (struct AsanFault){0};
}
@ -429,7 +497,7 @@ struct AsanFault __asan_check(const void *p, long n) {
if (n > 0) {
k = (intptr_t)p & 7;
s = SHADOW(p);
if (OverlapsShadowSpace(p, n)) {
if (__asan_overlaps_shadow_space(p, n)) {
return (struct AsanFault){kAsanProtected, s};
} else if (kisdangerous(s)) {
return (struct AsanFault){kAsanUnmapped, s};
@ -475,8 +543,9 @@ struct AsanFault __asan_check(const void *p, long n) {
struct AsanFault __asan_check_str(const char *p) {
uint64_t w;
signed char c, k, *s;
int g = __granularity();
s = SHADOW(p);
if (OverlapsShadowSpace(p, 1)) {
if (__asan_overlaps_shadow_space(p, 1)) {
return (struct AsanFault){kAsanProtected, s};
}
if (kisdangerous(s)) {
@ -494,7 +563,7 @@ struct AsanFault __asan_check_str(const char *p) {
++s;
}
for (;; ++s, p += 8) {
if (UNLIKELY(!((intptr_t)s & (FRAMESIZE - 1))) && kisdangerous(s)) {
if (UNLIKELY(!((intptr_t)s & (g - 1))) && kisdangerous(s)) {
return (struct AsanFault){kAsanUnmapped, s};
}
if ((c = *s) < 0) {
@ -749,9 +818,9 @@ void __asan_report_memory_origin(const unsigned char *addr, int size,
}
if (__executable_start <= addr && addr < _end) {
__asan_report_memory_origin_image((intptr_t)addr, size);
} else if (IsAutoFrame((intptr_t)addr >> 16)) {
if (_weaken(__asan_report_memory_origin_heap))
_weaken(__asan_report_memory_origin_heap)(addr, size);
/* } else if (IsAutoFrame((intptr_t)addr >> 16)) { */
/* if (_weaken(__asan_report_memory_origin_heap)) */
/* _weaken(__asan_report_memory_origin_heap)(addr, size); */
}
}
@ -766,9 +835,8 @@ static __wur __asan_die_f *__asan_report(const void *addr, int size,
int i;
wint_t c;
signed char t;
uint64_t x, y, z;
uint64_t z;
char *base, *q, *p = buf;
struct MemoryIntervals *m;
ftrace_enabled(-1);
kprintf("\n\e[J\e[1;31masan error\e[0m: %s %d-byte %s at %p shadow %p\n",
__asan_describe_access_poison(kind), size, message, addr,
@ -832,20 +900,20 @@ static __wur __asan_die_f *__asan_report(const void *addr, int size,
p = __asan_format_section(p, __executable_start, _etext, ".text", addr);
p = __asan_format_section(p, _etext, _edata, ".data", addr);
p = __asan_format_section(p, _end, _edata, ".bss", addr);
__mmi_lock();
for (m = &_mmi, i = 0; i < m->i; ++i) {
x = m->p[i].x;
y = m->p[i].y;
p = __asan_format_interval(p, x << 16, (y << 16) + (FRAMESIZE - 1));
z = (intptr_t)addr >> 16;
if (x <= z && z <= y)
__maps_lock();
struct Dll *e, *e2;
for (e = dll_first(__maps.used); e; e = e2) {
e2 = dll_next(__maps.used, e);
struct Map *map = MAP_CONTAINER(e);
p = __asan_format_interval(p, (uintptr_t)map->addr,
(uintptr_t)map->addr + map->size);
if (!__asan_overlaps_shadow_space(map->addr, map->size))
p = __asan_stpcpy(p, " ←address");
z = (((intptr_t)addr >> 3) + 0x7fff8000) >> 16;
if (x <= z && z <= y)
else
p = __asan_stpcpy(p, " ←shadow");
*p++ = '\n';
}
__mmi_unlock();
__maps_unlock();
*p = 0;
kprintf("%s", buf);
__asan_report_memory_origin(addr, size, kind);
@ -1033,95 +1101,16 @@ void __asan_after_dynamic_init(void) {
ASAN_LOG("__asan_after_dynamic_init()\n");
}
void __asan_map_shadow(uintptr_t p, size_t n) {
// assume _mmi.lock is held
void *addr;
int i, a, b;
size_t size;
int prot, flag;
struct DirectMap sm;
struct MemoryIntervals *m;
if (OverlapsShadowSpace((void *)p, n)) {
kprintf("error: %p size %'zu overlaps shadow space: %s\n", p, n,
DescribeBacktrace(__builtin_frame_address(0)));
_Exit(1);
}
m = &_mmi;
a = (0x7fff8000 + (p >> 3)) >> 16;
b = (0x7fff8000 + (p >> 3) + (n >> 3) + 0xffff) >> 16;
for (; a <= b; a += i) {
i = 1;
if (__asan_is_mapped(a)) {
continue;
}
for (; a + i <= b; ++i) {
if (__asan_is_mapped(a + i)) {
break;
}
}
size = (size_t)i << 16;
addr = (void *)ADDR_32_TO_48(a);
prot = PROT_READ | PROT_WRITE;
flag = MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS;
sm = sys_mmap(addr, size, prot, flag, -1, 0);
if (sm.addr == MAP_FAILED ||
__track_memory(m, a, a + i - 1, sm.maphandle, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, false, false, 0,
size) == -1) {
kprintf("error: could not map asan shadow memory\n");
__asan_die()();
__asan_unreachable();
}
__repstosb(addr, kAsanUnmapped, size);
}
__asan_unpoison((char *)p, n);
}
static textstartup void __asan_shadow_mapping(struct MemoryIntervals *m,
size_t i) {
uintptr_t x, y;
if (i < m->i) {
x = m->p[i].x;
y = m->p[i].y;
__asan_shadow_mapping(m, i + 1);
__asan_map_shadow(x << 16, (y - x + 1) << 16);
}
}
static textstartup void __asan_shadow_existing_mappings(void) {
__asan_shadow_mapping(&_mmi, 0);
if (!IsWindows()) {
int guard;
void *addr;
size_t size;
__get_main_stack(&addr, &size, &guard);
__asan_map_shadow((uintptr_t)addr, size);
__asan_poison(addr, guard, kAsanStackOverflow);
}
}
static size_t __asan_strlen(const char *s) {
size_t i = 0;
while (s[i])
++i;
return i;
}
forceinline ssize_t __write_str(const char *s) {
return sys_write(2, s, __asan_strlen(s));
}
void __asan_init(int argc, char **argv, char **envp, unsigned long *auxv) {
void __asan_init(void) {
static bool once;
if (!_cmpxchg(&once, false, true))
if (once)
return;
__asan_shadow_existing_mappings();
__asan_map_shadow((uintptr_t)__executable_start, _end - __executable_start);
__asan_map_shadow(0, 4096);
once = true;
__asan_shadow(0, 4096);
__asan_shadow(__maps.stack.addr, __maps.stack.size);
__asan_shadow(__executable_start, _end - __executable_start);
__asan_poison((void *)__veil("r", 0L), getauxval(AT_PAGESZ), kAsanNullPage);
if (!IsWindows()) {
sys_mprotect((void *)0x7fff8000, 0x10000, PROT_READ);
}
STRACE(" _ ____ _ _ _ ");
STRACE(" / \\ / ___| / \\ | \\ | |");
STRACE(" / _ \\ \\___ \\ / _ \\ | \\| |");

5
libc/intrin/asan.internal.h
View file

@ -46,7 +46,8 @@ void __asan_unpoison(void *, long);
void __asan_poison(void *, long, signed char);
void __asan_verify(const void *, size_t);
void __asan_verify_str(const char *);
void __asan_map_shadow(uintptr_t, size_t);
void __asan_shadow(void *, size_t);
void __asan_unshadow(void *, size_t);
bool __asan_is_valid(const void *, long) nosideeffect;
bool __asan_is_valid_str(const char *) nosideeffect;
bool __asan_is_valid_strlist(char *const *) nosideeffect;
@ -54,9 +55,9 @@ bool __asan_is_valid_iov(const struct iovec *, int) nosideeffect;
struct AsanFault __asan_check(const void *, long) nosideeffect;
struct AsanFault __asan_check_str(const char *) nosideeffect;
bool __asan_is_mapped(int);
int __asan_is_leaky(void *);
int __asan_print_trace(void *);
bool __asan_is_mapped(const void *);
__asan_die_f *__asan_die(void) __wur;
void __asan_memset(void *, char, size_t);
size_t __asan_get_heap_size(const void *);

9
libc/intrin/asaninit.S
View file

@ -19,14 +19,13 @@
#include "libc/macros.internal.h"
.init.start 303,_init_asan
#ifdef __x86_64__
#ifdef __SANITIZE_ADDRESS__
push %rdi
push %rsi
mov %r12,%rdi
mov %r13,%rsi
mov %r14,%rdx
mov %r15,%rcx
.weak __asan_init
call __asan_init
pop %rsi
pop %rdi
#endif /* __SANITIZE_ADDRESS__ */
#endif /* __x86_64__ */
.init.end 303,_init_asan

2
libc/intrin/describeflags.internal.h
View file

@ -21,7 +21,6 @@ const char *DescribeDnotifyFlags(char[80], int) libcesque;
const char *DescribeErrno(char[30], int) libcesque;
const char *DescribeFcntlCmd(char[20], int) libcesque;
const char *DescribeFlockType(char[12], int) libcesque;
const char *DescribeFrame(char[32], int) libcesque;
const char *DescribeFutexOp(char[64], int) libcesque;
const char *DescribeHow(char[12], int) libcesque;
const char *DescribeInOutInt64(char[23], ssize_t, int64_t *) libcesque;
@ -80,7 +79,6 @@ const char *DescribeWhichPrio(char[12], int) libcesque;
#define DescribeErrno(x) DescribeErrno(alloca(30), x)
#define DescribeFcntlCmd(x) DescribeFcntlCmd(alloca(20), x)
#define DescribeFlockType(x) DescribeFlockType(alloca(12), x)
#define DescribeFrame(x) DescribeFrame(alloca(32), x)
#define DescribeFutexOp(x) DescribeFutexOp(alloca(64), x)
#define DescribeHow(x) DescribeHow(alloca(12), x)
#define DescribeInOutInt64(rc, x) DescribeInOutInt64(alloca(23), rc, x)

72
libc/intrin/describeframe.c
View file

@ -1,72 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/dce.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/weaken.h"
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/winargs.internal.h"
#define UNSHADOW(x) ((int64_t)(MAX(0, (x) - 0x7fff8000)) << 3)
#define FRAME(x) ((int)((x) >> 16))
static const char *GetFrameName(int x) {
if (!x) {
return "null";
} else if (IsShadowFrame(x)) {
return "shadow";
} else if (IsAutoFrame(x)) {
return "automap";
} else if (IsFixedFrame(x)) {
return "fixed";
} else if (IsStaticStackFrame(x)) {
return "stack";
} else if (IsGfdsFrame(x)) {
return "g_fds";
} else if (IsNsyncFrame(x)) {
return "nsync";
} else if (IsZiposFrame(x)) {
return "zipos";
} else if (IsMemtrackFrame(x)) {
return "memtrack";
} else if (IsOldStackFrame(x)) {
return "system stack";
} else if (((GetStaticStackAddr(0) + GetStackSize()) >> 16) <= x &&
x <= ((GetStaticStackAddr(0) + GetStackSize() - 1) >> 16)) {
return "static stack";
} else if ((int)((intptr_t)__executable_start >> 16) <= x &&
x <= (int)(((intptr_t)_end - 1) >> 16)) {
return "image";
} else {
return "unknown";
}
}
const char *(DescribeFrame)(char buf[32], int x) {
if (IsShadowFrame(x)) {
ksnprintf(buf, 32, "%s %s %.8x", GetFrameName(x),
GetFrameName(FRAME(UNSHADOW(ADDR_32_TO_48(x)))),
FRAME(UNSHADOW(ADDR_32_TO_48(x))));
return buf;
} else {
return GetFrameName(x);
}
}

1
libc/intrin/describemapflags.c
View file

@ -24,7 +24,6 @@
const char *(DescribeMapFlags)(char buf[64], int x) {
const struct DescribeFlags kMapFlags[] = {
{MAP_STACK, "STACK"}, // order matters
{MAP_PRIVATE, "PRIVATE"}, //
{MAP_ANONYMOUS, "ANONYMOUS"}, //
{MAP_SHARED, "SHARED"}, //

2
libc/intrin/describemapping.c
View file

@ -29,8 +29,6 @@ static char DescribeMapType(int flags) {
return 'p';
case MAP_SHARED:
return 's';
case MAP_STACK:
return 'S';
default:
return '?';
}

2
libc/intrin/directmap-nt.c
View file

@ -86,7 +86,7 @@ TryAgain:
if ((dm.addr = MapViewOfFileEx(dm.maphandle, fl.flags2, off >> 32, off,
size, addr))) {
uint32_t oldprot;
if (VirtualProtect(addr, size, __prot2nt(prot, iscow), &oldprot)) {
if (VirtualProtect(dm.addr, size, __prot2nt(prot, iscow), &oldprot)) {
return dm;
}
UnmapViewOfFile(dm.addr);

17
libc/intrin/directmap.c
View file

@ -16,6 +16,7 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
@ -24,7 +25,9 @@
#include "libc/intrin/strace.internal.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/syslib.internal.h"
#include "libc/sysv/errfuns.h"
/**
* Obtains memory mapping directly from system.
@ -39,7 +42,10 @@
struct DirectMap sys_mmap(void *addr, size_t size, int prot, int flags, int fd,
int64_t off) {
struct DirectMap d;
if (IsXnuSilicon()) {
if ((__virtualsize += size) >= __virtualmax) {
d.maphandle = kNtInvalidHandleValue;
d.addr = (void *)enomem();
} else if (IsXnuSilicon()) {
long p = _sysret(__syslib->__mmap(addr, size, prot, flags, fd, off));
d.maphandle = kNtInvalidHandleValue;
d.addr = (void *)p;
@ -51,9 +57,10 @@ struct DirectMap sys_mmap(void *addr, size_t size, int prot, int flags, int fd,
} else {
d = sys_mmap_nt(addr, size, prot, flags, fd, off);
}
KERNTRACE("sys_mmap(%.12p /* %s */, %'zu, %s, %s, %d, %'ld) → {%.12p, %p}% m",
addr, DescribeFrame((intptr_t)addr >> 16), size,
DescribeProtFlags(prot), DescribeMapFlags(flags), fd, off, d.addr,
d.maphandle);
if (d.addr == MAP_FAILED)
__virtualsize -= size;
KERNTRACE("sys_mmap(%.12p, %'zu, %s, %s, %d, %'ld) → {%.12p, %p}% m", addr,
size, DescribeProtFlags(prot), DescribeMapFlags(flags), fd, off,
d.addr, d.maphandle);
return d;
}

8
libc/intrin/dll.c
View file

@ -28,7 +28,7 @@
*
* It's required that `elem` and `succ` aren't part of the same list.
*/
void dll_splice_after(struct Dll *elem, struct Dll *succ) {
privileged void dll_splice_after(struct Dll *elem, struct Dll *succ) {
struct Dll *tmp1, *tmp2;
tmp1 = elem->next;
tmp2 = succ->prev;
@ -43,7 +43,7 @@ void dll_splice_after(struct Dll *elem, struct Dll *succ) {
*
* @param list is a doubly-linked list, where `!*list` means empty
*/
void dll_remove(struct Dll **list, struct Dll *elem) {
privileged void dll_remove(struct Dll **list, struct Dll *elem) {
if (*list == elem) {
if ((*list)->prev == *list) {
*list = 0;
@ -66,7 +66,7 @@ void dll_remove(struct Dll **list, struct Dll *elem) {
* @param list is a doubly-linked list, where `!*list` means empty
* @param elem must not be a member of `list`, or null for no-op
*/
void dll_make_first(struct Dll **list, struct Dll *elem) {
privileged void dll_make_first(struct Dll **list, struct Dll *elem) {
if (elem) {
if (!*list) {
*list = elem->prev;
@ -85,7 +85,7 @@ void dll_make_first(struct Dll **list, struct Dll *elem) {
* @param list is a doubly-linked list, where `!*list` means empty
* @param elem must not be a member of `list`, or null for no-op
*/
void dll_make_last(struct Dll **list, struct Dll *elem) {
privileged void dll_make_last(struct Dll **list, struct Dll *elem) {
if (elem) {
dll_make_first(list, elem->next);
*list = elem;

14
libc/intrin/dll.h
View file

@ -14,38 +14,38 @@ struct Dll {
struct Dll *prev;
};
static inline void dll_init(struct Dll *e) {
forceinline void dll_init(struct Dll *e) {
e->next = e;
e->prev = e;
}
static inline int dll_is_alone(struct Dll *e) {
forceinline int dll_is_alone(struct Dll *e) {
return e->next == e && e->prev == e;
}
static inline int dll_is_empty(struct Dll *list) {
forceinline int dll_is_empty(struct Dll *list) {
return !list;
}
static inline struct Dll *dll_last(struct Dll *list) {
forceinline struct Dll *dll_last(struct Dll *list) {
return list;
}
static inline struct Dll *dll_first(struct Dll *list) {
forceinline struct Dll *dll_first(struct Dll *list) {
struct Dll *first = 0;
if (list)
first = list->next;
return first;
}
static inline struct Dll *dll_next(struct Dll *list, struct Dll *e) {
forceinline struct Dll *dll_next(struct Dll *list, struct Dll *e) {
struct Dll *next = 0;
if (e != list)
next = e->next;
return next;
}
static inline struct Dll *dll_prev(struct Dll *list, struct Dll *e) {
forceinline struct Dll *dll_prev(struct Dll *list, struct Dll *e) {
struct Dll *prev = 0;
if (e != list->next)
prev = e->prev;

68
libc/intrin/extend.c
View file

@ -16,51 +16,14 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/calls/struct/sigset.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/asancodes.h"
#include "libc/intrin/describebacktrace.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/log/libfatal.internal.h"
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
#define G FRAMESIZE
static void *_mapframe_impl(void *p, int f) {
int rc, prot, flags;
struct DirectMap dm;
prot = PROT_READ | PROT_WRITE;
flags = f | MAP_ANONYMOUS | MAP_FIXED;
if ((dm = sys_mmap(p, G, prot, flags, -1, 0)).addr == p) {
rc = __track_memory(&_mmi, (uintptr_t)p >> 16, (uintptr_t)p >> 16,
dm.maphandle, prot, flags, false, false, 0, G);
if (!rc) {
return p;
} else {
return 0;
}
} else {
return 0;
}
}
static void *_mapframe(void *p, int f) {
void *res;
// mmap isn't required to be @asyncsignalsafe but this is
BLOCK_SIGNALS;
__mmi_lock();
res = _mapframe_impl(p, f);
__mmi_unlock();
ALLOW_SIGNALS;
return res;
}
#define G __granularity()
/**
* Extends static allocation.
@ -84,32 +47,23 @@ static void *_mapframe(void *p, int f) {
*/
void *_extend(void *p, size_t n, void *e, int f, intptr_t h) {
char *q;
#ifndef NDEBUG
#ifdef __SANITIZE_ADDRESS__
if ((uintptr_t)SHADOW(p) & (G - 1))
notpossible;
__builtin_trap();
if ((uintptr_t)p + (G << kAsanScale) > h)
notpossible;
__builtin_trap();
#endif
// TODO(jart): Make this spin less in non-ASAN mode.
for (q = e; q < ((char *)p + n); q += 8) {
if (!((uintptr_t)q & (G - 1))) {
#ifndef NDEBUG
if (q + G > (char *)h)
notpossible;
#endif
if (!_mapframe(q, f))
__builtin_trap();
if (mmap(q, G, PROT_READ | PROT_WRITE, f | MAP_ANONYMOUS | MAP_FIXED, -1,
0) == MAP_FAILED)
return 0;
if (IsAsan()) {
if (!((uintptr_t)SHADOW(q) & (G - 1))) {
if (!_mapframe(SHADOW(q), f))
return 0;
__asan_poison(q, G << kAsanScale, kAsanProtected);
}
}
}
if (IsAsan()) {
*SHADOW(q) = 0;
}
}
return q;
}

26
libc/intrin/getmainstack.c
View file

@ -16,14 +16,14 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/struct/rlimit.h"
#include "libc/calls/struct/rlimit.internal.h"
#include "libc/dce.h"
#include "libc/intrin/getauxval.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/maps.h"
#include "libc/macros.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/rlim.h"
#include "libc/sysv/consts/rlimit.h"
@ -101,30 +101,22 @@ static size_t __get_stack_size(int pagesz, uintptr_t start, uintptr_t top) {
/**
* Returns approximate boundaries of main thread stack.
*
* This function works on every OS except Windows.
*/
void __get_main_stack(void **out_addr, size_t *out_size, int *out_guardsize) {
if (IsWindows()) {
*out_addr = (void *)GetStaticStackAddr(0);
*out_size = GetStaticStackSize();
*out_guardsize = getauxval(AT_PAGESZ);
return;
}
struct AddrSize __get_main_stack(void) {
int pagesz = getauxval(AT_PAGESZ);
uintptr_t start = (uintptr_t)__argv;
uintptr_t top = __get_main_top(pagesz);
uintptr_t bot = top - __get_stack_size(pagesz, start, top);
uintptr_t vdso = getauxval(AT_SYSINFO_EHDR);
if (vdso) {
struct AuxiliaryValue avdso = __getauxval(AT_SYSINFO_EHDR);
if (avdso.isfound) {
uintptr_t vdso = avdso.value;
if (vdso > start && vdso < top) {
top = vdso;
} else if (vdso < start && vdso >= bot) {
bot += vdso + pagesz * 2;
}
}
unassert(bot < top);
unassert(bot < start);
unassert(top > start);
*out_addr = (void *)bot;
*out_size = top - bot;
*out_guardsize = pagesz;
return (struct AddrSize){(char *)bot, top - bot};
}

5
libc/intrin/getsafesize.greg.c
View file

@ -37,11 +37,6 @@ privileged long __get_safe_size(long want, long extraspace) {
return want;
struct PosixThread *pt;
struct CosmoTib *tib = __get_tls_privileged();
if (!IsAutoFrame((uintptr_t)tib >> 16) &&
!(__executable_start <= (const unsigned char *)tib &&
(const unsigned char *)tib < _end)) {
return want;
}
long bottom, sp = GetStackPointer();
if ((char *)sp >= tib->tib_sigstack_addr &&
(char *)sp <= tib->tib_sigstack_addr + tib->tib_sigstack_size) {

22
libc/intrin/mmi.c → libc/intrin/granularity.c
View file

@ -1,7 +1,7 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
@ -16,13 +16,15 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/thread/thread.h"
#include "libc/dce.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/auxv.h"
#ifdef __x86_64__
__static_yoink("_init__mmi");
#endif
struct MemoryIntervals _mmi;
pthread_mutex_t __mmi_lock_obj = {._type = PTHREAD_MUTEX_RECURSIVE};
int __granularity(void) {
if (IsWindows())
return 65536;
static int res;
if (!res)
res = getauxval(AT_PAGESZ);
return res;
}

16
libc/intrin/mmi_lock.c → libc/intrin/klog.c
View file

@ -1,7 +1,7 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
@ -16,17 +16,5 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/memtrack.internal.h"
#include "libc/thread/thread.h"
// nsync depends on this non-nsync lock
extern pthread_mutex_t __mmi_lock_obj;
void __mmi_lock(void) {
pthread_mutex_lock(&__mmi_lock_obj);
}
void __mmi_unlock(void) {
pthread_mutex_unlock(&__mmi_lock_obj);
}
long __klog_handle;

86
libc/intrin/kprintf.greg.c
View file

@ -25,8 +25,11 @@
#include "libc/fmt/magnumstrs.internal.h"
#include "libc/intrin/asmflag.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/getenv.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/nomultics.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/log/internal.h"
@ -45,9 +48,11 @@
#include "libc/runtime/internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/runtime/symbols.internal.h"
#include "libc/serialize.h"
#include "libc/stdckdint.h"
#include "libc/stdio/sysparam.h"
#include "libc/str/str.h"
#include "libc/str/tab.internal.h"
#include "libc/str/utf16.h"
@ -117,7 +122,7 @@ __msabi extern typeof(SetLastError) *const __imp_SetLastError;
__msabi extern typeof(WriteFile) *const __imp_WriteFile;
// clang-format on
long __klog_handle;
extern long __klog_handle;
extern struct SymbolTable *__symtab;
__funline char *kadvance(char *p, char *e, long n) {
@ -141,20 +146,6 @@ __funline char *kemitquote(char *p, char *e, signed char t, unsigned c) {
return p;
}
__funline bool kiskernelpointer(const void *p) {
return 0x7f0000000000 <= (intptr_t)p && (intptr_t)p < 0x800000000000;
}
__funline bool kistextpointer(const void *p) {
return __executable_start <= (const unsigned char *)p &&
(const unsigned char *)p < _etext;
}
__funline bool kisimagepointer(const void *p) {
return __executable_start <= (const unsigned char *)p &&
(const unsigned char *)p < _end;
}
__funline bool kischarmisaligned(const char *p, signed char t) {
if (t == -1)
return (intptr_t)p & 1;
@ -163,55 +154,26 @@ __funline bool kischarmisaligned(const char *p, signed char t) {
return false;
}
__funline bool kismemtrackhosed(void) {
return !((_weaken(_mmi)->i <= _weaken(_mmi)->n) &&
(_weaken(_mmi)->p == _weaken(_mmi)->s ||
_weaken(_mmi)->p == (struct MemoryInterval *)kMemtrackStart));
}
privileged static bool kismapped(int x) {
// xxx: we can't lock because no reentrant locks yet
size_t m, r, l = 0;
if (!_weaken(_mmi))
return true;
if (kismemtrackhosed())
return false;
r = _weaken(_mmi)->i;
while (l < r) {
m = (l & r) + ((l ^ r) >> 1); // floor((a+b)/2)
if (_weaken(_mmi)->p[m].y < x) {
l = m + 1;
} else {
r = m;
privileged static bool32 kisdangerous_unlocked(const char *addr) {
struct Dll *e, *e2;
for (e = dll_first(__maps.used); e; e = e2) {
e2 = dll_next(__maps.used, e);
struct Map *map = MAP_CONTAINER(e);
if (map->addr <= addr && addr < map->addr + map->size) {
dll_remove(&__maps.used, e);
dll_make_first(&__maps.used, e);
return !(map->prot & PROT_READ);
}
}
if (l < _weaken(_mmi)->i && x >= _weaken(_mmi)->p[l].x) {
return !!(_weaken(_mmi)->p[l].prot & PROT_READ);
} else {
return false;
}
return true;
}
privileged bool32 kisdangerous(const void *p) {
int frame;
if (kisimagepointer(p))
return false;
if (kiskernelpointer(p))
return false;
if (IsOldStack(p))
return false;
if (IsLegalPointer(p)) {
frame = (uintptr_t)p >> 16;
if (IsStackFrame(frame))
return false;
if (kismapped(frame))
return false;
}
if (GetStackAddr() + GetGuardSize() <= (uintptr_t)p &&
(uintptr_t)p < GetStackAddr() + GetStackSize()) {
return false;
}
return true;
privileged bool32 kisdangerous(const void *addr) {
bool32 res;
__maps_lock();
res = kisdangerous_unlocked(addr);
__maps_unlock();
return res;
}
privileged static void klogclose(long fd) {
@ -463,10 +425,6 @@ privileged static size_t kformat(char *b, size_t n, const char *fmt,
unsigned long long x;
unsigned i, j, m, rem, sign, hash, cols, prec;
char c, *p, *e, pdot, zero, flip, dang, base, quot, uppr, ansi, z[128];
if (kistextpointer(b) || kisdangerous(b))
n = 0;
if (!kistextpointer(fmt))
fmt = "!!WONTFMT";
p = b;
f = fmt;
e = p + n; // assume if n was negative e < p will be the case

95
libc/intrin/maps.c Normal file
View file

@ -0,0 +1,95 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/maps.h"
#include "ape/sections.internal.h"
#include "libc/dce.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/strace.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/prot.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
#ifdef __x86_64__
__static_yoink("_init_maps");
#endif
struct Maps __maps;
void __maps_init(void) {
// record _start() stack mapping
if (!IsWindows()) {
struct AddrSize stack;
stack = __get_main_stack();
dll_init(&__maps.stack.elem);
__maps.stack.addr = stack.addr;
__maps.stack.size = stack.size;
__maps.stack.prot = (uintptr_t)ape_stack_prot;
__maps_insert(&__maps.stack);
}
// record .text and .data mappings
static struct Map text, data;
dll_init(&text.elem);
text.addr = (char *)__executable_start;
text.size = _etext - __executable_start;
text.prot = PROT_READ | PROT_EXEC;
int pagesz = getauxval(AT_PAGESZ);
uintptr_t ds = ((uintptr_t)_etext + pagesz - 1) & -pagesz;
if (ds < (uintptr_t)_end) {
dll_init(&data.elem);
data.addr = (char *)ds;
data.size = (uintptr_t)_end - ds;
data.prot = PROT_READ | PROT_WRITE;
__maps_insert(&data);
}
__maps_insert(&text);
}
privileged void __maps_lock(void) {
struct CosmoTib *tib;
if (!__threaded)
return;
if (!__tls_enabled)
return;
tib = __get_tls_privileged();
if (tib->tib_flags & TIB_FLAG_MAPLOCK)
return;
while (atomic_exchange_explicit(&__maps.lock, 1, memory_order_acquire)) {
#if defined(__GNUC__) && defined(__aarch64__)
__asm__ volatile("yield");
#elif defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__))
__asm__ volatile("pause");
#endif
}
tib->tib_flags |= TIB_FLAG_MAPLOCK;
}
privileged void __maps_unlock(void) {
struct CosmoTib *tib;
atomic_store_explicit(&__maps.lock, 0, memory_order_release);
if (__tls_enabled) {
tib = __get_tls_privileged();
tib->tib_flags &= ~TIB_FLAG_MAPLOCK;
}
}

49
libc/intrin/maps.h Normal file
View file

@ -0,0 +1,49 @@
#ifndef COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_
#define COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_
#include "libc/intrin/atomic.h"
#include "libc/intrin/dll.h"
#include "libc/thread/tls2.internal.h"
COSMOPOLITAN_C_START_
#define MAP_CONTAINER(e) DLL_CONTAINER(struct Map, elem, e)
struct Map {
_Atomic(struct Map *) next; /* for __maps.maps */
char *addr; /* granule aligned */
size_t size; /* must be nonzero */
struct Dll elem; /* for __maps.free */
int64_t off; /* -1 if anonymous */
int prot; /* memory protects */
int flags; /* memory map flag */
bool iscow; /* windows nt only */
bool readonlyfile; /* windows nt only */
intptr_t h; /* windows nt only */
};
struct Maps {
atomic_int lock;
_Atomic(struct Map *) maps;
struct Dll *free;
struct Map stack;
struct Dll *used;
};
struct AddrSize {
char *addr;
size_t size;
};
extern struct Maps __maps;
int maps_check(void);
void __maps_init(void);
void __maps_lock(void);
void __maps_unlock(void);
struct Map *__maps_alloc(void);
void __maps_free(struct Map *);
void __maps_insert(struct Map *);
void *__mmap(char *, size_t, int, int, int, int64_t);
struct AddrSize __get_main_stack(void);
COSMOPOLITAN_C_END_
#endif /* COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_ */

14
libc/intrin/mmi.init.S → libc/intrin/maps_init.S
View file

@ -1,7 +1,7 @@
/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│
│ vi: set noet ft=asm ts=8 sw=8 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
@ -16,10 +16,12 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/thread/thread.h"
#include "libc/macros.internal.h"
.init.start 200,_init__mmi
movb $16,_mmi+8
movl $_mmi+24,_mmi+16
.init.end 200,_init__mmi
.init.start 301,_init_maps
push %rdi
push %rsi
call __maps_init
pop %rsi
pop %rdi
.init.end 301,_init_maps

243
libc/intrin/memtrack.greg.c
View file

@ -1,243 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/log/libfatal.internal.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
#include "libc/mem/mem.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
static void *__shove_memory(struct MemoryInterval *d,
const struct MemoryInterval *s, int n) {
int i;
unassert(n >= 0);
if (d > s) {
for (i = n; i--;) {
d[i] = s[i];
}
} else {
for (i = 0; i < n; ++i) {
d[i] = s[i];
}
}
return d;
}
static void __remove_memory(struct MemoryIntervals *mm, int i, int n) {
unassert(i >= 0);
unassert(i + n <= mm->i);
__shove_memory(mm->p + i, mm->p + i + n, mm->i - (i + n));
mm->i -= n;
}
static bool __extend_memory(struct MemoryIntervals *mm) {
int prot, flags;
char *base, *shad;
size_t gran, size;
struct DirectMap dm;
gran = kMemtrackGran;
base = (char *)kMemtrackStart;
prot = PROT_READ | PROT_WRITE;
flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED;
if (mm->p == mm->s) {
// TODO(jart): How can we detect ASAN mode under GREG?
if (1 || IsAsan()) {
shad = (char *)(((intptr_t)base >> 3) + 0x7fff8000);
dm = sys_mmap(shad, gran >> 3, prot, flags, -1, 0);
if (!dm.addr)
return false;
}
dm = sys_mmap(base, gran, prot, flags, -1, 0);
if (!dm.addr)
return false;
__shove_memory(dm.addr, mm->p, mm->i);
mm->p = dm.addr;
mm->n = gran / sizeof(*mm->p);
} else {
size = ROUNDUP(mm->n * sizeof(*mm->p), gran);
base += size;
if (IsAsan()) {
shad = (char *)(((intptr_t)base >> 3) + 0x7fff8000);
dm = sys_mmap(shad, gran >> 3, prot, flags, -1, 0);
if (!dm.addr)
return false;
}
dm = sys_mmap(base, gran, prot, flags, -1, 0);
if (!dm.addr)
return false;
mm->n = (size + gran) / sizeof(*mm->p);
}
return true;
}
static int __mint_memory(struct MemoryIntervals *mm, int i) {
unassert(i >= 0);
unassert(i <= mm->i);
unassert(mm->n >= 0);
if (mm->i == mm->n && !__extend_memory(mm))
return enomem();
__shove_memory(mm->p + i + 1, mm->p + i, mm->i++ - i);
return 0;
}
static int __punch_memory(struct MemoryIntervals *mm, int x, int y, int i) {
if (__mint_memory(mm, i) == -1)
return -1;
mm->p[i + 0].size -= (size_t)(mm->p[i + 0].y - (x - 1)) * FRAMESIZE;
mm->p[i + 0].y = x - 1;
mm->p[i + 1].size -= (size_t)((y + 1) - mm->p[i + 1].x) * FRAMESIZE;
mm->p[i + 1].x = y + 1;
return 0;
}
int __untrack_memory(struct MemoryIntervals *mm, int x, int y,
void wf(struct MemoryIntervals *, int, int)) {
unsigned l, r;
unassert(y >= x);
if (!mm->i)
return 0;
// binary search for the lefthand side
l = __find_memory(mm, x);
if (l == mm->i)
return 0;
if (y < mm->p[l].x)
return 0;
// binary search for the righthand side
r = __find_memory(mm, y);
if (r == mm->i || (r > l && y < mm->p[r].x))
--r;
unassert(r >= l);
unassert(x <= mm->p[r].y);
// remove the middle of an existing map
//
// ----|mmmmmmmmmmmmmmmm|--------- before
// xxxxx
// ----|mmmm|-----|mmmmm|--------- after
//
// this isn't possible on windows because we track each
// 64kb segment on that platform using a separate entry
if (l == r && x > mm->p[l].x && y < mm->p[l].y) {
return __punch_memory(mm, x, y, l);
}
// trim the right side of the lefthand map
//
// ----|mmmmmmm|-------------- before
// xxxxx
// ----|mmmm|----------------- after
//
if (x > mm->p[l].x && x <= mm->p[l].y) {
unassert(y >= mm->p[l].y);
if (IsWindows())
return einval();
mm->p[l].size -= (size_t)(mm->p[l].y - (x - 1)) * FRAMESIZE;
mm->p[l].y = x - 1;
unassert(mm->p[l].x <= mm->p[l].y);
++l;
}
// trim the left side of the righthand map
//
// ------------|mmmmm|-------- before
// xxxxx
// ---------------|mm|-------- after
//
if (y >= mm->p[r].x && y < mm->p[r].y) {
unassert(x <= mm->p[r].x);
if (IsWindows())
return einval();
mm->p[r].size -= (size_t)((y + 1) - mm->p[r].x) * FRAMESIZE;
mm->p[r].x = y + 1;
unassert(mm->p[r].x <= mm->p[r].y);
--r;
}
if (l <= r) {
if (IsWindows() && wf) {
wf(mm, l, r);
}
__remove_memory(mm, l, r - l + 1);
}
return 0;
}
int __track_memory(struct MemoryIntervals *mm, int x, int y, long h, int prot,
int flags, bool readonlyfile, bool iscow, long offset,
long size) {
unsigned i;
unassert(y >= x);
i = __find_memory(mm, x);
// try to extend the righthand side of the lefthand entry
// we can't do that if we're tracking independent handles
// we can't do that if it's a file map with a small size!
if (i && x == mm->p[i - 1].y + 1 && h == mm->p[i - 1].h &&
prot == mm->p[i - 1].prot && flags == mm->p[i - 1].flags &&
mm->p[i - 1].size ==
(size_t)(mm->p[i - 1].y - mm->p[i - 1].x) * FRAMESIZE + FRAMESIZE) {
mm->p[i - 1].size += (size_t)(y - mm->p[i - 1].y) * FRAMESIZE;
mm->p[i - 1].y = y;
// if we filled the hole then merge the two mappings
if (i < mm->i && y + 1 == mm->p[i].x && h == mm->p[i].h &&
prot == mm->p[i].prot && flags == mm->p[i].flags) {
mm->p[i - 1].y = mm->p[i].y;
mm->p[i - 1].size += mm->p[i].size;
__remove_memory(mm, i, 1);
}
}
// try to extend the lefthand side of the righthand entry
// we can't do that if we're creating a smaller file map!
else if (i < mm->i && y + 1 == mm->p[i].x && h == mm->p[i].h &&
prot == mm->p[i].prot && flags == mm->p[i].flags &&
size == (size_t)(y - x) * FRAMESIZE + FRAMESIZE) {
mm->p[i].size += (size_t)(mm->p[i].x - x) * FRAMESIZE;
mm->p[i].x = x;
}
// otherwise, create a new entry and memmove the items
else {
if (__mint_memory(mm, i) == -1)
return -1;
mm->p[i].x = x;
mm->p[i].y = y;
mm->p[i].h = h;
mm->p[i].prot = prot;
mm->p[i].flags = flags;
mm->p[i].offset = offset;
mm->p[i].size = size;
mm->p[i].iscow = iscow;
mm->p[i].readonlyfile = readonlyfile;
}
return 0;
}

431
libc/intrin/mmap.c Normal file
View file

@ -0,0 +1,431 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "ape/sections.internal.h"
#include "libc/atomic.h"
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
#include "libc/calls/struct/sigset.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/nt/memory.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/runtime/zipos.internal.h"
#include "libc/stdio/sysparam.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/o.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
#include "libc/thread/thread.h"
#define WINBASE 0x100080040000 // TODO: Can we support Windows Vista again?
#define WINMAXX 0x200080000000
#define MAP_FIXED_NOREPLACE_linux 0x100000
#define PGUP(x) (((x) + granularity - 1) & -granularity)
static atomic_ulong rollo;
void __maps_free(struct Map *map) {
dll_make_last(&__maps.free, &map->elem);
}
void __maps_insert(struct Map *map) {
struct Map *last = __maps.maps;
if (last && //
map->addr == last->addr + last->size && //
map->flags == last->flags && //
map->prot == last->prot && //
map->off == last->off && //
map->h == last->h && //
map->off == -1) {
last->size += map->size;
dll_remove(&__maps.used, &last->elem);
dll_make_first(&__maps.used, &last->elem);
__maps_free(map);
} else {
dll_make_first(&__maps.used, &map->elem);
map->next = __maps.maps;
__maps.maps = map;
}
}
struct Map *__maps_alloc(void) {
struct Dll *e;
struct Map *map;
if ((e = dll_first(__maps.free))) {
dll_remove(&__maps.free, e);
map = MAP_CONTAINER(e);
map->next = 0;
return map;
}
int granularity = __granularity();
struct DirectMap sys = sys_mmap(0, granularity, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (sys.addr == MAP_FAILED)
return 0;
if (IsWindows())
CloseHandle(sys.maphandle);
map = sys.addr;
map->addr = MAP_FAILED;
dll_init(&map->elem);
for (int i = 1; i < granularity / sizeof(struct Map); ++i) {
dll_init(&map[i].elem);
__maps_free(map + i);
}
map->next = 0;
return map;
}
static bool __overlaps_existing_map(const char *addr, size_t size) {
for (struct Map *map = __maps.maps; map; map = map->next) {
if (MAX(addr, map->addr) < MIN(addr + size, map->addr + map->size))
return true;
}
return false;
}
static int __munmap_chunk(void *addr, size_t size) {
int rc = sys_munmap(addr, size);
if (IsAsan() && !rc)
__asan_unshadow(addr, size);
return rc;
}
static int __munmap(char *addr, size_t size, bool untrack_only) {
// validate arguments
int granularity = __granularity();
if (((uintptr_t)addr & (granularity - 1)) || //
!size || (uintptr_t)addr + size < size)
return einval();
// untrack and delete mapping
int rc = 0;
__maps_lock();
// we can't call strace, kprintf, or nothing
StartOver:;
struct Map *map = __maps.maps;
_Atomic(struct Map *) *prev = &__maps.maps;
while (map) {
char *map_addr = map->addr;
size_t map_size = map->size;
struct Map *next = map->next;
if (MAX(addr, map_addr) <
MIN(addr + PGUP(size), map_addr + PGUP(map_size))) {
if (addr <= map_addr && addr + PGUP(size) >= map_addr + PGUP(map_size)) {
// remove mapping completely
dll_remove(&__maps.used, &map->elem);
*prev = next;
map->size = 0;
map->addr = MAP_FAILED;
if (untrack_only) {
__maps_free(map);
} else {
if (!IsWindows()) {
if (__munmap_chunk(map_addr, map_size))
rc = -1;
} else {
if (!UnmapViewOfFile(map_addr))
rc = -1;
if (!CloseHandle(map->h))
rc = -1;
}
__maps_free(map);
goto StartOver;
}
map = next;
continue;
} else if (IsWindows()) {
// you can't carve up memory maps on windows. our mmap() makes
// this not a problem (for non-enormous memory maps) by making
// independent mappings for each 64 kb granule, under the hood
rc = einval();
} else if (addr <= map_addr) {
// shave off lefthand side of mapping
size_t left = addr + size - map_addr;
size_t right = map_addr + map_size - (addr + size);
map->addr += left;
map->size = right;
if (map->off != -1)
map->off += left;
if (!untrack_only) {
if (__munmap_chunk(map_addr, left) == -1)
rc = -1;
goto StartOver;
}
} else if (addr + PGUP(size) >= map_addr + PGUP(map_size)) {
// shave off righthand side of mapping
size_t left = addr - map_addr;
size_t right = map_addr + map_size - addr;
map->size = left;
if (!untrack_only) {
if (__munmap_chunk(addr, right) == -1)
rc = -1;
goto StartOver;
}
} else {
// punch hole in mapping
size_t left = addr - map_addr;
size_t middle = PGUP(size);
size_t right = map_size - middle - left;
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
leftmap->next = map;
leftmap->addr = map_addr;
leftmap->size = left;
leftmap->off = map->off;
leftmap->prot = map->prot;
leftmap->flags = map->flags;
map->addr += left + middle;
map->size = right;
if (map->off != -1)
map->off += left + middle;
dll_make_first(&__maps.used, &leftmap->elem);
*prev = leftmap;
if (!untrack_only) {
if (__munmap_chunk(addr, size) == -1)
rc = -1;
goto StartOver;
}
} else {
rc = -1;
}
}
}
prev = &map->next;
map = next;
}
__maps_unlock();
return rc;
}
static void *__mmap_chunk(void *addr, size_t size, int prot, int flags, int fd,
int64_t off, int granularity) {
// polyfill nuances of fixed mappings
int sysflags = flags;
bool noreplace = false;
bool should_untrack = false;
if (flags & MAP_FIXED_NOREPLACE) {
if (flags & MAP_FIXED)
return (void *)einval();
sysflags &= ~MAP_FIXED_NOREPLACE;
if (IsLinux()) {
noreplace = true;
sysflags |= MAP_FIXED_NOREPLACE_linux;
} else if (IsFreebsd() || IsNetbsd()) {
sysflags |= MAP_FIXED;
if (__overlaps_existing_map(addr, size))
return (void *)eexist();
} else {
noreplace = true;
}
} else if (flags & MAP_FIXED) {
should_untrack = true;
}
// allocate Map object
struct Map *map;
__maps_lock();
map = __maps_alloc();
__maps_unlock();
if (!map)
return MAP_FAILED;
// obtain mapping from operating system
int olderr = errno;
struct DirectMap res;
TryAgain:
res = sys_mmap(addr, size, prot, sysflags, fd, off);
if (res.addr == MAP_FAILED) {
if (IsWindows() && errno == EADDRNOTAVAIL) {
if (noreplace) {
errno = EEXIST;
} else if (should_untrack) {
sys_munmap(res.addr, size);
errno = olderr;
goto TryAgain;
} else {
addr += granularity;
errno = olderr;
goto TryAgain;
}
}
__maps_lock();
__maps_free(map);
__maps_unlock();
return MAP_FAILED;
}
// polyfill map fixed noreplace
// we assume non-linux gives us addr if it's free
// that's what linux (e.g. rhel7) did before noreplace
if (noreplace && res.addr != addr) {
sys_munmap(res.addr, size);
__maps_lock();
__maps_free(map);
__maps_unlock();
return (void *)eexist();
}
// untrack mapping we blew away
if (should_untrack)
__munmap(addr, size, true);
// track Map object
map->addr = res.addr;
map->size = size;
map->off = off;
map->prot = prot;
map->flags = flags;
map->h = res.maphandle;
if (IsWindows()) {
map->iscow = (flags & MAP_TYPE) != MAP_SHARED && fd != -1;
map->readonlyfile = (flags & MAP_TYPE) == MAP_SHARED && fd != -1 &&
(g_fds.p[fd].flags & O_ACCMODE) == O_RDONLY;
}
__maps_lock();
__maps_insert(map);
__maps_unlock();
return res.addr;
}
static void *__mmap_impl(char *addr, size_t size, int prot, int flags, int fd,
int64_t off, int granularity) {
// validate file map args
if (fd != -1) {
if (off & (granularity - 1))
return (void *)einval();
if (IsWindows()) {
if (!__isfdkind(fd, kFdFile))
return (void *)eacces();
if ((g_fds.p[fd].flags & O_ACCMODE) == O_WRONLY)
return (void *)eacces();
}
}
// mmap works fine on unix
if (!IsWindows())
return __mmap_chunk(addr, size, prot, flags, fd, off, granularity);
// if the concept of granularity wasn't exciting enough
if (!addr && !(flags & (MAP_FIXED | MAP_FIXED_NOREPLACE)))
addr = (char *)(WINBASE + atomic_fetch_add(&rollo, PGUP(size)) % WINMAXX);
// windows forbids unmapping a subset of a map once it's made
if (size <= granularity || size > 100 * 1024 * 1024)
return __mmap_chunk(addr, size, prot, flags, fd, off, granularity);
// so we create an separate map for each granule in the mapping
if (!(flags & MAP_FIXED)) {
while (__overlaps_existing_map(addr, size)) {
if (flags & MAP_FIXED_NOREPLACE)
return (void *)eexist();
addr += granularity;
}
}
char *res = addr;
while (size) {
char *got;
size_t amt = MIN(size, granularity);
got = __mmap_chunk(addr, amt, prot, flags, fd, off, granularity);
if (got != addr) {
if (got != MAP_FAILED)
__munmap(got, amt, false);
if (addr > res)
__munmap(res, addr - res, false);
errno = EAGAIN;
return MAP_FAILED;
}
size -= amt;
addr += amt;
off += amt;
}
return res;
}
void *__mmap(char *addr, size_t size, int prot, int flags, int fd,
int64_t off) {
char *res;
int granularity = __granularity();
// validate arguments
if (((uintptr_t)addr & (granularity - 1)) || //
!size || (uintptr_t)addr + size < size)
return (void *)einval();
if (size > 0x100000000000)
return (void *)enomem();
// normalize arguments
if (flags & MAP_ANONYMOUS) {
fd = -1;
off = 0;
size = PGUP(size);
}
// create memory mappping
if (!__isfdkind(fd, kFdZip)) {
res = __mmap_impl(addr, size, prot, flags, fd, off, granularity);
} else {
res = _weaken(__zipos_mmap)(
addr, size, prot, flags,
(struct ZiposHandle *)(uintptr_t)g_fds.p[fd].handle, off);
}
return res;
}
void *mmap(void *addr, size_t size, int prot, int flags, int fd, int64_t off) {
void *res;
res = __mmap(addr, size, prot, flags, fd, off);
if (IsAsan() && res != MAP_FAILED) {
__asan_shadow(res, size);
int granularity = __granularity();
if (size != PGUP(size))
__asan_poison(res + size, PGUP(size) - size, kAsanMmapSizeOverrun);
}
STRACE("mmap(%p, %'zu, %s, %s, %d, %'ld) → %p% m", addr, size,
DescribeProtFlags(prot), DescribeMapFlags(flags), fd, off, res);
return res;
}
int munmap(void *addr, size_t size) {
int rc;
rc = __munmap(addr, size, false);
STRACE("munmap(%p, %'zu) → %d% m", addr, size, rc);
return rc;
}
__weak_reference(mmap, mmap64);

215
libc/intrin/mprotect.c Normal file
View file

@ -0,0 +1,215 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/struct/sigset.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/maps.h"
#include "libc/intrin/strace.internal.h"
#include "libc/nt/memory.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/sysparam.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
#define PGUP(x) (((x) + pagesz - 1) & -pagesz)
static int __mprotect_chunk(char *addr, size_t size, int prot, bool iscow) {
if (!IsWindows())
return sys_mprotect(addr, size, prot);
uint32_t op;
if (!VirtualProtect(addr, size, __prot2nt(prot, iscow), &op))
return -1;
return 0;
}
int __mprotect(char *addr, size_t size, int prot) {
// unix checks prot before checking size
if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC))
return einval();
// make new technology consistent with unix
if (!size)
return 0;
// unix checks prot before checking size
int pagesz = getauxval(AT_PAGESZ);
if ((intptr_t)addr & (pagesz - 1))
return einval();
// change mappings
int rc = 0;
__maps_lock();
bool found = false;
struct Map *map = __maps.maps;
_Atomic(struct Map *) *prev = &__maps.maps;
while (map) {
char *map_addr = map->addr;
size_t map_size = map->size;
struct Map *next = map->next;
char *beg = MAX(addr, map_addr);
char *end = MIN(addr + PGUP(size), map_addr + PGUP(map_size));
if (beg < end) {
found = true;
if (addr <= map_addr && addr + PGUP(size) >= map_addr + PGUP(map_size)) {
// change protection of entire mapping
if (!__mprotect_chunk(map_addr, map_size, prot, map->iscow)) {
map->prot = prot;
} else {
rc = -1;
}
} else if (IsWindows()) {
// windows does allow changing protection at 4096 byte chunks
// however we currently don't have data structures that track
// this within the 64 kb map granules that can't be broken up
if (__mprotect_chunk(beg, end - beg, prot, map->iscow) == -1)
rc = -1;
} else if (addr <= map_addr) {
// cleave lefthand side of mapping
size_t left = addr + size - map_addr;
size_t right = map_addr + map_size - (addr + size);
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
if (!__mprotect_chunk(map_addr, left, prot, false)) {
leftmap->next = map;
leftmap->addr = map_addr;
leftmap->size = left;
leftmap->prot = prot;
leftmap->off = map->off;
leftmap->flags = map->flags;
map->addr += left;
map->size = right;
if (map->off != -1)
map->off += left;
dll_make_first(&__maps.used, &leftmap->elem);
*prev = leftmap;
} else {
__maps_free(leftmap);
rc = -1;
}
} else {
rc = -1;
}
} else if (addr + PGUP(size) >= map_addr + PGUP(map_size)) {
// cleave righthand side of mapping
size_t left = addr - map_addr;
size_t right = map_addr + map_size - addr;
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
if (!__mprotect_chunk(map_addr + left, right, prot, false)) {
leftmap->next = map;
leftmap->addr = map_addr;
leftmap->size = left;
leftmap->off = map->off;
leftmap->prot = map->prot;
leftmap->flags = map->flags;
map->addr += left;
map->size = right;
map->prot = prot;
if (map->off != -1)
map->off += left;
dll_make_first(&__maps.used, &leftmap->elem);
*prev = leftmap;
} else {
__maps_free(leftmap);
rc = -1;
}
} else {
rc = -1;
}
} else {
// punch hole in mapping
size_t left = addr - map_addr;
size_t middle = PGUP(size);
size_t right = map_size - middle - left;
struct Map *leftmap;
if ((leftmap = __maps_alloc())) {
struct Map *midlmap;
if ((midlmap = __maps_alloc())) {
if (!__mprotect_chunk(map_addr + left, middle, prot, false)) {
leftmap->next = midlmap;
leftmap->addr = map_addr;
leftmap->size = left;
leftmap->off = map->off;
leftmap->prot = map->prot;
leftmap->flags = map->flags;
midlmap->next = map;
midlmap->addr = map_addr + left;
midlmap->size = middle;
midlmap->off = map->off == -1 ? -1 : map->off + left;
midlmap->prot = prot;
midlmap->flags = map->flags;
map->addr += left + middle;
map->size = right;
if (map->off != -1)
map->off += left + middle;
dll_make_first(&__maps.used, &leftmap->elem);
dll_make_first(&__maps.used, &midlmap->elem);
*prev = leftmap;
} else {
__maps_free(midlmap);
__maps_free(leftmap);
rc = -1;
}
} else {
__maps_free(leftmap);
rc = -1;
}
} else {
rc = -1;
}
}
}
prev = &map->next;
map = next;
}
// allow user to change mappings unknown to cosmo runtime
if (!found)
rc = __mprotect_chunk(addr, size, prot, false);
__maps_unlock();
return rc;
}
/**
* Modifies restrictions on virtual memory address range.
*
* @param addr needs to be 4kb aligned
* @param prot can have PROT_{NONE,READ,WRITE,EXEC}
* @return 0 on success, or -1 w/ errno
* @raise ENOMEM on tracking memory oom
* @see mmap()
*/
int mprotect(void *addr, size_t size, int prot) {
int rc;
rc = __mprotect(addr, size, prot);
STRACE("mprotect(%p, %'zu, %s) → %d% m", addr, size, DescribeProtFlags(prot),
rc);
return rc;
}

47
libc/intrin/msync-nt.c Normal file
View file

@ -0,0 +1,47 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/syscall-nt.internal.h"
#include "libc/intrin/maps.h"
#include "libc/nt/memory.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/sysparam.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/errfuns.h"
textwindows int sys_msync_nt(char *addr, size_t size, int flags) {
int pagesz = getauxval(AT_PAGESZ);
size = (size + pagesz - 1) & -pagesz;
if ((uintptr_t)addr & (pagesz - 1))
return einval();
int rc = 0;
for (struct Map *map = __maps.maps; map; map = map->next) {
char *beg = MAX(addr, map->addr);
char *end = MIN(addr + size, map->addr + map->size);
if (beg < end)
if (!FlushViewOfFile(beg, end - beg))
rc = -1;
// TODO(jart): FlushFileBuffers too on g_fds handle if MS_SYNC?
}
return rc;
}

98
libc/intrin/msync.c Normal file
View file

@ -0,0 +1,98 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/sysv/consts/msync.h"
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/calls/cp.internal.h"
#include "libc/calls/syscall-nt.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/intrin/strace.internal.h"
#include "libc/macros.internal.h"
#include "libc/sysv/errfuns.h"
/**
* Synchronize memory mapping changes to disk.
*
* Without this, there's no guarantee memory is written back to disk.
* Particularly on RHEL5, OpenBSD, and Windows NT.
*
* @param addr needs to be 4096-byte page aligned
* @param flags needs MS_ASYNC or MS_SYNC and can have MS_INVALIDATE
* @return 0 on success or -1 w/ errno
* @raise ECANCELED if thread was cancelled in masked mode
* @raise EINTR if we needed to block and a signal was delivered instead
* @raise EINVAL if `MS_SYNC` and `MS_ASYNC` were both specified
* @raise EINVAL if unknown `flags` were passed
* @cancelationpoint
*/
int msync(void *addr, size_t size, int flags) {
int rc;
if ((flags & ~(MS_SYNC | MS_ASYNC | MS_INVALIDATE)) ||
(flags & (MS_SYNC | MS_ASYNC)) == (MS_SYNC | MS_ASYNC)) {
rc = einval();
goto Finished;
}
// According to POSIX, either MS_SYNC or MS_ASYNC must be specified
// in flags, and indeed failure to include one of these flags will
// cause msync() to fail on some systems. However, Linux permits a
// call to msync() that specifies neither of these flags, with
// semantics that are (currently) equivalent to specifying MS_ASYNC.
// ──Quoth msync(2) of Linux Programmer's Manual
int sysflags = flags;
sysflags = flags;
if (flags & MS_ASYNC) {
sysflags = MS_ASYNC;
} else if (flags & MS_SYNC) {
sysflags = MS_SYNC;
} else {
sysflags = MS_ASYNC;
}
if (flags & MS_INVALIDATE) {
sysflags |= MS_INVALIDATE;
}
// FreeBSD's manual says "The flags argument was both MS_ASYNC and
// MS_INVALIDATE. Only one of these flags is allowed." which makes
// following the POSIX recommendation somewhat difficult.
if (IsFreebsd()) {
if (sysflags == (MS_ASYNC | MS_INVALIDATE)) {
sysflags = MS_INVALIDATE;
}
}
// FreeBSD specifies MS_SYNC as 0 so we shift the Cosmo constants
if (IsFreebsd()) {
sysflags >>= 1;
}
BEGIN_CANCELATION_POINT;
if (!IsWindows()) {
rc = sys_msync(addr, size, sysflags);
} else {
rc = sys_msync_nt(addr, size, sysflags);
}
END_CANCELATION_POINT;
Finished:
STRACE("msync(%p, %'zu, %#x) → %d% m", addr, size, flags, rc);
return rc;
}

34
libc/intrin/printsystemmappings.greg.c → libc/intrin/munmap-metal.c
View file

@ -16,23 +16,25 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/sysv/consts/at.h"
#include "libc/sysv/consts/o.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/runtime/pc.internal.h"
#ifdef __x86_64__
void PrintSystemMappings(int outfd) {
int infd;
ssize_t rc;
char buf[64];
if (!IsWindows()) {
if ((infd = __sys_openat(AT_FDCWD, "/proc/self/maps", O_RDONLY, 0)) >= 0) {
sys_write(outfd, "\n", 1);
while ((rc = sys_read(infd, buf, sizeof(buf))) > 0) {
sys_write(outfd, buf, rc);
}
int sys_munmap_metal(void *addr, size_t size) {
size_t i;
uint64_t *e, paddr;
struct mman *mm = __get_mm();
uint64_t *pml4t = __get_pml4t();
for (i = 0; i < size; i += 4096) {
e = __get_virtual(mm, pml4t, (uint64_t)addr + i, false);
if (e) {
paddr = *e & PAGE_TA;
*e &= ~(PAGE_V | PAGE_RSRV);
invlpg((uint64_t)addr + i);
__unref_page(mm, pml4t, paddr);
}
sys_close(infd);
}
return 0;
}
#endif

48
libc/intrin/munmap-sysv.c Normal file
View file

@ -0,0 +1,48 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/dce.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/syslib.internal.h"
/**
* Unmaps memory directly with system.
*
* This function bypasses memtrack. Therefore it won't work on Windows,
* but it works on everything else including bare metal.
*
* @asyncsignalsafe
*/
int sys_munmap(void *p, size_t n) {
int rc;
if (IsXnuSilicon()) {
rc = _sysret(__syslib->__munmap(p, n));
} else if (IsMetal()) {
rc = sys_munmap_metal(p, n);
} else {
rc = __sys_munmap(p, n);
}
if (!rc)
__virtualsize -= n;
KERNTRACE("sys_munmap(%p, %'zu) → %d", p, n, rc);
return rc;
}

34
libc/intrin/nocolor.c
View file

@ -16,39 +16,19 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/dce.h"
#include "libc/log/internal.h"
#include "libc/runtime/runtime.h"
#define IsDumb(s) \
(s[0] == 'd' && s[1] == 'u' && s[2] == 'm' && s[3] == 'b' && !s[4])
/**
* Indicates if ANSI terminal colors are inappropriate.
*
* Normally this variable should be false. We only set it to true if
* we're running on an old version of Windows or the environment
* variable `TERM` is set to `dumb`.
*
* We think colors should be the norm, since most software is usually
* too conservative about removing them. Rather than using `isatty`
* consider using sed for instances where color must be removed:
*
* sed 's/\x1b\[[;[:digit:]]*m//g' <color.txt >uncolor.txt
*
* For some reason, important software is configured by default in many
* operating systems, to not only disable colors, but utf-8 too! Here's
* an example of how a wrapper script can fix that for `less`.
*
* #!/bin/sh
* LESSCHARSET=UTF-8 exec /usr/bin/less -RS "$@"
*
* Thank you for using colors!
*/
bool __nocolor;
__attribute__((__constructor__(20))) optimizesize textstartup void
__nocolor_init(int argc, char **argv, char **envp, intptr_t *auxv) {
char *s;
__nocolor = IsWindows() || ((s = getenv("TERM")) && IsDumb(s));
if ((s = getenv("TERM")))
if (s[0] == 'd' && //
s[1] == 'u' && //
s[2] == 'm' && //
s[3] == 'b' && //
s[4] == '\0')
__nocolor = true;
}

55
libc/intrin/printmaps.c Normal file
View file

@ -0,0 +1,55 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/fmt/conv.h"
#include "libc/fmt/itoa.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/maps.h"
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
/**
* Prints memory mappings.
*/
void __print_maps(void) {
int limit = 10;
long maptally = 0;
char mappingbuf[8], sb[16];
for (struct Map *map = __maps.maps; map; map = map->next) {
maptally += map->size;
kprintf("%012lx-%012lx %!s", map->addr, map->addr + map->size,
(DescribeMapping)(mappingbuf, map->prot, map->flags));
sizefmt(sb, map->size, 1024);
kprintf(" %!sb", sb);
if (map->h && map->h != -1)
kprintf(" h=%ld", map->h);
if (map->iscow)
kprintf(" cow");
if (map->readonlyfile)
kprintf(" readonlyfile");
kprintf("\n");
if (!--limit) {
kprintf("...\n");
break;
}
}
sizefmt(sb, maptally, 1024);
kprintf("# %!sb mapped memory\n", sb);
}

66
libc/intrin/printmemoryintervals.c
View file

@ -1,66 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/fmt/conv.h"
#include "libc/fmt/itoa.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
static bool IsNoteworthyHole(unsigned i, const struct MemoryIntervals *mm) {
// gaps between shadow frames aren't interesting
// the chasm from heap to stack ruins statistics
return !(
(IsShadowFrame(mm->p[i].y) || IsShadowFrame(mm->p[i + 1].x)) ||
(!IsStaticStackFrame(mm->p[i].y) && IsStaticStackFrame(mm->p[i + 1].x)));
}
void PrintMemoryIntervals(int fd, const struct MemoryIntervals *mm) {
long i, w, frames, maptally = 0;
char mappingbuf[8], framebuf[64], sb[16];
for (w = i = 0; i < mm->i; ++i) {
w = MAX(w, LengthInt64Thousands(mm->p[i].y + 1 - mm->p[i].x));
}
for (i = 0; i < mm->i; ++i) {
frames = mm->p[i].y + 1 - mm->p[i].x;
maptally += frames;
kprintf("%08x-%08x %s %'*ldx %s", mm->p[i].x, mm->p[i].y,
(DescribeMapping)(mappingbuf, mm->p[i].prot, mm->p[i].flags), w,
frames, (DescribeFrame)(framebuf, mm->p[i].x));
if (mm->p[i].iscow)
kprintf(" cow");
if (mm->p[i].readonlyfile)
kprintf(" readonlyfile");
sizefmt(sb, mm->p[i].size, 1024);
kprintf(" %sB", sb);
if (i + 1 < mm->i) {
frames = mm->p[i + 1].x - mm->p[i].y - 1;
if (frames && IsNoteworthyHole(i, mm)) {
sizefmt(sb, frames * FRAMESIZE, 1024);
kprintf(" w/ %sB hole", sb);
}
}
if (mm->p[i].h != -1) {
kprintf(" h=%ld", mm->p[i].h);
}
kprintf("\n");
}
sizefmt(sb, maptally * FRAMESIZE, 1024);
kprintf("# %sB total mapped memory\n", sb);
}

3
libc/intrin/pthread_tid.c
View file

@ -16,11 +16,14 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/dce.h"
#include "libc/intrin/atomic.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
int _pthread_tid(struct PosixThread *pt) {
if (IsWindows()) // xxx: fixme
return pt->ptid;
int tid = 0;
while (pt && !(tid = atomic_load_explicit(&pt->ptid, memory_order_acquire))) {
pthread_pause_np();

20
libc/intrin/findmemoryinterval.c → libc/intrin/virtualmax.c
View file

@ -1,7 +1,7 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ Copyright 2023 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
@ -16,19 +16,7 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
unsigned __find_memory(const struct MemoryIntervals *mm, int x) {
unsigned l, m, r;
l = 0;
r = mm->i;
while (l < r) {
m = (l & r) + ((l ^ r) >> 1); // floor((a+b)/2)
if (mm->p[m].y < x) {
l = m + 1;
} else {
r = m;
}
}
return l;
}
size_t __virtualmax = -1;
size_t __virtualsize = 0;