mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-07-26 12:30:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve memory safety

Browse source 
This commit makes numerous refinements to cosmopolitan memory handling.

The default stack size has been reduced from 2mb to 128kb. A new macro
is now provided so you can easily reconfigure the stack size to be any
value you want. Work around the breaking change by adding to your main:

    STATIC_STACK_SIZE(0x00200000);  // 2mb stack

If you're not sure how much stack you need, then you can use:

    STATIC_YOINK("stack_usage_logging");

After which you can `sort -nr o/$MODE/stack.log`. Based on the unit test
suite, nothing in the Cosmopolitan repository (except for Python) needs
a stack size greater than 30kb. There are also new macros for detecting
the size and address of the stack at runtime, e.g. GetStackAddr(). We
also now support sigaltstack() so if you want to see nice looking crash
reports whenever a stack overflow happens, you can put this in main():

    ShowCrashReports();

Under `make MODE=dbg` and `make MODE=asan` the unit testing framework
will now automatically print backtraces of memory allocations when
things like memory leaks happen. Bugs are now fixed in ASAN global
variable overrun detection. The memtrack and asan runtimes also handle
edge cases now. The new tools helped to identify a few memory leaks,
which are fixed by this change.

This change should fix an issue reported in #288 with ARG_MAX limits.
Fixing this doubled the performance of MKDEPS.COM and AR.COM yet again.
This commit is contained in:
Justine Tunney 2021-10-13 17:27:13 -07:00
parent a0b39f886c
commit 226aaf3547
317 changed files with 6474 additions and 3993 deletions
Download patch file Download diff file Expand all files Collapse all files

5
libc/runtime/abort-nt.c
View file

@ -16,10 +16,9 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/bits/pushpop.h"
#include "libc/calls/internal.h"
#include "libc/calls/struct/siginfo.h"
#include "libc/nt/enum/ctrlevent.h"
#include "libc/calls/typedef/sigaction_f.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/sig.h"

1
libc/runtime/arememoryintervalsok.c
View file

@ -19,6 +19,7 @@
#include "libc/runtime/memtrack.internal.h"
noasan bool AreMemoryIntervalsOk(const struct MemoryIntervals *mm) {
/* asan runtime depends on this function */
int i;
for (i = 0; i < mm->i; ++i) {
if (mm->p[i].y < mm->p[i].x) {

124
libc/runtime/cxaatexit.c
View file

@ -1,124 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/bits/weaken.h"
#include "libc/macros.internal.h"
#include "libc/mem/mem.h"
#include "libc/nexgen32e/bsf.h"
#include "libc/nexgen32e/bsr.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/errfuns.h"
static struct CxaAtexitBlocks {
struct CxaAtexitBlock {
unsigned mask;
struct CxaAtexitBlock *next;
struct CxaAtexit {
void *fp;
void *arg;
void *pred;
} p[ATEXIT_MAX];
} * p, root;
} __cxa_blocks;
/**
* Adds global destructor.
*
* Destructors are called in reverse order. They won't be called if the
* program aborts or _exit() is called. Invocations of this function are
* usually generated by the C++ compiler. Behavior is limitless if some
* other module has linked calloc().
*
* @param fp is void(*)(T)
* @param arg is passed to callback
* @param pred can be non-null for things like dso modules
* @return 0 on success or nonzero w/ errno
* @note folks have forked libc in past just to unbloat atexit()
*/
noasan int __cxa_atexit(void *fp, void *arg, void *pred) {
unsigned i;
struct CxaAtexitBlock *b, *b2;
_Static_assert(ATEXIT_MAX == CHAR_BIT * sizeof(b->mask), "");
b = __cxa_blocks.p;
if (!b) b = __cxa_blocks.p = &__cxa_blocks.root;
if (!~b->mask) {
if (weaken(calloc) &&
(b2 = weaken(calloc)(1, sizeof(struct CxaAtexitBlock)))) {
b2->next = b;
__cxa_blocks.p = b = b2;
} else {
return enomem();
}
}
i = bsr(~b->mask);
assert(i < ARRAYLEN(b->p));
b->mask |= 1u << i;
b->p[i].fp = fp;
b->p[i].arg = arg;
b->p[i].pred = pred;
return 0;
}
/**
* Triggers global destructors.
*
* They're called in LIFO order. If a destructor adds more destructors,
* then those destructors will be called immediately following, before
* iteration continues.
*
* @param pred can be null to match all
*/
void __cxa_finalize(void *pred) {
unsigned i, mask;
struct CxaAtexitBlock *b, *b2;
StartOver:
if ((b = __cxa_blocks.p)) {
for (;;) {
mask = b->mask;
while (mask) {
i = bsf(mask);
mask &= ~(1u << i);
if (!pred || pred == b->p[i].pred) {
b->mask &= ~(1u << i);
if (b->p[i].fp) {
((void (*)(void *))b->p[i].fp)(b->p[i].arg);
goto StartOver;
}
}
}
if (!pred) {
b2 = b->next;
if (b2) {
assert(b != &__cxa_blocks.root);
if (weaken(free)) {
weaken(free)(b);
}
}
__cxa_blocks.p = b2;
goto StartOver;
} else {
if (b->next) {
b = b->next;
} else {
break;
}
}
}
}
}

25
libc/runtime/cxaatexit.internal.h Normal file
View file

@ -0,0 +1,25 @@
#ifndef COSMOPOLITAN_LIBC_RUNTIME_CXAATEXIT_H_
#define COSMOPOLITAN_LIBC_RUNTIME_CXAATEXIT_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
#include "libc/stdio/stdio.h"
struct CxaAtexitBlocks {
struct CxaAtexitBlock {
unsigned mask;
struct CxaAtexitBlock *next;
struct CxaAtexit {
void *fp;
void *arg;
void *pred;
} p[ATEXIT_MAX];
} * p, root;
};
extern struct CxaAtexitBlocks __cxa_blocks;
void __cxa_printexits(FILE *, void *);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_RUNTIME_CXAATEXIT_H_ */

49
libc/runtime/describeframe.c Normal file
View file

@ -0,0 +1,49 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/log/libfatal.internal.h"
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#define ADDR(x) ((int64_t)((uint64_t)(x) << 32) >> 16)
#define UNSHADOW(x) ((int64_t)(MAX(0, (x)-0x7fff8000)) << 3)
#define FRAME(x) ((int)((x) >> 16))
noasan const char *DescribeFrame(int x) {
/* asan runtime depends on this function */
char *p;
static char buf[128];
if (IsShadowFrame(x)) {
p = buf;
p = __stpcpy(p, " shadow of ");
p = __fixcpy(p, UNSHADOW(ADDR(x)), 48);
return buf;
return " shadow ";
} else if (IsAutoFrame(x)) {
return " automap";
} else if (IsFixedFrame(x)) {
return " fixed ";
} else if (IsArenaFrame(x)) {
return " arena ";
} else if (IsStackFrame(x)) {
return " stack ";
} else {
return "";
}
}

40
libc/runtime/describemapping.c Normal file
View file

@ -0,0 +1,40 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/memtrack.internal.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
noasan char *DescribeMapping(int prot, int flags, char p[hasatleast 8]) {
/* asan runtime depends on this function */
p[0] = (prot & PROT_READ) ? 'r' : '-';
p[1] = (prot & PROT_WRITE) ? 'w' : '-';
p[2] = (prot & PROT_EXEC) ? 'x' : '-';
if (flags & MAP_PRIVATE) {
p[3] = 'p';
} else if (flags & MAP_SHARED) {
p[3] = 's';
} else {
p[3] = '?';
}
p[4] = (flags & MAP_ANONYMOUS) ? 'a' : 'f';
p[5] = (flags & MAP_GROWSDOWN) ? 'S' : '-';
p[6] = (flags & MAP_FIXED) ? 'F' : '-';
p[7] = 0;
return p;
}

1
libc/runtime/directmap-metal.c
View file

@ -29,6 +29,7 @@ static uint64_t sys_mmap_metal_break;
noasan struct DirectMap sys_mmap_metal(void *paddr, size_t size, int prot,
int flags, int fd, int64_t off) {
/* asan runtime depends on this function */
size_t i;
struct mman *mm;
struct DirectMap res;

1
libc/runtime/directmap-nt.c
View file

@ -32,6 +32,7 @@
textwindows noasan struct DirectMap sys_mmap_nt(void *addr, size_t size,
int prot, int flags,
int64_t handle, int64_t off) {
/* asan runtime depends on this function */
uint32_t got;
size_t i, upsize;
struct DirectMap dm;

11
libc/runtime/directmap.c
View file

@ -18,8 +18,11 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/internal.h"
#include "libc/calls/sysdebug.internal.h"
#include "libc/errno.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/str/str.h"
/**
* Obtains memory mapping directly from system.
@ -31,11 +34,15 @@
*/
noasan struct DirectMap sys_mmap(void *addr, size_t size, int prot, int flags,
int fd, int64_t off) {
/* asan runtime depends on this function */
char mode[8];
struct DirectMap dm;
if (!IsWindows() && !IsMetal()) {
dm.addr = __sys_mmap(addr, size, prot, flags, fd, off, off);
SYSDEBUG("sys_mmap(0x%x, 0x%x, %d, 0x%x, %d, %d) -> 0x%x", addr, size, prot,
flags, fd, off, dm.addr);
SYSDEBUG("sys_mmap(0x%p%s, 0x%x, %s, %d, %d) -> 0x%p %s", addr,
DescribeFrame((intptr_t)addr >> 16), size,
DescribeMapping(prot, flags, mode), (long)fd, off, dm.addr,
dm.addr != MAP_FAILED ? "" : strerror(errno));
dm.maphandle = kNtInvalidHandleValue;
return dm;
} else if (IsMetal()) {

2
libc/runtime/ftrace-hook.S
View file

@ -20,7 +20,7 @@
.privileged
ftrace_hook:
cmp $0,ftrace(%rip)
cmp $0,g_ftrace(%rip)
je 1f
ret
1: push %rbp

2
libc/runtime/ftrace.c
View file

@ -18,4 +18,4 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/runtime.h"
int ftrace;
int g_ftrace;

13
libc/runtime/ftracer.c
View file

@ -24,9 +24,11 @@
#include "libc/nexgen32e/rdtscp.h"
#include "libc/nexgen32e/stackframe.h"
#include "libc/nexgen32e/x86feature.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/symbols.internal.h"
#include "libc/stdio/stdio.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/o.h"
#pragma weak stderr
@ -49,7 +51,7 @@ static int g_lastsymbol;
static uint64_t laststamp;
static struct SymbolTable *g_symbols;
static privileged noinstrument noasan int GetNestingLevelImpl(
static privileged noinstrument noasan noubsan int GetNestingLevelImpl(
struct StackFrame *frame) {
int nesting = -2;
while (frame) {
@ -59,7 +61,7 @@ static privileged noinstrument noasan int GetNestingLevelImpl(
return MAX(0, nesting);
}
static privileged noinstrument noasan int GetNestingLevel(
static privileged noinstrument noasan noubsan int GetNestingLevel(
struct StackFrame *frame) {
int nesting;
nesting = GetNestingLevelImpl(frame);
@ -75,7 +77,8 @@ static privileged noinstrument noasan int GetNestingLevel(
* prologues of other functions. We assume those functions behave
* according to the System Five NexGen32e ABI.
*/
privileged noinstrument noasan void ftracer(void) {
privileged noinstrument noasan noubsan void ftracer(void) {
/* asan runtime depends on this function */
int symbol;
uint64_t stamp;
static bool noreentry;
@ -85,11 +88,11 @@ privileged noinstrument noasan void ftracer(void) {
stamp = rdtsc();
frame = __builtin_frame_address(0);
frame = frame->next;
if ((symbol = GetSymbol(g_symbols, frame->addr)) != -1 &&
if ((symbol = __get_symbol(g_symbols, frame->addr)) != -1 &&
symbol != g_lastsymbol) {
g_lastsymbol = symbol;
__printf("+ %*s%s %d\r\n", GetNestingLevel(frame) * 2, "",
GetSymbolName(g_symbols, symbol),
__get_symbol_name(g_symbols, symbol),
(long)(unsignedsubtract(stamp, laststamp) / 3.3));
laststamp = X86_HAVE(RDTSCP) ? rdtscp(0) : rdtsc();
}

42
libc/runtime/getsymbol.c Normal file
View file

@ -0,0 +1,42 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/symbols.internal.h"
privileged noinstrument noasan noubsan int __get_symbol(struct SymbolTable *t,
intptr_t a) {
/* asan runtime depends on this function */
unsigned l, m, r, n, k;
if (t) {
l = 0;
r = n = t->count;
k = a - t->addr_base;
while (l < r) {
m = (l + r) >> 1;
if (t->symbols[m].y < k) {
l = m + 1;
} else {
r = m;
}
}
if (l < n && t->symbols[l].x <= k && k <= t->symbols[l].y) {
return l;
}
}
return -1;
}

24
libc/runtime/atexit.c → libc/runtime/getsymbolname.c
View file

@ -1,7 +1,7 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
@ -16,18 +16,14 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/runtime.h"
#include "libc/runtime/symbols.internal.h"
/**
* Adds global destructor.
*
* Destructors are called in reverse order. They won't be called
* if the program aborts or _exit() is called. Invocations of this
* function are usually generated by the C++ compiler.
*
* @param rdi callback typed void(*)(T) (nullable)
* @return 0 on success or nonzero if out of space
*/
int atexit(void f(void)) {
return __cxa_atexit(f, 0, 0);
privileged noinstrument noasan noubsan char *__get_symbol_name(
struct SymbolTable *t, int s) {
/* asan runtime depends on this function */
if (t && s != -1) {
return t->name_base + t->names[s];
} else {
return 0;
}
}

13
libc/runtime/isheap.c
View file

@ -27,15 +27,6 @@
* @assume stack memory isn't stored beneath %rsp (-mno-red-zone)
*/
noasan bool _isheap(void *p) {
int x, i;
uintptr_t rsp;
asm("mov\t%%rsp,%0" : "=r"(rsp));
if (ROUNDDOWN(rsp, STACKSIZE) == ROUNDDOWN((intptr_t)p, STACKSIZE)) {
return false;
} else {
if ((intptr_t)p <= (intptr_t)_end) return false;
x = (intptr_t)p >> 16;
i = FindMemoryInterval(&_mmi, x);
return i < _mmi.i && x >= _mmi.p[i].x && x <= _mmi.p[i].y;
}
return kAutomapStart <= (intptr_t)p &&
(intptr_t)p < kAutomapStart + kAutomapSize;
}

35
libc/runtime/mapanon.c
View file

@ -33,19 +33,32 @@
* mmap(NULL, mapsize, PROT_READ | PROT_WRITE,
* MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
*
* Except it offers a small saving on code size.
* If mmap() fails, possibly because the parent process did this:
*
* if (!vfork()) {
* setrlimit(RLIMIT_AS, &(struct rlimit){maxbytes, maxbytes});
* execv(prog, (char *const[]){prog, 0});
* }
* wait(0);
*
* Then this function will call:
*
* __oom_hook(size);
*
* If it's linked. The LIBC_TESTLIB library provides an implementation,
* which can be installed as follows:
*
* int main() {
* InstallQuotaHandlers();
* // ...
* }
*
* That is performed automatically for unit test executables.
*/
void *mapanon(size_t size) {
noasan void *mapanon(size_t size) {
/* asan runtime depends on this function */
void *m;
if (!size || bsrl(size) >= 40) {
errno = EINVAL;
return MAP_FAILED;
}
m = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (m == MAP_FAILED) {
if (weaken(__oom_hook)) {
weaken(__oom_hook)(size);
}
}
if (m == MAP_FAILED && weaken(__oom_hook)) weaken(__oom_hook)(size);
return m;
}

9
libc/runtime/memtrack.c
View file

@ -37,6 +37,7 @@
static noasan void *MoveMemoryIntervals(struct MemoryInterval *d,
const struct MemoryInterval *s, int n) {
/* asan runtime depends on this function */
int i;
assert(n >= 0);
if (d > s) {
@ -53,6 +54,7 @@ static noasan void *MoveMemoryIntervals(struct MemoryInterval *d,
static noasan void RemoveMemoryIntervals(struct MemoryIntervals *mm, int i,
int n) {
/* asan runtime depends on this function */
assert(i >= 0);
assert(i + n <= mm->i);
MoveMemoryIntervals(mm->p + i, mm->p + i + n, mm->i - (i + n));
@ -60,6 +62,7 @@ static noasan void RemoveMemoryIntervals(struct MemoryIntervals *mm, int i,
}
static noasan void MapNewMappingArray(struct MemoryIntervals *mm) {
/* asan runtime depends on this function */
void *a;
int i, x, y, g;
size_t n, m, f;
@ -97,10 +100,10 @@ static noasan void MapNewMappingArray(struct MemoryIntervals *mm) {
}
}
}
flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED;
flags = MAP_ANONYMOUS | MAP_PRIVATE;
prot = PROT_READ | PROT_WRITE;
SYSDEBUG("MapNewMappingArray(0x%x, 0x%x) %d/%d/%d", a, m, i, mm->i, mm->n);
dm = sys_mmap(a, m, prot, flags, -1, 0);
dm = sys_mmap(a, m, prot, flags | MAP_FIXED, -1, 0);
if ((p = dm.addr) != MAP_FAILED) {
MoveMemoryIntervals(p, mm->p, i);
MoveMemoryIntervals(p + i + 1, mm->p + i, mm->i - i);
@ -126,6 +129,7 @@ static noasan void MapNewMappingArray(struct MemoryIntervals *mm) {
}
noasan int CreateMemoryInterval(struct MemoryIntervals *mm, int i) {
/* asan runtime depends on this function */
int rc;
rc = 0;
assert(i >= 0);
@ -189,6 +193,7 @@ noasan int ReleaseMemoryIntervals(struct MemoryIntervals *mm, int x, int y,
noasan int TrackMemoryInterval(struct MemoryIntervals *mm, int x, int y, long h,
int prot, int flags) {
/* asan runtime depends on this function */
unsigned i;
assert(y >= x);
assert(AreMemoryIntervalsOk(mm));

94
libc/runtime/memtrack.internal.h
View file

@ -4,12 +4,14 @@
#include "libc/macros.internal.h"
#include "libc/nt/enum/version.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
#define _kAutomapStart 0x0000100080000000 /* asan can't spread its poison */
#define _kAutomapSize 0x00000fff80000000 /* beyond the above mem address */
#define _kFixedmapStart 0x0000200000000000
#define _kAutomapStart 0x0000100080000000
#define _kAutomapSize 0x00000fff80000000
#define _kFixedmapStart 0x0000300000000000
#define _kFixedmapSize 0x00000fff80000000
/*
* TODO: Why can't we allocate addresses above 4GB on Windows 7 x64?
@ -18,8 +20,9 @@ COSMOPOLITAN_C_START_
#define MEMTRACK_ADDRESS(NORMAL, WIN7) \
(!(IsWindows() && NtGetVersion() < kNtVersionWindows10) ? NORMAL : WIN7)
#define kAutomapStart MEMTRACK_ADDRESS(_kAutomapStart, 0x10000000)
#define kAutomapSize MEMTRACK_ADDRESS(_kAutomapSize, 0x40000000)
#define kFixedmapStart MEMTRACK_ADDRESS(_kFixedmapStart, 0x50000000)
#define kAutomapSize MEMTRACK_ADDRESS(_kAutomapSize, 0x30000000)
#define kFixedmapStart MEMTRACK_ADDRESS(_kFixedmapStart, 0x40000000)
#define kFixedmapSize MEMTRACK_ADDRESS(_kFixedmapSize, 0x30000000)
struct MemoryInterval {
int x;
@ -37,6 +40,9 @@ struct MemoryIntervals {
extern hidden struct MemoryIntervals _mmi;
const char *DescribeFrame(int);
void PrintSystemMappings(int) hidden;
char *DescribeMapping(int, int, char[hasatleast 8]) hidden;
bool AreMemoryIntervalsOk(const struct MemoryIntervals *) nosideeffect hidden;
void PrintMemoryIntervals(int, const struct MemoryIntervals *) hidden;
int TrackMemoryInterval(struct MemoryIntervals *, int, int, long, int,
@ -46,8 +52,70 @@ int ReleaseMemoryIntervals(struct MemoryIntervals *, int, int,
void ReleaseMemoryNt(struct MemoryIntervals *, int, int) hidden;
int UntrackMemoryIntervals(void *, size_t) hidden;
static inline noasan unsigned FindMemoryInterval(
const struct MemoryIntervals *mm, int x) {
forceinline pureconst bool IsAutoFrame(int x) {
return (kAutomapStart >> 16) <= x &&
x <= ((kAutomapStart + (kAutomapSize - 1)) >> 16);
}
forceinline pureconst bool IsArenaFrame(int x) {
return 0x5000 <= x && x < 0x7ffe;
}
forceinline pureconst bool IsShadowFrame(int x) {
return 0x7fff <= x && x < 0x10008000;
}
forceinline pureconst bool IsStackFrame(int x) {
return (GetStaticStackAddr(0) >> 16) <= x &&
x <= ((GetStaticStackAddr(0) + (GetStackSize() - FRAMESIZE)) >> 16);
}
forceinline pureconst bool IsFixedFrame(int x) {
return (kFixedmapStart >> 16) <= x &&
x <= ((kFixedmapStart + (kFixedmapSize - 1)) >> 16);
}
forceinline pureconst bool OverlapsImageSpace(const void *p, size_t n) {
const unsigned char *start, *ender;
if (n) {
start = p;
ender = start + (n - 1);
return ((_base <= start && start < _end) ||
(_base <= ender && ender < _end) ||
(start < _base && _end <= ender));
} else {
return 0;
}
}
forceinline pureconst bool OverlapsArenaSpace(const void *p, size_t n) {
intptr_t x, y;
if (n) {
x = (intptr_t)p;
y = x + (n - 1);
return ((0x50000000 <= x && x <= 0x7ffdffff) ||
(0x50000000 <= y && y <= 0x7ffdffff) ||
(x < 0x50000000 && 0x7ffdffff < y));
} else {
return 0;
}
}
forceinline pureconst bool OverlapsShadowSpace(const void *p, size_t n) {
intptr_t x, y;
if (n) {
x = (intptr_t)p;
y = x + (n - 1);
return ((0x7fff0000 <= x && x <= 0x10007fffffff) ||
(0x7fff0000 <= y && y <= 0x10007fffffff) ||
(x < 0x7fff0000 && 0x10007fffffff < y));
} else {
return 0;
}
}
forceinline unsigned FindMemoryInterval(const struct MemoryIntervals *mm,
int x) {
unsigned l, m, r;
l = 0;
r = mm->i;
@ -62,6 +130,18 @@ static inline noasan unsigned FindMemoryInterval(
return l;
}
forceinline bool IsMemtracked(int x, int y) {
unsigned i;
i = FindMemoryInterval(&_mmi, x);
if (i == _mmi.i) return false;
if (!(_mmi.p[i].x <= x && x <= _mmi.p[i].y)) return false;
for (;;) {
if (y <= _mmi.p[i].y) return true;
if (++i == _mmi.i) return false;
if (_mmi.p[i].x != _mmi.p[i - 1].y + 1) return false;
}
}
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_RUNTIME_MEMTRACK_H_ */

250
libc/runtime/mmap.c
View file

@ -17,6 +17,7 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/bits/likely.h"
#include "libc/bits/weaken.h"
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
@ -24,6 +25,7 @@
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
#include "libc/rand/rand.h"
#include "libc/runtime/directmap.internal.h"
@ -34,83 +36,213 @@
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
#define IP(X) (intptr_t)(X)
#define VIP(X) (void *)IP(X)
#define ADDR(c) (void *)(IP(c) << 16)
#define ALIGNED(p) (!(IP(p) & (FRAMESIZE - 1)))
#define CANONICAL(p) (-0x800000000000 <= IP(p) && IP(p) <= 0x7fffffffffff)
#define IP(X) (intptr_t)(X)
#define VIP(X) (void *)IP(X)
#define SMALL(n) ((n) <= 0xffffffffffff)
#define ALIGNED(p) (!(IP(p) & (FRAMESIZE - 1)))
#define LEGAL(p) (-0x800000000000 <= IP(p) && IP(p) <= 0x7fffffffffff)
#define ADDR(x) ((int64_t)((uint64_t)(x) << 32) >> 16)
#define SHADE(x) (((intptr_t)(x) >> 3) + 0x7fff8000)
#define FRAME(x) ((int)((intptr_t)(x) >> 16))
forceinline wontreturn void Die(void) {
if (weaken(__die)) weaken(__die)();
abort();
}
noasan static bool IsMapped(char *p, size_t n) {
return OverlapsImageSpace(p, n) || IsMemtracked(FRAME(p), FRAME(p + (n - 1)));
}
noasan static bool NeedAutomap(char *p, size_t n) {
return !p || OverlapsArenaSpace(p, n) || OverlapsShadowSpace(p, n) ||
IsMapped(p, n);
}
noasan static bool ChooseInterval(int x, int n, int *res) {
int i;
if (_mmi.i) {
i = FindMemoryInterval(&_mmi, x);
if (i < _mmi.i) {
if (x + n < _mmi.p[i].x) {
*res = x;
return true;
}
while (++i < _mmi.i) {
if (_mmi.p[i].x - _mmi.p[i - 1].y > n) {
*res = _mmi.p[i - 1].y + 1;
return true;
}
}
}
if (INT_MAX - _mmi.p[i - 1].y >= n) {
*res = _mmi.p[i - 1].y + 1;
return true;
}
return false;
} else {
*res = x;
return true;
}
}
noasan static bool Automap(int n, int *res) {
*res = -1;
if (ChooseInterval(FRAME(kAutomapStart), n, res)) {
assert(*res >= FRAME(kAutomapStart));
if (*res + n <= FRAME(kAutomapStart + (kAutomapStart - 1))) {
return true;
} else {
SYSDEBUG("mmap(0x%p, 0x%x) ENOMEM (automap interval exhausted)",
ADDR(*res), ADDR(n + 1));
return false;
}
} else {
SYSDEBUG("mmap(0x%p, 0x%x) ENOMEM (automap failed)", ADDR(*res),
ADDR(n + 1));
return false;
}
}
/**
* Beseeches system for page-table entries.
* Beseeches system for page-table entries, e.g.
*
* char *p = mmap(NULL, 65536, PROT_READ | PROT_WRITE,
* MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
* munmap(p, 65536);
* char *m;
* m = mmap(NULL, FRAMESIZE, PROT_READ | PROT_WRITE,
* MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
* munmap(m, FRAMESIZE);
*
* @param addr optionally requests a particular virtual base address,
* which needs to be 64kb aligned if passed (for NT compatibility)
* @param size must be >0 and needn't be a multiple of FRAMESIZE
* @param prot can have PROT_READ, PROT_WRITE, PROT_EXEC, PROT_NONE, etc.
* @param addr should be 0 to let your memory manager choose address;
* unless MAP_FIXED or MAP_FIXED_NOREPLACE are specified in flags
* in which case this function will do precicely as you ask, even
* if p=0 (in which you need -fno-delete-null-pointer-checks); it
* needs to be 64kb aligned because it's a wise choice that sadly
* needs to be made mandatory because of Windows although you can
* use __sys_mmap() to circumvent it on System Five in which case
* runtime support services, e.g. asan memory safety, could break
* @param size must be >0 and needn't be a multiple of FRAMESIZE, but
* will be rounded up to FRAMESIZE automatically if MAP_ANONYMOUS
* is specified
* @param prot can have PROT_READ/PROT_WRITE/PROT_EXEC/PROT_NONE/etc.
* @param flags can have MAP_ANONYMOUS, MAP_SHARED, MAP_PRIVATE, etc.
* @param fd is an open()'d file descriptor whose contents shall be
* mapped, and is ignored if MAP_ANONYMOUS is specified
* @param fd is an open()'d file descriptor, whose contents shall be
* made available w/ automatic reading at the chosen address and
* must be -1 if MAP_ANONYMOUS is specified
* @param off specifies absolute byte index of fd's file for mapping,
* should be zero if MAP_ANONYMOUS is specified, and sadly needs
* to be 64kb aligned too
* @return virtual base address of new mapping, or MAP_FAILED w/ errno
*/
void *mmap(void *addr, size_t size, int prot, int flags, int fd, int64_t off) {
noasan void *mmap(void *addr, size_t size, int prot, int flags, int fd,
int64_t off) {
struct DirectMap dm;
int i, x, n, m, a, b, f;
SYSDEBUG("mmap(0x%x, 0x%x, %d, 0x%x, %d, %d)", addr, size, prot, flags, fd,
off);
if (!size) return VIP(einval());
if (size > 0x0000010000000000ull) return VIP(enomem());
if (!ALIGNED(off)) return VIP(einval());
if (!ALIGNED(addr)) return VIP(einval());
if (!CANONICAL(addr)) return VIP(einval());
if (!(!!(flags & MAP_ANONYMOUS) ^ (fd != -1))) return VIP(einval());
if (!(!!(flags & MAP_PRIVATE) ^ !!(flags & MAP_SHARED))) return VIP(einval());
if (fd == -1) size = ROUNDUP(size, FRAMESIZE);
if (IsWindows() && fd == -1) prot |= PROT_WRITE;
int a, b, i, f, m, n, x;
char mode[8], *p = addr;
if (UNLIKELY(!size)) {
SYSDEBUG("mmap(0x%p, 0x%x) EINVAL (size=0)", p, size);
return VIP(einval());
}
if (UNLIKELY(!SMALL(size))) {
SYSDEBUG("mmap(0x%p, 0x%x) EINVAL (size isn't 48-bit)", p, size);
return VIP(einval());
}
if (UNLIKELY(!LEGAL(p))) {
SYSDEBUG("mmap(0x%p, 0x%x) EINVAL (p isn't 48-bit)", p, size);
return VIP(einval());
}
if (UNLIKELY(!ALIGNED(p))) {
SYSDEBUG("mmap(0x%p, 0x%x) EINVAL (p isn't 64kb aligned)", p, size);
return VIP(einval());
}
if (UNLIKELY(fd < -1)) {
SYSDEBUG("mmap(0x%p, 0x%x, fd=%d) EBADF", p, size, (long)fd);
return VIP(ebadf());
}
if (UNLIKELY(!((fd != -1) ^ !!(flags & MAP_ANONYMOUS)))) {
SYSDEBUG("mmap(0x%p, 0x%x, %s, %d, %d) EINVAL (fd anonymous mismatch)", p,
size, DescribeMapping(prot, flags, mode), (long)fd, off);
return VIP(einval());
}
if (UNLIKELY(!(!!(flags & MAP_PRIVATE) ^ !!(flags & MAP_SHARED)))) {
SYSDEBUG("mmap(0x%p, 0x%x) EINVAL (MAP_SHARED ^ MAP_PRIVATE)", p, size);
return VIP(einval());
}
if (UNLIKELY(off < 0)) {
SYSDEBUG("mmap(0x%p, 0x%x, off=%d) EINVAL (neg off)", p, size, off);
return VIP(einval());
}
if (UNLIKELY(INT64_MAX - size < off)) {
SYSDEBUG("mmap(0x%p, 0x%x, off=%d) EINVAL (too large)", p, size, off);
return VIP(einval());
}
if (UNLIKELY(!ALIGNED(off))) {
SYSDEBUG("mmap(0x%p, 0x%x) EINVAL (p isn't 64kb aligned)", p, size);
return VIP(einval());
}
if ((flags & MAP_FIXED_NOREPLACE) && IsMapped(p, size)) {
if (OverlapsImageSpace(p, size)) {
SYSDEBUG("mmap(0x%p, 0x%x) EFAULT (overlaps image)", p, size);
} else {
SYSDEBUG("mmap(0x%p, 0x%x) EFAULT (overlaps existing)", p, size);
}
return VIP(efault());
}
SYSDEBUG("mmap(0x%p, 0x%x, %s, %d, %d)", p, size,
DescribeMapping(prot, flags, mode), (long)fd, off);
if (fd == -1) {
size = ROUNDUP(size, FRAMESIZE);
if (IsWindows()) {
prot |= PROT_WRITE; /* kludge */
}
}
n = FRAME(size) + !!(size & (FRAMESIZE - 1));
f = (flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED;
if (flags & MAP_FIXED) {
if (UntrackMemoryIntervals(addr, size) == -1) {
return MAP_FAILED;
x = FRAME(p);
if (IsWindows()) {
if (UntrackMemoryIntervals(p, size)) {
SYSDEBUG("FIXED UNTRACK FAILED %s", strerror(errno));
Die();
}
}
} else {
x = kAutomapStart >> 16;
n = ROUNDUP(size, FRAMESIZE) >> 16;
for (i = 0; i < _mmi.i; ++i) {
if (_mmi.p[i].y < x) continue;
if (__builtin_add_overflow(_mmi.p[i].y, n, &m)) return VIP(enomem());
if (_mmi.p[i].x > x + n - 1) break;
x = _mmi.p[i].y + 1;
}
if (x + n - 1 >= ((kAutomapStart + kAutomapSize) >> 16)) {
return (void *)(intptr_t)enomem();
}
addr = (void *)(intptr_t)((int64_t)x << 16);
} else if (!NeedAutomap(p, size)) {
x = FRAME(p);
} else if (!Automap(n, &x)) {
return VIP(enomem());
}
f = flags | MAP_FIXED;
p = (char *)ADDR(x);
if (IsOpenbsd() && (f & MAP_GROWSDOWN)) { /* openbsd:dubstack */
dm = sys_mmap(addr, size, prot, f & ~MAP_GROWSDOWN, fd, off);
if (dm.addr == MAP_FAILED) {
SYSDEBUG("sys_mmap failed");
return MAP_FAILED;
}
dm = sys_mmap(p, size, prot, f & ~MAP_GROWSDOWN, fd, off);
if (dm.addr == MAP_FAILED) return MAP_FAILED;
}
dm = sys_mmap(addr, size, prot, f, fd, off);
if (dm.addr == MAP_FAILED || dm.addr != addr) {
SYSDEBUG("sys_mmap failed");
dm = sys_mmap(p, size, prot, f, fd, off);
if (UNLIKELY(dm.addr == MAP_FAILED)) {
if (IsWindows() && (flags & MAP_FIXED)) {
SYSDEBUG("mmap(0x%p, 0x%x) -> %s (%s)", p, size, strerror(errno),
"can't recover from MAP_FIXED errors on Windows");
Die();
}
return MAP_FAILED;
}
a = ROUNDDOWN((intptr_t)addr, FRAMESIZE) >> 16;
b = ROUNDDOWN((intptr_t)addr + size - 1, FRAMESIZE) >> 16;
if (TrackMemoryInterval(&_mmi, a, b, dm.maphandle, prot, flags) == -1) {
abort();
if (UNLIKELY(dm.addr != p)) {
SYSDEBUG("KERNEL DIDN'T RESPECT MAP_FIXED");
Die();
}
if (weaken(__asan_map_shadow)) {
weaken(__asan_map_shadow)((uintptr_t)dm.addr, size);
if (!IsWindows() && (flags & MAP_FIXED)) {
if (UntrackMemoryIntervals(p, size)) {
SYSDEBUG("FIXED UNTRACK FAILED %s", strerror(errno));
Die();
}
}
return dm.addr;
if (TrackMemoryInterval(&_mmi, x, x + (n - 1), dm.maphandle, prot, flags)) {
if (sys_munmap(p, n) == -1) {
SYSDEBUG("TRACK MUNMAP FAILED %s", strerror(errno));
Die();
}
return MAP_FAILED;
}
if (weaken(__asan_map_shadow) && !OverlapsShadowSpace(p, size)) {
weaken(__asan_map_shadow)((intptr_t)p, size);
}
return p;
}

175
libc/runtime/mremap.c
View file

@ -16,18 +16,52 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/bits/likely.h"
#include "libc/bits/weaken.h"
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
#include "libc/calls/sysdebug.internal.h"
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/log/libfatal.internal.h"
#include "libc/macros.internal.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/mremap.h"
#include "libc/sysv/errfuns.h"
#define IP(X) (intptr_t)(X)
#define VIP(X) (void *)IP(X)
#define SMALL(n) ((n) <= 0xffffffffffff)
#define ALIGNED(p) (!(IP(p) & (FRAMESIZE - 1)))
#define LEGAL(p) (-0x800000000000 <= IP(p) && IP(p) <= 0x7fffffffffff)
#define ADDR(x) ((int64_t)((uint64_t)(x) << 32) >> 16)
#define SHADE(x) (((intptr_t)(x) >> 3) + 0x7fff8000)
#define FRAME(x) ((int)((intptr_t)(x) >> 16))
static size_t GetMapSize(size_t i, size_t *j) {
size_t n;
n = (size_t)(_mmi.p[i].y - _mmi.p[i].x + 1) << 16;
while (i + 1 < _mmi.i) {
if (_mmi.p[i + 1].x != _mmi.p[i].y + 1) break;
++i;
n += (size_t)(_mmi.p[i].y - _mmi.p[i].x + 1) << 16;
}
*j = i;
return n;
}
static bool MustMoveMap(intptr_t y, size_t j) {
return ADDR(_mmi.p[j].y) + FRAMESIZE > y ||
(j + 1 < _mmi.i && ADDR(_mmi.p[j + 1].x) < y);
}
/**
* Relocates mapping.
* Extends and/or relocates memory pages.
*
* @param p is old address
* @param n is old size
@ -36,27 +70,134 @@
* @param q is new address
*/
void *mremap(void *p, size_t n, size_t m, int f, ... /* void *q */) {
return VIP(enosys()); /* TODO: Implement Me! */
enosys();
return MAP_FAILED;
void *q;
va_list va;
if (!IsWindows()) {
if (!n) return VIP(einval());
if (!m) return VIP(einval());
if (!ALIGNED(p)) return VIP(einval());
n = ROUNDUP(n, FRAMESIZE);
m = ROUNDUP(m, FRAMESIZE);
if (f & MREMAP_FIXED) {
va_start(va, f);
q = va_arg(va, void *);
va_end(va);
if (!ALIGNED(q)) return VIP(einval());
size_t i, j, k;
struct DirectMap dm;
int a, b, prot, flags;
if (UNLIKELY(!m)) {
SYSDEBUG("mremap(0x%p, 0x%x, 0x%x, 0x%x) EINVAL (m=0)", p, n, m, f);
return VIP(einval());
}
if (UNLIKELY(!n)) {
SYSDEBUG("mremap(0x%p, 0x%x, 0x%x, 0x%x) EOPNOTSUPP (n=0)", p, n, m, f);
return VIP(eopnotsupp());
}
if (UNLIKELY(!ALIGNED(n))) {
SYSDEBUG("mremap(0x%p, 0x%x, 0x%x, 0x%x) EOPNOTSUPP (n align)", p, n, m, f);
return VIP(eopnotsupp());
}
if (UNLIKELY(!ALIGNED(m))) {
SYSDEBUG("mremap(0x%p, 0x%x, 0x%x, 0x%x) EOPNOTSUPP (n align)", p, n, m, f);
return VIP(eopnotsupp());
}
if (UNLIKELY(!ALIGNED(p))) {
SYSDEBUG("mremap(0x%p, 0x%x, 0x%x, 0x%x) EINVAL (64kb align)", p, n, m, f);
return VIP(einval());
}
if (UNLIKELY(!SMALL(n))) {
SYSDEBUG("mremap(0x%p, 0x%x, 0x%x, 0x%x) EINVAL (n too big)", p, n, m, f);
return VIP(enomem());
}
if (UNLIKELY(!SMALL(m))) {
SYSDEBUG("mremap(0x%p, 0x%x, 0x%x, 0x%x) EINVAL (m too big)", p, n, m, f);
return VIP(enomem());
}
if (f & ~(MREMAP_MAYMOVE | MREMAP_FIXED)) {
SYSDEBUG("mremap(0x%p, 0x%x, 0x%x, 0x%x) EINVAL (bad flag)", p, n, m, f);
return VIP(einval());
}
if (!IsMemtracked(FRAME(p), FRAME((intptr_t)p + (n - 1)))) {
SYSDEBUG("munmap(0x%x, 0x%x) EFAULT (interval not tracked)", p, n);
return VIP(efault());
}
SYSDEBUG("mremap(0x%p, 0x%x, 0x%x, 0x%x)", p, n, m, f);
i = FindMemoryInterval(&_mmi, FRAME(p));
if (i >= _mmi.i) return VIP(efault());
flags = _mmi.p[i].flags;
if (!(flags & MAP_ANONYMOUS)) {
return VIP(eopnotsupp()); /* TODO */
}
if (f & MREMAP_FIXED) {
va_start(va, f);
q = va_arg(va, void *);
va_end(va);
if (!ALIGNED(q)) return VIP(einval());
return VIP(eopnotsupp()); /* TODO */
}
prot = _mmi.p[i].prot;
for (k = i + 1; k <= j; ++k) {
prot |= _mmi.p[k].prot;
if (_mmi.p[k].flags != flags) {
return VIP(enomem());
}
}
if (m == n) {
return p;
} else if (m < n) {
if (munmap((char *)p + n, m - n) != -1) {
return p;
} else {
q = NULL;
if (!(f & MREMAP_MAYMOVE)) {
return MAP_FAILED;
}
} else if (!MustMoveMap(j, (intptr_t)p + n)) {
dm = sys_mmap((char *)p + n, m - n, prot, flags | MAP_FIXED, -1, 0);
if (dm.addr == MAP_FAILED) return 0;
if (TrackMemoryInterval(&_mmi, ((uintptr_t)p + n) >> 16,
((uintptr_t)p + m - FRAMESIZE) >> 16, dm.maphandle,
prot, flags) != -1) {
if (weaken(__asan_map_shadow)) {
weaken(__asan_map_shadow)((uintptr_t)dm.addr, m - n);
}
return p;
} else {
abort();
}
} else if (!(f & MREMAP_MAYMOVE)) {
return VIP(enomem());
} else if (IsLinux()) {
a = (uint32_t)(kAutomapStart >> 16);
i = FindMemoryInterval(&_mmi, a);
if (i < _mmi.i) {
for (; i + 1 < _mmi.i; ++i) {
if (((size_t)(_mmi.p[i + 1].x - _mmi.p[i].y - 1) << 16) >= m) {
break;
}
}
if (__builtin_add_overflow(_mmi.p[i].y, 1, &a)) {
return VIP(enomem());
}
}
return VIP(enosys());
if (__builtin_add_overflow(a, (int)((m >> 16) - 1), &b)) {
return VIP(enomem());
}
q = sys_mremap((void *)p, n, m, MREMAP_MAYMOVE | MREMAP_FIXED,
(void *)ADDR(a));
SYSDEBUG("sys_mremap(0x%p, 0x%x, 0x%x, 0x%x, 0x%x) -> 0x%p", p, n, m,
MREMAP_MAYMOVE | MREMAP_FIXED, ADDR(a));
if (q == MAP_FAILED) return 0;
if (ReleaseMemoryIntervals(&_mmi, (uintptr_t)p >> 16,
((uintptr_t)p + n - FRAMESIZE) >> 16, 0) != -1 &&
TrackMemoryInterval(&_mmi, a, b, -1, prot, flags) != -1) {
if (weaken(__asan_poison)) {
if (!OverlapsShadowSpace(p, n)) {
weaken(__asan_poison)((intptr_t)p, n, kAsanUnmapped);
}
if (!OverlapsShadowSpace(q, m)) {
weaken(__asan_map_shadow)((intptr_t)q, m);
}
}
return (void *)ADDR(a);
} else {
abort();
}
} else if ((q = mmap(0, m, prot, flags, -1, 0)) != MAP_FAILED) {
memcpy(q, p, n);
munmap(p, n);
return q;
} else {
return VIP(enosys());
return q;
}
}

2
libc/runtime/munmap-metal.c
View file

@ -19,7 +19,7 @@
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/pc.internal.h"
int sys_munmap_metal(void *addr, size_t size) {
noasan int sys_munmap_metal(void *addr, size_t size) {
size_t i;
uint64_t *e;
struct mman *mm;

33
libc/runtime/g_argc.S → libc/runtime/munmap-sysv.c
View file

@ -1,7 +1,7 @@
/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│
│vi: set et ft=asm ts=8 sw=8 fenc=utf-8 :vi│
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
@ -16,16 +16,19 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macros.internal.h"
#include "libc/notice.inc"
#include "libc/calls/internal.h"
#include "libc/calls/sysdebug.internal.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/memtrack.internal.h"
.initbss 300,_init_argc
// Global variable holding _start(argc) parameter.
__argc: .quad 0
.endobj __argc,globl
.previous
.init.start 300,_init_argc
mov %r12,%rax
stosq
.init.end 300,_init_argc
int sys_munmap(void *p, size_t n) {
int rc;
if (!IsMetal()) {
rc = __sys_munmap(p, n);
} else {
rc = sys_munmap_metal(p, n);
}
SYSDEBUG("sys_munmap(0x%p%s, 0x%x) -> %d", p,
DescribeFrame((intptr_t)p >> 16), n, (long)rc);
return rc;
}

90
libc/runtime/munmap.c
View file

@ -16,42 +16,100 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/bits/likely.h"
#include "libc/calls/internal.h"
#include "libc/calls/sysdebug.internal.h"
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/log/libfatal.internal.h"
#include "libc/macros.internal.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
#include "libc/sysv/errfuns.h"
#define IP(X) (intptr_t)(X)
#define ALIGNED(p) (!(IP(p) & (FRAMESIZE - 1)))
#define CANONICAL(p) (-0x800000000000 <= IP(p) && IP(p) <= 0x7fffffffffff)
#define IP(X) (intptr_t)(X)
#define SMALL(n) ((n) <= 0xffffffffffff)
#define ALIGNED(p) (!(IP(p) & (FRAMESIZE - 1)))
#define LEGAL(p) (-0x800000000000 <= IP(p) && IP(p) <= 0x7fffffffffff)
#define ADDR(x) ((int64_t)((uint64_t)(x) << 32) >> 16)
#define SHADE(x) (((intptr_t)(x) >> 3) + 0x7fff8000)
#define FRAME(x) ((int)((intptr_t)(x) >> 16))
/**
* Releases memory pages.
*
* This function may be used to punch holes in existing mappings, but your
* mileage may vary on Windows.
* This function may be used to punch holes in existing mappings, but
* your mileage may vary on Windows.
*
* @param addr is a pointer within any memory mapped region the process
* @param p is a pointer within any memory mapped region the process
* has permission to control, such as address ranges returned by
* mmap(), the program image itself, etc.
* @param size is the amount of memory to unmap, which needs to be a
* @param n is the number of bytes to be unmapped, and needs to be a
* multiple of FRAMESIZE for anonymous mappings, because windows
* and for files size needs to be perfect to the byte bc openbsd
* @return 0 on success, or -1 w/ errno
*/
noasan int munmap(void *addr, size_t size) {
noasan int munmap(void *v, size_t n) {
/* asan runtime depends on this function */
int rc;
/* TODO(jart): are we unmapping shadows? */
SYSDEBUG("munmap(0x%x, 0x%x)", addr, size);
if (!ALIGNED(addr) || !CANONICAL(addr) || !size) return einval();
if (UntrackMemoryIntervals(addr, size) == -1) return -1;
if (IsWindows()) return 0;
if (IsMetal()) sys_munmap_metal(addr, size);
SYSDEBUG("sys_munmap(0x%x, 0x%x)", addr, size);
return sys_munmap(addr, size);
char poison, *p = v;
intptr_t a, b, x, y;
if (UNLIKELY(!n)) {
SYSDEBUG("munmap(0x%p, 0x%x) %s (n=0)", p, n);
return einval();
}
if (UNLIKELY(!SMALL(n))) {
SYSDEBUG("munmap(0x%p, 0x%x) EINVAL (n isn't 48-bit)", p, n);
return einval();
}
if (UNLIKELY(!LEGAL(p))) {
SYSDEBUG("munmap(0x%p, 0x%x) EINVAL (p isn't 48-bit)", p, n);
return einval();
}
if (UNLIKELY(!LEGAL(p + (n - 1)))) {
SYSDEBUG("munmap(0x%p, 0x%x) EINVAL (p+(n-1) isn't 48-bit)", p, n);
return einval();
}
if (UNLIKELY(!ALIGNED(p))) {
SYSDEBUG("munmap(0x%p, 0x%x) EINVAL (p isn't 64kb aligned)", p, n);
return einval();
}
if (!IsMemtracked(FRAME(p), FRAME(p + (n - 1)))) {
SYSDEBUG("munmap(0x%p, 0x%x) EFAULT (interval not tracked)", p, n);
return efault();
}
SYSDEBUG("munmap(0x%p, 0x%x)", p, n);
if (UntrackMemoryIntervals(p, n) != -1) {
if (!IsWindows()) {
rc = sys_munmap(p, n);
if (rc != -1) {
if (IsAsan() && !OverlapsShadowSpace(p, n)) {
a = SHADE(p);
b = a + (n >> 3);
if (IsMemtracked(FRAME(a), FRAME(b - 1))) {
x = ROUNDUP(a, FRAMESIZE);
y = ROUNDDOWN(b, FRAMESIZE);
if (x < y) {
/* delete shadowspace if unmapping ≥512kb */
__repstosb((void *)a, kAsanUnmapped, x - a);
munmap((void *)x, y - x);
__repstosb((void *)y, kAsanUnmapped, b - y);
} else {
/* otherwise just poison and assume reuse */
__repstosb((void *)a, kAsanUnmapped, b - a);
}
} else {
SYSDEBUG("unshadow(0x%x, 0x%x) EFAULT", a, b - a);
}
}
}
return rc;
} else {
return 0; /* UntrackMemoryIntervals does it for NT */
}
} else {
return -1;
}
}

76
libc/runtime/opensymboltable.c
View file

@ -20,11 +20,16 @@
#include "libc/assert.h"
#include "libc/bits/bits.h"
#include "libc/calls/calls.h"
#include "libc/calls/sysdebug.internal.h"
#include "libc/dce.h"
#include "libc/elf/def.h"
#include "libc/elf/elf.h"
#include "libc/elf/scalar.h"
#include "libc/elf/struct/phdr.h"
#include "libc/elf/struct/shdr.h"
#include "libc/elf/struct/sym.h"
#include "libc/errno.h"
#include "libc/limits.h"
#include "libc/log/libfatal.internal.h"
#include "libc/macros.internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
@ -35,12 +40,72 @@
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
#define GetStr(tab, rva) ((char *)(tab) + (rva))
#define GetSection(e, s) ((void *)((intptr_t)(e) + (size_t)(s)->sh_offset))
#define GetShstrtab(e) GetSection(e, GetShdr(e, (e)->e_shstrndx))
#define GetSectionName(e, s) GetStr(GetShstrtab(e), (s)->sh_name)
#define GetPhdr(e, i) \
((Elf64_Phdr *)((intptr_t)(e) + (e)->e_phoff + \
(size_t)(e)->e_phentsize * (i)))
#define GetShdr(e, i) \
((Elf64_Shdr *)((intptr_t)(e) + (e)->e_shoff + \
(size_t)(e)->e_shentsize * (i)))
noasan static char *GetStrtab(Elf64_Ehdr *e, size_t *n) {
char *name;
Elf64_Half i;
Elf64_Shdr *shdr;
for (i = 0; i < e->e_shnum; ++i) {
shdr = GetShdr(e, i);
if (shdr->sh_type == SHT_STRTAB) {
name = GetSectionName(e, GetShdr(e, i));
if (name && !__strcmp(name, ".strtab")) {
if (n) *n = shdr->sh_size;
return GetSection(e, shdr);
}
}
}
return 0;
}
noasan static Elf64_Sym *GetSymtab(Elf64_Ehdr *e, Elf64_Xword *n) {
Elf64_Half i;
Elf64_Shdr *shdr;
for (i = e->e_shnum; i > 0; --i) {
shdr = GetShdr(e, i - 1);
if (shdr->sh_type == SHT_SYMTAB) {
if (shdr->sh_entsize != sizeof(Elf64_Sym)) continue;
if (n) *n = shdr->sh_size / shdr->sh_entsize;
return GetSection(e, shdr);
}
}
return 0;
}
noasan static void GetImageRange(Elf64_Ehdr *elf, intptr_t *x, intptr_t *y) {
unsigned i;
Elf64_Phdr *phdr;
intptr_t start, end, pstart, pend;
start = INTPTR_MAX;
end = 0;
for (i = 0; i < elf->e_phnum; ++i) {
phdr = GetPhdr(elf, i);
if (phdr->p_type != PT_LOAD) continue;
pstart = phdr->p_vaddr;
pend = phdr->p_vaddr + phdr->p_memsz;
if (pstart < start) start = pstart;
if (pend > end) end = pend;
}
if (x) *x = start;
if (y) *y = end;
}
/**
* Maps debuggable binary into memory and indexes symbol addresses.
*
* @return object freeable with CloseSymbolTable(), or NULL w/ errno
*/
struct SymbolTable *OpenSymbolTable(const char *filename) {
noasan struct SymbolTable *OpenSymbolTable(const char *filename) {
int fd;
void *map;
long *stp;
@ -60,8 +125,8 @@ struct SymbolTable *OpenSymbolTable(const char *filename) {
elf = map = mmap(0, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
if (map == MAP_FAILED) goto SystemError;
if (READ32LE(map) != READ32LE("\177ELF")) goto RaiseEnoexec;
if (!(name_base = GetElfStrs(map, st.st_size, &m))) goto RaiseEnobufs;
if (!(symtab = GetElfSymbolTable(map, st.st_size, &n))) goto RaiseEnobufs;
if (!(name_base = GetStrtab(map, &m))) goto RaiseEnobufs;
if (!(symtab = GetSymtab(map, &n))) goto RaiseEnobufs;
tsz = 0;
tsz += sizeof(struct SymbolTable);
tsz += sizeof(struct Symbol) * n;
@ -78,7 +143,7 @@ struct SymbolTable *OpenSymbolTable(const char *filename) {
t->mapsize = tsz;
t->names = (unsigned *)((char *)t + names_offset);
t->name_base = (char *)((char *)t + name_base_offset);
GetElfVirtualAddressRange(elf, st.st_size, &t->addr_base, &t->addr_end);
GetImageRange(elf, &t->addr_base, &t->addr_end);
memcpy(t->name_base, name_base, m);
--t->addr_end;
stp = (long *)((char *)t + stp_offset);
@ -122,6 +187,7 @@ RaiseEnobufs:
RaiseEnoexec:
errno = ENOEXEC;
SystemError:
SYSDEBUG("OpenSymbolTable() %s", strerror(errno));
if (map != MAP_FAILED) {
munmap(map, st.st_size);
}

50
libc/runtime/printmemoryintervals.c
View file

@ -16,27 +16,45 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/log/log.h"
#include "libc/fmt/itoa.h"
#include "libc/log/libfatal.internal.h"
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
#define ADDR(x) ((intptr_t)((int64_t)((uint64_t)(x) << 32) >> 16))
static bool IsNoteworthyHole(unsigned i, const struct MemoryIntervals *mm) {
// gaps between shadow frames aren't interesting
// the chasm from heap to stack ruins statistics
return !((IsArenaFrame(mm->p[i].y) && !IsArenaFrame(mm->p[i + 1].x)) ||
(IsShadowFrame(mm->p[i].y) || IsShadowFrame(mm->p[i + 1].x)) ||
(!IsStackFrame(mm->p[i].y) && IsStackFrame(mm->p[i + 1].x)));
}
void PrintMemoryIntervals(int fd, const struct MemoryIntervals *mm) {
int i, frames, maptally, gaptally;
maptally = 0;
gaptally = 0;
(dprintf)(fd, "%s%zd%s\r\n", "mm->i == ", mm->i, ";");
char *p, mode[8];
long i, w, frames, maptally = 0, gaptally = 0;
for (w = i = 0; i < mm->i; ++i) {
w = MAX(w, LengthInt64Thousands(mm->p[i].y + 1 - mm->p[i].x));
}
for (i = 0; i < mm->i; ++i) {
if (i && mm->p[i].x != mm->p[i - 1].y + 1) {
frames = mm->p[i].x - mm->p[i - 1].y - 1;
gaptally += frames;
(dprintf)(fd, "%s%,zd%s\r\n", "/* ", frames, " */");
}
frames = mm->p[i].y + 1 - mm->p[i].x;
maptally += frames;
(dprintf)(fd, "%s%3u%s0x%08x,0x%08x,%ld,%d,%d%s%,zd%s\r\n", "mm->p[", i,
"]=={", mm->p[i].x, mm->p[i].y, mm->p[i].h, mm->p[i].prot,
mm->p[i].flags, "}; /* ", frames, " */");
__printf("%0*x-%0*x %s %,*dx%s", 12, ADDR(mm->p[i].x), 12,
ADDR(mm->p[i].y + 1),
DescribeMapping(mm->p[i].prot, mm->p[i].flags, mode), w, frames,
DescribeFrame(mm->p[i].x));
if (i + 1 < _mmi.i) {
frames = mm->p[i + 1].x - mm->p[i].y - 1;
if (frames && IsNoteworthyHole(i, mm)) {
gaptally += frames;
__printf(" w/ %,d frame hole", frames);
}
}
if (mm->p[i].h != -1) {
__printf(" h=%d", mm->p[i].h);
}
__printf("\r\n");
}
(dprintf)(fd, "%s%,zd%s%,zd%s\r\n\r\n", "/* ", maptally, " frames mapped w/ ",
gaptally, " frames gapped */");
__printf("# %d frames mapped w/ %,d frames gapped\r\n", maptally, gaptally);
}

66
libc/runtime/program_executable_name.c
View file

@ -20,7 +20,9 @@
#include "libc/bits/bits.h"
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
#include "libc/calls/sysdebug.internal.h"
#include "libc/dce.h"
#include "libc/log/libfatal.internal.h"
#include "libc/macros.internal.h"
#include "libc/mem/alloca.h"
#include "libc/nt/runtime.h"
@ -32,6 +34,7 @@
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/ok.h"
#define SIZE 1024
#define CTL_KERN 1
#define KERN_PROC 14
#define KERN_PROC_PATHNAME_FREEBSD 12
@ -44,9 +47,9 @@
* guaranteed to exist, except on XNU and OpenBSD. It may be a symlink.
* It may be spoofed.
*/
char program_executable_name[1024];
char program_executable_name[SIZE];
static textwindows bool GetNtExePath(void) {
static textwindows bool GetNtExePath(char executable[SIZE]) {
uint64_t w;
wint_t x, y;
uint32_t i, j;
@ -60,44 +63,38 @@ static textwindows bool GetNtExePath(void) {
if (x == '\\') x = '/';
w = tpenc(x);
do {
program_executable_name[j] = w;
if (++j == sizeof(program_executable_name)) {
executable[j] = w;
if (++j == SIZE) {
return false;
}
} while ((w >>= 8));
}
program_executable_name[j] = 0;
executable[j] = 0;
return true;
}
textstartup void program_executable_name_init(int argc, char **argv,
char **envp, intptr_t *auxv) {
static textstartup void GetProgramExecutableName(char executable[SIZE],
char *p) {
char *t;
size_t m;
ssize_t n;
int cmd[4];
char *p, *t;
static bool once;
if (!cmpxchg(&once, 0, 1)) return;
if (IsWindows() && GetNtExePath()) return;
if (IsWindows() && GetNtExePath(executable)) return;
n = 0;
p = argv[0];
if (fileexists(p)) {
if (!_isabspath(p)) {
if (getcwd(program_executable_name,
sizeof(program_executable_name) - 1)) {
n = strlen(program_executable_name);
program_executable_name[n++] = '/';
if (getcwd(executable, SIZE - 1)) {
n = strlen(executable);
executable[n++] = '/';
}
}
} else if ((n = sys_readlinkat(AT_FDCWD, "/proc/self/exe",
program_executable_name,
sizeof(program_executable_name) - 1)) > 0) {
program_executable_name[n] = 0;
} else if ((n = sys_readlinkat(AT_FDCWD, "/proc/self/exe", executable,
SIZE - 1)) > 0) {
executable[n] = 0;
return;
} else if ((n = sys_readlinkat(AT_FDCWD, "/proc/curproc/file",
program_executable_name,
sizeof(program_executable_name) - 1)) > 0) {
program_executable_name[n] = 0;
} else if ((n = sys_readlinkat(AT_FDCWD, "/proc/curproc/file", executable,
SIZE - 1)) > 0) {
executable[n] = 0;
return;
} else if (IsFreebsd() || IsNetbsd()) {
cmd[0] = CTL_KERN;
@ -108,17 +105,28 @@ textstartup void program_executable_name_init(int argc, char **argv,
cmd[2] = KERN_PROC_PATHNAME_NETBSD;
}
cmd[3] = -1;
m = sizeof(program_executable_name);
if (sysctl(cmd, ARRAYLEN(cmd), program_executable_name, &m, 0, 0) != -1) {
m = SIZE;
if (sysctl(cmd, ARRAYLEN(cmd), executable, &m, 0, 0) != -1) {
return;
}
}
for (; *p; ++p) {
if (n + 1 < sizeof(program_executable_name)) {
program_executable_name[n++] = *p;
if (n + 1 < SIZE) {
executable[n++] = *p;
}
}
program_executable_name[n] = 0;
executable[n] = 0;
}
textstartup void program_executable_name_init(int argc, char **argv,
char **envp, intptr_t *auxv) {
static bool once;
char executable[SIZE];
if (!cmpxchg(&once, 0, 1)) return;
__stpcpy(program_executable_name, argv[0]);
GetProgramExecutableName(executable, argv[0]);
__stpcpy(program_executable_name, executable);
SYSDEBUG("GetProgramExecutableName() -> %s", program_executable_name);
}
const void *const program_executable_name_init_ctor[] initarray = {

48
libc/runtime/exit3.c → libc/runtime/quick_exit.c
View file

@ -16,38 +16,34 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/internal.h"
#include "libc/dce.h"
#include "libc/nexgen32e/vendor.internal.h"
#include "libc/bits/pushpop.h"
#include "libc/bits/weaken.h"
#include "libc/nt/console.h"
#include "libc/nt/enum/consolemodeflags.h"
#include "libc/nt/runtime.h"
#include "libc/nt/thunk/msabi.h"
#include "libc/sysv/consts/nr.h"
extern void(__msabi* __imp_ExitProcess)(uint32_t);
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/stdio.h"
/**
* Terminates process, ignoring destructors and atexit() handlers.
*
* When running on bare metal, this function will reboot your computer
* by hosing the interrupt descriptors and triple faulting the system.
* Exits process faster.
*
* @param exitcode is masked with 255
* @asyncsignalsafe
* @vforksafe
* @noreturn
*/
privileged wontreturn void _Exit(int exitcode) {
if ((!IsWindows() && !IsMetal()) || (IsMetal() && IsGenuineCosmo())) {
asm volatile("syscall"
: /* no outputs */
: "a"(__NR_exit_group), "D"(exitcode)
: "memory");
} else if (IsWindows()) {
__imp_ExitProcess(exitcode & 0xff);
wontreturn void quick_exit(int exitcode) {
const uintptr_t *p;
if (weaken(fflush)) {
weaken(fflush)(0);
}
asm("push\t$0\n\t"
"push\t$0\n\t"
"cli\n\t"
"lidt\t(%rsp)");
for (;;) asm("ud2");
for (p = __fini_array_end; p > __fini_array_start;) {
((void (*)(void))(*--p))();
}
if (SupportsWindows() && __ntconsolemode) {
SetConsoleMode(GetStdHandle(pushpop(kNtStdInputHandle)), __ntconsolemode);
SetConsoleMode(GetStdHandle(pushpop(kNtStdOutputHandle)),
kNtEnableProcessedOutput | kNtEnableWrapAtEolOutput |
kNtEnableVirtualTerminalProcessing);
}
_Exit(exitcode);
}

12
libc/runtime/runtime.h
View file

@ -8,14 +8,15 @@ COSMOPOLITAN_C_START_
typedef long jmp_buf[8] forcealign(CACHELINE);
extern int __argc; /* CRT */
extern char **__argv; /* CRT */
extern char **environ; /* CRT */
extern unsigned long *__auxv; /* CRT */
extern const int __argc; /* CRT */
extern char **const __argv; /* CRT */
extern char **const __envp; /* CRT */
extern unsigned long *const __auxv; /* CRT */
extern char program_executable_name[]; /* RII */
extern char *program_invocation_name; /* RII */
extern char *program_invocation_short_name; /* RII */
extern int ftrace; /* CRT */
extern int g_ftrace; /* CRT */
extern uint64_t g_syscount; /* RII */
extern const uint64_t kStartTsc; /* RII */
extern const char kTmpPath[]; /* RII */
@ -46,6 +47,7 @@ void _longjmp(jmp_buf, int) libcesque wontreturn paramsnonnull();
void exit(int) wontreturn;
void _exit(int) libcesque wontreturn;
void _Exit(int) libcesque wontreturn;
void quick_exit(int) wontreturn;
void abort(void) wontreturn noinstrument;
int __cxa_atexit(void *, void *, void *) libcesque;
int atfork(void *, void *) libcesque;
@ -81,7 +83,7 @@ int acct(const char *);
void longsort(long *, size_t);
bool _isheap(void *);
int NtGetVersion(void);
int NtGetVersion(void) pureconst;
long missingno();
void __oom_hook(size_t);
void __print(const void *, size_t);

79
libc/runtime/stack.h Normal file
View file

@ -0,0 +1,79 @@
#ifndef COSMOPOLITAN_LIBC_RUNTIME_STACK_H_
#define COSMOPOLITAN_LIBC_RUNTIME_STACK_H_
#include "ape/config.h"
#include "libc/dce.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
/**
* Tunes APE stack maximum size.
*
* This defaults to `STACKSIZE`. The bottom-most page will be protected
* to ensure your stack does not magically grow beyond this value. It's
* possible to detect stack overflows, by calling `ShowCrashReports()`.
*
* If you want to know how much stack your programs needs, then
*
* STATIC_YOINK("stack_usage_logging");
*
* will install an atexit() handler that appends to `o/$MODE/stack.log`
*
* @see libc/sysv/systemfive.S
* @see ape/ape.lds
*/
#define STATIC_STACK_SIZE(BYTES) \
STATIC_SYMBOL("ape_stack_memsz", _STACK_STRINGIFY(BYTES) _STACK_EXTRA)
/**
* Tunes APE stack virtual address.
*
* This defaults to `0x700000000000 - STACKSIZE`. The value defined by
* this macro will be respected, with two exceptions: (1) in MODE=tiny
* the operating system provided stack is used instead and (2) Windows
* Seven doesn't support 64-bit addresses so 0x10000000 - GetStackSize
* is used instead.
*
* @see libc/sysv/systemfive.S
* @see libc/nt/winmain.greg.c
* @see ape/ape.lds
*/
#define STATIC_STACK_ADDR(ADDR) \
STATIC_SYMBOL("ape_stack_vaddr", _STACK_STRINGIFY(ADDR))
#define _STACK_STRINGIFY(ADDR) #ADDR
#if IsAsan()
#define _STACK_EXTRA "*2"
#else
#define _STACK_EXTRA ""
#endif
#if defined(__GNUC__) && defined(__ELF__)
COSMOPOLITAN_C_START_
extern char ape_stack_memsz[] __attribute__((__weak__));
#define GetStackSize() ((uintptr_t)ape_stack_memsz)
/**
* Returns address of bottom of stack.
*/
#define GetStackAddr(ADDEND) \
(((intptr_t)__builtin_frame_address(0) & -GetStackSize()) + (ADDEND))
/**
* Returns preferred bottom address of stack.
*/
#define GetStaticStackAddr(ADDEND) \
({ \
intptr_t vAddr = 0; \
asm(".weak\tape_stack_vaddr\n\t" \
"movabs\t%1+ape_stack_vaddr,%0" \
: "=r"(vAddr) \
: "i"(ADDEND)); \
vAddr; \
})
COSMOPOLITAN_C_END_
#endif /* GNU ELF */
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_RUNTIME_STACK_H_ */

80
libc/runtime/stackuse.c Normal file
View file

@ -0,0 +1,80 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/fmt/itoa.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/o.h"
static char stacklog[1024];
static size_t NullLength(const char *s) {
typedef char xmm_t __attribute__((__vector_size__(16), __aligned__(16)));
size_t n;
xmm_t v, z = {0};
unsigned m, k = (uintptr_t)s & 15;
const xmm_t *p = (const xmm_t *)((uintptr_t)s & -16);
m = (__builtin_ia32_pmovmskb128(*p == z) ^ 0xffff) >> k << k;
while (!m) m = __builtin_ia32_pmovmskb128(*++p == z) ^ 0xffff;
n = (const char *)p + __builtin_ctzl(m) - s;
return n;
}
static textexit void LogStackUse(void) {
int i, fd;
bool quote;
char *p, *q;
size_t n, usage;
const char *stack;
fd = open(stacklog, O_APPEND | O_CREAT | O_WRONLY, 0644);
stack = (char *)GetStackAddr(0);
usage = GetStackSize() - (NullLength(stack + PAGESIZE) + PAGESIZE);
p = FormatUint64(stacklog, usage);
for (i = 0; i < __argc; ++i) {
n = strlen(__argv[i]);
if ((q = memchr(__argv[i], '\n', n))) n = q - __argv[i];
if (p - stacklog + 1 + 1 + n + 1 + 1 < sizeof(stacklog)) {
quote = !!memchr(__argv[i], ' ', n);
*p++ = ' ';
if (quote) *p++ = '\'';
p = mempcpy(p, __argv[i], n);
if (quote) *p++ = '\'';
} else {
break;
}
}
*p++ = '\n';
write(fd, stacklog, p - stacklog);
close(fd);
}
static textstartup void LogStackUseInit(void) {
if (IsTiny()) return;
if (isdirectory("o/" MODE) &&
getcwd(stacklog, sizeof(stacklog) - strlen("/o/" MODE "/stack.log"))) {
strcat(stacklog, "/o/" MODE "/stack.log");
atexit(LogStackUse);
}
}
const void *const stack_usage_logging[] initarray = {
LogStackUseInit,
};

34
libc/runtime/symbols.internal.h
View file

@ -1,8 +1,5 @@
#ifndef COSMOPOLITAN_LIBC_SYMBOLS_H_
#define COSMOPOLITAN_LIBC_SYMBOLS_H_
#include "libc/assert.h"
#include "libc/elf/elf.h"
#include "libc/runtime/ezmap.internal.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
@ -27,35 +24,8 @@ const char *FindDebugBinary(void);
struct SymbolTable *OpenSymbolTable(const char *);
int CloseSymbolTable(struct SymbolTable **);
int __hook(void *, struct SymbolTable *);
forceinline int GetSymbol(struct SymbolTable *t, intptr_t a) {
unsigned l, m, r, n, k;
if (t) {
l = 0;
r = n = t->count;
k = a - t->addr_base;
while (l < r) {
m = (l + r) >> 1;
if (t->symbols[m].y < k) {
l = m + 1;
} else {
r = m;
}
}
if (l < n && t->symbols[l].x <= k && k <= t->symbols[l].y) {
return l;
}
}
return -1;
}
forceinline char *GetSymbolName(struct SymbolTable *t, int s) {
if (t && s != -1) {
return t->name_base + t->names[s];
} else {
return 0;
}
}
int __get_symbol(struct SymbolTable *, intptr_t);
char *__get_symbol_name(struct SymbolTable *, int);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

30
libc/runtime/winmain.greg.c
View file

@ -89,12 +89,12 @@ static noasan textwindows noinstrument void MakeLongDoubleLongAgain(void) {
static noasan textwindows wontreturn noinstrument void WinMainNew(void) {
int64_t h;
int version;
size_t size;
int i, count;
uint64_t addr;
int64_t inhand;
struct WinArgs *wa;
const char16_t *env16;
intptr_t stackaddr, allocaddr;
size_t allocsize, argsize, stacksize;
extern char os asm("__hostos");
os = WINDOWS; /* madness https://news.ycombinator.com/item?id=21019722 */
version = NtGetPeb()->OSMajorVersion;
@ -115,19 +115,23 @@ static noasan textwindows wontreturn noinstrument void WinMainNew(void) {
}
_mmi.p = _mmi.s;
_mmi.n = OPEN_MAX;
addr = version < 10 ? 0x10000000 - STACKSIZE : 0x777000000000;
size = ROUNDUP(STACKSIZE + sizeof(struct WinArgs), FRAMESIZE);
MapViewOfFileExNuma((_mmi.p[0].h = CreateFileMappingNuma(
-1, &kNtIsInheritable, kNtPageExecuteReadwrite,
size >> 32, size, NULL, kNtNumaNoPreferredNode)),
kNtFileMapWrite | kNtFileMapExecute, 0, 0, size,
(void *)addr, kNtNumaNoPreferredNode);
_mmi.p[0].x = addr >> 16;
_mmi.p[0].y = (addr >> 16) + ((size >> 16) - 1);
argsize = ROUNDUP(sizeof(struct WinArgs), FRAMESIZE);
stackaddr = version < 10 ? 0x10000000 : GetStaticStackAddr(0);
stacksize = GetStackSize();
allocsize = argsize + stacksize;
allocaddr = stackaddr - argsize;
MapViewOfFileExNuma(
(_mmi.p[0].h = CreateFileMappingNuma(
-1, &kNtIsInheritable, kNtPageExecuteReadwrite, allocsize >> 32,
allocsize, NULL, kNtNumaNoPreferredNode)),
kNtFileMapWrite | kNtFileMapExecute, 0, 0, allocsize, (void *)allocaddr,
kNtNumaNoPreferredNode);
_mmi.p[0].x = allocaddr >> 16;
_mmi.p[0].y = (allocaddr >> 16) + ((allocsize >> 16) - 1);
_mmi.p[0].prot = PROT_READ | PROT_WRITE | PROT_EXEC;
_mmi.p[0].flags = MAP_PRIVATE | MAP_ANONYMOUS;
_mmi.i = 1;
wa = (struct WinArgs *)(addr + size - sizeof(struct WinArgs));
wa = (struct WinArgs *)allocaddr;
count = GetDosArgv(GetCommandLine(), wa->argblock, ARRAYLEN(wa->argblock),
wa->argv, ARRAYLEN(wa->argv));
for (i = 0; wa->argv[0][i]; ++i) {
@ -141,7 +145,7 @@ static noasan textwindows wontreturn noinstrument void WinMainNew(void) {
FreeEnvironmentStrings(env16);
wa->auxv[0][0] = pushpop(AT_EXECFN);
wa->auxv[0][1] = (intptr_t)wa->argv[0];
_jmpstack((char *)addr + STACKSIZE, cosmo, count, wa->argv, wa->envp,
_jmpstack((char *)stackaddr + stacksize, cosmo, count, wa->argv, wa->envp,
wa->auxv);
}