mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-07-01 08:48:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add x86_64-linux-gnu emulator

Browse source 
I wanted a tiny scriptable meltdown proof way to run userspace programs
and visualize how program execution impacts memory. It helps to explain
how things like Actually Portable Executable works. It can show you how
the GCC generated code is going about manipulating matrices and more. I
didn't feel fully comfortable with Qemu and Bochs because I'm not smart
enough to understand them. I wanted something like gVisor but with much
stronger levels of assurances. I wanted a single binary that'll run, on
all major operating systems with an embedded GPL barrier ZIP filesystem
that is tiny enough to transpile to JavaScript and run in browsers too.

https://justine.storage.googleapis.com/emulator625.mp4
This commit is contained in:
Justine Tunney 2020-08-25 04:23:25 -07:00
parent 467504308a
commit f4f4caab0e
1052 changed files with 65667 additions and 7825 deletions
Download patch file Download diff file Expand all files Collapse all files

36
libc/runtime/arememoryintervalsok.c Normal file
View file

@ -0,0 +1,36 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ This program is free software; you can redistribute it and/or modify │
│ it under the terms of the GNU General Public License as published by │
│ the Free Software Foundation; version 2 of the License. │
│ │
│ This program is distributed in the hope that it will be useful, but │
│ WITHOUT ANY WARRANTY; without even the implied warranty of │
│ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU │
│ General Public License for more details. │
│ │
│ You should have received a copy of the GNU General Public License │
│ along with this program; if not, write to the Free Software │
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/memtrack.h"
#include "libc/runtime/runtime.h"
bool AreMemoryIntervalsOk(const struct MemoryIntervals *mm) {
int i;
for (i = 0; i < mm->i; ++i) {
if (mm->p[i].y < mm->p[i].x) return false;
if (i) {
if (mm->h[i] || mm->h[i - 1]) {
if (mm->p[i].x <= mm->p[i - 1].y) return false;
} else {
if (mm->p[i].x <= mm->p[i - 1].y + 1) return false;
}
}
}
return true;
}

54
libc/runtime/asan.greg.c
View file

@ -19,31 +19,57 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/stdio/stdio.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/fileno.h"
struct SourceLocation {
const char *filename;
int line;
int column;
};
struct AccessInfo {
const uint8_t *addr;
const uint8_t *first_bad_addr;
size_t size;
bool iswrite;
unsigned long ip;
};
struct Global {
const uint8_t *addr;
size_t size;
size_t size_with_redzone;
const void *name;
const void *module_name;
unsigned long has_cxx_init;
struct kasan_source_location *location;
char *odr_indicator;
};
privileged void __asan_init(void) {
}
privileged void __asan_register_globals(uintptr_t a, uintptr_t b) {
privileged void __asan_version_mismatch_check_v8(void) {
}
privileged void __asan_unregister_globals(uintptr_t a, uintptr_t b) {
privileged void __asan_register_globals(struct Global globals[], int n) {
}
privileged void __asan_unregister_globals(struct Global globals[], int n) {
}
privileged void __asan_report_load_n(uint8_t *p, int n) {
}
privileged void __asan_report_store_n(uint8_t *p, int n) {
__asan_report_load_n(p, n);
}
privileged void __asan_loadN(uintptr_t ptr, size_t size) {
dprintf(STDERR_FILENO, "load %p %zu");
}
privileged void __asan_storeN(uintptr_t ptr, size_t size) {
dprintf(STDERR_FILENO, "store %p %zu");
}
privileged void __asan_report_load_n(uintptr_t ptr, size_t size) {
dprintf(STDERR_FILENO, "%s%zu%s%#p", "asan: load ", size, " bytes at ", ptr);
}
privileged void __asan_report_store_n(uintptr_t ptr, size_t size) {
dprintf(STDERR_FILENO, "%s%zu%s%#p", "asan: store ", size, " bytes at ", ptr);
}
privileged uintptr_t __asan_stack_malloc(size_t size, int classid) {
@ -54,9 +80,7 @@ privileged void __asan_stack_free(uintptr_t ptr, size_t size, int classid) {
}
privileged void __asan_handle_no_return(void) {
}
privileged void __asan_version_mismatch_check_v8(void) {
DebugBreak();
}
privileged void __asan_alloca_poison(uintptr_t addr, uintptr_t size) {

2
libc/runtime/assertfail.c
View file

@ -18,7 +18,7 @@
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/bits/bits.h"
#include "libc/bits/weaken.h"
#include "libc/calls/calls.h"
#include "libc/log/log.h"
#include "libc/macros.h"

16
libc/runtime/mappings.c → libc/runtime/atexit.c
View file

@ -17,6 +17,18 @@
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/mappings.h"
#include "libc/runtime/runtime.h"
struct Mappings _mm;
/**
* Adds global destructor.
*
* Destructors are called in reverse order. They won't be called
* if the program aborts or _exit() is called. Invocations of this
* function are usually generated by the C++ compiler.
*
* @param rdi callback typed void(*)(T) (nullable)
* @return 0 on success or nonzero if out of space
*/
int atexit(void f(void)) {
return __cxa_atexit(f, NULL, NULL);
}

91
libc/runtime/balloc.c
View file

@ -1,91 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ This program is free software; you can redistribute it and/or modify │
│ it under the terms of the GNU General Public License as published by │
│ the Free Software Foundation; version 2 of the License. │
│ │
│ This program is distributed in the hope that it will be useful, but │
│ WITHOUT ANY WARRANTY; without even the implied warranty of │
│ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU │
│ General Public License for more details. │
│ │
│ You should have received a copy of the GNU General Public License │
│ along with this program; if not, write to the Free Software │
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/bits/bits.h"
#include "libc/bits/popcnt.h"
#include "libc/bits/safemacros.h"
#include "libc/calls/calls.h"
#include "libc/runtime/buffer.h"
#include "libc/runtime/mappings.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/prot.h"
#define kGuard PAGESIZE
#define kGrain FRAMESIZE
/**
* Allocates page-guarded buffer.
*
* ┌─────────────────────────────────────┬s─i─g─s─e─g─v┐
* │ 𝑣₀..𝑣ₙ₋₁ │𝑣ₙ..𝑣ₙ₊₄₀₉₆₋₁│
* └─────────────────────────────────────┴s─i─g─s─e─g─v┘
*
* @param b is metadata object owned by caller, initialized to zero for
* first call; subsequent calls will resize
* @param a is alignment requirement in bytes, e.g. 1,2,4,8,16,...
* @param n is buffer size in bytes
* @return b->p
* @see ralloc()
*/
void *balloc(struct GuardedBuffer *b, unsigned a, size_t n) {
int rc;
char *p;
size_t mi, need;
struct AddrSize az;
struct MemoryCoord mc;
assert(a >= 1);
assert(a <= kGuard);
assert(kGuard < kGrain);
assert(popcnt(a) == 1);
assert(popcnt(kGuard) == 1);
assert(popcnt(kGrain) == 1);
assert(n < 0x800000000000ul - kGrain - kGuard);
if (n) {
az.addr = 0;
az.size = 0;
n = roundup(n, a);
need = roundup(n + kGuard, kGrain);
if (b->p) {
mc = ADDRSIZE_TO_COORD(b->p, 1);
mi = findmapping(mc.y);
if (mi && ISOVERLAPPING(mc, _mm.p[mi - 1])) {
az = COORD_TO_ADDRSIZE(_mm.p[mi - 1]);
if (az.size < need) {
rc = munmap(az.addr, az.size);
assert(rc != -1);
az.addr = NULL;
az.size = 0;
}
}
}
if (!az.size) {
if ((p = mapanon(need)) == MAP_FAILED) abort();
if (mprotect(p + need - kGuard, kGuard, PROT_NONE) == -1) abort();
} else {
p = az.addr;
}
b->p = p + need - kGuard - n;
}
return b->p;
}

11
libc/runtime/carsort.h Normal file
View file

@ -0,0 +1,11 @@
#ifndef COSMOPOLITAN_LIBC_RUNTIME_CARSORT_H_
#define COSMOPOLITAN_LIBC_RUNTIME_CARSORT_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
void carsort100(size_t, int32_t (*)[2]) paramsnonnull() nocallback nothrow;
void carsort1000(size_t, int32_t (*)[2]) paramsnonnull() nocallback nothrow;
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_RUNTIME_CARSORT_H_ */

44
libc/runtime/findmapping.c → libc/runtime/carsort100.c
View file

@ -17,33 +17,25 @@
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/mappings.h"
#include "libc/runtime/carsort.h"
#include "libc/str/str.h"
size_t findmapping_(int32_t k, const struct MemoryCoord *coords, size_t count) {
size_t l, r, m;
l = 0;
r = count;
while (l < r) {
m = (l + r) >> 1;
if (coords[m].x > k) {
r = m;
} else {
l = m + 1;
/**
* Sorts int32 key-value arrays of trivial size.
*
* @see test/libc/alg/carsort_test.c
* @see carsort1000() if larger
*/
textstartup void carsort100(size_t n, int32_t a[n][2]) {
int32_t t[2];
unsigned i, j;
for (i = 1; i < n; ++i) {
j = i;
memcpy(t, a[i], sizeof(t));
while (j > 0 && t[0] < a[j - 1][0]) {
memcpy(a[j], a[j - 1], sizeof(t));
--j;
}
memcpy(a[j], t, sizeof(t));
}
return l;
}
/**
* Returns appropriate rightmost index.
*/
size_t findmapping(int32_t k) {
return findmapping_(k, _mm.p, _mm.i);
}
/**
* Returns appropriate rightmost index.
*/
size_t pickmapping(int32_t k) {
return findmapping_(k, _mm.p, _mm.i);
}

37
libc/runtime/heapsortcar.c → libc/runtime/carsort1000.c
View file

@ -17,22 +17,21 @@
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/alg/alg.h"
#include "libc/runtime/carsort.h"
#include "libc/str/str.h"
static unsigned heapsortmax(int32_t (*A)[2], unsigned n, unsigned i, unsigned j,
unsigned k) {
unsigned m = i;
if (j < n && A[j][0] > A[m][0]) m = j;
if (k < n && A[k][0] > A[m][0]) m = k;
return m;
static textstartup size_t HeapSortMax(size_t n, int32_t A[n][2], long i, long j,
long k) {
if (j < n && A[j][0] > A[i][0]) i = j;
if (k < n && A[k][0] > A[i][0]) i = k;
return i;
}
static void heapsortdown(int32_t (*A)[2], unsigned n, unsigned i) {
unsigned j;
static textstartup void HeapSortDown(size_t n, int32_t A[n][2], long i) {
size_t j;
int32_t t[2];
for (;;) {
unsigned j = heapsortmax(A, n, i, 2 * i + 1, 2 * i + 2);
j = HeapSortMax(n, A, i, 2 * i + 1, 2 * i + 2);
if (j == i) break;
memcpy(t, A[i], sizeof(t));
memcpy(A[i], A[j], sizeof(t));
@ -42,25 +41,21 @@ static void heapsortdown(int32_t (*A)[2], unsigned n, unsigned i) {
}
/**
* Sorts key-values.
* Sorts int32 key-value arrays of nontrivial size.
*
* heapsortcar is a linearithmic / constant-memory / non-stable sorting
* function that's a tenth the size of the more generalized qsort() API.
* It can only sort arrays of 64-bit values, and comparisons happen by
* treating the lower 32-bits as a signed integer.
*
* @see test/libc/alg/heapsortcar_test.c
* @see test/libc/alg/carsort_test.c
* @see carsort100() if smaller
*/
void heapsortcar(int32_t (*A)[2], unsigned n) {
unsigned i;
textstartup void carsort1000(size_t n, int32_t A[n][2]) {
size_t i;
int32_t t[2];
for (i = ((n - 2) / 2) + 1; i > 0; i--) {
heapsortdown(A, n, i - 1);
HeapSortDown(n, A, i - 1);
}
for (i = 0; i < n; i++) {
memcpy(t, A[n - i - 1], sizeof(t));
memcpy(A[n - i - 1], A[0], sizeof(t));
memcpy(A[0], t, sizeof(t));
heapsortdown(A, n - i - 1, 0);
HeapSortDown(n - i - 1, A, 0);
}
}

6
libc/runtime/close_s.c
View file

@ -19,18 +19,18 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/bits/bits.h"
#include "libc/calls/calls.h"
#include "libc/runtime/mappings.h"
#include "libc/runtime/runtime.h"
/**
* Closes file descriptor.
*
* The caller's variable is made -1. Note that close(-1) is a no-op.
* The caller's variable is made -1 so subsequent calls are no-ops.
*
* @return 0 on success, or -1 w/ errno
* @asyncsignalsafe
*/
int close_s(int *fdp) {
int fd = -1;
return close(lockxchg(fdp, &fd));
if (lockxchg(fdp, &fd) == -1) return 0;
return close(fd);
}

14
libc/runtime/closesymboltable.c
View file

@ -17,21 +17,23 @@
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/runtime/ezmap.h"
#include "libc/runtime/symbols.h"
#include "libc/calls/calls.h"
/**
* Frees symbol table.
* @return 0 on success or -1 on system error
*/
int closesymboltable(struct SymbolTable **table) {
int rc = 0;
if (*table && (intptr_t)*table != (intptr_t)-1) {
struct SymbolTable *t = *table;
int rc;
struct SymbolTable *t;
rc = 0;
if (*table && *table != MAP_FAILED) {
t = *table;
*table = NULL;
if (unmapfile(&t->mf) == -1) rc = -1;
if (munmap(t, t->scratch) == -1) rc = -1;
rc |= unmapfile(&t->mf);
rc |= munmap(t, t->scratch);
}
return rc;
}

30
libc/runtime/munmap-thunk.S → libc/runtime/construct.S
View file

@ -17,17 +17,29 @@
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/internal.h"
#include "libc/macros.h"
.text.startup
.source __FILE__
munmap: push %rbp
/ Calls global initialization functions.
_construct:
push %rbp
mov %rsp,%rbp
push %rbx
push %rbx
ezlea _base,bx
call __munmap
pop %rbx
pop %rbx
pop %rbp
orb $RUNSTATE_INITIALIZED,g_runstate(%rip)
ezlea __init_array_start,ax # static ctors in forward order
.weak __init_array_start # could be called multiple times
ezlea __init_array_end,cx # idempotency recommended
.weak __init_array_end # @see ape/ape.lds
1: cmp %rax,%rcx
je 2f
push %rax
push %rcx
call *(%rax)
pop %rcx
pop %rax
add $8,%rax
jmp 1b
2: pop %rbp
ret
.endfn munmap,globl
.endfn _construct,globl

181
libc/runtime/cxaatexit.S
View file

@ -1,181 +0,0 @@
/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│
│vi: set et ft=asm ts=8 sw=8 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ This program is free software; you can redistribute it and/or modify │
│ it under the terms of the GNU General Public License as published by │
│ the Free Software Foundation; version 2 of the License. │
│ │
│ This program is distributed in the hope that it will be useful, but │
│ WITHOUT ANY WARRANTY; without even the implied warranty of │
│ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU │
│ General Public License for more details. │
│ │
│ You should have received a copy of the GNU General Public License │
│ along with this program; if not, write to the Free Software │
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "ape/relocations.h"
#include "libc/macros.h"
.section .text.exit,"ax",@progbits
.source __FILE__
/ Delegates to __cxa_atexit().
/
/ @param rdi callback typed void(*)(void) (nullable)
/ @return 0 on success or nonzero if out of space
atexit: xor %esi,%esi
xor %edx,%edx
/ 𝑠𝑙𝑖𝑑𝑒
/ Registers destructor to be called upon exit().
/
/ Destructors are called in reverse order. They won't be called
/ if the program aborts or _exit() is called. Invocations of this
/ function are usually generated by the C++ compiler.
/
/ @param rdi callback typed void(*)(T) (nullable)
/ @param rsi callback arg typed T (nullable)
/ @param rdx dso handle (nullable)
/ @return 0 on success or nonzero if out of space
/ @note folks have forked libc in past just to unbloat atexit()
/ @see news.ycombinator.com/item?id=20228082
/ @preinitsafe
__cxa_atexit:
push %rbp
mov %rsp,%rbp
.profilable
push %r15
ezlea g_cxa,cx
mov %rcx,%r15
mov (%r15),%rax # get state->block
mov 8(%r15),%rcx # get state->offset
test %rcx,%rcx
jz 2f
0: sub $24,%rcx
mov %rdx,(%rax,%rcx) # set cb->dso
mov %rsi,8(%rax,%rcx) # set cb->arg
mov %rdi,16(%rax,%rcx) # set cb->fn
mov %rcx,8(%r15) # set state->offset
xor %eax,%eax # success
1: pop %r15
pop %rbp
ret
2: .weak calloc
ezlea calloc,cx
test %rcx,%rcx
jz 1b # fail (no malloc)
push %rax
push %rdi
push %rsi
pushpop ATEXIT_MAX+1,%rdi
pushpop 16,%rsi
call *%rcx
pop %rsi
pop %rdi
pop %rcx # rax=new rcx=old
test %rax,%rax
jz 1b # fail (no memory)
mov $ATEXIT_MAX*8*3,%r8
mov %rax,(%r15) # set state->block
mov %rcx,(%rax,%r8) # set block->next
mov %r8,%rcx
jmp 0b
.endfn __cxa_atexit,globl
.endfn atexit,globl
/ Triggers destructors.
/
/ This implementation supports DSO handles, but is optimized for
/ cases when it's called only once by exit().
/
/ @param rdi is dso predicate or null to destroy all
/ @see libc/exit.c
__cxa_finalize:
push %rbp
mov %rsp,%rbp
.profilable
push %r14
push %r13
push %r12
mov g_cxa(%rip),%rsi
mov %rdi,%r14
0: mov %rsi,%r12 # loop through blocks
pushpop ATEXIT_MAX,%rcx
1: lodsq #→ dso # loop through callbacks
xchg %rax,%rdx
lodsq #→ arg
xchg %rax,%rdi
lodsq #→ fn
test %rax,%rax
jz 2f # ignore empty slots
test %r14,%r14
jmp 5f # null predicate match all
cmp %r14,%rdx
jne 2f # predicate mismatch
5: push %rsi
push %rcx
push %rcx
call *%rax
pop %rcx
pop %rcx
pop %rsi
xor %eax,%eax # clear slot (never reused)
mov %rax,-8(%rsi)
2: loop 1b
lodsq # get next block ptr
test %rax,%rax # don't free static block no. 1
jz 3f # which always has next == NULL
test %r14,%r14
jz 1f # don't free anything if just one dso
push %rax
mov %r12,%rdi
.weak free
call free # can't panic due to earlier test
1: pop %rsi
jmp 0b
3: pop %r12 # align stack for next call
test %r14,%r14 # no static dtor for dso exit
jnz 9f
ezlea __fini_array_end,ax # static dtors in reverse order
.weak __fini_array_end # could be called multiple times
ezlea __fini_array_start,cx # idempotency recommended
.weak __fini_array_start # or consider atexit()
8: sub $8,%rax # @see ape/ape.lds
cmp %rcx,%rax
jl 9f
push %rax
push %rcx
call *(%rax)
pop %rcx
pop %rax
jmp 8b
9: pop %r13
pop %r14
pop %rbp
ret
.endfn __cxa_finalize,globl,hidden
.bss
.align 16 # static/dynamic hybrid linked list
g_cxa: .quad 0 # last block ptr: (long (*)[32][3])
.quad 0 # block byte offset moves backwards
.endobj g_cxa
g_cxa_static:
.rept ATEXIT_MAX
.quad 0 # dso
.quad 0 # arg
.quad 0 # fn (or NULL for empty)
.endr
.quad 0 # next (always NULL in static block)
.endobj g_cxa_static
.previous
.init.start 300,_init_g_cxa
ezlea g_cxa,cx
lea g_cxa_static-g_cxa(%rcx),%rax
mov %rax,(%rcx)
movl $ATEXIT_MAX*8*3,8(%rcx)
.init.end 300,_init_g_cxa

122
libc/runtime/cxaatexit.c Normal file
View file

@ -0,0 +1,122 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ This program is free software; you can redistribute it and/or modify │
│ it under the terms of the GNU General Public License as published by │
│ the Free Software Foundation; version 2 of the License. │
│ │
│ This program is distributed in the hope that it will be useful, but │
│ WITHOUT ANY WARRANTY; without even the implied warranty of │
│ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU │
│ General Public License for more details. │
│ │
│ You should have received a copy of the GNU General Public License │
│ along with this program; if not, write to the Free Software │
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/bits/weaken.h"
#include "libc/macros.h"
#include "libc/mem/mem.h"
#include "libc/nexgen32e/bsf.h"
#include "libc/nexgen32e/bsr.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/errfuns.h"
static struct CxaAtexitBlocks {
struct CxaAtexitBlock {
unsigned mask;
struct CxaAtexitBlock *next;
struct CxaAtexit {
void *fp;
void *arg;
void *pred;
} p[ATEXIT_MAX];
} * p, root;
} __cxa_blocks;
/**
* Adds global destructor.
*
* Destructors are called in reverse order. They won't be called if the
* program aborts or _exit() is called. Invocations of this function are
* usually generated by the C++ compiler. Behavior is limitless if you
* choose to link calloc() and free().
*
* @param fp is void(*)(T)
* @param arg is passed to callback
* @param pred can be non-null for things like dso modules
* @return 0 on success or nonzero w/ errno
* @note folks have forked libc in past just to unbloat atexit()
*/
int __cxa_atexit(void *fp, void *arg, void *pred) {
unsigned i;
struct CxaAtexitBlock *b, *b2;
b = __cxa_blocks.p;
if (!b) b = __cxa_blocks.p = &__cxa_blocks.root;
if (!~b->mask) {
if (weaken(calloc) &&
(b2 = weaken(calloc)(1, sizeof(struct CxaAtexitBlock)))) {
b2->next = b;
__cxa_blocks.p = b = b2;
} else {
return enomem();
}
}
i = bsr(~b->mask);
assert(i < ARRAYLEN(b->p));
b->mask |= 1u << i;
b->p[i].fp = fp;
b->p[i].arg = arg;
b->p[i].pred = pred;
return 0;
}
/**
* Triggers global destructors.
*
* @param pred can be null to match all
* @note reentrant emptor
*/
void __cxa_finalize(void *pred) {
unsigned i;
unsigned long mask;
struct CxaAtexitBlock *b, *b2;
StartOver:
if ((b = __cxa_blocks.p)) {
for (;;) {
mask = b->mask;
while (mask) {
i = bsf(mask);
mask &= ~(1u << i);
if (!pred || pred == b->p[i].pred) {
b->mask &= ~(1u << i);
if (b->p[i].fp) {
((void (*)(void *))b->p[i].fp)(b->p[i].arg);
goto StartOver;
}
}
}
if (!pred) {
b2 = b->next;
if (b2) {
assert(b != &__cxa_blocks.root);
if (weaken(free)) {
weaken(free)(b);
}
}
__cxa_blocks.p = b2;
goto StartOver;
} else {
if (b->next) {
b = b->next;
} else {
break;
}
}
}
}
}

4
libc/runtime/defer.greg.c
View file

@ -53,10 +53,10 @@ void __defer(struct StackFrame *frame, void *fn, void *arg) {
frame2 = __builtin_frame_address(0);
assert(frame2->next == frame);
assert(PointerNotOwnedByParentStackFrame(frame2, frame, arg));
if (append(&__garbage,
if (append(&g_garbage,
(&(const struct Garbage){frame->next, (intptr_t)fn, (intptr_t)arg,
frame->addr})) != -1) {
atomic_store(&frame->addr, (intptr_t)&__gc);
atomic_store(&frame->addr, (intptr_t)&CollectGarbage);
} else {
abort();
}

88
libc/runtime/startmain.S → libc/runtime/executive.S
View file

@ -20,91 +20,37 @@
#include "libc/dce.h"
#include "libc/macros.h"
#include "libc/notice.inc"
#include "libc/runtime/internal.h"
#include "libc/sysv/consts/map.h"
#include "libc/dce.h"
#include "libc/runtime/mappings.h"
#include "libc/sysv/consts/prot.h"
.text.startup
.source __FILE__
/ Cosmopolitan process entrypoint.
/ Stack frame that owns process from spawn to exit.
/
/ @param r12 is argc
/ @param r13 is argv
/ @param r14 is environ
/ @param r15 is auxv
/ @param edi is argc
/ @param rsi is argv
/ @param rdx is environ
/ @param rcx is auxv
/ @noreturn
__executive:
.frame0
_executive:
push %rbp
mov %rsp,%rbp
ezlea _base,bx
#ifdef __FAST_MATH__
call __fast_math
#endif
call _init
#if IsModeDbg()
call _init # _init() is idempotent
#endif
/*
#if !IsTiny()
/ “Memory obfuscation for glibc, not for we”
/
/ 64kb stack w/ 4kb guard alongside tuning in libc/integral/c.inc
/ e.g. -Werror=frame-larger-than=4096 is intended to guarantee no
/ stack overflow possible. We like malloc and only cleverly avoid
/ its use at the lowest levels of the runtime stack, without MMU.
/ We like this practicee because it's how Google runs production.
mov $kStackCeiling-STACKSIZE,%rdi
mov $STACKSIZE,%esi
mov $PROT_READ|PROT_WRITE,%edx
mov MAP_ANONYMOUS,%ecx
or MAP_FIXED,%ecx
or MAP_PRIVATE,%ecx
mov $-1,%r8d
xor %r9d,%r9d
call mmap
cmp $-1,%eax
je abort
lea STACKSIZE(%rax),%rsp
xor %ebp,%ebp
mov %rax,%rdi
mov $PAGESIZE,%esi
mov $PROT_NONE,%edx
call mprotect
cmp $-1,%eax
je abort
#endif
*/
orl $RUNSTATE_INITIALIZED,g_runstate(%rip)
ezlea __init_array_start,ax # static ctors in forward order
.weak __init_array_start # could be called multiple times
ezlea __init_array_end,cx # idempotency recommended
.weak __init_array_end # @see ape/ape.lds
1: cmp %rax,%rcx
je 2f
push %rax
push %rcx
call *(%rax)
pop %rcx
pop %rax
add $8,%rax
jmp 1b
2: nop
#if !IsTrustworthy()
mov $PROT_READ,%edi
call __piro
#endif
mov %edi,%r12d
mov %rsi,%r13
mov %rdx,%r14
mov %rcx,%r15
call _spawn
mov %r12,%rdi
mov %r13,%rsi
mov %r14,%rdx
mov %r15,%rcx
.weak main
call main
mov %eax,%edi
call exit
.endfn __executive,weak,hidden
9: .endfn _executive,weak,hidden
ud2
#ifdef __PG__
/ Enables plaintext function tracing if --ftrace flag passed.

26
libc/runtime/mapanon-thunk.S → libc/runtime/exit.S
View file

@ -18,16 +18,22 @@
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macros.h"
.text.exit
.source __FILE__
mapanon:push %rbp
/ Exits program with grace.
/
/ @param %dil has exit code
/ @noreturn
exit: push %rbp
mov %rsp,%rbp
push %rbx
push %rbx
ezlea _base,bx
call __mapanon
pop %rbx
pop %rbx
pop %rbp
ret
.endfn mapanon,globl
push %rdi
push %rdi
xor %edi,%edi
call __cxa_finalize
pop %rdi
pop %rdi
call _Exit
.endfn exit,globl
ud2

12
libc/runtime/ezmap.c
View file

@ -17,6 +17,7 @@
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/bits/pushpop.h"
#include "libc/bits/safemacros.h"
#include "libc/calls/calls.h"
#include "libc/limits.h"
@ -48,11 +49,10 @@ int mapfileread(const char *filename, struct MappedFile *mf) {
* Releases resource returned by mapfileread().
*/
int unmapfile(struct MappedFile *mf) {
int rc = 0;
rc |= munmap(mf->addr, mf->size);
rc |= close(mf->fd);
mf->fd = -1;
mf->addr = MAP_FAILED;
mf->size = 0;
int rc;
rc = 0;
rc |= munmap_s(&mf->addr, mf->size);
rc |= close_s(&mf->fd);
pushmov(&mf->size, 0);
return rc;
}

6
libc/runtime/ezmap.h
View file

@ -23,13 +23,13 @@
COSMOPOLITAN_C_START_
struct MappedFile {
int64_t fd;
int fd;
void *addr;
size_t size;
};
int mapfileread(const char *filename, struct MappedFile *mf) hidden;
int unmapfile(struct MappedFile *mf) hidden;
int mapfileread(const char *, struct MappedFile *) hidden;
int unmapfile(struct MappedFile *) hidden;
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

24
libc/runtime/exit.c → libc/runtime/findmemoryinterval.c
View file

@ -17,14 +17,20 @@
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/memtrack.h"
/**
* Exits program with grace.
*/
noreturn textexit void exit(int rc) {
__cxa_finalize(NULL);
_exit(rc);
unreachable;
unsigned FindMemoryInterval(const struct MemoryIntervals *mm, int x) {
unsigned l, m, r;
l = 0;
r = mm->i;
while (l < r) {
m = (l + r) >> 1;
if (mm->p[m].x < x) {
l = m + 1;
} else {
r = m;
}
}
if (l && mm->p[l - 1].y >= x) --l;
return l;
}

1
libc/runtime/free_s.c
View file

@ -19,7 +19,6 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/bits/bits.h"
#include "libc/mem/mem.h"
#include "libc/runtime/mappings.h"
#include "libc/runtime/runtime.h"
/**

108
libc/runtime/ftrace.greg.c
View file

@ -98,116 +98,14 @@ privileged interruptfn void ftrace_hook(void) {
RESTORE_RBX();
}
/**
* Rewrites code in memory to log function calls.
*
* We do this by searching each function for the nop instruction
* inserted by GCC when we use the -pg -mnop-mcount flags. There's no
* risk of corrupting data since the linker scripts won't mix code and
* data.
*
* Modules built with -O3 and without the profiling flags might have
* these same nop instructions, but that shouldn't be problematic since
* they're only there for the puposes of aligning jumps, and therefore
* aren't actually executed. However codebases that use huge function
* alignments with wide-nop slides could pose minor issues. Further note
* that Cosmopolitan sources are almost never intentionally written to
* use code alignment, since we've only seen a few cases where it helps.
*
* @see ape/ape.lds
*/
privileged void ftrace_install(void) {
/* TODO(jart): Is -fschedule-insns2 so aggro that we need XED here? */
size_t i;
intptr_t addr;
sigset_t oldmask;
uint64_t code, mcode;
unsigned char *p, *pe;
const intptr_t kMcount = (intptr_t)&mcount;
const intptr_t kFtraceHook = (intptr_t)&ftrace_hook;
const intptr_t kProgramCodeStart = (intptr_t)&_ereal;
const intptr_t kPrivilegedStart = (intptr_t)&__privileged_start;
const bool kIsBinaryAligned = !(kPrivilegedStart & (PAGESIZE - 1));
g_buf[0] = '+';
g_buf[1] = ' ';
sigprocmask(SIG_BLOCK, &kSigsetFull, &oldmask);
if (__mprotect((void *)g_symbols->addr_base,
kPrivilegedStart - g_symbols->addr_base,
kIsBinaryAligned ? PROT_READ | PROT_WRITE
: PROT_READ | PROT_WRITE | PROT_EXEC) != -1) {
for (i = 0; i < g_symbols->count - 1; ++i) {
if (g_symbols->addr_base + g_symbols->symbols[i].addr_rva <
kProgramCodeStart) {
continue; /* skip over real mode symbols */
}
if (g_symbols->addr_base + g_symbols->symbols[i].addr_rva >=
kPrivilegedStart) {
break; /* stop before privileged symbols */
}
for (p = (unsigned char *)(g_symbols->addr_base +
g_symbols->symbols[i].addr_rva),
pe = (unsigned char *)(g_symbols->addr_base +
g_symbols->symbols[i + 1].addr_rva);
p < pe - 8; ++p) {
code = read64le(p);
/*
* Test for -mrecord-mcount (w/ -fpie or -fpic)
*
* nopw 0x00(%rax,%rax,1) ← morphed by package.com
* call *mcount(%rip) ← linked w/o -static
* addr32 call mcount ← relaxed w/ -static
* addr32 call mcount ← relaxed w/ -static
*
* Note that gcc refuses to insert the six byte nop.
*/
if ((code & 0x0000FFFFFFFFFFFF) == 0x0000441F0F66 ||
(code & 0x0000FFFFFFFFFFFF) ==
((((kMcount - ((intptr_t)&p[2] + 4)) << 16) | 0xE867) &
0x0000FFFFFFFFFFFF) ||
(code & 0x0000FFFFFFFFFFFF) ==
((((kMcount - ((intptr_t)&p[2] + 4)) << 16) | 0xFF15) &
0x0000FFFFFFFFFFFF)) {
p[0] = 0x67;
p[1] = 0xE8;
addr = kFtraceHook - ((intptr_t)&p[2] + 4);
p[2] = addr >> 000;
p[3] = addr >> 010;
p[4] = addr >> 020;
p[5] = addr >> 030;
break;
}
/*
* Test for -mnop-mcount (w/ -fno-pie)
*/
mcode = code & 0x000000FFFFFFFFFF;
if ((mcode == 0x00441F0F /* nopl 0x00(%eax,%eax,1) [canonical] */) ||
(mcode == 0x00041F0F67 /* nopl (%eax,%eax,1) [older gcc] */)) {
if (p[-1] != 0x66 /* nopw 0x0(%rax,%rax,1) [donotwant] */) {
p[0] = 0xE8 /* call Jvds */;
addr = kFtraceHook - ((intptr_t)&p[1] + 4);
p[1] = addr >> 000;
p[2] = addr >> 010;
p[3] = addr >> 020;
p[4] = addr >> 030;
}
break;
}
}
}
__mprotect((void *)g_symbols->addr_base,
kPrivilegedStart - g_symbols->addr_base, PROT_READ | PROT_EXEC);
}
sigprocmask(SIG_SETMASK, &oldmask, NULL);
}
/**
* Installs plaintext function tracer. Do not call.
* @see libc/runtime/_init.S for documentation
*/
textstartup void ftrace_init(void) {
g_buf[0] = '+';
g_buf[1] = ' ';
if ((g_symbols = opensymboltable(finddebugbinary()))) {
ftrace_install();
__hook(ftrace_hook, g_symbols);
}
}

18
libc/runtime/gc.h
View file

@ -21,7 +21,7 @@ struct StackFrame;
* @warning do not return a gc()'d pointer
* @warning do not realloc() with gc()'d pointer
*/
#define gc(THING) defer(weakfree, (THING))
#define gc(THING) defer((void *)weakfree, (void *)(THING))
/**
* Same as longjmp() but runs gc() / defer() destructors.
@ -31,14 +31,14 @@ void gclongjmp(jmp_buf, int) nothrow noreturn paramsnonnull();
/**
* Calls FN(ARG) when function returns.
*/
#define defer(FN, ARG) \
({ \
autotype(ARG) Arg = (ARG); \
/* prevent weird opts like tail call */ \
asm volatile("" : "+g"(Arg) : : "memory"); \
__defer(__builtin_frame_address(0), FN, Arg); \
asm volatile("" : "+g"(Arg) : : "memory"); \
Arg; \
#define defer(FN, ARG) \
({ \
autotype(ARG) Arg = (ARG); \
/* prevent weird opts like tail call */ \
asm volatile("" : "+g"(Arg) : : "memory"); \
__defer((struct StackFrame *)__builtin_frame_address(0), FN, Arg); \
asm volatile("" : "+g"(Arg) : : "memory"); \
Arg; \
})
void __defer(struct StackFrame *, void *, void *) hidden paramsnonnull((1, 2));

5
libc/runtime/getdosargv.c
View file

@ -17,6 +17,7 @@
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/bits/bits.h"
#include "libc/bits/pushpop.h"
#include "libc/bits/safemacros.h"
@ -38,11 +39,11 @@ struct DosArgv {
wint_t wc;
};
static inline textwindows void decodedosargv(struct DosArgv *st) {
static textwindows void decodedosargv(struct DosArgv *st) {
st->s += getutf16(st->s, &st->wc);
}
static inline textwindows void appenddosargv(struct DosArgv *st, wint_t wc) {
static textwindows void appenddosargv(struct DosArgv *st, wint_t wc) {
AppendChar(&st->p, st->pe, wc);
}

2
libc/runtime/getdosenviron.h
View file

@ -19,6 +19,7 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#ifndef COSMOPOLITAN_LIBC_DOSENVIRON_H_
#define COSMOPOLITAN_LIBC_DOSENVIRON_H_
#ifndef __STRICT_ANSI__
#include "libc/bits/safemacros.h"
#include "libc/str/appendchar.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
@ -64,4 +65,5 @@ static inline int getdosenviron(const char16_t *env, char *buf, size_t size,
}
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* !ANSI */
#endif /* COSMOPOLITAN_LIBC_DOSENVIRON_H_ */

2
libc/runtime/grow.c
View file

@ -20,11 +20,11 @@
#include "libc/assert.h"
#include "libc/bits/bits.h"
#include "libc/bits/safemacros.h"
#include "libc/bits/weaken.h"
#include "libc/conv/conv.h"
#include "libc/conv/sizemultiply.h"
#include "libc/macros.h"
#include "libc/mem/mem.h"
#include "libc/runtime/mappings.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
#include "libc/sysv/errfuns.h"

125
libc/runtime/hook.greg.c Normal file
View file

@ -0,0 +1,125 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ This program is free software; you can redistribute it and/or modify │
│ it under the terms of the GNU General Public License as published by │
│ the Free Software Foundation; version 2 of the License. │
│ │
│ This program is distributed in the hope that it will be useful, but │
│ WITHOUT ANY WARRANTY; without even the implied warranty of │
│ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU │
│ General Public License for more details. │
│ │
│ You should have received a copy of the GNU General Public License │
│ along with this program; if not, write to the Free Software │
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
#include "libc/calls/struct/sigset.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/symbols.h"
#include "libc/sysv/consts/prot.h"
/**
* Rewrites code in memory to hook function calls.
*
* We do this by searching each function for the nop instruction
* inserted by GCC when we use the -pg -mnop-mcount flags. There's no
* risk of corrupting data since the linker scripts won't mix code and
* data.
*
* Modules built with -O3 and without the profiling flags might have
* these same nop instructions, but that shouldn't be problematic since
* they're only there for the puposes of aligning jumps, and therefore
* aren't actually executed. However codebases that use huge function
* alignments with wide-nop slides could pose minor issues. Further note
* that Cosmopolitan sources are almost never intentionally written to
* use code alignment, since we've only seen a few cases where it helps.
*
* @see ape/ape.lds
*/
privileged void __hook(void ifunc(void), struct SymbolTable *symbols) {
size_t i;
intptr_t addr;
sigset_t oldmask;
uint64_t code, mcode;
unsigned char *p, *pe;
const intptr_t kMcount = (intptr_t)&mcount;
const intptr_t kProgramCodeStart = (intptr_t)&_ereal;
const intptr_t kPrivilegedStart = (intptr_t)&__privileged_start;
const bool kIsBinaryAligned = !(kPrivilegedStart & (PAGESIZE - 1));
sigprocmask(SIG_BLOCK, &kSigsetFull, &oldmask);
if (mprotect((void *)symbols->addr_base,
kPrivilegedStart - symbols->addr_base,
kIsBinaryAligned ? PROT_READ | PROT_WRITE
: PROT_READ | PROT_WRITE | PROT_EXEC) != -1) {
for (i = 0; i < symbols->count - 1; ++i) {
if (symbols->addr_base + symbols->symbols[i].addr_rva <
kProgramCodeStart) {
continue; /* skip over real mode symbols */
}
if (symbols->addr_base + symbols->symbols[i].addr_rva >=
kPrivilegedStart) {
break; /* stop before privileged symbols */
}
for (p = (unsigned char *)(symbols->addr_base +
symbols->symbols[i].addr_rva),
pe = (unsigned char *)(symbols->addr_base +
symbols->symbols[i + 1].addr_rva);
p < pe - 8; ++p) {
code = read64le(p);
/*
* Test for -mrecord-mcount (w/ -fpie or -fpic)
*
* nopw 0x00(%rax,%rax,1) ← morphed by package.com
* call *mcount(%rip) ← linked w/o -static
* addr32 call mcount ← relaxed w/ -static
* addr32 call mcount ← relaxed w/ -static
*
* Note that gcc refuses to insert the six byte nop.
*/
if ((code & 0x0000FFFFFFFFFFFF) == 0x0000441F0F66 ||
(code & 0x0000FFFFFFFFFFFF) ==
((((kMcount - ((intptr_t)&p[2] + 4)) << 16) | 0xE867) &
0x0000FFFFFFFFFFFF) ||
(code & 0x0000FFFFFFFFFFFF) ==
((((kMcount - ((intptr_t)&p[2] + 4)) << 16) | 0xFF15) &
0x0000FFFFFFFFFFFF)) {
p[0] = 0x67;
p[1] = 0xE8;
addr = (intptr_t)ifunc - ((intptr_t)&p[2] + 4);
p[2] = addr >> 000;
p[3] = addr >> 010;
p[4] = addr >> 020;
p[5] = addr >> 030;
break;
}
/*
* Test for -mnop-mcount (w/ -fno-pie)
*/
mcode = code & 0x000000FFFFFFFFFF;
if ((mcode == 0x00441F0F /* nopl 0x00(%eax,%eax,1) [canonical] */) ||
(mcode == 0x00041F0F67 /* nopl (%eax,%eax,1) [older gcc] */)) {
if (p[-1] != 0x66 /* nopw 0x0(%rax,%rax,1) [donotwant] */) {
p[0] = 0xE8 /* call Jvds */;
addr = (intptr_t)ifunc - ((intptr_t)&p[1] + 4);
p[1] = addr >> 000;
p[2] = addr >> 010;
p[3] = addr >> 020;
p[4] = addr >> 030;
}
break;
}
}
}
mprotect((void *)symbols->addr_base, kPrivilegedStart - symbols->addr_base,
PROT_READ | PROT_EXEC);
}
sigprocmask(SIG_SETMASK, &oldmask, NULL);
}

7
libc/runtime/internal.h
View file

@ -1,5 +1,6 @@
#ifndef COSMOPOLITAN_LIBC_RUNTIME_INTERNAL_H_
#define COSMOPOLITAN_LIBC_RUNTIME_INTERNAL_H_
#ifndef __STRICT_ANSI__
#include "libc/dce.h"
#include "libc/runtime/runtime.h"
@ -18,11 +19,12 @@ hidden extern unsigned g_runstate;
hidden extern void *g_stacktop;
void _init(void) hidden;
void __piro(int) hidden;
void _piro(int) hidden;
void *__cxa_finalize(void *) hidden;
int getdosargv(const char16_t *, char *, size_t, char **, size_t) hidden;
void _executive(int, char **, char **, long (*)[2]) hidden noreturn;
void __stack_chk_fail(void) noreturn relegated;
void __stack_chk_fail_local(void) noreturn relegated hidden;
int getdosargv(const char16_t *, char *, size_t, char **, size_t) hidden;
forceinline void AssertNeverCalledWhileTerminating(void) {
if (!NoDebug() && (g_runstate & RUNSTATE_TERMINATE)) {
@ -32,4 +34,5 @@ forceinline void AssertNeverCalledWhileTerminating(void) {
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* ANSI */
#endif /* COSMOPOLITAN_LIBC_RUNTIME_INTERNAL_H_ */

26
libc/runtime/isheap.c
View file

@ -17,22 +17,20 @@
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/mappings.h"
#include "libc/runtime/memtrack.h"
#include "libc/runtime/runtime.h"
/**
* Returns false if address can't be heap memory.
* Returns true if address isn't stack and was malloc'd or mmap'd.
*
* @assume stack addresses are always greater than heap addresses
* @assume stack memory isn't stored beneath %rsp (-mno-red-zone)
*/
bool isheap(void *p) {
size_t i;
struct MemoryCoord c;
if (!(kStackBottom <= (intptr_t)p && (intptr_t)p < kStackCeiling)) {
c = ADDRSIZE_TO_COORD(p, FRAMESIZE);
if ((i = findmapping(c.x))) {
return ISOVERLAPPING(_mm.p[i - 1], c);
} else {
return false;
}
} else {
return false;
}
int x, i;
register intptr_t rsp asm("rsp");
if ((intptr_t)p >= rsp) return false;
x = (intptr_t)p >> 16;
i = FindMemoryInterval(&_mmi, x);
return i < _mmi.i && x >= _mmi.p[i].x && x <= _mmi.p[i].y;
}

2
libc/runtime/mapanon.c
View file

@ -22,7 +22,7 @@
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
void *__mapanon(size_t mapsize) {
void *mapanon(size_t mapsize) {
return mmap(NULL, mapsize, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_NONBLOCK | MAP_ANONYMOUS, -1, 0);
}

78
libc/runtime/mappings.h
View file

@ -1,78 +0,0 @@
#ifndef COSMOPOLITAN_LIBC_RUNTIME_MAPPINGS_H_
#define COSMOPOLITAN_LIBC_RUNTIME_MAPPINGS_H_
#include "libc/dce.h"
#include "libc/macros.h"
#include "libc/runtime/runtime.h"
#define MMAP_MAX 300 /* TODO: crunch */
#define kStackCeiling 0x0000700000000000L
#define kStackBottom 0x0000600000000000L
#define kFixedMappingsStart 0x0000100000000000L /* cosmo won't auto-assign */
#define kFixedMappingsSize 0x0000100000000000L /* 16TB */
#define kMappingsStart 0x0000200000000000L /* cosmo auto-assigns here */
#define kMappingsSize 0x0000100000000000L /* 16TB */
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
#define ISOVERLAPPING(C1, C2) \
(((C1).x >= (C2).x && (C1).x <= (C2).y) || \
((C1).y >= (C2).x && (C1).y <= (C2).y))
#define ADDR_TO_COORD(ADDR) \
(int)(((intptr_t)(ADDR) & ~(FRAMESIZE - 1)) / FRAMESIZE)
#define COORD_TO_ADDR(COORD) (void *)((intptr_t)(COORD)*FRAMESIZE)
#define COORD_TO_SIZE(COORD) (void *)((intptr_t)(COORD)*FRAMESIZE)
#define ADDRSIZE_TO_COORD(ADDR, SIZE) \
((struct MemoryCoord){ \
.x = ADDR_TO_COORD(ADDR), \
.y = ADDR_TO_COORD(ADDR) + \
((unsigned)(ROUNDUP((SIZE), FRAMESIZE) / FRAMESIZE) - 1)})
#define COORD_TO_ADDRSIZE(COORD) \
((struct AddrSize){ \
.addr = COORD_TO_ADDR((COORD).x), \
.size = ((size_t)((COORD).y - (COORD).x + 1) * FRAMESIZE)})
#define GRANULATE_ADDRSIZE(ADDR, SIZE) \
do { \
struct AddrSize AdSiz; \
struct MemoryCoord MemCo; \
MemCo = ADDRSIZE_TO_COORD(*(ADDR), *(SIZE)); \
AdSiz = COORD_TO_ADDRSIZE(MemCo); \
*(ADDR) = AdSiz.addr; \
*(SIZE) = AdSiz.size; \
} while (0)
struct AddrSize {
void *addr;
size_t size;
};
/**
* Ordered inclusive 64kb-granular ranges on NexGen32e w/o PML5.
* c.𝑥 ≤ c.𝑦 so say c all.
* cₙ.𝑥 ≤ cₙ₊₁.𝑥 so say c all.
*/
struct Mappings {
size_t i;
struct MemoryCoord {
int32_t x, y;
} p[MMAP_MAX];
int64_t h[MMAP_MAX];
};
extern struct Mappings _mm;
bool isheap(void *);
size_t findmapping(int32_t);
size_t findmapping_(int32_t, const struct MemoryCoord *, size_t);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_RUNTIME_MAPPINGS_H_ */

117
libc/runtime/memtrack.c Normal file
View file

@ -0,0 +1,117 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ This program is free software; you can redistribute it and/or modify │
│ it under the terms of the GNU General Public License as published by │
│ the Free Software Foundation; version 2 of the License. │
│ │
│ This program is distributed in the hope that it will be useful, but │
│ WITHOUT ANY WARRANTY; without even the implied warranty of │
│ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU │
│ General Public License for more details. │
│ │
│ You should have received a copy of the GNU General Public License │
│ along with this program; if not, write to the Free Software │
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/macros.h"
#include "libc/runtime/memtrack.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
#include "libc/sysv/errfuns.h"
static void RemoveMemoryIntervals(struct MemoryIntervals *mm, int i, int n) {
assert(i >= 0);
assert(i + n <= mm->i);
memcpy(mm->p + i, mm->p + i + n,
(intptr_t)(mm->p + mm->i) - (intptr_t)(mm->p + i + n));
memcpy(mm->h + i, mm->h + i + n,
(intptr_t)(mm->h + mm->i) - (intptr_t)(mm->h + i + n));
mm->i -= n;
}
static void CreateMemoryInterval(struct MemoryIntervals *mm, int i) {
assert(i >= 0);
assert(i <= mm->i);
assert(mm->i < ARRAYLEN(mm->p));
memmove(mm->p + i + 1, mm->p + i,
(intptr_t)(mm->p + mm->i) - (intptr_t)(mm->p + i));
memmove(mm->h + i + 1, mm->h + i,
(intptr_t)(mm->h + mm->i) - (intptr_t)(mm->h + i));
++mm->i;
}
static int PunchHole(struct MemoryIntervals *mm, int x, int y, int i) {
if (mm->i == ARRAYLEN(mm->p)) return enomem();
CreateMemoryInterval(mm, i);
mm->p[i].y = x - 1;
mm->p[i + 1].x = y + 1;
return 0;
}
int ReleaseMemoryIntervals(struct MemoryIntervals *mm, int x, int y,
void wincb(struct MemoryIntervals *, int, int)) {
unsigned l, r;
assert(y >= x);
assert(AreMemoryIntervalsOk(mm));
if (!mm->i) return 0;
l = FindMemoryInterval(mm, x);
if (l == mm->i) return 0;
if (!l && y < mm->p[l].x) return 0;
if (y < mm->p[l].x) return 0;
r = FindMemoryInterval(mm, y);
if (r == mm->i || (r > l && y < mm->p[r].x)) --r;
assert(r >= l);
assert(x <= mm->p[r].y);
if (l == r && x > mm->p[l].x && y < mm->p[l].y) {
return PunchHole(mm, x, y, l);
}
if (x > mm->p[l].x && x <= mm->p[l].y) {
assert(y >= mm->p[l].y);
if (IsWindows()) return einval();
mm->p[l].y = x - 1;
assert(mm->p[l].x <= mm->p[l].y);
++l;
}
if (y >= mm->p[r].x && y < mm->p[r].y) {
assert(x <= mm->p[r].x);
if (IsWindows()) return einval();
mm->p[r].x = y + 1;
assert(mm->p[r].x <= mm->p[r].y);
--r;
}
if (l <= r) {
if (IsWindows() && wincb) {
wincb(mm, l, r);
}
RemoveMemoryIntervals(mm, l, r - l + 1);
}
return 0;
}
int TrackMemoryInterval(struct MemoryIntervals *mm, int x, int y, long h) {
unsigned i;
assert(y >= x);
assert(AreMemoryIntervalsOk(mm));
i = FindMemoryInterval(mm, x);
if (i && x == mm->p[i - 1].y + 1 && h == mm->h[i - 1]) {
mm->p[i - 1].y = y;
if (i < mm->i && y + 1 == mm->p[i].x && h == mm->h[i]) {
mm->p[i - 1].y = mm->p[i].y;
RemoveMemoryIntervals(mm, i, 1);
}
} else if (i < mm->i && y + 1 == mm->p[i].x && h == mm->h[i]) {
mm->p[i].x = x;
} else {
if (mm->i == ARRAYLEN(mm->p)) return enomem();
CreateMemoryInterval(mm, i);
mm->p[i].x = x;
mm->p[i].y = y;
mm->h[i] = h;
}
return 0;
}

33
libc/runtime/memtrack.h Normal file
View file

@ -0,0 +1,33 @@
#ifndef COSMOPOLITAN_LIBC_RUNTIME_MEMTRACK_H_
#define COSMOPOLITAN_LIBC_RUNTIME_MEMTRACK_H_
#include "libc/nexgen32e/vendor.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
#define kMappingsSize 0x100000000000 /* 16TB */
#define kMappingsStart (IsGenuineCosmo() ? 0x300000000000 : 0x200000000000)
#define kFixedMappingsStart 0x0000100000000000
#define kFixedMappingsSize kMappingsSize
struct MemoryIntervals {
int i;
struct MemoryInterval {
int x;
int y;
} p[32];
long h[32];
};
extern struct MemoryIntervals _mmi;
unsigned FindMemoryInterval(const struct MemoryIntervals *, int) nosideeffect;
bool AreMemoryIntervalsOk(const struct MemoryIntervals *) nosideeffect;
void PrintMemoryIntervals(int, const struct MemoryIntervals *);
int TrackMemoryInterval(struct MemoryIntervals *, int, int, long);
int ReleaseMemoryIntervals(struct MemoryIntervals *, int, int,
void (*)(struct MemoryIntervals *, int, int));
void ReleaseMemoryNt(struct MemoryIntervals *, int, int);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_RUNTIME_MEMTRACK_H_ */

35
libc/runtime/bfree.c → libc/runtime/memtracknt.c
View file

@ -18,23 +18,24 @@
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/runtime/buffer.h"
#include "libc/runtime/mappings.h"
#include "libc/nt/memory.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/memtrack.h"
#include "libc/runtime/runtime.h"
void bfree(struct GuardedBuffer *b) {
int mr;
size_t mi;
struct AddrSize az;
struct MemoryCoord mc;
if (b->p) {
mc = ADDRSIZE_TO_COORD(b->p, 1);
mi = findmapping(mc.y);
assert(mi > 0);
assert(ISOVERLAPPING(mc, _mm.p[mi - 1]));
az = COORD_TO_ADDRSIZE(_mm.p[mi - 1]);
mr = munmap(az.addr, az.size);
assert(mr != -1);
b->p = NULL;
static void *GetFrameAddr(int f) {
intptr_t a;
a = f;
a *= FRAMESIZE;
return (void *)a;
}
void ReleaseMemoryNt(struct MemoryIntervals *mm, int l, int r) {
int i, ok;
for (i = l; i <= r; ++i) {
ok = UnmapViewOfFile(GetFrameAddr(mm->p[i].x));
assert(ok);
ok = CloseHandle(mm->h[i]);
assert(ok);
}
}

3
libc/runtime/missioncritical.h
View file

@ -1,9 +1,11 @@
#ifndef COSMOPOLITAN_LIBC_INTERNAL_MISSIONCRITICAL_H_
#define COSMOPOLITAN_LIBC_INTERNAL_MISSIONCRITICAL_H_
#ifndef __STRICT_ANSI__
#include "libc/bits/bits.h"
#include "libc/calls/internal.h"
#include "libc/dce.h"
#include "libc/nexgen32e/nexgen32e.h"
#include "libc/nexgen32e/tinystrlen.h"
#include "libc/nt/console.h"
#include "libc/nt/enum/version.h"
#include "libc/nt/ntdll.h"
@ -140,4 +142,5 @@ interruptfn void __print(const void *, size_t);
#define RESTORE_RBX() /* disabled for now b/c clang */
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* !ANSI */
#endif /* COSMOPOLITAN_LIBC_INTERNAL_MISSIONCRITICAL_H_ */

150
libc/runtime/mmap.c
View file

@ -24,22 +24,33 @@
#include "libc/macros.h"
#include "libc/nt/memory.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/mappings.h"
#include "libc/rand/rand.h"
#include "libc/runtime/memtrack.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
#define VIP(X) (void *)(intptr_t)(X)
#define IP(X) (intptr_t)(X)
#define VIP(X) (void *)IP(X)
#define COORD(a) (int)(IP(a) >> 16)
#define ADDR(c) (void *)(IP(c) << 16)
#define ALIGNED(p) (!(IP(p) & (FRAMESIZE - 1)))
#define CANONICAL(p) (-0x800000000000 <= IP(p) && IP(p) <= 0x7fffffffffff)
#define LAST_COORD(a, n) (COORD(a) + (ROUNDUP(n, FRAMESIZE) >> 16) - 1)
struct DirectMap {
void *addr;
int64_t maphandle;
};
static textwindows struct DirectMap directmap$nt(void *addr, size_t size,
unsigned prot, unsigned flags,
int fd, int64_t off) {
struct DirectMap res;
struct MemoryIntervals _mmi;
static textwindows struct DirectMap DirectMapNt(void *addr, size_t size,
unsigned prot, unsigned flags,
int fd, int64_t off) {
struct DirectMap res; /* NT IS TORTURE */
if ((res.maphandle = CreateFileMappingNuma(
fd != -1 ? g_fds.p[fd].handle : kNtInvalidHandleValue,
&kNtIsInheritable, prot2nt(prot, flags), size >> 32, size, NULL,
@ -58,22 +69,48 @@ static textwindows struct DirectMap directmap$nt(void *addr, size_t size,
return res;
}
static struct DirectMap directmap(void *addr, size_t size, unsigned prot,
static struct DirectMap DirectMap(void *addr, size_t size, unsigned prot,
unsigned flags, int fd, int64_t off) {
if (!IsWindows()) {
return (struct DirectMap){mmap$sysv(addr, size, prot, flags, fd, off),
kNtInvalidHandleValue};
} else {
return directmap$nt(addr, size, prot, flags, fd, off);
return DirectMapNt(addr, size, prot, flags, fd, off);
}
}
static int UntrackMemoryIntervals(void *addr, size_t size) {
return ReleaseMemoryIntervals(&_mmi, COORD(addr), LAST_COORD(addr, size),
ReleaseMemoryNt);
}
/**
* Releases memory pages.
*
* @param addr is a pointer within any memory mapped region the process
* has permission to control, such as address ranges returned by
* mmap(), the program image itself, etc.
* @param size is the amount of memory to unmap, which needn't be a
* multiple of FRAMESIZE, and may be a subset of that which was
* mapped previously, and may punch holes in existing mappings,
* but your mileage may vary on windows
* @return 0 on success, or -1 w/ errno
*/
int munmap(void *addr, size_t size) {
int rc;
if (!ALIGNED(addr) || !CANONICAL(addr) || !size) return einval();
size = ROUNDUP(size, FRAMESIZE);
if (UntrackMemoryIntervals(addr, size) == -1) return -1;
if (IsWindows()) return 0;
return munmap$sysv(addr, size);
}
/**
* Beseeches system for page-table entries.
*
* @param addr optionally requests a particular virtual base address,
* which needs to be 64kb aligned if passed (for NT compatibility)
* @param size should be >0 and multiple of PAGESIZE
* @param size must be >0 and needn't be a multiple of FRAMESIZE
* @param prot can have PROT_READ, PROT_WRITE, PROT_EXEC, PROT_NONE, etc.
* @param flags can have MAP_ANONYMOUS, MAP_SHARED, MAP_PRIVATE, etc.
* @param fd is an open()'d file descriptor whose contents shall be
@ -83,73 +120,48 @@ static struct DirectMap directmap(void *addr, size_t size, unsigned prot,
* @return virtual base address of new mapping, or MAP_FAILED w/ errno
*/
void *mmap(void *addr, size_t size, int prot, int flags, int fd, int64_t off) {
size_t i;
intptr_t p;
int i;
long gap;
struct DirectMap dm;
struct MemoryCoord c;
p = (intptr_t)addr;
assert(!(0 < p && p < 0x200000));
assert(-0x800000000000L <= p && p <= 0x7fffffffffffL);
assert((flags & MAP_PRIVATE) ^ (flags & MAP_SHARED));
assert((flags & MAP_ANONYMOUS) ^ (fd != -1));
assert(off % PAGESIZE == 0);
assert(size > 0);
if (!(IsWindows() && fd != -1)) {
size = ROUNDUP(size, FRAMESIZE);
}
if (!size) return VIP(einval());
if (!ALIGNED(off)) return VIP(einval());
if (!ALIGNED(addr)) return VIP(einval());
if (!CANONICAL(addr)) return VIP(einval());
if (!(!!(flags & MAP_ANONYMOUS) ^ (fd != -1))) return VIP(einval());
if (!(!!(flags & MAP_PRIVATE) ^ !!(flags & MAP_SHARED))) return VIP(einval());
if (!(IsWindows() && fd != -1)) size = ROUNDUP(size, FRAMESIZE);
if (flags & MAP_FIXED) {
assert(addr != NULL);
assert((intptr_t)addr % FRAMESIZE == 0);
} else {
if (!addr) {
if (_mm.i) {
addr = COORD_TO_ADDR(_mm.p[_mm.i - 1].y + 1);
for (i = _mm.i; i; --i) {
if (_mm.p[i - 1].y + 1 + size / FRAMESIZE <=
ADDR_TO_COORD(kMappingsStart + kMappingsSize) &&
_mm.p[i - 1].y + 1 >= ADDR_TO_COORD(kMappingsStart)) {
addr = COORD_TO_ADDR(_mm.p[i - 1].y + 1);
break;
}
if (UntrackMemoryIntervals(addr, size) == -1) {
return MAP_FAILED;
}
} else if (_mmi.i) {
if (0 && IsModeDbg()) {
addr = VIP(rand64() & 0x00007ffffffff000);
} else {
for (i = _mmi.i - 1; i > 0; --i) {
gap = _mmi.p[i].x - _mmi.p[i - 1].y - 1;
assert(gap > 0);
if (gap >= (ROUNDUP(size, FRAMESIZE) >> 16)) {
addr = ADDR(_mmi.p[i - 1].y + 1);
break;
}
} else {
addr = VIP(kMappingsStart);
}
if (!addr) {
addr = ADDR(_mmi.p[_mmi.i - 1].y + 1);
}
}
addr = (void *)ROUNDDOWN((intptr_t)addr, FRAMESIZE);
}
if (_mm.i == MMAP_MAX) {
return VIP(enomem());
}
if (flags & MAP_FIXED) {
munmap(addr, size);
} else {
c = ADDRSIZE_TO_COORD(addr, size);
if ((i = findmapping(c.y)) && ISOVERLAPPING(c, _mm.p[i - 1])) {
return VIP(einval());
}
addr = VIP(kMappingsStart);
}
dm = directmap(addr, size, prot, flags | MAP_FIXED, fd, off);
if (dm.addr == MAP_FAILED) return MAP_FAILED;
i = findmapping(ADDR_TO_COORD(dm.addr));
if (i < _mm.i) {
memmove(&_mm.p[i + 1], &_mm.p[i],
(intptr_t)&_mm.p[_mm.i] - (intptr_t)&_mm.p[i]);
memmove(&_mm.h[i + 1], &_mm.h[i],
(intptr_t)&_mm.h[_mm.i] - (intptr_t)&_mm.h[i]);
assert((flags & MAP_FIXED) ||
(!isheap(addr) && !isheap((char *)addr + size - 1)));
dm = DirectMap(addr, size, prot, flags | MAP_FIXED, fd, off);
if (dm.addr == MAP_FAILED || dm.addr != addr) {
return MAP_FAILED;
}
if (TrackMemoryInterval(&_mmi, COORD(dm.addr), LAST_COORD(dm.addr, size),
dm.maphandle) == -1) {
_Exit(1);
}
_mm.p[i] = ADDRSIZE_TO_COORD(dm.addr, size);
_mm.h[i] = dm.maphandle;
_mm.i++;
assert((intptr_t)dm.addr % __BIGGEST_ALIGNMENT__ == 0);
return dm.addr;
}

21
libc/runtime/msync-nt.c
View file

@ -18,19 +18,24 @@
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/internal.h"
#include "libc/macros.h"
#include "libc/nt/files.h"
#include "libc/nt/memory.h"
#include "libc/runtime/mappings.h"
#include "libc/runtime/memtrack.h"
textwindows int msync$nt(void *addr, size_t size, int flags) {
size_t i, j;
struct MemoryCoord c;
int x, y, l, r, i;
if (!FlushViewOfFile(addr, size)) return winerr();
j = findmapping(ADDR_TO_COORD(addr));
c = ADDRSIZE_TO_COORD(addr, size);
for (i = j; i; --i) {
if (!ISOVERLAPPING(_mm.p[i - 1], c)) break;
FlushFileBuffers(_mm.h[i - 1]);
x = (intptr_t)addr >> 16;
y = x + (ROUNDUP(size, 65536) >> 16) - 1;
l = FindMemoryInterval(&_mmi, x);
r = FindMemoryInterval(&_mmi, y);
if (l && x <= _mmi.p[l - 1].y) --l;
if (r && y <= _mmi.p[r - 1].y) --r;
if (l < _mmi.i) {
for (i = l; i <= r; --i) {
FlushFileBuffers(_mmi.h[i - 1]);
}
}
return 0;
}

1
libc/runtime/msync.c
View file

@ -22,7 +22,6 @@
#include "libc/calls/internal.h"
#include "libc/dce.h"
#include "libc/macros.h"
#include "libc/runtime/mappings.h"
#include "libc/sysv/consts/msync.h"
/**

106
libc/runtime/munmap.c
View file

@ -1,106 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ This program is free software; you can redistribute it and/or modify │
│ it under the terms of the GNU General Public License as published by │
│ the Free Software Foundation; version 2 of the License. │
│ │
│ This program is distributed in the hope that it will be useful, but │
│ WITHOUT ANY WARRANTY; without even the implied warranty of │
│ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU │
│ General Public License for more details. │
│ │
│ You should have received a copy of the GNU General Public License │
│ along with this program; if not, write to the Free Software │
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
#include "libc/dce.h"
#include "libc/nt/memory.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/mappings.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/fileno.h"
/**
* Releases memory pages.
*
* @param addr is a pointer within any memory mapped region the process
* has permission to control, such as address ranges returned by
* mmap(), the program image itself, etc.
* @param size is the amount of memory to unmap, which should be a
* multiple of PAGESIZE, and may be a subset of that which was
* mapped previously
* @return 0 on success, or -1 w/ errno
*/
int __munmap(void *addr, size_t size) {
int rc;
intptr_t p;
size_t i, j;
struct AddrSize a;
struct MemoryCoord c, m;
p = (intptr_t)addr;
assert(!(0 < p && p < 0x200000));
assert(-0x800000000000L <= p && p <= 0x7fffffffffffL);
if (!size) return 0;
if (!addr || addr == MAP_FAILED) return 0;
if (addr == NULL && size <= 0x200000) return 0;
addr = (void *)ROUNDDOWN((intptr_t)addr, FRAMESIZE);
size = ROUNDUP(size, FRAMESIZE);
rc = 0;
c = ADDRSIZE_TO_COORD(addr, size);
j = findmapping(c.y);
for (i = j; i; --i) {
m = _mm.p[i - 1];
assert(m.x <= m.y);
assert(c.y >= m.x);
if (c.x > m.y) {
break;
} else if (c.x > m.x && c.y < m.y) {
/* remove middle portion */
assert(!"map hole punching not implemented");
} else if (c.x > m.x && c.y >= m.y) {
/* remove righthand portion */
assert(!"map hole punching not implemented");
/* _mm.p[i - 1].y = c.x - 1; */
/* i++; */
/* break; */
} else if (c.x <= m.x && c.y < m.y) {
/* remove lefthand portion */
assert(!"map hole punching not implemented");
/* _mm.p[i - 1].x = c.y + 1; */
/* j--; */
} else if ((m.x >= c.x && m.x <= c.y) && (m.y >= c.x && m.y <= c.y)) {
a = COORD_TO_ADDRSIZE(m);
if (!IsWindows()) {
rc |= munmap$sysv(a.addr, a.size);
} else {
if (!UnmapViewOfFile(a.addr)) rc = -1;
if (!CloseHandle(_mm.h[i - 1])) rc = -1;
}
} else {
assert(!"wut");
}
}
if (i < j) {
if (j < _mm.i) {
memmove(&_mm.p[i], &_mm.p[j],
(intptr_t)&_mm.p[_mm.i] - (intptr_t)&_mm.p[j]);
memmove(&_mm.h[i], &_mm.h[j],
(intptr_t)&_mm.h[_mm.i] - (intptr_t)&_mm.h[j]);
}
_mm.i -= j - i;
}
return rc;
}

3
libc/runtime/munmap_s.c
View file

@ -20,11 +20,10 @@
#include "libc/bits/bits.h"
#include "libc/bits/pushpop.h"
#include "libc/calls/calls.h"
#include "libc/runtime/mappings.h"
#include "libc/runtime/runtime.h"
/**
* Closes memory mapping, the Cosmopolitan way.
* Closes memory mapping.
*
* The caller's address holder is set to MAP_FAILED (-1) which is a
* no-op for subsequent invocations.

14
libc/runtime/opensymboltable.c
View file

@ -21,6 +21,7 @@
#include "libc/calls/calls.h"
#include "libc/elf/def.h"
#include "libc/elf/elf.h"
#include "libc/runtime/carsort.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/symbols.h"
#include "libc/sysv/consts/map.h"
@ -32,18 +33,19 @@
* @return object freeable with closesymboltable(), or NULL w/ errno
*/
struct SymbolTable *opensymboltable(const char *filename) {
struct SymbolTable *t = MAP_FAILED;
const Elf64_Sym *symtab;
unsigned i, j;
struct SymbolTable *t;
const Elf64_Sym *symtab, *sym;
t = MAP_FAILED;
if (filename && (t = mapanon(BIGPAGESIZE)) != MAP_FAILED &&
mapelfread(filename, &t->mf) &&
(t->name_base = getelfstringtable(t->elf, t->elfsize)) != NULL &&
(symtab = getelfsymboltable(t->elf, t->elfsize, &t->count)) &&
sizeof(struct SymbolTable) + sizeof(struct Symbol) * t->count <
(t->scratch = BIGPAGESIZE)) {
unsigned j = 0;
getelfvirtualaddressrange(t->elf, t->elfsize, &t->addr_base, &t->addr_end);
for (unsigned i = 0; i < t->count; ++i) {
const Elf64_Sym *sym = &symtab[i];
for (j = i = 0; i < t->count; ++i) {
sym = &symtab[i];
if (iselfsymbolcontent(sym) &&
(sym->st_value >= t->addr_base && sym->st_value <= t->addr_end)) {
t->symbols[j].addr_rva = (unsigned)(sym->st_value - t->addr_base);
@ -52,7 +54,7 @@ struct SymbolTable *opensymboltable(const char *filename) {
}
}
t->count = j;
heapsortcar((int32_t(*)[2])t->symbols, t->count);
carsort1000(t->count, (void *)t->symbols);
} else {
closesymboltable(&t);
}

13
libc/runtime/piro.c
View file

@ -33,17 +33,16 @@
╠──────────────────────────────────────────────────────▌▀▄─▐──▀▄─▐▄─▐▄▐▄─▐▄─▐▄─│
│ αcτµαlly pδrταblε εxεcµταblε § post-initialization read-only │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/bits/bits.h"
#include "libc/bits/safemacros.h"
#include "libc/bits/weaken.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
#include "libc/sysv/consts/prot.h"
#define getaddr(section) ((intptr_t)weakaddr(section))
static textstartup void __piro_protect(intptr_t start, intptr_t end, int prot) {
static textstartup void _piro_protect(intptr_t start, intptr_t end, int prot) {
ssize_t len = end - start;
if (len > 0 && start && start % PAGESIZE == 0 && len % PAGESIZE == 0) {
if (mprotect((void *)(unsigned long)start, len, prot) == -1) abort();
@ -62,10 +61,10 @@ static textstartup void __piro_protect(intptr_t start, intptr_t end, int prot) {
* @see ape/ape.lds
* @see libc/_start.S
*/
textstartup void __piro(int prot) {
textstartup void _piro(int prot) {
if (getaddr("main") < getaddr("__test_start")) {
__piro_protect(getaddr("__test_start"), getaddr("__test_end"), PROT_NONE);
_piro_protect(getaddr("__test_start"), getaddr("__test_end"), PROT_NONE);
}
__piro_protect(getaddr("__ro"), getaddr("_etext"), PROT_READ);
__piro_protect(getaddr("__piro_start"), getaddr("__piro_end"), prot);
_piro_protect(getaddr("__ro"), getaddr("_etext"), PROT_READ);
_piro_protect(getaddr("__piro_start"), getaddr("__piro_end"), prot);
}

45
libc/runtime/printmemoryintervals.c Normal file
View file

@ -0,0 +1,45 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ This program is free software; you can redistribute it and/or modify │
│ it under the terms of the GNU General Public License as published by │
│ the Free Software Foundation; version 2 of the License. │
│ │
│ This program is distributed in the hope that it will be useful, but │
│ WITHOUT ANY WARRANTY; without even the implied warranty of │
│ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU │
│ General Public License for more details. │
│ │
│ You should have received a copy of the GNU General Public License │
│ along with this program; if not, write to the Free Software │
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/log/log.h"
#include "libc/runtime/memtrack.h"
STATIC_YOINK("ntoa");
STATIC_YOINK("stoa");
void PrintMemoryIntervals(int fd, const struct MemoryIntervals *mm) {
int i, frames, maptally, gaptally;
maptally = 0;
gaptally = 0;
(dprintf)(fd, "%s%zd%s\n", "mm->i == ", mm->i, ";");
for (i = 0; i < mm->i; ++i) {
if (i && mm->p[i].x != mm->p[i - 1].y + 1) {
frames = mm->p[i].x - mm->p[i - 1].y - 1;
gaptally += frames;
(dprintf)(fd, "%s%,zd%s\n", "/* ", frames, " */");
}
frames = mm->p[i].y + 1 - mm->p[i].x;
maptally += frames;
(dprintf)(fd, "%s%3u%s0x%08x,0x%08x%s%,zd%s\n", "mm->p[", i, "]=={",
mm->p[i].x, mm->p[i].y, "}; /* ", frames, " */");
}
(dprintf)(fd, "%s%,zd%s%,zd%s\n\n", "/* ", maptally, " frames mapped w/ ",
gaptally, " frames gapped */");
}

2
libc/runtime/quick_exit.c
View file

@ -17,7 +17,7 @@
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/bits/bits.h"
#include "libc/bits/weaken.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/stdio.h"

42
libc/runtime/relocate.c Normal file
View file

@ -0,0 +1,42 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ This program is free software; you can redistribute it and/or modify │
│ it under the terms of the GNU General Public License as published by │
│ the Free Software Foundation; version 2 of the License. │
│ │
│ This program is distributed in the hope that it will be useful, but │
│ WITHOUT ANY WARRANTY; without even the implied warranty of │
│ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU │
│ General Public License for more details. │
│ │
│ You should have received a copy of the GNU General Public License │
│ along with this program; if not, write to the Free Software │
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/dce.h"
#include "libc/runtime/runtime.h"
/**
* Applies fixups to 64-bit rela addresses in binary.
*/
void __relocate(void) {
ptrdiff_t skew;
unsigned char **p;
p = __relo_start;
if (p != __relo_end) {
skew = (intptr_t)p - (intptr_t)*p;
do {
if (*p) {
*p += skew;
if (!NoDebug() && !(_base <= *p && *p < _end)) {
asm("int3");
}
}
} while (++p != __relo_end);
}
return;
}

54
libc/runtime/runtime.h
View file

@ -7,6 +7,8 @@ COSMOPOLITAN_C_START_
│ cosmopolitan § runtime ─╬─│┼
╚────────────────────────────────────────────────────────────────────────────│*/
struct SymbolTable;
struct StackFrame {
struct StackFrame *next;
intptr_t addr;
@ -14,31 +16,31 @@ struct StackFrame {
typedef long jmp_buf[8] aligned(CACHELINE);
extern int g_argc; /* CRT */
extern char **g_argv; /* CRT */
extern char **environ; /* CRT */
extern unsigned long *g_auxv; /* CRT */
extern jmp_buf g_winmain; /* CRT */
extern char *program_invocation_name; /* RII */
extern char *program_invocation_short_name; /* RII */
extern uint64_t g_syscount; /* RII */
extern const uint64_t kStartTsc; /* RII */
extern const char kTmpPath[]; /* RII */
extern const char kNtSystemDirectory[]; /* RII */
extern const char kNtWindowsDirectory[]; /* RII */
extern unsigned char _base[] aligned(PAGESIZE); /* αpε */
extern char _ehead aligned(PAGESIZE); /* αpε */
extern char _ereal; /* αpε */
extern char __privileged_start; /* αpε */
extern char __test_start; /* αpε */
extern char __ro; /* αpε */
extern char _etext aligned(PAGESIZE); /* αpε */
extern char __piro_start; /* αpε */
extern char _edata aligned(PAGESIZE); /* αpε */
extern char __piro_end; /* αpε */
extern char _end aligned(PAGESIZE); /* αpε */
extern uint8_t __zip_start[]; /* αpε */
extern uint8_t __zip_end[]; /* αpε */
extern int g_argc; /* CRT */
extern char **g_argv; /* CRT */
extern char **environ; /* CRT */
extern unsigned long *g_auxv; /* CRT */
extern jmp_buf g_winmain; /* CRT */
extern char *program_invocation_name; /* RII */
extern char *program_invocation_short_name; /* RII */
extern uint64_t g_syscount; /* RII */
extern const uint64_t kStartTsc; /* RII */
extern const char kTmpPath[]; /* RII */
extern const char kNtSystemDirectory[]; /* RII */
extern const char kNtWindowsDirectory[]; /* RII */
extern unsigned char _base[] aligned(PAGESIZE); /* αpε */
extern unsigned char _ehead[] aligned(PAGESIZE); /* αpε */
extern unsigned char _etext[] aligned(PAGESIZE); /* αpε */
extern unsigned char _edata[] aligned(PAGESIZE); /* αpε */
extern unsigned char _end[] aligned(PAGESIZE); /* αpε */
extern unsigned char _ereal; /* αpε */
extern unsigned char __privileged_start; /* αpε */
extern unsigned char __test_start; /* αpε */
extern unsigned char __ro; /* αpε */
extern unsigned char *__relo_start[]; /* αpε */
extern unsigned char *__relo_end[]; /* αpε */
extern uint8_t __zip_start[]; /* αpε */
extern uint8_t __zip_end[]; /* αpε */
long missingno();
void mcount(void);
@ -70,6 +72,8 @@ void loadxmm(void *);
void peekall(void);
int issetugid(void);
void weakfree(void *) libcesque;
void __hook(void (*)(void), struct SymbolTable *);
bool isheap(void *);
/*───────────────────────────────────────────────────────────────────────────│─╗
│ cosmopolitan § runtime » optimizations ─╬─│┼

2
libc/runtime/runtime.mk
View file

@ -61,6 +61,8 @@ $(LIBC_RUNTIME_A).pkg: \
o/$(MODE)/libc/runtime/asan.greg.o \
o/$(MODE)/libc/runtime/shadowargs.o \
o/$(MODE)/libc/runtime/hook.greg.o \
o/$(MODE)/libc/runtime/ftrace.greg.o \
o/$(MODE)/libc/runtime/__stack_chk_fail.o \
o/$(MODE)/libc/runtime/__stack_chk_guard.o: \
OVERRIDE_COPTS += \

51
libc/runtime/spawn.S Normal file
View file

@ -0,0 +1,51 @@
/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│
│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ This program is free software; you can redistribute it and/or modify │
│ it under the terms of the GNU General Public License as published by │
│ the Free Software Foundation; version 2 of the License. │
│ │
│ This program is distributed in the hope that it will be useful, but │
│ WITHOUT ANY WARRANTY; without even the implied warranty of │
│ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU │
│ General Public License for more details. │
│ │
│ You should have received a copy of the GNU General Public License │
│ along with this program; if not, write to the Free Software │
│ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA │
│ 02110-1301 USA │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/dce.h"
#include "libc/sysv/consts/prot.h"
#include "libc/macros.h"
/ Self-bootstraps process upon existence before calling main.
_spawn: push %rbp
mov %rsp,%rbp
/ Tune FPU settings if -ffast-math is somehow used systemically.
#ifdef __FAST_MATH__
call __fast_math
#endif
/ Call decentralized initialization assembly.
call _init
#if IsModeDbg()
call _init # _init() is idempotent
#endif
/ Call global initialization functions.
call _construct
/ Restricts .initbss memory so it's read-only after initialization.
/ TODO: Delete this unless there's measurable performance advantage.
#if !IsTrustworthy()
mov $PROT_READ,%edi
call _piro
#endif
pop %rbp
ret
.endfn _spawn,globl

3
libc/runtime/sysconf.h
View file

@ -13,8 +13,8 @@ COSMOPOLITAN_C_START_
long sysconf(int);
#if defined(__GNUC__) && !defined(__STRICT_ANSI__)
#define sysconf(X) __sysconf(X)
forceinline long __sysconf(int thing) {
switch (thing) {
case _SC_ARG_MAX:
@ -31,6 +31,7 @@ forceinline long __sysconf(int thing) {
return -1;
}
}
#endif /* GNU && !ANSI */
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

20
libc/runtime/winmain.greg.c
View file

@ -25,6 +25,7 @@
#include "libc/nt/runtime.h"
#include "libc/runtime/getdosenviron.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/missioncritical.h"
static void LoadFasterAndPreventHijacking(void) {
unsigned wrote;
@ -34,14 +35,15 @@ static void LoadFasterAndPreventHijacking(void) {
}
}
noreturn textwindows int WinMain(void *hInstance, void *hPrevInstance,
const char *lpCmdLine, int nCmdShow) {
textwindows int WinMain(void *hInstance, void *hPrevInstance,
const char *lpCmdLine, int nCmdShow) {
int i, count;
const char16_t *cmd16, *env16;
char *argarray[512], *envarray[512];
char argblock[ARG_MAX], envblock[ENV_MAX];
long auxarray[][2] = {{pushpop(0L), pushpop(0L)}};
char envblock[ENV_MAX], *envarray[512], argblock[ARG_MAX], *argarray[512];
LoadFasterAndPreventHijacking();
*(/*unconst*/ int *)&__hostos = pushpop(WINDOWS);
*(/*unconst*/ int *)&hostos = WINDOWS;
cmd16 = GetCommandLine();
env16 = GetEnvironmentStrings();
count = getdosargv(cmd16, argblock, ARG_MAX, argarray, 512);
@ -50,13 +52,5 @@ noreturn textwindows int WinMain(void *hInstance, void *hPrevInstance,
}
getdosenviron(env16, envblock, ENV_MAX, envarray, 512);
FreeEnvironmentStrings(env16);
register int argc asm("r12") = count;
register char **argv asm("r13") = argarray;
register char **envp asm("r14") = envarray;
register long(*auxv)[2] asm("r15") = auxarray;
asm volatile("jmp\t__executive"
: /* no outputs */
: "r"(argc), "r"(argv), "r"(envp), "r"(auxv)
: "memory", "cc");
unreachable;
_executive(count, argarray, envarray, auxarray);
}