mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-07-12 14:09:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Make more threading improvements

Browse source 
- ASAN memory morgue is now lockless
- Make C11 atomics header more portable
- Rewrote pthread keys support to be lockless
- Simplify Python's unicode table unpacking code
- Make crash report write(2) closer to being atomic
- Make it possible to strace/ftrace a single thread
- ASAN now checks nul-terminated strings fast and properly
- Windows fork() now restores TLS memory of calling thread
This commit is contained in:
Justine Tunney 2022-11-01 22:36:03 -07:00
parent d7b88734cd
commit e522aa3a07
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
189 changed files with 1363 additions and 1217 deletions
Download patch file Download diff file Expand all files Collapse all files

33
libc/intrin/__pid_exec.S Normal file
View file

@ -0,0 +1,33 @@
/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│
│vi: set et ft=asm ts=8 sw=8 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macros.internal.h"
#include "libc/notice.inc"
// __pid_exec is __pid that isn't updated on fork()
.initbss 301,_init__pid_exec
__pid_exec:
.quad 0
.endobj __pid_exec,globl
.previous
.init.start 301,_init__pid_exec
mov __pid(%rip),%rax
stosq
.init.end 301,_init__pid_exec

339
libc/intrin/asan.c
View file

@ -16,47 +16,35 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/calls/state.internal.h"
#include "libc/calls/struct/iovec.h"
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/asancodes.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/cmpxchg.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/kmalloc.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/leaky.internal.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/lockcmpxchg.h"
#include "libc/intrin/nomultics.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/intrin/tpenc.h"
#include "libc/intrin/weaken.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/internal.h"
#include "libc/log/libfatal.internal.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
#include "libc/mem/hook/hook.internal.h"
#include "libc/mem/mem.h"
#include "libc/mem/reverse.internal.h"
#include "libc/nexgen32e/gc.internal.h"
#include "libc/nexgen32e/stackframe.h"
#include "libc/nt/enum/version.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/runtime/symbols.internal.h"
#include "libc/str/str.h"
#include "libc/str/tab.internal.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/nr.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
#include "third_party/dlmalloc/dlmalloc.h"
@ -109,6 +97,8 @@ STATIC_YOINK("_init_asan");
* movq (%addr),%dst
*/
#define RBP __builtin_frame_address(0)
#define HOOK(HOOK, IMPL) \
do { \
if (_weaken(HOOK)) { \
@ -116,17 +106,13 @@ STATIC_YOINK("_init_asan");
} \
} while (0)
#define REQUIRE(FUNC) \
do { \
if (!_weaken(FUNC)) { \
kprintf("error: asan needs %s\n", #FUNC); \
__asan_die()(); \
__asan_unreachable(); \
} \
#define REQUIRE(FUNC) \
do { \
if (!_weaken(FUNC)) { \
__asan_require_failed(#FUNC); \
} \
} while (0)
typedef char xmm_t __attribute__((__vector_size__(16), __aligned__(1)));
struct AsanTrace {
uint32_t p[ASAN_TRACE_ITEMS]; // assumes linkage into 32-bit space
};
@ -161,20 +147,22 @@ struct AsanGlobal {
char *odr_indicator;
};
struct AsanMorgue {
unsigned i;
void *p[ASAN_MORGUE_ITEMS];
};
struct ReportOriginHeap {
const unsigned char *a;
int z;
};
static char __asan_bigbuf[FRAMESIZE];
static int __asan_noreentry;
static pthread_spinlock_t __asan_lock;
static struct AsanMorgue __asan_morgue;
static struct AsanMorgue {
_Atomic(unsigned) i;
_Atomic(void *) p[ASAN_MORGUE_ITEMS];
} __asan_morgue;
static bool __asan_once(void) {
bool want = false;
static _Atomic(bool) once;
return atomic_compare_exchange_strong_explicit(
&once, &want, true, memory_order_relaxed, memory_order_relaxed);
}
#define __asan_unreachable() \
do { \
@ -363,6 +351,12 @@ dontdiscard static __asan_die_f *__asan_die(void) {
}
}
static wontreturn void __asan_require_failed(const char *func) {
kprintf("error: asan needs %s\n", func);
__asan_die()();
__asan_unreachable();
}
void __asan_poison(void *p, long n, signed char t) {
signed char k, *s;
s = (signed char *)(((intptr_t)p >> 3) + 0x7fff8000);
@ -433,17 +427,14 @@ static struct AsanFault __asan_fault(const signed char *s, signed char dflt) {
}
static struct AsanFault __asan_checka(const signed char *s, long ndiv8) {
intptr_t a;
uint64_t w;
const signed char *e = s + ndiv8;
for (; ((intptr_t)s & 7) && s < e; ++s) {
if (*s) return __asan_fault(s - 1, kAsanHeapOverrun);
}
for (; s + 8 <= e; s += 8) {
if (UNLIKELY(!((a = (intptr_t)s) & 0xffff))) {
if (kisdangerous((void *)a)) {
return (struct AsanFault){kAsanUnmapped, s};
}
if (UNLIKELY(!((intptr_t)s & (FRAMESIZE - 1))) && kisdangerous(s)) {
return (struct AsanFault){kAsanUnmapped, s};
}
if ((w = ((uint64_t)(255 & s[0]) << 000 | (uint64_t)(255 & s[1]) << 010 |
(uint64_t)(255 & s[2]) << 020 | (uint64_t)(255 & s[3]) << 030 |
@ -462,25 +453,27 @@ static struct AsanFault __asan_checka(const signed char *s, long ndiv8) {
/**
* Checks validity of memory range.
*
* Normally this is abstracted by the compiler.
* This is normally abstracted by the compiler. In some cases, it may be
* desirable to perform an ASAN memory safety check explicitly, e.g. for
* system call wrappers that need to vet memory passed to the kernel, or
* string library routines that use the `noasan` keyword due to compiler
* generated ASAN being too costly. This function is fast especially for
* large memory ranges since this takes a few picoseconds for each byte.
*
* @param p is starting virtual address
* @param n is number of bytes to check
* @return kind is 0 on success or <0 on invalid
* @return kind is 0 on success or <0 if invalid
* @return shadow points to first poisoned shadow byte
* @note this takes 6 picoseconds per byte
*/
struct AsanFault __asan_check(const void *p, long n) {
intptr_t a;
struct AsanFault f;
signed char c, k, *s;
if (n > 0) {
k = (intptr_t)p & 7;
a = ((intptr_t)p >> 3) + 0x7fff8000;
s = (signed char *)a;
s = SHADOW(p);
if (OverlapsShadowSpace(p, n)) {
return (struct AsanFault){kAsanProtected, s};
} else if (kisdangerous((void *)a)) {
} else if (kisdangerous(s)) {
return (struct AsanFault){kAsanUnmapped, s};
}
if (UNLIKELY(k)) {
@ -509,12 +502,95 @@ struct AsanFault __asan_check(const void *p, long n) {
}
}
/**
* Checks validity of nul-terminated string.
*
* This is similar to `__asan_check(p, 1)` except it checks the validity
* of the entire string including its trailing nul byte, and goes faster
* than calling strlen() beforehand.
*
* @param p is starting virtual address
* @param n is number of bytes to check
* @return kind is 0 on success or <0 if invalid
* @return shadow points to first poisoned shadow byte
*/
struct AsanFault __asan_check_str(const char *p) {
uint64_t w;
struct AsanFault f;
signed char c, k, *s;
s = SHADOW(p);
if (OverlapsShadowSpace(p, 1)) {
return (struct AsanFault){kAsanProtected, s};
}
if (kisdangerous(s)) {
return (struct AsanFault){kAsanUnmapped, s};
}
if ((k = (intptr_t)p & 7)) {
do {
if ((c = *s) && c < k + 1) {
return __asan_fault(s, kAsanHeapOverrun);
}
if (!*p++) {
return (struct AsanFault){0};
}
} while ((k = (intptr_t)p & 7));
++s;
}
for (;; ++s, p += 8) {
if (UNLIKELY(!((intptr_t)s & (FRAMESIZE - 1))) && kisdangerous(s)) {
return (struct AsanFault){kAsanUnmapped, s};
}
if ((c = *s) < 0) {
return (struct AsanFault){c, s};
}
w = *(const uint64_t *)p;
if (!(w = ~w & (w - 0x0101010101010101) & 0x8080808080808080)) {
if (c > 0) {
return __asan_fault(s, kAsanHeapOverrun);
}
} else {
k = (unsigned)__builtin_ctzll(w) >> 3;
if (!c || c > k) {
return (struct AsanFault){0};
} else {
return __asan_fault(s, kAsanHeapOverrun);
}
}
}
}
/**
* Checks memory validity of memory region.
*/
bool __asan_is_valid(const void *p, long n) {
struct AsanFault f;
f = __asan_check(p, n);
return !f.kind;
}
/**
* Checks memory validity of nul-terminated string.
*/
bool __asan_is_valid_str(const char *p) {
struct AsanFault f;
f = __asan_check_str(p);
return !f.kind;
}
/**
* Checks memory validity of null-terminated nul-terminated string list.
*/
bool __asan_is_valid_strlist(char *const *p) {
for (;; ++p) {
if (!__asan_is_valid(p, sizeof(char *))) return false;
if (!*p) return true;
if (!__asan_is_valid_str(*p)) return false;
}
}
/**
* Checks memory validity of `iov` array and each buffer it describes.
*/
bool __asan_is_valid_iov(const struct iovec *iov, int iovlen) {
int i;
size_t size;
@ -532,15 +608,7 @@ bool __asan_is_valid_iov(const struct iovec *iov, int iovlen) {
}
}
bool __asan_is_valid_strlist(char *const *p) {
for (;; ++p) {
if (!__asan_is_valid(p, sizeof(char *))) return false;
if (!*p) return true;
if (!__asan_is_valid(*p, 1)) return false;
}
}
wint_t __asan_symbolize_access_poison(signed char kind) {
static wint_t __asan_symbolize_access_poison(signed char kind) {
switch (kind) {
case kAsanNullPage:
return L'∅';
@ -589,7 +657,7 @@ wint_t __asan_symbolize_access_poison(signed char kind) {
}
}
const char *__asan_describe_access_poison(signed char kind) {
static const char *__asan_describe_access_poison(signed char kind) {
switch (kind) {
case kAsanNullPage:
return "null pointer dereference";
@ -638,7 +706,7 @@ const char *__asan_describe_access_poison(signed char kind) {
}
}
dontdiscard static __asan_die_f *__asan_report_invalid_pointer(
static dontdiscard __asan_die_f *__asan_report_invalid_pointer(
const void *addr) {
kprintf("\n\e[J\e[1;31masan error\e[0m: this corruption at %p shadow %p\n",
addr, SHADOW(addr));
@ -702,7 +770,7 @@ static void __asan_report_memory_origin_image(intptr_t a, int z) {
}
}
static noasan void OnMemory(void *x, void *y, size_t n, void *a) {
static noasan void __asan_onmemory(void *x, void *y, size_t n, void *a) {
const unsigned char *p = x;
struct ReportOriginHeap *t = a;
if ((p <= t->a && t->a < p + n) ||
@ -720,7 +788,7 @@ static void __asan_report_memory_origin_heap(const unsigned char *a, int z) {
if (_weaken(malloc_inspect_all)) {
t.a = a;
t.z = z;
_weaken(malloc_inspect_all)(OnMemory, &t);
_weaken(malloc_inspect_all)(__asan_onmemory, &t);
} else {
kprintf("\tunknown please STATIC_YOINK(\"malloc_inspect_all\");\n");
}
@ -763,10 +831,10 @@ dontdiscard static __asan_die_f *__asan_report(const void *addr, int size,
wint_t c;
signed char t;
uint64_t x, y, z;
char *p, *q, *base;
char *p, *q, *buf, *base;
struct MemoryIntervals *m;
--__ftrace;
p = __asan_bigbuf;
ftrace_enabled(-1);
p = buf = kmalloc(1024 * 1024);
kprintf("\n\e[J\e[1;31masan error\e[0m: %s %d-byte %s at %p shadow %p\n",
__asan_describe_access_poison(kind), size, message, addr,
SHADOW(addr));
@ -840,17 +908,16 @@ dontdiscard static __asan_die_f *__asan_report(const void *addr, int size,
}
__mmi_unlock();
*p = 0;
kprintf("%s", __asan_bigbuf);
kprintf("%s", buf);
__asan_report_memory_origin(addr, size, kind);
kprintf("\nthe crash was caused by\n");
++__ftrace;
ftrace_enabled(+1);
return __asan_die();
}
void __asan_verify(const void *p, size_t n) {
static wontreturn void __asan_verify_failed(const void *p, size_t n,
struct AsanFault f) {
const char *q;
struct AsanFault f;
if (!(f = __asan_check(p, n)).kind) return;
q = UNSHADOW(f.shadow);
if ((uintptr_t)q != ((uintptr_t)p & -8) && (uintptr_t)q - (uintptr_t)p < n) {
n -= (uintptr_t)q - (uintptr_t)p;
@ -860,34 +927,40 @@ void __asan_verify(const void *p, size_t n) {
__asan_unreachable();
}
dontdiscard __asan_die_f *__asan_report_memory_fault(void *addr, int size,
const char *message) {
void __asan_verify(const void *p, size_t n) {
struct AsanFault f;
if (!(f = __asan_check(p, n)).kind) return;
__asan_verify_failed(p, n, f);
}
void __asan_verify_str(const char *p) {
struct AsanFault f;
if (!(f = __asan_check_str(p)).kind) return;
__asan_verify_failed(UNSHADOW(f.shadow), 8, f);
}
static dontdiscard __asan_die_f *__asan_report_memory_fault(
void *addr, int size, const char *message) {
return __asan_report(addr, size, message,
__asan_fault(SHADOW(addr), -128).kind);
}
void *__asan_morgue_add(void *p) {
int i;
void *r;
if (__threaded) pthread_spin_lock(&__asan_lock);
i = __asan_morgue.i++ & (ARRAYLEN(__asan_morgue.p) - 1);
r = __asan_morgue.p[i];
__asan_morgue.p[i] = p;
if (__threaded) pthread_spin_unlock(&__asan_lock);
return r;
static void *__asan_morgue_add(void *p) {
return atomic_exchange_explicit(
__asan_morgue.p + (atomic_fetch_add_explicit(&__asan_morgue.i, 1,
memory_order_acquire) &
(ARRAYLEN(__asan_morgue.p) - 1)),
p, memory_order_release);
}
static void __asan_morgue_flush(void) {
int i;
void *p;
if (__threaded) pthread_spin_lock(&__asan_lock);
unsigned i;
for (i = 0; i < ARRAYLEN(__asan_morgue.p); ++i) {
if (__asan_morgue.p[i] && _weaken(dlfree)) {
_weaken(dlfree)(__asan_morgue.p[i]);
if (atomic_load_explicit(__asan_morgue.p + i, memory_order_acquire)) {
_weaken(dlfree)(atomic_exchange_explicit(__asan_morgue.p + i, 0,
memory_order_release));
}
__asan_morgue.p[i] = 0;
}
if (__threaded) pthread_spin_unlock(&__asan_lock);
}
static size_t __asan_user_size(size_t n) {
@ -963,8 +1036,8 @@ static void __asan_trace(struct AsanTrace *bt, const struct StackFrame *bp) {
#define __asan_trace __asan_rawtrace
void *__asan_allocate(size_t a, size_t n, int underrun, int overrun,
struct AsanTrace *bt) {
static void *__asan_allocate(size_t a, size_t n, struct AsanTrace *bt,
int underrun, int overrun, int initializer) {
char *p;
size_t c;
struct AsanExtra *e;
@ -975,13 +1048,17 @@ void *__asan_allocate(size_t a, size_t n, int underrun, int overrun,
__asan_unpoison(p, n);
__asan_poison(p - 16, 16, underrun); /* see dlmalloc design */
__asan_poison(p + n, c - n, overrun);
__asan_memset(p, 0xF9, n);
__asan_memset(p, initializer, n);
__asan_write48(&e->size, n);
__asan_memcpy(&e->bt, bt, sizeof(*bt));
}
return p;
}
static void *__asan_allocate_heap(size_t a, size_t n, struct AsanTrace *bt) {
return __asan_allocate(a, n, bt, kAsanHeapUnderrun, kAsanHeapOverrun, 0xf9);
}
static struct AsanExtra *__asan_get_extra(const void *p, size_t *c) {
int f;
long x, n;
@ -1003,31 +1080,20 @@ static struct AsanExtra *__asan_get_extra(const void *p, size_t *c) {
size_t __asan_get_heap_size(const void *p) {
size_t n, c;
struct AsanExtra *e;
if ((e = __asan_get_extra(p, &c))) {
if (__asan_read48(e->size, &n)) {
return n;
} else {
return 0;
}
} else {
return 0;
if ((e = __asan_get_extra(p, &c)) && __asan_read48(e->size, &n)) {
return n;
}
return 0;
}
static size_t __asan_malloc_usable_size(const void *p) {
size_t n, c;
struct AsanExtra *e;
if ((e = __asan_get_extra(p, &c))) {
if (__asan_read48(e->size, &n)) {
return n;
} else {
__asan_report_invalid_pointer(p)();
__asan_unreachable();
}
} else {
__asan_report_invalid_pointer(p)();
__asan_unreachable();
if ((e = __asan_get_extra(p, &c)) && __asan_read48(e->size, &n)) {
return n;
}
__asan_report_invalid_pointer(p)();
__asan_unreachable();
}
int __asan_print_trace(void *p) {
@ -1122,7 +1188,7 @@ static void *__asan_realloc_nogrow(void *p, size_t n, size_t m,
static void *__asan_realloc_grow(void *p, size_t n, size_t m,
struct AsanTrace *bt) {
char *q;
if ((q = __asan_allocate(16, n, kAsanHeapUnderrun, kAsanHeapOverrun, bt))) {
if ((q = __asan_allocate_heap(16, n, bt))) {
__asan_memcpy(q, p, m);
__asan_deallocate(p, kAsanHeapRelocated);
}
@ -1136,48 +1202,41 @@ static void *__asan_realloc_impl(void *p, size_t n,
struct AsanExtra *e;
if ((e = __asan_get_extra(p, &c))) {
if (__asan_read48(e->size, &m)) {
if (n <= m) { /* shrink */
if (n <= m) { // shrink
__asan_poison((char *)p + n, m - n, kAsanHeapOverrun);
__asan_write48(&e->size, n);
return p;
} else if (n <= c - sizeof(struct AsanExtra)) { /* small growth */
} else if (n <= c - sizeof(struct AsanExtra)) { // small growth
__asan_unpoison((char *)p + m, n - m);
__asan_write48(&e->size, n);
return p;
} else { /* exponential growth */
} else { // exponential growth
return grow(p, n, m, &e->bt);
}
} else {
__asan_report_invalid_pointer(p)();
__asan_unreachable();
}
} else {
__asan_report_invalid_pointer(p)();
__asan_unreachable();
}
__asan_report_invalid_pointer(p)();
__asan_unreachable();
}
void *__asan_malloc(size_t size) {
struct AsanTrace bt;
__asan_trace(&bt, __builtin_frame_address(0));
return __asan_allocate(16, size, kAsanHeapUnderrun, kAsanHeapOverrun, &bt);
__asan_trace(&bt, RBP);
return __asan_allocate_heap(16, size, &bt);
}
void *__asan_memalign(size_t align, size_t size) {
struct AsanTrace bt;
__asan_trace(&bt, __builtin_frame_address(0));
return __asan_allocate(align, size, kAsanHeapUnderrun, kAsanHeapOverrun, &bt);
__asan_trace(&bt, RBP);
return __asan_allocate_heap(align, size, &bt);
}
void *__asan_calloc(size_t n, size_t m) {
char *p;
struct AsanTrace bt;
__asan_trace(&bt, __builtin_frame_address(0));
__asan_trace(&bt, RBP);
if (__builtin_mul_overflow(n, m, &n)) n = -1;
if ((p = __asan_allocate(16, n, kAsanHeapUnderrun, kAsanHeapOverrun, &bt))) {
__asan_memset(p, 0, n);
}
return p;
return __asan_allocate(16, n, &bt, kAsanHeapUnderrun, kAsanHeapOverrun, 0x00);
}
void *__asan_realloc(void *p, size_t n) {
@ -1190,32 +1249,25 @@ void *__asan_realloc(void *p, size_t n) {
return 0;
}
} else {
__asan_trace(&bt, __builtin_frame_address(0));
return __asan_allocate(16, n, kAsanHeapUnderrun, kAsanHeapOverrun, &bt);
__asan_trace(&bt, RBP);
return __asan_allocate_heap(16, n, &bt);
}
}
void *__asan_realloc_in_place(void *p, size_t n) {
if (p) {
return __asan_realloc_impl(p, n, __asan_realloc_nogrow);
} else {
return 0;
}
return p ? __asan_realloc_impl(p, n, __asan_realloc_nogrow) : 0;
}
int __asan_malloc_trim(size_t pad) {
__asan_morgue_flush();
if (_weaken(dlmalloc_trim)) {
return _weaken(dlmalloc_trim)(pad);
} else {
return 0;
}
return _weaken(dlmalloc_trim) ? _weaken(dlmalloc_trim)(pad) : 0;
}
void *__asan_stack_malloc(size_t size, int classid) {
struct AsanTrace bt;
__asan_trace(&bt, __builtin_frame_address(0));
return __asan_allocate(16, size, kAsanStackUnderrun, kAsanStackOverrun, &bt);
__asan_trace(&bt, RBP);
return __asan_allocate(16, size, &bt, kAsanStackUnderrun, kAsanStackOverrun,
0xf9);
}
void __asan_stack_free(char *p, size_t size, int classid) {
@ -1247,14 +1299,14 @@ void __asan_unregister_globals(struct AsanGlobal g[], int n) {
void __asan_evil(uint8_t *addr, int size, const char *s) {
struct AsanTrace tr;
__asan_rawtrace(&tr, __builtin_frame_address(0));
__asan_rawtrace(&tr, RBP);
kprintf("WARNING: ASAN bad %d byte %s at %x bt %x %x %x\n", size, s, addr,
tr.p[0], tr.p[1], tr.p[2], tr.p[3]);
}
void __asan_report_load(uint8_t *addr, int size) {
__asan_evil(addr, size, "load");
if (!__vforked && _lockcmpxchg(&__asan_noreentry, 0, 1)) {
if (!__vforked && __asan_once()) {
__asan_report_memory_fault(addr, size, "load")();
__asan_unreachable();
}
@ -1262,7 +1314,7 @@ void __asan_report_load(uint8_t *addr, int size) {
void __asan_report_store(uint8_t *addr, int size) {
__asan_evil(addr, size, "store");
if (!__vforked && _lockcmpxchg(&__asan_noreentry, 0, 1)) {
if (!__vforked && __asan_once()) {
__asan_report_memory_fault(addr, size, "store")();
__asan_unreachable();
}
@ -1412,7 +1464,7 @@ static textstartup void __asan_shadow_existing_mappings(void) {
textstartup void __asan_init(int argc, char **argv, char **envp,
intptr_t *auxv) {
static bool once;
if (!_lockcmpxchg(&once, false, true)) return;
if (!_cmpxchg(&once, false, true)) return;
if (IsWindows() && NtGetVersion() < kNtVersionWindows10) {
__write_str("error: asan binaries require windows10\r\n");
_Exitr(0); /* So `make MODE=dbg test` passes w/ Windows7 */
@ -1423,6 +1475,7 @@ textstartup void __asan_init(int argc, char **argv, char **envp,
if (_weaken(hook_malloc) || _weaken(hook_calloc) || _weaken(hook_realloc) ||
_weaken(hook_realloc_in_place) || _weaken(hook_free) ||
_weaken(hook_malloc_usable_size)) {
REQUIRE(dlfree);
REQUIRE(dlmemalign);
REQUIRE(dlmalloc_usable_size);
}

7
libc/intrin/asan.internal.h
View file

@ -19,13 +19,14 @@ struct AsanFault {
void __asan_unpoison(void *, long);
void __asan_poison(void *, long, signed char);
void __asan_verify(const void *, size_t);
void __asan_verify_str(const char *);
void __asan_map_shadow(uintptr_t, size_t);
bool __asan_is_valid(const void *, long) nosideeffect;
bool __asan_is_valid_strlist(char *const *) strlenesque;
bool __asan_is_valid_str(const char *) nosideeffect;
bool __asan_is_valid_strlist(char *const *) nosideeffect;
bool __asan_is_valid_iov(const struct iovec *, int) nosideeffect;
wint_t __asan_symbolize_access_poison(signed char) pureconst;
const char *__asan_describe_access_poison(signed char) pureconst;
struct AsanFault __asan_check(const void *, long) nosideeffect;
struct AsanFault __asan_check_str(const char *) nosideeffect;
void __asan_free(void *);
void *__asan_malloc(size_t);

18
libc/intrin/assertfail.greg.c
View file

@ -17,31 +17,29 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/atomic.h"
#include "libc/calls/state.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/lockcmpxchgp.h"
#include "libc/intrin/weaken.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/internal.h"
#include "libc/runtime/internal.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/symbols.internal.h"
/**
* Handles failure of assert() macro.
*/
relegated void __assert_fail(const char *expr, const char *file, int line) {
int me, owner;
static int sync;
static atomic_int once;
if (!__assert_disable) {
--__strace;
--__ftrace;
strace_enabled(-1);
ftrace_enabled(-1);
owner = 0;
me = sys_gettid();
kprintf("%s:%d: assert(%s) failed (tid %d)\n", file, line, expr, me);
if (__vforked || _lockcmpxchgp(&sync, &owner, me)) {
if (__vforked ||
atomic_compare_exchange_strong_explicit(
&once, &owner, me, memory_order_relaxed, memory_order_relaxed)) {
__restore_tty();
if (_weaken(ShowBacktrace)) {
_weaken(ShowBacktrace)(2, __builtin_frame_address(0));

92
libc/intrin/atomic.h
View file

@ -6,11 +6,9 @@
/**
* @fileoverview Cosmopolitan C11 Atomics Library
*
* - Forty-two different ways to say MOV.
* - Fourteen different ways to say XCHG.
* - Twenty different ways to say LOCK CMPXCHG.
*
* It's a lower level programming language than assembly!
* - Forty-two different ways to say MOV.
* - Fourteen different ways to say XCHG.
* - Twenty different ways to say LOCK CMPXCHG.
*
* @see libc/atomic.h
*/
@ -23,7 +21,7 @@
#define memory_order_acq_rel 4
#define memory_order_seq_cst 5
#define ATOMIC_VAR_INIT(value) (value)
#define ATOMIC_VAR_INIT(...) __VA_ARGS__
#define atomic_is_lock_free(obj) ((void)(obj), sizeof(obj) <= sizeof(void *))
#define atomic_flag atomic_bool
@ -113,27 +111,26 @@
#define atomic_store_explicit(pObject, desired, order) \
__atomic_store_n(pObject, desired, order)
#else
#elif (__GNUC__ + 0) * 100 + (__GNUC_MINOR__ + 0) >= 401
#define atomic_init(obj, value) ((void)(*(obj) = (value)))
#define atomic_thread_fence(order) __sync_synchronize()
#define atomic_signal_fence(order) __asm__ volatile("" ::: "memory")
#define __atomic_apply_stride(object, operand) \
(((__typeof__(__atomic_val(object)))0) + (operand))
#define atomic_compare_exchange_strong_explicit(object, expected, desired, \
success, failure) \
__extension__({ \
__typeof__(expected) __ep = (expected); \
__typeof__(*__ep) __e = *__ep; \
(void)(success); \
(void)(failure); \
(_Bool)((*__ep = __sync_val_compare_and_swap(object, __e, desired)) == \
__e); \
(((__typeof__(*(object)))0) + (operand))
#define atomic_compare_exchange_strong_explicit(object, expected, desired, \
success_order, failure_order) \
__extension__({ \
__typeof__(expected) __ep = (expected); \
__typeof__(*__ep) __e = *__ep; \
(void)(success_order); \
(void)(failure_order); \
(*__ep = __sync_val_compare_and_swap(object, __e, desired)) == __e; \
})
#define atomic_compare_exchange_weak_explicit(object, expected, desired, \
success, failure) \
atomic_compare_exchange_strong_explicit(object, expected, desired, success, \
failure)
#define atomic_compare_exchange_weak_explicit(object, expected, desired, \
success_order, failure_order) \
atomic_compare_exchange_strong_explicit(object, expected, desired, \
success_order, failure_order)
#if __has_builtin(__sync_swap)
#define atomic_exchange_explicit(object, desired, order) \
((void)(order), __sync_swap(object, desired))
@ -144,7 +141,7 @@
__typeof__(desired) __d = (desired); \
(void)(order); \
__sync_synchronize(); \
__sync_lock_test_and_set(&__atomic_val(__o), __d); \
__sync_lock_test_and_set(__o, __d); \
})
#endif
#define atomic_fetch_add_explicit(object, operand, order) \
@ -164,6 +161,57 @@
#define atomic_store_explicit(object, desired, order) \
((void)atomic_exchange_explicit(object, desired, order))
#elif defined(__GNUC__) && defined(__x86__) /* x86 with gcc 4.0 and earlier */
#define atomic_init(obj, value) ((void)(*(obj) = (value)))
#define atomic_thread_fence(order) __asm__ volatile("mfence" ::: "memory")
#define atomic_signal_fence(order) __asm__ volatile("" ::: "memory")
#define atomic_compare_exchange_strong_explicit(object, expected, desired, \
success_order, failure_order) \
__extension__({ \
char DidIt; \
__typeof__(object) IfThing = (object); \
__typeof__(IfThing) IsEqualToMe = (expected); \
__typeof__(*IfThing) ReplaceItWithMe = (desired), ax; \
(void)(success_order); \
(void)(failure_order); \
__asm__ volatile("lock cmpxchg\t%3,(%1)\n\t" \
"setz\t%b0" \
: "=q"(DidIt), "=r"(IfThing), "+a"(ax) \
: "r"(ReplaceItWithMe), "2"(*IsEqualToMe) \
: "memory", "cc"); \
*IsEqualToMe = ax; \
DidIt; \
})
#define atomic_compare_exchange_weak_explicit(object, expected, desired, \
success_order, failure_order) \
atomic_compare_exchange_strong_explicit(object, expected, desired, \
success_order, failure_order)
#define atomic_exchange_explicit(object, desired, order) \
__extension__({ \
__typeof__(object) __o = (object); \
__typeof__(*__o) __d = (desired); \
(void)(order); \
__asm__ volatile("xchg\t%0,%1" : "=r"(__d), "+m"(*__o) : "0"(__d)); \
__d; \
})
#define atomic_fetch_add_explicit(object, operand, order) \
__extension__({ \
__typeof__(object) __o = (object); \
__typeof__(*__o) __d = (desired); \
(void)(order); \
__asm__ volatile("lock xadd\t%0,%1" : "=r"(__d), "+m"(*__o) : "0"(__d)); \
__d; \
})
#define atomic_fetch_sub_explicit(object, operand, order) \
atomic_fetch_add_explicit(object, -(operand), order)
#define atomic_load_explicit(object, order) \
atomic_fetch_add_explicit(object, 0, order)
#define atomic_store_explicit(object, desired, order) \
((void)atomic_exchange_explicit(object, desired, order))
#else /* non-gcc or old gcc w/o x86 */
#error "atomic operations not supported with this compiler and/or architecture"
#endif
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

7
libc/intrin/cmpxchg.h
View file

@ -4,7 +4,7 @@
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
#if defined(__GNUC__) && !defined(__STRICT_ANSI__) && defined(__x86__)
#if defined(__GNUC__) && !defined(__STRICT_ANSI__) && !defined(__x86__)
#define _cmpxchg(IFTHING, ISEQUALTOME, REPLACEITWITHME) \
({ \
bool DidIt; \
@ -17,7 +17,10 @@ COSMOPOLITAN_C_START_
: "cc"); \
DidIt; \
})
#endif /* GNUC && !ANSI && x86 */
#else
#define _cmpxchg(IFTHING, ISEQUALTOME, REPLACEITWITHME) \
(*(IFTHING) == (ISEQUALTOME) ? (*(IFTHING) = (REPLACEITWITHME), 1) : 0)
#endif
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

2
libc/intrin/describeiovec.c
View file

@ -62,7 +62,7 @@ const char *(DescribeIovec)(char buf[N], ssize_t rc, const struct iovec *iov,
iov[i].iov_len);
}
append("%s}", iovlen > 2 ? ", ..." : "");
append("}");
return buf;
}

59
libc/intrin/describepollfds.c Normal file
View file

@ -0,0 +1,59 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/dce.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/limits.h"
#include "libc/macros.internal.h"
#include "libc/sock/struct/pollfd.h"
#include "libc/sock/struct/pollfd.internal.h"
#define N 300
#define append(...) o += ksnprintf(buf + o, N - o, __VA_ARGS__)
const char *(DescribePollFds)(char buf[N], ssize_t rc, struct pollfd *fds,
size_t nfds) {
char b64[64];
const char *d;
int i, j, o = 0;
if (!fds) return "NULL";
if ((!IsAsan() && kisdangerous(fds)) ||
(IsAsan() && !__asan_is_valid(fds, sizeof(*fds) * nfds))) {
ksnprintf(buf, N, "%p", fds);
return buf;
}
append("{");
for (i = 0; i < nfds; ++i) {
if (i) append(", ");
append("{%d, %s", fds[i].fd, (DescribePollFlags)(b64, fds[i].events));
if (rc >= 0) {
append(", [%s]", (DescribePollFlags)(b64, fds[i].revents));
}
append("}");
}
append("}");
return buf;
}

5
libc/intrin/directmap-metal.c
View file

@ -17,8 +17,8 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/macros.internal.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/pc.internal.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/map.h"
@ -39,8 +39,7 @@ noasan struct DirectMap sys_mmap_metal(void *paddr, size_t size, int prot,
size = ROUNDUP(size, 4096);
addr = (uint64_t)paddr;
if (!(flags & MAP_FIXED)) {
if (!addr)
addr = 4096;
if (!addr) addr = 4096;
for (i = 0; i < size; i += 4096) {
pte = __get_virtual(mm, pml4t, addr + i, false);
if (pte && (*pte & (PAGE_V | PAGE_RSRV))) {

10
libc/intrin/directmap-nt.c
View file

@ -19,26 +19,24 @@
#include "libc/assert.h"
#include "libc/calls/internal.h"
#include "libc/calls/state.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/nt/enum/filemapflags.h"
#include "libc/nt/enum/pageflags.h"
#include "libc/nt/memory.h"
#include "libc/nt/runtime.h"
#include "libc/nt/struct/processmemorycounters.h"
#include "libc/nt/struct/securityattributes.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/o.h"
textwindows struct DirectMap sys_mmap_nt(void *addr, size_t size, int prot,
int flags, int fd, int64_t off) {
size_t i;
bool iscow;
int iscow;
int64_t handle;
uint32_t oldprot;
struct DirectMap dm;
struct ProtectNt fl;
const struct NtSecurityAttributes *sec;
struct NtProcessMemoryCountersEx memcount;
if (fd != -1) {
handle = g_fds.p[fd].handle;
@ -85,10 +83,8 @@ textwindows struct DirectMap sys_mmap_nt(void *addr, size_t size, int prot,
size, addr))) {
if (VirtualProtect(addr, size, __prot2nt(prot, iscow), &oldprot)) {
return dm;
} else {
return dm;
UnmapViewOfFile(dm.addr);
}
UnmapViewOfFile(dm.addr);
}
CloseHandle(dm.maphandle);
}

4
libc/intrin/directmap.c
View file

@ -16,12 +16,12 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/strace.internal.h"
#include "libc/calls/syscall-sysv.internal.h"
#include "libc/errno.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/strace.internal.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/memtrack.internal.h"
/**

24
libc/intrin/directmap.internal.h Normal file
View file

@ -0,0 +1,24 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_DIRECTMAP_H_
#define COSMOPOLITAN_LIBC_INTRIN_DIRECTMAP_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
struct ProtectNt {
uint32_t flags1;
uint32_t flags2;
};
struct DirectMap {
void *addr;
int64_t maphandle;
};
struct DirectMap sys_mmap(void *, size_t, int, int, int, int64_t);
struct DirectMap sys_mmap_nt(void *, size_t, int, int, int, int64_t);
struct DirectMap sys_mmap_metal(void *, size_t, int, int, int, int64_t);
int sys_munmap_metal(void *, size_t);
int __prot2nt(int, int);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_INTRIN_DIRECTMAP_H_ */

2
libc/intrin/extend.c
View file

@ -22,8 +22,8 @@
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/asancodes.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/macros.internal.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/prot.h"

6
libc/intrin/ftrace.c
View file

@ -30,9 +30,9 @@
* short time period should say:
*
* void foo() {
* --__ftrace;
* ftrace_enabled(-1);
* bar();
* ++__ftrace;
* ftrace_enabled(+1);
* }
*
* This way you still have some flexibility to force function tracing,
@ -40,4 +40,4 @@
* though under normal circumstances, `__ftrace` should only be either
* zero or one.
*/
atomic_int __ftrace;
int __ftrace;

43
libc/intrin/ftrace_enabled.c Normal file
View file

@ -0,0 +1,43 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/runtime.h"
#include "libc/thread/tls.h"
/**
* Changes function call logging state for current thread.
*
* @param delta is added to enabled state
* @return enabled state before `delta` was applied
*/
int ftrace_enabled(int delta) {
int res;
struct CosmoTib *tib;
if (__tls_enabled) {
tib = __get_tls();
res = tib->tib_ftrace;
tib->tib_ftrace += delta;
if (!__ftrace && tib->tib_ftrace > 0) {
__ftrace = 1;
}
} else {
res = __ftrace;
__ftrace += delta;
}
return res;
}

6
libc/intrin/intrin.mk
View file

@ -44,6 +44,12 @@ $(LIBC_INTRIN_A).pkg: \
$(LIBC_INTRIN_A_OBJS) \
$(foreach x,$(LIBC_INTRIN_A_DIRECTDEPS),$($(x)_A).pkg)
# we can't use asan because:
# __strace_init() calls this before asan is initialized
o/$(MODE)/libc/intrin/strace_enabled.o: private \
OVERRIDE_COPTS += \
-fno-sanitize=address
# we can't use asan because:
# asan guard pages haven't been allocated yet
o/$(MODE)/libc/intrin/directmap.o \

18
libc/intrin/kmalloc.c
View file

@ -22,7 +22,6 @@
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/extend.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/sysv/consts/map.h"
@ -37,17 +36,18 @@ static struct {
pthread_spinlock_t lock;
} g_kmalloc;
static void kmalloc_lock(void) {
if (__threaded) pthread_spin_lock(&g_kmalloc.lock);
void __kmalloc_lock(void) {
pthread_spin_lock(&g_kmalloc.lock);
}
static void kmalloc_unlock(void) {
void __kmalloc_unlock(void) {
pthread_spin_unlock(&g_kmalloc.lock);
}
__attribute__((__constructor__)) static void kmalloc_init(void) {
pthread_atfork(kmalloc_lock, kmalloc_unlock, kmalloc_unlock);
}
#ifdef _NOPL0
#define __kmalloc_lock() _NOPL0("__threadcalls", __kmalloc_lock)
#define __kmalloc_unlock() _NOPL0("__threadcalls", __kmalloc_unlock)
#endif
/**
* Allocates permanent memory.
@ -66,7 +66,7 @@ void *kmalloc(size_t size) {
char *p, *e;
size_t i, n, t;
n = ROUNDUP(size + (IsAsan() * 8), KMALLOC_ALIGN);
kmalloc_lock();
__kmalloc_lock();
t = g_kmalloc.total;
e = g_kmalloc.endptr;
i = t;
@ -80,7 +80,7 @@ void *kmalloc(size_t size) {
} else {
p = 0;
}
kmalloc_unlock();
__kmalloc_unlock();
if (p) {
_unassert(!((intptr_t)(p + i) & (KMALLOC_ALIGN - 1)));
if (IsAsan()) __asan_poison(p + i + size, n - size, kAsanHeapOverrun);

3
libc/intrin/kmalloc.h
View file

@ -5,6 +5,9 @@ COSMOPOLITAN_C_START_
void *kmalloc(size_t) mallocesque attributeallocsize((1)) returnsaligned((8));
void __kmalloc_lock(void);
void __kmalloc_unlock(void);
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_INTRIN_KMALLOC_H_ */

1
libc/intrin/kprintf.greg.c
View file

@ -31,7 +31,6 @@
#include "libc/intrin/cmpxchg.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/lockcmpxchg.h"
#include "libc/intrin/nomultics.internal.h"
#include "libc/intrin/safemacros.internal.h"
#include "libc/intrin/weaken.h"

2
libc/intrin/memtrack.greg.c
View file

@ -22,13 +22,13 @@
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/bits.h"
#include "libc/intrin/directmap.internal.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/strace.internal.h"
#include "libc/log/libfatal.internal.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
#include "libc/mem/mem.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"

17
libc/intrin/mman.greg.c
View file

@ -137,8 +137,7 @@ static noasan textreal void __invert_memory(struct mman *mm, uint64_t *pml4t) {
for (i = 0; i < mm->e820n; ++i) {
uint64_t ps = mm->e820[i].addr, size = mm->e820[i].size;
/* ape/ape.S has already mapped the first 2 MiB of physical memory. */
if (ps < 0x200000 && ps + size <= 0x200000)
continue;
if (ps < 0x200000 && ps + size <= 0x200000) continue;
__invert_memory_area(mm, pml4t, ps, size, PAGE_RW | PAGE_XD);
}
}
@ -148,13 +147,13 @@ static noasan textreal void __invert_memory(struct mman *mm, uint64_t *pml4t) {
* so that assembly language routines can use it. This macro can be invoked
* from inside a function whose code is known to be emitted.
*/
#define export_offsetof(type, member) \
do { \
asm volatile(".globl \"" #type "::" #member "\"\n\t" \
".set \"" #type "::" #member "\",%c0" \
: /* no outputs */ \
: "i" (offsetof(type, member))); \
} while (0)
#define export_offsetof(type, member) \
do { \
asm volatile(".globl \"" #type "::" #member "\"\n\t" \
".set \"" #type "::" #member "\",%c0" \
: /* no outputs */ \
: "i"(offsetof(type, member))); \
} while (0)
noasan textreal void __setup_mman(struct mman *mm, uint64_t *pml4t) {
export_offsetof(struct mman, pc_drive_base_table);

9
libc/intrin/mmi_lock.c
View file

@ -31,12 +31,3 @@ void(__mmi_lock)(void) {
void(__mmi_unlock)(void) {
pthread_mutex_unlock(&__mmi_lock_obj);
}
void(__mmi_funlock)(void) {
bzero(&__mmi_lock_obj, sizeof(__mmi_lock_obj));
__mmi_lock_obj._type = PTHREAD_MUTEX_RECURSIVE;
}
__attribute__((__constructor__)) static void init(void) {
pthread_atfork(__mmi_lock, __mmi_unlock, __mmi_funlock);
}

4
libc/intrin/prot2nt.greg.c
View file

@ -16,11 +16,11 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/directmap.internal.h"
#include "libc/nt/enum/pageflags.h"
#include "libc/runtime/directmap.internal.h"
#include "libc/sysv/consts/prot.h"
privileged uint32_t __prot2nt(int prot, bool iscow) {
privileged int __prot2nt(int prot, int iscow) {
switch (prot & (PROT_READ | PROT_WRITE | PROT_EXEC)) {
case PROT_READ:
return kNtPageReadonly;

1
libc/intrin/pthread_atfork.c
View file

@ -74,6 +74,7 @@
* @param parent is run by fork() after forking happens in parent process
* @param child is run by fork() after forking happens in childe process
* @return 0 on success, or errno on error
* @raise ENOMEM if we require more vespene gas
*/
int pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) {
if (_weaken(_pthread_atfork)) {

57
libc/intrin/pthread_key_create.c
View file

@ -17,51 +17,40 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/errno.h"
#include "libc/intrin/bsr.h"
#include "libc/intrin/atomic.h"
#include "libc/runtime/runtime.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
/**
* Allocates TLS slot.
*
* If `dtor` is non-null, then it'll be called upon thread exit when the
* key's value is nonzero. The key's value is set to zero before it gets
* called. The ordering for multiple destructor calls is unspecified.
*
* @param key is set to the allocated key on success
* @param dtor specifies an optional destructor callback
* @return 0 on success, or errno on error
* @raise EAGAIN if `PTHREAD_KEYS_MAX` keys exist
*/
int pthread_key_create(pthread_key_t *key, pthread_key_dtor dtor) {
int i, j, rc = EAGAIN;
_pthread_key_lock();
for (i = 0; i < (PTHREAD_KEYS_MAX + 63) / 64; ++i) {
if (~_pthread_key_usage[i]) {
j = _bsrl(~_pthread_key_usage[i]);
_pthread_key_usage[i] |= 1ul << j;
_pthread_key_dtor[i * 64 + j] = dtor;
*key = i * 64 + j;
rc = 0;
break;
int i;
pthread_key_dtor expect;
if (!dtor) dtor = (pthread_key_dtor)-1;
for (i = 0; i < PTHREAD_KEYS_MAX; ++i) {
if (!(expect = atomic_load_explicit(_pthread_key_dtor + i,
memory_order_relaxed)) &&
atomic_compare_exchange_strong_explicit(_pthread_key_dtor + i, &expect,
dtor, memory_order_relaxed,
memory_order_relaxed)) {
*key = i;
return 0;
}
}
_pthread_key_unlock();
return rc;
}
void _pthread_key_lock(void) {
pthread_spin_lock(&_pthread_keys_lock);
}
void _pthread_key_unlock(void) {
pthread_spin_unlock(&_pthread_keys_lock);
}
static void _pthread_key_funlock(void) {
pthread_spin_init(&_pthread_keys_lock, 0);
}
static textexit void _pthread_key_atexit(void) {
_pthread_key_destruct(0);
return EAGAIN;
}
__attribute__((__constructor__)) static textstartup void _pthread_key_init() {
atexit(_pthread_key_atexit);
pthread_atfork(_pthread_key_lock, //
_pthread_key_unlock, //
_pthread_key_funlock);
atexit(_pthread_key_destruct);
}

17
libc/intrin/pthread_key_delete.c
View file

@ -17,22 +17,23 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
/**
* Deletes TLS slot.
*
* @param key was created by pthread_key_create()
* @return 0 on success, or errno on error
* @raise EINVAL if `key` isn't valid
*/
int pthread_key_delete(pthread_key_t key) {
int rc;
_pthread_key_lock();
uint64_t mask;
if (key < PTHREAD_KEYS_MAX) {
_pthread_key_usage[key / 64] &= ~(1ul << (key % 64));
_pthread_key_dtor[key] = 0;
rc = 0;
atomic_store_explicit(_pthread_key_dtor + key, 0, memory_order_relaxed);
return 0;
} else {
rc = EINVAL;
return EINVAL;
}
_pthread_key_unlock();
return rc;
}

38
libc/intrin/pthread_key_destruct.c
View file

@ -16,34 +16,30 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/bsr.h"
#include "libc/intrin/atomic.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
// TODO(jart): why does this even need a lock?
void _pthread_key_destruct(void *key[PTHREAD_KEYS_MAX]) {
int i, j;
uint64_t x;
void *value;
void _pthread_key_destruct(void) {
int i, j, gotsome;
void *val, **keys;
pthread_key_dtor dtor;
if (!__tls_enabled) return;
_pthread_key_lock();
if (!key) key = __get_tls()->tib_keys;
StartOver:
for (i = 0; i < (PTHREAD_KEYS_MAX + 63) / 64; ++i) {
x = _pthread_key_usage[i];
while (x) {
j = _bsrl(x);
if ((value = key[i * 64 + j]) && (dtor = _pthread_key_dtor[i * 64 + j])) {
key[i * 64 + j] = 0;
_pthread_key_unlock();
dtor(value);
_pthread_key_lock();
goto StartOver;
keys = __get_tls()->tib_keys;
for (j = 0; j < PTHREAD_DESTRUCTOR_ITERATIONS; ++j) {
for (gotsome = i = 0; i < PTHREAD_KEYS_MAX; ++i) {
if ((val = keys[i]) &&
(dtor = atomic_load_explicit(_pthread_key_dtor + i,
memory_order_relaxed)) &&
dtor != (pthread_key_dtor)-1) {
gotsome = 1;
keys[i] = 0;
dtor(val);
}
x &= ~(1ul << j);
}
if (!gotsome) {
break;
}
}
_pthread_key_unlock();
}

8
libc/intrin/pthread_keys.c
View file

@ -19,10 +19,4 @@
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
pthread_spinlock_t _pthread_keys_lock;
// bitset of tls key registrations
uint64_t _pthread_key_usage[(PTHREAD_KEYS_MAX + 63) / 64];
// pthread tls key destructors
pthread_key_dtor _pthread_key_dtor[PTHREAD_KEYS_MAX];
_Atomic(pthread_key_dtor) _pthread_key_dtor[PTHREAD_KEYS_MAX];

24
libc/intrin/restorewintty.c
View file

@ -22,24 +22,24 @@
#include "libc/nt/runtime.h"
#include "libc/runtime/internal.h"
uint32_t __winmainpid;
__msabi extern typeof(GetCurrentProcessId) *const __imp_GetCurrentProcessId;
__msabi extern typeof(SetConsoleMode) *const __imp_SetConsoleMode;
__msabi extern typeof(GetStdHandle) *const __imp_GetStdHandle;
const char kConsoleHandles[3] = {
extern uint32_t __pid_exec;
const unsigned char kConsoleHandles[3] = {
kNtStdInputHandle,
kNtStdOutputHandle,
kNtStdErrorHandle,
};
/**
* Puts cmd.exe gui back the way it was.
*/
noinstrument void _restorewintty(void) {
int i;
// Puts cmd.exe gui back the way it was.
privileged noinstrument void _restorewintty(void) {
if (!IsWindows()) return;
if (GetCurrentProcessId() == __winmainpid) {
for (i = 0; i < 3; ++i) {
SetConsoleMode(GetStdHandle(kConsoleHandles[i]), __ntconsolemode[i]);
}
__winmainpid = 0;
if (__imp_GetCurrentProcessId() != __pid_exec) return;
for (int i = 0; i < 3; ++i) {
__imp_SetConsoleMode(__imp_GetStdHandle(kConsoleHandles[i]),
__ntconsolemode[i]);
}
}

10
libc/intrin/strace.internal.h
View file

@ -16,11 +16,11 @@
COSMOPOLITAN_C_START_
#ifdef SYSDEBUG
#define STRACE(FMT, ...) \
do { \
if (UNLIKELY(__strace > 0)) { \
__stracef(STRACE_PROLOGUE FMT "\n", ##__VA_ARGS__); \
} \
#define STRACE(FMT, ...) \
do { \
if (UNLIKELY(__strace > 0) && strace_enabled(0) > 0) { \
__stracef(STRACE_PROLOGUE FMT "\n", ##__VA_ARGS__); \
} \
} while (0)
#else
#define STRACE(FMT, ...) (void)0

43
libc/intrin/strace_enabled.c Normal file
View file

@ -0,0 +1,43 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/runtime/runtime.h"
#include "libc/thread/tls.h"
/**
* Changes system call logging state for current thread.
*
* @param delta is added to enabled state
* @return enabled state before `delta` was applied
*/
int strace_enabled(int delta) {
int res;
struct CosmoTib *tib;
if (__tls_enabled) {
tib = __get_tls();
res = tib->tib_strace;
tib->tib_strace += delta;
if (!__strace && tib->tib_strace > 0) {
__strace = 1;
}
} else {
res = __strace;
__strace += delta;
}
return res;
}

5
libc/intrin/stracef.greg.c
View file

@ -19,10 +19,13 @@
#include "libc/intrin/kprintf.h"
#include "libc/intrin/strace.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/thread/tls.h"
privileged void __stracef(const char *fmt, ...) {
va_list v;
if (__strace <= 0) return;
if (__strace <= 0 || (__tls_enabled && __get_tls()->tib_strace <= 0)) {
return;
}
va_start(v, fmt);
kvprintf(fmt, v);
va_end(v);

3
libc/intrin/strlen.c
View file

@ -34,10 +34,9 @@ noasan size_t strlen(const char *s) {
xmm_t z = {0};
unsigned m, k = (uintptr_t)s & 15;
const xmm_t *p = (const xmm_t *)((uintptr_t)s & -16);
if (IsAsan()) __asan_verify(s, 1);
if (IsAsan()) __asan_verify_str(s);
m = __builtin_ia32_pmovmskb128(*p == z) >> k << k;
while (!m) m = __builtin_ia32_pmovmskb128(*++p == z);
n = (const char *)p + __builtin_ctzl(m) - s;
if (IsAsan()) __asan_verify(s, n);
return n;
}

2
libc/intrin/wsarecv.c
View file

@ -52,7 +52,7 @@ textwindows int WSARecv(
if (rc == -1) {
__winerr();
}
if (UNLIKELY(__strace > 0)) {
if (UNLIKELY(__strace > 0) && strace_enabled(0) > 0) {
kprintf(STRACE_PROLOGUE "WSARecv(%lu, [", s);
DescribeIovNt(inout_lpBuffers, dwBufferCount,
rc != -1 ? NumberOfBytesRecvd : 0);

3
libc/intrin/wsarecvfrom.c
View file

@ -23,6 +23,7 @@
#include "libc/intrin/strace.internal.h"
#include "libc/nt/thunk/msabi.h"
#include "libc/nt/winsock.h"
#include "libc/runtime/runtime.h"
__msabi extern typeof(WSARecvFrom) *const __imp_WSARecvFrom;
@ -54,7 +55,7 @@ textwindows int WSARecvFrom(
if (rc == -1) {
__winerr();
}
if (UNLIKELY(__strace > 0)) {
if (UNLIKELY(__strace > 0) && strace_enabled(0) > 0) {
kprintf(STRACE_PROLOGUE "WSARecvFrom(%lu, [", s);
DescribeIovNt(inout_lpBuffers, dwBufferCount,
rc != -1 ? NumberOfBytesRecvd : 0);