Make some systemic improvements

- add vdso dump utility
- tests now log stack usage
- rename g_ftrace to __ftrace
- make internal spinlocks go faster
- add conformant c11 atomics library
- function tracing now logs stack usage
- make function call tracing thread safe
- add -X unsecure (no ssl) mode to redbean
- munmap() has more consistent behavior now
- pacify fsync() calls on python unit tests
- make --strace flag work better in redbean
- start minimizing and documenting compiler flags
This commit is contained in:
Justine Tunney 2022-05-18 16:41:29 -07:00
parent c6bbca55e9
commit 9208c83f7a
141 changed files with 1948 additions and 1411 deletions

View file

@ -128,7 +128,7 @@ struct AsanSourceLocation {
};
struct AsanAccessInfo {
const uintptr_t addr;
const char *addr;
const uintptr_t first_bad_addr;
size_t size;
bool iswrite;
@ -136,7 +136,7 @@ struct AsanAccessInfo {
};
struct AsanGlobal {
const uintptr_t addr;
const char *addr;
size_t size;
size_t size_with_redzone;
const void *name;
@ -339,10 +339,10 @@ dontdiscard static __asan_die_f *__asan_die(void) {
}
}
void __asan_poison(long p, long n, signed char t) {
void __asan_poison(void *p, long n, signed char t) {
signed char k, *s;
s = (signed char *)((p >> 3) + 0x7fff8000);
if ((k = p & 7)) {
s = (signed char *)(((intptr_t)p >> 3) + 0x7fff8000);
if ((k = (intptr_t)p & 7)) {
if ((!*s && n >= 8 - k) || *s > k) *s = k;
n -= MIN(8 - k, n);
s += 1;
@ -354,10 +354,10 @@ void __asan_poison(long p, long n, signed char t) {
}
}
void __asan_unpoison(long p, long n) {
void __asan_unpoison(void *p, long n) {
signed char k, *s;
k = p & 7;
s = (signed char *)((p >> 3) + 0x7fff8000);
k = (intptr_t)p & 7;
s = (signed char *)(((intptr_t)p >> 3) + 0x7fff8000);
if (UNLIKELY(k)) {
if (k + n < 8) {
if (n > 0) *s = MAX(*s, k + n);
@ -737,9 +737,9 @@ dontdiscard static __asan_die_f *__asan_report(const void *addr, int size,
uint64_t x, y, z;
char *p, *q, *base;
struct MemoryIntervals *m;
++g_ftrace;
--__ftrace;
p = __fatalbuf;
kprintf("\n\e[J\e[1;31masan error\e[0m: %s %d-byte %s at %p shadow %p\n%s\n",
kprintf("\n\e[J\e[1;31masan error\e[0m: %s %d-byte %s at %p shadow %p\n",
__asan_describe_access_poison(kind), size, message, addr,
SHADOW(addr), __argv[0]);
if (0 < size && size < 80) {
@ -759,19 +759,19 @@ dontdiscard static __asan_die_f *__asan_report(const void *addr, int size,
for (c = i = 0; i < 80; ++i) {
if (!(t = __asan_check(base + i, 1).kind)) {
if (c != 32) {
p = __stpcpy(p, "\e[32m");
*p++ = '\e', *p++ = '[', *p++ = '3', *p++ = '2', *p++ = 'm';
c = 32;
}
*p++ = '.';
} else {
if (c != 31) {
p = __stpcpy(p, "\e[31m");
*p++ = '\e', *p++ = '[', *p++ = '3', *p++ = '1', *p++ = 'm';
c = 31;
}
p = __asan_utf8cpy(p, __asan_symbolize_access_poison(t));
}
}
p = __stpcpy(p, "\e[39m");
*p++ = '\e', *p++ = '[', *p++ = '3', *p++ = '9', *p++ = 'm';
*p++ = '\n';
for (i = 0; (intptr_t)(base + i) & 7; ++i) *p++ = ' ';
for (; i + 8 <= 80; i += 8) {
@ -814,7 +814,7 @@ dontdiscard static __asan_die_f *__asan_report(const void *addr, int size,
kprintf("%s", __fatalbuf);
__asan_report_memory_origin(addr, size, kind);
kprintf("\nthe crash was caused by\n");
--g_ftrace;
++__ftrace;
return __asan_die();
}
@ -840,7 +840,7 @@ dontdiscard __asan_die_f *__asan_report_memory_fault(void *addr, int size,
void *__asan_morgue_add(void *p) {
int i;
void *r;
_spinlock_optimistic(&__asan_lock);
_spinlock_cooperative(&__asan_lock);
i = __asan_morgue.i++ & (ARRAYLEN(__asan_morgue.p) - 1);
r = __asan_morgue.p[i];
__asan_morgue.p[i] = p;
@ -851,7 +851,7 @@ void *__asan_morgue_add(void *p) {
static void __asan_morgue_flush(void) {
int i;
void *p;
_spinlock_optimistic(&__asan_lock);
_spinlock_cooperative(&__asan_lock);
for (i = 0; i < ARRAYLEN(__asan_morgue.p); ++i) {
if (weaken(dlfree)) {
weaken(dlfree)(__asan_morgue.p[i]);
@ -943,9 +943,9 @@ static void *__asan_allocate(size_t a, size_t n, int underrun, int overrun,
if ((p = weaken(dlmemalign)(a, __asan_heap_size(n)))) {
c = weaken(dlmalloc_usable_size)(p);
e = (struct AsanExtra *)(p + c - sizeof(*e));
__asan_unpoison((uintptr_t)p, n);
__asan_poison((uintptr_t)p - 16, 16, underrun); /* see dlmalloc design */
__asan_poison((uintptr_t)p + n, c - n, overrun);
__asan_unpoison(p, n);
__asan_poison(p - 16, 16, underrun); /* see dlmalloc design */
__asan_poison(p + n, c - n, overrun);
__asan_memset(p, 0xF9, n);
__asan_write48(&e->size, n);
__asan_memcpy(&e->bt, bt, sizeof(*bt));
@ -1030,7 +1030,7 @@ static void __asan_deallocate(char *p, long kind) {
struct AsanExtra *e;
if ((e = __asan_get_extra(p, &c))) {
if (__asan_read48(e->size, &n)) {
__asan_poison((uintptr_t)p, c, kind);
__asan_poison(p, c, kind);
if (c <= ASAN_MORGUE_THRESHOLD) {
p = __asan_morgue_add(p);
}
@ -1084,11 +1084,11 @@ static void *__asan_realloc_impl(void *p, size_t n,
if ((e = __asan_get_extra(p, &c))) {
if (__asan_read48(e->size, &m)) {
if (n <= m) { /* shrink */
__asan_poison((uintptr_t)p + n, m - n, kAsanHeapOverrun);
__asan_poison((char *)p + n, m - n, kAsanHeapOverrun);
__asan_write48(&e->size, n);
return p;
} else if (n <= c - sizeof(struct AsanExtra)) { /* small growth */
__asan_unpoison((uintptr_t)p + m, n - m);
__asan_unpoison((char *)p + m, n - m);
__asan_write48(&e->size, n);
return p;
} else { /* exponential growth */
@ -1175,13 +1175,12 @@ void __asan_handle_no_return(void) {
void __asan_register_globals(struct AsanGlobal g[], int n) {
int i;
__asan_poison((intptr_t)g, sizeof(*g) * n, kAsanProtected);
__asan_poison(g, sizeof(*g) * n, kAsanProtected);
for (i = 0; i < n; ++i) {
__asan_poison(g[i].addr + g[i].size, g[i].size_with_redzone - g[i].size,
kAsanGlobalRedzone);
if (g[i].location) {
__asan_poison((intptr_t)g[i].location, sizeof(*g[i].location),
kAsanProtected);
__asan_poison(g[i].location, sizeof(*g[i].location), kAsanProtected);
}
}
}
@ -1228,15 +1227,15 @@ void __asan_report_store(uint8_t *addr, int size) {
}
}
void __asan_poison_stack_memory(uintptr_t addr, size_t size) {
void __asan_poison_stack_memory(char *addr, size_t size) {
__asan_poison(addr, size, kAsanStackFree);
}
void __asan_unpoison_stack_memory(uintptr_t addr, size_t size) {
void __asan_unpoison_stack_memory(char *addr, size_t size) {
__asan_unpoison(addr, size);
}
void __asan_alloca_poison(uintptr_t addr, uintptr_t size) {
void __asan_alloca_poison(char *addr, uintptr_t size) {
__asan_poison(addr - 32, 32, kAsanAllocaUnderrun);
__asan_poison(addr + size, 32, kAsanAllocaOverrun);
}
@ -1255,8 +1254,8 @@ void *__asan_get_current_fake_stack(void) {
return 0;
}
void __sanitizer_annotate_contiguous_container(long beg, long end, long old_mid,
long new_mid) {
void __sanitizer_annotate_contiguous_container(char *beg, char *end,
char *old_mid, char *new_mid) {
// the c++ stl uses this
// TODO(jart): make me faster
__asan_unpoison(beg, new_mid - beg);
@ -1317,11 +1316,11 @@ void __asan_map_shadow(uintptr_t p, size_t n) {
__repstosb((void *)(intptr_t)((int64_t)((uint64_t)a << 32) >> 16),
kAsanUnmapped, size);
}
__asan_unpoison((uintptr_t)p, n);
__asan_unpoison((char *)p, n);
}
static textstartup void __asan_shadow_string(char *s) {
__asan_map_shadow((uintptr_t)s, __strlen(s) + 1);
__asan_map_shadow((intptr_t)s, __strlen(s) + 1);
}
static textstartup void __asan_shadow_auxv(intptr_t *auxv) {
@ -1359,7 +1358,7 @@ static textstartup void __asan_shadow_mapping(struct MemoryIntervals *m,
static textstartup void __asan_shadow_existing_mappings(void) {
__asan_shadow_mapping(&_mmi, 0);
__asan_map_shadow(GetStackAddr(0), GetStackSize());
__asan_map_shadow((intptr_t)GetStackAddr(0), GetStackSize());
__asan_poison(GetStackAddr(0), PAGESIZE, kAsanStackOverflow);
}

View file

@ -16,8 +16,8 @@ struct AsanFault {
const signed char *shadow;
};
void __asan_unpoison(long, long);
void __asan_poison(long, long, signed char);
void __asan_unpoison(void *, long);
void __asan_poison(void *, long, signed char);
void __asan_verify(const void *, size_t);
void __asan_map_shadow(uintptr_t, size_t);
bool __asan_is_valid(const void *, long) nosideeffect;

View file

@ -33,8 +33,8 @@ relegated wontreturn void __assert_fail(const char *expr, const char *file,
int line) {
int rc;
static bool noreentry;
__strace = 0;
g_ftrace = 0;
--__strace;
--__ftrace;
kprintf("%s:%d: assert(%s) failed\n", file, line, expr);
if (_lockcmpxchg(&noreentry, false, true)) {
if (weaken(__die)) {

View file

@ -1,22 +0,0 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_ATOMIC_LOAD_H_
#define COSMOPOLITAN_LIBC_INTRIN_ATOMIC_LOAD_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
intptr_t atomic_load(void *, size_t);
#if defined(__GNUC__) && !defined(__STRICT_ANSI__)
#define atomic_load(MEM) \
({ \
autotype(MEM) Mem = (MEM); \
typeof(*Mem) Reg; \
asm("mov\t%1,%0" : "=r"(Reg) : "m"(*Mem)); \
Reg; \
})
#else
#define atomic_load(MEM) atomic_load(MEM, sizeof(*(MEM)))
#endif /* GNUC && !ANSI && x86 */
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_INTRIN_ATOMIC_LOAD_H_ */

View file

@ -1,54 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi
Copyright 2021 Justine Alexandra Roberts Tunney
Permission to use, copy, modify, and/or distribute this software for
any purpose with or without fee is hereby granted, provided that the
above copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/intrin/atomic_store.h"
/**
* Saves scalar to memory w/ one operation.
*
* This is guaranteed to happen in either one or zero operations,
* depending on whether or not it's possible for *(MEM) to be read
* afterwards. This macro only forbids compiler from using >1 ops.
*
* @param MEM is alignas(𝑘) uint𝑘_t[hasatleast 1] where 𝑘 {8,16,32,64}
* @param VAL is uint𝑘_t w/ better encoding for immediates (constexpr)
* @return VAL
* @note alignas(𝑘) on nexgen32e only needed for end of page gotcha
* @note alignas(𝑘) is implied if compiler knows type
* @note needed to defeat store tearing optimizations
* @see Intel Six-Thousand Page Manual Manual V.3A §8.2.3.1
* @see atomic_load()
*/
intptr_t(atomic_store)(void *p, intptr_t x, size_t n) {
switch (n) {
case 1:
__builtin_memcpy(p, &x, 1);
return x;
case 2:
__builtin_memcpy(p, &x, 2);
return x;
case 4:
__builtin_memcpy(p, &x, 4);
return x;
case 8:
__builtin_memcpy(p, &x, 8);
return x;
default:
return 0;
}
}

View file

@ -1,23 +0,0 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_ATOMIC_STORE_H_
#define COSMOPOLITAN_LIBC_INTRIN_ATOMIC_STORE_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
intptr_t atomic_store(void *, intptr_t, size_t);
#if defined(__GNUC__) && !defined(__STRICT_ANSI__)
#define atomic_store(MEM, VAL) \
({ \
autotype(VAL) Val = (VAL); \
typeof(&Val) Mem = (MEM); \
asm("mov%z1\t%1,%0" : "=m"(*Mem) : "r"(Val)); \
Val; \
})
#else
#define atomic_store(MEM, VAL) \
atomic_store(MEM, VAL, sizeof(*(MEM)) / (sizeof(*(MEM)) == sizeof(*(VAL))))
#endif /* __GNUC__ && !__STRICT_ANSI__ */
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_INTRIN_ATOMIC_STORE_H_ */

View file

@ -3,6 +3,7 @@
#include "libc/calls/struct/iovec.h"
#include "libc/calls/struct/rlimit.h"
#include "libc/calls/struct/sigaction.h"
#include "libc/calls/struct/sigaltstack.h"
#include "libc/calls/struct/sigset.h"
#include "libc/calls/struct/stat.h"
#include "libc/calls/struct/timespec.h"
@ -27,6 +28,7 @@ const char *DescribePollFlags(char *, size_t, int);
const char *DescribeStat(int, const struct stat *);
const char *DescribeDirfd(char[hasatleast 12], int);
const char *DescribeSigaction(char *, size_t, int, const struct sigaction *);
const char *DescribeSigaltstk(char *, size_t, int, const struct sigaltstack *);
const char *DescribeSigset(char *, size_t, int, const sigset_t *);
const char *DescribeRlimit(char *, size_t, int, const struct rlimit *);
const char *DescribeTimespec(char *, size_t, int, const struct timespec *);

View file

@ -30,17 +30,17 @@ noasan const char *DescribeFrame(int x) {
char *p;
static char buf[32];
if (IsShadowFrame(x)) {
ksnprintf(buf, sizeof(buf), " /*shadow:%.12p*/", UNSHADOW(ADDR(x)));
ksnprintf(buf, sizeof(buf), " shadow=%.8x", FRAME(UNSHADOW(ADDR(x))));
return buf;
return " /*shadow*/ ";
return " shadow ";
} else if (IsAutoFrame(x)) {
return " /*automap*/";
return " automap";
} else if (IsFixedFrame(x)) {
return " /*fixed*/ ";
return " fixed ";
} else if (IsArenaFrame(x)) {
return " /*arena*/ ";
return " arena ";
} else if (IsStaticStackFrame(x)) {
return " /*stack*/ ";
return " stack ";
} else {
return "";
}

View file

@ -26,8 +26,8 @@ const char *DescribeMapFlags(int x) {
_Alignas(char) static char mapflags[256];
const struct DescribeFlags kMapFlags[] = {
{MAP_STACK, "STACK"}, // order matters
{MAP_ANONYMOUS, "ANONYMOUS"}, //
{MAP_PRIVATE, "PRIVATE"}, //
{MAP_ANONYMOUS, "ANONYMOUS"}, //
{MAP_SHARED, "SHARED"}, //
{MAP_FIXED, "FIXED"}, //
{MAP_FIXED_NOREPLACE, "FIXED_NOREPLACE"}, //

View file

@ -18,4 +18,25 @@
*/
#include "libc/runtime/runtime.h"
int g_ftrace;
/**
* Function tracing enabled state.
*
* After ftrace_install() has been called, the logging of C function
* calls may be controlled by changing this variable. If `__ftrace` is
* greater than zero, functions are logged. Otherwise, they aren't.
*
* By convention, functions wishing to disable function tracing for a
* short time period should say:
*
* void foo() {
* --__ftrace;
* bar();
* ++__ftrace;
* }
*
* This way you still have some flexibility to force function tracing,
* by setting `__ftrace` to a higher number like `2` or `200`. Even
* though under normal circumstances, `__ftrace` should only be either
* zero or one.
*/
_Atomic(int) __ftrace;

View file

@ -66,50 +66,57 @@ o/$(MODE)/libc/intrin/kprintf.greg.o: \
o/$(MODE)/libc/intrin/tls.greg.o \
o/$(MODE)/libc/intrin/exit.greg.o \
o/$(MODE)/libc/intrin/errno.greg.o \
o/$(MODE)/libc/intrin/exit1.greg.o \
o/$(MODE)/libc/intrin/gettid.greg.o \
o/$(MODE)/libc/intrin/getenv.greg.o \
o/$(MODE)/libc/intrin/createfile.greg.o \
o/$(MODE)/libc/intrin/assertfail.greg.o \
o/$(MODE)/libc/intrin/reopenfile.greg.o \
o/$(MODE)/libc/intrin/deletefile.greg.o \
o/$(MODE)/libc/intrin/createpipe.greg.o \
o/$(MODE)/libc/intrin/closehandle.greg.o \
o/$(MODE)/libc/intrin/describeiov.greg.o \
o/$(MODE)/libc/intrin/openprocess.greg.o \
o/$(MODE)/libc/intrin/createthread.greg.o \
o/$(MODE)/libc/intrin/describestat.greg.o \
o/$(MODE)/libc/intrin/findnextfile.greg.o \
o/$(MODE)/libc/intrin/createprocess.greg.o \
o/$(MODE)/libc/intrin/findfirstfile.greg.o \
o/$(MODE)/libc/intrin/describeflags.greg.o \
o/$(MODE)/libc/intrin/describerlimit.greg.o \
o/$(MODE)/libc/intrin/removedirectory.greg.o \
o/$(MODE)/libc/intrin/createnamedpipe.greg.o \
o/$(MODE)/libc/intrin/unmapviewoffile.greg.o \
o/$(MODE)/libc/intrin/flushviewoffile.greg.o \
o/$(MODE)/libc/intrin/deviceiocontrol.greg.o \
o/$(MODE)/libc/intrin/createdirectory.greg.o \
o/$(MODE)/libc/intrin/flushfilebuffers.greg.o \
o/$(MODE)/libc/intrin/terminateprocess.greg.o \
o/$(MODE)/libc/intrin/describemapflags.greg.o \
o/$(MODE)/libc/intrin/describetimespec.greg.o \
o/$(MODE)/libc/intrin/getfileattributes.greg.o \
o/$(MODE)/libc/intrin/getexitcodeprocess.greg.o \
o/$(MODE)/libc/intrin/waitforsingleobject.greg.o \
o/$(MODE)/libc/intrin/setcurrentdirectory.greg.o \
o/$(MODE)/libc/intrin/mapviewoffileexnuma.greg.o \
o/$(MODE)/libc/intrin/createfilemappingnuma.greg.o \
o/$(MODE)/libc/intrin/waitformultipleobjects.greg.o \
o/$(MODE)/libc/intrin/generateconsolectrlevent.greg.o \
o/$(MODE)/libc/intrin/createfile.o \
o/$(MODE)/libc/intrin/reopenfile.o \
o/$(MODE)/libc/intrin/deletefile.o \
o/$(MODE)/libc/intrin/createpipe.o \
o/$(MODE)/libc/intrin/closehandle.o \
o/$(MODE)/libc/intrin/openprocess.o \
o/$(MODE)/libc/intrin/createthread.o \
o/$(MODE)/libc/intrin/findclose.o \
o/$(MODE)/libc/intrin/findnextfile.o \
o/$(MODE)/libc/intrin/createprocess.o \
o/$(MODE)/libc/intrin/findfirstfile.o \
o/$(MODE)/libc/intrin/removedirectory.o \
o/$(MODE)/libc/intrin/createsymboliclink.o \
o/$(MODE)/libc/intrin/createnamedpipe.o \
o/$(MODE)/libc/intrin/unmapviewoffile.o \
o/$(MODE)/libc/intrin/virtualprotect.o \
o/$(MODE)/libc/intrin/flushviewoffile.o \
o/$(MODE)/libc/intrin/createdirectory.o \
o/$(MODE)/libc/intrin/flushfilebuffers.o \
o/$(MODE)/libc/intrin/terminateprocess.o \
o/$(MODE)/libc/intrin/getfileattributes.o \
o/$(MODE)/libc/intrin/getexitcodeprocess.o \
o/$(MODE)/libc/intrin/waitforsingleobject.o \
o/$(MODE)/libc/intrin/setcurrentdirectory.o \
o/$(MODE)/libc/intrin/mapviewoffileex.o \
o/$(MODE)/libc/intrin/movefileex.o \
o/$(MODE)/libc/intrin/mapviewoffileexnuma.o \
o/$(MODE)/libc/intrin/createfilemapping.o \
o/$(MODE)/libc/intrin/createfilemappingnuma.o \
o/$(MODE)/libc/intrin/waitformultipleobjects.o \
o/$(MODE)/libc/intrin/generateconsolectrlevent.o \
o/$(MODE)/libc/intrin/kstarttsc.o \
o/$(MODE)/libc/intrin/nomultics.o \
o/$(MODE)/libc/intrin/ntconsolemode.o: \
OVERRIDE_CFLAGS += \
-Os \
-fwrapv \
-ffreestanding \
$(NO_MAGIC)
-fno-stack-protector \
-fno-sanitize=all
o/$(MODE)/libc/intrin/describeopenflags.greg.o: \
OVERRIDE_CPPFLAGS += \

View file

@ -40,6 +40,7 @@ kOpenFlags:
.e O_TRUNC,"TRUNC" //
.e O_CLOEXEC,"CLOEXEC" //
.e O_NONBLOCK,"NONBLOCK" //
.e O_DIRECTORY,"DIRECTORY" //
.e O_DIRECT,"DIRECT" // no-op on xnu/openbsd
.e O_APPEND,"APPEND" // weird on nt
.e O_TMPFILE,"TMPFILE" // linux, windows

View file

@ -1,7 +1,7 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi
/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│
vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi
Copyright 2021 Justine Alexandra Roberts Tunney
Copyright 2022 Justine Alexandra Roberts Tunney
Permission to use, copy, modify, and/or distribute this software for
any purpose with or without fee is hereby granted, provided that the
@ -16,38 +16,39 @@
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/intrin/atomic_load.h"
#include "libc/dce.h"
#include "libc/sysv/consts/nr.h"
#include "libc/macros.internal.h"
/**
* Reads scalar from memory w/ one operation.
*
* This macro is intended to prevent things like compiler load tearing
* optimizations.
*
* @param MEM is alignas(𝑘) uint𝑘_t[hasatleast 1] where 𝑘 ∈ {8,16,32,64}
* @return *(MEM)
* @note defeats compiler load tearing optimizations
* @note alignas(𝑘) is implied if compiler knows type
* @note alignas(𝑘) only avoids multi-core / cross-page edge cases
* @see Intel's Six-Thousand Page Manual V.3A §8.2.3.1
* @see atomic_store()
*/
intptr_t(atomic_load)(void *p, size_t n) {
intptr_t x = 0;
switch (n) {
case 1:
__builtin_memcpy(&x, p, 1);
return x;
case 2:
__builtin_memcpy(&x, p, 2);
return x;
case 4:
__builtin_memcpy(&x, p, 4);
return x;
case 8:
__builtin_memcpy(&x, p, 8);
return x;
default:
return 0;
}
}
// Asks kernel to let other threads be scheduled.
//
// @return 0 on success, or -1 w/ errno
sched_yield:
push %rbp
mov %rsp,%rbp
testb IsWindows()
jnz 1f
// UNIX Support
mov __NR_sched_yield,%eax
syscall
jmp 2f
// Windows Support
//
// A value of zero, together with the bAlertable parameter set to
// FALSE, causes the thread to relinquish the remainder of its time
// slice to any other thread that is ready to run, if there are no
// pending user APCs on the calling thread. If there are no other
// threads ready to run and no user APCs are queued, the function
// returns immediately, and the thread continues execution.
// Quoth MSDN
1: xor %ecx,%ecx
xor %edx,%edx
ntcall __imp_SleepEx
xor %eax,%eax
2: pop %rbp
ret
.endfn sched_yield,globl
.previous

View file

@ -1,13 +1,8 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_SPINLOCK_H_
#define COSMOPOLITAN_LIBC_INTRIN_SPINLOCK_H_
#include "libc/bits/weaken.h"
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/lockcmpxchg.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/log.h"
#include "libc/runtime/symbols.internal.h"
#if IsModeDbg() && !defined(_SPINLOCK_DEBUG)
#define _SPINLOCK_DEBUG
@ -15,13 +10,13 @@
#if defined(_SPINLOCK_DEBUG)
#define _spinlock(lock) _spinlock_ndebug(lock)
#define _spinlock_ndebug(lock) _spinlock_optimistic(lock)
#define _spinlock_ndebug(lock) _spinlock_cooperative(lock)
#elif defined(TINY)
#define _spinlock(lock) _spinlock_tiny(lock)
#define _spinlock_ndebug(lock) _spinlock_tiny(lock)
#else
#define _spinlock(lock) _spinlock_optimistic(lock)
#define _spinlock_ndebug(lock) _spinlock_optimistic(lock)
#define _spinlock(lock) _spinlock_cooperative(lock)
#define _spinlock_ndebug(lock) _spinlock_cooperative(lock)
#endif
#define _spunlock(lock) __atomic_clear(lock, __ATOMIC_RELAXED)
@ -41,15 +36,18 @@
} \
} while (0)
#define _spinlock_optimistic(lock) \
#define _spinlock_cooperative(lock) \
do { \
int __tries = 0; \
for (;;) { \
typeof(*(lock)) x; \
__atomic_load(lock, &x, __ATOMIC_RELAXED); \
if (!x && !_trylock(lock)) { \
break; \
} else { \
} else if (++__tries & 7) { \
__builtin_ia32_pause(); \
} else { \
sched_yield(); \
} \
} \
} while (0)
@ -57,7 +55,7 @@
#define _spinlock_debug(lock) \
do { \
typeof(*(lock)) me, owner; \
unsigned long warntries = 10000000; \
unsigned long warntries = 16777216; \
me = gettid(); \
if (!_lockcmpxchg(lock, 0, me)) { \
__atomic_load(lock, &owner, __ATOMIC_RELAXED); \
@ -71,7 +69,11 @@
kprintf("%s:%d: warning: possible deadlock on %s in %s()\n", \
__FILE__, __LINE__, #lock, __FUNCTION__); \
} \
__builtin_ia32_pause(); \
if (warntries & 7) { \
__builtin_ia32_pause(); \
} else { \
sched_yield(); \
} \
} \
} \
} while (0)

View file

@ -18,6 +18,7 @@
*/
#include "libc/calls/strace.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/runtime/runtime.h"
privileged void __stracef(const char *fmt, ...) {
va_list v;