mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-07-11 21:49:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Eliminate cyclic locks in runtime

Browse source 
This change introduces a new deadlock detector for Cosmo's POSIX threads
implementation. Error check mutexes will now track a DAG of nested locks
and report EDEADLK when a deadlock is theoretically possible. These will
occur rarely, but it's important for production hardening your code. You
don't even need to change your mutexes to use the POSIX error check mode
because `cosmocc -mdbg` will enable error checking on mutexes by default
globally. When cycles are found, an error message showing your demangled
symbols describing the strongly connected component are printed and then
the SIGTRAP is raised, which means you'll also get a backtrace if you're
using ShowCrashReports() too. This new error checker is so low-level and
so pure that it's able to verify the relationships of every libc runtime
lock, including those locks upon which the mutex implementation depends.
This commit is contained in:
Justine Tunney 2024-12-16 20:51:27 -08:00
parent 26c051c297
commit af7bd80430
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
141 changed files with 2094 additions and 1601 deletions
Download patch file Download diff file Expand all files Collapse all files

2
libc/intrin/__getenv.c
View file

@ -20,7 +20,7 @@
#include "libc/intrin/getenv.h"
#include "libc/intrin/kprintf.h"
privileged struct Env __getenv(char **p, const char *k) {
privileged optimizesize struct Env __getenv(char **p, const char *k) {
char *t;
int i, j;
for (i = 0; (t = p[i]); ++i) {

139
libc/intrin/bzero.c
View file

@ -16,155 +16,18 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/dce.h"
#include "libc/nexgen32e/nexgen32e.h"
#include "libc/nexgen32e/x86feature.h"
#include "libc/str/str.h"
typedef char xmm_t __attribute__((__vector_size__(16), __aligned__(1)));
typedef long long xmm_a __attribute__((__vector_size__(16), __aligned__(16)));
static void bzero128(char *p, size_t n) {
xmm_t v = {0};
if (n <= 32) {
*(xmm_t *)(p + n - 16) = v;
*(xmm_t *)p = v;
} else {
do {
n -= 32;
*(xmm_t *)(p + n) = v;
*(xmm_t *)(p + n + 16) = v;
} while (n > 32);
*(xmm_t *)(p + 16) = v;
*(xmm_t *)p = v;
}
}
#if defined(__x86_64__) && !defined(__chibicc__)
_Microarchitecture("avx") static void bzero_avx(char *p, size_t n) {
xmm_t v = {0};
if (n <= 32) {
*(xmm_t *)(p + n - 16) = v;
*(xmm_t *)p = v;
} else if (n >= 1024 && X86_HAVE(ERMS)) {
asm("rep stosb" : "+D"(p), "+c"(n), "=m"(*(char(*)[n])p) : "a"(0));
} else {
if (n < kHalfCache3 || !kHalfCache3) {
do {
n -= 32;
*(xmm_t *)(p + n) = v;
*(xmm_t *)(p + n + 16) = v;
} while (n > 32);
} else {
while ((uintptr_t)(p + n) & 15) {
p[--n] = 0;
}
do {
n -= 32;
__builtin_ia32_movntdq((xmm_a *)(p + n), (xmm_a)v);
__builtin_ia32_movntdq((xmm_a *)(p + n + 16), (xmm_a)v);
} while (n > 32);
asm("sfence");
}
*(xmm_t *)(p + 16) = v;
*(xmm_t *)p = v;
}
}
#endif
/**
* Sets memory to zero.
*
* bzero n=0 661 picoseconds
* bzero n=1 661 ps/byte 1,476 mb/s
* bzero n=2 330 ps/byte 2,952 mb/s
* bzero n=3 220 ps/byte 4,428 mb/s
* bzero n=4 165 ps/byte 5,904 mb/s
* bzero n=7 94 ps/byte 10,333 mb/s
* bzero n=8 41 ps/byte 23,618 mb/s
* bzero n=15 44 ps/byte 22,142 mb/s
* bzero n=16 20 ps/byte 47,236 mb/s
* bzero n=31 21 ps/byte 45,760 mb/s
* bzero n=32 20 ps/byte 47,236 mb/s
* bzero n=63 10 ps/byte 92,997 mb/s
* bzero n=64 15 ps/byte 62,982 mb/s
* bzero n=127 15 ps/byte 62,490 mb/s
* bzero n=128 10 ps/byte 94,473 mb/s
* bzero n=255 14 ps/byte 68,439 mb/s
* bzero n=256 9 ps/byte 105 gb/s
* bzero n=511 15 ps/byte 62,859 mb/s
* bzero n=512 11 ps/byte 83,976 mb/s
* bzero n=1023 15 ps/byte 61,636 mb/s
* bzero n=1024 10 ps/byte 88,916 mb/s
* bzero n=2047 9 ps/byte 105 gb/s
* bzero n=2048 8 ps/byte 109 gb/s
* bzero n=4095 8 ps/byte 115 gb/s
* bzero n=4096 8 ps/byte 118 gb/s
* bzero n=8191 7 ps/byte 129 gb/s
* bzero n=8192 7 ps/byte 130 gb/s
* bzero n=16383 6 ps/byte 136 gb/s
* bzero n=16384 6 ps/byte 137 gb/s
* bzero n=32767 6 ps/byte 140 gb/s
* bzero n=32768 6 ps/byte 141 gb/s
* bzero n=65535 15 ps/byte 64,257 mb/s
* bzero n=65536 15 ps/byte 64,279 mb/s
* bzero n=131071 15 ps/byte 63,166 mb/s
* bzero n=131072 15 ps/byte 63,115 mb/s
* bzero n=262143 15 ps/byte 62,052 mb/s
* bzero n=262144 15 ps/byte 62,097 mb/s
* bzero n=524287 15 ps/byte 61,699 mb/s
* bzero n=524288 15 ps/byte 61,674 mb/s
* bzero n=1048575 16 ps/byte 60,179 mb/s
* bzero n=1048576 15 ps/byte 61,330 mb/s
* bzero n=2097151 15 ps/byte 61,071 mb/s
* bzero n=2097152 15 ps/byte 61,065 mb/s
* bzero n=4194303 16 ps/byte 60,942 mb/s
* bzero n=4194304 16 ps/byte 60,947 mb/s
* bzero n=8388607 16 ps/byte 60,872 mb/s
* bzero n=8388608 16 ps/byte 60,879 mb/s
*
* @param p is memory address
* @param n is byte length
* @return p
* @asyncsignalsafe
*/
void bzero(void *p, size_t n) {
char *b;
uint64_t x;
b = p;
#ifdef __x86_64__
asm("xorl\t%k0,%k0" : "=r"(x));
#else
if (1) {
memset(p, 0, n);
return;
}
x = 0;
#endif
if (n <= 16) {
if (n >= 8) {
__builtin_memcpy(b, &x, 8);
__builtin_memcpy(b + n - 8, &x, 8);
} else if (n >= 4) {
__builtin_memcpy(b, &x, 4);
__builtin_memcpy(b + n - 4, &x, 4);
} else if (n) {
do {
asm volatile("" ::: "memory");
b[--n] = x;
} while (n);
}
#if defined(__x86_64__) && !defined(__chibicc__)
} else if (IsTiny()) {
asm("rep stosb" : "+D"(b), "+c"(n), "=m"(*(char(*)[n])b) : "a"(0));
return;
} else if (X86_HAVE(AVX)) {
bzero_avx(b, n);
#endif
} else {
bzero128(b, n);
}
memset(p, 0, n);
}
__weak_reference(bzero, explicit_bzero);

10
libc/intrin/cxalock.c
View file

@ -19,11 +19,7 @@
#include "libc/intrin/cxaatexit.h"
#include "libc/thread/thread.h"
static pthread_mutex_t __cxa_lock_obj;
void __cxa_wipe(void) {
pthread_mutex_init(&__cxa_lock_obj, 0);
}
pthread_mutex_t __cxa_lock_obj = PTHREAD_MUTEX_INITIALIZER;
void __cxa_lock(void) {
pthread_mutex_lock(&__cxa_lock_obj);
@ -32,7 +28,3 @@ void __cxa_lock(void) {
void __cxa_unlock(void) {
pthread_mutex_unlock(&__cxa_lock_obj);
}
__attribute__((__constructor__(60))) static textstartup void __cxa_init() {
pthread_atfork(__cxa_lock, __cxa_unlock, __cxa_wipe);
}

277
libc/intrin/deadlock.c Normal file
View file

@ -0,0 +1,277 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "ape/sections.internal.h"
#include "libc/assert.h"
#include "libc/atomic.h"
#include "libc/cosmo.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/maps.h"
#include "libc/macros.h"
#include "libc/str/str.h"
#include "libc/thread/lock.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
/**
* @fileoverview deadlock detector for statically allocated locks
*
* This module helps you spot multi-threading bugs in your program.
* High-level abstractions like mutexes are much easier to use than
* atomics, but they still carry their own non-obvious dangers. For
* example, nesting locks need to be nested in a consistent way and
* normal mutexes can't be required recursively. Normally this will
* cause your program to deadlock, i.e. hang indefinitely, but this
* module can detect such conditions and return errors instead, and
* better yet print helpful information when using `cosmocc -mdbg`.
*/
#define ABI privileged optimizesize
// building our visitor function using this optimizesize keyword shrinks
// the stack memory requirement from 7168 to 2048 bytes. totally amazing
// although please note this maximum isn't a hard limit. for normal mode
// builds your posix mandated mutex error checking will be less accurate
// but still helpful and reliable, although your cosmocc -mdbg will trap
// and report that you've run into the limit, so you can talk to justine
#define MAX_LOCKS 64
// cosmo's tib reserves space for 64 nested locks before things degrade.
// the cosmopolitan c runtime defines 16 locks, which are all registered
// with pthread_atfork(). it means you get to have 48 mutexes right now,
// and if you register all of them, then calling fork() will cause there
// to be 2080 edges in your lock graph. talk to justine if you need more
// because we're obviously going to need to find a way to make this grow
#define LOCK_EDGES_MAX 2080
// supported lock objects must define `void *_edges`
#define LOCK_EDGES_OFFSET 0
static_assert(offsetof(struct MapLock, edges) == LOCK_EDGES_OFFSET);
static_assert(offsetof(pthread_mutex_t, _edges) == LOCK_EDGES_OFFSET);
struct LockEdge {
struct LockEdge *next;
void *dest;
};
struct VisitedLock {
struct VisitedLock *next;
void *lock;
};
typedef _Atomic(struct LockEdge *) LockEdges;
static struct DeadlockDetector {
atomic_size_t edges_allocated;
struct LockEdge edges_memory[LOCK_EDGES_MAX];
} __deadlock;
forceinline struct CosmoTib *__deadlock_tls(void) {
return __get_tls_privileged();
}
forceinline LockEdges *get_lock_edges(void *lock) {
return (LockEdges *)((char *)lock + LOCK_EDGES_OFFSET);
}
forceinline struct LockEdge *load_lock_edges(LockEdges *edges) {
return atomic_load_explicit(edges, memory_order_relaxed);
}
ABI static int is_static_memory(void *lock) {
return _etext <= (unsigned char *)lock && (unsigned char *)lock < _end;
}
ABI static struct LockEdge *__deadlock_alloc(void) {
size_t edges_allocated =
atomic_load_explicit(&__deadlock.edges_allocated, memory_order_relaxed);
for (;;) {
if (edges_allocated == LOCK_EDGES_MAX) {
if (IsModeDbg()) {
kprintf("error: cosmo LOCK_EDGES_MAX needs to be increased\n");
DebugBreak();
}
return 0;
}
if (atomic_compare_exchange_weak_explicit(
&__deadlock.edges_allocated, &edges_allocated, edges_allocated + 1,
memory_order_relaxed, memory_order_relaxed))
return &__deadlock.edges_memory[edges_allocated];
}
}
ABI static void __deadlock_add_edge(void *from, void *dest) {
LockEdges *edges = get_lock_edges(from);
for (struct LockEdge *e = load_lock_edges(edges); e; e = e->next)
if (e->dest == dest)
return;
struct LockEdge *edge;
if ((edge = __deadlock_alloc())) {
edge->next = load_lock_edges(edges);
edge->dest = dest;
// we tolerate duplicate elements in the interest of performance.
// once an element is inserted, it's never removed. that's why we
// don't need need to worry about the aba problem. the cas itself
// is very important since it ensures inserted edges aren't lost.
for (;;)
if (atomic_compare_exchange_weak_explicit(edges, &edge->next, edge,
memory_order_relaxed,
memory_order_relaxed))
break;
}
}
ABI static bool __deadlock_visit(void *lock, struct VisitedLock *visited,
int notrap, int depth) {
if (++depth == MAX_LOCKS) {
if (IsModeDbg()) {
kprintf("error: too much recursion in deadlock detector\n");
DebugBreak();
}
return false;
}
for (struct VisitedLock *v = visited; v; v = v->next) {
if (v->lock == lock) {
if (IsModeDbg() && !notrap) {
// lock hierarchy violated!
//
// when you lock mutexes in a nested way, your locks must be
// nested in the same order globally. otherwise deadlocks might
// occur. for example, if you say in your first thread
//
// pthread_mutex_lock(&x);
// pthread_mutex_lock(&y);
// pthread_mutex_unlock(&y);
// pthread_mutex_unlock(&x);
//
// then in your second thread you say
//
// pthread_mutex_lock(&y);
// pthread_mutex_lock(&x);
// pthread_mutex_unlock(&x);
// pthread_mutex_unlock(&y);
//
// then a deadlock might happen, because {x→y, y→x} is cyclic!
// they don't happen often, but this is the kind of thing that
// matters if you want to build carrier grade production stuff
kprintf("error: cycle detected in directed graph of nested locks\n");
for (struct VisitedLock *v = visited; v; v = v->next)
kprintf("\t- %t\n", v->lock); // strongly connected component
DebugBreak();
}
return true;
}
}
LockEdges *edges = get_lock_edges(lock);
struct VisitedLock visit = {visited, lock};
for (struct LockEdge *e = load_lock_edges(edges); e; e = e->next)
if (__deadlock_visit(e->dest, &visit, notrap, depth))
return true;
return false;
}
/**
* Returns true if lock is already locked by calling thread.
*
* This function may return false negatives if we run out of TLS memory.
* That suboptimal condition will be reported in debug mode.
*
* @return 1 if lock is certainly owned by calling thread, 0 if lock is
* certainly not owned by calling thread, and -1 if we're uncertain
*/
ABI int __deadlock_tracked(void *lock) {
int full = 1;
int owned = 0;
struct CosmoTib *tib = __deadlock_tls();
for (int i = 0; i < ARRAYLEN(tib->tib_locks); ++i) {
full &= tib->tib_locks[i] != NULL;
owned |= tib->tib_locks[i] == lock;
}
if (full)
return -1;
if (!owned && !is_static_memory(lock))
return -1;
return owned;
}
/**
* Records that lock is held by thread.
* @param notrap can prevent error printing and debug breaking
* @asyncsignalsafe
*/
ABI void __deadlock_track(void *lock, int notrap) {
if (!notrap && !is_static_memory(lock))
return;
struct CosmoTib *tib = __deadlock_tls();
for (int i = 0; i < ARRAYLEN(tib->tib_locks); ++i) {
if (!tib->tib_locks[i]) {
tib->tib_locks[i] = lock;
return;
}
}
if (IsModeDbg()) {
kprintf("error: cosmo tls max lock depth needs to be increased!\n");
DebugBreak();
}
}
/**
* Records relationship for all held locks to `lock`.
* @param notrap can prevent error printing and debug breaking
* @asyncsignalsafe
*/
ABI void __deadlock_record(void *lock, int notrap) {
if (!notrap && !is_static_memory(lock))
return;
struct CosmoTib *tib = __deadlock_tls();
for (int i = 0; i < ARRAYLEN(tib->tib_locks); ++i)
if (tib->tib_locks[i] && tib->tib_locks[i] != lock)
__deadlock_add_edge(tib->tib_locks[i], lock);
}
/**
* Returns EDEADLK if locking `lock` could cause a deadlock.
* @param notrap can prevent error printing and debug breaking
* @asyncsignalsafe
*/
ABI int __deadlock_check(void *lock, int notrap) {
struct CosmoTib *tib = __deadlock_tls();
for (int i = 0; i < ARRAYLEN(tib->tib_locks); ++i) {
if (tib->tib_locks[i] == lock)
return 0;
if (tib->tib_locks[i]) {
struct VisitedLock visit = {0, tib->tib_locks[i]};
if (__deadlock_visit(lock, &visit, notrap, 0))
return EDEADLK;
}
}
return 0;
}
/**
* Records that lock isn't held by thread.
* @asyncsignalsafe
*/
ABI void __deadlock_untrack(void *lock) {
struct CosmoTib *tib = __deadlock_tls();
for (int i = 0; i < ARRAYLEN(tib->tib_locks); ++i)
tib->tib_locks[i] = tib->tib_locks[i] != lock ? tib->tib_locks[i] : 0;
}

186
libc/intrin/demangle.c
View file

@ -91,6 +91,8 @@ Copyright (c) 2024 Justine Tunney <jtunney@gmail.com>");
*
*/
#define ABI privileged optimizesize
#define DEMANGLE_NO_FLOATING_POINT
#define ASSERT(x) (void)0
@ -222,16 +224,18 @@ static int demangle_read_sname(struct demangle_data *);
static int demangle_read_subst(struct demangle_data *);
static int demangle_read_type(struct demangle_data *, struct type_delimit *);
static privileged size_t
ABI static size_t
demangle_strlen(const char *s)
{
size_t n = 0;
while (*s++)
while (*s++) {
asm volatile("" ::: "memory");
++n;
}
return n;
}
static privileged char *
ABI static char *
demangle_stpcpy(char *d, const char *s)
{
size_t i = 0;
@ -242,7 +246,7 @@ demangle_stpcpy(char *d, const char *s)
}
}
static privileged void *
ABI static void *
demangle_mempcpy(void *a, const void *b, size_t n)
{
char *d = a;
@ -252,14 +256,14 @@ demangle_mempcpy(void *a, const void *b, size_t n)
return d;
}
static privileged void *
ABI static void *
demangle_memcpy(void *a, const void *b, size_t n)
{
demangle_mempcpy(a, b, n);
return a;
}
static privileged int
ABI static int
demangle_strncmp(const char *a, const char *b, size_t n)
{
size_t i = 0;
@ -270,7 +274,7 @@ demangle_strncmp(const char *a, const char *b, size_t n)
return (a[i] & 0xff) - (b[i] & 0xff);
}
static privileged int
ABI static int
demangle_memcmp(const void *a, const void *b, size_t n)
{
int c;
@ -285,7 +289,7 @@ demangle_memcmp(const void *a, const void *b, size_t n)
return 0;
}
static privileged void
ABI static void
demangle_strlcpy(char *dst, const char *src, size_t dsize)
{
size_t remain;
@ -297,7 +301,7 @@ demangle_strlcpy(char *dst, const char *src, size_t dsize)
*dst = 0;
}
static privileged long
ABI static long
demangle_strtol(const char *s, int base)
{
static const uint8_t demangle_base36[80] = { 1, 2, 3, 4, 5, 6, 7, 8, 9,
@ -314,7 +318,7 @@ demangle_strtol(const char *s, int base)
return x;
}
static privileged char *
ABI static char *
demangle_strstr(const char *haystack, const char *needle)
{
size_t i;
@ -335,7 +339,7 @@ demangle_strstr(const char *haystack, const char *needle)
return 0;
}
static privileged char *
ABI static char *
demangle_utoa(char *p, unsigned long long x)
{
char t;
@ -356,7 +360,7 @@ demangle_utoa(char *p, unsigned long long x)
return p + i;
}
static privileged char *
ABI static char *
demangle_itoa(char *p, long long x)
{
if (x < 0)
@ -364,7 +368,7 @@ demangle_itoa(char *p, long long x)
return demangle_utoa(p, x);
}
static privileged void
ABI static void
demangle_free(struct demangle_data *h, void *ptr)
{
index_t base;
@ -381,7 +385,7 @@ demangle_free(struct demangle_data *h, void *ptr)
}
}
static privileged returnspointerwithnoaliases returnsnonnull void *
ABI static returnspointerwithnoaliases returnsnonnull void *
demangle_malloc(struct demangle_data *h, long a, long n)
{
long rem;
@ -438,7 +442,7 @@ demangle_malloc(struct demangle_data *h, long a, long n)
}
}
static privileged returnspointerwithnoaliases char *
ABI static returnspointerwithnoaliases char *
demangle_strdup(struct demangle_data *h, const char *s)
{
char *d = 0;
@ -450,7 +454,7 @@ demangle_strdup(struct demangle_data *h, const char *s)
return d;
}
static privileged void
ABI static void
demangle_vector_str_dest(struct demangle_data *h, struct vector_str *v)
{
int i;
@ -459,7 +463,7 @@ demangle_vector_str_dest(struct demangle_data *h, struct vector_str *v)
demangle_free(h, v->container);
}
static privileged void
ABI static void
demangle_vector_type_qualifier_dest(struct demangle_data *d,
struct vector_type_qualifier *v)
{
@ -467,7 +471,7 @@ demangle_vector_type_qualifier_dest(struct demangle_data *d,
demangle_vector_str_dest(d, &v->ext_name);
}
static privileged void
ABI static void
demangle_stack_str_init(struct stack_str *ss)
{
ss->str = ss->buf;
@ -476,7 +480,7 @@ demangle_stack_str_init(struct stack_str *ss)
ss->cap = sizeof(ss->buf);
}
static privileged void
ABI static void
demangle_stack_str_append(struct demangle_data *h, struct stack_str *ss,
const char *str, size_t len)
{
@ -499,7 +503,7 @@ demangle_stack_str_append(struct demangle_data *h, struct stack_str *ss,
#define demangle_stack_str_append_str(h, ss, s) \
demangle_stack_str_append(h, ss, s, demangle_strlen(s))
static privileged size_t
ABI static size_t
demangle_get_strlen_sum(struct demangle_data *h, const struct vector_str *v)
{
size_t i, len = 0;
@ -509,7 +513,7 @@ demangle_get_strlen_sum(struct demangle_data *h, const struct vector_str *v)
return len;
}
static privileged int
ABI static int
demangle_demangle_strncmp(const char *a, const char *b, size_t n)
{
size_t i = 0;
@ -527,7 +531,7 @@ demangle_demangle_strncmp(const char *a, const char *b, size_t n)
* @param l Length of the string.
* @return -1 at failed, 0 at not found, 1 at found.
*/
static privileged int
ABI static int
demangle_vector_str_find(struct demangle_data *h, const struct vector_str *v,
const char *o, size_t l)
{
@ -551,7 +555,7 @@ demangle_vector_str_find(struct demangle_data *h, const struct vector_str *v,
* @param l Length of the string.
* @return NULL at failed or NUL terminated new allocated string.
*/
static privileged char *
ABI static char *
demangle_vector_str_get_flat(struct demangle_data *ddata,
const struct vector_str *v, size_t *l)
{
@ -577,7 +581,7 @@ demangle_vector_str_get_flat(struct demangle_data *ddata,
return rtn;
}
static privileged void
ABI static void
demangle_vector_str_grow(struct demangle_data *ddata, struct vector_str *v)
{
size_t i, tmp_cap;
@ -605,7 +609,7 @@ demangle_vector_str_grow(struct demangle_data *ddata, struct vector_str *v)
* @brief Initialize vector_str.
* @return false at failed, true at success.
*/
static privileged void
ABI static void
demangle_vector_str_init(struct demangle_data *ddata, struct vector_str *v)
{
v->size = 0;
@ -621,7 +625,7 @@ demangle_vector_str_init(struct demangle_data *ddata, struct vector_str *v)
* @brief Remove last element in vector_str.
* @return false at failed, true at success.
*/
static privileged bool
ABI static bool
demangle_vector_str_pop(struct vector_str *v)
{
if (!v)
@ -641,7 +645,7 @@ demangle_vector_str_pop(struct vector_str *v)
* @brief Push back string to vector.
* @return false at failed, true at success.
*/
static privileged bool
ABI static bool
demangle_vector_str_push(struct demangle_data *ddata, struct vector_str *v,
const char *str, size_t len)
{
@ -665,7 +669,7 @@ demangle_vector_str_push(struct demangle_data *ddata, struct vector_str *v,
* @brief Push front org vector to det vector.
* @return false at failed, true at success.
*/
static privileged bool
ABI static bool
demangle_vector_str_push_vector_head(struct demangle_data *ddata,
struct vector_str *dst, struct vector_str *org)
{
@ -698,7 +702,7 @@ demangle_vector_str_push_vector_head(struct demangle_data *ddata,
* @brief Push org vector to the tail of det vector.
* @return false at failed, true at success.
*/
static privileged bool
ABI static bool
demangle_vector_str_push_vector(struct demangle_data *ddata,
struct vector_str *dst, struct vector_str *org)
{
@ -736,7 +740,7 @@ demangle_vector_str_push_vector(struct demangle_data *ddata,
* If r_len is not NULL, string length will be returned.
* @return NULL at failed or NUL terminated new allocated string.
*/
static privileged returnspointerwithnoaliases char *
ABI static returnspointerwithnoaliases char *
demangle_vector_str_substr(struct demangle_data *ddata,
const struct vector_str *v, size_t begin, size_t end, size_t *r_len)
{
@ -762,7 +766,7 @@ demangle_vector_str_substr(struct demangle_data *ddata,
return rtn;
}
static privileged int
ABI static int
demangle_vector_read_cmd_pop(struct vector_read_cmd *v)
{
if (!v->size)
@ -775,7 +779,7 @@ demangle_vector_read_cmd_pop(struct vector_read_cmd *v)
return 1;
}
static privileged void
ABI static void
demangle_vector_read_cmd_init(struct demangle_data *ddata,
struct vector_read_cmd *v)
{
@ -786,7 +790,7 @@ demangle_vector_read_cmd_init(struct demangle_data *ddata,
alignof(*v->r_container), sizeof(*v->r_container) * v->capacity);
}
static privileged void
ABI static void
demangle_data_init(struct demangle_data *d, const char *cur)
{
demangle_vector_str_init(d, &d->output);
@ -816,7 +820,7 @@ demangle_data_init(struct demangle_data *d, const char *cur)
d->last_sname = NULL;
}
static privileged int
ABI static int
demangle_push_str(struct demangle_data *ddata, const char *str, size_t len)
{
if (!str || !len)
@ -833,7 +837,7 @@ demangle_push_str(struct demangle_data *ddata, const char *str, size_t len)
}
#ifndef DEMANGLE_NO_FLOATING_POINT
static privileged int
ABI static int
demangle_push_fp(struct demangle_data *ddata,
char *decoder(struct demangle_data *, const char *, size_t))
{
@ -862,13 +866,13 @@ demangle_push_fp(struct demangle_data *ddata,
}
#endif // DEMANGLE_NO_FLOATING_POINT
static privileged int
ABI static int
demangle_pop_str(struct demangle_data *ddata)
{
return demangle_vector_str_pop(ddata->cur_output);
}
static privileged int
ABI static int
demangle_push_subst(struct demangle_data *ddata, const char *str, size_t len)
{
if (!str || !len)
@ -880,7 +884,7 @@ demangle_push_subst(struct demangle_data *ddata, const char *str, size_t len)
return 1;
}
static privileged int
ABI static int
demangle_push_subst_v(struct demangle_data *ddata, struct vector_str *v)
{
int rtn;
@ -900,7 +904,7 @@ demangle_push_subst_v(struct demangle_data *ddata, struct vector_str *v)
return rtn;
}
static privileged int
ABI static int
demangle_push_type_qualifier(struct demangle_data *ddata,
struct vector_type_qualifier *v, const char *type_str)
{
@ -1133,7 +1137,7 @@ demangle_push_type_qualifier(struct demangle_data *ddata,
return 1;
}
static privileged int
ABI static int
demangle_get_subst(struct demangle_data *ddata, size_t idx)
{
size_t len;
@ -1151,7 +1155,7 @@ demangle_get_subst(struct demangle_data *ddata, size_t idx)
return 1;
}
static privileged int
ABI static int
demangle_get_tmpl_param(struct demangle_data *ddata, size_t idx)
{
size_t len;
@ -1168,7 +1172,7 @@ demangle_get_tmpl_param(struct demangle_data *ddata, size_t idx)
return 1;
}
static privileged int
ABI static int
demangle_read_array(struct demangle_data *ddata)
{
size_t i, num_len, exp_len, p_idx, idx;
@ -1240,7 +1244,7 @@ demangle_read_array(struct demangle_data *ddata)
#ifndef DEMANGLE_NO_FLOATING_POINT
/* Simple hex to integer function used by decode_to_* function. */
static privileged int
ABI static int
hex_to_dec(char c)
{
switch (c) {
@ -1288,7 +1292,7 @@ hex_to_dec(char c)
* Todo
* Replace these functions to macro.
*/
static privileged returnspointerwithnoaliases char *
ABI static returnspointerwithnoaliases char *
decode_fp_to_double(struct demangle_data *ddata, const char *p, size_t len)
{
double f;
@ -1332,7 +1336,7 @@ again:
return rtn;
}
static privileged returnspointerwithnoaliases char *
ABI static returnspointerwithnoaliases char *
decode_fp_to_float(struct demangle_data *ddata, const char *p, size_t len)
{
size_t i, rtn_len, limit;
@ -1374,7 +1378,7 @@ again:
return rtn;
}
static privileged returnspointerwithnoaliases char *
ABI static returnspointerwithnoaliases char *
decode_fp_to_long_double(struct demangle_data *ddata, const char *p, size_t len)
{
long double f;
@ -1418,7 +1422,7 @@ again:
return rtn;
}
static privileged returnspointerwithnoaliases char *
ABI static returnspointerwithnoaliases char *
decode_fp_to_float128(struct demangle_data *ddata, const char *p, size_t len)
{
long double f;
@ -1475,7 +1479,7 @@ decode_fp_to_float128(struct demangle_data *ddata, const char *p, size_t len)
}
}
static privileged returnspointerwithnoaliases char *
ABI static returnspointerwithnoaliases char *
decode_fp_to_float80(struct demangle_data *ddata, const char *p, size_t len)
{
long double f;
@ -1538,7 +1542,7 @@ decode_fp_to_float80(struct demangle_data *ddata, const char *p, size_t len)
#endif // DEMANGLE_NO_FLOATING_POINT
static privileged int
ABI static int
demangle_read_expr_primary(struct demangle_data *ddata)
{
const char *num;
@ -1630,7 +1634,7 @@ demangle_read_expr_primary(struct demangle_data *ddata)
* http://gcc.gnu.org/bugzilla/show_bug.cgi?id=31775
* http://gcc.gnu.org/viewcvs?view=rev&revision=124467
*/
static privileged int
ABI static int
demangle_local_source_name(struct demangle_data *ddata)
{
/* L */
@ -1656,7 +1660,7 @@ demangle_local_source_name(struct demangle_data *ddata)
* read unqualified-name, unqualified name are operator-name, ctor-dtor-name,
* source-name
*/
static privileged int
ABI static int
demangle_read_uqname(struct demangle_data *ddata)
{
size_t len;
@ -2085,7 +2089,7 @@ demangle_read_uqname(struct demangle_data *ddata)
* Read template parameter that forms in 'T[number]_'.
* This function much like to read_subst but only for types.
*/
static privileged int
ABI static int
demangle_read_tmpl_param(struct demangle_data *ddata)
{
long nth;
@ -2116,7 +2120,7 @@ demangle_read_tmpl_param(struct demangle_data *ddata)
return 0;
}
static privileged int
ABI static int
demangle_vector_read_cmd_push(struct demangle_data *ddata,
struct vector_read_cmd *v, enum read_cmd cmd, void *data)
{
@ -2145,7 +2149,7 @@ demangle_vector_read_cmd_push(struct demangle_data *ddata,
return 1;
}
static privileged int
ABI static int
demangle_read_tmpl_arg(struct demangle_data *ddata)
{
if (*ddata->cur == '\0')
@ -2164,7 +2168,7 @@ demangle_read_tmpl_arg(struct demangle_data *ddata)
return demangle_read_type(ddata, NULL);
}
static privileged int
ABI static int
demangle_read_tmpl_args(struct demangle_data *ddata)
{
struct vector_str *v;
@ -2217,7 +2221,7 @@ demangle_read_tmpl_args(struct demangle_data *ddata)
return demangle_vector_read_cmd_pop(&ddata->cmd);
}
static privileged int
ABI static int
demangle_read_expression_trinary(struct demangle_data *ddata, const char *name1,
size_t len1, const char *name2, size_t len2)
{
@ -2236,7 +2240,7 @@ demangle_read_expression_trinary(struct demangle_data *ddata, const char *name1,
return demangle_read_expression(ddata);
}
static privileged int
ABI static int
demangle_read_expression_unary(struct demangle_data *ddata, const char *name,
size_t len)
{
@ -2248,7 +2252,7 @@ demangle_read_expression_unary(struct demangle_data *ddata, const char *name,
return demangle_push_str(ddata, name, len);
}
static privileged int
ABI static int
demangle_read_expression_binary(struct demangle_data *ddata, const char *name,
size_t len)
{
@ -2262,7 +2266,7 @@ demangle_read_expression_binary(struct demangle_data *ddata, const char *name,
return demangle_read_expression(ddata);
}
static privileged int
ABI static int
demangle_read_expression_impl(struct demangle_data *ddata)
{
if (*ddata->cur == '\0')
@ -2544,7 +2548,7 @@ demangle_read_expression_impl(struct demangle_data *ddata)
return 0;
}
static privileged int
ABI static int
demangle_read_expression(struct demangle_data *ddata)
{
if (ddata->depth == MAX_DEPTH)
@ -2555,7 +2559,7 @@ demangle_read_expression(struct demangle_data *ddata)
return res;
}
static privileged int
ABI static int
demangle_read_expression_flat(struct demangle_data *ddata, char **str)
{
struct vector_str *output;
@ -2584,7 +2588,7 @@ demangle_read_expression_flat(struct demangle_data *ddata, char **str)
}
/* size, capacity, ext_name */
static privileged void
ABI static void
demangle_vector_type_qualifier_init(struct demangle_data *ddata,
struct vector_type_qualifier *v)
{
@ -2600,7 +2604,7 @@ demangle_vector_type_qualifier_init(struct demangle_data *ddata,
demangle_vector_str_init(ddata, &v->ext_name);
}
static privileged struct read_cmd_item *
ABI static struct read_cmd_item *
demangle_vector_read_cmd_find(struct vector_read_cmd *v, enum read_cmd dst)
{
int i;
@ -2615,7 +2619,7 @@ demangle_vector_read_cmd_find(struct vector_read_cmd *v, enum read_cmd dst)
return 0;
}
static privileged int
ABI static int
demangle_read_function(struct demangle_data *ddata, int *ext_c,
struct vector_type_qualifier *v)
{
@ -2751,7 +2755,7 @@ demangle_read_function(struct demangle_data *ddata, int *ext_c,
return 1;
}
static privileged int
ABI static int
demangle_read_offset_number(struct demangle_data *ddata)
{
bool negative;
@ -2787,7 +2791,7 @@ demangle_read_offset_number(struct demangle_data *ddata)
return 1;
}
static privileged int
ABI static int
demangle_read_nv_offset(struct demangle_data *ddata)
{
if (!DEM_PUSH_STR(ddata, "offset : "))
@ -2796,7 +2800,7 @@ demangle_read_nv_offset(struct demangle_data *ddata)
return demangle_read_offset_number(ddata);
}
static privileged int
ABI static int
demangle_read_v_offset(struct demangle_data *ddata)
{
if (!DEM_PUSH_STR(ddata, "offset : "))
@ -2812,7 +2816,7 @@ demangle_read_v_offset(struct demangle_data *ddata)
}
/* read offset, offset are nv-offset, v-offset */
static privileged int
ABI static int
demangle_read_offset(struct demangle_data *ddata)
{
if (*ddata->cur == 'h') {
@ -2826,7 +2830,7 @@ demangle_read_offset(struct demangle_data *ddata)
return 0;
}
static privileged int
ABI static int
demangle_read_type_flat(struct demangle_data *ddata, char **str)
{
struct vector_str *output;
@ -2858,7 +2862,7 @@ demangle_read_type_flat(struct demangle_data *ddata, char **str)
* read number
* number ::= [n] <decimal>
*/
static privileged int
ABI static int
demangle_read_number(struct demangle_data *ddata, long *rtn)
{
long len, negative_factor;
@ -2887,7 +2891,7 @@ demangle_read_number(struct demangle_data *ddata, long *rtn)
return 1;
}
static privileged int
ABI static int
demangle_read_number_as_string(struct demangle_data *ddata, char **str)
{
long n;
@ -2904,7 +2908,7 @@ demangle_read_number_as_string(struct demangle_data *ddata, char **str)
return 1;
}
static privileged int
ABI static int
demangle_read_encoding_impl(struct demangle_data *ddata)
{
char *name, *type, *num_str;
@ -3113,7 +3117,7 @@ demangle_read_encoding_impl(struct demangle_data *ddata)
}
/* read encoding, encoding are function name, data name, special-name */
static privileged int
ABI static int
demangle_read_encoding(struct demangle_data *ddata)
{
if (ddata->depth == MAX_DEPTH)
@ -3124,7 +3128,7 @@ demangle_read_encoding(struct demangle_data *ddata)
return res;
}
static privileged int
ABI static int
demangle_read_local_name(struct demangle_data *ddata)
{
struct vector_str local_name;
@ -3205,7 +3209,7 @@ demangle_read_local_name(struct demangle_data *ddata)
return 1;
}
static privileged int
ABI static int
demangle_read_nested_name(struct demangle_data *ddata)
{
struct stack_str v;
@ -3293,7 +3297,7 @@ next:
return 1;
}
static privileged int
ABI static int
demangle_read_name_impl(struct demangle_data *ddata)
{
struct stack_str v;
@ -3355,7 +3359,7 @@ clean:
return rtn;
}
static privileged int
ABI static int
demangle_read_name(struct demangle_data *ddata)
{
if (ddata->depth == MAX_DEPTH)
@ -3366,7 +3370,7 @@ demangle_read_name(struct demangle_data *ddata)
return res;
}
static privileged int
ABI static int
demangle_read_name_flat(struct demangle_data *ddata, char **str)
{
struct vector_str *output;
@ -3394,7 +3398,7 @@ demangle_read_name_flat(struct demangle_data *ddata, char **str)
return 1;
}
static privileged int
ABI static int
demangle_read_pointer_to_member(struct demangle_data *ddata,
struct vector_type_qualifier *v)
{
@ -3454,7 +3458,7 @@ clean1:
}
/* read source-name, source-name is <len> <ID> */
static privileged int
ABI static int
demangle_read_sname(struct demangle_data *ddata)
{
size_t lim;
@ -3485,7 +3489,7 @@ demangle_read_sname(struct demangle_data *ddata)
return 1;
}
static privileged int
ABI static int
demangle_read_subst_stdtmpl(struct demangle_data *ddata, const char *str)
{
struct vector_str *output;
@ -3523,7 +3527,7 @@ demangle_read_subst_stdtmpl(struct demangle_data *ddata, const char *str)
return 1;
}
static privileged int
ABI static int
demangle_read_subst_std(struct demangle_data *ddata)
{
struct vector_str *output, v;
@ -3574,7 +3578,7 @@ demangle_read_subst_std(struct demangle_data *ddata)
return 1;
}
static privileged int
ABI static int
demangle_read_subst(struct demangle_data *ddata)
{
long nth;
@ -3702,7 +3706,7 @@ demangle_read_subst(struct demangle_data *ddata)
return 0;
}
static privileged int
ABI static int
demangle_vector_type_qualifier_push(struct demangle_data *ddata,
struct vector_type_qualifier *v, enum type_qualifier t)
{
@ -3731,7 +3735,7 @@ demangle_vector_type_qualifier_push(struct demangle_data *ddata,
return 1;
}
static privileged int
ABI static int
demangle_read_type_impl(struct demangle_data *ddata, struct type_delimit *td)
{
struct vector_type_qualifier v;
@ -4254,7 +4258,7 @@ clean:
return 0;
}
static privileged int
ABI static int
demangle_read_type(struct demangle_data *ddata, struct type_delimit *td)
{
if (ddata->depth == MAX_DEPTH)
@ -4265,7 +4269,7 @@ demangle_read_type(struct demangle_data *ddata, struct type_delimit *td)
return res;
}
static privileged int
ABI static int
demangle_copy_output(struct demangle_data *ddata, char *buf,
const struct vector_str *v, size_t buflen)
{
@ -4288,14 +4292,14 @@ demangle_copy_output(struct demangle_data *ddata, char *buf,
return -1;
}
static privileged int
ABI static int
demangle_failure(char *buf, const char *org, size_t buflen)
{
demangle_strlcpy(buf, org, buflen);
return -1;
}
static privileged int
ABI static int
demangle(struct demangle_data *ddata, char *buf, const char *org, size_t buflen)
{
struct vector_str ret_type;
@ -4447,7 +4451,7 @@ demangle(struct demangle_data *ddata, char *buf, const char *org, size_t buflen)
* @return bytes of output name or -1 upon error or truncation
* @asyncsignalsafe
*/
privileged int
ABI int
__demangle(char *buf, const char *org, size_t buflen)
{
struct demangle_data ddata[1];
@ -4461,7 +4465,7 @@ __demangle(char *buf, const char *org, size_t buflen)
*
* This means it starts with either "_Z" or "_GLOBAL__I_".
*/
privileged int
ABI int
__is_mangled(const char *org)
{
if (!org)

9
libc/intrin/describebacktrace.c
View file

@ -24,13 +24,15 @@
#define N 160
privileged static bool IsDangerous(const void *ptr) {
#define ABI privileged optimizesize
ABI static bool IsDangerous(const void *ptr) {
if (_weaken(kisdangerous))
return _weaken(kisdangerous)(ptr);
return false;
}
privileged static char *FormatHex(char *p, unsigned long x) {
ABI static char *FormatHex(char *p, unsigned long x) {
int k = x ? (__builtin_clzl(x) ^ 63) + 1 : 1;
k = (k + 3) & -4;
while (k > 0)
@ -39,8 +41,7 @@ privileged static char *FormatHex(char *p, unsigned long x) {
return p;
}
privileged dontinstrument const char *_DescribeBacktrace(
char buf[N], const struct StackFrame *fr) {
ABI const char *_DescribeBacktrace(char buf[N], const struct StackFrame *fr) {
char *p = buf;
char *pe = p + N;
bool gotsome = false;

2
libc/intrin/fds.c
View file

@ -44,6 +44,7 @@
#include "libc/sysv/consts/o.h"
#include "libc/sysv/consts/prot.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
#define OPEN_MAX 16
@ -86,6 +87,7 @@ static textwindows void SetupWinStd(struct Fds *fds, int i, uint32_t x) {
}
textstartup void __init_fds(int argc, char **argv, char **envp) {
struct Fds *fds;
fds = &g_fds;
fds->n = 4;

3
libc/intrin/getsafesize.greg.c
View file

@ -22,7 +22,6 @@
#include "libc/runtime/stack.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/tls.h"
#include "libc/thread/tls2.internal.h"
/**
* Computes safer buffer size for alloca().
@ -32,7 +31,7 @@
* @return number of bytes to use for your buffer, or negative if the
* allocation would likely cause a stack overflow
*/
privileged long __get_safe_size(long want, long extraspace) {
privileged optimizesize long __get_safe_size(long want, long extraspace) {
if (!__tls_enabled)
return want;
struct PosixThread *pt;

28
libc/intrin/kprintf.greg.c
View file

@ -65,10 +65,11 @@
#include "libc/sysv/consts/o.h"
#include "libc/sysv/consts/prot.h"
#include "libc/thread/tls.h"
#include "libc/thread/tls2.internal.h"
#include "libc/vga/vga.internal.h"
#include "libc/wctype.h"
#define ABI privileged optimizesize
#define STACK_ERROR "kprintf error: stack is about to overflow\n"
#define KGETINT(x, va, t, s) \
@ -159,7 +160,7 @@ __funline bool kischarmisaligned(const char *p, signed char t) {
return false;
}
privileged bool32 kisdangerous(const void *addr) {
ABI bool32 kisdangerous(const void *addr) {
bool32 res = true;
__maps_lock();
if (__maps.maps) {
@ -175,7 +176,7 @@ privileged bool32 kisdangerous(const void *addr) {
return res;
}
privileged static void klogclose(long fd) {
ABI static void klogclose(long fd) {
#ifdef __x86_64__
long ax = __NR_close;
asm volatile("syscall"
@ -192,7 +193,7 @@ privileged static void klogclose(long fd) {
#endif
}
privileged static long klogfcntl(long fd, long cmd, long arg) {
ABI static long klogfcntl(long fd, long cmd, long arg) {
#ifdef __x86_64__
char cf;
long ax = __NR_fcntl;
@ -224,7 +225,7 @@ privileged static long klogfcntl(long fd, long cmd, long arg) {
#endif
}
privileged static long klogopen(const char *path) {
ABI static long klogopen(const char *path) {
long dirfd = AT_FDCWD;
long flags = O_WRONLY | O_CREAT | O_APPEND;
long mode = 0600;
@ -263,7 +264,7 @@ privileged static long klogopen(const char *path) {
}
// returns log handle or -1 if logging shouldn't happen
privileged long kloghandle(void) {
ABI long kloghandle(void) {
// kprintf() needs to own a file descriptor in case apps closes stderr
// our close() and dup() implementations will trigger this initializer
// to minimize a chance that the user accidentally closes their logger
@ -342,7 +343,7 @@ privileged long kloghandle(void) {
}
#ifdef __x86_64__
privileged void _klog_serial(const char *b, size_t n) {
ABI void _klog_serial(const char *b, size_t n) {
size_t i;
uint16_t dx;
unsigned char al;
@ -362,7 +363,7 @@ privileged void _klog_serial(const char *b, size_t n) {
}
#endif /* __x86_64__ */
privileged void klog(const char *b, size_t n) {
ABI void klog(const char *b, size_t n) {
#ifdef __x86_64__
long h;
uint32_t wrote;
@ -420,8 +421,7 @@ privileged void klog(const char *b, size_t n) {
#endif
}
privileged static size_t kformat(char *b, size_t n, const char *fmt,
va_list va) {
ABI static size_t kformat(char *b, size_t n, const char *fmt, va_list va) {
int si;
wint_t t, u;
const char *abet;
@ -1033,7 +1033,7 @@ privileged static size_t kformat(char *b, size_t n, const char *fmt,
* @asyncsignalsafe
* @vforksafe
*/
privileged size_t ksnprintf(char *b, size_t n, const char *fmt, ...) {
ABI size_t ksnprintf(char *b, size_t n, const char *fmt, ...) {
size_t m;
va_list v;
va_start(v, fmt);
@ -1052,7 +1052,7 @@ privileged size_t ksnprintf(char *b, size_t n, const char *fmt, ...) {
* @asyncsignalsafe
* @vforksafe
*/
privileged size_t kvsnprintf(char *b, size_t n, const char *fmt, va_list v) {
ABI size_t kvsnprintf(char *b, size_t n, const char *fmt, va_list v) {
return kformat(b, n, fmt, v);
}
@ -1063,7 +1063,7 @@ privileged size_t kvsnprintf(char *b, size_t n, const char *fmt, va_list v) {
* @asyncsignalsafe
* @vforksafe
*/
privileged void kvprintf(const char *fmt, va_list v) {
ABI void kvprintf(const char *fmt, va_list v) {
#pragma GCC push_options
#pragma GCC diagnostic ignored "-Walloca-larger-than="
long size = __get_safe_size(8000, 8000);
@ -1149,7 +1149,7 @@ privileged void kvprintf(const char *fmt, va_list v) {
* @asyncsignalsafe
* @vforksafe
*/
privileged void kprintf(const char *fmt, ...) {
ABI void kprintf(const char *fmt, ...) {
// system call support runtime depends on this function
// function tracing runtime depends on this function
// asan runtime depends on this function

29
libc/intrin/localtime_lock.c Normal file
View file

@ -0,0 +1,29 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "third_party/tz/lock.h"
pthread_mutex_t __localtime_lock_obj = PTHREAD_MUTEX_INITIALIZER;
void __localtime_lock(void) {
pthread_mutex_lock(&__localtime_lock_obj);
}
void __localtime_unlock(void) {
pthread_mutex_unlock(&__localtime_lock_obj);
}

68
libc/intrin/maps.c
View file

@ -19,14 +19,15 @@
#include "libc/intrin/maps.h"
#include "ape/sections.internal.h"
#include "libc/calls/state.internal.h"
#include "libc/cosmo.h"
#include "libc/dce.h"
#include "libc/intrin/describebacktrace.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/maps.h"
#include "libc/nexgen32e/rdtsc.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/stack.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/prot.h"
#include "libc/thread/lock.h"
@ -34,6 +35,12 @@
__static_yoink("_init_maps");
#endif
#define ABI privileged optimizespeed
// take great care if you enable this
// especially if you're using --ftrace too
#define DEBUG_MAPS_LOCK 0
struct Maps __maps;
void __maps_add(struct Map *map) {
@ -65,6 +72,10 @@ void __maps_stack(char *stackaddr, int pagesz, int guardsize, size_t stacksize,
void __maps_init(void) {
int pagesz = __pagesize;
// initialize lemur64 rng
__maps.rand = 2131259787901769494;
__maps.rand ^= rdtsc();
// record _start() stack mapping
if (!IsWindows()) {
struct AddrSize stack;
@ -88,7 +99,16 @@ void __maps_init(void) {
__maps_adder(&text, pagesz);
}
privileged bool __maps_lock(void) {
#if DEBUG_MAPS_LOCK
privileged static void __maps_panic(const char *msg) {
// it's only safe to pass a format string. if we use directives such
// as %s, %t etc. then kprintf() will recursively call __maps_lock()
kprintf(msg);
DebugBreak();
}
#endif
ABI bool __maps_lock(void) {
int me;
uint64_t word, lock;
struct CosmoTib *tib;
@ -101,24 +121,35 @@ privileged bool __maps_lock(void) {
me = atomic_load_explicit(&tib->tib_tid, memory_order_acquire);
if (me <= 0)
return false;
word = atomic_load_explicit(&__maps.lock, memory_order_relaxed);
word = atomic_load_explicit(&__maps.lock.word, memory_order_relaxed);
for (;;) {
if (MUTEX_OWNER(word) == me) {
if (atomic_compare_exchange_weak_explicit(
&__maps.lock, &word, MUTEX_INC_DEPTH(word), memory_order_relaxed,
memory_order_relaxed))
&__maps.lock.word, &word, MUTEX_INC_DEPTH(word),
memory_order_relaxed, memory_order_relaxed))
return true;
continue;
}
#if DEBUG_MAPS_LOCK
if (__deadlock_tracked(&__maps.lock) == 1)
__maps_panic("error: maps lock already held\n");
if (__deadlock_check(&__maps.lock, 1))
__maps_panic("error: maps lock is cyclic\n");
#endif
word = 0;
lock = MUTEX_LOCK(word);
lock = MUTEX_SET_OWNER(lock, me);
if (atomic_compare_exchange_weak_explicit(&__maps.lock, &word, lock,
if (atomic_compare_exchange_weak_explicit(&__maps.lock.word, &word, lock,
memory_order_acquire,
memory_order_relaxed))
memory_order_relaxed)) {
#if DEBUG_MAPS_LOCK
__deadlock_track(&__maps.lock, 0);
__deadlock_record(&__maps.lock, 0);
#endif
return false;
}
for (;;) {
word = atomic_load_explicit(&__maps.lock, memory_order_relaxed);
word = atomic_load_explicit(&__maps.lock.word, memory_order_relaxed);
if (MUTEX_OWNER(word) == me)
break;
if (!word)
@ -127,7 +158,7 @@ privileged bool __maps_lock(void) {
}
}
privileged void __maps_unlock(void) {
ABI void __maps_unlock(void) {
int me;
uint64_t word;
struct CosmoTib *tib;
@ -140,16 +171,25 @@ privileged void __maps_unlock(void) {
me = atomic_load_explicit(&tib->tib_tid, memory_order_acquire);
if (me <= 0)
return;
word = atomic_load_explicit(&__maps.lock, memory_order_relaxed);
word = atomic_load_explicit(&__maps.lock.word, memory_order_relaxed);
#if DEBUG_MAPS_LOCK
if (__deadlock_tracked(&__maps.lock) == 0)
__maps_panic("error: maps lock not owned by caller\n");
#endif
for (;;) {
if (MUTEX_DEPTH(word)) {
if (atomic_compare_exchange_weak_explicit(
&__maps.lock, &word, MUTEX_DEC_DEPTH(word), memory_order_relaxed,
memory_order_relaxed))
&__maps.lock.word, &word, MUTEX_DEC_DEPTH(word),
memory_order_relaxed, memory_order_relaxed))
break;
}
if (atomic_compare_exchange_weak_explicit(
&__maps.lock, &word, 0, memory_order_release, memory_order_relaxed))
if (atomic_compare_exchange_weak_explicit(&__maps.lock.word, &word, 0,
memory_order_release,
memory_order_relaxed)) {
#if DEBUG_MAPS_LOCK
__deadlock_untrack(&__maps.lock);
#endif
break;
}
}
}

9
libc/intrin/maps.h
View file

@ -3,7 +3,6 @@
#include "libc/intrin/atomic.h"
#include "libc/intrin/tree.h"
#include "libc/runtime/runtime.h"
#include "libc/thread/tls2.internal.h"
COSMOPOLITAN_C_START_
#define MAPS_RETRY ((void *)-1)
@ -26,9 +25,15 @@ struct Map {
};
};
struct MapLock {
void *edges;
_Atomic(uint64_t) word;
};
struct Maps {
uint128_t rand;
struct Tree *maps;
_Atomic(uint64_t) lock;
struct MapLock lock;
_Atomic(uintptr_t) freed;
size_t count;
size_t pages;

7
libc/intrin/mmap.c
View file

@ -34,7 +34,6 @@
#include "libc/nt/runtime.h"
#include "libc/runtime/runtime.h"
#include "libc/runtime/zipos.internal.h"
#include "libc/stdio/rand.h"
#include "libc/stdio/sysparam.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/mremap.h"
@ -42,7 +41,7 @@
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/errfuns.h"
#define MMDEBUG IsModeDbg()
#define MMDEBUG 0
#define MAX_SIZE 0x0ff800000000ul
#define MAX_TRIES 50
@ -404,7 +403,9 @@ static int __munmap(char *addr, size_t size) {
void *__maps_randaddr(void) {
uintptr_t addr;
addr = _rand64();
__maps_lock();
addr = (__maps.rand *= 15750249268501108917ull) >> 64;
__maps_unlock();
addr &= 0x3fffffffffff;
addr |= 0x004000000000;
addr &= -__gransize;

101
libc/intrin/pthread_atfork_actual.c
View file

@ -1,101 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/atomic.h"
#include "libc/calls/state.internal.h"
#include "libc/cosmo.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/dll.h"
#include "libc/intrin/strace.h"
#include "libc/macros.h"
#include "libc/proc/proc.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
struct AtFork {
struct AtFork *p[2];
atfork_f f[3];
};
static struct AtForks {
pthread_spinlock_t lock;
struct AtFork *list;
struct AtFork pool[64];
atomic_int allocated;
} _atforks;
static void _pthread_onfork(int i, const char *op) {
struct AtFork *a;
if (!i)
pthread_spin_lock(&_atforks.lock);
for (a = _atforks.list; a; a = a->p[!i]) {
if (a->f[i]) {
STRACE("pthread_atfork(%s, %t)", op, a->f[i]);
a->f[i]();
}
_atforks.list = a;
}
if (i)
pthread_spin_unlock(&_atforks.lock);
}
void _pthread_onfork_prepare(void) {
_pthread_onfork(0, "prepare");
}
void _pthread_onfork_parent(void) {
_pthread_onfork(1, "parent");
}
void _pthread_onfork_child(void) {
_pthread_onfork(2, "child");
}
static struct AtFork *_pthread_atfork_alloc(void) {
int i, n = ARRAYLEN(_atforks.pool);
if (atomic_load_explicit(&_atforks.allocated, memory_order_relaxed) < n &&
(i = atomic_fetch_add(&_atforks.allocated, 1)) < n) {
return _atforks.pool + i;
} else {
return 0;
}
}
int _pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) {
int rc;
struct AtFork *a;
if (!(a = _pthread_atfork_alloc()))
return ENOMEM;
a->f[0] = prepare;
a->f[1] = parent;
a->f[2] = child;
pthread_spin_lock(&_atforks.lock);
a->p[0] = 0;
a->p[1] = _atforks.list;
if (_atforks.list)
_atforks.list->p[0] = a;
_atforks.list = a;
pthread_spin_unlock(&_atforks.lock);
rc = 0;
return rc;
}

70
libc/intrin/pthread_atfork.c → libc/intrin/pthread_mutex_consistent.c
View file

@ -1,7 +1,7 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
@ -16,62 +16,26 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/intrin/weaken.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/cosmo.h"
#include "libc/dce.h"
#include "libc/intrin/atomic.h"
#include "libc/thread/lock.h"
#include "libc/thread/thread.h"
/**
* Registers fork() handlers.
* Recovers mutex whose owner died.
*
* Parent and child functions are called in the same order they're
* registered. Prepare functions are called in reverse order.
*
* Here's an example of how pthread_atfork() can be used:
*
* static struct {
* pthread_once_t once;
* pthread_mutex_t lock;
* // data structures...
* } g_lib;
*
* static void lib_wipe(void) {
* pthread_mutex_init(&g_lib.lock, 0);
* }
*
* static void lib_lock(void) {
* pthread_mutex_lock(&g_lib.lock);
* }
*
* static void lib_unlock(void) {
* pthread_mutex_unlock(&g_lib.lock);
* }
*
* static void lib_setup(void) {
* lib_wipe();
* pthread_atfork(lib_lock, lib_unlock, lib_wipe);
* }
*
* static void lib_init(void) {
* pthread_once(&g_lib.once, lib_setup);
* }
*
* void lib(void) {
* lib_init();
* lib_lock();
* // do stuff...
* lib_unlock();
* }
*
* @param prepare is run by fork() before forking happens
* @param parent is run by fork() after forking happens in parent process
* @param child is run by fork() after forking happens in childe process
* @return 0 on success, or errno on error
* @raise ENOMEM if we require more vespene gas
*/
int pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) {
if (_weaken(_pthread_atfork)) {
return _weaken(_pthread_atfork)(prepare, parent, child);
} else {
return 0;
}
int pthread_mutex_consistent(pthread_mutex_t *mutex) {
// The POSIX concept of robust mutexes is a bit cray. So let's change
// things up a bit. Rather than implementing all those goofy behaviors
// we shall simply use this function to weasel around the ownership
// check in pthread_mutex_unlock().
uint64_t word = atomic_load_explicit(&mutex->_word, memory_order_relaxed);
if (IsModeDbg() || MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK)
__deadlock_track(mutex, 0);
return 0;
}

2
libc/intrin/pthread_mutex_init.c
View file

@ -24,7 +24,7 @@
* pthread_mutex_t lock;
* pthread_mutexattr_t attr;
* pthread_mutexattr_init(&attr);
* pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL);
* pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT);
* pthread_mutex_init(&lock, &attr);
* pthread_mutexattr_destroy(&attr);
* // ...

282
libc/intrin/pthread_mutex_lock.c
View file

@ -24,62 +24,82 @@
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/describeflags.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/strace.h"
#include "libc/intrin/weaken.h"
#include "libc/macros.h"
#include "libc/runtime/internal.h"
#include "libc/thread/lock.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
#include "third_party/nsync/mu.h"
static errno_t pthread_mutex_lock_normal_success(pthread_mutex_t *mutex,
uint64_t word) {
if (IsModeDbg() || MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK) {
__deadlock_track(mutex, MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK);
__deadlock_record(mutex, MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK);
}
return 0;
}
// see "take 3" algorithm in "futexes are tricky" by ulrich drepper
// slightly improved to attempt acquiring multiple times b4 syscall
static void pthread_mutex_lock_drepper(atomic_int *futex, char pshare) {
int word = 0;
static int pthread_mutex_lock_drepper(pthread_mutex_t *mutex, uint64_t word,
bool is_trylock) {
int val = 0;
if (atomic_compare_exchange_strong_explicit(
futex, &word, 1, memory_order_acquire, memory_order_acquire))
return;
LOCKTRACE("acquiring pthread_mutex_lock_drepper(%t)...", futex);
if (word == 1)
word = atomic_exchange_explicit(futex, 2, memory_order_acquire);
&mutex->_futex, &val, 1, memory_order_acquire, memory_order_acquire))
return pthread_mutex_lock_normal_success(mutex, word);
if (is_trylock)
return EBUSY;
LOCKTRACE("acquiring pthread_mutex_lock_drepper(%t)...", mutex);
if (val == 1)
val = atomic_exchange_explicit(&mutex->_futex, 2, memory_order_acquire);
BLOCK_CANCELATION;
while (word > 0) {
cosmo_futex_wait(futex, 2, pshare, 0, 0);
word = atomic_exchange_explicit(futex, 2, memory_order_acquire);
while (val > 0) {
cosmo_futex_wait(&mutex->_futex, 2, MUTEX_PSHARED(word), 0, 0);
val = atomic_exchange_explicit(&mutex->_futex, 2, memory_order_acquire);
}
ALLOW_CANCELATION;
return pthread_mutex_lock_normal_success(mutex, word);
}
static errno_t pthread_mutex_lock_recursive(pthread_mutex_t *mutex,
uint64_t word) {
uint64_t word, bool is_trylock) {
uint64_t lock;
int backoff = 0;
int me = gettid();
bool once = false;
for (;;) {
if (MUTEX_OWNER(word) == me) {
if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) {
if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) {
if (atomic_compare_exchange_weak_explicit(
&mutex->_word, &word, MUTEX_INC_DEPTH(word),
memory_order_relaxed, memory_order_relaxed))
return 0;
continue;
} else {
return EAGAIN;
}
if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) {
if (atomic_compare_exchange_weak_explicit(
&mutex->_word, &word, MUTEX_INC_DEPTH(word),
memory_order_relaxed, memory_order_relaxed))
return 0;
continue;
} else {
return EDEADLK;
return EAGAIN;
}
}
if (IsModeDbg())
__deadlock_check(mutex, 0);
word = MUTEX_UNLOCK(word);
lock = MUTEX_LOCK(word);
lock = MUTEX_SET_OWNER(lock, me);
if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
memory_order_acquire,
memory_order_relaxed)) {
if (IsModeDbg()) {
__deadlock_track(mutex, 0);
__deadlock_record(mutex, 0);
}
mutex->_pid = __pid;
return 0;
}
if (is_trylock)
return EBUSY;
if (!once) {
LOCKTRACE("acquiring pthread_mutex_lock_recursive(%t)...", mutex);
once = true;
@ -97,25 +117,33 @@ static errno_t pthread_mutex_lock_recursive(pthread_mutex_t *mutex,
#if PTHREAD_USE_NSYNC
static errno_t pthread_mutex_lock_recursive_nsync(pthread_mutex_t *mutex,
uint64_t word) {
uint64_t word,
bool is_trylock) {
int me = gettid();
for (;;) {
if (MUTEX_OWNER(word) == me) {
if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) {
if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) {
if (atomic_compare_exchange_weak_explicit(
&mutex->_word, &word, MUTEX_INC_DEPTH(word),
memory_order_relaxed, memory_order_relaxed))
return 0;
continue;
} else {
return EAGAIN;
}
if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) {
if (atomic_compare_exchange_weak_explicit(
&mutex->_word, &word, MUTEX_INC_DEPTH(word),
memory_order_relaxed, memory_order_relaxed))
return 0;
continue;
} else {
return EDEADLK;
return EAGAIN;
}
}
_weaken(nsync_mu_lock)((nsync_mu *)mutex->_nsyncx);
if (IsModeDbg())
__deadlock_check(mutex, 0);
if (!is_trylock) {
_weaken(nsync_mu_lock)((nsync_mu *)mutex->_nsync);
} else {
if (!_weaken(nsync_mu_trylock)((nsync_mu *)mutex->_nsync))
return EBUSY;
}
if (IsModeDbg()) {
__deadlock_track(mutex, 0);
__deadlock_record(mutex, 0);
}
word = MUTEX_UNLOCK(word);
word = MUTEX_LOCK(word);
word = MUTEX_SET_OWNER(word, me);
@ -126,69 +154,82 @@ static errno_t pthread_mutex_lock_recursive_nsync(pthread_mutex_t *mutex,
}
#endif
static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex) {
uint64_t word;
static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex,
bool is_trylock) {
uint64_t word = atomic_load_explicit(&mutex->_word, memory_order_relaxed);
// get current state of lock
word = atomic_load_explicit(&mutex->_word, memory_order_relaxed);
// handle recursive mutexes
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_RECURSIVE) {
#if PTHREAD_USE_NSYNC
if (_weaken(nsync_mu_lock) &&
MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE) {
return pthread_mutex_lock_recursive_nsync(mutex, word, is_trylock);
} else {
return pthread_mutex_lock_recursive(mutex, word, is_trylock);
}
#else
return pthread_mutex_lock_recursive(mutex, word, is_trylock);
#endif
}
// check if normal mutex is already owned by calling thread
if (!is_trylock &&
(MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK ||
(IsModeDbg() && MUTEX_TYPE(word) == PTHREAD_MUTEX_DEFAULT))) {
if (__deadlock_tracked(mutex) == 1) {
if (IsModeDbg() && MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) {
kprintf("error: attempted to lock non-recursive mutex that's already "
"held by the calling thread: %t\n",
mutex);
DebugBreak();
}
return EDEADLK;
}
}
// check if locking will create cycle in lock graph
if (IsModeDbg() || MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK)
if (__deadlock_check(mutex, MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK))
return EDEADLK;
#if PTHREAD_USE_NSYNC
// use superior mutexes if possible
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL && //
MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && //
if (MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE &&
_weaken(nsync_mu_lock)) {
// on apple silicon we should just put our faith in ulock
// otherwise *nsync gets struck down by the eye of sauron
if (!IsXnuSilicon()) {
_weaken(nsync_mu_lock)((nsync_mu *)mutex);
return 0;
if (!is_trylock) {
_weaken(nsync_mu_lock)((nsync_mu *)mutex->_nsync);
return pthread_mutex_lock_normal_success(mutex, word);
} else {
if (_weaken(nsync_mu_trylock)((nsync_mu *)mutex->_nsync))
return pthread_mutex_lock_normal_success(mutex, word);
return EBUSY;
}
}
}
#endif
// handle normal mutexes
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) {
pthread_mutex_lock_drepper(&mutex->_futex, MUTEX_PSHARED(word));
return 0;
}
// handle recursive and error checking mutexes
#if PTHREAD_USE_NSYNC
if (_weaken(nsync_mu_lock) &&
MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE) {
return pthread_mutex_lock_recursive_nsync(mutex, word);
} else {
return pthread_mutex_lock_recursive(mutex, word);
}
#else
return pthread_mutex_lock_recursive(mutex, word);
#endif
// isc licensed non-recursive mutex implementation
return pthread_mutex_lock_drepper(mutex, word, is_trylock);
}
/**
* Locks mutex.
*
* Here's an example of using a normal mutex:
* Locks mutex, e.g.
*
* pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
* pthread_mutex_lock(&lock);
* // do work...
* pthread_mutex_unlock(&lock);
* pthread_mutex_destroy(&lock);
*
* Cosmopolitan permits succinct notation for normal mutexes:
*
* pthread_mutex_t lock = {0};
* pthread_mutex_lock(&lock);
* // do work...
* pthread_mutex_unlock(&lock);
*
* Here's an example of the proper way to do recursive mutexes:
* The long way to do that is:
*
* pthread_mutex_t lock;
* pthread_mutexattr_t attr;
* pthread_mutexattr_init(&attr);
* pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
* pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT);
* pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_PRIVATE);
* pthread_mutex_init(&lock, &attr);
* pthread_mutexattr_destroy(&attr);
* pthread_mutex_lock(&lock);
@ -196,28 +237,99 @@ static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex) {
* pthread_mutex_unlock(&lock);
* pthread_mutex_destroy(&lock);
*
* This function does nothing in vfork() children.
* The following non-POSIX initializers are also provided by cosmo libc:
*
* You can debug locks the acquisition of locks by building your program
* with `cosmocc -mdbg` and passing the `--strace` flag to your program.
* This will cause a line to be logged each time a mutex or spin lock is
* locked or unlocked. When locking, this is printed after the lock gets
* acquired. The entry to the lock operation will be logged too but only
* if the lock couldn't be immediately acquired. Lock logging works best
* when `mutex` refers to a static variable, in which case its name will
* be printed in the log.
* - `PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP`
* - `PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP`
* - `PTHREAD_SIGNAL_SAFE_MUTEX_INITIALIZER_NP`
* - `PTHREAD_NORMAL_MUTEX_INITIALIZER_NP`
*
* Locking a mutex that's already locked by the calling thread will make
* the thread hang indefinitely, i.e. it's a deadlock condition. You can
* use `PTHREAD_MUTEX_RECURSIVE` to allow recursive locking, which could
* result in somewhat less performance. An alternative solution is using
* the `PTHREAD_MUTEX_ERRORCHECK` mode, which raises `EDEADLK` for that.
*
* If a thread locks a mutex while other mutexes are already locked then
* you need to observe a consistent global ordering, otherwise deadlocks
* might occur. The Cosmopolitan runtime can detect these cycles quickly
* so you can fix your code before it becomes an issue. With error check
* mode, an EPERM will be returned. If your app is using `cosmocc -mdbg`
* then an error message will be printed including the demangled symbols
* of the mutexes in the strongly connected component that was detected.
* Please note that, even for debug builds mutexes set to explicitly use
* the `PTHREAD_MUTEX_ERRORCHECK` mode will return an error code instead
* which means the cosmo debug mode only influences undefined behaviors.
*
* Cosmopolitan only supports error checking on mutexes stored in static
* memory, i.e. your `mutex` pointer must point inside the .data or .bss
* sections of your executable. When compiling your programs using -mdbg
* all your locks will gain error checking automatically. When deadlocks
* are detected an error message will be printed and a SIGTRAP signal is
* raised, which may be ignored to force EDEADLK and EPERM to be raised.
*
* Using `cosmocc -mdbg` also enhances `--strace` with information about
* mutexes. First, locks and unlocks will be logged. Since the lock line
* only appears after the lock is acquired, that might mean you'll never
* get an indication about a lock that takes a very long time to acquire
* so, whenever a lock can't immediately be acquired, a second line gets
* printed *before* the lock is acquired to let you know that the thread
* is waiting for a particular lock. If your mutex object resides within
* static memory, then its demangled symbol name will be printed. If you
* call ShowCrashReports() at the beginning of your main() function then
* you'll also see a backtrace when a locking violation occurs. When the
* symbols in the violation error messages show up as numbers, and it is
* desirable to see demangled symbols without enabling full crash report
* functionality the GetSymbolTable() function may be called for effect.
*
* If you use `PTHREAD_MUTEX_NORMAL`, instead of `PTHREAD_MUTEX_DEFAULT`
* then deadlocking is actually defined behavior according to POSIX.1 so
* the helpfulness of `cosmocc -mdbg` will be somewhat weakened.
*
* If your `mutex` object resides in `MAP_SHARED` memory, then undefined
* behavior will happen unless you use `PTHREAD_PROCESS_SHARED` mode, if
* the lock is used by multiple processes.
*
* This function does nothing when the process is in vfork() mode.
*
* @return 0 on success, or error number on failure
* @raise EDEADLK if mutex is recursive and locked by another thread
* @raise EDEADLK if mutex is non-recursive and locked by current thread
* @raise EDEADLK if cycle is detected in global nested lock graph
* @raise EAGAIN if maximum recursive locks is exceeded
* @see pthread_spin_lock()
* @vforksafe
*/
errno_t pthread_mutex_lock(pthread_mutex_t *mutex) {
if (!__vforked) {
errno_t err = pthread_mutex_lock_impl(mutex);
if (__tls_enabled && !__vforked) {
errno_t err = pthread_mutex_lock_impl(mutex, false);
LOCKTRACE("pthread_mutex_lock(%t) → %s", mutex, DescribeErrno(err));
return err;
} else {
LOCKTRACE("skipping pthread_mutex_lock(%t) due to vfork", mutex);
LOCKTRACE("skipping pthread_mutex_lock(%t) due to runtime state", mutex);
return 0;
}
}
/**
* Attempts acquiring lock.
*
* Unlike pthread_mutex_lock() this function won't block and instead
* returns an error immediately if the lock couldn't be acquired.
*
* @return 0 if lock was acquired, otherwise an errno
* @raise EBUSY if lock is currently held by another thread
* @raise EAGAIN if maximum number of recursive locks is held
* @raise EDEADLK if `mutex` is `PTHREAD_MUTEX_ERRORCHECK` and the
* current thread already holds this mutex
*/
errno_t pthread_mutex_trylock(pthread_mutex_t *mutex) {
if (__tls_enabled && !__vforked) {
errno_t err = pthread_mutex_lock_impl(mutex, true);
LOCKTRACE("pthread_mutex_trylock(%t) → %s", mutex, DescribeErrno(err));
return err;
} else {
LOCKTRACE("skipping pthread_mutex_trylock(%t) due to runtime state", mutex);
return 0;
}
}

152
libc/intrin/pthread_mutex_trylock.c
View file

@ -1,152 +0,0 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2023 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/weaken.h"
#include "libc/runtime/internal.h"
#include "libc/thread/lock.h"
#include "libc/thread/thread.h"
#include "third_party/nsync/mu.h"
static errno_t pthread_mutex_trylock_drepper(atomic_int *futex) {
int word = 0;
if (atomic_compare_exchange_strong_explicit(
futex, &word, 1, memory_order_acquire, memory_order_acquire))
return 0;
return EBUSY;
}
static errno_t pthread_mutex_trylock_recursive(pthread_mutex_t *mutex,
uint64_t word) {
uint64_t lock;
int me = gettid();
for (;;) {
if (MUTEX_OWNER(word) == me) {
if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) {
if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) {
if (atomic_compare_exchange_weak_explicit(
&mutex->_word, &word, MUTEX_INC_DEPTH(word),
memory_order_relaxed, memory_order_relaxed))
return 0;
continue;
} else {
return EAGAIN;
}
} else {
return EDEADLK;
}
}
word = MUTEX_UNLOCK(word);
lock = MUTEX_LOCK(word);
lock = MUTEX_SET_OWNER(lock, me);
if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
memory_order_acquire,
memory_order_relaxed)) {
mutex->_pid = __pid;
return 0;
}
return EBUSY;
}
}
static errno_t pthread_mutex_trylock_recursive_nsync(pthread_mutex_t *mutex,
uint64_t word) {
int me = gettid();
for (;;) {
if (MUTEX_OWNER(word) == me) {
if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) {
if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) {
if (atomic_compare_exchange_weak_explicit(
&mutex->_word, &word, MUTEX_INC_DEPTH(word),
memory_order_relaxed, memory_order_relaxed))
return 0;
continue;
} else {
return EAGAIN;
}
} else {
return EDEADLK;
}
}
if (_weaken(nsync_mu_trylock)((nsync_mu *)mutex->_nsyncx)) {
word = MUTEX_UNLOCK(word);
word = MUTEX_LOCK(word);
word = MUTEX_SET_OWNER(word, me);
mutex->_word = word;
mutex->_pid = __pid;
return 0;
} else {
return EBUSY;
}
}
}
/**
* Attempts acquiring lock.
*
* Unlike pthread_mutex_lock() this function won't block and instead
* returns an error immediately if the lock couldn't be acquired.
*
* @return 0 if lock was acquired, otherwise an errno
* @raise EAGAIN if maximum number of recursive locks is held
* @raise EBUSY if lock is currently held in read or write mode
* @raise EINVAL if `mutex` doesn't refer to an initialized lock
* @raise EDEADLK if `mutex` is `PTHREAD_MUTEX_ERRORCHECK` and the
* current thread already holds this mutex
*/
errno_t pthread_mutex_trylock(pthread_mutex_t *mutex) {
// get current state of lock
uint64_t word = atomic_load_explicit(&mutex->_word, memory_order_relaxed);
#if PTHREAD_USE_NSYNC
// use superior mutexes if possible
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL &&
MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && //
_weaken(nsync_mu_trylock)) {
// on apple silicon we should just put our faith in ulock
// otherwise *nsync gets struck down by the eye of sauron
if (!IsXnuSilicon()) {
if (_weaken(nsync_mu_trylock)((nsync_mu *)mutex)) {
return 0;
} else {
return EBUSY;
}
}
}
#endif
// handle normal mutexes
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL)
return pthread_mutex_trylock_drepper(&mutex->_futex);
// handle recursive and error checking mutexes
#if PTHREAD_USE_NSYNC
if (_weaken(nsync_mu_trylock) &&
MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE) {
return pthread_mutex_trylock_recursive_nsync(mutex, word);
} else {
return pthread_mutex_trylock_recursive(mutex, word);
}
#else
return pthread_mutex_trylock_recursive(mutex, word);
#endif
}

101
libc/intrin/pthread_mutex_unlock.c
View file

@ -22,6 +22,8 @@
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/describeflags.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/strace.h"
#include "libc/intrin/weaken.h"
#include "libc/runtime/internal.h"
@ -61,8 +63,11 @@ static errno_t pthread_mutex_unlock_recursive(pthread_mutex_t *mutex,
// actually unlock the mutex
if (atomic_compare_exchange_weak_explicit(
&mutex->_word, &word, MUTEX_UNLOCK(word), memory_order_release,
memory_order_relaxed))
memory_order_relaxed)) {
if (IsModeDbg())
__deadlock_untrack(mutex);
return 0;
}
}
}
@ -89,63 +94,85 @@ static errno_t pthread_mutex_unlock_recursive_nsync(pthread_mutex_t *mutex,
// actually unlock the mutex
mutex->_word = MUTEX_UNLOCK(word);
_weaken(nsync_mu_unlock)((nsync_mu *)mutex->_nsyncx);
_weaken(nsync_mu_unlock)((nsync_mu *)mutex->_nsync);
if (IsModeDbg())
__deadlock_untrack(mutex);
return 0;
}
}
#endif
/**
* Releases mutex.
*
* This function does nothing in vfork() children.
*
* @return 0 on success or error number on failure
* @raises EPERM if in error check mode and not owned by caller
* @vforksafe
*/
errno_t pthread_mutex_unlock(pthread_mutex_t *mutex) {
uint64_t word;
static errno_t pthread_mutex_unlock_impl(pthread_mutex_t *mutex) {
uint64_t word = atomic_load_explicit(&mutex->_word, memory_order_relaxed);
if (__vforked) {
LOCKTRACE("skipping pthread_mutex_lock(%t) due to vfork", mutex);
return 0;
// check if mutex isn't held by calling thread
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK || IsModeDbg()) {
if (__deadlock_tracked(mutex) == 0) {
if (IsModeDbg() && MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) {
kprintf("error: unlock mutex not owned by calling thread: %t\n", mutex);
DebugBreak();
}
return EPERM;
}
}
LOCKTRACE("pthread_mutex_unlock(%t)", mutex);
// get current state of lock
word = atomic_load_explicit(&mutex->_word, memory_order_relaxed);
// handle recursive mutexes
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_RECURSIVE) {
#if PTHREAD_USE_NSYNC
if (_weaken(nsync_mu_unlock) &&
MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE) {
return pthread_mutex_unlock_recursive_nsync(mutex, word);
} else {
return pthread_mutex_unlock_recursive(mutex, word);
}
#else
return pthread_mutex_unlock_recursive(mutex, word);
#endif
}
#if PTHREAD_USE_NSYNC
// use superior mutexes if possible
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL && //
MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && //
if (MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE && //
_weaken(nsync_mu_unlock)) {
// on apple silicon we should just put our faith in ulock
// otherwise *nsync gets struck down by the eye of sauron
if (!IsXnuSilicon()) {
_weaken(nsync_mu_unlock)((nsync_mu *)mutex);
_weaken(nsync_mu_unlock)((nsync_mu *)mutex->_nsync);
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK || IsModeDbg())
__deadlock_untrack(mutex);
return 0;
}
}
#endif
// implement barebones normal mutexes
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) {
pthread_mutex_unlock_drepper(&mutex->_futex, MUTEX_PSHARED(word));
pthread_mutex_unlock_drepper(&mutex->_futex, MUTEX_PSHARED(word));
if (MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK || IsModeDbg())
__deadlock_untrack(mutex);
return 0;
}
/**
* Releases mutex.
*
* POSIX.1 says it's undefined behavior to unlock a mutex that wasn't
* locked by the calling thread. Therefore, if `mutex` isn't locked, or
* it is locked and the thing that locked it was a different thread or
* process, then you should expect your program to deadlock or crash.
*
* This function does nothing in vfork() children.
*
* @return 0 on success or error number on failure
* @raises EPERM if mutex ownership isn't acceptable
* @vforksafe
*/
errno_t pthread_mutex_unlock(pthread_mutex_t *mutex) {
if (__tls_enabled && !__vforked) {
errno_t err = pthread_mutex_unlock_impl(mutex);
LOCKTRACE("pthread_mutex_unlock(%t) → %s", mutex, DescribeErrno(err));
return err;
} else {
LOCKTRACE("skipping pthread_mutex_lock(%t) due to runtime state", mutex);
return 0;
}
// handle recursive and error checking mutexes
#if PTHREAD_USE_NSYNC
if (_weaken(nsync_mu_unlock) &&
MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE) {
return pthread_mutex_unlock_recursive_nsync(mutex, word);
} else {
return pthread_mutex_unlock_recursive(mutex, word);
}
#else
return pthread_mutex_unlock_recursive(mutex, word);
#endif
}

33
libc/intrin/pthread_mutex_wipe_np.c Normal file
View file

@ -0,0 +1,33 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/str/str.h"
#include "libc/thread/lock.h"
#include "libc/thread/thread.h"
/**
* Unlocks mutex from child process after fork.
*/
int pthread_mutex_wipe_np(pthread_mutex_t *mutex) {
void *edges = mutex->_edges;
uint64_t word = mutex->_word;
bzero(mutex, sizeof(*mutex));
mutex->_word = MUTEX_UNLOCK(word);
mutex->_edges = edges;
return 0;
}

1
libc/intrin/pthread_mutexattr_gettype.c
View file

@ -23,6 +23,7 @@
* Gets mutex type.
*
* @param type will be set to one of these on success
* - `PTHREAD_MUTEX_DEFAULT`
* - `PTHREAD_MUTEX_NORMAL`
* - `PTHREAD_MUTEX_RECURSIVE`
* - `PTHREAD_MUTEX_ERRORCHECK`

2
libc/intrin/pthread_mutexattr_settype.c
View file

@ -24,6 +24,7 @@
* Sets mutex type.
*
* @param type can be one of
* - `PTHREAD_MUTEX_DEFAULT`
* - `PTHREAD_MUTEX_NORMAL`
* - `PTHREAD_MUTEX_RECURSIVE`
* - `PTHREAD_MUTEX_ERRORCHECK`
@ -32,6 +33,7 @@
*/
errno_t pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type) {
switch (type) {
case PTHREAD_MUTEX_DEFAULT:
case PTHREAD_MUTEX_NORMAL:
case PTHREAD_MUTEX_RECURSIVE:
case PTHREAD_MUTEX_ERRORCHECK:

6
libc/intrin/pthreadlock.c
View file

@ -18,12 +18,12 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/thread/posixthread.internal.h"
pthread_mutex_t _pthread_lock_obj = PTHREAD_MUTEX_INITIALIZER;
pthread_mutex_t __pthread_lock_obj = PTHREAD_MUTEX_INITIALIZER;
void _pthread_lock(void) {
pthread_mutex_lock(&_pthread_lock_obj);
pthread_mutex_lock(&__pthread_lock_obj);
}
void _pthread_unlock(void) {
pthread_mutex_unlock(&_pthread_lock_obj);
pthread_mutex_unlock(&__pthread_lock_obj);
}

7
libc/intrin/rand64.c
View file

@ -27,7 +27,7 @@
static int _rand64_pid;
static unsigned __int128 _rand64_pool;
pthread_mutex_t _rand64_lock_obj = PTHREAD_SIGNAL_SAFE_MUTEX_INITIALIZER_NP;
pthread_mutex_t __rand64_lock_obj = PTHREAD_MUTEX_INITIALIZER;
/**
* Returns nondeterministic random data.
@ -38,12 +38,11 @@ pthread_mutex_t _rand64_lock_obj = PTHREAD_SIGNAL_SAFE_MUTEX_INITIALIZER_NP;
*
* @see rdseed(), rdrand(), rand(), random(), rngset()
* @note this function passes bigcrush and practrand
* @asyncsignalsafe
*/
uint64_t _rand64(void) {
void *p;
uint128_t s;
pthread_mutex_lock(&_rand64_lock_obj);
pthread_mutex_lock(&__rand64_lock_obj);
if (__pid == _rand64_pid) {
s = _rand64_pool; // normal path
} else {
@ -64,6 +63,6 @@ uint64_t _rand64(void) {
_rand64_pid = __pid;
}
_rand64_pool = (s *= 15750249268501108917ull); // lemur64
pthread_mutex_unlock(&_rand64_lock_obj);
pthread_mutex_unlock(&__rand64_lock_obj);
return s >> 64;
}

5
libc/intrin/sig.c
View file

@ -73,6 +73,8 @@ struct SignalFrame {
ucontext_t ctx;
};
extern pthread_mutex_t __sig_worker_lock;
static textwindows bool __sig_ignored_by_default(int sig) {
return sig == SIGURG || //
sig == SIGCONT || //
@ -667,9 +669,6 @@ textwindows int __sig_check(void) {
return res;
}
// this mutex is needed so execve() can shut down the signal worker
pthread_mutex_t __sig_worker_lock;
// background thread for delivering inter-process signals asynchronously
// this checks for undelivered process-wide signals, once per scheduling
// quantum, which on windows should be every ~15ms or so, unless somehow

8
libc/intrin/sigblock.c
View file

@ -16,14 +16,20 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/sysv/consts/sig.h"
#include "libc/calls/sig.internal.h"
#include "libc/calls/struct/sigset.internal.h"
#include "libc/dce.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/weaken.h"
#include "libc/sysv/consts/sig.h"
#include "libc/thread/tls.h"
// since there's so many c library interfaces and system call wrappers
// that always need to block signals we avoid the distraction of their
// ftrace and strace output being muddied with sigprocmask lines. it's
// usually better that sigprocmask only strace the user is calling it.
// plus, since we have a very specific use case, this code goes faster
struct Signals __sig;
sigset_t __sig_block(void) {

22
libc/intrin/siglock.c Normal file
View file

@ -0,0 +1,22 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/thread/thread.h"
// this mutex is needed so execve() can shut down the signal worker
pthread_mutex_t __sig_worker_lock = PTHREAD_MUTEX_INITIALIZER;

32
libc/intrin/sigprocmask-nt.c
View file

@ -16,8 +16,6 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/atomic.h"
#include "libc/calls/sig.internal.h"
#include "libc/calls/struct/sigset.h"
#include "libc/intrin/atomic.h"
@ -28,37 +26,25 @@
#ifdef __x86_64__
textwindows int __sig_mask(int how, const sigset_t *neu, sigset_t *old) {
// validate api usage
if (how != SIG_BLOCK && how != SIG_UNBLOCK && how != SIG_SETMASK) {
if (how != SIG_BLOCK && how != SIG_UNBLOCK && how != SIG_SETMASK)
return einval();
}
// get address of sigset to modify
atomic_ulong *mask = &__get_tls()->tib_sigmask;
// handle read-only case
sigset_t oldmask;
atomic_ulong *mask = &__get_tls()->tib_sigmask;
if (neu) {
if (how == SIG_BLOCK) {
oldmask = atomic_fetch_or_explicit(mask, *neu, memory_order_acq_rel);
oldmask = atomic_fetch_or(mask, *neu);
} else if (how == SIG_UNBLOCK) {
oldmask = atomic_fetch_and_explicit(mask, ~*neu, memory_order_acq_rel);
} else { // SIG_SETMASK
oldmask = atomic_exchange_explicit(mask, *neu, memory_order_acq_rel);
oldmask = atomic_fetch_and(mask, ~*neu);
} else {
oldmask = atomic_exchange(mask, *neu);
}
if (_weaken(__sig_check)) {
if (_weaken(__sig_check))
_weaken(__sig_check)();
}
} else {
oldmask = atomic_load_explicit(mask, memory_order_acquire);
oldmask = atomic_load(mask);
}
// return old signal mask to caller
if (old) {
if (old)
*old = oldmask;
}
return 0;
}

9
libc/intrin/sigprocmask.c
View file

@ -16,18 +16,12 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
#include "libc/calls/sig.internal.h"
#include "libc/calls/struct/sigset.h"
#include "libc/calls/struct/sigset.internal.h"
#include "libc/dce.h"
#include "libc/fmt/itoa.h"
#include "libc/intrin/describeflags.h"
#include "libc/intrin/strace.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/sig.h"
#include "libc/sysv/errfuns.h"
/**
* Changes signal blocking state of calling thread, e.g.:
@ -55,9 +49,8 @@ int sigprocmask(int how, const sigset_t *opt_set, sigset_t *opt_out_oldset) {
} else {
rc = sys_sigprocmask(how, opt_set, opt_out_oldset ? &old : 0);
}
if (rc != -1 && opt_out_oldset) {
if (rc != -1 && opt_out_oldset)
*opt_out_oldset = old;
}
STRACE("sigprocmask(%s, %s, [%s]) → %d% m", DescribeHow(how),
DescribeSigset(0, opt_set), DescribeSigset(rc, opt_out_oldset), rc);
return rc;

7
libc/intrin/flushers.c → libc/intrin/sigvar.c
View file

@ -1,7 +1,7 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
@ -16,7 +16,6 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/stdio/fflush.internal.h"
#include "libc/calls/sig.internal.h"
pthread_mutex_t __fflush_lock_obj;
struct StdioFlush __fflush;
struct Signals __sig;

95
libc/intrin/stdio.c Normal file
View file

@ -0,0 +1,95 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/weaken.h"
#include "libc/mem/mem.h"
#include "libc/stdio/internal.h"
#define STDIO_FILE_USE_AFTER_FREE 1
#define CORRUPT_STDIO_FILE_OBJECT 1
struct Stdio __stdio = {
.lock = PTHREAD_MUTEX_INITIALIZER,
};
void __stdio_lock(void) {
pthread_mutex_lock(&__stdio.lock);
}
void __stdio_unlock(void) {
pthread_mutex_unlock(&__stdio.lock);
}
static int refchk(int refs) {
unassert(refs != STDIO_FILE_USE_AFTER_FREE);
unassert(refs < CORRUPT_STDIO_FILE_OBJECT);
return refs;
}
void __stdio_ref(FILE *f) {
refchk(atomic_fetch_sub_explicit(&f->refs, 1, memory_order_relaxed));
}
static void __stdio_unref_impl(FILE *f, bool should_lock) {
int refs = atomic_load_explicit(&f->refs, memory_order_relaxed);
for (;;) {
refchk(refs);
if (refs) {
if (atomic_compare_exchange_strong_explicit(&f->refs, &refs, refs + 1,
memory_order_acq_rel,
memory_order_relaxed))
return;
continue;
}
if (should_lock) {
__stdio_lock();
if ((refs = atomic_load_explicit(&f->refs, memory_order_relaxed))) {
__stdio_unlock();
continue;
}
}
if (!atomic_compare_exchange_strong_explicit(
&f->refs, &refs, 1, memory_order_acq_rel, memory_order_relaxed)) {
if (should_lock)
__stdio_unlock();
continue;
}
dll_remove(&__stdio.files, &f->elem);
if (should_lock)
__stdio_unlock();
break;
}
if (_weaken(free)) {
_weaken(free)(f->getln);
if (f->freebuf)
_weaken(free)(f->buf);
if (f->freethis)
_weaken(free)(f);
}
}
void __stdio_unref(FILE *f) {
__stdio_unref_impl(f, true);
}
void __stdio_unref_unlocked(FILE *f) {
__stdio_unref_impl(f, false);
}

5
libc/intrin/sys_gettid.greg.c
View file

@ -25,7 +25,10 @@
__msabi extern typeof(GetCurrentThreadId) *const __imp_GetCurrentThreadId;
int sys_gettid(void) {
// it's important that this be noinstrument because the child process
// created by fork() needs to update this value quickly, since ftrace
// will deadlock __maps_lock() if the wrong tid is accidentally used.
dontinstrument int sys_gettid(void) {
int64_t wut;
#ifdef __x86_64__
int tid;

54
libc/intrin/tls.c Normal file
View file

@ -0,0 +1,54 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2024 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/thread/tls.h"
#include "libc/dce.h"
/**
* Returns location of thread information block.
*
* This should be favored over __get_tls() for .privileged code that
* can't be self-modified by __enable_tls().
*/
privileged optimizespeed struct CosmoTib *__get_tls_privileged(void) {
#if defined(__x86_64__)
char *tib, *lin = (char *)0x30;
if (IsNetbsd() || IsOpenbsd()) {
asm("mov\t%%fs:(%1),%0" : "=a"(tib) : "r"(lin) : "memory");
} else {
asm("mov\t%%gs:(%1),%0" : "=a"(tib) : "r"(lin) : "memory");
if (IsWindows())
tib = *(char **)(tib + 0x1480 + __tls_index * 8);
}
return (struct CosmoTib *)tib;
#elif defined(__aarch64__)
return __get_tls();
#endif
}
#if defined(__x86_64__)
privileged optimizespeed struct CosmoTib *__get_tls_win32(void) {
char *tib, *lin = (char *)0x30;
asm("mov\t%%gs:(%1),%0" : "=a"(tib) : "r"(lin) : "memory");
tib = *(char **)(tib + 0x1480 + __tls_index * 8);
return (struct CosmoTib *)tib;
}
privileged void __set_tls_win32(void *tls) {
asm("mov\t%1,%%gs:%0" : "=m"(*((long *)0x1480 + __tls_index)) : "r"(tls));
}
#endif

4
libc/intrin/winerr.greg.c
View file

@ -24,7 +24,7 @@
#include "libc/nt/runtime.h"
#include "libc/sock/internal.h"
#include "libc/sysv/errfuns.h"
#include "libc/thread/tls2.internal.h"
#include "libc/thread/tls.h"
/**
* Return path for failed Win32 API calls.
@ -32,7 +32,7 @@
* @return -1 w/ few exceptions
* @note this is a code-size saving device
*/
privileged int64_t __winerr(void) {
privileged optimizesize int64_t __winerr(void) {
errno_t e;
if (IsWindows()) {
e = __dos2errno(__imp_GetLastError());

4
libc/intrin/wintlsinit.c
View file

@ -21,7 +21,7 @@
#include "libc/nt/thunk/msabi.h"
#include "libc/runtime/runtime.h"
#include "libc/thread/tls.h"
#include "libc/thread/tls2.internal.h"
#ifdef __x86_64__
__msabi extern typeof(GetCurrentThreadId) *const __imp_GetCurrentThreadId;
@ -41,3 +41,5 @@ textwindows dontinstrument void __bootstrap_tls(struct CosmoTib *tib,
tib->tib_tid = __imp_GetCurrentThreadId();
__set_tls_win32(tib);
}
#endif /* __x86_64__ */