mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-07-01 16:58:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use *NSYNC for POSIX threads locking APIs

Browse source 
Condition variables, barriers, and r/w locks now work very well.
This commit is contained in:
Justine Tunney 2022-09-11 11:02:07 -07:00
parent 3de35e196c
commit b5cb71ab84
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
197 changed files with 3734 additions and 3817 deletions
Download patch file Download diff file Expand all files Collapse all files

35
libc/intrin/asan.c
View file

@ -25,6 +25,7 @@
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/asancodes.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/leaky.internal.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/lockcmpxchg.h"
#include "libc/intrin/nomultics.internal.h"
@ -170,7 +171,7 @@ struct ReportOriginHeap {
};
static int __asan_noreentry;
static pthread_mutex_t __asan_lock;
static pthread_spinlock_t __asan_lock;
static struct AsanMorgue __asan_morgue;
#define __asan_unreachable() \
@ -867,25 +868,25 @@ dontdiscard __asan_die_f *__asan_report_memory_fault(void *addr, int size,
void *__asan_morgue_add(void *p) {
int i;
void *r;
if (__threaded) pthread_mutex_lock(&__asan_lock);
if (__threaded) pthread_spin_lock(&__asan_lock);
i = __asan_morgue.i++ & (ARRAYLEN(__asan_morgue.p) - 1);
r = __asan_morgue.p[i];
__asan_morgue.p[i] = p;
if (__threaded) pthread_mutex_unlock(&__asan_lock);
if (__threaded) pthread_spin_unlock(&__asan_lock);
return r;
}
static void __asan_morgue_flush(void) {
int i;
void *p;
if (__threaded) pthread_mutex_lock(&__asan_lock);
if (__threaded) pthread_spin_lock(&__asan_lock);
for (i = 0; i < ARRAYLEN(__asan_morgue.p); ++i) {
if (__asan_morgue.p[i] && weaken(dlfree)) {
weaken(dlfree)(__asan_morgue.p[i]);
}
__asan_morgue.p[i] = 0;
}
if (__threaded) pthread_mutex_unlock(&__asan_lock);
if (__threaded) pthread_spin_unlock(&__asan_lock);
}
static size_t __asan_user_size(size_t n) {
@ -1052,6 +1053,30 @@ int __asan_print_trace(void *p) {
return 0;
}
// Returns true if `p` was allocated by an IGNORE_LEAKS(function).
int __asan_is_leaky(void *p) {
int sym;
size_t c, i, n;
intptr_t f, *l;
struct AsanExtra *e;
struct SymbolTable *st;
if (!weaken(GetSymbolTable)) notpossible;
if (!(e = __asan_get_extra(p, &c))) return 0;
if (!__asan_read48(e->size, &n)) return 0;
if (!__asan_is_mapped((((intptr_t)p >> 3) + 0x7fff8000) >> 16)) return 0;
if (!(st = GetSymbolTable())) return 0;
for (i = 0; i < ARRAYLEN(e->bt.p) && e->bt.p[i]; ++i) {
if ((sym = weaken(__get_symbol)(st, e->bt.p[i])) == -1) continue;
f = st->addr_base + st->symbols[sym].x;
for (l = _leaky_start; l < _leaky_end; ++l) {
if (f == *l) {
return 1;
}
}
}
return 0;
}
static void __asan_deallocate(char *p, long kind) {
size_t c, n;
struct AsanExtra *e;

2
libc/intrin/asan.internal.h
View file

@ -3,6 +3,7 @@
#include "libc/calls/struct/iovec.h"
#include "libc/intrin/asancodes.h"
#include "libc/macros.internal.h"
#include "libc/thread/thread.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
@ -29,6 +30,7 @@ struct AsanFault __asan_check(const void *, long) nosideeffect;
void __asan_free(void *);
void *__asan_malloc(size_t);
int __asan_is_leaky(void *);
int __asan_malloc_trim(size_t);
int __asan_print_trace(void *);
void *__asan_calloc(size_t, size_t);

3
libc/intrin/describeflags.internal.h
View file

@ -17,7 +17,7 @@ const char *DescribeCapability(char[20], int);
const char *DescribeClockName(char[32], int);
const char *DescribeDirfd(char[12], int);
const char *DescribeFrame(char[32], int);
const char *DescribeFutexOp(int);
const char *DescribeFutexOp(char[64], int);
const char *DescribeFutexResult(char[12], int);
const char *DescribeHow(char[12], int);
const char *DescribeMapFlags(char[64], int);
@ -60,6 +60,7 @@ const char *DescribeWhence(char[12], int);
#define DescribeClockName(x) DescribeClockName(alloca(32), x)
#define DescribeDirfd(x) DescribeDirfd(alloca(12), x)
#define DescribeFrame(x) DescribeFrame(alloca(32), x)
#define DescribeFutexOp(x) DescribeFutexOp(alloca(64), x)
#define DescribeFutexResult(x) DescribeFutexResult(alloca(12), x)
#define DescribeHow(x) DescribeHow(alloca(12), x)
#define DescribeMapFlags(x) DescribeMapFlags(alloca(64), x)

49
libc/intrin/describefutexop.c
View file

@ -16,16 +16,47 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/fmt/itoa.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/futex.h"
const char *DescribeFutexOp(int x) {
if (x == FUTEX_WAIT) return "FUTEX_WAIT";
if (x == FUTEX_WAKE) return "FUTEX_WAKE";
if (x == FUTEX_REQUEUE) return "FUTEX_REQUEUE";
// order matters (the private bit might be zero)
if (x == FUTEX_WAIT_PRIVATE) return "FUTEX_WAIT_PRIVATE";
if (x == FUTEX_WAKE_PRIVATE) return "FUTEX_WAKE_PRIVATE";
if (x == FUTEX_REQUEUE_PRIVATE) return "FUTEX_REQUEUE_PRIVATE";
return "FUTEX_???";
const char *(DescribeFutexOp)(char buf[64], int x) {
bool priv = false;
if (x & FUTEX_PRIVATE_FLAG) {
priv = true;
x &= ~FUTEX_PRIVATE_FLAG;
}
bool real = false;
if (x & FUTEX_CLOCK_REALTIME) {
real = true;
x &= ~FUTEX_CLOCK_REALTIME;
}
char *p = buf;
if (x == FUTEX_WAIT) {
p = stpcpy(p, "FUTEX_WAIT");
} else if (x == FUTEX_WAKE) {
p = stpcpy(p, "FUTEX_WAKE");
} else if (x == FUTEX_REQUEUE) {
p = stpcpy(p, "FUTEX_REQUEUE");
} else if (x == FUTEX_WAIT_BITSET) {
p = stpcpy(p, "FUTEX_WAIT_BITSET");
} else {
p = stpcpy(p, "FUTEX_");
p = FormatUint32(p, x);
}
if (priv) {
p = stpcpy(p, "_PRIVATE");
}
if (real) {
p = stpcpy(p, "|FUTEX_CLOCK_REALTIME");
}
return buf;
}

8
libc/intrin/extend.c
View file

@ -37,11 +37,12 @@ static void _mapframe(void *p) {
if ((dm = sys_mmap(p, G, prot, flags, -1, 0)).addr != p) {
notpossible;
}
if (TrackMemoryInterval(&_mmi, (uintptr_t)p >> 16,
((uintptr_t)p + G - 1) >> 16, dm.maphandle, prot,
flags, false, false, 0, G)) {
__mmi_lock();
if (TrackMemoryInterval(&_mmi, (uintptr_t)p >> 16, (uintptr_t)p >> 16,
dm.maphandle, prot, flags, false, false, 0, G)) {
notpossible;
}
__mmi_unlock();
}
/**
@ -80,5 +81,6 @@ noasan void *_extend(void *p, size_t n, void *e, intptr_t h) {
*SHADOW(q) = 0;
}
}
asm("mfence");
return q;
}

3
libc/intrin/ftrace.c
View file

@ -16,6 +16,7 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/atomic.h"
#include "libc/runtime/runtime.h"
/**
@ -39,4 +40,4 @@
* though under normal circumstances, `__ftrace` should only be either
* zero or one.
*/
_Atomic(int) __ftrace;
atomic_int __ftrace;

12
libc/intrin/futex.internal.h
View file

@ -1,12 +0,0 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_FUTEX_INTERNAL_H_
#define COSMOPOLITAN_LIBC_INTRIN_FUTEX_INTERNAL_H_
#include "libc/calls/struct/timespec.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
int _futex_wait(void *, int, char, struct timespec *) hidden;
int _futex_wake(void *, int, char) hidden;
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_INTRIN_FUTEX_INTERNAL_H_ */

4
libc/intrin/g_fds.c
View file

@ -19,11 +19,11 @@
#include "libc/calls/extend.internal.h"
#include "libc/calls/internal.h"
#include "libc/calls/state.internal.h"
#include "libc/thread/thread.h"
#include "libc/intrin/pushpop.h"
#include "libc/intrin/weaken.h"
#include "libc/nt/runtime.h"
#include "libc/sysv/consts/o.h"
#include "libc/thread/thread.h"
STATIC_YOINK("_init_g_fds");
@ -41,7 +41,7 @@ static textwindows dontinline void SetupWinStd(struct Fds *fds, int i, int x) {
textstartup void InitializeFileDescriptors(void) {
struct Fds *fds;
__fds_lock_obj.type = PTHREAD_MUTEX_RECURSIVE;
__fds_lock_obj._type = PTHREAD_MUTEX_RECURSIVE;
fds = VEIL("r", &g_fds);
fds->p = fds->e = (void *)0x6fe000040000;
fds->n = 4;

2
libc/intrin/kprintf.greg.c
View file

@ -24,6 +24,8 @@
#include "libc/errno.h"
#include "libc/fmt/divmod10.internal.h"
#include "libc/fmt/fmt.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/asancodes.h"
#include "libc/intrin/bits.h"
#include "libc/intrin/cmpxchg.h"
#include "libc/intrin/kprintf.h"

39
libc/intrin/leaky.S Normal file
View file

@ -0,0 +1,39 @@
/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│
│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macros.internal.h"
// Decentralized section for leaky functions.
.section .piro.relo.sort.leaky.1,"aw",@nobits
.type _leaky_start,@object
.type _leaky_end,@object
.globl _leaky_start,_leaky_end
.hidden _leaky_start,_leaky_end
.byte 0
.align __SIZEOF_POINTER__
.underrun
_leaky_start:
.previous/*
...
decentralized content
...
*/.section .piro.relo.sort.leaky.3,"aw",@nobits
_leaky_end:
.quad 0
.overrun
.previous

16
libc/intrin/leaky.internal.h Normal file
View file

@ -0,0 +1,16 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_LEAKY_INTERNAL_H_
#define COSMOPOLITAN_LIBC_INTRIN_LEAKY_INTERNAL_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
#define IGNORE_LEAKS(FUNC) \
STATIC_YOINK("_leaky_start"); \
void *_leaky_##FUNC[] _Section(".piro.relo.sort.leaky.2." #FUNC \
",\"aw\",@init_array #") = {FUNC}
extern intptr_t _leaky_end[] __attribute__((__weak__));
extern intptr_t _leaky_start[] __attribute__((__weak__));
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_INTRIN_LEAKY_INTERNAL_H_ */

69
libc/intrin/lengthuint64.c Normal file
View file

@ -0,0 +1,69 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/fmt/itoa.h"
#include "libc/nexgen32e/nexgen32e.h"
/**
* Returns `len(str(x))` where x is an unsigned 64-bit integer.
*/
unsigned LengthUint64(uint64_t x) {
unsigned w;
if (x) {
w = kTensIndex[63 ^ __builtin_clzll(x)];
w += x >= kTens[w];
return w;
} else {
return 1;
}
}
/**
* Returns `len(str(x))` where x is a signed 64-bit integer.
*/
unsigned LengthInt64(int64_t x) {
if (x >= 0) {
return LengthUint64(x);
} else {
return 1 + LengthUint64(-(uint64_t)x);
}
}
/**
* Returns decimal string length of uint64 w/ thousands separators.
*/
unsigned LengthUint64Thousands(uint64_t x) {
unsigned w;
w = LengthUint64(x);
w += (w - 1) / 3;
return w;
}
/**
* Returns decimal string length of int64 w/ thousands separators.
*/
unsigned LengthInt64Thousands(int64_t x) {
unsigned w;
if (x >= 0) {
w = LengthUint64(x);
return w + (w - 1) / 3;
} else {
w = LengthUint64(-(uint64_t)x);
return 1 + w + (w - 1) / 3;
}
}

9
libc/intrin/memtrack.greg.c
View file

@ -17,14 +17,14 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/intrin/bits.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/weaken.h"
#include "libc/calls/calls.h"
#include "libc/calls/strace.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/asan.internal.h"
#include "libc/intrin/bits.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/weaken.h"
#include "libc/log/libfatal.internal.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
@ -201,6 +201,9 @@ int TrackMemoryInterval(struct MemoryIntervals *mm, int x, int y, long h,
unsigned i;
#if IsModeDbg()
assert(y >= x);
if (!AreMemoryIntervalsOk(mm)) {
PrintMemoryIntervals(2, mm);
}
assert(AreMemoryIntervalsOk(mm));
#endif

2
libc/intrin/mmi.init.S
View file

@ -22,5 +22,5 @@
.init.start 200,_init__mmi
movb $OPEN_MAX,_mmi+8
movl $_mmi+24,_mmi+16
movb $PTHREAD_MUTEX_RECURSIVE,__mmi_lock_obj(%rip)
movb $PTHREAD_MUTEX_RECURSIVE,__mmi_lock_obj+16(%rip)
.init.end 200,_init__mmi

4
libc/intrin/mmi_lock.c
View file

@ -16,8 +16,10 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/thread/thread.h"
#include "libc/runtime/memtrack.internal.h"
#include "libc/thread/thread.h"
// this lock currently needs to be (1) recursive and (2) not nsync
extern pthread_mutex_t __mmi_lock_obj;

67
libc/intrin/printmemoryintervals.c Normal file
View file

@ -0,0 +1,67 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/fmt/itoa.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/kprintf.h"
#include "libc/macros.internal.h"
#include "libc/runtime/memtrack.internal.h"
#define ADDR(x) ((intptr_t)((int64_t)((uint64_t)(x) << 32) >> 16))
static bool IsNoteworthyHole(unsigned i, const struct MemoryIntervals *mm) {
// gaps between shadow frames aren't interesting
// the chasm from heap to stack ruins statistics
return !(
(IsArenaFrame(mm->p[i].y) && !IsArenaFrame(mm->p[i + 1].x)) ||
(IsShadowFrame(mm->p[i].y) || IsShadowFrame(mm->p[i + 1].x)) ||
(!IsStaticStackFrame(mm->p[i].y) && IsStaticStackFrame(mm->p[i + 1].x)));
}
void PrintMemoryIntervals(int fd, const struct MemoryIntervals *mm) {
char *p, mappingbuf[8], framebuf[32];
long i, w, frames, maptally = 0, gaptally = 0;
for (w = i = 0; i < mm->i; ++i) {
w = MAX(w, LengthInt64Thousands(mm->p[i].y + 1 - mm->p[i].x));
}
for (i = 0; i < mm->i; ++i) {
frames = mm->p[i].y + 1 - mm->p[i].x;
maptally += frames;
kprintf("%08x-%08x %s %'*ldx%s", mm->p[i].x, mm->p[i].y,
(DescribeMapping)(mappingbuf, mm->p[i].prot, mm->p[i].flags), w,
frames, (DescribeFrame)(framebuf, mm->p[i].x));
if (mm->p[i].iscow) kprintf(" cow");
if (mm->p[i].readonlyfile) kprintf(" readonlyfile");
if (mm->p[i].size !=
(size_t)(mm->p[i].y - mm->p[i].x) * FRAMESIZE + FRAMESIZE) {
kprintf(" size=%'zu", mm->p[i].size);
}
if (i + 1 < mm->i) {
frames = mm->p[i + 1].x - mm->p[i].y - 1;
if (frames && IsNoteworthyHole(i, mm)) {
gaptally += frames;
kprintf(" w/ %'ld frame hole", frames);
}
}
if (mm->p[i].h != -1) {
kprintf(" h=%ld", mm->p[i].h);
}
kprintf("\n");
}
kprintf("# %ld frames mapped w/ %'ld frames gapped\n", maptally, gaptally);
}

31
libc/intrin/pthread_getspecific.c Normal file
View file

@ -0,0 +1,31 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
/**
* Gets value of TLS slot for current thread.
*/
void *pthread_getspecific(pthread_key_t key) {
if (0 <= key && key < PTHREAD_KEYS_MAX) {
return _pthread_keys[key];
} else {
return 0;
}
}

48
libc/intrin/wait0.c → libc/intrin/pthread_key_create.c
View file

@ -16,33 +16,37 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/calls.h"
#include "libc/dce.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/futex.internal.h"
#include "libc/errno.h"
#include "libc/nexgen32e/bsr.h"
#include "libc/runtime/runtime.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
#include "libc/intrin/wait0.internal.h"
#include "libc/linux/futex.h"
#include "libc/thread/tls.h"
/**
* Blocks until memory location becomes zero.
*
* This is intended to be used on the child thread id, which is updated
* by the _spawn() system call when a thread terminates. The purpose of
* this operation is to know when it's safe to munmap() a threads stack
* Allocates TLS slot.
*/
void _wait0(const int *ctid) {
int x;
for (;;) {
if (!(x = atomic_load_explicit(ctid, memory_order_relaxed))) {
int pthread_key_create(pthread_key_t *key, pthread_key_dtor dtor) {
int i, j, rc = EAGAIN;
pthread_spin_lock(&_pthread_keys_lock);
for (i = 0; i < (PTHREAD_KEYS_MAX + 63) / 64; ++i) {
if (~_pthread_key_usage[i]) {
j = bsrl(~_pthread_key_usage[i]);
_pthread_key_usage[i] |= 1ul << j;
_pthread_key_dtor[i * 64 + j] = dtor;
*key = i * 64 + j;
rc = 0;
break;
} else {
_futex_wait(ctid, x, PTHREAD_PROCESS_SHARED, &(struct timespec){2});
}
}
if (IsOpenbsd()) {
// TODO(jart): Why do we need it? It's not even perfect.
// What's up with all these OpenBSD flakes??
pthread_yield();
}
pthread_spin_unlock(&_pthread_keys_lock);
return rc;
}
static textexit void _pthread_key_atexit(void) {
_pthread_key_destruct(0);
}
__attribute__((__constructor__)) static textstartup void _pthread_key_init() {
atexit(_pthread_key_atexit);
}

40
libc/intrin/futex_wait.c → libc/intrin/pthread_key_delete.c
View file

@ -16,35 +16,23 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/strace.internal.h"
#include "libc/calls/struct/timespec.h"
#include "libc/calls/struct/timespec.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/futex.internal.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
#include "libc/sysv/consts/futex.h"
int _futex(void *, int, int, struct timespec *) hidden;
int _futex_wait(void *addr, int expect, char pshared,
struct timespec *timeout) {
int op, ax, pf;
if (IsLinux() || IsOpenbsd()) {
pf = pshared == PTHREAD_PROCESS_PRIVATE ? FUTEX_PRIVATE_FLAG : 0;
op = FUTEX_WAIT | pf;
ax = _futex(addr, op, expect, timeout);
if (SupportsLinux() && pf && ax == -ENOSYS) {
// RHEL5 doesn't support FUTEX_PRIVATE_FLAG
op = FUTEX_WAIT;
ax = _futex(addr, op, expect, timeout);
}
if (IsOpenbsd() && ax > 0) ax = -ax; // yes openbsd does this w/o cf
STRACE("futex(%t, %s, %d, %s) → %s", addr, DescribeFutexOp(op), expect,
DescribeTimespec(0, timeout), DescribeFutexResult(ax));
return ax;
/**
* Deletes TLS slot.
*/
int pthread_key_delete(pthread_key_t key) {
int rc;
pthread_spin_lock(&_pthread_keys_lock);
if (key < PTHREAD_KEYS_MAX) {
_pthread_key_usage[key / 64] &= ~(1ul << (key % 64));
_pthread_key_dtor[key] = 0;
rc = 0;
} else {
return pthread_yield();
rc = EINVAL;
}
pthread_spin_unlock(&_pthread_keys_lock);
return rc;
}

48
libc/intrin/pthread_key_destruct.c Normal file
View file

@ -0,0 +1,48 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/nexgen32e/bsr.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
void _pthread_key_destruct(void *key[PTHREAD_KEYS_MAX]) {
int i, j;
uint64_t x;
void *value;
pthread_key_dtor dtor;
if (!__tls_enabled) return;
pthread_spin_lock(&_pthread_keys_lock);
if (!key) key = _pthread_keys;
StartOver:
for (i = 0; i < (PTHREAD_KEYS_MAX + 63) / 64; ++i) {
x = _pthread_key_usage[i];
while (x) {
j = bsrl(x);
if ((value = key[i * 64 + j]) && (dtor = _pthread_key_dtor[i * 64 + j])) {
key[i * 64 + j] = 0;
pthread_spin_unlock(&_pthread_keys_lock);
dtor(value);
pthread_spin_lock(&_pthread_keys_lock);
goto StartOver;
}
x &= ~(1ul << j);
}
}
pthread_spin_unlock(&_pthread_keys_lock);
}

31
libc/intrin/pthread_keys.c Normal file
View file

@ -0,0 +1,31 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
pthread_spinlock_t _pthread_keys_lock;
// tls value slots for pthread keys api
_Thread_local void *_pthread_keys[PTHREAD_KEYS_MAX];
// bitset of tls key registrations
uint64_t _pthread_key_usage[(PTHREAD_KEYS_MAX + 63) / 64];
// pthread tls key destructors
pthread_key_dtor _pthread_key_dtor[PTHREAD_KEYS_MAX];

4
libc/intrin/pthread_mutex_init.c
View file

@ -27,8 +27,8 @@
int pthread_mutex_init(pthread_mutex_t *mutex,
const pthread_mutexattr_t *attr) {
*mutex = (pthread_mutex_t){
attr ? attr->type : 0,
attr ? attr->pshared : 0,
._type = attr ? attr->_type : 0,
._pshared = attr ? attr->_pshared : 0,
};
return 0;
}

40
libc/intrin/pthread_mutex_lock.c
View file

@ -16,12 +16,14 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/futex.internal.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/weaken.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
#include "third_party/nsync/mu.h"
/**
* Locks mutex.
@ -60,28 +62,18 @@
int pthread_mutex_lock(pthread_mutex_t *mutex) {
int c, d, t;
if (mutex->type == PTHREAD_MUTEX_NORMAL) {
// From Futexes Are Tricky Version 1.1 § Mutex, Take 3;
// Ulrich Drepper, Red Hat Incorporated, June 27, 2004.
c = 0;
if (!atomic_compare_exchange_strong_explicit(
&mutex->lock, &c, 1, memory_order_acquire, memory_order_relaxed)) {
if (c != 2) {
c = atomic_exchange_explicit(&mutex->lock, 2, memory_order_acquire);
}
while (c) {
_futex_wait(&mutex->lock, 2, mutex->pshared, 0);
c = atomic_exchange_explicit(&mutex->lock, 2, memory_order_acquire);
}
}
if (LIKELY(__tls_enabled && //
mutex->_type == PTHREAD_MUTEX_NORMAL && //
mutex->_pshared == PTHREAD_PROCESS_PRIVATE && //
weaken(nsync_mu_lock))) {
weaken(nsync_mu_lock)((nsync_mu *)mutex);
return 0;
}
t = gettid();
if (mutex->type == PTHREAD_MUTEX_ERRORCHECK) {
c = atomic_load_explicit(&mutex->lock, memory_order_relaxed);
if (mutex->_type == PTHREAD_MUTEX_ERRORCHECK) {
c = atomic_load_explicit(&mutex->_lock, memory_order_relaxed);
if ((c & 0x000fffff) == t) {
assert(!"deadlock");
return EDEADLK;
}
}
@ -90,29 +82,29 @@ int pthread_mutex_lock(pthread_mutex_t *mutex) {
c = 0;
d = 0x10100000 | t;
if (atomic_compare_exchange_weak_explicit(
&mutex->lock, &c, d, memory_order_acquire, memory_order_relaxed)) {
&mutex->_lock, &c, d, memory_order_acquire, memory_order_relaxed)) {
break;
} else {
if ((c & 0x000fffff) == t) {
if ((c & 0x0ff00000) < 0x0ff00000) {
c = atomic_fetch_add_explicit(&mutex->lock, 0x00100000,
c = atomic_fetch_add_explicit(&mutex->_lock, 0x00100000,
memory_order_relaxed);
break;
} else {
assert(!"recurse");
return EAGAIN;
}
}
if ((c & 0xf0000000) == 0x10000000) {
d = 0x20000000 | c;
if (atomic_compare_exchange_weak_explicit(&mutex->lock, &c, d,
if (atomic_compare_exchange_weak_explicit(&mutex->_lock, &c, d,
memory_order_acquire,
memory_order_relaxed)) {
c = d;
}
}
if ((c & 0xf0000000) == 0x20000000) {
_futex_wait(&mutex->lock, c, mutex->pshared, 0);
// _futex_wait(&mutex->_lock, c, mutex->_pshared, 0);
pthread_yield();
}
}
}

25
libc/intrin/pthread_mutex_trylock.c
View file

@ -20,7 +20,11 @@
#include "libc/calls/calls.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/weaken.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
#include "third_party/nsync/mu.h"
/**
* Locks mutex if it isn't locked already.
@ -35,10 +39,21 @@
int pthread_mutex_trylock(pthread_mutex_t *mutex) {
int c, d, t;
if (mutex->type == PTHREAD_MUTEX_NORMAL) {
if (LIKELY(__tls_enabled && //
mutex->_type == PTHREAD_MUTEX_NORMAL && //
mutex->_pshared == PTHREAD_PROCESS_PRIVATE && //
weaken(nsync_mu_trylock))) {
if (weaken(nsync_mu_trylock)((nsync_mu *)mutex)) {
return 0;
} else {
return EBUSY;
}
}
if (mutex->_type == PTHREAD_MUTEX_NORMAL) {
c = 0;
if (atomic_compare_exchange_strong_explicit(
&mutex->lock, &c, 1, memory_order_acquire, memory_order_relaxed)) {
&mutex->_lock, &c, 1, memory_order_acquire, memory_order_relaxed)) {
return 0;
} else {
return EBUSY;
@ -49,14 +64,14 @@ int pthread_mutex_trylock(pthread_mutex_t *mutex) {
t = gettid();
d = 0x10100000 | t;
if (!atomic_compare_exchange_strong_explicit(
&mutex->lock, &c, d, memory_order_acquire, memory_order_relaxed)) {
if ((c & 0x000fffff) != t || mutex->type == PTHREAD_MUTEX_ERRORCHECK) {
&mutex->_lock, &c, d, memory_order_acquire, memory_order_relaxed)) {
if ((c & 0x000fffff) != t || mutex->_type == PTHREAD_MUTEX_ERRORCHECK) {
return EBUSY;
}
if ((c & 0x0ff00000) == 0x0ff00000) {
return EAGAIN;
}
atomic_fetch_add_explicit(&mutex->lock, 0x00100000, memory_order_relaxed);
atomic_fetch_add_explicit(&mutex->_lock, 0x00100000, memory_order_relaxed);
}
return 0;

30
libc/intrin/pthread_mutex_unlock.c
View file

@ -16,12 +16,14 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/futex.internal.h"
#include "libc/intrin/likely.h"
#include "libc/intrin/weaken.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
#include "third_party/nsync/mu.h"
/**
* Releases mutex.
@ -32,30 +34,28 @@
int pthread_mutex_unlock(pthread_mutex_t *mutex) {
int c, t;
if (mutex->type == PTHREAD_MUTEX_NORMAL) {
// From Futexes Are Tricky Version 1.1 § Mutex, Take 3;
// Ulrich Drepper, Red Hat Incorporated, June 27, 2004.
if ((c = atomic_fetch_sub(&mutex->lock, 1)) != 1) {
atomic_store_explicit(&mutex->lock, 0, memory_order_release);
_futex_wake(&mutex->lock, 1, mutex->pshared);
}
if (LIKELY(__tls_enabled && //
mutex->_type == PTHREAD_MUTEX_NORMAL && //
mutex->_pshared == PTHREAD_PROCESS_PRIVATE && //
weaken(nsync_mu_unlock))) {
weaken(nsync_mu_unlock)((nsync_mu *)mutex);
return 0;
}
t = gettid();
if (mutex->type == PTHREAD_MUTEX_ERRORCHECK) {
c = atomic_load_explicit(&mutex->lock, memory_order_relaxed);
if (mutex->_type == PTHREAD_MUTEX_ERRORCHECK) {
c = atomic_load_explicit(&mutex->_lock, memory_order_relaxed);
if ((c & 0x000fffff) != t) {
assert(!"permlock");
return EPERM;
}
}
c = atomic_fetch_sub(&mutex->lock, 0x00100000);
c = atomic_fetch_sub(&mutex->_lock, 0x00100000);
if ((c & 0x0ff00000) == 0x00100000) {
atomic_store_explicit(&mutex->lock, 0, memory_order_release);
atomic_store_explicit(&mutex->_lock, 0, memory_order_release);
if ((c & 0xf0000000) == 0x20000000) {
_futex_wake(&mutex->lock, 1, mutex->pshared);
// _futex_wake(&mutex->_lock, 1, mutex->_pshared);
pthread_yield();
}
}

25
libc/intrin/pthread_once.c
View file

@ -16,11 +16,12 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/weaken.h"
#include "libc/thread/thread.h"
#include "third_party/nsync/once.h"
#define INIT 0
#define CALLING 1
@ -44,28 +45,30 @@
* @return 0 on success, or errno on error
*/
int pthread_once(pthread_once_t *once, void init(void)) {
char old;
switch ((old = atomic_load_explicit(&once->lock, memory_order_relaxed))) {
uint32_t old;
if (weaken(nsync_run_once)) {
weaken(nsync_run_once)((nsync_once *)once, init);
return 0;
}
switch ((old = atomic_load_explicit(&once->_lock, memory_order_relaxed))) {
case INIT:
if (atomic_compare_exchange_strong_explicit(&once->lock, &old, CALLING,
if (atomic_compare_exchange_strong_explicit(&once->_lock, &old, CALLING,
memory_order_acquire,
memory_order_relaxed)) {
init();
atomic_store(&once->lock, FINISHED);
break;
atomic_store(&once->_lock, FINISHED);
return 0;
}
// fallthrough
case CALLING:
do {
pthread_yield();
} while (atomic_load_explicit(&once->lock, memory_order_relaxed) ==
} while (atomic_load_explicit(&once->_lock, memory_order_relaxed) ==
CALLING);
break;
return 0;
case FINISHED:
break;
return 0;
default:
assert(!"bad once");
return EINVAL;
}
return 0;
}

31
libc/intrin/futex_wake.c → libc/intrin/pthread_setspecific.c
View file

@ -16,31 +16,18 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/strace.internal.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/describeflags.internal.h"
#include "libc/intrin/futex.internal.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
#include "libc/sysv/consts/futex.h"
int _futex(void *, int, int) hidden;
int _futex_wake(void *addr, int count, char pshared) {
int op, ax, pf;
if (IsLinux() || IsOpenbsd()) {
pf = pshared == PTHREAD_PROCESS_PRIVATE ? FUTEX_PRIVATE_FLAG : 0;
op = FUTEX_WAKE | pf;
ax = _futex(addr, op, count);
if (SupportsLinux() && pf && ax == -ENOSYS) {
// RHEL5 doesn't support FUTEX_PRIVATE_FLAG
op = FUTEX_WAKE;
ax = _futex(addr, op, count);
}
STRACE("futex(%t, %s, %d) → %s", addr, DescribeFutexOp(op), count,
DescribeFutexResult(ax));
return ax;
} else {
/**
* Sets value of TLS slot for current thread.
*/
int pthread_setspecific(pthread_key_t key, void *val) {
if (0 <= key && key < PTHREAD_KEYS_MAX) {
_pthread_keys[key] = val;
return 0;
} else {
return EINVAL;
}
}

10
libc/intrin/wait0.internal.h
View file

@ -1,10 +0,0 @@
#ifndef COSMOPOLITAN_LIBC_INTRIN_WAIT0_H_
#define COSMOPOLITAN_LIBC_INTRIN_WAIT0_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
void _wait0(const int *) hidden;
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_LIBC_INTRIN_WAIT0_H_ */

21
libc/intrin/wantcrashreports.c Normal file
View file

@ -0,0 +1,21 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/log/internal.h"
bool _wantcrashreports;