Make more threading improvements

- ASAN memory morgue is now lockless
- Make C11 atomics header more portable
- Rewrote pthread keys support to be lockless
- Simplify Python's unicode table unpacking code
- Make crash report write(2) closer to being atomic
- Make it possible to strace/ftrace a single thread
- ASAN now checks nul-terminated strings fast and properly
- Windows fork() now restores TLS memory of calling thread
This commit is contained in:
Justine Tunney 2022-11-01 22:36:03 -07:00
parent d7b88734cd
commit e522aa3a07
No known key found for this signature in database
GPG key ID: BE714B4575D6E328
189 changed files with 1363 additions and 1217 deletions

View file

@ -6,11 +6,9 @@
/**
* @fileoverview Cosmopolitan C11 Atomics Library
*
* - Forty-two different ways to say MOV.
* - Fourteen different ways to say XCHG.
* - Twenty different ways to say LOCK CMPXCHG.
*
* It's a lower level programming language than assembly!
* - Forty-two different ways to say MOV.
* - Fourteen different ways to say XCHG.
* - Twenty different ways to say LOCK CMPXCHG.
*
* @see libc/atomic.h
*/
@ -23,7 +21,7 @@
#define memory_order_acq_rel 4
#define memory_order_seq_cst 5
#define ATOMIC_VAR_INIT(value) (value)
#define ATOMIC_VAR_INIT(...) __VA_ARGS__
#define atomic_is_lock_free(obj) ((void)(obj), sizeof(obj) <= sizeof(void *))
#define atomic_flag atomic_bool
@ -113,27 +111,26 @@
#define atomic_store_explicit(pObject, desired, order) \
__atomic_store_n(pObject, desired, order)
#else
#elif (__GNUC__ + 0) * 100 + (__GNUC_MINOR__ + 0) >= 401
#define atomic_init(obj, value) ((void)(*(obj) = (value)))
#define atomic_thread_fence(order) __sync_synchronize()
#define atomic_signal_fence(order) __asm__ volatile("" ::: "memory")
#define __atomic_apply_stride(object, operand) \
(((__typeof__(__atomic_val(object)))0) + (operand))
#define atomic_compare_exchange_strong_explicit(object, expected, desired, \
success, failure) \
__extension__({ \
__typeof__(expected) __ep = (expected); \
__typeof__(*__ep) __e = *__ep; \
(void)(success); \
(void)(failure); \
(_Bool)((*__ep = __sync_val_compare_and_swap(object, __e, desired)) == \
__e); \
(((__typeof__(*(object)))0) + (operand))
#define atomic_compare_exchange_strong_explicit(object, expected, desired, \
success_order, failure_order) \
__extension__({ \
__typeof__(expected) __ep = (expected); \
__typeof__(*__ep) __e = *__ep; \
(void)(success_order); \
(void)(failure_order); \
(*__ep = __sync_val_compare_and_swap(object, __e, desired)) == __e; \
})
#define atomic_compare_exchange_weak_explicit(object, expected, desired, \
success, failure) \
atomic_compare_exchange_strong_explicit(object, expected, desired, success, \
failure)
#define atomic_compare_exchange_weak_explicit(object, expected, desired, \
success_order, failure_order) \
atomic_compare_exchange_strong_explicit(object, expected, desired, \
success_order, failure_order)
#if __has_builtin(__sync_swap)
#define atomic_exchange_explicit(object, desired, order) \
((void)(order), __sync_swap(object, desired))
@ -144,7 +141,7 @@
__typeof__(desired) __d = (desired); \
(void)(order); \
__sync_synchronize(); \
__sync_lock_test_and_set(&__atomic_val(__o), __d); \
__sync_lock_test_and_set(__o, __d); \
})
#endif
#define atomic_fetch_add_explicit(object, operand, order) \
@ -164,6 +161,57 @@
#define atomic_store_explicit(object, desired, order) \
((void)atomic_exchange_explicit(object, desired, order))
#elif defined(__GNUC__) && defined(__x86__) /* x86 with gcc 4.0 and earlier */
#define atomic_init(obj, value) ((void)(*(obj) = (value)))
#define atomic_thread_fence(order) __asm__ volatile("mfence" ::: "memory")
#define atomic_signal_fence(order) __asm__ volatile("" ::: "memory")
#define atomic_compare_exchange_strong_explicit(object, expected, desired, \
success_order, failure_order) \
__extension__({ \
char DidIt; \
__typeof__(object) IfThing = (object); \
__typeof__(IfThing) IsEqualToMe = (expected); \
__typeof__(*IfThing) ReplaceItWithMe = (desired), ax; \
(void)(success_order); \
(void)(failure_order); \
__asm__ volatile("lock cmpxchg\t%3,(%1)\n\t" \
"setz\t%b0" \
: "=q"(DidIt), "=r"(IfThing), "+a"(ax) \
: "r"(ReplaceItWithMe), "2"(*IsEqualToMe) \
: "memory", "cc"); \
*IsEqualToMe = ax; \
DidIt; \
})
#define atomic_compare_exchange_weak_explicit(object, expected, desired, \
success_order, failure_order) \
atomic_compare_exchange_strong_explicit(object, expected, desired, \
success_order, failure_order)
#define atomic_exchange_explicit(object, desired, order) \
__extension__({ \
__typeof__(object) __o = (object); \
__typeof__(*__o) __d = (desired); \
(void)(order); \
__asm__ volatile("xchg\t%0,%1" : "=r"(__d), "+m"(*__o) : "0"(__d)); \
__d; \
})
#define atomic_fetch_add_explicit(object, operand, order) \
__extension__({ \
__typeof__(object) __o = (object); \
__typeof__(*__o) __d = (desired); \
(void)(order); \
__asm__ volatile("lock xadd\t%0,%1" : "=r"(__d), "+m"(*__o) : "0"(__d)); \
__d; \
})
#define atomic_fetch_sub_explicit(object, operand, order) \
atomic_fetch_add_explicit(object, -(operand), order)
#define atomic_load_explicit(object, order) \
atomic_fetch_add_explicit(object, 0, order)
#define atomic_store_explicit(object, desired, order) \
((void)atomic_exchange_explicit(object, desired, order))
#else /* non-gcc or old gcc w/o x86 */
#error "atomic operations not supported with this compiler and/or architecture"
#endif
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */