Make more threading improvements

- ASAN memory morgue is now lockless
- Make C11 atomics header more portable
- Rewrote pthread keys support to be lockless
- Simplify Python's unicode table unpacking code
- Make crash report write(2) closer to being atomic
- Make it possible to strace/ftrace a single thread
- ASAN now checks nul-terminated strings fast and properly
- Windows fork() now restores TLS memory of calling thread

libc/x/x.h

@@ -48,8 +48,8 @@ char *utf32to8(const wchar_t *, size_t, size_t *) dontdiscard;
 char *xhomedir(void) dontdiscard;
 char *xstripext(const char *) dontdiscard;
 char *xstripexts(const char *) dontdiscard;
-void *xload(bool *, void **, const void *, size_t, size_t);
-void *xloadzd(bool *, void **, const void *, size_t, size_t, size_t, size_t,
+void *xload(_Atomic(void *) *, const void *, size_t, size_t);
+void *xloadzd(_Atomic(void *) *, const void *, size_t, size_t, size_t, size_t,
               uint32_t);
 int rmrf(const char *);
 char *xbasename(const char *) paramsnonnull()

libc/x/xload.c

@@ -16,10 +16,9 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/intrin/lockcmpxchg.h"
-#include "libc/log/check.h"
-#include "libc/mem/mem.h"
-#include "libc/runtime/runtime.h"
+#include "libc/intrin/atomic.h"
+#include "libc/intrin/kmalloc.h"
+#include "libc/runtime/internal.h"
 #include "libc/x/x.h"
 #include "third_party/zlib/zlib.h"
 
@@ -27,43 +26,30 @@
  * Inflates data once atomically, e.g.
  *
  *     void *GetData(void) {
- *       static char once;
- *       static void *ptr;
+ *       static _Atomic(void *) ptr;
  *       static const unsigned char rodata[] = {...};
- *       if (once) return ptr;
- *       return xload(&once, &ptr, rodata, 112, 1024);
+ *       return xload(&ptr, rodata, 112, 1024);
  *     }
  *
  * The above is an example of how this helper may be used to have lazy
  * loading of big infrequently accessed image data.
  *
- * @param o points to your static init guard
- * @param t points to your static pointer holder
+ * @param a points to your static pointer holder
  * @param p is read-only data compressed using raw deflate
  * @param n is byte length of deflated data
  * @param m is byte length of inflated data
  * @return pointer to inflated data
  * @threadsafe
  */
-void *xload(bool *o, void **t, const void *p, size_t n, size_t m) {
-  void *q;
-  z_stream zs;
-  q = malloc(m);
-  zs.zfree = 0;
-  zs.zalloc = 0;
-  zs.next_in = p;
-  zs.avail_in = n;
-  zs.total_in = n;
-  zs.avail_out = m;
-  zs.total_out = m;
-  zs.next_out = (void *)q;
-  inflateInit2(&zs, -MAX_WBITS);
-  inflate(&zs, Z_NO_FLUSH);
-  if (_lockcmpxchg(t, 0, q)) {
-    __cxa_atexit(free, q, 0);
-  } else {
-    free(q);
+void *xload(_Atomic(void *) * a, const void *p, size_t n, size_t m) {
+  void *r, *z;
+  if ((r = atomic_load_explicit(a, memory_order_acquire))) return r;
+  if (!(r = kmalloc(m))) return 0;
+  if (__inflate(r, m, p, n)) return 0;
+  z = 0;
+  if (!atomic_compare_exchange_strong_explicit(a, &z, r, memory_order_release,
+                                               memory_order_relaxed)) {
+    r = z;
   }
-  *o = true;
-  return *t;
+  return r;
 }

libc/x/xloadzd.c

@@ -18,7 +18,8 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/assert.h"
 #include "libc/fmt/leb128.h"
-#include "libc/intrin/lockcmpxchg.h"
+#include "libc/intrin/atomic.h"
+#include "libc/intrin/kmalloc.h"
 #include "libc/mem/mem.h"
 #include "libc/nexgen32e/crc32.h"
 #include "libc/runtime/internal.h"
@@ -29,8 +30,7 @@
 /**
  * Loads δzd encoded data once atomically.
  *
- * @param o points to your static init guard
- * @param t points to your static pointer holder
+ * @param a points to your static pointer holder
  * @param p is read-only data compressed using raw deflate
  * @param n is byte length of deflated data
  * @param m is byte length of inflated data
@@ -40,17 +40,23 @@
  * @return pointer to decoded data
  * @threadsafe
  */
-void *xloadzd(bool *o, void **t, const void *p, size_t n, size_t m, size_t c,
+void *xloadzd(_Atomic(void *) * a, const void *p, size_t n, size_t m, size_t c,
               size_t z, uint32_t s) {
-  void *r;
   size_t i;
-  z_stream zs;
   char *q, *b;
+  void *r, *g;
   int64_t x, y;
+  if ((r = atomic_load_explicit(a, memory_order_acquire))) return r;
   _unassert(z == 2 || z == 4);
-  b = q = malloc(m);
-  __inflate(q, m, p, n);
-  r = memalign(z, c * z);
+  if (!(b = q = malloc(m))) return 0;
+  if (__inflate(q, m, p, n)) {
+    free(q);
+    return 0;
+  }
+  if (!(r = kmalloc(c * z))) {
+    free(q);
+    return 0;
+  }
   for (x = i = 0; i < c; ++i) {
     b += unzleb64(b, 10, &y);
     x += y;
@@ -61,12 +67,11 @@ void *xloadzd(bool *o, void **t, const void *p, size_t n, size_t m, size_t c,
     }
   }
   free(q);
-  _npassert(crc32_z(0, r, c * z) == s);
-  if (_lockcmpxchg(t, 0, r)) {
-    __cxa_atexit(free, r, 0);
-  } else {
-    free(q);
+  assert(crc32_z(0, r, c * z) == s);
+  g = 0;
+  if (!atomic_compare_exchange_strong_explicit(a, &g, r, memory_order_relaxed,
+                                               memory_order_relaxed)) {
+    r = g;
   }
-  *o = true;
-  return *t;
+  return r;
 }