Make more threading improvements

- ASAN memory morgue is now lockless - Make C11 atomics header more portable - Rewrote pthread keys support to be lockless - Simplify Python's unicode table unpacking code - Make crash report write(2) closer to being atomic - Make it possible to strace/ftrace a single thread - ASAN now checks nul-terminated strings fast and properly - Windows fork() now restores TLS memory of calling thread
2025-07-04 18:28:30 +00:00 · 2022-11-01 22:36:03 -07:00 · 2022-11-01 22:36:03 -07:00 · e522aa3a07
commit e522aa3a07
parent d7b88734cd
189 changed files with 1363 additions and 1217 deletions
--- a/libc/log/backtrace2.c
+++ b/libc/log/backtrace2.c
@ -17,38 +17,25 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/calls/calls.h"
-#include "libc/calls/internal.h"
 #include "libc/calls/struct/rusage.internal.h"
 #include "libc/calls/syscall-sysv.internal.h"
 #include "libc/calls/syscall_support-sysv.internal.h"
 #include "libc/dce.h"
 #include "libc/errno.h"
-#include "libc/fmt/conv.h"
-#include "libc/fmt/fmt.h"
 #include "libc/fmt/itoa.h"
 #include "libc/intrin/kprintf.h"
 #include "libc/intrin/promises.internal.h"
-#include "libc/intrin/safemacros.internal.h"
-#include "libc/intrin/strace.internal.h"
 #include "libc/intrin/weaken.h"
 #include "libc/log/backtrace.internal.h"
 #include "libc/log/color.internal.h"
 #include "libc/log/log.h"
-#include "libc/mem/alg.h"
-#include "libc/mem/bisectcarleft.internal.h"
-#include "libc/mem/gc.internal.h"
 #include "libc/nexgen32e/gc.internal.h"
+#include "libc/nexgen32e/stackframe.h"
 #include "libc/runtime/runtime.h"
-#include "libc/runtime/stack.h"
 #include "libc/runtime/symbols.internal.h"
-#include "libc/stdio/append.h"
-#include "libc/stdio/stdio.h"
 #include "libc/str/str.h"
-#include "libc/sysv/consts/fileno.h"
 #include "libc/sysv/consts/o.h"
-#include "libc/sysv/consts/sig.h"
 #include "libc/thread/tls.h"
-#include "libc/x/x.h"

 #define kBacktraceMaxFrames 128
 #define kBacktraceBufSize   ((kBacktraceMaxFrames - 1) * (18 + 1))
@ -58,7 +45,6 @@ static void ShowHint(const char *s) {
 }

 static int PrintBacktraceUsingAddr2line(int fd, const struct StackFrame *bp) {
-  bool ok;
  ssize_t got;
  intptr_t addr;
  size_t i, j, gi;
@ -110,6 +96,9 @@ static int PrintBacktraceUsingAddr2line(int fd, const struct StackFrame *bp) {
  garbage = __tls_enabled ? __get_tls()->tib_garbages : 0;
  gi = garbage ? garbage->i : 0;
  for (frame = bp; frame && i < kBacktraceMaxFrames - 1; frame = frame->next) {
+    if (kisdangerous(frame)) {
+      return -1;
+    }
    addr = frame->addr;
    if (addr == _weakaddr("__gc")) {
      do {
@ -158,7 +147,7 @@ static int PrintBacktraceUsingAddr2line(int fd, const struct StackFrame *bp) {
        got -= p3 - p1;
        p1 += p3 - p1;
      } else {
-        write(2, p1, got);
+        sys_write(2, p1, got);
        break;
      }
    }
@ -189,12 +178,12 @@ static int PrintBacktrace(int fd, const struct StackFrame *bp) {
 void ShowBacktrace(int fd, const struct StackFrame *bp) {
 #ifdef __FNO_OMIT_FRAME_POINTER__
  /* asan runtime depends on this function */
-  --__ftrace;
-  --__strace;
+  ftrace_enabled(-1);
+  strace_enabled(-1);
  if (!bp) bp = __builtin_frame_address(0);
  PrintBacktrace(fd, bp);
-  ++__strace;
-  ++__ftrace;
+  strace_enabled(+1);
+  ftrace_enabled(+1);
 #else
  (fprintf)(stderr, "ShowBacktrace() needs these flags to show C backtrace:\n"
                    "\t-D__FNO_OMIT_FRAME_POINTER__\n"
--- a/libc/log/backtrace3.c
+++ b/libc/log/backtrace3.c
@ -58,6 +58,10 @@ noinstrument noasan int PrintBacktraceUsingSymbols(int fd,
  garbage = __tls_enabled ? __get_tls()->tib_garbages : 0;
  gi = garbage ? garbage->i : 0;
  for (i = 0, frame = bp; frame; frame = frame->next) {
+    if (kisdangerous(frame)) {
+      kprintf("<dangerous frame>\n");
+      break;
+    }
    if (++i == LIMIT) {
      kprintf("<truncated backtrace>\n");
      break;
--- a/libc/log/checkfail.c
+++ b/libc/log/checkfail.c
@ -45,8 +45,8 @@ relegated void __check_fail(const char *suffix, const char *opstr,
  size_t i;
  va_list va;
  char hostname[32];
-  --__strace;
-  --__ftrace;
+  strace_enabled(-1);
+  ftrace_enabled(-1);
  e = errno;
  __start_fatal(file, line);
  __stpcpy(hostname, "unknown");
--- a/libc/log/die.c
+++ b/libc/log/die.c
@ -16,11 +16,13 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/atomic.h"
 #include "libc/calls/calls.h"
+#include "libc/calls/state.internal.h"
 #include "libc/calls/syscall-sysv.internal.h"
 #include "libc/dce.h"
+#include "libc/intrin/atomic.h"
 #include "libc/intrin/kprintf.h"
-#include "libc/intrin/lockcmpxchgp.h"
 #include "libc/log/backtrace.internal.h"
 #include "libc/log/internal.h"
 #include "libc/log/libfatal.internal.h"
@ -40,10 +42,12 @@ STATIC_YOINK("_idt");
 relegated wontreturn void __die(void) {
  /* asan runtime depends on this function */
  int me, owner;
-  static int sync;
+  static atomic_int once;
  owner = 0;
  me = sys_gettid();
-  if (_lockcmpxchgp(&sync, &owner, me)) {
+  if (__vforked ||
+      atomic_compare_exchange_strong_explicit(
+          &once, &owner, me, memory_order_relaxed, memory_order_relaxed)) {
    __restore_tty();
    if (IsDebuggerPresent(false)) {
      DebugBreak();
--- a/libc/log/leaks.c
+++ b/libc/log/leaks.c
@ -19,8 +19,8 @@
 #include "libc/dce.h"
 #include "libc/intrin/asan.internal.h"
 #include "libc/intrin/bits.h"
+#include "libc/intrin/cmpxchg.h"
 #include "libc/intrin/kprintf.h"
-#include "libc/intrin/lockcmpxchg.h"
 #include "libc/intrin/strace.internal.h"
 #include "libc/mem/mem.h"
 #include "libc/runtime/internal.h"
@ -82,7 +82,7 @@ static noasan bool HasLeaks(void) {
 noasan void CheckForMemoryLeaks(void) {
  struct mallinfo mi;
  if (!IsAsan()) return;  // we need traces to exclude leaky
-  if (!_lockcmpxchg(&once, false, true)) {
+  if (!_cmpxchg(&once, false, true)) {
    kprintf("CheckForMemoryLeaks() may only be called once\n");
    exit(1);
  }
--- a/libc/log/log_retrace.c
+++ b/libc/log/log_retrace.c
@ -20,5 +20,5 @@
 #include "libc/runtime/runtime.h"

 noinstrument void _log_retrace(void) {
-  ++__ftrace;
+  ftrace_enabled(+1);
 }
--- a/libc/log/log_untrace.c
+++ b/libc/log/log_untrace.c
@ -20,5 +20,5 @@
 #include "libc/runtime/runtime.h"

 noinstrument void _log_untrace(void) {
-  --__ftrace;
+  ftrace_enabled(-1);
 }
--- a/libc/log/oncrash.c
+++ b/libc/log/oncrash.c
@ -16,6 +16,8 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/assert.h"
+#include "libc/atomic.h"
 #include "libc/calls/calls.h"
 #include "libc/calls/state.internal.h"
 #include "libc/calls/struct/sigaction.h"
@ -25,9 +27,9 @@
 #include "libc/errno.h"
 #include "libc/fmt/itoa.h"
 #include "libc/intrin/asan.internal.h"
+#include "libc/intrin/atomic.h"
+#include "libc/intrin/kmalloc.h"
 #include "libc/intrin/kprintf.h"
-#include "libc/intrin/lockcmpxchg.h"
-#include "libc/intrin/lockcmpxchgp.h"
 #include "libc/intrin/strace.internal.h"
 #include "libc/intrin/weaken.h"
 #include "libc/log/backtrace.internal.h"
@ -118,14 +120,12 @@ static char *HexCpy(char p[hasatleast 17], uint64_t x, uint8_t k) {
  return p;
 }

-relegated static void ShowGeneralRegisters(ucontext_t *ctx) {
+relegated static char *ShowGeneralRegisters(char *p, ucontext_t *ctx) {
  int64_t x;
  const char *s;
  size_t i, j, k;
  long double st;
-  char *p, buf[128];
-  p = buf;
-  kprintf("\n");
+  *p++ = '\n';
  for (i = 0, j = 0, k = 0; i < ARRAYLEN(kGregNames); ++i) {
    if (j > 0) *p++ = ' ';
    if (!(s = kGregNames[(unsigned)kGregOrder[i]])[2]) *p++ = ' ';
@ -145,25 +145,25 @@ relegated static void ShowGeneralRegisters(ucontext_t *ctx) {
      if (x < 0) x = -x, *p++ = '-';
      p = FormatUint64(p, x / 1000), *p++ = '.';
      p = FormatUint64(p, x % 1000);
-      *p = 0;
-      kprintf("%s\n", buf);
-      p = buf;
+      *p++ = '\n';
    }
  }
  DescribeCpuFlags(
      p, ctx->uc_mcontext.eflags,
      ctx->uc_mcontext.fpregs ? ctx->uc_mcontext.fpregs->swd : 0,
      ctx->uc_mcontext.fpregs ? ctx->uc_mcontext.fpregs->mxcsr : 0);
-  kprintf("%s\n", buf);
+  *p++ = '\n';
+  return p;
 }

-relegated static void ShowSseRegisters(ucontext_t *ctx) {
+relegated static char *ShowSseRegisters(char *p, ucontext_t *ctx) {
  size_t i;
-  char *p, buf[128];
  if (ctx->uc_mcontext.fpregs) {
-    kprintf("\n");
+    *p++ = '\n';
    for (i = 0; i < 8; ++i) {
-      p = buf;
+      *p++ = 'X';
+      *p++ = 'M';
+      *p++ = 'M';
      if (i >= 10) {
        *p++ = i / 10 + '0';
        *p++ = i % 10 + '0';
@ -185,10 +185,10 @@ relegated static void ShowSseRegisters(ucontext_t *ctx) {
      *p++ = ' ';
      p = HexCpy(p, ctx->uc_mcontext.fpregs->xmm[i + 8].u64[1], 64);
      p = HexCpy(p, ctx->uc_mcontext.fpregs->xmm[i + 8].u64[0], 64);
-      *p = 0;
-      kprintf("XMM%s\n", buf);
+      *p++ = '\n';
    }
  }
+  return p;
 }

 void ShowCrashReportHook(int, int, int, struct siginfo *, ucontext_t *);
@ -196,10 +196,10 @@ void ShowCrashReportHook(int, int, int, struct siginfo *, ucontext_t *);
 relegated void ShowCrashReport(int err, int sig, struct siginfo *si,
                               ucontext_t *ctx) {
  int i;
-  char *p;
+  size_t n;
  char host[64];
+  char *p, *buf;
  struct utsname names;
-  static char buf[4096];
  if (_weaken(ShowCrashReportHook)) {
    ShowCrashReportHook(2, err, sig, si, ctx);
  }
@ -210,24 +210,31 @@ relegated void ShowCrashReport(int err, int sig, struct siginfo *si,
  stpcpy(host, "unknown");
  gethostname(host, sizeof(host));
  uname(&names);
-  p = buf;
  errno = err;
-  kprintf("\n%serror%s: Uncaught %G (%s) on %s pid %d tid %d\n"
-          "  %s\n"
-          "  %m\n"
-          "  %s %s %s %s\n",
-          !__nocolor ? "\e[30;101m" : "", !__nocolor ? "\e[0m" : "", sig,
-          (ctx && (ctx->uc_mcontext.rsp >= GetStaticStackAddr(0) &&
-                   ctx->uc_mcontext.rsp <= GetStaticStackAddr(0) + PAGESIZE))
-              ? "Stack Overflow"
-              : GetSiCodeName(sig, si->si_code),
-          host, getpid(), gettid(), program_invocation_name, names.sysname,
-          names.version, names.nodename, names.release);
+  // TODO(jart): Buffer the WHOLE crash report with backtrace for atomic write.
+  _npassert((p = buf = kmalloc((n = 1024 * 1024))));
+  p += ksnprintf(
+      p, n,
+      "\n%serror%s: Uncaught %G (%s) on %s pid %d tid %d\n"
+      "  %s\n"
+      "  %m\n"
+      "  %s %s %s %s\n",
+      !__nocolor ? "\e[30;101m" : "", !__nocolor ? "\e[0m" : "", sig,
+      (ctx && (ctx->uc_mcontext.rsp >= GetStaticStackAddr(0) &&
+               ctx->uc_mcontext.rsp <= GetStaticStackAddr(0) + PAGESIZE))
+          ? "Stack Overflow"
+          : GetSiCodeName(sig, si->si_code),
+      host, getpid(), gettid(), program_invocation_name, names.sysname,
+      names.version, names.nodename, names.release);
  if (ctx) {
-    ShowGeneralRegisters(ctx);
-    ShowSseRegisters(ctx);
-    kprintf("\n");
+    p = ShowGeneralRegisters(p, ctx);
+    p = ShowSseRegisters(p, ctx);
+    *p++ = '\n';
+    write(2, buf, p - buf);
    ShowFunctionCalls(ctx);
+  } else {
+    *p++ = '\n';
+    write(2, buf, p - buf);
  }
  kprintf("\n");
  if (!IsWindows()) __print_maps();
@ -235,7 +242,7 @@ relegated void ShowCrashReport(int err, int sig, struct siginfo *si,
  if (__argv) {
    for (i = 0; i < __argc; ++i) {
      if (!__argv[i]) continue;
-      if (IsAsan() && !__asan_is_valid(__argv[i], 1)) continue;
+      if (IsAsan() && !__asan_is_valid_str(__argv[i])) continue;
      kprintf("%s ", __argv[i]);
    }
  }
@ -279,17 +286,19 @@ static wontreturn relegated noinstrument void __minicrash(int sig,
 * @vforksafe
 */
 relegated void __oncrash(int sig, struct siginfo *si, ucontext_t *ctx) {
+  bool bZero;
  intptr_t rip;
  int me, owner;
  int gdbpid, err;
-  static int sync;
-  static bool _notpossible;
+  static atomic_int once;
+  static atomic_bool once2;
  STRACE("__oncrash rip %x", ctx->uc_mcontext.rip);
-  --__ftrace;
-  --__strace;
+  ftrace_enabled(-1);
+  strace_enabled(-1);
  owner = 0;
  me = sys_gettid();
-  if (__vforked || _lockcmpxchgp(&sync, &owner, me)) {
+  if (atomic_compare_exchange_strong_explicit(
+          &once, &owner, me, memory_order_relaxed, memory_order_relaxed)) {
    if (!__vforked) {
      rip = ctx ? ctx->uc_mcontext.rip : 0;
      err = errno;
@ -310,9 +319,9 @@ relegated void __oncrash(int sig, struct siginfo *si, ucontext_t *ctx) {
        ShowCrashReport(err, sig, si, ctx);
        _Exitr(128 + sig);
      }
-      sync = 0;
+      atomic_store_explicit(&once, 0, memory_order_relaxed);
    } else {
-      sync = 0;
+      atomic_store_explicit(&once, 0, memory_order_relaxed);
      __minicrash(sig, si, ctx, "WHILE VFORKED");
    }
  } else if (sig == SIGTRAP) {
@ -320,7 +329,9 @@ relegated void __oncrash(int sig, struct siginfo *si, ucontext_t *ctx) {
    goto ItsATrap;
  } else if (owner == me) {
    // we crashed while generating a crash report
-    if (_lockcmpxchg(&_notpossible, false, true)) {
+    bZero = false;
+    if (atomic_compare_exchange_strong_explicit(
+            &once2, &bZero, true, memory_order_relaxed, memory_order_relaxed)) {
      __minicrash(sig, si, ctx, "WHILE CRASHING");
    } else {
      // somehow __minicrash() crashed not possible
@ -335,6 +346,6 @@ relegated void __oncrash(int sig, struct siginfo *si, ucontext_t *ctx) {
    _Exit1(8);
  }
 ItsATrap:
-  ++__strace;
-  ++__ftrace;
+  strace_enabled(+1);
+  ftrace_enabled(+1);
 }
--- a/libc/log/showcrashreports.c
+++ b/libc/log/showcrashreports.c
@ -68,7 +68,7 @@ relegated void RestoreDefaultCrashSignalHandlers(void) {
  int e;
  size_t i;
  sigset_t ss;
-  --__strace;
+  strace_enabled(-1);
  sigemptyset(&ss);
  sigprocmask(SIG_SETMASK, &ss, NULL);
  for (i = 0; i < ARRAYLEN(kCrashSigs); ++i) {
@ -78,7 +78,7 @@ relegated void RestoreDefaultCrashSignalHandlers(void) {
      errno = e;
    }
  }
-  ++__strace;
+  strace_enabled(+1);
 }

 static void FreeSigAltStack(void *p) {
--- a/libc/log/vflogf.c
+++ b/libc/log/vflogf.c
@ -95,7 +95,7 @@ void(vflogf)(unsigned level, const char *file, int line, FILE *f,
  if (!f) f = __log_file;
  if (!f) return;
  flockfile(f);
-  --__strace;
+  strace_enabled(-1);

  // We display TIMESTAMP.MICROS normally. However, when we log multiple
  // times in the same second, we display TIMESTAMP+DELTAMICROS instead.
@ -138,6 +138,6 @@ void(vflogf)(unsigned level, const char *file, int line, FILE *f,
    unreachable;
  }

-  ++__strace;
+  strace_enabled(+1);
  funlockfile(f);
 }