Make garbage collection thread safe

- You can now use _gc(malloc()) in multithreaded programs
- This change fixes a bug where fork() on NT disabled TLS
- Fixed TLS code morphing on XNU/NT, for R8-R15 registers

examples/compress.c

@@ -8,6 +8,7 @@
 ╚─────────────────────────────────────────────────────────────────*/
 #endif
 #include "libc/assert.h"
+#include "libc/calls/calls.h"
 #include "libc/errno.h"
 #include "libc/fmt/conv.h"
 #include "libc/log/check.h"

examples/decompress.c

@@ -8,6 +8,7 @@
 ╚─────────────────────────────────────────────────────────────────*/
 #endif
 #include "libc/assert.h"
+#include "libc/calls/calls.h"
 #include "libc/errno.h"
 #include "libc/mem/mem.h"
 #include "libc/runtime/gc.internal.h"

libc/integral/c.inc

@@ -591,10 +591,10 @@ typedef struct {
 #endif
 #endif
 
-#define notpossible \
-  do {              \
-    asm("ud2");     \
-    unreachable;    \
+#define notpossible  \
+  do {               \
+    asm("ud2\nnop"); \
+    unreachable;     \
   } while (0)
 
 #define donothing \

libc/intrin/asan.c

@@ -40,7 +40,9 @@
 #include "libc/mem/mem.h"
 #include "libc/mem/reverse.internal.h"
 #include "libc/nexgen32e/gc.internal.h"
+#include "libc/nexgen32e/gettls.h"
 #include "libc/nexgen32e/stackframe.h"
+#include "libc/nexgen32e/threaded.h"
 #include "libc/nt/enum/version.h"
 #include "libc/nt/runtime.h"
 #include "libc/runtime/directmap.internal.h"
@@ -56,6 +58,7 @@
 #include "libc/sysv/consts/nr.h"
 #include "libc/sysv/consts/prot.h"
 #include "libc/sysv/errfuns.h"
+#include "libc/thread/thread.h"
 #include "third_party/dlmalloc/dlmalloc.h"
 
 STATIC_YOINK("_init_asan");
@@ -936,7 +939,7 @@ static void __asan_trace(struct AsanTrace *bt, const struct StackFrame *bp) {
   size_t i, gi;
   intptr_t addr;
   struct Garbages *garbage;
-  garbage = weaken(__garbage);
+  garbage = __tls_enabled ? ((cthread_t)__get_tls())->garbages : 0;
   gi = garbage ? garbage->i : 0;
   for (f1 = -1, i = 0; bp && i < ARRAYLEN(bt->p); ++i, bp = bp->next) {
     if (f1 != (f2 = ((intptr_t)bp >> 16))) {

libc/intrin/tlsisrequired.c

@@ -0,0 +1,25 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/nexgen32e/threaded.h"
+
+void TlsIsRequired(void) {
+  if (!__tls_enabled) {
+    notpossible;
+  }
+}

libc/log/backtrace2.c

@@ -37,6 +37,8 @@
 #include "libc/mem/alg.h"
 #include "libc/mem/bisectcarleft.internal.h"
 #include "libc/nexgen32e/gc.internal.h"
+#include "libc/nexgen32e/gettls.h"
+#include "libc/nexgen32e/threaded.h"
 #include "libc/runtime/gc.internal.h"
 #include "libc/runtime/runtime.h"
 #include "libc/runtime/stack.h"
@@ -47,6 +49,7 @@
 #include "libc/sysv/consts/fileno.h"
 #include "libc/sysv/consts/o.h"
 #include "libc/sysv/consts/sig.h"
+#include "libc/thread/thread.h"
 #include "libc/x/x.h"
 
 #define kBacktraceMaxFrames 128
@@ -106,7 +109,7 @@ static int PrintBacktraceUsingAddr2line(int fd, const struct StackFrame *bp) {
   argv[i++] = "-a"; /* filter out w/ shell script wrapper for old versions */
   argv[i++] = "-pCife";
   argv[i++] = debugbin;
-  garbage = weaken(__garbage);
+  garbage = __tls_enabled ? ((cthread_t)__get_tls())->garbages : 0;
   gi = garbage ? garbage->i : 0;
   for (frame = bp; frame && i < kBacktraceMaxFrames - 1; frame = frame->next) {
     addr = frame->addr;

libc/log/backtrace3.c

@@ -16,21 +16,24 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/mem/bisectcarleft.internal.h"
 #include "libc/assert.h"
-#include "libc/intrin/weaken.h"
 #include "libc/calls/calls.h"
 #include "libc/fmt/fmt.h"
 #include "libc/fmt/itoa.h"
 #include "libc/intrin/kprintf.h"
+#include "libc/intrin/weaken.h"
 #include "libc/log/backtrace.internal.h"
 #include "libc/macros.internal.h"
+#include "libc/mem/bisectcarleft.internal.h"
 #include "libc/nexgen32e/gc.internal.h"
+#include "libc/nexgen32e/gettls.h"
 #include "libc/nexgen32e/stackframe.h"
+#include "libc/nexgen32e/threaded.h"
 #include "libc/runtime/memtrack.internal.h"
 #include "libc/runtime/runtime.h"
 #include "libc/runtime/symbols.internal.h"
 #include "libc/str/str.h"
+#include "libc/thread/thread.h"
 
 #define LIMIT 100
 
@@ -54,7 +57,7 @@ noinstrument noasan int PrintBacktraceUsingSymbols(int fd,
   struct Garbages *garbage;
   const struct StackFrame *frame;
   if (!bp) bp = __builtin_frame_address(0);
-  garbage = weaken(__garbage);
+  garbage = __tls_enabled ? ((cthread_t)__get_tls())->garbages : 0;
   gi = garbage ? garbage->i : 0;
   for (i = 0, frame = bp; frame; frame = frame->next) {
     if (++i == LIMIT) {

libc/log/log.mk

@@ -69,6 +69,7 @@ o/$(MODE)/libc/log/watch.o: private			\
 		OVERRIDE_CFLAGS +=			\
 			-ffreestanding
 
+o/$(MODE)/libc/log/watch.o				\
 o/$(MODE)/libc/log/attachdebugger.o			\
 o/$(MODE)/libc/log/checkaligned.o			\
 o/$(MODE)/libc/log/checkfail.o				\

libc/log/printgarbage.c

@@ -20,8 +20,11 @@
 #include "libc/intrin/kprintf.h"
 #include "libc/log/log.h"
 #include "libc/nexgen32e/gc.internal.h"
+#include "libc/nexgen32e/gettls.h"
+#include "libc/nexgen32e/threaded.h"
 #include "libc/stdio/stdio.h"
-/* clang-format off */
+#include "libc/thread/thread.h"
+// clang-format off
 
 /**
  * Prints list of deferred operations on shadow stack.
@@ -30,24 +33,25 @@ void PrintGarbage(void) {
   size_t i;
   char name[19];
   const char *symbol;
+  struct Garbages *g;
   kprintf("\n");
   kprintf("                         SHADOW STACK @ %p\n", __builtin_frame_address(0));
   kprintf("garbage ent. parent frame     original ret        callback              arg        \n");
   kprintf("------------ ------------ ------------------ ------------------ ------------------\n");
-  if (__garbage.i) {
-    for (i = __garbage.i; i--;) {
-      symbol = __get_symbol_by_addr(__garbage.p[i].ret);
+  if ((g = __tls_enabled ? ((cthread_t)__get_tls())->garbages:0) && g->i) {
+    for (i = g->i; i--;) {
+      symbol = __get_symbol_by_addr(g->p[i].ret);
       if (symbol) {
         ksnprintf(name, sizeof(name), "%s", symbol);
       } else {
-        ksnprintf(name, sizeof(name), "%#014lx", __garbage.p[i].ret);
+        ksnprintf(name, sizeof(name), "%#014lx", g->p[i].ret);
       }
       kprintf("%12lx %12lx %18s %18s %#18lx\n",
-              __garbage.p + i,
-              __garbage.p[i].frame,
+              g->p + i,
+              g->p[i].frame,
               name,
-              __get_symbol_by_addr(__garbage.p[i].fn),
-              __garbage.p[i].arg);
+              __get_symbol_by_addr(g->p[i].fn),
+              g->p[i].arg);
     }
   } else {
     kprintf("%12s %12s %18s %18s %18s\n","empty","-","-","-","-");

libc/mem/_gc_free.c

@@ -0,0 +1,24 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/mem/mem.h"
+#include "libc/runtime/gc.h"
+
+void _gc_free(void *p) {
+  free(p);
+}

libc/mem/defer.greg.c

@@ -1,85 +0,0 @@
-/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
-│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
-╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2020 Justine Alexandra Roberts Tunney                              │
-│                                                                              │
-│ Permission to use, copy, modify, and/or distribute this software for         │
-│ any purpose with or without fee is hereby granted, provided that the         │
-│ above copyright notice and this permission notice appear in all copies.      │
-│                                                                              │
-│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
-│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
-│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
-│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
-│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
-│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
-│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
-│ PERFORMANCE OF THIS SOFTWARE.                                                │
-╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/assert.h"
-#include "libc/intrin/likely.h"
-#include "libc/intrin/weaken.h"
-#include "libc/calls/calls.h"
-#include "libc/mem/mem.h"
-#include "libc/nexgen32e/gc.internal.h"
-#include "libc/runtime/gc.internal.h"
-#include "libc/runtime/runtime.h"
-#include "libc/str/str.h"
-
-forceinline bool PointerNotOwnedByParentStackFrame(struct StackFrame *frame,
-                                                   struct StackFrame *parent,
-                                                   void *ptr) {
-  return !(((intptr_t)ptr > (intptr_t)frame) &&
-           ((intptr_t)ptr < (intptr_t)parent));
-}
-
-static void __garbage_destroy(void) {
-  if (weaken(free)) {
-    weaken(free)(__garbage.p);
-  }
-  bzero(&__garbage, sizeof(__garbage));
-}
-
-void __deferer(struct StackFrame *frame, void *fn, void *arg) {
-  size_t n2;
-  struct Garbage *p2;
-  if (UNLIKELY(__garbage.i == __garbage.n)) {
-    p2 = __garbage.p;
-    n2 = __garbage.n + (__garbage.n >> 1);
-    if (__garbage.p != __garbage.initmem) {
-      if (!weaken(realloc)) return;
-      if (!(p2 = weaken(realloc)(p2, n2 * sizeof(*p2)))) return;
-    } else {
-      if (!weaken(malloc)) return;
-      if (!(p2 = weaken(malloc)(n2 * sizeof(*p2)))) return;
-      memcpy(p2, __garbage.p, __garbage.n * sizeof(*p2));
-      atexit(__garbage_destroy);
-    }
-    __garbage.p = p2;
-    __garbage.n = n2;
-  }
-  __garbage.p[__garbage.i].frame = frame;
-  __garbage.p[__garbage.i].fn = (intptr_t)fn;
-  __garbage.p[__garbage.i].arg = (intptr_t)arg;
-  __garbage.p[__garbage.i].ret = frame->addr;
-  __garbage.i++;
-  frame->addr = (intptr_t)__gc;
-}
-
-/**
- * Adds destructor to garbage shadow stack.
- *
- * @param frame is passed automatically by wrapper macro
- * @param fn takes one argument
- * @param arg is passed to fn(arg)
- * @return arg
- */
-void __defer(struct StackFrame *frame, void *fn, void *arg) {
-  struct StackFrame *f;
-  if (!arg) return;
-  f = __builtin_frame_address(0);
-  assert(__garbage.n);
-  assert(f->next == frame);
-  assert(PointerNotOwnedByParentStackFrame(f, frame, arg));
-  __deferer(frame, fn, arg);
-}

libc/mem/gc.c

@@ -16,9 +16,93 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/assert.h"
+#include "libc/intrin/kprintf.h"
+#include "libc/intrin/likely.h"
+#include "libc/mem/mem.h"
+#include "libc/nexgen32e/gc.internal.h"
+#include "libc/nexgen32e/gettls.h"
 #include "libc/nexgen32e/stackframe.h"
-#include "libc/runtime/gc.h"
+#include "libc/nexgen32e/threaded.h"
 #include "libc/runtime/runtime.h"
+#include "libc/str/str.h"
+#include "libc/thread/thread.h"
+
+static inline bool PointerNotOwnedByParentStackFrame(struct StackFrame *frame,
+                                                     struct StackFrame *parent,
+                                                     void *ptr) {
+  return !(((intptr_t)ptr > (intptr_t)frame) &&
+           ((intptr_t)ptr < (intptr_t)parent));
+}
+
+static void TeardownGc(void) {
+  int i;
+  cthread_t tls;
+  struct Garbages *g;
+  if (__tls_enabled) {
+    tls = (cthread_t)__get_tls();
+    if ((g = tls->garbages)) {
+      // exit() currently doesn't use _gclongjmp() like pthread_exit()
+      // so we need to run the deferred functions manually.
+      while (g->i) {
+        --g->i;
+        ((void (*)(intptr_t))g->p[g->i].fn)(g->p[g->i].arg);
+      }
+      free(g->p);
+      free(g);
+    }
+  }
+}
+
+__attribute__((__constructor__)) static void InitializeGc(void) {
+  atexit(TeardownGc);
+}
+
+// add item to garbage shadow stack.
+// then rewrite caller's return address on stack.
+static void DeferFunction(struct StackFrame *frame, void *fn, void *arg) {
+  int n2;
+  cthread_t tls;
+  struct Garbage *p2;
+  struct Garbages *g;
+  TlsIsRequired();
+  tls = (cthread_t)__get_tls();
+  g = tls->garbages;
+  if (UNLIKELY(!g)) {
+    if (!(g = malloc(sizeof(struct Garbages)))) notpossible;
+    g->i = 0;
+    g->n = 4;
+    if (!(g->p = malloc(g->n * sizeof(struct Garbage)))) {
+      kprintf("malloc failed\n");
+      notpossible;
+    }
+    tls->garbages = g;
+  } else if (UNLIKELY(g->i == g->n)) {
+    p2 = g->p;
+    n2 = g->n + (g->n >> 1);
+    if ((p2 = realloc(p2, n2 * sizeof(*p2)))) {
+      g->p = p2;
+      g->n = n2;
+    } else {
+      notpossible;
+    }
+  }
+  g->p[g->i].frame = frame;
+  g->p[g->i].fn = (intptr_t)fn;
+  g->p[g->i].arg = (intptr_t)arg;
+  g->p[g->i].ret = frame->addr;
+  g->i++;
+  frame->addr = (intptr_t)__gc;
+}
+
+// the gnu extension macros for _gc / _defer point here
+void __defer(void *rbp, void *fn, void *arg) {
+  struct StackFrame *f, *frame = rbp;
+  f = __builtin_frame_address(0);
+  assert(f->next == frame);
+  assert(PointerNotOwnedByParentStackFrame(f, frame, arg));
+  DeferFunction(frame, fn, arg);
+}
 
 /**
  * Frees memory when function returns.
@@ -38,11 +122,12 @@
  * @warning do not realloc() with gc()'d pointer
  * @warning be careful about static keyword due to impact of inlining
  * @note you should use -fno-omit-frame-pointer
+ * @threadsafe
  */
 void *(_gc)(void *thing) {
   struct StackFrame *frame;
   frame = __builtin_frame_address(0);
-  __deferer(frame->next, _weakfree, thing);
+  DeferFunction(frame->next, _weakfree, thing);
   return thing;
 }
 
@@ -57,10 +142,11 @@ void *(_gc)(void *thing) {
  * @warning do not realloc() with gc()'d pointer
  * @warning be careful about static keyword due to impact of inlining
  * @note you should use -fno-omit-frame-pointer
+ * @threadsafe
  */
 void *(_defer)(void *fn, void *arg) {
   struct StackFrame *frame;
   frame = __builtin_frame_address(0);
-  __deferer(frame->next, fn, arg);
+  DeferFunction(frame->next, fn, arg);
   return arg;
 }

libc/nexgen32e/gc.S

@@ -20,8 +20,6 @@
 #include "libc/dce.h"
 #include "libc/notice.inc"
 
-#define INITIAL_CAPACITY 4
-
 	nop
 
 //	Invokes deferred function calls.
@@ -34,9 +32,12 @@
 //
 //	@param	rax,rdx,xmm0,xmm1,st0,st1 is return value
 //	@see	test/libc/runtime/gc_test.c
-__gc:	decq	__garbage(%rip)
-	mov	__garbage(%rip),%r8
-	mov	__garbage+16(%rip),%r9
+//	@threadsafe
+__gc:	mov	%fs:0,%rcx			# __get_tls()
+	mov	0x18(%rcx),%rcx			# cthread_t::garbages
+	decl	(%rcx)				# ++g->i
+	mov	(%rcx),%r8d			# r8 = g->i
+	mov	8(%rcx),%r9			# r9 = g->p
 	js	9f
 	shl	$5,%r8
 	lea	(%r9,%r8),%r8
@@ -55,25 +56,5 @@ __gc:	decq	__garbage(%rip)
 	mov	-8(%rbp),%rax
 	leave
 	ret
-9:	hlt
+9:	ud2
 	.endfn	__gc,globl,hidden
-
-	.bss
-	.align	8
-__garbage:
-	.quad	0				# garbage.i
-	.quad	0				# garbage.n
-	.quad	0				# garbage.p
-	.rept	INITIAL_CAPACITY
-	.quad	0				# garbage.p[𝑖].frame
-	.quad	0				# garbage.p[𝑖].fn
-	.quad	0				# garbage.p[𝑖].arg
-	.quad	0				# garbage.p[𝑖].ret
-	.endr
-	.endobj	__garbage,globl,hidden
-	.previous
-
-	.init.start 100,_init_garbage
-	movb	$INITIAL_CAPACITY,__garbage+8(%rip)
-	movl	$__garbage+24,__garbage+16(%rip)
-	.init.end 100,_init_garbage

libc/nexgen32e/gc.internal.h

@@ -12,13 +12,10 @@ struct Garbage {
 };
 
 struct Garbages {
-  size_t i, n;
+  int i, n;
   struct Garbage *p;
-  struct Garbage initmem[1];
 };
 
-extern struct Garbages __garbage;
-
 int64_t __gc(void);
 
 COSMOPOLITAN_C_END_

libc/nexgen32e/gclongjmp.S

@@ -28,29 +28,33 @@
 //	@param	esi is returned by setjmp() invocation (coerced nonzero)
 //	@assume	system five nexgen32e abi conformant
 //	@see	examples/ctrlc.c
+//	@threadsafe
 //	@noreturn
 _gclongjmp:
 	push	%rbp
 	mov	%rsp,%rbp
 	.profilable
-	lea	__garbage(%rip),%r12
-	mov	(%r12),%r13			# garbage.i
-	test	%r13,%r13
+	mov	%fs:0,%r12			# __get_tls()
+	mov	0x18(%r12),%r12			# cthread_t::garbages
+	test	%r12,%r12
+	jz	0f
+	movl	(%r12),%r13d			# garbages.i
+	test	%r13d,%r13d
 	jnz	.L.unwind.destructors
 0:	jmp	longjmp
 .L.unwind.destructors:
 	push	%rdi
 	push	%rsi
-	mov	16(%r12),%r14			# garbage.p
+	mov	8(%r12),%r14			# garbages.p
 	mov	(%rdi),%r15			# jmp_buf[0] is new %rsp
 	shl	$5,%r13				# log2(sizeof(struct Garbage))
 1:	sub	$32,%r13			# 𝑖--
 	js	2f
-	cmp	(%r14,%r13),%r15		# new %rsp > garbage.p[𝑖].frame
+	cmp	(%r14,%r13),%r15		# new %rsp > garbages.p[𝑖].frame
 	jbe	2f
-	mov	16(%r14,%r13),%rdi		# garbage.p[𝑖].arg
-	callq	*8(%r14,%r13)			# garbage.p[𝑖].fn
-	decq	(%r12)				# garbage.i--
+	mov	16(%r14,%r13),%rdi		# garbages.p[𝑖].arg
+	callq	*8(%r14,%r13)			# garbages.p[𝑖].fn
+	decl	(%r12)				# garbages.i--
 	jmp	1b
 2:	pop	%rsi
 	pop	%rdi

libc/nexgen32e/threaded.h

@@ -7,6 +7,7 @@ extern int __threaded;
 extern bool __tls_enabled;
 extern unsigned __tls_index;
 
+void TlsIsRequired(void);
 void *__initialize_tls(char[64]);
 void __install_tls(char[64]);
 

libc/runtime/enable_tls.c

@@ -230,7 +230,7 @@ privileged void __enable_tls(void) {
 
       // we're checking for the following expression:
       //   0144 == p[0] &&           // %fs
-      //   0110 == p[1] &&           // rex.w (64-bit operand size)
+      //   0110 == (p[1] & 0373) &&  // rex.w (and ignore rex.r)
       //   (0213 == p[2] ||          // mov reg/mem → reg (word-sized)
       //   0003 == p[2]) &&          // add reg/mem → reg (word-sized)
       //   0004 == (p[3] & 0307) &&  // mod/rm (4,reg,0) means sib → reg
@@ -239,7 +239,7 @@ privileged void __enable_tls(void) {
       //   0000 == p[6] &&           // displacement
       //   0000 == p[7] &&           // displacement
       //   0000 == p[8]              // displacement
-      w = READ64LE(p) & READ64LE("\377\377\377\307\377\377\377\377");
+      w = READ64LE(p) & READ64LE("\377\373\377\307\377\377\377\377");
       if ((w == READ64LE("\144\110\213\004\045\000\000\000") ||
            w == READ64LE("\144\110\003\004\045\000\000\000")) &&
           !p[8]) {

libc/runtime/fork-nt.c

@@ -29,6 +29,7 @@
 #include "libc/mem/alloca.h"
 #include "libc/mem/mem.h"
 #include "libc/nexgen32e/nt2sysv.h"
+#include "libc/nexgen32e/threaded.h"
 #include "libc/nt/console.h"
 #include "libc/nt/createfile.h"
 #include "libc/nt/enum/accessmask.h"
@@ -88,7 +89,6 @@ static inline textwindows ssize_t ForkIo(int64_t h, char *p, size_t n,
 static dontinline textwindows bool ForkIo2(int64_t h, void *buf, size_t n,
                                            bool32 (*fn)(), const char *sf) {
   ssize_t rc = ForkIo(h, buf, n, fn);
-  __tls_enabled = false;  // prevent tls crash in kprintf
   NTTRACE("%s(%ld, %p, %'zu) → %'zd% m", sf, h, buf, n, rc);
   return rc != -1;
 }
@@ -98,6 +98,7 @@ static dontinline textwindows bool WriteAll(int64_t h, void *buf, size_t n) {
 }
 
 static textwindows dontinline void ReadOrDie(int64_t h, void *buf, size_t n) {
+  __tls_enabled = false;  // prevent tls crash in kprintf
   if (!ForkIo2(h, buf, n, ReadFile, "ReadFile")) {
     AbortFork("ReadFile");
   }

libc/runtime/gc.h

@@ -1,27 +1,24 @@
 #ifndef COSMOPOLITAN_LIBC_RUNTIME_GC_H_
 #define COSMOPOLITAN_LIBC_RUNTIME_GC_H_
-#include "libc/calls/calls.h"
-#include "libc/nexgen32e/stackframe.h"
-#include "libc/runtime/runtime.h"
 #if !(__ASSEMBLER__ + __LINKER__ + 0)
 COSMOPOLITAN_C_START_
 
-void *_gc(void *) hidden;
-void *_defer(void *, void *) hidden;
-void __defer(struct StackFrame *, void *, void *) hidden;
-void __deferer(struct StackFrame *, void *, void *) hidden;
-void _gclongjmp(jmp_buf, int) dontthrow wontreturn;
+void *_gc(void *);
+void *_defer(void *, void *);
+void __defer(void *, void *, void *);
+void _gclongjmp(void *, int) dontthrow wontreturn;
+void _gc_free(void *);
 
 #if defined(__GNUC__) && !defined(__STRICT_ANSI__)
-#define _gc(THING) _defer((void *)_weakfree, (void *)(THING))
-#define _defer(FN, ARG)                                                \
-  ({                                                                   \
-    autotype(ARG) Arg = (ARG);                                         \
-    /* prevent weird opts like tail call */                            \
-    asm volatile("" : "+g"(Arg) : : "memory");                         \
-    __defer((struct StackFrame *)__builtin_frame_address(0), FN, Arg); \
-    asm volatile("" : "+g"(Arg) : : "memory");                         \
-    Arg;                                                               \
+#define _gc(THING) _defer((void *)_gc_free, (void *)(THING))
+#define _defer(FN, ARG)                           \
+  ({                                              \
+    autotype(ARG) Arg = (ARG);                    \
+    /* prevent weird opts like tail call */       \
+    asm volatile("" : "+g"(Arg) : : "memory");    \
+    __defer(__builtin_frame_address(0), FN, Arg); \
+    asm volatile("" : "+g"(Arg) : : "memory");    \
+    Arg;                                          \
   })
 #endif /* defined(__GNUC__) && !defined(__STRICT_ANSI__) */
 

libc/thread/pthread_create.c

@@ -16,6 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/assert.h"
 #include "libc/calls/calls.h"
 #include "libc/errno.h"
 #include "libc/intrin/asan.internal.h"
@@ -24,7 +25,9 @@
 #include "libc/intrin/wait0.internal.h"
 #include "libc/intrin/weaken.h"
 #include "libc/mem/mem.h"
+#include "libc/nexgen32e/gc.internal.h"
 #include "libc/nexgen32e/gettls.h"
+#include "libc/nexgen32e/threaded.h"
 #include "libc/runtime/runtime.h"
 #include "libc/sysv/consts/clone.h"
 #include "libc/sysv/consts/map.h"
@@ -56,6 +59,7 @@ static int PosixThread(void *arg, int tid) {
   if (weaken(_pthread_key_destruct)) {
     weaken(_pthread_key_destruct)(0);
   }
+  cthread_ungarbage();
   if (atomic_load_explicit(&pt->status, memory_order_acquire) ==
       kPosixThreadDetached) {
     atomic_store_explicit(&pt->status, kPosixThreadZombie,
@@ -108,6 +112,7 @@ int pthread_create(pthread_t *thread, const pthread_attr_t *attr,
   int rc, e = errno;
   struct PosixThread *pt;
   pthread_attr_t default_attr;
+  TlsIsRequired();
   _pthread_zombies_decimate();
 
   // default attributes

libc/thread/pthread_exit.c

@@ -18,6 +18,7 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/intrin/pthread.h"
 #include "libc/nexgen32e/gettls.h"
+#include "libc/runtime/gc.h"
 #include "libc/thread/posixthread.internal.h"
 #include "libc/thread/spawn.h"
 #include "libc/thread/thread.h"
@@ -38,7 +39,7 @@ wontreturn void pthread_exit(void *rc) {
   struct PosixThread *pt;
   if ((pt = ((cthread_t)__get_tls())->pthread)) {
     pt->rc = rc;
-    longjmp(pt->exiter, 1);
+    _gclongjmp(pt->exiter, 1);
   } else {
     _Exit1((int)(intptr_t)rc);
   }

libc/thread/spawn.c

@@ -30,6 +30,7 @@
 #include "libc/sysv/consts/clone.h"
 #include "libc/sysv/consts/map.h"
 #include "libc/sysv/consts/prot.h"
+#include "libc/sysv/errfuns.h"
 #include "libc/thread/spawn.h"
 #include "libc/thread/thread.h"
 
@@ -54,6 +55,20 @@
 #define _TIBZ sizeof(struct cthread_descriptor_t)
 #define _MEMZ ROUNDUP(_TLSZ + _TIBZ, alignof(struct cthread_descriptor_t))
 
+struct spawner {
+  int (*fun)(void *, int);
+  void *arg;
+};
+
+static int Spawner(void *arg, int tid) {
+  int rc;
+  struct spawner *spawner = arg;
+  rc = spawner->fun(spawner->arg, tid);
+  cthread_ungarbage();
+  free(spawner);
+  return 0;
+}
+
 /**
  * Spawns thread, e.g.
  *
@@ -78,6 +93,9 @@
  */
 int _spawn(int fun(void *, int), void *arg, struct spawn *opt_out_thread) {
   struct spawn *th, ths;
+  struct spawner *spawner;
+  TlsIsRequired();
+  if (!fun) return einval();
 
   // we need to to clobber the output memory before calling clone, since
   // there's no guarantee clone() won't suspend the parent, and focus on
@@ -102,11 +120,14 @@ int _spawn(int fun(void *, int), void *arg, struct spawn *opt_out_thread) {
     return -1;
   }
 
-  if (clone(fun, th->stk, GetStackSize() - 16 /* openbsd:stackbound */,
+  spawner = malloc(sizeof(struct spawner));
+  spawner->fun = fun;
+  spawner->arg = arg;
+  if (clone(Spawner, th->stk, GetStackSize() - 16 /* openbsd:stackbound */,
             CLONE_VM | CLONE_THREAD | CLONE_FS | CLONE_FILES | CLONE_SIGHAND |
                 CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_SETTID |
                 CLONE_CHILD_CLEARTID,
-            arg, &th->ptid, th->tib, th->ctid) == -1) {
+            spawner, &th->ptid, th->tib, th->ctid) == -1) {
     _freestack(th->stk);
     free(th->tls);
     return -1;

libc/thread/thread.h

@@ -28,7 +28,7 @@ struct FtraceTls {  /* 16 */
 struct cthread_descriptor_t {
   struct cthread_descriptor_t *self;  /* 0x00 */
   struct FtraceTls ftrace;            /* 0x08 */
-  void *garbages;                     /* 0x10 */
+  void *garbages;                     /* 0x18 */
   locale_t locale;                    /* 0x20 */
   pthread_t pthread;                  /* 0x28 */
   struct cthread_descriptor_t *self2; /* 0x30 */
@@ -64,6 +64,7 @@ int cthread_sem_wait(cthread_sem_t *, int, const struct timespec *);
 int cthread_sem_signal(cthread_sem_t *);
 int cthread_memory_wait32(int *, int, const struct timespec *);
 int cthread_memory_wake32(int *, int);
+void cthread_ungarbage(void);
 
 COSMOPOLITAN_C_END_
 #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

libc/thread/ungarbage.c

@@ -1,7 +1,7 @@
 /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
 │vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
 ╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2021 Justine Alexandra Roberts Tunney                              │
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
 │                                                                              │
 │ Permission to use, copy, modify, and/or distribute this software for         │
 │ any purpose with or without fee is hereby granted, provided that the         │
@@ -16,27 +16,19 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/log/log.h"
+#include "libc/assert.h"
+#include "libc/mem/mem.h"
 #include "libc/nexgen32e/gc.internal.h"
-#include "libc/stdio/stdio.h"
-/* clang-format off */
+#include "libc/nexgen32e/gettls.h"
+#include "libc/thread/thread.h"
 
-/**
- * Prints list of deferred operations on shadow stack w/o symbols.
- */
-void PrintGarbageNumeric(FILE *f) {
-  size_t i;
-  f = stderr;
-  fprintf(f, "\n");
-  fprintf(f, "                          SHADOW STACK @ 0x%016lx\n", __builtin_frame_address(0));
-  fprintf(f, " garbage entry  parent frame     original ret        callback              arg        \n");
-  fprintf(f, "-------------- -------------- ------------------ ------------------ ------------------\n");
-  for (i = __garbage.i; i--;) {
-    fprintf(f, "0x%012lx 0x%012lx 0x%016lx 0x%016lx 0x%016lx\n",
-            __garbage.p + i,
-            __garbage.p[i].frame,
-            __garbage.p[i].ret,
-            __garbage.p[i].fn,
-            __garbage.p[i].arg);
+void cthread_ungarbage(void) {
+  struct Garbages *g;
+  if ((g = ((cthread_t)__get_tls())->garbages)) {
+    // _pthread_exit() uses _gclongjmp() so if this assertion fails,
+    // then the likely cause is the thread used gc() with longjmp().
+    assert(!g->i);
+    free(g->p);
+    free(g);
   }
 }

test/libc/calls/open_test.c

@@ -16,6 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/calls.h"
 #include "libc/calls/internal.h"
 #include "libc/dce.h"
 #include "libc/errno.h"

test/libc/nexgen32e/gclongjmp_test.c

@@ -20,12 +20,14 @@
 #include "libc/mem/mem.h"
 #include "libc/nexgen32e/gc.internal.h"
 #include "libc/nexgen32e/nexgen32e.h"
+#include "libc/runtime/gc.h"
 #include "libc/runtime/gc.internal.h"
 #include "libc/runtime/runtime.h"
 #include "libc/stdio/stdio.h"
 #include "libc/str/str.h"
 #include "libc/testlib/ezbench.h"
 #include "libc/testlib/testlib.h"
+#include "libc/thread/spawn.h"
 #include "libc/x/x.h"
 
 #define GC(x) _defer(Free, x)
@@ -77,6 +79,45 @@ TEST(gclongjmp, test) {
   free(x);
 }
 
+void crawl(const char *path) {
+  const char *dir;
+  if (!strcmp(path, "/") || !strcmp(path, ".")) return;
+  crawl(_gc(xdirname(path)));
+}
+
+int Worker(void *arg, int tid) {
+  crawl("a/b/c/d/a/b/c/d/a/b/c/d/a/b/c/d/a/b/c/d/a/b/c/d");
+  return 0;
+}
+
+TEST(gc, torture) {
+  int i, n = 32;
+  struct spawn *t = gc(malloc(sizeof(struct spawn) * n));
+  for (i = 0; i < n; ++i) ASSERT_SYS(0, 0, _spawn(Worker, 0, t + i));
+  for (i = 0; i < n; ++i) EXPECT_SYS(0, 0, _join(t + i));
+}
+
+void crawl2(jmp_buf jb, const char *path) {
+  const char *dir;
+  if (!strcmp(path, "/") || !strcmp(path, ".")) _gclongjmp(jb, 1);
+  crawl2(jb, _gc(xdirname(path)));
+}
+
+int Worker2(void *arg, int tid) {
+  jmp_buf jb;
+  if (!setjmp(jb)) {
+    crawl2(jb, "a/b/c/d/a/b/c/d/a/b/c/d/a/b/c/d/a/b/c/d/a/b/c/d");
+  }
+  return 0;
+}
+
+TEST(_gclongjmp, torture) {
+  int i, n = 32;
+  struct spawn *t = gc(malloc(sizeof(struct spawn) * n));
+  for (i = 0; i < n; ++i) ASSERT_SYS(0, 0, _spawn(Worker2, 0, t + i));
+  for (i = 0; i < n; ++i) EXPECT_SYS(0, 0, _join(t + i));
+}
+
 dontinline void F1(void) {
   /* 3x slower than F2() but sooo worth it */
   gc(malloc(16));

test/libc/nexgen32e/test.mk

@@ -36,6 +36,7 @@ TEST_LIBC_NEXGEN32E_DIRECTDEPS =				\
 	LIBC_STR						\
 	LIBC_STUBS						\
 	LIBC_SYSV						\
+	LIBC_THREAD						\
 	LIBC_TESTLIB						\
 	LIBC_X							\
 	TOOL_VIZ_LIB						\

test/libc/sock/sendrecvmsg_test.c

@@ -17,6 +17,7 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/assert.h"
+#include "libc/calls/calls.h"
 #include "libc/calls/struct/iovec.h"
 #include "libc/dce.h"
 #include "libc/errno.h"

test/libc/sock/test.mk

@@ -34,6 +34,7 @@ TEST_LIBC_SOCK_DIRECTDEPS =					\
 	LIBC_STR						\
 	LIBC_STUBS						\
 	LIBC_SYSV						\
+	LIBC_LOG						\
 	LIBC_SYSV_CALLS						\
 	LIBC_TESTLIB						\
 	LIBC_X							\

test/libc/stdio/fgetln_test.c

@@ -16,6 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/calls.h"
 #include "libc/mem/mem.h"
 #include "libc/runtime/gc.internal.h"
 #include "libc/stdio/stdio.h"

third_party/quickjs/call.c

@@ -25,6 +25,7 @@
 #include "libc/assert.h"
 #include "libc/mem/mem.h"
 #include "libc/runtime/gc.internal.h"
+#include "libc/runtime/runtime.h"
 #include "third_party/quickjs/internal.h"
 
 asm(".ident\t\"\\n\\n\

third_party/smallz4/smallz4cat.c

@@ -29,6 +29,7 @@
 #include "libc/calls/calls.h"
 #include "libc/mem/mem.h"
 #include "libc/runtime/gc.internal.h"
+#include "libc/runtime/runtime.h"
 #include "libc/stdio/stdio.h"
 #include "libc/str/str.h"
 

third_party/tidy/config.c

@@ -18,6 +18,7 @@
 #include "libc/calls/calls.h"
 #include "libc/runtime/gc.internal.h"
 #include "libc/mem/mem.h"
+#include "libc/runtime/runtime.h"
 #include "third_party/tidy/tags.h"
 
 #ifdef WINDOWS_OS

tool/build/lib/elfwriter.c

@@ -17,6 +17,7 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/assert.h"
+#include "libc/calls/calls.h"
 #include "libc/elf/def.h"
 #include "libc/log/check.h"
 #include "libc/mem/arraylist2.internal.h"

tool/build/lib/psk.c

@@ -16,10 +16,12 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/calls.h"
 #include "libc/calls/struct/stat.h"
 #include "libc/log/check.h"
 #include "libc/mem/mem.h"
 #include "libc/runtime/gc.internal.h"
+#include "libc/runtime/runtime.h"
 #include "libc/sysv/consts/o.h"
 #include "libc/x/x.h"
 #include "tool/build/lib/psk.h"

tool/hash/hash.mk

@@ -22,6 +22,7 @@ TOOL_HASH_DIRECTDEPS =				\
 	LIBC_RUNTIME				\
 	LIBC_STDIO				\
 	LIBC_STR				\
+	LIBC_MEM				\
 	LIBC_STUBS
 
 TOOL_HASH_DEPS :=				\

tool/viz/printimage.c

@@ -21,6 +21,7 @@
 #include "dsp/scale/scale.h"
 #include "dsp/tty/quant.h"
 #include "dsp/tty/tty.h"
+#include "libc/calls/calls.h"
 #include "libc/calls/ioctl.h"
 #include "libc/calls/struct/stat.h"
 #include "libc/calls/struct/winsize.h"