Make threads faster and more reliable

This change doubles the performance of thread spawning. That's thanks to
our new stack manager, which allows us to avoid zeroing stacks. It gives
us 15µs spawns rather than 30µs spawns on Linux. Also, pthread_exit() is
faster now, since it doesn't need to acquire the pthread GIL. On NetBSD,
that helps us avoid allocating too many semaphores. Even if that happens
we're now able to survive semaphores running out and even memory running
out, when allocating *NSYNC waiter objects. I found a lot more rare bugs
in the POSIX threads runtime that could cause things to crash, if you've
got dozens of threads all spawning and joining dozens of threads. I want
cosmo to be world class production worthy for 2025 so happy holidays all

third_party/dlmalloc/dlmalloc.c

@@ -45,7 +45,6 @@
 #define USE_LOCKS 2
 #define MALLOC_INSPECT_ALL 1
 #define ABORT_ON_ASSERT_FAILURE 0
-#define LOCK_AT_FORK 1
 #define NO_MALLOC_STATS 1
 
 #if IsModeDbg()

third_party/dlmalloc/init.inc

@@ -3,38 +3,38 @@
 #include "libc/nexgen32e/rdtsc.h"
 #include "libc/runtime/runtime.h"
 
-/* ---------------------------- setting mparams -------------------------- */
-
-#if LOCK_AT_FORK
-#if ONLY_MSPACES
-
 void dlmalloc_pre_fork(void) {
+#if ONLY_MSPACES
   mstate h;
   for (unsigned i = ARRAYLEN(g_heaps); i--;)
     if ((h = atomic_load_explicit(&g_heaps[i], memory_order_acquire)))
       ACQUIRE_LOCK(&h->mutex);
+#else
+  ACQUIRE_LOCK(&(gm)->mutex);
+#endif
 }
 
 void dlmalloc_post_fork_parent(void) {
+#if ONLY_MSPACES
   mstate h;
   for (unsigned i = 0; i < ARRAYLEN(g_heaps); ++i)
     if ((h = atomic_load_explicit(&g_heaps[i], memory_order_acquire)))
       RELEASE_LOCK(&h->mutex);
+#else
+  RELEASE_LOCK(&(gm)->mutex);
+#endif
 }
 
 void dlmalloc_post_fork_child(void) {
+#if ONLY_MSPACES
   mstate h;
   for (unsigned i = 0; i < ARRAYLEN(g_heaps); ++i)
     if ((h = atomic_load_explicit(&g_heaps[i], memory_order_acquire)))
-      (void)REFRESH_LOCK(&h->mutex);
-}
-
+      REFRESH_LOCK(&h->mutex);
 #else
-void dlmalloc_pre_fork(void)         { ACQUIRE_LOCK(&(gm)->mutex); }
-void dlmalloc_post_fork_parent(void) { RELEASE_LOCK(&(gm)->mutex); }
-void dlmalloc_post_fork_child(void)  { (void)REFRESH_LOCK(&(gm)->mutex); }
-#endif /* ONLY_MSPACES */
-#endif /* LOCK_AT_FORK */
+  REFRESH_LOCK(&(gm)->mutex);
+#endif
+}
 
 /* Initialize mparams */
 __attribute__((__constructor__(49))) int init_mparams(void) {

third_party/dlmalloc/platform.inc

@@ -151,10 +151,6 @@
   ========================================================================
 */
 
-#ifndef LOCK_AT_FORK
-#define LOCK_AT_FORK 0
-#endif
-
 /* ------------------- size_t and alignment properties -------------------- */
 
 /* The byte and bit size of a size_t */

third_party/nsync/common.c

@@ -40,6 +40,7 @@
 #include "third_party/nsync/mu_semaphore.h"
 #include "third_party/nsync/mu_semaphore.internal.h"
 #include "libc/intrin/kprintf.h"
+#include "libc/intrin/strace.h"
 #include "third_party/nsync/wait_s.internal.h"
 __static_yoink("nsync_notice");
 
@@ -179,10 +180,10 @@ static waiter *free_waiters_pop (void) {
 	return w;
 }
 
-static void free_waiters_populate (void) {
+static bool free_waiters_populate (void) {
 	int n;
 	if (IsNetbsd ()) {
-		// netbsd needs a real file descriptor per semaphore
+		// netbsd semaphores are file descriptors
 		n = 1;
 	} else {
 		n = __pagesize / sizeof(waiter);
@@ -192,14 +193,17 @@ static void free_waiters_populate (void) {
 				MAP_PRIVATE | MAP_ANONYMOUS,
 				-1, 0);
 	if (waiters == MAP_FAILED)
-		nsync_panic_ ("out of memory\n");
+		return false;
 	for (size_t i = 0; i < n; ++i) {
 		waiter *w = &waiters[i];
 		w->tag = WAITER_TAG;
 		w->nw.tag = NSYNC_WAITER_TAG;
 		if (!nsync_mu_semaphore_init (&w->sem)) {
-			if (!i)
-				nsync_panic_ ("out of semaphores\n");
+			if (!i) {
+				// netbsd can run out of semaphores
+				munmap (waiters, n * sizeof (waiter));
+				return false;
+			}
 			break;
 		}
 		w->nw.sem = &w->sem;
@@ -208,6 +212,7 @@ static void free_waiters_populate (void) {
 		dll_init (&w->same_condition);
 		free_waiters_push (w);
 	}
+	return true;
 }
 
 /* -------------------------------- */
@@ -232,11 +237,18 @@ void nsync_waiter_destroy (void *v) {
 waiter *nsync_waiter_new_ (void) {
 	waiter *w;
 	waiter *tw;
+	unsigned attempts = 0;
+	bool out_of_semaphores = false;
 	tw = waiter_for_thread;
 	w = tw;
 	if (w == NULL || (w->flags & (WAITER_RESERVED|WAITER_IN_USE)) != WAITER_RESERVED) {
-		while (!(w = free_waiters_pop ()))
-			free_waiters_populate ();
+		while (!(w = free_waiters_pop ())) {
+			if (!out_of_semaphores)
+				if (!free_waiters_populate ())
+					out_of_semaphores = true;
+			if (out_of_semaphores)
+				attempts = pthread_delay_np (&free_waiters, attempts);
+		}
 		if (tw == NULL) {
 			w->flags |= WAITER_RESERVED;
 			waiter_for_thread = w;

third_party/nsync/mu_semaphore_sem.c

@@ -33,7 +33,6 @@
 #include "third_party/nsync/time.h"
 #include "third_party/nsync/mu_semaphore.h"
 #include "libc/intrin/atomic.h"
-#include "libc/atomic.h"
 #include "third_party/nsync/time.h"
 
 /**
@@ -83,8 +82,9 @@ void nsync_mu_semaphore_sem_fork_child (void) {
 	for (f = atomic_load_explicit (&g_sems, memory_order_relaxed); f; f = f->next) {
 		int rc = sys_close (f->id);
 		STRACE ("close(%ld) → %d", f->id, rc);
-		ASSERT (nsync_mu_semaphore_sem_create (f));
 	}
+	for (f = atomic_load_explicit (&g_sems, memory_order_relaxed); f; f = f->next)
+		ASSERT (nsync_mu_semaphore_sem_create (f));
 }
 
 /* Initialize *s; the initial value is 0. */
@@ -92,7 +92,7 @@ bool nsync_mu_semaphore_init_sem (nsync_semaphore *s) {
 	struct sem *f = (struct sem *) s;
 	if (!nsync_mu_semaphore_sem_create (f))
 		return false;
-	sems_push(f);
+	sems_push (f);
 	return true;
 }