Fix fork waiter leak in nsync

This change fixes a bug where nsync waiter objects would leak. It'd mean
that long-running programs like runitd would run out of file descriptors
on NetBSD where waiter objects have ksem file descriptors. On other OSes
this bug is mostly harmless since the worst that can happen with a futex
is to leak a little bit of ram. The bug was caused because tib_nsync was
sneaking back in after the finalization code had cleared it. This change
refactors the thread exiting code to handle nsync teardown appropriately
and in making this change I found another issue, which is that user code
which is buggy, and tries to exit without joining joinable threads which
haven't been detached, would result in a deadlock. That doesn't sound so
bad, except the main thread is a joinable thread. So this deadlock would
be triggered in ways that put libc at fault. So we now auto-join threads
and libc will log a warning to --strace when that happens for any thread

third_party/nsync/common.c

@@ -17,21 +17,18 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/atomic.h"
 #include "libc/calls/calls.h"
-#include "libc/calls/syscall-sysv.internal.h"
+#include "libc/calls/calls.h"
 #include "libc/dce.h"
-#include "libc/intrin/directmap.h"
+#include "libc/fmt/itoa.h"
 #include "libc/intrin/dll.h"
-#include "libc/intrin/extend.h"
-#include "libc/nt/enum/filemapflags.h"
-#include "libc/nt/enum/pageflags.h"
-#include "libc/nt/memory.h"
-#include "libc/nt/runtime.h"
-#include "libc/runtime/memtrack.internal.h"
+#include "libc/intrin/kprintf.h"
+#include "libc/intrin/weaken.h"
 #include "libc/runtime/runtime.h"
 #include "libc/stdalign.h"
-#include "libc/stdalign.h"
+#include "libc/str/str.h"
 #include "libc/sysv/consts/map.h"
 #include "libc/sysv/consts/prot.h"
+#include "libc/thread/posixthread.internal.h"
 #include "libc/thread/thread.h"
 #include "libc/thread/tls.h"
 #include "third_party/nsync/atomic.h"
@@ -39,8 +36,7 @@
 #include "third_party/nsync/common.internal.h"
 #include "third_party/nsync/mu_semaphore.h"
 #include "third_party/nsync/mu_semaphore.internal.h"
-#include "libc/intrin/kprintf.h"
-#include "libc/intrin/strace.h"
+#include "libc/intrin/cxaatexit.h"
 #include "third_party/nsync/wait_s.internal.h"
 __static_yoink("nsync_notice");
 
@@ -139,6 +135,9 @@ waiter *nsync_dll_waiter_samecond_ (struct Dll *e) {
 
 /* -------------------------------- */
 
+// TODO(jart): enforce in dbg mode once off-by-one flake is fixed
+#define DETECT_WAITER_LEAKS 0
+
 #define MASQUE 0x00fffffffffffff8
 #define PTR(x) ((uintptr_t)(x) & MASQUE)
 #define TAG(x) ROL((uintptr_t)(x) & ~MASQUE, 8)
@@ -147,6 +146,54 @@ waiter *nsync_dll_waiter_samecond_ (struct Dll *e) {
 #define ROR(x, n) (((x) >> (n)) | ((x) << (64 - (n))))
 
 static atomic_uintptr_t free_waiters;
+static _Atomic(waiter *) all_waiters;
+
+#if DETECT_WAITER_LEAKS
+static atomic_int all_waiters_count;
+static atomic_int free_waiters_count;
+#endif
+
+static waiter *get_waiter_for_thread (void) {
+	return __get_tls()->tib_nsync;
+}
+
+static bool set_waiter_for_thread (waiter *w) {
+	__get_tls()->tib_nsync = w;
+	return (true);
+}
+
+#if DETECT_WAITER_LEAKS
+__attribute__((__destructor__)) static void reconcile_waiters (void) {
+	// we can't perform this check if using exit() with threads
+	if (!pthread_orphan_np ())
+		return;
+	waiter *w;
+	if ((w = get_waiter_for_thread ())) {
+		nsync_waiter_destroy_ (w);
+		set_waiter_for_thread (0);
+	}
+	if (all_waiters_count != free_waiters_count) {
+		char ibuf[2][12];
+		FormatInt32 (ibuf[0], all_waiters_count);
+		FormatInt32 (ibuf[1], free_waiters_count);
+		tinyprint (2, "error: nsync panic: all_waiter_count (",
+			   ibuf[0], ") != free_waiters_count (", ibuf[1],
+			   ")\n", NULL);
+		_Exit (156);
+	}
+}
+#endif
+
+static void all_waiters_push (waiter *w) {
+	w->next_all = atomic_load_explicit (&all_waiters, memory_order_relaxed);
+	while (!atomic_compare_exchange_weak_explicit (&all_waiters, &w->next_all, w,
+						       memory_order_acq_rel,
+						       memory_order_relaxed))
+		pthread_pause_np ();
+#if DETECT_WAITER_LEAKS
+	++all_waiters_count;
+#endif
+}
 
 static void free_waiters_push (waiter *w) {
 	uintptr_t tip;
@@ -154,14 +201,16 @@ static void free_waiters_push (waiter *w) {
 	tip = atomic_load_explicit (&free_waiters, memory_order_relaxed);
 	for (;;) {
 		w->next_free = (waiter *) PTR (tip);
-		if (atomic_compare_exchange_weak_explicit (&free_waiters,
-							   &tip,
+		if (atomic_compare_exchange_weak_explicit (&free_waiters, &tip,
 							   ABA (w, TAG (tip) + 1),
 							   memory_order_release,
 							   memory_order_relaxed))
 			break;
 		pthread_pause_np ();
 	}
+#if DETECT_WAITER_LEAKS
+	++free_waiters_count;
+#endif
 }
 
 static waiter *free_waiters_pop (void) {
@@ -169,15 +218,18 @@ static waiter *free_waiters_pop (void) {
 	uintptr_t tip;
 	tip = atomic_load_explicit (&free_waiters, memory_order_relaxed);
 	while ((w = (waiter *) PTR (tip))) {
-		if (atomic_compare_exchange_weak_explicit (&free_waiters,
-							   &tip,
+		if (atomic_compare_exchange_weak_explicit (&free_waiters, &tip,
 							   ABA (w->next_free, TAG (tip) + 1),
 							   memory_order_acquire,
 							   memory_order_relaxed))
 			break;
 		pthread_pause_np ();
 	}
-	return w;
+#if DETECT_WAITER_LEAKS
+	if (w)
+		--free_waiters_count;
+#endif
+	return (w);
 }
 
 static bool free_waiters_populate (void) {
@@ -193,7 +245,7 @@ static bool free_waiters_populate (void) {
 				MAP_PRIVATE | MAP_ANONYMOUS,
 				-1, 0);
 	if (waiters == MAP_FAILED)
-		return false;
+		return (false);
 	for (size_t i = 0; i < n; ++i) {
 		waiter *w = &waiters[i];
 		w->tag = WAITER_TAG;
@@ -202,7 +254,7 @@ static bool free_waiters_populate (void) {
 			if (!i) {
 				// netbsd can run out of semaphores
 				munmap (waiters, n * sizeof (waiter));
-				return false;
+				return (false);
 			}
 			break;
 		}
@@ -211,47 +263,31 @@ static bool free_waiters_populate (void) {
 		w->nw.flags = NSYNC_WAITER_FLAG_MUCV;
 		dll_init (&w->same_condition);
 		free_waiters_push (w);
+		all_waiters_push (w);
 	}
-	return true;
+	return (true);
 }
 
 /* -------------------------------- */
 
-#define waiter_for_thread __get_tls()->tib_nsync
-
-void nsync_waiter_destroy (void *v) {
-	waiter *w = (waiter *) v;
-	/* Reset waiter_for_thread in case another thread-local variable reuses
-	   the waiter in its destructor while the waiter is taken by the other
-	   thread from free_waiters. This can happen as the destruction order
-	   of thread-local variables can be arbitrary in some platform e.g.
-	   POSIX. */
-	waiter_for_thread = NULL;
-	ASSERT ((w->flags & (WAITER_RESERVED|WAITER_IN_USE)) == WAITER_RESERVED);
-	w->flags &= ~WAITER_RESERVED;
-	free_waiters_push (w);
-}
-
 /* Return a pointer to an unused waiter struct.
    Ensures that the enclosed timer is stopped and its channel drained. */
 waiter *nsync_waiter_new_ (void) {
 	waiter *w;
 	waiter *tw;
-	unsigned attempts = 0;
 	bool out_of_semaphores = false;
-	tw = waiter_for_thread;
-	w = tw;
+	w = tw = get_waiter_for_thread ();
 	if (w == NULL || (w->flags & (WAITER_RESERVED|WAITER_IN_USE)) != WAITER_RESERVED) {
 		while (!(w = free_waiters_pop ())) {
 			if (!out_of_semaphores)
 				if (!free_waiters_populate ())
 					out_of_semaphores = true;
 			if (out_of_semaphores)
-				attempts = pthread_delay_np (&free_waiters, attempts);
+				pthread_yield_np ();
 		}
 		if (tw == NULL) {
-			w->flags |= WAITER_RESERVED;
-			waiter_for_thread = w;
+			if (set_waiter_for_thread (w))
+				w->flags |= WAITER_RESERVED;
 		}
 	}
 	w->flags |= WAITER_IN_USE;
@@ -261,14 +297,67 @@ waiter *nsync_waiter_new_ (void) {
 /* Return an unused waiter struct *w to the free pool. */
 void nsync_waiter_free_ (waiter *w) {
 	ASSERT ((w->flags & WAITER_IN_USE) != 0);
+	w->wipe_mu = NULL;
+	w->wipe_cv = NULL;
 	w->flags &= ~WAITER_IN_USE;
 	if ((w->flags & WAITER_RESERVED) == 0) {
+		if (w == get_waiter_for_thread ())
+			set_waiter_for_thread (0);
 		free_waiters_push (w);
-		if (w == waiter_for_thread)
-			waiter_for_thread = 0;
 	}
 }
 
+/* Destroys waiter associated with dead thread. */
+void nsync_waiter_destroy_ (void *v) {
+	waiter *w = (waiter *) v;
+	ASSERT ((w->flags & (WAITER_RESERVED|WAITER_IN_USE)) == WAITER_RESERVED);
+	w->flags &= ~WAITER_RESERVED;
+	free_waiters_push (w);
+}
+
+/* Ravages nsync waiters/locks/conds after fork(). */
+void nsync_waiter_wipe_ (void) {
+	int n = 0;
+	waiter *w;
+	waiter *next;
+	waiter *prev = 0;
+	waiter *wall = atomic_load_explicit (&all_waiters, memory_order_relaxed);
+	for (w = wall; w; w = w->next_all)
+		nsync_mu_semaphore_destroy (&w->sem);
+	for (w = wall; w; w = next) {
+		next = w->next_all;
+		w->tag = 0;
+		w->flags = 0;
+		w->nw.tag = 0;
+		w->nw.flags = NSYNC_WAITER_FLAG_MUCV;
+		atomic_init(&w->nw.waiting, 0);
+		w->l_type = 0;
+		bzero (&w->cond, sizeof (w->cond));
+		dll_init (&w->same_condition);
+		if (w->wipe_mu)
+			bzero (w->wipe_mu, sizeof (*w->wipe_mu));
+		if (w->wipe_cv)
+			bzero (w->wipe_cv, sizeof (*w->wipe_cv));
+		if (!nsync_mu_semaphore_init (&w->sem))
+			continue;  /* leak it */
+		w->next_free = prev;
+		w->next_all = prev;
+		prev = w;
+		++n;
+	}
+#if DETECT_WAITER_LEAKS
+	atomic_init (&all_waiters_count, n);
+	atomic_init (&free_waiters_count, n);
+#else
+	(void)n;
+#endif
+	atomic_init (&free_waiters, prev);
+	atomic_init (&all_waiters, prev);
+	for (struct Dll *e = dll_first (_pthread_list); e;
+	     e = dll_next (_pthread_list, e))
+		POSIXTHREAD_CONTAINER (e)->tib->tib_nsync = 0;
+}
+
 /* ====================================================================================== */
 
 /* writer_type points to a lock_type that describes how to manipulate a mu for a writer. */

third_party/nsync/common.internal.h

@@ -154,7 +154,7 @@ extern lock_type *nsync_reader_type_;
 
 /* ---------- */
 
-/* Hold a pair of  condition function and its argument. */
+/* Hold a pair of condition function and its argument. */
 struct wait_condition_s {
   int (*f)(const void *v);
   const void *v;
@@ -191,18 +191,19 @@ struct wait_condition_s {
     ATM_STORE_REL (&w.waiting, 0);
     nsync_mu_semaphore_v (&w.sem); */
 typedef struct waiter_s {
-  uint32_t tag; /* debug DLL_NSYNC_WAITER, DLL_WAITER, DLL_WAITER_SAMECOND */
-  int flags;    /* see WAITER_* bits below */
-  nsync_semaphore sem;       /* Thread waits on this semaphore. */
-  struct nsync_waiter_s nw;  /* An embedded nsync_waiter_s. */
-  struct nsync_mu_s_ *cv_mu; /* pointer to nsync_mu associated with a cv wait */
-  lock_type
-      *l_type; /* Lock type of the mu, or nil if not associated with a mu. */
-  nsync_atomic_uint32_ remove_count; /* count of removals from queue */
+  uint32_t tag;                      /* Debug DLL_NSYNC_WAITER, DLL_WAITER, DLL_WAITER_SAMECOND. */
+  int flags;                         /* See WAITER_* bits below. */
+  nsync_semaphore sem;               /* Thread waits on this semaphore. */
+  struct nsync_waiter_s nw;          /* An embedded nsync_waiter_s. */
+  struct nsync_mu_s_ *cv_mu;         /* Pointer to nsync_mu associated with a cv wait. */
+  lock_type *l_type;                 /* Lock type of the mu, or nil if not associated with a mu. */
+  nsync_atomic_uint32_ remove_count; /* Monotonic count of removals from queue. */
   struct wait_condition_s cond;      /* A condition on which to acquire a mu. */
-  struct Dll same_condition;         /* Links neighbours in nw.q with same
-                                        non-nil condition. */
+  struct Dll same_condition;         /* Links neighbours in nw.q with same non-nil condition. */
+  struct waiter_s * next_all;
   struct waiter_s * next_free;
+  struct nsync_mu_s_ *wipe_mu;
+  struct nsync_cv_s_ *wipe_cv;
 } waiter;
 static const uint32_t WAITER_TAG = 0x0590239f;
 static const uint32_t NSYNC_WAITER_TAG = 0x726d2ba9;

third_party/nsync/mem/nsync_cv.c

@@ -286,6 +286,8 @@ int nsync_cv_wait_with_deadline_generic (nsync_cv *pcv, void *pmu,
 	IGNORE_RACES_START ();
 
 	c.w = nsync_waiter_new_ ();
+	c.w->wipe_cv = pcv;
+	c.w->wipe_mu = pmu;
 	c.clock = clock;
 	c.abs_deadline = abs_deadline;
 	c.cancel_note = cancel_note;

third_party/nsync/mu.c

@@ -57,6 +57,7 @@ void nsync_mu_lock_slow_ (nsync_mu *mu, waiter *w, uint32_t clear, lock_type *l_
 	w->cond.f = NULL; /* Not using a conditional critical section. */
 	w->cond.v = NULL;
 	w->cond.eq = NULL;
+	w->wipe_mu = mu;
 	w->l_type = l_type;
 	zero_to_acquire = l_type->zero_to_acquire;
 	if (clear != 0) {
@@ -202,6 +203,7 @@ void nsync_mu_rlock (nsync_mu *mu) {
 		    !atomic_compare_exchange_strong_explicit (&mu->word, &old_word,
 							      (old_word+MU_RADD_TO_ACQUIRE) & ~MU_RCLEAR_ON_ACQUIRE,
 							      memory_order_acquire, memory_order_relaxed)) {
+			LOCKTRACE("acquiring nsync_mu_rlock(%t)...", mu);
 			waiter *w = nsync_waiter_new_ ();
 			nsync_mu_lock_slow_ (mu, w, 0, nsync_reader_type_);
 			nsync_waiter_free_ (w);

third_party/nsync/mu_semaphore.c

@@ -30,6 +30,15 @@ bool nsync_mu_semaphore_init (nsync_semaphore *s) {
 	}
 }
 
+/* Destroy *s. */
+void nsync_mu_semaphore_destroy (nsync_semaphore *s) {
+	if (IsNetbsd ()) {
+		return nsync_mu_semaphore_destroy_sem (s);
+	} else {
+		return nsync_mu_semaphore_destroy_futex (s);
+	}
+}
+
 /* Wait until the count of *s exceeds 0, and decrement it. If POSIX cancellations
    are currently disabled by the thread, then this function always succeeds. When
    they're enabled in MASKED mode, this function may return ECANCELED. Otherwise,

third_party/nsync/mu_semaphore.h

@@ -10,6 +10,9 @@ typedef struct nsync_semaphore_s_ {
 /* Initialize *s; the initial value is 0. */
 bool nsync_mu_semaphore_init(nsync_semaphore *s);
 
+/* Destroy *s. */
+void nsync_mu_semaphore_destroy(nsync_semaphore *s);
+
 /* Wait until the count of *s exceeds 0, and decrement it. */
 errno_t nsync_mu_semaphore_p(nsync_semaphore *s);
 

third_party/nsync/mu_semaphore.internal.h

@@ -5,19 +5,16 @@
 COSMOPOLITAN_C_START_
 
 bool nsync_mu_semaphore_init_futex(nsync_semaphore *);
+void nsync_mu_semaphore_destroy_futex(nsync_semaphore *);
 errno_t nsync_mu_semaphore_p_futex(nsync_semaphore *);
 errno_t nsync_mu_semaphore_p_with_deadline_futex(nsync_semaphore *, int, nsync_time);
 void nsync_mu_semaphore_v_futex(nsync_semaphore *);
 
 bool nsync_mu_semaphore_init_sem(nsync_semaphore *);
+void nsync_mu_semaphore_destroy_sem(nsync_semaphore *);
 errno_t nsync_mu_semaphore_p_sem(nsync_semaphore *);
 errno_t nsync_mu_semaphore_p_with_deadline_sem(nsync_semaphore *, int, nsync_time);
 void nsync_mu_semaphore_v_sem(nsync_semaphore *);
 
-bool nsync_mu_semaphore_init_gcd(nsync_semaphore *);
-errno_t nsync_mu_semaphore_p_gcd(nsync_semaphore *);
-errno_t nsync_mu_semaphore_p_with_deadline_gcd(nsync_semaphore *, int, nsync_time);
-void nsync_mu_semaphore_v_gcd(nsync_semaphore *);
-
 COSMOPOLITAN_C_END_
 #endif /* COSMOPOLITAN_THIRD_PARTY_NSYNC_MU_SEMAPHORE_INTERNAL_H_ */

third_party/nsync/mu_semaphore_futex.c

@@ -51,6 +51,9 @@ bool nsync_mu_semaphore_init_futex (nsync_semaphore *s) {
 	return true;
 }
 
+void nsync_mu_semaphore_destroy_futex (nsync_semaphore *s) {
+}
+
 /* Wait until the count of *s exceeds 0, and decrement it. If POSIX cancellations
    are currently disabled by the thread, then this function always succeeds. When
    they're enabled in MASKED mode, this function may return ECANCELED. Otherwise,

third_party/nsync/mu_semaphore_sem.c

@@ -43,23 +43,14 @@
 
 struct sem {
 	int64_t id;
-	struct sem *next;
 };
 
-static _Atomic(struct sem *) g_sems;
-
 static nsync_semaphore *sem_big_enough_for_sem = (nsync_semaphore *) (uintptr_t)(1 /
 	(sizeof (struct sem) <= sizeof (*sem_big_enough_for_sem)));
 
-static void sems_push (struct sem *f) {
-	f->next = atomic_load_explicit (&g_sems, memory_order_relaxed);
-	while (!atomic_compare_exchange_weak_explicit (&g_sems, &f->next, f,
-						       memory_order_acq_rel,
-						       memory_order_relaxed))
-		pthread_pause_np ();
-}
-
-static bool nsync_mu_semaphore_sem_create (struct sem *f) {
+/* Initialize *s; the initial value is 0. */
+bool nsync_mu_semaphore_init_sem (nsync_semaphore *s) {
+	struct sem *f = (struct sem *) s;
 	int rc;
 	int lol;
 	f->id = 0;
@@ -77,23 +68,10 @@ static bool nsync_mu_semaphore_sem_create (struct sem *f) {
 	return true;
 }
 
-void nsync_mu_semaphore_sem_fork_child (void) {
-	struct sem *f;
-	for (f = atomic_load_explicit (&g_sems, memory_order_relaxed); f; f = f->next) {
-		int rc = sys_close (f->id);
-		STRACE ("close(%ld) → %d", f->id, rc);
-	}
-	for (f = atomic_load_explicit (&g_sems, memory_order_relaxed); f; f = f->next)
-		ASSERT (nsync_mu_semaphore_sem_create (f));
-}
-
-/* Initialize *s; the initial value is 0. */
-bool nsync_mu_semaphore_init_sem (nsync_semaphore *s) {
+/* Destroys *s. */
+void nsync_mu_semaphore_destroy_sem (nsync_semaphore *s) {
 	struct sem *f = (struct sem *) s;
-	if (!nsync_mu_semaphore_sem_create (f))
-		return false;
-	sems_push (f);
-	return true;
+	sys_close (f->id);
 }
 
 /* Wait until the count of *s exceeds 0, and decrement it. If POSIX cancellations

third_party/nsync/wait_s.internal.h

@@ -20,7 +20,7 @@ struct nsync_waiter_s {
 /* set if waiter is embedded in Mu/CV's internal structures */
 #define NSYNC_WAITER_FLAG_MUCV 0x1
 
-void nsync_waiter_destroy(void *);
+void nsync_waiter_destroy_(void *);
 
 COSMOPOLITAN_C_END_
 #endif /* COSMOPOLITAN_LIBC_THREAD_WAIT_INTERNAL_H_ */