Introduce ctl::set and ctl::map

We now have a C++ red-black tree implementation that implements standard
template library compatible APIs while compiling 10x faster than libcxx.
It's not as beautiful as the red-black tree implementation in Plinko but
this will get the job done and the test proves it upholds all invariants

This change also restores CheckForMemoryLeaks() support and fixes a real
actual bug I discovered with Doug Lea's dlmalloc_inspect_all() function.

libc/calls/fcntl-nt.c

@@ -28,11 +28,11 @@
 #include "libc/calls/wincrash.internal.h"
 #include "libc/errno.h"
 #include "libc/intrin/kprintf.h"
-#include "libc/intrin/leaky.internal.h"
 #include "libc/intrin/weaken.h"
 #include "libc/limits.h"
 #include "libc/log/backtrace.internal.h"
 #include "libc/macros.internal.h"
+#include "libc/mem/leaks.h"
 #include "libc/mem/mem.h"
 #include "libc/nt/createfile.h"
 #include "libc/nt/enum/fileflagandattributes.h"
@@ -75,7 +75,7 @@ static textwindows struct FileLock *NewFileLock(void) {
     fl = g_locks.free;
     g_locks.free = fl->next;
   } else {
-    unassert((fl = _weaken(malloc)(sizeof(*fl))));
+    unassert((fl = may_leak(_weaken(malloc)(sizeof(*fl)))));
   }
   bzero(fl, sizeof(*fl));
   fl->next = g_locks.list;
@@ -83,8 +83,6 @@ static textwindows struct FileLock *NewFileLock(void) {
   return fl;
 }
 
-IGNORE_LEAKS(NewFileLock)
-
 static textwindows void FreeFileLock(struct FileLock *fl) {
   fl->next = g_locks.free;
   g_locks.free = fl;

libc/intrin/leaky.internal.h

@@ -1,12 +0,0 @@
-#ifndef COSMOPOLITAN_LIBC_INTRIN_LEAKY_INTERNAL_H_
-#define COSMOPOLITAN_LIBC_INTRIN_LEAKY_INTERNAL_H_
-#include "libc/dce.h"
-COSMOPOLITAN_C_START_
-
-#define IGNORE_LEAKS(FUNC)
-
-extern intptr_t _leaky_end[] __attribute__((__weak__));
-extern intptr_t _leaky_start[] __attribute__((__weak__));
-
-COSMOPOLITAN_C_END_
-#endif /* COSMOPOLITAN_LIBC_INTRIN_LEAKY_INTERNAL_H_ */

libc/isystem/cosmo.h

@@ -43,6 +43,7 @@
 #include "libc/intrin/weaken.h"
 #include "libc/mem/critbit0.h"
 #include "libc/mem/gc.h"
+#include "libc/mem/leaks.h"
 #include "libc/nexgen32e/rdtsc.h"
 #include "libc/nexgen32e/stackframe.h"
 #include "libc/nexgen32e/x86feature.h"

libc/log/showcrashreports.c

@@ -20,7 +20,6 @@
 #include "libc/calls/struct/sigaction.h"
 #include "libc/calls/struct/sigaltstack.h"
 #include "libc/calls/struct/sigset.h"
-#include "libc/intrin/leaky.internal.h"
 #include "libc/log/internal.h"
 #include "libc/mem/mem.h"
 #include "libc/runtime/runtime.h"
@@ -93,5 +92,3 @@ void ShowCrashReports(void) {
   InstallCrashHandler(SIGABRT, 0);
   InstallCrashHandler(SIGSEGV, SA_ONSTACK);
 }
-
-IGNORE_LEAKS(ShowCrashReports)

libc/mem/leaks.c

@@ -16,10 +16,79 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/cxxabi.h"
+#include "libc/intrin/cxaatexit.internal.h"
+#include "libc/intrin/dll.h"
+#include "libc/intrin/kprintf.h"
+#include "libc/intrin/weaken.h"
+#include "libc/mem/mem.h"
+#include "libc/nt/typedef/imagetlscallback.h"
+#include "libc/runtime/runtime.h"
+#include "libc/thread/posixthread.internal.h"
+#include "libc/thread/thread.h"
+
+#define LEAK_CONTAINER(e) DLL_CONTAINER(struct Leak, elem, e)
+
+struct Leak {
+  void *alloc;
+  struct Dll elem;
+};
+
+static int leak_count;
+static struct Dll *leaks;
+static struct Dll *freaks;
+static pthread_mutex_t lock;
+
+void __may_leak(void *alloc) {
+  if (!alloc)
+    return;
+  pthread_mutex_lock(&lock);
+  if (dll_is_empty(freaks)) {
+    int g = __granularity();
+    struct Leak *p = _mapanon(g);
+    int n = g / sizeof(struct Leak);
+    for (int i = 0; i < n; ++i) {
+      dll_init(&p[i].elem);
+      dll_make_first(&freaks, &p[i].elem);
+    }
+  }
+  struct Dll *e = dll_first(freaks);
+  LEAK_CONTAINER(e)->alloc = alloc;
+  dll_remove(&freaks, e);
+  dll_make_first(&leaks, e);
+  pthread_mutex_unlock(&lock);
+}
+
+static void visitor(void *start, void *end, size_t used_bytes, void *arg) {
+  if (!used_bytes)
+    return;
+  for (struct Dll *e = dll_first(leaks); e; e = dll_next(leaks, e))
+    if (start == LEAK_CONTAINER(e)->alloc)
+      return;
+  kprintf("error: leaked %'zu byte allocation at %p\n", used_bytes, start);
+  ++leak_count;
+}
 
 /**
  * Tests for memory leaks.
  */
 void CheckForMemoryLeaks(void) {
-  // TODO(jart): give me a new lease on life
+
+  // validate usage of this api
+  if (_weaken(_pthread_decimate))
+    _weaken(_pthread_decimate)();
+  if (!pthread_orphan_np())
+    kprintf("warning: called CheckForMemoryLeaks() from non-orphaned thread\n");
+
+  // call dynamic global destructors
+  __cxa_thread_finalize();
+  __cxa_finalize(0);
+
+  // check for leaks
+  malloc_inspect_all(visitor, 0);
+  if (leak_count) {
+    kprintf("loser: you forgot to call free %'d time%s\n", leak_count,
+            leak_count == 1 ? "" : "s");
+    _exit(73);
+  }
 }

libc/mem/leaks.h

@@ -0,0 +1,22 @@
+#ifndef COSMOPOLITAN_LIBC_MEM_LEAKS_H_
+#define COSMOPOLITAN_LIBC_MEM_LEAKS_H_
+#include "libc/intrin/weaken.h"
+COSMOPOLITAN_C_START_
+
+void CheckForMemoryLeaks(void) libcesque;
+
+/**
+ * Declares that allocation needn't be freed.
+ *
+ * This function does nothing if CheckForMemoryLeaks() hasn't been
+ * linked into the binary.
+ */
+forceinline void *may_leak(void *__p) {
+  void __may_leak(void *) libcesque;
+  if (_weaken(__may_leak))
+    _weaken(__may_leak)(__p);
+  return __p;
+}
+
+COSMOPOLITAN_C_END_
+#endif /* COSMOPOLITAN_LIBC_MEM_LEAKS_H_ */

libc/mem/putenv.c

@@ -17,10 +17,10 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/intrin/getenv.internal.h"
-#include "libc/intrin/leaky.internal.h"
 #include "libc/intrin/strace.internal.h"
 #include "libc/macros.internal.h"
 #include "libc/mem/internal.h"
+#include "libc/mem/leaks.h"
 #include "libc/mem/mem.h"
 #include "libc/runtime/runtime.h"
 #include "libc/sysv/errfuns.h"
@@ -42,7 +42,7 @@ static char **__growenv(char **a) {
     a = environ;
   n = a ? __lenenv(a) : 0;
   c = MAX(8ul, n) << 1;
-  if ((b = malloc(c * sizeof(char *)))) {
+  if ((b = may_leak(malloc(c * sizeof(char *))))) {
     if (a) {
       for (p = b; *a;) {
         *p++ = *a++;
@@ -59,8 +59,6 @@ static char **__growenv(char **a) {
   }
 }
 
-IGNORE_LEAKS(__growenv)
-
 int __putenv(char *s, bool overwrite) {
   char **p;
   struct Env e;

libc/mem/setenv.c

@@ -16,9 +16,9 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/intrin/leaky.internal.h"
 #include "libc/intrin/strace.internal.h"
 #include "libc/mem/internal.h"
+#include "libc/mem/leaks.h"
 #include "libc/mem/mem.h"
 #include "libc/runtime/runtime.h"
 #include "libc/str/str.h"
@@ -39,7 +39,8 @@ int setenv(const char *name, const char *value, int overwrite) {
   size_t n, m;
   if (!name || !*name || !value || strchr(name, '='))
     return einval();
-  if ((s = malloc((n = strlen(name)) + 1 + (m = strlen(value)) + 1))) {
+  if ((s = may_leak(
+           malloc((n = strlen(name)) + 1 + (m = strlen(value)) + 1)))) {
     memcpy(mempcpy(mempcpy(s, name, n), "=", 1), value, m + 1);
     rc = __putenv(s, overwrite);
   } else {
@@ -48,5 +49,3 @@ int setenv(const char *name, const char *value, int overwrite) {
   STRACE("setenv(%#s, %#s, %hhhd) → %d% m", name, value, overwrite, rc);
   return rc;
 }
-
-IGNORE_LEAKS(setenv)

libc/proc/proc.c

@@ -27,9 +27,9 @@
 #include "libc/errno.h"
 #include "libc/fmt/wintime.internal.h"
 #include "libc/intrin/dll.h"
-#include "libc/intrin/leaky.internal.h"
 #include "libc/intrin/strace.internal.h"
 #include "libc/intrin/weaken.h"
+#include "libc/mem/leaks.h"
 #include "libc/mem/mem.h"
 #include "libc/nt/accounting.h"
 #include "libc/nt/enum/processaccess.h"
@@ -283,7 +283,7 @@ textwindows struct Proc *__proc_new(void) {
     }
     if (!proc) {
       if (_weaken(malloc)) {
-        proc = _weaken(malloc)(sizeof(struct Proc));
+        proc = may_leak(_weaken(malloc)(sizeof(struct Proc)));
       } else {
         enomem();
         return 0;
@@ -297,8 +297,6 @@ textwindows struct Proc *__proc_new(void) {
   return proc;
 }
 
-IGNORE_LEAKS(__proc_new)
-
 /**
  * Adds process to active list.
  *

libc/runtime/cxa_thread_atexit.c

@@ -17,7 +17,12 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/intrin/cxaatexit.internal.h"
+#include "libc/intrin/weaken.h"
 #include "libc/mem/mem.h"
+#include "libc/nexgen32e/gc.internal.h"
+#include "libc/thread/posixthread.internal.h"
+#include "libc/thread/tls.h"
+#include "third_party/nsync/wait_s.internal.h"
 
 struct Dtor {
   void *fun;
@@ -27,18 +32,58 @@ struct Dtor {
 
 static _Thread_local struct Dtor *__cxa_thread_atexit_list;
 
+static void __cxa_thread_unkey(struct CosmoTib *tib) {
+  void *val;
+  int i, j, gotsome;
+  pthread_key_dtor dtor;
+  for (j = 0; j < PTHREAD_DESTRUCTOR_ITERATIONS; ++j) {
+    for (gotsome = i = 0; i < PTHREAD_KEYS_MAX; ++i) {
+      if ((val = tib->tib_keys[i]) &&
+          (dtor = atomic_load_explicit(_pthread_key_dtor + i,
+                                       memory_order_relaxed)) &&
+          dtor != (pthread_key_dtor)-1) {
+        gotsome = 1;
+        tib->tib_keys[i] = 0;
+        dtor(val);
+      }
+    }
+    if (!gotsome) {
+      break;
+    }
+  }
+}
+
+static void _pthread_ungarbage(struct CosmoTib *tib) {
+  struct Garbages *g;
+  while ((g = tib->tib_garbages)) {
+    tib->tib_garbages = 0;
+    while (g->i--) {
+      ((void (*)(intptr_t))g->p[g->i].fn)(g->p[g->i].arg);
+    }
+    _weaken(free)(g->p);
+    _weaken(free)(g);
+  }
+}
+
 void __cxa_thread_finalize(void) {
   struct Dtor *dtor;
   while ((dtor = __cxa_thread_atexit_list)) {
     __cxa_thread_atexit_list = dtor->next;
     ((void (*)(void *))dtor->fun)(dtor->arg);
-    free(dtor);
+    _weaken(free)(dtor);
   }
+  struct CosmoTib *tib = __get_tls();
+  __cxa_thread_unkey(tib);
+  if (tib->tib_nsync)
+    _weaken(nsync_waiter_destroy)(tib->tib_nsync);
+  _pthread_ungarbage(tib);
 }
 
 int __cxa_thread_atexit_impl(void *fun, void *arg, void *dso_symbol) {
   struct Dtor *dtor;
-  if (!(dtor = malloc(sizeof(struct Dtor))))
+  if (!_weaken(malloc))
+    return -1;
+  if (!(dtor = _weaken(malloc)(sizeof(struct Dtor))))
     return -1;
   dtor->fun = fun;
   dtor->arg = arg;

libc/runtime/exit.c

@@ -40,14 +40,10 @@ wontreturn void exit(int exitcode) {
   STRACE("exit(%d)", exitcode);
 
   // call thread local c++ object destructors
-  if (_weaken(__cxa_thread_finalize)) {
-    _weaken(__cxa_thread_finalize)();
-  }
+  __cxa_thread_finalize();
 
   // call atexit() and __cxa_atexit() destructors
-  if (_weaken(__cxa_finalize)) {
-    _weaken(__cxa_finalize)(NULL);
-  }
+  __cxa_finalize(NULL);
 
   // call __destructor__ and finiarray destructors
   const uintptr_t *p;

libc/runtime/runtime.h

@@ -109,7 +109,6 @@ void __paginate_file(int, const char *) libcesque;
 void _weakfree(void *) libcesque;
 void *_mapanon(size_t) attributeallocsize((1)) mallocesque libcesque;
 void *_mapshared(size_t) attributeallocsize((1)) mallocesque libcesque;
-void CheckForMemoryLeaks(void) libcesque;
 void CheckForFileLeaks(void) libcesque;
 void __enable_threads(void) libcesque;
 void __oom_hook(size_t) libcesque;

libc/testlib/testmain.c

@@ -33,6 +33,7 @@
 #include "libc/intrin/weaken.h"
 #include "libc/log/log.h"
 #include "libc/macros.internal.h"
+#include "libc/mem/leaks.h"
 #include "libc/mem/mem.h"
 #include "libc/nexgen32e/nexgen32e.h"
 #include "libc/runtime/runtime.h"

libc/thread/posixthread.internal.h

@@ -110,8 +110,6 @@ void _pthread_lock(void) libcesque;
 void _pthread_onfork_child(void) libcesque;
 void _pthread_onfork_parent(void) libcesque;
 void _pthread_onfork_prepare(void) libcesque;
-void _pthread_ungarbage(void) libcesque;
-void _pthread_unkey(struct CosmoTib *) libcesque;
 void _pthread_unlock(void) libcesque;
 void _pthread_unref(struct PosixThread *) libcesque;
 void _pthread_unwind(struct PosixThread *) libcesque;

libc/thread/pthread_atfork.c

@@ -23,7 +23,6 @@
 #include "libc/errno.h"
 #include "libc/intrin/atomic.h"
 #include "libc/intrin/dll.h"
-#include "libc/intrin/leaky.internal.h"
 #include "libc/intrin/strace.internal.h"
 #include "libc/macros.internal.h"
 #include "libc/proc/proc.internal.h"
@@ -100,5 +99,3 @@ int _pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) {
   rc = 0;
   return rc;
 }
-
-IGNORE_LEAKS(_pthread_atfork)

libc/thread/pthread_exit.c

@@ -44,27 +44,6 @@ void _pthread_unwind(struct PosixThread *pt) {
   }
 }
 
-void _pthread_unkey(struct CosmoTib *tib) {
-  void *val;
-  int i, j, gotsome;
-  pthread_key_dtor dtor;
-  for (j = 0; j < PTHREAD_DESTRUCTOR_ITERATIONS; ++j) {
-    for (gotsome = i = 0; i < PTHREAD_KEYS_MAX; ++i) {
-      if ((val = tib->tib_keys[i]) &&
-          (dtor = atomic_load_explicit(_pthread_key_dtor + i,
-                                       memory_order_relaxed)) &&
-          dtor != (pthread_key_dtor)-1) {
-        gotsome = 1;
-        tib->tib_keys[i] = 0;
-        dtor(val);
-      }
-    }
-    if (!gotsome) {
-      break;
-    }
-  }
-}
-
 /**
  * Terminates current POSIX thread.
  *
@@ -111,14 +90,7 @@ wontreturn void pthread_exit(void *rc) {
 
   // free resources
   _pthread_unwind(pt);
-  if (_weaken(__cxa_thread_finalize)) {
-    _weaken(__cxa_thread_finalize)();
-  }
-  _pthread_unkey(tib);
-  if (tib->tib_nsync) {
-    nsync_waiter_destroy(tib->tib_nsync);
-  }
-  _pthread_ungarbage();
+  __cxa_thread_finalize();
   _pthread_decimate();
 
   // run atexit handlers if orphaned thread

libc/thread/pthread_ungarbage.c

@@ -1,35 +0,0 @@
-/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
-│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
-╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
-│                                                                              │
-│ Permission to use, copy, modify, and/or distribute this software for         │
-│ any purpose with or without fee is hereby granted, provided that the         │
-│ above copyright notice and this permission notice appear in all copies.      │
-│                                                                              │
-│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
-│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
-│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
-│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
-│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
-│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
-│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
-│ PERFORMANCE OF THIS SOFTWARE.                                                │
-╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/mem/mem.h"
-#include "libc/nexgen32e/gc.internal.h"
-#include "libc/thread/tls.h"
-
-void _pthread_ungarbage(void) {
-  struct Garbages *g;
-  struct CosmoTib *tib;
-  tib = __get_tls();
-  while ((g = tib->tib_garbages)) {
-    tib->tib_garbages = 0;
-    while (g->i--) {
-      ((void (*)(intptr_t))g->p[g->i].fn)(g->p[g->i].arg);
-    }
-    free(g->p);
-    free(g);
-  }
-}