Get rid of kmalloc()

This changes *NSYNC to allocate waiters on the stack so our locks don't
need to depend on dynamic memory. This make our runtiem simpler, and it
also fixes bugs with thread cancellation support.

libc/calls/execve.c

@@ -71,7 +71,7 @@ int execve(const char *prog, char *const argv[], char *const envp[]) {
       rc = _weaken(sys_pledge_linux)(__execpromises, __pledge_mode);
     }
     if (!rc) {
-      if (_weaken(__zipos_parseuri) &&
+      if (0 && _weaken(__zipos_parseuri) &&
           (_weaken(__zipos_parseuri)(prog, &uri) != -1)) {
         rc = _weaken(__zipos_open)(&uri, O_RDONLY | O_CLOEXEC);
         if (rc != -1) {

libc/calls/fcntl-nt.c

@@ -25,12 +25,13 @@
 #include "libc/calls/syscall_support-nt.internal.h"
 #include "libc/calls/wincrash.internal.h"
 #include "libc/errno.h"
-#include "libc/intrin/kmalloc.h"
 #include "libc/intrin/kprintf.h"
+#include "libc/intrin/leaky.internal.h"
 #include "libc/intrin/weaken.h"
 #include "libc/limits.h"
 #include "libc/log/backtrace.internal.h"
 #include "libc/macros.internal.h"
+#include "libc/mem/mem.h"
 #include "libc/nt/createfile.h"
 #include "libc/nt/enum/fileflagandattributes.h"
 #include "libc/nt/enum/filelockflags.h"
@@ -72,7 +73,7 @@ static textwindows struct FileLock *NewFileLock(void) {
     fl = g_locks.free;
     g_locks.free = fl->next;
   } else {
-    fl = kmalloc(sizeof(*fl));
+    unassert((fl = _weaken(malloc)(sizeof(*fl))));
   }
   bzero(fl, sizeof(*fl));
   fl->next = g_locks.list;
@@ -80,6 +81,8 @@ static textwindows struct FileLock *NewFileLock(void) {
   return fl;
 }
 
+IGNORE_LEAKS(NewFileLock)
+
 static textwindows void FreeFileLock(struct FileLock *fl) {
   fl->next = g_locks.free;
   g_locks.free = fl;
@@ -129,6 +132,10 @@ static textwindows int sys_fcntl_nt_lock(struct Fd *f, int fd, int cmd,
   int64_t pos, off, len, end;
   struct FileLock *fl, *ft, **flp;
 
+  if (!_weaken(malloc)) {
+    return enomem();
+  }
+
   l = (struct flock *)arg;
   len = l->l_len;
   off = l->l_start;

libc/calls/ntspawn.c

@@ -87,8 +87,6 @@ textwindows int ntspawn(
   block = NULL;
   _init_sigchld();
   if (__mkntpath(prog, prog16) == -1) return -1;
-  // we can't call malloc() because we're higher in the topological order
-  // we can't call kmalloc() because fork() calls this when kmalloc is locked
   if ((handle = CreateFileMapping(-1, 0, pushpop(kNtPageReadwrite), 0,
                                   sizeof(*block), 0)) &&
       (block = MapViewOfFileEx(handle, kNtFileMapRead | kNtFileMapWrite, 0, 0,

libc/calls/syscall-sysv.internal.h

@@ -87,6 +87,7 @@ i32 sys_sem_close(i64);
 i32 sys_sem_destroy(i64);
 i32 sys_sem_getvalue(i64, u32 *);
 i32 sys_sem_init(u32, i64 *);
+i32 sys_sem_destroy(i64);
 i32 sys_sem_open(const char *, int, u32, i64 *);
 i32 sys_sem_post(i64);
 i32 sys_sem_trywait(i64);

libc/calls/tcsetattr-nt.c

@@ -24,6 +24,7 @@
 #include "libc/calls/termios.internal.h"
 #include "libc/calls/ttydefaults.h"
 #include "libc/dce.h"
+#include "libc/errno.h"
 #include "libc/intrin/describeflags.internal.h"
 #include "libc/intrin/nomultics.internal.h"
 #include "libc/intrin/strace.internal.h"
@@ -261,6 +262,7 @@ textwindows int tcsetattr_nt(int fd, int opt, const struct termios *tio) {
 __attribute__((__constructor__)) static void tcsetattr_nt_init(void) {
   if (!getenv("TERM")) {
     setenv("TERM", "xterm-256color", true);
+    errno = 0;  // ignore malloc not linked
   }
 }
 

libc/calls/tkill.c

@@ -98,16 +98,43 @@ static dontinline textwindows int __tkill_nt(int tid, int sig,
   }
 }
 
-static int __tkill_m1(int tid, int sig, struct CosmoTib *tib) {
-  struct PosixThread *pt = (struct PosixThread *)__get_tls()->tib_pthread;
-  return __syslib->pthread_kill(pt->next, sig);
+static int __tkill_posix(int tid, int sig, struct CosmoTib *tib) {
+
+  // avoid lock when killing self
+  int me;
+  struct PosixThread *pt;
+  pt = (struct PosixThread *)__get_tls()->tib_pthread;
+  me = atomic_load_explicit(&__get_tls()->tib_tid, memory_order_relaxed);
+  if (tid == me && (!tib || tib == __get_tls())) {
+    return __syslib->pthread_kill(pt->next, sig);
+  }
+
+  // otherwise look for matching thread
+  struct Dll *e;
+  pthread_spin_lock(&_pthread_lock);
+  for (e = dll_first(_pthread_list); e; e = dll_next(_pthread_list, e)) {
+    enum PosixThreadStatus status;
+    struct PosixThread *pt = POSIXTHREAD_CONTAINER(e);
+    int rhs = atomic_load_explicit(&pt->tib->tib_tid, memory_order_acquire);
+    if (rhs <= 0 || tid != rhs) continue;
+    if (tib && tib != pt->tib) continue;
+    status = atomic_load_explicit(&pt->status, memory_order_acquire);
+    pthread_spin_unlock(&_pthread_lock);
+    if (status < kPosixThreadTerminated) {
+      return __syslib->pthread_kill(pt->next, sig);
+    } else {
+      return 0;
+    }
+  }
+  pthread_spin_unlock(&_pthread_lock);
+  return esrch();
 }
 
 // OpenBSD has an optional `tib` parameter for extra safety.
 int __tkill(int tid, int sig, void *tib) {
   int rc;
   if (IsXnuSilicon()) {
-    return __tkill_m1(tid, sig, tib);
+    return __tkill_posix(tid, sig, tib);
   } else if (IsLinux() || IsXnu() || IsFreebsd() || IsOpenbsd() || IsNetbsd()) {
     rc = sys_tkill(tid, sig, tib);
   } else if (IsWindows()) {

libc/intrin/asan.c

@@ -27,7 +27,6 @@
 #include "libc/intrin/bits.h"
 #include "libc/intrin/cmpxchg.h"
 #include "libc/intrin/directmap.internal.h"
-#include "libc/intrin/kmalloc.h"
 #include "libc/intrin/kprintf.h"
 #include "libc/intrin/leaky.internal.h"
 #include "libc/intrin/likely.h"
@@ -835,10 +834,9 @@ static __wur __asan_die_f *__asan_report(const void *addr, int size,
   wint_t c;
   signed char t;
   uint64_t x, y, z;
-  char *p, *q, *buf, *base;
   struct MemoryIntervals *m;
+  char buf[8192], *base, *q, *p = buf;
   ftrace_enabled(-1);
-  p = buf = kmalloc(1024 * 1024);
   kprintf("\n\e[J\e[1;31masan error\e[0m: %s %d-byte %s at %p shadow %p\n",
           __asan_describe_access_poison(kind), size, message, addr,
           SHADOW(addr));

libc/intrin/describeframe.c

@@ -43,8 +43,6 @@ static const char *GetFrameName(int x) {
     return "g_fds";
   } else if (IsZiposFrame(x)) {
     return "zipos";
-  } else if (IsKmallocFrame(x)) {
-    return "kmalloc";
   } else if (IsMemtrackFrame(x)) {
     return "memtrack";
   } else if (IsOldStackFrame(x)) {

libc/intrin/kmalloc.c

@@ -1,95 +0,0 @@
-/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
-│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
-╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
-│                                                                              │
-│ Permission to use, copy, modify, and/or distribute this software for         │
-│ any purpose with or without fee is hereby granted, provided that the         │
-│ above copyright notice and this permission notice appear in all copies.      │
-│                                                                              │
-│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
-│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
-│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
-│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
-│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
-│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
-│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
-│ PERFORMANCE OF THIS SOFTWARE.                                                │
-╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/intrin/kmalloc.h"
-#include "libc/assert.h"
-#include "libc/atomic.h"
-#include "libc/calls/syscall-sysv.internal.h"
-#include "libc/dce.h"
-#include "libc/intrin/asan.internal.h"
-#include "libc/intrin/atomic.h"
-#include "libc/intrin/describebacktrace.internal.h"
-#include "libc/intrin/extend.internal.h"
-#include "libc/log/libfatal.internal.h"
-#include "libc/macros.internal.h"
-#include "libc/runtime/memtrack.internal.h"
-#include "libc/sysv/consts/map.h"
-#include "libc/thread/thread.h"
-#include "libc/thread/tls.h"
-
-#define KMALLOC_ALIGN sizeof(intptr_t)
-
-static struct {
-  char *endptr;
-  size_t total;
-  pthread_spinlock_t lock;
-} g_kmalloc;
-
-void __kmalloc_lock(void) {
-  pthread_spin_lock(&g_kmalloc.lock);
-}
-
-void __kmalloc_unlock(void) {
-  pthread_spin_unlock(&g_kmalloc.lock);
-}
-
-#ifdef _NOPL0
-#define __kmalloc_lock()   _NOPL0("__threadcalls", __kmalloc_lock)
-#define __kmalloc_unlock() _NOPL0("__threadcalls", __kmalloc_unlock)
-#endif
-
-/**
- * Allocates permanent memory.
- *
- * The code malloc() depends upon uses this function to allocate memory.
- * The returned memory can't be freed, and leak detection is impossible.
- * This function panics when memory isn't available.
- *
- * Memory returned by this function is aligned on the word size, and as
- * such, kmalloc() shouldn't be used for vector operations.
- *
- * @return zero-initialized memory on success, or null w/ errno
- * @raise ENOMEM if we require more vespene gas
- */
-dontasan void *kmalloc(size_t size) {
-  char *p, *e;
-  size_t i, n, t;
-  n = ROUNDUP(size + (IsAsan() * 8), KMALLOC_ALIGN);
-  __kmalloc_lock();
-  t = g_kmalloc.total;
-  e = g_kmalloc.endptr;
-  i = t;
-  t += n;
-  p = (char *)kMemtrackKmallocStart;
-  if (!e) e = p;
-  if ((e = _extend(p, t, e, MAP_PRIVATE,
-                   kMemtrackKmallocStart + kMemtrackKmallocSize))) {
-    g_kmalloc.endptr = e;
-    g_kmalloc.total = t;
-  } else {
-    p = 0;
-  }
-  __kmalloc_unlock();
-  if (p) {
-    unassert(!((intptr_t)(p + i) & (KMALLOC_ALIGN - 1)));
-    if (IsAsan()) __asan_poison(p + i + size, n - size, kAsanHeapOverrun);
-    return p + i;
-  } else {
-    return 0;
-  }
-}

libc/intrin/kmalloc.h

@@ -1,17 +0,0 @@
-#ifndef COSMOPOLITAN_LIBC_INTRIN_KMALLOC_H_
-#define COSMOPOLITAN_LIBC_INTRIN_KMALLOC_H_
-#ifdef _COSMO_SOURCE
-#define kmalloc __kmalloc
-#if !(__ASSEMBLER__ + __LINKER__ + 0)
-COSMOPOLITAN_C_START_
-
-void *kmalloc(size_t)
-mallocesque attributeallocsize((1)) returnsaligned((8));
-
-void __kmalloc_lock(void);
-void __kmalloc_unlock(void);
-
-COSMOPOLITAN_C_END_
-#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
-#endif /* _COSMO_SOURCE */
-#endif /* COSMOPOLITAN_LIBC_INTRIN_KMALLOC_H_ */

libc/intrin/leaky.internal.h

@@ -1,12 +1,17 @@
 #ifndef COSMOPOLITAN_LIBC_INTRIN_LEAKY_INTERNAL_H_
 #define COSMOPOLITAN_LIBC_INTRIN_LEAKY_INTERNAL_H_
+#include "libc/dce.h"
 #if !(__ASSEMBLER__ + __LINKER__ + 0)
 COSMOPOLITAN_C_START_
 
+#if IsAsan()
 #define IGNORE_LEAKS(FUNC)                                        \
-  __static_yoink("_leaky_start");                                   \
+  __static_yoink("_leaky_start");                                 \
   void *_leaky_##FUNC[] _Section(".piro.relo.sort.leaky.2." #FUNC \
-                                 ",\"aw\",@init_array #") = {FUNC}
+                                 ",\"aw\",@init_array #") = {FUNC};
+#else
+#define IGNORE_LEAKS(FUNC)
+#endif
 
 extern intptr_t _leaky_end[] __attribute__((__weak__));
 extern intptr_t _leaky_start[] __attribute__((__weak__));

libc/intrin/pthread_mutex_trylock.c

@@ -19,6 +19,7 @@
 #include "libc/calls/calls.h"
 #include "libc/errno.h"
 #include "libc/intrin/atomic.h"
+#include "libc/intrin/kprintf.h"
 #include "libc/intrin/weaken.h"
 #include "libc/thread/thread.h"
 #include "libc/thread/tls.h"

libc/intrin/putenv.c

@@ -17,11 +17,14 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/intrin/getenv.internal.h"
-#include "libc/intrin/kmalloc.h"
+#include "libc/intrin/leaky.internal.h"
 #include "libc/intrin/strace.internal.h"
+#include "libc/intrin/weaken.h"
 #include "libc/macros.internal.h"
 #include "libc/mem/internal.h"
+#include "libc/mem/mem.h"
 #include "libc/runtime/runtime.h"
+#include "libc/sysv/errfuns.h"
 
 #define ToUpper(c) ((c) >= 'a' && (c) <= 'z' ? (c) - 'a' + 'A' : (c))
 
@@ -40,7 +43,7 @@ static char **GrowEnviron(char **a) {
   if (!a) a = environ;
   n = a ? GetEnvironLen(a) : 0;
   c = MAX(16ul, n) << 1;
-  if ((b = kmalloc(c * sizeof(char *)))) {
+  if (_weaken(malloc) && (b = _weaken(malloc)(c * sizeof(char *)))) {
     if (a) {
       for (p = b; *a;) {
         *p++ = *a++;
@@ -51,10 +54,13 @@ static char **GrowEnviron(char **a) {
     capacity = c;
     return b;
   } else {
+    enomem();
     return 0;
   }
 }
 
+IGNORE_LEAKS(GrowEnviron)
+
 int PutEnvImpl(char *s, bool overwrite) {
   char **p;
   struct Env e;

libc/intrin/setenv.c

@@ -16,9 +16,11 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/intrin/kmalloc.h"
+#include "libc/intrin/leaky.internal.h"
 #include "libc/intrin/strace.internal.h"
+#include "libc/intrin/weaken.h"
 #include "libc/mem/internal.h"
+#include "libc/mem/mem.h"
 #include "libc/runtime/runtime.h"
 #include "libc/str/str.h"
 #include "libc/sysv/errfuns.h"
@@ -43,9 +45,13 @@ int setenv(const char *name, const char *value, int overwrite) {
   }
   n = strlen(name);
   m = strlen(value);
-  if (!(s = kmalloc(n + 1 + m + 1))) return -1;
+  if (!_weaken(malloc) || !(s = _weaken(malloc)(n + 1 + m + 1))) {
+    return enomem();
+  }
   memcpy(mempcpy(mempcpy(s, name, n), "=", 1), value, m + 1);
   rc = PutEnvImpl(s, overwrite);
   STRACE("setenv(%#s, %#s, %d) → %d% m", name, value, overwrite, rc);
   return rc;
 }
+
+IGNORE_LEAKS(setenv)

libc/log/oncrash_amd64.c

@@ -32,7 +32,6 @@
 #include "libc/intrin/atomic.h"
 #include "libc/intrin/describebacktrace.internal.h"
 #include "libc/intrin/describeflags.internal.h"
-#include "libc/intrin/kmalloc.h"
 #include "libc/intrin/kprintf.h"
 #include "libc/intrin/strace.internal.h"
 #include "libc/intrin/weaken.h"
@@ -207,9 +206,9 @@ void ShowCrashReportHook(int, int, int, struct siginfo *, ucontext_t *);
 relegated void ShowCrashReport(int err, int sig, struct siginfo *si,
                                ucontext_t *ctx) {
   int i;
-  size_t n;
+  char *p;
   char host[64];
-  char *p, *buf;
+  char buf[3000];
   struct utsname names;
   if (_weaken(ShowCrashReportHook)) {
     ShowCrashReportHook(2, err, sig, si, ctx);
@@ -223,9 +222,9 @@ relegated void ShowCrashReport(int err, int sig, struct siginfo *si,
   uname(&names);
   errno = err;
   // TODO(jart): Buffer the WHOLE crash report with backtrace for atomic write.
-  npassert((p = buf = kmalloc((n = 1024 * 1024))));
+  p = buf;
   p += ksnprintf(
-      p, n,
+      p, 10000,
       "\n%serror%s: Uncaught %G (%s) on %s pid %d tid %d\n"
       "  %s\n"
       "  %s\n"

libc/runtime/fork-nt.c

@@ -68,7 +68,6 @@
 extern int64_t __wincrashearly;
 bool32 __onntconsoleevent(uint32_t);
 void sys_setitimer_nt_reset(void);
-void kmalloc_unlock(void);
 
 static textwindows wontreturn void AbortFork(const char *func) {
 #ifdef SYSDEBUG

libc/runtime/memtrack.internal.h

@@ -12,19 +12,17 @@
 #if !(__ASSEMBLER__ + __LINKER__ + 0)
 COSMOPOLITAN_C_START_
 
-#define kAutomapStart         0x100080040000
-#define kAutomapSize          (kMemtrackStart - kAutomapStart)
-#define kMemtrackStart        0x1fe7fffc0000
-#define kMemtrackSize         (0x1ffffffc0000 - kMemtrackStart)
-#define kFixedmapStart        0x300000040000
-#define kFixedmapSize         (0x400000040000 - kFixedmapStart)
-#define kMemtrackFdsStart     0x6fe000040000
-#define kMemtrackFdsSize      (0x6feffffc0000 - kMemtrackFdsStart)
-#define kMemtrackZiposStart   0x6fd000040000
-#define kMemtrackZiposSize    (0x6fdffffc0000 - kMemtrackZiposStart)
-#define kMemtrackKmallocStart 0x6fc000040000
-#define kMemtrackKmallocSize  (0x6fcffffc0000 - kMemtrackKmallocStart)
-#define kMemtrackGran         (!IsAsan() ? FRAMESIZE : FRAMESIZE * 8)
+#define kAutomapStart       0x100080040000
+#define kAutomapSize        (kMemtrackStart - kAutomapStart)
+#define kMemtrackStart      0x1fe7fffc0000
+#define kMemtrackSize       (0x1ffffffc0000 - kMemtrackStart)
+#define kFixedmapStart      0x300000040000
+#define kFixedmapSize       (0x400000040000 - kFixedmapStart)
+#define kMemtrackFdsStart   0x6fe000040000
+#define kMemtrackFdsSize    (0x6feffffc0000 - kMemtrackFdsStart)
+#define kMemtrackZiposStart 0x6fd000040000
+#define kMemtrackZiposSize  (0x6fdffffc0000 - kMemtrackZiposStart)
+#define kMemtrackGran       (!IsAsan() ? FRAMESIZE : FRAMESIZE * 8)
 
 struct MemoryInterval {
   int x;
@@ -132,11 +130,6 @@ forceinline pureconst bool IsZiposFrame(int x) {
          x <= (int)((kMemtrackZiposStart + kMemtrackZiposSize - 1) >> 16);
 }
 
-forceinline pureconst bool IsKmallocFrame(int x) {
-  return (int)(kMemtrackKmallocStart >> 16) <= x &&
-         x <= (int)((kMemtrackKmallocStart + kMemtrackKmallocSize - 1) >> 16);
-}
-
 forceinline pureconst bool IsShadowFrame(int x) {
   return 0x7fff <= x && x < 0x10008000;
 }

libc/runtime/memtrack64.txt

@@ -1808,7 +1808,7 @@
 6f900000-6f9fffff 64gb   free
 6fa00000-6fafffff 64gb   free
 6fb00000-6fbfffff 64gb   free
-6fc00004-6fcffffb 64gb   kmalloc
+6fc00004-6fcffffb 64gb   free
 6fd00004-6fdffffb 64gb   zipos
 6fe00004-6feffffb 64gb   g_fds
 6ff00004-70000003 64gb   free

libc/runtime/syslib.internal.h

@@ -44,6 +44,7 @@ struct Syslib {
   dispatch_time_t (*dispatch_walltime)(const struct timespec *, int64_t);
   /* v2 (2023-09-10) */
   long (*pthread_self)(void);
+  void (*dispatch_release)(dispatch_semaphore_t);
 };
 
 extern struct Syslib *__syslib;

libc/runtime/zipos-get.c

@@ -23,7 +23,6 @@
 #include "libc/cosmo.h"
 #include "libc/fmt/conv.h"
 #include "libc/intrin/cmpxchg.h"
-#include "libc/intrin/kmalloc.h"
 #include "libc/intrin/promises.internal.h"
 #include "libc/intrin/strace.internal.h"
 #include "libc/macros.internal.h"
@@ -91,7 +90,7 @@ static int __zipos_compare_names(const void *a, const void *b, void *c) {
 static void __zipos_generate_index(struct Zipos *zipos) {
   size_t c, i;
   zipos->records = GetZipCdirRecords(zipos->cdir);
-  zipos->index = kmalloc(zipos->records * sizeof(size_t));
+  zipos->index = _mapanon(zipos->records * sizeof(size_t));
   for (i = 0, c = GetZipCdirOffset(zipos->cdir); i < zipos->records;
        ++i, c += ZIP_CFILE_HDRSIZE(zipos->map + c)) {
     zipos->index[i] = c;

libc/stdio/alloc.c

@@ -16,49 +16,26 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/intrin/atomic.h"
-#include "libc/intrin/kmalloc.h"
+#include "libc/mem/mem.h"
 #include "libc/stdio/internal.h"
-#include "libc/stdio/stdio.h"
-#include "libc/str/str.h"
 #include "libc/thread/thread.h"
 
-static _Atomic(FILE *) __stdio_freelist;
-
 FILE *__stdio_alloc(void) {
   FILE *f;
-  f = atomic_load_explicit(&__stdio_freelist, memory_order_acquire);
-  while (f) {
-    if (atomic_compare_exchange_weak_explicit(
-            &__stdio_freelist, &f,
-            atomic_load_explicit((_Atomic(struct FILE *) *)&f->next,
-                                 memory_order_acquire),
-            memory_order_release, memory_order_relaxed)) {
-      atomic_store_explicit((_Atomic(struct FILE *) *)&f->next, 0,
-                            memory_order_release);
-      break;
-    }
-  }
-  if (!f) {
-    f = kmalloc(sizeof(FILE));
-  }
-  if (f) {
-    f->lock._type = PTHREAD_MUTEX_RECURSIVE;
+  if ((f = calloc(1, sizeof(FILE)))) {
+    pthread_mutexattr_t attr;
+    pthread_mutexattr_init(&attr);
+    pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
+    pthread_mutex_init(&f->lock, &attr);
+    pthread_mutexattr_destroy(&attr);
+    f->dynamic = 1;
   }
   return f;
 }
 
 void __stdio_free(FILE *f) {
-  FILE *g;
-  bzero(f, sizeof(*f));
-  g = atomic_load_explicit(&__stdio_freelist, memory_order_acquire);
-  for (;;) {
-    atomic_store_explicit((_Atomic(struct FILE *) *)&f->next, g,
-                          memory_order_release);
-    if (atomic_compare_exchange_weak_explicit(&__stdio_freelist, &g, f,
-                                              memory_order_release,
-                                              memory_order_relaxed)) {
-      break;
-    }
+  pthread_mutex_destroy(&f->lock);
+  if (f->dynamic) {
+    free(f);
   }
 }

libc/stdio/internal.h

@@ -9,21 +9,22 @@
 COSMOPOLITAN_C_START_
 
 struct FILE {
-  uint8_t bufmode;      /* 0x00 _IOFBF, etc. (ignored if fd=-1) */
-  char noclose;         /* 0x01 for fake dup() todo delete! */
-  uint32_t iomode;      /* 0x04 O_RDONLY, etc. (ignored if fd=-1) */
-  int32_t state;        /* 0x08 0=OK, -1=EOF, >0=errno */
-  int fd;               /* 0x0c ≥0=fd, -1=closed|buffer */
-  uint32_t beg;         /* 0x10 */
-  uint32_t end;         /* 0x14 */
-  char *buf;            /* 0x18 */
-  uint32_t size;        /* 0x20 */
-  uint32_t nofree;      /* 0x24 */
-  int pid;              /* 0x28 */
-  char *getln;          /* 0x30 */
-  pthread_mutex_t lock; /* 0x38 */
-  struct FILE *next;    /* 0x48 */
-  char mem[BUFSIZ];     /* 0x50 */
+  uint8_t bufmode; /* _IOFBF, etc. (ignored if fd=-1) */
+  char noclose;    /* for fake dup() todo delete! */
+  char dynamic;    /* did malloc() create this object? */
+  uint32_t iomode; /* O_RDONLY, etc. (ignored if fd=-1) */
+  int32_t state;   /* 0=OK, -1=EOF, >0=errno */
+  int fd;          /* ≥0=fd, -1=closed|buffer */
+  uint32_t beg;
+  uint32_t end;
+  char *buf;
+  uint32_t size;
+  uint32_t nofree;
+  int pid;
+  char *getln;
+  pthread_mutex_t lock;
+  struct FILE *next;
+  char mem[BUFSIZ];
 };
 
 extern uint64_t g_rando;

libc/thread/pthread_atfork.c

@@ -17,13 +17,17 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/assert.h"
+#include "libc/atomic.h"
 #include "libc/calls/state.internal.h"
+#include "libc/cosmo.h"
+#include "libc/dce.h"
 #include "libc/errno.h"
 #include "libc/intrin/atomic.h"
 #include "libc/intrin/dll.h"
-#include "libc/intrin/kmalloc.h"
+#include "libc/intrin/leaky.internal.h"
 #include "libc/mem/mem.h"
 #include "libc/runtime/memtrack.internal.h"
+#include "libc/runtime/runtime.h"
 #include "libc/str/str.h"
 #include "libc/thread/posixthread.internal.h"
 #include "libc/thread/thread.h"
@@ -61,11 +65,9 @@ void _pthread_onfork_prepare(void) {
   pthread_spin_lock(&_pthread_lock);
   __fds_lock();
   __mmi_lock();
-  __kmalloc_lock();
 }
 
 void _pthread_onfork_parent(void) {
-  __kmalloc_unlock();
   __mmi_unlock();
   __fds_unlock();
   pthread_spin_unlock(&_pthread_lock);
@@ -85,7 +87,6 @@ void _pthread_onfork_child(void) {
   atomic_store_explicit(&pt->cancelled, false, memory_order_relaxed);
 
   // wipe core runtime locks
-  __kmalloc_unlock();
   pthread_mutexattr_init(&attr);
   pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
   pthread_mutex_init(&__mmi_lock_obj, &attr);
@@ -105,7 +106,7 @@ void _pthread_onfork_child(void) {
 int _pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) {
   int rc;
   struct AtFork *a;
-  if (!(a = kmalloc(sizeof(struct AtFork)))) return ENOMEM;
+  if (!(a = malloc(sizeof(struct AtFork)))) return ENOMEM;
   a->f[0] = prepare;
   a->f[1] = parent;
   a->f[2] = child;
@@ -118,3 +119,5 @@ int _pthread_atfork(atfork_f prepare, atfork_f parent, atfork_f child) {
   rc = 0;
   return rc;
 }
+
+IGNORE_LEAKS(_pthread_atfork)

libc/thread/tls.h

@@ -27,7 +27,7 @@ struct CosmoTib {
   _Atomic(int32_t) tib_tid;       /* 0x38 transitions -1 → tid → 0 */
   int32_t tib_errno;              /* 0x3c */
   uint64_t tib_flags;             /* 0x40 */
-  void *tib_nsync;
+  long __padding;
   int tib_ftrace;       /* inherited */
   int tib_strace;       /* inherited */
   uint64_t tib_sigmask; /* inherited */

libc/x/xload.c

@@ -17,7 +17,7 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/intrin/atomic.h"
-#include "libc/intrin/kmalloc.h"
+#include "libc/mem/mem.h"
 #include "libc/runtime/internal.h"
 #include "libc/x/x.h"
 #include "third_party/zlib/zlib.h"
@@ -44,7 +44,7 @@
 void *xload(_Atomic(void *) *a, const void *p, size_t n, size_t m) {
   void *r, *z;
   if ((r = atomic_load_explicit(a, memory_order_acquire))) return r;
-  if (!(r = kmalloc(m))) return 0;
+  if (!(r = malloc(m))) return 0;
   if (__inflate(r, m, p, n)) return 0;
   z = 0;
   if (!atomic_compare_exchange_strong_explicit(a, &z, r, memory_order_release,