Make mmap() scalable

It's now possible to create thousands of thousands of sparse independent
memory mappings, without any slowdown. The memory manager is better with
tracking memory protection now, particularly on Windows in a precise way
that can be restored during fork(). You now have the highest quality mem
manager possible. It's even better than some OSes like XNU, where mmap()
is implemented as an O(n) operation which means sadly things aren't much
improved over there. With this change the llamafile HTTP server endpoint
at /tokenize with a prompt of 50 tokens is now able to handle 2.6m r/sec

libc/calls/setrlimit.c

@@ -85,7 +85,7 @@ int setrlimit(int resource, const struct rlimit *rlim) {
     rc = efault();
   } else if (IsXnuSilicon()) {
     rc = _sysret(__syslib->__setrlimit(resource, rlim));
-  } else if (!IsWindows()) {
+  } else if (!IsWindows() && !(IsNetbsd() && resource == RLIMIT_AS)) {
     rc = sys_setrlimit(resource, rlim);
   } else if (resource == RLIMIT_STACK) {
     rc = enotsup();

libc/intrin/BUILD.mk

@@ -62,6 +62,10 @@ o/$(MODE)/libc/intrin/kprintf.o: private		\
 			-Wframe-larger-than=128		\
 			-Walloca-larger-than=128
 
+o/$(MODE)/libc/intrin/tree.o: private			\
+		CFLAGS +=				\
+			-ffunction-sections
+
 o//libc/intrin/memmove.o: private			\
 		CFLAGS +=				\
 			-fno-toplevel-reorder

libc/intrin/describemapping.c

@@ -24,7 +24,7 @@
 static char DescribeMapType(int flags) {
   switch (flags & MAP_TYPE) {
     case MAP_FILE:
-      return 'f';
+      return '-';
     case MAP_PRIVATE:
       return 'p';
     case MAP_SHARED:
@@ -47,7 +47,7 @@ const char *(DescribeMapping)(char p[8], int prot, int flags) {
   DescribeProt(p, prot);
   p[3] = DescribeMapType(flags);
   p[4] = (flags & MAP_ANONYMOUS) ? 'a' : '-';
-  p[5] = (flags & MAP_FIXED) ? 'F' : '-';
+  p[5] = (flags & MAP_FIXED) ? 'f' : '-';
   p[6] = 0;
   return p;
 }

libc/intrin/describeprotflags.c

@@ -20,12 +20,12 @@
 #include "libc/macros.internal.h"
 #include "libc/sysv/consts/prot.h"
 
-static const struct DescribeFlags kProtFlags[] = {
+const char *(DescribeProtFlags)(char buf[48], int x) {
+  const struct DescribeFlags kProtFlags[] = {
       {PROT_READ, "READ"},    //
       {PROT_WRITE, "WRITE"},  //
       {PROT_EXEC, "EXEC"},    //
+      {PROT_GUARD, "GUARD"},  //
   };
-
-const char *(DescribeProtFlags)(char buf[48], int x) {
   return DescribeFlags(buf, 48, kProtFlags, ARRAYLEN(kProtFlags), "PROT_", x);
 }

libc/intrin/describerlimit.c

@@ -18,8 +18,11 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/calls/struct/rlimit.h"
 #include "libc/dce.h"
+#include "libc/fmt/itoa.h"
 #include "libc/intrin/kprintf.h"
 #include "libc/intrin/strace.internal.h"
+#include "libc/str/str.h"
+#include "libc/sysv/consts/rlim.h"
 
 const char *DescribeRlimit(char buf[64], int rc, const struct rlimit *rlim) {
   if (rc == -1)
@@ -29,7 +32,18 @@ const char *DescribeRlimit(char buf[64], int rc, const struct rlimit *rlim) {
   if (kisdangerous(rlim)) {
     ksnprintf(buf, 64, "%p", rlim);
   } else {
-    ksnprintf(buf, 64, "{%'ld, %'ld}", rlim->rlim_cur, rlim->rlim_max);
+    char str[2][21];
+    if (rlim->rlim_cur == RLIM_INFINITY) {
+      strcpy(str[0], "RLIM_INFINITY");
+    } else {
+      FormatInt64(str[0], rlim->rlim_cur);
+    }
+    if (rlim->rlim_max == RLIM_INFINITY) {
+      strcpy(str[1], "RLIM_INFINITY");
+    } else {
+      FormatInt64(str[1], rlim->rlim_max);
+    }
+    ksnprintf(buf, 64, "{%s, %s}", str[0], str[1]);
   }
   return buf;
 }

libc/intrin/directmap-nt.c

@@ -86,9 +86,8 @@ TryAgain:
     if ((dm.addr = MapViewOfFileEx(dm.maphandle, fl.flags2, off >> 32, off,
                                    size, addr))) {
       uint32_t oldprot;
-      if (VirtualProtect(dm.addr, size, __prot2nt(prot, iscow), &oldprot)) {
+      if (VirtualProtect(dm.addr, size, __prot2nt(prot, iscow), &oldprot))
         return dm;
-      }
       UnmapViewOfFile(dm.addr);
     }
     CloseHandle(dm.maphandle);

libc/intrin/dll.c

@@ -28,7 +28,7 @@
  *
  * It's required that `elem` and `succ` aren't part of the same list.
  */
-privileged void dll_splice_after(struct Dll *elem, struct Dll *succ) {
+void dll_splice_after(struct Dll *elem, struct Dll *succ) {
   struct Dll *tmp1, *tmp2;
   tmp1 = elem->next;
   tmp2 = succ->prev;
@@ -43,7 +43,7 @@ privileged void dll_splice_after(struct Dll *elem, struct Dll *succ) {
  *
  * @param list is a doubly-linked list, where `!*list` means empty
  */
-privileged void dll_remove(struct Dll **list, struct Dll *elem) {
+void dll_remove(struct Dll **list, struct Dll *elem) {
   if (*list == elem) {
     if ((*list)->prev == *list) {
       *list = 0;
@@ -66,7 +66,7 @@ privileged void dll_remove(struct Dll **list, struct Dll *elem) {
  * @param list is a doubly-linked list, where `!*list` means empty
  * @param elem must not be a member of `list`, or null for no-op
  */
-privileged void dll_make_first(struct Dll **list, struct Dll *elem) {
+void dll_make_first(struct Dll **list, struct Dll *elem) {
   if (elem) {
     if (!*list) {
       *list = elem->prev;
@@ -85,7 +85,7 @@ privileged void dll_make_first(struct Dll **list, struct Dll *elem) {
  * @param list is a doubly-linked list, where `!*list` means empty
  * @param elem must not be a member of `list`, or null for no-op
  */
-privileged void dll_make_last(struct Dll **list, struct Dll *elem) {
+void dll_make_last(struct Dll **list, struct Dll *elem) {
   if (elem) {
     dll_make_first(list, elem->next);
     *list = elem;

libc/intrin/kprintf.greg.c

@@ -25,7 +25,6 @@
 #include "libc/fmt/magnumstrs.internal.h"
 #include "libc/intrin/asmflag.h"
 #include "libc/intrin/atomic.h"
-#include "libc/intrin/dll.h"
 #include "libc/intrin/getenv.internal.h"
 #include "libc/intrin/kprintf.h"
 #include "libc/intrin/likely.h"
@@ -154,27 +153,18 @@ __funline bool kischarmisaligned(const char *p, signed char t) {
   return false;
 }
 
-privileged static bool32 kisdangerous_unlocked(const char *addr) {
-  struct Dll *e;
-  if ((e = dll_first(__maps.used))) {
-    do {
-      struct Map *map = MAP_CONTAINER(e);
-      if (map->addr <= addr && addr < map->addr + map->size) {
-        dll_remove(&__maps.used, e);
-        dll_make_first(&__maps.used, e);
-        return !(map->prot & PROT_READ);
-      }
-    } while ((e = dll_next(__maps.used, e)));
-    return true;
-  } else {
-    return false;
-  }
-}
-
 privileged bool32 kisdangerous(const void *addr) {
-  bool32 res;
+  bool32 res = true;
   __maps_lock();
-  res = kisdangerous_unlocked(addr);
+  if (__maps.maps) {
+    struct Map *map;
+    if ((map = __maps_floor(addr)))
+      if ((const char *)addr >= map->addr &&
+          (const char *)addr < map->addr + map->size)
+        res = false;
+  } else {
+    res = false;
+  }
   __maps_unlock();
   return res;
 }

libc/intrin/maps.c

@@ -20,6 +20,7 @@
 #include "ape/sections.internal.h"
 #include "libc/dce.h"
 #include "libc/intrin/dll.h"
+#include "libc/intrin/maps.h"
 #include "libc/runtime/runtime.h"
 #include "libc/runtime/stack.h"
 #include "libc/sysv/consts/auxv.h"
@@ -32,8 +33,7 @@ __static_yoink("_init_maps");
 struct Maps __maps;
 
 void __maps_add(struct Map *map) {
-  dll_init(&map->elem);
+  tree_insert(&__maps.maps, &map->tree, __maps_compare);
-  dll_make_first(&__maps.used, &map->elem);
   ++__maps.count;
 }
 

libc/intrin/maps.h

@@ -1,36 +1,41 @@
-#ifndef COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_
+#ifndef COSMOPOLITAN_MAPS_H_
-#define COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_
+#define COSMOPOLITAN_MAPS_H_
 #include "libc/intrin/atomic.h"
 #include "libc/intrin/dll.h"
+#include "libc/intrin/tree.h"
+#include "libc/runtime/runtime.h"
 #include "libc/thread/tls2.internal.h"
 COSMOPOLITAN_C_START_
 
-#define MAP_CONTAINER(e) DLL_CONTAINER(struct Map, elem, e)
+#define MAP_TREE_CONTAINER(e) TREE_CONTAINER(struct Map, tree, e)
+#define MAP_FREE_CONTAINER(e) DLL_CONTAINER(struct Map, free, e)
 
 struct Map {
   char *addr;        /* granule aligned */
   size_t size;       /* must be nonzero */
-  struct Dll elem;   /* for __maps.free */
+  int64_t off;       /* ignore for anon */
-  int64_t off;       /* -1 if anonymous */
   int prot;          /* memory protects */
   int flags;         /* memory map flag */
   bool iscow;        /* windows nt only */
   bool readonlyfile; /* windows nt only */
   unsigned visited;  /* used for checks */
+  unsigned oldprot;  /* in windows fork */
   intptr_t hand;     /* windows nt only */
+  union {
+    struct Tree tree;
+    struct Dll free;
+  };
 };
 
 struct Maps {
-  unsigned mono;
   atomic_int lock;
+  struct Tree *maps;
   struct Dll *free;
-  struct Dll *used;
   size_t count;
   size_t pages;
+  atomic_ulong rollo;
   struct Map stack;
   struct Map guard;
-  bool once;
-  atomic_ulong rollo;
 };
 
 struct AddrSize {
@@ -45,10 +50,37 @@ bool __maps_lock(void);
 void __maps_check(void);
 void __maps_unlock(void);
 void __maps_add(struct Map *);
-struct Map *__maps_alloc(void);
 void __maps_free(struct Map *);
+struct Map *__maps_alloc(void);
+struct Map *__maps_floor(const char *);
 void __maps_stack(char *, int, int, size_t, int, intptr_t);
+int __maps_compare(const struct Tree *, const struct Tree *);
 struct AddrSize __get_main_stack(void);
 
+forceinline optimizespeed int __maps_search(const void *key,
+                                            const struct Tree *node) {
+  const char *addr = (const char *)key;
+  const struct Map *map = (const struct Map *)MAP_TREE_CONTAINER(node);
+  if (addr < map->addr)
+    return +1;
+  if (addr >= map->addr + map->size)
+    return -1;
+  return 0;
+}
+
+static struct Map *__maps_next(struct Map *map) {
+  struct Tree *node;
+  if ((node = tree_next(&map->tree)))
+    return MAP_TREE_CONTAINER(node);
+  return 0;
+}
+
+static struct Map *__maps_first(void) {
+  struct Tree *node;
+  if ((node = tree_first(__maps.maps)))
+    return MAP_TREE_CONTAINER(node);
+  return 0;
+}
+
 COSMOPOLITAN_C_END_
-#endif /* COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_ */
+#endif /* COSMOPOLITAN_MAPS_H_ */

libc/intrin/mmap.c

@@ -67,20 +67,31 @@
       struct StackFrame *bp = __builtin_frame_address(0);                 \
       kprintf("%!s:%d: assertion failed: %!s\n", __FILE__, __LINE__, #x); \
       kprintf("bt %!s\n", (DescribeBacktrace)(bt, bp));                   \
-      __print_maps();                                                     \
+      __print_maps(0);                                                    \
-      _Exit(99);                                                          \
+      __builtin_trap();                                                   \
     }                                                                     \
   } while (0)
 #endif
 
+int __maps_compare(const struct Tree *ra, const struct Tree *rb) {
+  const struct Map *a = (const struct Map *)MAP_TREE_CONTAINER(ra);
+  const struct Map *b = (const struct Map *)MAP_TREE_CONTAINER(rb);
+  return (a->addr > b->addr) - (a->addr < b->addr);
+}
+
+privileged optimizespeed struct Map *__maps_floor(const char *addr) {
+  struct Tree *node;
+  if ((node = tree_floor(__maps.maps, addr, __maps_search)))
+    return MAP_TREE_CONTAINER(node);
+  return 0;
+}
+
 static bool overlaps_existing_map(const char *addr, size_t size, int pagesz) {
-  for (struct Dll *e = dll_first(__maps.used); e;
+  struct Map *map;
-       e = dll_next(__maps.used, e)) {
+  if ((map = __maps_floor(addr)))
-    struct Map *map = MAP_CONTAINER(e);
     if (MAX(addr, map->addr) <
         MIN(addr + PGUP(size), map->addr + PGUP(map->size)))
       return true;
-  }
   return false;
 }
 
@@ -89,74 +100,59 @@ void __maps_check(void) {
   size_t maps = 0;
   size_t pages = 0;
   int pagesz = getpagesize();
-  unsigned id = ++__maps.mono;
+  static unsigned mono;
-  for (struct Dll *e = dll_first(__maps.used); e;
+  unsigned id = ++mono;
-       e = dll_next(__maps.used, e)) {
+  for (struct Map *map = __maps_first(); map; map = __maps_next(map)) {
-    struct Map *map = MAP_CONTAINER(e);
     ASSERT(map->addr != MAP_FAILED);
     ASSERT(map->visited != id);
     ASSERT(map->size);
     map->visited = id;
     pages += (map->size + getpagesize() - 1) / getpagesize();
     maps += 1;
+    struct Map *next;
+    if ((next = __maps_next(map))) {
+      ASSERT(map->addr < next->addr);
+      ASSERT(
+          !(MAX(map->addr, next->addr) <
+            MIN(map->addr + PGUP(map->size), next->addr + PGUP(next->size))));
+    }
   }
   ASSERT(maps = __maps.count);
   ASSERT(pages == __maps.pages);
-  for (struct Dll *e = dll_first(__maps.used); e;
-       e = dll_next(__maps.used, e)) {
-    struct Map *m1 = MAP_CONTAINER(e);
-    for (struct Dll *f = dll_next(__maps.used, e); f;
-         f = dll_next(__maps.used, f)) {
-      struct Map *m2 = MAP_CONTAINER(f);
-      ASSERT(MAX(m1->addr, m2->addr) >=
-             MIN(m1->addr + PGUP(m1->size), m2->addr + PGUP(m2->size)));
-    }
-  }
 #endif
 }
 
 void __maps_free(struct Map *map) {
   map->size = 0;
   map->addr = MAP_FAILED;
-  ASSERT(dll_is_alone(&map->elem));
+  dll_init(&map->free);
-  dll_make_last(&__maps.free, &map->elem);
+  dll_make_first(&__maps.free, &map->free);
 }
 
 static void __maps_insert(struct Map *map) {
-  struct Dll *e = dll_first(__maps.used);
-  struct Map *last = e ? MAP_CONTAINER(e) : 0;
   __maps.pages += (map->size + getpagesize() - 1) / getpagesize();
-  if (last && !IsWindows() &&                  //
+  struct Map *floor = __maps_floor(map->addr);
-      map->addr == last->addr + last->size &&  //
+  if (floor && !IsWindows() &&                 //
+      map->addr + map->size == floor->addr &&  //
       (map->flags & MAP_ANONYMOUS) &&          //
-      map->flags == last->flags &&             //
+      map->flags == floor->flags &&            //
-      map->prot == last->prot) {
+      map->prot == floor->prot) {
-    last->size += map->size;
+    floor->addr -= map->size;
-    dll_remove(&__maps.used, &last->elem);
+    floor->size += map->size;
-    dll_make_first(&__maps.used, &last->elem);
-    __maps_free(map);
-  } else if (last && !IsWindows() &&                 //
-             map->addr + map->size == last->addr &&  //
-             (map->flags & MAP_ANONYMOUS) &&         //
-             map->flags == last->flags &&            //
-             map->prot == last->prot) {
-    last->addr -= map->size;
-    last->size += map->size;
-    dll_remove(&__maps.used, &last->elem);
-    dll_make_first(&__maps.used, &last->elem);
     __maps_free(map);
+    __maps_check();
   } else {
     __maps_add(map);
-  }
     __maps_check();
   }
+}
 
 struct Map *__maps_alloc(void) {
   struct Dll *e;
   struct Map *map;
   if ((e = dll_first(__maps.free))) {
     dll_remove(&__maps.free, e);
-    map = MAP_CONTAINER(e);
+    map = MAP_FREE_CONTAINER(e);
     return map;
   }
   int granularity = __granularity();
@@ -168,11 +164,8 @@ struct Map *__maps_alloc(void) {
     CloseHandle(sys.maphandle);
   map = sys.addr;
   map->addr = MAP_FAILED;
-  dll_init(&map->elem);
+  for (int i = 1; i < granularity / sizeof(struct Map); ++i)
-  for (int i = 1; i < granularity / sizeof(struct Map); ++i) {
-    dll_init(&map[i].elem);
     __maps_free(map + i);
-  }
   return map;
 }
 
@@ -190,26 +183,26 @@ static int __munmap(char *addr, size_t size, bool untrack_only) {
 
   // untrack mappings
   int rc = 0;
-  struct Dll *cur;
+  struct Map *map;
-  struct Dll *next;
+  struct Map *next;
-  struct Dll *delete = 0;
+  struct Dll *deleted = 0;
   if (__maps_lock()) {
     __maps_unlock();
     return edeadlk();
   }
-  for (cur = dll_first(__maps.used); cur; cur = next) {
+  for (map = __maps_floor(addr); map; map = next) {
-    next = dll_next(__maps.used, cur);
+    next = __maps_next(map);
-    struct Map *map = MAP_CONTAINER(cur);
     char *map_addr = map->addr;
     size_t map_size = map->size;
-    if (MAX(addr, map_addr) < MIN(addr + size, map_addr + PGUP(map_size))) {
+    if (!(MAX(addr, map_addr) < MIN(addr + size, map_addr + PGUP(map_size))))
+      break;
     if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) {
       // remove mapping completely
-        dll_remove(&__maps.used, cur);
+      tree_remove(&__maps.maps, &map->tree);
-        dll_make_first(&delete, cur);
+      dll_init(&map->free);
+      dll_make_first(&deleted, &map->free);
       __maps.pages -= (map_size + pagesz - 1) / pagesz;
       __maps.count -= 1;
-        __maps_check();
     } else if (IsWindows()) {
       // you can't carve up memory maps on windows. our mmap() makes
       // this not a problem (for non-enormous memory maps) by making
@@ -231,8 +224,8 @@ static int __munmap(char *addr, size_t size, bool untrack_only) {
         __maps.pages -= (left + pagesz - 1) / pagesz;
         leftmap->addr = map_addr;
         leftmap->size = left;
-          dll_make_first(&delete, &leftmap->elem);
+        dll_init(&leftmap->free);
-          __maps_check();
+        dll_make_first(&deleted, &leftmap->free);
       } else {
         rc = -1;
       }
@@ -246,8 +239,8 @@ static int __munmap(char *addr, size_t size, bool untrack_only) {
         __maps.pages -= (right + pagesz - 1) / pagesz;
         rightmap->addr = addr;
         rightmap->size = right;
-          dll_make_first(&delete, &rightmap->elem);
+        dll_init(&rightmap->free);
-          __maps_check();
+        dll_make_first(&deleted, &rightmap->free);
       } else {
         rc = -1;
       }
@@ -269,27 +262,27 @@ static int __munmap(char *addr, size_t size, bool untrack_only) {
           map->size = right;
           if (!(map->flags & MAP_ANONYMOUS))
             map->off += left + middle;
-            dll_make_first(&__maps.used, &leftmap->elem);
+          tree_insert(&__maps.maps, &leftmap->tree, __maps_compare);
           __maps.pages -= (middle + pagesz - 1) / pagesz;
           __maps.count += 1;
           middlemap->addr = addr;
           middlemap->size = size;
-            dll_make_first(&delete, &middlemap->elem);
+          dll_init(&middlemap->free);
+          dll_make_first(&deleted, &middlemap->free);
+        } else {
+          rc = -1;
+        }
+      } else {
+        rc = -1;
+      }
+    }
     __maps_check();
-          } else {
-            rc = -1;
-          }
-        } else {
-          rc = -1;
-        }
-      }
-    }
   }
   __maps_unlock();
 
   // delete mappings
-  for (struct Dll *e = dll_first(delete); e; e = dll_next(delete, e)) {
+  for (struct Dll *e = dll_first(deleted); e; e = dll_next(deleted, e)) {
-    struct Map *map = MAP_CONTAINER(e);
+    struct Map *map = MAP_FREE_CONTAINER(e);
     if (!untrack_only) {
       if (!IsWindows()) {
         if (sys_munmap(map->addr, map->size))
@@ -305,12 +298,12 @@ static int __munmap(char *addr, size_t size, bool untrack_only) {
   }
 
   // free mappings
-  if (!dll_is_empty(delete)) {
+  if (!dll_is_empty(deleted)) {
     __maps_lock();
     struct Dll *e;
-    while ((e = dll_first(delete))) {
+    while ((e = dll_first(deleted))) {
-      dll_remove(&delete, e);
+      dll_remove(&deleted, e);
-      __maps_free(MAP_CONTAINER(e));
+      __maps_free(MAP_FREE_CONTAINER(e));
     }
     __maps_check();
     __maps_unlock();
@@ -350,6 +343,7 @@ static void *__mmap_chunk(void *addr, size_t size, int prot, int flags, int fd,
     __maps_unlock();
     return (void *)edeadlk();
   }
+  __maps_check();
   map = __maps_alloc();
   __maps_unlock();
   if (!map)

libc/intrin/mprotect.c

@@ -25,6 +25,7 @@
 #include "libc/intrin/kprintf.h"
 #include "libc/intrin/maps.h"
 #include "libc/intrin/strace.internal.h"
+#include "libc/intrin/tree.h"
 #include "libc/nt/memory.h"
 #include "libc/runtime/internal.h"
 #include "libc/runtime/runtime.h"
@@ -51,7 +52,7 @@ static int __mprotect_chunk(char *addr, size_t size, int prot, bool iscow) {
 int __mprotect(char *addr, size_t size, int prot) {
 
   // unix checks prot before checking size
-  if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC))
+  if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC | PROT_GUARD))
     return einval();
 
   // make new technology consistent with unix
@@ -68,19 +69,18 @@ int __mprotect(char *addr, size_t size, int prot) {
 
   // change mappings
   int rc = 0;
-  struct Dll *cur;
   bool found = false;
   if (__maps_lock()) {
     __maps_unlock();
     return edeadlk();
   }
-  for (cur = dll_first(__maps.used); cur; cur = dll_next(__maps.used, cur)) {
+  for (struct Map *map = __maps_floor(addr); map; map = __maps_next(map)) {
-    struct Map *map = MAP_CONTAINER(cur);
     char *map_addr = map->addr;
     size_t map_size = map->size;
     char *beg = MAX(addr, map_addr);
     char *end = MIN(addr + size, map_addr + PGUP(map_size));
-    if (beg < end) {
+    if (beg >= end)
+      break;
     found = true;
     if (addr <= map_addr && addr + size >= map_addr + PGUP(map_size)) {
       // change protection of entire mapping
@@ -109,7 +109,7 @@ int __mprotect(char *addr, size_t size, int prot) {
           map->hand = -1;
           if (!(map->flags & MAP_ANONYMOUS))
             map->off += left;
-            dll_make_first(&__maps.used, &leftmap->elem);
+          tree_insert(&__maps.maps, &leftmap->tree, __maps_compare);
           __maps.count += 1;
           __maps_check();
         } else {
@@ -140,7 +140,7 @@ int __mprotect(char *addr, size_t size, int prot) {
           map->hand = -1;
           if (!(map->flags & MAP_ANONYMOUS))
             map->off += left;
-            dll_make_first(&__maps.used, &leftmap->elem);
+          tree_insert(&__maps.maps, &leftmap->tree, __maps_compare);
           __maps.count += 1;
           __maps_check();
         } else {
@@ -179,8 +179,8 @@ int __mprotect(char *addr, size_t size, int prot) {
             map->hand = -1;
             if (!(map->flags & MAP_ANONYMOUS))
               map->off += left + middle;
-              dll_make_first(&__maps.used, &midlmap->elem);
+            tree_insert(&__maps.maps, &leftmap->tree, __maps_compare);
-              dll_make_first(&__maps.used, &leftmap->elem);
+            tree_insert(&__maps.maps, &midlmap->tree, __maps_compare);
             __maps.count += 2;
             __maps_check();
           } else {
@@ -197,7 +197,6 @@ int __mprotect(char *addr, size_t size, int prot) {
       }
     }
   }
-  }
 
   // allow user to change mappings unknown to cosmo runtime
   if (!found)

libc/intrin/msync-nt.c

@@ -34,10 +34,10 @@ textwindows int sys_msync_nt(char *addr, size_t size, int flags) {
     return einval();
 
   int rc = 0;
-  __maps_lock();
+  if (__maps_lock()) {
-  for (struct Dll *e = dll_first(__maps.used); e;
+    rc = edeadlk();
-       e = dll_next(__maps.used, e)) {
+  } else {
-    struct Map *map = MAP_CONTAINER(e);
+    for (struct Map *map = __maps_floor(addr); map; map = __maps_next(map)) {
       char *beg = MAX(addr, map->addr);
       char *end = MIN(addr + size, map->addr + map->size);
       if (beg < end)
@@ -45,6 +45,7 @@ textwindows int sys_msync_nt(char *addr, size_t size, int flags) {
           rc = -1;
       // TODO(jart): FlushFileBuffers too on g_fds handle if MS_SYNC?
     }
+  }
   __maps_unlock();
 
   return rc;

libc/intrin/printmaps.c

@@ -29,16 +29,11 @@
 /**
  * Prints memory mappings.
  */
-void __print_maps(void) {
+void __print_maps(size_t limit) {
-  int limit = 15;
-  long maptally = 0;
   char mappingbuf[8], sb[16];
   __maps_lock();
-  struct Dll *e, *e2;
+  for (struct Tree *e = tree_first(__maps.maps); e; e = tree_next(e)) {
-  for (e = dll_first(__maps.used); e; e = e2) {
+    struct Map *map = MAP_TREE_CONTAINER(e);
-    e2 = dll_next(__maps.used, e);
-    struct Map *map = MAP_CONTAINER(e);
-    maptally += map->size;
     kprintf("%012lx-%012lx %!s", map->addr, map->addr + map->size,
             (DescribeMapping)(mappingbuf, map->prot, map->flags));
     sizefmt(sb, map->size, 1024);

libc/intrin/prot2nt.greg.c

@@ -42,6 +42,8 @@ privileged int __prot2nt(int prot, int iscow) {
         return kNtPageExecuteReadwrite;
       }
     default:
+      if (prot & PROT_GUARD)
+        return kNtPageReadwrite | kNtPageGuard;
       return kNtPageNoaccess;
   }
 }

libc/intrin/pthread_delay_np.c

@@ -0,0 +1,40 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2024 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/intrin/strace.internal.h"
+#include "libc/thread/thread.h"
+
+/**
+ * Delays execution for brief moment.
+ *
+ * @param symbol may be used to strace names of static locks
+ * @param backoff should start at zero and be feed back in
+ * @return new value for backoff
+ */
+int pthread_delay_np(const void *symbol, int backoff) {
+  if (backoff < 7) {
+    volatile int i;
+    for (i = 0; i != 1 << backoff; i++) {
+    }
+    backoff++;
+  } else {
+    STRACE("pthread_delay_np(%t)", symbol);
+    pthread_yield_np();
+  }
+  return backoff;
+}

libc/intrin/pthread_mutex_lock.c

@@ -18,8 +18,10 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/calls/calls.h"
 #include "libc/calls/state.internal.h"
+#include "libc/dce.h"
 #include "libc/errno.h"
 #include "libc/intrin/atomic.h"
+#include "libc/intrin/describeflags.internal.h"
 #include "libc/intrin/strace.internal.h"
 #include "libc/intrin/weaken.h"
 #include "libc/runtime/internal.h"
@@ -27,6 +29,66 @@
 #include "libc/thread/thread.h"
 #include "third_party/nsync/mu.h"
 
+static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex) {
+  int me;
+  int backoff = 0;
+  uint64_t word, lock;
+
+  // get current state of lock
+  word = atomic_load_explicit(&mutex->_word, memory_order_relaxed);
+
+  // use fancy nsync mutex if possible
+  if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL &&        //
+      MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE &&  //
+      _weaken(nsync_mu_lock)) {
+    _weaken(nsync_mu_lock)((nsync_mu *)mutex);
+    return 0;
+  }
+
+  // implement barebones normal mutexes
+  if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) {
+    for (;;) {
+      word = MUTEX_UNLOCK(word);
+      lock = MUTEX_LOCK(word);
+      if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
+                                                memory_order_acquire,
+                                                memory_order_relaxed))
+        return 0;
+      backoff = pthread_delay_np(mutex, backoff);
+    }
+  }
+
+  // implement recursive mutexes
+  me = gettid();
+  for (;;) {
+    if (MUTEX_OWNER(word) == me) {
+      if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) {
+        if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) {
+          if (atomic_compare_exchange_weak_explicit(
+                  &mutex->_word, &word, MUTEX_INC_DEPTH(word),
+                  memory_order_relaxed, memory_order_relaxed))
+            return 0;
+          continue;
+        } else {
+          return EAGAIN;
+        }
+      } else {
+        return EDEADLK;
+      }
+    }
+    word = MUTEX_UNLOCK(word);
+    lock = MUTEX_LOCK(word);
+    lock = MUTEX_SET_OWNER(lock, me);
+    if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
+                                              memory_order_acquire,
+                                              memory_order_relaxed)) {
+      mutex->_pid = __pid;
+      return 0;
+    }
+    backoff = pthread_delay_np(mutex, backoff);
+  }
+}
+
 /**
  * Locks mutex.
  *
@@ -65,65 +127,10 @@
  * @vforksafe
  */
 errno_t pthread_mutex_lock(pthread_mutex_t *mutex) {
-  int me;
-  uint64_t word, lock;
-
-  LOCKTRACE("pthread_mutex_lock(%t)", mutex);
-
   if (__vforked)
     return 0;
-
+  LOCKTRACE("acquiring %t...", mutex);
-  // get current state of lock
+  errno_t err = pthread_mutex_lock_impl(mutex);
-  word = atomic_load_explicit(&mutex->_word, memory_order_relaxed);
+  LOCKTRACE("pthread_mutex_lock(%t) → %s", mutex, DescribeErrno(err));
-
+  return err;
-  // use fancy nsync mutex if possible
-  if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL &&        //
-      MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE &&  //
-      _weaken(nsync_mu_lock)) {
-    _weaken(nsync_mu_lock)((nsync_mu *)mutex);
-    return 0;
-  }
-
-  // implement barebones normal mutexes
-  if (MUTEX_TYPE(word) == PTHREAD_MUTEX_NORMAL) {
-    for (;;) {
-      word = MUTEX_UNLOCK(word);
-      lock = MUTEX_LOCK(word);
-      if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
-                                                memory_order_acquire,
-                                                memory_order_relaxed))
-        return 0;
-      pthread_pause_np();
-    }
-  }
-
-  // implement recursive mutexes
-  me = gettid();
-  for (;;) {
-    if (MUTEX_OWNER(word) == me) {
-      if (MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) {
-        if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) {
-          if (atomic_compare_exchange_weak_explicit(
-                  &mutex->_word, &word, MUTEX_INC_DEPTH(word),
-                  memory_order_relaxed, memory_order_relaxed))
-            return 0;
-          continue;
-        } else {
-          return EAGAIN;
-        }
-      } else {
-        return EDEADLK;
-      }
-    }
-    word = MUTEX_UNLOCK(word);
-    lock = MUTEX_LOCK(word);
-    lock = MUTEX_SET_OWNER(lock, me);
-    if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
-                                              memory_order_acquire,
-                                              memory_order_relaxed)) {
-      mutex->_pid = __pid;
-      return 0;
-    }
-    pthread_pause_np();
-  }
 }

libc/intrin/pthread_mutex_trylock.c

@@ -17,6 +17,7 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/calls/calls.h"
+#include "libc/dce.h"
 #include "libc/errno.h"
 #include "libc/intrin/atomic.h"
 #include "libc/intrin/weaken.h"

libc/intrin/pthread_mutex_unlock.c

@@ -17,6 +17,7 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/calls/calls.h"
+#include "libc/dce.h"
 #include "libc/errno.h"
 #include "libc/intrin/atomic.h"
 #include "libc/intrin/strace.internal.h"

libc/intrin/pthread_syshand.c

@@ -29,6 +29,6 @@ intptr_t _pthread_syshand(struct PosixThread *pt) {
     syshand = atomic_load_explicit(&pt->tib->tib_syshand, memory_order_acquire);
     if (syshand)
       return syshand;
-    pthread_pause_np();
+    pthread_yield_np();
   }
 }

libc/intrin/pthread_tid.c

@@ -24,6 +24,6 @@
 int _pthread_tid(struct PosixThread *pt) {
   int tid = 0;
   while (pt && !(tid = atomic_load_explicit(&pt->ptid, memory_order_acquire)))
-    pthread_pause_np();
+    pthread_yield_np();
   return tid;
 }

libc/intrin/rbtree.c

@@ -1,312 +0,0 @@
-// Copyright 2024 Justine Alexandra Roberts Tunney
-//
-// Permission to use, copy, modify, and/or distribute this software for
-// any purpose with or without fee is hereby granted, provided that the
-// above copyright notice and this permission notice appear in all copies.
-//
-// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
-// WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
-// AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
-// DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
-// PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
-// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-// PERFORMANCE OF THIS SOFTWARE.
-
-#include "rbtree.h"
-#include "libc/dce.h"
-#include "libc/str/str.h"
-
-#define RBTREE_DEBUG
-
-struct rbtree *rbtree_next(struct rbtree *node) {
-  if (!node)
-    return 0;
-  if (node->right)
-    return rbtree_first(node->right);
-  struct rbtree *parent = node->parent;
-  while (parent && node == parent->right) {
-    node = parent;
-    parent = parent->parent;
-  }
-  return parent;
-}
-
-struct rbtree *rbtree_prev(struct rbtree *node) {
-  if (!node)
-    return 0;
-  if (rbtree_get_left(node))
-    return rbtree_last(rbtree_get_left(node));
-  struct rbtree *parent = node->parent;
-  while (parent && node == rbtree_get_left(parent)) {
-    node = parent;
-    parent = parent->parent;
-  }
-  return parent;
-}
-
-struct rbtree *rbtree_first(struct rbtree *node) {
-  while (node && rbtree_get_left(node))
-    node = rbtree_get_left(node);
-  return node;
-}
-
-struct rbtree *rbtree_last(struct rbtree *node) {
-  while (node && node->right)
-    node = node->right;
-  return node;
-}
-
-static void rbtree_rotate_left(struct rbtree **root, struct rbtree *x) {
-  struct rbtree *y = x->right;
-  x->right = rbtree_get_left(y);
-  if (rbtree_get_left(y))
-    rbtree_get_left(y)->parent = x;
-  y->parent = x->parent;
-  if (!x->parent) {
-    *root = y;
-  } else if (x == rbtree_get_left(x->parent)) {
-    rbtree_set_left(x->parent, y);
-  } else {
-    x->parent->right = y;
-  }
-  rbtree_set_left(y, x);
-  x->parent = y;
-}
-
-static void rbtree_rotate_right(struct rbtree **root, struct rbtree *y) {
-  struct rbtree *x = rbtree_get_left(y);
-  rbtree_set_left(y, x->right);
-  if (x->right)
-    x->right->parent = y;
-  x->parent = y->parent;
-  if (!y->parent) {
-    *root = x;
-  } else if (y == y->parent->right) {
-    y->parent->right = x;
-  } else {
-    rbtree_set_left(y->parent, x);
-  }
-  x->right = y;
-  y->parent = x;
-}
-
-static void rbtree_insert_fixup(struct rbtree **root, struct rbtree *node) {
-  rbtree_set_red(node, 1);
-  while (node != *root && rbtree_get_red(node->parent)) {
-    if (node->parent == rbtree_get_left(node->parent->parent)) {
-      struct rbtree *uncle = node->parent->parent->right;
-      if (uncle && rbtree_get_red(uncle)) {
-        rbtree_set_red(node->parent, 0);
-        rbtree_set_red(uncle, 0);
-        rbtree_set_red(node->parent->parent, 1);
-        node = node->parent->parent;
-      } else {
-        if (node == node->parent->right) {
-          node = node->parent;
-          rbtree_rotate_left(root, node);
-        }
-        rbtree_set_red(node->parent, 0);
-        rbtree_set_red(node->parent->parent, 1);
-        rbtree_rotate_right(root, node->parent->parent);
-      }
-    } else {
-      struct rbtree *uncle = rbtree_get_left(node->parent->parent);
-      if (uncle && rbtree_get_red(uncle)) {
-        rbtree_set_red(node->parent, 0);
-        rbtree_set_red(uncle, 0);
-        rbtree_set_red(node->parent->parent, 1);
-        node = node->parent->parent;
-      } else {
-        if (node == rbtree_get_left(node->parent)) {
-          node = node->parent;
-          rbtree_rotate_right(root, node);
-        }
-        rbtree_set_red(node->parent, 0);
-        rbtree_set_red(node->parent->parent, 1);
-        rbtree_rotate_left(root, node->parent->parent);
-      }
-    }
-  }
-  rbtree_set_red(*root, 0);
-}
-
-void rbtree_insert(struct rbtree **root, struct rbtree *node,
-                   rbtree_cmp_f *cmp) {
-  bzero(node, sizeof(*node));
-  if (!*root) {
-    *root = node;
-  } else {
-    struct rbtree *search = *root;
-    struct rbtree *parent = 0;
-    do {
-      parent = search;
-      if (cmp(node, search) < 0) {
-        search = rbtree_get_left(search);
-      } else {
-        search = search->right;
-      }
-    } while (search);
-    if (cmp(node, parent) < 0) {
-      rbtree_set_left(parent, node);
-    } else {
-      parent->right = node;
-    }
-    node->parent = parent;
-    rbtree_insert_fixup(root, node);
-  }
-}
-
-static void rbtree_transplant(struct rbtree **root, struct rbtree *u,
-                              struct rbtree *v) {
-  if (!u->parent) {
-    *root = v;
-  } else if (u == rbtree_get_left(u->parent)) {
-    rbtree_set_left(u->parent, v);
-  } else {
-    u->parent->right = v;
-  }
-  if (v)
-    v->parent = u->parent;
-}
-
-static void rbtree_remove_fixup(struct rbtree **root, struct rbtree *node,
-                                struct rbtree *parent) {
-  while (node != *root && (!node || !rbtree_get_red(node))) {
-    if (node == rbtree_get_left(parent)) {
-      struct rbtree *sibling = parent->right;
-      if (rbtree_get_red(sibling)) {
-        rbtree_set_red(sibling, 0);
-        rbtree_set_red(parent, 1);
-        rbtree_rotate_left(root, parent);
-        sibling = parent->right;
-      }
-      if ((!rbtree_get_left(sibling) ||
-           !rbtree_get_red(rbtree_get_left(sibling))) &&
-          (!sibling->right || !rbtree_get_red(sibling->right))) {
-        rbtree_set_red(sibling, 1);
-        node = parent;
-        parent = node->parent;
-      } else {
-        if (!sibling->right || !rbtree_get_red(sibling->right)) {
-          rbtree_set_red(rbtree_get_left(sibling), 0);
-          rbtree_set_red(sibling, 1);
-          rbtree_rotate_right(root, sibling);
-          sibling = parent->right;
-        }
-        rbtree_set_red(sibling, rbtree_get_red(parent));
-        rbtree_set_red(parent, 0);
-        rbtree_set_red(sibling->right, 0);
-        rbtree_rotate_left(root, parent);
-        node = *root;
-        break;
-      }
-    } else {
-      struct rbtree *sibling = rbtree_get_left(parent);
-      if (rbtree_get_red(sibling)) {
-        rbtree_set_red(sibling, 0);
-        rbtree_set_red(parent, 1);
-        rbtree_rotate_right(root, parent);
-        sibling = rbtree_get_left(parent);
-      }
-      if ((!sibling->right || !rbtree_get_red(sibling->right)) &&
-          (!rbtree_get_left(sibling) ||
-           !rbtree_get_red(rbtree_get_left(sibling)))) {
-        rbtree_set_red(sibling, 1);
-        node = parent;
-        parent = node->parent;
-      } else {
-        if (!rbtree_get_left(sibling) ||
-            !rbtree_get_red(rbtree_get_left(sibling))) {
-          rbtree_set_red(sibling->right, 0);
-          rbtree_set_red(sibling, 1);
-          rbtree_rotate_left(root, sibling);
-          sibling = rbtree_get_left(parent);
-        }
-        rbtree_set_red(sibling, rbtree_get_red(parent));
-        rbtree_set_red(parent, 0);
-        rbtree_set_red(rbtree_get_left(sibling), 0);
-        rbtree_rotate_right(root, parent);
-        node = *root;
-        break;
-      }
-    }
-  }
-  if (node)
-    rbtree_set_red(node, 0);
-}
-
-void rbtree_remove(struct rbtree **root, struct rbtree *node) {
-  struct rbtree *y = node;
-  struct rbtree *x = 0;
-  struct rbtree *x_parent = 0;
-  int y_original_color = rbtree_get_red(y);
-  if (!rbtree_get_left(node)) {
-    x = node->right;
-    rbtree_transplant(root, node, node->right);
-    x_parent = node->parent;
-  } else if (!node->right) {
-    x = rbtree_get_left(node);
-    rbtree_transplant(root, node, rbtree_get_left(node));
-    x_parent = node->parent;
-  } else {
-    y = rbtree_first(node->right);
-    y_original_color = rbtree_get_red(y);
-    x = y->right;
-    if (y->parent == node) {
-      if (x)
-        x->parent = y;
-      x_parent = y;
-    } else {
-      rbtree_transplant(root, y, y->right);
-      y->right = node->right;
-      y->right->parent = y;
-      x_parent = y->parent;
-    }
-    rbtree_transplant(root, node, y);
-    rbtree_set_left(y, rbtree_get_left(node));
-    rbtree_get_left(y)->parent = y;
-    rbtree_set_red(y, rbtree_get_red(node));
-  }
-  if (!y_original_color)
-    rbtree_remove_fixup(root, x, x_parent);
-}
-
-struct rbtree *rbtree_get(const struct rbtree *node, const struct rbtree *key,
-                          rbtree_cmp_f *cmp) {
-  while (node) {
-    int c = cmp(key, node);
-    if (c < 0) {
-      node = rbtree_get_left(node);
-    } else if (c > 0) {
-      node = node->right;
-    } else {
-      return (struct rbtree *)node;
-    }
-  }
-  return 0;
-}
-
-struct rbtree *rbtree_floor(const struct rbtree *node, const struct rbtree *key,
-                            rbtree_cmp_f *cmp) {
-  while (node) {
-    if (cmp(key, node) < 0) {
-      node = rbtree_get_left(node);
-    } else {
-      node = node->right;
-    }
-  }
-  return (struct rbtree *)node;
-}
-
-struct rbtree *rbtree_ceil(const struct rbtree *node, const struct rbtree *key,
-                           rbtree_cmp_f *cmp) {
-  while (node) {
-    if (cmp(node, key) < 0) {
-      node = rbtree_get_left(node);
-    } else {
-      node = node->right;
-    }
-  }
-  return (struct rbtree *)node;
-}

libc/intrin/rbtree.h

@@ -1,57 +0,0 @@
-#ifdef _COSMO_SOURCE
-#ifndef COSMOPOLITAN_RBTREE_H_
-#define COSMOPOLITAN_RBTREE_H_
-#define rbtree_ceil   __rbtree_ceil
-#define rbtree_first  __rbtree_first
-#define rbtree_floor  __rbtree_floor
-#define rbtree_get    __rbtree_get
-#define rbtree_insert __rbtree_insert
-#define rbtree_last   __rbtree_last
-#define rbtree_next   __rbtree_next
-#define rbtree_prev   __rbtree_prev
-#define rbtree_remove __rbtree_remove
-COSMOPOLITAN_C_START_
-
-#define RBTREE_CONTAINER(t, f, p) ((t *)(((char *)(p)) - offsetof(t, f)))
-
-struct rbtree {
-  uintptr_t word;
-  struct rbtree *right;
-  struct rbtree *parent;
-};
-
-typedef int rbtree_cmp_f(const struct rbtree *, const struct rbtree *);
-
-static inline struct rbtree *rbtree_get_left(const struct rbtree *node) {
-  return (struct rbtree *)(node->word & -2);
-}
-
-static inline void rbtree_set_left(struct rbtree *node, struct rbtree *left) {
-  node->word = (uintptr_t)left | (node->word & 1);
-}
-
-static inline int rbtree_get_red(const struct rbtree *node) {
-  return node->word & 1;
-}
-
-static inline void rbtree_set_red(struct rbtree *node, int red) {
-  node->word &= -2;
-  node->word |= red;
-}
-
-struct rbtree *rbtree_next(struct rbtree *) libcesque;
-struct rbtree *rbtree_prev(struct rbtree *) libcesque;
-struct rbtree *rbtree_first(struct rbtree *) libcesque;
-struct rbtree *rbtree_last(struct rbtree *) libcesque;
-void rbtree_remove(struct rbtree **, struct rbtree *) libcesque;
-void rbtree_insert(struct rbtree **, struct rbtree *, rbtree_cmp_f *) libcesque;
-struct rbtree *rbtree_get(const struct rbtree *, const struct rbtree *,
-                          rbtree_cmp_f *) libcesque;
-struct rbtree *rbtree_ceil(const struct rbtree *, const struct rbtree *,
-                           rbtree_cmp_f *) libcesque;
-struct rbtree *rbtree_floor(const struct rbtree *, const struct rbtree *,
-                            rbtree_cmp_f *) libcesque;
-
-COSMOPOLITAN_C_END_
-#endif /* COSMOPOLITAN_RBTREE_H_ */
-#endif /* _COSMO_SOURCE */

libc/intrin/tree.c

@@ -0,0 +1,271 @@
+// Copyright 2024 Justine Alexandra Roberts Tunney
+//
+// Permission to use, copy, modify, and/or distribute this software for
+// any purpose with or without fee is hereby granted, provided that the
+// above copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
+// WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
+// AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
+// DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
+// PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include "tree.h"
+
+struct Tree *tree_last(struct Tree *node) {
+  while (node && node->right)
+    node = node->right;
+  return node;
+}
+
+struct Tree *tree_first(struct Tree *node) {
+  while (node && tree_get_left(node))
+    node = tree_get_left(node);
+  return node;
+}
+
+struct Tree *tree_next(struct Tree *node) {
+  if (!node)
+    return 0;
+  if (node->right)
+    return tree_first(node->right);
+  struct Tree *parent = node->parent;
+  while (parent && node == parent->right) {
+    node = parent;
+    parent = parent->parent;
+  }
+  return parent;
+}
+
+struct Tree *tree_prev(struct Tree *node) {
+  struct Tree *parent;
+  if (!node)
+    return 0;
+  if (tree_get_left(node))
+    return tree_last(tree_get_left(node));
+  parent = node->parent;
+  while (parent && node == tree_get_left(parent)) {
+    node = parent;
+    parent = parent->parent;
+  }
+  return parent;
+}
+
+static void tree_rotate_left(struct Tree **root, struct Tree *x) {
+  struct Tree *y = x->right;
+  x->right = tree_get_left(y);
+  if (tree_get_left(y))
+    tree_get_left(y)->parent = x;
+  y->parent = x->parent;
+  if (!x->parent) {
+    *root = y;
+  } else if (x == tree_get_left(x->parent)) {
+    tree_set_left(x->parent, y);
+  } else {
+    x->parent->right = y;
+  }
+  tree_set_left(y, x);
+  x->parent = y;
+}
+
+static void tree_rotate_right(struct Tree **root, struct Tree *y) {
+  struct Tree *x = tree_get_left(y);
+  tree_set_left(y, x->right);
+  if (x->right)
+    x->right->parent = y;
+  x->parent = y->parent;
+  if (!y->parent) {
+    *root = x;
+  } else if (y == y->parent->right) {
+    y->parent->right = x;
+  } else {
+    tree_set_left(y->parent, x);
+  }
+  y->parent = x;
+  x->right = y;
+}
+
+static void tree_rebalance_insert(struct Tree **root, struct Tree *node) {
+  struct Tree *uncle;
+  tree_set_red(node, 1);
+  while (node != *root && tree_get_red(node->parent)) {
+    if (node->parent == tree_get_left(node->parent->parent)) {
+      uncle = node->parent->parent->right;
+      if (uncle && tree_get_red(uncle)) {
+        tree_set_red(node->parent, 0);
+        tree_set_red(uncle, 0);
+        tree_set_red(node->parent->parent, 1);
+        node = node->parent->parent;
+      } else {
+        if (node == node->parent->right) {
+          node = node->parent;
+          tree_rotate_left(root, node);
+        }
+        tree_set_red(node->parent, 0);
+        tree_set_red(node->parent->parent, 1);
+        tree_rotate_right(root, node->parent->parent);
+      }
+    } else {
+      uncle = tree_get_left(node->parent->parent);
+      if (uncle && tree_get_red(uncle)) {
+        tree_set_red(node->parent, 0);
+        tree_set_red(uncle, 0);
+        tree_set_red(node->parent->parent, 1);
+        node = node->parent->parent;
+      } else {
+        if (node == tree_get_left(node->parent)) {
+          node = node->parent;
+          tree_rotate_right(root, node);
+        }
+        tree_set_red(node->parent, 0);
+        tree_set_red(node->parent->parent, 1);
+        tree_rotate_left(root, node->parent->parent);
+      }
+    }
+  }
+  tree_set_red(*root, 0);
+}
+
+void tree_insert(struct Tree **root, struct Tree *node, tree_cmp_f *cmp) {
+  struct Tree *search, *parent;
+  node->word = 0;
+  node->right = 0;
+  node->parent = 0;
+  if (!*root) {
+    *root = node;
+  } else {
+    search = *root;
+    parent = 0;
+    do {
+      parent = search;
+      if (cmp(node, search) < 0) {
+        search = tree_get_left(search);
+      } else {
+        search = search->right;
+      }
+    } while (search);
+    if (cmp(node, parent) < 0) {
+      tree_set_left(parent, node);
+    } else {
+      parent->right = node;
+    }
+    node->parent = parent;
+    tree_rebalance_insert(root, node);
+  }
+}
+
+static void tree_transplant(struct Tree **root, struct Tree *u,
+                            struct Tree *v) {
+  if (!u->parent) {
+    *root = v;
+  } else if (u == tree_get_left(u->parent)) {
+    tree_set_left(u->parent, v);
+  } else {
+    u->parent->right = v;
+  }
+  if (v)
+    v->parent = u->parent;
+}
+
+static void tree_rebalance_remove(struct Tree **root, struct Tree *node,
+                                  struct Tree *parent) {
+  struct Tree *sibling;
+  while (node != *root && (!node || !tree_get_red(node))) {
+    if (node == tree_get_left(parent)) {
+      sibling = parent->right;
+      if (tree_get_red(sibling)) {
+        tree_set_red(sibling, 0);
+        tree_set_red(parent, 1);
+        tree_rotate_left(root, parent);
+        sibling = parent->right;
+      }
+      if ((!tree_get_left(sibling) || !tree_get_red(tree_get_left(sibling))) &&
+          (!sibling->right || !tree_get_red(sibling->right))) {
+        tree_set_red(sibling, 1);
+        node = parent;
+        parent = node->parent;
+      } else {
+        if (!sibling->right || !tree_get_red(sibling->right)) {
+          tree_set_red(tree_get_left(sibling), 0);
+          tree_set_red(sibling, 1);
+          tree_rotate_right(root, sibling);
+          sibling = parent->right;
+        }
+        tree_set_red(sibling, tree_get_red(parent));
+        tree_set_red(parent, 0);
+        tree_set_red(sibling->right, 0);
+        tree_rotate_left(root, parent);
+        node = *root;
+        break;
+      }
+    } else {
+      sibling = tree_get_left(parent);
+      if (tree_get_red(sibling)) {
+        tree_set_red(sibling, 0);
+        tree_set_red(parent, 1);
+        tree_rotate_right(root, parent);
+        sibling = tree_get_left(parent);
+      }
+      if ((!sibling->right || !tree_get_red(sibling->right)) &&
+          (!tree_get_left(sibling) || !tree_get_red(tree_get_left(sibling)))) {
+        tree_set_red(sibling, 1);
+        node = parent;
+        parent = node->parent;
+      } else {
+        if (!tree_get_left(sibling) || !tree_get_red(tree_get_left(sibling))) {
+          tree_set_red(sibling->right, 0);
+          tree_set_red(sibling, 1);
+          tree_rotate_left(root, sibling);
+          sibling = tree_get_left(parent);
+        }
+        tree_set_red(sibling, tree_get_red(parent));
+        tree_set_red(parent, 0);
+        tree_set_red(tree_get_left(sibling), 0);
+        tree_rotate_right(root, parent);
+        node = *root;
+        break;
+      }
+    }
+  }
+  if (node)
+    tree_set_red(node, 0);
+}
+
+void tree_remove(struct Tree **root, struct Tree *node) {
+  struct Tree *x = 0;
+  struct Tree *y = node;
+  struct Tree *x_parent = 0;
+  int y_original_color = tree_get_red(y);
+  if (!tree_get_left(node)) {
+    x = node->right;
+    tree_transplant(root, node, node->right);
+    x_parent = node->parent;
+  } else if (!node->right) {
+    x = tree_get_left(node);
+    tree_transplant(root, node, tree_get_left(node));
+    x_parent = node->parent;
+  } else {
+    y = tree_first(node->right);
+    y_original_color = tree_get_red(y);
+    x = y->right;
+    if (y->parent == node) {
+      if (x)
+        x->parent = y;
+      x_parent = y;
+    } else {
+      tree_transplant(root, y, y->right);
+      y->right = node->right;
+      y->right->parent = y;
+      x_parent = y->parent;
+    }
+    tree_transplant(root, node, y);
+    tree_set_left(y, tree_get_left(node));
+    tree_get_left(y)->parent = y;
+    tree_set_red(y, tree_get_red(node));
+  }
+  if (!y_original_color)
+    tree_rebalance_remove(root, x, x_parent);
+}

libc/intrin/tree.h

@@ -0,0 +1,91 @@
+#ifndef COSMOPOLITAN_TREE_H_
+#define COSMOPOLITAN_TREE_H_
+#define tree_first  __tree_first
+#define tree_insert __tree_insert
+#define tree_last   __tree_last
+#define tree_next   __tree_next
+#define tree_prev   __tree_prev
+#define tree_remove __tree_remove
+COSMOPOLITAN_C_START_
+
+#define TREE_CONTAINER(t, f, p) ((t *)(((char *)(p)) - offsetof(t, f)))
+
+struct Tree {
+  uintptr_t word;
+  struct Tree *right;
+  struct Tree *parent;
+};
+
+typedef int tree_search_f(const void *, const struct Tree *);
+typedef int tree_cmp_f(const struct Tree *, const struct Tree *);
+
+forceinline struct Tree *tree_get_left(const struct Tree *node) {
+  return (struct Tree *)(node->word & -2);
+}
+
+static inline void tree_set_left(struct Tree *node, struct Tree *left) {
+  node->word = (uintptr_t)left | (node->word & 1);
+}
+
+static inline int tree_get_red(const struct Tree *node) {
+  return node->word & 1;
+}
+
+static inline void tree_set_red(struct Tree *node, int red) {
+  node->word &= -2;
+  node->word |= red;
+}
+
+forceinline optimizespeed struct Tree *tree_floor(const struct Tree *node,
+                                                  const void *key,
+                                                  tree_search_f *cmp) {
+  struct Tree *left = 0;
+  while (node) {
+    if (cmp(key, node) >= 0) {
+      left = (struct Tree *)node;
+      node = tree_get_left(node);
+    } else {
+      node = node->right;
+    }
+  }
+  return left;
+}
+
+static inline struct Tree *tree_ceil(const struct Tree *node, const void *key,
+                                     tree_search_f *cmp) {
+  struct Tree *right = 0;
+  while (node) {
+    if (cmp(key, node) < 0) {
+      right = (struct Tree *)node;
+      node = tree_get_left(node);
+    } else {
+      node = node->right;
+    }
+  }
+  return right;
+}
+
+static inline struct Tree *tree_get(const struct Tree *node, const void *key,
+                                    tree_search_f *cmp) {
+  while (node) {
+    int c = cmp(key, node);
+    if (c < 0) {
+      node = tree_get_left(node);
+    } else if (c > 0) {
+      node = node->right;
+    } else {
+      return (struct Tree *)node;
+    }
+  }
+  return 0;
+}
+
+struct Tree *tree_next(struct Tree *) libcesque;
+struct Tree *tree_prev(struct Tree *) libcesque;
+struct Tree *tree_first(struct Tree *) libcesque;
+struct Tree *tree_last(struct Tree *) libcesque;
+void tree_remove(struct Tree **, struct Tree *) libcesque;
+void tree_insert(struct Tree **, struct Tree *, tree_cmp_f *) libcesque;
+
+COSMOPOLITAN_C_END_
+#endif /* COSMOPOLITAN_TREE_H_ */

libc/log/oncrash_amd64.c

@@ -241,14 +241,10 @@ static relegated void ShowCrashReport(int err, int sig, siginfo_t *si,
     klog(buf, p - buf);
   }
   kprintf("\n");
-  if (!IsWindows()) {
+  __print_maps(15);
-    __print_maps();
+  if (__argv)
-  }
+    for (i = 0; i < __argc; ++i)
-  if (__argv) {
-    for (i = 0; i < __argc; ++i) {
       kprintf("%s ", __argv[i]);
-    }
-  }
   kprintf("\n");
 }
 

libc/proc/fork-nt.c

@@ -30,6 +30,7 @@
 #include "libc/intrin/kprintf.h"
 #include "libc/intrin/maps.h"
 #include "libc/intrin/strace.internal.h"
+#include "libc/intrin/tree.h"
 #include "libc/intrin/weaken.h"
 #include "libc/macros.internal.h"
 #include "libc/nt/createfile.h"
@@ -95,7 +96,7 @@ static inline textwindows ssize_t ForkIo(int64_t h, char *p, size_t n,
   size_t i;
   uint32_t x;
   for (i = 0; i < n; i += x) {
-    if (!f(h, p + i, n - i, &x, NULL))
+    if (!f(h, p + i, n - i, &x, 0))
       return __winerr();
     if (!x)
       break;
@@ -109,10 +110,11 @@ static dontinline textwindows ssize_t ForkIo2(
     const char *sf, bool ischild) {
   ssize_t rc = ForkIo(h, buf, n, fn);
   if (ischild) {
-    __tls_enabled_set(false);  // prevent tls crash in kprintf
+    // prevent crashes
+    __tls_enabled_set(false);
     __pid = __imp_GetCurrentProcessId();
     __klog_handle = 0;
-    __maps.used = 0;
+    __maps.maps = 0;
   }
   NTTRACE("%s(%ld, %p, %'zu) → %'zd% m", sf, h, buf, n, rc);
   return rc;
@@ -121,9 +123,11 @@ static dontinline textwindows ssize_t ForkIo2(
 static dontinline textwindows bool WriteAll(int64_t h, void *buf, size_t n) {
   bool ok;
   ok = ForkIo2(h, buf, n, (void *)WriteFile, "WriteFile", false) != -1;
-  if (!ok)
+  if (!ok) {
-    AbortFork("WriteAll");
+    STRACE("fork() failed in parent due to WriteAll(%ld, %p, %'zu) → %u", h,
-  // Sleep(10);
+           buf, n, GetLastError());
+    __print_maps(0);
+  }
   return ok;
 }
 
@@ -185,30 +189,6 @@ static textwindows void *Malloc(size_t size) {
   return HeapAlloc(GetProcessHeap(), 0, size);
 }
 
-static textwindows void Free(void *addr) {
-  HeapFree(GetProcessHeap(), 0, addr);
-}
-
-static int CountMaps(struct Dll *maps) {
-  int count = 0;
-  for (struct Dll *e = dll_first(maps); e; e = dll_next(maps, e))
-    ++count;
-  return count;
-}
-
-static struct Map **SortMaps(struct Dll *maps, int count) {
-  int j, i = 0;
-  struct Map **sorted = Malloc(count * sizeof(struct Map *));
-  for (struct Dll *e = dll_first(maps); e; e = dll_next(maps, e)) {
-    struct Map *map = MAP_CONTAINER(e);
-    for (j = i; j > 0 && sorted[j - 1]->addr > map->addr; --j)
-      sorted[j] = sorted[j - 1];
-    sorted[j] = map;
-    ++i;
-  }
-  return sorted;
-}
-
 textwindows void WinMainForked(void) {
   jmp_buf jb;
   int64_t reader;
@@ -233,35 +213,30 @@ textwindows void WinMainForked(void) {
   ReadOrDie(reader, jb, sizeof(jb));
 
   // read memory mappings from parent process
-  int n = 0;
+  struct Tree *maps = 0;
-  struct Dll *maps = 0;
   for (;;) {
     struct Map *map = Malloc(sizeof(struct Map));
     ReadOrDie(reader, map, sizeof(struct Map));
-    if (map->addr == MAP_FAILED) {
+    if (map->addr == MAP_FAILED)
-      Free(map);
       break;
+    tree_insert(&maps, &map->tree, __maps_compare);
   }
-    dll_init(&map->elem);
-    dll_make_first(&maps, &map->elem);
-    ++n;
-  }
-
-  // created sorted array of maps
-  struct Map **sorted = SortMaps(maps, n);
 
   // map memory into process
   int granularity = __granularity();
-  for (int i = 0; i < n; ++i) {
+  for (struct Tree *e = tree_first(maps); e; e = tree_next(e)) {
-    struct Map *map = sorted[i];
+    struct Map *map = MAP_TREE_CONTAINER(e);
     if ((uintptr_t)map->addr & (granularity - 1))
       continue;
-    size_t size = map->size;
     // get true length in case mprotect() chopped up actual win32 map
-    for (int j = i + 1;
+    size_t size = map->size;
-         j < n && sorted[j]->hand == -1 && map->addr + size == sorted[j]->addr;
+    for (struct Tree *e2 = tree_next(e); e2; e2 = tree_next(e2)) {
-         ++j) {
+      struct Map *map2 = MAP_TREE_CONTAINER(e2);
-      size += sorted[j]->size;
+      if (map2->hand == -1 && map->addr + size == map2->addr) {
+        size += map2->size;
+      } else {
+        break;
+      }
     }
     // obtain the most permissive access possible
     unsigned prot, access;
@@ -295,11 +270,11 @@ textwindows void WinMainForked(void) {
 
   // fixup memory manager
   __maps.free = 0;
-  __maps.used = 0;
+  __maps.maps = 0;
   __maps.count = 0;
   __maps.pages = 0;
-  for (int i = 0; i < n; ++i) {
+  for (struct Tree *e = tree_first(maps); e; e = tree_next(e)) {
-    struct Map *map = sorted[i];
+    struct Map *map = MAP_TREE_CONTAINER(e);
     __maps.count += 1;
     __maps.pages += (map->size + getpagesize() - 1) / getpagesize();
     unsigned old_protect;
@@ -307,8 +282,7 @@ textwindows void WinMainForked(void) {
                         &old_protect))
       AbortFork("VirtualProtect");
   }
-  Free(sorted);
+  __maps.maps = maps;
-  __maps.used = maps;
   __maps_init();
 
   // mitosis complete
@@ -393,19 +367,13 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
       if (spawnrc != -1) {
         CloseHandle(procinfo.hThread);
         ok = WriteAll(writer, jb, sizeof(jb));
-        int count = 0;
         // this list will be populated with the maps we're transferring
-        struct Dll *e2, *maps = 0;
+        for (struct Map *map = __maps_first(); ok && map;
-        for (struct Dll *e = dll_first(__maps.used); ok && e; e = e2) {
+             map = __maps_next(map)) {
-          e2 = dll_next(__maps.used, e);
-          struct Map *map = MAP_CONTAINER(e);
           if (MAX((char *)__executable_start, map->addr) <
               MIN((char *)_end, map->addr + map->size))
             continue;  // executable image is loaded by windows
-          dll_remove(&__maps.used, e);
-          dll_make_last(&maps, e);
           ok = WriteAll(writer, map, sizeof(*map));
-          ++count;
         }
         // send a terminating Map struct to child
         if (ok) {
@@ -415,40 +383,44 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
         }
         // now write content of each map to child
         int granularity = __granularity();
-        struct Map **sorted = SortMaps(maps, count);
+        for (struct Map *map = __maps_first(); ok && map;
-        uint32_t *old_protect = Malloc(count * 4);
+             map = __maps_next(map)) {
-        for (int i = 0; ok && i < count; ++i) {
-          struct Map *map = sorted[i];
           // we only need to worry about the base mapping
           if ((uintptr_t)map->addr & (granularity - 1))
             continue;
+          if (MAX((char *)__executable_start, map->addr) <
+              MIN((char *)_end, map->addr + map->size))
+            continue;  // executable image is loaded by windows
           // shared mappings don't need to be copied
           if ((map->flags & MAP_TYPE) == MAP_SHARED)
             continue;
           // get true length in case mprotect() chopped up actual win32 map
-          int j;
           size_t size = map->size;
-          for (j = i + 1; j < count && sorted[j]->hand == -1 &&
+          for (struct Map *map2 = __maps_next(map); map2;
-                          map->addr + size == sorted[j]->addr;
+               map2 = __maps_next(map2)) {
-               ++j) {
+            if (map2->hand == -1 && map->addr + size == map2->addr) {
-            size += sorted[j]->size;
+              size += map2->size;
+            } else {
+              break;
             }
-          for (int k = i; ok && k < j; ++k)
+          }
-            if (!(sorted[k]->prot & PROT_READ))
+          for (struct Map *map2 = map; ok && map2; map2 = __maps_next(map2)) {
+            if (!(map2->prot & PROT_READ))
+              if (map->addr >= map2->addr && map->addr < map->addr + size)
                 ok = VirtualProtect(
-                  sorted[k]->addr, sorted[k]->size,
+                    map2->addr, map2->size,
-                  __prot2nt(sorted[k]->prot | PROT_READ, map->iscow),
+                    __prot2nt(map2->prot | PROT_READ, map2->iscow),
-                  &old_protect[k]);
+                    &map2->oldprot);
+          }
           if (ok)
             ok = WriteAll(writer, map->addr, size);
-          for (int k = i; ok && k < j; ++k)
+          for (struct Map *map2 = map; ok && map2; map2 = __maps_next(map2)) {
-            if (!(sorted[k]->prot & PROT_READ))
+            if (!(map2->prot & PROT_READ))
-              ok = VirtualProtect(sorted[k]->addr, sorted[k]->size,
+              if (map->addr >= map2->addr && map->addr < map->addr + size)
-                                  old_protect[k], &old_protect[k]);
+                ok = VirtualProtect(map2->addr, map2->size, map2->oldprot,
+                                    &map2->oldprot);
+          }
         }
-        Free(old_protect);
-        Free(sorted);
-        dll_make_first(&__maps.used, maps);
         if (ok)
           ok = WriteAll(writer, __data_start, __data_end - __data_start);
         if (ok)
@@ -466,6 +438,7 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
         } else {
           TerminateProcess(procinfo.hProcess, SIGKILL);
           CloseHandle(procinfo.hProcess);
+          rc = -1;
         }
       }
     }
@@ -473,9 +446,8 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
       CloseHandle(reader);
     if (writer != -1)
       CloseHandle(writer);
-    if (rc == -1 && errno != ENOMEM) {
+    if (rc == -1 && errno != ENOMEM)
       eagain();  // posix fork() only specifies two errors
-    }
   } else {
     rc = 0;
     // re-apply code morphing for thread-local storage

libc/proc/fork.c

@@ -22,6 +22,7 @@
 #include "libc/calls/state.internal.h"
 #include "libc/calls/struct/sigset.h"
 #include "libc/calls/struct/sigset.internal.h"
+#include "libc/calls/struct/timespec.h"
 #include "libc/calls/syscall-nt.internal.h"
 #include "libc/calls/syscall-sysv.internal.h"
 #include "libc/dce.h"
@@ -45,7 +46,6 @@
 #include "libc/thread/posixthread.internal.h"
 #include "libc/thread/tls.h"
 
-extern pthread_mutex_t nsync_waiters_mu;
 extern pthread_mutex_t _pthread_lock_obj;
 
 static void _onfork_prepare(void) {
@@ -54,11 +54,10 @@ static void _onfork_prepare(void) {
   _pthread_lock();
   __maps_lock();
   __fds_lock();
-  pthread_mutex_lock(&nsync_waiters_mu);
+  LOCKTRACE("READY TO ROCK AND ROLL");
 }
 
 static void _onfork_parent(void) {
-  pthread_mutex_unlock(&nsync_waiters_mu);
   __fds_unlock();
   __maps_unlock();
   _pthread_unlock();
@@ -68,7 +67,6 @@ static void _onfork_parent(void) {
 
 static void _onfork_child(void) {
   __fds_lock_obj = (pthread_mutex_t)PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
-  nsync_waiters_mu = (pthread_mutex_t)PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
   _pthread_lock_obj = (pthread_mutex_t)PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
   atomic_store_explicit(&__maps.lock, 0, memory_order_relaxed);
   atomic_store_explicit(&__get_tls()->tib_relock_maps, 0, memory_order_relaxed);
@@ -77,7 +75,9 @@ static void _onfork_child(void) {
 }
 
 int _fork(uint32_t dwCreationFlags) {
+  long micros;
   struct Dll *e;
+  struct timespec started;
   int ax, dx, tid, parent;
   parent = __pid;
   BLOCK_SIGNALS;
@@ -85,11 +85,13 @@ int _fork(uint32_t dwCreationFlags) {
     __proc_lock();
   if (__threaded)
     _onfork_prepare();
+  started = timespec_real();
   if (!IsWindows()) {
     ax = sys_fork();
   } else {
     ax = sys_fork_nt(dwCreationFlags);
   }
+  micros = timespec_tomicros(timespec_sub(timespec_real(), started));
   if (!ax) {
 
     // get new process id
@@ -136,15 +138,14 @@ int _fork(uint32_t dwCreationFlags) {
     // run user fork callbacks
     if (__threaded)
       _onfork_child();
-    STRACE("fork() → 0 (child of %d)", parent);
+    STRACE("fork() → 0 (child of %d; took %ld us)", parent, micros);
   } else {
     // this is the parent process
-    if (__threaded) {
+    if (__threaded)
       _onfork_parent();
-    }
     if (IsWindows())
       __proc_unlock();
-    STRACE("fork() → %d% m", ax);
+    STRACE("fork() → %d% m (took %ld us)", ax, micros);
   }
   ALLOW_SIGNALS;
   return ax;

libc/proc/posix_spawn.c

@@ -482,9 +482,8 @@ errno_t posix_spawn(int *pid, const char *path,
                     const posix_spawn_file_actions_t *file_actions,
                     const posix_spawnattr_t *attrp, char *const argv[],
                     char *const envp[]) {
-  if (IsWindows()) {
+  if (IsWindows())
     return posix_spawn_nt(pid, path, file_actions, attrp, argv, envp);
-  }
   int pfds[2];
   bool use_pipe;
   volatile int status = 0;
@@ -516,66 +515,55 @@ errno_t posix_spawn(int *pid, const char *path,
         sigaction(sig, &dfl, 0);
       }
     }
-    if (flags & POSIX_SPAWN_SETSID) {
+    if (flags & POSIX_SPAWN_SETSID)
       setsid();
-    }
+    if ((flags & POSIX_SPAWN_SETPGROUP) && setpgid(0, (*attrp)->pgroup))
-    if ((flags & POSIX_SPAWN_SETPGROUP) && setpgid(0, (*attrp)->pgroup)) {
       goto ChildFailed;
-    }
+    if ((flags & POSIX_SPAWN_RESETIDS) && setgid(getgid()))
-    if ((flags & POSIX_SPAWN_RESETIDS) && setgid(getgid())) {
       goto ChildFailed;
-    }
+    if ((flags & POSIX_SPAWN_RESETIDS) && setuid(getuid()))
-    if ((flags & POSIX_SPAWN_RESETIDS) && setuid(getuid())) {
       goto ChildFailed;
-    }
     if (file_actions) {
       struct _posix_faction *a;
       for (a = *file_actions; a; a = a->next) {
         if (use_pipe && pfds[1] == a->fildes) {
           int p2;
-          if ((p2 = dup(pfds[1])) == -1) {
+          if ((p2 = dup(pfds[1])) == -1)
             goto ChildFailed;
-          }
           lost_cloexec = true;
           close(pfds[1]);
           pfds[1] = p2;
         }
         switch (a->action) {
           case _POSIX_SPAWN_CLOSE:
-            if (close(a->fildes)) {
+            if (close(a->fildes))
               goto ChildFailed;
-            }
             break;
           case _POSIX_SPAWN_DUP2:
-            if (dup2(a->fildes, a->newfildes) == -1) {
+            if (dup2(a->fildes, a->newfildes) == -1)
               goto ChildFailed;
-            }
             break;
           case _POSIX_SPAWN_OPEN: {
             int t;
-            if ((t = openat(AT_FDCWD, a->path, a->oflag, a->mode)) == -1) {
+            if ((t = openat(AT_FDCWD, a->path, a->oflag, a->mode)) == -1)
               goto ChildFailed;
-            }
             if (t != a->fildes) {
               if (dup2(t, a->fildes) == -1) {
                 close(t);
                 goto ChildFailed;
               }
-              if (close(t)) {
+              if (close(t))
                 goto ChildFailed;
             }
-            }
             break;
           }
           case _POSIX_SPAWN_CHDIR:
-            if (chdir(a->path) == -1) {
+            if (chdir(a->path) == -1)
               goto ChildFailed;
-            }
             break;
           case _POSIX_SPAWN_FCHDIR:
-            if (fchdir(a->fildes) == -1) {
+            if (fchdir(a->fildes) == -1)
               goto ChildFailed;
-            }
             break;
           default:
             __builtin_unreachable();
@@ -583,18 +571,14 @@ errno_t posix_spawn(int *pid, const char *path,
       }
     }
     if (IsLinux() || IsFreebsd() || IsNetbsd()) {
-      if (flags & POSIX_SPAWN_SETSCHEDULER) {
+      if (flags & POSIX_SPAWN_SETSCHEDULER)
         if (sched_setscheduler(0, (*attrp)->schedpolicy,
-                               &(*attrp)->schedparam) == -1) {
+                               &(*attrp)->schedparam) == -1)
           goto ChildFailed;
-        }
+      if (flags & POSIX_SPAWN_SETSCHEDPARAM)
-      }
+        if (sched_setparam(0, &(*attrp)->schedparam))
-      if (flags & POSIX_SPAWN_SETSCHEDPARAM) {
-        if (sched_setparam(0, &(*attrp)->schedparam)) {
           goto ChildFailed;
     }
-      }
-    }
     if (flags & POSIX_SPAWN_SETRLIMIT) {
       int rlimset = (*attrp)->rlimset;
       while (rlimset) {
@@ -608,9 +592,8 @@ errno_t posix_spawn(int *pid, const char *path,
         }
       }
     }
-    if (lost_cloexec) {
+    if (lost_cloexec)
       fcntl(pfds[1], F_SETFD, FD_CLOEXEC);
-    }
     if (flags & POSIX_SPAWN_SETSIGMASK) {
       childmask = (*attrp)->sigmask;
     } else {
@@ -636,9 +619,8 @@ errno_t posix_spawn(int *pid, const char *path,
     if (!use_pipe) {
       res = status;
     } else {
-      if (can_clobber) {
+      if (can_clobber)
         atomic_store_explicit(&has_vfork, true, memory_order_release);
-      }
       res = 0;
       read(pfds[0], &res, sizeof(res));
     }
@@ -651,9 +633,8 @@ errno_t posix_spawn(int *pid, const char *path,
   } else {
     res = errno;
   }
-  if (use_pipe) {
+  if (use_pipe)
     close(pfds[0]);
-  }
 ParentFailed:
   sigprocmask(SIG_SETMASK, &oldmask, 0);
   pthread_setcancelstate(cs, 0);

libc/runtime/mapstack.c

@@ -50,9 +50,10 @@ void *NewCosmoStack(void) {
     if (IsOpenbsd() && __sys_mmap(p, n, PROT_READ | PROT_WRITE,
                                   MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD |
                                       MAP_STACK_OPENBSD,
-                                  -1, 0, 0) != p) {
+                                  -1, 0, 0) != p)
+      notpossible;
+    if (mprotect(p, GetGuardSize(), PROT_NONE | PROT_GUARD))
       notpossible;
-    }
     return p;
   } else {
     return 0;

libc/runtime/runtime.h

@@ -91,7 +91,7 @@ void ShowCrashReports(void) libcesque;
 int ftrace_install(void) libcesque;
 int ftrace_enabled(int) libcesque;
 int strace_enabled(int) libcesque;
-void __print_maps(void) libcesque;
+void __print_maps(size_t) libcesque;
 void __printargs(const char *) libcesque;
 /* builtin sh-like system/popen dsl */
 int _cocmd(int, char **, char **) libcesque;

libc/stdio/printargs.c

@@ -468,7 +468,7 @@ textstartup void __printargs(const char *prologue) {
 
   PRINT("");
   PRINT("MEMTRACK");
-  __print_maps();
+  __print_maps(0);
 
   PRINT("");
   PRINT("TERMIOS");

libc/sysv/consts.sh

@@ -292,6 +292,7 @@ syscon	mprot	PROT_NONE				0			0			0			0			0			0			0			0			# mmap, mprotect, unix
 syscon	mprot	PROT_READ				1			1			1			1			1			1			1			1			# mmap, mprotect, unix consensus
 syscon	mprot	PROT_WRITE				2			2			2			2			2			2			2			2			# mmap, mprotect, unix consensus
 syscon	mprot	PROT_EXEC				4			4			4			4			4			4			4			4			# mmap, mprotect, unix consensus
+syscon	mprot	PROT_GUARD				0			0			0			0			0			0			0			0x100			# mmap, mprotect, unix consensus
 
 #	mremap() flags
 #	the revolutionary praxis of realloc()

libc/sysv/consts/PROT_GUARD.S

@@ -0,0 +1,2 @@
+#include "libc/sysv/consts/syscon.internal.h"
+.syscon mprot,PROT_GUARD,0,0,0,0,0,0,0,0x100

libc/sysv/consts/prot.h

@@ -3,10 +3,7 @@
 #if !(__ASSEMBLER__ + __LINKER__ + 0)
 COSMOPOLITAN_C_START_
 
-extern const int PROT_NONE;
+extern const int PROT_GUARD;
-extern const int PROT_READ;
-extern const int PROT_WRITE;
-extern const int PROT_EXEC;
 
 COSMOPOLITAN_C_END_
 #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

libc/thread/pthread_create.c

@@ -229,21 +229,11 @@ static errno_t pthread_create_impl(pthread_t *thread,
               -1, 0, 0) != pt->pt_attr.__stackaddr) {
         notpossible;
       }
-      if (pt->pt_attr.__guardsize) {
+      if (pt->pt_attr.__guardsize)
-        if (!IsWindows()) {
         if (mprotect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize,
-                       PROT_NONE)) {
+                     PROT_NONE | PROT_GUARD))
           notpossible;
     }
-        } else {
-          uint32_t oldattr;
-          if (!VirtualProtect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize,
-                              kNtPageReadwrite | kNtPageGuard, &oldattr)) {
-            notpossible;
-          }
-        }
-      }
-    }
     if (!pt->pt_attr.__stackaddr || pt->pt_attr.__stackaddr == MAP_FAILED) {
       rc = errno;
       _pthread_free(pt, false);

libc/thread/thread.h

@@ -190,6 +190,7 @@ int pthread_spin_trylock(pthread_spinlock_t *) libcesque paramsnonnull();
 int pthread_spin_unlock(pthread_spinlock_t *) libcesque paramsnonnull();
 int pthread_testcancel_np(void) libcesque;
 int pthread_tryjoin_np(pthread_t, void **) libcesque;
+int pthread_delay_np(const void *, int) libcesque;
 int pthread_yield_np(void) libcesque;
 int pthread_yield(void) libcesque;
 pthread_id_np_t pthread_getthreadid_np(void) libcesque;

test/ctl/set_test.cc

@@ -19,9 +19,9 @@
 #include "ctl/set.h"
 #include "libc/mem/leaks.h"
 
-// #include <set>
+#include <set>
-// #define ctl std
+#define ctl std
-// #define check() size()
+#define check() size()
 
 int
 rand32(void)
@@ -148,6 +148,16 @@ main()
         s.check();
     }
 
+    {
+        // Test lower_bound and upper_bound
+        ctl::set<int> s{ 1, 3, 4, 5, 7, 9 };
+        auto lower = s.lower_bound(4);
+        auto upper = s.upper_bound(4);
+        if (*lower != 4 || *upper != 5)
+            return 18;
+        s.check();
+    }
+
     {
         // Test emplace
         ctl::set<ctl::pair<int, int>> s;

test/libc/calls/raise_test.c

@@ -20,7 +20,6 @@
 #include "libc/calls/struct/sigaction.h"
 #include "libc/calls/struct/siginfo.h"
 #include "libc/dce.h"
-#include "libc/intrin/kprintf.h"
 #include "libc/runtime/runtime.h"
 #include "libc/sysv/consts/sa.h"
 #include "libc/sysv/consts/sicode.h"

test/libc/calls/setrlimit_test.c

@@ -118,10 +118,11 @@ TEST(setrlimit, testFileSizeLimit) {
   EXPECT_EQ(0, WTERMSIG(wstatus));
 }
 
-int SetKernelEnforcedMemoryLimit(size_t n) {
+int SetMemoryLimit(size_t n) {
-  struct rlimit rlim;
+  struct rlimit rlim = {0};
   getrlimit(RLIMIT_AS, &rlim);
   rlim.rlim_cur = n;
+  rlim.rlim_max = n;
   return setrlimit(RLIMIT_AS, &rlim);
 }
 
@@ -129,27 +130,20 @@ TEST(setrlimit, testMemoryLimit) {
   char *p;
   bool gotsome;
   int i, wstatus;
-  if (IsXnu())
-    return;
-  if (IsOpenbsd())
-    return;  // simply too slow until mmap() becomes O(logn)
   ASSERT_NE(-1, (wstatus = xspawn(0)));
   if (wstatus == -2) {
-    ASSERT_EQ(0, SetKernelEnforcedMemoryLimit(MEM));
+    ASSERT_EQ(0, SetMemoryLimit(MEM));
-    for (gotsome = false, i = 0; i < (MEM * 2) / __granularity(); ++i) {
+    for (gotsome = false, i = 0; i < (MEM * 2) / getpagesize(); ++i) {
-      p = mmap(0, __granularity(), PROT_READ | PROT_WRITE,
+      p = mmap(0, getpagesize(), PROT_READ | PROT_WRITE,
-               MAP_ANONYMOUS | MAP_PRIVATE | MAP_POPULATE, -1, 0);
+               MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
       if (p != MAP_FAILED) {
         gotsome = true;
       } else {
-        if (!IsNetbsd()) {
-          // TODO(jart): what's going on with NetBSD?
         ASSERT_TRUE(gotsome);
-        }
         ASSERT_EQ(ENOMEM, errno);
         _Exit(0);
       }
-      rngset(p, __granularity(), _rand64, -1);
+      rngset(p, getpagesize(), _rand64, -1);
     }
     _Exit(1);
   }

test/libc/intrin/mmap_scalability_test.c

@@ -0,0 +1,76 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2024 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/calls.h"
+#include "libc/calls/struct/timespec.h"
+#include "libc/dce.h"
+#include "libc/intrin/kprintf.h"
+#include "libc/runtime/runtime.h"
+#include "libc/sysv/consts/map.h"
+#include "libc/sysv/consts/prot.h"
+
+#define BENCH(ITERATIONS, WORK_PER_RUN, CODE)                                 \
+  do {                                                                        \
+    struct timespec start = timespec_real();                                  \
+    for (int __i = 0; __i < ITERATIONS; ++__i) {                              \
+      asm volatile("" ::: "memory");                                          \
+      CODE;                                                                   \
+    }                                                                         \
+    long long work = (WORK_PER_RUN) * (ITERATIONS);                           \
+    double nanos =                                                            \
+        (timespec_tonanos(timespec_sub(timespec_real(), start)) + work - 1) / \
+        (double)work;                                                         \
+    kprintf("%'20ld ns %2dx %s\n", (long)nanos, (ITERATIONS), #CODE);         \
+  } while (0)
+
+void *randaddr(void) {
+  static unsigned long lcg = 1;
+  lcg *= 6364136223846793005;
+  lcg += 1442695040888963407;
+  return (void *)(lcg >> 48 << 28);
+}
+
+void map_unmap_one_page(void) {
+  void *p;
+  if ((p = mmap(randaddr(), 1, PROT_READ | PROT_WRITE,
+                MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)) == MAP_FAILED)
+    __builtin_trap();
+  if (munmap(p, 1))
+    __builtin_trap();
+}
+
+int main() {
+  kprintf("\n");
+  BENCH(1000, 1, map_unmap_one_page());
+
+  // macos doesn't scale
+  if (IsXnu())
+    return 0;
+
+  // create lots of sparse mappings to put
+  // weight on __maps.maps. red-black tree
+  int n = 10000;
+  kprintf("%20s creating %d sparse maps...\n", "", n);
+  for (int i = 0; i < n; ++i) {
+    if (mmap(randaddr(), 1, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS,
+             -1, 0) == MAP_FAILED)
+      __builtin_trap();
+  }
+
+  BENCH(1000, 1, map_unmap_one_page());
+}

test/libc/intrin/mmap_test.c

@@ -461,7 +461,23 @@ void BenchUnmap(void) {
     __builtin_trap();
 }
 
+void BenchBigMmap(void) {
+  void *p;
+  p = mmap(0, 101 * 1024 * 1024, PROT_READ | PROT_WRITE,
+           MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+  if (p == MAP_FAILED)
+    __builtin_trap();
+  ptrs[count++] = p;
+}
+
+void BenchBigMunmap(void) {
+  if (munmap(ptrs[--count], 101 * 1024 * 1024))
+    __builtin_trap();
+}
+
 BENCH(mmap, bench) {
   EZBENCH2("mmap", donothing, BenchMmapPrivate());
   EZBENCH2("munmap", donothing, BenchUnmap());
+  // EZBENCH2("big mmap", donothing, BenchBigMmap());
+  // EZBENCH2("big munmap", donothing, BenchBigMunmap());
 }

test/libc/intrin/tree_test.c

@@ -13,16 +13,18 @@
 // TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 // PERFORMANCE OF THIS SOFTWARE.
 
-#include "libc/intrin/rbtree.h"
+#include "libc/intrin/tree.h"
 #include "libc/intrin/kprintf.h"
+#include "libc/intrin/maps.h"
 #include "libc/macros.internal.h"
+#include "libc/mem/mem.h"
 #include "libc/runtime/runtime.h"
 #include "libc/stdio/rand.h"
 
-#define NUMBER_CONTAINER(e) RBTREE_CONTAINER(struct number, elem, e)
+#define NUMBER_CONTAINER(e) TREE_CONTAINER(struct number, elem, e)
 
-void rbtree_checker(const struct rbtree *node, const struct rbtree *parent,
+void tree_checker(const struct Tree *node, const struct Tree *parent,
-                    int black_count, int *black_height, rbtree_cmp_f *cmp) {
+                  int black_count, int *black_height, tree_cmp_f *cmp) {
   if (!node) {
     // Leaf nodes are considered black
     if (*black_height == -1) {
@@ -37,89 +39,101 @@ void rbtree_checker(const struct rbtree *node, const struct rbtree *parent,
     // ILLEGAL TREE: Parent link is incorrect
     __builtin_trap();
   if (parent) {
-    if (rbtree_get_left(parent) == node && cmp(parent, node) < 0)
+    if (tree_get_left(parent) == node && cmp(parent, node) < 0)
       // ILLEGAL TREE: Binary search property violated on left child
       __builtin_trap();
     if (parent->right == node && cmp(node, parent) < 0)
       // ILLEGAL TREE: Binary search property violated on right child
       __builtin_trap();
   }
-  if (!rbtree_get_red(node)) {
+  if (!tree_get_red(node)) {
     black_count++;
-  } else if (parent && rbtree_get_red(parent)) {
+  } else if (parent && tree_get_red(parent)) {
     // ILLEGAL TREE: Red node has red child
     __builtin_trap();
   }
-  rbtree_checker(rbtree_get_left(node), node, black_count, black_height, cmp);
+  tree_checker(tree_get_left(node), node, black_count, black_height, cmp);
-  rbtree_checker(node->right, node, black_count, black_height, cmp);
+  tree_checker(node->right, node, black_count, black_height, cmp);
 }
 
-void rbtree_check(struct rbtree *root, rbtree_cmp_f *cmp) {
+void tree_check(struct Tree *root, tree_cmp_f *cmp) {
   if (root) {
-    if (rbtree_get_red(root))
+    if (tree_get_red(root))
       // ILLEGAL TREE: root node must be black
       __builtin_trap();
     int black_height = -1;
-    rbtree_checker(root, 0, 0, &black_height, cmp);
+    tree_checker(root, 0, 0, &black_height, cmp);
   }
 }
 
 struct number {
-  int number;
+  long number;
-  struct rbtree elem;
+  struct Tree elem;
 };
 
-int number_compare(const struct rbtree *ra, const struct rbtree *rb) {
+int number_search(const void *ra, const struct Tree *rb) {
+  long a = (long)ra;
+  const struct number *b = NUMBER_CONTAINER(rb);
+  return (a > b->number) - (a < b->number);
+}
+
+int number_compare(const struct Tree *ra, const struct Tree *rb) {
   const struct number *a = NUMBER_CONTAINER(ra);
   const struct number *b = NUMBER_CONTAINER(rb);
   return (a->number > b->number) - (a->number < b->number);
 }
 
 struct number *number_new(int number) {
-  static int used;
+  struct number *res;
-  static struct number heap[8192];
+  if ((res = malloc(sizeof(struct number))))
-  if (used == ARRAYLEN(heap))
-    return 0;
-  struct number *res = &heap[used++];
     res->number = number;
   return res;
 }
 
-struct rbtree *tree = 0;
+struct Tree *tree = 0;
 
 void print(void) {
-  for (struct rbtree *e = rbtree_first(tree); e; e = rbtree_next(e))
+  for (struct Tree *e = tree_first(tree); e; e = tree_next(e))
     kprintf("%3d", NUMBER_CONTAINER(e)->number);
   kprintf("\n");
 }
 
 void print_reversed(void) {
-  for (struct rbtree *e = rbtree_last(tree); e; e = rbtree_prev(e))
+  for (struct Tree *e = tree_last(tree); e; e = tree_prev(e))
     kprintf("%3d", NUMBER_CONTAINER(e)->number);
   kprintf("\n");
 }
 
 void simple_test(void) {
-  static const int kNumba[] = {74, 53, 96, 70, 34, 95, 30, 2,  89, 46,
+  static const long kNumba[] = {74, 53, 96, 70, 34, 95, 30, 2,  96, 46,
-                               23, 2,  52, 0,  34, 12, 90, 95, 32, 65};
+                                23, 2,  52, 0,  34, 94, 90, 95, 32, 65};
   for (int i = 0; i < 20; ++i) {
     int number = kNumba[i];
     kprintf("%3d", number);
-    rbtree_insert(&tree, &number_new(number)->elem, number_compare);
+    tree_insert(&tree, &number_new(number)->elem, number_compare);
-    rbtree_check(tree, number_compare);
+    tree_check(tree, number_compare);
   }
   kprintf("\n");
   print();
   print_reversed();
   for (int i = 0; i < 20; ++i) {
-    rbtree_remove(&tree, rbtree_get(tree, &(&(struct number){kNumba[i]})->elem,
+    tree_remove(&tree, tree_get(tree, (void *)kNumba[i], number_search));
-                                    number_compare));
+    tree_check(tree, number_compare);
-    rbtree_check(tree, number_compare);
     print();
   }
 }
 
 int main() {
   ShowCrashReports();
+  tree_check(__maps.maps, __maps_compare);
+  kprintf("\n");
+  __print_maps(15);
+  kprintf("\n");
   simple_test();
+  tree_check(__maps.maps, __maps_compare);
+  for (int i = 0; i < 100000; ++i)
+    tree_insert(&tree, &number_new(rand())->elem, number_compare);
+  tree_check(tree, number_compare);
+  tree_check(__maps.maps, __maps_compare);
+  __print_maps(15);
 }

test/libc/thread/pthread_atfork_test.c

@@ -16,8 +16,11 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/atomic.h"
 #include "libc/calls/calls.h"
 #include "libc/dce.h"
+#include "libc/intrin/atomic.h"
+#include "libc/intrin/kprintf.h"
 #include "libc/mem/gc.h"
 #include "libc/mem/mem.h"
 #include "libc/runtime/internal.h"
@@ -66,7 +69,7 @@ TEST(pthread_atfork, test) {
   EXITS(0);
 }
 
-pthread_mutex_t mu;
+pthread_mutex_t mu = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
 
 void mu_lock(void) {
   pthread_mutex_lock(&mu);
@@ -80,8 +83,12 @@ void mu_wipe(void) {
   pthread_mutex_init(&mu, 0);
 }
 
+static atomic_bool once;
+
 void *Worker(void *arg) {
   for (int i = 0; i < 20; ++i) {
+    if (!atomic_exchange(&once, true))
+      __print_maps(0);
     mu_lock();
     usleep(20);
     mu_unlock();
@@ -99,14 +106,14 @@ void *Worker(void *arg) {
 }
 
 TEST(pthread_atfork, fork_exit_torture) {
+  if (!IsFreebsd())
+    return;
   mu_wipe();
   pthread_atfork(mu_lock, mu_unlock, mu_wipe);
   int i, n = 4;
   pthread_t *t = gc(malloc(sizeof(pthread_t) * n));
-  for (i = 0; i < n; ++i) {
+  for (i = 0; i < n; ++i)
     ASSERT_EQ(0, pthread_create(t + i, 0, Worker, 0));
-  }
+  for (i = 0; i < n; ++i)
-  for (i = 0; i < n; ++i) {
     ASSERT_EQ(0, pthread_join(t[i], 0));
 }
-}

third_party/nsync/common.c

@@ -18,6 +18,7 @@
 #include "libc/calls/calls.h"
 #include "libc/calls/syscall-sysv.internal.h"
 #include "libc/dce.h"
+#include "libc/intrin/directmap.internal.h"
 #include "libc/intrin/dll.h"
 #include "libc/intrin/extend.internal.h"
 #include "libc/nt/enum/filemapflags.h"
@@ -25,21 +26,17 @@
 #include "libc/nt/memory.h"
 #include "libc/nt/runtime.h"
 #include "libc/runtime/memtrack.internal.h"
+#include "libc/runtime/runtime.h"
+#include "libc/stdalign.internal.h"
+#include "libc/stdalign.internal.h"
 #include "libc/sysv/consts/map.h"
 #include "libc/sysv/consts/prot.h"
+#include "libc/thread/thread.h"
 #include "libc/thread/tls.h"
 #include "third_party/nsync/atomic.h"
 #include "third_party/nsync/atomic.internal.h"
 #include "third_party/nsync/common.internal.h"
 #include "third_party/nsync/mu_semaphore.h"
-#include "third_party/nsync/races.internal.h"
-#include "libc/runtime/runtime.h"
-#include "libc/runtime/runtime.h"
-#include "libc/sysv/consts/map.h"
-#include "libc/nt/runtime.h"
-#include "libc/intrin/directmap.internal.h"
-#include "libc/thread/thread.h"
-#include "libc/dce.h"
 #include "third_party/nsync/wait_s.internal.h"
 __static_yoink("nsync_notice");
 
@@ -97,33 +94,15 @@ __static_yoink("nsync_notice");
    distinct wakeup conditions were high.  So clients are advised to resort to
    condition variables if they have many distinct wakeup conditions. */
 
-/* Used in spinloops to delay resumption of the loop.
-   Usage:
-       unsigned attempts = 0;
-       while (try_something) {
-	  attempts = nsync_spin_delay_ (attempts);
-       } */
-unsigned nsync_spin_delay_ (unsigned attempts) {
-	if (attempts < 7) {
-		volatile int i;
-		for (i = 0; i != 1 << attempts; i++) {
-		}
-		attempts++;
-	} else {
-		nsync_yield_ ();
-	}
-	return (attempts);
-}
-
 /* Spin until (*w & test) == 0, then atomically perform *w = ((*w | set) &
    ~clear), perform an acquire barrier, and return the previous value of *w.
    */
 uint32_t nsync_spin_test_and_set_ (nsync_atomic_uint32_ *w, uint32_t test,
-				   uint32_t set, uint32_t clear) {
+				   uint32_t set, uint32_t clear, void *symbol) {
 	unsigned attempts = 0; /* CV_SPINLOCK retry count */
 	uint32_t old = ATM_LOAD (w);
 	while ((old & test) != 0 || !ATM_CAS_ACQ (w, old, (old | set) & ~clear)) {
-		attempts = nsync_spin_delay_ (attempts);
+		attempts = pthread_delay_np (symbol, attempts);
 		old = ATM_LOAD (w);
 	}
 	return (old);
@@ -156,33 +135,69 @@ waiter *nsync_dll_waiter_samecond_ (struct Dll *e) {
 
 /* -------------------------------- */
 
-static void *nsync_malloc (size_t size) {
+static _Atomic(waiter *) free_waiters;
-	void *res;
+
-	if (!IsWindows ()) {
+static void free_waiters_push (waiter *w) {
-		// too much of a performance hit to track
+	int backoff = 0;
-		res = __sys_mmap ((void *)0x7110000000, size,
+	w->next_free = atomic_load_explicit (&free_waiters, memory_order_relaxed);
-				  PROT_READ | PROT_WRITE,
+	while (!atomic_compare_exchange_weak_explicit (&free_waiters, &w->next_free, w,
-				  MAP_PRIVATE | MAP_ANONYMOUS,
+						       memory_order_acq_rel, memory_order_relaxed))
-				  -1, 0, 0);
+		backoff = pthread_delay_np(free_waiters, backoff);
+}
+
+static void free_waiters_populate (void) {
+	int n;
+	if (IsNetbsd () || IsXnuSilicon ()) {
+		// netbsd needs one file descriptor per semaphore (!!)
+		// tim cook wants us to use his grand central dispatch
+		n = 1;
 	} else {
-		// must be tracked for fork() resurrection
+		n = getpagesize() / sizeof(waiter);
-		res = mmap (0, size,
+	}
+	waiter *waiters = mmap (0, n * sizeof(waiter),
 				PROT_READ | PROT_WRITE,
 				MAP_PRIVATE | MAP_ANONYMOUS,
 				-1, 0);
-	}
+	if (waiters == MAP_FAILED)
-	if (res == MAP_FAILED)
 		nsync_panic_ ("out of memory\n");
-	return res;
+	for (size_t i = 0; i < n; ++i) {
+		waiter *w = &waiters[i];
+		w->tag = WAITER_TAG;
+		w->nw.tag = NSYNC_WAITER_TAG;
+		if (!nsync_mu_semaphore_init (&w->sem)) {
+			if (!i)
+				nsync_panic_ ("out of semaphores\n");
+			break;
+		}
+		w->nw.sem = &w->sem;
+		dll_init (&w->nw.q);
+		NSYNC_ATOMIC_UINT32_STORE_ (&w->nw.waiting, 0);
+		w->nw.flags = NSYNC_WAITER_FLAG_MUCV;
+		ATM_STORE (&w->remove_count, 0);
+		dll_init (&w->same_condition);
+		w->flags = 0;
+		free_waiters_push (w);
+	}
+}
+
+static waiter *free_waiters_pop (void) {
+	waiter *w;
+	int backoff = 0;
+	for (;;) {
+		if ((w = atomic_load_explicit (&free_waiters, memory_order_relaxed))) {
+			if (atomic_compare_exchange_weak_explicit (&free_waiters, &w, w->next_free,
+								   memory_order_acq_rel, memory_order_relaxed))
+				return w;
+			backoff = pthread_delay_np(free_waiters, backoff);
+		} else {
+			free_waiters_populate ();
+		}
+	}
+	return w;
 }
 
 /* -------------------------------- */
 
-static struct Dll *free_waiters = NULL;
-
-/* free_waiters points to a doubly-linked list of free waiter structs. */
-pthread_mutex_t nsync_waiters_mu = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
-
 #define waiter_for_thread __get_tls()->tib_nsync
 
 void nsync_waiter_destroy (void *v) {
@@ -193,45 +208,20 @@ void nsync_waiter_destroy (void *v) {
 	   of thread-local variables can be arbitrary in some platform e.g.
 	   POSIX. */
 	waiter_for_thread = NULL;
-	IGNORE_RACES_START ();
 	ASSERT ((w->flags & (WAITER_RESERVED|WAITER_IN_USE)) == WAITER_RESERVED);
 	w->flags &= ~WAITER_RESERVED;
-	pthread_mutex_lock (&nsync_waiters_mu);
+	free_waiters_push (w);
-	dll_make_first (&free_waiters, &w->nw.q);
-	pthread_mutex_unlock (&nsync_waiters_mu);
-	IGNORE_RACES_END ();
 }
 
 /* Return a pointer to an unused waiter struct.
    Ensures that the enclosed timer is stopped and its channel drained. */
 waiter *nsync_waiter_new_ (void) {
-	struct Dll *q;
-	waiter *tw;
 	waiter *w;
+	waiter *tw;
 	tw = waiter_for_thread;
 	w = tw;
 	if (w == NULL || (w->flags & (WAITER_RESERVED|WAITER_IN_USE)) != WAITER_RESERVED) {
-		w = NULL;
+		w = free_waiters_pop ();
-		pthread_mutex_lock (&nsync_waiters_mu);
-		q = dll_first (free_waiters);
-		if (q != NULL) { /* If free list is non-empty, dequeue an item. */
-			dll_remove (&free_waiters, q);
-			w = DLL_WAITER (q);
-		}
-		pthread_mutex_unlock (&nsync_waiters_mu);
-		if (w == NULL) { /* If free list was empty, allocate an item. */
-			w = (waiter *) nsync_malloc (sizeof (*w));
-			w->tag = WAITER_TAG;
-			w->nw.tag = NSYNC_WAITER_TAG;
-			nsync_mu_semaphore_init (&w->sem);
-			w->nw.sem = &w->sem;
-			dll_init (&w->nw.q);
-			NSYNC_ATOMIC_UINT32_STORE_ (&w->nw.waiting, 0);
-			w->nw.flags = NSYNC_WAITER_FLAG_MUCV;
-			ATM_STORE (&w->remove_count, 0);
-			dll_init (&w->same_condition);
-			w->flags = 0;
-		}
 		if (tw == NULL) {
 			w->flags |= WAITER_RESERVED;
 			waiter_for_thread = w;
@@ -246,9 +236,7 @@ void nsync_waiter_free_ (waiter *w) {
 	ASSERT ((w->flags & WAITER_IN_USE) != 0);
 	w->flags &= ~WAITER_IN_USE;
 	if ((w->flags & WAITER_RESERVED) == 0) {
-		pthread_mutex_lock (&nsync_waiters_mu);
+		free_waiters_push (w);
-		dll_make_first (&free_waiters, &w->nw.q);
-		pthread_mutex_unlock (&nsync_waiters_mu);
 		if (w == waiter_for_thread)
 			waiter_for_thread = 0;
 	}

third_party/nsync/common.internal.h

@@ -24,19 +24,11 @@ void nsync_yield_(void);
 /* Retrieve the per-thread cache of the waiter object.  Platform specific. */
 void *nsync_per_thread_waiter_(void (*dest)(void *));
 
-/* Used in spinloops to delay resumption of the loop.
-   Usage:
-       unsigned attempts = 0;
-       while (try_something) {
-          attempts = nsync_spin_delay_ (attempts);
-       } */
-unsigned nsync_spin_delay_(unsigned attempts);
-
 /* Spin until (*w & test) == 0, then atomically perform *w = ((*w | set) &
    ~clear), perform an acquire barrier, and return the previous value of *w.
    */
 uint32_t nsync_spin_test_and_set_(nsync_atomic_uint32_ *w, uint32_t test,
-                                  uint32_t set, uint32_t clear);
+                                  uint32_t set, uint32_t clear, void *symbol);
 
 /* Abort after printing the nul-temrinated string s[]. */
 void nsync_panic_(const char *s) wontreturn;
@@ -210,6 +202,7 @@ typedef struct waiter_s {
   struct wait_condition_s cond;      /* A condition on which to acquire a mu. */
   struct Dll same_condition;         /* Links neighbours in nw.q with same
                                         non-nil condition. */
+  struct waiter_s * next_free;
 } waiter;
 static const uint32_t WAITER_TAG = 0x0590239f;
 static const uint32_t NSYNC_WAITER_TAG = 0x726d2ba9;

third_party/nsync/mem/nsync_cv.c

@@ -193,7 +193,7 @@ static int nsync_cv_wait_with_deadline_impl_ (struct nsync_cv_wait_with_deadline
 			/* A timeout or cancellation occurred, and no wakeup.
 			   Acquire *pcv's spinlock, and confirm.  */
 			c->old_word = nsync_spin_test_and_set_ (&c->pcv->word, CV_SPINLOCK,
-								CV_SPINLOCK, 0);
+								CV_SPINLOCK, 0, c->cv_mu);
 			/* Check that w wasn't removed from the queue after we
 			   checked above, but before we acquired the spinlock.
 			   The test of remove_count confirms that the waiter *w
@@ -226,7 +226,7 @@ static int nsync_cv_wait_with_deadline_impl_ (struct nsync_cv_wait_with_deadline
                            has not yet set the waiting field to zero; a
                            cancellation or timeout may prevent this thread
                            from blocking above on the semaphore.  */
-			attempts = nsync_spin_delay_ (attempts);
+			attempts = pthread_delay_np (c->cv_mu, attempts);
 		}
 	}
 	if (c->cv_mu != NULL && c->w->cv_mu == NULL) { /* waiter was moved to *pmu's queue, and woken. */
@@ -323,7 +323,7 @@ int nsync_cv_wait_with_deadline_generic (nsync_cv *pcv, void *pmu,
 	}
 
 	/* acquire spinlock, set non-empty */
-	c.old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK|CV_NON_EMPTY, 0);
+	c.old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK|CV_NON_EMPTY, 0, pmu);
 	dll_make_last (&pcv->waiters, &c.w->nw.q);
 	c.remove_count = ATM_LOAD (&c.w->remove_count);
 	/* Release the spin lock. */
@@ -355,7 +355,7 @@ void nsync_cv_signal (nsync_cv *pcv) {
 		int all_readers = 0;
 		/* acquire spinlock */
 		uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK,
-							      CV_SPINLOCK, 0);
+							      CV_SPINLOCK, 0, pcv);
 		if (!dll_is_empty (pcv->waiters)) {
 			/* Point to first waiter that enqueued itself, and
 			   detach it from all others.  */
@@ -438,7 +438,7 @@ void nsync_cv_broadcast (nsync_cv *pcv) {
 		int all_readers;
 		struct Dll *to_wake_list = NULL;   /* waiters that we will wake */
 		/* acquire spinlock */
-		nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0);
+		nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0, pcv);
 		p = NULL;
 		next = NULL;
 		all_readers = 1;
@@ -497,7 +497,7 @@ static nsync_time cv_ready_time (void *v, struct nsync_waiter_s *nw) {
 static int cv_enqueue (void *v, struct nsync_waiter_s *nw) {
 	nsync_cv *pcv = (nsync_cv *) v;
 	/* acquire spinlock */
-	uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0);
+	uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0, pcv);
 	dll_make_last (&pcv->waiters, &nw->q);
 	ATM_STORE (&nw->waiting, 1);
 	/* Release spinlock. */
@@ -509,7 +509,7 @@ static int cv_dequeue (void *v, struct nsync_waiter_s *nw) {
 	nsync_cv *pcv = (nsync_cv *) v;
 	int was_queued = 0;
 	/* acquire spinlock */
-	uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0);
+	uint32_t old_word = nsync_spin_test_and_set_ (&pcv->word, CV_SPINLOCK, CV_SPINLOCK, 0, pcv);
 	if (ATM_LOAD_ACQ (&nw->waiting) != 0) {
 		dll_remove (&pcv->waiters, &nw->q);
 		ATM_STORE (&nw->waiting, 0);

third_party/nsync/mem/nsync_debug.c

@@ -197,7 +197,7 @@ static char *emit_mu_state (struct emit_buf *b, nsync_mu *mu,
         word = ATM_LOAD (&mu->word);
         if ((word & MU_WAITING) != 0 && print_waiters &&  /* can benefit from lock */
 	    (blocking || (word & MU_SPINLOCK) == 0)) {  /* willing, or no need to wait */
-                word = nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK, MU_SPINLOCK, 0);
+                word = nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK, MU_SPINLOCK, 0, mu);
                 acquired = 1;
         }
         readers = word / MU_RLOCK;
@@ -234,7 +234,7 @@ static char *emit_cv_state (struct emit_buf *b, nsync_cv *cv,
         word = ATM_LOAD (&cv->word);
         if ((word & CV_NON_EMPTY) != 0 && print_waiters &&  /* can benefit from lock */
 	    (blocking || (word & CV_SPINLOCK) == 0)) {  /* willing, or no need to wait */
-                word = nsync_spin_test_and_set_ (&cv->word, CV_SPINLOCK, CV_SPINLOCK, 0);
+                word = nsync_spin_test_and_set_ (&cv->word, CV_SPINLOCK, CV_SPINLOCK, 0, cv);
                 acquired = 1;
         }
         emit_print (b, "cv 0x%i -> 0x%i = {", (uintptr_t) cv, word);

third_party/nsync/mem/nsync_mu_wait.c

@@ -21,6 +21,7 @@
 #include "third_party/nsync/common.internal.h"
 #include "third_party/nsync/mu_semaphore.h"
 #include "third_party/nsync/races.internal.h"
+#include "libc/thread/thread.h"
 #include "third_party/nsync/wait_s.internal.h"
 __static_yoink("nsync_notice");
 
@@ -84,7 +85,7 @@ static int mu_try_acquire_after_timeout_or_cancel (nsync_mu *mu, lock_type *l_ty
 			ATM_CAS_RELACQ (&mu->word, old_word,
 					old_word|MU_WRITER_WAITING);
 		}
-		spin_attempts = nsync_spin_delay_ (spin_attempts);
+		spin_attempts = pthread_delay_np (mu, spin_attempts);
 		old_word = ATM_LOAD (&mu->word);
 	}
 	/* Check that w wasn't removed from the queue after our caller checked,
@@ -194,7 +195,7 @@ int nsync_mu_wait_with_deadline (nsync_mu *mu,
 
 		/* Acquire spinlock. */
 		old_word = nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK,
-			MU_SPINLOCK|MU_WAITING|has_condition, MU_ALL_FALSE);
+			MU_SPINLOCK|MU_WAITING|has_condition, MU_ALL_FALSE, mu);
 		had_waiters = ((old_word & (MU_DESIG_WAKER | MU_WAITING)) == MU_WAITING);
 		/* Queue the waiter. */
 		if (first_wait) {
@@ -244,7 +245,7 @@ int nsync_mu_wait_with_deadline (nsync_mu *mu,
 			}
 
 			if (ATM_LOAD (&w->nw.waiting) != 0) {
-				attempts = nsync_spin_delay_ (attempts); /* will ultimately yield */
+				attempts = pthread_delay_np (mu, attempts); /* will ultimately yield */
 			}
 		}
 

third_party/nsync/mem/nsync_once.c

@@ -21,6 +21,7 @@
 #include "third_party/nsync/mu_semaphore.h"
 #include "third_party/nsync/once.h"
 #include "third_party/nsync/races.internal.h"
+#include "libc/thread/thread.h"
 #include "third_party/nsync/wait_s.internal.h"
 __static_yoink("nsync_notice");
 
@@ -92,7 +93,7 @@ static void nsync_run_once_impl (nsync_once *once, struct once_sync_s *s,
 				deadline = nsync_time_add (nsync_time_now (), nsync_time_ms (attempts));
 				nsync_cv_wait_with_deadline (&s->once_cv, &s->once_mu, deadline, NULL);
 			} else {
-				attempts = nsync_spin_delay_ (attempts);
+				attempts = pthread_delay_np (once, attempts);
 			}
 		}
 		if (s != NULL) {

third_party/nsync/mu.c

@@ -22,6 +22,7 @@
 #include "third_party/nsync/common.internal.h"
 #include "third_party/nsync/mu_semaphore.h"
 #include "third_party/nsync/races.internal.h"
+#include "libc/thread/thread.h"
 #include "third_party/nsync/wait_s.internal.h"
 __static_yoink("nsync_notice");
 
@@ -120,7 +121,7 @@ void nsync_mu_lock_slow_ (nsync_mu *mu, waiter *w, uint32_t clear, lock_type *l_
 			   about waiting writers or long waiters. */
 			zero_to_acquire &= ~(MU_WRITER_WAITING | MU_LONG_WAIT);
 		}
-		attempts = nsync_spin_delay_ (attempts);
+		attempts = pthread_delay_np (mu, attempts);
 	}
 	ALLOW_CANCELATION;
 }
@@ -393,7 +394,7 @@ void nsync_mu_unlock_slow_ (nsync_mu *mu, lock_type *l_type) {
 				   released above. */
 				if (testing_conditions) {
 					nsync_spin_test_and_set_ (&mu->word, MU_SPINLOCK,
-								  MU_SPINLOCK, 0);
+								  MU_SPINLOCK, 0, mu);
 				}
 
 				/* add the new_waiters to the last of the waiters. */
@@ -444,7 +445,7 @@ void nsync_mu_unlock_slow_ (nsync_mu *mu, lock_type *l_type) {
 			}
 			return;
 		}
-		attempts = nsync_spin_delay_ (attempts);
+		attempts = pthread_delay_np (mu, attempts);
 	}
 }
 

third_party/nsync/mu_semaphore.c

@@ -27,7 +27,7 @@ __static_yoink("nsync_notice");
 #define PREFER_GCD_OVER_ULOCK 1
 
 /* Initialize *s; the initial value is 0. */
-void nsync_mu_semaphore_init (nsync_semaphore *s) {
+bool nsync_mu_semaphore_init (nsync_semaphore *s) {
 	if (PREFER_GCD_OVER_ULOCK && IsXnuSilicon ()) {
 		return nsync_mu_semaphore_init_gcd (s);
 	} else if (IsNetbsd ()) {

third_party/nsync/mu_semaphore.h

@@ -8,7 +8,7 @@ typedef struct nsync_semaphore_s_ {
 } nsync_semaphore;
 
 /* Initialize *s; the initial value is 0. */
-void nsync_mu_semaphore_init(nsync_semaphore *s);
+bool nsync_mu_semaphore_init(nsync_semaphore *s);
 
 /* Wait until the count of *s exceeds 0, and decrement it. */
 errno_t nsync_mu_semaphore_p(nsync_semaphore *s);

third_party/nsync/mu_semaphore.internal.h

@@ -4,17 +4,17 @@
 #include "third_party/nsync/time.h"
 COSMOPOLITAN_C_START_
 
-void nsync_mu_semaphore_init_futex(nsync_semaphore *);
+bool nsync_mu_semaphore_init_futex(nsync_semaphore *);
 errno_t nsync_mu_semaphore_p_futex(nsync_semaphore *);
 errno_t nsync_mu_semaphore_p_with_deadline_futex(nsync_semaphore *, nsync_time);
 void nsync_mu_semaphore_v_futex(nsync_semaphore *);
 
-void nsync_mu_semaphore_init_sem(nsync_semaphore *);
+bool nsync_mu_semaphore_init_sem(nsync_semaphore *);
 errno_t nsync_mu_semaphore_p_sem(nsync_semaphore *);
 errno_t nsync_mu_semaphore_p_with_deadline_sem(nsync_semaphore *, nsync_time);
 void nsync_mu_semaphore_v_sem(nsync_semaphore *);
 
-void nsync_mu_semaphore_init_gcd(nsync_semaphore *);
+bool nsync_mu_semaphore_init_gcd(nsync_semaphore *);
 errno_t nsync_mu_semaphore_p_gcd(nsync_semaphore *);
 errno_t nsync_mu_semaphore_p_with_deadline_gcd(nsync_semaphore *, nsync_time);
 void nsync_mu_semaphore_v_gcd(nsync_semaphore *);

third_party/nsync/mu_semaphore_futex.c

@@ -43,9 +43,10 @@ static nsync_semaphore *sem_big_enough_for_futex = (nsync_semaphore *) (uintptr_
 	(sizeof (struct futex) <= sizeof (*sem_big_enough_for_futex)));
 
 /* Initialize *s; the initial value is 0. */
-void nsync_mu_semaphore_init_futex (nsync_semaphore *s) {
+bool nsync_mu_semaphore_init_futex (nsync_semaphore *s) {
 	struct futex *f = (struct futex *) s;
 	f->i = 0;
+	return true;
 }
 
 /* Wait until the count of *s exceeds 0, and decrement it. If POSIX cancellations

third_party/nsync/mu_semaphore_gcd.c

@@ -85,8 +85,8 @@ static errno_t nsync_dispatch_semaphore_wait (nsync_semaphore *s,
 }
 
 /* Initialize *s; the initial value is 0.  */
-void nsync_mu_semaphore_init_gcd (nsync_semaphore *s) {
+bool nsync_mu_semaphore_init_gcd (nsync_semaphore *s) {
-	*(dispatch_semaphore_t *)s = dispatch_semaphore_create (0);
+	return !!(*(dispatch_semaphore_t *)s = dispatch_semaphore_create (0));
 }
 
 /* Wait until the count of *s exceeds 0, and decrement it. If POSIX cancellations

third_party/nsync/mu_semaphore_sem.c

@@ -54,14 +54,22 @@ static struct {
 static nsync_semaphore *sem_big_enough_for_sem = (nsync_semaphore *) (uintptr_t)(1 /
 	(sizeof (struct sem) <= sizeof (*sem_big_enough_for_sem)));
 
-static void nsync_mu_semaphore_sem_create (struct sem *f) {
+static bool nsync_mu_semaphore_sem_create (struct sem *f) {
+	int rc;
 	int lol;
 	f->id = 0;
-	ASSERT (!sys_sem_init (0, &f->id));
+	rc = sys_sem_init (0, &f->id);
-	if ((lol = __sys_fcntl (f->id, F_DUPFD_CLOEXEC, 50)) >= 50) {
+	STRACE ("sem_init(0, [%ld]) → %d", f->id, rc);
-		sys_close (f->id);
+	if (rc != 0)
+		return false;
+	lol = __sys_fcntl (f->id, F_DUPFD_CLOEXEC, 50);
+	STRACE ("fcntl(%ld, F_DUPFD_CLOEXEC, 50) → %d", f->id, lol);
+	if (lol >= 50) {
+		rc = sys_close (f->id);
+		STRACE ("close(%ld) → %d", f->id, rc);
 		f->id = lol;
 	}
+	return true;
 }
 
 static void nsync_mu_semaphore_sem_fork_child (void) {
@@ -69,8 +77,12 @@ static void nsync_mu_semaphore_sem_fork_child (void) {
 	struct sem *f;
 	for (e = dll_first (g_sems.list); e; e = dll_next (g_sems.list, e)) {
 		f = SEM_CONTAINER (e);
-		sys_close (f->id);
+		int rc = sys_close (f->id);
-		nsync_mu_semaphore_sem_create (f);
+		STRACE ("close(%ld) → %d", f->id, rc);
+	}
+	for (e = dll_first (g_sems.list); e; e = dll_next (g_sems.list, e)) {
+		f = SEM_CONTAINER (e);
+		ASSERT (nsync_mu_semaphore_sem_create (f));
 	}
 	(void) pthread_spin_init (&g_sems.lock, 0);
 }
@@ -80,15 +92,16 @@ static void nsync_mu_semaphore_sem_init (void) {
 }
 
 /* Initialize *s; the initial value is 0. */
-void nsync_mu_semaphore_init_sem (nsync_semaphore *s) {
+bool nsync_mu_semaphore_init_sem (nsync_semaphore *s) {
 	struct sem *f = (struct sem *) s;
-	nsync_mu_semaphore_sem_create (f);
+	if (!nsync_mu_semaphore_sem_create (f))
+		return false;
 	cosmo_once (&g_sems.once, nsync_mu_semaphore_sem_init);
 	pthread_spin_lock (&g_sems.lock);
 	dll_init (&f->list);
 	dll_make_first (&g_sems.list, &f->list);
 	pthread_spin_unlock (&g_sems.lock);
-	STRACE ("sem_init(0, [%ld]) → 0", f->id);
+	return true;
 }
 
 /* Wait until the count of *s exceeds 0, and decrement it. If POSIX cancellations

tool/build/runitd.c

@@ -137,7 +137,7 @@
 #define LOGF(LVL, FMT, ...)                                                   \
   do {                                                                        \
     if (g_log_level >= LOG_LEVEL_##LVL) {                                     \
-      kprintf("%r" #LVL " %6P %'18T %s:%d " FMT "\n", __FILE__, __LINE__, \
+      kprintf("%r" #LVL " %6P %6H %'18T %s:%d " FMT "\n", __FILE__, __LINE__, \
               ##__VA_ARGS__);                                                 \
     }                                                                         \
   } while (0)
@@ -443,12 +443,18 @@ void FreeClient(struct Client *client) {
 void *ClientWorker(void *arg) {
   uint32_t crc;
   sigset_t sigmask;
+  struct timespec ts0;
+  struct timespec ts1;
+  struct timespec ts2;
   int events, wstatus;
   struct Client *client = arg;
   uint32_t namesize, filesize;
   char *addrstr, *origname;
   unsigned char msg[4 + 1 + 4 + 4 + 4];
 
+  ts0 = timespec_real();
+  ts1 = timespec_real();
+
   SetupPresharedKeySsl(MBEDTLS_SSL_IS_SERVER, g_psk);
   defer(FreeClient, client);
 
@@ -458,9 +464,15 @@ void *ClientWorker(void *arg) {
   EzHandshake();
   addrstr = DescribeAddress(&client->addr);
   DEBUF("%s %s %s", DescribeAddress(&g_servaddr), "accepted", addrstr);
+  DEBUF("it took %'zu us to handshake client",
+        timespec_tomicros(timespec_sub(timespec_real(), ts1)));
 
   // get the executable
+  ts1 = timespec_real();
+  ts2 = timespec_real();
   Recv(client, msg, sizeof(msg));
+  DEBUF("it took %'zu us to receive #1",
+        timespec_tomicros(timespec_sub(timespec_real(), ts2)));
   if (READ32BE(msg) != RUNITD_MAGIC) {
     WARNF("%s magic mismatch!", addrstr);
     pthread_exit(0);
@@ -473,11 +485,19 @@ void *ClientWorker(void *arg) {
   filesize = READ32BE(msg + 9);
   crc = READ32BE(msg + 13);
   origname = gc(calloc(1, namesize + 1));
+  ts2 = timespec_real();
   Recv(client, origname, namesize);
+  DEBUF("it took %'zu us to receive #2",
+        timespec_tomicros(timespec_sub(timespec_real(), ts2)));
   VERBF("%s sent %#s (%'u bytes @ %#s)", addrstr, origname, filesize,
         client->tmpexepath);
   char *exedata = gc(malloc(filesize));
+  ts2 = timespec_real();
   Recv(client, exedata, filesize);
+  DEBUF("it took %'zu us to receive #3",
+        timespec_tomicros(timespec_sub(timespec_real(), ts2)));
+  DEBUF("it took %'zu us to receive executable from network",
+        timespec_tomicros(timespec_sub(timespec_real(), ts1)));
   if (crc32_z(0, exedata, filesize) != crc) {
     WARNF("%s crc mismatch! %#s", addrstr, origname);
     pthread_exit(0);
@@ -488,6 +508,7 @@ void *ClientWorker(void *arg) {
   // condition can happen, where etxtbsy is raised by our execve
   // we're using o_cloexec so it's guaranteed to fix itself fast
   // thus we use an optimistic approach to avoid expensive locks
+  ts1 = timespec_real();
   sprintf(client->tmpexepath, "o/%s.XXXXXX",
           basename(stripext(gc(strdup(origname)))));
   int exefd = openatemp(AT_FDCWD, client->tmpexepath, 0, O_CLOEXEC, 0700);
@@ -510,6 +531,8 @@ void *ClientWorker(void *arg) {
     WARNF("%s failed to close %#s due to %m", addrstr, origname);
     pthread_exit(0);
   }
+  DEBUF("it took %'zu us to write executable to disk",
+        timespec_tomicros(timespec_sub(timespec_real(), ts1)));
 
   // do the args
   int i = 0;
@@ -560,8 +583,11 @@ RetryOnEtxtbsyRaceCondition:
   posix_spawn_file_actions_adddup2(&spawnfila, g_bogusfd, 0);
   posix_spawn_file_actions_adddup2(&spawnfila, client->pipe[1], 1);
   posix_spawn_file_actions_adddup2(&spawnfila, client->pipe[1], 2);
+  ts1 = timespec_real();
   err = posix_spawn(&client->pid, client->tmpexepath, &spawnfila, &spawnattr,
                     args, environ);
+  DEBUF("it took %'zu us to call posix_spawn",
+        timespec_tomicros(timespec_sub(timespec_real(), ts1)));
   if (err) {
     if (err == ETXTBSY) {
       goto RetryOnEtxtbsyRaceCondition;
@@ -599,8 +625,11 @@ RetryOnEtxtbsyRaceCondition:
     fds[0].events = POLLIN;
     fds[1].fd = client->pipe[0];
     fds[1].events = POLLIN;
+    ts1 = timespec_real();
     events = poll(fds, ARRAYLEN(fds),
                   timespec_tomillis(timespec_sub(deadline, now)));
+    DEBUF("it took %'zu us to call poll",
+          timespec_tomicros(timespec_sub(timespec_real(), ts1)));
     if (events == -1) {
       if (errno == EINTR) {
         INFOF("poll interrupted");
@@ -615,7 +644,10 @@ RetryOnEtxtbsyRaceCondition:
       if (fds[0].revents) {
         int received;
         char buf[512];
+        ts1 = timespec_real();
         received = mbedtls_ssl_read(&ezssl, buf, sizeof(buf));
+        DEBUF("it took %'zu us to call mbedtls_ssl_read",
+              timespec_tomicros(timespec_sub(timespec_real(), ts1)));
         if (!received) {
           WARNF("%s client disconnected so killing worker %d", origname,
                 client->pid);
@@ -640,7 +672,10 @@ RetryOnEtxtbsyRaceCondition:
       }
       if (fds[1].revents) {
         char buf[512];
+        ts1 = timespec_real();
         ssize_t got = read(client->pipe[0], buf, sizeof(buf));
+        DEBUF("it took %'zu us to call read",
+              timespec_tomicros(timespec_sub(timespec_real(), ts1)));
         if (got == -1) {
           WARNF("got %s reading %s output", strerror(errno), origname);
           goto HangupClientAndTerminateJob;
@@ -658,7 +693,10 @@ RetryOnEtxtbsyRaceCondition:
 WaitAgain:
   DEBUF("waitpid");
   struct rusage rusage;
+  ts1 = timespec_real();
   int wrc = wait4(client->pid, &wstatus, 0, &rusage);
+  DEBUF("it took %'zu us to call wait4",
+        timespec_tomicros(timespec_sub(timespec_real(), ts1)));
   if (wrc == -1) {
     if (errno == EINTR) {
       WARNF("waitpid interrupted; killing %s pid %d", origname, client->pid);
@@ -711,13 +749,18 @@ WaitAgain:
     AppendResourceReport(&client->output, &rusage, "\n");
     PrintProgramOutput(client);
   }
+  ts1 = timespec_real();
   SendProgramOutput(client);
   SendExitMessage(exitcode);
   mbedtls_ssl_close_notify(&ezssl);
+  DEBUF("it took %'zu us to send result to client",
+        timespec_tomicros(timespec_sub(timespec_real(), ts1)));
   if (etxtbsy_tries > 1) {
     WARNF("encountered %d ETXTBSY race conditions spawning %s",
           etxtbsy_tries - 1, origname);
   }
+  DEBUF("it took %'zu us TO DO EVERYTHING",
+        timespec_tomicros(timespec_sub(timespec_real(), ts0)));
   pthread_exit(0);
 }