Rewrite memory manager

Actually Portable Executable now supports Android. Cosmo's old mmap code
required a 47 bit address space. The new implementation is very agnostic
and supports both smaller address spaces (e.g. embedded) and even modern
56-bit PML5T paging for x86 which finally came true on Zen4 Threadripper

Cosmopolitan no longer requires UNIX systems to observe the Windows 64kb
granularity; i.e. sysconf(_SC_PAGE_SIZE) will now report the host native
page size. This fixes a longstanding POSIX conformance issue, concerning
file mappings that overlap the end of file. Other aspects of conformance
have been improved too, such as the subtleties of address assignment and
and the various subtleties surrounding MAP_FIXED and MAP_FIXED_NOREPLACE

On Windows, mappings larger than 100 megabytes won't be broken down into
thousands of independent 64kb mappings. Support for MAP_STACK is removed
by this change; please use NewCosmoStack() instead.

Stack overflow avoidance is now being implemented using the POSIX thread
APIs. Please use GetStackBottom() and GetStackAddr(), instead of the old
error-prone GetStackAddr() and HaveStackMemory() APIs which are removed.

ape/aarch64.lds

@@ -1,6 +1,7 @@
 /*-*- mode: ld-script; indent-tabs-mode: nil; tab-width: 2; coding: utf-8   -*-│
 │ vi: set et sts=2 sw=2 fenc=utf-8                                         :vi │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/sysv/consts/prot.h"
 #include "libc/thread/tls.h"
 
 ENTRY(_start)
@@ -285,9 +286,13 @@ SECTIONS {
 
 ape_stack_vaddr = DEFINED(ape_stack_vaddr) ? ape_stack_vaddr : 0x700000000000;
 ape_stack_memsz = DEFINED(ape_stack_memsz) ? ape_stack_memsz : 8 * 1024 * 1024;
-ape_stack_align = DEFINED(ape_stack_align) ? ape_stack_align : 16;
+ape_stack_align = DEFINED(ape_stack_align) ? MAX(ape_stack_align, 16) : 16;
+ape_stack_prot = PROT_READ | PROT_WRITE;
 ape_stack_round = -ape_stack_align;
 
+ASSERT(ape_stack_align < 1048576, "stack align too big");
+ASSERT(!(ape_stack_align & (ape_stack_align - 1)), "stack align must be two power");
+
 _tls_size = _tbss_end - _tdata_start;
 _tdata_size = _tdata_end - _tdata_start;
 _tbss_size = _tbss_end - _tbss_start;

ape/ape.lds

@@ -588,9 +588,12 @@ ape_stack_vaddr = DEFINED(ape_stack_vaddr) ? ape_stack_vaddr : 0x700000000000;
 ape_stack_paddr = ape_ram_paddr + ape_ram_filesz;
 ape_stack_filesz = 0;
 ape_stack_memsz = DEFINED(ape_stack_memsz) ? ape_stack_memsz : 8 * 1024 * 1024;
-ape_stack_align = DEFINED(ape_stack_align) ? ape_stack_align : 16;
+ape_stack_align = DEFINED(ape_stack_align) ? MAX(ape_stack_align, 16) : 16;
 ape_stack_round = -ape_stack_align;
 
+ASSERT(ape_stack_align < 1048576, "stack align too big");
+ASSERT(!(ape_stack_align & (ape_stack_align - 1)), "stack align must be two power");
+
 ape_note_offset = ape_cod_offset + (ape_note - ape_cod_vaddr);
 ape_note_filesz = ape_note_end - ape_note;
 ape_note_memsz = ape_note_filesz;

ctl/README.md

@@ -12,7 +12,7 @@ This package provides an STL alternative that's designed to:
 
 This code is experimental and mostly for Justine to save herself from
 the STL trainwreck. That doesn't mean cosmocc users aren't welcome to
-use tihs too. You shouldn't expect the design to not change later on.
+use this too. You shouldn't expect the design to not change later on.
 
 ## Rationale
 

examples/nesemu1.cc

@@ -137,7 +137,7 @@ struct Action {
 
 struct Audio {
   size_t i;
-  int16_t p[FRAMESIZE];
+  int16_t p[65536];
 };
 
 struct Status {

libc/calls/finddebugbinary.c

@@ -27,7 +27,6 @@
 #include "libc/elf/tinyelf.internal.h"
 #include "libc/errno.h"
 #include "libc/intrin/directmap.internal.h"
-#include "libc/intrin/kprintf.h"
 #include "libc/nt/memory.h"
 #include "libc/nt/runtime.h"
 #include "libc/runtime/runtime.h"

libc/calls/virtualmax.S

@@ -1,37 +0,0 @@
-/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8     -*-│
-│ vi: set noet ft=asm ts=8 sw=8 fenc=utf-8                                 :vi │
-╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
-│                                                                              │
-│ Permission to use, copy, modify, and/or distribute this software for         │
-│ any purpose with or without fee is hereby granted, provided that the         │
-│ above copyright notice and this permission notice appear in all copies.      │
-│                                                                              │
-│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
-│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
-│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
-│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
-│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
-│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
-│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
-│ PERFORMANCE OF THIS SOFTWARE.                                                │
-╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/macros.internal.h"
-
-	.initbss 201,_init___virtualmax
-
-//	Maximum amount of virtual memory in bytes.
-//
-//	mmap() will return ENOMEM once this is reached.
-//
-//	By default no limit is imposed.
-__virtualmax:
-	.quad	0
-	.endobj	__virtualmax,globl
-	.previous
-
-	.init.start 201,_init___virtualmax
-	push	$-1
-	pop	%rax
-	stosq
-	.init.end 201,_init___virtualmax

libc/integral/normalize.inc

@@ -84,7 +84,6 @@
 #endif
 
 #ifdef _COSMO_SOURCE
-#define FRAMESIZE 65536
 #define _PAGESIZE 4096
 #endif
 

libc/intrin/BUILD.mk

@@ -66,8 +66,7 @@ o/$(MODE)/libc/intrin/asan.o: private			\
 		CFLAGS +=				\
 			-O2				\
 			-finline			\
-			-finline-functions		\
+			-finline-functions
-			-fpatchable-function-entry=0,0
 
 o//libc/intrin/memmove.o: private			\
 		CFLAGS +=				\

libc/intrin/asan.c

@@ -16,6 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "ape/sections.internal.h"
 #include "libc/calls/state.internal.h"
 #include "libc/calls/struct/rlimit.h"
 #include "libc/calls/struct/rlimit.internal.h"
@@ -30,6 +31,7 @@
 #include "libc/intrin/kprintf.h"
 #include "libc/intrin/leaky.internal.h"
 #include "libc/intrin/likely.h"
+#include "libc/intrin/maps.h"
 #include "libc/intrin/strace.internal.h"
 #include "libc/intrin/weaken.h"
 #include "libc/log/libfatal.internal.h"
@@ -94,13 +96,6 @@ __static_yoink("_init_asan");
 
 #define ASAN_LOG(...) (void)0  // kprintf(__VA_ARGS__)
 
-#define REQUIRE(FUNC)               \
-  do {                              \
-    if (!_weaken(FUNC)) {           \
-      __asan_require_failed(#FUNC); \
-    }                               \
-  } while (0)
-
 struct AsanSourceLocation {
   char *filename;
   int line;
@@ -161,7 +156,7 @@ static char *__asan_stpcpy(char *d, const char *s) {
   }
 }
 
-void __asan_memset(void *p, char c, size_t n) {
+dontinstrument void __asan_memset(void *p, char c, size_t n) {
   char *b;
   size_t i;
   uint64_t x;
@@ -313,20 +308,16 @@ static wontreturn void __asan_require_failed(const char *func) {
   __asan_unreachable();
 }
 
-void __asan_poison(void *p, long n, signed char t) {
+static bool __asan_overlaps_shadow_space(const void *p, size_t n) {
-  signed char k, *s;
+  intptr_t BegA, EndA, BegB, EndB;
-  s = (signed char *)(((intptr_t)p >> 3) + 0x7fff8000);
+  if (n) {
-  if ((k = (intptr_t)p & 7)) {
+    BegA = (intptr_t)p;
-    if ((!*s && n >= 8 - k) || *s > k)
+    EndA = BegA + n;
-      *s = k;
+    BegB = 0x7fff0000;
-    n -= MIN(8 - k, n);
+    EndB = 0x100080000000;
-    s += 1;
+    return MAX(BegA, BegB) < MIN(EndA, EndB);
-  }
+  } else {
-  __asan_memset(s, t, n >> 3);
+    return 0;
-  if ((k = n & 7)) {
-    s += n >> 3;
-    if (*s < 0 || (*s > 0 && *s <= k))
-      *s = t;
   }
 }
 
@@ -353,17 +344,92 @@ void __asan_unpoison(void *p, long n) {
   }
 }
 
-bool __asan_is_mapped(int x) {
+dontinstrument void __asan_poison(void *p, long n, signed char t) {
-  // xxx: we can't lock because no reentrant locks yet
+  signed char k, *s;
-  int i;
+  s = (signed char *)(((intptr_t)p >> 3) + 0x7fff8000);
-  bool res;
+  if ((k = (intptr_t)p & 7)) {
-  __mmi_lock();
+    if ((!*s && n >= 8 - k) || *s > k)
-  i = __find_memory(&_mmi, x);
+      *s = k;
-  res = i < _mmi.i && x >= _mmi.p[i].x;
+    n -= MIN(8 - k, n);
-  __mmi_unlock();
+    s += 1;
+  }
+  __asan_memset(s, t, n >> 3);
+  if ((k = n & 7)) {
+    s += n >> 3;
+    if (*s < 0 || (*s > 0 && *s <= k))
+      *s = t;
+  }
+}
+
+static void __asan_poison_safe(char *p, long n, signed char t) {
+  int granularity = __granularity();
+  int chunk = granularity << 3;
+  while (n > 0) {
+    char *block = (char *)((uintptr_t)p & -chunk);
+    signed char *shadow = (signed char *)(((uintptr_t)block >> 3) + 0x7fff8000);
+    if (__asan_is_mapped(shadow)) {
+      char *start = MAX(p, block);
+      size_t bytes = MIN(n, chunk);
+      __asan_poison(start, bytes, t);
+    }
+    p += chunk;
+    n -= chunk;
+  }
+}
+
+privileged static bool __asan_is_mapped_unlocked(const char *addr) {
+  struct Dll *e, *e2;
+  for (e = dll_first(__maps.used); e; e = e2) {
+    e2 = dll_next(__maps.used, e);
+    struct Map *map = MAP_CONTAINER(e);
+    if (map->addr <= addr && addr < map->addr + map->size) {
+      dll_remove(&__maps.used, e);
+      dll_make_first(&__maps.used, e);
+      return true;
+    }
+  }
+  return false;
+}
+
+privileged bool __asan_is_mapped(const void *addr) {
+  bool32 res;
+  __maps_lock();
+  res = __asan_is_mapped_unlocked(addr);
+  __maps_unlock();
   return res;
 }
 
+static void __asan_ensure_shadow_is_mapped(void *addr, size_t size) {
+  int m = 0x7fff8000;
+  int g = __granularity();
+  size_t n = (size + 7) & -8;
+  char *p = (char *)((((uintptr_t)addr >> 3) + m) & -g);
+  char *pe = (char *)(((((uintptr_t)(addr + n) >> 3) + m) + g - 1) & -g);
+  for (char *pm = p; p < pe; p = pm) {
+    pm += g;
+    if (!__asan_is_mapped(p)) {
+      while (pm < pe && !__asan_is_mapped(pm))
+        pm += g;
+      if (__mmap(p, pm - p, PROT_READ | PROT_WRITE,
+                 MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) != MAP_FAILED)
+        __asan_memset(p, kAsanUnmapped, pm - p);
+    }
+  }
+}
+
+void __asan_shadow(void *addr, size_t size) {
+  if (!__asan_overlaps_shadow_space(addr, size)) {
+    __asan_ensure_shadow_is_mapped(addr, size);
+    __asan_unpoison(addr, size);
+  }
+}
+
+void __asan_unshadow(void *addr, size_t size) {
+  if (!__asan_overlaps_shadow_space(addr, size)) {
+    __asan_poison_safe(addr, size, kAsanUnmapped);
+  }
+}
+
 static bool __asan_is_image(const unsigned char *p) {
   return __executable_start <= p && p < _end;
 }
@@ -387,13 +453,15 @@ static struct AsanFault __asan_fault(const signed char *s, signed char dflt) {
 
 static struct AsanFault __asan_checka(const signed char *s, long ndiv8) {
   uint64_t w;
+  int g = __granularity();
+  const signed char *o = s;
   const signed char *e = s + ndiv8;
   for (; ((intptr_t)s & 7) && s < e; ++s) {
     if (*s)
       return __asan_fault(s - 1, kAsanHeapOverrun);
   }
   for (; s + 8 <= e; s += 8) {
-    if (UNLIKELY(!((intptr_t)s & (FRAMESIZE - 1))) && kisdangerous(s)) {
+    if (UNLIKELY(!((intptr_t)s & (g - 1))) && kisdangerous(s)) {
       return (struct AsanFault){kAsanUnmapped, s};
     }
     if ((w = READ64LE(s))) {
@@ -403,7 +471,7 @@ static struct AsanFault __asan_checka(const signed char *s, long ndiv8) {
   }
   for (; s < e; ++s) {
     if (*s)
-      return __asan_fault(s - 1, kAsanHeapOverrun);
+      return __asan_fault(s > o ? s - 1 : s, kAsanHeapOverrun);
   }
   return (struct AsanFault){0};
 }
@@ -429,7 +497,7 @@ struct AsanFault __asan_check(const void *p, long n) {
   if (n > 0) {
     k = (intptr_t)p & 7;
     s = SHADOW(p);
-    if (OverlapsShadowSpace(p, n)) {
+    if (__asan_overlaps_shadow_space(p, n)) {
       return (struct AsanFault){kAsanProtected, s};
     } else if (kisdangerous(s)) {
       return (struct AsanFault){kAsanUnmapped, s};
@@ -475,8 +543,9 @@ struct AsanFault __asan_check(const void *p, long n) {
 struct AsanFault __asan_check_str(const char *p) {
   uint64_t w;
   signed char c, k, *s;
+  int g = __granularity();
   s = SHADOW(p);
-  if (OverlapsShadowSpace(p, 1)) {
+  if (__asan_overlaps_shadow_space(p, 1)) {
     return (struct AsanFault){kAsanProtected, s};
   }
   if (kisdangerous(s)) {
@@ -494,7 +563,7 @@ struct AsanFault __asan_check_str(const char *p) {
     ++s;
   }
   for (;; ++s, p += 8) {
-    if (UNLIKELY(!((intptr_t)s & (FRAMESIZE - 1))) && kisdangerous(s)) {
+    if (UNLIKELY(!((intptr_t)s & (g - 1))) && kisdangerous(s)) {
       return (struct AsanFault){kAsanUnmapped, s};
     }
     if ((c = *s) < 0) {
@@ -749,9 +818,9 @@ void __asan_report_memory_origin(const unsigned char *addr, int size,
   }
   if (__executable_start <= addr && addr < _end) {
     __asan_report_memory_origin_image((intptr_t)addr, size);
-  } else if (IsAutoFrame((intptr_t)addr >> 16)) {
+    /* } else if (IsAutoFrame((intptr_t)addr >> 16)) { */
-    if (_weaken(__asan_report_memory_origin_heap))
+    /*   if (_weaken(__asan_report_memory_origin_heap)) */
-      _weaken(__asan_report_memory_origin_heap)(addr, size);
+    /*     _weaken(__asan_report_memory_origin_heap)(addr, size); */
   }
 }
 
@@ -766,9 +835,8 @@ static __wur __asan_die_f *__asan_report(const void *addr, int size,
   int i;
   wint_t c;
   signed char t;
-  uint64_t x, y, z;
+  uint64_t z;
   char *base, *q, *p = buf;
-  struct MemoryIntervals *m;
   ftrace_enabled(-1);
   kprintf("\n\e[J\e[1;31masan error\e[0m: %s %d-byte %s at %p shadow %p\n",
           __asan_describe_access_poison(kind), size, message, addr,
@@ -832,20 +900,20 @@ static __wur __asan_die_f *__asan_report(const void *addr, int size,
   p = __asan_format_section(p, __executable_start, _etext, ".text", addr);
   p = __asan_format_section(p, _etext, _edata, ".data", addr);
   p = __asan_format_section(p, _end, _edata, ".bss", addr);
-  __mmi_lock();
+  __maps_lock();
-  for (m = &_mmi, i = 0; i < m->i; ++i) {
+  struct Dll *e, *e2;
-    x = m->p[i].x;
+  for (e = dll_first(__maps.used); e; e = e2) {
-    y = m->p[i].y;
+    e2 = dll_next(__maps.used, e);
-    p = __asan_format_interval(p, x << 16, (y << 16) + (FRAMESIZE - 1));
+    struct Map *map = MAP_CONTAINER(e);
-    z = (intptr_t)addr >> 16;
+    p = __asan_format_interval(p, (uintptr_t)map->addr,
-    if (x <= z && z <= y)
+                               (uintptr_t)map->addr + map->size);
+    if (!__asan_overlaps_shadow_space(map->addr, map->size))
       p = __asan_stpcpy(p, " ←address");
-    z = (((intptr_t)addr >> 3) + 0x7fff8000) >> 16;
+    else
-    if (x <= z && z <= y)
       p = __asan_stpcpy(p, " ←shadow");
     *p++ = '\n';
   }
-  __mmi_unlock();
+  __maps_unlock();
   *p = 0;
   kprintf("%s", buf);
   __asan_report_memory_origin(addr, size, kind);
@@ -1033,95 +1101,16 @@ void __asan_after_dynamic_init(void) {
   ASAN_LOG("__asan_after_dynamic_init()\n");
 }
 
-void __asan_map_shadow(uintptr_t p, size_t n) {
+void __asan_init(void) {
-  // assume _mmi.lock is held
-  void *addr;
-  int i, a, b;
-  size_t size;
-  int prot, flag;
-  struct DirectMap sm;
-  struct MemoryIntervals *m;
-  if (OverlapsShadowSpace((void *)p, n)) {
-    kprintf("error: %p size %'zu overlaps shadow space: %s\n", p, n,
-            DescribeBacktrace(__builtin_frame_address(0)));
-    _Exit(1);
-  }
-  m = &_mmi;
-  a = (0x7fff8000 + (p >> 3)) >> 16;
-  b = (0x7fff8000 + (p >> 3) + (n >> 3) + 0xffff) >> 16;
-  for (; a <= b; a += i) {
-    i = 1;
-    if (__asan_is_mapped(a)) {
-      continue;
-    }
-    for (; a + i <= b; ++i) {
-      if (__asan_is_mapped(a + i)) {
-        break;
-      }
-    }
-    size = (size_t)i << 16;
-    addr = (void *)ADDR_32_TO_48(a);
-    prot = PROT_READ | PROT_WRITE;
-    flag = MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS;
-    sm = sys_mmap(addr, size, prot, flag, -1, 0);
-    if (sm.addr == MAP_FAILED ||
-        __track_memory(m, a, a + i - 1, sm.maphandle, PROT_READ | PROT_WRITE,
-                       MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, false, false, 0,
-                       size) == -1) {
-      kprintf("error: could not map asan shadow memory\n");
-      __asan_die()();
-      __asan_unreachable();
-    }
-    __repstosb(addr, kAsanUnmapped, size);
-  }
-  __asan_unpoison((char *)p, n);
-}
-
-static textstartup void __asan_shadow_mapping(struct MemoryIntervals *m,
-                                              size_t i) {
-  uintptr_t x, y;
-  if (i < m->i) {
-    x = m->p[i].x;
-    y = m->p[i].y;
-    __asan_shadow_mapping(m, i + 1);
-    __asan_map_shadow(x << 16, (y - x + 1) << 16);
-  }
-}
-
-static textstartup void __asan_shadow_existing_mappings(void) {
-  __asan_shadow_mapping(&_mmi, 0);
-  if (!IsWindows()) {
-    int guard;
-    void *addr;
-    size_t size;
-    __get_main_stack(&addr, &size, &guard);
-    __asan_map_shadow((uintptr_t)addr, size);
-    __asan_poison(addr, guard, kAsanStackOverflow);
-  }
-}
-
-static size_t __asan_strlen(const char *s) {
-  size_t i = 0;
-  while (s[i])
-    ++i;
-  return i;
-}
-
-forceinline ssize_t __write_str(const char *s) {
-  return sys_write(2, s, __asan_strlen(s));
-}
-
-void __asan_init(int argc, char **argv, char **envp, unsigned long *auxv) {
   static bool once;
-  if (!_cmpxchg(&once, false, true))
+  if (once)
     return;
-  __asan_shadow_existing_mappings();
+  once = true;
-  __asan_map_shadow((uintptr_t)__executable_start, _end - __executable_start);
+
-  __asan_map_shadow(0, 4096);
+  __asan_shadow(0, 4096);
+  __asan_shadow(__maps.stack.addr, __maps.stack.size);
+  __asan_shadow(__executable_start, _end - __executable_start);
   __asan_poison((void *)__veil("r", 0L), getauxval(AT_PAGESZ), kAsanNullPage);
-  if (!IsWindows()) {
-    sys_mprotect((void *)0x7fff8000, 0x10000, PROT_READ);
-  }
   STRACE("    _    ____    _    _   _ ");
   STRACE("   / \\  / ___|  / \\  | \\ | |");
   STRACE("  / _ \\ \\___ \\ / _ \\ |  \\| |");

libc/intrin/asan.internal.h

@@ -46,7 +46,8 @@ void __asan_unpoison(void *, long);
 void __asan_poison(void *, long, signed char);
 void __asan_verify(const void *, size_t);
 void __asan_verify_str(const char *);
-void __asan_map_shadow(uintptr_t, size_t);
+void __asan_shadow(void *, size_t);
+void __asan_unshadow(void *, size_t);
 bool __asan_is_valid(const void *, long) nosideeffect;
 bool __asan_is_valid_str(const char *) nosideeffect;
 bool __asan_is_valid_strlist(char *const *) nosideeffect;
@@ -54,9 +55,9 @@ bool __asan_is_valid_iov(const struct iovec *, int) nosideeffect;
 struct AsanFault __asan_check(const void *, long) nosideeffect;
 struct AsanFault __asan_check_str(const char *) nosideeffect;
 
-bool __asan_is_mapped(int);
 int __asan_is_leaky(void *);
 int __asan_print_trace(void *);
+bool __asan_is_mapped(const void *);
 __asan_die_f *__asan_die(void) __wur;
 void __asan_memset(void *, char, size_t);
 size_t __asan_get_heap_size(const void *);

libc/intrin/asaninit.S

@@ -19,14 +19,13 @@
 #include "libc/macros.internal.h"
 
 	.init.start 303,_init_asan
+#ifdef __x86_64__
+#ifdef __SANITIZE_ADDRESS__
 	push	%rdi
 	push	%rsi
-	mov	%r12,%rdi
-	mov	%r13,%rsi
-	mov	%r14,%rdx
-	mov	%r15,%rcx
-	.weak	__asan_init
 	call	__asan_init
 	pop	%rsi
 	pop	%rdi
+#endif /* __SANITIZE_ADDRESS__ */
+#endif /* __x86_64__ */
 	.init.end 303,_init_asan

libc/intrin/describeflags.internal.h

@@ -21,7 +21,6 @@ const char *DescribeDnotifyFlags(char[80], int) libcesque;
 const char *DescribeErrno(char[30], int) libcesque;
 const char *DescribeFcntlCmd(char[20], int) libcesque;
 const char *DescribeFlockType(char[12], int) libcesque;
-const char *DescribeFrame(char[32], int) libcesque;
 const char *DescribeFutexOp(char[64], int) libcesque;
 const char *DescribeHow(char[12], int) libcesque;
 const char *DescribeInOutInt64(char[23], ssize_t, int64_t *) libcesque;
@@ -80,7 +79,6 @@ const char *DescribeWhichPrio(char[12], int) libcesque;
 #define DescribeErrno(x)             DescribeErrno(alloca(30), x)
 #define DescribeFcntlCmd(x)          DescribeFcntlCmd(alloca(20), x)
 #define DescribeFlockType(x)         DescribeFlockType(alloca(12), x)
-#define DescribeFrame(x)             DescribeFrame(alloca(32), x)
 #define DescribeFutexOp(x)           DescribeFutexOp(alloca(64), x)
 #define DescribeHow(x)               DescribeHow(alloca(12), x)
 #define DescribeInOutInt64(rc, x)    DescribeInOutInt64(alloca(23), rc, x)

libc/intrin/describeframe.c

@@ -1,72 +0,0 @@
-/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
-│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
-╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2021 Justine Alexandra Roberts Tunney                              │
-│                                                                              │
-│ Permission to use, copy, modify, and/or distribute this software for         │
-│ any purpose with or without fee is hereby granted, provided that the         │
-│ above copyright notice and this permission notice appear in all copies.      │
-│                                                                              │
-│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
-│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
-│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
-│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
-│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
-│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
-│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
-│ PERFORMANCE OF THIS SOFTWARE.                                                │
-╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/dce.h"
-#include "libc/intrin/describeflags.internal.h"
-#include "libc/intrin/kprintf.h"
-#include "libc/intrin/weaken.h"
-#include "libc/macros.internal.h"
-#include "libc/runtime/memtrack.internal.h"
-#include "libc/runtime/runtime.h"
-#include "libc/runtime/winargs.internal.h"
-
-#define UNSHADOW(x) ((int64_t)(MAX(0, (x) - 0x7fff8000)) << 3)
-#define FRAME(x)    ((int)((x) >> 16))
-
-static const char *GetFrameName(int x) {
-  if (!x) {
-    return "null";
-  } else if (IsShadowFrame(x)) {
-    return "shadow";
-  } else if (IsAutoFrame(x)) {
-    return "automap";
-  } else if (IsFixedFrame(x)) {
-    return "fixed";
-  } else if (IsStaticStackFrame(x)) {
-    return "stack";
-  } else if (IsGfdsFrame(x)) {
-    return "g_fds";
-  } else if (IsNsyncFrame(x)) {
-    return "nsync";
-  } else if (IsZiposFrame(x)) {
-    return "zipos";
-  } else if (IsMemtrackFrame(x)) {
-    return "memtrack";
-  } else if (IsOldStackFrame(x)) {
-    return "system stack";
-  } else if (((GetStaticStackAddr(0) + GetStackSize()) >> 16) <= x &&
-             x <= ((GetStaticStackAddr(0) + GetStackSize() - 1) >> 16)) {
-    return "static stack";
-  } else if ((int)((intptr_t)__executable_start >> 16) <= x &&
-             x <= (int)(((intptr_t)_end - 1) >> 16)) {
-    return "image";
-  } else {
-    return "unknown";
-  }
-}
-
-const char *(DescribeFrame)(char buf[32], int x) {
-  if (IsShadowFrame(x)) {
-    ksnprintf(buf, 32, "%s %s %.8x", GetFrameName(x),
-              GetFrameName(FRAME(UNSHADOW(ADDR_32_TO_48(x)))),
-              FRAME(UNSHADOW(ADDR_32_TO_48(x))));
-    return buf;
-  } else {
-    return GetFrameName(x);
-  }
-}

libc/intrin/describemapflags.c

@@ -24,7 +24,6 @@
 
 const char *(DescribeMapFlags)(char buf[64], int x) {
   const struct DescribeFlags kMapFlags[] = {
-      {MAP_STACK, "STACK"},                      // order matters
       {MAP_PRIVATE, "PRIVATE"},                  //
       {MAP_ANONYMOUS, "ANONYMOUS"},              //
       {MAP_SHARED, "SHARED"},                    //

libc/intrin/describemapping.c

@@ -29,8 +29,6 @@ static char DescribeMapType(int flags) {
       return 'p';
     case MAP_SHARED:
       return 's';
-    case MAP_STACK:
-      return 'S';
     default:
       return '?';
   }

libc/intrin/directmap-nt.c

@@ -86,7 +86,7 @@ TryAgain:
     if ((dm.addr = MapViewOfFileEx(dm.maphandle, fl.flags2, off >> 32, off,
                                    size, addr))) {
       uint32_t oldprot;
-      if (VirtualProtect(addr, size, __prot2nt(prot, iscow), &oldprot)) {
+      if (VirtualProtect(dm.addr, size, __prot2nt(prot, iscow), &oldprot)) {
         return dm;
       }
       UnmapViewOfFile(dm.addr);

libc/intrin/directmap.c

@@ -16,6 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/calls.h"
 #include "libc/calls/syscall-sysv.internal.h"
 #include "libc/dce.h"
 #include "libc/errno.h"
@@ -24,7 +25,9 @@
 #include "libc/intrin/strace.internal.h"
 #include "libc/nt/runtime.h"
 #include "libc/runtime/memtrack.internal.h"
+#include "libc/runtime/runtime.h"
 #include "libc/runtime/syslib.internal.h"
+#include "libc/sysv/errfuns.h"
 
 /**
  * Obtains memory mapping directly from system.
@@ -39,7 +42,10 @@
 struct DirectMap sys_mmap(void *addr, size_t size, int prot, int flags, int fd,
                           int64_t off) {
   struct DirectMap d;
-  if (IsXnuSilicon()) {
+  if ((__virtualsize += size) >= __virtualmax) {
+    d.maphandle = kNtInvalidHandleValue;
+    d.addr = (void *)enomem();
+  } else if (IsXnuSilicon()) {
     long p = _sysret(__syslib->__mmap(addr, size, prot, flags, fd, off));
     d.maphandle = kNtInvalidHandleValue;
     d.addr = (void *)p;
@@ -51,9 +57,10 @@ struct DirectMap sys_mmap(void *addr, size_t size, int prot, int flags, int fd,
   } else {
     d = sys_mmap_nt(addr, size, prot, flags, fd, off);
   }
-  KERNTRACE("sys_mmap(%.12p /* %s */, %'zu, %s, %s, %d, %'ld) → {%.12p, %p}% m",
+  if (d.addr == MAP_FAILED)
-            addr, DescribeFrame((intptr_t)addr >> 16), size,
+    __virtualsize -= size;
-            DescribeProtFlags(prot), DescribeMapFlags(flags), fd, off, d.addr,
+  KERNTRACE("sys_mmap(%.12p, %'zu, %s, %s, %d, %'ld) → {%.12p, %p}% m", addr,
-            d.maphandle);
+            size, DescribeProtFlags(prot), DescribeMapFlags(flags), fd, off,
+            d.addr, d.maphandle);
   return d;
 }

libc/intrin/dll.c

@@ -28,7 +28,7 @@
  *
  * It's required that `elem` and `succ` aren't part of the same list.
  */
-void dll_splice_after(struct Dll *elem, struct Dll *succ) {
+privileged void dll_splice_after(struct Dll *elem, struct Dll *succ) {
   struct Dll *tmp1, *tmp2;
   tmp1 = elem->next;
   tmp2 = succ->prev;
@@ -43,7 +43,7 @@ void dll_splice_after(struct Dll *elem, struct Dll *succ) {
  *
  * @param list is a doubly-linked list, where `!*list` means empty
  */
-void dll_remove(struct Dll **list, struct Dll *elem) {
+privileged void dll_remove(struct Dll **list, struct Dll *elem) {
   if (*list == elem) {
     if ((*list)->prev == *list) {
       *list = 0;
@@ -66,7 +66,7 @@ void dll_remove(struct Dll **list, struct Dll *elem) {
  * @param list is a doubly-linked list, where `!*list` means empty
  * @param elem must not be a member of `list`, or null for no-op
  */
-void dll_make_first(struct Dll **list, struct Dll *elem) {
+privileged void dll_make_first(struct Dll **list, struct Dll *elem) {
   if (elem) {
     if (!*list) {
       *list = elem->prev;
@@ -85,7 +85,7 @@ void dll_make_first(struct Dll **list, struct Dll *elem) {
  * @param list is a doubly-linked list, where `!*list` means empty
  * @param elem must not be a member of `list`, or null for no-op
  */
-void dll_make_last(struct Dll **list, struct Dll *elem) {
+privileged void dll_make_last(struct Dll **list, struct Dll *elem) {
   if (elem) {
     dll_make_first(list, elem->next);
     *list = elem;

libc/intrin/dll.h

@@ -14,38 +14,38 @@ struct Dll {
   struct Dll *prev;
 };
 
-static inline void dll_init(struct Dll *e) {
+forceinline void dll_init(struct Dll *e) {
   e->next = e;
   e->prev = e;
 }
 
-static inline int dll_is_alone(struct Dll *e) {
+forceinline int dll_is_alone(struct Dll *e) {
   return e->next == e && e->prev == e;
 }
 
-static inline int dll_is_empty(struct Dll *list) {
+forceinline int dll_is_empty(struct Dll *list) {
   return !list;
 }
 
-static inline struct Dll *dll_last(struct Dll *list) {
+forceinline struct Dll *dll_last(struct Dll *list) {
   return list;
 }
 
-static inline struct Dll *dll_first(struct Dll *list) {
+forceinline struct Dll *dll_first(struct Dll *list) {
   struct Dll *first = 0;
   if (list)
     first = list->next;
   return first;
 }
 
-static inline struct Dll *dll_next(struct Dll *list, struct Dll *e) {
+forceinline struct Dll *dll_next(struct Dll *list, struct Dll *e) {
   struct Dll *next = 0;
   if (e != list)
     next = e->next;
   return next;
 }
 
-static inline struct Dll *dll_prev(struct Dll *list, struct Dll *e) {
+forceinline struct Dll *dll_prev(struct Dll *list, struct Dll *e) {
   struct Dll *prev = 0;
   if (e != list->next)
     prev = e->prev;

libc/intrin/extend.c

@@ -16,51 +16,14 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/assert.h"
 #include "libc/calls/calls.h"
-#include "libc/calls/struct/sigset.internal.h"
 #include "libc/dce.h"
-#include "libc/errno.h"
 #include "libc/intrin/asan.internal.h"
-#include "libc/intrin/asancodes.h"
+#include "libc/runtime/runtime.h"
-#include "libc/intrin/describebacktrace.internal.h"
-#include "libc/intrin/directmap.internal.h"
-#include "libc/log/libfatal.internal.h"
-#include "libc/macros.internal.h"
-#include "libc/runtime/memtrack.internal.h"
 #include "libc/sysv/consts/map.h"
 #include "libc/sysv/consts/prot.h"
 
-#define G FRAMESIZE
+#define G __granularity()
-
-static void *_mapframe_impl(void *p, int f) {
-  int rc, prot, flags;
-  struct DirectMap dm;
-  prot = PROT_READ | PROT_WRITE;
-  flags = f | MAP_ANONYMOUS | MAP_FIXED;
-  if ((dm = sys_mmap(p, G, prot, flags, -1, 0)).addr == p) {
-    rc = __track_memory(&_mmi, (uintptr_t)p >> 16, (uintptr_t)p >> 16,
-                        dm.maphandle, prot, flags, false, false, 0, G);
-    if (!rc) {
-      return p;
-    } else {
-      return 0;
-    }
-  } else {
-    return 0;
-  }
-}
-
-static void *_mapframe(void *p, int f) {
-  void *res;
-  // mmap isn't required to be @asyncsignalsafe but this is
-  BLOCK_SIGNALS;
-  __mmi_lock();
-  res = _mapframe_impl(p, f);
-  __mmi_unlock();
-  ALLOW_SIGNALS;
-  return res;
-}
 
 /**
  * Extends static allocation.
@@ -84,32 +47,23 @@ static void *_mapframe(void *p, int f) {
  */
 void *_extend(void *p, size_t n, void *e, int f, intptr_t h) {
   char *q;
-#ifndef NDEBUG
+
+#ifdef __SANITIZE_ADDRESS__
   if ((uintptr_t)SHADOW(p) & (G - 1))
-    notpossible;
+    __builtin_trap();
   if ((uintptr_t)p + (G << kAsanScale) > h)
-    notpossible;
+    __builtin_trap();
 #endif
-  // TODO(jart): Make this spin less in non-ASAN mode.
+
   for (q = e; q < ((char *)p + n); q += 8) {
     if (!((uintptr_t)q & (G - 1))) {
-#ifndef NDEBUG
       if (q + G > (char *)h)
-        notpossible;
+        __builtin_trap();
-#endif
+      if (mmap(q, G, PROT_READ | PROT_WRITE, f | MAP_ANONYMOUS | MAP_FIXED, -1,
-      if (!_mapframe(q, f))
+               0) == MAP_FAILED)
         return 0;
-      if (IsAsan()) {
-        if (!((uintptr_t)SHADOW(q) & (G - 1))) {
-          if (!_mapframe(SHADOW(q), f))
-            return 0;
-          __asan_poison(q, G << kAsanScale, kAsanProtected);
-        }
-      }
-    }
-    if (IsAsan()) {
-      *SHADOW(q) = 0;
     }
   }
+
   return q;
 }

libc/intrin/findmemoryinterval.c

@@ -1,34 +0,0 @@
-/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
-│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
-╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
-│                                                                              │
-│ Permission to use, copy, modify, and/or distribute this software for         │
-│ any purpose with or without fee is hereby granted, provided that the         │
-│ above copyright notice and this permission notice appear in all copies.      │
-│                                                                              │
-│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
-│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
-│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
-│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
-│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
-│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
-│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
-│ PERFORMANCE OF THIS SOFTWARE.                                                │
-╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/runtime/memtrack.internal.h"
-
-unsigned __find_memory(const struct MemoryIntervals *mm, int x) {
-  unsigned l, m, r;
-  l = 0;
-  r = mm->i;
-  while (l < r) {
-    m = (l & r) + ((l ^ r) >> 1);  // floor((a+b)/2)
-    if (mm->p[m].y < x) {
-      l = m + 1;
-    } else {
-      r = m;
-    }
-  }
-  return l;
-}

libc/intrin/getmainstack.c

@@ -16,14 +16,14 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/assert.h"
 #include "libc/calls/struct/rlimit.h"
 #include "libc/calls/struct/rlimit.internal.h"
 #include "libc/dce.h"
+#include "libc/intrin/getauxval.internal.h"
 #include "libc/intrin/kprintf.h"
+#include "libc/intrin/maps.h"
 #include "libc/macros.internal.h"
 #include "libc/runtime/runtime.h"
-#include "libc/runtime/stack.h"
 #include "libc/sysv/consts/auxv.h"
 #include "libc/sysv/consts/rlim.h"
 #include "libc/sysv/consts/rlimit.h"
@@ -101,30 +101,22 @@ static size_t __get_stack_size(int pagesz, uintptr_t start, uintptr_t top) {
 
 /**
  * Returns approximate boundaries of main thread stack.
+ *
+ * This function works on every OS except Windows.
  */
-void __get_main_stack(void **out_addr, size_t *out_size, int *out_guardsize) {
+struct AddrSize __get_main_stack(void) {
-  if (IsWindows()) {
-    *out_addr = (void *)GetStaticStackAddr(0);
-    *out_size = GetStaticStackSize();
-    *out_guardsize = getauxval(AT_PAGESZ);
-    return;
-  }
   int pagesz = getauxval(AT_PAGESZ);
   uintptr_t start = (uintptr_t)__argv;
   uintptr_t top = __get_main_top(pagesz);
   uintptr_t bot = top - __get_stack_size(pagesz, start, top);
-  uintptr_t vdso = getauxval(AT_SYSINFO_EHDR);
+  struct AuxiliaryValue avdso = __getauxval(AT_SYSINFO_EHDR);
-  if (vdso) {
+  if (avdso.isfound) {
+    uintptr_t vdso = avdso.value;
     if (vdso > start && vdso < top) {
       top = vdso;
     } else if (vdso < start && vdso >= bot) {
       bot += vdso + pagesz * 2;
     }
   }
-  unassert(bot < top);
+  return (struct AddrSize){(char *)bot, top - bot};
-  unassert(bot < start);
-  unassert(top > start);
-  *out_addr = (void *)bot;
-  *out_size = top - bot;
-  *out_guardsize = pagesz;
 }

libc/intrin/getsafesize.greg.c

@@ -37,11 +37,6 @@ privileged long __get_safe_size(long want, long extraspace) {
     return want;
   struct PosixThread *pt;
   struct CosmoTib *tib = __get_tls_privileged();
-  if (!IsAutoFrame((uintptr_t)tib >> 16) &&
-      !(__executable_start <= (const unsigned char *)tib &&
-        (const unsigned char *)tib < _end)) {
-    return want;
-  }
   long bottom, sp = GetStackPointer();
   if ((char *)sp >= tib->tib_sigstack_addr &&
       (char *)sp <= tib->tib_sigstack_addr + tib->tib_sigstack_size) {

libc/intrin/granularity.c

@@ -1,7 +1,7 @@
 /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
 │ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
 ╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│ Copyright 2024 Justine Alexandra Roberts Tunney                              │
 │                                                                              │
 │ Permission to use, copy, modify, and/or distribute this software for         │
 │ any purpose with or without fee is hereby granted, provided that the         │
@@ -16,14 +16,15 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/runtime/memtrack.internal.h"
+#include "libc/dce.h"
 #include "libc/runtime/runtime.h"
+#include "libc/sysv/consts/auxv.h"
 
-/**
+int __granularity(void) {
- * Prints memory mappings to stderr.
+  if (IsWindows())
- */
+    return 65536;
-void __print_maps(void) {
+  static int res;
-  __mmi_lock();
+  if (!res)
-  PrintMemoryIntervals(2, &_mmi);
+    res = getauxval(AT_PAGESZ);
-  __mmi_unlock();
+  return res;
 }

libc/intrin/klog.c

@@ -1,7 +1,7 @@
 /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
 │ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
 ╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│ Copyright 2024 Justine Alexandra Roberts Tunney                              │
 │                                                                              │
 │ Permission to use, copy, modify, and/or distribute this software for         │
 │ any purpose with or without fee is hereby granted, provided that the         │
@@ -16,12 +16,5 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/runtime/memtrack.internal.h"
 
-size_t __get_memtrack_size(struct MemoryIntervals *mm) {
+long __klog_handle;
-  size_t i, n;
-  for (n = i = 0; i < mm->i; ++i) {
-    n += ((size_t)(mm->p[i].y - mm->p[i].x) + 1) << 16;
-  }
-  return n;
-}

libc/intrin/kprintf.greg.c

@@ -25,8 +25,11 @@
 #include "libc/fmt/magnumstrs.internal.h"
 #include "libc/intrin/asmflag.h"
 #include "libc/intrin/atomic.h"
+#include "libc/intrin/dll.h"
 #include "libc/intrin/getenv.internal.h"
+#include "libc/intrin/kprintf.h"
 #include "libc/intrin/likely.h"
+#include "libc/intrin/maps.h"
 #include "libc/intrin/nomultics.internal.h"
 #include "libc/intrin/weaken.h"
 #include "libc/log/internal.h"
@@ -45,9 +48,11 @@
 #include "libc/runtime/internal.h"
 #include "libc/runtime/memtrack.internal.h"
 #include "libc/runtime/runtime.h"
+#include "libc/runtime/stack.h"
 #include "libc/runtime/symbols.internal.h"
 #include "libc/serialize.h"
 #include "libc/stdckdint.h"
+#include "libc/stdio/sysparam.h"
 #include "libc/str/str.h"
 #include "libc/str/tab.internal.h"
 #include "libc/str/utf16.h"
@@ -117,7 +122,7 @@ __msabi extern typeof(SetLastError) *const __imp_SetLastError;
 __msabi extern typeof(WriteFile) *const __imp_WriteFile;
 // clang-format on
 
-long __klog_handle;
+extern long __klog_handle;
 extern struct SymbolTable *__symtab;
 
 __funline char *kadvance(char *p, char *e, long n) {
@@ -141,20 +146,6 @@ __funline char *kemitquote(char *p, char *e, signed char t, unsigned c) {
   return p;
 }
 
-__funline bool kiskernelpointer(const void *p) {
-  return 0x7f0000000000 <= (intptr_t)p && (intptr_t)p < 0x800000000000;
-}
-
-__funline bool kistextpointer(const void *p) {
-  return __executable_start <= (const unsigned char *)p &&
-         (const unsigned char *)p < _etext;
-}
-
-__funline bool kisimagepointer(const void *p) {
-  return __executable_start <= (const unsigned char *)p &&
-         (const unsigned char *)p < _end;
-}
-
 __funline bool kischarmisaligned(const char *p, signed char t) {
   if (t == -1)
     return (intptr_t)p & 1;
@@ -163,55 +154,26 @@ __funline bool kischarmisaligned(const char *p, signed char t) {
   return false;
 }
 
-__funline bool kismemtrackhosed(void) {
+privileged static bool32 kisdangerous_unlocked(const char *addr) {
-  return !((_weaken(_mmi)->i <= _weaken(_mmi)->n) &&
+  struct Dll *e, *e2;
-           (_weaken(_mmi)->p == _weaken(_mmi)->s ||
+  for (e = dll_first(__maps.used); e; e = e2) {
-            _weaken(_mmi)->p == (struct MemoryInterval *)kMemtrackStart));
+    e2 = dll_next(__maps.used, e);
-}
+    struct Map *map = MAP_CONTAINER(e);
-
+    if (map->addr <= addr && addr < map->addr + map->size) {
-privileged static bool kismapped(int x) {
+      dll_remove(&__maps.used, e);
-  // xxx: we can't lock because no reentrant locks yet
+      dll_make_first(&__maps.used, e);
-  size_t m, r, l = 0;
+      return !(map->prot & PROT_READ);
-  if (!_weaken(_mmi))
-    return true;
-  if (kismemtrackhosed())
-    return false;
-  r = _weaken(_mmi)->i;
-  while (l < r) {
-    m = (l & r) + ((l ^ r) >> 1);  // floor((a+b)/2)
-    if (_weaken(_mmi)->p[m].y < x) {
-      l = m + 1;
-    } else {
-      r = m;
     }
   }
-  if (l < _weaken(_mmi)->i && x >= _weaken(_mmi)->p[l].x) {
+  return true;
-    return !!(_weaken(_mmi)->p[l].prot & PROT_READ);
-  } else {
-    return false;
-  }
 }
 
-privileged bool32 kisdangerous(const void *p) {
+privileged bool32 kisdangerous(const void *addr) {
-  int frame;
+  bool32 res;
-  if (kisimagepointer(p))
+  __maps_lock();
-    return false;
+  res = kisdangerous_unlocked(addr);
-  if (kiskernelpointer(p))
+  __maps_unlock();
-    return false;
+  return res;
-  if (IsOldStack(p))
-    return false;
-  if (IsLegalPointer(p)) {
-    frame = (uintptr_t)p >> 16;
-    if (IsStackFrame(frame))
-      return false;
-    if (kismapped(frame))
-      return false;
-  }
-  if (GetStackAddr() + GetGuardSize() <= (uintptr_t)p &&
-      (uintptr_t)p < GetStackAddr() + GetStackSize()) {
-    return false;
-  }
-  return true;
 }
 
 privileged static void klogclose(long fd) {
@@ -463,10 +425,6 @@ privileged static size_t kformat(char *b, size_t n, const char *fmt,
   unsigned long long x;
   unsigned i, j, m, rem, sign, hash, cols, prec;
   char c, *p, *e, pdot, zero, flip, dang, base, quot, uppr, ansi, z[128];
-  if (kistextpointer(b) || kisdangerous(b))
-    n = 0;
-  if (!kistextpointer(fmt))
-    fmt = "!!WONTFMT";
   p = b;
   f = fmt;
   e = p + n;  // assume if n was negative e < p will be the case

libc/intrin/maps.c

@@ -0,0 +1,95 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2024 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/intrin/maps.h"
+#include "ape/sections.internal.h"
+#include "libc/dce.h"
+#include "libc/intrin/dll.h"
+#include "libc/intrin/kprintf.h"
+#include "libc/intrin/strace.internal.h"
+#include "libc/runtime/runtime.h"
+#include "libc/runtime/stack.h"
+#include "libc/sysv/consts/auxv.h"
+#include "libc/sysv/consts/prot.h"
+#include "libc/thread/thread.h"
+#include "libc/thread/tls.h"
+
+#ifdef __x86_64__
+__static_yoink("_init_maps");
+#endif
+
+struct Maps __maps;
+
+void __maps_init(void) {
+
+  // record _start() stack mapping
+  if (!IsWindows()) {
+    struct AddrSize stack;
+    stack = __get_main_stack();
+    dll_init(&__maps.stack.elem);
+    __maps.stack.addr = stack.addr;
+    __maps.stack.size = stack.size;
+    __maps.stack.prot = (uintptr_t)ape_stack_prot;
+    __maps_insert(&__maps.stack);
+  }
+
+  // record .text and .data mappings
+  static struct Map text, data;
+  dll_init(&text.elem);
+  text.addr = (char *)__executable_start;
+  text.size = _etext - __executable_start;
+  text.prot = PROT_READ | PROT_EXEC;
+  int pagesz = getauxval(AT_PAGESZ);
+  uintptr_t ds = ((uintptr_t)_etext + pagesz - 1) & -pagesz;
+  if (ds < (uintptr_t)_end) {
+    dll_init(&data.elem);
+    data.addr = (char *)ds;
+    data.size = (uintptr_t)_end - ds;
+    data.prot = PROT_READ | PROT_WRITE;
+    __maps_insert(&data);
+  }
+  __maps_insert(&text);
+}
+
+privileged void __maps_lock(void) {
+  struct CosmoTib *tib;
+  if (!__threaded)
+    return;
+  if (!__tls_enabled)
+    return;
+  tib = __get_tls_privileged();
+  if (tib->tib_flags & TIB_FLAG_MAPLOCK)
+    return;
+  while (atomic_exchange_explicit(&__maps.lock, 1, memory_order_acquire)) {
+#if defined(__GNUC__) && defined(__aarch64__)
+    __asm__ volatile("yield");
+#elif defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__))
+    __asm__ volatile("pause");
+#endif
+  }
+  tib->tib_flags |= TIB_FLAG_MAPLOCK;
+}
+
+privileged void __maps_unlock(void) {
+  struct CosmoTib *tib;
+  atomic_store_explicit(&__maps.lock, 0, memory_order_release);
+  if (__tls_enabled) {
+    tib = __get_tls_privileged();
+    tib->tib_flags &= ~TIB_FLAG_MAPLOCK;
+  }
+}

libc/intrin/maps.h

@@ -0,0 +1,49 @@
+#ifndef COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_
+#define COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_
+#include "libc/intrin/atomic.h"
+#include "libc/intrin/dll.h"
+#include "libc/thread/tls2.internal.h"
+COSMOPOLITAN_C_START_
+
+#define MAP_CONTAINER(e) DLL_CONTAINER(struct Map, elem, e)
+
+struct Map {
+  _Atomic(struct Map *) next; /* for __maps.maps */
+  char *addr;                 /* granule aligned */
+  size_t size;                /* must be nonzero */
+  struct Dll elem;            /* for __maps.free */
+  int64_t off;                /* -1 if anonymous */
+  int prot;                   /* memory protects */
+  int flags;                  /* memory map flag */
+  bool iscow;                 /* windows nt only */
+  bool readonlyfile;          /* windows nt only */
+  intptr_t h;                 /* windows nt only */
+};
+
+struct Maps {
+  atomic_int lock;
+  _Atomic(struct Map *) maps;
+  struct Dll *free;
+  struct Map stack;
+  struct Dll *used;
+};
+
+struct AddrSize {
+  char *addr;
+  size_t size;
+};
+
+extern struct Maps __maps;
+
+int maps_check(void);
+void __maps_init(void);
+void __maps_lock(void);
+void __maps_unlock(void);
+struct Map *__maps_alloc(void);
+void __maps_free(struct Map *);
+void __maps_insert(struct Map *);
+void *__mmap(char *, size_t, int, int, int, int64_t);
+struct AddrSize __get_main_stack(void);
+
+COSMOPOLITAN_C_END_
+#endif /* COSMOPOLITAN_LIBC_RUNTIME_MAPS_H_ */

libc/intrin/maps_init.S

@@ -1,7 +1,7 @@
 /*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8     -*-│
 │ vi: set noet ft=asm ts=8 sw=8 fenc=utf-8                                 :vi │
 ╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2021 Justine Alexandra Roberts Tunney                              │
+│ Copyright 2024 Justine Alexandra Roberts Tunney                              │
 │                                                                              │
 │ Permission to use, copy, modify, and/or distribute this software for         │
 │ any purpose with or without fee is hereby granted, provided that the         │
@@ -16,10 +16,12 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/thread/thread.h"
 #include "libc/macros.internal.h"
 
-	.init.start 200,_init__mmi
+	.init.start 301,_init_maps
-	movb	$16,_mmi+8
+	push	%rdi
-	movl	$_mmi+24,_mmi+16
+	push	%rsi
-	.init.end 200,_init__mmi
+	call	__maps_init
+	pop	%rsi
+	pop	%rdi
+	.init.end 301,_init_maps

libc/intrin/memtrack.greg.c

@@ -1,243 +0,0 @@
-/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
-│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
-╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2020 Justine Alexandra Roberts Tunney                              │
-│                                                                              │
-│ Permission to use, copy, modify, and/or distribute this software for         │
-│ any purpose with or without fee is hereby granted, provided that the         │
-│ above copyright notice and this permission notice appear in all copies.      │
-│                                                                              │
-│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
-│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
-│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
-│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
-│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
-│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
-│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
-│ PERFORMANCE OF THIS SOFTWARE.                                                │
-╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/assert.h"
-#include "libc/calls/calls.h"
-#include "libc/dce.h"
-#include "libc/errno.h"
-#include "libc/intrin/asan.internal.h"
-#include "libc/intrin/directmap.internal.h"
-#include "libc/intrin/strace.internal.h"
-#include "libc/log/libfatal.internal.h"
-#include "libc/log/log.h"
-#include "libc/macros.internal.h"
-#include "libc/mem/mem.h"
-#include "libc/runtime/memtrack.internal.h"
-#include "libc/runtime/runtime.h"
-#include "libc/str/str.h"
-#include "libc/sysv/consts/map.h"
-#include "libc/sysv/consts/prot.h"
-#include "libc/sysv/errfuns.h"
-
-static void *__shove_memory(struct MemoryInterval *d,
-                            const struct MemoryInterval *s, int n) {
-  int i;
-  unassert(n >= 0);
-  if (d > s) {
-    for (i = n; i--;) {
-      d[i] = s[i];
-    }
-  } else {
-    for (i = 0; i < n; ++i) {
-      d[i] = s[i];
-    }
-  }
-  return d;
-}
-
-static void __remove_memory(struct MemoryIntervals *mm, int i, int n) {
-  unassert(i >= 0);
-  unassert(i + n <= mm->i);
-  __shove_memory(mm->p + i, mm->p + i + n, mm->i - (i + n));
-  mm->i -= n;
-}
-
-static bool __extend_memory(struct MemoryIntervals *mm) {
-  int prot, flags;
-  char *base, *shad;
-  size_t gran, size;
-  struct DirectMap dm;
-  gran = kMemtrackGran;
-  base = (char *)kMemtrackStart;
-  prot = PROT_READ | PROT_WRITE;
-  flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED;
-  if (mm->p == mm->s) {
-    // TODO(jart): How can we detect ASAN mode under GREG?
-    if (1 || IsAsan()) {
-      shad = (char *)(((intptr_t)base >> 3) + 0x7fff8000);
-      dm = sys_mmap(shad, gran >> 3, prot, flags, -1, 0);
-      if (!dm.addr)
-        return false;
-    }
-    dm = sys_mmap(base, gran, prot, flags, -1, 0);
-    if (!dm.addr)
-      return false;
-    __shove_memory(dm.addr, mm->p, mm->i);
-    mm->p = dm.addr;
-    mm->n = gran / sizeof(*mm->p);
-  } else {
-    size = ROUNDUP(mm->n * sizeof(*mm->p), gran);
-    base += size;
-    if (IsAsan()) {
-      shad = (char *)(((intptr_t)base >> 3) + 0x7fff8000);
-      dm = sys_mmap(shad, gran >> 3, prot, flags, -1, 0);
-      if (!dm.addr)
-        return false;
-    }
-    dm = sys_mmap(base, gran, prot, flags, -1, 0);
-    if (!dm.addr)
-      return false;
-    mm->n = (size + gran) / sizeof(*mm->p);
-  }
-  return true;
-}
-
-static int __mint_memory(struct MemoryIntervals *mm, int i) {
-  unassert(i >= 0);
-  unassert(i <= mm->i);
-  unassert(mm->n >= 0);
-  if (mm->i == mm->n && !__extend_memory(mm))
-    return enomem();
-  __shove_memory(mm->p + i + 1, mm->p + i, mm->i++ - i);
-  return 0;
-}
-
-static int __punch_memory(struct MemoryIntervals *mm, int x, int y, int i) {
-  if (__mint_memory(mm, i) == -1)
-    return -1;
-  mm->p[i + 0].size -= (size_t)(mm->p[i + 0].y - (x - 1)) * FRAMESIZE;
-  mm->p[i + 0].y = x - 1;
-  mm->p[i + 1].size -= (size_t)((y + 1) - mm->p[i + 1].x) * FRAMESIZE;
-  mm->p[i + 1].x = y + 1;
-  return 0;
-}
-
-int __untrack_memory(struct MemoryIntervals *mm, int x, int y,
-                     void wf(struct MemoryIntervals *, int, int)) {
-  unsigned l, r;
-  unassert(y >= x);
-  if (!mm->i)
-    return 0;
-  // binary search for the lefthand side
-  l = __find_memory(mm, x);
-  if (l == mm->i)
-    return 0;
-  if (y < mm->p[l].x)
-    return 0;
-
-  // binary search for the righthand side
-  r = __find_memory(mm, y);
-  if (r == mm->i || (r > l && y < mm->p[r].x))
-    --r;
-  unassert(r >= l);
-  unassert(x <= mm->p[r].y);
-
-  // remove the middle of an existing map
-  //
-  // ----|mmmmmmmmmmmmmmmm|--------- before
-  //           xxxxx
-  // ----|mmmm|-----|mmmmm|--------- after
-  //
-  // this isn't possible on windows because we track each
-  // 64kb segment on that platform using a separate entry
-  if (l == r && x > mm->p[l].x && y < mm->p[l].y) {
-    return __punch_memory(mm, x, y, l);
-  }
-
-  // trim the right side of the lefthand map
-  //
-  // ----|mmmmmmm|-------------- before
-  //           xxxxx
-  // ----|mmmm|----------------- after
-  //
-  if (x > mm->p[l].x && x <= mm->p[l].y) {
-    unassert(y >= mm->p[l].y);
-    if (IsWindows())
-      return einval();
-    mm->p[l].size -= (size_t)(mm->p[l].y - (x - 1)) * FRAMESIZE;
-    mm->p[l].y = x - 1;
-    unassert(mm->p[l].x <= mm->p[l].y);
-    ++l;
-  }
-
-  // trim the left side of the righthand map
-  //
-  // ------------|mmmmm|-------- before
-  //           xxxxx
-  // ---------------|mm|-------- after
-  //
-  if (y >= mm->p[r].x && y < mm->p[r].y) {
-    unassert(x <= mm->p[r].x);
-    if (IsWindows())
-      return einval();
-    mm->p[r].size -= (size_t)((y + 1) - mm->p[r].x) * FRAMESIZE;
-    mm->p[r].x = y + 1;
-    unassert(mm->p[r].x <= mm->p[r].y);
-    --r;
-  }
-
-  if (l <= r) {
-    if (IsWindows() && wf) {
-      wf(mm, l, r);
-    }
-    __remove_memory(mm, l, r - l + 1);
-  }
-  return 0;
-}
-
-int __track_memory(struct MemoryIntervals *mm, int x, int y, long h, int prot,
-                   int flags, bool readonlyfile, bool iscow, long offset,
-                   long size) {
-  unsigned i;
-  unassert(y >= x);
-  i = __find_memory(mm, x);
-
-  // try to extend the righthand side of the lefthand entry
-  // we can't do that if we're tracking independent handles
-  // we can't do that if it's a file map with a small size!
-  if (i && x == mm->p[i - 1].y + 1 && h == mm->p[i - 1].h &&
-      prot == mm->p[i - 1].prot && flags == mm->p[i - 1].flags &&
-      mm->p[i - 1].size ==
-          (size_t)(mm->p[i - 1].y - mm->p[i - 1].x) * FRAMESIZE + FRAMESIZE) {
-    mm->p[i - 1].size += (size_t)(y - mm->p[i - 1].y) * FRAMESIZE;
-    mm->p[i - 1].y = y;
-    // if we filled the hole then merge the two mappings
-    if (i < mm->i && y + 1 == mm->p[i].x && h == mm->p[i].h &&
-        prot == mm->p[i].prot && flags == mm->p[i].flags) {
-      mm->p[i - 1].y = mm->p[i].y;
-      mm->p[i - 1].size += mm->p[i].size;
-      __remove_memory(mm, i, 1);
-    }
-  }
-
-  // try to extend the lefthand side of the righthand entry
-  // we can't do that if we're creating a smaller file map!
-  else if (i < mm->i && y + 1 == mm->p[i].x && h == mm->p[i].h &&
-           prot == mm->p[i].prot && flags == mm->p[i].flags &&
-           size == (size_t)(y - x) * FRAMESIZE + FRAMESIZE) {
-    mm->p[i].size += (size_t)(mm->p[i].x - x) * FRAMESIZE;
-    mm->p[i].x = x;
-  }
-
-  // otherwise, create a new entry and memmove the items
-  else {
-    if (__mint_memory(mm, i) == -1)
-      return -1;
-    mm->p[i].x = x;
-    mm->p[i].y = y;
-    mm->p[i].h = h;
-    mm->p[i].prot = prot;
-    mm->p[i].flags = flags;
-    mm->p[i].offset = offset;
-    mm->p[i].size = size;
-    mm->p[i].iscow = iscow;
-    mm->p[i].readonlyfile = readonlyfile;
-  }
-
-  return 0;
-}

libc/intrin/mmap.c

@@ -0,0 +1,431 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2024 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "ape/sections.internal.h"
+#include "libc/atomic.h"
+#include "libc/calls/calls.h"
+#include "libc/calls/internal.h"
+#include "libc/calls/struct/sigset.internal.h"
+#include "libc/calls/syscall-sysv.internal.h"
+#include "libc/dce.h"
+#include "libc/errno.h"
+#include "libc/intrin/asan.internal.h"
+#include "libc/intrin/atomic.h"
+#include "libc/intrin/describeflags.internal.h"
+#include "libc/intrin/directmap.internal.h"
+#include "libc/intrin/dll.h"
+#include "libc/intrin/maps.h"
+#include "libc/intrin/strace.internal.h"
+#include "libc/intrin/weaken.h"
+#include "libc/nt/memory.h"
+#include "libc/nt/runtime.h"
+#include "libc/runtime/runtime.h"
+#include "libc/runtime/stack.h"
+#include "libc/runtime/zipos.internal.h"
+#include "libc/stdio/sysparam.h"
+#include "libc/sysv/consts/auxv.h"
+#include "libc/sysv/consts/map.h"
+#include "libc/sysv/consts/o.h"
+#include "libc/sysv/consts/prot.h"
+#include "libc/sysv/errfuns.h"
+#include "libc/thread/thread.h"
+
+#define WINBASE 0x100080040000  // TODO: Can we support Windows Vista again?
+#define WINMAXX 0x200080000000
+
+#define MAP_FIXED_NOREPLACE_linux 0x100000
+
+#define PGUP(x) (((x) + granularity - 1) & -granularity)
+
+static atomic_ulong rollo;
+
+void __maps_free(struct Map *map) {
+  dll_make_last(&__maps.free, &map->elem);
+}
+
+void __maps_insert(struct Map *map) {
+  struct Map *last = __maps.maps;
+  if (last &&                                  //
+      map->addr == last->addr + last->size &&  //
+      map->flags == last->flags &&             //
+      map->prot == last->prot &&               //
+      map->off == last->off &&                 //
+      map->h == last->h &&                     //
+      map->off == -1) {
+    last->size += map->size;
+    dll_remove(&__maps.used, &last->elem);
+    dll_make_first(&__maps.used, &last->elem);
+    __maps_free(map);
+  } else {
+    dll_make_first(&__maps.used, &map->elem);
+    map->next = __maps.maps;
+    __maps.maps = map;
+  }
+}
+
+struct Map *__maps_alloc(void) {
+  struct Dll *e;
+  struct Map *map;
+  if ((e = dll_first(__maps.free))) {
+    dll_remove(&__maps.free, e);
+    map = MAP_CONTAINER(e);
+    map->next = 0;
+    return map;
+  }
+  int granularity = __granularity();
+  struct DirectMap sys = sys_mmap(0, granularity, PROT_READ | PROT_WRITE,
+                                  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  if (sys.addr == MAP_FAILED)
+    return 0;
+  if (IsWindows())
+    CloseHandle(sys.maphandle);
+  map = sys.addr;
+  map->addr = MAP_FAILED;
+  dll_init(&map->elem);
+  for (int i = 1; i < granularity / sizeof(struct Map); ++i) {
+    dll_init(&map[i].elem);
+    __maps_free(map + i);
+  }
+  map->next = 0;
+  return map;
+}
+
+static bool __overlaps_existing_map(const char *addr, size_t size) {
+  for (struct Map *map = __maps.maps; map; map = map->next) {
+    if (MAX(addr, map->addr) < MIN(addr + size, map->addr + map->size))
+      return true;
+  }
+  return false;
+}
+
+static int __munmap_chunk(void *addr, size_t size) {
+  int rc = sys_munmap(addr, size);
+  if (IsAsan() && !rc)
+    __asan_unshadow(addr, size);
+  return rc;
+}
+
+static int __munmap(char *addr, size_t size, bool untrack_only) {
+
+  // validate arguments
+  int granularity = __granularity();
+  if (((uintptr_t)addr & (granularity - 1)) ||  //
+      !size || (uintptr_t)addr + size < size)
+    return einval();
+
+  // untrack and delete mapping
+  int rc = 0;
+  __maps_lock();
+  // we can't call strace, kprintf, or nothing
+StartOver:;
+  struct Map *map = __maps.maps;
+  _Atomic(struct Map *) *prev = &__maps.maps;
+  while (map) {
+    char *map_addr = map->addr;
+    size_t map_size = map->size;
+    struct Map *next = map->next;
+    if (MAX(addr, map_addr) <
+        MIN(addr + PGUP(size), map_addr + PGUP(map_size))) {
+      if (addr <= map_addr && addr + PGUP(size) >= map_addr + PGUP(map_size)) {
+        // remove mapping completely
+        dll_remove(&__maps.used, &map->elem);
+        *prev = next;
+        map->size = 0;
+        map->addr = MAP_FAILED;
+        if (untrack_only) {
+          __maps_free(map);
+        } else {
+          if (!IsWindows()) {
+            if (__munmap_chunk(map_addr, map_size))
+              rc = -1;
+          } else {
+            if (!UnmapViewOfFile(map_addr))
+              rc = -1;
+            if (!CloseHandle(map->h))
+              rc = -1;
+          }
+          __maps_free(map);
+          goto StartOver;
+        }
+        map = next;
+        continue;
+      } else if (IsWindows()) {
+        // you can't carve up memory maps on windows. our mmap() makes
+        // this not a problem (for non-enormous memory maps) by making
+        // independent mappings for each 64 kb granule, under the hood
+        rc = einval();
+      } else if (addr <= map_addr) {
+        // shave off lefthand side of mapping
+        size_t left = addr + size - map_addr;
+        size_t right = map_addr + map_size - (addr + size);
+        map->addr += left;
+        map->size = right;
+        if (map->off != -1)
+          map->off += left;
+        if (!untrack_only) {
+          if (__munmap_chunk(map_addr, left) == -1)
+            rc = -1;
+          goto StartOver;
+        }
+      } else if (addr + PGUP(size) >= map_addr + PGUP(map_size)) {
+        // shave off righthand side of mapping
+        size_t left = addr - map_addr;
+        size_t right = map_addr + map_size - addr;
+        map->size = left;
+        if (!untrack_only) {
+          if (__munmap_chunk(addr, right) == -1)
+            rc = -1;
+          goto StartOver;
+        }
+      } else {
+        // punch hole in mapping
+        size_t left = addr - map_addr;
+        size_t middle = PGUP(size);
+        size_t right = map_size - middle - left;
+        struct Map *leftmap;
+        if ((leftmap = __maps_alloc())) {
+          leftmap->next = map;
+          leftmap->addr = map_addr;
+          leftmap->size = left;
+          leftmap->off = map->off;
+          leftmap->prot = map->prot;
+          leftmap->flags = map->flags;
+          map->addr += left + middle;
+          map->size = right;
+          if (map->off != -1)
+            map->off += left + middle;
+          dll_make_first(&__maps.used, &leftmap->elem);
+          *prev = leftmap;
+          if (!untrack_only) {
+            if (__munmap_chunk(addr, size) == -1)
+              rc = -1;
+            goto StartOver;
+          }
+        } else {
+          rc = -1;
+        }
+      }
+    }
+    prev = &map->next;
+    map = next;
+  }
+  __maps_unlock();
+
+  return rc;
+}
+
+static void *__mmap_chunk(void *addr, size_t size, int prot, int flags, int fd,
+                          int64_t off, int granularity) {
+
+  // polyfill nuances of fixed mappings
+  int sysflags = flags;
+  bool noreplace = false;
+  bool should_untrack = false;
+  if (flags & MAP_FIXED_NOREPLACE) {
+    if (flags & MAP_FIXED)
+      return (void *)einval();
+    sysflags &= ~MAP_FIXED_NOREPLACE;
+    if (IsLinux()) {
+      noreplace = true;
+      sysflags |= MAP_FIXED_NOREPLACE_linux;
+    } else if (IsFreebsd() || IsNetbsd()) {
+      sysflags |= MAP_FIXED;
+      if (__overlaps_existing_map(addr, size))
+        return (void *)eexist();
+    } else {
+      noreplace = true;
+    }
+  } else if (flags & MAP_FIXED) {
+    should_untrack = true;
+  }
+
+  // allocate Map object
+  struct Map *map;
+  __maps_lock();
+  map = __maps_alloc();
+  __maps_unlock();
+  if (!map)
+    return MAP_FAILED;
+
+  // obtain mapping from operating system
+  int olderr = errno;
+  struct DirectMap res;
+TryAgain:
+  res = sys_mmap(addr, size, prot, sysflags, fd, off);
+  if (res.addr == MAP_FAILED) {
+    if (IsWindows() && errno == EADDRNOTAVAIL) {
+      if (noreplace) {
+        errno = EEXIST;
+      } else if (should_untrack) {
+        sys_munmap(res.addr, size);
+        errno = olderr;
+        goto TryAgain;
+      } else {
+        addr += granularity;
+        errno = olderr;
+        goto TryAgain;
+      }
+    }
+    __maps_lock();
+    __maps_free(map);
+    __maps_unlock();
+    return MAP_FAILED;
+  }
+
+  // polyfill map fixed noreplace
+  // we assume non-linux gives us addr if it's free
+  // that's what linux (e.g. rhel7) did before noreplace
+  if (noreplace && res.addr != addr) {
+    sys_munmap(res.addr, size);
+    __maps_lock();
+    __maps_free(map);
+    __maps_unlock();
+    return (void *)eexist();
+  }
+
+  // untrack mapping we blew away
+  if (should_untrack)
+    __munmap(addr, size, true);
+
+  // track Map object
+  map->addr = res.addr;
+  map->size = size;
+  map->off = off;
+  map->prot = prot;
+  map->flags = flags;
+  map->h = res.maphandle;
+  if (IsWindows()) {
+    map->iscow = (flags & MAP_TYPE) != MAP_SHARED && fd != -1;
+    map->readonlyfile = (flags & MAP_TYPE) == MAP_SHARED && fd != -1 &&
+                        (g_fds.p[fd].flags & O_ACCMODE) == O_RDONLY;
+  }
+  __maps_lock();
+  __maps_insert(map);
+  __maps_unlock();
+
+  return res.addr;
+}
+
+static void *__mmap_impl(char *addr, size_t size, int prot, int flags, int fd,
+                         int64_t off, int granularity) {
+
+  // validate file map args
+  if (fd != -1) {
+    if (off & (granularity - 1))
+      return (void *)einval();
+    if (IsWindows()) {
+      if (!__isfdkind(fd, kFdFile))
+        return (void *)eacces();
+      if ((g_fds.p[fd].flags & O_ACCMODE) == O_WRONLY)
+        return (void *)eacces();
+    }
+  }
+
+  // mmap works fine on unix
+  if (!IsWindows())
+    return __mmap_chunk(addr, size, prot, flags, fd, off, granularity);
+
+  // if the concept of granularity wasn't exciting enough
+  if (!addr && !(flags & (MAP_FIXED | MAP_FIXED_NOREPLACE)))
+    addr = (char *)(WINBASE + atomic_fetch_add(&rollo, PGUP(size)) % WINMAXX);
+
+  // windows forbids unmapping a subset of a map once it's made
+  if (size <= granularity || size > 100 * 1024 * 1024)
+    return __mmap_chunk(addr, size, prot, flags, fd, off, granularity);
+
+  // so we create an separate map for each granule in the mapping
+  if (!(flags & MAP_FIXED)) {
+    while (__overlaps_existing_map(addr, size)) {
+      if (flags & MAP_FIXED_NOREPLACE)
+        return (void *)eexist();
+      addr += granularity;
+    }
+  }
+  char *res = addr;
+  while (size) {
+    char *got;
+    size_t amt = MIN(size, granularity);
+    got = __mmap_chunk(addr, amt, prot, flags, fd, off, granularity);
+    if (got != addr) {
+      if (got != MAP_FAILED)
+        __munmap(got, amt, false);
+      if (addr > res)
+        __munmap(res, addr - res, false);
+      errno = EAGAIN;
+      return MAP_FAILED;
+    }
+    size -= amt;
+    addr += amt;
+    off += amt;
+  }
+  return res;
+}
+
+void *__mmap(char *addr, size_t size, int prot, int flags, int fd,
+             int64_t off) {
+  char *res;
+  int granularity = __granularity();
+
+  // validate arguments
+  if (((uintptr_t)addr & (granularity - 1)) ||  //
+      !size || (uintptr_t)addr + size < size)
+    return (void *)einval();
+  if (size > 0x100000000000)
+    return (void *)enomem();
+
+  // normalize arguments
+  if (flags & MAP_ANONYMOUS) {
+    fd = -1;
+    off = 0;
+    size = PGUP(size);
+  }
+
+  // create memory mappping
+  if (!__isfdkind(fd, kFdZip)) {
+    res = __mmap_impl(addr, size, prot, flags, fd, off, granularity);
+  } else {
+    res = _weaken(__zipos_mmap)(
+        addr, size, prot, flags,
+        (struct ZiposHandle *)(uintptr_t)g_fds.p[fd].handle, off);
+  }
+
+  return res;
+}
+
+void *mmap(void *addr, size_t size, int prot, int flags, int fd, int64_t off) {
+  void *res;
+  res = __mmap(addr, size, prot, flags, fd, off);
+  if (IsAsan() && res != MAP_FAILED) {
+    __asan_shadow(res, size);
+    int granularity = __granularity();
+    if (size != PGUP(size))
+      __asan_poison(res + size, PGUP(size) - size, kAsanMmapSizeOverrun);
+  }
+  STRACE("mmap(%p, %'zu, %s, %s, %d, %'ld) → %p% m", addr, size,
+         DescribeProtFlags(prot), DescribeMapFlags(flags), fd, off, res);
+  return res;
+}
+
+int munmap(void *addr, size_t size) {
+  int rc;
+  rc = __munmap(addr, size, false);
+  STRACE("munmap(%p, %'zu) → %d% m", addr, size, rc);
+  return rc;
+}
+
+__weak_reference(mmap, mmap64);

libc/intrin/mmi.c

@@ -1,28 +0,0 @@
-/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
-│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
-╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2020 Justine Alexandra Roberts Tunney                              │
-│                                                                              │
-│ Permission to use, copy, modify, and/or distribute this software for         │
-│ any purpose with or without fee is hereby granted, provided that the         │
-│ above copyright notice and this permission notice appear in all copies.      │
-│                                                                              │
-│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
-│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
-│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
-│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
-│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
-│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
-│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
-│ PERFORMANCE OF THIS SOFTWARE.                                                │
-╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/macros.internal.h"
-#include "libc/runtime/memtrack.internal.h"
-#include "libc/thread/thread.h"
-
-#ifdef __x86_64__
-__static_yoink("_init__mmi");
-#endif
-
-struct MemoryIntervals _mmi;
-pthread_mutex_t __mmi_lock_obj = {._type = PTHREAD_MUTEX_RECURSIVE};

libc/intrin/mprotect.c

@@ -0,0 +1,215 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2020 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/struct/sigset.internal.h"
+#include "libc/calls/syscall-sysv.internal.h"
+#include "libc/dce.h"
+#include "libc/intrin/describeflags.internal.h"
+#include "libc/intrin/directmap.internal.h"
+#include "libc/intrin/dll.h"
+#include "libc/intrin/kprintf.h"
+#include "libc/intrin/maps.h"
+#include "libc/intrin/strace.internal.h"
+#include "libc/nt/memory.h"
+#include "libc/runtime/internal.h"
+#include "libc/runtime/runtime.h"
+#include "libc/stdio/sysparam.h"
+#include "libc/sysv/consts/auxv.h"
+#include "libc/sysv/consts/prot.h"
+#include "libc/sysv/errfuns.h"
+
+#define PGUP(x) (((x) + pagesz - 1) & -pagesz)
+
+static int __mprotect_chunk(char *addr, size_t size, int prot, bool iscow) {
+
+  if (!IsWindows())
+    return sys_mprotect(addr, size, prot);
+
+  uint32_t op;
+  if (!VirtualProtect(addr, size, __prot2nt(prot, iscow), &op))
+    return -1;
+
+  return 0;
+}
+
+int __mprotect(char *addr, size_t size, int prot) {
+
+  // unix checks prot before checking size
+  if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC))
+    return einval();
+
+  // make new technology consistent with unix
+  if (!size)
+    return 0;
+
+  // unix checks prot before checking size
+  int pagesz = getauxval(AT_PAGESZ);
+  if ((intptr_t)addr & (pagesz - 1))
+    return einval();
+
+  // change mappings
+  int rc = 0;
+  __maps_lock();
+  bool found = false;
+  struct Map *map = __maps.maps;
+  _Atomic(struct Map *) *prev = &__maps.maps;
+  while (map) {
+    char *map_addr = map->addr;
+    size_t map_size = map->size;
+    struct Map *next = map->next;
+    char *beg = MAX(addr, map_addr);
+    char *end = MIN(addr + PGUP(size), map_addr + PGUP(map_size));
+    if (beg < end) {
+      found = true;
+      if (addr <= map_addr && addr + PGUP(size) >= map_addr + PGUP(map_size)) {
+        // change protection of entire mapping
+        if (!__mprotect_chunk(map_addr, map_size, prot, map->iscow)) {
+          map->prot = prot;
+        } else {
+          rc = -1;
+        }
+      } else if (IsWindows()) {
+        // windows does allow changing protection at 4096 byte chunks
+        // however we currently don't have data structures that track
+        // this within the 64 kb map granules that can't be broken up
+        if (__mprotect_chunk(beg, end - beg, prot, map->iscow) == -1)
+          rc = -1;
+      } else if (addr <= map_addr) {
+        // cleave lefthand side of mapping
+        size_t left = addr + size - map_addr;
+        size_t right = map_addr + map_size - (addr + size);
+        struct Map *leftmap;
+        if ((leftmap = __maps_alloc())) {
+          if (!__mprotect_chunk(map_addr, left, prot, false)) {
+            leftmap->next = map;
+            leftmap->addr = map_addr;
+            leftmap->size = left;
+            leftmap->prot = prot;
+            leftmap->off = map->off;
+            leftmap->flags = map->flags;
+            map->addr += left;
+            map->size = right;
+            if (map->off != -1)
+              map->off += left;
+            dll_make_first(&__maps.used, &leftmap->elem);
+            *prev = leftmap;
+          } else {
+            __maps_free(leftmap);
+            rc = -1;
+          }
+        } else {
+          rc = -1;
+        }
+      } else if (addr + PGUP(size) >= map_addr + PGUP(map_size)) {
+        // cleave righthand side of mapping
+        size_t left = addr - map_addr;
+        size_t right = map_addr + map_size - addr;
+        struct Map *leftmap;
+        if ((leftmap = __maps_alloc())) {
+          if (!__mprotect_chunk(map_addr + left, right, prot, false)) {
+            leftmap->next = map;
+            leftmap->addr = map_addr;
+            leftmap->size = left;
+            leftmap->off = map->off;
+            leftmap->prot = map->prot;
+            leftmap->flags = map->flags;
+            map->addr += left;
+            map->size = right;
+            map->prot = prot;
+            if (map->off != -1)
+              map->off += left;
+            dll_make_first(&__maps.used, &leftmap->elem);
+            *prev = leftmap;
+          } else {
+            __maps_free(leftmap);
+            rc = -1;
+          }
+        } else {
+          rc = -1;
+        }
+      } else {
+        // punch hole in mapping
+        size_t left = addr - map_addr;
+        size_t middle = PGUP(size);
+        size_t right = map_size - middle - left;
+        struct Map *leftmap;
+        if ((leftmap = __maps_alloc())) {
+          struct Map *midlmap;
+          if ((midlmap = __maps_alloc())) {
+            if (!__mprotect_chunk(map_addr + left, middle, prot, false)) {
+              leftmap->next = midlmap;
+              leftmap->addr = map_addr;
+              leftmap->size = left;
+              leftmap->off = map->off;
+              leftmap->prot = map->prot;
+              leftmap->flags = map->flags;
+              midlmap->next = map;
+              midlmap->addr = map_addr + left;
+              midlmap->size = middle;
+              midlmap->off = map->off == -1 ? -1 : map->off + left;
+              midlmap->prot = prot;
+              midlmap->flags = map->flags;
+              map->addr += left + middle;
+              map->size = right;
+              if (map->off != -1)
+                map->off += left + middle;
+              dll_make_first(&__maps.used, &leftmap->elem);
+              dll_make_first(&__maps.used, &midlmap->elem);
+              *prev = leftmap;
+            } else {
+              __maps_free(midlmap);
+              __maps_free(leftmap);
+              rc = -1;
+            }
+          } else {
+            __maps_free(leftmap);
+            rc = -1;
+          }
+        } else {
+          rc = -1;
+        }
+      }
+    }
+    prev = &map->next;
+    map = next;
+  }
+
+  // allow user to change mappings unknown to cosmo runtime
+  if (!found)
+    rc = __mprotect_chunk(addr, size, prot, false);
+
+  __maps_unlock();
+  return rc;
+}
+
+/**
+ * Modifies restrictions on virtual memory address range.
+ *
+ * @param addr needs to be 4kb aligned
+ * @param prot can have PROT_{NONE,READ,WRITE,EXEC}
+ * @return 0 on success, or -1 w/ errno
+ * @raise ENOMEM on tracking memory oom
+ * @see mmap()
+ */
+int mprotect(void *addr, size_t size, int prot) {
+  int rc;
+  rc = __mprotect(addr, size, prot);
+  STRACE("mprotect(%p, %'zu, %s) → %d% m", addr, size, DescribeProtFlags(prot),
+         rc);
+  return rc;
+}

libc/intrin/msync-nt.c

@@ -16,24 +16,32 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/assert.h"
+#include "libc/calls/syscall-nt.internal.h"
-#include "libc/intrin/strace.internal.h"
+#include "libc/intrin/maps.h"
 #include "libc/nt/memory.h"
 #include "libc/nt/runtime.h"
-#include "libc/runtime/memtrack.internal.h"
 #include "libc/runtime/runtime.h"
+#include "libc/stdio/sysparam.h"
+#include "libc/sysv/consts/auxv.h"
+#include "libc/sysv/errfuns.h"
 
-static inline void *GetFrameAddr(int f) {
+textwindows int sys_msync_nt(char *addr, size_t size, int flags) {
-  intptr_t a;
-  a = f;
-  a *= FRAMESIZE;
-  return (void *)a;
-}
 
-void __release_memory_nt(struct MemoryIntervals *mm, int l, int r) {
+  int pagesz = getauxval(AT_PAGESZ);
-  int i;
+  size = (size + pagesz - 1) & -pagesz;
-  for (i = l; i <= r; ++i) {
+
-    UnmapViewOfFile(GetFrameAddr(mm->p[i].x));
+  if ((uintptr_t)addr & (pagesz - 1))
-    CloseHandle(mm->p[i].h);
+    return einval();
+
+  int rc = 0;
+  for (struct Map *map = __maps.maps; map; map = map->next) {
+    char *beg = MAX(addr, map->addr);
+    char *end = MIN(addr + size, map->addr + map->size);
+    if (beg < end)
+      if (!FlushViewOfFile(beg, end - beg))
+        rc = -1;
+    // TODO(jart): FlushFileBuffers too on g_fds handle if MS_SYNC?
   }
+
+  return rc;
 }

libc/intrin/munmap-sysv.c

@@ -21,6 +21,7 @@
 #include "libc/intrin/describeflags.internal.h"
 #include "libc/intrin/directmap.internal.h"
 #include "libc/intrin/strace.internal.h"
+#include "libc/runtime/runtime.h"
 #include "libc/runtime/syslib.internal.h"
 
 /**
@@ -40,7 +41,8 @@ int sys_munmap(void *p, size_t n) {
   } else {
     rc = __sys_munmap(p, n);
   }
-  KERNTRACE("sys_munmap(%p /* %s */, %'zu) → %d", p,
+  if (!rc)
-            DescribeFrame((intptr_t)p >> 16), n, rc);
+    __virtualsize -= n;
+  KERNTRACE("sys_munmap(%p, %'zu) → %d", p, n, rc);
   return rc;
 }

libc/intrin/nocolor.c

@@ -16,39 +16,19 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/dce.h"
 #include "libc/log/internal.h"
 #include "libc/runtime/runtime.h"
 
-#define IsDumb(s) \
-  (s[0] == 'd' && s[1] == 'u' && s[2] == 'm' && s[3] == 'b' && !s[4])
-
-/**
- * Indicates if ANSI terminal colors are inappropriate.
- *
- * Normally this variable should be false. We only set it to true if
- * we're running on an old version of Windows or the environment
- * variable `TERM` is set to `dumb`.
- *
- * We think colors should be the norm, since most software is usually
- * too conservative about removing them. Rather than using `isatty`
- * consider using sed for instances where color must be removed:
- *
- *      sed 's/\x1b\[[;[:digit:]]*m//g' <color.txt >uncolor.txt
- *
- * For some reason, important software is configured by default in many
- * operating systems, to not only disable colors, but utf-8 too! Here's
- * an example of how a wrapper script can fix that for `less`.
- *
- *      #!/bin/sh
- *      LESSCHARSET=UTF-8 exec /usr/bin/less -RS "$@"
- *
- * Thank you for using colors!
- */
 bool __nocolor;
 
 __attribute__((__constructor__(20))) optimizesize textstartup void
 __nocolor_init(int argc, char **argv, char **envp, intptr_t *auxv) {
   char *s;
-  __nocolor = IsWindows() || ((s = getenv("TERM")) && IsDumb(s));
+  if ((s = getenv("TERM")))
+    if (s[0] == 'd' &&  //
+        s[1] == 'u' &&  //
+        s[2] == 'm' &&  //
+        s[3] == 'b' &&  //
+        s[4] == '\0')
+      __nocolor = true;
 }

libc/intrin/printmaps.c

@@ -16,22 +16,40 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/fmt/conv.h"
+#include "libc/fmt/itoa.h"
+#include "libc/intrin/describeflags.internal.h"
+#include "libc/intrin/kprintf.h"
+#include "libc/intrin/maps.h"
 #include "libc/macros.internal.h"
 #include "libc/runtime/memtrack.internal.h"
 #include "libc/runtime/runtime.h"
 
-// TODO(jart): DELETE
-
 /**
- * Returns true if address isn't stack and was malloc'd or mmap'd.
+ * Prints memory mappings.
- *
- * @assume stack addresses are always greater than heap addresses
- * @assume stack memory isn't stored beneath %rsp (-mno-red-zone)
- * @deprecated
  */
-optimizesize bool32 _isheap(const void *p) {
+void __print_maps(void) {
-  intptr_t x, y;
+  int limit = 10;
-  x = kAutomapStart;
+  long maptally = 0;
-  y = x + kAutomapSize;
+  char mappingbuf[8], sb[16];
-  return x <= (intptr_t)p && (intptr_t)p < y;
+  for (struct Map *map = __maps.maps; map; map = map->next) {
+    maptally += map->size;
+    kprintf("%012lx-%012lx %!s", map->addr, map->addr + map->size,
+            (DescribeMapping)(mappingbuf, map->prot, map->flags));
+    sizefmt(sb, map->size, 1024);
+    kprintf(" %!sb", sb);
+    if (map->h && map->h != -1)
+      kprintf(" h=%ld", map->h);
+    if (map->iscow)
+      kprintf(" cow");
+    if (map->readonlyfile)
+      kprintf(" readonlyfile");
+    kprintf("\n");
+    if (!--limit) {
+      kprintf("...\n");
+      break;
+    }
+  }
+  sizefmt(sb, maptally, 1024);
+  kprintf("# %!sb mapped memory\n", sb);
 }

libc/intrin/printmemoryintervals.c

@@ -1,66 +0,0 @@
-/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
-│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
-╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2020 Justine Alexandra Roberts Tunney                              │
-│                                                                              │
-│ Permission to use, copy, modify, and/or distribute this software for         │
-│ any purpose with or without fee is hereby granted, provided that the         │
-│ above copyright notice and this permission notice appear in all copies.      │
-│                                                                              │
-│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
-│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
-│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
-│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
-│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
-│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
-│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
-│ PERFORMANCE OF THIS SOFTWARE.                                                │
-╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/fmt/conv.h"
-#include "libc/fmt/itoa.h"
-#include "libc/intrin/describeflags.internal.h"
-#include "libc/intrin/kprintf.h"
-#include "libc/macros.internal.h"
-#include "libc/runtime/memtrack.internal.h"
-
-static bool IsNoteworthyHole(unsigned i, const struct MemoryIntervals *mm) {
-  // gaps between shadow frames aren't interesting
-  // the chasm from heap to stack ruins statistics
-  return !(
-      (IsShadowFrame(mm->p[i].y) || IsShadowFrame(mm->p[i + 1].x)) ||
-      (!IsStaticStackFrame(mm->p[i].y) && IsStaticStackFrame(mm->p[i + 1].x)));
-}
-
-void PrintMemoryIntervals(int fd, const struct MemoryIntervals *mm) {
-  long i, w, frames, maptally = 0;
-  char mappingbuf[8], framebuf[64], sb[16];
-  for (w = i = 0; i < mm->i; ++i) {
-    w = MAX(w, LengthInt64Thousands(mm->p[i].y + 1 - mm->p[i].x));
-  }
-  for (i = 0; i < mm->i; ++i) {
-    frames = mm->p[i].y + 1 - mm->p[i].x;
-    maptally += frames;
-    kprintf("%08x-%08x %s %'*ldx %s", mm->p[i].x, mm->p[i].y,
-            (DescribeMapping)(mappingbuf, mm->p[i].prot, mm->p[i].flags), w,
-            frames, (DescribeFrame)(framebuf, mm->p[i].x));
-    if (mm->p[i].iscow)
-      kprintf(" cow");
-    if (mm->p[i].readonlyfile)
-      kprintf(" readonlyfile");
-    sizefmt(sb, mm->p[i].size, 1024);
-    kprintf(" %sB", sb);
-    if (i + 1 < mm->i) {
-      frames = mm->p[i + 1].x - mm->p[i].y - 1;
-      if (frames && IsNoteworthyHole(i, mm)) {
-        sizefmt(sb, frames * FRAMESIZE, 1024);
-        kprintf(" w/ %sB hole", sb);
-      }
-    }
-    if (mm->p[i].h != -1) {
-      kprintf(" h=%ld", mm->p[i].h);
-    }
-    kprintf("\n");
-  }
-  sizefmt(sb, maptally * FRAMESIZE, 1024);
-  kprintf("# %sB total mapped memory\n", sb);
-}

libc/intrin/printsystemmappings.greg.c

@@ -1,38 +0,0 @@
-/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
-│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
-╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2021 Justine Alexandra Roberts Tunney                              │
-│                                                                              │
-│ Permission to use, copy, modify, and/or distribute this software for         │
-│ any purpose with or without fee is hereby granted, provided that the         │
-│ above copyright notice and this permission notice appear in all copies.      │
-│                                                                              │
-│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
-│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
-│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
-│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
-│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
-│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
-│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
-│ PERFORMANCE OF THIS SOFTWARE.                                                │
-╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/calls/syscall-sysv.internal.h"
-#include "libc/dce.h"
-#include "libc/runtime/memtrack.internal.h"
-#include "libc/sysv/consts/at.h"
-#include "libc/sysv/consts/o.h"
-
-void PrintSystemMappings(int outfd) {
-  int infd;
-  ssize_t rc;
-  char buf[64];
-  if (!IsWindows()) {
-    if ((infd = __sys_openat(AT_FDCWD, "/proc/self/maps", O_RDONLY, 0)) >= 0) {
-      sys_write(outfd, "\n", 1);
-      while ((rc = sys_read(infd, buf, sizeof(buf))) > 0) {
-        sys_write(outfd, buf, rc);
-      }
-    }
-    sys_close(infd);
-  }
-}

libc/intrin/pthread_tid.c

@@ -16,11 +16,14 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/dce.h"
 #include "libc/intrin/atomic.h"
 #include "libc/thread/posixthread.internal.h"
 #include "libc/thread/thread.h"
 
 int _pthread_tid(struct PosixThread *pt) {
+  if (IsWindows())  // xxx: fixme
+    return pt->ptid;
   int tid = 0;
   while (pt && !(tid = atomic_load_explicit(&pt->ptid, memory_order_acquire))) {
     pthread_pause_np();

libc/intrin/virtualmax.c

@@ -17,8 +17,6 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/runtime/runtime.h"
-#ifndef __x86_64__
 
-size_t __virtualmax;
+size_t __virtualmax = -1;
-
+size_t __virtualsize = 0;
-#endif /* __x86_64__ */

libc/log/backtrace.internal.h

@@ -1,19 +1,9 @@
 #ifndef COSMOPOLITAN_LIBC_LOG_BACKTRACE_H_
 #define COSMOPOLITAN_LIBC_LOG_BACKTRACE_H_
 #include "libc/nexgen32e/stackframe.h"
-#include "libc/runtime/memtrack.internal.h"
 #include "libc/runtime/symbols.internal.h"
 COSMOPOLITAN_C_START_
 
-forceinline pureconst bool IsValidStackFramePointer(struct StackFrame *x) {
-  /* assumes __mmi_lock() is held */
-  return IsLegalPointer(x) && !((uintptr_t)x & 15) &&
-         (IsStaticStackFrame((uintptr_t)x >> 16) ||
-          IsOldStackFrame((uintptr_t)x >> 16) ||
-          /* lua coroutines need this */
-          IsMemtracked((uintptr_t)x >> 16, (uintptr_t)x >> 16));
-}
-
 void ShowBacktrace(int, const struct StackFrame *);
 int PrintBacktraceUsingSymbols(int, const struct StackFrame *,
                                struct SymbolTable *);

libc/log/leaks.c

@@ -126,7 +126,6 @@ dontasan void CheckForMemoryLeaks(void) {
     malloc_inspect_all(OnMemory, 0);
     kprintf("\n");
     /* __print_maps(); */
-    /* PrintSystemMappings(2); */
     /* PrintGarbage(); */
     _Exit(78);
   }

libc/log/oncrash_amd64.c

@@ -245,7 +245,6 @@ static relegated void ShowCrashReport(int err, int sig, siginfo_t *si,
   if (!IsWindows()) {
     __print_maps();
   }
-  /* PrintSystemMappings(2); */
   if (__argv) {
     for (i = 0; i < __argc; ++i) {
       kprintf("%s ", __argv[i]);
@@ -269,6 +268,8 @@ static inline void SpinUnlock(atomic_uint *lock) {
 
 relegated void __oncrash(int sig, siginfo_t *si, void *arg) {
   static atomic_uint lock;
+  ftrace_enabled(-1);
+  strace_enabled(-1);
   BLOCK_CANCELATION;
   SpinLock(&lock);
   int err = errno;
@@ -291,6 +292,8 @@ relegated void __oncrash(int sig, siginfo_t *si, void *arg) {
 
   SpinUnlock(&lock);
   ALLOW_CANCELATION;
+  strace_enabled(+1);
+  ftrace_enabled(+1);
 }
 
 #endif /* __x86_64__ */

libc/log/oncrash_arm64.c

@@ -33,6 +33,7 @@
 #include "libc/calls/ucontext.h"
 #include "libc/cosmo.h"
 #include "libc/cxxabi.h"
+#include "libc/dce.h"
 #include "libc/errno.h"
 #include "libc/intrin/atomic.h"
 #include "libc/intrin/describebacktrace.internal.h"
@@ -211,8 +212,6 @@ static relegated void __oncrash_impl(int sig, siginfo_t *si, ucontext_t *ctx) {
   struct utsname names = {0};
   struct Buffer b[1] = {{buf, size}};
   b->p[b->i++] = '\n';
-  ftrace_enabled(-1);
-  strace_enabled(-1);
   __restore_tty();
   uname(&names);
   gethostname(host, sizeof(host));
@@ -390,6 +389,8 @@ static inline void SpinUnlock(atomic_uint *lock) {
 
 relegated void __oncrash(int sig, siginfo_t *si, void *arg) {
   static atomic_uint lock;
+  ftrace_enabled(-1);
+  strace_enabled(-1);
   BLOCK_CANCELATION;
   SpinLock(&lock);
   __oncrash_impl(sig, si, arg);
@@ -416,6 +417,8 @@ relegated void __oncrash(int sig, siginfo_t *si, void *arg) {
 
   SpinUnlock(&lock);
   ALLOW_CANCELATION;
+  strace_enabled(+1);
+  ftrace_enabled(+1);
 }
 
 #endif /* __aarch64__ */

libc/mem/BUILD.mk

@@ -54,8 +54,7 @@ o/$(MODE)/libc/mem/asan.o: private			\
 			-fno-sanitize=all		\
 			-fno-stack-protector		\
 			-Wframe-larger-than=4096	\
-			-Walloca-larger-than=4096	\
+			-Walloca-larger-than=4096
-			-fpatchable-function-entry=0,0
 
 # make asan stack traces shorter
 o/$(MODE)/libc/mem/asanthunk.o: private			\

libc/mem/asan.c

@@ -123,21 +123,13 @@ static void *__asan_allocate_heap(size_t a, size_t n, struct AsanTrace *bt) {
 }
 
 static struct AsanExtra *__asan_get_extra(const void *p, size_t *c) {
-  int f;
+  long n;
-  long x, n;
+  if (kisdangerous(p))
-  struct AsanExtra *e;
-  f = (intptr_t)p >> 16;
-  if (!kisdangerous(p) && (n = dlmalloc_usable_size((void *)p)) > sizeof(*e) &&
-      !ckd_add(&x, (intptr_t)p, n) && x <= 0x800000000000 &&
-      (LIKELY(f == (int)((x - 1) >> 16)) || !kisdangerous((void *)(x - 1))) &&
-      (LIKELY(f == (int)((x = x - sizeof(*e)) >> 16)) ||
-       __asan_is_mapped(x >> 16)) &&
-      !(x & (_Alignof(struct AsanExtra) - 1))) {
-    *c = n;
-    return (struct AsanExtra *)x;
-  } else {
     return 0;
-  }
+  if ((n = dlmalloc_usable_size((void *)p)) < sizeof(struct AsanExtra))
+    return 0;
+  *c = n;
+  return (struct AsanExtra *)(p + n - sizeof(struct AsanExtra));
 }
 
 // Returns true if `p` was allocated by an IGNORE_LEAKS(function).
@@ -153,7 +145,7 @@ int __asan_is_leaky(void *p) {
     return 0;
   if (!__asan_read48(e->size, &n))
     return 0;
-  if (!__asan_is_mapped((((intptr_t)p >> 3) + 0x7fff8000) >> 16))
+  if (!__asan_is_mapped((char *)(((intptr_t)p >> 3) + 0x7fff8000)))
     return 0;
   if (!(st = GetSymbolTable()))
     return 0;
@@ -182,7 +174,7 @@ int __asan_print_trace(void *p) {
     return -1;
   }
   kprintf("\n%p %,lu bytes [asan]", (char *)p, n);
-  if (!__asan_is_mapped((((intptr_t)p >> 3) + 0x7fff8000) >> 16))
+  if (!__asan_is_mapped((char *)(((intptr_t)p >> 3) + 0x7fff8000)))
     kprintf(" (shadow not mapped?!)");
   for (i = 0; i < ARRAYLEN(e->bt.p) && e->bt.p[i]; ++i)
     kprintf("\n%*lx %t", 12, e->bt.p[i], e->bt.p[i]);

libc/mem/pvalloc.c

@@ -18,6 +18,7 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/errno.h"
 #include "libc/mem/mem.h"
+#include "libc/runtime/runtime.h"
 #include "libc/stdckdint.h"
 
 /**
@@ -31,9 +32,9 @@
  * @see valloc()
  */
 void *pvalloc(size_t n) {
-  if (ckd_add(&n, n, FRAMESIZE - 1)) {
+  if (ckd_add(&n, n, __granularity() - 1)) {
     errno = ENOMEM;
     return 0;
   }
-  return memalign(FRAMESIZE, n & -FRAMESIZE);
+  return memalign(__granularity(), n & -__granularity());
 }

libc/mem/tinymalloc.inc

@@ -14,6 +14,7 @@
 // PERFORMANCE OF THIS SOFTWARE.
 
 #include "libc/assert.h"
+#include "libc/dce.h"
 #include "libc/errno.h"
 #include "libc/mem/mem.h"
 #include "libc/stdalign.internal.h"
@@ -28,7 +29,7 @@
 #define TINYMALLOC_MAX_ALIGN 4096
 #endif
 
-#ifndef MODE_DBG /* don't interfere with asan malloc */
+#if !IsAsan()
 
 alignas(TINYMALLOC_MAX_ALIGN) static struct {
   char memory[TINYMALLOC_MAX_BYTES];
@@ -167,4 +168,4 @@ OutOfMemory:
   return 0;
 }
 
-#endif /* MODE_DBG */
+#endif /* IsAsan() */

libc/mem/valloc.c

@@ -17,6 +17,7 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/mem/mem.h"
+#include "libc/runtime/runtime.h"
 
 /**
  * Allocates granular aligned memory, i.e.
@@ -28,5 +29,5 @@
  * @see pvalloc()
  */
 void *valloc(size_t n) {
-  return memalign(FRAMESIZE, n);
+  return memalign(__granularity(), n);
 }

libc/proc/fork-nt.c

@@ -17,14 +17,17 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "ape/sections.internal.h"
+#include "libc/assert.h"
 #include "libc/calls/internal.h"
 #include "libc/calls/sig.internal.h"
 #include "libc/calls/state.internal.h"
 #include "libc/calls/syscall_support-nt.internal.h"
 #include "libc/errno.h"
 #include "libc/fmt/itoa.h"
+#include "libc/intrin/atomic.h"
 #include "libc/intrin/directmap.internal.h"
 #include "libc/intrin/kprintf.h"
+#include "libc/intrin/maps.h"
 #include "libc/intrin/strace.internal.h"
 #include "libc/intrin/weaken.h"
 #include "libc/macros.internal.h"
@@ -41,11 +44,14 @@
 #include "libc/nt/runtime.h"
 #include "libc/nt/signals.h"
 #include "libc/nt/struct/ntexceptionpointers.h"
+#include "libc/nt/thread.h"
+#include "libc/nt/thunk/msabi.h"
 #include "libc/proc/ntspawn.h"
 #include "libc/proc/proc.internal.h"
 #include "libc/runtime/internal.h"
 #include "libc/runtime/memtrack.internal.h"
 #include "libc/runtime/runtime.h"
+#include "libc/runtime/stack.h"
 #include "libc/runtime/symbols.internal.h"
 #include "libc/str/str.h"
 #include "libc/sysv/consts/at.h"
@@ -55,10 +61,13 @@
 #include "libc/sysv/consts/sig.h"
 #include "libc/sysv/errfuns.h"
 #include "libc/thread/itimer.internal.h"
+#include "libc/thread/posixthread.internal.h"
 #include "libc/thread/tls.h"
 #ifdef __x86_64__
 
+extern long __klog_handle;
 void WipeKeystrokes(void);
+__msabi extern typeof(GetCurrentProcessId) *const __imp_GetCurrentProcessId;
 
 static textwindows wontreturn void AbortFork(const char *func) {
 #if SYSDEBUG
@@ -98,8 +107,11 @@ static dontinline textwindows bool ForkIo2(int64_t h, void *buf, size_t n,
                                                         struct NtOverlapped *),
                                            const char *sf, bool ischild) {
   ssize_t rc = ForkIo(h, buf, n, fn);
-  if (ischild)
+  if (ischild) {
     __tls_enabled_set(false);  // prevent tls crash in kprintf
+    __pid = __imp_GetCurrentProcessId();
+    __klog_handle = 0;
+  }
   NTTRACE("%s(%ld, %p, %'zu) → %'zd% m", sf, h, buf, n, rc);
   return rc != -1;
 }
@@ -183,16 +195,17 @@ static __msabi textwindows int OnForkCrash(struct NtExceptionPointers *ep) {
   TerminateThisProcess(SIGSTKFLT);
 }
 
+static textwindows void *Malloc(size_t size) {
+  return HeapAlloc(GetProcessHeap(), 0, size);
+}
+
 textwindows void WinMainForked(void) {
   jmp_buf jb;
   int64_t reader;
   int64_t savetsc;
-  char *addr, *shad;
+  struct Map *map;
-  uint64_t size, upsize;
+  uint32_t varlen, oldprot;
-  struct MemoryInterval *maps;
   char16_t fvar[21 + 1 + 21 + 1];
-  uint32_t i, varlen, oldprot, savepid;
-  long mapcount, mapcapacity, specialz;
   struct Fds *fds = __veil("r", &g_fds);
 
   // check to see if the process was actually forked
@@ -200,7 +213,7 @@ textwindows void WinMainForked(void) {
   varlen = GetEnvironmentVariable(u"_FORK", fvar, ARRAYLEN(fvar));
   if (!varlen || varlen >= ARRAYLEN(fvar))
     return;
-  NTTRACE("WinMainForked()");
+  /* STRACE("WinMainForked()"); */
   SetEnvironmentVariable(u"_FORK", NULL);
 #if SYSDEBUG
   int64_t oncrash = AddVectoredExceptionHandler(1, (void *)OnForkCrash);
@@ -208,64 +221,57 @@ textwindows void WinMainForked(void) {
   ParseInt(fvar, &reader);
 
   // read the cpu state from the parent process & plus
-  // read the list of mappings from the parent process
-  // this is stored in a special secretive memory map!
-  // read ExtendMemoryIntervals for further details :|
-  maps = (void *)kMemtrackStart;
   ReadOrDie(reader, jb, sizeof(jb));
-  ReadOrDie(reader, &mapcount, sizeof(_mmi.i));
-  ReadOrDie(reader, &mapcapacity, sizeof(_mmi.n));
-  specialz = ROUNDUP(mapcapacity * sizeof(_mmi.p[0]), kMemtrackGran);
-  ViewOrDie(MapOrDie(kNtPageReadwrite, specialz), kNtFileMapWrite, 0, specialz,
-            maps);
-  ReadOrDie(reader, maps, mapcount * sizeof(_mmi.p[0]));
-  if (IsAsan()) {
-    shad = (char *)(((intptr_t)maps >> 3) + 0x7fff8000);
-    size = ROUNDUP(specialz >> 3, FRAMESIZE);
-    ViewOrDie(MapOrDie(kNtPageReadwrite, size), kNtFileMapWrite, 0, size, shad);
-    ReadOrDie(reader, shad, (mapcount * sizeof(_mmi.p[0])) >> 3);
-  }
 
-  // read the heap mappings from the parent process
+  // read memory mappings from parent process
-  for (i = 0; i < mapcount; ++i) {
+  struct Map *maps = __maps.maps;
-    addr = (char *)((uint64_t)maps[i].x << 16);
+  for (;;) {
-    size = maps[i].size;
+    map = Malloc(sizeof(*map));
-    if ((maps[i].flags & MAP_TYPE) != MAP_SHARED) {
+    ReadOrDie(reader, map, sizeof(*map));
-      upsize = ROUNDUP(size, FRAMESIZE);
+    if ((map->flags & MAP_TYPE) != MAP_SHARED) {
       // we don't need to close the map handle because sys_mmap_nt
       // doesn't mark it inheritable across fork() for MAP_PRIVATE
-      ViewOrDie((maps[i].h = MapOrDie(kNtPageExecuteReadwrite, upsize)),
+      ViewOrDie((map->h = MapOrDie(kNtPageExecuteReadwrite, map->size)),
-                kNtFileMapWrite | kNtFileMapExecute, 0, upsize, addr);
+                kNtFileMapWrite | kNtFileMapExecute, 0, map->size, map->addr);
-      ReadOrDie(reader, addr, size);
+      ReadOrDie(reader, map->addr, map->size);
     } else {
       // we can however safely inherit MAP_SHARED with zero copy
-      ViewOrDie(maps[i].h,
+      ViewOrDie(map->h,
-                maps[i].readonlyfile ? kNtFileMapRead | kNtFileMapExecute
+                map->readonlyfile ? kNtFileMapRead | kNtFileMapExecute
-                                     : kNtFileMapWrite | kNtFileMapExecute,
+                                  : kNtFileMapWrite | kNtFileMapExecute,
-                maps[i].offset, size, addr);
+                map->off, map->size, map->addr);
     }
+    dll_init(&map->elem);
+    bool isdone = !map->next;
+    map->next = maps;
+    maps = map;
+    if (isdone)
+      break;
   }
 
   // read the .data and .bss program image sections
-  savepid = __pid;
   savetsc = kStartTsc;
   ReadOrDie(reader, __data_start, __data_end - __data_start);
   ReadOrDie(reader, __bss_start, __bss_end - __bss_start);
-  __pid = savepid;
   kStartTsc = savetsc;
   __threaded = false;
   __tls_index = 0;
   __tls_enabled_set(false);
 
-  // apply fixups and reapply memory protections
+  // fixup memory manager
-  _mmi.p = maps;
+  __maps.free = 0;
-  _mmi.n = specialz / sizeof(_mmi.p[0]);
+  __maps.used = 0;
-  for (i = 0; i < mapcount; ++i) {
+  __maps.maps = maps;
-    if (!VirtualProtect((void *)((uint64_t)maps[i].x << 16), maps[i].size,
+  dll_init(&__maps.stack.elem);
-                        __prot2nt(maps[i].prot, maps[i].iscow), &oldprot)) {
+  dll_make_first(&__maps.used, &__maps.stack.elem);
+  for (struct Map *map = maps; map; map = map->next) {
+    dll_make_last(&__maps.used, &map->elem);
+    if (!VirtualProtect(map->addr, map->size, __prot2nt(map->prot, map->iscow),
+                        &oldprot)) {
       AbortFork("VirtualProtect");
     }
   }
+  __maps_init();
 
   // mitosis complete
   if (!CloseHandle(reader)) {
@@ -295,7 +301,7 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
   jmp_buf jb;
   uint32_t op;
   char **args;
-  int i, rc = -1;
+  int rc = -1;
   struct Proc *proc;
   struct CosmoTib *tib;
   char16_t pipename[64];
@@ -304,10 +310,10 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
   struct NtProcessInformation procinfo;
   char *p, forkvar[6 + 21 + 1 + 21 + 1];
   tib = __get_tls();
-  ftrace_enabled(-1);
-  strace_enabled(-1);
   if (!(proc = __proc_new()))
     return -1;
+  ftrace_enabled(-1);
+  strace_enabled(-1);
   if (!setjmp(jb)) {
     reader = CreateNamedPipe(__create_pipe_name(pipename), kNtPipeAccessInbound,
                              kNtPipeTypeByte | kNtPipeReadmodeByte, 1, PIPE_BUF,
@@ -324,6 +330,7 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
       startinfo.hStdError = g_fds.p[2].handle;
       args = __argv;
 #if SYSDEBUG
+      int i;
       // If --strace was passed to this program, then propagate it the
       // forked process since the flag was removed by __intercept_flag
       if (strace_enabled(0) > 0) {
@@ -350,22 +357,17 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
           &procinfo});
       if (spawnrc != -1) {
         CloseHandle(procinfo.hThread);
-        ok = WriteAll(writer, jb, sizeof(jb)) &&
+        ok = WriteAll(writer, jb, sizeof(jb));
-             WriteAll(writer, &_mmi.i, sizeof(_mmi.i)) &&
+        for (struct Map *map = __maps.maps; ok && map; map = map->next) {
-             WriteAll(writer, &_mmi.n, sizeof(_mmi.n)) &&
+          if (MAX((char *)__executable_start, map->addr) <
-             WriteAll(writer, _mmi.p, _mmi.i * sizeof(_mmi.p[0]));
+              MIN((char *)_end, map->addr + map->size))
-        if (IsAsan() && ok) {
+            continue;  // executable image is loaded by windows
-          ok = WriteAll(writer, (char *)(((intptr_t)_mmi.p >> 3) + 0x7fff8000),
+          ok = WriteAll(writer, map, sizeof(*map));
-                        (_mmi.i * sizeof(_mmi.p[0])) >> 3);
+          if (ok && (map->flags & MAP_TYPE) != MAP_SHARED) {
-        }
-        for (i = 0; i < _mmi.i && ok; ++i) {
-          if ((_mmi.p[i].flags & MAP_TYPE) != MAP_SHARED) {
-            char *p = (char *)((uint64_t)_mmi.p[i].x << 16);
             // XXX: forking destroys thread guard pages currently
-            VirtualProtect(
+            VirtualProtect(map->addr, map->size,
-                p, _mmi.p[i].size,
+                           __prot2nt(map->prot | PROT_READ, map->iscow), &op);
-                __prot2nt(_mmi.p[i].prot | PROT_READ, _mmi.p[i].iscow), &op);
+            ok = WriteAll(writer, map->addr, map->size);
-            ok = WriteAll(writer, p, _mmi.p[i].size);
           }
         }
         if (ok)
@@ -416,6 +418,9 @@ textwindows int sys_fork_nt(uint32_t dwCreationFlags) {
     if (_weaken(__itimer_wipe)) {
       _weaken(__itimer_wipe)();
     }
+    // notify pthread join
+    atomic_store_explicit(&_pthread_static.ptid, GetCurrentThreadId(),
+                          memory_order_release);
   }
   if (rc == -1) {
     dll_make_first(&__proc.free, &proc->elem);

libc/proc/fork.c

@@ -26,6 +26,8 @@
 #include "libc/dce.h"
 #include "libc/intrin/atomic.h"
 #include "libc/intrin/dll.h"
+#include "libc/intrin/kprintf.h"
+#include "libc/intrin/maps.h"
 #include "libc/intrin/strace.internal.h"
 #include "libc/intrin/weaken.h"
 #include "libc/nt/files.h"
@@ -43,32 +45,29 @@
 #include "libc/thread/tls.h"
 
 static void _onfork_prepare(void) {
-  if (_weaken(_pthread_onfork_prepare)) {
+  if (_weaken(_pthread_onfork_prepare))
     _weaken(_pthread_onfork_prepare)();
-  }
   _pthread_lock();
+  __maps_lock();
   __fds_lock();
-  __mmi_lock();
 }
 
 static void _onfork_parent(void) {
-  __mmi_unlock();
   __fds_unlock();
+  __maps_unlock();
   _pthread_unlock();
-  if (_weaken(_pthread_onfork_parent)) {
+  if (_weaken(_pthread_onfork_parent))
     _weaken(_pthread_onfork_parent)();
-  }
 }
 
 static void _onfork_child(void) {
   pthread_mutexattr_t attr;
   pthread_mutexattr_init(&attr);
   pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
-  extern pthread_mutex_t __mmi_lock_obj;
-  pthread_mutex_init(&__mmi_lock_obj, &attr);
   pthread_mutex_init(&__fds_lock_obj, &attr);
   pthread_mutexattr_destroy(&attr);
   _pthread_init();
+  __maps_unlock();
   if (_weaken(_pthread_onfork_child)) {
     _weaken(_pthread_onfork_child)();
   }
@@ -81,9 +80,8 @@ int _fork(uint32_t dwCreationFlags) {
   BLOCK_SIGNALS;
   if (IsWindows())
     __proc_lock();
-  if (__threaded) {
+  if (__threaded)
     _onfork_prepare();
-  }
   if (!IsWindows()) {
     ax = sys_fork();
   } else {

libc/runtime/clone.c

@@ -47,6 +47,7 @@
 #include "libc/runtime/syslib.internal.h"
 #include "libc/sock/internal.h"
 #include "libc/stdalign.internal.h"
+#include "libc/stdio/sysparam.h"
 #include "libc/str/str.h"
 #include "libc/sysv/consts/arch.h"
 #include "libc/sysv/consts/clone.h"
@@ -80,27 +81,25 @@
 struct CloneArgs {
   alignas(16) union {
     struct {
-      int tid;
+      atomic_int tid;
       int this;
     };
-    uint32_t utid;
     int64_t tid64;
   };
-  int *ptid;
+  atomic_int *ptid;
-  int *ctid;
+  atomic_int *ctid;
-  int *ztid;
+  atomic_int *ztid;
   char *tls;
   int (*func)(void *, int);
   void *arg;
+  long sp;
 };
 
 int sys_set_tls(uintptr_t, void *);
-int __stack_call(void *, int, long, long, int (*)(void *, int), void *);
+int __stack_call(void *, int, long, long, int (*)(void *, int), long);
 
-static struct CloneArgs *AllocateCloneArgs(char *stk, size_t stksz) {
+static long AlignStack(long sp, char *stk, long stksz, int mal) {
-  return (struct CloneArgs *)(((uintptr_t)(stk + stksz) -
+  return sp & -mal;
-                               sizeof(struct CloneArgs)) &
-                              -16);
 }
 
 #ifdef __x86_64__
@@ -120,8 +119,8 @@ WinThreadEntry(int rdi,                            // rcx
   int rc;
   if (wt->tls)
     __set_tls_win32(wt->tls);
-  *wt->ctid = wt->tid;
+  *wt->ctid = GetCurrentThreadId();
-  rc = __stack_call(wt->arg, wt->tid, 0, 0, wt->func, wt);
+  rc = __stack_call(wt->arg, wt->tid, 0, 0, wt->func, wt->sp);
   // we can now clear ctid directly since we're no longer using our own
   // stack memory, which can now be safely free'd by the parent thread.
   *wt->ztid = 0;
@@ -134,24 +133,31 @@ WinThreadEntry(int rdi,                            // rcx
 
 static textwindows errno_t CloneWindows(int (*func)(void *, int), char *stk,
                                         size_t stksz, int flags, void *arg,
-                                        void *tls, int *ptid, int *ctid) {
+                                        void *tls, atomic_int *ptid,
+                                        atomic_int *ctid) {
+  long sp;
   int64_t h;
+  uint32_t utid;
   struct CloneArgs *wt;
-  wt = AllocateCloneArgs(stk, stksz);
+  sp = (intptr_t)stk + stksz;
+  sp = AlignStack(sp, stk, stksz, 16);
+  sp -= sizeof(struct CloneArgs);
+  sp &= -alignof(struct CloneArgs);
+  wt = (struct CloneArgs *)sp;
   wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
   wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid;
   wt->func = func;
   wt->arg = arg;
   wt->tls = flags & CLONE_SETTLS ? tls : 0;
+  wt->sp = sp;
   if ((h = CreateThread(&kNtIsInheritable, 65536, (void *)WinThreadEntry, wt,
-                        kNtStackSizeParamIsAReservation, &wt->utid))) {
+                        kNtStackSizeParamIsAReservation, &utid))) {
+    if (flags & CLONE_PARENT_SETTID)
+      *ptid = utid;
     if (flags & CLONE_SETTLS) {
       struct CosmoTib *tib = tls;
       atomic_store_explicit(&tib->tib_syshand, h, memory_order_release);
     }
-    if (flags & CLONE_PARENT_SETTID) {
-      *ptid = wt->tid;
-    }
     return 0;
   } else {
     return __dos2errno(GetLastError());
@@ -222,14 +228,26 @@ XnuThreadMain(void *pthread,                    // rdi
 }
 
 static errno_t CloneXnu(int (*fn)(void *), char *stk, size_t stksz, int flags,
-                        void *arg, void *tls, int *ptid, int *ctid) {
+                        void *arg, void *tls, atomic_int *ptid,
+                        atomic_int *ctid) {
+
+  // perform this weird mandatory system call once
   static bool once;
-  struct CloneArgs *wt;
   if (!once) {
     npassert(sys_bsdthread_register(XnuThreadThunk, 0, 0, 0, 0, 0, 0) != -1);
     once = true;
   }
-  wt = AllocateCloneArgs(stk, stksz);
+
+  // setup stack for thread
+  long sp;
+  struct CloneArgs *wt;
+  sp = (intptr_t)stk + stksz;
+  sp = AlignStack(sp, stk, stksz, 16);
+  sp -= sizeof(struct CloneArgs);
+  sp &= -alignof(struct CloneArgs);
+  wt = (struct CloneArgs *)sp;
+
+  // pass parameters to new thread via xnu
   wt->ptid = flags & CLONE_PARENT_SETTID ? ptid : &wt->tid;
   wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
   wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid;
@@ -260,8 +278,8 @@ static wontreturn void OpenbsdThreadMain(void *p) {
 }
 
 static errno_t CloneOpenbsd(int (*func)(void *, int), char *stk, size_t stksz,
-                            int flags, void *arg, void *tls, int *ptid,
+                            int flags, void *arg, void *tls, atomic_int *ptid,
-                            int *ctid) {
+                            atomic_int *ctid) {
   int rc;
   intptr_t sp;
   struct __tfork *tf;
@@ -273,11 +291,12 @@ static errno_t CloneOpenbsd(int (*func)(void *, int), char *stk, size_t stksz,
   sp -= sizeof(struct CloneArgs);
   sp &= -alignof(struct CloneArgs);
   wt = (struct CloneArgs *)sp;
+  sp = AlignStack(sp, stk, stksz, 16);
   wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
   wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid;
   wt->arg = arg;
   wt->func = func;
-  tf->tf_stack = (char *)wt - 8;
+  tf->tf_stack = (char *)sp - 8;
   tf->tf_tcb = flags & CLONE_SETTLS ? tls : 0;
   tf->tf_tid = &wt->tid;
   if ((rc = __tfork_thread(tf, sizeof(*tf), OpenbsdThreadMain, wt)) >= 0) {
@@ -297,7 +316,7 @@ static errno_t CloneOpenbsd(int (*func)(void *, int), char *stk, size_t stksz,
 static wontreturn void NetbsdThreadMain(void *arg,                 // rdi
                                         int (*func)(void *, int),  // rsi
                                         int *tid,                  // rdx
-                                        int *ctid,                 // rcx
+                                        atomic_int *ctid,          // rcx
                                         int *ztid) {               // r9
   int ax, dx;
   // TODO(jart): Why are we seeing flakes where *tid is zero?
@@ -316,11 +335,13 @@ static wontreturn void NetbsdThreadMain(void *arg,                 // rdi
 }
 
 static int CloneNetbsd(int (*func)(void *, int), char *stk, size_t stksz,
-                       int flags, void *arg, void *tls, int *ptid, int *ctid) {
+                       int flags, void *arg, void *tls, atomic_int *ptid,
+                       atomic_int *ctid) {
   // NetBSD has its own clone() and it works, but it's technically a
   // second-class API, intended to help Linux folks migrate to this.
+  int ax;
   bool failed;
-  int ax, *tid;
+  atomic_int *tid;
   intptr_t dx, sp;
   static bool once;
   struct ucontext_netbsd *ctx;
@@ -335,16 +356,16 @@ static int CloneNetbsd(int (*func)(void *, int), char *stk, size_t stksz,
     npassert(!failed);
     once = true;
   }
-  sp = (intptr_t)(stk + stksz);
+  sp = (intptr_t)stk + stksz;
 
   // allocate memory for tid
-  sp -= sizeof(int);
+  sp -= sizeof(atomic_int);
-  sp = sp & -alignof(int);
+  sp = sp & -alignof(atomic_int);
-  tid = (int *)sp;
+  tid = (atomic_int *)sp;
   *tid = 0;
 
   // align the stack
-  sp = sp & -16;
+  sp = AlignStack(sp, stk, stksz, 16);
 
   // simulate call to misalign stack and ensure backtrace looks good
   sp -= 8;
@@ -439,11 +460,16 @@ static wontreturn void FreebsdThreadMain(void *p) {
 }
 
 static errno_t CloneFreebsd(int (*func)(void *, int), char *stk, size_t stksz,
-                            int flags, void *arg, void *tls, int *ptid,
+                            int flags, void *arg, void *tls, atomic_int *ptid,
-                            int *ctid) {
+                            atomic_int *ctid) {
+  long sp;
   int64_t tid;
   struct CloneArgs *wt;
-  wt = AllocateCloneArgs(stk, stksz);
+  sp = (intptr_t)stk + stksz;
+  sp -= sizeof(struct CloneArgs);
+  sp &= -alignof(struct CloneArgs);
+  wt = (struct CloneArgs *)sp;
+  sp = AlignStack(sp, stk, stksz, 16);
   wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
   wt->ztid = flags & CLONE_CHILD_CLEARTID ? ctid : &wt->tid;
   wt->tls = tls;
@@ -453,7 +479,7 @@ static errno_t CloneFreebsd(int (*func)(void *, int), char *stk, size_t stksz,
       .start_func = FreebsdThreadMain,
       .arg = wt,
       .stack_base = stk,
-      .stack_size = (uintptr_t)wt - (uintptr_t)stk,
+      .stack_size = sp - (long)stk,
       .tls_base = flags & CLONE_SETTLS ? tls : 0,
       .tls_size = 64,
       .child_tid = &wt->tid64,
@@ -492,15 +518,16 @@ static void *SiliconThreadMain(void *arg) {
   struct CloneArgs *wt = arg;
   asm volatile("mov\tx28,%0" : /* no outputs */ : "r"(wt->tls));
   *wt->ctid = wt->this;
-  __stack_call(wt->arg, wt->this, 0, 0, wt->func, wt);
+  __stack_call(wt->arg, wt->this, 0, 0, wt->func, wt->sp);
   *wt->ztid = 0;
   ulock_wake(UL_COMPARE_AND_WAIT | ULF_WAKE_ALL, wt->ztid, 0);
   return 0;
 }
 
 static errno_t CloneSilicon(int (*fn)(void *, int), char *stk, size_t stksz,
-                            int flags, void *arg, void *tls, int *ptid,
+                            int flags, void *arg, void *tls, atomic_int *ptid,
-                            int *ctid) {
+                            atomic_int *ctid) {
+  long sp;
   void *attr;
   errno_t res;
   unsigned tid;
@@ -508,7 +535,11 @@ static errno_t CloneSilicon(int (*fn)(void *, int), char *stk, size_t stksz,
   size_t babystack;
   struct CloneArgs *wt;
   static atomic_uint tids;
-  wt = AllocateCloneArgs(stk, stksz);
+  sp = (intptr_t)stk + stksz;
+  sp -= sizeof(struct CloneArgs);
+  sp &= -alignof(struct CloneArgs);
+  wt = (struct CloneArgs *)sp;
+  sp = AlignStack(sp, stk, stksz, 16);
   tid = atomic_fetch_add_explicit(&tids, 1, memory_order_acq_rel);
   wt->this = tid = (tid & (kMaxThreadIds - 1)) + kMinThreadId;
   wt->ctid = flags & CLONE_CHILD_SETTID ? ctid : &wt->tid;
@@ -516,6 +547,7 @@ static errno_t CloneSilicon(int (*fn)(void *, int), char *stk, size_t stksz,
   wt->tls = flags & CLONE_SETTLS ? tls : 0;
   wt->func = fn;
   wt->arg = arg;
+  wt->sp = sp;
   babystack = __syslib->__pthread_stack_min;
 #pragma GCC push_options
 #pragma GCC diagnostic ignored "-Walloca-larger-than="
@@ -545,16 +577,16 @@ struct LinuxCloneArgs {
   int (*func)(void *, int);
   void *arg;
   char *tls;
-  int ctid;
+  atomic_int ctid;
 };
 
-int sys_clone_linux(int flags,   // rdi
+int sys_clone_linux(int flags,         // rdi
-                    long sp,     // rsi
+                    long sp,           // rsi
-                    int *ptid,   // rdx
+                    atomic_int *ptid,  // rdx
-                    int *ctid,   // rcx
+                    atomic_int *ctid,  // rcx
-                    void *tls,   // r8
+                    void *tls,         // r8
-                    void *func,  // r9
+                    void *func,        // r9
-                    void *arg);  // 8(rsp)
+                    void *arg);        // 8(rsp)
 
 static int LinuxThreadEntry(void *arg, int tid) {
   struct LinuxCloneArgs *wt = arg;
@@ -563,19 +595,21 @@ static int LinuxThreadEntry(void *arg, int tid) {
 }
 
 static int CloneLinux(int (*func)(void *arg, int rc), char *stk, size_t stksz,
-                      int flags, void *arg, void *tls, int *ptid, int *ctid) {
+                      int flags, void *arg, void *tls, atomic_int *ptid,
+                      atomic_int *ctid) {
   int rc;
   long sp;
   struct LinuxCloneArgs *wt;
-  sp = (intptr_t)(stk + stksz);
+  sp = (intptr_t)stk + stksz;
   sp -= sizeof(struct LinuxCloneArgs);
+  sp &= -alignof(struct LinuxCloneArgs);
+  wt = (struct LinuxCloneArgs *)sp;
   // align the stack
 #ifdef __aarch64__
-  sp = sp & -128;  // for kernel 4.6 and earlier
+  sp = AlignStack(sp, stk, stksz, 128);  // for kernel <=4.6
 #else
-  sp = sp & -16;
+  sp = AlignStack(sp, stk, stksz, 16);
 #endif
-  wt = (struct LinuxCloneArgs *)sp;
 #ifdef __x86_64__
   if (flags & CLONE_SETTLS) {
     flags &= ~CLONE_SETTLS;

libc/runtime/cosmo2.c

@@ -21,6 +21,7 @@
 #include "libc/calls/syscall-sysv.internal.h"
 #include "libc/dce.h"
 #include "libc/errno.h"
+#include "libc/intrin/maps.h"
 #include "libc/intrin/strace.internal.h"
 #include "libc/limits.h"
 #include "libc/macros.internal.h"
@@ -162,13 +163,8 @@ wontreturn textstartup void cosmo(long *sp, struct Syslib *m1, char *exename,
   // needed by kisdangerous()
   __pid = sys_getpid().ax;
 
-  // initialize memory manager
-  _mmi.i = 0;
-  _mmi.p = _mmi.s;
-  _mmi.n = ARRAYLEN(_mmi.s);
-  __virtualmax = -1;
-
   // initialize file system
+  __maps_init();
   __init_fds(argc, argv, envp);
 
   // prepend cwd to executable path

libc/runtime/enable_tls.c

@@ -25,6 +25,7 @@
 #include "libc/intrin/atomic.h"
 #include "libc/intrin/dll.h"
 #include "libc/intrin/getenv.internal.h"
+#include "libc/intrin/kprintf.h"
 #include "libc/intrin/weaken.h"
 #include "libc/macros.internal.h"
 #include "libc/nt/files.h"
@@ -251,7 +252,7 @@ textstartup void __enable_tls(void) {
   _pthread_static.pt_flags = PT_STATIC;
   dll_init(&_pthread_static.list);
   _pthread_list = &_pthread_static.list;
-  atomic_store_explicit(&_pthread_static.ptid, tid, memory_order_relaxed);
+  atomic_store_explicit(&_pthread_static.ptid, tid, memory_order_release);
 
   // ask the operating system to change the x86 segment register
   __set_tls(tib);

libc/runtime/ftrace_install.c

@@ -16,8 +16,8 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/calls.h"
 #include "libc/fmt/itoa.h"
-#include "libc/intrin/kprintf.h"
 #include "libc/runtime/internal.h"
 #include "libc/runtime/runtime.h"
 #include "libc/runtime/stack.h"
@@ -28,7 +28,7 @@ textstartup int ftrace_install(void) {
     ftrace_stackdigs = LengthInt64Thousands(GetStackSize());
     return __hook(ftrace_hook, GetSymbolTable());
   } else {
-    kprintf("error: --ftrace failed to open symbol table\n");
+    tinyprint(2, "error: --ftrace failed to open symbol table\n", NULL);
     return -1;
   }
 }

libc/runtime/getavphyspages.c

@@ -23,5 +23,5 @@ long __get_avphys_pages(void) {
   struct sysinfo si;
   if (sysinfo(&si) == -1)
     return -1;
-  return (((int64_t)si.freeram + si.bufferram) * si.mem_unit) / FRAMESIZE;
+  return (((int64_t)si.freeram + si.bufferram) * si.mem_unit) / __granularity();
 }

libc/runtime/getpagesize.c

@@ -23,5 +23,5 @@
  * @see sysconf(_SC_PAGE_SIZE) which is portable
  */
 int getpagesize(void) {
-  return FRAMESIZE;
+  return __granularity();
 }

libc/runtime/getphyspages.c

@@ -23,5 +23,5 @@ long __get_phys_pages(void) {
   struct sysinfo si;
   if (sysinfo(&si) == -1)
     return -1;
-  return ((int64_t)si.totalram * si.mem_unit) / FRAMESIZE;
+  return ((int64_t)si.totalram * si.mem_unit) / __granularity();
 }

libc/runtime/getsymboltable.c

@@ -63,7 +63,7 @@ static struct SymbolTable *GetSymbolTableFromZip(struct Zipos *zipos) {
       (cf = GetZipFile(zipos, ".symtab")) != -1) {
     lf = GetZipCfileOffset(zipos->map + cf);
     size = GetZipLfileUncompressedSize(zipos->map + lf);
-    size2 = ROUNDUP(size, FRAMESIZE);
+    size2 = ROUNDUP(size, __granularity());
     if ((res = _mapanon(size2))) {
       switch (ZIP_LFILE_COMPRESSIONMETHOD(zipos->map + lf)) {
         case kZipCompressionNone:

libc/runtime/grow.c

@@ -16,6 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "ape/sections.internal.h"
 #include "libc/assert.h"
 #include "libc/fmt/conv.h"
 #include "libc/intrin/weaken.h"
@@ -31,6 +32,13 @@
 #define GUARANTEE_TERMINATOR 1
 #define INITIAL_CAPACITY     (32 - GUARANTEE_TERMINATOR)
 
+static bool isheap(const void *p) {
+  if (__executable_start <= (const unsigned char *)p &&
+      (const unsigned char *)p < _end)
+    return false;
+  return true;
+}
+
 bool __grow(void *pp, size_t *capacity, size_t itemsize, size_t extra) {
   void **p, *p1, *p2;
   size_t n1, n2;
@@ -39,8 +47,8 @@ bool __grow(void *pp, size_t *capacity, size_t itemsize, size_t extra) {
   p = (void **)pp;
   unassert(itemsize);
   unassert((*p && *capacity) || (!*p && !*capacity));
-  unassert(!_isheap(*p) || ((intptr_t)*p & 15) == 0);
+  unassert(!isheap(*p) || ((intptr_t)*p & 15) == 0);
-  p1 = _isheap(*p) ? *p : NULL;
+  p1 = isheap(*p) ? *p : NULL;
   p2 = NULL;
   n1 = *capacity;
   n2 = (*p ? n1 + (n1 >> 1) : MAX(4, INITIAL_CAPACITY / itemsize)) + extra;

libc/runtime/internal.h

@@ -6,7 +6,7 @@
 #include "libc/runtime/runtime.h"
 
 #define STACK_CEIL 0x700000000000ul
-#define STACK_SIZE FRAMESIZE
+#define STACK_SIZE 65536
 
 #define RUNLEVEL_MALLOC 1
 
@@ -41,7 +41,6 @@ long _setstack(void *, void *, ...);
 int GetDosArgv(const char16_t *, char *, size_t, char **, size_t);
 int GetDosEnviron(const char16_t *, char *, size_t, char **, size_t);
 bool __intercept_flag(int *, char *[], const char *);
-int sys_mprotect_nt(void *, size_t, int);
 int __inflate(void *, size_t, const void *, size_t);
 void *__mmap_unlocked(void *, size_t, int, int, int, int64_t);
 int __munmap_unlocked(char *, size_t);

libc/runtime/ismemtracked.greg.c

@@ -1,43 +0,0 @@
-/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
-│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
-╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
-│                                                                              │
-│ Permission to use, copy, modify, and/or distribute this software for         │
-│ any purpose with or without fee is hereby granted, provided that the         │
-│ above copyright notice and this permission notice appear in all copies.      │
-│                                                                              │
-│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
-│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
-│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
-│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
-│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
-│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
-│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
-│ PERFORMANCE OF THIS SOFTWARE.                                                │
-╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/runtime/memtrack.internal.h"
-
-static inline bool IsMemtrackedImpl(int x, int y) {
-  unsigned i;
-  i = __find_memory(&_mmi, x);
-  if (i == _mmi.i)
-    return false;
-  if (x < _mmi.p[i].x)
-    return false;
-  for (;;) {
-    if (y <= _mmi.p[i].y)
-      return true;
-    if (++i == _mmi.i)
-      return false;
-    if (_mmi.p[i].x != _mmi.p[i - 1].y + 1)
-      return false;
-  }
-}
-
-bool IsMemtracked(int x, int y) {
-  /* assumes __mmi_lock() is held */
-  bool res;
-  res = IsMemtrackedImpl(x, y);
-  return res;
-}

libc/runtime/mapanon.c

@@ -22,7 +22,6 @@
 #include "libc/runtime/runtime.h"
 #include "libc/sysv/consts/map.h"
 #include "libc/sysv/consts/prot.h"
-#include "libc/sysv/errfuns.h"
 
 /**
  * Helper function for allocating anonymous mapping.
@@ -59,11 +58,9 @@
 void *_mapanon(size_t size) {
   void *m;
   m = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-  if (m != MAP_FAILED) {
+  if (m != MAP_FAILED)
     return m;
-  }
+  if (errno == ENOMEM && _weaken(__oom_hook))
-  if (errno == ENOMEM && _weaken(__oom_hook)) {
     _weaken(__oom_hook)(size);
-  }
   return 0;
 }

libc/runtime/mapstack.c

@@ -17,6 +17,7 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/calls/calls.h"
+#include "libc/calls/syscall-sysv.internal.h"
 #include "libc/dce.h"
 #include "libc/intrin/asan.internal.h"
 #include "libc/intrin/asancodes.h"
@@ -27,6 +28,9 @@
 #include "libc/sysv/consts/map.h"
 #include "libc/sysv/consts/prot.h"
 
+#define MAP_ANON_OPENBSD  0x1000
+#define MAP_STACK_OPENBSD 0x4000
+
 /**
  * Allocates stack.
  *
@@ -42,13 +46,17 @@
  */
 void *NewCosmoStack(void) {
   char *p;
-  if ((p = mmap(0, GetStackSize(), PROT_READ | PROT_WRITE,
+  size_t n = GetStackSize() + (uintptr_t)ape_stack_align;
-                MAP_ANONYMOUS |
+  if ((p = mmap(0, n, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1,
-                    (IsAarch64() && IsLinux() && IsQemuUser() ? MAP_PRIVATE
+                0)) != MAP_FAILED) {
-                                                              : MAP_STACK),
+    if (IsOpenbsd() && __sys_mmap(p, n, PROT_READ | PROT_WRITE,
-                -1, 0)) != MAP_FAILED) {
+                                  MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD |
+                                      MAP_STACK_OPENBSD,
+                                  -1, 0, 0) != p) {
+      notpossible;
+    }
     if (IsAsan()) {
-      __asan_poison(p + GetStackSize() - 16, 16, kAsanStackOverflow);
+      __asan_poison(p + n - 16, 16, kAsanStackOverflow);
       __asan_poison(p, getauxval(AT_PAGESZ), kAsanStackOverflow);
     }
     return p;

libc/runtime/memtrack.internal.h

@@ -1,189 +1,22 @@
 #ifndef COSMOPOLITAN_LIBC_RUNTIME_MEMTRACK_H_
 #define COSMOPOLITAN_LIBC_RUNTIME_MEMTRACK_H_
-#include "ape/sections.internal.h"
-#include "libc/dce.h"
-#include "libc/macros.internal.h"
-#include "libc/runtime/runtime.h"
-#include "libc/runtime/stack.h"
-#include "libc/sysv/consts/ss.h"
-#include "libc/thread/tls.h"
 COSMOPOLITAN_C_START_
 
-#define kAutomapStart       0x100080040000
+#ifndef __SANITIZE_ADDRESS__
-#define kAutomapSize        (kMemtrackStart - kAutomapStart)
+#define kFixedmapStart      0x300000000
-#define kMemtrackStart      0x1fe7fffc0000
+#define kFixedmapSize       (0x400000000 - kFixedmapStart)
-#define kMemtrackSize       (0x1ffffffc0000 - kMemtrackStart)
+#define kMemtrackFdsStart   0x6fe000000
+#define kMemtrackFdsSize    (0x6ff000000 - kMemtrackFdsStart)
+#define kMemtrackZiposStart 0x6fd000000
+#define kMemtrackZiposSize  (0xafe000000 - kMemtrackZiposStart)
+#else
 #define kFixedmapStart      0x300000040000
 #define kFixedmapSize       (0x400000040000 - kFixedmapStart)
-#define kMemtrackNsyncStart 0x6fc000040000
-#define kMemtrackNsyncSize  (0x6fcffffc0000 - kMemtrackNsyncStart)
 #define kMemtrackFdsStart   0x6fe000040000
 #define kMemtrackFdsSize    (0x6feffffc0000 - kMemtrackFdsStart)
 #define kMemtrackZiposStart 0x6fd000040000
 #define kMemtrackZiposSize  (0x6fdffffc0000 - kMemtrackZiposStart)
-#define kMemtrackGran       (!IsAsan() ? FRAMESIZE : FRAMESIZE * 8)
-
-struct MemoryInterval {
-  int x;
-  int y;
-  long h;
-  long size;
-  long offset;
-  int flags;
-  char prot;
-  bool iscow;
-  bool readonlyfile;
-};
-
-struct MemoryIntervals {
-  size_t i, n;
-  struct MemoryInterval *p;
-  struct MemoryInterval s[16];
-};
-
-extern struct MemoryIntervals _mmi;
-
-void __mmi_lock(void);
-void __mmi_unlock(void);
-bool IsMemtracked(int, int);
-void PrintSystemMappings(int);
-unsigned __find_memory(const struct MemoryIntervals *, int) nosideeffect;
-bool __check_memtrack(const struct MemoryIntervals *) nosideeffect;
-void PrintMemoryIntervals(int, const struct MemoryIntervals *);
-int __track_memory(struct MemoryIntervals *, int, int, long, int, int, bool,
-                   bool, long, long);
-int __untrack_memory(struct MemoryIntervals *, int, int,
-                     void (*)(struct MemoryIntervals *, int, int));
-void __release_memory_nt(struct MemoryIntervals *, int, int);
-int __untrack_memories(void *, size_t);
-size_t __get_memtrack_size(struct MemoryIntervals *) nosideeffect;
-
-#ifdef __x86_64__
-/*
- * AMD64 has 48-bit signed pointers (PML4T)
- * AMD64 is trying to go bigger, i.e. 57-bit (PML5T)
- * LINUX forbids userspace from leveraging negative pointers
- * Q-EMU may impose smaller vaspaces emulating AMD on non-AMD
- *
- * Having "signed pointers" means these top sixteen bits
- *
- *     0x0000000000000000
- *       ^^^^
- *
- * must be
- *
- *   - 0000 for positive pointers
- *   - FFFF for negative pointers
- *
- * otherwise the instruction using the faulty pointer will fault.
- */
-#define IsLegalPointer(p) \
-  (-0x800000000000 <= (intptr_t)(p) && (intptr_t)(p) <= 0x7fffffffffff)
-#define ADDR_32_TO_48(x) (intptr_t)((uint64_t)(int)(x) << 16)
-#elif defined(__aarch64__)
-/*
- * ARM64 has 48-bit unsigned pointers (Armv8.0-A)
- * ARM64 can possibly go bigger, i.e. 52-bit (Armv8.2-A)
- * ARM64 can impose arbitrarily smaller vaspaces, e.g. 40/44-bit
- * APPLE in their limitless authoritarianism forbids 32-bit pointers
- */
-#define IsLegalPointer(p) ((uintptr_t)(p) <= 0xffffffffffff)
-#define ADDR_32_TO_48(x)  (uintptr_t)((uint64_t)(uint32_t)(x) << 16)
-#else
-/* RISC-V Sipeed Nezha has 39-bit vaspace */
-#error "unsupported architecture"
 #endif
 
-forceinline pureconst bool IsLegalSize(uint64_t n) {
-  /* subtract frame size so roundup is safe */
-  return n <= 0x800000000000 - FRAMESIZE;
-}
-
-forceinline pureconst bool IsAutoFrame(int x) {
-  return (int)(kAutomapStart >> 16) <= x &&
-         x <= (int)((kAutomapStart + kAutomapSize - 1) >> 16);
-}
-
-forceinline pureconst bool IsMemtrackFrame(int x) {
-  return (int)(kAutomapStart >> 16) <= x &&
-         x <= (int)((kAutomapStart + kAutomapSize - 1) >> 16);
-}
-
-forceinline pureconst bool IsGfdsFrame(int x) {
-  return (int)(kMemtrackFdsStart >> 16) <= x &&
-         x <= (int)((kMemtrackFdsStart + kMemtrackFdsSize - 1) >> 16);
-}
-
-forceinline pureconst bool IsNsyncFrame(int x) {
-  return (int)(kMemtrackNsyncStart >> 16) <= x &&
-         x <= (int)((kMemtrackNsyncStart + kMemtrackNsyncSize - 1) >> 16);
-}
-
-forceinline pureconst bool IsZiposFrame(int x) {
-  return (int)(kMemtrackZiposStart >> 16) <= x &&
-         x <= (int)((kMemtrackZiposStart + kMemtrackZiposSize - 1) >> 16);
-}
-
-forceinline pureconst bool IsShadowFrame(int x) {
-  return 0x7fff <= x && x < 0x10008000;
-}
-
-forceinline pureconst bool IsStaticStackFrame(int x) {
-  intptr_t stack = GetStaticStackAddr(0);
-  return (int)(stack >> 16) <= x &&
-         x <= (int)((stack + (GetStackSize() - FRAMESIZE)) >> 16);
-}
-
-forceinline pureconst bool IsStackFrame(int x) {
-  intptr_t stack = GetStackAddr();
-  return (int)(stack >> 16) <= x &&
-         x <= (int)((stack + (GetStackSize() - FRAMESIZE)) >> 16);
-}
-
-forceinline pureconst bool IsOldStack(const void *x) {
-  size_t foss_stack_size = 8ul * 1024 * 1024;
-  uintptr_t top = __oldstack + foss_stack_size;
-  uintptr_t bot = __oldstack - foss_stack_size;
-  return bot <= (uintptr_t)x && (uintptr_t)x < top;
-}
-
-forceinline pureconst bool IsOldStackFrame(int x) {
-  size_t foss_stack_size = 8ul * 1024 * 1024;
-  uintptr_t top = __oldstack + foss_stack_size;
-  uintptr_t bot = __oldstack - foss_stack_size;
-  return (int)(bot >> 16) <= x && x <= (int)((top >> 16) - 1);
-}
-
-forceinline pureconst bool IsFixedFrame(int x) {
-  return (kFixedmapStart >> 16) <= x &&
-         x <= ((kFixedmapStart + (kFixedmapSize - 1)) >> 16);
-}
-
-forceinline pureconst bool OverlapsImageSpace(const void *p, size_t n) {
-  const unsigned char *BegA, *EndA, *BegB, *EndB;
-  if (n) {
-    BegA = p;
-    EndA = BegA + n;
-    BegB = __executable_start;
-    EndB = _end;
-    return MAX(BegA, BegB) < MIN(EndA, EndB);
-  } else {
-    return 0;
-  }
-}
-
-forceinline pureconst bool OverlapsShadowSpace(const void *p, size_t n) {
-  intptr_t BegA, EndA, BegB, EndB;
-  if (n) {
-    BegA = (intptr_t)p;
-    EndA = BegA + n;
-    BegB = 0x7fff0000;
-    EndB = 0x100080000000;
-    return MAX(BegA, BegB) < MIN(EndA, EndB);
-  } else {
-    return 0;
-  }
-}
-
 COSMOPOLITAN_C_END_
 #endif /* COSMOPOLITAN_LIBC_RUNTIME_MEMTRACK_H_ */

libc/runtime/mmap.c

@@ -1,503 +0,0 @@
-/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
-│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
-╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2020 Justine Alexandra Roberts Tunney                              │
-│                                                                              │
-│ Permission to use, copy, modify, and/or distribute this software for         │
-│ any purpose with or without fee is hereby granted, provided that the         │
-│ above copyright notice and this permission notice appear in all copies.      │
-│                                                                              │
-│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
-│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
-│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
-│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
-│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
-│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
-│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
-│ PERFORMANCE OF THIS SOFTWARE.                                                │
-╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/assert.h"
-#include "libc/calls/calls.h"
-#include "libc/calls/internal.h"
-#include "libc/calls/struct/sigset.h"
-#include "libc/calls/syscall-sysv.internal.h"
-#include "libc/dce.h"
-#include "libc/errno.h"
-#include "libc/intrin/asan.internal.h"
-#include "libc/intrin/asancodes.h"
-#include "libc/intrin/bsr.h"
-#include "libc/intrin/describeflags.internal.h"
-#include "libc/intrin/directmap.internal.h"
-#include "libc/intrin/kprintf.h"
-#include "libc/intrin/likely.h"
-#include "libc/intrin/safemacros.internal.h"
-#include "libc/intrin/strace.internal.h"
-#include "libc/intrin/weaken.h"
-#include "libc/limits.h"
-#include "libc/log/backtrace.internal.h"
-#include "libc/log/libfatal.internal.h"
-#include "libc/log/log.h"
-#include "libc/macros.internal.h"
-#include "libc/nt/enum/memflags.h"
-#include "libc/nt/enum/pageflags.h"
-#include "libc/nt/memory.h"
-#include "libc/nt/process.h"
-#include "libc/nt/runtime.h"
-#include "libc/nt/struct/processmemorycounters.h"
-#include "libc/runtime/internal.h"
-#include "libc/runtime/memtrack.internal.h"
-#include "libc/runtime/runtime.h"
-#include "libc/runtime/zipos.internal.h"
-#include "libc/stdckdint.h"
-#include "libc/stdio/rand.h"
-#include "libc/str/str.h"
-#include "libc/sysv/consts/auxv.h"
-#include "libc/sysv/consts/map.h"
-#include "libc/sysv/consts/o.h"
-#include "libc/sysv/consts/prot.h"
-#include "libc/sysv/consts/sig.h"
-#include "libc/sysv/consts/ss.h"
-#include "libc/sysv/errfuns.h"
-#include "libc/thread/thread.h"
-
-#define MAP_ANONYMOUS_linux   0x00000020
-#define MAP_ANONYMOUS_openbsd 0x00001000
-#define MAP_GROWSDOWN_linux   0x00000100
-#define MAP_STACK_freebsd     0x00000400
-#define MAP_STACK_openbsd     0x00004000
-
-#define IP(X)      (intptr_t)(X)
-#define VIP(X)     (void *)IP(X)
-#define ALIGNED(p) (!(IP(p) & (FRAMESIZE - 1)))
-#define SHADE(x)   (((intptr_t)(x) >> 3) + 0x7fff8000)
-#define FRAME(x)   ((int)((intptr_t)(x) >> 16))
-
-static inline pureconst unsigned long __rounddown2pow(unsigned long x) {
-  return x ? 1ul << bsrl(x) : 0;
-}
-
-static wontreturn void __mmap_die(const char *s) {
-  if (_weaken(__die))
-    _weaken(__die)();
-  STRACE("%s %m", s);
-  _Exit(199);
-}
-
-static inline bool __overlaps_existing_mapping(char *p, size_t n) {
-  int a, b, i;
-  unassert(n > 0);
-  a = FRAME(p);
-  b = FRAME(p + (n - 1));
-  i = __find_memory(&_mmi, a);
-  if (i < _mmi.i) {
-    if (a <= _mmi.p[i].x && _mmi.p[i].x <= b)
-      return true;
-    if (a <= _mmi.p[i].y && _mmi.p[i].y <= b)
-      return true;
-    if (_mmi.p[i].x <= a && b <= _mmi.p[i].y)
-      return true;
-  }
-  return false;
-}
-
-static bool __choose_memory(int x, int n, int align, int *res) {
-  // TODO: improve performance
-  int i, start, end;
-  unassert(align > 0);
-  if (_mmi.i) {
-
-    // find the start of the automap memory region
-    i = __find_memory(&_mmi, x);
-    if (i < _mmi.i) {
-
-      // check to see if there's space available before the first entry
-      if (!ckd_add(&start, x, align - 1)) {
-        start &= -align;
-        if (!ckd_add(&end, start, n - 1)) {
-          if (end < _mmi.p[i].x) {
-            *res = start;
-            return true;
-          }
-        }
-      }
-
-      // check to see if there's space available between two entries
-      while (++i < _mmi.i) {
-        if (!ckd_add(&start, _mmi.p[i - 1].y, 1) &&
-            !ckd_add(&start, start, align - 1)) {
-          start &= -align;
-          if (!ckd_add(&end, start, n - 1)) {
-            if (end < _mmi.p[i].x) {
-              *res = start;
-              return true;
-            }
-          }
-        }
-      }
-    }
-
-    // otherwise append after the last entry if space is available
-    if (!ckd_add(&start, _mmi.p[i - 1].y, 1) &&
-        !ckd_add(&start, start, align - 1)) {
-      start &= -align;
-      if (!ckd_add(&end, start, n - 1)) {
-        *res = start;
-        return true;
-      }
-    }
-
-  } else {
-    // if memtrack is empty, then just assign the requested address
-    // assuming it doesn't overflow
-    if (!ckd_add(&start, x, align - 1)) {
-      start &= -align;
-      if (!ckd_add(&end, start, n - 1)) {
-        *res = start;
-        return true;
-      }
-    }
-  }
-
-  return false;
-}
-
-static bool __auto_map(int count, int align, int *res) {
-  return __choose_memory(FRAME(kAutomapStart), count, align, res) &&
-         *res + count <= FRAME(kAutomapStart + (kAutomapSize - 1));
-}
-
-static void *__finish_memory(void *addr, size_t size, int prot, int flags,
-                             int fd, int64_t off, int f, int x, int n,
-                             struct DirectMap dm) {
-  if (!IsWindows() && (flags & MAP_FIXED)) {
-    if (__untrack_memories(addr, size)) {
-      __mmap_die("FIXED UNTRACK FAILED");
-    }
-  }
-  if (__track_memory(&_mmi, x, x + (n - 1), dm.maphandle, prot, flags, false,
-                     false, off, size)) {
-    if (sys_munmap(addr, n) == -1) {
-      __mmap_die("TRACK MUNMAP FAILED");
-    }
-    return MAP_FAILED;
-  }
-  if (_weaken(__asan_map_shadow) && !OverlapsShadowSpace(addr, size)) {
-    _weaken(__asan_map_shadow)((intptr_t)addr, size);
-  }
-  return addr;
-}
-
-static void *__map_memory(void *addr, size_t size, int prot, int flags, int fd,
-                          int64_t off, int f, int x, int n) {
-  struct DirectMap dm;
-  dm = sys_mmap(addr, size, prot, f, fd, off);
-  if (VERY_UNLIKELY(dm.addr == MAP_FAILED)) {
-    if (IsWindows() && (flags & MAP_FIXED)) {
-      __mmap_die("can't recover from MAP_FIXED errors on Windows");
-    }
-    return MAP_FAILED;
-  }
-  if (VERY_UNLIKELY(dm.addr != addr)) {
-    __mmap_die("KERNEL DIDN'T RESPECT MAP_FIXED");
-  }
-  return __finish_memory(addr, size, prot, flags, fd, off, f, x, n, dm);
-}
-
-/**
- * Maps memory from system, one frame at a time.
- *
- * This is useful on Windows since it allows us to partially unmap or
- * punch holes into existing mappings.
- */
-static textwindows dontinline void *__map_memories(char *addr, size_t size,
-                                                   int prot, int flags, int fd,
-                                                   int64_t off, int f, int x,
-                                                   int n) {
-  size_t i, m;
-  int64_t oi, sz;
-  struct DirectMap dm;
-  bool iscow, readonlyfile;
-  m = (size_t)(n - 1) << 16;
-  unassert(m < size);
-  unassert(m + FRAMESIZE >= size);
-  oi = fd == -1 ? 0 : off + m;
-  sz = size - m;
-  dm = sys_mmap(addr + m, sz, prot, f, fd, oi);
-  if (dm.addr == MAP_FAILED)
-    return MAP_FAILED;
-  iscow = (flags & MAP_TYPE) != MAP_SHARED && fd != -1;
-  readonlyfile = (flags & MAP_TYPE) == MAP_SHARED && fd != -1 &&
-                 (g_fds.p[fd].flags & O_ACCMODE) == O_RDONLY;
-  if (__track_memory(&_mmi, x + (n - 1), x + (n - 1), dm.maphandle, prot, flags,
-                     readonlyfile, iscow, oi, sz) == -1) {
-    __mmap_die("__map_memories unrecoverable #1");
-  }
-  for (i = 0; i < m; i += FRAMESIZE) {
-    oi = fd == -1 ? 0 : off + i;
-    sz = FRAMESIZE;
-    dm = sys_mmap(addr + i, sz, prot, f, fd, oi);
-    if (dm.addr == MAP_FAILED ||
-        __track_memory(&_mmi, x + i / FRAMESIZE, x + i / FRAMESIZE,
-                       dm.maphandle, prot, flags, readonlyfile, iscow, oi,
-                       sz) == -1) {
-      __mmap_die("__map_memories unrecoverable #2");
-    }
-  }
-  if (_weaken(__asan_map_shadow) && !OverlapsShadowSpace(addr, size)) {
-    _weaken(__asan_map_shadow)((intptr_t)addr, size);
-  }
-  return addr;
-}
-
-inline void *__mmap_unlocked(void *addr, size_t size, int prot, int flags,
-                             int fd, int64_t off) {
-  char *p = addr;
-  struct DirectMap dm;
-  size_t requested_size;
-  bool needguard, clashes;
-  int a, f, n, x, pagesize;
-  size_t virtualused, virtualneed;
-
-  if (VERY_UNLIKELY(!size)) {
-    STRACE("can't mmap zero bytes");
-    return VIP(einval());
-  }
-
-  if (VERY_UNLIKELY(!ALIGNED(p))) {
-    STRACE("cosmo mmap is 64kb aligned");
-    return VIP(einval());
-  }
-
-  if (VERY_UNLIKELY(!IsLegalSize(size))) {
-    STRACE("mmap size isn't legal");
-    return VIP(einval());
-  }
-
-  if (VERY_UNLIKELY(!IsLegalPointer(p))) {
-    STRACE("mmap addr isn't 48-bit");
-    return VIP(einval());
-  }
-
-  if ((flags & (MAP_SHARED | MAP_PRIVATE)) == (MAP_SHARED | MAP_PRIVATE)) {
-    flags = MAP_SHARED;  // cf. MAP_SHARED_VALIDATE
-  }
-
-  requested_size = size;
-  pagesize = getauxval(AT_PAGESZ);
-  if (flags & MAP_ANONYMOUS) {
-    fd = -1;
-    off = 0;
-    size = ROUNDUP(size, FRAMESIZE);
-    if ((flags & MAP_TYPE) == MAP_FILE) {
-      STRACE("need MAP_PRIVATE or MAP_SHARED");
-      return VIP(einval());
-    }
-  } else if (__isfdkind(fd, kFdZip)) {
-    STRACE("mmap fd is zipos handle");
-    return VIP(einval());
-  } else if (VERY_UNLIKELY(off < 0)) {
-    STRACE("mmap negative offset");
-    return VIP(einval());
-  } else if (off & ((IsWindows() ? FRAMESIZE : pagesize) - 1)) {
-    STRACE("mmap offset isn't properly aligned");
-    return VIP(einval());
-  } else if (VERY_UNLIKELY(INT64_MAX - size < off)) {
-    STRACE("mmap too large");
-    return VIP(einval());
-  }
-
-  if (__virtualmax < LONG_MAX &&
-      (ckd_add(&virtualneed, (virtualused = __get_memtrack_size(&_mmi)),
-               size) ||
-       virtualneed > __virtualmax)) {
-    STRACE("mmap %'zu size + %'zu inuse exceeds virtual memory limit %'zu",
-           size, virtualused, __virtualmax);
-    return VIP(enomem());
-  }
-
-  clashes = OverlapsImageSpace(p, size) || __overlaps_existing_mapping(p, size);
-
-  if ((flags & MAP_FIXED_NOREPLACE) == MAP_FIXED_NOREPLACE && clashes) {
-    STRACE("mmap noreplace overlaps existing");
-    return VIP(eexist());
-  }
-
-  if (ckd_add(&n, (int)(size >> 16), (int)!!(size & (FRAMESIZE - 1)))) {
-    STRACE("mmap range overflows");
-    return VIP(einval());
-  }
-
-  a = MAX(1, __rounddown2pow(size) >> 16);
-  f = (flags & ~MAP_FIXED_NOREPLACE) | MAP_FIXED;
-  if (flags & MAP_FIXED) {
-    x = FRAME(p);
-    if (IsWindows()) {
-      if (__untrack_memories(p, size)) {
-        __mmap_die("FIXED UNTRACK FAILED");
-      }
-    }
-  } else if (p && !clashes && !OverlapsShadowSpace(p, size)) {
-    x = FRAME(p);
-  } else if (!__auto_map(n, a, &x)) {
-    STRACE("automap has no room for %d frames with %d alignment", n, a);
-    return VIP(enomem());
-  }
-
-  needguard = false;
-  p = (char *)ADDR_32_TO_48(x);
-  if ((f & MAP_TYPE) == MAP_STACK) {
-    if (~f & MAP_ANONYMOUS) {
-      STRACE("MAP_STACK must be anonymous");
-      return VIP(einval());
-    }
-    f &= ~MAP_TYPE;
-    f |= MAP_PRIVATE;
-    if (IsOpenbsd()) {  // openbsd:dubstack
-      // on openbsd this is less about scalability of threads, and more
-      // about defining the legal intervals for the RSP register. sadly
-      // openbsd doesn't let us create a new fixed stack mapping. but..
-      // openbsd does allow us to overwrite existing fixed mappings, to
-      // authorize its usage as a stack.
-      if (sys_mmap(p, size, prot, f, fd, off).addr == MAP_FAILED) {
-        return MAP_FAILED;
-      }
-      f |= MAP_STACK_openbsd;
-      needguard = true;
-    } else if (IsLinux()) {
-      // make sure there's no existing stuff existing between our stack
-      // starting page and the bottom guard page, since that would stop
-      // our stack page from growing down.
-      npassert(!sys_munmap(p, size));
-      int guardsize = pagesize, e = errno;
-      if ((dm = sys_mmap(p + size - guardsize, guardsize, prot,
-                         f | MAP_GROWSDOWN_linux, fd, off))
-              .addr != MAP_FAILED) {
-        npassert(sys_mmap(p, pagesize, PROT_NONE,
-                          MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)
-                     .addr == p);
-        dm.addr = p;
-        p = __finish_memory(p, size, prot, flags, fd, off, f, x, n, dm);
-        if (IsAsan() && p != MAP_FAILED) {
-          __asan_poison(p, pagesize, kAsanStackOverflow);
-        }
-        return p;
-      } else if (errno == ENOTSUP) {
-        // WSL and Blink don't support MAP_GROWSDOWN
-        needguard = true;
-        errno = e;
-      }
-    } else {
-      if (IsFreebsd()) {
-        f |= MAP_STACK_freebsd;
-      }
-      needguard = true;
-    }
-  }
-
-  if (!IsWindows()) {
-    p = __map_memory(p, size, prot, flags, fd, off, f, x, n);
-  } else {
-    p = __map_memories(p, size, prot, flags, fd, off, f, x, n);
-  }
-
-  if (p != MAP_FAILED) {
-    if (IsAsan()) {
-      __asan_poison(p + requested_size, size - requested_size,
-                    kAsanMmapSizeOverrun);
-    }
-    if (needguard) {
-      if (!IsWindows()) {
-        unassert(!mprotect(p, pagesize, PROT_NONE));
-        if (IsAsan()) {
-          __asan_poison(p, pagesize, kAsanStackOverflow);
-        }
-      } else {
-        uint32_t oldattr;
-        unassert(VirtualProtect(p, pagesize, kNtPageReadwrite | kNtPageGuard,
-                                &oldattr));
-      }
-    }
-  }
-
-  return p;
-}
-
-/**
- * Creates virtual memory, e.g.
- *
- *     char *m;
- *     m = mmap(NULL, FRAMESIZE, PROT_READ | PROT_WRITE,
- *              MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
- *     munmap(m, FRAMESIZE);
- *
- * @param addr should be 0 to let your memory manager choose address;
- *     unless MAP_FIXED or MAP_FIXED_NOREPLACE are specified in flags
- *     in which case this function will do precicely as you ask, even
- *     if p=0 (in which you need -fno-delete-null-pointer-checks); it
- *     needs to be 64kb aligned because it's a wise choice that sadly
- *     needs to be made mandatory because of Windows although you can
- *     use __sys_mmap() to circumvent it on System Five in which case
- *     runtime support services, e.g. asan memory safety, could break
- * @param size must be >0 otherwise EINVAL is raised
- * @param prot can have PROT_READ/PROT_WRITE/PROT_EXEC/PROT_NONE/etc.
- * @param flags should have one of the following masked by `MAP_TYPE`
- *     - `MAP_FILE` in which case `MAP_ANONYMOUS` shouldn't be used
- *     - `MAP_PRIVATE` for copy-on-write behavior of writeable pages
- *     - `MAP_SHARED` to create shared memory between processes
- *     - `MAP_STACK` to create a grows-down alloc, where a guard page
- *       is automatically protected at the bottom, sized as AT_PAGESZ
- *     Your `flags` may optionally bitwise or any of the following:
- *     - `MAP_ANONYMOUS` in which case `fd` and `off` are ignored
- *     - `MAP_FIXED` in which case `addr` becomes more than a hint
- *     - `MAP_FIXED_NOREPLACE` to protect existing mappings; this is
- *       always polyfilled by mmap() which tracks its own memory and
- *       removed before passing to the kernel, in order to support
- *       old versions; if you believe mappings exist which only the
- *       kernel knows, then this flag may be passed to sys_mmap() on
- *       Linux 4.17+ and FreeBSD (where it has multiple bits)
- *     - `MAP_CONCEAL` is FreeBSD/NetBSD/OpenBSD-only
- *     - `MAP_NORESERVE` is Linux/XNU/NetBSD-only
- *     - `MAP_POPULATE` is Linux/FreeBSD-only
- *     - `MAP_NONBLOCK` is Linux-only
- *     - `MAP_NOSYNC` is FreeBSD-only
- *     - `MAP_INHERIT` is NetBSD-only
- *     - `MAP_LOCKED` is Linux-only
- * @param fd is an open()'d file descriptor, whose contents shall be
- *     made available w/ automatic reading at the chosen address
- * @param off specifies absolute byte index of fd's file for mapping,
- *     should be zero if MAP_ANONYMOUS is specified, which SHOULD be
- *     aligned to FRAMESIZE
- * @return virtual base address of new mapping, or MAP_FAILED w/ errno
- */
-void *mmap(void *addr, size_t size, int prot, int flags, int fd, int64_t off) {
-  void *res;
-#if SYSDEBUG
-  size_t toto = 0;
-#if _KERNTRACE || _NTTRACE
-  if (IsWindows()) {
-    STRACE("mmap(%p, %'zu, %s, %s, %d, %'ld) → ...", addr, size,
-           DescribeProtFlags(prot), DescribeMapFlags(flags), fd, off);
-  }
-#endif
-#endif
-  __mmi_lock();
-  if (!__isfdkind(fd, kFdZip)) {
-    res = __mmap_unlocked(addr, size, prot, flags, fd, off);
-  } else {
-    res = _weaken(__zipos_mmap)(
-        addr, size, prot, flags,
-        (struct ZiposHandle *)(intptr_t)g_fds.p[fd].handle, off);
-  }
-#if SYSDEBUG
-  toto = __strace > 0 ? __get_memtrack_size(&_mmi) : 0;
-#endif
-  __mmi_unlock();
-#if SYSDEBUG
-  STRACE("mmap(%p, %'zu, %s, %s, %d, %'ld) → %p% m (%'zu bytes total)", addr,
-         size, DescribeProtFlags(prot), DescribeMapFlags(flags), fd, off, res,
-         toto);
-#endif
-  return res;
-}
-
-__strong_reference(mmap, mmap64);

libc/runtime/mprotect-nt.greg.c

@@ -1,69 +0,0 @@
-/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
-│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
-╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
-│                                                                              │
-│ Permission to use, copy, modify, and/or distribute this software for         │
-│ any purpose with or without fee is hereby granted, provided that the         │
-│ above copyright notice and this permission notice appear in all copies.      │
-│                                                                              │
-│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
-│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
-│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
-│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
-│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
-│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
-│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
-│ PERFORMANCE OF THIS SOFTWARE.                                                │
-╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/calls/struct/sigset.internal.h"
-#include "libc/intrin/directmap.internal.h"
-#include "libc/nt/memory.h"
-#include "libc/runtime/internal.h"
-#include "libc/runtime/memtrack.internal.h"
-
-textwindows int sys_mprotect_nt(void *addr, size_t size, int prot) {
-  int rc = 0;
-  unsigned i;
-  uint32_t op;
-  char *a, *b, *x, *y, *p;
-  BLOCK_SIGNALS;
-  __mmi_lock();
-  size = (size + 4095) & -4096;
-  p = addr;
-  i = __find_memory(&_mmi, (intptr_t)p >> 16);
-  if (i == _mmi.i || (!i && p + size <= (char *)ADDR_32_TO_48(_mmi.p[0].x))) {
-    // memory isn't in memtrack
-    // let's just trust the user then
-    // it's probably part of the executable
-    if (!VirtualProtect(addr, size, __prot2nt(prot, false), &op)) {
-      rc = -1;
-    }
-  } else {
-    // memory is in memtrack, so use memtrack, to do dimensioning
-    // we unfortunately must do something similar to this for cow
-    for (; i < _mmi.i; ++i) {
-      x = (char *)ADDR_32_TO_48(_mmi.p[i].x);
-      y = (char *)ADDR_32_TO_48(_mmi.p[i].y) + 65536;
-      if ((x <= p && p < y) || (x < p + size && p + size <= y) ||
-          (p < x && y < p + size)) {
-        if (p <= x && p + size >= y) {
-          _mmi.p[i].prot = prot;
-        } else {
-          _mmi.p[i].prot |= prot;
-        }
-        a = MIN(MAX(p, x), y);
-        b = MAX(MIN(p + size, y), x);
-        if (!VirtualProtect(a, b - a, __prot2nt(prot, _mmi.p[i].iscow), &op)) {
-          rc = -1;
-          break;
-        }
-      } else {
-        break;
-      }
-    }
-  }
-  __mmi_unlock();
-  ALLOW_SIGNALS;
-  return rc;
-}

libc/runtime/mprotect.c

@@ -1,56 +0,0 @@
-/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
-│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
-╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2020 Justine Alexandra Roberts Tunney                              │
-│                                                                              │
-│ Permission to use, copy, modify, and/or distribute this software for         │
-│ any purpose with or without fee is hereby granted, provided that the         │
-│ above copyright notice and this permission notice appear in all copies.      │
-│                                                                              │
-│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
-│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
-│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
-│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
-│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
-│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
-│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
-│ PERFORMANCE OF THIS SOFTWARE.                                                │
-╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/calls/syscall-sysv.internal.h"
-#include "libc/dce.h"
-#include "libc/errno.h"
-#include "libc/intrin/describeflags.internal.h"
-#include "libc/intrin/likely.h"
-#include "libc/intrin/strace.internal.h"
-#include "libc/runtime/internal.h"
-#include "libc/runtime/runtime.h"
-#include "libc/sysv/consts/auxv.h"
-#include "libc/sysv/consts/prot.h"
-#include "libc/sysv/errfuns.h"
-
-/**
- * Modifies restrictions on virtual memory address range.
- *
- * @param addr needs to be 4kb aligned
- * @param prot can have PROT_{NONE,READ,WRITE,EXEC,GROWSDOWN,GROWSUP}
- * @return 0 on success, or -1 w/ errno
- * @see mmap()
- */
-int mprotect(void *addr, size_t size, int prot) {
-  int64_t rc;
-  if (prot &
-      ~(PROT_READ | PROT_WRITE | PROT_EXEC | PROT_GROWSDOWN | PROT_GROWSUP)) {
-    rc = einval();  // unix checks prot before checking size
-  } else if (!size) {
-    return 0;  // make new technology consistent with unix
-  } else if (UNLIKELY((intptr_t)addr & (getauxval(AT_PAGESZ) - 1))) {
-    rc = einval();  // unix checks prot before checking size
-  } else if (!IsWindows()) {
-    rc = sys_mprotect(addr, size, prot);
-  } else {
-    rc = sys_mprotect_nt(addr, size, prot);
-  }
-  STRACE("mprotect(%p, %'zu, %s) → %d% m", addr, size, DescribeProtFlags(prot),
-         rc);
-  return rc;
-}

libc/runtime/msync-nt.c

@@ -1,50 +0,0 @@
-/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
-│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
-╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2020 Justine Alexandra Roberts Tunney                              │
-│                                                                              │
-│ Permission to use, copy, modify, and/or distribute this software for         │
-│ any purpose with or without fee is hereby granted, provided that the         │
-│ above copyright notice and this permission notice appear in all copies.      │
-│                                                                              │
-│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
-│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
-│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
-│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
-│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
-│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
-│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
-│ PERFORMANCE OF THIS SOFTWARE.                                                │
-╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/macros.internal.h"
-#include "libc/nt/files.h"
-#include "libc/nt/memory.h"
-#include "libc/runtime/memtrack.internal.h"
-#include "libc/sysv/consts/map.h"
-#include "libc/sysv/consts/msync.h"
-#include "libc/sysv/consts/o.h"
-#include "libc/sysv/consts/prot.h"
-
-textwindows int sys_msync_nt(char *addr, size_t size, int flags) {
-  int i, rc = 0;
-  char *a, *b, *x, *y;
-  __mmi_lock();
-  for (i = __find_memory(&_mmi, (intptr_t)addr >> 16); i < _mmi.i; ++i) {
-    x = (char *)ADDR_32_TO_48(_mmi.p[i].x);
-    y = x + _mmi.p[i].size;
-    if ((x <= addr && addr < y) || (x < addr + size && addr + size <= y) ||
-        (addr < x && y < addr + size)) {
-      a = MIN(MAX(addr, x), y);
-      b = MAX(MIN(addr + size, y), x);
-      if (!FlushViewOfFile(a, b - a)) {
-        rc = -1;
-        break;
-      }
-      // TODO(jart): FlushFileBuffers too on g_fds handle if MS_SYNC?
-    } else {
-      break;
-    }
-  }
-  __mmi_unlock();
-  return rc;
-}

libc/runtime/munmap.c

@@ -1,160 +0,0 @@
-/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
-│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
-╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2020 Justine Alexandra Roberts Tunney                              │
-│                                                                              │
-│ Permission to use, copy, modify, and/or distribute this software for         │
-│ any purpose with or without fee is hereby granted, provided that the         │
-│ above copyright notice and this permission notice appear in all copies.      │
-│                                                                              │
-│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
-│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
-│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
-│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
-│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
-│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
-│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
-│ PERFORMANCE OF THIS SOFTWARE.                                                │
-╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/assert.h"
-#include "libc/calls/state.internal.h"
-#include "libc/calls/syscall-sysv.internal.h"
-#include "libc/dce.h"
-#include "libc/errno.h"
-#include "libc/intrin/asan.internal.h"
-#include "libc/intrin/directmap.internal.h"
-#include "libc/intrin/likely.h"
-#include "libc/intrin/strace.internal.h"
-#include "libc/log/backtrace.internal.h"
-#include "libc/log/libfatal.internal.h"
-#include "libc/log/log.h"
-#include "libc/macros.internal.h"
-#include "libc/runtime/internal.h"
-#include "libc/runtime/memtrack.internal.h"
-#include "libc/runtime/runtime.h"
-#include "libc/str/str.h"
-#include "libc/sysv/errfuns.h"
-
-#define IP(X)      (intptr_t)(X)
-#define ALIGNED(p) (!(IP(p) & (FRAMESIZE - 1)))
-#define FRAME(x)   ((int)((intptr_t)(x) >> 16))
-
-static void __munmap_shadow(char *p, size_t n) {
-  intptr_t a, b, x, y;
-  KERNTRACE("__munmap_shadow(%p, %'zu)", p, n);
-  a = ((intptr_t)p >> 3) + 0x7fff8000;
-  b = a + (n >> 3);
-  if (IsMemtracked(FRAME(a), FRAME(b - 1))) {
-    x = ROUNDUP(a, FRAMESIZE);
-    y = ROUNDDOWN(b, FRAMESIZE);
-    if (0 && x < y) {
-      // delete shadowspace if unmapping ≥512kb. in practice it has
-      // to be >1mb since we can only unmap it if it's aligned, and
-      // as such we poison the edges if there are any.
-      __repstosb((void *)a, kAsanUnmapped, x - a);
-      __munmap_unlocked((void *)x, y - x);
-      __repstosb((void *)y, kAsanUnmapped, b - y);
-    } else {
-      // otherwise just poison and assume reuse
-      __repstosb((void *)a, kAsanUnmapped, b - a);
-    }
-  } else {
-    STRACE("unshadow(%.12p, %p) EFAULT", a, b - a);
-  }
-}
-
-// our api supports doing things like munmap(0, 0x7fffffffffff) but some
-// platforms (e.g. openbsd) require that we know the specific intervals
-// or else it returns EINVAL. so we munmap a piecewise.
-static void __munmap_impl(char *p, size_t n) {
-  char *q;
-  size_t m;
-  intptr_t a, b, c;
-  int i, l, r, beg, end;
-  KERNTRACE("__munmap_impl(%p, %'zu)", p, n);
-  l = FRAME(p);
-  r = FRAME(p + n - 1);
-  i = __find_memory(&_mmi, l);
-  for (; i < _mmi.i && r >= _mmi.p[i].x; ++i) {
-    if (l >= _mmi.p[i].x && r <= _mmi.p[i].y) {
-
-      // it's contained within the entry
-      beg = l;
-      end = r;
-    } else if (l <= _mmi.p[i].x && r >= _mmi.p[i].x) {
-
-      // it overlaps with the lefthand side of the entry
-      beg = _mmi.p[i].x;
-      end = MIN(r, _mmi.p[i].y);
-    } else if (l <= _mmi.p[i].y && r >= _mmi.p[i].y) {
-
-      // it overlaps with the righthand side of the entry
-      beg = MAX(_mmi.p[i].x, l);
-      end = _mmi.p[i].y;
-    } else {
-      __builtin_unreachable();
-    }
-    // openbsd even requires that if we mapped, for instance a 5 byte
-    // file, that we be sure to call munmap(file, 5). let's abstract!
-    a = ADDR_32_TO_48(beg);
-    b = ADDR_32_TO_48(end) + FRAMESIZE;
-    c = ADDR_32_TO_48(_mmi.p[i].x) + _mmi.p[i].size;
-    q = (char *)a;
-    m = MIN(b, c) - a;
-    if (!IsWindows()) {
-      npassert(!sys_munmap(q, m));
-    } else {
-      // Handled by __untrack_memories() on Windows
-    }
-    if (IsAsan() && !OverlapsShadowSpace(p, n)) {
-      __munmap_shadow(q, m);
-    }
-  }
-}
-
-int __munmap_unlocked(char *p, size_t n) {
-  unassert(!__vforked);
-  if (UNLIKELY(!n)) {
-    STRACE("munmap n is 0");
-    return einval();
-  }
-  if (UNLIKELY(!IsLegalSize(n))) {
-    STRACE("munmap n isn't 48-bit");
-    return einval();
-  }
-  if (UNLIKELY(!IsLegalPointer(p))) {
-    STRACE("munmap p isn't 48-bit");
-    return einval();
-  }
-  if (UNLIKELY(!IsLegalPointer(p + (n - 1)))) {
-    STRACE("munmap p+(n-1) isn't 48-bit");
-    return einval();
-  }
-  if (UNLIKELY(!ALIGNED(p))) {
-    STRACE("munmap(%p) isn't 64kb aligned", p);
-    return einval();
-  }
-  __munmap_impl(p, n);
-  return __untrack_memories(p, n);
-}
-
-/**
- * Releases memory pages.
- *
- * @param p is the beginning of the memory region to unmap
- * @param n is the number of bytes to be unmapped
- * @return 0 on success, or -1 w/ errno
- * @raises EINVAL if `n == 0`
- * @raises EINVAL if `n` isn't 48-bit
- * @raises EINVAL if `p+(n-1)` isn't 48-bit
- * @raises EINVAL if `p` isn't 65536-byte aligned
- */
-int munmap(void *p, size_t n) {
-  int rc;
-  __mmi_lock();
-  rc = __munmap_unlocked(p, n);
-  size_t toto = __strace > 0 ? __get_memtrack_size(&_mmi) : 0;
-  __mmi_unlock();
-  STRACE("munmap(%.12p, %'zu) → %d% m (%'zu bytes total)", p, n, rc, toto);
-  return rc;
-}

libc/runtime/opensymboltable.greg.c

@@ -31,6 +31,7 @@
 #include "libc/runtime/runtime.h"
 #include "libc/runtime/symbols.internal.h"
 #include "libc/str/str.h"
+#include "libc/sysv/consts/auxv.h"
 #include "libc/sysv/consts/map.h"
 #include "libc/sysv/consts/o.h"
 #include "libc/sysv/consts/prot.h"
@@ -48,6 +49,7 @@ static struct SymbolTable *OpenSymbolTableImpl(const char *filename) {
   size_t n, m, tsz, size;
   const Elf64_Sym *symtab, *sym;
   ptrdiff_t names_offset, name_base_offset, stp_offset;
+  long pagesz = getauxval(AT_PAGESZ);
   map = MAP_FAILED;
   if ((fd = open(filename, O_RDONLY | O_CLOEXEC)) == -1)
     return 0;
@@ -73,11 +75,11 @@ static struct SymbolTable *OpenSymbolTableImpl(const char *filename) {
   tsz += sizeof(unsigned) * n;
   name_base_offset = tsz;
   tsz += m;
-  tsz = ROUNDUP(tsz, FRAMESIZE);
+  tsz = ROUNDUP(tsz, pagesz);
   stp_offset = tsz;
   size = tsz;
   tsz += sizeof(const Elf64_Sym *) * n;
-  tsz = ROUNDUP(tsz, FRAMESIZE);
+  tsz = ROUNDUP(tsz, pagesz);
   t = mmap(0, tsz, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
   if (t == MAP_FAILED)
     goto SystemError;
@@ -128,7 +130,7 @@ static struct SymbolTable *OpenSymbolTableImpl(const char *filename) {
     ++j;
   }
   t->count = j;
-  munmap(stp, ROUNDUP(sizeof(const Elf64_Sym *) * n, FRAMESIZE));
+  munmap(stp, sizeof(const Elf64_Sym *) * n);
   munmap(map, filesize);
   close(fd);
   return t;

libc/runtime/runtime.h

@@ -19,9 +19,9 @@ typedef unsigned long jmp_buf[18];
 typedef unsigned long jmp_buf[26];
 #endif
 
-void mcount(void);
+void mcount(void) libcesque;
-int daemon(int, int);
+int daemon(int, int) libcesque;
-unsigned long getauxval(unsigned long);
+unsigned long getauxval(unsigned long) libcesque;
 int setjmp(jmp_buf)
 libcesque returnstwice paramsnonnull();
 void longjmp(jmp_buf, int) libcesque wontreturn paramsnonnull();
@@ -37,28 +37,28 @@ void quick_exit(int) wontreturn;
 void abort(void) wontreturn;
 int atexit(void (*)(void)) paramsnonnull() libcesque;
 char *getenv(const char *) paramsnonnull() __wur nosideeffect libcesque;
-int putenv(char *);
+int putenv(char *) libcesque;
-int setenv(const char *, const char *, int);
+int setenv(const char *, const char *, int) libcesque;
-int unsetenv(const char *);
+int unsetenv(const char *) libcesque;
-int clearenv(void);
+int clearenv(void) libcesque;
-void fpreset(void);
+void fpreset(void) libcesque;
-void *mmap(void *, uint64_t, int32_t, int32_t, int32_t, int64_t);
+void *mmap(void *, uint64_t, int32_t, int32_t, int32_t, int64_t) libcesque;
-int munmap(void *, uint64_t);
+int munmap(void *, uint64_t) libcesque;
-int mprotect(void *, uint64_t, int);
+int mprotect(void *, uint64_t, int) libcesque;
-int msync(void *, size_t, int);
+int msync(void *, size_t, int) libcesque;
-int mlock(const void *, size_t);
+int mlock(const void *, size_t) libcesque;
-int munlock(const void *, size_t);
+int munlock(const void *, size_t) libcesque;
-long gethostid(void);
+long gethostid(void) libcesque;
-int sethostid(long);
+int sethostid(long) libcesque;
-char *getlogin(void);
+char *getlogin(void) libcesque;
-int getlogin_r(char *, size_t);
+int getlogin_r(char *, size_t) libcesque;
-int login_tty(int);
+int login_tty(int) libcesque;
-int getpagesize(void);
+int getpagesize(void) libcesque;
-int syncfs(int) dontthrow;
+int syncfs(int) dontthrow libcesque;
-int vhangup(void);
+int vhangup(void) libcesque;
-int getdtablesize(void);
+int getdtablesize(void) libcesque;
-int sethostname(const char *, size_t);
+int sethostname(const char *, size_t) libcesque;
-int acct(const char *);
+int acct(const char *) libcesque;
 
 #if defined(_GNU_SOURCE) || defined(_COSMO_SOURCE)
 extern char **environ;
@@ -80,62 +80,62 @@ extern uint64_t kStartTsc;
 extern const char kNtSystemDirectory[];
 extern const char kNtWindowsDirectory[];
 extern size_t __virtualmax;
+extern size_t __virtualsize;
 extern size_t __stackmax;
 extern bool32 __isworker;
 /* utilities */
-void _intsort(int *, size_t);
+void _intsort(int *, size_t) libcesque;
-void _longsort(long *, size_t);
+void _longsort(long *, size_t) libcesque;
 /* diagnostics */
-void ShowCrashReports(void);
+void ShowCrashReports(void) libcesque;
-int ftrace_install(void);
+int ftrace_install(void) libcesque;
-int ftrace_enabled(int);
+int ftrace_enabled(int) libcesque;
-int strace_enabled(int);
+int strace_enabled(int) libcesque;
-void __print_maps(void);
+void __print_maps(void) libcesque;
-void __printargs(const char *);
+void __printargs(const char *) libcesque;
 /* builtin sh-like system/popen dsl */
-int _cocmd(int, char **, char **);
+int _cocmd(int, char **, char **) libcesque;
 /* executable program */
-char *GetProgramExecutableName(void);
+char *GetProgramExecutableName(void) libcesque;
-char *GetInterpreterExecutableName(char *, size_t);
+char *GetInterpreterExecutableName(char *, size_t) libcesque;
-int __open_executable(void);
+int __open_executable(void) libcesque;
 /* execution control */
-int verynice(void);
+int verynice(void) libcesque;
-void __warn_if_powersave(void);
+void __warn_if_powersave(void) libcesque;
-void _Exit1(int) libcesque wontreturn;
+void _Exit1(int) libcesque wontreturn libcesque;
-void __paginate(int, const char *);
+void __paginate(int, const char *) libcesque;
-void __paginate_file(int, const char *);
+void __paginate_file(int, const char *) libcesque;
 /* memory management */
-void _weakfree(void *);
+void _weakfree(void *) libcesque;
-void *_mapanon(size_t) attributeallocsize((1)) mallocesque;
+void *_mapanon(size_t) attributeallocsize((1)) mallocesque libcesque;
-void *_mapshared(size_t) attributeallocsize((1)) mallocesque;
+void *_mapshared(size_t) attributeallocsize((1)) mallocesque libcesque;
-void CheckForMemoryLeaks(void);
+void CheckForMemoryLeaks(void) libcesque;
-void CheckForFileLeaks(void);
+void CheckForFileLeaks(void) libcesque;
-bool32 _isheap(const void *);
+void __enable_threads(void) libcesque;
-void __enable_threads(void);
+void __oom_hook(size_t) libcesque;
-void __oom_hook(size_t);
 /* code morphing */
-void __morph_begin(void);
+void __morph_begin(void) libcesque;
-void __morph_end(void);
+void __morph_end(void) libcesque;
-void __jit_begin(void);
+void __jit_begin(void) libcesque;
-void __jit_end(void);
+void __jit_end(void) libcesque;
-void __clear_cache(void *, void *);
+void __clear_cache(void *, void *) libcesque;
 /* portability */
-bool32 IsGenuineBlink(void);
+bool32 IsGenuineBlink(void) libcesque;
-bool32 IsCygwin(void);
+bool32 IsCygwin(void) libcesque;
-const char *GetCpuidOs(void);
+const char *GetCpuidOs(void) libcesque;
-const char *GetCpuidEmulator(void);
+const char *GetCpuidEmulator(void) libcesque;
-void GetCpuidBrand(char[13], uint32_t);
+void GetCpuidBrand(char[13], uint32_t) libcesque;
-long __get_rlimit(int);
+long __get_rlimit(int) libcesque;
-const char *__describe_os(void);
+const char *__describe_os(void) libcesque;
-long __get_sysctl(int, int);
+long __get_sysctl(int, int) libcesque;
-int __get_arg_max(void) pureconst;
+int __granularity(void) pureconst libcesque;
-int __get_cpu_count(void) pureconst;
+int __get_arg_max(void) pureconst libcesque;
-long __get_avphys_pages(void) pureconst;
+int __get_cpu_count(void) pureconst libcesque;
-long __get_phys_pages(void) pureconst;
+long __get_avphys_pages(void) pureconst libcesque;
-long __get_minsigstksz(void) pureconst;
+long __get_phys_pages(void) pureconst libcesque;
-void __get_main_stack(void **, size_t *, int *);
+long __get_minsigstksz(void) pureconst libcesque;
-long __get_safe_size(long, long);
+long __get_safe_size(long, long) libcesque;
-char *__get_tmpdir(void);
+char *__get_tmpdir(void) libcesque;
 forceinline int __trace_disabled(int x) {
   return 0;
 }

libc/runtime/stack.h

@@ -14,17 +14,6 @@
  */
 #define GetGuardSize() 16384
 
-/**
- * Align APE main thread stack at startup.
- *
- * You need this in your main program module:
- *
- *     STATIC_STACK_ALIGN(GetStackSize());
- *
- * If you want to use GetStackAddr() and HaveStackMemory() safely on
- * your main thread in your process. It causes crt.S to waste a tiny
- * amount of memory to ensure those macros go extremely fast.
- */
 #define STATIC_STACK_ALIGN(BYTES) \
   _STACK_SYMBOL("ape_stack_align", _STACK_STRINGIFY(BYTES) _STACK_EXTRA)
 
@@ -62,29 +51,12 @@ COSMOPOLITAN_C_START_
 extern char ape_stack_prot[] __attribute__((__weak__));
 extern char ape_stack_memsz[] __attribute__((__weak__));
 extern char ape_stack_align[] __attribute__((__weak__));
+extern char ape_stack_round[] __attribute__((__weak__));
 
-/**
+uintptr_t GetStackBottom(void) pureconst;
- * Returns address of bottom of current stack.
- *
- * This always works on threads. If you want it to work on the main
- * process too, then you'll need STATIC_STACK_ALIGN(GetStackSize())
- * which will burn O(256kb) of memory to ensure thread invariants.
- */
-#define GetStackAddr() ((GetStackPointer() - 1) & -GetStackSize())
 
 #define GetStaticStackSize() ((uintptr_t)ape_stack_memsz)
 
-/**
- * Returns true if at least `n` bytes of stack are available.
- *
- * This always works on threads. If you want it to work on the main
- * process too, then you'll need STATIC_STACK_ALIGN(GetStackSize())
- * which will burn O(256kb) of memory to ensure thread invariants,
- * which make this check exceedingly fast.
- */
-#define HaveStackMemory(n) \
-  (GetStackPointer() >= GetStackAddr() + GetGuardSize() + (n))
-
 /**
  * Extends stack memory by poking large allocations.
  *
@@ -137,8 +109,6 @@ int FreeCosmoStack(void *) libcesque;
             : "i"(ADDEND));                 \
     vAddr;                                  \
   })
-#else
-#define GetStaticStackAddr(ADDEND) (GetStackAddr() + ADDEND)
 #endif
 
 #define GetStackPointer()           \

libc/runtime/sysconf.c

@@ -25,9 +25,11 @@
 #include "libc/limits.h"
 #include "libc/macros.internal.h"
 #include "libc/runtime/clktck.h"
+#include "libc/intrin/maps.h"
 #include "libc/runtime/runtime.h"
 #include "libc/runtime/sysconf.h"
 #include "libc/sysv/consts/_posix.h"
+#include "libc/sysv/consts/auxv.h"
 #include "libc/sysv/consts/limits.h"
 #include "libc/sysv/consts/rlimit.h"
 #include "libc/sysv/consts/ss.h"
@@ -58,7 +60,7 @@ long sysconf(int name) {
     case _SC_CLK_TCK:
       return CLK_TCK;
     case _SC_PAGESIZE:
-      return FRAMESIZE;
+      return __granularity();
     case _SC_ARG_MAX:
       return __get_arg_max();
     case _SC_SIGSTKSZ:

libc/runtime/untrackmemoryintervals.c

@@ -1,31 +0,0 @@
-/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
-│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
-╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2020 Justine Alexandra Roberts Tunney                              │
-│                                                                              │
-│ Permission to use, copy, modify, and/or distribute this software for         │
-│ any purpose with or without fee is hereby granted, provided that the         │
-│ above copyright notice and this permission notice appear in all copies.      │
-│                                                                              │
-│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
-│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
-│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
-│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
-│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
-│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
-│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
-│ PERFORMANCE OF THIS SOFTWARE.                                                │
-╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/assert.h"
-#include "libc/dce.h"
-#include "libc/macros.internal.h"
-#include "libc/runtime/memtrack.internal.h"
-
-int __untrack_memories(void *addr, size_t size) {
-  int a, b;
-  unassert(size > 0);
-  a = ROUNDDOWN((intptr_t)addr, FRAMESIZE) >> 16;
-  b = ROUNDDOWN((intptr_t)addr + size - 1, FRAMESIZE) >> 16;
-  return __untrack_memory(&_mmi, a, b,
-                          SupportsWindows() ? __release_memory_nt : 0);
-}

libc/runtime/winmain.greg.c

@@ -20,6 +20,8 @@
 #include "libc/calls/internal.h"
 #include "libc/calls/sig.internal.h"
 #include "libc/calls/syscall_support-nt.internal.h"
+#include "libc/intrin/dll.h"
+#include "libc/intrin/maps.h"
 #include "libc/intrin/nomultics.internal.h"
 #include "libc/intrin/weaken.h"
 #include "libc/limits.h"
@@ -192,28 +194,26 @@ static abi wontreturn void WinInit(const char16_t *cmdline) {
   __imp_AddVectoredExceptionHandler(true, (void *)OnWinCrash);
 
   // allocate memory for stack and argument block
-  _mmi.p = _mmi.s;
+  char *stackaddr = (char *)GetStaticStackAddr(0);
-  _mmi.n = ARRAYLEN(_mmi.s);
-  uintptr_t stackaddr = GetStaticStackAddr(0);
   size_t stacksize = GetStaticStackSize();
   __imp_MapViewOfFileEx(
-      (_mmi.p[0].h = __imp_CreateFileMappingW(
+      (__maps.stack.h = __imp_CreateFileMappingW(
            -1, 0, kNtPageExecuteReadwrite, stacksize >> 32, stacksize, NULL)),
-      kNtFileMapWrite | kNtFileMapExecute, 0, 0, stacksize, (void *)stackaddr);
+      kNtFileMapWrite | kNtFileMapExecute, 0, 0, stacksize, stackaddr);
   int prot = (intptr_t)ape_stack_prot;
   if (~prot & PROT_EXEC) {
     uint32_t old;
-    __imp_VirtualProtect((void *)stackaddr, stacksize, kNtPageReadwrite, &old);
+    __imp_VirtualProtect(stackaddr, stacksize, kNtPageReadwrite, &old);
   }
   uint32_t oldattr;
-  __imp_VirtualProtect((void *)stackaddr, GetGuardSize(),
+  __imp_VirtualProtect(stackaddr, GetGuardSize(),
                        kNtPageReadwrite | kNtPageGuard, &oldattr);
-  _mmi.p[0].x = stackaddr >> 16;
+  __maps.stack.addr = stackaddr;
-  _mmi.p[0].y = (stackaddr >> 16) + ((stacksize - 1) >> 16);
+  __maps.stack.size = stacksize;
-  _mmi.p[0].prot = prot;
+  __maps.stack.prot = prot;
-  _mmi.p[0].flags = 0x00000026;  // stack+anonymous
+  __maps.maps = &__maps.stack;
-  _mmi.p[0].size = stacksize;
+  dll_init(&__maps.stack.elem);
-  _mmi.i = 1;
+  dll_make_first(&__maps.used, &__maps.stack.elem);
   struct WinArgs *wa =
       (struct WinArgs *)(stackaddr + (stacksize - sizeof(struct WinArgs)));
 
@@ -300,7 +300,7 @@ static abi wontreturn void WinInit(const char16_t *cmdline) {
 
   // handover control to cosmopolitan runtime
   __stack_call(count, wa->argv, wa->envp, wa->auxv, cosmo,
-               stackaddr + (stacksize - sizeof(struct WinArgs)));
+               (uintptr_t)(stackaddr + (stacksize - sizeof(struct WinArgs))));
 }
 
 abi int64_t WinMain(int64_t hInstance, int64_t hPrevInstance,

libc/runtime/zipos-get.c

@@ -62,7 +62,7 @@ static void __zipos_dismiss(uint8_t *map, const uint8_t *cdir, long pg) {
   }
 
   // unmap the executable portion beneath the local files
-  mo = ROUNDDOWN(lo, FRAMESIZE);
+  mo = ROUNDDOWN(lo, __granularity());
   if (mo)
     munmap(map, mo);
 

libc/runtime/zipos-mmap.c

@@ -20,6 +20,7 @@
 #include "libc/calls/struct/iovec.h"
 #include "libc/dce.h"
 #include "libc/errno.h"
+#include "libc/intrin/maps.h"
 #include "libc/intrin/strace.internal.h"
 #include "libc/runtime/internal.h"
 #include "libc/runtime/runtime.h"
@@ -37,7 +38,7 @@
  * Map zipos file into memory. See mmap.
  *
  * @param addr should be 0 or a compatible address
- * @param size must be >0 and will be rounded up to FRAMESIZE
+ * @param size must be >0 and will be rounded up to granularity
  *     automatically.
  * @param prot can have PROT_READ/PROT_WRITE/PROT_EXEC/PROT_NONE/etc.
  * @param flags cannot have `MAP_SHARED` or `MAP_ANONYMOUS`, there is
@@ -76,7 +77,7 @@ void *__zipos_mmap(void *addr, size_t size, int prot, int flags,
   flags |= MAP_PRIVATE | MAP_ANONYMOUS;
 
   const int tempProt = !IsXnu() ? prot | PROT_WRITE : PROT_WRITE;
-  void *outAddr = __mmap_unlocked(addr, size, tempProt, flags, -1, 0);
+  void *outAddr = __mmap(addr, size, tempProt, flags, -1, 0);
   if (outAddr == MAP_FAILED) {
     return MAP_FAILED;
   }
@@ -96,7 +97,7 @@ void *__zipos_mmap(void *addr, size_t size, int prot, int flags,
   } while (0);
 
   const int e = errno;
-  __munmap_unlocked(outAddr, size);
+  munmap(outAddr, size);
   errno = e;
   strace_enabled(+1);
   return MAP_FAILED;

libc/runtime/zipos-stat-impl.c

@@ -20,6 +20,7 @@
 #include "libc/calls/struct/stat.h"
 #include "libc/intrin/atomic.h"
 #include "libc/intrin/safemacros.internal.h"
+#include "libc/runtime/runtime.h"
 #include "libc/runtime/zipos.internal.h"
 #include "libc/str/str.h"
 #include "libc/sysv/consts/s.h"
@@ -30,7 +31,7 @@ int __zipos_stat_impl(struct Zipos *zipos, size_t cf, struct stat *st) {
   bzero(st, sizeof(*st));
   st->st_nlink = 1;
   st->st_dev = zipos->dev;
-  st->st_blksize = FRAMESIZE;
+  st->st_blksize = __granularity();
   if (cf == ZIPOS_SYNTHETIC_DIRECTORY) {
     st->st_mode = S_IFDIR | (0555 & ~atomic_load_explicit(
                                         &__umask, memory_order_acquire));

libc/stdio/printargs.c

@@ -464,11 +464,6 @@ textstartup void __printargs(const char *prologue) {
   PRINT(" ☼ %s = %#s", "GetProgramExecutableName", GetProgramExecutableName());
   PRINT(" ☼ %s = %#s", "GetInterpreterExecutableName",
         GetInterpreterExecutableName(u.path, sizeof(u.path)));
-  PRINT(" ☼ %s = %p", "GetStackSize()", GetStackSize());
-  PRINT(" ☼ %s = %p", "GetGuardSize()", GetGuardSize());
-  PRINT(" ☼ %s = %p", "GetStackAddr()", GetStackAddr());
-  PRINT(" ☼ %s = %p", "GetStaticStackSize()", GetStaticStackSize());
-  PRINT(" ☼ %s = %p", "GetStaticStackAddr(0)", GetStaticStackAddr(0));
   PRINT(" ☼ %s = %p", "__builtin_frame_address(0)", __builtin_frame_address(0));
 
   PRINT("");

libc/sysv/consts.sh

@@ -219,10 +219,9 @@ syscon	mmap	MAP_FILE				0			0			0			0			0			0			0			0			# consensus
 syscon	mmap	MAP_SHARED				1			1			1			1			1			1			1			1			# forced consensus & faked nt
 syscon	mmap	MAP_SHARED_VALIDATE			3			3			1			1			1			1			1			1			# weird linux thing
 syscon	mmap	MAP_PRIVATE				2			2			2			2			2			2			2			2			# forced consensus & faked nt
-syscon	mmap	MAP_STACK				6			6			6			6			6			6			6			6			# our definition
 syscon	mmap	MAP_TYPE				15			15			15			15			15			15			15			15			# mask for type of mapping
 syscon	mmap	MAP_FIXED				0x00000010		0x00000010		0x00000010		0x00000010		0x00000010		0x00000010		0x00000010		0x00000010		# unix consensus; openbsd appears to forbid; faked nt
-syscon	mmap	MAP_FIXED_NOREPLACE			0x08000000		0x08000000		0x00004010		0x00004010		0x08000000		0x08000000		0x08000000		0x08000000     		# handled and defined by cosmo runtime; 0x100000 on linux 4.7+; MAP_FIXED|MAP_EXCL on FreeBSD
+syscon	mmap	MAP_FIXED_NOREPLACE			0x08000000		0x08000000		0x00004000		0x00004000		0x08000000		0x08000000		0x08000000		0x08000000     		# handled and defined by cosmo runtime; 0x100000 on linux 4.7+; MAP_FIXED|MAP_EXCL on FreeBSD
 syscon	mmap	MAP_ANONYMOUS				0x00000020		0x00000020		0x00001000		0x00001000		0x00001000		0x00001000		0x00001000		0x00000020		# bsd consensus; faked nt
 syscon	mmap	MAP_GROWSDOWN				0x00000100		0x00000100		0			0			0			0			0			0			# use MAP_STACK; abstracted by MAP_STACK; may be passed to __sys_mmap() for low-level Linux fiddling
 syscon	mmap	MAP_LOCKED				0x00002000		0x00002000		0			0			0			0			0			0
@@ -293,8 +292,6 @@ syscon	mprot	PROT_NONE				0			0			0			0			0			0			0			0			# mmap, mprotect, unix
 syscon	mprot	PROT_READ				1			1			1			1			1			1			1			1			# mmap, mprotect, unix consensus
 syscon	mprot	PROT_WRITE				2			2			2			2			2			2			2			2			# mmap, mprotect, unix consensus
 syscon	mprot	PROT_EXEC				4			4			4			4			4			4			4			4			# mmap, mprotect, unix consensus
-syscon	mprot	PROT_GROWSDOWN				0x01000000		0x01000000		0			0			0			0			0			0			# intended for mprotect; see MAP_GROWSDOWN for mmap() (todo: what was 0x01000000 on nt)
-syscon	mprot	PROT_GROWSUP				0x02000000		0x02000000		0			0			0			0			0			0			# intended for mprotect; see MAP_GROWSDOWN for mmap()
 
 #	mremap() flags
 #	the revolutionary praxis of realloc()

libc/sysv/consts/MAP_FIXED_NOREPLACE.S

@@ -1,2 +1,2 @@
 #include "libc/sysv/consts/syscon.internal.h"
-.syscon mmap,MAP_FIXED_NOREPLACE,0x08000000,0x08000000,0x00004010,0x00004010,0x08000000,0x08000000,0x08000000,0x08000000
+.syscon mmap,MAP_FIXED_NOREPLACE,0x08000000,0x08000000,0x00004000,0x00004000,0x08000000,0x08000000,0x08000000,0x08000000

libc/sysv/consts/MAP_STACK.S

@@ -1,2 +0,0 @@
-#include "libc/sysv/consts/syscon.internal.h"
-.syscon mmap,MAP_STACK,6,6,6,6,6,6,6,6

libc/sysv/consts/PROT_GROWSDOWN.S

@@ -1,2 +0,0 @@
-#include "libc/sysv/consts/syscon.internal.h"
-.syscon mprot,PROT_GROWSDOWN,0x01000000,0x01000000,0,0,0,0,0,0

libc/sysv/consts/PROT_GROWSUP.S

@@ -1,2 +0,0 @@
-#include "libc/sysv/consts/syscon.internal.h"
-.syscon mprot,PROT_GROWSUP,0x02000000,0x02000000,0,0,0,0,0,0

libc/sysv/consts/map.h

@@ -33,7 +33,6 @@ COSMOPOLITAN_C_END_
 #define MAP_FILE    0
 #define MAP_SHARED  1
 #define MAP_PRIVATE 2
-#define MAP_STACK   6
 #define MAP_TYPE    15
 #define MAP_FIXED   16
 

libc/sysv/consts/prot.h

@@ -7,8 +7,6 @@ extern const int PROT_NONE;
 extern const int PROT_READ;
 extern const int PROT_WRITE;
 extern const int PROT_EXEC;
-extern const int PROT_GROWSDOWN;
-extern const int PROT_GROWSUP;
 
 COSMOPOLITAN_C_END_
 #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

libc/sysv/systemfive.S

@@ -16,7 +16,6 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/intrin/strace.internal.h"
 #include "libc/dce.h"
 #include "libc/macros.internal.h"
 #include "libc/thread/pt.internal.h"
@@ -378,30 +377,8 @@ _init_systemfive_magnums:
 	jnz	3b
 	xchg	%rbx,%rax
 	stosq
-#if SYSDEBUG
-	inc	%r8d
-#endif
 	jmp	2b
 5:	nop
-#if SYSDEBUG
-	push	%rdi
-	push	%rsi
-	push	%r8
-	call	__describe_os
-	mov	%rax,%rdx
-	pop	%r8
-	.weak	__stracef
-	ezlea	__stracef,ax
-	test	%rax,%rax
-	jz	5f
-	ezlea	.Llog,di
-	mov	%r8d,%esi
-	push	%rax
-	call	*%rax
-	pop	%rax
-5:	pop	%rsi
-	pop	%rdi
-#endif /* DEBUGSYS */
 	pop	%rdi
 	pop	%rsi
 	pop	%rbx
@@ -447,11 +424,3 @@ _init_systemfive_sigsys:
 _init_systemfive_done:
 	nop
 	.init.end 300,_init_systemfive,globl,hidden
-
-#if SYSDEBUG
-	.rodata.str1.1
-.Llog:	.ascii	STRACE_PROLOGUE
-	.ascii	"bell system five system call support"
-	.asciz	" %'u magnums loaded on %s\n"
-	.previous
-#endif /* DEBUGSYS */

libc/thread/getstackbottom.c

@@ -1,7 +1,7 @@
 /*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
 │ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
 ╞══════════════════════════════════════════════════════════════════════════════╡
-│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│ Copyright 2024 Justine Alexandra Roberts Tunney                              │
 │                                                                              │
 │ Permission to use, copy, modify, and/or distribute this software for         │
 │ any purpose with or without fee is hereby granted, provided that the         │
@@ -16,17 +16,17 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/runtime/memtrack.internal.h"
 #include "libc/thread/thread.h"
 
-// nsync depends on this non-nsync lock
+uintptr_t GetStackBottom(void) {
-
+  char *bottom;
-extern pthread_mutex_t __mmi_lock_obj;
+  void *stackaddr;
-
+  pthread_attr_t tattr;
-void __mmi_lock(void) {
+  size_t stacksize, guardsize;
-  pthread_mutex_lock(&__mmi_lock_obj);
+  pthread_getattr_np(pthread_self(), &tattr);
-}
+  pthread_attr_getstack(&tattr, &stackaddr, &stacksize);
-
+  pthread_attr_getguardsize(&tattr, &guardsize);
-void __mmi_unlock(void) {
+  bottom = stackaddr;
-  pthread_mutex_unlock(&__mmi_lock_obj);
+  bottom += guardsize;
+  return (uintptr_t)bottom;
 }

libc/thread/openbsd.internal.h

@@ -4,7 +4,7 @@ COSMOPOLITAN_C_START_
 
 struct __tfork {
   void *tf_tcb;
-  int32_t *tf_tid;
+  _Atomic(int) *tf_tid;
   void *tf_stack;
 };
 

libc/thread/pthread_atfork.c

@@ -16,7 +16,6 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/assert.h"
 #include "libc/atomic.h"
 #include "libc/calls/state.internal.h"
 #include "libc/cosmo.h"
@@ -25,8 +24,8 @@
 #include "libc/intrin/atomic.h"
 #include "libc/intrin/dll.h"
 #include "libc/intrin/leaky.internal.h"
+#include "libc/intrin/strace.internal.h"
 #include "libc/macros.internal.h"
-#include "libc/mem/mem.h"
 #include "libc/proc/proc.internal.h"
 #include "libc/runtime/runtime.h"
 #include "libc/str/str.h"
@@ -42,18 +41,19 @@ struct AtFork {
 static struct AtForks {
   pthread_spinlock_t lock;
   struct AtFork *list;
-  struct AtFork pool[8];
+  struct AtFork pool[64];
   atomic_int allocated;
 } _atforks;
 
-static void _pthread_onfork(int i) {
+static void _pthread_onfork(int i, const char *op) {
   struct AtFork *a;
-  unassert(0 <= i && i <= 2);
   if (!i)
     pthread_spin_lock(&_atforks.lock);
   for (a = _atforks.list; a; a = a->p[!i]) {
-    if (a->f[i])
+    if (a->f[i]) {
+      STRACE("pthread_atfork(%s, %t)", op, a->f[i]);
       a->f[i]();
+    }
     _atforks.list = a;
   }
   if (i)
@@ -61,15 +61,15 @@ static void _pthread_onfork(int i) {
 }
 
 void _pthread_onfork_prepare(void) {
-  _pthread_onfork(0);
+  _pthread_onfork(0, "prepare");
 }
 
 void _pthread_onfork_parent(void) {
-  _pthread_onfork(1);
+  _pthread_onfork(1, "parent");
 }
 
 void _pthread_onfork_child(void) {
-  _pthread_onfork(2);
+  _pthread_onfork(2, "child");
 }
 
 static struct AtFork *_pthread_atfork_alloc(void) {
@@ -78,7 +78,7 @@ static struct AtFork *_pthread_atfork_alloc(void) {
       (i = atomic_fetch_add(&_atforks.allocated, 1)) < n) {
     return _atforks.pool + i;
   } else {
-    return malloc(sizeof(struct AtFork));
+    return 0;
   }
 }
 

libc/thread/pthread_create.c

@@ -28,6 +28,7 @@
 #include "libc/intrin/bsr.h"
 #include "libc/intrin/describeflags.internal.h"
 #include "libc/intrin/dll.h"
+#include "libc/intrin/kprintf.h"
 #include "libc/intrin/strace.internal.h"
 #include "libc/intrin/weaken.h"
 #include "libc/log/internal.h"
@@ -64,10 +65,6 @@ __static_yoink("_pthread_onfork_prepare");
 __static_yoink("_pthread_onfork_parent");
 __static_yoink("_pthread_onfork_child");
 
-/* #ifndef MODE_DBG */
-/* __static_yoink("threaded_dlmalloc"); */
-/* #endif */
-
 #define MAP_ANON_OPENBSD  0x1000
 #define MAP_STACK_OPENBSD 0x4000
 
@@ -76,7 +73,8 @@ void _pthread_free(struct PosixThread *pt, bool isfork) {
   if (pt->pt_flags & PT_STATIC)
     return;
   if (pt->pt_flags & PT_OWNSTACK) {
-    unassert(!munmap(pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize));
+    unassert(!munmap(pt->pt_attr.__stackaddr,
+                     pt->pt_attr.__stacksize + (uintptr_t)ape_stack_align));
   }
   if (!isfork) {
     uint64_t syshand =
@@ -149,7 +147,7 @@ static int FixupCustomStackOnOpenbsd(pthread_attr_t *attr) {
   uintptr_t x, y;
   int e, rc, pagesz;
   pagesz = getauxval(AT_PAGESZ);
-  n = attr->__stacksize;
+  n = attr->__stacksize + (uintptr_t)ape_stack_align;
   x = (uintptr_t)attr->__stackaddr;
   y = ROUNDUP(x, pagesz);
   n -= y - x;
@@ -159,7 +157,7 @@ static int FixupCustomStackOnOpenbsd(pthread_attr_t *attr) {
                  MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD,
                  -1, 0, 0) == (void *)y) {
     attr->__stackaddr = (void *)y;
-    attr->__stacksize = n;
+    attr->__stacksize = n - (uintptr_t)ape_stack_align;
     return 0;
   } else {
     rc = errno;
@@ -214,48 +212,37 @@ static errno_t pthread_create_impl(pthread_t *thread,
     }
   } else {
     // cosmo is managing the stack
-    // 1. in mono repo optimize for tiniest stack possible
-    // 2. in public world optimize to *work* regardless of memory
-    int granularity = FRAMESIZE;
     int pagesize = getauxval(AT_PAGESZ);
     pt->pt_attr.__guardsize = ROUNDUP(pt->pt_attr.__guardsize, pagesize);
-    pt->pt_attr.__stacksize = ROUNDUP(pt->pt_attr.__stacksize, granularity);
+    pt->pt_attr.__stacksize = pt->pt_attr.__stacksize;
     if (pt->pt_attr.__guardsize + pagesize > pt->pt_attr.__stacksize) {
       _pthread_free(pt, false);
       return EINVAL;
     }
-    if (pt->pt_attr.__guardsize == pagesize &&
+    pt->pt_attr.__stackaddr =
-        !(IsAarch64() && IsLinux() && IsQemuUser())) {
+        mmap(0, pt->pt_attr.__stacksize + (uintptr_t)ape_stack_align,
-      // MAP_GROWSDOWN doesn't work very well on qemu-aarch64
+             PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-      pt->pt_attr.__stackaddr =
+    if (pt->pt_attr.__stackaddr != MAP_FAILED) {
-          mmap(0, pt->pt_attr.__stacksize, PROT_READ | PROT_WRITE,
+      if (IsOpenbsd() &&
-               MAP_STACK | MAP_ANONYMOUS, -1, 0);
+          __sys_mmap(
-    } else {
+              pt->pt_attr.__stackaddr,
-      pt->pt_attr.__stackaddr =
+              pt->pt_attr.__stacksize + (uintptr_t)ape_stack_align,
-          mmap(0, pt->pt_attr.__stacksize, PROT_READ | PROT_WRITE,
+              PROT_READ | PROT_WRITE,
-               MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+              MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD,
-      if (pt->pt_attr.__stackaddr != MAP_FAILED) {
+              -1, 0, 0) != pt->pt_attr.__stackaddr) {
-        if (IsOpenbsd() &&
+        notpossible;
-            __sys_mmap(
+      }
-                pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize,
+      if (pt->pt_attr.__guardsize) {
-                PROT_READ | PROT_WRITE,
+        if (!IsWindows()) {
-                MAP_PRIVATE | MAP_FIXED | MAP_ANON_OPENBSD | MAP_STACK_OPENBSD,
+          if (mprotect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize,
-                -1, 0, 0) != pt->pt_attr.__stackaddr) {
+                       PROT_NONE)) {
-          notpossible;
+            notpossible;
-        }
+          }
-        if (pt->pt_attr.__guardsize) {
+        } else {
-          if (!IsWindows()) {
+          uint32_t oldattr;
-            if (mprotect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize,
+          if (!VirtualProtect(pt->pt_attr.__stackaddr, pt->pt_attr.__guardsize,
-                         PROT_NONE)) {
+                              kNtPageReadwrite | kNtPageGuard, &oldattr)) {
-              notpossible;
+            notpossible;
-            }
-          } else {
-            uint32_t oldattr;
-            if (!VirtualProtect(pt->pt_attr.__stackaddr,
-                                pt->pt_attr.__guardsize,
-                                kNtPageReadwrite | kNtPageGuard, &oldattr)) {
-              notpossible;
-            }
           }
         }
       }
@@ -305,8 +292,7 @@ static errno_t pthread_create_impl(pthread_t *thread,
   _pthread_unlock();
 
   // launch PosixThread(pt) in new thread
-  if ((rc = clone(PosixThread, pt->pt_attr.__stackaddr,
+  if ((rc = clone(PosixThread, pt->pt_attr.__stackaddr, pt->pt_attr.__stacksize,
-                  pt->pt_attr.__stacksize - (IsOpenbsd() ? 16 : 0),
                   CLONE_VM | CLONE_THREAD | CLONE_FS | CLONE_FILES |
                       CLONE_SIGHAND | CLONE_SYSVSEM | CLONE_SETTLS |
                       CLONE_PARENT_SETTID | CLONE_CHILD_SETTID |

libc/thread/pthread_getattr_np.c

@@ -20,10 +20,10 @@
 #include "libc/calls/struct/rlimit.h"
 #include "libc/dce.h"
 #include "libc/intrin/atomic.h"
+#include "libc/intrin/maps.h"
 #include "libc/limits.h"
 #include "libc/macros.internal.h"
 #include "libc/runtime/runtime.h"
-#include "libc/runtime/stack.h"
 #include "libc/str/str.h"
 #include "libc/sysv/consts/auxv.h"
 #include "libc/sysv/consts/rlim.h"
@@ -73,8 +73,9 @@ errno_t pthread_getattr_np(pthread_t thread, pthread_attr_t *attr) {
       __builtin_unreachable();
   }
   if (!attr->__stacksize && (pt->pt_flags & PT_STATIC)) {
-    __get_main_stack(&attr->__stackaddr, &attr->__stacksize,
+    attr->__stackaddr = __maps.stack.addr;
-                     &attr->__guardsize);
+    attr->__stacksize = __maps.stack.size;
+    attr->__guardsize = 0;
   }
   return 0;
 }

libc/thread/tls.h

@@ -4,6 +4,7 @@
 #define TLS_ALIGNMENT 64
 
 #define TIB_FLAG_VFORKED 1
+#define TIB_FLAG_MAPLOCK 2
 
 #if !(__ASSEMBLER__ + __LINKER__ + 0)
 COSMOPOLITAN_C_START_

net/https/finishcertificate.c

@@ -18,6 +18,7 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/log/log.h"
 #include "libc/mem/gc.h"
+#include "libc/runtime/runtime.h"
 #include "libc/x/xasprintf.h"
 #include "net/https/https.h"
 
@@ -26,7 +27,7 @@ struct Cert FinishCertificate(struct Cert *ca, mbedtls_x509write_cert *wcert,
   int i, n, rc;
   unsigned char *p;
   mbedtls_x509_crt *cert;
-  p = malloc((n = FRAMESIZE));
+  p = malloc((n = __granularity()));
   i = mbedtls_x509write_crt_der(wcert, p, n, GenerateHardRandom, 0);
   if (i < 0)
     FATALF("write key (grep -0x%04x)", -i);

net/turfwar/turfwar.c

@@ -122,7 +122,7 @@
 #define SOCK_MAX           100     // max length of socket queue
 #define MSG_BUF            512     // small response lookaside
 
-#define INBUF_SIZE  FRAMESIZE
+#define INBUF_SIZE  65536
 #define OUTBUF_SIZE 8192
 
 #define TB_BYTES (1u << TB_CIDR)