Make mappings unlimited on NT

This change might also fix fork() in certain cases on NT.

libc/runtime/arememoryintervalsok.c

@@ -16,7 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/runtime/memtrack.h"
+#include "libc/runtime/memtrack.internal.h"
 #include "libc/runtime/runtime.h"
 
 noasan bool AreMemoryIntervalsOk(const struct MemoryIntervals *mm) {

libc/runtime/findmemoryinterval.c

@@ -16,7 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/runtime/memtrack.h"
+#include "libc/runtime/memtrack.internal.h"
 
 noasan unsigned FindMemoryInterval(const struct MemoryIntervals *mm, int x) {
   unsigned l, m, r;

libc/runtime/fork-nt.c

@@ -26,6 +26,7 @@
 #include "libc/nexgen32e/nt2sysv.h"
 #include "libc/nt/dll.h"
 #include "libc/nt/enum/filemapflags.h"
+#include "libc/nt/enum/memflags.h"
 #include "libc/nt/enum/pageflags.h"
 #include "libc/nt/enum/startf.h"
 #include "libc/nt/enum/wt.h"
@@ -37,7 +38,7 @@
 #include "libc/nt/synchronization.h"
 #include "libc/nt/thread.h"
 #include "libc/runtime/directmap.internal.h"
-#include "libc/runtime/memtrack.h"
+#include "libc/runtime/memtrack.internal.h"
 #include "libc/runtime/runtime.h"
 #include "libc/str/str.h"
 #include "libc/sysv/consts/map.h"
@@ -46,6 +47,51 @@
 #include "libc/sysv/consts/sig.h"
 #include "libc/sysv/errfuns.h"
 
+static textwindows noasan void NtDebug(const char *fmt, ...) {
+  return;
+  int i;
+  va_list va;
+  uint32_t w, u;
+  const char *s;
+  unsigned long d;
+  char c, b[256], *p;
+  va_start(va, fmt);
+  for (p = b;;) {
+    switch ((c = *fmt++)) {
+      case '\0':
+        va_end(va);
+        WriteFile(GetStdHandle(kNtStdErrorHandle), b, p - b, &w, 0);
+        return;
+      case '%':
+        switch ((c = *fmt++)) {
+          case 's':
+            for (s = va_arg(va, const char *); s && *s;) *p++ = *s++;
+            break;
+          case 'd':
+            d = va_arg(va, unsigned long);
+            for (i = 16; i--;) {
+              u = (d >> (i * 4)) & 0xf;
+              if (u < 10) {
+                c = '0' + u;
+              } else {
+                u -= 10;
+                c = 'a' + u;
+              }
+              *p++ = c;
+            }
+            break;
+          default:
+            *p++ = c;
+            break;
+        }
+        break;
+      default:
+        *p++ = c;
+        break;
+    }
+  }
+}
+
 static textwindows noasan char16_t *ParseInt(char16_t *p, int64_t *x) {
   *x = 0;
   while (*p == ' ') p++;
@@ -56,33 +102,42 @@ static textwindows noasan char16_t *ParseInt(char16_t *p, int64_t *x) {
   return p;
 }
 
-static noinline textwindows noasan void ForkIo(int64_t h, void *buf, size_t n,
+static noinline textwindows noasan bool ForkIo(int64_t h, void *buf, size_t n,
                                                bool32 (*f)()) {
   char *p;
   size_t i;
   uint32_t x;
   for (p = buf, i = 0; i < n; i += x) {
-    f(h, p + i, n - i, &x, NULL);
+    if (!f(h, p + i, n - i, &x, NULL)) {
+      return false;
+    }
   }
+  return true;
 }
 
 static noinline textwindows noasan void WriteAll(int64_t h, void *buf,
                                                  size_t n) {
-  ForkIo(h, buf, n, WriteFile);
+  if (!ForkIo(h, buf, n, WriteFile)) {
+    NtDebug("fork() WriteFile(%zu) failed %d\n", n, GetLastError());
+  }
 }
 
 static textwindows noinline noasan void ReadAll(int64_t h, void *buf,
                                                 size_t n) {
-  ForkIo(h, buf, n, ReadFile);
+  if (!ForkIo(h, buf, n, ReadFile)) {
+    NtDebug("fork() ReadFile(%zu) failed %d\n", n, GetLastError());
+  }
 }
 
 textwindows noasan void WinMainForked(void) {
   void *addr;
   jmp_buf jb;
+  long mapcount;
   uint64_t size;
   uint32_t i, varlen;
   struct DirectMap dm;
   int64_t reader, writer;
+  struct MemoryInterval *maps;
   char16_t var[21 + 1 + 21 + 1];
   varlen = GetEnvironmentVariable(u"_FORK", var, ARRAYLEN(var));
   if (!varlen) return;
@@ -90,29 +145,33 @@ textwindows noasan void WinMainForked(void) {
   SetEnvironmentVariable(u"_FORK", NULL);
   ParseInt(ParseInt(var, &reader), &writer);
   ReadAll(reader, jb, sizeof(jb));
-  ReadAll(reader, &_mmi.i, sizeof(_mmi.i));
-  for (i = 0; i < _mmi.i; ++i) {
-    ReadAll(reader, &_mmi.p[i], sizeof(_mmi.p[i]));
-    addr = (void *)((uint64_t)_mmi.p[i].x << 16);
-    size = ((uint64_t)(_mmi.p[i].y - _mmi.p[i].x) << 16) + FRAMESIZE;
-    if (_mmi.p[i].flags & MAP_PRIVATE) {
-      CloseHandle(_mmi.p[i].h);
-      _mmi.p[i].h =
-          sys_mmap_nt(addr, size, _mmi.p[i].prot, _mmi.p[i].flags, -1, 0)
-              .maphandle;
+  ReadAll(reader, &mapcount, sizeof(_mmi.i));
+  maps = GlobalAlloc(0, mapcount * sizeof(*_mmi.p));
+  ReadAll(reader, maps, mapcount * sizeof(*_mmi.p));
+  for (i = 0; i < mapcount; ++i) {
+    addr = (void *)((uint64_t)maps[i].x << 16);
+    size = ((uint64_t)(maps[i].y - maps[i].x) << 16) + FRAMESIZE;
+    if (maps[i].flags & MAP_PRIVATE) {
+      CloseHandle(maps[i].h);
+      maps[i].h =
+          sys_mmap_nt(addr, size, maps[i].prot, maps[i].flags, -1, 0).maphandle;
       ReadAll(reader, addr, size);
     } else {
       MapViewOfFileExNuma(
-          _mmi.p[i].h,
-          (_mmi.p[i].prot & PROT_WRITE)
+          maps[i].h,
+          (maps[i].prot & PROT_WRITE)
               ? kNtFileMapWrite | kNtFileMapExecute | kNtFileMapRead
               : kNtFileMapExecute | kNtFileMapRead,
           0, 0, size, addr, kNtNumaNoPreferredNode);
     }
   }
-  ReadAll(reader, _edata, _end - _edata);
+  ReadAll(reader, _etext, _end - _etext);
+  for (i = 0; i < mapcount; ++i) {
+    _mmi.p[i].h = maps[i].h;
+  }
   CloseHandle(reader);
   CloseHandle(writer);
+  GlobalFree(maps);
   if (weaken(__wincrash_nt)) {
     AddVectoredExceptionHandler(1, (void *)weaken(__wincrash_nt));
   }
@@ -156,14 +215,14 @@ textwindows int sys_fork_nt(void) {
         }
         WriteAll(writer, jb, sizeof(jb));
         WriteAll(writer, &_mmi.i, sizeof(_mmi.i));
+        WriteAll(writer, _mmi.p, _mmi.i * sizeof(*_mmi.p));
         for (i = 0; i < _mmi.i; ++i) {
-          WriteAll(writer, &_mmi.p[i], sizeof(_mmi.p[i]));
           if (_mmi.p[i].flags & MAP_PRIVATE) {
             WriteAll(writer, (void *)((uint64_t)_mmi.p[i].x << 16),
                      ((uint64_t)(_mmi.p[i].y - _mmi.p[i].x) << 16) + FRAMESIZE);
           }
         }
-        WriteAll(writer, _edata, _end - _edata);
+        WriteAll(writer, _etext, _end - _etext);
         CloseHandle(writer);
       } else {
         rc = -1;

libc/runtime/isheap.c

@@ -17,7 +17,7 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/macros.internal.h"
-#include "libc/runtime/memtrack.h"
+#include "libc/runtime/memtrack.internal.h"
 #include "libc/runtime/runtime.h"
 
 /**

libc/runtime/memtrack.c

@@ -17,9 +17,12 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/assert.h"
+#include "libc/bits/bits.h"
+#include "libc/bits/weaken.h"
 #include "libc/dce.h"
 #include "libc/macros.internal.h"
-#include "libc/runtime/memtrack.h"
+#include "libc/mem/mem.h"
+#include "libc/runtime/memtrack.internal.h"
 #include "libc/runtime/runtime.h"
 #include "libc/str/str.h"
 #include "libc/sysv/errfuns.h"
@@ -167,18 +170,38 @@ static noasan void RemoveMemoryIntervals(struct MemoryIntervals *mm, int i,
   mm->i -= n;
 }
 
-static noasan void CreateMemoryInterval(struct MemoryIntervals *mm, int i) {
+static noasan int CreateMemoryInterval(struct MemoryIntervals *mm, int i) {
+  int rc;
+  void *p;
+  size_t n;
+  static bool noreentry;
+  rc = 0;
   assert(i >= 0);
   assert(i <= mm->i);
-  assert(mm->i < ARRAYLEN(mm->p));
+  assert(mm->i < mm->n);
   MoveMemoryNoAsan(mm->p + i + 1, mm->p + i,
                    (intptr_t)(mm->p + mm->i) - (intptr_t)(mm->p + i));
-  ++mm->i;
+  if (++mm->i > (mm->n >> 1) && cmpxchg(&noreentry, false, true)) {
+    n = mm->n << 1;
+    p = weaken(malloc) ? weaken(malloc)(n * sizeof(*mm->p)) : 0;
+    if (p) {
+      memcpy(p, mm->p, mm->i * sizeof(*mm->p));
+      if (mm->p != mm->s && weaken(free)) {
+        weaken(free)(mm->p);
+      }
+      mm->p = p;
+      mm->n = n;
+    } else {
+      rc = enomem();
+    }
+    noreentry = false;
+  }
+  return 0;
 }
 
 static noasan int PunchHole(struct MemoryIntervals *mm, int x, int y, int i) {
-  if (mm->i == ARRAYLEN(mm->p)) return enomem();
-  CreateMemoryInterval(mm, i);
+  if (mm->i == mm->n) return enomem();
+  if (CreateMemoryInterval(mm, i) == -1) return -1;
   mm->p[i].y = x - 1;
   mm->p[i + 1].x = y + 1;
   return 0;
@@ -242,8 +265,8 @@ noasan int TrackMemoryInterval(struct MemoryIntervals *mm, int x, int y, long h,
              prot == mm->p[i].prot && flags == mm->p[i].flags) {
     mm->p[i].x = x;
   } else {
-    if (mm->i == ARRAYLEN(mm->p)) return enomem();
-    CreateMemoryInterval(mm, i);
+    if (mm->i == mm->n) return enomem();
+    if (CreateMemoryInterval(mm, i) == -1) return -1;
     mm->p[i].x = x;
     mm->p[i].y = y;
     mm->p[i].h = h;

libc/runtime/memtrack.internal.h

@@ -21,15 +21,18 @@ COSMOPOLITAN_C_START_
 #define kAutomapSize   MEMTRACK_ADDRESS(_kAutomapSize, 0x40000000)
 #define kFixedmapStart MEMTRACK_ADDRESS(_kFixedmapStart, 0x40000000)
 
+struct MemoryInterval {
+  int x;
+  int y;
+  long h;
+  int prot;
+  int flags;
+};
+
 struct MemoryIntervals {
-  long i;
-  struct MemoryInterval {
-    int x;
-    int y;
-    long h;
-    int prot;
-    int flags;
-  } p[128];
+  long i, n;
+  struct MemoryInterval *p;
+  struct MemoryInterval s[OPEN_MAX];
 };
 
 extern hidden struct MemoryIntervals _mmi;

libc/runtime/memtracknt.c

@@ -19,7 +19,7 @@
 #include "libc/assert.h"
 #include "libc/nt/memory.h"
 #include "libc/nt/runtime.h"
-#include "libc/runtime/memtrack.h"
+#include "libc/runtime/memtrack.internal.h"
 #include "libc/runtime/runtime.h"
 
 static noasan void *GetFrameAddr(int f) {

libc/runtime/mmap.c

@@ -26,7 +26,7 @@
 #include "libc/macros.internal.h"
 #include "libc/rand/rand.h"
 #include "libc/runtime/directmap.internal.h"
-#include "libc/runtime/memtrack.h"
+#include "libc/runtime/memtrack.internal.h"
 #include "libc/runtime/runtime.h"
 #include "libc/str/str.h"
 #include "libc/sysv/consts/map.h"

libc/runtime/mmi.c

@@ -16,6 +16,8 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/runtime/memtrack.h"
+#include "libc/runtime/memtrack.internal.h"
+
+STATIC_YOINK("_init__mmi");
 
 struct MemoryIntervals _mmi;

libc/runtime/mmi.init.S

@@ -0,0 +1,24 @@
+/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8     -*-│
+│vi: set et ft=asm ts=8 tw=8 fenc=utf-8                                     :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2021 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/macros.internal.h"
+
+	.init.start 200,_init__mmi
+	movb	$OPEN_MAX,_mmi+8
+	movl	$_mmi+24,_mmi+16
+	.init.end 200,_init__mmi

libc/runtime/msync-nt.c

@@ -20,7 +20,7 @@
 #include "libc/macros.internal.h"
 #include "libc/nt/files.h"
 #include "libc/nt/memory.h"
-#include "libc/runtime/memtrack.h"
+#include "libc/runtime/memtrack.internal.h"
 
 textwindows int sys_msync_nt(void *addr, size_t size, int flags) {
   int x, y, l, r, i;

libc/runtime/munmap.c

@@ -20,7 +20,7 @@
 #include "libc/dce.h"
 #include "libc/macros.internal.h"
 #include "libc/runtime/directmap.internal.h"
-#include "libc/runtime/memtrack.h"
+#include "libc/runtime/memtrack.internal.h"
 #include "libc/runtime/runtime.h"
 #include "libc/sysv/errfuns.h"
 

libc/runtime/printmemoryintervals.c

@@ -18,7 +18,7 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/calls/calls.h"
 #include "libc/log/log.h"
-#include "libc/runtime/memtrack.h"
+#include "libc/runtime/memtrack.internal.h"
 
 void PrintMemoryIntervals(int fd, const struct MemoryIntervals *mm) {
   int i, frames, maptally, gaptally;

libc/runtime/untrackmemoryintervals.c

@@ -18,7 +18,7 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/dce.h"
 #include "libc/macros.internal.h"
-#include "libc/runtime/memtrack.h"
+#include "libc/runtime/memtrack.internal.h"
 
 int UntrackMemoryIntervals(void *addr, size_t size) {
   int a, b;

libc/runtime/winmain.greg.c

@@ -38,7 +38,7 @@
 #include "libc/nt/struct/teb.h"
 #include "libc/runtime/directmap.internal.h"
 #include "libc/runtime/internal.h"
-#include "libc/runtime/memtrack.h"
+#include "libc/runtime/memtrack.internal.h"
 #include "libc/runtime/runtime.h"
 #include "libc/sock/internal.h"
 
@@ -108,6 +108,8 @@ static noasan textwindows wontreturn void WinMainNew(void) {
                    kNtEnableProcessedOutput | kNtEnableWrapAtEolOutput |
                        kNtEnableVirtualTerminalProcessing);
   }
+  _mmi.p = _mmi.s;
+  _mmi.n = OPEN_MAX;
   addr = version < 10 ? 0xff00000 : 0x777000000000;
   size = ROUNDUP(STACKSIZE + sizeof(struct WinArgs), FRAMESIZE);
   MapViewOfFileExNuma((_mmi.p[0].h = CreateFileMappingNuma(