Fix many thread and file descriptor issues on Windows

The greenbean web server now works nearly perfectly on Windows with over
1000 threads. But some synchronization issues still remain which prevent
us from going over nine thousand.

examples/greenbean.c

@@ -100,7 +100,7 @@ int workers;
 int messages;
 int connections;
 const char *status;
-volatile int closingtime;
+_Atomic(int) closingtime;
 
 int Worker(void *id) {
   int server, yes = 1;
@@ -273,8 +273,7 @@ int main(int argc, char *argv[]) {
                        MAP_STACK | MAP_ANONYMOUS, -1, 0);
     CHECK_NE(-1, clone(Worker, stack, GetStackSize(),
                        CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_FILES |
-                           CLONE_SIGHAND | CLONE_SETTLS | CLONE_CHILD_SETTID |
+                           CLONE_SIGHAND | CLONE_SETTLS,
-                           CLONE_CHILD_CLEARTID,
                        (void *)(intptr_t)i, 0, tls, 64, 0));
   }
   status = "";

libc/calls/dup-nt.c

@@ -48,26 +48,33 @@ textwindows int sys_dup_nt(int oldfd, int newfd, int flags, int start) {
   }
 
   // allocate a new file descriptor
-  if (newfd == -1) {
+  for (;;) {
-    if ((newfd = __reservefd_unlocked(start)) == -1) {
+    if (newfd == -1) {
-      _spunlock(&__fds_lock);
+      if ((newfd = __reservefd_unlocked(start)) == -1) {
-      return -1;
+        _spunlock(&__fds_lock);
+        return -1;
+      }
+      break;
+    } else {
+      if (__ensurefds_unlocked(newfd) == -1) {
+        _spunlock(&__fds_lock);
+        return -1;
+      }
+      if (g_fds.p[newfd].kind) {
+        _spunlock(&__fds_lock);
+        close(newfd);
+        _spinlock(&__fds_lock);
+      }
+      if (!g_fds.p[newfd].kind) {
+        g_fds.p[newfd].kind = kFdReserved;
+        break;
+      }
     }
-  } else {
-    if (__ensurefds_unlocked(newfd) == -1) {
-      _spunlock(&__fds_lock);
-      return -1;
-    }
-    if (g_fds.p[newfd].kind) {
-      _spunlock(&__fds_lock);
-      close(newfd);
-      _spinlock(&__fds_lock);
-    }
-    g_fds.p[newfd].kind = kFdReserved;
   }
 
   handle = g_fds.p[oldfd].handle;
   proc = GetCurrentProcess();
+
   if (DuplicateHandle(proc, handle, proc, &g_fds.p[newfd].handle, 0, true,
                       kNtDuplicateSameAccess)) {
     g_fds.p[newfd].kind = g_fds.p[oldfd].kind;
@@ -82,7 +89,7 @@ textwindows int sys_dup_nt(int oldfd, int newfd, int flags, int start) {
     }
     rc = newfd;
   } else {
-    __releasefd(newfd);
+    __releasefd_unlocked(newfd);
     rc = __winerr();
   }
 

libc/calls/internal.h

@@ -66,7 +66,7 @@ struct Fd {
 };
 
 struct Fds {
-  size_t f; /* lowest free slot */
+  int f;    /* lowest free slot */
   size_t n; /* monotonic capacity */
   struct Fd *p;
   struct Fd __init_p[OPEN_MAX];

libc/calls/interrupts-nt.c

@@ -23,16 +23,35 @@
 #include "libc/calls/strace.internal.h"
 #include "libc/calls/struct/sigaction.h"
 #include "libc/dce.h"
-#include "libc/intrin/spinlock.h"
+#include "libc/intrin/lockcmpxchgp.h"
+#include "libc/nexgen32e/threaded.h"
+
+_Alignas(64) static int rlock;
+
+static privileged inline bool AcquireInterruptPollLock(void) {
+  // any thread can poll for interrupts
+  // but it's wasteful to have every single thread doing it
+  int me, owner, tries;
+  if (!__threaded) return true;
+  me = gettid();
+  owner = 0;
+  if (_lockcmpxchgp(&rlock, &owner, me)) return true;
+  return owner == me;
+}
+
+static textwindows inline void ReleaseInterruptPollLock(void) {
+  int zero = 0;
+  __atomic_store(&rlock, &zero, __ATOMIC_RELAXED);
+}
 
 textwindows bool _check_interrupts(bool restartable, struct Fd *fd) {
   bool res;
   if (__time_critical) return false;
-  if (_trylock(&__fds_lock)) return false;
+  if (!AcquireInterruptPollLock()) return false;
   if (weaken(_check_sigalrm)) weaken(_check_sigalrm)();
   if (weaken(_check_sigchld)) weaken(_check_sigchld)();
   if (fd && weaken(_check_sigwinch)) weaken(_check_sigwinch)(fd);
   res = weaken(__sig_check) && weaken(__sig_check)(restartable);
-  _spunlock(&__fds_lock);
+  ReleaseInterruptPollLock();
   return res;
 }

libc/calls/ioctl_tiocgwinsz-nt.c

@@ -22,6 +22,7 @@
 #include "libc/calls/strace.internal.h"
 #include "libc/calls/struct/termios.h"
 #include "libc/calls/struct/winsize.h"
+#include "libc/intrin/spinlock.h"
 #include "libc/log/log.h"
 #include "libc/nt/console.h"
 #include "libc/nt/enum/startf.h"
@@ -31,13 +32,15 @@
 #include "libc/sysv/errfuns.h"
 
 textwindows int ioctl_tiocgwinsz_nt(struct Fd *fd, struct winsize *ws) {
-  int i, e;
+  int i, e, rc;
   uint32_t mode;
   struct Fd *fds[3];
   struct NtStartupInfo startinfo;
   struct NtConsoleScreenBufferInfoEx sbinfo;
+  rc = -1;
   e = errno;
   if (ws) {
+    _spinlock(&__fds_lock);
     fds[0] = fd, fds[1] = g_fds.p + 1, fds[2] = g_fds.p + 0;
     GetStartupInfo(&startinfo);
     for (i = 0; i < ARRAYLEN(fds); ++i) {
@@ -51,14 +54,16 @@ textwindows int ioctl_tiocgwinsz_nt(struct Fd *fd, struct winsize *ws) {
             ws->ws_xpixel = 0;
             ws->ws_ypixel = 0;
             errno = e;
-            return 0;
+            rc = 0;
+            break;
           } else if (startinfo.dwFlags & kNtStartfUsecountchars) {
             ws->ws_col = startinfo.dwXCountChars;
             ws->ws_row = startinfo.dwYCountChars;
             ws->ws_xpixel = 0;
             ws->ws_ypixel = 0;
             errno = e;
-            return 0;
+            rc = 0;
+            break;
           } else {
             __winerr();
           }
@@ -69,8 +74,9 @@ textwindows int ioctl_tiocgwinsz_nt(struct Fd *fd, struct winsize *ws) {
         ebadf();
       }
     }
+    _spunlock(&__fds_lock);
   } else {
     efault();
   }
-  return -1;
+  return rc;
 }

libc/calls/pipe-nt.c

@@ -49,9 +49,11 @@ textwindows int sys_pipe_nt(int pipefd[2], unsigned flags) {
   } else {
     mode = kNtPipeTypeMessage | kNtPipeReadmodeMessage;
   }
-  if ((hin = CreateNamedPipe(
+  _spunlock(&__fds_lock);
-           pipename, kNtPipeAccessInbound | kNtFileFlagOverlapped, mode, 1,
+  hin = CreateNamedPipe(pipename, kNtPipeAccessInbound | kNtFileFlagOverlapped,
-           PIPE_BUF, PIPE_BUF, 0, &kNtIsInheritable)) != -1) {
+                        mode, 1, PIPE_BUF, PIPE_BUF, 0, &kNtIsInheritable);
+  _spinlock(&__fds_lock);
+  if (hin != -1) {
     if ((hout = CreateFile(pipename, kNtGenericWrite, 0, &kNtIsInheritable,
                            kNtOpenExisting, kNtFileFlagOverlapped, 0)) != -1) {
       g_fds.p[reader].kind = kFdFile;

libc/calls/reservefd.c

@@ -29,33 +29,38 @@
 #include "libc/str/str.h"
 #include "libc/sysv/errfuns.h"
 
+// XXX: until we can add read locks to all the code that uses g_fds.p
+//      (right now we only have write locks) we need to keep old copies
+//      of g_fds.p around after it's been extended, so that threads
+//      which are using an fd they de facto own can continue reading
+static void FreeOldFdsArray(void *p) {
+  weaken(free)(p);
+}
+
 /**
  * Grows file descriptor array memory if needed.
  */
 int __ensurefds_unlocked(int fd) {
   size_t n1, n2;
   struct Fd *p1, *p2;
+  if (fd < g_fds.n) return fd;
+  STRACE("__ensurefds(%d) extending", fd);
+  if (!weaken(malloc)) return emfile();
+  p1 = g_fds.p;
   n1 = g_fds.n;
-  if (fd >= n1) {
+  if (p1 == g_fds.__init_p) {
-    STRACE("__ensurefds(%d) extending", fd);
+    if (!(p2 = weaken(malloc)(sizeof(g_fds.__init_p)))) return -1;
-    if (weaken(malloc)) {
+    memcpy(p2, p1, sizeof(g_fds.__init_p));
-      // TODO(jart): we need a semaphore for this
+    g_fds.p = p1 = p2;
-      p1 = g_fds.p;
-      n2 = fd + (fd >> 1);
-      if ((p2 = weaken(malloc)(n2 * sizeof(*p1)))) {
-        memcpy(p2, p1, n1 * sizeof(*p1));
-        g_fds.p = p2;
-        g_fds.n = n2;
-        if (p1 != g_fds.__init_p) {
-          __cxa_atexit(free, p1, 0);
-        }
-      } else {
-        fd = enomem();
-      }
-    } else {
-      fd = emfile();
-    }
   }
+  n2 = n1;
+  while (n2 <= fd) n2 *= 2;
+  if (!(p2 = weaken(malloc)(n2 * sizeof(*p1)))) return -1;
+  __cxa_atexit(FreeOldFdsArray, p1, 0);
+  memcpy(p2, p1, n1 * sizeof(*p1));
+  bzero(p2 + n1, (p2 + n2) - (p2 + n1));
+  g_fds.p = p2;
+  g_fds.n = n2;
   return fd;
 }
 
@@ -74,7 +79,7 @@ int __ensurefds(int fd) {
  */
 int __reservefd_unlocked(int start) {
   int fd;
-  for (fd = g_fds.f; fd < g_fds.n; ++fd) {
+  for (fd = MAX(start, g_fds.f); fd < g_fds.n; ++fd) {
     if (!g_fds.p[fd].kind) {
       break;
     }
@@ -100,10 +105,10 @@ int __reservefd(int start) {
  * Closes non-stdio file descriptors to free dynamic memory.
  */
 static void FreeFds(void) {
-  int i;
+  int i, keep = 3;
-  NTTRACE("FreeFds()");
+  STRACE("FreeFds()");
   _spinlock(&__fds_lock);
-  for (i = 3; i < g_fds.n; ++i) {
+  for (i = keep; i < g_fds.n; ++i) {
     if (g_fds.p[i].kind) {
       _spunlock(&__fds_lock);
       close(i);
@@ -111,8 +116,11 @@ static void FreeFds(void) {
     }
   }
   if (g_fds.p != g_fds.__init_p) {
-    memcpy(g_fds.__init_p, g_fds.p, sizeof(*g_fds.p) * 3);
+    bzero(g_fds.__init_p, sizeof(g_fds.__init_p));
-    weaken(free)(g_fds.p);
+    memcpy(g_fds.__init_p, g_fds.p, sizeof(*g_fds.p) * keep);
+    if (weaken(free)) {
+      weaken(free)(g_fds.p);
+    }
     g_fds.p = g_fds.__init_p;
     g_fds.n = ARRAYLEN(g_fds.__init_p);
   }

libc/calls/sigchld-nt.c

@@ -21,6 +21,7 @@
 #include "libc/calls/sig.internal.h"
 #include "libc/calls/strace.internal.h"
 #include "libc/dce.h"
+#include "libc/intrin/spinlock.h"
 #include "libc/nt/enum/wait.h"
 #include "libc/nt/runtime.h"
 #include "libc/nt/synchronization.h"
@@ -38,7 +39,10 @@ void _check_sigchld(void) {
   int pids[64];
   uint32_t i, n;
   int64_t handles[64];
-  if (!(n = __sample_pids(pids, handles, true))) return;
+  _spinlock(&__fds_lock);
+  n = __sample_pids(pids, handles, true);
+  _spunlock(&__fds_lock);
+  if (!n) return;
   i = WaitForMultipleObjects(n, handles, false, 0);
   if (i == kNtWaitTimeout) return;
   if (i == kNtWaitFailed) {
@@ -53,8 +57,10 @@ void _check_sigchld(void) {
   if (__sighandflags[SIGCHLD] & SA_NOCLDWAIT) {
     STRACE("SIGCHILD SA_NOCLDWAIT fd=%d handle=%ld", pids[i], handles[i]);
     CloseHandle(handles[i]);
-    __releasefd_unlocked(pids[i]);
+    __releasefd(pids[i]);
   }
+  _spinlock(&__fds_lock);
   g_fds.p[pids[i]].zombie = true;
+  _spunlock(&__fds_lock);
   __sig_add(SIGCHLD, CLD_EXITED);
 }

libc/intrin/_spinlock_debug_4.c

@@ -16,24 +16,30 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/bits/weaken.h"
 #include "libc/calls/calls.h"
 #include "libc/intrin/kprintf.h"
 #include "libc/intrin/lockcmpxchgp.h"
 #include "libc/intrin/spinlock.h"
+#include "libc/log/log.h"
 #include "libc/nexgen32e/rdtsc.h"
+#include "libc/runtime/internal.h"
 #include "libc/time/clockstonanos.internal.h"
 
-void _spinlock_debug_4(void *lockptr, const char *lockname, const char *file,
+privileged void _spinlock_debug_4(int *lock, const char *lockname,
-                       int line, const char *func) {
+                                  const char *file, int line,
+                                  const char *func) {
   unsigned i;
+  int me, owner;
   uint64_t ts1, ts2;
-  int me, owner, *lock = lockptr;
   me = gettid();
   owner = 0;
   if (!_lockcmpxchgp(lock, &owner, me)) {
     if (owner == me) {
-      kprintf("%s:%d: warning: lock re-entry on %s in %s()\n", file, line,
+      kprintf("%s:%d: error: lock re-entry on %s in %s()\n", file, line,
               lockname, func);
+      if (weaken(__die)) weaken(__die)();
+      __restorewintty();
       _Exit(1);
     }
     i = 0;

libc/intrin/_trylock_debug_4.c

@@ -16,41 +16,27 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/calls/calls.h"
+#include "libc/bits/weaken.h"
 #include "libc/intrin/kprintf.h"
 #include "libc/intrin/lockcmpxchgp.h"
 #include "libc/intrin/spinlock.h"
-#include "libc/nexgen32e/rdtsc.h"
+#include "libc/log/log.h"
-#include "libc/time/clockstonanos.internal.h"
+#include "libc/runtime/internal.h"
+#include "libc/runtime/runtime.h"
 
-void _spinlock_debug_1(void *lockptr, const char *lockname, const char *file,
+privileged int _trylock_debug_4(int *lock, const char *lockname,
-                       int line, const char *func) {
+                                const char *file, int line, const char *func) {
-  unsigned i;
+  int owner = 0;
-  uint64_t ts1, ts2;
+  int me = gettid();
-  int me, owner, *lock = lockptr;
+  if (_lockcmpxchgp(lock, &owner, me)) {
-  me = gettid();
+    return 0;
-  owner = 0;
+  } else if (owner != me) {
-  if (!_lockcmpxchgp(lock, &owner, me)) {
+    return owner;
-    if (owner == me) {
+  } else {
-      kprintf("%s:%d: warning: possible re-entry on lock %s in %s()\n", file,
+    kprintf("%s:%d: error: lock re-entry on %s in %s()\n", file, line, lockname,
-              line, lockname, func);
+            func);
-    }
+    if (weaken(__die)) weaken(__die)();
-    i = 0;
+    __restorewintty();
-    ts1 = rdtsc();
+    _Exit(1);
-    for (;;) {
-      owner = 0;
-      if (_lockcmpxchgp(lock, &owner, me)) break;
-      ts2 = rdtsc();
-      if (ClocksToNanos(ts1, ts2) > 1000000000ul) {
-        ts1 = ts2;
-        kprintf("%s:%d: warning: slow lock on %s in %s()\n", file, line,
-                lockname, func);
-      }
-      if (++i & 7) {
-        __builtin_ia32_pause();
-      } else {
-        sched_yield();
-      }
-    }
   }
 }

libc/intrin/gettid.greg.c

@@ -20,6 +20,10 @@
 #include "libc/dce.h"
 #include "libc/nexgen32e/threaded.h"
 #include "libc/nt/thread.h"
+#include "libc/nt/thunk/msabi.h"
+#include "libc/runtime/internal.h"
+
+__msabi extern typeof(GetCurrentThreadId) *const __imp_GetCurrentThreadId;
 
 /**
  * Returns current thread id.
@@ -38,7 +42,7 @@ privileged int gettid(void) {
   }
 
   if (IsWindows()) {
-    return GetCurrentThreadId();
+    return __imp_GetCurrentThreadId();
   }
 
   if (IsLinux()) {
@@ -83,5 +87,5 @@ privileged int gettid(void) {
     return wut;  // narrowing intentional
   }
 
-  return getpid();
+  return __pid;
 }

libc/intrin/intrin.mk

@@ -74,10 +74,23 @@ o/$(MODE)/libc/intrin/kprintf.greg.o:			\
 			-fno-sanitize=all		\
 			-fno-stack-protector
 
+# we can't use compiler magic because:
+#   spinlocks are called very early in initialization
+#   e.g. __cxa_atexit()
+o/$(MODE)/libc/intrin/gettid.greg.o			\
+o/$(MODE)/libc/intrin/_trylock_debug_4.o		\
+o/$(MODE)/libc/intrin/_spinlock_debug_4.o:		\
+		OVERRIDE_CFLAGS +=			\
+			-fwrapv				\
+			-x-no-pg			\
+			-mno-fentry			\
+			-ffreestanding			\
+			-fno-sanitize=all		\
+			-fno-stack-protector
+
 o/$(MODE)/libc/intrin/tls.greg.o			\
 o/$(MODE)/libc/intrin/exit.greg.o			\
 o/$(MODE)/libc/intrin/exit1.greg.o			\
-o/$(MODE)/libc/intrin/gettid.greg.o			\
 o/$(MODE)/libc/intrin/getenv.greg.o			\
 o/$(MODE)/libc/intrin/assertfail.greg.o			\
 o/$(MODE)/libc/intrin/describeiov.greg.o		\

libc/intrin/once.h

@@ -2,20 +2,20 @@
 #define COSMOPOLITAN_LIBC_INTRIN_ONCE_H_
 #include "libc/intrin/spinlock.h"
 
-#define _once(x)                       \
+#define _once(x)                      \
-  ({                                   \
+  ({                                  \
-    typeof(x) oncerc;                  \
+    typeof(x) oncerc;                 \
-    static bool once;                  \
+    static bool once;                 \
-    static typeof(oncerc) onceresult;  \
+    static typeof(oncerc) onceresult; \
-    _Alignas(64) static char oncelock; \
+    _Alignas(64) static int oncelock; \
-    _spinlock(&oncelock);              \
+    _spinlock(&oncelock);             \
-    if (once) {                        \
+    if (once) {                       \
-      oncerc = onceresult;             \
+      oncerc = onceresult;            \
-    } else {                           \
+    } else {                          \
-      oncerc = onceresult = x;         \
+      oncerc = onceresult = x;        \
-    }                                  \
+    }                                 \
-    _spunlock(&oncelock);              \
+    _spunlock(&oncelock);             \
-    oncerc;                            \
+    oncerc;                           \
   })
 
 #endif /* COSMOPOLITAN_LIBC_INTRIN_ONCE_H_ */

libc/intrin/releasefd.c

@@ -20,13 +20,6 @@
 #include "libc/intrin/spinlock.h"
 #include "libc/macros.internal.h"
 
-void __releasefd_unlocked(int fd) {
-  if (0 <= fd && fd < g_fds.n) {
-    g_fds.p[fd].kind = 0;
-    g_fds.f = MIN(fd, g_fds.f);
-  }
-}
-
 void __releasefd(int fd) {
   _spinlock(&__fds_lock);
   __releasefd_unlocked(fd);

libc/intrin/releasefd_unlocked.c

@@ -0,0 +1,27 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/internal.h"
+#include "libc/macros.internal.h"
+
+void __releasefd_unlocked(int fd) {
+  if (0 <= fd && fd < g_fds.n) {
+    g_fds.p[fd].kind = 0;
+    g_fds.f = MIN(fd, g_fds.f);
+  }
+}

libc/intrin/sched_yield.S

@@ -19,6 +19,7 @@
 #include "libc/dce.h"
 #include "libc/sysv/consts/nr.h"
 #include "libc/macros.internal.h"
+.privileged
 
 //	Asks kernel to let other threads be scheduled.
 //

libc/intrin/spinlock.h

@@ -4,6 +4,10 @@
 #include "libc/calls/calls.h"
 #include "libc/intrin/lockcmpxchg.h"
 #include "libc/intrin/lockcmpxchgp.h"
+/*───────────────────────────────────────────────────────────────────────────│─╗
+│ cosmopolitan § spinlocks                                                 ─╬─│┼
+╚────────────────────────────────────────────────────────────────────────────│─╝
+  privileged unsophisticated locking subroutines */
 
 #if IsModeDbg() && !defined(_SPINLOCK_DEBUG)
 #define _SPINLOCK_DEBUG
@@ -12,15 +16,27 @@
 #if defined(_SPINLOCK_DEBUG)
 #define _spinlock(lock)        _spinlock_ndebug(lock)
 #define _spinlock_ndebug(lock) _spinlock_cooperative(lock)
+#define _trylock(lock)         _trylock_debug(lock)
+#define _seizelock(lock)       _seizelock_impl(lock, gettid())
 #elif defined(TINY)
 #define _spinlock(lock)        _spinlock_tiny(lock)
 #define _spinlock_ndebug(lock) _spinlock_tiny(lock)
+#define _trylock(lock)         _trylock_inline(lock)
+#define _seizelock(lock)       _seizelock_impl(lock, 1)
 #else
 #define _spinlock(lock)        _spinlock_cooperative(lock)
 #define _spinlock_ndebug(lock) _spinlock_cooperative(lock)
+#define _trylock(lock)         _trylock_inline(lock)
+#define _seizelock(lock)       _seizelock_impl(lock, 1)
 #endif
 
-#define _trylock(lock) __atomic_test_and_set(lock, __ATOMIC_SEQ_CST)
+#define _trylock_inline(lock) __atomic_test_and_set(lock, __ATOMIC_SEQ_CST)
+
+#define _trylock_debug(lock) \
+  _trylock_debug_4(lock, #lock, __FILE__, __LINE__, __FUNCTION__)
+
+#define _spinlock_debug(lock) \
+  _spinlock_debug_4(lock, #lock, __FILE__, __LINE__, __FUNCTION__)
 
 #define _spunlock(lock)                             \
   do {                                              \
@@ -29,19 +45,19 @@
     __atomic_store(__lock, &__x, __ATOMIC_RELAXED); \
   } while (0)
 
-#define _seizelock(lock)                            \
+#define _seizelock_impl(lock, value)                \
   do {                                              \
     autotype(lock) __lock = (lock);                 \
-    typeof(*__lock) __x = 1;                        \
+    typeof(*__lock) __x = (value);                  \
     __atomic_store(__lock, &__x, __ATOMIC_RELEASE); \
   } while (0)
 
-#define _spinlock_tiny(lock)        \
+#define _spinlock_tiny(lock)          \
-  do {                              \
+  do {                                \
-    autotype(lock) __lock = (lock); \
+    autotype(lock) __lock = (lock);   \
-    while (_trylock(__lock)) {      \
+    while (_trylock_inline(__lock)) { \
-      __builtin_ia32_pause();       \
+      __builtin_ia32_pause();         \
-    }                               \
+    }                                 \
   } while (0)
 
 #define _spinlock_cooperative(lock)                  \
@@ -51,7 +67,7 @@
     int __tries = 0;                                 \
     for (;;) {                                       \
       __atomic_load(__lock, &__x, __ATOMIC_RELAXED); \
-      if (!__x && !_trylock(__lock)) {               \
+      if (!__x && !_trylock_inline(__lock)) {        \
         break;                                       \
       } else if (++__tries & 7) {                    \
         __builtin_ia32_pause();                      \
@@ -61,21 +77,7 @@
     }                                                \
   } while (0)
 
-void _spinlock_debug_1(void *, const char *, const char *, int, const char *);
+int _trylock_debug_4(int *, const char *, const char *, int, const char *);
-void _spinlock_debug_4(void *, const char *, const char *, int, const char *);
+void _spinlock_debug_4(int *, const char *, const char *, int, const char *);
-
-#define _spinlock_debug(lock)                                             \
-  do {                                                                    \
-    switch (sizeof(*(lock))) {                                            \
-      case 1:                                                             \
-        _spinlock_debug_1(lock, #lock, __FILE__, __LINE__, __FUNCTION__); \
-        break;                                                            \
-      case 4:                                                             \
-        _spinlock_debug_4(lock, #lock, __FILE__, __LINE__, __FUNCTION__); \
-        break;                                                            \
-      default:                                                            \
-        assert(!"unsupported size");                                      \
-    }                                                                     \
-  } while (0)
 
 #endif /* COSMOPOLITAN_LIBC_INTRIN_SPINLOCK_H_ */

libc/intrin/virtualprotect.c

@@ -24,6 +24,11 @@
 
 __msabi extern typeof(VirtualProtect) *const __imp_VirtualProtect;
 
+static const char *DescribeVpFlags(uint32_t *x) {
+  if (!x) return "n/a";
+  return DescribeNtPageFlags(*x);
+}
+
 /**
  * Protects memory on the New Technology.
  * @note this wrapper takes care of ABI, STRACE(), and __winerr()
@@ -34,13 +39,9 @@ textwindows bool32 VirtualProtect(void *lpAddress, uint64_t dwSize,
   bool32 bOk;
   char oldbuf[64];
   bOk = __imp_VirtualProtect(lpAddress, dwSize, flNewProtect, lpflOldProtect);
-  if (bOk) {
+  if (!bOk) __winerr();
-    __stpcpy(oldbuf, DescribeNtPageFlags(*lpflOldProtect));
-  } else {
-    __winerr();
-    __stpcpy(oldbuf, "n/a");
-  }
   NTTRACE("VirtualProtect(%p, %'zu, %s, [%s]) → %hhhd% m", lpAddress, dwSize,
-          DescribeNtPageFlags(flNewProtect), oldbuf, bOk);
+          DescribeNtPageFlags(flNewProtect),
+          DescribeVpFlags(bOk ? lpflOldProtect : 0), bOk);
   return bOk;
 }

libc/nexgen32e/threaded.h

@@ -23,9 +23,9 @@ void __install_tls(char[hasatleast 64]);
 static noasan inline char *__get_tls(void) {
   char *tib, *lin = (char *)0x30;
   if (IsLinux() || IsFreebsd() || IsNetbsd() || IsOpenbsd()) {
-    asm("mov\t%%fs:(%1),%0" : "=a"(tib) : "r"(lin));
+    asm("mov\t%%fs:(%1),%0" : "=a"(tib) : "r"(lin) : "memory");
   } else {
-    asm("mov\t%%gs:(%1),%0" : "=a"(tib) : "r"(lin));
+    asm("mov\t%%gs:(%1),%0" : "=a"(tib) : "r"(lin) : "memory");
     if (IsWindows()) {
       tib = *(char **)(tib + 0x1480 + __tls_index * 8);
     }

libc/rand/rand64.c

@@ -31,7 +31,7 @@
 
 static int thepid;
 static uint128_t thepool;
-_Alignas(64) static char rand64_lock;
+_Alignas(64) static int rand64_lock;
 
 /**
  * Returns nondeterministic random data.

libc/runtime/cosmo.S

@@ -135,6 +135,16 @@ cosmo:	push	%rbp
 	.init.end 306,_init_ftrace
 #endif
 
+#if IsAsan()
+	.init.start 306,_init_symbols
+	push	%rdi
+	push	%rsi
+	call	__init_symbols
+	pop	%rsi
+	pop	%rdi
+	.init.end 306,_init_symbols
+#endif
+
 #if IsModeDbg()
 #ifdef SYSDEBUG
 	.init.start 307,_init_printargs

libc/runtime/getsymboltable.c

@@ -30,7 +30,7 @@
 #include "libc/zip.h"
 #include "libc/zipos/zipos.internal.h"
 
-static char g_lock;
+static int g_lock;
 hidden struct SymbolTable *__symtab;  // for kprintf
 
 /**

libc/runtime/hook.greg.c

@@ -59,6 +59,7 @@ privileged noinstrument noasan int __hook(void *ifunc,
   intptr_t kMcount = (intptr_t)&mcount;
   intptr_t kProgramCodeStart = (intptr_t)_ereal;
   intptr_t kPrivilegedStart = (intptr_t)__privileged_addr;
+  if (!symbols) return -1;
   __morph_begin();
   for (i = 0; i < symbols->count; ++i) {
     if (symbols->addr_base + symbols->symbols[i].x < kProgramCodeStart) {

libc/runtime/symbols.c

@@ -0,0 +1,30 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/bits/weaken.h"
+#include "libc/dce.h"
+#include "libc/log/backtrace.internal.h"
+#include "libc/log/log.h"
+#include "libc/runtime/runtime.h"
+#include "libc/runtime/symbols.internal.h"
+
+void __init_symbols(void) {
+  if (__strace || (IsAsan() && weaken(__die))) {
+    GetSymbolTable();
+  }
+}

libc/sock/accept-nt.c

@@ -19,6 +19,7 @@
 #include "libc/calls/internal.h"
 #include "libc/calls/sig.internal.h"
 #include "libc/intrin/kprintf.h"
+#include "libc/intrin/spinlock.h"
 #include "libc/mem/mem.h"
 #include "libc/nt/files.h"
 #include "libc/nt/struct/pollfd.h"
@@ -44,7 +45,9 @@ textwindows int sys_accept_nt(struct Fd *fd, void *addr, uint32_t *addrsize,
   for (;;) {
     if (!WSAPoll(&(struct sys_pollfd_nt){fd->handle, POLLIN}, 1,
                  __SIG_POLLING_INTERVAL_MS)) {
-      if (_check_interrupts(true, g_fds.p)) return eintr();
+      if (_check_interrupts(true, g_fds.p)) {
+        return eintr();
+      }
       continue;
     }
     if ((h = WSAAccept(fd->handle, addr, (int32_t *)addrsize, 0, 0)) != -1) {
@@ -54,7 +57,8 @@ textwindows int sys_accept_nt(struct Fd *fd, void *addr, uint32_t *addrsize,
       if ((!(flags & SOCK_NONBLOCK) ||
            __sys_ioctlsocket_nt(h, FIONBIO, (uint32_t[]){1}) != -1) &&
           (sockfd2 = calloc(1, sizeof(struct SockFd)))) {
-        if ((client = __reservefd(-1)) != -1) {
+        _spinlock(&__fds_lock);
+        if ((client = __reservefd_unlocked(-1)) != -1) {
           sockfd2->family = sockfd->family;
           sockfd2->type = sockfd->type;
           sockfd2->protocol = sockfd->protocol;
@@ -63,8 +67,10 @@ textwindows int sys_accept_nt(struct Fd *fd, void *addr, uint32_t *addrsize,
           g_fds.p[client].mode = 0140666;
           g_fds.p[client].handle = h;
           g_fds.p[client].extra = (uintptr_t)sockfd2;
+          _spunlock(&__fds_lock);
           return client;
         }
+        _spunlock(&__fds_lock);
         free(sockfd2);
       }
       __sys_closesocket_nt(h);

libc/sock/epoll.c

@@ -36,6 +36,7 @@
 #include "libc/calls/internal.h"
 #include "libc/dce.h"
 #include "libc/errno.h"
+#include "libc/intrin/spinlock.h"
 #include "libc/limits.h"
 #include "libc/macros.internal.h"
 #include "libc/mem/mem.h"
@@ -1324,7 +1325,8 @@ static textwindows dontinline int sys_epoll_create1_nt(uint32_t flags) {
   struct PortState *port_state;
   struct TsTreeNode *tree_node;
   if (wepoll_init() < 0) return -1;
-  if ((fd = __reservefd(-1)) == -1) return -1;
+  fd = __reservefd(-1);
+  if (fd == -1) return -1;
   port_state = port_new(&ephnd);
   if (!port_state) {
     __releasefd(fd);
@@ -1338,10 +1340,12 @@ static textwindows dontinline int sys_epoll_create1_nt(uint32_t flags) {
     __releasefd(fd);
     return -1;
   }
+  _spinlock(&__fds_lock);
   g_fds.p[fd].kind = kFdEpoll;
   g_fds.p[fd].handle = ephnd;
   g_fds.p[fd].flags = flags;
   g_fds.p[fd].mode = 0140666;
+  _spunlock(&__fds_lock);
   return fd;
 }
 

libc/sock/socket-nt.c

@@ -44,7 +44,8 @@ textwindows int sys_socket_nt(int family, int type, int protocol) {
   int64_t h;
   struct SockFd *sockfd;
   int fd, oflags, truetype;
-  if ((fd = __reservefd(-1)) == -1) return -1;
+  fd = __reservefd(-1);
+  if (fd == -1) return -1;
   truetype = type & ~(SOCK_CLOEXEC | SOCK_NONBLOCK);
   if ((h = WSASocket(family, truetype, protocol, NULL, 0,
                      kNtWsaFlagOverlapped)) != -1) {

libc/sock/socketpair-nt.c

@@ -30,10 +30,10 @@
 #include "libc/sysv/errfuns.h"
 
 textwindows int sys_socketpair_nt(int family, int type, int proto, int sv[2]) {
-  int64_t hpipe, h1, h2;
-  int reader, writer, oflags;
-  char16_t pipename[64];
   uint32_t mode;
+  char16_t pipename[64];
+  int64_t hpipe, h1, h2;
+  int rc, reader, writer, oflags;
 
   // Supports only AF_UNIX
   if (family != AF_UNIX) {
@@ -53,9 +53,13 @@ textwindows int sys_socketpair_nt(int family, int type, int proto, int sv[2]) {
   }
 
   CreatePipeName(pipename);
-  if ((reader = __reservefd(-1)) == -1) return -1;
+  _spinlock(&__fds_lock);
-  if ((writer = __reservefd(-1)) == -1) {
+  reader = __reservefd_unlocked(-1);
-    __releasefd(reader);
+  writer = __reservefd_unlocked(-1);
+  _spunlock(&__fds_lock);
+  if (reader == -1 || writer == -1) {
+    if (reader != -1) __releasefd(reader);
+    if (writer != -1) __releasefd(writer);
     return -1;
   }
   if ((hpipe = CreateNamedPipe(
@@ -68,28 +72,33 @@ textwindows int sys_socketpair_nt(int family, int type, int proto, int sv[2]) {
 
   h1 = CreateFile(pipename, kNtGenericWrite | kNtGenericRead, 0,
                   &kNtIsInheritable, kNtOpenExisting, kNtFileFlagOverlapped, 0);
-  if (h1 == -1) {
-    CloseHandle(hpipe);
-    __releasefd(writer);
-    __releasefd(reader);
-    return -1;
-  }
 
   _spinlock(&__fds_lock);
 
-  g_fds.p[reader].kind = kFdFile;
+  if (h1 != -1) {
-  g_fds.p[reader].flags = oflags;
-  g_fds.p[reader].mode = 0140444;
-  g_fds.p[reader].handle = hpipe;
 
-  g_fds.p[writer].kind = kFdFile;
+    g_fds.p[reader].kind = kFdFile;
-  g_fds.p[writer].flags = oflags;
+    g_fds.p[reader].flags = oflags;
-  g_fds.p[writer].mode = 0140222;
+    g_fds.p[reader].mode = 0140444;
-  g_fds.p[writer].handle = h1;
+    g_fds.p[reader].handle = hpipe;
+
+    g_fds.p[writer].kind = kFdFile;
+    g_fds.p[writer].flags = oflags;
+    g_fds.p[writer].mode = 0140222;
+    g_fds.p[writer].handle = h1;
+
+    sv[0] = reader;
+    sv[1] = writer;
+
+    rc = 0;
+  } else {
+    CloseHandle(hpipe);
+    __releasefd_unlocked(writer);
+    __releasefd_unlocked(reader);
+    rc = -1;
+  }
 
   _spunlock(&__fds_lock);
 
-  sv[0] = reader;
+  return rc;
-  sv[1] = writer;
-  return 0;
 }

libc/zipos/get.c

@@ -75,7 +75,7 @@ struct Zipos *__zipos_get(void) {
   const char *progpath;
   static struct Zipos zipos;
   uint8_t *map, *base, *cdir;
-  _Alignas(64) static char lock;
+  _Alignas(64) static int lock;
   _spinlock(&lock);
   if (!once) {
     sigfillset(&neu);

libc/zipos/open.c

@@ -25,6 +25,7 @@
 #include "libc/calls/struct/stat.h"
 #include "libc/dce.h"
 #include "libc/errno.h"
+#include "libc/intrin/spinlock.h"
 #include "libc/macros.internal.h"
 #include "libc/mem/mem.h"
 #include "libc/nexgen32e/crc32.h"
@@ -126,11 +127,14 @@ static int __zipos_load(struct Zipos *zipos, size_t cf, unsigned flags,
   if (h->mem) {
     if ((fd = IsWindows() ? __reservefd(-1) : dup(2)) != -1) {
       if (__ensurefds(fd) != -1) {
+        _spinlock(&__fds_lock);
         h->handle = g_fds.p[fd].handle;
         g_fds.p[fd].kind = kFdZip;
         g_fds.p[fd].handle = (intptr_t)h;
         g_fds.p[fd].flags = flags | O_CLOEXEC;
         g_fds.p[fd].mode = mode;
+        g_fds.p[fd].extra = 0;
+        _spunlock(&__fds_lock);
         return fd;
       }
       close(fd);

test/libc/calls/open_test.c

@@ -16,8 +16,10 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/internal.h"
 #include "libc/dce.h"
 #include "libc/errno.h"
+#include "libc/macros.internal.h"
 #include "libc/sysv/consts/o.h"
 #include "libc/testlib/testlib.h"
 #include "libc/x/x.h"
@@ -88,3 +90,26 @@ TEST(open, testOpenExistingForAppendWriteOnly_seeksToEnd) {
   EXPECT_STREQ("hello", buf);
   EXPECT_SYS(0, 0, close(3));
 }
+
+int CountFds(void) {
+  int i, count;
+  for (count = i = 0; i < g_fds.n; ++i) {
+    if (g_fds.p[i].kind) {
+      ++count;
+    }
+  }
+  return count;
+}
+
+TEST(open, lotsOfFds) {
+  if (!IsWindows()) return;
+  int i, n = 200;
+  ASSERT_SYS(0, 0, xbarf("hello.txt", "hello", -1));
+  for (i = 3; i < n; ++i) {
+    EXPECT_EQ(i, CountFds());
+    EXPECT_SYS(0, i, open("hello.txt", O_RDONLY));
+  }
+  for (i = 3; i < n; ++i) {
+    EXPECT_SYS(0, 0, close(i));
+  }
+}

test/libc/runtime/clone_test.c

@@ -36,7 +36,8 @@
 #include "libc/time/time.h"
 
 char *stack, *tls;
-int x, me, tid, thechilde, childetid;
+int x, me, tid, *childetid;
+_Atomic(int) thechilde;
 
 void SetUp(void) {
   x = 0;
@@ -44,6 +45,7 @@ void SetUp(void) {
   tls = calloc(1, 64);
   __initialize_tls(tls);
   *(int *)(tls + 0x3c) = 31337;
+  childetid = (int *)(tls + 0x0038);
   ASSERT_NE(MAP_FAILED, (stack = mmap(0, GetStackSize(), PROT_READ | PROT_WRITE,
                                       MAP_STACK | MAP_ANONYMOUS, -1, 0)));
 }
@@ -77,26 +79,25 @@ int CloneTest1(void *arg) {
     ASSERT_EQ(31337, errno);
   }
   ASSERT_EQ(23, (intptr_t)arg);
-  thechilde = gettid();
   ASSERT_NE(gettid(), getpid());
-  ASSERT_EQ(gettid(), childetid);  // CLONE_CHILD_SETTID
+  ASSERT_EQ(gettid(), *childetid);  // CLONE_CHILD_SETTID
   return 0;
 }
 
 TEST(clone, test1) {
   int ptid = 0;
-  _seizelock(&childetid);
+  *childetid = -1;
+  _seizelock(childetid);
   ASSERT_NE(-1, (tid = clone(CloneTest1, stack, GetStackSize(),
                              CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_FILES |
                                  CLONE_SIGHAND | CLONE_PARENT_SETTID |
                                  CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID |
                                  CLONE_SETTLS,
-                             (void *)23, &ptid, tls, 64, &childetid)));
+                             (void *)23, &ptid, tls, 64, childetid)));
-  _spinlock(&childetid);  // CLONE_CHILD_CLEARTID
+  _spinlock(childetid);  // CLONE_CHILD_CLEARTID
   ASSERT_EQ(tid, ptid);
   ASSERT_EQ(42, x);
   ASSERT_NE(me, tid);
-  ASSERT_EQ(tid, thechilde);
   ASSERT_EQ(0, errno);
   errno = 31337;
   ASSERT_EQ(31337, errno);
@@ -115,13 +116,13 @@ int CloneTestSys(void *arg) {
 TEST(clone, tlsSystemCallsErrno_wontClobberMainThreadBecauseTls) {
   ASSERT_EQ(0, errno);
   ASSERT_EQ(31337, *(int *)(tls + 0x3c));
-  _seizelock(&childetid);
+  _seizelock(childetid);
   ASSERT_NE(-1, (tid = clone(CloneTestSys, stack, GetStackSize(),
                              CLONE_THREAD | CLONE_VM | CLONE_FS | CLONE_FILES |
                                  CLONE_SIGHAND | CLONE_CHILD_SETTID |
                                  CLONE_CHILD_CLEARTID | CLONE_SETTLS,
-                             (void *)23, 0, tls, 64, &childetid)));
+                             (void *)23, 0, tls, 64, childetid)));
-  _spinlock(&childetid);  // CLONE_CHILD_CLEARTID
+  _spinlock(childetid);  // CLONE_CHILD_CLEARTID
   ASSERT_EQ(0, errno);
   ASSERT_EQ(EFAULT, *(int *)(tls + 0x3c));
 }