Make some more fixups

libc/intrin/cxaatexit.c

@@ -19,8 +19,6 @@
 #include "libc/assert.h"
 #include "libc/bits/weaken.h"
 #include "libc/calls/strace.internal.h"
-#include "libc/intrin/pthread.h"
-#include "libc/intrin/spinlock.h"
 #include "libc/macros.internal.h"
 #include "libc/mem/mem.h"
 #include "libc/nexgen32e/bsr.h"
@@ -30,8 +28,6 @@
 
 STATIC_YOINK("__cxa_finalize");
 
-static pthread_mutex_t __cxa_lock;
-
 /**
  * Adds global destructor.
  *
@@ -51,7 +47,7 @@ noasan int __cxa_atexit(void *fp, void *arg, void *pred) {
   unsigned i;
   struct CxaAtexitBlock *b, *b2;
   _Static_assert(ATEXIT_MAX == CHAR_BIT * sizeof(b->mask), "");
-  pthread_mutex_lock(&__cxa_lock);
+  __cxa_lock();
   b = __cxa_blocks.p;
   if (!b) b = __cxa_blocks.p = &__cxa_blocks.root;
   if (!~b->mask) {
@@ -60,7 +56,7 @@ noasan int __cxa_atexit(void *fp, void *arg, void *pred) {
       b2->next = b;
       __cxa_blocks.p = b = b2;
     } else {
-      pthread_mutex_unlock(&__cxa_lock);
+      __cxa_unlock();
       return enomem();
     }
   }
@@ -70,6 +66,6 @@ noasan int __cxa_atexit(void *fp, void *arg, void *pred) {
   b->p[i].fp = fp;
   b->p[i].arg = arg;
   b->p[i].pred = pred;
-  pthread_mutex_unlock(&__cxa_lock);
+  __cxa_unlock();
   return 0;
 }

libc/intrin/cxafinalize.c

@@ -34,9 +34,12 @@
  * @param pred can be null to match all
  */
 void __cxa_finalize(void *pred) {
+  void *fp, *arg;
   unsigned i, mask;
   struct CxaAtexitBlock *b, *b2;
 StartOver:
+  __cxa_lock();
+StartOverLocked:
   if ((b = __cxa_blocks.p)) {
     for (;;) {
       mask = b->mask;
@@ -45,9 +48,11 @@ StartOver:
         mask &= ~(1u << i);
         if (!pred || pred == b->p[i].pred) {
           b->mask &= ~(1u << i);
-          if (b->p[i].fp) {
-            STRACE("__cxa_finalize(%t, %p)", b->p[i].fp, b->p[i].arg);
-            ((void (*)(void *))b->p[i].fp)(b->p[i].arg);
+          if ((fp = b->p[i].fp)) {
+            arg = b->p[i].arg;
+            __cxa_unlock();
+            STRACE("__cxa_finalize(%t, %p)", fp, arg);
+            ((void (*)(void *))fp)(arg);
             goto StartOver;
           }
         }
@@ -61,7 +66,7 @@ StartOver:
           }
         }
         __cxa_blocks.p = b2;
-        goto StartOver;
+        goto StartOverLocked;
       } else {
         if (b->next) {
           b = b->next;
@@ -71,4 +76,5 @@ StartOver:
       }
     }
   }
+  __cxa_unlock();
 }

libc/intrin/cxalock.c

@@ -0,0 +1,21 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/runtime/cxaatexit.internal.h"
+
+pthread_mutex_t __cxa_lock_obj;

libc/intrin/intrin.mk

@@ -42,10 +42,6 @@ $(LIBC_INTRIN_A).pkg:					\
 		$(LIBC_INTRIN_A_OBJS)			\
 		$(foreach x,$(LIBC_INTRIN_A_DIRECTDEPS),$($(x)_A).pkg)
 
-$(LIBC_INTRIN_A_OBJS):					\
-		OVERRIDE_CFLAGS +=			\
-			-foptimize-sibling-calls
-
 # we can't use asan and ubsan because:
 #   this is asan and ubsan
 o/$(MODE)/libc/intrin/asan.o				\
@@ -76,20 +72,6 @@ o/$(MODE)/libc/intrin/kprintf.greg.o:			\
 			-fno-sanitize=all		\
 			-fno-stack-protector
 
-# we can't use compiler magic because:
-#   spinlocks are called very early in initialization
-#   e.g. __cxa_atexit()
-o/$(MODE)/libc/intrin/gettid.greg.o			\
-o/$(MODE)/libc/intrin/_trylock_debug_4.o		\
-o/$(MODE)/libc/intrin/_spinlock_debug_4.o:		\
-		OVERRIDE_CFLAGS +=			\
-			-fwrapv				\
-			-x-no-pg			\
-			-mno-fentry			\
-			-ffreestanding			\
-			-fno-sanitize=all		\
-			-fno-stack-protector
-
 o/$(MODE)/libc/intrin/tls.greg.o			\
 o/$(MODE)/libc/intrin/exit.greg.o			\
 o/$(MODE)/libc/intrin/exit1.greg.o			\

libc/intrin/kprintf.greg.c

@@ -55,36 +55,6 @@
 
 extern hidden struct SymbolTable *__symtab;
 
-struct Timestamps {
-  unsigned long long birth;
-  unsigned long long start;
-};
-
-unsigned long long __kbirth;  // see fork-nt.c
-
-privileged static struct Timestamps kenter(void) {
-  struct Timestamps ts;
-  ts.start = rdtsc();
-  ts.birth = __kbirth;
-  if (!ts.birth) {
-    ts.birth = kStartTsc;
-    if (!ts.birth) ts.birth = 1;
-    _lockcmpxchg(&__kbirth, 0, ts.birth);
-  }
-  return ts;
-}
-
-privileged static void kleave(struct Timestamps ts) {
-  uint64_t finish, elapse, adjust;
-  finish = rdtsc();
-  elapse = unsignedsubtract(finish, ts.start);
-  adjust = ts.birth + elapse;
-  if (!adjust) adjust = 1;
-  if (__kbirth == ts.birth) {
-    _lockcmpxchg(&__kbirth, ts.birth, adjust);  // ignore overlapping intervals
-  }
-}
-
 privileged static inline char *kadvance(char *p, char *e, long n) {
   intptr_t t = (intptr_t)p;
   if (__builtin_add_overflow(t, n, &t)) t = (intptr_t)e;
@@ -213,8 +183,8 @@ privileged static void klog(const char *b, size_t n) {
   }
 }
 
-privileged static size_t kformat(char *b, size_t n, const char *fmt, va_list va,
-                                 struct Timestamps ts) {
+privileged static size_t kformat(char *b, size_t n, const char *fmt,
+                                 va_list va) {
   int si;
   wint_t t, u;
   const char *abet;
@@ -333,7 +303,7 @@ privileged static size_t kformat(char *b, size_t n, const char *fmt, va_list va,
           continue;
 
         case 'T':
-          x = ClocksToNanos(ts.start, ts.birth) % 86400000000000;
+          x = ClocksToNanos(rdtsc(), kStartTsc) % 86400000000000;
           goto FormatUnsigned;
 
         case 'P':
@@ -766,37 +736,8 @@ privileged static size_t kformat(char *b, size_t n, const char *fmt, va_list va,
 privileged size_t ksnprintf(char *b, size_t n, const char *fmt, ...) {
   size_t m;
   va_list v;
-  struct Timestamps t;
-  t = kenter();
   va_start(v, fmt);
-  m = kformat(b, n, fmt, v, t);
-  va_end(v);
-  kleave(t);
-  return m;
-}
-
-/**
- * Privileged snprintf() w/o timestamp feature.
- *
- * This provides a marginal performance boost, but it means %T can no
- * longer be used.
- *
- *     snprintf(".")       l:        25𝑐         8𝑛𝑠
- *     kusnprintf(".")     l:        22𝑐         7𝑛𝑠
- *     ksnprintf(".")      l:        54𝑐        17𝑛𝑠
- *
- * @param b is buffer, and guaranteed a NUL-terminator if `n>0`
- * @param n is number of bytes available in buffer
- * @return length of output excluding NUL, which may exceed `n`
- * @see kprintf() for documentation
- * @asyncsignalsafe
- * @vforksafe
- */
-privileged size_t kusnprintf(char *b, size_t n, const char *fmt, ...) {
-  size_t m;
-  va_list v;
-  va_start(v, fmt);
-  m = kformat(b, n, fmt, v, (struct Timestamps){0});
+  m = kformat(b, n, fmt, v);
   va_end(v);
   return m;
 }
@@ -812,12 +753,7 @@ privileged size_t kusnprintf(char *b, size_t n, const char *fmt, ...) {
  * @vforksafe
  */
 privileged size_t kvsnprintf(char *b, size_t n, const char *fmt, va_list v) {
-  size_t m;
-  struct Timestamps t;
-  t = kenter();
-  m = kformat(b, n, fmt, v, t);
-  kleave(t);
-  return m;
+  return kformat(b, n, fmt, v);
 }
 
 /**
@@ -830,12 +766,9 @@ privileged size_t kvsnprintf(char *b, size_t n, const char *fmt, va_list v) {
 privileged void kvprintf(const char *fmt, va_list v) {
   size_t n;
   char b[4000];
-  struct Timestamps t;
   if (!v) return;
-  t = kenter();
-  n = kformat(b, sizeof(b), fmt, v, t);
+  n = kformat(b, sizeof(b), fmt, v);
   klog(b, MIN(n, sizeof(b) - 1));
-  kleave(t);
 }
 
 /**

libc/intrin/kprintf.h

@@ -3,11 +3,8 @@
 #if !(__ASSEMBLER__ + __LINKER__ + 0)
 COSMOPOLITAN_C_START_
 
-extern unsigned long long __kbirth;
-
 void kprintf(const char *, ...);
 size_t ksnprintf(char *, size_t, const char *, ...);
-size_t kusnprintf(char *, size_t, const char *, ...);
 void kvprintf(const char *, va_list);
 size_t kvsnprintf(char *, size_t, const char *, va_list);
 bool kisdangerous(const void *);

libc/intrin/pthread_mutex_lock.c

@@ -26,7 +26,7 @@
 /**
  * Acquires mutex.
  */
-int pthread_mutex_lock(pthread_mutex_t *mutex) {
+noasan noubsan int pthread_mutex_lock(pthread_mutex_t *mutex) {
   int me, owner;
   unsigned tries;
   if (__threaded) {

libc/intrin/pthread_mutex_unlock.c

@@ -27,7 +27,7 @@
 /**
  * Releases mutex.
  */
-int pthread_mutex_unlock(pthread_mutex_t *mutex) {
+noasan noubsan int pthread_mutex_unlock(pthread_mutex_t *mutex) {
   int owner;
   bool shouldunlock;
   assert(mutex->reent > 0);

libc/runtime/cxaatexit.internal.h

@@ -1,5 +1,6 @@
 #ifndef COSMOPOLITAN_LIBC_RUNTIME_CXAATEXIT_H_
 #define COSMOPOLITAN_LIBC_RUNTIME_CXAATEXIT_H_
+#include "libc/intrin/pthread.h"
 #include "libc/stdio/stdio.h"
 #if !(__ASSEMBLER__ + __LINKER__ + 0)
 COSMOPOLITAN_C_START_
@@ -16,10 +17,14 @@ struct CxaAtexitBlocks {
   } * p, root;
 };
 
+extern pthread_mutex_t __cxa_lock_obj;
 extern struct CxaAtexitBlocks __cxa_blocks;
 
 void __cxa_printexits(FILE *, void *);
 
+#define __cxa_lock()   pthread_mutex_lock(&__cxa_lock_obj)
+#define __cxa_unlock() pthread_mutex_unlock(&__cxa_lock_obj)
+
 COSMOPOLITAN_C_END_
 #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
 #endif /* COSMOPOLITAN_LIBC_RUNTIME_CXAATEXIT_H_ */

libc/runtime/fork-nt.c

@@ -139,12 +139,11 @@ textwindows void WinMainForked(void) {
   char *addr, *shad;
   struct DirectMap dm;
   uint64_t size, upsize;
+  int64_t oncrash, savetsc;
   struct MemoryInterval *maps;
   char16_t fvar[21 + 1 + 21 + 1];
-  int64_t oncrash, savetsc, savebir;
   uint32_t i, varlen, oldprot, savepid;
   long mapcount, mapcapacity, specialz;
-  extern uint64_t ts asm("kStartTsc");
 
   // check to see if the process was actually forked
   // this variable should have the pipe handle numba
@@ -198,13 +197,11 @@ textwindows void WinMainForked(void) {
 
   // read the .data and .bss program image sections
   savepid = __pid;
-  savebir = __kbirth;
-  savetsc = ts;
+  savetsc = kStartTsc;
   ReadOrDie(reader, __data_start, __data_end - __data_start);
   ReadOrDie(reader, __bss_start, __bss_end - __bss_start);
   __pid = savepid;
-  __kbirth = savebir;
-  ts = savetsc;
+  kStartTsc = savetsc;
 
   // apply fixups and reapply memory protections
   _mmi.p = maps;

libc/runtime/ismemtracked.greg.c

@@ -31,6 +31,7 @@ static inline bool IsMemtrackedImpl(int x, int y) {
 }
 
 bool IsMemtracked(int x, int y) {
+  /* assumes __mmi_lock() is held */
   bool res;
   res = IsMemtrackedImpl(x, y);
   return res;

libc/runtime/runtime.h

@@ -20,7 +20,7 @@ extern _Atomic(int) __strace;                       /* SYS */
 extern char *program_invocation_name;               /* RII */
 extern char *program_invocation_short_name;         /* RII */
 extern uint64_t __syscount;                         /* RII */
-extern const uint64_t kStartTsc;                    /* RII */
+extern uint64_t kStartTsc;                          /* RII */
 extern const char kTmpPath[];                       /* RII */
 extern const char kNtSystemDirectory[];             /* RII */
 extern const char kNtWindowsDirectory[];            /* RII */

libc/runtime/winmain.greg.c

@@ -275,9 +275,8 @@ __msabi textwindows int64_t WinMain(int64_t hInstance, int64_t hPrevInstance,
                                     const char *lpCmdLine, int64_t nCmdShow) {
   const char16_t *cmdline;
   extern char os asm("__hostos");
-  extern uint64_t ts asm("kStartTsc");
   os = WINDOWS; /* madness https://news.ycombinator.com/item?id=21019722 */
-  ts = rdtsc();
+  kStartTsc = rdtsc();
   __pid = GetCurrentProcessId();
 #if !IsTiny()
   __wincrashearly = AddVectoredExceptionHandler(1, (void *)OnEarlyWinCrash);